diff --git a/.github/workflows/build-msi.yml b/.github/workflows/build-msi.yml new file mode 100644 index 0000000..895c42d --- /dev/null +++ b/.github/workflows/build-msi.yml @@ -0,0 +1,128 @@ +# Builds TpF2BigMaps.msi the way a release does: installer\build_msi.ps1 +# (build.bat -> out\tpf2_bigmap.dll, the vendored shared binaries, wix build, +# then an administrative-install validation). A push to main, a v* tag or a manual run +# gets the MSI and its checksum as an artifact; a v* tag also gets a draft GitHub +# release with them attached. Nothing builds on dev or pull requests. +# +# build.bat finds the runner's Visual Studio through vswhere, so no Build Tools +# path is assumed. WiX v7 is installed as a .NET global tool; passing +# -AcceptWixEula means the repository owner accepts the WiX Open Source +# Maintenance Fee terms (https://wixtoolset.org/osmf/) for CI builds. +# +# The runner has no tpf2-multiplayer checkout beside this one, so the script's +# own PluginHost.wxs drift check would be skipped; the "Shared fragment" step +# fetches the multiplayer copy at the release recorded in vendor\VENDORED.md +# and compares it instead (line endings aside, like the script does). +name: Build MSI + +on: + push: + branches: [main] + tags: ['v*'] + # docs and every Markdown file, release notes included, change no byte of the + # package. A release commit always touches installer/VERSION, so a tag is + # never filtered out by this. + paths-ignore: ['docs/**', '**/*.md'] + workflow_dispatch: + +permissions: + contents: write # release upload on tags; read-only is enough for everything else + +concurrency: + group: msi-${{ github.ref }} + cancel-in-progress: true + +jobs: + # ONE BUILD PER RELEASE. A release is a tag on a commit that main is then + # fast-forwarded to; both pushes would build the same bytes. The tag goes up + # first (tools/ci_run.py, the release recipe), so a main push finds its tag's + # run here and steps aside. A main push nobody tagged still builds. + check: + runs-on: ubuntu-latest + outputs: + skip: ${{ steps.dup.outputs.skip }} + steps: + - id: dup + env: + GH_TOKEN: ${{ github.token }} + run: | + skip=false + if [ "${{ github.ref }}" = "refs/heads/main" ]; then + tag=$(gh api "repos/${{ github.repository }}/actions/runs?head_sha=${{ github.sha }}&per_page=30" \ + --jq '[.workflow_runs[] | select(.event == "push" and (.head_branch | startswith("v")) and (.id != ${{ github.run_id }}))] | .[0].head_branch // empty') + if [ -n "$tag" ]; then skip=true; echo "commit ${{ github.sha }} is release $tag: its tag run builds it"; fi + fi + echo "skip=$skip" >> "$GITHUB_OUTPUT" + + msi: + needs: check + if: needs.check.outputs.skip != 'true' + runs-on: windows-2022 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + - name: Version + id: version + shell: pwsh + run: | + $v = (Get-Content installer\VERSION -Raw).Trim() + if ($v -notmatch '^\d+\.\d+\.\d+$') { throw "installer\VERSION is not a three-part version: '$v'" } + "version=$v" >> $env:GITHUB_OUTPUT + $ref = "${{ github.ref }}" + if ($ref -like 'refs/tags/v*' -and $ref -ne "refs/tags/v$v") { + throw "tag $ref does not match installer\VERSION ($v): bump VERSION before tagging" + } + "installer\VERSION = $v" + + - name: Shared fragment matches the vendored multiplayer release + shell: pwsh + run: | + $rel = (Select-String -Path installer\vendor\VENDORED.md -Pattern '^source release:\s*(v\S+)').Matches[0].Groups[1].Value + if (-not $rel) { throw "installer\vendor\VENDORED.md names no source release" } + $url = "https://raw.githubusercontent.com/silver2127/tpf2-multiplayer/$rel/installer/PluginHost.wxs" + "comparing installer\PluginHost.wxs with $url" + $theirs = (Invoke-WebRequest -Uri $url -UseBasicParsing).Content -replace "`r`n", "`n" + $ours = [IO.File]::ReadAllText("installer\PluginHost.wxs") -replace "`r`n", "`n" + if ($ours -cne $theirs) { throw "installer\PluginHost.wxs differs from the tpf2-multiplayer copy at $rel -- the two packages must share it" } + "PluginHost.wxs matches tpf2-multiplayer $rel" + + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' + + - name: WiX Toolset v7 + run: | + dotnet tool install --global wix + wix --version + + - name: Build the MSI + run: powershell -ExecutionPolicy Bypass -File installer\build_msi.ps1 -AcceptWixEula -Validate + + - name: Checksums + shell: pwsh + working-directory: installer/out + run: | + $msi = "TpF2BigMaps-${{ steps.version.outputs.version }}.msi" + Get-FileHash $msi -Algorithm SHA256 | + ForEach-Object { "{0} {1}" -f $_.Hash.ToLower(), (Split-Path $_.Path -Leaf) } | + Set-Content SHA256SUMS.txt -Encoding ascii + Get-Content SHA256SUMS.txt + + - uses: actions/upload-artifact@v4 + with: + name: TpF2BigMaps-${{ github.sha }} + path: | + installer/out/TpF2BigMaps-${{ steps.version.outputs.version }}.msi + installer/out/SHA256SUMS.txt + if-no-files-found: error + + - name: Release + if: startsWith(github.ref, 'refs/tags/v') + uses: softprops/action-gh-release@v2 + with: + files: | + installer/out/TpF2BigMaps-${{ steps.version.outputs.version }}.msi + installer/out/SHA256SUMS.txt + draft: true + generate_release_notes: true diff --git a/README.md b/README.md index 285ef38..cff8daf 100644 --- a/README.md +++ b/README.md @@ -5,9 +5,35 @@ Maps larger than Transport Fever 2's New Game menu will build. **Native Linux:** an experimental port is available under `linux/`. See [Linux installation](docs/linux/INSTALL.md) and [port scope and evidence](docs/linux/PORT.md). It supports large-map controls through 512 tiles per edge and six sparse density -presets for towns and industries. Linux dev.3 adds opt-in lossless terrain RAM -compression, faster saves and an exact SIMD terrain scan. Material compression, -the remaining Windows speed optimizations and depth 12/13 are not yet included. +presets for towns and industries. Linux dev.3 adds lossless terrain RAM +compression, faster saves and an exact SIMD terrain scan. Linux dev.4 adds lazy zero terrain tiles, compressed-content deduplication and +travel-time controls. Alignment batching, block/material paging, adaptive memory +policy and depth 12/13 remain unported; see [integration evidence](docs/linux/PORT.md#windows-integration-bd0d85f-linux-dev4-partial). +The Windows integration through `9180629` adds native offline sidecar-format, +UTF-8 I/O and fingerprint-discovery validation; live sidecar serving and the +adaptive pager policy remain unported. See [current integration evidence](docs/linux/PORT.md#windows-integration-9180629-partial). +Integration `926113b` also ports graceful octree fallback: native requests for +`octree_depth=12`/`13` load the verified depth-11 root and retain the 512-tile +ceiling. This does not add larger-root support or make larger-root saves safe. +See [fallback evidence](docs/linux/PORT.md#windows-integration-926113b). +The Windows minimap from `cdfee2a` is not yet available on native Linux; +see the [integration evidence](docs/linux/PORT.md#minimap-integration-cdfee2a-partial). + +Compression is enabled by default as of integration `b755f65`; set +`terrain_cache_compress=0` to disable it. See [latest integration](docs/linux/PORT.md#windows-integration-b755f65-partial). + +The following upstream sync and feature descriptions concern the Windows branch; +this native branch retains the Linux scope described above. + +**Big Maps is developed in +[tpf2-multiplayer](https://github.com/silver2127/tpf2-multiplayer/tree/dev/bigmap), +under `bigmap/`, and this repository is kept in sync with it automatically.** Every +change made there lands here too, so this is the place to get Big Maps on its own, +without multiplayer. TpF2 Multiplayer 0.7 and later already include Big Maps. +Please open issues and pull requests against `dev` in tpf2-multiplayer: a change +made only here is overwritten by the next sync. The exception is the standalone +installer (`installer/`, `.github/`, `tools/vendor_host.ps1`, +`tools/test_config_msi.py`), which lives only in this repository. Experimental [generation performance modes](docs/generation-performance.md) add a configurable placement budget and conservative Desert terrain-buffer @@ -22,17 +48,32 @@ overflow above approximately 185 km separation. See sites and offline validation; an in-game regeneration check is still pending. A native plugin for the **tpf2mp plugin host**. It carries no multiplayer code -and has no build-time dependency on the host tree — only the vendored -`src/tpf2mp_plugin.h`, which is the whole ABI. +and has no build-time dependency on the host tree, only `src/tpf2mp_plugin.h`, +the whole ABI, which the sync copies from tpf2-multiplayer. -Target: **Transport Fever 2 build 35924** (Steam, 2024-12-11, the last release). -Every address here was measured on it, each site is byte-verified before it is -patched, and the plugin refuses to patch anything else. +Target: **Transport Fever 2 build 35924**, in both of its builds: Steam +(2024-12-11) and GOG (2024-12-12). Every address was measured on both, each site +is byte-verified before it is patched, and the plugin refuses to patch anything +else. -**GOG is not supported** (since 0.2.0). The installer refuses a GOG folder, and the -New Game menu rows and density levels exist only for the Steam binary. The GOG -addresses 0.1.1 added for the size ladder, street raster and octree are still in -`src/bigmap.cpp`, but nothing tests or maintains them. +The two builds share their code shape but not their addresses — the same +function sits at a different RVA in each — so the plugin keeps a pair of +constants per patched site and byte-verifies the one it is about to write. At +start it reports which build it found: + +``` +[host] tpf2_bigmap: game build is the GOG 2024-12-12 binary -- all three sites byte-verify; using the GOG layout +``` + +Some features remain Steam-only because their sites were never measured on the +GOG build, or because the replacement is a Steam code shape. They **degrade with +a log line** rather than failing the load: the density levels, the added size +dropdown rows, the placement/instance/material experiments, and the minimap. +`octree_depth=12`/`13` degrades the same way — see below. + +To install on GOG, use **`TpF2BigMaps--gog.msi`** instead of the +standard package: the standard one refuses a GOG game folder, because the +installer's folder check knows only the Steam executable. --- @@ -225,15 +266,34 @@ b2 0b mov dl, 0xb ; depth 11 ceiling remains **512 tiles (131 km)** with 128 m leaves. The default patch is applied only when the configuration asks for a size over 256 tiles. -**Experimental actual depths 12 and 13 are available.** Set `octree_depth=13` +**Actual depths 12 and 13 are available, and tested.** Set `octree_depth=13` and `max_tiles=2048` for **2,048-tile (524.288 km) edge capacity**, or depth 12 and `max_tiles=1024` for 262.144 km. Both retain **128 m leaves**. The patch assigns compact IDs to levels 11/12 and updates the renderer's level decoder. It is byte-verified and tested offline against original engine insertion -instructions, but **not yet validated in a running game**. Heightmap area -limits still apply, so the longest maps must be narrow. See -[the implementation and test notes](docs/octree-depth12.md) for configuration, -evidence and remaining live checks. Defaults retain depth 11. +instructions, and the higher depths have been **tested in play and work fine**: +large maps at depth 13 created, played, saved and reloaded on Steam, in single +player and in multiplayer (Windows players with the native Linux dedicated +server, which supports depths 12/13 since 2026-09-23), with no duplicate street +nodes and no assertion. On GOG the fallback below was validated in a running +game (a 57 x 57 km map created, played and reloaded). Heightmap area limits +still apply, so the longest maps must be narrow. Every multiplayer peer needs +the same `octree_depth`. See +[the implementation and test notes](docs/octree-depth12.md) for configuration +and evidence. + +**Steam 35924 only.** Depths 12 and 13 replace two prologues the GOG build does +not share, so there `octree_depth=12`/`13` fall back to the depth-11 root +instead of refusing to load: the plugin still loads, the ceiling stays 512 tiles, +and the log says which depth was asked for and what went in. + +``` +[host] tpf2_bigmap: octree: octree_depth=13 is Steam 35924 only -- this build keeps depth 11 and loads, + so the edge ceiling stays 512 tiles, not 2048 +``` + +That fallback is why the shipped config can keep `octree_depth=13` for both +builds: Steam gets 2,048-tile edges, GOG gets the valid 512-tile root and works. Terrain LOD at the edge was **not** traced to the same limit. The only terrain-side 32,768 is an asymmetric legacy vertex packer (tiles −128..895), @@ -463,9 +523,13 @@ stock. **Download `TpF2BigMaps-.msi` from the [latest release](https://github.com/silver2127/tpf2-bigmap/releases) and run it.** -It finds the Transport Fever 2 folder Steam registered, asks you to confirm it, +It finds the Transport Fever 2 folder (from Steam's own uninstall entry, or from +the folder a previous install remembered), asks you to confirm it, and puts these in place: +If you play with TpF2 Multiplayer 0.7 or later, you already have Big Maps: do not +install this package as well. Installing TpF2 Multiplayer removes it. + | file | what | | --- | --- | | `alut.dll` | the proxy the game loads in place of its own (the original is kept as `alut_real.dll`) | @@ -482,6 +546,19 @@ value, removed on uninstall) — that is what makes a big map load in about a minute instead of a quarter of an hour; the measurements are in the [multiplayer installer README](https://github.com/silver2127/tpf2-multiplayer/blob/main/installer/README.md#segment-heap). +### GOG + +Use **`TpF2BigMaps--gog.msi`**. It is the same package with one +difference — it tells the folder check that the GOG build of the game is a +supported target — so it validates the folder exactly like the standard one +instead of refusing it. A standard package can do the same from the command line +with `TPF2_ALLOW_GOG=1`; without it a GOG folder is refused, because the check +knows only the Steam executable and would rather say so than install into a game +it cannot patch. + +On GOG the folder dialog starts on the Steam default, which does not exist +there: point it at the game folder once, and every later install remembers it. + Then: **New Game**. The size dropdown has rows after the stock sizes, from 32 x 32 km up to 128 x 128 km, with *experimental map sizes* on or off, and *Towns*, *Number of industries* and *Industry density target* have six more levels @@ -492,25 +569,32 @@ the way it shapes the stock ones. To set a shape yourself, add a `octree=1`, and the area within the street-raster budget (`street_raster=1` scales the cell to keep it there). -### Installing alongside TpF2 Multiplayer - -Both packages work in either order and can be removed in either order. They -share the proxy and the plugin host, and both installers declare those under -the **same component GUIDs** (`installer/PluginHost.wxs`, byte-identical in both -repositories), so Windows Installer reference-counts them: the second install -finds them present, the first uninstall leaves them for the other, and only the -last one out puts the game's own `alut.dll` back. The custom actions that park -and restore `alut.dll` check that count too, so uninstalling one product never -restores the stock library out from under the other. - -Each product keeps its own config — this one in `plugins\tpf2_bigmap.cfg`, which -the host merges over `tpf2mp.cfg` — so neither installer touches a file the -other owns. - -`installer\test_coexist.ps1` proves all of it against a throwaway folder with -the real `msiexec` transactions (both orders, both directions, the shared -registry value tracked and restored). It needs an elevated PowerShell because -the packages are per-machine. +### With TpF2 Multiplayer + +TpF2 Multiplayer 0.7 and later ship Big Maps themselves and remove this package +when they install (its UpgradeCode is listed in their MSI), so the plugin has one +owner. Before 0.7 the two packages coexisted: they shared the proxy and the +plugin host under the **same component GUIDs** (`installer/PluginHost.wxs`), so +Windows Installer reference-counted them and only the last one out put the game's +own `alut.dll` back. `installer\test_coexist.ps1` proves that against a +throwaway folder with the real `msiexec` transactions (it needs an elevated +PowerShell, because the packages are per-machine). + +### Virus-scanner findings + +The DLLs and the MSI are not code-signed, so any rule of the form *unsigned +image loaded* matches them wherever the game loads them. One report (a mod site's +sandbox scan, 2026-09-20) went further and matched *Unsigned Image Loaded Into +LSASS Process*: its event named `C:\p250ps71\dll\OrqiEaoo.dll`, a randomly +named file that is the sandbox's own monitoring hook, injected into every +process including `lsass.exe`. Nothing here loads into any process but the +game: the installer places the files in the table above and two registry +strings under `HKLM\SOFTWARE\silver2127\TpF2 Big Maps`, registers no service, +no security package, no `AppInit_DLLs`, and the plugin sets no hooks and opens +no other process. The one custom action, on full uninstall only, runs +`rundll32` on the plugin to put the stock `base_mod.lua` back. Compare the +SHA-256 in such a report with the release assets' digests on the GitHub +release page; they will not match. ### Uninstall @@ -557,6 +641,14 @@ checkout's build outputs instead (dev only). `build_msi.ps1` refuses to build if [OSMF EULA](https://wixtoolset.org/osmf/); `-AcceptWixEula` passes it per-invocation and nothing accepts it for you. +GitHub Actions runs the same script (`.github/workflows/build-msi.yml`): every push to `dev` or `main` and +every pull request builds `TpF2BigMaps-.msi` and `SHA256SUMS.txt` on a `windows-2022` runner from the +vendored shared binaries and keeps them as the run's artifact; a `v*` tag (which must equal `installer\VERSION`) +also creates a draft GitHub release with them attached, ready to be edited and published. The runner has no +multiplayer checkout beside this one, so the workflow compares `PluginHost.wxs` with the copy at the release +named in `installer\vendor\VENDORED.md` instead. The workflow passes `-AcceptWixEula`, which is the +repository owner accepting the WiX terms for those builds. + ## Verifying it worked `%LOCALAPPDATA%\tpf2mp\data\tpf2mp_host.log` shows the hook lines, the @@ -567,42 +659,139 @@ and read `numTilesX`/`numTilesY` back out of the `.sav` header with the snippet near the top of this file — that proves the value survived generation *and* serialization, which is stronger than trusting the log. +The same log carries the pagers' `resident target` lines: one at start with the +auto budgets for this machine's RAM, a few during the load, and then quiet. See +*Reading the log* under Memory. + ## Extended map ratios New Game ratios can now extend through **1:20** with `max_ratio=20` (Steam). Both stock and added size rows are supported; map-edge and heightmap limits still apply. See [map-ratios.md](docs/map-ratios.md) for dimensions and validation. -## Experimental gameplay RAM reduction - -**Lossless 1 m cache compression is implemented separately**: set -`terrain_cache_spacing_m=1`, `terrain_cache_compress=1` and optionally -`terrain_cache_hot_mb=1024`, then restart. `terrain_cache_warm_mb=4096` provides -a temporary larger resident allowance during generation/bulk allocation. -Version 2 improves lossless encoding, reuses unchanged compressed tiles, and -shares compressed COW copies. Native ownership/concurrency tests, world loading -and two connected rail builds followed by save/reload pass. Runtime terrain backing settled near -2.55 GiB on the 114x570-tile world. It preserves every sample and the octree. -See [terrain-compression.md](docs/terrain-compression.md) for measured codec -results, the fixed-address paging mechanism and validation status. - -The renderer upload mismatch is patched: 131x131 buffered CPU samples are -expanded temporarily to the stock 259x259 GPU upload, including borders. -**The 2 m experiment is discontinued after repeated terrain fragments, large -tile seams and a rail construction crash. Keep the active setting at 1 m.** -An alignment-coordinate bridge was deployed; the subsequent construction -prototype was only built and unit-tested, and is not a validated gameplay fix. - -`terrain_cache_spacing_m=2` uses a 2 m derived terrain height cache when creating -or loading a world. On the measured 114x570-tile map, the calculated cache -payload saving is 5.98 GiB. The 4 m source heightmap and octree depth remain -unchanged; fine terrain alignment and deformation become coarser. - -Use `1` and reload to restore 1 m caches. `0` preserves the saved/default -resolution. Steam 35924 only, experimental; full gameplay verification is -pending. See [gameplay-memory.md](docs/gameplay-memory.md) for evidence, tests -and rollback. Build and run `python tools/test_terrain_cache.py` to check the -hook and original game allocation instructions without modifying the game. +## Performance optimizations + +A big map strains the stock engine in two ways. It runs out of memory, because +per-tile structures are sized for maps a tenth as large. It also loads slowly, +because the load re-derives the whole terrain from scratch. Everything below is +switched in `plugins\tpf2_bigmap.cfg` (restart to apply; `0` restores stock). +Every item is **lossless**: it produces the same bytes the stock code would, +and most are checked bit-for-bit against the game's own machine code by a test +in `tools\`. So none of them can desync a multiplayer session, and peers with +different settings stay in sync. The one exception, `placement_attempts`, only +changes how a *new* map is generated. Switches the cfg marks "Steam 35924" keep +the stock code on any other build. + +### Memory + +| Optimization | Switch (shipped) | How it works | +| --- | --- | --- | +| Terrain compression pager | `terrain_cache_compress=1` | The 1 m height cache (a 257x257 block per tile, kept for the whole session) goes through a pager. Tiles near the camera stay raw. Cold tiles are compressed with planar prediction (left + up - upper-left), zigzag residuals and a per-tile rANS coder, to about 7% of their raw size. They are decommitted, and decoded back **at the same address** when the engine touches them (about 0.2 ms a tile), so the engine never knows. Budgets adapt to the machine's RAM: see the next section. On native Linux the same pager runs on userfaultfd. [terrain-compression.md](docs/terrain-compression.md) | +| Material-cell paging | `material_cache_compress=1` | The renderer keeps a 67,601-byte material-index cell per tile forever (4.13 GiB on a 256x256 map). Cold cells are held compressed (about 20% of raw) and restored in place on access. [material-grid-lifetime.md](docs/material-grid-lifetime.md) | +| Alignment batching | `alignment_batch_tiles=512` | On a save load the engine cuts every road, track and construction into the terrain of the *whole map* in one call. It holds millions of intermediate blocks until the end, which was an "Out of memory" assert on a 94 GiB machine. The plugin splits that call into batches of N tiles (compute, publish, free), which is what the engine already does frame by frame in play. Peak private memory on a 207,360-tile save went from **35 GiB to 8.3 GiB**. [alignment-batch.md](docs/alignment-batch.md) | +| Lazy zero | `terrain_lazy_zero=1` | A load allocates every tile of both terrain versions up front, all zero, then fills them over 20-30 s. Backing each allocation immediately committed up to 27 GiB before any terrain existed. Now a tile gets its memory on first touch, so commit follows the fill. | +| Tile dedup | `terrain_dedup=1` | During a load both terrain versions exist, and every tile has exactly one byte-identical twin in the other (measured on a 256x256 save). When an evicted tile matches a stored blob (two independent 64-bit hashes must agree), it shares that blob. The second twin costs a hash instead of an encode and is stored once. | +| Instance-list shrink | `instance_shrink=1` | While a new world is created, the tree and scenery instance lists of each 64 m cell are trimmed to their size, dropping the growth slack the game would otherwise keep all session. Contents and order are unchanged; it uses the game's own allocator. | +| Generation buffer reuse | `python tools\install_generation_memory.py` | This one is opt-in and a Lua pass rather than a switch. It rewrites the stock New Game terrain generators so that temporary full-map float buffers whose lifetimes do not overlap share storage. The pass examines the completed op list and leaves op order, parameters and seeds alone. Desert goes from 18 buffers to 15 and Temperate from 10 to 9, which is 4 GiB per buffer at 228 x 1140 tiles. `--restore` undoes it. [generation-performance.md](docs/generation-performance.md) | + +### Load time and CPU + +| Optimization | Switch (shipped) | How it works | +| --- | --- | --- | +| Terrain sidecar | `terrain_sidecar=1` | Every save (manual and autosave) also writes `.terr`, the finished, aligned 1 m height cache, fingerprinted with the `.sav`'s hash. Loading that same save decodes each tile from the sidecar the moment the engine creates it, and skips both the refine and the alignment cut for it. On a 256x256 save that skips two alignment passes of about 14 s each. A missing or non-matching sidecar just loads stock. [terrain-sidecar.md](docs/terrain-sidecar.md) | +| SSE2 terrain refine | `terrain_refine_fast=1` | This is the bicubic refine from the 4 m base heightmap to the 1 m cache, which runs for every tile on load and after every terrain edit. It is rewritten with SSE2 and gives the same bits: about **3.4x** faster (58 to 17 ns a sample). [terrain-refine.md](docs/terrain-refine.md) | +| SSE2 min/max scan | `terrain_minmax_fast=1` | The per-tile `CalcMinMaxHeight` scan during terrain publication becomes an 8-lane SSE2 reduction, and the height-block copy becomes one `memcpy` a row. The scan is about **24x** faster. [terrain-minmax.md](docs/terrain-minmax.md) | +| Faster alignment blend | `terrain_align_fast=1` | `CalculateHeightMod`, which re-applies road/track/construction cuts to a height block, gets a pooled scratch buffer instead of an allocation per call and an SSE2 blend. It is **2.6x to 4.7x** faster per block. [terrain-alignment-speed.md](docs/terrain-alignment-speed.md) | +| Material-index loop | `material_index_fast=1` | This was the largest single game-code hotspot in a world-entry profile. Stock walks a tile region repeatedly, eight material layers per pass. The replacement finishes all layers for one pixel before moving on, so it computes the interpolation coordinates and dither thresholds once. Layer priority, overlap and float order are preserved, and odd geometry falls back to stock. [world-entry-performance.md](docs/world-entry-performance.md) | +| Fewer placement attempts | `placement_attempts=50` | The town/industry placement worker's inner optimisation budget goes from 200 tries to 50 (75% fewer). Spacing, slope, water checks and requested counts are unchanged. Placement can come out slightly less even. This affects only new-map generation, never a loaded save. [generation-performance.md](docs/generation-performance.md) | +| Faster saves | `save_fast=1` | zstd level 1 instead of 3, and a 64 KiB save input buffer instead of 128 bytes. Compression is **2.79x** faster for files about 9% larger; the format is unchanged. [save-performance.md](docs/save-performance.md) | +| Native minimap | `minimap=1` | The minimap's terrain picture is rendered natively and handed to the game as one texture. Its cost is the picture's pixel count, not the map size: there is no UI widget per cell and no Lua height sampling (the Workshop minimap needed ~43 million Lua calls at 256x256). [minimap.md](docs/minimap.md) | + +`world_entry_timings=1` logs the time and memory of each world-entry stage to +`tpf2mp_host.log`, which is how the items above were found and measured. + +Retired or held back: `terrain_blocks` paged the alignment blocks through a +small pager and is superseded by batching (off; kept as the fallback). The 2 m +derived cache (`terrain_cache_spacing_m=2`) was discontinued after terrain +seams and a crash. `terrain_cow_share` is still experimental and off. + +## Memory: what a big map costs, and how the plugin keeps it in check + +**What the game itself needs.** Most of a big map's memory is the engine's own +data, and no setting here changes that. Measured 2026-09-20: a freshly generated +large stock map was **15 GiB private** at the end of generation before the +plugin's caches held a byte, and a 44 MB save of a mid-sized world loaded to +about 6.6 GiB. A 16 GiB machine cannot hold a 128 x 128 world however the caches +are set; a 32 GiB one holds it at around 20 GB. + +**What the plugin adds, and controls.** The 1 m terrain cache and the material +grid are the two engine structures that scale with the map (see +[terrain-compression.md](docs/terrain-compression.md) and +[material-grid-lifetime.md](docs/material-grid-lifetime.md)). The plugin routes both through +pagers that keep a *resident target* of tiles uncompressed and hold the rest +compressed, losslessly, decoding a tile when the engine touches it (about +0.2 ms each). Shipped on since 0.3.1. Once a second each pager sets its target +from four rules, in this order: + +1. **Loading allowance.** While a world generates or loads, everything live may + stay resident up to what free RAM and free commit allow, less the machine's + headroom (`PagerHeadroom`: a seventh of RAM, 2..12 GiB). The loader re-reads + what was just evicted, so evicting during a load only makes it slower. +2. **Steady state.** Afterwards the target ramps down by 1/16 a second toward + the *hot* budget, grows by 1/8 when the engine faults 300 or more evicted + tiles back in per second (`cold restores/s` in the log), and holds between. + Since 2026-09-20 the target the stutter feedback drove it to is kept as a + **working-set floor** that decays by 1/256 per quiet second (halves in about + three minutes) instead of sawtoothing back to hot every second the camera + rests -- on a 32 GiB machine that sawtooth was 1,000-2,000 decodes a second. +3. **The cap.** `terrain_cache_max_mb` bounds the target: **0 = automatic, a + quarter of RAM clamped to 4096..8192 MiB** (16 GiB: 4096; 32 GiB and up: + 8192), the same size on every machine that can afford it; `-1` removes it, + and the target then fills RAM down to the headroom, which is smoothest and + is what made the game 20 GB on a 32 GiB machine. `material_cache_max_mb` + is the same for material cells, automatic = a quarter of the terrain cap. A + cap below the map's working set is paid in decodes: at 300 or more + `cold restores/s` it is costing frames (a 4096 cap did, on a freshly + generated large map). +4. **Commit pressure.** When free commit (RAM plus page file) drops under + `CommitTightBytes` (an eighth of RAM, 2..10 GiB) both pagers throttle to + 256 MiB and evict urgently; when free RAM drops under the headroom they + shrink by 1/8 a second. The throttle is **sticky**: it holds until free commit + clears the threshold by more than the pager itself gives back, and for at + least 30 s. Without that, on a machine with no page file sitting 1 GiB under + the threshold, the pager's own 3 GiB release cleared it, it re-expanded, and + the flag set again -- 74 flips in one session, each re-inflating the terrain + in front of the camera: a stutter at every close zoom (2026-09-20). + +**Budgets and knobs** (`plugins\tpf2_bigmap.cfg`, restart to apply): + +| key | default | meaning | +| --- | --- | --- | +| `terrain_cache_hot_mb` | `0` = auto: RAM/30, 256..4096 | the steady-state floor (94 GiB: 3195; 32 GiB: 1092) | +| `terrain_cache_warm_mb` | `-1` = auto: RAM/12, up to 8192 | the allowance while loading | +| `terrain_cache_max_mb` | `0` = auto: RAM/4, 4096..8192 | the cap; `-1` = none | +| `material_cache_hot_mb` / `_warm_mb` / `_max_mb` | auto | the same three for material cells (hot RAM/180, 96..1024; warm RAM/48, up to 4096; cap a quarter of the terrain cap) | +| `terrain_cache_evict_per_s` | `4000` | eviction ceiling at rest; `0` = unlimited | +| `simulate_physical_mb` | `0` = off | **rig-only**: the policy sizes itself for a machine of this RAM, free RAM and free commit scaled to the real machine's fractions; the engine is not constrained. `32768` tests the 32 GiB sizing on a bigger PC | + +**Reading the log** (`%LOCALAPPDATA%\tpf2mp\data\tpf2mp_host.log`): a +`resident target A -> B MiB (... commit_tight=N pressure=N free=M MiB)` line +per change of target (a handful per session is normal; dozens means the +throttle is flapping), `N cold restores/s, resident target A -> B MiB (working +set exceeds the budget)` when the engine is re-reading evicted tiles, and the +`world entry:` lines with the process's private and resident size at each +stage of a load. + +**If the game is bigger than you want.** Set `terrain_cache_max_mb` lower and +watch `cold restores/s`; below 100 the cap is free, above 300 you are paying +frames for RAM. The rest is the map: pick a smaller size. + +The experiments this replaced are kept for the record in +[gameplay-memory.md](docs/gameplay-memory.md): the 2 m derived cache +(`terrain_cache_spacing_m=2`, discontinued after terrain fragments, tile seams +and a rail construction crash -- keep `1`), and the early fixed budgets. The +renderer upload mismatch fix (131x131 buffered CPU samples expanded to the +stock 259x259 GPU upload) is part of the 1 m cache and stays. ## Faster autosaves and manual saves @@ -614,6 +803,17 @@ Save contents and format are unchanged. Set `0` and restart to restore stock compression. See [save-performance.md](docs/save-performance.md) for validation and limitations. +## Travel-time limits + +`travel_time_limit_s` and `cargo_path_time_s` rewrite the two `.rdata` cells +that hold the urbansim's limits (Steam 35924): 1200 s, the 20 minutes at 1x +within which a cargo type must be able to reach a consumer's station (the same +cell bounds people's paths and destination choice), and 6000 s, the longest path +the path builder gives a cargo item. Values are game seconds, 0 keeps stock, +others clamp to 60..86400. Byte-verified; `python tools\test_travel_time.py`. +Shipped off: on a big map, set `travel_time_limit_s=3600` to let demand reach +towns an hour apart. + ## Licence MIT. See `LICENSE`. diff --git a/build.bat b/build.bat index bc261a6..80c0a64 100644 --- a/build.bat +++ b/build.bat @@ -82,6 +82,22 @@ if /i "%1"=="-decode-exp" ( cl /nologo /O2 /MT /W3 /EHsc /FAs /Faout\decode_experiments.asm tools\decode_experiments.cpp /Fo:out\decode_experiments.obj /Fe:out\decode_experiments.exe || exit /b 1 exit /b ) +if /i "%1"=="-alignbatch-test" ( + cl /nologo /O2 /MT /W3 /EHsc tools\test_alignment_batch.cpp /Fo:out\test_alignment_batch.obj /Fe:out\test_alignment_batch.exe || exit /b 1 + exit /b +) +if /i "%1"=="-serve-test" ( + cl /nologo /O2 /MT /W3 /EHsc tools\test_terrain_serve.cpp /Fo:out\test_terrain_serve.obj /Fe:out\test_terrain_serve.exe || exit /b 1 + exit /b +) +if /i "%1"=="-sidecar-test" ( + cl /nologo /O2 /MT /W3 /EHsc tools\test_terrain_sidecar.cpp /Fo:out\test_terrain_sidecar.obj /Fe:out\test_terrain_sidecar.exe || exit /b 1 + exit /b +) +if /i "%1"=="-smallpager-test" ( + cl /nologo /O2 /MT /W3 /EHsc tools\test_small_pager.cpp /Fo:out\test_small_pager.obj /Fe:out\test_small_pager.exe || exit /b 1 + exit /b +) if /i "%1"=="-matpager-test" ( cl /nologo /O2 /MT /W3 /EHsc tools\test_material_pager.cpp /Fo:out\test_material_pager.obj /Fe:out\test_material_pager.exe || exit /b 1 exit /b @@ -95,18 +111,70 @@ if /i "%1"=="-codec-bench" ( exit /b ) +REM The minimap game script travels inside the DLL (src\minimap.h writes it out). +powershell -NoProfile -ExecutionPolicy Bypass -File tools\embed_lua.ps1 -In mod\minimap\bigmap_minimap.lua -Out out\minimap_lua.inc -Name kMinimapLua || exit /b 1 cl /nologo /O2 /MT /W3 /EHsc /c src\bigmap.cpp /Fo:out\bigmap.obj || exit /b 1 link /nologo /DLL /MAP:out\tpf2_bigmap.map /OUT:out\tpf2_bigmap.dll out\bigmap.obj || exit /b 1 echo BUILD BIGMAP OK -if /i "%1"=="-deploy" ( - if not exist "%LOCALAPPDATA%\tpf2mp\data\plugins" mkdir "%LOCALAPPDATA%\tpf2mp\data\plugins" - copy /y out\tpf2_bigmap.dll "%LOCALAPPDATA%\tpf2mp\data\plugins\" >nul ^ - && echo DEPLOYED to %LOCALAPPDATA%\tpf2mp\data\plugins ^ - || echo DEPLOY SKIPPED ^(dll locked by a running game^) - REM A datadir deploy must carry the cfg too: the plugin host scans the - REM datadir plugins folder first, and a plugin there with no cfg next to it - REM silently runs on built-in defaults (no size ladder, no octree). - if exist cfg\tpf2_bigmap.cfg copy /y cfg\tpf2_bigmap.cfg "%LOCALAPPDATA%\tpf2mp\data\plugins\" >nul -) +if /i "%1"=="-deploy" call :deploy %2 || exit /b 1 endlocal +exit /b 0 + +REM ---- -deploy: one install, not two --------------------------------------- +REM The plugin host scans the datadir FIRST, so a datadir copy silently shadows a +REM game-folder one -- and the two drift: the datadir gets this build and the repo +REM cfg, while the game folder keeps whatever the user edited. A subroutine, not +REM a parenthesised block: cmd expands a block before it runs, so %GAMEDIR% set +REM inside it read as nothing, and a game folder under "Program Files (x86)" +REM closed the block at its ")" ("...Transport was unexpected at this time") -- +REM even on a plain build that never asked to deploy. +:deploy +call :find_game_dir +if not defined GAMEDIR goto :deploy_copy +if not exist "%GAMEDIR%\plugins\tpf2_bigmap.dll" goto :deploy_copy +echo. +echo DEPLOY REFUSED: a game-folder install already exists: +echo %GAMEDIR%\plugins\tpf2_bigmap.dll +echo The plugin host scans %%LOCALAPPDATA%%\tpf2mp\data\plugins FIRST, so a +echo datadir copy would shadow it and run against the repo cfg, not yours. +echo Remove one of the two installs, or pass -deploy-force to overwrite anyway. +echo. +if /i not "%~1"=="-deploy-force" exit /b 1 +:deploy_copy +if not exist "%LOCALAPPDATA%\tpf2mp\data\plugins" mkdir "%LOCALAPPDATA%\tpf2mp\data\plugins" +copy /y out\tpf2_bigmap.dll "%LOCALAPPDATA%\tpf2mp\data\plugins\" >nul && echo DEPLOYED to %LOCALAPPDATA%\tpf2mp\data\plugins || echo DEPLOY SKIPPED: dll locked by a running game +REM A datadir deploy must carry the cfg too: the plugin host scans the datadir +REM plugins folder first, and a plugin there with no cfg next to it silently runs +REM on built-in defaults (no size ladder, no octree). +if exist cfg\tpf2_bigmap.cfg copy /y cfg\tpf2_bigmap.cfg "%LOCALAPPDATA%\tpf2mp\data\plugins\" >nul +exit /b 0 + +REM ---- locate the game folder, for the -deploy shadow check ---------------- +REM A subroutine, not inline: inside a parenthesised block cmd expands the whole +REM block before it runs, so `if not defined GAMEDIR` would keep testing the +REM pre-loop value and the last registry hit would win, not the first. +:find_game_dir +set "GAMEDIR=" +if defined TPF2_GAMEDIR if exist "%TPF2_GAMEDIR%\TransportFever2.exe" set "GAMEDIR=%TPF2_GAMEDIR%" +if defined GAMEDIR goto :eof +for /f "tokens=2,*" %%a in ('reg query "HKLM\SOFTWARE\WOW6432Node\GOG.com\Games" /s /v path 2^>nul ^| findstr /i "path"') do ( + if not defined GAMEDIR if exist "%%b\TransportFever2.exe" set "GAMEDIR=%%b" +) +if defined GAMEDIR goto :eof +for /f "tokens=2,*" %%a in ('reg query "HKLM\SOFTWARE\GOG.com\Games" /s /v path 2^>nul ^| findstr /i "path"') do ( + if not defined GAMEDIR if exist "%%b\TransportFever2.exe" set "GAMEDIR=%%b" +) +if defined GAMEDIR goto :eof +REM %ProgramFiles(x86)% must not appear inside the parenthesised list: its ")" +REM closes the list early ("\Steam\steamapps\common\Transport was unexpected at +REM this time"), which failed every build on a machine where no GOG registry +REM entry answered first. Expanded into a variable before the loop instead. +set "PF86=%ProgramFiles(x86)%" +for %%d in ( + "C:\Games\Transport Fever 2" + "%PF86%\Steam\steamapps\common\Transport Fever 2" + "%ProgramFiles%\Steam\steamapps\common\Transport Fever 2" + "%PF86%\GOG Galaxy\Games\Transport Fever 2" +) do if not defined GAMEDIR if exist "%%~d\TransportFever2.exe" set "GAMEDIR=%%~d" +goto :eof diff --git a/cfg/tpf2_bigmap.cfg b/cfg/tpf2_bigmap.cfg index 7dbc6e4..941f14f 100644 --- a/cfg/tpf2_bigmap.cfg +++ b/cfg/tpf2_bigmap.cfg @@ -56,6 +56,17 @@ terrain_align_fast=1 # order are unchanged. Saved games load through another path. 0 = stock. instance_shrink=1 +# Steam 35924, experimental: an in-game minimap that works on the largest maps. +# Adds a map button to the main toolbar through a GUI-only game script that the +# plugin writes to res\config\game_script\bigmap_minimap.lua (and deletes again +# when this is 0 or the plugin is uninstalled). The terrain picture is rendered +# natively and handed to the game as a texture, so its cost is the picture's +# pixel count, not the map size: no widget per pixel and no line limit. Towns +# and industries are clickable; click the map to move the camera. Pure UI: it +# cannot affect the simulation, saves or multiplayer. On since 0.5.2; 0 = no +# minimap. +minimap=1 + # enabled=0 stops the host loading the DLL at all (checked before LoadLibrary, # so a disabled plugin's DllMain never runs). enabled=1 @@ -284,7 +295,7 @@ tiles_y=0 # ever runs -- so this is the only thing standing between a fat-fingered config # and a map that eats the machine. # -# 512 is the default octree ceiling. Experimental octree_depth=12 allows +# 512 is the default octree ceiling. octree_depth=12 allows # 1024-tile edges; octree_depth=13 allows 2048-tile edges, both retaining # 128 m leaf cells. Heightmap pixel products # above INT_MAX are reduced (a 1024-square map does not fit). WITHOUT @@ -344,15 +355,18 @@ cell_budget_millions=1500 # file, and the game's own load-time merge fails on them. Regenerate. octree=1 -# EXPERIMENTAL: 12 enables actual depth 12 and a +-131072 m root; +# 12 enables actual depth 12 and a +-131072 m root; # 13 enables depth 13 and +-262144 m. Both keep 128 m leaves. # Edge capacities: 1024 tiles / 262.144 km, or 2048 tiles / 524.288 km. -# Steam 35924 only. Requires max_tiles=1024 or 2048, respectively, -# and a larger size_format entry to offer larger +# Steam 35924 only. On the GOG build neither prologue is at a verified address, +# so 12 and 13 degrade to the depth-11 root: the plugin still loads, the ceiling +# stays 512 tiles, and the log says so. Requires max_tiles=1024 or 2048, +# respectively, and a larger size_format entry to offer larger # maps. This option applies at load even if the menu's configured sizes are small. # Keep it enabled whenever loading a world made for the larger root; multiplayer -# participants need matching patches/settings. Save/load and live rendering of -# larger worlds are NOT yet validated. See docs/octree-depth12.md. +# participants need the same octree_depth. Depths 12/13 are tested in play +# (create, play, save/reload, multiplayer incl. the native Linux dedicated +# server) and work fine. See docs/octree-depth12.md. # 11 retains the existing behavior. Only 11, 12 and 13 are accepted. octree_depth=13 @@ -373,12 +387,39 @@ terrain_cache_compress=1 # clamped 256..4096 MiB; warm ~RAM/12, clamped to hot..8192 MiB; the material # pager uses RAM/180 and RAM/48). Explicit values override. warm_mb 0 disables # the loading allowance. +# Once a world is loaded the target does not snap to hot_mb: it drops by at +# most 1/16 per second, holds or grows while the engine keeps faulting +# evicted tiles back in, and drifts back toward hot_mb when the working set +# fits. The ceiling is hot_mb plus 3/4 of the room (free RAM plus what the +# pager holds) above a reserve of a quarter of that room or the machine's +# headroom (RAM/7, 2..12 GiB), whichever is larger; free RAM under the +# headroom shrinks the target by 1/8 per second. hot_mb is the floor. terrain_cache_hot_mb=0 # Temporary resident target during generation or bulk tile allocation (0..8192). # Kept through late loading until a fresh gameplay UI frame (matching menu DLL). # Old menu DLL fallback: 3 minute tail. Disabled below 8 GiB available RAM. # Set 0 to disable the temporary allowance. terrain_cache_warm_mb=-1 +# Cap on the resident 1 m tile target, in MiB. 0 = automatic: a quarter of RAM +# clamped to 4096..8192 (16 GiB: 4096; 32 GiB and up: 8192), the same size on +# every machine that can afford it. -1 = no cap: the policy above then fills +# RAM down to the machine's headroom while the engine keeps re-reading tiles. +# A cap is paid with decodes (about 0.2 ms each); watch `cold restores/s` in +# the log, at 300 or more it is costing frames (a 4096 cap did, on a freshly +# generated big map, 2026-09-20). Most of a big map's memory is the engine's +# own -- 15 GiB before the pager held a byte -- and no cap here changes that. +# hot_mb stays the floor. +terrain_cache_max_mb=0 +# Ceiling on evictions per second while nothing is loading and the commit +# charge is not tight (0 = unlimited). Each eviction is a protection change, +# an encode and an unmap; draining a loaded world at several thousand per +# second made the game stutter even from background threads (measured +# 2026-09-17). The rate actually used adapts below this ceiling: evictions +# get a quarter of one core (their measured cost sets the cap, so a slow +# machine evicts less and a fast one more), and any gameplay frame of 60 ms +# or more halves the rate for a while (needs the multiplayer menu DLL for +# the frame signal). Loading bursts and a tight commit charge ignore it. +terrain_cache_evict_per_s=4000 # EXPERIMENTAL, MEASUREMENT STAGE: share the height-cache section between the # two CTerrain versions instead of copying it. Loading a save briefly holds both @@ -395,6 +436,57 @@ terrain_cache_warm_mb=-1 # version is released. 0 = the stock eager copy. terrain_cow_share=0 +# MEASUREMENT ONLY: hash every live terrain tile and log how many are +# byte-identical to another (every 10 s while loading, every 2 min otherwise). +# Decides whether content dedup between the two CTerrain versions of a save +# load is worth building: both allocate 65,536 tiles each on a 256x256 map from +# the same save data. Costs a few seconds of one background thread per probe; +# changes nothing. Read the "terrain dedup probe:" lines in tpf2mp_host.log. +terrain_dedup_probe=0 + +# Content dedup of terrain tiles (Steam 35924, needs terrain_cache_compress=1). +# MEASURED 2026-09-17 on a 256x256 save load: for the whole time both CTerrain +# versions exist, every one of the 131,072 tiles has exactly one byte-identical +# twin in the other version (64,922 pairs at 131,072 live, 719 still unfilled), +# and after the load the surviving 65,536 are all distinct. With this on, an +# eviction whose bytes match a blob already stored shares that blob instead of +# encoding it again: the second twin of each pair costs a hash instead of an +# encode, and the compressed copy is kept once. Two independent 64-bit hashes +# must agree before anything is shared. 0 = every eviction encodes. +terrain_dedup=1 + +# Fresh terrain tiles start without memory behind them. Measured 2026-09-17: +# a save load allocates every tile of both versions first, all zero (112,678 +# of 131,072 still zero at one probe), and fills them over the next 20-30 s; +# a section per allocation committed up to 27 GiB before any terrain existed, +# faster than eviction could shed it, and the commit charge hit the limit +# (the game's own "Out of memory" assert). With this on a tile gets its +# section on the first touch instead, so commit follows the fill. 0 = a +# section per allocation, as before. +terrain_lazy_zero=1 + +# The terrain alignment pass at load computes a 257x257 work block for every +# tile and keeps all of them until the publication pass has copied each into +# its tile (measured 2026-09-17 with an allocation trace: 99,014 blocks, +# 19.2 GiB, plus 37,354 alignment result vectors, 6.7 GiB, all alive at the +# moment the game ran out of memory). With this on both go through the +# small pager (page-granular spans: reserved on allocation, committed on the +# first touch, compressed and decommitted when over budget, restored on the +# next access), and their release is caught at the CRT free import. Measured +# 2026-09-17 on a second map (LONGBOI): 3.3 million result vectors and ~10 +# million work vectors of ~16 KB, the game's own 34 GiB private peak. +# Needs terrain_cache_compress=1. Superseded by alignment_batch_tiles below, +# which keeps the live set to one batch and makes this cost only (three +# system calls per block, ~13 million blocks on that map): off by default, +# kept as the fallback for a build where batching is off. 0 = heap. +terrain_blocks=0 +# Resident budget of the small pager in MiB. Deliberately no loading +# allowance (measured 2026-09-17: with one it kept 26.7 GiB of blocks +# resident during the load, the very peak it removes); under a tight commit +# charge it drains to 64 MiB. Raise it on a machine with commit to spare to +# keep more blocks resident and shorten the publication pass. +small_cache_hot_mb=1024 + # EXPERIMENTAL (Steam only): lossless paging of the terrain renderer's # material-index cells. The game keeps one 67,601-byte cell per tile for the # whole session (measured 4.13 GiB on a 256x256 map); cold cells are kept @@ -405,6 +497,17 @@ terrain_cow_share=0 material_cache_compress=1 material_cache_hot_mb=0 material_cache_warm_mb=-1 +# RIG-ONLY: pretend the machine has this much RAM, in MiB (0 = off). The pager +# policy (thresholds, headroom, caps, budgets) then sizes itself for that +# machine, with free RAM and free commit scaled to the fractions the real +# machine shows. The engine is not constrained. 32768 tests a 32 GiB PC here. +simulate_physical_mb=0 +# Cap on the resident material-cell target, in MiB. 0 = automatic: a quarter +# of the terrain cap (2048 on 32 GiB or more). -1 = no cap. +material_cache_max_mb=0 +# Same ramp-down, hold-on-faults and adaptive eviction rate as the terrain +# cache (measured 2026-09-17: the old snap evicted 93,000 cells in 30 s). +material_cache_evict_per_s=4000 # Steam New Game map ratios, 1:1 through 1:N (5..20). 20 adds 1:6..1:20. # Extra ratios work with stock and added size rows. Area is approximately kept @@ -416,3 +519,59 @@ max_ratio=20 # of 128 bytes. Applies to automatic and manual saves; files may be larger. # Serialization and save format are unchanged. 0 restores stock on restart. save_fast=1 + +# Steam 35924: the travel-time limits of the urbansim, in game seconds +# (730 s = one game year at 1x; the stock values are 20 and 100 minutes). +# travel_time_limit_s stock 1200: the "stations reachable within the +# limit" search for every cargo type, passengers +# included, and people's own path and destination +# choice. A cargo type only sees consumers whose +# station it can reach within this. +# cargo_path_time_s stock 6000: the longest path the path builder will +# give a cargo item, and the stock-list target picker. +# 0 keeps stock; other values are clamped to 60..86400. Two 4-byte data +# cells, byte-verified before the rewrite. Gameplay changes: longer trips +# become acceptable to people and cargo, so demand spreads further. +travel_time_limit_s=0 +cargo_path_time_s=0 + +# Steam 35924: the terrain alignment update runs in batches of this many +# tiles. In play the engine updates the few tiles a build touched, computing +# every road/track/construction cut on the thread pool and then publishing +# them into the height cache. On a save load it does the same for the WHOLE +# map in one call, holding every intermediate block until the end: measured +# 2026-09-17 on a 207,360-tile map, 3.3 million result vectors and ~10 +# million work vectors alive at once, the game's own 34 GiB private peak and +# an "Out of memory" assert on a 94 GiB machine with no page file. With this +# set, one call of the whole map becomes calls of N tiles (compute, publish, +# free per batch), which is the per-frame behaviour the engine already has. +# Byte-verified; the game's own tile set is walked read-only and never +# modified. 0 = stock (one call). +alignment_batch_tiles=512 + +# Terrain sidecar: serve a save's finished 1 m height cache on load. +# A sidecar file written beside the save holds the aligned cache; when it +# matches the save being loaded, every tile it holds is decoded into the +# tile the moment CTerrain::AddTile creates it, and the load's refine and +# alignment publication into that tile are skipped (the sidecar IS that +# result). Tiles the file lacks, and any save without a matching file, load +# stock. Needs terrain_cache_compress=1 (the tile arena carries the mark). +# 0 = never apply a sidecar. +terrain_sidecar=1 + +# Write the sidecar. 1 = every save gets ".terr" beside it: the terrain is +# captured at the start of the save (the world is frozen for its length), and the +# file is stamped with the finished .sav's hash and renamed into place when the +# engine returns. A load finds its sidecar by that hash, so a COPY of the save +# under another name (multiplayer's mp_shared.sav) uses it too. A sidecar whose +# .sav is gone (a rotated autosave) is deleted at the next write. Costs save time +# and disk (about a fifth of the raw 1 m cache). 0 = read sidecars, never write. +terrain_sidecar_write=1 + +# Optional size limit: no sidecar for a map with more grid records (tiles) than +# this. The capture is +# about 0.17 ms and 23 KiB per tile, on EVERY save, autosaves included: +# 8,712 tiles 1.5 s 190 MiB 65,536 tiles (256 x 256) 11 s 1.4 GiB +# 0 = no limit (the default: a big map's saves take longer and its loads are +# what the sidecar is for). Set a number to skip the sidecar above that size. +terrain_sidecar_max_tiles=0 diff --git a/docs/alignment-batch.md b/docs/alignment-batch.md new file mode 100644 index 0000000..77b8c9b --- /dev/null +++ b/docs/alignment-batch.md @@ -0,0 +1,102 @@ +# The load-time memory spike: the terrain alignment pass, and batching it + +Steam 35924. Everything below was measured on 2026-09-17 on a 94 GiB machine +with **no page file**, so the commit limit equals physical RAM (93.6 GB) and +about 44 GB of it is committed by other programs and the kernel before the +game starts. Maps: `LONGMAPSAVE` and `LONGBOI`, both 207,360 tiles (two +CTerrain versions of 103,680). + +## What the game does when a save loads + +The save holds the 4 m base heightmap and every alignment (the triangle +mesh with weights that a road, track, station or building cuts into the +ground), not the finished 1 m height cache. On load the engine rebuilds the +cache with the bicubic refine and then re-applies every alignment: +`ecs::TerrainAlignmentSystem`'s update (`0xaac810`) hands the set of terrain +blocks dirtied since the last frame (`this+0xa0`, a `std::set`, one entry per +sub-tile block: its `CVec2i` and a vector of the 16-byte alignment items that +touched it; node constructor `0x6fba90`, 0x40 bytes) to `UpdateSubterrains` +(`0xaad6c0`), which for **every entry** gathers the block's alignments, +extracts a work block (`terrain_util::GetBlock`, `0x3c4a20`) and computes a +result block (`CalculateHeightMod`) per region on the thread pool +(`ThreadPool::LoopImpl` at `0xaadc50`), and only then publishes all results +into the height caches (`0x33cd10` at `0xaadcf5`) and frees them +(`0x3b08f0`). In play the set is a few entries. On a save load it is the whole +map. + +## Measured + +| | LONGMAPSAVE (ETW trace of the 18:10 crash) | LONGBOI (in-process counters) | +|---|---|---| +| entries in the dirty set | | 1,658,880 (8 per tile) | +| result vectors (resize at `0xaac4d9`) | 37,354 whole-tile | 3,317,760 | +| work blocks (`GetBlock` -> `0x310230`) | 99,014 whole-tile | ~9.9 million | +| live at the peak | 31.6 GiB, 25.8 of it these blocks | private 34.2-35.8 GiB | +| result | "Out of memory" assert | assert, or survived at the edge | + +The ETW attribution is `tools/re/etl_alloc.py` in the multiplayer repository +(VirtualAlloc events with stacks, paired with their releases); the +in-process counters are the `block_calls= block_sized= result_resizes=` +fields of the pager log line. + +## What did not fix it + +- **Tile pager: lazy zero tiles, dedup, throttle, section retry.** Necessary + (the pager itself was adding up to 19 GiB of sections on top, and its + restore failure at the commit limit was one crash), but the engine's own + 34 GiB remained. +- **Routing the blocks through the tile pager** (`terrain_blocks`, first + version): never matched, the blocks are not tile-sized on LONGBOI. +- **A small pager** (`small_pager.h`, page-granular spans, `small_codec.h`): + caught every block, restored 4.9 million of them correctly, but eviction + could not keep pace with ~50,000 allocations a second against 24 engine + threads on one lock; the peak stayed at 35 GiB unless the producers were + throttled. Kept, off by default (`terrain_blocks=0`). + +## The fix: `alignment_batch_tiles` (`src/alignment_batch.h`) + +A detour on `UpdateSubterrains` (17-byte steal, byte-verified together with +the caller's call site and the iterator's `isnil` test) walks the game's set +read-only with the same MSVC `_Tree` successor step the function uses, +copies each entry's 32-byte value (coordinate plus the vector's three +pointers, which the callee only reads), and calls the original once per +batch of `alignment_batch_tiles` entries (default 512) with a degenerate tree +of its own (a left chain, head as nil). Compute and publication alternate +per batch, which is the per-frame behaviour the engine already has; the +game's set and its vectors are untouched and freed by the caller's own erase. + +Result on LONGBOI (19:30 load): + +| | before | with batching | +|---|---|---| +| peak private commit | 34.2-35.8 GiB | **8.3 GiB** | +| alignment pass | one call, 1,658,880 entries | 3,240 calls of 512 | +| throttle waits, restore retries | dozens to hundreds | 0 | +| load | assert, or survived at the edge | finished | + +The offline test (`tools/test_alignment_batch.cpp`, `build.bat +-alignbatch-test`) builds a `std::set` of entries with real vectors in the +same MSVC layout, checks the walk against the STL's own iteration, the +batches' concatenation, pass-through for small sets and batch 0, chains of +one and exact multiples, and the byte anchors in the real executable. + +## Open + +- Load time was not timed against stock on the same map; the pass now + publishes 3,240 times, each with its own per-call overhead. +- The alignment pass is still necessary work. A sidecar file holding the + finished 1 m cache (`src/terrain_sidecar.h`, `docs/terrain-sidecar.md`) + skips its result on load: `src/terrain_serve.h` post-hooks + `CTerrain::AddTile` (`0x33cb60`, the only place a tile record comes to + life; it returns void, so the record is found by entity from a rotating + cursor), decodes the saved cache into the fresh vector while it is still + private after AddTile's own detach, and marks the tile arena slot `served`; + the block-copy replacement (`0x30a540`) then skips every copy into a served + slot, so the refine and the pass still compute but publish nothing over it. + Counters ride on the pager's 30 s line (`sidecar: add_tile= applied= + copies_skipped=`). Whether the compute itself can be skipped per served + tile depends on mapping a dirty-set block to its tile, not yet done; the + pass's wall time (`alignment pass: ... ms`) says what that would save. + The sidecar's SaveGame/LoadGame hooks and fingerprint are the other + session's; until they load a file, `terrain_sidecar=1` is inert. + (Wired 2026-09-21: `src/terrain_sidecar_io.h`, see `docs/terrain-sidecar.md`.) diff --git a/docs/linux/INSTALL.md b/docs/linux/INSTALL.md index a247e2c..caf858b 100644 --- a/docs/linux/INSTALL.md +++ b/docs/linux/INSTALL.md @@ -4,7 +4,7 @@ For Steam Transport Fever 2 Linux build 35924. This is a native `.so` plugin, not a Wine/Proton DLL. The package includes the shared native plugin host; multiplayer is not required. -1. Close the game and extract `tpf2-bigmap-0.4.0-linux-dev.3.tar.gz`. +1. Close the game and extract `tpf2-bigmap-0.4.0-linux-dev.4.tar.gz`. 2. In the extracted directory, run `bash install.sh` without sudo. 3. Use the Steam launch-options line printed by the installer. If multiplayer is installed, keep its existing launch options: it loads this plugin too. @@ -21,7 +21,7 @@ Linux defaults are placed in `.cfg.example`. Use the plugin when loading worlds that need its expanded octree. Back up important saves before testing. Larger maps need substantially more RAM and -generation time. Optional terrain compression reduces settled RAM use, but does +generation time. Default-on terrain compression reduces settled RAM use, but does not eliminate the peak memory needed while loading or generating a world. ## Supported scope @@ -33,13 +33,13 @@ not eliminate the peak memory needed while loading or generating a world. The largest square is 510 tiles; the preview distance calculation imposes a separate diagonal bound until its Linux overflow fix is ported. - Exact SSE2 terrain min/max scan and faster zstd saves, enabled by default. -- Experimental lossless terrain compression using Linux userfaultfd, opt-in. +- Experimental lossless terrain compression using Linux userfaultfd, enabled by default. - Byte/build guards and a shared host that coexists with native multiplayer. Depth 12/13 (1024/2048-tile edges), material compression, terrain copy sharing, generation buffer reuse and the remaining renderer optimizations are not ported in this build. Do not use the Windows configuration: its depth-13 -setting is rejected explicitly. See PORT.md for evidence and validation limits. +setting falls back to depth 11 and does not support larger-root saves. See PORT.md for evidence and validation limits. ## Sparse density presets @@ -72,8 +72,7 @@ unchanged; files can be larger. `terrain_minmax_fast=1` uses an exact SSE2 scan for terrain height bounds. Both default to on even with an older config. Set either to 0 and restart to disable it. -To try terrain RAM compression, add or change these keys under `[tpf2_bigmap]` -in your installed config and restart: +Terrain RAM compression defaults to these settings under `[tpf2_bigmap]`: ```ini terrain_cache_compress=1 @@ -86,19 +85,37 @@ The plugin reserves a large virtual address range; use resident memory (RSS), not virtual size (VIRT), when comparing RAM use. Compressed data and other game allocations still need RAM. More cache space can reduce decompression activity. -Compression is **off by default**. It requires a Linux kernel that permits +Compression is **on by default**, including when the key is missing. Existing +configs with an explicit `terrain_cache_compress=0` remain off after upgrade. It requires a Linux kernel that permits user-mode userfaultfd with missing-page and write-protection support. Unsupported systems log `terrain compression unavailable` and continue without compression; no root permissions or kernel setting changes are required by the installer. The backend handles userspace faults only: a kernel operation directly accessing an evicted terrain page is outside its supported path. The tested game paths are loading, rendering, simulation, track construction and save/reload; other mods, -GPU drivers and long sessions need further testing. See PORT.md for details. +GPU drivers and long sessions need further testing. A kernel-origin fault can +cause SIGBUS: set `terrain_cache_compress=0` and restart if this occurs. See +PORT.md for details. One 128x128-tile save settled at 4.98 GiB RSS versus 7.78 GiB with dev.2, using a 1 GiB terrain budget. Loading peaks were about 9.8 GiB in both runs. This is a single-machine comparison, not a guaranteed saving or a frame-rate benchmark. +## Incremental memory changes and travel controls (dev.4) + +Inside the terrain pager, `terrain_lazy_zero=1` delays terrain pages until +first access and `terrain_dedup=1` shares identical compressed content. Set either +to 0 to opt out. `terrain_dedup_probe=1` logs a hash-group census every 120 seconds; +it defaults off and can briefly block writers while hashing resident tiles. +The pager still uses a fixed hot budget. Windows adaptive budgets, pressure +backpressure, alignment batching and small/material paging are not available. + +`travel_time_limit_s` and `cargo_path_time_s` default to 0 (stock 1200 and 6000 +seconds). Positive values clamp to 60..86400 game seconds. Restart after changing +them. Both are byte-verified native data patches. Dev.4 passes offline tests; +its lab launch failed before reaching the game, so the older gameplay and memory +measurements above do not validate these additions. See PORT.md for evidence. + ## Remove Run `bash uninstall.sh` (with the same `--prefix` if supplied). It removes only diff --git a/docs/linux/PORT.md b/docs/linux/PORT.md index 2c24573..e50767f 100644 --- a/docs/linux/PORT.md +++ b/docs/linux/PORT.md @@ -2,6 +2,9 @@ Baseline Windows source: bigmap commit `4f0de6f` (0.4.0). Linux ELF GNU build-id: `3a0e156390b0e6f1e372051c24802c8493ae454a`. +Current integration: `b755f653a2e5067a6ca34e89df64dcabe039a052` (partial; see +the final section). Earlier sections describe historical defaults and limits. + The host checks this identity; the plugin checks every patch before publishing any change. `tools/linux/verify_game.py GAME_ELF` independently checks all sites. @@ -171,3 +174,654 @@ Windows block-copy optimization is not part of this change. - Testing used the isolated native lab with the multiplayer Lua mod retained, but no second peer. Multiplayer synchronization and long-session stability remain untested for these additions. + +## Minimap integration cdfee2a (partial) + +Windows source integrated: `cdfee2a12511b05a2a7f67bc67ca3a818e140feb` +(2026-09-16). The Windows implementation, Lua GUI, embedding tool and tests +merge unchanged. **The native Linux minimap is not implemented.** Linux does +not install the GUI script or intercept its image tokens. `minimap=0` remains +the default; `minimap=1` logs an explicit unsupported-feature diagnostic and +leaves the existing large-map features available. No new game patches are +installed, and the existing twenty-site verification manifest is unchanged. + +### Static investigation + +The Linux signature/xref exports and actual build 35924 ELF were examined with +Capstone. All following addresses are investigation evidence, not enabled hook +sites. They must not be treated as a completed port or substituted into the +Windows implementation. + +| Linux RVA | Established evidence | +| --- | --- | +| `0xcf5f20` | Signature identifies `CTerrain::BaseGetVertices(CVec2i) const`. `rdi` is terrain, `rsi` packs x/y. At `0xcf5f38`, `48 8b 57 18` loads terrain+0x18. Instructions at `0xcf5f4b..0xcf5f6e` subtract header x/y at +0/+4, multiply by header width +8, load cells at +0x10, use a 40-byte cell stride and check entity -1. | +| `0x1046540` | Signature identifies `CGameUI::CreateConstructionMenu`; SysV this is rdi. Entry bytes `f3 0f 1e fa 55 48 89 e5 41 57 41 56 41 55`. At `0x10465dc`, `48 8b 87 50 04 00 00` loads UI+0x450. This alone does not establish the accessor's vtable or the returned state layout. | +| `0x30b26c0` | Signature identifies raw `ImageView::SetImage(int,int,int,const vector&,bool)`. Prologue saves channels from esi, width from edx, height from ecx, vector from r8 and bool from r9d. It loads the vector's begin pointer and passes it to `0x34c65a0` at `0x30b281e`. Texture upload/copy lifetime has not been established through that callee. | +| `0x30b2ab0` | Nearby candidate examined for the image-path overload. It saves this from rdi, argument pointer from rsi and flag from edx. However, at `0x30b2b6e..0x30b2bb2` it reads string data/length at argument+0/+8 **and** +0x20/+0x28, then a field at +0x40. Thus it is not proven to accept the single-string argument used by the Windows detour. Do not hook it with a libstdc++ string-only prototype. | + +The adjacent raw-image constructor at `0x30b2950` calls `0x30b26c0`. +Terrain code near `0xcf6474..0xcf6495` reads levels at +0x28 and resolution +at +0x2c/+0x30, consistent with part of the Windows sampling layout, but this +is insufficient to certify the entire terrain sampler. Searching TerrainPtr +signatures also led to the ImportHeightmap callback at `0x10a1190`; its +0x450 field is a widget double, not evidence for CGameUI's terrain accessor. + +### Not ported / evidence still required + +- Resolve the Lua `ImageView:setImage` binding to the actual Linux string + overload and prove its argument ownership and delegation to the image + resource overload. The candidate above failed the single-string check. +- Trace UI+0x450 through the Linux accessor and current game state to terrain; + prove the virtual slot and terrain offset instead of importing Windows + vtable slot 1/state+0x20. Check replacement across loading another world. +- Complete the terrain origin/height-scale/water-field evidence and raw texture + upload lifetime before calling the sampler or freeing uploaded pixel data. +- Then implement Linux guarded hooks, script synchronization and native renderer + integration, with synthetic ABI/render tests. The merged Windows renderer + and Windows ctypes/DLL tests are not a native Linux implementation. + +Validation for this partial integration: `tools/linux/build.sh` and +`python3 tools/linux/verify_game.py GAME_ELF`. The native config test checks +that requesting the unsupported minimap logs a warning and writes only the same existing patch sites and lengths as the default configuration. No live game +validation was attempted, as required by this job. + +## Minimap company map integration d99c054 (shared code only) + +Windows source integrated: `d99c0549793721d9dc17c67ec28a1f9161ac0398` +(2026-09-16, "minimap: company colours and names from the multiplayer mod's +map"). The commit changes only the shared Lua GUI +(`mod/minimap/bigmap_minimap.lua`) and the Windows minimap test +(`tools/test_minimap.py`). Both merge unchanged. The GUI now reads +`mp_company_map.txt` (`me=`, then `==`) +beside `mp_company_cfg.txt` and uses the creation-order guess only when that +file is absent. + +The commit has no platform-specific code: no hook, byte pattern, address or +struct offset. So there is nothing to reverse engineer, no Linux patch site is +added and the twenty-site manifest is unchanged. As recorded for `cdfee2a` +above, native Linux still does not install the minimap GUI or implement its +renderer, so this change has no effect on Linux yet. It will apply unchanged +once the minimap is ported. + +Validation: `tools/linux/build.sh` (four CTest suites pass) and +`tools/linux/verify_game.py` (build-id and 20 sites pass). The merged script +parses under system Lua 5.2 (`luac -p`). A standalone run of the new +map-file parser on the test's sample file gave the expected player-to-company +mapping, local company and unescaped names. `tools/test_minimap.py` was not run +because it loads the Windows `out/tpf2_bigmap.dll` and needs `pefile`/`lupa`. + +## Windows integration bd0d85f (Linux dev.4, partial) + +Integrated on 2026-09-22: the oldest twenty Windows commits, `6121934` through +`bd0d85f2b01f9564f6a9b77aecef684d7e0026af`. The original baseline above remains +historical; this is an incremental merge into `linux-native`. No conflicts were +present. Windows sources and tests are preserved. The merge is staged, not +committed. The remaining 27 Windows commits are outside this integration. + +| Windows commits | Native disposition | +| --- | --- | +| `6121934`, `a75f83e`, `ae3027e` | Native terrain deduplication and diagnostic hash-group census; Windows measurement documentation merged unchanged. | +| `4b39aad`, `a0805be`, `5271b75`, `48946d2`, `c4aa846`, `09fd641`, `35317e7` | Working-set/loading budgets, section retry, commit pressure and adaptive eviction policy remain unported; investigation below. | +| `101d36e` | Both travel-time controls ported to verified Linux data cells. | +| `9431584` | Lazy zero terrain allocations through userfaultfd missing-page handling. | +| `8f4e6a9`, `52ede02`, `cce148a`, `0d2c47e`, `b528366` | Block routing, block diagnostics, small pager and commit backpressure remain unported. Shared block codec builds and is tested natively. | +| `c76db05`, `30cb13a`, `bd0d85f` | Batching implementation and corrected 32-byte values retained for Windows; Linux tree/lifetime investigation below. Windows measurements are not Linux results. | + +### Native pager changes + +`terrain_lazy_zero=1` leaves a newly allocated slot missing, with explicit zero +metadata and no blob. Its first read or write copies one zero-filled stride using +UFFDIO_COPY, then removes write protection and wakes the faulting threads. +Untouched release does not restore pages. `terrain_lazy_zero=0` retains eager +UFFDIO_ZEROPAGE initialization. Both preserve the existing CTerrain allocation +and release ABI; no new game allocation hooks or offsets are introduced. + +`terrain_dedup=1` indexes immutable packed blobs by the codec hash. Before sharing, +the policy worker decodes a candidate and compares **all 132,098 sample bytes** +against the write-protected resident tile. Hash collisions cannot authorize a +false share. A hit takes a reference and skips encoding; a miss creates a blob. +The index is non-owning, removes the entry on final release, and uses a separate +mutex with slot-before-index lock ordering. Restore always creates private +resident pages and drops its blob reference, even for a read. This differs from +Windows' retained read-only restored views, but preserves exact contents and +independent lifetimes. Packed byte statistics count shared mappings once. +Index/metadata allocation failure refuses that eviction and unprotects the tile. + +`terrain_dedup_probe=1` reports hashed, skipped, distinct, duplicate, zero, pair +and largest-group counts every 120 seconds. Cold tiles use their stored hashes; +resident tiles are write-protected during hashing; lazy zeros need no restore. +This is a diagnostic census of hash groups, not an atomic world snapshot or an +exact equality proof. Unlike Windows, there is no ten-second loading cadence or +low-half diagnostic. There is no verified native loading signal. Probing can +stall writers and defaults off. Dedup and lazy zero default on **only inside the +opt-in terrain compression backend**; terrain compression itself still defaults +off. No Linux load-speed or memory-saving measurement is claimed for dev.4. + +### Travel-time RE and guards + +The actual ELF's `.rodata` has the same two float values at different addresses: + +| Linux RVA | Stock bytes | Setting / consumers | +| --- | --- | --- | +| `0x43029a8` | `00 00 96 44` (1200.f) | `travel_time_limit_s`: PathFactory, destination scoring, reachable-station BFS | +| `0x43029a4` | `00 80 bb 45` (6000.f) | `cargo_path_time_s`: PathFactory and StockListSystem | + +A complete `objdump -d -M intel` scan found four RIP-relative readers of 1200: +`0x14e1779`, `0x14fe79e`, `0x1500013`, `0x1500233`; and two of 6000: +`0x14e178d`, `0x172cc06`. Their instruction bytes respectively are +`f3 0f 10 15 27 12 e2 02`, `f3 0f 10 2d 02 42 e0 02`, +`f3 0f 10 05 8d 29 e0 02`, `f3 0f 10 05 6d 27 e0 02`, +`f3 0f 10 25 0f 12 e2 02`, `f3 0f 10 05 96 5d bd 02`. +The reader instructions are evidence, not patched sites. + +PathFactory.cpp assert/signature references anchor `0x14e13c0`: it loads 1200 +into xmm2, stores the stack limit at rbp-0xbb8, conditionally replaces it with +6000 from xmm4, then passes that limit in xmm0 at `0x14e1825`. +The destination_util.cpp function `0x14fe200` compares accumulated time against +1200 at `0x14fe7a6`. The two destination tasks at `0x14fff50`/`0x1500170` pass +1200 in xmm0 to `0x1558870` at `0x1500022`/`0x1500242`. That callee's source and +signature references at `0x155953c`/`0x1559548` identify +`simulation_util::path_finder::GetReachableStationsBFS(int,float,...)`. +StockListSystem.cpp signatures identify `0x172c970` as `Produce`; its load at +`0x172cc06` supplies 6000 in xmm0. These independently establish the shared +constants' roles without importing Windows addresses or struct layouts. + +Only the two four-byte data cells are changed. SysV floating arguments continue +through the stock instructions; no trampoline, object layout or ownership +contract changes. Values <=0 leave stock; positive settings clamp to 60..86400. +Both sites participate in the existing verify-all-before-write plan and rollback. +A mismatch refuses initialization before publishing patches. The ELF manifest +now has 22 sites. Live gameplay effects remain untested (launch failure below). + +### Not ported: static attempts and missing live proof + +**Alignment batching.** Signature exports identify `0x173cbe0` as the thread-pool +loop for `TerrainAlignmentSystem::UpdateSubterrains(const std::map>&)`. Actual disassembly traces its caller to `0x173dae0`: +`rdi=self` is saved in r14 at `0x173daec`, `rsi=map` in r12 at `0x173daf3`. +It reads the leftmost node at map+0x18 (`0x173db87`), uses map+8 as sentinel +(`0x173db8c`), and calls libstdc++ `_Rb_tree_increment` at `0x173e018`. +Its direct caller at `0x173e443` has bytes `e8 98 f6 ff ff`. The loop call is +`0x173e0a6 -> 0x173cbe0`; publication calls `0xcf56d0` at `0x173e16f`, +followed by delete calls including `0x173e180` and `0x173e1a9`. +The caller clears the tree rooted at self+0xa0 (header self+0xa8, count +0xc8). +This is not the MSVC head/isnil layout. No fake native tree is published. +Still required: live map nodes and full 32-byte values, vector ownership across +thread-pool completion, and proof that publishing one batch cannot affect the +computation of later batches. The lab launch failed before these probes could +attach, so `alignment_batch_tiles` remains disabled with a request diagnostic. + +**Work/result blocks and small pager.** Source/signature anchors and disassembly +locate Linux `terrain_util::GetBlock` at `0xdb55c0`. It squares the dimension at +`0xdb587b`, doubles the sample count at `0xdb5881`, and calls operator new +(`0x6dbce0`) with bytes in rdi at `0xdb589e` (`e8 3d 64 92 ff`). It stores the +three pointers at rbp-0xa0/-0x98/-0x90 and zeros the uint16 samples in a loop. +A delete call occurs at `0xdb5bfe -> 0x6dbcd0`. Windows' vector-constructor +return-address filter and CRT free-IAT hook therefore cannot simply be moved. +The native publication helper `0x173ed50` was also disassembled; it deletes at +`0x173ee94`. Allocation escape/exception paths, all result resizes and final +owners are not proven. Without a live load to observe them, intercepting global +delete could hand arena pointers to an unguarded native free. `terrain_blocks` +remains disabled with a diagnostic; its counters, small-span pager, fixed budget +and pressure throttle are absent. The portable `BlockCodec` is retained and +validated independently; that does not claim that the small pager is ported. + +**Budgets, rate control and backpressure.** The Windows implementation relies on +GlobalMemoryStatusEx free commit, page-file-backed section creation, world-entry +state, bulk allocation timestamps and the menu DLL's `Tpf2mpLastGameUiTick`. +The Linux source and installed menu `.so` exports were examined: no corresponding +export exists (game UI/frame state is local). Linux uses anonymous +MAP_NORESERVE memory and UFFDIO_COPY, not section creation. This host reports +`vm.overcommit_memory=0`, `overcommit_ratio=50`; CommitLimit minus Committed_AS +is not the Windows section-allocation contract. Importing the 10/12 GiB thresholds +without validating pressure and fault progress would misrepresent protection +against OOM. No native load/pressure/frame measurements were possible after the +lab failed. The Linux pager retains its fixed hot budget, two-second age and +bounded scan; there is no loading allowance to ramp down, measured-cost/frame +rate cap, pressure-driven cap override, restore retry or backpressure. Explicit +Windows warm/rate settings log a diagnostic. Material paging was already absent; +its matching policy changes remain absent too. Needed next: a native gameplay +stamp/loading signal, validated Linux memory-pressure inputs and partial-copy +failure/retry tests under a constrained lab process before enabling the policy. + +### Build, tests and live attempt + +`tools/linux/build.sh` passes five suites under the soldier SDK: bigmap, density, +pager, terrain codec and the new shared block codec suite. The userfaultfd test +actually ran, rather than skipping. New checks cover untouched page residency, +first-read/write zeros, untouched release, eager opt-out, cold and resident +censuses, sixteen shared pairs, private writes to twins, concurrent restores, +write/eviction races and final blob cleanup. Travel tests cover clamping, disabled +values, byte mismatch before writes and rollback after a failed data write. +The block codec checks zero/ramp/random data, boundary sizes, truncation, hash +corruption and capacity rejection. ELF build-id and all 22 patch sites pass. + +The lab's share/tpf2mp and mp_lockstep_1 directories were backed up with `cp -a` +to `.before-port`. The built plugin and test config were installed only in the +native actor. The official launcher at +`/home/topsnek/tpf2-multiplayer/tools/sandbox/tpf2mp-lab` was used because this +bigmap clone has no sandbox launcher. It failed immediately with +`bwrap: setting up uid map: Permission denied`. Two attempts using the same lab +mount/launch specification with system bwrap (including PRESSURE_VESSEL_BWRAP) +reached pressure-vessel but failed to create its nested namespace. A final +attempt with the final build through the original launcher failed identically. +No kernel/security policy was changed. No game process, title menu, renderer, +Vulkan device, save load or gdb session was reached; no live success is claimed. +All attempted runs exited immediately, below the three-minute limit. No Steam +process was restarted, no save was modified, and no input events were sent. + +The actor share tree was restored from its backup; the unchanged mod tree was +compared to its backup. Recursive comparisons are empty. No launched process +remains. Raw disassembly, launch errors, build/verification logs, actor logs/data, +test configuration and restore comparisons are retained in this job's +`meta/live/`. Existing actor log contents predate these failed launches and +must not be mistaken for new gameplay observations. + +## Windows integration 26bced4 (partial) + +Integrated 2026-09-22: twenty Windows commits `dc7264d` through +`26bced4b98893809bf9bd7960a0acec7b9ab244a`, based on the preceding `bd0d85f` +integration. No merge conflicts. Seven pending Windows commits remain outside +this batch. Windows implementations, release assets and MSI CI are retained; +the merge is staged and uncommitted. + +| Commits | Native disposition | +| --- | --- | +| `dc7264d`, `915aec1` | Alignment handle, pass timing and test merge; native batching remains unported. | +| `e62e43b`, `0511bb5`, `96340f9`, `059f2f1`, `da9d033`, `ff2fd60` | Shared sidecar file API now compiles and is tested on Linux; engine grid adapter and save/load integration are not enabled. | +| `1c3c713`, `4d696f3` | Windows RE/measurements merge as documentation, not native evidence. | +| `5bf8aaf`, `8e42635` | AddTile serving, publication suppression and pass-end release remain unported. | +| `f7c6395`, `585921b`, `a448c32` | Windows releases retained. Native minimap remains off/unavailable; pager policy portion remains unported. | +| `2d03251`, `7407127`, `ce3feb7`, `24a020a` | Adaptive caps, simulated memory, throttle hysteresis and working-set floor remain unported. | +| `26bced4` | Windows MSI/draft-release workflow retained unchanged; no publishing performed. | + +### Portable sidecar validation + +`src/terrain_sidecar.h` uses scoped `OpenFile`/`CompareExtension` helpers: +MSVC keeps `fopen_s`/`_stricmp`; Linux uses `fopen`/`strcasecmp`. Test exports +retain `dllexport` on Windows. The file format, fingerprint, streaming index, +lazy per-tile decode and fallback contract remain unchanged. No Windows game +address is used by the native plugin. This header's raw `GridOf`/`VectorOf` +still describe the **Windows** layout; it is compiled only by an offline native +test, not included by the native plugin. + +Both incoming synthetic tests allocated only 0x20 bytes for a control block +with a 24-byte vector at +0x10: writing `end` overran it by eight bytes. They +now allocate 0x28. The sidecar test is a sixth CTest suite, with assertions +explicitly enabled in Release. It restores 824 tiles exactly, checks holes, +wrong-sized caches, mismatched fingerprints/dimensions, truncation, corrupted +blobs, per-tile fallback, windowed indices, and changed-save rejection. The +same test also passes host ASan/UBSan. This validates the shared file API with +synthetic Windows-layout objects, not Linux game ownership. + +### Static RE: Linux differs at the shared pointer + +Examined the actual build 35924 ELF (build-id remains +`3a0e156390b0e6f1e372051c24802c8493ae454a`) with objdump and the signature +exports. These are investigation sites, **not new patches**: + +| Site | Evidence | +| --- | --- | +| `0x173db00` | `48 8b 7f 08`, loads system+8 into rdi, followed at `0x173db13` by call to signature-identified `CTerrain::GetTileCache` (`0xcf5960`). Confirms the alignment system's terrain field statically. | +| `0xcf71d0` | AddTile candidate: `endbr64; push rbp; mov rbp,rsp; push r15`; `0xcf71da: 49 89 ff` saves SysV this/rdi in r15, `0xcf71e8: 41 89 f5` saves entity/esi in r13d. Its profiling string at `0x3f25667` is `CTerrain`. | +| `0xcf73f2..0xcf7419` | Loads terrain+0x18 grid, subtracts grid origin, multiplies by width, scales record index by 40, stores entity (`45 89 2c 24`) in record+0. This links the candidate to AddTile independently of its generic profiling label. | +| `0xcf75e4..0xcf7608` | Loads record+0x10 shared ownership control, tests reference count at control+8 against 1, then loads **record+8 directly as the vector**, with last at vector+8 and first at vector+0. | +| `0xcf774a`, `0xcf77ce`, `0xcf77d3` | Detached copy: vector is new control+0x10 (`4d 8d 6e 10`); stores that vector pointer at record+8 (`4d 89 6c 24 08`) and control separately at record+0x10 (`4d 89 74 24 10`). | +| `0xcf7696` | Existing guarded default-append call, vector in rdi and extra sample count in rsi. AddTile increments record version at `0xcf762f`. | +| `0xcf4f78..0xcf4f8d` | Independent detach/read path checks record+0x10 refcount, then reads record+8 and dereferences the vector's first pointer. Corroborates that adding Windows' extra +0x10 here would be wrong. | + +The prior native map traversal evidence at `0x173dae0` was rechecked. The +publication call at `0x173e16f` still targets `0xcf56d0`. No assumption that +Windows and Linux shared_ptr layouts agree is made. No hook or byte manifest +entry was added; all 22 existing guarded sites remain verified. + +### Not ported: attempts and remaining evidence + +**Alignment/sidecar runtime:** the Linux AddTile/vector and terrain-field +investigation above made progress, but the lab could not start (below). Live +private ownership after AddTile, worker completion, both terrain versions' +creation order, save-time ownership and lifetime across loads remain unproven. +The Linux publication/copy paths must be traced before suppressing any writes; +a persistent served flag must not suppress later gameplay edits. The native +batching prerequisite remains absent, including the timing and pass-end callback. +No live system pointer is retained. Save-path resolution and completed-save +triggers also remain absent. In this exact incoming Windows snapshot, +`ArmForLoad` and `WriteForSave` have definitions but **no production callers**; +future Windows wiring is outside this batch. No end-to-end sidecar load claim +is made for either platform here. + +**Memory policy:** inspected both Windows workers and Linux's UFFD backend. +Windows uses section/free-commit accounting and a gameplay tick; Linux uses +anonymous MAP_NORESERVE/UFFDIO_COPY, retains a fixed resident target and has no +material pager. The installed menu library's dynamic exports still provide no +`Tpf2mpLastGameUiTick`. This host's overcommit mode is 0, ratio 50; measured +MemTotal/MemAvailable/CommitLimit/Committed_AS are retained in the evidence. +CommitLimit minus Committed_AS is not a validated replacement for Windows free +section commit. The failed live launch prevented loading, frame/fault feedback, +constrained-memory progress and recovery measurements. Importing the formulas +alone would not validate throttle safety. Sized headroom, caps, simulation, +hysteresis, working-set floor and associated material policy stay disabled. +Needed: native pressure inputs, loading/frame signals, recoverable UFFD copy +failures and live constrained-process measurements. No Linux memory or load-time +improvement is claimed. + +Explicit `terrain_sidecar=1`, nonzero `terrain_cache_max_mb`, +`material_cache_max_mb` or `simulate_physical_mb` now log native diagnostics. +Tests assert that these requests add no patch sites or writes. Linux defaults +remain minimap=0, sidecar=0 and fixed terrain_cache_hot_mb=1024; compression is +still opt-in. Zero cap/simulation settings do not enable an automatic policy. + +### Validation and live attempt + +`tools/linux/build.sh`: six suites pass, including the real userfaultfd pager +test (not skipped). `tools/linux/verify_game.py`: build-id and all 22 sites pass. +Host ASan/UBSan sidecar test passes. Windows DLL/serve/alignment tests and the +MSI workflow were not executed on this Linux host. + +Backed up both actor share/tpf2mp and mod directories with `cp -a` to +`.before-port`, installed the built plugin and Linux configuration in the +native actor, then ran the official tpf2-multiplayer lab launcher (this clone +has no launcher). It exited immediately with `bwrap: setting up uid map: +Permission denied`. No title menu, renderer/Vulkan device, save load, gameplay +or gdb attachment was reached. No input was sent, save modified, or Steam +process touched. The failure was well within the three-minute time box. +The share directory was restored; both restored share and unchanged mod compare +identically to backups. No lab game process remains. Actor logs/data were +copied for completeness, but pre-existing gameplay logs are not evidence of +this run. Disassembly, launch/build/test logs, memory evidence, process check +and empty restoration diffs are in this job's `meta/live/`. + +## Windows integration 9180629 (partial) + +Integrated 2026-09-22: the final seven Windows commits `5ca9f16` through +`918062927018b3397450e83ddc5ec5210a914153`, following `26bced4` above. +Resolved and staged `src/terrain_sidecar.h`; the merge remains uncommitted. +The original baseline is unchanged as historical context. + +| Commits | Native disposition | +| --- | --- | +| `5ca9f16` | README memory measurements and scanner notes retained as Windows documentation. | +| `3e20ce3`, `757d35d` | Windows release 0.5.3, vendored DLLs, MSI library discovery and CI retained. Linux installs into the shared host prefix (`--prefix` supported), not the game folder; no Windows DLL is substituted for a native host. | +| `7053712` | Portable UTF-8 sidecar I/O, fingerprint stamping and sibling discovery ported and tested. Engine save/load capture, backend path resolution and orphan sweeping remain unported. | +| `80a6c63` | Live-terrain selection remains unported pending ownership proof. | +| `3c9fc62` | Shared grid reader and synthetic tests now dereference record+8 directly as the vector, agreeing with Linux disassembly below. | +| `9180629` | Windows load-only fault sleep retained; native pager has no commit throttle or equivalent loading signal. | + +### Portable code and regression coverage + +The conflict joined Windows wide UTF-8 path handling with the prior POSIX CRT +port. Windows keeps `MultiByteToWideChar`, `_wfopen_s`, `_wremove` and its +FindFirstFileW search. Linux opens UTF-8 path bytes directly, removes files via +`remove`, and uses `opendir`/`readdir` to search sibling `.terr` headers. POSIX +suffix matching is case-sensitive. Discovery prefers the requested file and +leaves the path unchanged on failure or insufficient output capacity. File +format/version and codec are unchanged; `Refingerprint` checks the header hash +before stamping. Neither the shared grid nor these helpers is used by the live +native plugin yet. + +The seventh CTest suite, `sidecar_io`, tests non-ASCII directories, unstamped +rejection, stamping and exact restoration, discovery under a different name, +foreign/zero fingerprints, bounded buffers, corrupt headers and removal. Its +vector object is independent of the control block, catching accidental extra +indirection. The existing 824-tile sidecar test retains the corrected 0x28-byte +synthetic control allocation and now uses the incoming shared_ptr layout. +Explicit `terrain_sidecar_write=1` or nonzero `terrain_sidecar_max_tiles` requests +now produce the same unsupported diagnostic as `terrain_sidecar=1`; native +config tests verify no added patches or writes for each request. + +### Static RE: save/load ABI and terrain layout + +Signature/source exports and actual build 35924 ELF disassembly establish the +following **investigation sites, not installed hooks**. No guarded patch was +added; the manifest remains 22 sites. + +| Linux RVA | Evidence / contract | +| --- | --- | +| `0xc7ec00` SaveGame | Serializer.cpp signature matches the nine explicit Windows parameters. Entry bytes `f3 0f 1e fa 55 48 89 e5 41 57 41 56 41 55 41 54`. At `0xc7ec2a`, `4c 8b 65 10` loads SaveGameId from rbp+0x10 (entry rsp+8), **argument 7**, not Windows' hidden-return argument 8. Flag is rbp+0x18; monitor rbp+0x20. Epilogue at `0xc7fd9d` loads edx and `0xc7fda3` loads rax, returning the aggregate in registers. A Windows void-pointer/hidden-return detour would corrupt this ABI. | +| `0xc7ca40` LoadGame | Serializer.cpp unique_ptr-return signature. `0xc7ca55: 49 89 cd` saves rcx in r13; `0xc7caeb..0xc7caf3` reads id+0x28 length and id+0x20 data for the logged save name. The result pointer is saved from rdi at `0xc7ca7e`, context from rsi at `0xc7ca96`, modRep from rdx at `0xc7caa8`. SaveGameId is argument 4 in **rcx**, not Windows r9. | +| `0x3341ef0` GetSavegameInfo | StandardSaveGameBackend.cpp signature. Hidden result rdi, backend rsi, id rdx (saved as r14 at `0x3341efc`). Reads path data/length at id+0/+8, name at +0x20/+0x28, namespace at +0x40. Empty path calls `0x33467f0` with backend+8 and id+0x40 at `0x3341f52..0x3341f5b`; returned entry directory is read at +0x28/+0x30. `.sav` is appended via char-string `_M_append` at `0x3342076`; its literal is `0x3f249c8`. This is native libstdc++ narrow string handling, not MSVC wstring SSO. | +| `0x18ba890`, `0x18ba270` | Both save and load call the first then the second. First: `endbr64; mov rax,[rip+0x41984e5]; ret`, global cell `0x5a52d80`. Second: `f3 0f 1e fa 48 8b 3f 48 8b 07 ff 60 20`, forwarding through impl vtable+0x20, **not Windows +0x18**. Concrete backend identity/lifetime still needs live proof. | +| `0xcf75e4..0xcf7608` | Rechecked AddTile: ownership count comes from record+0x10, vector directly from record+8, samples from vector+0, end from vector+8. Detached copy stores control+0x10 as record+8 at `0xcf77ce` (`4d 89 6c 24 08`) and the control as record+0x10 at `0xcf77d3` (`4d 89 74 24 10`). The corrected Windows reader agrees with this Linux layout; the earlier documentation's purported Windows-only extra indirection was the upstream bug. | + +### Not ported and remaining proof + +SaveGame/LoadGame hooks, current-terrain capture and serving still require live +proof of both terrain versions' lifetimes, synchronization during SaveGame, +private writable vectors after AddTile, publication suppression and pass-end +release. Native alignment batching/serving prerequisites remain absent. Windows +SEH probing of potentially freed pointers is not portable ownership validation; +we do not retain or dereference stale native candidates. Backend selection and +completed-save/error handling must also be proven before enabling temporary-file +capture, rename or automatic orphan deletion. Static functions above were found, +but the attempted lab launch failed before gdb could attach. + +The native UFFD backend was reviewed for `9180629`: faults restore with +UFFDIO_COPY and contain no budget/commit-pressure sleep. Its 10 ms sleep belongs +to the eviction policy worker. There is no material or small pager. Therefore +Linux does not currently have the gameplay-freezing throttle this commit fixes. +Porting the full adaptive policy remains outstanding: the menu library still +exports no Tpf2mpLastGameUiTick, and Linux anonymous overcommit (mode 0, ratio +50) does not implement Windows section/free-commit accounting. The failed lab +launch prevents load/frame and constrained-memory recovery measurements. No +Windows thresholds, loading guesses or unverified hook addresses were enabled. + +### Validation and live attempt + +`tools/linux/build.sh` passes all **seven** soldier SDK CTest suites, including +actual userfaultfd execution (not skipped). ELF verification passes the build-id +and all 22 guarded sites. Host ASan/UBSan runs of both sidecar tests pass. Windows +DLL tests, MSI and CI were not executed on Linux. + +Both actor directories were compared with their existing `.before-port` backups +and freshly copied with `cp -a`. The final plugin and Linux config were installed +in the actor's `share/tpf2mp/data/plugins/`. The official launcher from +`/home/topsnek/tpf2-multiplayer/tools/sandbox/tpf2mp-lab` (absent in this clone) +was run with the native lab root. It failed immediately with +`bwrap: setting up uid map: Permission denied`, including the final-build retry. +No title menu, Vulkan device, save, gameplay or gdb session was reached. No +input events were sent, Steam was untouched and no save was modified. No live +performance or correctness result is claimed. Both directories compare equal +to their backups after restoration; no lab executable remains running. +Disassembly, test logs, launch error and restoration comparisons are retained +in this integration job's `meta/live/`; copied actor logs/data are pre-existing +and do not establish new gameplay observations. + +## Windows integration 926113b + +Integrated 2026-09-23: Windows merge commit +`926113b78bcfacb3e487a98298d6d6ae32e9f53e` (PR #5, GOG support for 0.5.2), +including its merged branch commits and the Program Files (x86) batch fix. +This is one first-parent integration after `9180629`; later pending Windows +commits are excluded. No conflicts were present. The merge stays staged and +uncommitted. The baseline above remains historical. + +### Native disposition + +Windows GOG now falls back from requested octree depth 12/13 to its verified +11 root. Native Linux had the corresponding refusal despite already having a +verified depth-11 implementation. It now accepts requests 11, 12 and 13; +12/13 log the fallback when octree expansion is enabled. Other values still +fail before any patch writes. Native defaults remain 11/512. + +The existing single `cap` calculation limits the menu, explicit dimensions and +configured size claims to the installed root: at most 512 tiles, at most 256 +with octree disabled, at most 180 with raster disabled, and any smaller user +cap. Heightmap constraints remain applied. No depth-12/13 Linux hook is added; +a save requiring those roots is still unsupported. Unknown ELF builds and +byte mismatches still fail closed. Windows sources, GOG MSI property and custom +action DLL, vendor provenance, deployment shadow guard and batch quoting fix +are retained unchanged. Linux's build script never deploys; its installer uses +the native shared prefix and preserves existing configuration and host. There +is no native game-folder DLL deployment or MSI/registry counterpart to add. +The Windows README's GOG observations are upstream results, not Linux tests. + +### Static evidence + +Rechecked the actual lab ELF and all 22 manifest sites. At `0xa84230`, +`49 8b 7e 48` loads the octree receiver into SysV rdi; at `0xa84234`, +`f3 0f 10 05 48 7c 40 03` loads the original extent into xmm0; at `0xa8423c`, +`be 0a 00 00 00` sets depth 10 in esi. The call at `0xa84241` is +`e8 3a 23 c2 00`, targeting `0x16a6580`. The existing verified 13-byte patch +redirects the extent to private 65536.f and sets esi=11. Fallback uses precisely +that patch, with no new address, byte pattern, struct offset or ABI assumption. +The original menu and resize anchors are documented above. Disassembly and +build-id/site verification output are archived in this job's `meta/live/`. + +### Validation and live attempt + +`tools/linux/build.sh` passes all seven CTest suites, including the real UFFD +pager test. Expanded native config tests cover depths 11/12/13, smaller user +caps, disabled octree/raster, bounded menu and explicit dimensions, invalid +values, identical patch site/length sets and the emitted depth-11 immediate. +A mismatch specifically at the octree site refuses depths 12 and 13 with zero +writes. The independent verifier passes the GNU build-id and all 22 sites. +Native installer checks pass checksums, standalone host loading, upgrade config +preservation, multiplayer coexistence and uninstall/save preservation. An initial +installer-test invocation omitted its required package argument and exited before +running tests; rerunning against a fixture of the built files passed. Windows +DLL tests, batch execution and MSI installation were not run on Linux. + +Both native actor directories were backed up with `cp -a` to +`.before-port-926113b`. Installed the built plugin and Linux config with only +`octree_depth=13` and `max_tiles=2048` changed, then used the official launcher +at `/home/topsnek/tpf2-multiplayer/tools/sandbox/tpf2mp-lab` (absent in this +clone). It failed immediately: `bwrap: setting up uid map: Permission denied`. +No game, title menu, Vulkan device, save load, gameplay or gdb session was +reached; no live fallback success is claimed. No input, Steam operation or +save modification was performed. Both directories were restored and recursive +comparisons are empty; no TransportFever2 process remains. Launch output, +test config, disassembly, tests, restoration/process checks and copied actor +logs/data are in `meta/live/`. Copied actor gameplay data predates this attempt. + +### Not ported + +No newly applicable behavior from this integration remains unported. GOG PE +addresses and MSI acceptance apply to Windows only; native support remains the +identified Steam ELF. Actual native depth-12/13 roots and previously recorded +minimap/sidecar/adaptive-policy gaps remain outside this incremental change. +The live startup check is blocked by the lab's namespace failure, not reported +as a successful runtime validation. + + +## Windows integration b755f65 (partial) + +Integrated 2026-09-26: eleven Windows commits `c7e8e30` through +`b755f653a2e5067a6ca34e89df64dcabe039a052`, after `926113b` above. +The README conflict retains the Linux scope and the upstream standalone-sync +and unified Windows installer descriptions. All changes are staged; the merge +is not committed. The initial Windows baseline remains historical. + +| Commits | Native disposition | +| --- | --- | +| `c7e8e30`, `26b4ca3` | Unified Windows packaging and standalone mirror documentation retained. Native packaging continues to use the shared prefix and preserve installed configs; the synchronized standalone installer and ABI header remain. No native publishing workflow added. | +| `1c6fccc`, `9cba049`, `9f6a3e2` | GOG support, batch quoting and graceful octree fallback already integrated through `926113b`. Native depth 12/13 requests still install depth 11; unchanged guarded sites and caps. | +| `2e2abe5`, `4ef7656` | Upstream play results and performance catalog retained, with explicit Linux scope notices. Separate native dedicated-server depth-13 results do not validate this standalone port. | +| `a2756cc` | Compression on in the shipped native config and missing-key default. Explicit 0 and unavailable-userfaultfd stock fallback preserved. No version change. | +| `e19eec1` | Thirty-second diagnostics now include elapsed-time faults/s and evictions/s. Recency eviction and automatic headroom/working-set budgets remain unported; see below. | +| `45dd855` | POSIX autosave discovery helper and native tests added. It is not connected to native SaveGame hooks, which remain absent. Windows save hooks and tests remain intact. | +| `b755f65` | Shared sidecar reader/release synchronization ported with a native reader/writer mutex and concurrency regression coverage. No live sidecar hook enabled. | + +### Native changes and contracts + +Compression uses the existing byte-verified AddTile allocation and control-block +dispose sites; neither the SysV ABI nor the patch manifest changes. Both the +shipped key and `cfgBool` fallback are 1. Existing installations with explicit 0 +remain off because the installer preserves configuration. Unsupported userfaultfd +setup still logs and leaves allocation hooks off. `UFFD_USER_MODE_ONLY` cannot +serve kernel-origin accesses to missing pages: a GPU driver/mod can cause SIGBUS; +set `terrain_cache_compress=0` and restart in that case. This supersedes earlier +sections' opt-in defaults, not their validation limitations. + +The sidecar header uses Windows SRW locks on Windows and +`std::shared_timed_mutex` on Linux. Loaded/Has and the entire tile decode hold +shared ownership; BeginApply publication/reset and EndApply use exclusive +ownership. EndApply moves state out under the lock, frees it afterward and +returns true to exactly one releaser. The merged Windows pass callback logs +only for that releaser. This guards file lifetime, not the game's terrain +lifetime; native serving remains disabled. BeginApply calls still require the +load owner to serialize whole load transitions, as in upstream. + +`linux/sidecar_files.h` implements the new discovery rule on POSIX UTF-8 paths: +exactly one regular `.sav` with mtime at least call start minus two seconds. +Directories, symlinks and `.sav.lua` are excluded; suffix matching is +case-sensitive. Failure, ambiguity and insufficient output capacity preserve +the caller's output. No rename, orphan deletion or game-file mutation is +performed by the helper. Timestamps alone do not prove save/terrain ownership; +future hook wiring must serialize saves and validate success before stamping. + +Native pager status now samples cumulative counters every 30 seconds and divides +their deltas by actual steady-clock elapsed time. `faults/s` includes lazy-zero +faults and write-protection events, not just cold restores. The counters do not +claim that resident reads are observable. + +### Static investigation and missing live proof + +Re-read Linux signature exports and disassembled the actual lab ELF. No new +patch site is enabled; build-id and all 22 existing sites pass verification. +The following are investigation evidence, not hooks: + +- SaveGame `0xc7ec00`: `4c 8b 65 10` at `0xc7ec2a` loads the seventh SysV + argument (SaveGameId) from rbp+0x10. At `0xc7ec95`/`0xc7ec9a`, the name is + read from id+0x20/+0x28. `0xc7fd9d` and `0xc7fda3` return the aggregate in + edx/rax. Windows hidden-return and argument placement cannot be reused. +- LoadGame `0xc7ca40`: id is saved from rcx, with the hidden result in rdi; + the logged name uses the native 32-byte string layout. Save/load both use + the backend path already documented under `9180629`. +- GetSavegameInfo `0x3341ef0`: hidden result rdi, backend rsi, id rdx; + name/path strings and the `.sav` append agree with the earlier analysis. + Backend forwarding at `0x18ba270` ends in `ff 60 20`, vtable+0x20. +- AddTile detached copy: `4d 8d 6e 10` at `0xcf774a` constructs vector + object control+0x10; `4d 89 6c 24 08` at `0xcf77ce` stores it at record+8, + and `4d 89 74 24 10` at `0xcf77d3` stores the control at record+0x10. + This supports the synthetic grid layout, but not cross-thread ownership. + +**Not ported: autosave/sidecar engine integration.** The lab startup failed +before a process could be attached with gdb. Save success/rotation, actual backend +identity, terrain snapshot ownership, AddTile writable ownership and completion +of all alignment workers remain unproven live. Native alignment batching and +publication suppression are still absent. The new lock and file helper are +native offline components only; automatic autosave sidecars remain unavailable. +The native unsupported-setting diagnostic is retained. + +**Not ported: recency eviction and adaptive budgets (`e19eec1`).** Compared +`src/pager_impl.inl` with `linux/terrain_pager.h`. Native Serve updates touched +on UFFD events; Policy scans up to 256 round-robin candidates and skips those +less than two seconds old. Resident reads and ordinary unprotected writes are +invisible. The backend has no recency queue or repeated-fault retention, and its +fixed 1024 MiB config is not the upstream server's automatic RAM/30 policy. +Windows section/commit-pressure rules do not establish Linux headroom safety. +The failed live launch prevented steady-state fault/eviction, simulation-rate, +load-peak and constrained-memory recovery measurements. Rates are now exposed, +but an unvalidated automatic budget or claim of eliminating thrash is not shipped. +Needed: native policy tests for hot-set retention and pressure recovery plus +live large-save measurements. Existing larger-root/minimap gaps are unchanged. + +### Validation and live attempt + +`tools/linux/build.sh` passes all seven soldier SDK CTest suites, including real +UFFD allocation/eviction/restore tests (not skipped). Native config regression +checks cover missing compression key, explicit 0 and explicit 1; repeated fake +ELF host initialization intercepts pager startup while the separate pager suite +runs the real backend. Existing cap, byte-guard and rollback coverage still passes. +The sidecar I/O suite adds 200 races with four private-grid readers and eight +releasers: exactly one release, each decode intact or refused, plus repeated +BeginApply/reset and malformed fingerprint rejection. Discovery tests cover +UTF-8 paths, zero/one/two candidates, old saves, nanosecond tolerance boundaries, +metadata/directories/symlinks and short output buffers. ASan/UBSan passes (1,940 concurrent decodes intact in that run). The native +installer fixture passes checksums, host loading, config-preserving upgrade, +multiplayer coexistence and uninstall/save preservation. Logs and final build +output are archived with this job's evidence. + +Both native actor directories were backed up with `cp -a` to +`.before-port-b755f65` (and `.before-port-b755f65-final` for the final-build +retry); the built plugin and default-on config were installed +only in the actor. Ran the official launcher from +`/home/topsnek/tpf2-multiplayer/tools/sandbox/tpf2mp-lab` (absent in this clone), +with a 180-second timeout. Both launches failed immediately with +`bwrap: setting up uid map: Permission denied`. No game process, title menu, +Vulkan device, save load, gameplay or gdb attachment was reached. No input was +sent, Steam operation performed or save changed. Both directories were restored +and recursive comparisons are empty. Actor logs/data copied into `meta/live/` +are pre-existing, not evidence of a successful new run. Disassembly, launch +output, tests and restoration evidence are retained there. Windows DLL/MSI +tests were not run on Linux. diff --git a/docs/load-speed-todo.md b/docs/load-speed-todo.md index 8831e1d..ac68a90 100644 --- a/docs/load-speed-todo.md +++ b/docs/load-speed-todo.md @@ -49,11 +49,12 @@ the title menu (frame loop or overlay busy-wait; not load-specific). hook does fire (gameplay) every single share was written, 710/710, making it pure overhead. Peak was 38.96 GB against 39.02 GB with it off. See `docs/terrain-cow-sharing.md` section 0c. -- [ ] **Content dedup of identical tiles** (replaces the item above). Both - CTerrain versions allocate 65,536 tiles each from the same save data, so they - are probably byte-identical. `terrain_codec.h` already computes a 64-bit - content hash. Cheap test first: hash both versions' tiles during one load and - count collisions; build dedup only if that is near 65,536. +- [x] **Content dedup of identical tiles** (replaces the item above). MEASURED + September 17 with `terrain_dedup_probe`: at 131,072 live tiles, 64,922 exact + pairs and 719 still-unfilled zero tiles; after the load, 65,536 distinct. + Built as `terrain_dedup` (evictions share an identical stored blob instead of + encoding); see `docs/terrain-cow-sharing.md` section 0d. In-game hit count and + load time not yet measured. - [x] **`0x30a55c` identified**: the middle pdata chunk of the uint16 height block copy `0x30a540` (row-by-row, one word per iteration, 132 KB per tile copy). Replaced with a per-row memcpy in `terrain_minmax_fast`. diff --git a/docs/minimap-native-preview.md b/docs/minimap-native-preview.md new file mode 100644 index 0000000..3354a0e --- /dev/null +++ b/docs/minimap-native-preview.md @@ -0,0 +1,599 @@ +# Native map preview (`UI::MapPreviewComp`) as an in-game minimap (Steam 35924) + +Read-only reverse engineering, September 16, 2026. The game was not launched, +attached to or patched, and nothing under the Steam directory was modified. Executable +SHA256 `782b904a8f7bbdac1f7a18528f1a5c778691e5aa3087c37c351bf6912585175c`. All +addresses are RVAs relative to `0x140000000`. + +Labels, as in `runtime-memory-audit.md`: + +- **MEASURED**: observed in a running game. **No in-game measurements were taken for + this document.** A few claims come from shipped data files (style sheets, climate + scripts). Those are marked **FILE**. +- **DERIVED**: read from decompiled or disassembled code. Correct if the reading is + correct, but not observed at run time. +- **GUESS**: an inference that the code does not pin down, such as a name, a meaning + or a performance figure. + +## Answers in brief + +1. `UI::MapPreviewComp` (vtable `3019868`, ctor `63a190`, 0x498 bytes) is a + `ContentView`. It turns a `Map` struct (the map generator's output) into an RGBA + `ImageView`, adds town and industry marker `ImageView`s on top, and shows a + "{length} x {width} km" text. Whatever `std::function` generator it is given fills + the `Map` on a worker thread. The component has no `TerrainToolkit` or generator + state of its own. [DERIVED] +2. `CreateTextureData` (`63b9a0`) produces the image. It is point-sampled from the + `Map`'s full-resolution float heightmap and is at most about 1024 px on its long + side. Land uses a climate height ramp, a hillshade and a tree tint. Water uses a + depth ramp. The image is uploaded as a fresh raw RGBA8 texture through + `ImageView::SetImage(int,int,int,const vector&,bool)` (`22b4350`) and + `tex_util::LoadRaw` (`25a0f40`). Neither `CreateTextureAsync` nor `ReloadTexture` is + on this path. A refresh builds a new `ImageView` and a new texture. [DERIVED] +3. Markers are child `ImageView` components in a `CFloatingLayout` over the image, at + normalized positions with y flipped. They are not drawn into the texture. [DERIVED] +4. **Partly.** It already runs in game: all six non-menu users sit in + `UI::CGameUI::CreateConstructionMenu` (`5a2900`), the in-game map-editor terrain + tools. There it is fed from the live world by `map_util::MakeMap` (`360ce0`). As + used there it does not scale. `MakeMap` materialises the whole 4 m heightmap as + floats: 1.0 GiB at 256² tiles, 4.0 GiB at 512², and a 32-bit overflow above 724² + tiles. The component keeps that `Map` alive and rebuilds its whole UI subtree on + every refresh. The reusable pieces are lower down: `ImageView::SetImage` (raw), + `IRenderContext::UpdateTexture`-like slot `0x218`, and `CTerrain::BaseGetVertices` + (`33d130`). [DERIVED] +5. Lua can reach none of this. `MapPreviewComp`, `TextureTriangleBuffer`, + `EmissionLayerComp` and `DebugTextureView` have no sol usertype. The Lua + `ImageView` usertype has `new(string)` and `setImage(string, bool)` only, and both + resolve a **file path**. No Lua entry point takes pixels or a `TextureId`. [DERIVED] +6. The cheapest native height reader is `CTerrain::BaseGetVertices(CVec2i tile)` + (`33d130`, the ECS `TerrainTileHeightmap`, 65x65 uint16). Heights in metres are + `v * CTerrain+0x34 + CTerrain+0x3c`. The water level is `CTerrain+0x40`. The tile + rectangle is at `*(CTerrain+0x18)`. [DERIVED] + +## 1. `UI::MapPreviewComp` + +### Identity + +| Item | Value | Evidence | Class | +|---|---|---|---| +| RTTI / vtable | `.?AVMapPreviewComp@UI@@`, vtable `3019868`, 40 slots | `vftables.csv`, `vtable_dump.csv` | DERIVED | +| Base class | `UI::ContentView`: vtable identical to `388b050` except slot 0 (dtor `63b780`) and slot 33 (`63bdb0`, per-frame update) | `vtable_dump.csv` | DERIVED | +| Source file | `src\Game\UI\Components\MapPreviewComp.cpp`; asserts in `63ca90` and `63b9a0` | strings `3019b10` | DERIVED | +| Object size | 0x498 (`operator new(0x498)` at `66d90f` before the ctor call at `66d941`, and in `5b71b0` and `52f1b0`) | disassembly | DERIVED | +| Constructor | `63a190(this, const terrain::ClimateRep& climates, CVec2i size, bool flag)` | decompile; the size is packed in `r8` (`66d933`) and passed to `22810d0`, which converts it to floats and calls component vslot `0xc0` (min size). `flag` goes to `22819c0` on the text box | DERIVED (meaning of `flag`: GUESS, text visibility) | +| Destructor | `63b780`: destroys the std::function at +0x458, then the impl (`63b610`) | decompile | DERIVED | + +### Field layout (the parts that matter) + +`MapPreviewComp` (0x498): + +| Offset | Field | Evidence | +|---|---|---| +| +0x018 | core pointer (base `CComponent`). `*(core+0x60)` is the `FileSystem` used by `Update` | `63bdb0` → `5f0ef0` | +| +0x440 | `ContentView*`, the centred progress/error slot. It is also item 2 of the floating layout | ctor, `63c6a0` | +| +0x448 | `Impl*` (0xa0 bytes, below) | ctor | +| +0x450 | `const terrain::ClimateRep*` | ctor, `63bdb0` | +| +0x458 | `std::function`, the marker-overlay factory; impl pointer at +0x490 | dtor, `66d96e`, `7c5100` | + +`Impl` (0xa0): + +| Offset | Field | Evidence | +|---|---|---| +| +0x00 | `std::string`, the progress text | `63c4f0` | +| +0x20 | `std::function&, UpdateProgress&, UpdatePreview&, Map*)>`, the generator (impl at +0x58) | `63c4f0`, `63a010` | +| +0x60 / +0x68 | `std::shared_ptr` (a new 0x168-byte `Map` for each generation, `63b830`) | ctor, `63b830`, `63b610` | +| +0x70 | async task (0x40 bytes, ctor `31c6d0`). Byte +0 is the cancel flag, byte +2 means "result pending", +8 is the progress, and an `exception_ptr` sits at +0x30 | `63bdb0`, `31cb20`, `63c4f0` | +| +0x78 | `TextView*` ("main-menu" style) | ctor | +| +0x80 | `CFloatingLayout*`. Items 0 and 1 are `ContentView`s at (-2,-2); item 0 holds the image. Item 2 is `this+0x440` at (0.5, 0.5) | ctor, `63ca90`, `63c6a0` | +| +0x88 | `std::vector`, the RGBA pixel buffer. **It is kept after upload** | `63bdb0` passes `impl+0x88` to `Update` | + +### Who creates it and how it is fed + +| Creator | Call site | Feeds it with | Class | +|---|---|---|---| +| `UI::CMenuUI::CreatePageNewGame` (`66c2b0`) | `66d941`: `MapPreviewComp(*(CMenuUI+0x590), {512,512}, true)`, then the marker factory `7c5100(false,false)` is moved into +0x458 | `65b620` ("Generating map...") calls `63c4f0` with the New Game generator lambda (`644b30` → `63f200`, `do_call` `67d8c0` → `6568b0` → `35f480`, which reads `TerrainGeneratorDesc` and reports `UpdateProgress`) | DERIVED (roles of the individual combo boxes: GUESS) | +| `UI::ExportMapComp` (`52f1b0`) | `52fb3b`, size `DAT_144133390` = (512,512) in `.data` | `5315d0` ("Generating preview..."): `MakeMap(out, ?, ?, terrain, engine, heightmap=1, towns=0, industries=0, trees=0)` at `53163f`, then a generator that copies the ready `Map` (`52e9c0`) | DERIVED | +| `GenerateHeightmapComp` (`5b71b0`) | `5b7334` | lambda `5ba740` ("Generating map...") | DERIVED | +| `GenerateIndustriesComp` (`5bef50`) | `5bf15c` | lambda `5c11c0` ("Generating industries...") | DERIVED | +| `GenerateStreetsComp` (`5c4580`) | `5c4ce7` | `5c76e0` ("Generating streets...") | DERIVED | +| `GenerateTownsComp` (`5cbc80`) | `5cbe9a` | `5ce260` ("Generating towns...") | DERIVED | +| `ImportHeightmapComp` (`5e79c0`) | `5e7fed` | `5e95f0` ("Reading heightmap...") | DERIVED | + +The six map-editor components are constructed only by +`UI::CGameUI::CreateConstructionMenu` (`5a2900`, called from `CGameUI::CreateUI` +`569f00` at `56e8ca`). They are gated on CGameUI bytes `+0x50d`/`+0x50e`/`+0x50f`; +`ExportMapComp` needs `+0x50f`. [DERIVED; the flags' meaning (map-editor capabilities) +is a GUESS.] The in-game style sheet also styles the component: +`construction-menu.lua` has `!construct-editor-menu MapPreviewComp` (350x350) and +`!construct-menu MapPreviewComp` (600x600) [FILE]. + +**Public entry point.** `63c4f0(this, const std::function&, const +std::string& text)` (name GUESS: `StartGeneration`). It stores the generator and the +text. If no task is running, `63c6a0` resets the `Map` (`63b830`), places a progress +bar in layout item 2 and starts the task. The task binds the generator to the `Map*` +(`63a010`). If a task is already running, it sets the cancel flag and the restart +byte. [DERIVED] + +**Per-frame slot 33 (`63bdb0`).** It returns at once when `impl+0x70` is null. When +the task has finished and nothing is pending, it calls `Update` (`63ca90`) on the UI +thread and frees the task. [DERIVED] + +### `Map` (0x168; ctor `35bcb0`, filled by `MakeMap` `360ce0`) + +| Offset | Field | Evidence | Class | +|---|---|---|---| +| +0x00 / +0x04 | int tiles X / Y | `63ca90`, `360ce0` (from `CTerrain` `33d540`) | DERIVED | +| +0x08 | int `levels` (base; samples per tile = `1< heightmap`, `((X<` (16 B: int id, CVec3f pos) | `63ca90` tree loop | DERIVED | +| +0x118 | towns, 0x80 B each: +0x00 `CVec2f pos`, +0x08 name, +0x78 int existing-entity or -1 | `7c4560`, `35cf20` | DERIVED | +| +0x130 | industries, 0x38 B each: +0x00 `CVec2f pos`, +0x10 name, +0x30 int existing or -1 | `7c42b0` | DERIVED | +| +0x148 | `std::string` climate id | `63ca90` (`ResTypeRep` lookup) | DERIVED | + +The two 0x40-byte objects at +0x38 and +0xa8 and the vectors at +0x78, +0xe8 and ++0x100 were not identified (GUESS: assets and generator-internal lists). + +## 2. The texture + +### Size [DERIVED, `63ca90`] + +``` +hmW = (X << levels) + 1 hmH = (Y << levels) + 1 +pw = 1 << ceil(log2(X << levels)) ph = 1 << ceil(log2(Y << levels)) +fact = max(pw,ph) <= 1024 ? 1 : floor(max(pw,ph) / 1024 + 0.5) // assert texSizeFact > 0 +texW = pw / fact texH = ph / fact +UV max = (hmW / (texW*fact), hmH / (texH*fact)) -> ImageView::SetCustomTextureCoordinates (22b4300) +``` + +`fact` is also the **sampling stride**: texel (x,y) reads heightmap sample +(`fact*x`, `fact*y`) with no filtering. + +| Map (tiles) | levels | heightmap samples | texture | stride | metres per texel | +|---|---|---:|---|---:|---:| +| 114x228 | 6 | 7,297 x 14,593 | 512 x 1024 | 16 | 64 | +| 256x256 | 6 | 16,385² | 1024 x 1024 | 16 | 64 | +| 512x512 | 6 | 32,769² | 1024 x 1024 | 32 | 128 | + +(DERIVED, assuming levels = 6 and resolution 4 m, as in `runtime-memory-audit.md`.) + +### `CreateTextureData` (`63b9a0`) + +Full signature (string `30199b0`): +`CreateTextureData(int hmW, int hmH, const CVec3f& res, float waterLevel, const +vector& heights, int texW, int texH, int fact, const +terrain::PreviewColoring& pc, const std::function& +colorFn, vector& out, int nChannels)`, with the assert `nChannels == 4`. + +The row loop is `ThreadPool::LoopImpl` (`639420`), one row per item, with chunks of +`texH/64` rows. The body is `63aa90`. The UI thread blocks until the loop finishes. +[DERIVED] + +Before the call, `Update` zeroes `out` and adds 2 to byte R of the texel under every +`MapTree` (`63ca90`). Per texel with `h = heights[hmW*fact*y + fact*x]` [DERIVED]: + +- **Water (`h < waterLevel`):** + `t = clamp((h - wl) / (-100 - wl), 0, 1)`, + `rgb = lerp(pc.waterColor0, pc.waterColor1, t)`. No shading. +- **Land:** + - Normal from central differences on immediate neighbours (±1 sample, not ±`fact`): + `n = normalize(-(h[x+1]-h[x-1])*(0.5/res.x)*2, -(h[y+1]-h[y-1])*(0.5/res.y)*2, 1)`. + - Sun term `s = max(0, n·(-1,1,1)/√3)`. + - Ambient term `a = (3*n.z + Σ max(0, n·d_i)) / (3 + Σ d_i.z)` over 24 hemisphere + directions (3 rings at `θ=(r+0.5)/3·π/2`, 8 azimuths). + - Base colour `c = colorFn(h, x, y)`. + - Tree tint from the pre-accumulated byte `k`: `t = max(0,k-0.75)*0.75/(k+1)`. + - `rgb = lerp(c, pc.treeColor, t) * (a*pc.ambientColor + s*pc.sunColor) * 0.5`. +- Channels: R, G, B = `floor(v*255+0.5)`, A = 255. Format RGBA8 (4 channels). + +### `terrain::PreviewColoring` (climate `mapColoring`) [DERIVED from reader `36f090`, default ctor `36d810`] + +| Offset | Lua key | Default | +|---|---|---| +| +0x00..+0x38 | `std::variant>`, index byte at +0x38: `texture = {levels, fileName}` (0) or `levels = {{color, height}}` (1) | index 1: (93,112,66)@0 m, (64,78,50)@100 m, (242,235,220)@550 m | +| +0x40 | `waterColor0` | (100,135,158) | +| +0x4c | `waterColor1` | (60,80,91) | +| +0x58 | `ambientColor` | (204,230,255) | +| +0x64 | `sunColor` | (255,255,204) | +| +0x70 | `treeColor` | (24,36,16) | + +- The `levels` colour function (`63e1d0`) binary-searches the height and interpolates + linearly, clamping at both ends [DERIVED]. +- The `texture` variant loads `res/textures/` (must be `.tga`) and + `res/textures/terrain/noise.tga`, and samples through `63a800` [DERIVED, sampler not + decompiled]. +- Shipped climates [FILE]: `temperate.clima.lua` sets only `ambientColor`/`sunColor` + and so gets the default ramp. `dry.clima.lua` and `tropical.clima.lua` define + `levels`, and `tropical` also defines water colours. + +### Upload, and updating after creation [DERIVED] + +1. `Update` allocates an `ImageView` (0x488) and calls the raw-pixel ctor + `22b2f90(this, 4, texW, texH, const vector&, CVec2i frames=(-1,-1), int + numFrames=-1)` (`63d84a`). +2. The ctor calls `ImageView::SetImage(int nChannels, int w, int h, const + vector&, bool validate)` (`22b4350`, assert `size == 4 || size == 3`): + - It releases any previous texture (`22b4240`). + - It builds a `TextureCreateInfo` (`22b4160`, `25a0c60`). + - It calls `tex_util::LoadRaw(IRenderContext&, TextureFormat (RGB_8=2, RGBA_8=3), + w, h, const uint8*, const TextureCreateInfo&)` (`25a0f40`). That is + `IRenderContext` vslot `0x1f8` = `CreateRawTexture(const void*, TextureFormat, + CVec2i, const TextureCreateInfo&)`: GL `257c180`, Vulkan `25d29c0`. Vulkan + `vulkan_util::LoadRawTexture` (`25f3f00`) runs synchronously, and only the data + pointer (`vector.begin`) is read. + - It stores a 0x14-byte `Texture {TextureId id; int 0; int w; int h; int + channels}` at `ImageView+0x440`. + - The render context comes from `engine::GetGlobalGLContext()` (`daf90`, global + `0x4331bb8`), field `+0x38`. +3. `SetCustomTextureCoordinates` (`22b4300`), the style name `map-preview-image`, + and `ContentView::SetContent` (`2286020`) into layout item 0. +4. **After creation:** + - `MapPreviewComp` never updates the texture in place. Each generation creates a + new `ImageView`. + - Calling `ImageView::SetImage` again destroys and recreates the texture + (`22b4240` → `25a1700` → vslot `0x200`, destroy a `vector`). + - The render context does have an in-place sub-rectangle update: vslot **`0x218`**. + GL `25840d0` does `glPixelStorei(UNPACK_ALIGNMENT,1)`, `glBindTexture`, then + `glTexSubImage2D(GL_TEXTURE_2D, 0, off.x, off.y, size.x, size.y, , + , data)`. Vulkan `25dd130` does a staging upload through `25f6440` + (`CmdBegin`, `CmdLayoutTransition`, `CmdImageCopy`). + - The engine uses it in game for `engine::ColorMap::UpdateSubData` (`d8760`), + reached from `UI::EmissionLayerComp::UpdateTexture` (`52e050`). Call shape: + `rc->vtbl[0x218](rc, TextureId, CVec2i offset (r8), CVec2i size (r9), const + void* data ([rsp+0x20]))`. The pixel layout must match the texture's creation + format. + - `OpenGLRenderContext::ReloadTexture` (vslot `0x208`, GL `2583870`) takes a + `unique_ptr` and checks `textureData.reloadFn`. It serves + file-backed and streaming textures and is not needed here (GUESS). + `CreateTextureAsync` is not on this path. + +## 3. Markers [DERIVED] + +- The `std::function` at +0x458 is + `_Func_impl` (factory `7c5100(bool existingTowns, bool + existingIndustries)`, `do_call` `7c5170` → `7c4bb0`). +- `7c4bb0` builds one `CFloatingLayout` (`22ae540`). It adds + `IndustryMarkers(map+0x130)` (`7c42b0`) and `TownMarkers(map+0x118)` (`7c4560`), each + its own `CFloatingLayout`. +- Each marker is `new ImageView("ui/icons/main-menu/map_town.tga" | + "map_industry.tga")` (path ctor `22b30e0`): + - style name `MapPreviewComp::TownMarker` / `::IndustryMarker`; + - tooltip from the name (component vslot `0xd8`); + - style class `existing` when the flag is set and the entity id is ≥ 0; + - `CFloatingLayout::AddItem(marker, u, v)` (`22ae690`). +- Position mapping, lambda `do_call` `5c9a90` (COMDAT-shared): + +``` +origin = (-(X/2)*(1<vtbl[1](acc)`, then `CTerrain* = *(state+0x20)` and `ecs::Engine* = + *(state+0x28)`. +- In `CGameUI` the accessor is `*(CGameUI+0x450)` (`5a2900` passes `param_1[0x8a]`). + The accessor class was not identified. +- The in-game `ClimateRep` is `*(*(*(CGameUI+0x448)+0x150)+0xd8)` (`5a2900` → `5b71b0` + argument). + +### Why it does not scale as used + +| Cost | 114x228 | 256² | 512² | Evidence | Class | +|---|---:|---:|---:|---|---| +| `MakeMap` float heightmap `(64X+1)(64Y+1)*4` | 406 MiB | 1.00 GiB | 4.00 GiB | `360ce0`, `35d3e0` | DERIVED | +| ...kept by the component (`shared_ptr` at impl+0x60) until the next generation or destruction | same | same | same | `63b830`, `63b610` | DERIVED | +| RGBA pixel buffer kept at impl+0x88 | 2 MB | 4 MB | 4 MB | `63bdb0`, `63ca90` | DERIVED | +| Heightmap pass: 16- or 32-sample stride, one full-map pass per refresh | | | | `63aa90` | DERIVED | +| Per refresh: a new `ImageView` + texture + every marker `ImageView` | | | | `63ca90` | DERIVED | + +Further points [DERIVED unless marked]: + +- **The sample count is 32-bit.** `MakeMap` computes it with `imul edx, eax; movsxd` + (`360ec6`), so any map with `(64X+1)(64Y+1) > 2^31-1` (square maps above 724² tiles) + gets a wrapped, undersized vector. The fill lambda also indexes with int arithmetic. + GUESS: heap corruption. +- **Side finding.** `industry_util::CreateAndAddIndustry` (`92c1d0`) calls `MakeMap` + with `heightmap=1, towns=1, industries=1` at `92c295` (bytes at `92c26d..92c281`). + If that path runs when an industry spawns during play (GUESS; its caller `152e90` is + a `GameSim.cpp` "Connect industry" lambda), every industry creation on a 512² map + allocates a transient 4.0 GiB float heightmap. The spawn-decision lambda `152ab0` + asks for industries only. **Verify before trusting.** It matters for bigmap + regardless of the minimap. +- It has no camera rectangle, no roads, rails or water meshes, and no click handling. + The layout is the New Game widget: text, a progress bar during generation, and a + "Map generation failed" error slot (`63c0b0`). + +### Hard dependencies + +| Dependency | New-Game-only? | Notes | +|---|---|---| +| `terrain::ClimateRep&` | No: `CMenuUI+0x590` in the menu, `GameRes+0xd8` in game | only for `PreviewColoring` | +| `Map` producer (`std::function`) | No | any function that fills `Map*`. `TerrainToolkit` / `ScriptGenerator` live only in the New Game lambda (`67d8c0` → `6568b0` → `35f480`), not in the component | +| `FileSystem` (`core+0x60`) | No | only for the `texture` colouring variant | +| Style names `map-preview-image`, marker styles | FILE: marker styles live in `main-menu.lua`; `construction-menu.lua` styles the component | GUESS: whether `main-menu.lua` rules apply in game | +| Global `ThreadPool` (`23827c0`) | No | row loop and async task | + +### Reusable lower-level pieces + +| Piece | RVA | Use | +|---|---|---| +| `ImageView` raw ctor | `22b2f90` | native-owned image component | +| `ImageView::SetImage(raw)` | `22b4350` | (re)create the texture from RGBA bytes on an existing `ImageView` | +| `ImageView::SetCustomTextureCoordinates` | `22b4300` | crop padding | +| `ImageView::SetImage(string)` | `22b4610` | the only method Lua reaches; detour target | +| `Texture` struct at `ImageView+0x440` | | `TextureId` at +0 | +| `GetGlobalGLContext` global | `0x4331bb8` (+0x38 = `IRenderContext*`) | render context | +| `IRenderContext` vslot `0x218` | GL `25840d0`, VK `25dd130` | in-place sub-rectangle upload | +| `CreateTextureData` | `63b9a0` | reference algorithm. Callable in principle, but it needs game-ABI `std::function` and `vector` objects and a pre-sized buffer. Reimplementing it (section 2) is simpler | +| `map_util::MakeMap` with `heightmap=false` | `360ce0` | towns and industries as `Map` records | +| `UI::DebugTextureView` | ctor `22a88e0` (0x458), `TextureId` at +0x440 | draws any `TextureId`, but through a debug-only draw `249a9c0` (single caller) with scale, flip and threshold fields. GUESS: may not show RGBA faithfully. `ImageView` uses the common quad `249b030` (12 callers) | + +### Minimal native call sequences + +**A. Raw texture behind an `ImageView`** (recommended core) [DERIVED call shapes] + +Run on the UI thread, inside a call where the `ImageView` is known to be alive: + +``` +// create or replace +struct { uint8_t* b, *e, *c; } px = { buf, buf + W*H*4, buf + W*H*4 }; // only b is read +ImageView_SetImage_raw(iv /*rcx*/, 4, W, H, &px, true); // 22b4350 +// in-place region update later +TextureId id = (*(Texture**)(iv + 0x440))->id; +IRenderContext* rc = *(IRenderContext**)(*(void**)(base + 0x4331bb8) + 0x38); +rc->vtbl[0x218/8](rc, id, CVec2i{x,y} /*r8*/, CVec2i{w,h} /*r9*/, rgba /*[rsp+20]*/); +``` + +**B. The component itself with a reduced `Map`** [DERIVED; std::function ABI GUESS] + +``` +p = operator_new(0x498); // 2bf3a80 +MapPreviewComp_ctor(p, climateRep, CVec2i{512,512}, true); // 63a190 +move marker std::function from 7c5100(tmp,false,false) into p+0x458 +Map m: X,Y; levels=2; resolution=(64,64,zScale); offset; waterLevel; + heightmap (4X+1)(4Y+1) floats sampled natively (16 MB at 512²); + towns/industries from MakeMap(&tmp, ?, ?, terrain, engine, 0, 1, 1, 0) on the UI thread +StartGeneration(p, std::function{ copy m into Map* }, "…") // 63c4f0 +insert p into a layout (no Lua route); slot 33 does the rest +``` + +Keep `resolution.xy * (1<@UI@@`): +- Components and layouts: `AbsoluteLayout`, `AbstractSlider`, `Button`, + `CAbstractLayout`, `CBoxLayout`, `CComponent`, `CFloatingLayout`, `Chart`, + `CheckBox`, `ColorChooser`, `ColorChooserButton`, `ColorPicker`, `ComboBox`, + `ContentView`, `CProgressBar`, `CRendererComponent`, `CTextInputField`, + `DoubleSpinBox`, `ILayout`, `ILayoutItem`, `ImageView`, `LayoutBase`, + `LineRenderView`, `List`, `ScrollArea`, `Slider`, `Slider2`, `SpinBox`, + `StationGroupDisplayComp`, `Table`, `TabWidget`, `TextView`, `ToggleButton`, + `ToggleButtonGroup`, `Window`. +- Non-components: `CCore`, `CGameUI`, `CameraController`, `ViewManager`. +- Registration of `ImageView` is in `RegisterUsertypesGui1` (`18fb6e0`). + +**Not bound:** `MapPreviewComp`, `TextureTriangleBuffer`, `EmissionLayerComp`, +`DebugTextureView`. No metatable string exists for any of them. + +- `TextureTriangleBuffer` (`868680` `SetTexture(TextureId)`) is not a UI component. It + owns GPU vertex and index buffers (`867270`), and its users are + `brush_util::Brush::UpdateBrush` (`485550`) and + `track_render_util::AddOneWayTextureTriangles`. GUESS: world-space overlays. + +**Lua `ImageView` usertype** (`usertype_metatable` at `42cb270`): +- `"new"`: factory `18db550` → path ctor `22b30e0(this, path, 0, 0, -1, -1, -1)`. +- `"setImage"`: lambda `1863370` → `22b4610(this, std::string path, bool)`. That + resolves `tex_load::ResolveTex` (`259d4c0`, empty string becomes + `ui/missing_image.tga`) → `22b4700` → `TextureManager::AddRef(TextureDesc)` + (`2490820`). +- Two read-only properties with 7- and 9-character names that were not resolved. +- Base classes `CComponent`, `ILayoutItem`. + +The legacy `game.gui` table (`113f530`) exposes `imageView_create` (`1139200` → +`22b30e0`) and `imageView_setImage` (`112a770` → `22b4610`), which are also path-only. +**No Lua path accepts raw pixels or a `TextureId`.** The only Lua-reachable code that +touches a texture is the path string passed to `22b4610`. + +`LineRenderView::DoRender` (`61a9f0`) rebuilds a two-vertex-per-line array every frame +from the line vector at +0x458. No cap was found in it or in the start of the UI line +draw `24995d0`; where the reported ~5,000-line limit comes from was not established. + +## 6. Live world data for the native side + +### `CTerrain` fields [DERIVED: getters `33d270`, `33d280`, `33d330`, `33d340`, `33d540`, `33d560`, `33d770`, `33d790`, `33d7f0`] + +| Offset | Meaning | +|---|---| +| +0x08 | `ecs::Engine*` (used by `BaseGetVertices`) | +| +0x18 | tile grid header `{int x0, y0, W, H; cells* at +0x10}`, cells 0x28 bytes, tile entity id at cell+0 (-1 = none) | +| +0x28 | int base levels (6 → 65x65 per tile) | +| +0x2c..+0x37 | `CVec3f` base resolution (x, y metres per base sample; z height scale) | +| +0x38 | int high levels (8 → 257x257 cache) | +| +0x3c | float height offset | +| +0x40 | float **water level** (also `ITiledTerrain` slot 10, `156a30` / `34e1d0`) | +| +0x50 | component type index of `TerrainTileHeightmap` | + +- Map extent: `W*256 m x H*256 m`, tile rectangle `[x0, x0+W) x [y0, y0+H)`. +- World origin as the preview uses it: `(-(W/2)*256, -(H/2)*256)` [DERIVED from + `63ca90`/`7c4bb0`; GUESS for odd W or H, verify against a known position]. + +### Height readers + +| Reader | What it returns | Paging interaction | Class | +|---|---|---|---| +| **`CTerrain::BaseGetVertices(CVec2i tile)`** `33d130` | `const vector&`, 65x65 base heightmap from the ECS `TerrainTileHeightmap`. Lock-free component lookup (`engine+0x88[cti]`, dense +0x68 or paged +0x80). Asserts if the cell id is -1 | not paged by the plugin (the pager covers the 257² cache only, `terrain_pager.h`) | DERIVED | +| metres | `v * *(float*)(CTerrain+0x34) + *(float*)(CTerrain+0x3c)` (exactly what `MakeMap` does, `35d3e0`) | | DERIVED | +| `sub_terrain_util::GetHeightmap(int srcLevels, const vector&, int dstLevels, x0, y0, x1, y1, uint16* out, stride, outX, outY)` `3ac330` | resampling to a coarser level (`3acf50`) or the same level (`30a540`) | pure CPU | DERIVED | +| 257² cache via `33d580` cell +0x08 | 1 m heights | faults the bigmap pager | DERIVED | + +Water in the editor can also be `ecs::component::WaterMesh` entities +(`EngineTerrain::GetWaterMeshes` `2f7ed0`). The preview ignores them and uses the +global level only [DERIVED]. Lakes placed at other levels would need those meshes +(GUESS). + +**Cost estimate** (GUESS, not measured): +- A 1024² minimap of a 512² map needs 2x2 samples per tile: 262,144 + `BaseGetVertices` calls and about 1 M sample reads, roughly 20-80 ms + single-threaded, dominated by cache misses on 8.4 KB tile vectors. +- 2048² (4x4 per tile) is about 4 M reads. +- Only terraformed tiles need re-sampling later. Tile publication `33cd10` (already + patched by `terrain_minmax.h`) is the natural dirty signal. + +**Which `CTerrain`.** The two game states each hold a `CTerrain` (`33d240` copies the +grid, `runtime-memory-audit.md`). The UI reads through the accessor chain in section 4 +(`*(CGameUI+0x450)` → vslot 1 → +0x20). Capturing a raw `CTerrain*` once risks reading +the version the simulation is writing (GUESS). Re-derive it per use. + +**Getting `CGameUI`.** +- `5741d0` (`CGameUI` per-frame update) is already detoured by `tpf2_menu.dll` + (`tpf2-multiplayer/native/src/menu_hook.cpp`, 21-byte steal). A second + `verifyBytes`-guarded hook there would fail when both are loaded. +- Candidate instead: `CreateConstructionMenu` `5a2900`, which runs once per world with + `this` = `CGameUI`. Its prologue `40 55 56 57 41 54 41 55 41 56 41 57 48 8d ac 24 c0 + fa ff ff` is 20 bytes with no RIP-relative operand. [DERIVED] + +Towns, industries, the camera and the street/track network: use Lua (`api.engine`). +`MakeMap(towns/industries only)` is a native alternative but is not cheaper for a few +hundred entities (GUESS). + +## 7. Ranked implementation approaches (512² tiles and beyond) + +### 1. Native RGBA texture behind a Lua-owned `ImageView` (magic `setImage` path). **Recommended.** + +**Native side:** +- Detour `ImageView::SetImage(string)` at `22b4610`. Steal 20 bytes + `40 53 56 57 48 81 ec 90 00 00 00 48 c7 44 24 20 fe ff ff ff`; the next + instruction at `22b4624` is RIP-relative. It has 49 calling functions (70 call + sites), so the non-magic path must be one length check plus a `memcmp`. +- On a magic string, e.g. `bigmap:minimap` (15 chars or fewer, so it stays in the SSO + buffer): + 1. Overwrite the string with a real small UI texture path using the game's + `basic_string::assign` (`83270`), and call the original. That keeps by-value + string ownership stock. + 2. Call `ImageView::SetImage(raw)` `22b4350` with the plugin's RGBA buffer. +- On `bigmap:minimap:update`, push dirty rectangles through `IRenderContext` vslot + `0x218` using `*(ImageView+0x440)->id`. +- Heights come from `CTerrain::BaseGetVertices` `33d130` through the accessor chain. + Colouring reimplements section 2: height ramp from the climate, water depth ramp, + hillshade. +- Roads and rails can be rasterized into the same texture (no line cap) once their + data reaches native code (see risks). + +**Lua side:** +- Window, `api.gui.comp.ImageView.new("")`, then + `setImage("bigmap:minimap", false)` on open and a timed `bigmap:minimap:update`. +- Town and industry markers and the camera rectangle as child components in a + `FloatingLayout` or `AbsoluteLayout`. +- Click → world position → camera move. +- Refresh cadence is Lua's choice. Every native texture write happens synchronously + inside that Lua call, on a live `ImageView`. + +**Risk:** medium-low. Raw-texture `ImageView`s created on the UI thread have in-game +precedent (the map editor). + +**What could break:** +- A game update moves `22b4610`, `22b4350`, `ImageView+0x440`, vslot `0x218` or + `0x4331bb8`. `verifyBytes` catches code, not field offsets. +- Vulkan in-place updates may differ from GL (unmeasured); the fallback is a full + `SetImage` at about 4-16 MB per refresh. +- The texture dies with the `ImageView`, so native code must never cache the + `TextureId`. +- A wrong `CTerrain` (sim-side) read. +- No native road/rail reader exists yet: needs ECS `BaseEdge`/`BaseNode` layouts or a + Lua→file channel. + +### 2. The same texture in a native-owned UI + +- `ImageView` raw ctor `22b2f90` plus native window and layout insertion into + `CGameUI`, with native markers and input. +- Lua does nothing. +- **Risk:** medium-high. The window, layout, input and lifetime APIs were not + researched. +- **What could break:** UI tree ownership (double free when the game tears the UI + down), style sheets, input focus. + +### 3. `MapPreviewComp` with a reduced `Map` + +Sequence 4.B: +- `levels=2`, resolution 64 m, 16 MB of floats at 512². +- `MakeMap(towns, industries)` only, run on the UI thread before `StartGeneration` + `63c4f0`. +- The engine's own colouring and markers are reused. Hosting is still native, because + Lua cannot reference the component. + +**Risk:** medium. + +**What could break:** +- A game-ABI `std::function` must be built in the plugin (layout stable since + VS2015 — GUESS). +- Every refresh rebuilds the `ImageView`, texture and markers and flashes a progress + bar. +- The `Map` plus a 4 MB pixel copy stay resident. +- At `levels=2` the hillshade neighbour step (one sample, 64 m) equals the sampling + stride, so shading is not aliased the way it is at `levels=6` (4 m neighbours + under a 128 m stride). There is still no camera overlay, roads or in-place updates. +- Marker styles may not apply in game (FILE vs GUESS). + +### 4. File-backed texture for a plain Lua `ImageView` + +- Native writes a TGA into a mod `res/textures` folder; Lua calls `setImage(path)`. +- **Risk:** medium. +- **What could break:** + - `TextureManager::AddRef` caches by `TextureDesc` (`2490820`), so a refresh needs a + new file name each time, and old textures accumulate until released. + - `ReloadTexture` needs a `reloadFn` (GL `2583870`) and has no Lua route. + - Disk I/O of 4 MB or more per refresh; stale images. + +### 5. `MapPreviewComp` fed like the map editor (`MakeMap` with `heightmap=true`). **Rejected.** + +4.0 GiB of floats at 512², kept by the component; a 32-bit sample count above 724² +tiles; full UI rebuild per refresh [DERIVED]. + +### 6. A second top-down `CRendererComponent`. **Rejected (GUESS).** + +It renders the 3D scene again, with far-plane and LOD limits over 131 km and a large +frame cost. It is not researched here. + +## Open questions and how to check them read-only + +1. **Vulkan in-place update.** Does vslot `0x218` on an `ImageView`-owned raw texture + display correctly? It needs an in-game test build; code reading cannot settle it. +2. **The industry-spawn `MakeMap` (`92c1d0`).** Log calls and `(X,Y)` on a 256² save. + A working-set spike of about 1 GiB per industry creation would confirm it. +3. **World origin for odd tile counts.** Compare a known town position from + `api.engine` with the formula in section 3. +4. **The accessor class behind `*(CGameUI+0x450)`.** Confirm that its vslot 1 returns + the UI-side (read) game state. + +## Reproduction + +Decompiles are in the session scratchpad `mm\b1`..`mm\b6`, each with its `tN.txt` +target list. They were generated with `tpf2-multiplayer/tools/ghidra/run.ps1 +DecompileTargets.java 240 -ProjDir `. Vtable slots, prologues and float constants were read from +the executable with `pefile` + `capstone` (read-only). String and call-edge lookups +used `C:\tools\ghidra_out\{strings,funcsig,func2src,call_edges,vftables,vtable_dump}.csv`. diff --git a/docs/minimap.md b/docs/minimap.md new file mode 100644 index 0000000..c83113f --- /dev/null +++ b/docs/minimap.md @@ -0,0 +1,165 @@ +# In-game minimap (`minimap=1`) + +Built September 16, 2026. Tested offline (`tools/test_minimap.py`); **not yet run +in game**. Reverse engineering: `minimap-native-preview.md`. + +## Why a new one + +The Workshop "Minimap" mod (3256290611) cannot be patched into working on big +maps. Four things in it are architectural: + +- It finds the map edge with a linear scan that stops at 50,000 m. A 320-tile map + is +-40,960 m and a 512-tile map +-65,536 m. +- It samples heights every 10 m from Lua: about 43 million calls at 256x256 tiles. +- It paints terrain as one UI component per cell. +- It caps roads and rails at about 5,000 lines. + +## Design + +| Part | Where | What it does | +|---|---|---| +| Game script | `mod/minimap/bigmap_minimap.lua`, embedded in the DLL by `tools/embed_lua.ps1`, written to `res/config/game_script/bigmap_minimap.lua` at start | Toolbar button, window, town and industry markers, camera circle, click-to-move, layer toggles, Refresh. GUI only: no `update`, `save` or `load` | +| Request | Lua: `imageView:setImage("bigmap:minimap?x0=&y0=&x1=&y1=&w=&h=&relief=&g=")` | Lua probes the map extent (binary search on `isValidCoordinate`) and owns the world-to-picture mapping, so the image, the markers and the clicks share one definition | +| Detour | `ImageView::SetImage(string)` `0x22b4610` | A 14-byte prefix check for every other image. On a token: the placeholder `ui/icons/main-menu/map_town.tga` is written into the string's own buffer (or an empty path if it does not fit), the original consumes it, then the texture is replaced | +| Terrain | `*(CGameUI+0x450)` -> `vtbl[1]` -> `+0x20` = `CTerrain`; `CGameUI` captured by a 25-byte thunk on `CreateConstructionMenu` `0x5a2900` | The thunk stores `rcx` and jumps on without touching the stack, so the function's unknown arity does not matter | +| Heights | `CTerrain::BaseGetVertices` `0x33d130` (65x65 uint16 per tile) | Not paged by the terrain pager. Empty cells are skipped before the call, which would assert | +| Colour | reimplementation of `CreateTextureData` `0x63b9a0` | Default climate ramp, water depth ramp, hillshade (`relief`), 24-direction ambient term. Shaded on up to 8 threads | +| Upload | `ImageView::SetImage(raw)` `0x22b4350`, RGBA8 | The path the engine's own map preview uses; synchronous | + +Twelve sites are byte-verified before anything is hooked, including the getters +that pin every `CTerrain` offset the renderer reads. Anything that fails leaves +the game untouched and deletes the script, so no button appears without working +hooks. The script is only ever deleted if its first line is the plugin's marker. + +## Offline tests (`tools/test_minimap.py`) + +- The token parser, including 16 rejected forms. +- Every rendered height against an independent model of the engine layout: + origin, packed tile argument, row = y, south in row 0, a missing tile and + pixels outside the map. Every colour is checked to within 1 across water, + land, outside and the multithreaded path. +- The accessor chain, including the vtable range check. +- The detour: non-tokens pass through untouched. Tokens keep their buffer and + capacity, and the uploaded bytes equal a direct render. +- The thunk: executed with four register and two stack arguments. +- The installer against the Steam executable, plus refusal paths. +- Script sync, including never touching a foreign file. +- The Lua script under a mocked game API, plus the multiplayer repo's luacheck. + +## First in-game session (September 16): what it showed + +On a 192x192-tile save at a 175% UI scale (3038x1918 screen) the terrain +rendered correctly and fast (`1024x1024 px ... in 40 ms`), but the window showed +only the lower half of an oversized picture, the markers crowded its top-left +57%, and the camera circle sat off its true position. Measured from the +screenshot, the cause was two unit systems: + +- `setMinimumSize` takes **UI units**, multiplied by the UI scale on screen: a + 900-unit picture was 1,576 px wide. +- `AbsoluteLayout` rect positions, `getContentRect()` and `getMousePos()` are + **screen pixels**: markers placed at 0..900 spread over 0..900 px. The stock + `selectortooltip.lua` mixes these three the same way. +- `AbsoluteLayout` appears to read `rect.y` as the item's **vertical centre**. A + picture at `y = 0` showed exactly its lower half (786 of 1,576 px). The + Workshop minimap placed full-height views at `y = height/2`. INFERRED from that + evidence, not from code. + +The script now measures the UI scale from the map container's content rect, +places everything in screen pixels, and checks the picture's measured position +once per session. If the `rect.y` rule is the other way, it rebuilds with the +other rule. Clicks use `getMousePos()` minus the picture's rect. The toolbar +icon is `ui/button/medium/terrain@2x.tga` (from `res/textures/ui/ui.zip`), not +the Workshop mod's `map@2x.tga`. The log line +`layout: UI scale S (guessed G), AbsoluteLayout y is the ...` records what was +found. + +## Roads, tracks, stations, companies and industry types (September 16) + +All of these are drawn into the picture natively, so a network of any size has +no line cap and no widget per edge. + +- **Gathering (Lua).** When a map build's layout is confirmed, the script reads + every `BASE_EDGE` and then every `STATION` on a 6 ms per-frame budget, showing + "Loading network N%". For each edge it records the class (track, or a street's + `categories[1]`: urban, country or highway; one-way types are told apart by + name), both node positions and both Hermite tangents. For each station it + records the type (rail, road, water or air from the construction file name), + centre, direction and half length. Roadside stops are edge objects, placed via + `getEdgeObject2EdgeMap`. Owners come from `PLAYER_OWNED`. +- **Transport.** The network travels once as text after the token's header: + `P index r g b`, `E class colour x0 y0 tx0 ty0 x1 y1 tx1 ty1`, and + `S class colour x y ux uy half`. The plugin caches it under the header's `net` + id. `layers` (1 roads, 2 tracks, 4 stations) and `hide` (a bit mask of owner + palette indices) re-render from the cache with a header-only request. A request + whose id does not match the cache draws no network, so another world's network + never shows. +- **Drawing (native).** Anti-aliased strokes along each edge's cubic Hermite + curve, with the engine's own tangents. Draw order is roads (urban 70% blend, + country, highway), then tracks, then stations as outlined bars. +- **Companies.** In the multiplayer mod's companies mode, owners map to company + ids through the lobby's `mp_company_cfg.txt`. This machine's player is its + company; the other engine players, in ascending id order, follow the roster + order they were created in. Players that own nothing still count, so the + mapping does not shift. Colours are the lobby chip colours (the + `CM.cmCompanyColor` algorithm, copied). Without a roster, two or more owners + still get distinct colours (this player first). A lone player's network keeps + the classic track colour and shows no company legend. +- **Industry types.** Each industry construction type is listed with the icon of + the cargo it produces and a count, and can be toggled. Markers use the same + icon. The cargo is resolved per type before any marker is placed, in this + order: what any industry of the type has produced + (`game.interface.getEntity(simBuilding).itemsProduced`); the first + `output = { CARGO = n }` table in the type's own construction file, read from + each mod root on `package.path` (`/res/scripts/?.lua`); a built-in table of + the 16 stock industries (their files are zipped in `construction.zip`, so they + cannot be read); what one has taken in; else `map_industry@2x.tga`. +- **Legend.** Show toggles for towns, industries, roads, tracks, stations and the + camera; one row per company, with its colour and a toggle; one row per + industry type. + +A real bug found offline on the way: Lua 5.2's `%d` goes through a 32-bit C +`long` on Windows and refused the network id. Ids and masks are now formatted +with `%.0f`. + +## To confirm in game + +The offline mock cannot settle these. Each is logged or visible: + +1. **The script loads.** The stdout line `[bigmap minimap] installed` should + appear and a map button should show on the main toolbar. This proves + `res/config/game_script` files load without a mod entry. +2. **The picture renders.** Look for `minimap: rendered WxH px ... in N ms` in + `tpf2mp_host.log`. The log line prints the tile rectangle and water level. +3. **Orientation.** North should be up and the picture should not be mirrored. + Check a coast or a lake against the camera. Row 0 is assumed to be the south + edge (`ImageView::DoRender` `0x22b3860`). +4. **Marker alignment.** Town icons should sit on the towns in the picture. + `AbsoluteLayout` rectangles are assumed to be top-left and in UI pixels. The + old Workshop mod placed its line view at `y = height/2`, which hints that + `LineRenderView` may use another origin; if the camera circle is offset + vertically, that is the cause. +5. **Click frame.** The first three map clicks log `event pos` and `screen`. If + `event pos` is not relative to the picture, clicks will move the camera to + the wrong place. +6. **Camera API.** `setCameraData` should move the camera. If it does not, the + script falls back to `game.gui.setCamera` and logs when both fail. +7. **Cost.** Record the render time on a 256x256 and a 512x512 map. + +8. **Network gathering time** on a large map: the `network N: ... in S s` line. + Lua reads about five components per edge. If this is too slow on a 512x512 + network, the next step is a native ECS reader. +9. **Company mapping** in a companies-mode session: every company's tracks + should show in its lobby chip colour, and its legend row should say the + right company number. +10. **Cargo icons:** each industry type should show its cargo. First session + (September 16): some markers were blank. Only industries that had produced + something reported a cargo, markers placed before their type found one + kept the fallback, and that fallback (`map_industry.tga`, likewise + `map_town.tga`) is not a file; `ui.zip` ships only the `@2x` names. Now fixed + as described above. The stdout line `industry icons: N from production, + N from mod files, N from the stock list, N from inputs, N generic (files)` + names any type still generic. +11. **Roadside stops** should appear as small bars on their streets. + +Known limits in this version: temperate terrain colours on every climate; +industry markers capped at 4,000; a network gathered by Lua (see point 8). diff --git a/docs/octree-depth12.md b/docs/octree-depth12.md index 52e7813..6182b1a 100644 --- a/docs/octree-depth12.md +++ b/docs/octree-depth12.md @@ -1,3 +1,7 @@ +> Native Linux branch: the upstream server results below describe a separate +> implementation. This standalone port still installs depth 11 (512-tile edge +> cap), including when 12/13 are requested. See [PORT.md](linux/PORT.md). + # Experimental actual octree depths 12 and 13 Implemented for Steam build 35924, tested offline on 2026-09-12. **Not yet @@ -6,6 +10,11 @@ leaf cells; it does not substitute coarser leaves. Depth 12 uses a root of ±131,072 m (1,024-tile / 262.144 km edges). Depth 13 uses ±262,144 m (2,048-tile / 524.288 km edges). +**Steam 35924 only.** On the GOG build neither replacement prologue is at a +verified address, so `octree_depth=12`/`13` degrade to the depth-11 root there: +the plugin loads instead of refusing, the edge ceiling stays 512 tiles, and the +log line says which depth was asked for and what went in. + ## Configuration and build Run `build.bat` to produce `out/tpf2_bigmap.dll`. The build does not deploy. @@ -14,7 +23,10 @@ In the existing `[tpf2_bigmap]` configuration section, set `octree=1`, to the desired dimensions, for example `size7_format0=1280x32` with `size_label7=Depth13 test`. Replace existing assignments rather than duplicating keys. This example crosses the old root boundary with a relatively narrow map. -The shipped configuration retains `octree_depth=11` and `max_tiles=512`. +The shipped `cfg\tpf2_bigmap.cfg` sets `octree_depth=13` and `max_tiles=2048` +(its size ladder runs 128 to 512 tiles). On the GOG build those two values are +inert: 12 and 13 fall back to the depth-11 root described below, so the ceiling +there is 512 tiles. Depth 12/13 is installed even when the configured menu sizes are small, because loading a world does not pass through the menu sizing hook. Keep the setting @@ -117,14 +129,15 @@ built DLL. It checks: depth-10/11 compatibility, explicit size caps and pixel-count boundary cases, and 5,120 generated ratio shapes within INT_MAX. -`python tools/test_newgame_menu.py` also passes. No live game, full renderer, -save/reload or multiplayer validation has been performed. Emulator allocation -stubs do not validate allocator behavior or concurrency in the running engine. - -Before relying on the patch, generate a fresh narrow map exceeding 1,024 tiles for depth 13, -build and remove roads and constructions beyond ±131,072 m and in all four -corners, move vehicles across that boundary, inspect near/far render culling, -then save and reload repeatedly. Check for duplicate-node repairs, missing -objects and crashes. Repeat loading a stock world in the same process and -multiplayer replication with matching builds. Existing damaged saves are not -repaired by this patch. Depth 14 and higher are explicitly unsupported. +`python tools/test_newgame_menu.py` also passes. + +**Tested in play (2026-09-23): the higher depths work fine.** Large maps at +depth 13 have been created, played, saved and reloaded on Steam, in single +player and in multiplayer -- Windows players together with the native Linux +dedicated server, which gained depths 12/13 the same day +(`docs/re/crossplatform/OCTREE13_LINUX_REPORT.md` in the multiplayer repo) -- +with no duplicate-node repairs, no assertion and no missing objects. On GOG the +depth-11 fallback was validated in a running game (a 57 x 57 km map created, +played and reloaded). Every multiplayer peer needs the same `octree_depth`. +Existing damaged saves are not repaired by this patch. Depth 14 and higher are +explicitly unsupported. diff --git a/docs/release-0.5.0.md b/docs/release-0.5.0.md new file mode 100644 index 0000000..8e439ce --- /dev/null +++ b/docs/release-0.5.0.md @@ -0,0 +1,13 @@ +Big saves load without the memory spike, plus an in-game minimap and travel-time settings. + +**This release overwrites existing `plugins/tpf2_bigmap.cfg`, including user edits, on installation, upgrade and repair.** Close the game before installing. It shares the proxy and plugin host with TpF2 Multiplayer 0.6.1; installing either package updates both. + +Measured on a 207,360-tile save on a 94 GiB machine with no page file: the game's private memory peak while loading fell from 34 to 36 GiB down to 8.3 GiB, and the "Out of memory" assert that ended those loads is gone. + +- **The load-time memory spike is fixed** (`alignment_batch_tiles=512`). On a save load the engine rebuilt the 1 m height cache for the whole map in one terrain alignment update, holding every intermediate block until the end: 3.3 million result vectors and about 10 million work vectors alive at once on a 207,360-tile save. The update now runs in batches of 512 blocks, computing, publishing and freeing per batch, which is the per-frame behaviour the engine already has in play. The game's own tile set is walked read-only and never modified. Details in `docs/alignment-batch.md`. +- **In-game minimap** (`minimap=1`, off by default). A native-rendered minimap for maps too large for the game's own, with company colours and names when TpF2 Multiplayer is installed. +- **Travel-time limits** (`travel_time_limit_s`, `cargo_path_time_s`, both `0` = stock). The engine's 20 minute passenger and 100 minute cargo path limits become settings; set them to, say, `6000` to let people and cargo take routes across a very large map. +- **Terrain pager tuning, all on by default.** Content deduplication of the two engine terrain versions (`terrain_dedup`: measured on a real load, 53% of evictions share an already compressed copy); lazy zero tiles (a fresh tile gets its backing on first touch, not at allocation); an eviction rate that adapts to its measured cost and to gameplay frame stalls, with a per-second cap; a steady-state budget that follows the working set instead of snapping to the hot size after a load; a loading reserve that leaves the engine half of free memory (12 GiB floor) and backs off while the system commit charge is tight; and a retry when the system refuses a section at the commit limit instead of a crash. +- Groundwork for a terrain sidecar (a save's finished 1 m cache written beside it and served on load) ships inert: nothing writes or reads a sidecar file yet. A compressed small-block pager for the alignment pass (`terrain_blocks`) is kept but off; batching superseded it. + +The performance and octree options remain experimental. Report problems with `%LOCALAPPDATA%\tpf2mp\data\tpf2mp_host.log` attached. diff --git a/docs/release-0.5.1.md b/docs/release-0.5.1.md new file mode 100644 index 0000000..9278ddc --- /dev/null +++ b/docs/release-0.5.1.md @@ -0,0 +1,13 @@ +Fixes awful performance on 32 GB machines after a big save loads. + +**This release overwrites existing `plugins/tpf2_bigmap.cfg`, including user edits, on installation, upgrade and repair.** Close the game before installing. It shares the proxy and plugin host with TpF2 Multiplayer 0.6.1; installing either package updates both. + +The terrain and material pagers kept a flat 12 GiB of RAM in reserve, a figure measured on a 94 GiB machine with no page file. On a 32 GiB machine a loaded big save leaves less than that free, so the pagers pinned their resident sets at the minimum (about 1 GiB of terrain tiles), evicted everything else, and the engine faulted the evicted tiles straight back in through a decode each, thousands per second: the game ran, but everything stuttered. The same flat figure also declared the commit charge "tight" for the whole session on a machine with a system-managed page file, which throttled the pagers on top. + +- **Headroom sized to the machine.** The reserve is a seventh of installed RAM, clamped to 2 to 12 GiB: 12 GiB on 94 GiB as before, 4.6 GiB on 32 GiB, 2.3 GiB on 16 GiB. The commit-tight threshold follows the same idea (an eighth, 2 to 10 GiB). +- **The pager counts what it holds as its own room.** The resident ceiling used to be computed from free RAM alone, so a pager that had filled memory during the load saw little free RAM, shrank to its floor, and then could not grow back. It now budgets from free RAM plus its own resident set, keeping a quarter of that (or the headroom) free. +- **Memory pressure shrinks it.** When free RAM drops under the headroom the target shrinks by an eighth per second and eviction turns urgent, whatever the decode feedback says. Paging the engine out is worse than any decode. + +On a 32 GiB machine with a 207,360-tile save the terrain pager now settles around 9 to 10 GiB resident with 4 to 5 GiB free instead of 1 GiB resident and a decode storm. The new `pressure=` and `free=` fields on the `resident target` log lines show the state; `cold restores/s` lines at 300 or more mean the working set still does not fit. + +Everything else is 0.5.0. diff --git a/docs/release-0.5.2.md b/docs/release-0.5.2.md new file mode 100644 index 0000000..a05320e --- /dev/null +++ b/docs/release-0.5.2.md @@ -0,0 +1,7 @@ +The in-game minimap ships on. + +**This release overwrites existing `plugins/tpf2_bigmap.cfg`, including user edits, on installation, upgrade and repair.** Close the game before installing. It shares the proxy and plugin host with TpF2 Multiplayer 0.6.1; installing either package updates both. + +- **Minimap on by default** (`minimap=1`). 0.5.0 introduced the native-rendered minimap for maps too large for the game's own, with company colours and names when TpF2 Multiplayer is installed, but shipped it off, and every install of the package reset a hand-edited `minimap=1` back to off. Set `minimap=0` in `plugins/tpf2_bigmap.cfg` to remove the map button again. + +Everything else is 0.5.1 (the 32 GB machine fix). diff --git a/docs/release-0.5.3.md b/docs/release-0.5.3.md new file mode 100644 index 0000000..33090f1 --- /dev/null +++ b/docs/release-0.5.3.md @@ -0,0 +1,12 @@ +The installer finds the game on any drive, and the pager policy from the September measurements ships + +**This release overwrites existing `plugins/tpf2_bigmap.cfg`, including user edits, on installation, upgrade and repair.** Close the game before installing. It shares the proxy, the plugin host and the installer's custom actions with TpF2 Multiplayer 0.6.1.14 (vendored from that release); installing either package updates both. + +- **The installer finds the game in any Steam library, on any drive.** The folder page was pre-filled from Steam's registration of the game or from the folder a previous install remembered, and otherwise showed the stock `C:\Program Files (x86)\Steam` path; a game in a second library meant browsing by hand. When neither holds `TransportFever2.exe`, the installer now reads Steam's own library list (`libraryfolders.vdf`) and takes the first library that has it. Shared with TpF2 Multiplayer. +- **Zoomed-in stutter on a big map** (`terrain pager 3195 <-> 256 MiB` flapping in the host log): the commit-pressure throttle is now sticky, holding until free commit clears the threshold by more than the pager itself gives back, and for at least 30 s. On a machine without a page file the pager's own release cleared the threshold, it re-expanded, and the flag set again, 74 times in one session. +- **The resident caps are the same size on every machine that can afford them**: `terrain_cache_max_mb=0` means a quarter of RAM clamped to 4096..8192 MiB (a 4 GiB cap sat under a fresh large map's working set and cost frames in decodes), `material_cache_max_mb=0` a quarter of that. `-1` removes a cap. +- **A working-set floor.** The level the stutter feedback drove the resident target to is kept and decays over minutes, instead of the target sawtoothing back to the hot budget every quiet second (1,000-2,000 decodes a second on a 32 GiB machine while the camera rested). +- `simulate_physical_mb` (rig only) makes the policy size itself for a smaller machine; the engine is not constrained. +- Releases are built on GitHub Actions and checksummed; the README describes the pagers as shipped. + +The pager changes were measured on the developer's machine; if a 32 GB machine behaves differently, the `resident target` lines in `%LOCALAPPDATA%\tpf2mp\data\tpf2mp_host.log` say what the policy did. diff --git a/docs/terrain-compression.md b/docs/terrain-compression.md index f5b2d9d..a1841e5 100644 --- a/docs/terrain-compression.md +++ b/docs/terrain-compression.md @@ -7,6 +7,52 @@ both rail segments intact. Extended gameplay remains untested. This is separate from the discontinued 2 m experiment. Source heightmaps, derived 257x257 samples, physical tile dimensions and octree depth are unchanged. +For the native branch in this repository, see [PORT.md](linux/PORT.md#windows-integration-b755f65-partial). +Compression now defaults on, but the pager still uses its fixed 1024 MiB budget +and round-robin scan with a two-second grace period. The upstream server +measurements below are not tests of this branch. The new 30-second log reports +faults/s (including initial zero-page faults) and evictions/s; adaptive sizing +and recency eviction remain outstanding. + +## Native Linux: ON by default for everyone (the owner's call, 2026-09-22) + +The Linux port (`bigmap/linux/`, the userfaultfd pager) shipped +`terrain_cache_compress=0` as an opt-in, because `UFFD_USER_MODE_ONLY` cannot +serve a kernel-origin fault on an evicted tile. Without it the native game holds +every tile raw: the dedicated server (31 GB) was OOM-killed at 29.3 GB loading a +~52,000-tile map that Windows holds at 8-10 GB. The owner: "turn it on for +everyone". + +- `bigmap/linux/tpf2_bigmap.cfg`: `terrain_cache_compress=1`, and the code's + built-in default for a missing key is 1 too. +- It stays a cfg switch (`terrain_cache_compress=0` turns it off) and still falls + back to stock paths when userfaultfd setup fails. +- The Linux docs (`bigmap/docs/linux/PORT.md`, `INSTALL.md`) say it is on by + default, name the kernel-fault risk and how to turn it off if a graphics driver + or mod trips it (SIGBUS in the log). +- Released within 0.7: no version change. Compression changes no simulation + result, so peers with and without it stay in sync. + +### The Linux pager thrashes a running big map (measured 2026-09-22) + +On the dedicated server with the ~52,000-tile map and the automatic budget (about +1 GiB hot; 1.9 GiB resident), the running game faulted **~790 tiles a second** +(23,700 faults in 30 s) and evicted as many again, and the engine stretched its +200 ms batch to 400 ms: the session ran at half speed, 30+ units behind a joiner. +The Linux policy evicts round-robin ("not an LRU cache: ordinary reads of +resident pages do not refresh their age", `bigmap/docs/linux/PORT.md`), so it keeps +evicting the tiles the simulation touches every step. FOR THE LINUX PORT: + +- Evict by recency, like the Windows pager: a tile faulted in (or written) is + young; the scan skips young tiles and takes the oldest cold ones. Without a + cheap access bit (userfaultfd sees only faults), age = time since its last + fault-in; a tile that faulted twice within a few seconds is hot and stays. +- The automatic hot budget (RAM / 30) is far too small for a running big map; + size it from the machine and the live tile count like Windows' + `AutoTerrainBudgets` headroom, and count faults per second in the 30 s status + line so thrashing is visible. +- Stopgap on the server: `terrain_cache_hot_mb=6144` in its cfg. + ## Format 3 and pager capacity (September 15, built and tested offline, NOT deployed) Three changes, all in `src/terrain_codec.h` and `src/terrain_pager.h`: @@ -293,3 +339,31 @@ RAM minus 12 GiB and never below the configured warm allowance (`TerrainBudgetMB`, both pagers). Compression then happens after loading on the below-normal eviction threads. Tested offline (`test_terrain_compression.py` budget cases); load time with this policy not yet measured. + +## 2026-09-17: headroom sized to the machine (0.5.1) + +A 32 GiB machine ran a loaded 207,360-tile save at 19.4 GB in the game with +awful performance. Cause: the flat 12 GiB reserve above (MEASURED on this +94 GiB, no-page-file box) is more than a 32 GiB machine has free once the +save is in, so `TerrainBudgetSteady`'s ceiling fell to the hot budget +(1 GiB), the pager evicted everything else, and the engine faulted the +evicted tiles back in through a decode each; the flat 10 GiB commit-tight +threshold also held on a machine with a system-managed page file, which +throttled the pagers on top. + +Changes (`terrain_compression.h`, both pagers): + +- `PagerHeadroom(physical)` = RAM/7 clamped to 2..12 GiB replaces the flat + 12 GiB in the loading reserve and the steady ceiling; `CommitTightBytes` = + RAM/8 clamped to 2..10 GiB replaces the flat 10 GiB. +- The steady ceiling budgets from *room* = free RAM plus the pager's own + target (what it could own), reserving a quarter of the room or the + headroom, and hands the terrain pager 3/4 of the rest, the material pager + 1/2. Counting free RAM alone let a full pager starve itself. +- Pressure: free RAM under the headroom shrinks the target by 1/8 per second + and sets the urgent eviction flag, regardless of the decode feedback. + +Offline (`test_terrain_compression.py`): a 32 GiB machine after a load with +12 GiB held and 4 GiB free settles at 9,652 MiB instead of 1,092. Not yet +measured on a 32 GiB machine; the `resident target` log lines carry +`pressure=` and `free=` for that. diff --git a/docs/terrain-cow-sharing.md b/docs/terrain-cow-sharing.md index 318e699..e2e0e39 100644 --- a/docs/terrain-cow-sharing.md +++ b/docs/terrain-cow-sharing.md @@ -111,6 +111,64 @@ byte-identical. The lever is therefore **content dedup, not copy-on-write**, and next test, no new machinery: hash both versions' tiles during a load and report how many collide. Only build dedup if that number is near 65,536. +## 0d. MEASURED (September 17, 256x256 load): the two versions ARE identical, and dedup is built + +`terrain_dedup_probe=1` (this commit) hashes every live tile from the pager +worker and counts hash groups. One load of `New Game6464`, probes every 10 s +while loading: + +| live | hashed (resident / cold) | distinct | zero tiles | pairs | largest group | note | +|---|---|---|---|---|---|---| +| 24,775 | 24,767 (16,120 / 8,647) | 1 | 24,767 | 0 | 24,767 | first version allocating, nothing filled yet | +| 131,072 | 131,072 (71,376 / 59,696) | 1 | 131,072 | 0 | 131,072 | both versions allocated, all still zero | +| 131,072 | 131,072 (72,386 / 58,686) | 9,965 | 112,678 | 8,430 | 112,678 | filling: the filled tiles already come in pairs | +| **131,072** | **131,072 (43,619 / 87,453)** | **65,432** | **719** | **64,922** | 719 | **both versions full: every filled tile has exactly one twin** | +| 131,072 | 119,042 (17,007 / 102,035) | 65,536 | 0 | 53,506 | 2 | second version being released mid-probe, still pure pairs | +| 65,536 | 65,536 (9,347 / 56,189) | 65,536 | 0 | 0 | 1 | after the load: one version, every tile distinct | + +So the lever named at the end of 0c is real: 65,536 of the 131,072 tiles held +during a load are byte-for-byte copies of the other version, and outside the +load window nothing is duplicated (later probes during play: 0-2 pairs). + +**Built: `terrain_dedup=1`** (`src/pager_impl.inl`, `Evict`). An eviction +hashes the tile before encoding (the codec's own 64-bit hash plus a second, +independent one) and looks the pair up in an index of stored blobs. A hit +shares the existing blob through the Clone mechanism (one immutable blob, +reference counted, dropped by the first write) and skips the encode; a miss +encodes and registers the new blob. The index is open addressing over blob +pointers with tombstones, rebuilt from the slot table when half full. The +blob leaves the index with its last owner. Both pager tests pass, including +eight writers allocating twins against a forced evictor. + +What it saves: the second twin of every pair costs a hash (~60 us) instead of +an encode, and its compressed bytes are kept once. In the run above the pager +encoded 59,696 tiles while both versions existed; with the index, every twin +whose partner was already cold is a hit. What it does NOT save: resident +backing. A twin is evicted when the budget says so, exactly as before; the +8.25 GiB of the second version stays resident on a machine whose loading +allowance keeps it. Freeing it early would mean evicting twins regardless of +budget, which needs a hash per candidate and re-decodes if the engine reads +the released version; not built. + +**Measured in the game (September 17, `LONGMAPSAVE`, 103,680 tiles per +version, 207,360 live during the load, 94 GiB machine with the commit charge at +its limit for most of the load):** + +| point | evictions | real encodes | reused (blob kept) | dedup hits | +|---|---|---|---|---| +| mid-load, 207,360 live, dedup ON | 165,646 | 13,732 | 0 | 151,928 | +| same map, same point, dedup OFF (16:38 run) | 135,637 | 135,637 | 0 | - | +| load complete, 103,680 live, dedup ON | 498,951 | 104,780 | 130,993 | 263,192 | + +Over the whole load only 21% of evictions encoded anything; 53% shared a blob +that already existed. Early in the load most hits are the still-unfilled +zero tiles the loading policy evicts (one blob for all of them); later they are +the second twins. `failures=0`, `dedup_rebuilds=0`. Load time was not timed on +this run (the load ran with `commit_tight=1` flapping the resident target +between 256 MiB and 8 GiB every second, which dominates it; that back-off is +older than this change). Compressed bytes at the end: 1,093 MiB for 45,402 cold +tiles. + ## 1. What the engine actually does (DERIVED from the machine code) The height cache of one tile is a `std::vector` (66,049 samples, diff --git a/docs/terrain-sidecar.md b/docs/terrain-sidecar.md new file mode 100644 index 0000000..589a918 --- /dev/null +++ b/docs/terrain-sidecar.md @@ -0,0 +1,155 @@ +# Terrain sidecar: reuse a save's aligned 1 m cache on load + +The load-time terrain alignment pass (`docs/alignment-batch.md`) rebuilds the +1 m height cache every load: the bicubic refine turns the 4 m base heightmap +into a 1 m base cache, then the alignment pass cuts every road, track and +construction into it. Batching (`alignment_batch_tiles`) removed its memory +spike (35 GiB -> 8.3 GiB on a 207,360-tile save), so what remains is its +**time**. The finished cache is a pure function of the save, so writing it +beside the `.sav` and restoring it on load of the *same* save skips both the +refine and the pass. + +## Status + +**Wired since 2026-09-21** (`src/terrain_sidecar_io.h`): every save writes +`.terr` and every load arms it. Before that nothing wrote or armed a +sidecar and `terrain_sidecar=1` was inert. How it works now: + +- **SaveGame** (`0x2e97c0`) is detoured. The terrain is captured at the + detour's *entry*, into `.terr.tmp` with fingerprint 0: the world is + frozen for the length of the call, so the capture is exactly what the save + holds. When the engine returns and the `.sav`'s write time has moved, the file + is stamped with the new `.sav`'s hash (`Refingerprint`) and renamed over + `.terr`. A failed save leaves no sidecar. Capturing after the engine + returned would race the first alignment of the resumed simulation. +- **LoadGame** (`0x2e5ec0`) is detoured: it clears `g_alignmentTerrain`, resolves + the save's path, hashes the `.sav` and arms the sidecar (`ArmForLoad`). The + first `AddTile` of the load opens it, as before. +- **The path** comes from the `SaveGameId` both functions receive + (`{ wstring path; string name; string ns }`): `\.sav`, and for an + id with no path the directory `StandardSaveGameBackend` keeps for `ns` + (`0xbb23c0` app, `0xbb2db0` backend, `0x2471640` directory lookup, the + backend's vftable checked against `0x38e5758`). All five sites are + byte-verified; paths are UTF-8 and opened wide. +- **Found by fingerprint.** Multiplayer loads a *copy* of the host's save as + `mp_shared.sav`; the copy has the same bytes, so when `.terr` is absent + or foreign the loader takes any `.terr` in that folder whose header carries + the save's hash (32 bytes read per candidate). +- **Which terrain.** `g_alignmentTerrain`, trusted only when its grid is sane + and every full tile's cache lies in the terrain pager's arena. +- **Orphans.** Autosaves rotate; after each write a `.terr` whose `.sav` is gone + is deleted. +- **Cost** (synthetic tiles, 2026-09-21): 0.17 ms and 22.7 KiB per tile, 17.6 % + of raw. 8,712 tiles: 1.5 s, 190 MiB. 65,536 tiles: 11 s, 1.4 GiB, on every + save. There is no size limit by default (the owner's call, 2026-09-21: a slow + save is fine, the load is what matters); `terrain_sidecar_max_tiles=` skips + the sidecar above n grid records. + `terrain_sidecar_write=0` reads sidecars without writing them. +- **Test:** `python tools\test_sidecar_io.py` (16 checks, a non-ASCII folder). +- **Autosaves (fixed 2026-09-22).** An autosave's file is not `.sav` + (the engine names and rotates it: `autosave__.sav`), so the + expected file never changed and every autosave's sidecar was discarded ("the + save was not written"), on the host and in the Sandboxie box alike; manual + saves were fine. Now, when the expected `.sav` did not change, the one `.sav` + in that folder written during the call takes the sidecar (two or more: none, + so a sidecar never carries another save's hash). A hot join's shared save is + such an autosave. `tools/test_sidecar_io.py` section 5. +- **Not sent to joiners, on purpose.** On a 66,248-tile map the `.terr` is + 1.6 GB beside a 755 MB save, and what it can save is part of the alignment + pass: 8.9-10.4 s of a 185 s world entry (road connections alone took 125 s). + Sending it would make a join slower even on a fast link, and far slower + through a Steam relay; the lobby also holds a transfer in memory. A joiner + loads stock; the sidecar serves the host's own loads. +- **Not yet measured in a game.** What a served load saves is the refine's and + the pass's *publication* into served tiles; their compute still runs. The + first real numbers should come from the `terrain sidecar:` log lines and the + `alignment pass:` timings of a load with and without the file. A save sent to + another machine (the dedicated server) needs its `.terr` sent with it. + +The rest of this section is the groundwork as first recorded. + + +Built and tested here: the **file format**, the **grid walk** that reads and +restores every tile's height cache, and the **codec** (the variable-length +`BlockCodec` from `small_codec.h`). `tools/test_terrain_sidecar.cpp` builds a +synthetic grid to the game's exact layout and checks a full round trip, plus +every refusal path. Measured on synthetic terrain-like tiles: **17.6 % of raw**, +about 22 KB per 257x257 tile, so a full 207,360-tile map is roughly **1.2 GiB** +on disk. + +Not yet integrated (needs the pass, which `src/alignment_batch.h` owns): + +1. **Fingerprint.** Key the file to the save by a 64-bit hash of the `.sav` + bytes, captured in a `SaveGame` (`0x2e97c0`) post-hook and recomputed in a + `LoadGame` (`0x2e5ec0`) pre-hook. Same bytes => same aligned terrain, so a + match is exact; any mismatch (edited save, plugin-less save, or a *different + save of the same map* — same base heightmap, different roads) makes `Apply` + a no-op and the pass runs. A content hash of the base heightmap alone is + NOT enough: it would false-accept a different save of the same map and serve + the wrong cuts, so the key must be tied to the save's identity. + + The save's resolved path is reachable exactly as the multiplayer menu DLL's + `AutoLoadCall` does it (native/src/menu_hook.cpp ~2219): the argument is a + `SaveGameId { std::wstring path@0x00, name@0x20, namespace@0x40 }` (SSO); + `app = APP_ACCESSOR()`, `mgr = *(app + 200)` is the save manager, and + `SAVEINFO_GET(info, mgr, id)` fills a 0x110-byte save-info struct that + carries the absolute `.sav` path. `LoadGame` (`0x2e5ec0`) receives that + SaveGameId directly; `SaveGame` (`0x2e97c0`) has it too. Hash the resolved + `.sav` file and place/read `.terr` beside it. Store the hash in + `static uint64_t TerrainSidecar::g_saveFingerprint`, which `Write` and + `BeginApply` already take as their `fingerprint` argument. +2. **Write trigger.** After a save completes, walk `g_alignmentTerrain` + (exposed by the batch detour, `CTerrain` from `self+8`) and write the file. +3. **Read trigger and short-circuit.** The pass CREATES the tiles (`AddTile`, + `0x33cb60`, is called from the alignment system), so the sidecar cannot + replace it wholesale. Two options, to decide with the pass owner: + - Let the pass create the tiles but make its per-block compute a no-op when + a valid sidecar is loaded, then `Apply` the saved caches. Smallest change, + but it lives inside the pass. + - Serve the saved cache from the refine detour and skip the rasterise. This + is what "the sidecar replaces what the refine produces" would mean, but + the refine yields *base* heights and the pass cuts them afterwards, so it + only works if the cut is also suppressed for served tiles. + +## Two CTerrain versions, one image + +A save load holds two CTerrain versions transiently, each with the full tile +grid (measured: a 256x256 save is 65,536 tiles per version, 131,072 AddTile +calls, two alignment passes of ~13.9 s each = 27.8 s). One sidecar image serves +both, and no second grid is stored, because: + +- The two versions' tile caches are byte-identical tile for tile. This is the + same fact the content-dedup measured (every tile has exactly one byte-identical + twin in the other version); it is why `terrain_dedup` works. +- Both grids share dimensions and origin, so `RecordIndex(x, y)` is the same in + both, and `ApplyTile(GridOf(versionB), index)` decodes the same blob into + version B's tile correctly. + +All serving happens at AddTile, and every AddTile (both versions) completes +before either alignment pass runs, so `g_load` need only live from the first +AddTile to the last; `EndApply` after the first pass is correct today, after the +last pass is strictly safe. At save time there is normally one CTerrain, so +`WriteForSave` captures the single live version, which is the state both +transient versions converge to on the next load. + +## File format + +Little-endian. `TERR`, version 1. + +``` +FileHeader { u32 magic; u32 version; u64 fingerprint; i32 nx, ny; u32 tiles; u64 headerHash; } +repeated: TileHeader { u32 recordIndex; u32 bytes; } then `bytes` of BlockCodec blob +``` + +`headerHash` catches a torn write; each tile's `BlockCodec` blob carries its own +64-bit content hash, so a flipped byte fails decode rather than restoring wrong +terrain. `Apply` refuses the file unless `magic`, `version`, `headerHash`, +`fingerprint` and `nx`/`ny` all match, only writes into a record whose vector is +already `Samples` long (so it never allocates a tile), and returns -1 on a +corrupt-but-matching file so the caller discards it and lets the pass run. + +## Grid layout (RE, Steam 35924) + +`CTerrain + 0x18` -> grid `{ i32 x0@0, i32 y0@4, i32 nx@8, i32 ny@0xc, void* records@0x10 }`; +each record is 40 bytes `{ i32 entity@0, {u16* first, *last, *end}@8, i32 version@0x20 }`. +Confirmed from `CTerrain::GetTile` (`0x33d580`) and `AddTile` (`0x33cb60`). diff --git a/installer/Package.wxs b/installer/Package.wxs index 744e521..32a9d77 100644 --- a/installer/Package.wxs +++ b/installer/Package.wxs @@ -67,6 +67,19 @@ lets them cross into the elevated half of a UAC-prompted install. --> + + + + + + + @@ -159,7 +172,9 @@ only moves on when TPF2_GAMEDIR_OK is not "0". The stock set cannot be extended in place: its Next-button events carry no condition, so a DoAction added beside them could never stop the wizard. Identical to - the TpF2 Multiplayer copy. --> + the TpF2 Multiplayer copy. The check itself is what differs between the + two builds of this package: the GOG variant sets TPF2_ALLOW_GOG so + tpf2ca.dll accepts the GOG executable as well as Steam's. --> diff --git a/installer/PluginHost.wxs b/installer/PluginHost.wxs index d095696..89663db 100644 --- a/installer/PluginHost.wxs +++ b/installer/PluginHost.wxs @@ -67,6 +67,7 @@ + @@ -78,7 +79,16 @@ + + + + +