diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3b23f69..0b6a938 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -53,7 +53,7 @@ repos: - '--fix' id: 'ruff-check' repo: 'https://github.com/astral-sh/ruff-pre-commit' - rev: 'v0.16.5' + rev: 'v0.16.4' - hooks: - id: 'yapf' name: 'check Python files are formatted' diff --git a/.vscode/dictionary.txt b/.vscode/dictionary.txt index ebcbdd3..82f9fca 100644 --- a/.vscode/dictionary.txt +++ b/.vscode/dictionary.txt @@ -61,6 +61,7 @@ bitrock bmax bmin bootram +boxalpha bpms bpps bsky @@ -69,9 +70,11 @@ bsum byteswap cabdata calcsize +cameracollision cand caplog capsys +cardboardalpha cbar cdda cdiner @@ -80,6 +83,7 @@ cdrommsf cdtitle cgbi chara +charactercollision checkerboard chunksize cigam @@ -88,6 +92,7 @@ ciwmodel clen clut cluts +cmpheader cmpr cmpxchg codepoint @@ -104,6 +109,7 @@ coredata coveragerc coverallsapp crdt +cryptheader cryptoff cryptsize cstring @@ -112,6 +118,7 @@ ctypes cuda cuebin cupy +cutoff darksphere dashcam dataarray @@ -223,6 +230,7 @@ gigayears globaltoc gltf gluvs +gognitti gojuon gojūon gojūon's @@ -308,6 +316,7 @@ lanczos lastexitcode launchable ldrsb +lerp lextudio lhuf libaudio @@ -316,6 +325,8 @@ libjsonnet librarys libsg libsonnet +lightmap +lightmaps linecap linejoin linters @@ -333,11 +344,14 @@ marm maskable maxcolors maxidx +maxpane +maxpayne mbin medi mediump megayears memcpy +memoryfile mido midwaydec minlen @@ -374,6 +388,7 @@ nlist nloop nmip nocolon +nodraw nofixprebinding noie nojekyll @@ -453,6 +468,7 @@ prov pset psmt putalpha +putdata putpalette pvrpack pvrt @@ -495,6 +511,7 @@ reflec reflecbeatplus refpack regen +renderable resgroup rgba rgbaq @@ -524,6 +541,7 @@ scancode scancodes scei scel +schar schemafile schi schl @@ -542,6 +560,7 @@ settimg sevenzip sfbk sgeo +sharedtextures shdr shellcheck shellformat @@ -565,6 +584,8 @@ skmple skuk skybox skyboxes +skybox's +slerp slus smlt smpl @@ -588,6 +609,7 @@ ssaid ssha ssre sszl +startpoint statusline stcol stcycl @@ -620,6 +642,7 @@ swizzles syft symtab sysex +systemtime szlen tabindex tatsh @@ -634,6 +657,7 @@ texoff texsize textlength teyu +tf's thps timecents titletranslit @@ -671,11 +695,13 @@ undrafted unframeable unparseable unshield +unshields unsplittable unswizzle unswizzled untile untiled +unwalkable usemtl userprofile usrdir @@ -716,6 +742,7 @@ xform xmap xoff xsecond +xyzw yapf yapfignore yarnrc diff --git a/README.md b/README.md index 84a44b5..26c8aa2 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,7 @@ Run `dade --help` to list the games, and `dade --help` to list a game's s | `dade incoming` | _Incoming_ (PC and Dreamcast) | Rage Software / Interplay | | `dade jubeatplus` | _jubeat plus_ (iOS) | Konami | | `dade marmalade` | Any Marmalade SDK title (Derbh, IwResGroup) | Marmalade / Ideaworks | +| `dade maxpane` | _Max Payne_ (PC) | Remedy Entertainment | | `dade misc` | Formats belonging to no single game | — | | `dade monopoly08` | _Monopoly_ (2008, multi-platform) | Electronic Arts | | `dade rbplus` | _REFLEC BEAT plus_ (iOS) | Konami | @@ -455,6 +456,85 @@ pieces are given one of each, rather than all of them at once. The retail disc converts clean, so `--ignore-failures` is not needed for it; pass it to log and skip an asset that will not convert instead of stopping. +## Max Payne + +```shell +dade maxpane ras-list MAXPAYNE.ISO +dade maxpane ras-extract MAXPAYNE.ISO -o extracted +``` + +Readers for the RAS (Remedy Archive System) containers the game loads everything from. The argument +may be a `.ras` archive, a `.mpm` mod package, a directory (searched recursively), an InstallShield +`DATA1.CAB`, an ISO, or the `.cue` of a cue/bin pair. + +A retail disc needs both routes at once, which the extractor takes care of: the level archives sit +loose on the disc, while the shared game database is inside `DATA1.CAB` and is unpacked with +[`unshield`](https://github.com/twogood/unshield). A cabinet is skipped with a warning when +`unshield` is missing, so the loose archives still come out. + +Members are stored back to back with no offset field, so the directory doubles as an integrity +check; `ras-list` reports an archive as `intact` when the header, both tables, and every stored size +account for the file exactly. + +Every member is LZSS-compressed and the archive tables are encrypted, both handled transparently. +Pass `--raw` to `ras-extract` to keep the `RA->` and `RC->` wrappers. + +```shell +dade maxpane inspect-tags extracted/data/database/levels/part1/Part1_Level6.ldb +``` + +`inspect-tags` decodes the tagged `R_MemoryFile` stream that every custom asset is built from, +naming each value's type. The walk stops where a level leaves tagged territory, which is where its +first untagged string begins. + +```shell +dade maxpane ldb2glb extracted/data/database/levels -o glb +dade maxpane ldb-textures extracted/data/database/levels -o textures +``` + +```shell +dade maxpane ldb2glb extracted/data/database/levels -D extracted/data/database -o glb +``` + +`ldb2glb` converts levels to binary glTF, one `.glb` per `.ldb`, in parallel across every core. +Pass `--database` and the NPCs and pickups are drawn with their own models, read from the game's +`skins` and `level_items` directories; without it they are written as named empty nodes. +Each file carries the level's architecture, its props, the game's own texture coordinates, and every +embedded image. + +Every clip a prop can play -- a door swinging either way, a lift rising, a fan turning -- comes out +as a named glTF animation, so a viewer can list and play them. A level stores a clip as two poses +and two curves, one giving the distance travelled in world units and the other how far the prop has +turned; both are baked into keyframes on the way out, and a clip that moves nothing is dropped. + +The baked lighting is written too: each level's atlases are embedded, each face names the one that +lights it, and the second coordinate set addresses it. It goes in glTF's occlusion slot, which is +the closest the format has to a lightmap, so a viewer wanting the game's own look should multiply +that texture's colour into the base rather than treat it as ambient occlusion. + +The sky is written out. A level's `skybox` faces are what closes it off wherever it opens to the +air, and leaving them out puts a hole through every street; they get a flat unlit colour, because +the sky the game drew came from the renderer rather than from the level. Their placeholder image is +never used, and neither is `dummy`'s, which stays dropped. + +Graffiti, signage and switchable surfaces come off their walls slightly. A level lays each of them +in exactly the plane of what it covers, and nothing in the file marks which is which, because the +engine walked its BSP and never drew both at once. A viewer draws the whole level and has only a +depth buffer, so `dade.maxpane.decals` works the layering out from the geometry and lifts each +covered face about eight millimetres along its normal. + +Four things about the format are easy to get backwards. A face's corner count is not its number of +sides -- the editor drops extra corners along edges it shares with other faces -- so triangulating +from corner nought can start with a straight line, and taking the winding from that turns 822 of +the shipped faces inside out. A material's second string is the +material's _name_, not a filename, and only the level's category table says which image it draws +with; matching on filename instead leaves a fifth of a level's faces untextured. Level architecture +is already in world space, keeping its transform only as the editor's pivot, while an animated prop +is placed by its transform -- applying both the same way moves the architecture twice. And a model's +texture coordinates are stored with V running negative and are meant to be used exactly as written, +Direct3D's wrapping doing the rest; negating them to get a tidy `0..1` range turns every skin upside +down, which shows on a face and nowhere else. + ## Extreme-G, Interstate '76, and Tony Hawk's Pro Skater 2 ```shell diff --git a/dade/common/lz.py b/dade/common/lz.py index ca86252..33632df 100644 --- a/dade/common/lz.py +++ b/dade/common/lz.py @@ -1,13 +1,15 @@ """ -Okumura LZSS decompressor, binary ``_0`` variant, as used by both Extreme-G games. +Okumura LZSS decompressor, binary ``_0`` variant, as used by both Extreme-G games and Max Payne. -The ring buffer is 4096 bytes and zero-filled, the write cursor starts at ``0xFEE``, and control -bytes are consumed least-significant bit first. A set control bit introduces one literal byte; a -clear bit introduces a two-byte match where the offset is ``b0 | ((b1 & 0xF0) << 4)`` and the -length is ``(b1 & 0x0F) + 3``. The stream carries no end marker: decoding stops once the caller's -expected output size is reached. +The ring buffer is 4096 bytes, the write cursor starts at ``0xFEE``, and control bytes are consumed +least-significant bit first. A set control bit introduces one literal byte; a clear bit introduces a +two-byte match where the offset is ``b0 | ((b1 & 0xF0) << 4)`` and the length is +``(b1 & 0x0F) + 3``. The stream carries no end marker: decoding stops once the caller's expected +output size is reached. -The PC port stores the identical byte stream, so this decoder serves both platforms unchanged. +The Extreme-G PC port stores the identical byte stream, so this decoder serves both platforms +unchanged. Max Payne's ``RA->`` records use the same encoding but prime the ring buffer with spaces +rather than NULs, which only matters for matches that reach back before the first literal. """ from __future__ import annotations @@ -26,7 +28,11 @@ _RING_MASK = RING_SIZE - 1 -def decompress_lzss0(data: bytes, start: int, decompressed_size: int) -> tuple[bytes, int]: +def decompress_lzss0(data: bytes, + start: int, + decompressed_size: int, + *, + fill: int = 0) -> tuple[bytes, int]: """ Decompress an LZSS ``_0`` stream. @@ -39,6 +45,8 @@ def decompress_lzss0(data: bytes, start: int, decompressed_size: int) -> tuple[b decompressed_size : int Number of bytes to produce. Decoding stops as soon as this many bytes exist, so the output may overshoot by up to fifteen bytes inside the final match and is truncated. + fill : int + Byte value used to prime the ring buffer. Extreme-G uses ``0``; Max Payne uses ``0x20``. Returns ------- @@ -47,7 +55,7 @@ def decompress_lzss0(data: bytes, start: int, decompressed_size: int) -> tuple[b :py:class:`IndexError` propagates if the stream runs past the end of *data*. """ out = bytearray() - ring = bytearray(RING_SIZE) + ring = bytearray([fill]) * RING_SIZE cursor = RING_START control = 0 pos = start diff --git a/dade/common/tools.py b/dade/common/tools.py index 46e9354..546e41a 100644 --- a/dade/common/tools.py +++ b/dade/common/tools.py @@ -1,18 +1,22 @@ """ -Location of the native helper tools the converters shell out to. +Location and invocation of the native helper tools the converters shell out to. A tool is resolved from an explicit override first, then from the ambient override published by -:py:func:`dade.common.context.tool_path`, and finally from ``PATH``. Invocation stays with the -game that owns the tool, because the argument lists and environment handling are tool-specific. +:py:func:`dade.common.context.tool_path`, and finally from ``PATH``. """ from __future__ import annotations from pathlib import Path from shutil import which +import logging +import os +import subprocess as sp from dade.common.context import tool_path -__all__ = ('ToolNotFoundError', 'locate_tool') +__all__ = ('ToolNotFoundError', 'find_unshield', 'locate_tool', 'run_unshield') + +log = logging.getLogger(__name__) class ToolNotFoundError(Exception): @@ -49,3 +53,45 @@ def locate_tool(name: str, override: Path | None = None) -> Path: return Path(found) msg = f'Could not find `{name}`. Put it on PATH or pass `--{name}-path`.' raise ToolNotFoundError(msg) + + +def find_unshield(override: Path | None = None) -> Path: + """ + Locate the ``unshield`` binary. + + Parameters + ---------- + override : Path | None + An explicit path that takes precedence over the context override and ``PATH``. + + Returns + ------- + Path + The resolved path to the binary. + """ + return locate_tool('unshield', override) + + +def run_unshield(cabinet: Path, output_dir: Path) -> None: + """ + Extract an InstallShield cabinet with ``unshield``. + + Parameters + ---------- + cabinet : Path + The ``DATA1.CAB`` file to unpack. Its ``DATA1.HDR`` and ``DATA2.CAB`` siblings must sit + beside it. + output_dir : Path + The directory the contents are extracted to. + """ + unshield = find_unshield() + env = dict(os.environ) + lib_dir = unshield.parent.parent / 'lib' + if lib_dir.is_dir(): + existing = env.get('LD_LIBRARY_PATH', '') + env['LD_LIBRARY_PATH'] = f'{lib_dir}{os.pathsep}{existing}' if existing else str(lib_dir) + log.debug('Extracting `%s` to `%s`.', cabinet, output_dir) + sp.run((str(unshield), '-d', str(output_dir), 'x', str(cabinet)), + check=True, + env=env, + capture_output=True) diff --git a/dade/incoming/sources.py b/dade/incoming/sources.py index bc18e05..b3ebd8e 100644 --- a/dade/incoming/sources.py +++ b/dade/incoming/sources.py @@ -8,7 +8,9 @@ import logging import subprocess as sp -from .tools import run_gdiextract, run_unshield +from dade.common.tools import run_unshield + +from .tools import run_gdiextract __all__ = ('PreparedSource', 'SourceError', 'prepare_source') diff --git a/dade/incoming/tools.py b/dade/incoming/tools.py index c7e0381..2661503 100644 --- a/dade/incoming/tools.py +++ b/dade/incoming/tools.py @@ -3,7 +3,6 @@ from typing import TYPE_CHECKING import logging -import os import subprocess as sp from dade.common.tools import ToolNotFoundError, locate_tool @@ -11,8 +10,7 @@ if TYPE_CHECKING: from pathlib import Path -__all__ = ('ToolNotFoundError', 'find_gdiextract', 'find_spvr2png', 'find_unshield', - 'run_gdiextract', 'run_unshield') +__all__ = ('ToolNotFoundError', 'find_gdiextract', 'find_spvr2png', 'run_gdiextract') log = logging.getLogger(__name__) @@ -51,47 +49,6 @@ def find_gdiextract(override: Path | None = None) -> Path: return locate_tool('gdiextract', override) -def find_unshield(override: Path | None = None) -> Path: - """ - Locate the ``unshield`` binary. - - Parameters - ---------- - override : Path | None - An explicit path that takes precedence over the context override and ``PATH``. - - Returns - ------- - Path - The resolved path to the binary. - """ - return locate_tool('unshield', override) - - -def run_unshield(cabinet: Path, output_dir: Path) -> None: - """ - Extract an InstallShield cabinet with ``unshield``. - - Parameters - ---------- - cabinet : Path - The ``DATA1.CAB`` file to unpack. - output_dir : Path - The directory the contents are extracted to. - """ - unshield = find_unshield() - env = dict(os.environ) - lib_dir = unshield.parent.parent / 'lib' - if lib_dir.is_dir(): - existing = env.get('LD_LIBRARY_PATH', '') - env['LD_LIBRARY_PATH'] = f'{lib_dir}{os.pathsep}{existing}' if existing else str(lib_dir) - log.debug('Extracting `%s` to `%s`.', cabinet, output_dir) - sp.run((str(unshield), '-d', str(output_dir), 'x', str(cabinet)), - check=True, - env=env, - capture_output=True) - - def run_gdiextract(gdi: Path, output_dir: Path) -> None: """ Extract the file system from a Dreamcast GDI with ``gdiextract``. diff --git a/dade/main.py b/dade/main.py index ee2e2db..9c181cb 100644 --- a/dade/main.py +++ b/dade/main.py @@ -23,6 +23,7 @@ from .incoming.main import main as incoming_extract from .jubeatplus.main import jubeatplus as jubeatplus_group from .marmalade.main import marm as marmalade_group +from .maxpane.main import cli as maxpane_group from .misc.main import misc as misc_group from .monopoly08.main import main as monopoly_extract from .rbplus.main import rbplus as rbplus_group @@ -94,6 +95,7 @@ def main() -> None: main.add_command(_incoming()) main.add_command(jubeatplus_group, name='jubeatplus') main.add_command(marmalade_group, name='marmalade') +main.add_command(maxpane_group, name='maxpane') main.add_command(misc_group, name='misc') main.add_command(_monopoly08()) main.add_command(rbplus_group, name='rbplus') diff --git a/dade/maxpane/__init__.py b/dade/maxpane/__init__.py new file mode 100644 index 0000000..17248b8 --- /dev/null +++ b/dade/maxpane/__init__.py @@ -0,0 +1,8 @@ +""" +Max Payne (Remedy Entertainment, 2001) asset toolkit. + +Readers for the formats built on Remedy's ``rl`` library: the RAS (Remedy Archive System) ``.ras`` +containers and the ``.mpm`` mod packages that share their layout, the seeded stream cipher and +``RA->``/``RC->`` block wrappers guarding their contents, and the tagged ``R_MemoryFile`` streams +that carry every custom asset, including ``.ldb`` levels and ``.ai`` movement networks. +""" diff --git a/dade/maxpane/blocks.py b/dade/maxpane/blocks.py new file mode 100644 index 0000000..e80b656 --- /dev/null +++ b/dade/maxpane/blocks.py @@ -0,0 +1,156 @@ +""" +The ``RA->`` and ``RC->`` block wrappers Remedy's ``rl`` library puts around stored data. + +``RA->`` marks an LZSS-compressed block and ``RC->`` an encrypted one. Either may wrap the other, +so :py:func:`unwrap` peels layers until neither magic matches. Every member of the shipped archives +carries exactly one ``RA->`` layer. +""" +from __future__ import annotations + +import struct + +from dade.common.lz import decompress_lzss0 + +from .crypto import decrypt + +__all__ = ('COMPRESSED_MAGIC', 'ENCRYPTED_MAGIC', 'RING_FILL', 'decompress', 'decrypt_block', + 'is_compressed', 'is_encrypted', 'unwrap') + +COMPRESSED_MAGIC = b'RA->' +"""Magic introducing an LZSS-compressed block, from ``R_File::CMPHEADER``. + +:meta hide-value: +""" +ENCRYPTED_MAGIC = b'RC->' +"""Magic introducing an encrypted block, from ``R_File::CRYPTHEADER``. + +:meta hide-value: +""" +RING_FILL = 0x20 +"""Byte Max Payne primes the LZSS ring buffer with. + +:meta hide-value: +""" + +_COMPRESSED_HEADER_SIZE = 12 +_ENCRYPTED_HEADER_SIZE = 16 + + +def is_compressed(data: bytes) -> bool: + """ + Report whether *data* begins with a ``RA->`` block. + + Parameters + ---------- + data : bytes + Candidate block. + + Returns + ------- + bool + :py:obj:`True` when the magic matches. + """ + return data[:4] == COMPRESSED_MAGIC + + +def is_encrypted(data: bytes) -> bool: + """ + Report whether *data* begins with a ``RC->`` block. + + Parameters + ---------- + data : bytes + Candidate block. + + Returns + ------- + bool + :py:obj:`True` when the magic matches. + """ + return data[:4] == ENCRYPTED_MAGIC + + +def decompress(data: bytes) -> bytes: + """ + Decompress a ``RA->`` block. + + Parameters + ---------- + data : bytes + A block starting with :py:data:`COMPRESSED_MAGIC`, followed by the decompressed size and + the packed size as unsigned 32-bit integers. + + Returns + ------- + bytes + The decompressed payload. + + Raises + ------ + ValueError + If *data* does not start with :py:data:`COMPRESSED_MAGIC`. + """ + if not is_compressed(data): + msg = f'Not a compressed block: {data[:4]!r}.' + raise ValueError(msg) + raw_size, _ = struct.unpack_from(' bytes: + """ + Decrypt a ``RC->`` block. + + Parameters + ---------- + data : bytes + A block starting with :py:data:`ENCRYPTED_MAGIC`, followed by the plaintext size, a + reserved dword, and the signed seed, all 32 bits wide. + + Returns + ------- + bytes + The plaintext. + + Raises + ------ + ValueError + If *data* does not start with :py:data:`ENCRYPTED_MAGIC`. + """ + if not is_encrypted(data): + msg = f'Not an encrypted block: {data[:4]!r}.' + raise ValueError(msg) + size, _, seed = struct.unpack_from(' tuple[bytes, tuple[str, ...]]: + """ + Remove every ``RA->`` and ``RC->`` layer from a block. + + Parameters + ---------- + data : bytes + Possibly wrapped block. + + Returns + ------- + tuple[bytes, tuple[str, ...]] + The unwrapped payload and the layers removed, outermost first, each ``'lzss'`` or + ``'crypt'``. + """ + layers: list[str] = [] + while len(data) >= _COMPRESSED_HEADER_SIZE: + # An encrypted block's header is the longer of the two, so a buffer that is long enough to + # be a compressed one and starts with `RC->` is still too short to take apart. + if is_encrypted(data): + if len(data) < _ENCRYPTED_HEADER_SIZE: + break + data = decrypt_block(data) + layers.append('crypt') + elif is_compressed(data): + data = decompress(data) + layers.append('lzss') + else: + break + return data, tuple(layers) diff --git a/dade/maxpane/commands/__init__.py b/dade/maxpane/commands/__init__.py new file mode 100644 index 0000000..965628f --- /dev/null +++ b/dade/maxpane/commands/__init__.py @@ -0,0 +1 @@ +"""Command exports.""" diff --git a/dade/maxpane/commands/inspect_tags.py b/dade/maxpane/commands/inspect_tags.py new file mode 100644 index 0000000..32ceb82 --- /dev/null +++ b/dade/maxpane/commands/inspect_tags.py @@ -0,0 +1,49 @@ +"""``dade maxpane inspect-tags`` - summarise the tagged stream inside a Max Payne asset.""" +from __future__ import annotations + +from collections import Counter +from pathlib import Path +import logging + +import click + +from dade.maxpane.blocks import unwrap +from dade.maxpane.memoryfile import BasicType, iter_values + +from .utils import debug_option + +__all__ = ('inspect_tags',) + +log = logging.getLogger(__name__) + + +@click.command(name='inspect-tags') +@click.argument('asset', type=click.Path(exists=True, dir_okay=False, path_type=Path)) +@click.option('-n', '--limit', default=32, help='Number of leading values to print.', type=int) +@debug_option +def inspect_tags(asset: Path, limit: int) -> None: + """ + Decode the tagged R_MemoryFile stream at the start of ASSET. + + Accepts a loose file or one still wrapped in RA-> or RC-> blocks. Walks from the first byte and + stops where the stream leaves tagged territory, which happens as soon as untagged bulk data + such as a string or lightmap begins. Prints where it stopped so the boundary is visible. + """ + data, layers = unwrap(asset.read_bytes()) + if layers: + click.echo(f'Unwrapped: {" -> ".join(layers)}.') + histogram: Counter[str] = Counter() + reached = 0 + for index, value in enumerate(iter_values(data)): + histogram[BasicType(value.tag).name] += 1 + reached = value.end + if index < limit: + shown = (repr(value.payload.decode('latin-1')) + if value.tag == BasicType.STRING else value.payload.hex(' ')) + click.echo(f' 0x{value.offset:08x} {BasicType(value.tag).name:<10s} {shown}') + # An asset that unwrapped to nothing has been walked in full, vacuously. + covered = 100.0 * reached / len(data) if data else 100.0 + click.echo(f'{asset.name}: {len(data)} bytes, walked to {reached} ({covered:.2f}%).') + click.echo(f'By tag: {dict(histogram.most_common())}') + if reached < len(data): + click.echo(f'Stopped on 0x{data[reached]:02x} at offset {reached}.') diff --git a/dade/maxpane/commands/ldb2glb.py b/dade/maxpane/commands/ldb2glb.py new file mode 100644 index 0000000..9417f57 --- /dev/null +++ b/dade/maxpane/commands/ldb2glb.py @@ -0,0 +1,163 @@ +"""``dade maxpane ldb2glb`` - convert levels to binary glTF.""" +from __future__ import annotations + +from concurrent.futures import ProcessPoolExecutor +from pathlib import Path +from typing import NamedTuple +import logging + +import click + +from dade.common.workers import default_jobs +from dade.maxpane.blocks import unwrap +from dade.maxpane.gltf import build_glb +from dade.maxpane.ldb import InvalidLevelError, read_level + +from .models import load_models +from .utils import debug_option + +__all__ = ('ldb2glb',) + +log = logging.getLogger(__name__) + + +class _Result(NamedTuple): + """Outcome of converting one level.""" + + name: str + """The level's file name.""" + meshes: int + """Number of placed meshes written.""" + props: int + """Number of animated props written.""" + faces: int + """Number of faces written.""" + images: int + """Number of images embedded.""" + placements: int + """Number of NPC and pickup placements written.""" + models: int + """Number of those placements drawn with a real model.""" + clips: int + """Number of prop animations written.""" + error: str + """Failure message, empty when the level converted.""" + + +def _convert(job: tuple[Path, Path, Path | None]) -> _Result: + """ + Convert one level, reporting failure rather than raising. + + Runs in a worker process, so the outcome has to be picklable and exceptions have to be turned + into data. + + Parameters + ---------- + job : tuple[Path, Path, Path | None] + The level to read, the directory to write into, and the game database to take NPC and + pickup models from. + + Returns + ------- + _Result + What was written, or the failure message. + """ + level, output_dir, database = job + try: + parsed = read_level(unwrap(level.read_bytes())[0]) + models = load_models(database, parsed) if database else {} + payload = build_glb(parsed, models=models, name=level.stem) + except (IndexError, InvalidLevelError, ValueError) as e: + return _Result(clips=0, + error=str(e), + faces=0, + images=0, + meshes=0, + models=0, + name=level.name, + placements=0, + props=0) + destination = output_dir / f'{level.stem}.glb' + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(payload) + meshes = parsed.mesh.meshes if parsed.mesh else () + props = parsed.props.meshes if parsed.props else () + faces = sum(len(m.faces) for m in (*meshes, *props)) + drawn = {f'character:{c.skin}' for c in parsed.characters if f'character:{c.skin}' in models} + drawn |= {f'item:{i.item}' for i in parsed.items if f'item:{i.item}' in models} + clips = parsed.props.animations if parsed.props else () + return _Result(clips=sum(1 for prop in clips for c in prop if c.start != c.end), + error='', + faces=faces or len(parsed.geometry.polygons), + images=len(parsed.textures), + meshes=len(meshes), + models=len(drawn), + name=level.name, + placements=len(parsed.characters) + len(parsed.items), + props=len(props)) + + +@click.command(name='ldb2glb') +@click.argument('levels', nargs=-1, required=True, type=click.Path(exists=True, path_type=Path)) +@click.option('-o', + '--output-dir', + default='.', + help='Directory to write the .glb files into.', + type=click.Path(file_okay=False, path_type=Path)) +@click.option('-j', + '--jobs', + default=0, + help='Worker processes to convert with. Defaults to the CPU count.', + type=int) +@click.option('-D', + '--database', + default=None, + help="The game's data/database directory, to draw NPCs and pickups with their own " + 'models. Without it they are written as empty named nodes.', + type=click.Path(exists=True, file_okay=False, path_type=Path)) +@click.option('--ignore-failures', is_flag=True, help='Log and skip a level that will not read.') +@debug_option +def ldb2glb(levels: tuple[Path, ...], output_dir: Path, jobs: int, database: Path | None, *, + ignore_failures: bool) -> None: + """ + Convert each LEVELS ``.ldb`` to a ``.glb`` holding its geometry. + + A directory is searched recursively for ``.ldb`` files. Levels are accepted loose or still + wrapped in a RA-> block, so a file taken straight out of an archive works. + + Each placed mesh becomes its own node with its own transform, keeping props separate from the + architecture, and every face is textured with the image the game gives it. Coordinates are + passed through unchanged because levels are already Y-up. + + Pass ``--database`` to draw the NPCs and pickups with their own models, read from the game's + ``skins`` and ``level_items`` directories. + + Reading a level is processor-bound, so levels are converted in parallel processes. + """ # noqa: DOC501 + found: list[Path] = [] + for level in levels: + found.extend(sorted(level.rglob('*.ldb')) if level.is_dir() else [level]) + if not found: + click.echo('No .ldb files found.', err=True) + raise click.Abort + workers = min(jobs or default_jobs(), len(found)) + results: list[_Result] = [] + if workers > 1: + with ProcessPoolExecutor(max_workers=workers) as pool: + results.extend(pool.map(_convert, [(level, output_dir, database) for level in found])) + else: + results.extend(_convert((level, output_dir, database)) for level in found) + converted = 0 + for result in results: + if result.error: + if not ignore_failures: + click.echo(f'{result.name}: {result.error}', err=True) + raise click.Abort + log.warning('Skipping `%s`: %s', result.name, result.error) + continue + converted += 1 + click.echo(f'{result.name}: {result.meshes} meshes, {result.props} props, ' + f'{result.faces} faces, {result.images} images, ' + f'{result.placements} placements ({result.models} modelled), ' + f'{result.clips} clips') + click.echo(f'{converted}/{len(found)} levels converted into {output_dir}.') diff --git a/dade/maxpane/commands/ldb_textures.py b/dade/maxpane/commands/ldb_textures.py new file mode 100644 index 0000000..85135ca --- /dev/null +++ b/dade/maxpane/commands/ldb_textures.py @@ -0,0 +1,85 @@ +"""``dade maxpane ldb-textures`` - write out the images a level embeds.""" +from __future__ import annotations + +from pathlib import PurePosixPath, PureWindowsPath +import logging +import pathlib + +import click + +from dade.maxpane.blocks import unwrap +from dade.maxpane.ldb import InvalidLevelError, read_textures + +from .utils import debug_option + +__all__ = ('ldb_textures',) + +log = logging.getLogger(__name__) + + +def _relative(path: str) -> PurePosixPath: + """ + Turn an authored Windows path into a safe relative one. + + Parameters + ---------- + path : str + The path as stored in the level. + + Returns + ------- + PurePosixPath + The path with any drive letter, leading separator and upward step removed, so it can only + ever name somewhere under the output directory. + """ + windows = PureWindowsPath(path) + # A level's texture paths are whatever the artist's machine had, so they are not to be trusted + # with where a file lands: `..` in one of them would write outside the directory asked for. + parts = [ + part for part in windows.parts if part not in {'\\', '/', '.', '..'} and ':' not in part + ] + return PurePosixPath(*parts) if parts else PurePosixPath('texture') + + +@click.command(name='ldb-textures') +@click.argument('levels', + nargs=-1, + required=True, + type=click.Path(exists=True, path_type=pathlib.Path)) +@click.option('-o', + '--output-dir', + default='.', + help='Directory to write the images into.', + type=click.Path(file_okay=False, path_type=pathlib.Path)) +@click.option('--flat', is_flag=True, help='Write every image into one directory.') +@debug_option +def ldb_textures(levels: tuple[pathlib.Path, ...], output_dir: pathlib.Path, *, flat: bool) -> None: + """ + Write out the images embedded in each LEVELS ``.ldb``. + + A directory is searched recursively. Each image is stored in the level exactly as the artist + saved it, so the bytes are written through untouched and keep their original extension. Images + land under the directory tree of the path they were authored at, with the drive letter dropped; + pass --flat to put them all in one directory instead. + """ # noqa: DOC501 + found: list[pathlib.Path] = [] + for level in levels: + found.extend(sorted(level.rglob('*.ldb')) if level.is_dir() else [level]) + if not found: + click.echo('No .ldb files found.', err=True) + raise click.Abort + written = 0 + for level in found: + try: + textures = read_textures(unwrap(level.read_bytes())[0]) + except (IndexError, InvalidLevelError, ValueError) as e: + click.echo(f'{level.name}: {e}', err=True) + raise click.Abort from e + for texture in textures: + relative = _relative(texture.path) + destination = output_dir / (relative.name if flat else relative) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(texture.data) + written += 1 + click.echo(f'{level.name}: {len(textures)} images.') + click.echo(f'{written} images written to {output_dir}.') diff --git a/dade/maxpane/commands/models.py b/dade/maxpane/commands/models.py new file mode 100644 index 0000000..fa8f3fb --- /dev/null +++ b/dade/maxpane/commands/models.py @@ -0,0 +1,137 @@ +"""Find the models a level's NPCs and pickups are drawn with, and read them off disk.""" +from __future__ import annotations + +from typing import TYPE_CHECKING +import logging +import re + +from dade.maxpane.blocks import unwrap +from dade.maxpane.model import InvalidModelError, read_model +from dade.maxpane.typing import TextureImage + +if TYPE_CHECKING: + from pathlib import Path + + from dade.maxpane.typing import Level, Model + +__all__ = ('load_models',) + +log = logging.getLogger(__name__) + +_EXPORT_DATA = re.compile(r'ExportData\s*=\s*([^;]+);', re.IGNORECASE) +"""How a pickup's ``.txt`` names the file holding its geometry.""" + + +def _skin(database: Path, skin: str) -> Path | None: + """ + Find a character's model. + + Parameters + ---------- + database : pathlib.Path + The ``data/database`` directory. + skin : str + The skin's directory name, as the level records it. + + Returns + ------- + pathlib.Path | None + The model, or :py:obj:`None` when the directory holds none. + """ + directory = database / 'skins' / skin + if not directory.is_dir(): + return None + found = sorted(p for p in directory.iterdir() if p.suffix.lower() == '.kfs') + # A skin ships one model per level of detail; the nearest is the one to draw. + return next((p for p in found if p.stem.lower().endswith('_l0')), found[0] if found else None) + + +def _item(database: Path, item: str) -> Path | None: + """ + Find a pickup's model, which its script names rather than the directory listing. + + Parameters + ---------- + database : pathlib.Path + The ``data/database`` directory. + item : str + The pickup's directory name, as the level records it. + + Returns + ------- + pathlib.Path | None + The model, or :py:obj:`None` when the script does not name a readable one. + """ + script = database / 'level_items' / f'{item}.txt' + if not script.is_file(): + return None + match = _EXPORT_DATA.search(script.read_text('latin-1')) + if match is None: + return None + model = database / 'level_items' / item / match.group(1).strip() + return model if model.is_file() else None + + +def _read(path: Path) -> Model | None: + """ + Read one model and the images its materials name. + + The model carries a search path -- its own ``textures`` directory, then the shared one beside + it -- and the images are looked up along it in order. + + Parameters + ---------- + path : pathlib.Path + The model file. + + Returns + ------- + Model | None + The model with its images filled in, or :py:obj:`None` when it will not read. + """ + try: + model = read_model(unwrap(path.read_bytes())[0]) + except (IndexError, InvalidModelError, ValueError): + log.warning('Could not read the model `%s`.', path) + return None + textures = [] + for file in dict.fromkeys(model.materials.values()): + for folder in model.search: + candidate = path.parent / folder / file + if candidate.is_file(): + textures.append(TextureImage(data=candidate.read_bytes(), kind=0, path=file)) + break + else: + log.debug('No image named `%s` for `%s`.', file, path.name) + return model._replace(textures=tuple(textures)) + + +def load_models(database: Path, level: Level) -> dict[str, Model]: + """ + Read every model a level needs, keyed by the node label its placements get. + + Parameters + ---------- + database : pathlib.Path + The game's ``data/database`` directory. + level : Level + The level whose NPCs and pickups are to be drawn. + + Returns + ------- + dict[str, Model] + Node label to model, missing an entry for anything that could not be found. + """ + wanted = {f'character:{c.skin}': ('skins', c.skin) for c in level.characters} + wanted.update({f'item:{i.item}': ('items', i.item) for i in level.items}) + out: dict[str, Model] = {} + for label, (kind, name) in wanted.items(): + path = _skin(database, name) if kind == 'skins' else _item(database, name) + if path is None: + log.debug('No model for `%s`.', label) + continue + model = _read(path) + if model is not None: + out[label] = model + log.debug('Read %d of %d models.', len(out), len(wanted)) + return out diff --git a/dade/maxpane/commands/ras_extract.py b/dade/maxpane/commands/ras_extract.py new file mode 100644 index 0000000..77566c9 --- /dev/null +++ b/dade/maxpane/commands/ras_extract.py @@ -0,0 +1,98 @@ +"""``dade maxpane ras-extract`` - unpack a RAS archive, MPM package, or disc image.""" +from __future__ import annotations + +from fnmatch import fnmatch +from pathlib import Path +import logging + +import click + +from dade.common.exceptions import InvalidFormatError +from dade.maxpane.ras import InvalidArchiveError, member_bytes, read_directory + +from .sources import NoArchivesFoundError, iter_archives +from .utils import debug_option + +__all__ = ('ras_extract',) + +log = logging.getLogger(__name__) + + +def _under(output_dir: Path, path: str) -> Path | None: + r""" + Place one member's stored path inside the output directory, or refuse to. + + A member's path is whatever the archive says, and nothing stops an archive naming + ``..\..\.ssh\authorized_keys``. Anything that would land outside the directory the caller + asked for is dropped rather than written somewhere it was not wanted. + + Parameters + ---------- + output_dir : Path + Directory the caller asked members to be written into. + path : str + The member's path, as the archive stores it. + + Returns + ------- + Path | None + Where to write the member, or :py:obj:`None` when its path escapes *output_dir*. + """ + root = output_dir.resolve() + destination = (root / path).resolve() + return destination if destination.is_relative_to(root) else None + + +def _unpack(label: str, data: bytes, patterns: tuple[str, ...], output_dir: Path, *, + raw: bool) -> tuple[int, int]: + selected = [ + entry for entry in read_directory(data).entries + if not patterns or any(fnmatch(entry.path, pattern) for pattern in patterns) + ] + log.debug('Extracting %d members from `%s`.', len(selected), label) + total = written = 0 + for entry in selected: + if (destination := _under(output_dir, entry.path)) is None: + log.warning('Skipping `%s`: it would be written outside the output directory.', + entry.path) + continue + destination.parent.mkdir(parents=True, exist_ok=True) + payload = member_bytes(data, entry, raw=raw) + destination.write_bytes(payload) + total += len(payload) + written += 1 + return total, written + + +@click.command(name='ras-extract') +@click.argument('source', type=click.Path(exists=True, path_type=Path)) +@click.argument('patterns', nargs=-1) +@click.option('-o', + '--output-dir', + default='.', + help='Directory to write members into.', + type=click.Path(file_okay=False, path_type=Path)) +@click.option('--raw', is_flag=True, help='Keep the RA-> and RC-> wrappers.') +@debug_option +def ras_extract(source: Path, patterns: tuple[str, ...], output_dir: Path, *, raw: bool) -> None: + """ + Extract members of the archives named by SOURCE into the output directory. + + SOURCE is a RAS archive, an MPM mod package, an ISO image, or the cue sheet of a cue/bin pair. + Every archive on a disc image is extracted, all into the same tree, which reproduces the layout + the game itself sees because the archives share one namespace. + + PATTERNS are globs matched against each member's full in-archive path. Every member is + extracted when none are given. + """ # noqa: DOC501 + total = 0 + count = 0 + try: + for label, data in iter_archives(source): + written_bytes, written_members = _unpack(label, data, patterns, output_dir, raw=raw) + total += written_bytes + count += written_members + except (InvalidArchiveError, InvalidFormatError, NoArchivesFoundError) as e: + click.echo(str(e), err=True) + raise click.Abort from e + click.echo(f'{count} members, {total} bytes written to {output_dir}.') diff --git a/dade/maxpane/commands/ras_list.py b/dade/maxpane/commands/ras_list.py new file mode 100644 index 0000000..4245e14 --- /dev/null +++ b/dade/maxpane/commands/ras_list.py @@ -0,0 +1,67 @@ +"""``dade maxpane ras-list`` - show the directory of a RAS archive, MPM package, or disc image.""" +from __future__ import annotations + +from collections import Counter +from pathlib import Path +import json +import logging + +import click + +from dade.common.exceptions import InvalidFormatError +from dade.maxpane.ras import InvalidArchiveError, read_directory + +from .sources import NoArchivesFoundError, iter_archives +from .utils import debug_option + +__all__ = ('ras_list',) + +log = logging.getLogger(__name__) + + +def _describe(label: str, data: bytes, report: dict[str, list[dict[str, object]]] | None) -> None: + contents = read_directory(data) + if report is not None: + report[label] = [{ + 'modified': entry.modified, + 'path': entry.path, + 'size': entry.size, + 'stored_size': entry.stored_size + } for entry in contents.entries] + return + histogram = Counter( + entry.name.rsplit('.', 1)[-1].lower() if '.' in entry.name else '' + for entry in contents.entries) + intact = 'intact' if contents.data_end == len(data) else 'TRUNCATED' + click.echo(f'{label}: v{contents.header.version:.2f}, ' + f'{contents.header.file_count} members in ' + f'{contents.header.directory_count} directories, {intact}.') + click.echo(f'By extension: {dict(histogram.most_common())}') + for entry in contents.entries: + click.echo(f' {entry.size:>10d} {entry.modified or "":<24s} {entry.path}') + + +@click.command(name='ras-list') +@click.argument('source', type=click.Path(exists=True, path_type=Path)) +@click.option('--json', 'as_json', is_flag=True, help='Print the directory as JSON.') +@debug_option +def ras_list(source: Path, *, as_json: bool) -> None: + """ + List the members of the archives named by SOURCE. + + SOURCE is a RAS archive, an MPM mod package, an ISO image, or the cue sheet of a cue/bin pair. + Every archive on a disc image is listed. For each archive this prints the format version, + member and directory counts, a histogram of member extensions, and one line per member. + + An archive is reported intact when its header, both tables, and every stored size together + account for the file exactly. + """ # noqa: DOC501 + report: dict[str, list[dict[str, object]]] = {} + try: + for label, data in iter_archives(source): + _describe(label, data, report if as_json else None) + except (InvalidArchiveError, InvalidFormatError, NoArchivesFoundError) as e: + click.echo(str(e), err=True) + raise click.Abort from e + if as_json: + click.echo(json.dumps(report, indent=2, sort_keys=True)) diff --git a/dade/maxpane/commands/sources.py b/dade/maxpane/commands/sources.py new file mode 100644 index 0000000..212ea34 --- /dev/null +++ b/dade/maxpane/commands/sources.py @@ -0,0 +1,130 @@ +"""Resolve a command-line argument into the archives it names.""" +from __future__ import annotations + +from pathlib import Path, PurePosixPath +from tempfile import TemporaryDirectory +from typing import TYPE_CHECKING +import logging +import re +import subprocess as sp + +from dade.common.disc import open_image +from dade.common.tools import ToolNotFoundError, run_unshield +from dade.maxpane.ras import MAGIC + +if TYPE_CHECKING: + from collections.abc import Iterator + +__all__ = ('ARCHIVE_SUFFIXES', 'CABINET_NAME', 'NoArchivesFoundError', 'iter_archives') + +log = logging.getLogger(__name__) + +_CABINET_PART_RE = re.compile(r'^data\d*\.(cab|hdr)$', re.IGNORECASE) + +ARCHIVE_SUFFIXES = ('.ras', '.mpm') +"""Suffixes identifying an archive, matched case-insensitively. + +:meta hide-value: +""" +CABINET_NAME = 'data1.cab' +"""Name of the InstallShield cabinet holding the archives that are not loose on the disc. + +:meta hide-value: +""" + + +class NoArchivesFoundError(ValueError): + """Raised when a source holds no RAS archive.""" + + +def _is_archive_name(name: str) -> bool: + return name.lower().endswith(ARCHIVE_SUFFIXES) + + +def _is_archive_file(path: Path) -> bool: + with path.open('rb') as handle: + return handle.read(len(MAGIC)) == MAGIC + + +def _iter_directory(root: Path) -> Iterator[tuple[str, bytes]]: + for path in sorted(root.rglob('*')): + if path.is_file() and _is_archive_name(path.name): + log.debug('Reading `%s`.', path) + yield str(path.relative_to(root)), path.read_bytes() + + +def _iter_cabinet(cabinet: Path) -> Iterator[tuple[str, bytes]]: + with TemporaryDirectory() as work_dir: + try: + run_unshield(cabinet, Path(work_dir)) + except ToolNotFoundError: + log.warning('Skipping `%s`: unshield is not installed.', cabinet.name) + return + except sp.CalledProcessError: + log.warning('Skipping `%s`: unshield failed to unpack it.', cabinet.name) + return + yield from _iter_directory(Path(work_dir)) + + +def _iter_image(source: Path) -> Iterator[tuple[str, bytes]]: + image = open_image(source) + paths = [path for path, _ in image.iter_files()] + for path in paths: + if _is_archive_name(path): + log.debug('Reading `%s` from the image.', path) + yield path, image.read_file(path) + for cabinet in (path for path in paths if path.lower().endswith(CABINET_NAME)): + # unshield reads the volumes and header sitting beside the cabinet, so they have to be + # staged together. Nothing else on the disc is copied out. + directory = PurePosixPath(cabinet).parent + with TemporaryDirectory() as staging: + for path in paths: + name = PurePosixPath(path) + if name.parent == directory and _CABINET_PART_RE.match(name.name): + (Path(staging) / name.name).write_bytes(image.read_file(path)) + yield from _iter_cabinet(Path(staging) / PurePosixPath(cabinet).name) + + +def iter_archives(source: Path) -> Iterator[tuple[str, bytes]]: + """ + Yield every archive reachable from *source*. + + A file beginning with the RAS magic is yielded as-is. A directory is scanned recursively. A + disc image, either an ISO or the ``.cue`` of a cue/bin pair, yields the archives lying loose on + it and then those inside its InstallShield cabinet, which is unpacked to a temporary directory. + Max Payne's retail disc needs both: the levels are loose but the shared game database is in the + cabinet. + + A cabinet is skipped with a warning when ``unshield`` is missing or fails, so the archives that + were reachable are still returned. + + Parameters + ---------- + source : Path + An archive, a directory, an InstallShield cabinet, or a disc image. + + Yields + ------ + tuple[str, bytes] + A label for messages and the whole archive. + + Raises + ------ + NoArchivesFoundError + If no archive could be reached. + """ + found = 0 + if source.is_dir(): + members: Iterator[tuple[str, bytes]] = _iter_directory(source) + elif source.suffix.lower() == '.cab' or source.name.lower() == CABINET_NAME: + members = _iter_cabinet(source) + elif _is_archive_file(source): + members = iter(((source.name, source.read_bytes()),)) + else: + members = _iter_image(source) + for label, data in members: + found += 1 + yield label, data + if not found: + msg = f'No RAS archives found in `{source}`.' + raise NoArchivesFoundError(msg) diff --git a/dade/maxpane/commands/utils.py b/dade/maxpane/commands/utils.py new file mode 100644 index 0000000..9260937 --- /dev/null +++ b/dade/maxpane/commands/utils.py @@ -0,0 +1,12 @@ +"""Shared helpers for the Max Payne commands.""" +from __future__ import annotations + +import bascom + +__all__ = ('debug_option',) + +debug_option = bascom.debug_option({'dade.common': {}, 'dade.maxpane': {}}) +"""Attach ``-d/--debug`` to a leaf command and route it through :py:func:`bascom.setup_logging`. + +:meta hide-value: +""" diff --git a/dade/maxpane/crypto.py b/dade/maxpane/crypto.py new file mode 100644 index 0000000..c0ef9ba --- /dev/null +++ b/dade/maxpane/crypto.py @@ -0,0 +1,90 @@ +""" +The seeded stream cipher guarding Max Payne's RAS archives and their wrapped blocks. + +Transcribed from ``R_File::decryptWithSeed`` in ``rl.dll``. Each byte is rotated left by its index +modulo five, exclusive-ORed with a value derived from that index, then offset by the low byte of a +Wichmann-Hill generator stepped once per byte. The generator is written the way the original +compiler emitted it, because archive seeds are large enough that ``171 * seed`` overflows a signed +32-bit integer and the wrapped result is part of the key schedule. +""" +from __future__ import annotations + +__all__ = ('decrypt', 'next_seed') + +_MASK32 = 0xFFFFFFFF +_SIGN_BIT = 0x80000000 +_WRAP = 0x100000000 +_MAGIC = 0xB92143FB - _WRAP +"""Signed multiplier the compiler substituted for division by 177. + +:meta hide-value: +""" +_MODULUS = 30269 +"""Wichmann-Hill modulus, equal to ``171 * 177 + 2``. + +:meta hide-value: +""" +_DIVISOR_SHIFT = 7 +"""Right shift paired with :py:data:`_MAGIC` to complete the division. + +:meta hide-value: +""" + + +def _signed(value: int) -> int: + value &= _MASK32 + return value - _WRAP if value >= _SIGN_BIT else value + + +def next_seed(seed: int) -> int: + """ + Advance the cipher's Wichmann-Hill generator by one step. + + Parameters + ---------- + seed : int + Current generator state. + + Returns + ------- + int + The next state, as a signed 32-bit integer. + """ + state = _signed(seed) + quotient = _signed((_MAGIC * state) >> 32) + quotient = _signed(quotient + state) + quotient = _signed(quotient >> _DIVISOR_SHIFT) + quotient = _signed(quotient + ((quotient & _MASK32) >> 31)) + quotient = _signed(quotient * _MODULUS) + scaled = _signed(_signed(state * 8) - state) + scaled = _signed(state + _signed(scaled * 8)) + scaled = _signed(scaled + _signed(scaled * 2)) + return _signed(scaled - quotient) + + +def decrypt(data: bytes, seed: int) -> bytes: + """ + Decrypt a block with the given seed. + + Parameters + ---------- + data : bytes + Ciphertext. The keystream always restarts at index zero, so callers must pass whole + blocks rather than slices of one. + seed : int + Signed cipher seed. Zero is promoted to one. + + Returns + ------- + bytes + The plaintext, the same length as *data*. + """ + if seed == 0: + seed = 1 + out = bytearray(len(data)) + for index, byte in enumerate(data): + rotation = index % 5 + rotated = ((byte << rotation) | (byte >> (8 - rotation))) & 0xFF if rotation else byte + seed = next_seed(seed) + out[index] = (((((index & 0xFF) + 3) & 0xFF) * 6 & 0xFF) ^ rotated) + (seed & 0xFF) & 0xFF + return bytes(out) diff --git a/dade/maxpane/decals.py b/dade/maxpane/decals.py new file mode 100644 index 0000000..6a83932 --- /dev/null +++ b/dade/maxpane/decals.py @@ -0,0 +1,189 @@ +"""Pull coplanar surfaces apart so a viewer's depth buffer can tell them apart.""" +from __future__ import annotations + +from collections import defaultdict +from typing import TYPE_CHECKING + +if TYPE_CHECKING: # pragma: no cover + from collections.abc import Sequence + + from .typing import Vector3 + +__all__ = ('DECAL_STEP', 'layer_faces') + +DECAL_STEP = 1.0 / 128.0 +"""How far apart to hold two surfaces that share a plane, in level units. + +A level unit is about a metre, so this lifts a graffiti tag eight millimetres off its wall: far +too little to see, and far enough that a depth buffer stops guessing which one is in front.""" + +_NORMAL_STEPS = 128.0 +"""Grid the normal is rounded onto when deciding whether two faces face the same way.""" + +_AWAY_STEPS = 16.0 +"""Grid a plane's distance from the origin is rounded onto.""" + +_CELL = 4.0 +"""Side of the square the in-plane index buckets faces into.""" + +_TOUCH = 0.03 +"""Overlap two boxes must share before they count as stacked rather than merely adjacent. + +Level architecture is tiled, so a wall is many faces whose boxes meet along their edges. Without +this every tile would read as covering its neighbour.""" + +_FLAT = 1e-9 +"""Below this a normal is too short to say which way a face points.""" + +_PAIR = 2 +"""Faces a plane needs before anything on it can be covering anything else.""" + + +def _frame(normal: Vector3) -> tuple[Vector3, Vector3]: + """ + Pick two axes spanning the plane a normal describes. + + Parameters + ---------- + normal : Vector3 + A unit normal. + + Returns + ------- + tuple[Vector3, Vector3] + Two perpendicular unit vectors lying in the plane. + """ + x, y, z = normal + # Cross with whichever axis the normal leans on least, so the result is never degenerate. + other = (0.0, 0.0, 1.0) if abs(z) < abs(x) or abs(z) < abs(y) else (1.0, 0.0, 0.0) + ax = y * other[2] - z * other[1] + ay = z * other[0] - x * other[2] + az = x * other[1] - y * other[0] + length = (ax * ax + ay * ay + az * az) ** 0.5 or 1.0 + first = (ax / length, ay / length, az / length) + return first, (y * first[2] - z * first[1], z * first[0] - x * first[2], + x * first[1] - y * first[0]) + + +def _profile(normal: Vector3, + corners: Sequence[Vector3]) -> tuple[tuple[int, ...], tuple[float, ...]] | None: + """ + Reduce a face to the plane it lies in and the box it covers within that plane. + + Parameters + ---------- + normal : Vector3 + The face's outward normal, which need not be unit length. + corners : collections.abc.Sequence[Vector3] + The face's corners. + + Returns + ------- + tuple[tuple[int, ...], tuple[float, ...]] | None + The quantised plane and the box, or :py:obj:`None` when the face has no plane. + """ + length = sum(v * v for v in normal) ** 0.5 + if length < _FLAT or not corners: + return None + unit = (normal[0] / length, normal[1] / length, normal[2] / length) + first, second = _frame(unit) + flat = [(sum(f * p for f, p in zip(first, corner, strict=True)), + sum(s * p for s, p in zip(second, corner, strict=True))) for corner in corners] + away = sum(n * p for n, p in zip(unit, corners[0], strict=True)) + # A face and one turned to face the other way never hide each other, so the sign is kept. + plane = (*(round(v * _NORMAL_STEPS) for v in unit), round(away * _AWAY_STEPS)) + box = (min(p[0] for p in flat), min(p[1] for p in flat), max( + p[0] for p in flat), max(p[1] for p in flat)) + return plane, box + + +def _stacked(one: Sequence[float], two: Sequence[float]) -> bool: + """ + Say whether two in-plane boxes cover a shared patch rather than merely meeting. + + Parameters + ---------- + one : collections.abc.Sequence[float] + A box as minimum and maximum on both axes. + two : collections.abc.Sequence[float] + The other box. + + Returns + ------- + bool + :py:obj:`True` when the two share more than a seam. + """ + return (min(one[2], two[2]) - max(one[0], two[0]) > _TOUCH + and min(one[3], two[3]) - max(one[1], two[1]) > _TOUCH) + + +def _cells(box: Sequence[float]) -> list[tuple[int, int]]: + """ + List the index squares a box touches. + + Parameters + ---------- + box : collections.abc.Sequence[float] + A box as minimum and maximum on both axes. + + Returns + ------- + list[tuple[int, int]] + The squares, as integer coordinates. + """ + return [(x, y) for x in range(int(box[0] // _CELL), + int(box[2] // _CELL) + 1) + for y in range(int(box[1] // _CELL), + int(box[3] // _CELL) + 1)] + + +def layer_faces(surfaces: Sequence[tuple[Vector3, Sequence[Vector3]]]) -> list[int]: + """ + Work out how far off its plane each face has to sit to stop fighting the ones behind it. + + A level draws graffiti, signs and stains as their own polygons laid exactly on the wall, and + keeps every variant of a switchable surface -- a television showing static or a programme, a + neon sign lit or dark -- in the same place. The engine chose between them and drew what was + left in tree order, so none of it ever fought. A viewer that draws the whole level at once has + only its depth buffer to go on, and two surfaces at the same depth flicker against each other. + + Faces are considered in order of the area they cover, so the surface underneath keeps the + plane the level gave it and only the smaller things laid over it move. + + Parameters + ---------- + surfaces : collections.abc.Sequence[tuple[Vector3, collections.abc.Sequence[Vector3]]] + Each face's outward normal and its corners, both in the space they will be drawn in. + + Returns + ------- + list[int] + How many steps of :py:data:`DECAL_STEP` to lift each face along its normal, one entry per + face given, and zero for anything nothing else is stacked on. + """ + boxes: dict[int, tuple[float, ...]] = {} + planes: defaultdict[tuple[int, ...], list[int]] = defaultdict(list) + for index, (normal, corners) in enumerate(surfaces): + found = _profile(normal, corners) + if found is not None: + planes[found[0]].append(index) + boxes[index] = found[1] + out = [0] * len(surfaces) + for members in planes.values(): + if len(members) < _PAIR: + continue + grid: defaultdict[tuple[int, int], list[int]] = defaultdict(list) + widest = sorted(members, + key=lambda i: -(boxes[i][2] - boxes[i][0]) * (boxes[i][3] - boxes[i][1])) + for index in widest: + box = boxes[index] + squares = _cells(box) + below = -1 + for square in squares: + for other in grid[square]: + if _stacked(box, boxes[other]): + below = max(below, out[other]) + out[index] = below + 1 + for square in squares: + grid[square].append(index) + return out diff --git a/dade/maxpane/gltf.py b/dade/maxpane/gltf.py new file mode 100644 index 0000000..a082412 --- /dev/null +++ b/dade/maxpane/gltf.py @@ -0,0 +1,1337 @@ +""" +Turn Max Payne level geometry into a binary glTF (``.glb``). + +A level stores its geometry twice. The BSP faces split it for visibility, and a static mesh +container holds the geometry the game actually draws: a corner array shared by every mesh, then one +entry per placed mesh with its own vertices, normals and transform. The second form carries +Remedy's own texture coordinates and keeps props as separate placed objects, so it is used whenever +it can be read; the BSP faces are the fallback and are written untextured. + +Faces are convex and are triangulated as fans, which is exact for convex polygons. Each fan is +wound to agree with the face's stored normal so that back-face culling shows a level from the +inside when the camera sits outside it, which is what a level viewer wants. + +Each material's image is embedded. Targa images are re-encoded as PNG because glTF only carries PNG +and JPEG; JPEG data is embedded as it was stored. + +Level coordinates are Y-up, which is glTF's convention, so positions are written through unchanged. +""" +from __future__ import annotations + +from io import BytesIO +from types import MappingProxyType +from typing import TYPE_CHECKING, Any +import json +import logging +import math +import struct + +from PIL import Image + +from .decals import DECAL_STEP, layer_faces + +if TYPE_CHECKING: + from collections.abc import Callable, Mapping, Sequence + + from .typing import ( + Corner, + Level, + Model, + Placement, + Polygon, + PropAnimation, + RenderMesh, + StaticMesh, + TextureImage, + Vector3, + ) + +__all__ = ('GLB_MAGIC', 'build_glb') + +log = logging.getLogger(__name__) + +GLB_MAGIC = b'glTF' +"""Magic starting every binary glTF. + +:meta hide-value: +""" + +_VERSION = 2 +_JSON_CHUNK = b'JSON' +_BIN_CHUNK = b'BIN\x00' +_FLOAT = 5126 +_UNSIGNED_INT = 5125 +_ARRAY_BUFFER = 34962 +_ELEMENT_ARRAY_BUFFER = 34963 +_TRIANGLES = 4 +_REPEAT = 10497 +_JPEG_MAGIC = b'\xff\xd8' +_PNG_MAGIC = b'\x89PNG' +_FALLBACK_COLOUR = (0.72, 0.72, 0.74, 1.0) +_NODRAW = 'nodraw' +_MAX_KEYFRAMES = 24 +"""Keyframes kept per clip. The stored curves carry up to 256 samples of a smooth ease, which is +far more than the motion needs. + +:meta hide-value: +""" +_STILL = 1e-6 +"""Below this a channel's two poses are the same and the channel is not worth writing. + +:meta hide-value: +""" +_SLERP_LINEAR = 0.9995 +"""Above this the two rotations are close enough that a straight walk is indistinguishable, and +taking the arc would divide by nearly zero. + +:meta hide-value: +""" +_NON_DRAWING = frozenset({'cameracollision', 'dummy'}) +"""Material categories the engine does not draw. + +Levels name these outright: anything ending in ``nodraw`` is collision geometry, and ``dummy`` +carries Remedy's placeholder image. Keeping them would paper a level in magenta ``DUMMY`` text. + +:meta hide-value: +""" + +_SKY = 'skybox' +"""Category on the faces that close a level off where it opens to the sky. + +These carry a placeholder image too -- teal ``SKYBOX`` text -- but unlike ``dummy`` they cannot +just be dropped. They are the only thing between a courtyard, an alley or a stretch of street and +nothing at all, so leaving them out puts a hole through the level wherever the game showed sky. +They are written with a flat colour instead, which a viewer can swap for a real sky. + +:meta hide-value: +""" + +_SKY_COLOUR = (0.29, 0.33, 0.40, 1.0) +"""Stand-in for the sky. Max Payne draws its own sky from the renderer's settings rather than from +anything the level stores, so there is nothing in the file to read: this is the dull overcast +blue-grey the game's nights are lit by. + +:meta hide-value: +""" + +_FLAT_FAN = 1e-9 +"""Below this a fan triangle is a straight line and says nothing about which way its face points. + +:meta hide-value: +""" + +_UNLIT = 'KHR_materials_unlit' +"""Extension marking a material that takes its colour straight from the base colour. + +:meta hide-value: +""" + + +def _pad(data: bytes, fill: bytes) -> bytes: + return data + fill * (-len(data) % 4) + + +def _draws(level: Level, material_id: int) -> bool: + """ + Report whether the engine draws faces using a material. + + Parameters + ---------- + level : Level + The level the material belongs to. + material_id : int + Identifier a face references. + + Returns + ------- + bool + :py:obj:`False` for collision-only and placeholder materials. + """ + material = level.materials.get(material_id) + if material is None: + return True + category = material.category.lower() + return not (category.endswith(_NODRAW) or category in _NON_DRAWING) + + +def _wind(fan: list[tuple[int, int, int]], positions: Sequence[Vector3], + normal: Vector3) -> list[tuple[int, int, int]]: + """ + Reverse a fan whose corner order disagrees with its normal. + + The decision is taken from the first triangle that has any area. Levels put extra corners part + way along a face's edges, where one polygon meets several, and a face whose first three corners + are three of those is a straight line with no side to face. Reading the winding off it points + the whole face the wrong way, and a floor drawn from below is a hole in the level seen from + above. + + Parameters + ---------- + fan : list[tuple[int, int, int]] + Triangles as written. + positions : collections.abc.Sequence[Vector3] + Positions the triangles index. + normal : Vector3 + The face's outward normal. + + Returns + ------- + list[tuple[int, int, int]] + The fan, reversed when needed so it winds counter-clockwise seen from the front. + """ + for triangle in fan: + a, b, c = (positions[i] for i in triangle) + u = (b[0] - a[0], b[1] - a[1], b[2] - a[2]) + v = (c[0] - a[0], c[1] - a[1], c[2] - a[2]) + cross = (u[1] * v[2] - u[2] * v[1], u[2] * v[0] - u[0] * v[2], u[0] * v[1] - u[1] * v[0]) + facing = sum(x * n for x, n in zip(cross, normal, strict=True)) + if abs(facing) <= _FLAT_FAN: + continue + return fan if facing > 0.0 else [(z, y, x) for x, y, z in fan] + return fan + + +def _image_payload(texture: TextureImage) -> tuple[bytes, str] | None: + """ + Prepare an image for embedding, re-encoding Targa as PNG. + + Parameters + ---------- + texture : TextureImage + The image as the level stored it. + + Returns + ------- + tuple[bytes, str] | None + The bytes and their MIME type, or :py:obj:`None` if the image cannot be decoded. + """ + if texture.data[:2] == _JPEG_MAGIC: + return texture.data, 'image/jpeg' + if texture.data[:4] == _PNG_MAGIC: + return texture.data, 'image/png' + try: + with Image.open(BytesIO(texture.data)) as image: + buffer = BytesIO() + mode = 'RGBA' if image.mode in {'RGBA', 'LA', 'P'} else 'RGB' + image.convert(mode).save(buffer, format='PNG') + except OSError: + log.warning('Could not decode `%s`.', texture.path) + return None + return buffer.getvalue(), 'image/png' + + +_OPAQUE = 240 +_CLEAR = 15 +_SOFT_FRACTION = 0.05 +_MASK_MODE = 'MASK' +_ALPHA_CUTOFF = 0.5 + + +def _compose_alpha(colour: bytes, mask: bytes) -> tuple[tuple[bytes, str], str] | None: + """ + Put a mask's brightness into a colour image's alpha channel. + + The alpha mode follows the mask itself. A cutout -- foliage, a chain-link fence, a neon sign cut + from its background -- is almost entirely black or white, and reads best as ``MASK`` because it + needs no depth sorting. A mask with a real gradient, such as smoked glass or water, has to + blend. + + Parameters + ---------- + colour : bytes + The colour image as the level stored it. + mask : bytes + The mask image as the level stored it. + + Returns + ------- + tuple[tuple[bytes, str], str] | None + The PNG and its MIME type, and the glTF alpha mode, or :py:obj:`None` when either image + cannot be decoded. + """ + try: + payload, soft = _apply_mask(colour, mask) + except OSError: + return None + return (payload, 'image/png'), 'BLEND' if soft > _SOFT_FRACTION else _MASK_MODE + + +def _apply_mask(colour: bytes, mask: bytes) -> tuple[bytes, float]: + """ + Decode both images, put the mask into the colour's alpha, and measure the mask. + + Parameters + ---------- + colour : bytes + The colour image as the level stored it. + mask : bytes + The mask image as the level stored it. + + Returns + ------- + tuple[bytes, float] + The composed PNG and the share of mask pixels that are neither clear nor opaque. + """ + with Image.open(BytesIO(colour)) as base, Image.open(BytesIO(mask)) as cover: + rgb = base.convert('RGB') + grey = cover.convert('L') + if grey.size != rgb.size: + grey = grey.resize(rgb.size) + histogram = grey.histogram() + rgb.putalpha(grey) + buffer = BytesIO() + rgb.save(buffer, format='PNG') + return buffer.getvalue(), sum(histogram[_CLEAR + 1:_OPAQUE]) / max(sum(histogram), 1) + + +def _node_matrix(transform: Sequence[float]) -> list[float]: + """ + Turn a stored four-by-three transform into a glTF node matrix. + + The transform is conjugated by the same depth mirror applied to vertices, which keeps the + rotation's determinant positive so no renderer has to reverse winding for it. + + Parameters + ---------- + transform : collections.abc.Sequence[float] + Three basis rows then a translation, as ``M_Matrix4x3`` stores them. + + Returns + ------- + list[float] + Sixteen floats in the column-major order glTF expects. + """ + a, b, c, t = (transform[0:3], transform[3:6], transform[6:9], transform[9:12]) + return [ + a[0], a[1], -a[2], 0.0, b[0], b[1], -b[2], 0.0, -c[0], -c[1], c[2], 0.0, t[0], t[1], -t[2], + 1.0 + ] + + +def _place(matrix: Sequence[float], point: Vector3) -> Vector3: + """ + Put a point through a column-major node matrix. + + Parameters + ---------- + matrix : collections.abc.Sequence[float] + Sixteen floats from :py:func:`_node_matrix`. + point : Vector3 + A point in the node's own space. + + Returns + ------- + Vector3 + The point in the scene's space. + """ + x, y, z = point + return (matrix[0] * x + matrix[4] * y + matrix[8] * z + matrix[12], + matrix[1] * x + matrix[5] * y + matrix[9] * z + matrix[13], + matrix[2] * x + matrix[6] * y + matrix[10] * z + matrix[14]) + + +def _turn(matrix: Sequence[float], vector: Vector3) -> Vector3: + """ + Put a direction through a column-major node matrix, leaving the translation out. + + Parameters + ---------- + matrix : collections.abc.Sequence[float] + Sixteen floats from :py:func:`_node_matrix`. + vector : Vector3 + A direction in the node's own space. + + Returns + ------- + Vector3 + The direction in the scene's space. + """ + x, y, z = vector + return (matrix[0] * x + matrix[4] * y + matrix[8] * z, + matrix[1] * x + matrix[5] * y + matrix[9] * z, + matrix[2] * x + matrix[6] * y + matrix[10] * z) + + +def _decompose(transform: Sequence[float]) -> tuple[list[float], list[float], list[float]]: + """ + Split a stored transform into the translation, rotation and scale a glTF node animates on. + + A node carrying a ``matrix`` cannot be animated, so an animated prop has to be written as three + separate properties instead. The split is taken from :py:func:`_node_matrix`'s output so a prop + lands in exactly the same place whether or not it moves. + + Parameters + ---------- + transform : collections.abc.Sequence[float] + Three basis rows then a translation, as ``M_Matrix4x3`` stores them. + + Returns + ------- + tuple[list[float], list[float], list[float]] + The translation, the rotation as an ``xyzw`` quaternion, and the scale. + """ + matrix = _node_matrix(transform) + columns = [matrix[0:3], matrix[4:7], matrix[8:11]] + scale = [math.sqrt(sum(v * v for v in column)) or 1.0 for column in columns] + # A negative determinant would make the rotation a reflection, which no quaternion can hold; + # the mirror conjugation is built to avoid one, and folding it into the scale keeps the node + # correct if a level ever contains it. + left, middle, right = (columns[i] for i in range(3)) + determinant = (left[0] * (middle[1] * right[2] - middle[2] * right[1]) - left[1] * + (middle[0] * right[2] - middle[2] * right[0]) + left[2] * + (middle[0] * right[1] - middle[1] * right[0])) + if determinant < 0: + scale[0] = -scale[0] + rotation = [[columns[c][r] / scale[c] for c in range(3)] for r in range(3)] + return [matrix[12], matrix[13], matrix[14]], _quaternion(rotation), scale + + +def _quaternion(rotation: Sequence[Sequence[float]]) -> list[float]: + """ + Turn a rotation matrix into an ``xyzw`` quaternion. + + Parameters + ---------- + rotation : collections.abc.Sequence[collections.abc.Sequence[float]] + Three rows of three, orthonormal. + + Returns + ------- + list[float] + The quaternion, normalised. + """ + trace = rotation[0][0] + rotation[1][1] + rotation[2][2] + if trace > 0.0: + scale = math.sqrt(trace + 1.0) * 2.0 + out = [(rotation[2][1] - rotation[1][2]) / scale, (rotation[0][2] - rotation[2][0]) / scale, + (rotation[1][0] - rotation[0][1]) / scale, 0.25 * scale] + else: + # Pivot on the largest diagonal entry, which keeps the divisor away from zero. + axis = max(range(3), key=lambda i: rotation[i][i]) + other, third = (axis + 1) % 3, (axis + 2) % 3 + scale = math.sqrt(1.0 + rotation[axis][axis] - rotation[other][other] - + rotation[third][third]) * 2.0 + out = [0.0, 0.0, 0.0, (rotation[third][other] - rotation[other][third]) / scale] + out[axis] = 0.25 * scale + out[other] = (rotation[other][axis] + rotation[axis][other]) / scale + out[third] = (rotation[third][axis] + rotation[axis][third]) / scale + length = math.sqrt(sum(v * v for v in out)) or 1.0 + return [v / length for v in out] + + +def _keyframes(duration: float, curve: Sequence[float], + total: float) -> tuple[list[float], list[float]]: + """ + Thin one curve to a manageable number of keyframes and normalise it. + + Parameters + ---------- + duration : float + How long the clip runs. + curve : collections.abc.Sequence[float] + The curve's samples, evenly spaced across the duration. + total : float + What the curve's last sample means as a whole, so the samples come out as fractions. + + Returns + ------- + tuple[list[float], list[float]] + The keyframe times and the fraction of the motion done at each. + """ + # Round the step up so that adding the final sample back cannot push the count over. + step = max(1, -(-(len(curve) - 1) // (_MAX_KEYFRAMES - 1))) + picked = list(range(0, len(curve), step)) + if picked[-1] != len(curve) - 1: + picked.append(len(curve) - 1) + last = max(len(curve) - 1, 1) + return ([duration * index / last + for index in picked], [curve[index] / total for index in picked]) + + +def _lerp(start: Sequence[float], end: Sequence[float], at: float) -> list[float]: + """ + Walk straight from one vector to another. + + Parameters + ---------- + start : collections.abc.Sequence[float] + The vector at nought. + end : collections.abc.Sequence[float] + The vector at one. + at : float + How far along. + + Returns + ------- + list[float] + The vector at *at*. + """ + return [a + (b - a) * at for a, b in zip(start, end, strict=True)] + + +def _slerp(start: Sequence[float], end: Sequence[float], at: float) -> list[float]: + """ + Walk from one quaternion to another along the shorter arc. + + Straight interpolation would make a door swing at an uneven rate and, for the wide arcs the + game uses, visibly cut the corner. + + Parameters + ---------- + start : collections.abc.Sequence[float] + The rotation at nought, as ``xyzw``. + end : collections.abc.Sequence[float] + The rotation at one. + at : float + How far along. + + Returns + ------- + list[float] + The rotation at *at*, normalised. + """ + dot = sum(a * b for a, b in zip(start, end, strict=True)) + # A quaternion and its negation are the same rotation; flipping picks the shorter way round. + target = list(end) if dot >= 0.0 else [-v for v in end] + dot = abs(dot) + if dot > _SLERP_LINEAR: + out = _lerp(start, target, at) + else: + angle = math.acos(dot) + near = math.sin((1.0 - at) * angle) / math.sin(angle) + far = math.sin(at * angle) / math.sin(angle) + out = [a * near + b * far for a, b in zip(start, target, strict=True)] + length = math.sqrt(sum(v * v for v in out)) or 1.0 + return [v / length for v in out] + + +def _mirror(vector: Vector3) -> Vector3: + """ + Mirror a vector along the depth axis. + + Max Payne is a Direct3D game and stores a left-handed world; glTF is right-handed. Without the + conversion every level comes out as its own mirror image, which only shows up on signage: the + Choir Communications billboard reads backwards. + + Parameters + ---------- + vector : Vector3 + A position or normal in the level's own space. + + Returns + ------- + Vector3 + The vector with its depth component negated. + """ + return (vector[0], vector[1], -vector[2]) + + +class _Document: + """ + Builds a glTF document as meshes are added. + + Parameters + ---------- + level : Level + The level being converted. Its images are embedded up front so materials can refer to them. + """ + def __init__(self, level: Level) -> None: + self._level = level + self._blob = bytearray() + self._by_path: dict[str, int] = {} + self._by_material: dict[tuple[int, int], int] = {} + self._sky_material: int | None = None + self._by_lightmap: dict[int, int | None] = {} + self.views: list[dict[str, Any]] = [] + self.accessors: list[dict[str, Any]] = [] + self.meshes: list[dict[str, Any]] = [] + self.nodes: list[dict[str, Any]] = [] + self.materials: list[dict[str, Any]] = [] + self.images: list[dict[str, Any]] = [] + self.textures: list[dict[str, Any]] = [] + self.animations: list[dict[str, Any]] = [] + self._raw = {texture.path: texture.data for texture in level.textures} + self._masked: dict[tuple[str, str], tuple[int, str]] = {} + self._by_model: dict[str, int] = {} + # A mask is only ever read through the material that names it, so embedding it on its own + # would double its weight for nothing. + masks = {m.alpha for m in level.materials.values() if m.alpha} + for texture in level.textures: + if texture.path in masks: + continue + payload = _image_payload(texture) + if payload is None: + continue + self._by_path.setdefault(texture.path, self._add_texture(payload, texture.path)) + + def add_animation(self, node: int, mesh: str, clip: PropAnimation) -> bool: + """ + Add one clip as a glTF animation driving a node's translation and rotation. + + The level drives the two separately: one curve gives the distance travelled in world units + and the other how far the prop has turned, each with its own sample count. Both are walked + and baked into keyframes, which leaves nothing for a viewer to interpolate differently, and + is why the samples are thinned first -- a crane's curve carries 4096 of them. + + A channel whose two poses agree is left out, so a hinged door gets rotation alone. + + Parameters + ---------- + node : int + Index of the node the clip drives. + mesh : str + The prop's name, which the clip's name is appended to. + clip : PropAnimation + The clip. + + Returns + ------- + bool + Whether the clip drove anything. + """ + start, end = _decompose(clip.start), _decompose(clip.end) + travel = math.dist(start[0], end[0]) + turn = 1.0 - abs(sum(a * b for a, b in zip(start[1], end[1], strict=True))) + + def slide(at: float) -> list[float]: + return _lerp(start[0], end[0], at) + + def spin(at: float) -> list[float]: + return _slerp(start[1], end[1], at) + + channels: list[dict[str, Any]] = [] + samplers: list[dict[str, Any]] = [] + walks = ( + ('translation', 'VEC3', clip.distance, travel, travel, slide), + ('rotation', 'VEC4', clip.turn, turn, 1.0, spin), + ) + for path, kind, curve, moves, total, pose in walks: + if moves <= _STILL or not curve: + continue + times, progress = _keyframes(clip.duration, curve, total) + values = [pose(at) for at in progress] + size = len(values[0]) + samplers.append({ + 'input': + self.accessor(struct.pack(f'<{len(times)}f', *times), + None, + componentType=_FLOAT, + count=len(times), + max=[max(times)], + min=[min(times)], + type='SCALAR'), + 'interpolation': + 'LINEAR', + 'output': + self.accessor(b''.join(struct.pack(f'<{size}f', *value) for value in values), + None, + componentType=_FLOAT, + count=len(values), + type=kind) + }) + channels.append({'sampler': len(samplers) - 1, 'target': {'node': node, 'path': path}}) + if not channels: + return False + self.animations.append({ + 'channels': channels, + # The prop and the clip are both named by the level, and a clip name is only unique + # within its prop, so the two together are what a viewer can list and a script can + # match: `DO_Animate("open1")` sent to `::Sanctum Corridor::door01.DO`. + 'name': f'{mesh}/{clip.name}', + 'samplers': samplers + }) + return True + + def settle(self, node: int, matrix: Sequence[float]) -> None: + """ + Put a node back on a matrix after its clips turned out to drive nothing. + + Parameters + ---------- + node : int + Index of the node. + matrix : collections.abc.Sequence[float] + The node's four-by-three transform. + """ + entry = self.nodes[node] + for key in ('rotation', 'scale', 'translation'): + entry.pop(key, None) + entry['matrix'] = _node_matrix(matrix) + + def _add_texture(self, payload: tuple[bytes, str], name: str) -> int: + """ + Embed one image and give it a texture. + + Parameters + ---------- + payload : tuple[bytes, str] + The encoded image and its MIME type. + name : str + Name for the image. + + Returns + ------- + int + Index into the document's textures. + """ + self.images.append({ + 'bufferView': self._add_view(payload[0]), + 'mimeType': payload[1], + 'name': name + }) + self.textures.append({'sampler': 0, 'source': len(self.images) - 1}) + return len(self.textures) - 1 + + def masked_texture(self, colour: str, mask: str) -> tuple[int, str] | None: + """ + Compose a colour image with its alpha mask, reusing the result across materials. + + Parameters + ---------- + colour : str + Path of the colour image. + mask : str + Path of the mask image. + + Returns + ------- + tuple[int, str] | None + The texture index and the glTF alpha mode, or :py:obj:`None` when either image cannot + be decoded. + """ + if (found := self._masked.get((colour, mask))) is not None: + return found + composed = _compose_alpha(self._raw.get(colour, b''), self._raw.get(mask, b'')) + if composed is None: + log.warning('Could not mask `%s` with `%s`.', colour, mask) + return None + payload, mode = composed + index = self._add_texture(payload, f'{colour} + {mask}') + self._masked[colour, mask] = (index, mode) + return index, mode + + def _add_view(self, payload: bytes, target: int | None = None) -> int: + self._blob.extend(b'\x00' * (-len(self._blob) % 4)) + view: dict[str, Any] = { + 'buffer': 0, + 'byteLength': len(payload), + 'byteOffset': len(self._blob) + } + if target is not None: + view['target'] = target + self._blob.extend(payload) + self.views.append(view) + return len(self.views) - 1 + + def accessor(self, payload: bytes, target: int | None, **extra: Any) -> int: + """ + Append attribute or index data and describe it with an accessor. + + Parameters + ---------- + payload : bytes + The raw data. + target : int | None + glTF buffer target, or :py:obj:`None` for data a vertex puller never reads, such as an + animation's keyframes. + extra : Any + Remaining accessor fields. + + Returns + ------- + int + Index into the document's accessors. + """ + self.accessors.append({'bufferView': self._add_view(payload, target), **extra}) + return len(self.accessors) - 1 + + def model_material(self, name: str, model: Model, images: Mapping[str, TextureImage]) -> int: + """ + Add a material belonging to a model rather than to the level. + + A model brings its own library, naming an image file that the caller was asked to read in + beside it. Materials are keyed by that file name so two NPCs sharing a texture share it here + too. + + Parameters + ---------- + name : str + The material's name in the model's library. + model : Model + The model the material belongs to. + images : collections.abc.Mapping[str, TextureImage] + The model's images, keyed by lowercased file name. + + Returns + ------- + int + Index into the document's materials. + """ + file = model.materials.get(name, '') + if (found := self._by_model.get(file.lower())) is not None: + return found + pbr: dict[str, Any] = {'metallicFactor': 0.0, 'roughnessFactor': 0.9} + texture = images.get(file.lower()) + payload = _image_payload(texture) if texture else None + if payload is None: + pbr['baseColorFactor'] = list(_FALLBACK_COLOUR) + else: + pbr['baseColorTexture'] = {'index': self._add_texture(payload, file), 'texCoord': 0} + self.materials.append({ + 'doubleSided': False, + 'name': f'{name} ({file})' if file else name, + 'pbrMetallicRoughness': pbr + }) + self._by_model[file.lower()] = len(self.materials) - 1 + return len(self.materials) - 1 + + def material(self, key: tuple[int, int]) -> int: + """ + Return the glTF material for a level material and lightmap pair, creating it once. + + A face names both the material it draws with and which of the level's baked lighting + atlases lights it, so the two together decide the material a primitive needs. The atlas + goes in the occlusion slot on the second coordinate set: glTF has no slot that multiplies a + baked lightmap into the base colour, and occlusion is the one a plain viewer darkens with + rather than ignoring. A viewer after the game's own look should multiply that texture's + colour into the base rather than treating it as ambient occlusion. + + Parameters + ---------- + key : tuple[int, int] + The material identifier a face references and the atlas that lights it. + + Returns + ------- + int + Index into the document's materials. + """ + if key in self._by_material: + return self._by_material[key] + material_id, lightmap = key + material = self._level.materials.get(material_id) + pbr: dict[str, Any] = {'metallicFactor': 0.0, 'roughnessFactor': 0.9} + if material and material.category.lower() == _SKY: + return self._sky() + mode = '' + index = None + if material and material.alpha: + if (masked := self.masked_texture(material.image, material.alpha)) is not None: + index, mode = masked + elif material: + index = self._by_path.get(material.image) + if index is None: + pbr['baseColorFactor'] = list(_FALLBACK_COLOUR) + else: + pbr['baseColorTexture'] = {'index': index, 'texCoord': 0} + name = material.texture if material else f'material_{material_id}' + entry: dict[str, Any] = { + # A cut-out is a flat card meant to be seen from either side, so it cannot be culled. + 'doubleSided': bool(mode), + 'name': name, + 'pbrMetallicRoughness': pbr + } + if (atlas := self._lightmap(lightmap)) is not None: + entry['occlusionTexture'] = {'index': atlas, 'texCoord': 1} + entry['name'] = f'{name} + lightmap {lightmap}' + if mode: + entry['alphaMode'] = mode + if mode == _MASK_MODE: + entry['alphaCutoff'] = _ALPHA_CUTOFF + self.materials.append(entry) + self._by_material[key] = len(self.materials) - 1 + return len(self.materials) - 1 + + def _sky(self) -> int: + """ + Give every skybox face one flat material, made once and shared. + + Returns + ------- + int + Index into the document's materials. + """ + if self._sky_material is None: + self.materials.append({ + # Sky is not a surface and must not shade, so it is written unlit. A viewer that + # does not know the extension still gets the same colour, just lit. + 'extensions': { + _UNLIT: {} + }, + 'name': _SKY, + 'pbrMetallicRoughness': { + 'baseColorFactor': list(_SKY_COLOUR), + 'metallicFactor': 0.0, + 'roughnessFactor': 1.0 + } + }) + self._sky_material = len(self.materials) - 1 + return self._sky_material + + def _lightmap(self, index: int) -> int | None: + """ + Embed one baked lighting atlas, reusing it across every material that names it. + + Parameters + ---------- + index : int + Index into the level's atlases. + + Returns + ------- + int | None + The texture index, or :py:obj:`None` when the level has no such atlas or it will not + decode. + """ + if not 0 <= index < len(self._level.lightmaps): + return None + if index in self._by_lightmap: + return self._by_lightmap[index] + payload = _image_payload(self._level.lightmaps[index]) + found = None if payload is None else self._add_texture(payload, f'lightmap_{index}') + self._by_lightmap[index] = found + return found + + def finish(self, name: str) -> bytes: + """ + Serialise the document. + + Parameters + ---------- + name : str + Name for the scene. + + Returns + ------- + bytes + A complete binary glTF. + """ + binary = _pad(bytes(self._blob), b'\x00') + document: dict[str, Any] = { + 'accessors': self.accessors, + 'asset': { + 'generator': 'dade maxpane', + 'version': '2.0' + }, + 'bufferViews': self.views, + 'buffers': [{ + 'byteLength': len(binary) + }], + 'materials': self.materials, + 'meshes': self.meshes, + 'nodes': self.nodes, + 'scene': 0, + 'scenes': [{ + 'name': name, + 'nodes': list(range(len(self.nodes))) + }] + } + if self.images: + document['images'] = self.images + document['samplers'] = [{'wrapS': _REPEAT, 'wrapT': _REPEAT}] + document['textures'] = self.textures + if self.animations: + document['animations'] = self.animations + if self._sky_material is not None: + document['extensionsUsed'] = [_UNLIT] + chunk = _pad(json.dumps(document, separators=(',', ':')).encode(), b' ') + length = 12 + 8 + len(chunk) + 8 + len(binary) + return b''.join((GLB_MAGIC, struct.pack(' int: + """ + Add one mesh and the node that places it. + + Parameters + ---------- + document : _Document + The document being built. + positions : collections.abc.Sequence[Vector3] + Vertex positions. + normals : collections.abc.Sequence[Vector3] + One normal per position. + coords : collections.abc.Sequence[tuple[float, float]] + One texture coordinate per position, empty when the mesh is untextured. + groups : dict[Any, list[tuple[int, int, int]]] + Triangles grouped by whatever decides their material. + name : str + Name for the node and mesh. + matrix : collections.abc.Sequence[float] | None + The mesh's four-by-three transform, or :py:obj:`None` to leave the node at the origin. + material : collections.abc.Callable[[Any], int] | None + Turns a group's key into a document material index. Defaults to the level's own materials. + lightmap_coords : collections.abc.Sequence[tuple[float, float]] + One lightmap coordinate per position, empty when the mesh has none. + animate : bool + Write the node's placement as translation, rotation and scale rather than as one matrix. A + node carrying a matrix cannot be animated. + + Returns + ------- + int + Index of the node that was added. + """ + material = material or document.material + attributes = { + 'NORMAL': + document.accessor(b''.join(struct.pack('<3f', *v) for v in normals), + _ARRAY_BUFFER, + componentType=_FLOAT, + count=len(normals), + type='VEC3'), + 'POSITION': + document.accessor(b''.join(struct.pack('<3f', *v) for v in positions), + _ARRAY_BUFFER, + componentType=_FLOAT, + count=len(positions), + max=[max(v[i] for v in positions) for i in range(3)], + min=[min(v[i] for v in positions) for i in range(3)], + type='VEC3') + } + for slot, values in (('TEXCOORD_0', coords), ('TEXCOORD_1', lightmap_coords)): + if not values: + continue + attributes[slot] = document.accessor(b''.join(struct.pack('<2f', *v) for v in values), + _ARRAY_BUFFER, + componentType=_FLOAT, + count=len(values), + type='VEC2') + primitives = [] + for material_id, triangles in sorted(groups.items()): + flat = [index for triangle in triangles for index in triangle] + primitives.append({ + 'attributes': + attributes, + 'indices': + document.accessor(struct.pack(f'<{len(flat)}I', *flat), + _ELEMENT_ARRAY_BUFFER, + componentType=_UNSIGNED_INT, + count=len(flat), + max=[max(flat)], + min=[min(flat)], + type='SCALAR'), + 'material': + material(material_id), + 'mode': + _TRIANGLES + }) + document.meshes.append({'name': name, 'primitives': primitives}) + node: dict[str, Any] = {'mesh': len(document.meshes) - 1, 'name': name} + if matrix is not None: + if animate: + translation, rotation, scale = _decompose(matrix) + node.update(rotation=rotation, scale=scale, translation=translation) + else: + node['matrix'] = _node_matrix(matrix) + document.nodes.append(node) + return len(document.nodes) - 1 + + +def _add_static_mesh( + document: _Document, + level: Level, + mesh: StaticMesh, + corners: Sequence[Corner], + name: str, + *, + placed: bool = False, + clips: Sequence[PropAnimation] = (), + lifts: Mapping[int, int] = MappingProxyType({}) +) -> None: + """ + Add one mesh, expanding its shared corners into vertices. + + Faces the engine does not draw are left out, so collision volumes and the skybox's placeholder + image do not paper over the level. + + Vertices arrive in their room's space and :py:attr:`StaticMesh.transform` is the transform the + reader worked out for that room from the level's exit graph, so it always has to be applied. + + Parameters + ---------- + document : _Document + The document being built. + level : Level + The level the mesh belongs to, consulted for which materials draw. + mesh : StaticMesh + The mesh to add. + corners : collections.abc.Sequence[Corner] + The container's shared corner array. + name : str + Name for the node and mesh. + placed : bool + Apply :py:attr:`StaticMesh.transform`. + clips : collections.abc.Sequence[PropAnimation] + Animations the mesh can play. A mesh with any is written so it can be animated. + lifts : collections.abc.Mapping[int, int] + How far off its plane each face has to sit, in steps of + :py:data:`dade.maxpane.decals.DECAL_STEP`, keyed by the face's index. Faces that stay put + may be left out. + """ + positions: list[Vector3] = [] + normals: list[Vector3] = [] + coords: list[tuple[float, float]] = [] + baked: list[tuple[float, float]] = [] + groups: dict[tuple[int, int], list[tuple[int, int, int]]] = {} + for at, face in enumerate(mesh.faces): + if not _draws(level, face.material): + continue + base = len(positions) + end = face.first_corner + face.corner_count + if end > len(corners): + return + # Anything laid over another surface rides a little way up its own normal, so that a + # viewer drawing both at once does not have to guess which is in front. + step = lifts.get(at, 0) * DECAL_STEP + lift = _mirror(face.normal) + for corner in corners[face.first_corner:end]: + if corner.position >= len(mesh.positions): + return + x, y, z = _mirror(mesh.positions[corner.position]) + positions.append((x + lift[0] * step, y + lift[1] * step, z + lift[2] * step)) + normals.append(_mirror(mesh.normals[corner.position])) + coords.append(corner.uv) + baked.append(corner.lightmap_uv) + fan = [(base, base + i, base + i + 1) for i in range(1, face.corner_count - 1)] + # A face names both its material and the atlas that lights it, and a primitive can carry + # only one of each, so the two together group the triangles. + groups.setdefault((face.material, face.lightmap), []).extend( + _wind(fan, positions, _mirror(face.normal))) + if not groups: + return + moving = [clip for clip in clips if clip.start != clip.end] + node = _add_mesh(document, + positions, + normals, + coords, + groups, + name, + mesh.transform if placed else None, + animate=bool(moving), + lightmap_coords=baked) + written = sum(document.add_animation(node, name, clip) for clip in moving) + if moving and not written and placed: + # Nothing drove the node after all, so it may as well carry a matrix like the rest. + document.settle(node, mesh.transform) + + +def _add_bsp(document: _Document, polygons: Sequence[Polygon], vertices: Sequence[Vector3], + name: str) -> None: + """ + Add the BSP faces as one untextured node. + + Parameters + ---------- + document : _Document + The document being built. + polygons : collections.abc.Sequence[Polygon] + The level's faces. + vertices : collections.abc.Sequence[Vector3] + The level's vertex pool. + name : str + Name for the node and mesh. + """ + mirrored = [_mirror(vertex) for vertex in vertices] + normals: list[Vector3] = [(0.0, 1.0, 0.0)] * len(mirrored) + triangles: list[tuple[int, int, int]] = [] + for polygon in polygons: + first = polygon.first_vertex + normal = _mirror(polygon.normal) + for index in range(first, first + polygon.vertex_count): + normals[index] = normal + fan = [(first, first + i, first + i + 1) for i in range(1, polygon.vertex_count - 1)] + triangles.extend(_wind(fan, mirrored, normal)) + _add_mesh(document, mirrored, normals, (), {(-1, -1): triangles}, name, None) + + +def _add_model(document: _Document, model: Model, placement: Placement, label: str) -> None: + """ + Add one NPC or pickup as real geometry. + + The model arrives in its own space with its own material library, so its images are embedded + beside the level's and the placement's transform puts it where the level asked for it. + + Parameters + ---------- + document : _Document + The document being built. + model : Model + The model to add, with its images already read in. + placement : Placement + Where the object stands. + label : str + Prefix naming what the node holds, such as ``character:transit_cop``. + """ + images = {texture.path.lower(): texture for texture in model.textures} + for index, mesh in enumerate(model.meshes): + positions: list[Vector3] = [] + normals: list[Vector3] = [] + coords: list[tuple[float, float]] = [] + groups: dict[int, list[tuple[int, int, int]]] = {} + for face in mesh.faces: + base = len(positions) + for corner in range(3): + point = mesh.positions[face.positions[corner]] + positions.append(_mirror(point)) + normals.append( + _mirror(mesh.normals[face.positions[corner]]) if mesh.normals else (0.0, 1.0, + 0.0)) + coords.append(mesh.coords[face.coords[corner]] if mesh.coords else (0.0, 0.0)) + groups.setdefault(face.material, []).append((base, base + 2, base + 1)) + if not groups: + continue + name = f'{label} {placement.name}'.strip() + _add_mesh( + document, + positions, + normals, + coords, + groups, + name if len(model.meshes) == 1 else f'{name} #{index}', + placement.transform, + material=lambda key, mesh=mesh: document.model_material( # type: ignore[misc] + mesh.materials[key] if key < len(mesh.materials) else '', model, images)) + + +def _add_placement(document: _Document, placement: Placement, label: str) -> None: + """ + Add one NPC or pickup as an empty node. + + The models live outside the level, under ``data/database/skins`` and + ``data/database/level_items``, so what the level itself supplies is where each one stands and + which model belongs there. Carrying that as a named, positioned node keeps the placement usable + without the model. + + Parameters + ---------- + document : _Document + The document being built. + placement : Placement + Where the object stands. + label : str + Prefix naming what the node holds, such as ``character:transit_cop``. + """ + document.nodes.append({ + 'matrix': _node_matrix(placement.transform), + 'name': f'{label} {placement.name}'.strip() + }) + + +def _lift_decals(level: Level, containers: Sequence[RenderMesh]) -> list[list[dict[int, int]]]: + """ + Work out which faces have to come off their plane, across the whole level at once. + + Whether two surfaces fight depends on where they end up in the scene, not on which mesh they + were stored in, so every drawn face is gathered in scene space first and layered together. See + :py:func:`dade.maxpane.decals.layer_faces` for what the layering means. + + Parameters + ---------- + level : Level + The level being written, consulted for which materials draw. + containers : collections.abc.Sequence[RenderMesh] + The containers whose meshes will be written, in the order they will be written in. + + Returns + ------- + list[list[dict[int, int]]] + One entry per container, one per mesh within it, mapping a face's index to how many steps + of :py:data:`dade.maxpane.decals.DECAL_STEP` it has to rise. Faces that stay put are left + out. + """ + surfaces: list[tuple[Vector3, list[Vector3]]] = [] + origins: list[tuple[int, int, int]] = [] + for group, container in enumerate(containers): + for index, mesh in enumerate(container.meshes): + matrix = _node_matrix(mesh.transform) + for at, face in enumerate(mesh.faces): + if not _draws(level, face.material): + continue + end = face.first_corner + face.corner_count + if end > len(container.corners) or any( + corner.position >= len(mesh.positions) + for corner in container.corners[face.first_corner:end]): + continue + surfaces.append((_turn(matrix, _mirror(face.normal)), [ + _place(matrix, _mirror(mesh.positions[corner.position])) + for corner in container.corners[face.first_corner:end] + ])) + origins.append((group, index, at)) + out: list[list[dict[int, int]]] = [[{} for _ in c.meshes] for c in containers] + for layer, (group, index, at) in zip(layer_faces(surfaces), origins, strict=True): + if layer: + out[group][index][at] = layer + return out + + +def build_glb(level: Level, + *, + name: str = 'level', + models: Mapping[str, Model] | None = None) -> bytes: + """ + Build a ``.glb`` from a level. + + Parameters + ---------- + level : Level + A level from :py:func:`dade.maxpane.ldb.read_level`. + name : str + Name given to the scene and its nodes. + models : collections.abc.Mapping[str, Model] | None + Models to draw the NPCs and pickups with, keyed by the node label the placement gets, such + as ``character:transit_cop``. A placement with no model is written as an empty node. + + Returns + ------- + bytes + A complete binary glTF. + + Raises + ------ + ValueError + If the level has nothing to draw. + """ + document = _Document(level) + containers = [(c, p) for c, p in ((level.mesh, name), (level.props, f'{name}_prop')) if c] + lifts = _lift_decals(level, [container for container, _prefix in containers]) + for group, (container, prefix) in enumerate(containers): + for index, mesh in enumerate(container.meshes): + label = container.names[index] if index < len(container.names) else f'{prefix}_{index}' + clips = container.animations[index] if index < len(container.animations) else () + _add_static_mesh(document, + level, + mesh, + container.corners, + label, + clips=clips, + lifts=lifts[group][index], + placed=True) + found = models or {} + for placement, label in ([(c.placement, f'character:{c.skin}') + for c in level.characters] + [(i.placement, f'item:{i.item}') + for i in level.items]): + model = found.get(label) + if model is not None and any(m.faces for m in model.meshes): + _add_model(document, model, placement, label) + else: + _add_placement(document, placement, label) + if not any('mesh' in node for node in document.nodes): + if not level.geometry.polygons: + msg = 'The level has no faces.' + raise ValueError(msg) + _add_bsp(document, level.geometry.polygons, level.geometry.vertices, name) + return document.finish(name) diff --git a/dade/maxpane/ldb.py b/dade/maxpane/ldb.py new file mode 100644 index 0000000..b109986 --- /dev/null +++ b/dade/maxpane/ldb.py @@ -0,0 +1,1547 @@ +""" +Reader for the world geometry at the head of a ``.ldb`` level. + +``X_LevelDBExportLevel::vf03`` in ``MaxED.exe`` writes four arrays before anything else, and the +first two are the whole of the level's static geometry: a pool of vertices, then a table of convex +faces. Each face records where its corners start in the pool and how many there are, and the runs +are contiguous and exhaustive, so the pool is a flat de-indexed corner list rather than something +an index buffer addresses. + +That is checkable, and :py:func:`read_geometry` checks it: the vertex counts must sum to exactly +the pool size. On ``Part1_Level6.ldb`` that is 15333 faces over 53263 vertices, with every +consecutive pair contiguous. + +Y is up. On a shipped level the Y axis carries far fewer distinct values than X or Z and they +cluster hard, which is what floors and ceilings at fixed heights look like. glTF is also Y-up, so +no axis conversion is needed. + +The two arrays that follow are the level BSP, and after them come the containers; neither is needed +for geometry and neither is read here. +""" +from __future__ import annotations + +from typing import TYPE_CHECKING, NamedTuple +import logging +import struct + +from .memoryfile import TAG_SIZES, BasicType, read_int, read_string, read_vector3 +from .typing import ( + Character, + Corner, + Level, + LevelGeometry, + LevelItem, + Material, + MeshFace, + Placement, + Polygon, + PropAnimation, + RenderMesh, + StaticMesh, + TextureImage, +) + +if TYPE_CHECKING: + from collections.abc import Callable, Sequence + + from .typing import Vector3 + +__all__ = ('MAX_POLYGON_VERTICES', 'MIN_POLYGON_VERTICES', 'InvalidLevelError', 'read_geometry', + 'read_textures') + +log = logging.getLogger(__name__) + +MIN_POLYGON_VERTICES = 3 +"""Fewest corners a face may have. + +:meta hide-value: +""" +MAX_POLYGON_VERTICES = 64 +"""Most corners a face may have. Shipped levels stay at or below eight; the ceiling only guards +against a desynchronised read. + +:meta hide-value: +""" + + +class InvalidLevelError(ValueError): + """Raised when a buffer is not a readable level.""" + + +def _read_array_count(data: bytes, offset: int, label: str) -> tuple[int, int]: + if offset >= len(data) or data[offset] != BasicType.ARRAY: + got = f'0x{data[offset]:02x}' if offset < len(data) else 'end of file' + msg = f'Expected an array marker for {label} at offset {offset}, got {got}.' + raise InvalidLevelError(msg) + return read_int(data, offset + 1) + + +def read_geometry(data: bytes) -> LevelGeometry: + """ + Read a level's vertex pool and face table. + + Parameters + ---------- + data : bytes + A decompressed ``.ldb``. + + Returns + ------- + LevelGeometry + The vertices and faces. + + Raises + ------ + InvalidLevelError + If the leading arrays are missing or malformed, or the faces do not account for the vertex + pool exactly. + """ + count, offset = _read_array_count(data, 0, 'the vertex pool') + vertices: list[Vector3] = [] + for _ in range(count): + vertex, offset = read_vector3(data, offset) + vertices.append(vertex) + count, offset = _read_array_count(data, offset, 'the face table') + polygons: list[Polygon] = [] + total = 0 + for _ in range(count): + first, offset = read_int(data, offset) + corners, offset = read_int(data, offset) + polygon_index, offset = read_int(data, offset) + mesh_index, offset = read_int(data, offset) + normal, offset = read_vector3(data, offset) + origin, offset = read_vector3(data, offset) + if not MIN_POLYGON_VERTICES <= corners <= MAX_POLYGON_VERTICES: + msg = f'Face {len(polygons)} claims {corners} corners.' + raise InvalidLevelError(msg) + if first < 0 or first + corners > len(vertices): + msg = (f'Face {len(polygons)} spans vertices {first}..{first + corners}, ' + f'outside a pool of {len(vertices)}.') + raise InvalidLevelError(msg) + total += corners + polygons.append( + Polygon(first_vertex=first, + mesh_index=mesh_index, + normal=normal, + origin=origin, + polygon_index=polygon_index, + vertex_count=corners)) + if total != len(vertices): + msg = (f'Faces account for {total} vertices but the pool holds {len(vertices)}; ' + 'the read is out of step.') + raise InvalidLevelError(msg) + log.debug('Read %d faces over %d vertices.', len(polygons), len(vertices)) + return LevelGeometry(polygons=tuple(polygons), vertices=tuple(vertices)) + + +def _skip_preamble(data: bytes) -> int: + """ + Step over the four leading arrays and the format version. + + Parameters + ---------- + data : bytes + A decompressed level. + + Returns + ------- + int + The offset of the first container. + """ + count, offset = _read_array_count(data, 0, 'the vertex pool') + for _ in range(count): + _, offset = read_vector3(data, offset) + count, offset = _read_array_count(data, offset, 'the face table') + for _ in range(count): + for _ in range(4): + _, offset = read_int(data, offset) + _, offset = read_vector3(data, offset) + _, offset = read_vector3(data, offset) + count, offset = _read_array_count(data, offset, 'the BSP') + for _ in range(count): + _, offset = read_vector3(data, offset) + _, offset = read_vector3(data, offset) + for _ in range(6): + _, offset = read_int(data, offset) + count, offset = _read_array_count(data, offset, 'the BSP face list') + for _ in range(count): + _, offset = read_int(data, offset) + _version, offset = read_int(data, offset) + return offset + + +def read_textures(data: bytes) -> tuple[TextureImage, ...]: + """ + Read the images a level embeds. + + Each texture is stored as a complete image file, byte for byte as the artist saved it, under + the absolute path it was authored at. ``X_LevelDBTextureImage``'s writer emits a format code, + a byte count, and then the file, so nothing needs decoding here. + + Parameters + ---------- + data : bytes + A decompressed ``.ldb``. + + Returns + ------- + tuple[TextureImage, ...] + The textures, in stored order. + + Raises + ------ + InvalidLevelError + If the level's leading arrays are malformed, or a texture runs past the end of the file. + """ + offset = _skip_preamble(data) + count, offset = read_int(data, offset) + textures: list[TextureImage] = [] + for _ in range(count): + path, offset = read_string(data, offset) + kind, offset = read_int(data, offset) + length, offset = read_int(data, offset) + if offset + length > len(data): + msg = f'Texture {path!r} claims {length} bytes but the file ends.' + raise InvalidLevelError(msg) + textures.append(TextureImage(data=data[offset:offset + length], kind=kind, path=path)) + offset += length + log.debug('Read %d textures.', len(textures)) + return tuple(textures) + + +def _read_material_map(data: bytes, offset: int) -> tuple[dict[int, Material], int]: + """ + Read the identifier-to-material map. + + Parameters + ---------- + data : bytes + A decompressed level. + offset : int + Offset of the map marker. + + Returns + ------- + tuple[dict[int, Material], int] + The materials and the offset just past the map. + + Raises + ------ + InvalidLevelError + If the map or one of its entries is not where it should be. + """ + if data[offset] != BasicType.MAP: + msg = f'Expected a map marker at offset {offset}, got 0x{data[offset]:02x}.' + raise InvalidLevelError(msg) + count, offset = read_int(data, offset + 1) + materials: dict[int, Material] = {} + for _ in range(count): + key, offset = read_int(data, offset) + if data[offset] != BasicType.PAIR: + msg = f'Expected a pair marker at offset {offset}, got 0x{data[offset]:02x}.' + raise InvalidLevelError(msg) + category, offset = read_string(data, offset + 1) + texture, offset = read_string(data, offset) + materials[key] = Material(alpha='', category=category, image='', texture=texture) + return materials, offset + + +def _resolve_images(materials: dict[int, Material], images: dict[str, tuple[str, str]], + textures: Sequence[TextureImage]) -> dict[int, Material]: + """ + Fill in each material's colour and alpha image paths from the category table. + + The category entry's second path is the colour and its third is the alpha mask. Across the 29 + shipped levels every entry's colour path is embedded, and 592 of the 4060 entries name a second, + different image for the mask: ``cardboardalpha`` draws with ``cardboard10a.JPG`` and takes its + alpha from ``boxalpha01c.JPG``. The third path is never a fallback for the second. + + Parameters + ---------- + materials : dict[int, Material] + Materials as read from the map, with no images yet. + images : dict[str, tuple[str, str]] + Lowercased material name to its colour path and alpha path. + textures : collections.abc.Sequence[TextureImage] + The level's embedded images. + + Returns + ------- + dict[int, Material] + The same materials, each carrying the paths of the images it draws with. + """ + embedded = {texture.path.lower(): texture.path for texture in textures} + resolved: dict[int, Material] = {} + for key, material in materials.items(): + colour, mask = images.get(material.texture.lower(), ('', '')) + image = embedded.get(colour.lower(), '') + alpha = embedded.get(mask.lower(), '') + resolved[key] = material._replace(alpha='' if alpha == image else alpha, image=image) + missing = sum(1 for material in resolved.values() if not material.image) + if missing: + log.debug('%d of %d materials name no embedded image.', missing, len(resolved)) + return resolved + + +_MATRIX_FLOATS = 12 +_MATRIX_SIZE = 1 + 4 * _MATRIX_FLOATS +_MIN_CORNERS = 100 +_MAX_CORNERS = 4_000_000 +_MAX_MESHES = 100_000 +_MAX_PLACEMENTS = 100_000 +_PROBE_CORNERS = 64 +"""Corners read before a candidate container is parsed in full. + +The array marker's byte value occurs constantly inside float payloads, so a level yields tens of +thousands of candidates. Four corners let hundreds through, and parsing each of those in full costs +seconds; sixty-four rejects effectively all of them for a fixed, tiny cost. + +:meta hide-value: +""" + + +def _expect_tag(data: bytes, offset: int, tag: int) -> int: + if offset >= len(data) or data[offset] != tag: + got = f'0x{data[offset]:02x}' if offset < len(data) else 'end of file' + msg = f'Expected tag 0x{tag:02x} at offset {offset}, got {got}.' + raise InvalidLevelError(msg) + return offset + 1 + + +def _read_vector2(data: bytes, offset: int) -> tuple[tuple[float, float], int]: + offset = _expect_tag(data, offset, BasicType.VECTOR2) + return struct.unpack_from('<2f', data, offset), offset + 8 + + +def _read_float(data: bytes, offset: int) -> tuple[float, int]: + offset = _expect_tag(data, offset, BasicType.FLOAT) + return struct.unpack_from(' tuple[list[Corner], int]: + """ + Read the corner array a static mesh container shares between its meshes. + + Each corner is written by ``FUN_005ef230``: a position index, a texture coordinate, a lightmap + coordinate, a packed colour and a flag. + + Parameters + ---------- + data : bytes + A decompressed level. + offset : int + Offset of the first corner. + count : int + Number of corners to read. + + Returns + ------- + tuple[list[Corner], int] + The corners and the offset just past them. + """ + out: list[Corner] = [] + for _ in range(count): + position, offset = read_int(data, offset) + uv, offset = _read_vector2(data, offset) + lightmap_uv, offset = _read_vector2(data, offset) + colour, offset = read_int(data, offset) + offset = _expect_tag(data, offset, BasicType.BOOL) + 1 + out.append(Corner(colour=colour, lightmap_uv=lightmap_uv, position=position, uv=uv)) + return out, offset + + +def _read_faces(data: bytes, offset: int) -> tuple[list[MeshFace], int]: + """ + Read a mesh polygon container. + + ``X_LevelDBExportMeshPolygonContainer::vf01`` writes a count, then a key and a polygon per + entry, and closes with a map of integers to vectors that carries no pair markers. + + Parameters + ---------- + data : bytes + A decompressed level. + offset : int + Offset of the polygon count. + + Returns + ------- + tuple[list[MeshFace], int] + The faces and the offset just past the container. + """ + count, offset = read_int(data, offset) + out: list[MeshFace] = [] + for _ in range(count): + _key, offset = read_int(data, offset) + first, offset = read_int(data, offset) + corners, offset = read_int(data, offset) + normal, offset = read_vector3(data, offset) + flags, offset = read_int(data, offset) + material, offset = read_int(data, offset) + lightmap, offset = read_int(data, offset) + _u, offset = _read_float(data, offset) + _v, offset = _read_float(data, offset) + _spare, offset = read_int(data, offset) + out.append( + MeshFace(corner_count=corners, + first_corner=first, + flags=flags, + lightmap=lightmap, + material=material, + normal=normal)) + offset = _expect_tag(data, offset, BasicType.MAP) + extra, offset = read_int(data, offset) + for _ in range(extra): + _, offset = read_int(data, offset) + _, offset = read_vector3(data, offset) + return out, offset + + +def _read_static_mesh(data: bytes, offset: int) -> tuple[StaticMesh, int]: + """ + Read one placed mesh. + + ``X_LevelDBExportStaticMesh::vf01`` writes a position array, an array-marked normal array, a + four-by-three transform, and then the mesh's polygon container. + + Parameters + ---------- + data : bytes + A decompressed level. + offset : int + Offset of the position count. + + Returns + ------- + tuple[StaticMesh, int] + The mesh and the offset just past it. + """ + count, offset = read_int(data, offset) + positions: list[Vector3] = [] + for _ in range(count): + vertex, offset = read_vector3(data, offset) + positions.append(vertex) + offset = _expect_tag(data, offset, BasicType.ARRAY) + count, offset = read_int(data, offset) + normals: list[Vector3] = [] + for _ in range(count): + vertex, offset = read_vector3(data, offset) + normals.append(vertex) + offset = _expect_tag(data, offset, BasicType.MATRIX4X3) + transform = struct.unpack_from(f'<{_MATRIX_FLOATS}f', data, offset) + offset += 4 * _MATRIX_FLOATS + faces, offset = _read_faces(data, offset) + return StaticMesh(faces=tuple(faces), + normals=tuple(normals), + positions=tuple(positions), + transform=transform), offset + + +def _read_corner_count(data: bytes, offset: int, *, minimum: int = _MIN_CORNERS) -> tuple[int, int]: + """ + Read a mesh container's corner count and reject an implausible one. + + The scan for the static mesh container tries every array marker in the file, so this doubles as + the cheap first filter there: the marker's byte value occurs constantly inside float payloads + and almost every hit fails here. The prop container is found by walking rather than scanning, + so it passes a *minimum* of zero and only needs the upper bound. + + Parameters + ---------- + data : bytes + A decompressed level. + offset : int + Offset of the container's array marker. + minimum : int + Counts at or below this are rejected. + + Returns + ------- + tuple[int, int] + The count and the offset just past it. + + Raises + ------ + InvalidLevelError + If the marker is missing or the count is outside the plausible range. + """ + offset = _expect_tag(data, offset, BasicType.ARRAY) + count, offset = read_int(data, offset) + if not minimum < count < _MAX_CORNERS: + msg = f'Implausible corner count {count} at offset {offset}.' + raise InvalidLevelError(msg) + return count, offset + + +def _read_mesh_container(data: bytes, offset: int) -> tuple[RenderMesh, int]: + """ + Read a static mesh container. + + Parameters + ---------- + data : bytes + A decompressed level. + offset : int + Offset of the container's array marker. + + Returns + ------- + tuple[RenderMesh, int] + The mesh and the offset just past the container. + + Raises + ------ + InvalidLevelError + If the structure does not hold at *offset*. + """ + count, offset = _read_corner_count(data, offset) + corners, offset = _read_corners(data, offset, count) + count, offset = read_int(data, offset) + if not 0 <= count < _MAX_MESHES: + msg = f'Implausible mesh count {count} at offset {offset}.' + raise InvalidLevelError(msg) + meshes: list[StaticMesh] = [] + keys: list[int] = [] + for _ in range(count): + key, offset = read_int(data, offset) + mesh, offset = _read_static_mesh(data, offset) + meshes.append(mesh) + keys.append(key) + return RenderMesh(corners=tuple(corners), keys=tuple(keys), meshes=tuple(meshes)), offset + + +def _find_mesh_container(data: bytes, offset: int) -> tuple[RenderMesh, int] | None: + """ + Locate the static mesh container. + + The exit container sits between the lightmaps and the meshes and is not read here, so the + meshes' position is not derivable; instead every array marker is tried and the first whose + whole structure parses is taken. A level holds one such container, so the search stops at the + first hit rather than looking for a better one. + + Parameters + ---------- + data : bytes + A decompressed level. + offset : int + Offset to start searching from. + + Returns + ------- + tuple[RenderMesh, int] | None + The mesh and the offset just past the container, or :py:obj:`None` when no candidate + parses. + """ + marker = bytes((BasicType.ARRAY,)) + position = data.find(marker, offset) + while position >= 0: + found = _try_mesh_container(data, position) + if found is not None and found[0].meshes: + return found + position = data.find(marker, position + 1) + return None + + +def _try_mesh_container(data: bytes, position: int) -> tuple[RenderMesh, int] | None: + """ + Attempt to read a static mesh container at one candidate offset. + + Parameters + ---------- + data : bytes + A decompressed level. + position : int + Offset of a candidate array marker. + + Returns + ------- + tuple[RenderMesh, int] | None + The mesh and the offset just past it, or :py:obj:`None` when the structure does not hold + here. + """ + try: + _count, probe = _read_corner_count(data, position) + _read_corners(data, probe, _PROBE_CORNERS) + return _read_mesh_container(data, position) + except (IndexError, InvalidLevelError, struct.error, ValueError): + return None + + +def _skip_material_index(data: bytes, offset: int) -> int: + """ + Step over the map that indexes materials the other way round. + + Parameters + ---------- + data : bytes + A decompressed level. + offset : int + Offset of the map marker. + + Returns + ------- + int + The offset just past the map. + + Raises + ------ + InvalidLevelError + If the map or one of its entries is not where it should be. + """ + if data[offset] != BasicType.MAP: + msg = f'Expected a map marker at offset {offset}, got 0x{data[offset]:02x}.' + raise InvalidLevelError(msg) + count, offset = read_int(data, offset + 1) + for _ in range(count): + if data[offset] != BasicType.PAIR: + msg = f'Expected a pair marker at offset {offset}, got 0x{data[offset]:02x}.' + raise InvalidLevelError(msg) + _, offset = read_string(data, offset + 1) + _, offset = read_string(data, offset) + _, offset = read_int(data, offset) + return offset + + +def _read_material_categories(data: bytes, offset: int) -> tuple[dict[str, tuple[str, str]], int]: + r""" + Read the table naming the image behind each material. + + ``X_LevelDBExportMaterialCategory::read`` writes three strings per entry: the material's name, + the path of the image it draws with, and a fallback path pointing at the artist's source file. + The name is what the material map stores, and it only looks like a filename -- + ``PAINTING22CB.JPG`` is the name of the material that draws with ``...\\Painting22b.jpg``. This + table is the only link between the two, so matching on filename gets a fifth of the level's + faces wrong. + + Parameters + ---------- + data : bytes + A decompressed level. + offset : int + Offset of the category count. + + Returns + ------- + tuple[dict[str, tuple[str, str]], int] + Lowercased material name to its image path and fallback path, and the offset just past the + categories. + """ + count, offset = read_int(data, offset) + images: dict[str, tuple[str, str]] = {} + for _ in range(count): + _category, offset = read_string(data, offset) + materials, offset = read_int(data, offset) + for _ in range(materials): + name, offset = read_string(data, offset) + path, offset = read_string(data, offset) + fallback, offset = read_string(data, offset) + offset += 2 * (1 + TAG_SIZES[BasicType.BOOL]) + images[name.lower()] = (path, fallback) + return images, offset + + +def _read_lightmaps(data: bytes, offset: int) -> tuple[tuple[TextureImage, ...], int]: + """ + Read the baked lighting atlases. + + Each is an uncompressed 24-bit Targa of 256 by 256 pixels -- 196626 bytes, header included -- + written the same way the level's other images are: an identifier, a format code, a byte count, + then the file. :py:attr:`Corner.lightmap_uv` addresses them and already arrives in the nought to + one range they want. + + Parameters + ---------- + data : bytes + A decompressed level. + offset : int + Offset of the entry count. + + Returns + ------- + tuple[tuple[TextureImage, ...], int] + The atlases and the offset just past the container. + """ + count, offset = _read_entry_count(data, offset, 'lightmap') + out: list[TextureImage] = [] + for _ in range(count): + key, offset = read_int(data, offset) + kind, offset = read_int(data, offset) + length, offset = read_int(data, offset) + out.append( + TextureImage(data=data[offset:offset + length], kind=kind, path=f'lightmap_{key}.tga')) + offset += length + return tuple(out), offset + + +_IDENTITY = (1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0) +_ROOM_ARRAYS = 10 +_ROOM_TRAILING = 4 + + +def _compose(a: Sequence[float], b: Sequence[float]) -> tuple[float, ...]: + """ + Compose two four-by-three transforms, applying *a* then *b*. + + Parameters + ---------- + a : collections.abc.Sequence[float] + The transform applied first. + b : collections.abc.Sequence[float] + The transform applied second. + + Returns + ------- + tuple[float, ...] + Twelve floats: three basis rows then a translation. + """ + out = [ + sum(a[row * 3 + k] * b[k * 3 + col] for k in range(3)) for row in range(3) + for col in range(3) + ] + out.extend(sum(a[9 + k] * b[k * 3 + col] for k in range(3)) + b[9 + col] for col in range(3)) + return tuple(out) + + +def _read_any_array(data: bytes, offset: int) -> tuple[list[str | int], int]: + """ + Read an array whose element type is told by the first element's tag. + + The room record mixes arrays of names with arrays of identifiers and gives no type marker. An + empty array consumes nothing either way, so peeking at the first element settles it. + + Parameters + ---------- + data : bytes + A decompressed level. + offset : int + Offset of the array marker. + + Returns + ------- + tuple[list[str | int], int] + The elements and the offset just past the array. + """ + offset = _expect_tag(data, offset, BasicType.ARRAY) + count, offset = read_int(data, offset) + out: list[str | int] = [] + for _ in range(count): + value: str | int + if data[offset] == BasicType.STRING: + value, offset = read_string(data, offset) + else: + value, offset = read_int(data, offset) + out.append(value) + return out, offset + + +def _read_exits(data: bytes, offset: int) -> tuple[dict[str, tuple[tuple[float, ...], str]], int]: + """ + Read the exits, which are what hold a level together. + + ``X_LevelDBExportExit::read`` writes the portal polygon and its normal in the room's own space, + a ``mat4x3`` mapping that space into the room on the other side, two indices, the name of the + matching exit over there, and a remap of the portal's corners. The two sides of one doorway + carry inverse transforms. + + Parameters + ---------- + data : bytes + A decompressed level. + offset : int + Offset of the entry count. + + Returns + ------- + tuple[dict[str, tuple[tuple[float, ...], str]], int] + Exit name to its transform and its partner's name, and the offset just past the container. + """ + count, offset = _read_entry_count(data, offset, 'exit') + exits: dict[str, tuple[tuple[float, ...], str]] = {} + for _ in range(count): + key, offset = read_string(data, offset) + corners, offset = read_int(data, offset) + for _ in range(corners + 1): + offset = _expect_tag(data, offset, BasicType.VECTOR3) + TAG_SIZES[BasicType.VECTOR3] + offset = _expect_tag(data, offset, BasicType.MATRIX4X3) + transform = struct.unpack_from(f'<{_MATRIX_FLOATS}f', data, offset) + offset += 4 * _MATRIX_FLOATS + for _ in range(2): + _, offset = read_int(data, offset) + partner, offset = read_string(data, offset) + offset = _expect_tag(data, offset, BasicType.ARRAY) + groups, offset = read_int(data, offset) + for _ in range(groups): + _, offset = _read_any_array(data, offset) + exits[key] = (transform, partner) + return exits, offset + + +def _read_rooms(data: bytes, offset: int) -> tuple[list[tuple[list[int], str]], int]: + """ + Read the rooms: the identifiers of the meshes each one owns, and its name. + + ``X_LevelDBExportRoom::read`` writes ten arrays -- the room's mesh identifiers, its exits, its + objects, its props, its pickups and so on -- then the room's name and five trailing scalars. + + Parameters + ---------- + data : bytes + A decompressed level. + offset : int + Offset of the entry count. + + Returns + ------- + tuple[list[tuple[list[int], str]], int] + Each room's mesh identifiers and name, and the offset just past the container. + """ + count, offset = _read_entry_count(data, offset, 'room') + rooms: list[tuple[list[int], str]] = [] + for _ in range(count): + _key, offset = read_int(data, offset) + arrays: list[list[str | int]] = [] + for _ in range(_ROOM_ARRAYS): + values, offset = _read_any_array(data, offset) + arrays.append(values) + name, offset = read_string(data, offset) + offset = _expect_tag(data, offset, BasicType.FLOAT) + TAG_SIZES[BasicType.FLOAT] + for _ in range(_ROOM_TRAILING): + _, offset = read_int(data, offset) + rooms.append(([i for i in arrays[0] if isinstance(i, int)], name)) + return rooms, offset + + +def _place_rooms(exits: dict[str, tuple[tuple[float, ...], str]], + rooms: Sequence[tuple[list[int], str]]) -> dict[str, tuple[float, ...]]: + """ + Give every room a world transform by walking the exit graph. + + A level is not stored as one space: each room is modelled about its own origin, so without this + they all pile onto each other. On ``Part1_Level1.ldb`` 545 of the 703 room pairs overlap by more + than half the smaller room's volume before the walk and 15 after it. + + Rooms unreachable from the first are given their own component rather than dropped, so nothing + disappears when a level's graph is not fully connected. + + Parameters + ---------- + exits : dict[str, tuple[tuple[float, ...], str]] + Exit name to its transform and its partner's name. + rooms : collections.abc.Sequence[tuple[list[int], str]] + Each room's mesh identifiers and name. + + Returns + ------- + dict[str, tuple[float, ...]] + Room name to the transform placing it in the world. + """ + graph: dict[str, list[tuple[str, tuple[float, ...]]]] = {} + for key, (transform, partner) in exits.items(): + if partner in exits: + graph.setdefault(key.rsplit('::', 1)[0], []).append((partner.rsplit('::', + 1)[0], transform)) + placed: dict[str, tuple[float, ...]] = {} + for _ids, start in rooms: + if start in placed: + continue + placed[start] = _IDENTITY + queue = [start] + while queue: + here = queue.pop() + for there, transform in graph.get(here, ()): + if there in placed: + continue + placed[there] = _compose(transform, placed[here]) + queue.append(there) + return placed + + +def _read_entry_count(data: bytes, offset: int, label: str) -> tuple[int, int]: + """ + Read a container's entry count and reject an implausible one. + + The containers are walked in sequence, so one misread ripples into every one that follows; the + count is the first place that shows, which is why each container checks it before believing it. + + Parameters + ---------- + data : bytes + A decompressed level. + offset : int + Offset of the count. + label : str + What is being counted, for the message. + + Returns + ------- + tuple[int, int] + The count and the offset just past it. + + Raises + ------ + InvalidLevelError + If the count is negative or beyond anything a level holds. + """ + count, offset = read_int(data, offset) + if not 0 <= count < _MAX_PLACEMENTS: + msg = f'Implausible {label} count {count} at offset {offset}.' + raise InvalidLevelError(msg) + return count, offset + + +def _read_placement(data: bytes, offset: int) -> tuple[Placement, int]: + """ + Read the head every placed object shares. + + ``X_LevelDBLevelObject::read`` writes a name, the transform placing the object, a second + transform, an identifier, and the room the object belongs to. + + Parameters + ---------- + data : bytes + A decompressed level. + offset : int + Offset of the object's name. + + Returns + ------- + tuple[Placement, int] + The placement and the offset just past it. + """ + name, offset = read_string(data, offset) + offset = _expect_tag(data, offset, BasicType.MATRIX4X3) + transform = struct.unpack_from(f'<{_MATRIX_FLOATS}f', data, offset) + offset = _expect_tag(data, offset + 4 * _MATRIX_FLOATS, BasicType.MATRIX4X3) + offset += 4 * _MATRIX_FLOATS + _identifier, offset = read_int(data, offset) + room, offset = read_string(data, offset) + return Placement(name=name, room=room, transform=transform), offset + + +def _read_string_array(data: bytes, offset: int) -> int: + """ + Step over an array of strings. + + Parameters + ---------- + data : bytes + A decompressed level. + offset : int + Offset of the array marker. + + Returns + ------- + int + The offset just past the array. + """ + offset = _expect_tag(data, offset, BasicType.ARRAY) + count, offset = read_int(data, offset) + for _ in range(count): + _, offset = read_string(data, offset) + return offset + + +def _skip_properties(data: bytes, offset: int) -> int: + """ + Step over one property bag: an array of strings, a map of arrays, then another array. + + Parameters + ---------- + data : bytes + A decompressed level. + offset : int + Offset of the first array marker. + + Returns + ------- + int + The offset just past the bag. + """ + offset = _read_string_array(data, offset) + offset = _expect_tag(data, offset, BasicType.MAP) + count, offset = read_int(data, offset) + for _ in range(count): + _, offset = read_string(data, offset) + offset = _read_string_array(data, offset) + return _read_string_array(data, offset) + + +def _skip_static_light(data: bytes, offset: int) -> int: + """ + Step over one static light: the placement, an orientation, then ten floats. + + Parameters + ---------- + data : bytes + A decompressed level. + offset : int + Offset of the light. + + Returns + ------- + int + The offset just past the light. + """ + _placement, offset = _read_placement(data, offset) + offset = _expect_tag(data, offset, BasicType.MATRIX3) + TAG_SIZES[BasicType.MATRIX3] + for _ in range(10): + offset = _expect_tag(data, offset, BasicType.FLOAT) + TAG_SIZES[BasicType.FLOAT] + return offset + + +def _skip_startpoint(data: bytes, offset: int) -> int: + """ + Step over one start point: the placement then an integer. + + Parameters + ---------- + data : bytes + A decompressed level. + offset : int + Offset of the start point. + + Returns + ------- + int + The offset just past the start point. + """ + _placement, offset = _read_placement(data, offset) + _, offset = read_int(data, offset) + return offset + + +def _skip_fsm(data: bytes, offset: int) -> int: + """ + Step over one state machine. + + ``X_LevelDBExportFSM::read`` writes the placement, an array of strings, a string, a property + bag, then three maps whose values are themselves property bags. + + Parameters + ---------- + data : bytes + A decompressed level. + offset : int + Offset of the machine. + + Returns + ------- + int + The offset just past the machine. + """ + _placement, offset = _read_placement(data, offset) + offset = _read_string_array(data, offset) + _, offset = read_string(data, offset) + offset = _skip_properties(data, offset) + for _ in range(3): + offset = _expect_tag(data, offset, BasicType.MAP) + count, offset = read_int(data, offset) + for _ in range(count): + _, offset = read_string(data, offset) + offset = _skip_properties(data, offset) + return offset + + +def _skip_trigger(data: bytes, offset: int) -> int: + """ + Step over one trigger: the placement, a float, then an integer. + + Parameters + ---------- + data : bytes + A decompressed level. + offset : int + Offset of the trigger. + + Returns + ------- + int + The offset just past the trigger. + """ + _placement, offset = _read_placement(data, offset) + offset = _expect_tag(data, offset, BasicType.FLOAT) + TAG_SIZES[BasicType.FLOAT] + _, offset = read_int(data, offset) + return offset + + +def _read_curve(data: bytes, offset: int) -> tuple[tuple[float, ...], int]: + """ + Read one animation curve: a three-integer header, a sample count, and that many floats. + + Parameters + ---------- + data : bytes + A decompressed level. + offset : int + Offset of the curve. + + Returns + ------- + tuple[tuple[float, ...], int] + The samples and the offset just past the curve. + """ + for _ in range(3): + _, offset = read_int(data, offset) + count, offset = read_int(data, offset) + samples = [] + for _ in range(count): + offset = _expect_tag(data, offset, BasicType.FLOAT) + samples.append(struct.unpack_from(' tuple[PropAnimation, int]: + """ + Read one prop animation. + + ``X_LevelDBExportDynamicMeshAnimation::read`` writes a duration, the transform the prop starts + at, the one it ends at, three arrays of script lines, then two curves. The scripts are what + chain clips together -- a door's ``open1`` ends by firing ``DO_Animate(stop1)`` -- and are not + needed to draw the motion, so they are stepped over. + + The two curves are separate channels, each sampled evenly across the duration. The first is the + distance travelled in world units -- its last sample matches the gap between the two poses on + 4701 of the 4704 moving clips -- and the second is how far the prop has turned, from nought to + one, reaching exactly one on 3535 of the 3536 that turn. They carry different sample counts, + so a prop that both slides and turns eases the two differently. + + Parameters + ---------- + data : bytes + A decompressed level. + offset : int + Offset of the animation. + name : str + The clip's name, which the container read just before this. + + Returns + ------- + tuple[PropAnimation, int] + The clip and the offset just past it. + """ + offset = _expect_tag(data, offset, BasicType.FLOAT) + duration = struct.unpack_from(' tuple[RenderMesh, int]: + """ + Read the animated props. + + The container mirrors the static mesh one -- a corner pool shared by every prop, then one entry + per prop -- but keys its entries by name and follows each mesh with the placement, the prop's + animations, six flags and four integers. + + Props nest: ``::a5::Gas_Bottle_small1::valve.DO`` is a child of ``::a5::Gas_Bottle_small1``, and + the mesh's own transform is relative to that parent while the placement's is the accumulated + world one. They agree for a prop with no parent, which is most of them, and disagree on 1356 of + the 5946 props across the shipped levels -- by as much as 953 units on + ``Startup_level.ldb``. The placement's is the one to keep, so it replaces the mesh's here and + the exporter can treat every prop the same way. + + Parameters + ---------- + data : bytes + A decompressed level. + offset : int + Offset of the container's array marker. + + Returns + ------- + tuple[RenderMesh, int] + The props and the offset just past the container. + """ + count, offset = _read_corner_count(data, offset, minimum=-1) + corners, offset = _read_corners(data, offset, count) + count, offset = _read_entry_count(data, offset, 'prop') + meshes: list[StaticMesh] = [] + names: list[str] = [] + clips: list[tuple[PropAnimation, ...]] = [] + for _ in range(count): + name, offset = read_string(data, offset) + mesh, offset = _read_static_mesh(data, offset) + placement, offset = _read_placement(data, offset) + mesh = mesh._replace(transform=placement.transform) + animations, offset = read_int(data, offset) + found: list[PropAnimation] = [] + for _ in range(animations): + clip, offset = read_string(data, offset) + animation, offset = _read_animation(data, offset, clip) + found.append(animation) + offset += _DYNAMIC_MESH_FLAGS * (1 + TAG_SIZES[BasicType.BOOL]) + for _ in range(_DYNAMIC_MESH_TRAILING): + _, offset = read_int(data, offset) + meshes.append(mesh) + names.append(name) + clips.append(tuple(found)) + return RenderMesh(animations=tuple(clips), + corners=tuple(corners), + meshes=tuple(meshes), + names=tuple(names)), offset + + +def _read_object_container(data: bytes, offset: int, + element: Callable[[bytes, int], int]) -> tuple[list[Placement], int]: + """ + Read one container of placed objects. + + ``R_Container::read`` writes the entry count, then per entry a key and the element itself. + Every element begins with the placement, so *element* is only asked to step over the rest. + + Parameters + ---------- + data : bytes + A decompressed level. + offset : int + Offset of the entry count. + element : collections.abc.Callable[[bytes, int], int] + Reads one whole element and returns the offset just past it. + + Returns + ------- + tuple[list[Placement], int] + The placements and the offset just past the container. + + """ + count, offset = _read_entry_count(data, offset, 'placement') + out: list[Placement] = [] + for _ in range(count): + _key, offset = read_string(data, offset) + placement, _ = _read_placement(data, offset) + offset = element(data, offset) + out.append(placement) + return out, offset + + +def _read_characters(data: bytes, offset: int) -> tuple[tuple[Character, ...], int]: + """ + Read the NPC placements. + + ``X_LevelDBExportCharacter::read`` follows the placement with the skin's directory name and four + arrays of strings holding the character's scripted behaviour. + + Parameters + ---------- + data : bytes + A decompressed level. + offset : int + Offset of the entry count. + + Returns + ------- + tuple[tuple[Character, ...], int] + The characters and the offset just past the container. + + """ + count, offset = _read_entry_count(data, offset, 'character') + out: list[Character] = [] + for _ in range(count): + _key, offset = read_string(data, offset) + placement, offset = _read_placement(data, offset) + skin, offset = read_string(data, offset) + for _ in range(4): + offset = _read_string_array(data, offset) + out.append(Character(placement=placement, skin=skin)) + return tuple(out), offset + + +def _read_items(data: bytes, offset: int) -> tuple[tuple[LevelItem, ...], int]: + """ + Read the pickups. + + ``X_LevelDBExportLevelItem::read`` follows the placement with the item's directory name. + + Parameters + ---------- + data : bytes + A decompressed level. + offset : int + Offset of the entry count. + + Returns + ------- + tuple[tuple[LevelItem, ...], int] + The pickups and the offset just past the container. + + """ + count, offset = _read_entry_count(data, offset, 'item') + out: list[LevelItem] = [] + for _ in range(count): + _key, offset = read_string(data, offset) + placement, offset = _read_placement(data, offset) + item, offset = read_string(data, offset) + out.append(LevelItem(item=item, placement=placement)) + return tuple(out), offset + + +def _skip_static_point_light(data: bytes, offset: int) -> int: + """ + Step over one static point light: the placement then six floats. + + Parameters + ---------- + data : bytes + A decompressed level. + offset : int + Offset of the light. + + Returns + ------- + int + The offset just past the light. + """ + _placement, offset = _read_placement(data, offset) + for _ in range(6): + offset = _expect_tag(data, offset, BasicType.FLOAT) + TAG_SIZES[BasicType.FLOAT] + return offset + + +def _read_indexed_container(data: bytes, offset: int, element: Callable[[bytes, int], int]) -> int: + """ + Step over a container that keys its entries by number rather than by name. + + Parameters + ---------- + data : bytes + A decompressed level. + offset : int + Offset of the entry count. + element : collections.abc.Callable[[bytes, int], int] + Reads one element and returns the offset just past it. + + Returns + ------- + int + The offset just past the container. + """ + count, offset = _read_entry_count(data, offset, 'entry') + for _ in range(count): + _key, offset = read_int(data, offset) + offset = element(data, offset) + return offset + + +def _room_of(name: str) -> str: + """ + Take the room out of an object's name. + + Every placed object is named ``::room::rest``, and that prefix is the only link back to its + room; the object's own room field is left empty on the shipped levels. + + Parameters + ---------- + name : str + The object's name. + + Returns + ------- + str + The room's name, or an empty string when the name does not carry one. + """ + parts = name.split('::') + return f'::{parts[1]}' if len(parts) > 2 else '' # noqa: PLR2004 + + +class _Tail(NamedTuple): + """Everything read after the static meshes, already placed in the world.""" + + mesh: RenderMesh + props: RenderMesh | None + characters: tuple[Character, ...] + items: tuple[LevelItem, ...] + + +def _read_tail_containers( + data: bytes, offset: int +) -> tuple[tuple[Character, ...], RenderMesh, tuple[LevelItem, ...], list[tuple[list[int], str]]]: + """ + Read every container after the static meshes, in the order the level writes them. + + Parameters + ---------- + data : bytes + A decompressed level. + offset : int + Offset just past the static mesh container. + + Returns + ------- + tuple[tuple[Character, ...], RenderMesh, tuple[LevelItem, ...], list[tuple[list[int], str]]] + The characters, props, pickups and rooms, each still in its own room's space. + """ + for element in (_skip_static_light, _skip_startpoint, _skip_fsm): + _, offset = _read_object_container(data, offset, element) + characters, offset = _read_characters(data, offset) + _, offset = _read_object_container(data, offset, _skip_trigger) + props, offset = _read_dynamic_meshes(data, offset) + items, offset = _read_items(data, offset) + offset = _read_indexed_container(data, offset, _skip_static_point_light) + rooms, offset = _read_rooms(data, offset) + return characters, props, items, rooms + + +def _place_props(props: RenderMesh, placed: dict[str, tuple[float, ...]]) -> RenderMesh: + """ + Move every prop, and both ends of each of its clips, into its room's place in the world. + + Parameters + ---------- + props : RenderMesh + The props, still in their own rooms' spaces. + placed : dict[str, tuple[float, ...]] + Room name to the transform placing it. + + Returns + ------- + RenderMesh + The same props, placed. + """ + meshes = [] + clips = [] + for index, (mesh, name) in enumerate(zip(props.meshes, props.names, strict=True)): + room = placed.get(_room_of(name), _IDENTITY) + meshes.append(mesh._replace(transform=_compose(mesh.transform, room))) + found = props.animations[index] if index < len(props.animations) else () + clips.append( + tuple( + clip._replace(end=_compose(clip.end, room), start=_compose(clip.start, room)) + for clip in found)) + return props._replace(animations=tuple(clips), meshes=tuple(meshes)) + + +def _read_tail(data: bytes, offset: int, mesh: RenderMesh, exits: dict[str, tuple[tuple[float, ...], + str]]) -> _Tail: + """ + Read the containers that follow the static meshes and place everything in one world. + + ``X_LevelDBExportLevel::read`` takes them in a fixed order: static lights, start points, state + machines, characters, triggers, animated props, pickups, static point lights, then rooms. The + rooms come last, which is why the whole tail has to be walked before anything can be placed. + + A failure anywhere costs the rest of the tail. The level still renders without it, so the walk + gives up and hands back what it has rather than raising. + + Parameters + ---------- + data : bytes + A decompressed level. + offset : int + Offset just past the static mesh container. + mesh : RenderMesh + The static meshes, still in their own rooms' spaces. + exits : dict[str, tuple[tuple[float, ...], str]] + The level's exits, which say how the rooms fit together. + + Returns + ------- + _Tail + The meshes, props, characters and pickups, placed where the level's exits put them. + """ + characters: tuple[Character, ...] = () + items: tuple[LevelItem, ...] = () + props: RenderMesh | None = None + try: + characters, props, items, rooms = _read_tail_containers(data, offset) + except (IndexError, InvalidLevelError, struct.error, ValueError): + log.warning('Could not walk the tail containers; the level is left unassembled.') + return _Tail(characters=characters, items=items, mesh=mesh, props=props) + placed = _place_rooms(exits, rooms) + by_key = {identifier: placed.get(name, _IDENTITY) for ids, name in rooms for identifier in ids} + return _Tail(characters=tuple( + c._replace(placement=c.placement._replace(transform=_compose( + c.placement.transform, placed.get(_room_of(c.placement.name), _IDENTITY)))) + for c in characters), + items=tuple( + i._replace(placement=i.placement._replace(transform=_compose( + i.placement.transform, placed.get(_room_of(i.placement.name), _IDENTITY)))) + for i in items), + mesh=mesh._replace(meshes=tuple( + m._replace(transform=by_key.get(key, _IDENTITY)) + for m, key in zip(mesh.meshes, mesh.keys, strict=True))), + props=props and _place_props(props, placed)) + + +def read_level(data: bytes) -> Level: + """ + Read a level's geometry, images, materials and per-face material assignment. + + Parameters + ---------- + data : bytes + A decompressed ``.ldb``. + + Returns + ------- + Level + Everything decoded so far. + + Raises + ------ + InvalidLevelError + If any structure is not where the format says it should be. + """ + geometry = read_geometry(data) + offset = _skip_preamble(data) + count, offset = read_int(data, offset) + textures: list[TextureImage] = [] + for _ in range(count): + path, offset = read_string(data, offset) + kind, offset = read_int(data, offset) + length, offset = read_int(data, offset) + if offset + length > len(data): + msg = f'Texture {path!r} claims {length} bytes but the file ends.' + raise InvalidLevelError(msg) + textures.append(TextureImage(data=data[offset:offset + length], kind=kind, path=path)) + offset += length + materials, offset = _read_material_map(data, offset) + offset = _skip_material_index(data, offset) + images, offset = _read_material_categories(data, offset) + materials = _resolve_images(materials, images, textures) + lightmaps, offset = _read_lightmaps(data, offset) + log.debug('Read %d textures and %d materials.', len(textures), len(materials)) + exits: dict[str, tuple[tuple[float, ...], str]] = {} + try: + exits, offset = _read_exits(data, offset) + found: tuple[RenderMesh, int] | None = _read_mesh_container(data, offset) + except (IndexError, InvalidLevelError, struct.error, ValueError): + log.warning('Could not read the exits; searching for the meshes instead.') + found = _find_mesh_container(data, offset) + if found is None: + log.warning('Could not locate the renderable meshes; falling back to the BSP faces.') + return Level(geometry=geometry, + lightmaps=lightmaps, + materials=materials, + mesh=None, + textures=tuple(textures)) + mesh, offset = found + mesh, props, characters, items = _read_tail(data, offset, mesh, exits) + log.debug('Read %d props, %d characters and %d items.', + len(props.meshes) if props else 0, len(characters), len(items)) + return Level(characters=characters, + geometry=geometry, + items=items, + lightmaps=lightmaps, + materials=materials, + mesh=mesh, + props=props, + textures=tuple(textures)) diff --git a/dade/maxpane/main.py b/dade/maxpane/main.py new file mode 100644 index 0000000..b7497c6 --- /dev/null +++ b/dade/maxpane/main.py @@ -0,0 +1,24 @@ +"""Root Click group and subcommand wiring for the Max Payne tools.""" +from __future__ import annotations + +import click + +from .commands.inspect_tags import inspect_tags +from .commands.ldb2glb import ldb2glb +from .commands.ldb_textures import ldb_textures +from .commands.ras_extract import ras_extract +from .commands.ras_list import ras_list + +__all__ = ('cli',) + + +@click.group(name='maxpane', context_settings={'help_option_names': ('-h', '--help')}) +def cli() -> None: + """Extract and decode Max Payne (Remedy Entertainment) assets.""" + + +cli.add_command(inspect_tags) +cli.add_command(ldb2glb) +cli.add_command(ldb_textures) +cli.add_command(ras_extract) +cli.add_command(ras_list) diff --git a/dade/maxpane/memoryfile.py b/dade/maxpane/memoryfile.py new file mode 100644 index 0000000..bdb302f --- /dev/null +++ b/dade/maxpane/memoryfile.py @@ -0,0 +1,322 @@ +""" +Decoder for the tagged ``R_MemoryFile`` streams carrying every custom Max Payne asset. + +``R_MemoryFile::writeTagged`` prefixes each value with a one-byte ``BasicType`` tag and then writes +a byte count the caller chooses, so a tag does not encode its own length in general. In practice +each tag is used with one width, which :py:data:`TAG_SIZES` records. + +Integers are stored in a narrowest-signed-fit form: ``operator<<`` measures the magnitude and picks +a one-, two-, three-, or four-byte encoding, with the tag identifying the width. That is why the +same logical field is two bytes in one level and three in another. + +Chunked assets place a :py:data:`BasicType.CHUNK` tag before a 12-byte header giving an identifier, +a version, and a size that counts the 13-byte header itself. Levels and movement networks are flat +tagged streams and use no chunks. +""" +from __future__ import annotations + +from enum import IntEnum +from typing import TYPE_CHECKING +import struct + +from .typing import TaggedValue + +if TYPE_CHECKING: + from collections.abc import Iterator, Mapping + +__all__ = ('CHUNK_HEADER_SIZE', 'TAG_SIZES', 'BasicType', 'iter_values', 'read_chunk_header', + 'read_int', 'read_string', 'read_vector3') + + +class BasicType(IntEnum): + """ + Tag values written by ``R_MemoryFile::writeTag``. + + The scalar entries were read from the ``operator<<`` overloads exported by ``rl.dll``; the + compact integer entries come from the width selection inside those overloads. + """ + + LONG = 0x00 + """Signed long, four bytes.""" + ULONG = 0x01 + """Unsigned long, four bytes.""" + INT32 = 0x02 + """Signed integer needing the full four bytes.""" + UINT32 = 0x03 + """Unsigned integer needing the full four bytes.""" + SHORT = 0x04 + """Signed short, two bytes.""" + USHORT = 0x05 + """Unsigned short, two bytes.""" + CHAR = 0x06 + """Character, one byte.""" + SCHAR = 0x07 + """Signed character, one byte.""" + UCHAR = 0x08 + """Unsigned character, one byte.""" + FLOAT = 0x09 + """Single-precision float, four bytes.""" + DOUBLE = 0x0A + """Double-precision float, eight bytes.""" + CHUNK = 0x0C + """Chunk header of twelve bytes.""" + STRING = 0x0D + """String. The payload length is not implied by the tag.""" + BOOL = 0x0E + """Boolean, one byte.""" + UINT24 = 0x0F + """Unsigned integer compacted to three bytes.""" + UINT16 = 0x10 + """Unsigned integer compacted to two bytes.""" + UINT8 = 0x11 + """Unsigned integer compacted to one byte.""" + INT24 = 0x12 + """Signed integer compacted to three bytes.""" + INT16 = 0x13 + """Signed integer compacted to two bytes.""" + INT8 = 0x14 + """Signed integer compacted to one byte.""" + VECTOR2 = 0x15 + """Two floats.""" + VECTOR3 = 0x16 + """Three floats.""" + VECTOR4 = 0x17 + """Four floats.""" + MATRIX2 = 0x18 + """Two-by-two matrix of floats.""" + MATRIX3 = 0x19 + """Three-by-three matrix of floats.""" + MATRIX4X3 = 0x1A + """Four-by-three matrix of floats.""" + MATRIX4 = 0x1B + """Four-by-four matrix of floats.""" + ARRAY = 0x1C + """Marker introducing a counted array. Carries no payload of its own.""" + MAP = 0x1F + """Marker introducing a counted map. Carries no payload of its own; a count follows, then that + many entries of a key and a :py:attr:`PAIR`.""" + PAIR = 0x25 + """Marker introducing the two halves of a map entry. Carries no payload of its own.""" + FLOAT16 = 0x26 + """Half-precision float, two bytes.""" + + +CHUNK_HEADER_SIZE = 13 +"""Size of a chunk tag plus its twelve-byte header. + +:meta hide-value: +""" + +TAG_SIZES: Mapping[int, int] = { + BasicType.ARRAY: 0, + BasicType.BOOL: 1, + BasicType.CHAR: 1, + BasicType.CHUNK: 12, + BasicType.DOUBLE: 8, + BasicType.FLOAT: 4, + BasicType.FLOAT16: 2, + BasicType.INT8: 1, + BasicType.INT16: 2, + BasicType.INT24: 3, + BasicType.INT32: 4, + BasicType.LONG: 4, + BasicType.MATRIX2: 16, + BasicType.MATRIX3: 36, + BasicType.MATRIX4: 64, + BasicType.MAP: 0, + BasicType.MATRIX4X3: 48, + BasicType.PAIR: 0, + BasicType.SCHAR: 1, + BasicType.SHORT: 2, + BasicType.UCHAR: 1, + BasicType.UINT8: 1, + BasicType.UINT16: 2, + BasicType.UINT24: 3, + BasicType.UINT32: 4, + BasicType.ULONG: 4, + BasicType.USHORT: 2, + BasicType.VECTOR2: 8, + BasicType.VECTOR3: 12, + BasicType.VECTOR4: 16 +} +"""Payload width in bytes for every tag whose width is fixed. + +:py:attr:`BasicType.STRING` is absent because its length is chosen by the caller. + +:meta hide-value: +""" + +_SIGNED_TAGS = frozenset({ + BasicType.INT8, BasicType.INT16, BasicType.INT24, BasicType.INT32, BasicType.LONG, + BasicType.SHORT, BasicType.SCHAR +}) + +_INTEGER_TAGS = frozenset({ + BasicType.BOOL, BasicType.CHAR, BasicType.INT8, BasicType.INT16, BasicType.INT24, + BasicType.INT32, BasicType.LONG, BasicType.SCHAR, BasicType.SHORT, BasicType.UCHAR, + BasicType.UINT8, BasicType.UINT16, BasicType.UINT24, BasicType.UINT32, BasicType.ULONG, + BasicType.USHORT +}) +"""Tags whose payload is a whole number, and so the only ones an integer can be read from. + +Every other tag in :py:data:`TAG_SIZES` has a width too, which is not the same thing: reading a +``FLOAT`` as an integer gives its bit pattern, and reading a ``MAP``, whose width is nought, gives +a silent zero.""" + + +def read_chunk_header(data: bytes, offset: int = 0) -> tuple[int, int, int]: + """ + Read a chunk tag and its header. + + Parameters + ---------- + data : bytes + The stream. + offset : int + Byte offset of the :py:attr:`BasicType.CHUNK` tag. + + Returns + ------- + tuple[int, int, int] + The chunk identifier, its version, and its size. The size counts the + :py:data:`CHUNK_HEADER_SIZE` header bytes, so the payload is that many bytes shorter. + + Raises + ------ + ValueError + If the tag at *offset* is not :py:attr:`BasicType.CHUNK`. + """ + if data[offset] != BasicType.CHUNK: + msg = f'Not a chunk tag at offset {offset}: 0x{data[offset]:02x}.' + raise ValueError(msg) + return struct.unpack_from('<3I', data, offset + 1) + + +def read_int(data: bytes, offset: int) -> tuple[int, int]: + """ + Read one tagged integer, whatever width it was compacted to. + + Parameters + ---------- + data : bytes + The stream. + offset : int + Byte offset of the tag. + + Returns + ------- + tuple[int, int] + The value and the offset just past it. + + Raises + ------ + ValueError + If the tag at *offset* is not an integer tag. + """ + tag = data[offset] + if tag not in _INTEGER_TAGS: + msg = f'Not an integer tag at offset {offset}: 0x{tag:02x}.' + raise ValueError(msg) + width = TAG_SIZES[tag] + end = offset + 1 + width + return int.from_bytes(data[offset + 1:end], 'little', signed=tag in _SIGNED_TAGS), end + + +def read_string(data: bytes, offset: int) -> tuple[str, int]: + """ + Read one tagged string. + + A string is the :py:attr:`BasicType.STRING` tag, then a tagged integer giving its length, then + that many raw bytes. The length is compacted like any other integer, so it is one to four bytes + wide and carries its own tag. + + Parameters + ---------- + data : bytes + The stream. + offset : int + Byte offset of the tag. + + Returns + ------- + tuple[str, int] + The string and the offset just past it. + + Raises + ------ + ValueError + If the tag at *offset* is not :py:attr:`BasicType.STRING`. + """ + if data[offset] != BasicType.STRING: + msg = f'Not a string tag at offset {offset}: 0x{data[offset]:02x}.' + raise ValueError(msg) + length, start = read_int(data, offset + 1) + return data[start:start + length].decode('latin-1'), start + length + + +def read_vector3(data: bytes, offset: int) -> tuple[tuple[float, float, float], int]: + """ + Read one tagged three-component vector. + + Parameters + ---------- + data : bytes + The stream. + offset : int + Byte offset of the tag. + + Returns + ------- + tuple[tuple[float, float, float], int] + The vector and the offset just past it. + + Raises + ------ + ValueError + If the tag at *offset* is not :py:attr:`BasicType.VECTOR3`. + """ + if data[offset] != BasicType.VECTOR3: + msg = f'Not a vector tag at offset {offset}: 0x{data[offset]:02x}.' + raise ValueError(msg) + return struct.unpack_from('<3f', data, offset + 1), offset + 13 + + +def iter_values(data: bytes, offset: int = 0) -> Iterator[TaggedValue]: + """ + Walk a tagged stream until an unknown tag or the end of the buffer. + + Bulk data such as lightmap pixels is written untagged, so a walk stops wherever the stream + leaves tagged territory. The caller can resume by passing a later offset. Strings are followed + correctly because they carry their own length. + + Parameters + ---------- + data : bytes + The stream. + offset : int + Byte offset to start at. + + Yields + ------ + TaggedValue + Each tag and its payload, in order. A string's payload is its bytes without the length. + """ + while offset < len(data): + tag = data[offset] + if tag == BasicType.STRING: + try: + text, end = read_string(data, offset) + except (IndexError, ValueError): + return + if end > len(data): + return + yield TaggedValue(end=end, offset=offset, payload=text.encode('latin-1'), tag=tag) + offset = end + continue + if (width := TAG_SIZES.get(tag)) is None or offset + 1 + width > len(data): + return + yield TaggedValue(end=offset + 1 + width, + offset=offset, + payload=bytes(data[offset + 1:offset + 1 + width]), + tag=tag) + offset += 1 + width diff --git a/dade/maxpane/model.py b/dade/maxpane/model.py new file mode 100644 index 0000000..3c6ef42 --- /dev/null +++ b/dade/maxpane/model.py @@ -0,0 +1,576 @@ +r""" +Reader for the ``.kfs`` and ``.kf2`` models NPCs, pickups and weapons are drawn with. + +Both are chunked ``R_MemoryFile`` streams written by the 3ds max exporter, and they carry the same +information in two encodings. A ``.kfs`` skin uses version 0 chunks, where everything is tagged and +faces index positions and texture coordinates separately; a ``.kf2`` object uses version 1, where +the counts are tagged but the vertex and coordinate data are packed float arrays and a flat +sixteen-bit index buffer draws them. The chunk identifiers are shared, so one reader handles both. + +Models are Z-up, the convention of the tool that exported them, while the game and glTF are both +Y-up; positions and normals are rotated on the way out so a character stands up. Texture +coordinates are not touched at all: V runs negative, and the game hands it to Direct3D as written +and lets wrapping sort it out. + +A model does not embed its images. It carries a search path -- always ``textures`` then +``..\\sharedtextures`` -- and its materials name files to be found along it, so a caller that wants +the model textured has to read those off disk itself. +""" +from __future__ import annotations + +from typing import TYPE_CHECKING +import logging +import struct + +from .memoryfile import CHUNK_HEADER_SIZE, TAG_SIZES, BasicType, read_int, read_string +from .typing import Model, ModelFace, ModelMesh + +if TYPE_CHECKING: + from collections.abc import Iterator, Sequence + + from .typing import Vector3 + +__all__ = ('InvalidModelError', 'read_model') + +log = logging.getLogger(__name__) + +_LIBRARY = 0x0001000F +"""Material library: a search path, a count, then that many material chunks.""" +_MATERIAL = 0x00010010 +_TEXTURE = 0x00010011 +_MESH = 0x00010005 +_NODE = 0x00010000 +_POSITIONS = 0x00010006 +_FACES = 0x00010007 +_FACE = 0x00010008 +_MATERIALS = 0x0001000C +_COORDS = 0x0001000E + +_PACKED = 1 +"""Chunk version that packs its payload instead of tagging every value.""" +_FLOAT_SIZE = 4 +_INDEX_SIZE = 2 +"""Width of one entry in a packed mesh's index buffer.""" +_TRIANGLE = 3 +_MAX_ELEMENTS = 4_000_000 + + +class InvalidModelError(ValueError): + """Raised when a buffer is not a readable model.""" + + +def _chunks(data: bytes, offset: int, end: int) -> Iterator[tuple[int, int, int, int]]: + """ + Walk the chunks laid consecutively between two offsets. + + Parameters + ---------- + data : bytes + A decompressed model. + offset : int + Offset of the first chunk tag. + end : int + Offset to stop at. + + Yields + ------ + tuple[int, int, int, int] + The chunk's identifier and version, and the offsets of its body and of its end. + + Raises + ------ + InvalidModelError + If a chunk's size does not fit inside *end*. + """ + while offset < end: + if data[offset] != BasicType.CHUNK: + return + if offset + CHUNK_HEADER_SIZE > end: + msg = f'A chunk header at offset {offset} runs past the end of the stream.' + raise InvalidModelError(msg) + identifier, version, size = struct.unpack_from('<3I', data, offset + 1) + if size < CHUNK_HEADER_SIZE or offset + size > end: + msg = f'Chunk 0x{identifier:08x} at offset {offset} claims {size} bytes.' + raise InvalidModelError(msg) + yield identifier, version, offset + CHUNK_HEADER_SIZE, offset + size + offset += size + + +def _read_count(data: bytes, offset: int) -> tuple[int, int]: + """ + Read an element count and reject an implausible one. + + Parameters + ---------- + data : bytes + A decompressed model. + offset : int + Offset of the count. + + Returns + ------- + tuple[int, int] + The count and the offset just past it. + + Raises + ------ + InvalidModelError + If the count is negative or beyond anything a model holds. + """ + count, offset = read_int(data, offset) + if not 0 <= count < _MAX_ELEMENTS: + msg = f'Implausible element count {count} at offset {offset}.' + raise InvalidModelError(msg) + return count, offset + + +def _upright(x: float, y: float, z: float) -> Vector3: + """ + Turn an exporter-space vector into a game-space one. + + Parameters + ---------- + x : float + Sideways. + y : float + Depth in exporter space. + z : float + Up in exporter space. + + Returns + ------- + Vector3 + The same vector with the up axis moved from Z to Y. + """ + return (x, -z, y) + + +def _read_vectors(data: bytes, offset: int, count: int, end: int, *, + packed: bool) -> tuple[list[Vector3], int]: + """ + Read a run of three-component vectors, tagged or packed. + + Parameters + ---------- + data : bytes + A decompressed model. + offset : int + Offset of the first vector. + count : int + How many to read. + end : int + Offset the run has to finish inside, being the end of the chunk holding it. + packed : bool + Read raw floats rather than tagged vectors. + + Returns + ------- + tuple[list[Vector3], int] + The vectors and the offset just past them. + + Raises + ------ + InvalidModelError + If the run does not fit inside its chunk, or a tagged one holds something that is not a + vector. + """ + out: list[Vector3] = [] + stride = 3 * _FLOAT_SIZE + # The count comes out of the file, so on its own it says only how much to read, not how much + # there is. Unchecked it reads whatever follows the chunk and calls it geometry. + if offset + count * (stride if packed else 1 + stride) > end: + msg = f'A run of {count} vectors at offset {offset} does not fit inside its chunk.' + raise InvalidModelError(msg) + if packed: + out.extend( + _upright(*struct.unpack_from('<3f', data, offset + index * stride)) + for index in range(count)) + return out, offset + count * stride + for _ in range(count): + if data[offset] != BasicType.VECTOR3: + msg = f'Expected a vector at offset {offset}, got 0x{data[offset]:02x}.' + raise InvalidModelError(msg) + out.append(_upright(*struct.unpack_from('<3f', data, offset + 1))) + offset += 1 + stride + return out, offset + + +def _read_face_indices(data: bytes, offset: int, end: int) -> list[tuple[int, ...]]: + """ + Read the per-face index chunks a version 0 mesh writes. + + Parameters + ---------- + data : bytes + A decompressed model. + offset : int + Offset of the first face chunk. + end : int + Offset to stop at. + + Returns + ------- + list[tuple[int, ...]] + One tuple of indices per face. + """ + out: list[tuple[int, ...]] = [] + for identifier, _version, body, tail in _chunks(data, offset, end): + if identifier != _FACE: + break + count, cursor = _read_count(data, body) + indices = [] + for _ in range(count): + value, cursor = read_int(data, cursor) + indices.append(value) + out.append(tuple(indices)) + offset = tail + return out + + +def _skip_face_chunks(data: bytes, offset: int, end: int) -> int: + """ + Step over the per-face index chunks to whatever follows them. + + Parameters + ---------- + data : bytes + A decompressed model. + offset : int + Offset of the first face chunk. + end : int + Offset to stop at. + + Returns + ------- + int + The offset just past the last face chunk. + """ + while offset < end and data[offset] == BasicType.CHUNK: + _identifier, _version, _body, tail = next(_chunks(data, offset, end)) + offset = tail + return offset + + +def _read_materials(data: bytes, offset: int, end: int) -> tuple[dict[str, str], tuple[str, ...]]: + """ + Read the material library: what each material is called and which image it draws with. + + Parameters + ---------- + data : bytes + A decompressed model. + offset : int + Offset of the library's body. + end : int + Offset to stop at. + + Returns + ------- + tuple[dict[str, str], tuple[str, ...]] + Material name to image file name, and the directories to look for those images in. + """ + search, offset = read_string(data, offset) + _count, offset = read_int(data, offset) + materials: dict[str, str] = {} + for identifier, _version, body, tail in _chunks(data, offset, end): + if identifier != _MATERIAL: + continue + name, cursor = read_string(data, body) + image = '' + for inner, _v, inner_body, _inner_tail in _chunks(data, _skip_to_chunk(data, cursor, tail), + tail): + if inner != _TEXTURE: + continue + _slot, at = read_string(data, inner_body) + for _ in range(2): + _, at = read_int(data, at) + files, at = read_int(data, at) + for _ in range(files): + image, at = read_string(data, at) + break + break + materials[name] = image + offset = tail + return materials, tuple(part for part in search.replace('\\', '/').split(';') if part) + + +def _skip_to_chunk(data: bytes, offset: int, end: int) -> int: + """ + Step over tagged values until a chunk begins. + + A material writes a long run of flags and colours before its texture chunk, and none of it is + needed here, so the values are stepped over by their tag widths rather than decoded. + + Parameters + ---------- + data : bytes + A decompressed model. + offset : int + Offset to start from. + end : int + Offset to stop at. + + Returns + ------- + int + The offset of the next chunk, or *end* when there is none. + """ + while offset < end and data[offset] != BasicType.CHUNK: + if data[offset] == BasicType.STRING: + _, offset = read_string(data, offset) + continue + width = TAG_SIZES.get(data[offset]) + if width is None: + return end + offset += 1 + width + return offset + + +def _read_mesh(data: bytes, offset: int, end: int) -> ModelMesh: + """ + Read one mesh, in either encoding. + + Parameters + ---------- + data : bytes + A decompressed model. + offset : int + Offset of the mesh's body. + end : int + Offset to stop at. + + Returns + ------- + ModelMesh + The mesh. + """ + name = '' + positions: list[Vector3] = [] + normals: list[Vector3] = [] + coords: list[tuple[float, float]] = [] + position_faces: list[tuple[int, ...]] = [] + coord_faces: list[tuple[int, ...]] = [] + materials: tuple[str, ...] = () + face_materials: list[int] = [] + # The mesh chunk's own version does not decide the encoding: a skin's mesh is version 1 with + # version 0 arrays inside it, and an object's is version 2 with version 1 arrays. Each array + # chunk says for itself whether it is packed. + for identifier, inner, body, tail in _chunks(data, offset, end): + packed = inner >= _PACKED + if identifier == _NODE: + name, _ = read_string(data, body) + elif identifier == _POSITIONS: + count, cursor = _read_count(data, body) + positions, cursor = _read_vectors(data, cursor, count, tail, packed=packed) + if packed: + normals, _ = _read_vectors(data, cursor, count, tail, packed=True) + elif identifier == _FACES: + position_faces = _read_faces(data, body, tail, packed=packed) + elif identifier == _COORDS: + coord_faces, coords = _read_coords(data, body, tail, packed=packed) + elif identifier == _MATERIALS: + materials, face_materials = _read_mesh_materials(data, body, tail) + return _assemble(name, positions, normals, coords, position_faces, coord_faces, materials, + face_materials) + + +def _read_faces(data: bytes, offset: int, end: int, *, packed: bool) -> list[tuple[int, ...]]: + """ + Read the triangles: a flat index buffer when packed, one chunk per face otherwise. + + Parameters + ---------- + data : bytes + A decompressed model. + offset : int + Offset of the chunk's body. + end : int + Offset to stop at. + packed : bool + Read a sixteen-bit index buffer rather than per-face chunks. + + Returns + ------- + list[tuple[int, ...]] + One tuple of indices per face. + + Raises + ------ + InvalidModelError + If a packed index buffer does not fit inside its chunk. + """ + count, cursor = _read_count(data, offset) + if not packed: + return _read_face_indices(data, cursor, end) + if cursor + count * _INDEX_SIZE > end: + msg = f'An index buffer of {count} at offset {cursor} does not fit inside its chunk.' + raise InvalidModelError(msg) + indices = struct.unpack_from(f'<{count}H', data, cursor) + return [tuple(indices[at:at + _TRIANGLE]) for at in range(0, count - 2, _TRIANGLE)] + + +def _read_coords(data: bytes, offset: int, end: int, *, + packed: bool) -> tuple[list[tuple[int, ...]], list[tuple[float, float]]]: + """ + Read one texture coordinate set. + + A packed set stores one coordinate per vertex, so its faces are the position faces and no + per-face indices are written. A tagged set writes per-face indices first and its own pool after. + + Coordinates are stored as three-component vectors whose second component is V and whose third + is unused, and they go to Direct3D untouched, so they are kept exactly as written. V is + normally negative, which wraps to the same texel as ``1 + v``; U goes negative too on skins + that mirror a face across the head, so wrapping is required either way. + + Parameters + ---------- + data : bytes + A decompressed model. + offset : int + Offset of the chunk's body. + end : int + Offset to stop at. + packed : bool + Read a packed pool rather than per-face chunks. + + Returns + ------- + tuple[list[tuple[int, ...]], list[tuple[float, float]]] + The per-face indices, empty when the set is per-vertex, and the coordinates. + """ + _set, cursor = read_int(data, offset) + count, cursor = _read_count(data, cursor) + if packed: + vectors, _ = _read_vectors(data, cursor, count, end, packed=True) + return [], [(u, w) for u, _v, w in vectors] + faces = _read_face_indices(data, cursor, end) + cursor = _skip_face_chunks(data, cursor, end) + for _ in range(2): + _, cursor = read_int(data, cursor) + count, cursor = _read_count(data, cursor) + vectors, _ = _read_vectors(data, cursor, count, end, packed=False) + return faces, [(u, w) for u, _v, w in vectors] + + +def _read_mesh_materials(data: bytes, offset: int, end: int) -> tuple[tuple[str, ...], list[int]]: + """ + Read the material names a mesh uses and which one each face draws with. + + Parameters + ---------- + data : bytes + A decompressed model. + offset : int + Offset of the chunk's body. + end : int + Offset to stop at. + + Returns + ------- + tuple[tuple[str, ...], list[int]] + The names, and one index per face where the mesh writes them. + """ + count, cursor = _read_count(data, offset) + names = [] + for _ in range(count): + name, cursor = read_string(data, cursor) + names.append(name) + if cursor >= end or data[cursor] == BasicType.CHUNK: + return tuple(names), [] + faces, cursor = _read_count(data, cursor) + out = [] + for _ in range(faces): + value, cursor = read_int(data, cursor) + out.append(value) + return tuple(names), out + + +def _assemble(name: str, positions: list[Vector3], normals: list[Vector3], + coords: list[tuple[float, float]], position_faces: list[tuple[int, ...]], + coord_faces: list[tuple[int, ...]], materials: tuple[str, ...], + face_materials: list[int]) -> ModelMesh: + """ + Put the separately-stored parts of a mesh together, dropping anything malformed. + + Parameters + ---------- + name : str + The mesh's name. + positions : list[Vector3] + Vertex positions. + normals : list[Vector3] + One normal per position, or empty. + coords : list[tuple[float, float]] + Texture coordinates. + position_faces : list[tuple[int, ...]] + Position indices per face. + coord_faces : list[tuple[int, ...]] + Coordinate indices per face, empty when the coordinates are per-vertex. + materials : tuple[str, ...] + Material names the faces index. + face_materials : list[int] + One material index per face, or empty when every face uses the first. + + Returns + ------- + ModelMesh + The assembled mesh. + """ + + # An index has to be in range at both ends: a negative one is in range for Python and picks a + # vertex from the far end of the pool, which is silently wrong geometry rather than an error. + def holds(indices: Sequence[int], pool: Sequence[object]) -> bool: + return len(indices) == _TRIANGLE and all(0 <= i < len(pool) for i in indices) + + faces: list[ModelFace] = [] + for index, triangle in enumerate(position_faces): + if not holds(triangle, positions): + continue + mapped = coord_faces[index] if index < len(coord_faces) else triangle + if not holds(mapped, coords): + mapped = (0, 0, 0) + material = face_materials[index] if index < len(face_materials) else 0 + faces.append( + ModelFace(coords=(mapped[0], mapped[1], mapped[2]), + material=material if 0 <= material < len(materials) else 0, + positions=(triangle[0], triangle[1], triangle[2]))) + return ModelMesh(coords=tuple(coords), + faces=tuple(faces), + materials=materials, + name=name, + normals=tuple(normals), + positions=tuple(positions)) + + +def read_model(data: bytes) -> Model: + """ + Read a ``.kfs`` skin or ``.kf2`` object. + + Parameters + ---------- + data : bytes + A decompressed model. + + Returns + ------- + Model + The meshes and the material library. + + Raises + ------ + InvalidModelError + If the file is not a chunked stream, or a chunk runs past the end. + """ + if not data or data[0] != BasicType.CHUNK: + msg = 'Not a chunked model.' + raise InvalidModelError(msg) + materials: dict[str, str] = {} + search: tuple[str, ...] = () + meshes: list[ModelMesh] = [] + for identifier, _version, body, tail in _chunks(data, 0, len(data)): + if identifier == _LIBRARY: + materials, search = _read_materials(data, body, tail) + elif identifier == _MESH: + meshes.append(_read_mesh(data, body, tail)) + log.debug('Read %d meshes and %d materials.', len(meshes), len(materials)) + return Model(materials=materials, meshes=tuple(meshes), search=search) diff --git a/dade/maxpane/ras.py b/dade/maxpane/ras.py new file mode 100644 index 0000000..ea3432b --- /dev/null +++ b/dade/maxpane/ras.py @@ -0,0 +1,277 @@ +""" +Reader for RAS (Remedy Archive System) archives. + +``.ras`` archives and the ``.mpm`` mod packages share one layout. + +The 44-byte header starts with the magic and a signed cipher seed in the clear; the remaining 36 +bytes are encrypted with that seed and give the file and directory counts, the size of each table, +the format version, a CRC32 of the header with its own CRC field zeroed, and the writer identity. +The file table follows, then the directory table, each encrypted with the same seed restarting its +keystream at index zero. + +Both tables are sequences of a NUL-terminated name followed by fixed-width fields: 40 bytes for a +file (two sizes, two reserved dwords, the owning directory index, another reserved dword, and a +``SYSTEMTIME``) and 16 bytes for a directory (a ``SYSTEMTIME`` alone). Members carry no offset +field because payloads are stored back to back in file-table order, which makes +``header + tables + sum of stored sizes == archive size`` an exact integrity check. +""" +from __future__ import annotations + +from typing import TYPE_CHECKING +import logging +import struct + +from .blocks import unwrap +from .crypto import decrypt +from .typing import ArchiveHeader, RASContents, RASDirectory, RASEntry + +if TYPE_CHECKING: + from collections.abc import Iterator + +__all__ = ('ARCHIVE_VERSION', 'HEADER_SIZE', 'MAGIC', 'InvalidArchiveError', 'is_intact', + 'iter_members', 'member_bytes', 'read_directory', 'read_header') + +log = logging.getLogger(__name__) + +MAGIC = b'RAS\x00' +"""Magic at the start of every archive. + +:meta hide-value: +""" +ARCHIVE_VERSION = 1.2 +"""The only format version known to exist, from ``R_File::RAS_ARCHIVE_VERSION``. + +:meta hide-value: +""" +HEADER_SIZE = 0x2C +"""Size of the archive header in bytes. + +:meta hide-value: +""" + +_FILE_FIELDS_SIZE = 40 +_SYSTEMTIME_SIZE = 16 +_VERSION_TOLERANCE = 1e-6 + + +class InvalidArchiveError(ValueError): + """Raised when a buffer is not a readable RAS archive.""" + + +def _system_time(data: bytes, offset: int) -> str | None: + year, month, _, day, hour, minute, second, millisecond = struct.unpack_from('<8H', data, offset) + if not year: + return None + return (f'{year:04d}-{month:02d}-{day:02d} ' + f'{hour:02d}:{minute:02d}:{second:02d}.{millisecond:03d}') + + +def _read_name(data: bytes, offset: int) -> tuple[str, int]: + end = data.index(b'\x00', offset) + return data[offset:end].decode('latin-1'), end + 1 + + +def read_header(data: bytes) -> ArchiveHeader: + """ + Decode and decrypt the archive header. + + Parameters + ---------- + data : bytes + At least the first :py:data:`HEADER_SIZE` bytes of an archive. + + Returns + ------- + ArchiveHeader + The decrypted header. + + Raises + ------ + InvalidArchiveError + If the magic is wrong or the version is not :py:data:`ARCHIVE_VERSION`. + """ + if data[:4] != MAGIC: + msg = f'Not a RAS archive: {data[:4]!r}.' + raise InvalidArchiveError(msg) + if len(data) < HEADER_SIZE: + msg = f'A RAS archive is at least {HEADER_SIZE} bytes; this one is {len(data)}.' + raise InvalidArchiveError(msg) + seed = struct.unpack_from(' _VERSION_TOLERANCE: + msg = f'Unsupported RAS archive version {version:.2f}.' + raise InvalidArchiveError(msg) + return ArchiveHeader(archiver_id=struct.unpack_from(' tuple[list[RASDirectory], list[RASEntry], int]: + """ + Walk both decrypted tables. + + Parameters + ---------- + header : ArchiveHeader + The decrypted header, for its two counts. + file_table : bytes + The decrypted file table. + directory_table : bytes + The decrypted directory table. + cursor : int + Offset of the first member's payload. + + Returns + ------- + tuple[list[RASDirectory], list[RASEntry], int] + The directories, the entries, and the offset just past the last member's payload. + + Raises + ------ + InvalidArchiveError + If an entry names a directory the archive does not hold. + """ + directories: list[RASDirectory] = [] + offset = 0 + for _ in range(header.directory_count): + name, offset = _read_name(directory_table, offset) + directories.append(RASDirectory(modified=_system_time(directory_table, offset), name=name)) + offset += _SYSTEMTIME_SIZE + entries: list[RASEntry] = [] + offset = 0 + for _ in range(header.file_count): + name, offset = _read_name(file_table, offset) + size, stored_size, _, directory, _, _ = struct.unpack_from('<6I', file_table, offset) + if directory >= len(directories): + msg = f'`{name}` names directory {directory} of {len(directories)}.' + raise InvalidArchiveError(msg) + modified = _system_time(file_table, offset + 24) + offset += _FILE_FIELDS_SIZE + entries.append( + RASEntry(directory=directory, + modified=modified, + name=name, + offset=cursor, + path=(directories[directory].name + name).replace('\\', '/').lstrip('/'), + size=size, + stored_size=stored_size)) + cursor += stored_size + return directories, entries, cursor + + +def read_directory(data: bytes) -> RASContents: + """ + Decode the header and both tables. + + Parameters + ---------- + data : bytes + At least the header and both tables. The payload itself is not touched. + + Returns + ------- + RASContents + The header, the directory table, the file table, and the computed end of the payload. + + Raises + ------ + InvalidArchiveError + If the header will not read, the buffer is too short to hold both tables, or an entry + names a directory the archive does not hold. + """ + header = read_header(data) + table_start = HEADER_SIZE + header.file_table_size + # Slicing a short buffer silently gives back a short table, and the walk over it then fails + # somewhere inside a name or a field rather than saying the archive is cut off. + tables_end = table_start + header.directory_table_size + if len(data) < tables_end: + msg = f'The tables need {tables_end} bytes; the archive is {len(data)}.' + raise InvalidArchiveError(msg) + file_table = decrypt(bytes(data[HEADER_SIZE:table_start]), header.seed) + directory_table = decrypt(bytes(data[table_start:table_start + header.directory_table_size]), + header.seed) + cursor = table_start + header.directory_table_size + # A table long enough to hold what the header promised can still be nonsense inside it: a name + # with no terminator, or one long enough to leave no room for the fields behind it. Those come + # back from the readers as `ValueError` or `struct.error`, which say nothing about archives. + try: + directories, entries, cursor = _read_tables(header, file_table, directory_table, cursor) + except InvalidArchiveError: + raise + except (ValueError, struct.error) as e: + msg = f'The tables will not read: {e}' + raise InvalidArchiveError(msg) from e + return RASContents(data_end=cursor, + directories=tuple(directories), + entries=tuple(entries), + header=header) + + +def member_bytes(data: bytes, entry: RASEntry, *, raw: bool = False) -> bytes: + """ + Slice one member out of an archive. + + Parameters + ---------- + data : bytes + The whole archive. + entry : RASEntry + Member to read, from :py:func:`read_directory` over the same buffer. + raw : bool + Return the stored bytes without removing ``RA->`` or ``RC->`` wrappers. + + Returns + ------- + bytes + The member's contents. + """ + stored = bytes(data[entry.offset:entry.offset + entry.stored_size]) + return stored if raw else unwrap(stored)[0] + + +def iter_members(data: bytes, *, raw: bool = False) -> Iterator[tuple[RASEntry, bytes]]: + """ + Yield every member of an archive with its contents. + + Parameters + ---------- + data : bytes + The whole archive. + raw : bool + Return stored bytes without removing ``RA->`` or ``RC->`` wrappers. + + Yields + ------ + tuple[RASEntry, bytes] + Each member and its contents, in stored order. + """ + for entry in read_directory(data).entries: + yield entry, member_bytes(data, entry, raw=raw) + + +def is_intact(data: bytes) -> bool: + """ + Check that the tables account for every byte of the archive. + + Parameters + ---------- + data : bytes + The whole archive. + + Returns + ------- + bool + :py:obj:`True` when the computed end of the payload matches the buffer length. + """ + return read_directory(data).data_end == len(data) diff --git a/dade/maxpane/typing.py b/dade/maxpane/typing.py new file mode 100644 index 0000000..8e66130 --- /dev/null +++ b/dade/maxpane/typing.py @@ -0,0 +1,339 @@ +"""Typed data structures shared across :py:mod:`dade.maxpane`.""" +from __future__ import annotations + +from typing import NamedTuple, TypeAlias + +__all__ = ('ArchiveHeader', 'Corner', 'Level', 'LevelGeometry', 'Material', 'MeshFace', 'Polygon', + 'RASContents', 'RASDirectory', 'RASEntry', 'RenderMesh', 'StaticMesh', 'TaggedValue', + 'TextureImage', 'Vector3') + +Vector3: TypeAlias = 'tuple[float, float, float]' +"""A point or direction in level space, where one unit is about a metre. + +Nothing in a level says so outright, but the skins settle it: ``gognitti_vinnie_l0.kfs`` is a man +1.88 units tall, and the doors he walks through are a little over two.""" + + +class ArchiveHeader(NamedTuple): + """The decrypted 44-byte header at the start of every RAS archive.""" + + seed: int + """Signed cipher seed, stored in the clear, keying the rest of the archive.""" + file_count: int + """Number of entries in the file table.""" + directory_count: int + """Number of entries in the directory table.""" + file_table_size: int + """Size of the file table in bytes.""" + directory_table_size: int + """Size of the directory table in bytes.""" + version: float + """Archive format version. Only ``1.20`` is known to exist.""" + crc: int + """CRC32 of the header with this field zeroed.""" + file_crc: int + """CRC32 of the decrypted file table.""" + directory_crc: int + """CRC32 of the decrypted directory table.""" + archiver_id: int + """Writer identity checked by ``R_File::verifyArchiverID``. Always ``3`` in shipped data.""" + + +class RASDirectory(NamedTuple): + """One entry of a RAS directory table.""" + + name: str + """Backslash-delimited path as stored, including leading and trailing separators.""" + modified: str | None + """Modification time as ``YYYY-MM-DD HH:MM:SS.mmm``, or :py:obj:`None` when unset.""" + + +class RASEntry(NamedTuple): + """One member of a RAS archive.""" + + name: str + """Member name without any directory part.""" + path: str + """Full slash-delimited path, formed by joining the member's directory with its name.""" + size: int + """Size in bytes after any ``RA->`` or ``RC->`` wrapper is removed.""" + stored_size: int + """Number of bytes occupied in the archive.""" + offset: int + """Absolute byte offset of the member's payload, derived from the preceding members.""" + directory: int + """Index into the archive's directory table.""" + modified: str | None + """Modification time as ``YYYY-MM-DD HH:MM:SS.mmm``, or :py:obj:`None` when unset.""" + + +class RASContents(NamedTuple): + """Everything decoded from an archive's header and tables.""" + + data_end: int + """Offset one past the last member's payload. Equals the archive size when intact.""" + directories: tuple[RASDirectory, ...] + """The directory table, in stored order.""" + entries: tuple[RASEntry, ...] + """The file table, in stored order.""" + header: ArchiveHeader + """The decrypted archive header.""" + + +class Polygon(NamedTuple): + """One convex face of a level's world geometry.""" + + first_vertex: int + """Index of the face's first corner in the level's vertex pool.""" + vertex_count: int + """Number of corners, between three and eight. The corners are the pool entries from + :py:attr:`first_vertex` onwards, so faces never share pool entries.""" + polygon_index: int + """Index into a table that is not decoded yet. The name is provisional: on + ``Part1_Level6.ldb`` it takes 4201 values over 15333 faces, and faces sharing a value do not + reliably share a normal, so it is not a plane identifier.""" + mesh_index: int + """Index into a smaller table that is not decoded yet, also provisionally named. It takes 5 + values on ``Part1_Level6.ldb`` and 39 on ``Part1_Level1.ldb``, splitting a level into a handful + of large face sets, which is what :py:func:`dade.maxpane.gltf.build_glb` groups primitives + by.""" + normal: Vector3 + """Outward face normal.""" + origin: Vector3 + """A point on the face's plane.""" + + +class LevelGeometry(NamedTuple): + """The world geometry at the head of a ``.ldb``.""" + + vertices: tuple[Vector3, ...] + """The vertex pool, already de-indexed: every face owns a contiguous run of it.""" + polygons: tuple[Polygon, ...] + """The faces, in stored order.""" + + +class TaggedValue(NamedTuple): + """One value decoded from a tagged ``R_MemoryFile`` stream.""" + + offset: int + """Byte offset of the tag within the stream.""" + end: int + """Byte offset just past the value, which is where the next tag begins.""" + tag: int + """Raw ``BasicType`` tag byte.""" + payload: bytes + """Value bytes without the tag. Empty for markers such as ``ARRAY``; for a string this is the + text alone, with its length prefix removed.""" + + +class TextureImage(NamedTuple): + """One image a level carries, stored as a complete file.""" + + path: str + """Absolute path the artist authored the image at, used as the texture's key.""" + kind: int + """Format code written beside the image. ``0`` accompanies Targa data and ``4`` JPEG.""" + data: bytes + """The image file, byte for byte.""" + + +class Material(NamedTuple): + """One material binding a surface category to an image.""" + + category: str + """Surface category, such as ``wood`` or ``metal``. Drives footstep and impact sounds.""" + texture: str + """The material's name, as the artist typed it. Usually looks like a filename but is not one: + ``BOOKSHELF01_128X256.JPG`` names the material that draws with + ``bookshelf01_256x256.jpg``. Use :py:attr:`image` to reach the picture.""" + image: str + """Path of the embedded image the material draws with, matching a + :py:attr:`TextureImage.path` exactly, or an empty string when the level's category table does + not name one.""" + alpha: str = '' + """Path of the embedded image holding the material's alpha, or an empty string when it is + opaque. The colour images are JPEG or 8-bit PCX and carry no alpha channel of their own, so a + material that needs one -- foliage, fences, neon signs, glass -- names a second image whose + brightness is the mask.""" + + +class Level(NamedTuple): + """Everything decoded from a ``.ldb``.""" + + geometry: LevelGeometry + """The vertex pool and face table.""" + textures: tuple[TextureImage, ...] + """The embedded images.""" + materials: dict[int, Material] + """Material identifier to material.""" + mesh: RenderMesh | None + """The renderable mesh, or :py:obj:`None` when it could not be located.""" + props: RenderMesh | None = None + """The animated props -- doors, elevators, breakables -- which the editor keeps apart from the + architecture because each can be driven by an animation.""" + characters: tuple[Character, ...] = () + """The NPCs the level spawns.""" + items: tuple[LevelItem, ...] = () + """The pickups the level holds.""" + lightmaps: tuple[TextureImage, ...] = () + """The baked lighting atlases, addressed by :py:attr:`Corner.lightmap_uv`. Each is an + uncompressed 256 by 256 Targa.""" + + +class Corner(NamedTuple): + """One polygon corner of a level's renderable mesh.""" + + uv: tuple[float, float] + """Texture coordinate, as the game itself stores it.""" + lightmap_uv: tuple[float, float] + """Second coordinate set, addressing the level's lightmap atlases.""" + colour: int + """Packed vertex colour.""" + position: int + """Index into :py:attr:`RenderMesh.positions`.""" + + +class MeshFace(NamedTuple): + """One convex face of a level's renderable mesh.""" + + first_corner: int + """Index of the face's first corner in :py:attr:`RenderMesh.corners`.""" + corner_count: int + """Number of corners the face owns, taken consecutively.""" + material: int + """Identifier into :py:attr:`Level.materials`.""" + normal: Vector3 + """Outward face normal.""" + lightmap: int = -1 + """Index into :py:attr:`Level.lightmaps` of the atlas that lights the face, addressed by + :py:attr:`Corner.lightmap_uv`. Every level's faces cover its atlas range exactly.""" + flags: int = 0 + """The face's rendering flags.""" + + +class StaticMesh(NamedTuple): + """One placed mesh: level architecture, a prop, or anything else built in the editor.""" + + positions: tuple[Vector3, ...] + """Vertex positions in the mesh's own space, before :py:attr:`transform`.""" + normals: tuple[Vector3, ...] + """One normal per position.""" + transform: tuple[float, ...] + """Twelve floats: three basis rows then a translation, placing the mesh in the level.""" + faces: tuple[MeshFace, ...] + """The mesh's faces, whose corners index :py:attr:`RenderMesh.corners`.""" + + +class PropAnimation(NamedTuple): + """One clip a prop can play: a door swinging open, a lift rising, a fan turning.""" + + name: str + """The clip's name, such as ``open1``. A prop's clips chain by name from their scripts.""" + duration: float + """How long the clip runs, in seconds.""" + start: tuple[float, ...] + """The prop's transform at the beginning, as twelve floats.""" + end: tuple[float, ...] + """The prop's transform at the end, as twelve floats.""" + distance: tuple[float, ...] + """How far the prop has travelled, in world units, sampled evenly across :py:attr:`duration`. + Its last sample is the whole distance from :py:attr:`start` to :py:attr:`end`, which holds on + 4701 of the 4704 moving clips in the shipped levels.""" + turn: tuple[float, ...] + """How far the prop has turned, from zero to one, sampled evenly across :py:attr:`duration`. + The two curves are separate channels with their own sample counts, and a prop that both slides + and turns eases them differently.""" + + +class RenderMesh(NamedTuple): + """A level's renderable geometry, carrying the game's own texture coordinates.""" + + corners: tuple[Corner, ...] + """Polygon corners shared by every mesh, each naming a position in its own mesh.""" + meshes: tuple[StaticMesh, ...] + """The placed meshes, in stored order.""" + names: tuple[str, ...] = () + """One editor name per mesh, where the container stores them. The static mesh container keys + its meshes by number and leaves this empty; the dynamic mesh container keys them by name.""" + keys: tuple[int, ...] = () + """One identifier per mesh, where the container keys them by number. The room table and the + BSP's faces both name meshes by these.""" + animations: tuple[tuple[PropAnimation, ...], ...] = () + """The clips each mesh can play, in the same order as :py:attr:`meshes`. Only the animated + props have any.""" + + +class Placement(NamedTuple): + """Where the editor put one thing that is not level architecture. + + ``X_LevelDBLevelObject::read`` gives every placed object the same head: a name, the transform + that places it, a second transform, an identifier, and the room it belongs to. + """ + + name: str + """The editor's name for the placement, such as ``::a5::e1``.""" + transform: tuple[float, ...] + """Twelve floats: three basis rows then a translation.""" + room: str + """Name of the room the object belongs to, empty when it belongs to none.""" + + +class Character(NamedTuple): + """One NPC the level spawns.""" + + placement: Placement + """Where the character stands.""" + skin: str + """Directory name under ``data/database/skins``, such as ``gognitti_vinnie``.""" + + +class LevelItem(NamedTuple): + """One pickup: a weapon, ammunition, or a painkiller bottle.""" + + placement: Placement + """Where the pickup sits.""" + item: str + """Directory name under ``data/database/level_items``, such as ``ammo_ingram``.""" + + +class ModelFace(NamedTuple): + """One triangle of a model.""" + + positions: tuple[int, int, int] + """Indices into :py:attr:`ModelMesh.positions`.""" + coords: tuple[int, int, int] + """Indices into :py:attr:`ModelMesh.coords`, empty-safe when the mesh has none.""" + material: int + """Index into :py:attr:`ModelMesh.materials`.""" + + +class ModelMesh(NamedTuple): + """One mesh of a model.""" + + name: str + """The mesh's name in the exporter that wrote it.""" + positions: tuple[Vector3, ...] + """Vertex positions, converted from the exporter's Z-up space to the game's Y-up one.""" + normals: tuple[Vector3, ...] + """One normal per position, or empty when the file stores none.""" + coords: tuple[tuple[float, float], ...] + """Texture coordinates exactly as stored. Both components run outside ``0..1``, V almost + always negative, so the sampler has to wrap them the way Direct3D did.""" + faces: tuple[ModelFace, ...] + """The mesh's triangles.""" + materials: tuple[str, ...] + """Names of the materials the faces index, as they appear in :py:attr:`Model.materials`.""" + + +class Model(NamedTuple): + """A ``.kfs`` skin or ``.kf2`` object: the models NPCs, pickups and weapons are drawn with.""" + + meshes: tuple[ModelMesh, ...] + """The meshes, in stored order.""" + materials: dict[str, str] + """Material name to the file name of the image it draws with.""" + search: tuple[str, ...] = () + """Directories the model expects its images to be found in, relative to its own.""" + textures: tuple[TextureImage, ...] = () + """The images themselves. A model does not embed them, so a caller that wants the model + textured has to read them off disk and put them here.""" diff --git a/docs/api/index.rst b/docs/api/index.rst index a6967ec..8fa946f 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -25,6 +25,7 @@ and :doc:`../utilities`. incoming jubeatplus marmalade + maxpane monopoly08 rbplus rhythmin diff --git a/docs/api/maxpane.rst b/docs/api/maxpane.rst new file mode 100644 index 0000000..5345087 --- /dev/null +++ b/docs/api/maxpane.rst @@ -0,0 +1,60 @@ +Max Payne +========= + +Readers for the formats Remedy Entertainment built on its ``rl`` library for *Max Payne*: the RAS +archives and MPM mod packages, the seeded stream cipher and LZSS block wrappers guarding them, the +tagged ``R_MemoryFile`` streams every custom asset is written as, and the levels themselves. + +``dade.maxpane.blocks`` +----------------------- + +.. automodule:: dade.maxpane.blocks + :members: + +``dade.maxpane.crypto`` +----------------------- + +.. automodule:: dade.maxpane.crypto + :members: + +``dade.maxpane.decals`` +----------------------- + +.. automodule:: dade.maxpane.decals + :members: + +``dade.maxpane.gltf`` +--------------------- + +.. automodule:: dade.maxpane.gltf + :members: + +``dade.maxpane.ldb`` +-------------------- + +.. automodule:: dade.maxpane.ldb + :members: + +``dade.maxpane.memoryfile`` +--------------------------- + +.. automodule:: dade.maxpane.memoryfile + :members: + +``dade.maxpane.model`` +---------------------- + +.. automodule:: dade.maxpane.model + :members: + +``dade.maxpane.ras`` +-------------------- + +.. automodule:: dade.maxpane.ras + :members: + +``dade.maxpane.typing`` +----------------------- + +.. automodule:: dade.maxpane.typing + :members: diff --git a/docs/index.rst b/docs/index.rst index e31ee3d..f478425 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,8 +12,8 @@ dade The games currently supported are *Incoming* (PC and Dreamcast), *Amplitude* and *FreQuency* (PS2), *Tone Sphere*, *jubeat plus* and *pop'n rhythmin* (iOS), any Marmalade SDK title, BitRock / InstallBuilder installers, *Monopoly* (2008), *Extreme-G* and *Extreme-G 2* (N64 and - PC), *Interstate '76*, *The Sopranos: Road to Respect* (PS2), and *Tony Hawk's Pro Skater 2* - (PC). + PC), *Interstate '76*, *Max Payne*, *The Sopranos: Road to Respect* (PS2), and *Tony Hawk's Pro + Skater 2* (PC). .. toctree:: :caption: Command reference diff --git a/tests/common/test_tools.py b/tests/common/test_tools.py new file mode 100644 index 0000000..118e0df --- /dev/null +++ b/tests/common/test_tools.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from dade.common.context import using_tool_paths +from dade.common.tools import run_unshield + +if TYPE_CHECKING: + from pathlib import Path + + from pytest_mock import MockerFixture + import pytest + + +def test_run_unshield_with_lib(tmp_path: Path, mocker: MockerFixture, + monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv('LD_LIBRARY_PATH', raising=False) + build = tmp_path / 'build' + (build / 'src').mkdir(parents=True) + (build / 'lib').mkdir() + binary = build / 'src' / 'unshield' + binary.write_text('x') + run = mocker.patch('dade.common.tools.sp.run') + with using_tool_paths({'unshield': binary}): + run_unshield(tmp_path / 'DATA1.CAB', tmp_path / 'out') + env = run.call_args.kwargs['env'] + assert env['LD_LIBRARY_PATH'] == str(build / 'lib') + + +def test_run_unshield_lib_appends(tmp_path: Path, mocker: MockerFixture, + monkeypatch: pytest.MonkeyPatch) -> None: + build = tmp_path / 'build' + (build / 'src').mkdir(parents=True) + (build / 'lib').mkdir() + binary = build / 'src' / 'unshield' + binary.write_text('x') + monkeypatch.setenv('LD_LIBRARY_PATH', '/existing') + run = mocker.patch('dade.common.tools.sp.run') + with using_tool_paths({'unshield': binary}): + run_unshield(tmp_path / 'c.cab', tmp_path / 'out') + assert run.call_args.kwargs['env']['LD_LIBRARY_PATH'].endswith(':/existing') + + +def test_run_unshield_no_lib(tmp_path: Path, mocker: MockerFixture) -> None: + binary = tmp_path / 'unshield' + binary.write_text('x') + run = mocker.patch('dade.common.tools.sp.run') + with using_tool_paths({'unshield': binary}): + run_unshield(tmp_path / 'c.cab', tmp_path / 'out') + run.assert_called_once() diff --git a/tests/incoming/test_tools.py b/tests/incoming/test_tools.py index 2a88a63..db4b9bc 100644 --- a/tests/incoming/test_tools.py +++ b/tests/incoming/test_tools.py @@ -5,7 +5,7 @@ import pytest from dade.common.context import using_tool_paths -from dade.incoming.tools import ToolNotFoundError, find_spvr2png, run_gdiextract, run_unshield +from dade.incoming.tools import ToolNotFoundError, find_spvr2png, run_gdiextract if TYPE_CHECKING: from pathlib import Path @@ -44,44 +44,6 @@ def test_locate_not_found(mocker: MockerFixture) -> None: find_spvr2png() -def test_run_unshield_with_lib(tmp_path: Path, mocker: MockerFixture, - monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.delenv('LD_LIBRARY_PATH', raising=False) - build = tmp_path / 'build' - (build / 'src').mkdir(parents=True) - (build / 'lib').mkdir() - binary = build / 'src' / 'unshield' - binary.write_text('x') - run = mocker.patch('dade.incoming.tools.sp.run') - with using_tool_paths({'unshield': binary}): - run_unshield(tmp_path / 'DATA1.CAB', tmp_path / 'out') - env = run.call_args.kwargs['env'] - assert env['LD_LIBRARY_PATH'] == str(build / 'lib') - - -def test_run_unshield_lib_appends(tmp_path: Path, mocker: MockerFixture, - monkeypatch: pytest.MonkeyPatch) -> None: - build = tmp_path / 'build' - (build / 'src').mkdir(parents=True) - (build / 'lib').mkdir() - binary = build / 'src' / 'unshield' - binary.write_text('x') - monkeypatch.setenv('LD_LIBRARY_PATH', '/existing') - run = mocker.patch('dade.incoming.tools.sp.run') - with using_tool_paths({'unshield': binary}): - run_unshield(tmp_path / 'c.cab', tmp_path / 'out') - assert run.call_args.kwargs['env']['LD_LIBRARY_PATH'].endswith(':/existing') - - -def test_run_unshield_no_lib(tmp_path: Path, mocker: MockerFixture) -> None: - binary = tmp_path / 'unshield' - binary.write_text('x') - run = mocker.patch('dade.incoming.tools.sp.run') - with using_tool_paths({'unshield': binary}): - run_unshield(tmp_path / 'c.cab', tmp_path / 'out') - run.assert_called_once() - - def test_run_gdiextract(tmp_path: Path, mocker: MockerFixture) -> None: binary = tmp_path / 'gdiextract' binary.write_text('x') diff --git a/tests/maxpane/conftest.py b/tests/maxpane/conftest.py new file mode 100644 index 0000000..0546dc9 --- /dev/null +++ b/tests/maxpane/conftest.py @@ -0,0 +1,659 @@ +"""Fixtures for the Max Payne tests.""" +from __future__ import annotations + +from itertools import starmap +from typing import TYPE_CHECKING +import math +import struct +import zlib + +import pytest + +from dade.maxpane.crypto import next_seed + +if TYPE_CHECKING: + from collections.abc import Callable, Sequence + +_ARCHIVER_ID = 3 +_SYSTEMTIME = struct.pack('<8H', 2001, 7, 3, 11, 17, 54, 28, 0) +_LEVEL_VERSION = 32 +"""The only level format version the game reads. `X_LevelDBExportLevel::vf00` throws on any +other.""" + + +def _encrypt(data: bytes, seed: int) -> bytes: + """ + Invert :py:func:`dade.maxpane.crypto.decrypt` so tests can build archives. + + Parameters + ---------- + data : bytes + Plaintext. + seed : int + Signed cipher seed. + + Returns + ------- + bytes + Ciphertext that decrypts back to *data*. + """ + if seed == 0: + seed = 1 + out = bytearray(len(data)) + for index, byte in enumerate(data): + seed = next_seed(seed) + rotated = ((byte - (seed & 0xFF)) & 0xFF) ^ ((((index & 0xFF) + 3) & 0xFF) * 6 & 0xFF) + rotation = index % 5 + out[index] = (((rotated >> rotation) | (rotated << + (8 - rotation))) & 0xFF if rotation else rotated) + return bytes(out) + + +@pytest.fixture +def encrypt_ras() -> Callable[[bytes, int], bytes]: + """ + Expose the archive cipher's inverse. + + Returns + ------- + collections.abc.Callable[[bytes, int], bytes] + A callable taking plaintext and a seed and returning ciphertext. + """ + return _encrypt + + +@pytest.fixture +def make_ras() -> Callable[..., bytes]: + """ + Build a RAS archive in memory. + + Returns + ------- + collections.abc.Callable[..., bytes] + A callable returning a complete archive. + """ + def build(members: Sequence[tuple[str, bytes]] = (('a.txt', b'hello'), ('b.bin', b'world')), + *, + directories: Sequence[str] = ('\\', '\\data\\'), + directory: int | None = None, + modified: bool = True, + seed: int = 0x1234, + terminate: bool = True, + version: float = 1.2) -> bytes: + stamp = _SYSTEMTIME if modified else bytes(16) + # Every member sits in the last directory unless the caller names one, which is how an + # index the archive cannot honour gets written. + named = len(directories) - 1 if directory is None else directory + file_table = bytearray() + for name, payload in members: + # A name with no terminator is what a corrupt table looks like from the reader's side. + file_table += name.encode() + (b'\x00' if terminate else b'') + file_table += struct.pack('<6I', len(payload), len(payload), 0, named, 0, _ARCHIVER_ID) + file_table += stamp + directory_table = bytearray() + for name in directories: + directory_table += name.encode() + b'\x00' + stamp + fields = struct.pack('<4IfIIII', len(members), len(directories), len(file_table), + len(directory_table), version, 0, zlib.crc32(bytes(file_table)), + zlib.crc32(bytes(directory_table)), _ARCHIVER_ID) + return (b'RAS\x00' + struct.pack(' bytes: + return bytes((tag,)) + payload + + +def _int(value: int) -> bytes: + """Write an integer the way ``operator<<`` compacts it.""" + magnitude = abs(value) + for mask, tag, width in ((0xFF800000, 0x02, 4), (0x007F8000, 0x12, 3), (0x00007F80, 0x13, 2)): + if magnitude & mask: + return _tag(tag, value.to_bytes(width, 'little', signed=True)) + return _tag(0x14, value.to_bytes(1, 'little', signed=True)) + + +def _vec3(x: float, y: float, z: float) -> bytes: + return _tag(0x16, struct.pack('<3f', x, y, z)) + + +def _vec2(u: float, v: float) -> bytes: + return _tag(0x15, struct.pack('<2f', u, v)) + + +def _string(text: str) -> bytes: + return _tag(0x0D) + _int(len(text)) + text.encode('latin-1') + + +_IDENTITY = (1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0) + +_HALF_TURN = (-1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, -1.0) +"""Half a turn about the up axis. Its trace is negative, which is the case a quaternion has to +pivot on rather than take straight.""" +_REFLECTED = (-1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0) +"""A left-handed basis, which no quaternion can hold on its own.""" +_TILT = (math.cos(math.radians(2)), 0.0, -math.sin(math.radians(2)), 0.0, 1.0, 0.0, + math.sin(math.radians(2)), 0.0, math.cos(math.radians(2))) +"""Two degrees about the up axis: far enough to be worth a channel, close enough that walking the +arc between the two rotations would divide by nearly zero.""" +_STRETCHED = (2.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0) +"""Twice as wide, with no rotation. A clip ending here moves nothing a channel can carry.""" + + +def _matrix(translation: tuple[float, float, float] = (0.0, 0.0, 0.0), + basis: Sequence[float] = _IDENTITY) -> bytes: + return _tag(0x1A, struct.pack('<12f', *basis, *translation)) + + +def _placement(name: str = '', translation: tuple[float, float, float] = (0.0, 0.0, 0.0)) -> bytes: + """Write the head every placed object shares: name, two transforms, an identifier, a room.""" + return _string(name) + _matrix(translation) + _matrix() + _int(0) + _string('') + + +def _string_array(values: Sequence[str] = ('line',)) -> bytes: + return _tag(0x1C) + _int(len(values)) + b''.join(_string(v) for v in values) + + +def _properties() -> bytes: + """0x005e7c00: a string array, a map of arrays, then another string array.""" + return (_string_array() + _tag(0x1F) + _int(1) + _string('key') + _string_array() + + _string_array()) + + +def _properties_map() -> bytes: + """0x005e9c70: a map whose values are whole property bags.""" + return _tag(0x1F) + _int(1) + _string('state') + _properties() + + +def _object_container(entries: Sequence[bytes]) -> bytes: + return _int(len(entries)) + b''.join(entries) + + +def _placement_containers(characters: Sequence[tuple[str, str, tuple[float, float, float]]], + items: Sequence[tuple[str, str, tuple[float, float, float]]], + props: bytes, + *, + corrupt: str = '') -> bytes: + """ + Write the containers the level keeps after its static meshes. + + ``X_LevelDBExportLevel::vf00`` takes them in this order: static lights, start points, state + machines, characters, triggers, animated props, then pickups. + + Parameters + ---------- + characters : collections.abc.Sequence[tuple[str, str, tuple[float, float, float]]] + Key, skin and position per NPC. + items : collections.abc.Sequence[tuple[str, str, tuple[float, float, float]]] + Key, item name and position per pickup. + props : bytes + An encoded dynamic mesh container. + corrupt : str + ``'placements'`` writes a count no level could hold, so the walk has to abandon the tail. + + Returns + ------- + bytes + The encoded containers. + """ + out = bytearray() + if corrupt == 'placements': + return bytes(_tag(0x02, struct.pack(' bytes: + """ + Write the exits, which are what say how the rooms fit together. + + Parameters + ---------- + exits : collections.abc.Sequence[tuple[str, str, tuple[float, float, float]]] + This exit's name, its partner's name, and the translation into the partner's room. + + Returns + ------- + bytes + The encoded container. + """ + out = bytearray(_int(len(exits))) + for name, partner, translation in exits: + out += _string(name) + out += _tag(0x11, b'\x04') + for corner in ((0.0, 0.0, 0.0), (1.0, 0.0, 0.0), (1.0, 1.0, 0.0), (0.0, 1.0, 0.0)): + out += _vec3(*corner) + out += _vec3(0.0, 0.0, 1.0) + out += _matrix(translation) + out += _int(0) + _int(1) + out += _string(partner) + out += _tag(0x1C) + _int(1) + _tag(0x1C) + _int(4) + b''.join(_int(i) for i in range(4)) + return bytes(out) + + +def _room_container(rooms: Sequence[tuple[int, Sequence[int], str]]) -> bytes: + """ + Write the point light container, then the rooms. + + Parameters + ---------- + rooms : collections.abc.Sequence[tuple[int, collections.abc.Sequence[int], str]] + Each room's key, the mesh keys it owns, and its name. + + Returns + ------- + bytes + Both encoded containers. + """ + # One static point light, so the walk has to step over a placement and six floats. + out = bytearray(_int(1) + _int(0) + _placement() + _tag(0x09, struct.pack(' bytes: + """ + Build a static mesh container holding meshes of independent triangles. + + Parameters + ---------- + triangles : int + Number of triangles per mesh. The container needs enough corners to be recognised, so keep + this comfortably above the reader's minimum. + materials : collections.abc.Sequence[int] + Material identifier per triangle, cycled if shorter than *triangles*. + mesh_count : int | None + Override the mesh count written ahead of the meshes, to exercise the reader's guard. + meshes : int + Number of meshes actually written. They share the one corner array. + extra : collections.abc.Sequence[int] + Keys for the map of extra vectors that closes each face array. + layout : str + ``'wind_back'`` stores a normal that opposes the corner order, so the exporter has to + reverse each fan. ``'stacked'`` puts every triangle in one plane on top of the last instead + of one above the other, which is how a level lays graffiti and signage over a wall. + ``'collinear'`` puts every triangle's corners on one line, so no triangle says which way + its face points. + corrupt : str + ``'faces'`` points the last face past the end of the corner array, ``'corners'`` points the + last corner past the end of the position pool. Both are exporter guards, not reader ones. + + Returns + ------- + bytes + The encoded container. + """ + out = bytearray(_corner_pool(triangles * 3, corrupt=corrupt)) + out += _int(meshes if mesh_count is None else mesh_count) + for mesh in range(meshes): + out += _int(mesh) # Its key. + out += _mesh_body(triangles, materials, mesh, extra=extra, layout=layout, corrupt=corrupt) + return bytes(out) + + +def _corner_pool(corners: int, *, corrupt: str = '') -> bytes: + out = bytearray(_tag(0x1C) + _int(corners)) + for index in range(corners): + position = corners + 1 if corrupt == 'corners' and index == corners - 1 else index + out += (_int(position) + _vec2(index / max(corners, 1), 0.5) + _vec2(0.0, 0.0) + + _tag(0x11, b'\xff') + _tag(0x0E, b'\x00')) + return bytes(out) + + +def _mesh_body( + triangles: int, + materials: Sequence[int], + mesh: int = 0, + *, + extra: Sequence[int] = (3,), + layout: str = '', + corrupt: str = '', + lightmap: int = 0, + translation: tuple[float, float, float] = (0.0, 0.0, 0.0)) -> bytes: + """One mesh's positions, normals, transform and faces, without its key.""" + corners = triangles * 3 + # After the exporter's depth mirror each triangle's corners wind about +Y, so a +Y normal + # leaves the fan as written and a -Y one forces it to be reversed. + normal = (0.0, -1.0, 0.0) if layout == 'wind_back' else (0.0, 1.0, 0.0) + # A collinear face has no side to face, which is what the exporter's winding has to survive. + corner = (((0.0, 0.0, 0.0), (1.0, 0.0, 0.0), (2.0, 0.0, 0.0)) if layout == 'collinear' else + ((0.0, 0.0, 0.0), (1.0, 0.0, 0.0), (0.0, 0.0, 1.0))) + out = bytearray(_int(corners)) + for index in range(corners): + x, y, z = corner[index % 3] + out += _vec3(x + mesh, 0.0 if layout == 'stacked' else float(index // 3), z + y) + out += _tag(0x1C) + _int(corners) + for _ in range(corners): + out += _vec3(0.0, 0.0, 1.0) + out += _matrix(translation) + out += _int(triangles) + for index in range(triangles): + first = corners if corrupt == 'faces' and index == triangles - 1 else index * 3 + out += (_int(index) + _int(first) + _int(3) + _vec3(*normal) + _tag(0x11, b'\x01') + + _int(materials[index % len(materials)]) + _int(lightmap) + + _tag(0x09, struct.pack(' bytes: + """ + 0x005f56e0: a duration, two transforms, three script arrays, then two float curves. + + The first curve is the distance travelled in world units and the second is how far the prop has + turned, from nought to one; both run straight from one end to the other here. + """ + travel = math.dist(start, end) + out = bytearray(_tag(0x09, struct.pack(' bytes: + """ + Build the container holding the animated props. + + Parameters + ---------- + props : collections.abc.Sequence[tuple[str, tuple[float, float, float]]] + Name and placement per prop. + materials : collections.abc.Sequence[int] + Material identifier per triangle, cycled. + triangles : int + Triangles per prop. + animations : int + Animations attached to each prop. + swing : tuple[float, float, float] + Where each clip ends, relative to where the prop starts. Leave it at the origin for a clip + that moves nothing, which the exporter drops. + spin : collections.abc.Sequence[float] + The basis each clip ends on. + samples : int + Samples in each clip's curve. + + Returns + ------- + bytes + The encoded container. + """ + out = bytearray(_corner_pool(triangles * 3)) + out += _int(len(props)) + for name, translation in props: + out += _string(name) + out += _mesh_body(triangles, materials, translation=translation) + out += _placement(name, translation) + out += _int(animations) + # A clip's poses are absolute, not relative: the shipped door's start matrix carries the + # same translation as the prop's own placement. + moved = (translation[0] + swing[0], translation[1] + swing[1], translation[2] + swing[2]) + for index in range(animations): + out += _string(f'clip{index}') + _animation( + translation, moved, samples=samples, spin=spin) + out += _tag(0x0E, b'\x01') * 6 + out += _int(0) * 4 + return bytes(out) + + +@pytest.fixture +def bases() -> dict[str, Sequence[float]]: + """ + Expose the bases a clip can end on. + + Returns + ------- + dict[str, collections.abc.Sequence[float]] + ``half_turn`` rotates the prop about the up axis, ``tilt`` barely does, ``reflected`` is + left-handed, and ``stretched`` only changes the prop's size. + """ + return { + 'half_turn': _HALF_TURN, + 'reflected': _REFLECTED, + 'stretched': _STRETCHED, + 'tilt': _TILT + } + + +@pytest.fixture +def make_mesh_container() -> Callable[..., bytes]: + """ + Build a static mesh container on its own. + + Returns + ------- + collections.abc.Callable[..., bytes] + A callable returning the encoded container. + """ + return _static_mesh_container + + +@pytest.fixture +def make_ldb() -> Callable[..., bytes]: + """ + Build a minimal level in memory. + + Returns + ------- + collections.abc.Callable[..., bytes] + A callable returning a decompressed ``.ldb``. + """ + def build(*, + faces: Sequence[Sequence[tuple[float, float, float]]] = ( + ((0.0, 0.0, 0.0), (1.0, 0.0, 0.0), (1.0, 0.0, 1.0), (0.0, 0.0, 1.0)), + ((0.0, 2.0, 0.0), (1.0, 2.0, 0.0), (1.0, 2.0, 1.0)), + ), + mesh_indices: Sequence[int] = (7, 9), + textures: Sequence[tuple[str, int, bytes]] = (('C:\\A.TGA', 0, b'\x00\x01\x02'),), + materials: Sequence[tuple[int, str, + str]] = ((7, 'wood', 'A.TGA'), (9, 'metal', 'B.JPG')), + triangles: int = 40, + face_materials: Sequence[int] = (7, 9), + categories: Sequence[tuple[str, Sequence[tuple[str, str, str]]]] | None = None, + lightmaps: Sequence[tuple[int, int, bytes]] = (), + complete: bool = True, + bsp: tuple[int, int] = (0, 0), + meshes: int = 1, + layout: str = '', + corrupt: str = '', + characters: Sequence[tuple[str, str, tuple[float, float, + float]]] = (('::room::e1', 'transit_cop', + (1.0, 2.0, 3.0)),), + items: Sequence[tuple[str, str, tuple[float, float, + float]]] = (('::room::ammo', 'ammo_ingram', + (4.0, 5.0, 6.0)),), + props: Sequence[tuple[str, tuple[float, float, + float]]] = (('::room::door.DO', (7.0, 8.0, 9.0)),), + motion: tuple[tuple[float, float, float], Sequence[float], + int] = ((0.0, 0.0, 0.0), _IDENTITY, 2), + placements: bool = True, + world: tuple[Sequence[tuple[str, str, tuple[float, float, float]]], + Sequence[tuple[int, Sequence[int], str]]] = ((), ((0, (0,), '::room'),)), + junk: bytes = b'') -> bytes: + out = bytearray() + vertices = [corner for face in faces for corner in face] + out += _tag(0x1C) + _int(len(vertices)) + for corner in vertices: + out += _vec3(*corner) + out += _tag(0x1C) + _int(len(faces)) + first = 0 + for index, (face, mesh) in enumerate(zip(faces, mesh_indices, strict=True)): + out += _int(first) + _int(len(face)) + _int(index) + _int(mesh) + out += _vec3(0.0, 1.0, 0.0) + _vec3(*face[0]) + first += len(face) + out += _tag(0x1C) + _int(bsp[0]) + for _ in range(bsp[0]): + out += _vec3(0.0, 1.0, 0.0) + _vec3(0.0, 0.0, 0.0) + out += b''.join(_int(0) for _ in range(6)) + out += _tag(0x1C) + _int(bsp[1]) + out += b''.join(_int(0) for _ in range(bsp[1])) + out += _int(_LEVEL_VERSION) + out += _int(len(textures)) + for path, kind, blob in textures: + out += _string(path) + _int(kind) + _int(len(blob)) + blob + if not complete: + return bytes(out) + out += _tag(0x1F) + _int(len(materials)) + for key, category, texture in materials: + out += _int(key) + _tag(0x25) + _string(category) + _string(texture) + out += _tag(0x1F) + _int(len(materials)) + for key, category, texture in materials: + out += _tag(0x25) + _string(category) + _string(texture) + _int(key) + if categories is None: + # A level's category table is what ties a material's name to an embedded image, so by + # default give every material an entry naming the texture whose basename matches. + by_base = {p.replace('\\', '/').rsplit('/', 1)[-1].lower(): p for p, _, _ in textures} + categories = [(category, ((texture, by_base.get(texture.lower(), ''), ''),)) + for _, category, texture in materials] + out += _int(len(categories)) + for category, entries in categories: + out += _string(category) + _int(len(entries)) + for a, b, c in entries: + out += _string(a) + _string(b) + _string(c) + _tag(0x0E, b'\x00') * 2 + out += _int(len(lightmaps)) + for key, kind, blob in lightmaps: + out += _int(key) + _int(kind) + _int(len(blob)) + blob + out += junk + if triangles and face_materials: + out += _exit_container(world[0]) + out += _static_mesh_container(triangles, + face_materials, + meshes=meshes, + layout=layout, + corrupt=corrupt) + if placements: + out += _placement_containers(characters, + items, + _dynamic_mesh_container(props, + face_materials or (0,), + samples=motion[2], + spin=motion[1], + swing=motion[0]), + corrupt=corrupt) + out += _room_container(world[1]) + return bytes(out) + + return build + + +def _chunk(identifier: int, version: int, body: bytes) -> bytes: + return _tag(0x0C) + struct.pack('<3I', identifier, version, len(body) + 13) + body + + +def _model_material(name: str, image: str) -> bytes: + """One entry of a model's material library, with the texture chunk it ends on.""" + body = _string(name) + _tag(0x0E, b'\x00') * 5 + _int(2) + _int(0) + _int(0) + body += b''.join(_tag(0x09, struct.pack(' bytes: + return _chunk(0x00010008, 0, _int(len(indices)) + b''.join(_int(i) for i in indices)) + + +@pytest.fixture +def make_model() -> Callable[..., bytes]: + """ + Build a model in memory, in either of the two encodings. + + Returns + ------- + collections.abc.Callable[..., bytes] + A callable returning a decompressed ``.kfs`` or ``.kf2``. + """ + def build(*, + packed: bool = False, + name: str = 'body', + positions: Sequence[tuple[float, float, float]] = ((0.0, 0.0, 0.0), (1.0, 0.0, 0.0), + (0.0, 0.0, 1.0)), + faces: Sequence[Sequence[int]] = ((0, 1, 2),), + coords: Sequence[tuple[float, float, + float]] = ((0.0, 0.0, 0.0), (1.0, 0.0, 0.0), (0.0, -1.0, 0.0)), + coord_faces: Sequence[Sequence[int]] = ((0, 1, 2),), + materials: Sequence[tuple[str, str]] = (('Skin', 'skin.png'),), + face_materials: Sequence[int] = (0,), + search: str = 'textures;..\\sharedtextures') -> bytes: + library = _string(search) + _int(len(materials)) + library += b''.join(starmap(_model_material, materials)) + if packed: + vertices = _tag(0x11, bytes((len(positions),))) + vertices += b''.join(struct.pack('<3f', *p) for p in positions) + vertices += b''.join(struct.pack('<3f', 0.0, 0.0, 1.0) for _ in positions) + flat = [i for face in faces for i in face] + triangles = _tag(0x10, struct.pack(' bytes: + stream = make_lzss(payload) + return b'RA->' + struct.pack(' bytes: + return b'RC->' + struct.pack(' None: + assert is_compressed(b'RA->1234') + assert not is_compressed(b'RC->1234') + + +def test_is_encrypted() -> None: + assert is_encrypted(b'RC->1234') + assert not is_encrypted(b'RA->1234') + + +def test_decompress(make_lzss: Callable[[bytes], bytes]) -> None: + payload = b'Nothing to lose. ' * 9 + assert decompress(_compressed(payload, make_lzss)) == payload + + +def test_decompress_rejects_a_foreign_block() -> None: + with pytest.raises(ValueError, match='Not a compressed block'): + decompress(b'RC->' + bytes(16)) + + +def test_decompress_primes_the_ring_with_spaces() -> None: + # A match reaching back before any literal reads the primed ring, which is the only place the + # fill byte is observable. + stream = bytes((0xFE, 0x00, 0x00)) + block = b'RA->' + struct.pack(' None: + payload = b'a poem about a lightbulb' + assert decrypt_block(_encrypted(payload, 0x2A, encrypt_ras)) == payload + + +def test_decrypt_block_rejects_a_foreign_block() -> None: + with pytest.raises(ValueError, match='Not an encrypted block'): + decrypt_block(b'RA->' + bytes(16)) + + +def test_unwrap_peels_compression(make_lzss: Callable[[bytes], bytes]) -> None: + payload = b'a bullet time' * 4 + data, layers = unwrap(_compressed(payload, make_lzss)) + assert data == payload + assert layers == ('lzss',) + + +def test_unwrap_peels_encryption(encrypt_ras: Callable[[bytes, int], bytes]) -> None: + payload = b'the flesh of fallen angels' * 2 + data, layers = unwrap(_encrypted(payload, 0x5150, encrypt_ras)) + assert data == payload + assert layers == ('crypt',) + + +def test_unwrap_peels_both(make_lzss: Callable[[bytes], bytes], + encrypt_ras: Callable[[bytes, int], bytes]) -> None: + payload = b'a graphic novel' * 8 + nested = _compressed(payload, make_lzss) + data, layers = unwrap(_encrypted(nested, 0x99, encrypt_ras)) + assert data == payload + assert layers == ('crypt', 'lzss') + + +def test_unwrap_leaves_plain_data_alone() -> None: + assert unwrap(b'no wrapper here at all') == (b'no wrapper here at all', ()) + + +def test_unwrap_stops_on_an_encrypted_block_too_short_to_read() -> None: + # Long enough to be a compressed block, too short to be the encrypted one it claims to be. + data = b'RC->' + bytes(8) + assert unwrap(data) == (data, ()) diff --git a/tests/maxpane/test_commands.py b/tests/maxpane/test_commands.py new file mode 100644 index 0000000..87afafd --- /dev/null +++ b/tests/maxpane/test_commands.py @@ -0,0 +1,387 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING +import json +import struct +import subprocess as sp + +import pytest + +from dade.common.tools import ToolNotFoundError +from dade.maxpane.commands.inspect_tags import inspect_tags +from dade.maxpane.commands.ldb2glb import ldb2glb +from dade.maxpane.commands.ldb_textures import ldb_textures +from dade.maxpane.commands.ras_extract import ras_extract +from dade.maxpane.commands.ras_list import ras_list +from dade.maxpane.commands.sources import NoArchivesFoundError, iter_archives +from dade.maxpane.main import cli +from dade.maxpane.memoryfile import BasicType + +if TYPE_CHECKING: + from collections.abc import Callable, Iterator, Mapping + from pathlib import Path + + from click.testing import CliRunner + from pytest_mock import MockerFixture + + +class _StubImage: + """Stands in for an ISO 9660 image so the source walk is tested without one.""" + def __init__(self, files: Mapping[str, bytes]) -> None: + self._files = dict(files) + + def iter_files(self) -> Iterator[tuple[str, int]]: + """ + Yield each file's path and size. + + Yields + ------ + tuple[str, int] + The path and size, sorted by path. + """ + for path in sorted(self._files): + yield path, len(self._files[path]) + + def read_file(self, path: str) -> bytes: + """ + Return a file's contents. + + Parameters + ---------- + path : str + The path to read. + + Returns + ------- + bytes + The contents. + """ + return self._files[path] + + +def test_cli_lists_its_commands(runner: CliRunner) -> None: + result = runner.invoke(cli, ('--help',)) + assert result.exit_code == 0 + assert 'ras-extract' in result.output + assert 'ras-list' in result.output + + +def test_ras_list(runner: CliRunner, tmp_path: Path, make_ras: Callable[..., bytes]) -> None: + archive = tmp_path / 'x_data.ras' + archive.write_bytes(make_ras()) + result = runner.invoke(ras_list, (str(archive),)) + assert result.exit_code == 0 + assert 'v1.20, 2 members in 2 directories, intact' in result.output + assert 'data/a.txt' in result.output + + +def test_ras_list_json(runner: CliRunner, tmp_path: Path, make_ras: Callable[..., bytes]) -> None: + archive = tmp_path / 'x_data.ras' + archive.write_bytes(make_ras()) + result = runner.invoke(ras_list, (str(archive), '--json')) + assert result.exit_code == 0 + assert json.loads(result.output)['x_data.ras'][0]['path'] == 'data/a.txt' + + +def test_ras_list_reports_truncation(runner: CliRunner, tmp_path: Path, + make_ras: Callable[..., bytes]) -> None: + archive = tmp_path / 'x_data.ras' + archive.write_bytes(make_ras()[:-1]) + assert 'TRUNCATED' in runner.invoke(ras_list, (str(archive),)).output + + +def test_ras_list_rejects_a_foreign_file(runner: CliRunner, tmp_path: Path) -> None: + other = tmp_path / 'not.ras' + other.write_bytes(b'\x00' * 4096) + result = runner.invoke(ras_list, (str(other),)) + assert result.exit_code != 0 + + +def test_ras_extract(runner: CliRunner, tmp_path: Path, make_ras: Callable[..., bytes]) -> None: + archive = tmp_path / 'x_data.ras' + archive.write_bytes(make_ras()) + out = tmp_path / 'out' + result = runner.invoke(ras_extract, (str(archive), '-o', str(out))) + assert result.exit_code == 0 + assert (out / 'data' / 'a.txt').read_bytes() == b'hello' + assert (out / 'data' / 'b.bin').read_bytes() == b'world' + + +def test_ras_extract_filters_by_pattern(runner: CliRunner, tmp_path: Path, + make_ras: Callable[..., bytes]) -> None: + archive = tmp_path / 'x_data.ras' + archive.write_bytes(make_ras()) + out = tmp_path / 'out' + result = runner.invoke(ras_extract, (str(archive), '*.txt', '-o', str(out))) + assert result.exit_code == 0 + assert (out / 'data' / 'a.txt').is_file() + assert not (out / 'data' / 'b.bin').exists() + + +def test_ras_extract_rejects_a_foreign_file(runner: CliRunner, tmp_path: Path) -> None: + other = tmp_path / 'not.ras' + other.write_bytes(b'\x00' * 4096) + assert runner.invoke(ras_extract, (str(other), '-o', str(tmp_path / 'out'))).exit_code != 0 + + +def test_inspect_tags(runner: CliRunner, tmp_path: Path) -> None: + asset = tmp_path / 'level.ldb' + asset.write_bytes( + bytes((BasicType.ARRAY,)) + bytes((BasicType.INT16,)) + b'\x01\x00' + + bytes((BasicType.VECTOR3,)) + struct.pack('<3f', 1.0, 2.0, 3.0)) + result = runner.invoke(inspect_tags, (str(asset),)) + assert result.exit_code == 0 + assert 'ARRAY' in result.output + assert 'VECTOR3' in result.output + assert 'walked to 17 (100.00%)' in result.output + + +def test_inspect_tags_reports_where_it_stopped(runner: CliRunner, tmp_path: Path) -> None: + asset = tmp_path / 'level.ldb' + asset.write_bytes(bytes((BasicType.INT8,)) + b'\x01' + bytes((BasicType.STRING,)) + b'name') + result = runner.invoke(inspect_tags, (str(asset),)) + assert result.exit_code == 0 + assert 'Stopped on 0x0d at offset 2.' in result.output + + +def test_inspect_tags_unwraps_first(runner: CliRunner, tmp_path: Path, + make_lzss: Callable[[bytes], bytes]) -> None: + payload = bytes((BasicType.INT8,)) + b'\x07' + stream = make_lzss(payload) + asset = tmp_path / 'level.ldb' + asset.write_bytes(b'RA->' + struct.pack(' None: + asset = tmp_path / 'level.ldb' + asset.write_bytes((bytes((BasicType.INT8,)) + b'\x01') * 5) + result = runner.invoke(inspect_tags, (str(asset), '-n', '2')) + assert result.exit_code == 0 + assert result.output.count('INT8') == 3 # Two listed values plus the histogram line. + + +def test_iter_archives_scans_a_directory_recursively(tmp_path: Path, + make_ras: Callable[..., bytes]) -> None: + (tmp_path / 'levels').mkdir() + (tmp_path / 'levels' / 'x_level1.ras').write_bytes(make_ras()) + (tmp_path / 'mod.MPM').write_bytes(make_ras()) + (tmp_path / 'readme.txt').write_bytes(b'ignored') + assert sorted( + label for label, _ in iter_archives(tmp_path)) == ['levels/x_level1.ras', 'mod.MPM'] + + +def test_iter_archives_raises_when_nothing_is_found(tmp_path: Path) -> None: + (tmp_path / 'readme.txt').write_bytes(b'nothing here') + with pytest.raises(NoArchivesFoundError, match='No RAS archives found'): + list(iter_archives(tmp_path)) + + +def test_iter_archives_reads_loose_archives_from_an_image(tmp_path: Path, mocker: MockerFixture, + make_ras: Callable[..., bytes]) -> None: + archive = make_ras() + mocker.patch('dade.maxpane.commands.sources.open_image', + return_value=_StubImage({ + 'DISK1/LEVELS/X_LEVEL1.RAS': archive, + 'DISK1/SETUP.EXE': b'stub' + })) + image = tmp_path / 'disc.iso' + image.write_bytes(b'not a ras') + assert [label for label, _ in iter_archives(image)] == ['DISK1/LEVELS/X_LEVEL1.RAS'] + + +def test_iter_archives_unshields_a_cabinet_on_an_image(tmp_path: Path, mocker: MockerFixture, + make_ras: Callable[..., bytes]) -> None: + staged: dict[str, list[str]] = {} + + def fake_unshield(cabinet: Path, output_dir: Path) -> None: + staged['siblings'] = sorted(path.name for path in cabinet.parent.iterdir()) + (output_dir / 'x_data.ras').write_bytes(make_ras()) + + mocker.patch('dade.maxpane.commands.sources.run_unshield', side_effect=fake_unshield) + mocker.patch('dade.maxpane.commands.sources.open_image', + return_value=_StubImage({ + 'DISK1/DATA1.CAB': b'ISc(', + 'DISK1/DATA1.HDR': b'hdr', + 'DISK1/DATA2.CAB': b'vol', + 'DISK1/LEVELS/X_LEVEL1.RAS': make_ras(), + 'DISK1/SETUP.EXE': b'stub' + })) + image = tmp_path / 'disc.iso' + image.write_bytes(b'not a ras') + labels = [label for label, _ in iter_archives(image)] + assert labels == ['DISK1/LEVELS/X_LEVEL1.RAS', 'x_data.ras'] + assert staged['siblings'] == ['DATA1.CAB', 'DATA1.HDR', 'DATA2.CAB'] + + +def test_iter_archives_skips_a_cabinet_without_unshield(tmp_path: Path, + mocker: MockerFixture) -> None: + mocker.patch('dade.maxpane.commands.sources.run_unshield', + side_effect=ToolNotFoundError('missing')) + cabinet = tmp_path / 'data1.cab' + cabinet.write_bytes(b'ISc(') + with pytest.raises(NoArchivesFoundError): + list(iter_archives(cabinet)) + + +def test_iter_archives_skips_a_cabinet_unshield_cannot_read(tmp_path: Path, + mocker: MockerFixture) -> None: + mocker.patch('dade.maxpane.commands.sources.run_unshield', + side_effect=sp.CalledProcessError(1, 'unshield')) + cabinet = tmp_path / 'data1.cab' + cabinet.write_bytes(b'ISc(') + with pytest.raises(NoArchivesFoundError): + list(iter_archives(cabinet)) + + +def test_iter_archives_reads_a_cabinet(tmp_path: Path, mocker: MockerFixture, + make_ras: Callable[..., bytes]) -> None: + def fake_unshield(cabinet: Path, output_dir: Path) -> None: + (output_dir / 'x_data.ras').write_bytes(make_ras()) + + mocker.patch('dade.maxpane.commands.sources.run_unshield', side_effect=fake_unshield) + cabinet = tmp_path / 'data1.cab' + cabinet.write_bytes(b'ISc(') + assert [label for label, _ in iter_archives(cabinet)] == ['x_data.ras'] + + +def test_ldb2glb(runner: CliRunner, tmp_path: Path, make_ldb: Callable[..., bytes]) -> None: + level = tmp_path / 'Part1_Level6.ldb' + level.write_bytes(make_ldb()) + out = tmp_path / 'out' + result = runner.invoke(ldb2glb, (str(level), '-o', str(out))) + assert result.exit_code == 0 + assert (out / 'Part1_Level6.glb').read_bytes()[:4] == b'glTF' + assert ('1 meshes, 1 props, 42 faces, 1 images, 2 placements (0 modelled), 0 clips' + in result.output) + + +def test_ldb2glb_draws_the_placements_from_a_database(runner: CliRunner, tmp_path: Path, + make_ldb: Callable[..., bytes], + make_model: Callable[..., bytes]) -> None: + level = tmp_path / 'Part1_Level6.ldb' + level.write_bytes(make_ldb()) + database = tmp_path / 'database' + skin = database / 'skins' / 'transit_cop' + skin.mkdir(parents=True) + (skin / 'transit_cop_l0.kfs').write_bytes(make_model()) + result = runner.invoke(ldb2glb, (str(level), '-o', str(tmp_path / 'out'), '-D', str(database))) + assert result.exit_code == 0 + assert '2 placements (1 modelled)' in result.output + + +def test_ldb2glb_accepts_a_compressed_level(runner: CliRunner, tmp_path: Path, + make_ldb: Callable[..., bytes], + make_lzss: Callable[[bytes], bytes]) -> None: + payload = make_ldb() + stream = make_lzss(payload) + level = tmp_path / 'wrapped.ldb' + level.write_bytes(b'RA->' + struct.pack(' None: + (tmp_path / 'levels').mkdir() + (tmp_path / 'levels' / 'a.ldb').write_bytes(make_ldb()) + result = runner.invoke(ldb2glb, (str(tmp_path), '-o', str(tmp_path / 'out'))) + assert result.exit_code == 0 + assert '1/1 levels converted' in result.output + + +def test_ldb2glb_without_any_levels(runner: CliRunner, tmp_path: Path) -> None: + (tmp_path / 'empty').mkdir() + result = runner.invoke(ldb2glb, (str(tmp_path / 'empty'),)) + assert result.exit_code != 0 + assert 'No .ldb files found.' in result.output + + +def test_ldb2glb_aborts_on_a_bad_level(runner: CliRunner, tmp_path: Path) -> None: + level = tmp_path / 'bad.ldb' + level.write_bytes(b'\x14\x02\x14\x02') + assert runner.invoke(ldb2glb, (str(level), '-o', str(tmp_path / 'o'))).exit_code != 0 + + +def test_ldb2glb_can_skip_a_bad_level(runner: CliRunner, tmp_path: Path, + make_ldb: Callable[..., bytes]) -> None: + (tmp_path / 'good.ldb').write_bytes(make_ldb()) + (tmp_path / 'bad.ldb').write_bytes(b'\x14\x02\x14\x02') + result = runner.invoke(ldb2glb, + (str(tmp_path), '-o', str(tmp_path / 'out'), '--ignore-failures')) + assert result.exit_code == 0 + assert '1/2 levels converted' in result.output + + +def test_ldb_textures(runner: CliRunner, tmp_path: Path, make_ldb: Callable[..., bytes]) -> None: + level = tmp_path / 'a.ldb' + level.write_bytes(make_ldb(textures=(('X:\\PROJECTS\\T\\WALL.TGA', 0, b'\x00\x01'),))) + out = tmp_path / 'tex' + result = runner.invoke(ldb_textures, (str(level), '-o', str(out))) + assert result.exit_code == 0 + assert (out / 'PROJECTS' / 'T' / 'WALL.TGA').read_bytes() == b'\x00\x01' + assert '1 images written' in result.output + + +def test_ldb_textures_flat(runner: CliRunner, tmp_path: Path, make_ldb: Callable[..., + bytes]) -> None: + level = tmp_path / 'a.ldb' + level.write_bytes(make_ldb(textures=(('X:\\PROJECTS\\T\\WALL.TGA', 0, b'\x00\x01'),))) + out = tmp_path / 'tex' + assert runner.invoke(ldb_textures, (str(level), '-o', str(out), '--flat')).exit_code == 0 + assert (out / 'WALL.TGA').is_file() + + +def test_ldb_textures_bare_name(runner: CliRunner, tmp_path: Path, + make_ldb: Callable[..., bytes]) -> None: + level = tmp_path / 'a.ldb' + level.write_bytes(make_ldb(textures=(('C:\\', 0, b'\x00'),))) + out = tmp_path / 'tex' + assert runner.invoke(ldb_textures, (str(level), '-o', str(out))).exit_code == 0 + assert (out / 'texture').is_file() + + +def test_ldb_textures_without_any_levels(runner: CliRunner, tmp_path: Path) -> None: + (tmp_path / 'empty').mkdir() + result = runner.invoke(ldb_textures, (str(tmp_path / 'empty'),)) + assert result.exit_code != 0 + assert 'No .ldb files found.' in result.output + + +def test_ldb_textures_aborts_on_a_bad_level(runner: CliRunner, tmp_path: Path) -> None: + level = tmp_path / 'bad.ldb' + level.write_bytes(b'\x14\x02\x14\x02') + assert runner.invoke(ldb_textures, (str(level), '-o', str(tmp_path / 'o'))).exit_code != 0 + + +def test_ras_extract_refuses_to_write_outside_the_output_directory( + runner: CliRunner, tmp_path: Path, make_ras: Callable[..., bytes]) -> None: + # An archive names its own member paths, and nothing stops one naming its way back out. + archive = tmp_path / 'x_data.ras' + archive.write_bytes(make_ras(directories=('\\', '\\..\\..\\'))) + out = tmp_path / 'out' + result = runner.invoke(ras_extract, (str(archive), '-o', str(out))) + assert result.exit_code == 0 + assert not (tmp_path.parent / 'a.txt').exists() + assert not list(out.rglob('a.txt')) + + +def test_ldb_textures_refuses_a_path_that_climbs_out(runner: CliRunner, tmp_path: Path, + make_ldb: Callable[..., bytes]) -> None: + level = tmp_path / 'a.ldb' + level.write_bytes(make_ldb(textures=(('X:\\..\\..\\WALL.TGA', 0, b'\x00\x01'),))) + out = tmp_path / 'tex' + result = runner.invoke(ldb_textures, (str(level), '-o', str(out))) + assert result.exit_code == 0 + # The upward steps are dropped, so the image lands directly under the output directory. + assert (out / 'WALL.TGA').read_bytes() == b'\x00\x01' + assert not (tmp_path.parent / 'WALL.TGA').exists() + + +def test_inspect_tags_on_an_empty_asset(runner: CliRunner, tmp_path: Path) -> None: + asset = tmp_path / 'empty.bin' + asset.write_bytes(b'') + result = runner.invoke(inspect_tags, (str(asset),)) + assert result.exit_code == 0 + assert '0 bytes, walked to 0 (100.00%)' in result.output diff --git a/tests/maxpane/test_crypto.py b/tests/maxpane/test_crypto.py new file mode 100644 index 0000000..1783d38 --- /dev/null +++ b/tests/maxpane/test_crypto.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from dade.maxpane.crypto import decrypt, next_seed + +if TYPE_CHECKING: + from collections.abc import Callable + + +def test_next_seed_matches_wichmann_hill() -> None: + state = 4242 + for _ in range(64): + expected = 171 * state - 30269 * (state // 177) + state = next_seed(state) + assert state == expected + + +def test_next_seed_wraps_like_the_original() -> None: + assert next_seed(1239061428) == -13999155 + + +def test_decrypt_round_trips(encrypt_ras: Callable[[bytes, int], bytes]) -> None: + payload = bytes(range(256)) * 3 + assert decrypt(encrypt_ras(payload, 0x46AA8D54), 0x46AA8D54) == payload + + +def test_decrypt_promotes_a_zero_seed(encrypt_ras: Callable[[bytes, int], bytes]) -> None: + payload = b'the sudden silence' + assert decrypt(encrypt_ras(payload, 0), 0) == payload + assert decrypt(encrypt_ras(payload, 1), 0) == payload + + +def test_decrypt_is_position_dependent(encrypt_ras: Callable[[bytes, int], bytes]) -> None: + cipher = encrypt_ras(b'\x00' * 16, 99) + assert len(set(cipher)) > 1 + + +def test_decrypt_empty() -> None: + assert decrypt(b'', 7) == b'' diff --git a/tests/maxpane/test_decals.py b/tests/maxpane/test_decals.py new file mode 100644 index 0000000..1b8b95c --- /dev/null +++ b/tests/maxpane/test_decals.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from dade.maxpane.decals import layer_faces + +_FLOOR = ((0.0, 0.0, 0.0), (8.0, 0.0, 0.0), (8.0, 0.0, 8.0), (0.0, 0.0, 8.0)) +_RUG = ((2.0, 0.0, 2.0), (5.0, 0.0, 2.0), (5.0, 0.0, 5.0), (2.0, 0.0, 5.0)) +_UP = (0.0, 1.0, 0.0) + + +def test_layer_faces_leaves_a_lone_face_alone() -> None: + assert layer_faces([(_UP, _FLOOR)]) == [0] + + +def test_layer_faces_lifts_the_smaller_of_two_stacked_faces() -> None: + assert layer_faces([(_UP, _FLOOR), (_UP, _RUG)]) == [0, 1] + + +def test_layer_faces_keeps_the_wider_face_on_its_plane_whichever_comes_first() -> None: + # The rug is given first, but the floor is what the level built and has to stay put. + assert layer_faces([(_UP, _RUG), (_UP, _FLOOR)]) == [1, 0] + + +def test_layer_faces_stacks_three_deep() -> None: + coaster = ((3.0, 0.0, 3.0), (4.0, 0.0, 3.0), (4.0, 0.0, 4.0), (3.0, 0.0, 4.0)) + assert layer_faces([(_UP, _FLOOR), (_UP, _RUG), (_UP, coaster)]) == [0, 1, 2] + + +def test_layer_faces_ignores_faces_that_only_meet_along_an_edge() -> None: + # Architecture is tiled, so neighbouring wall panels share a seam and nothing more. + right = ((8.0, 0.0, 0.0), (16.0, 0.0, 0.0), (16.0, 0.0, 8.0), (8.0, 0.0, 8.0)) + assert layer_faces([(_UP, _FLOOR), (_UP, right)]) == [0, 0] + + +def test_layer_faces_ignores_a_face_on_a_different_plane() -> None: + ceiling = tuple((x, 4.0, z) for x, _y, z in _RUG) + assert layer_faces([(_UP, _FLOOR), (_UP, ceiling)]) == [0, 0] + + +def test_layer_faces_ignores_a_face_turned_the_other_way() -> None: + # A viewer never sees both sides of one surface at once, so back to back is not a clash. + assert layer_faces([(_UP, _FLOOR), ((0.0, -1.0, 0.0), _RUG)]) == [0, 0] + + +def test_layer_faces_skips_a_face_with_no_plane() -> None: + assert layer_faces([((0.0, 0.0, 0.0), _FLOOR), (_UP, ()), (_UP, _FLOOR)]) == [0, 0, 0] + + +def test_layer_faces_handles_a_wall_that_leans_on_z() -> None: + # The in-plane frame picks a different helper axis for a normal that leans on Z. + wall = ((0.0, 0.0, 0.0), (0.0, 8.0, 0.0), (8.0, 8.0, 0.0), (8.0, 0.0, 0.0)) + tag = ((2.0, 2.0, 0.0), (2.0, 5.0, 0.0), (5.0, 5.0, 0.0), (5.0, 2.0, 0.0)) + assert layer_faces([((0.0, 0.0, 1.0), wall), ((0.0, 0.0, 1.0), tag)]) == [0, 1] diff --git a/tests/maxpane/test_gltf.py b/tests/maxpane/test_gltf.py new file mode 100644 index 0000000..cf62cb4 --- /dev/null +++ b/tests/maxpane/test_gltf.py @@ -0,0 +1,623 @@ +from __future__ import annotations + +from io import BytesIO +from typing import TYPE_CHECKING, Any +import json +import struct + +import pytest + +from dade.maxpane.decals import DECAL_STEP +from dade.maxpane.gltf import GLB_MAGIC, build_glb +from dade.maxpane.ldb import read_level +from dade.maxpane.typing import Level, LevelGeometry, Model + +if TYPE_CHECKING: + from collections.abc import Callable + + +def _parse(glb: bytes) -> tuple[dict[str, Any], bytes]: + assert glb[:4] == GLB_MAGIC + version, length = struct.unpack_from(' bytes: + view = document['bufferViews'][document['accessors'][index]['bufferView']] + return binary[view['byteOffset']:view['byteOffset'] + view['byteLength']] + + +def test_build_glb_structure(make_ldb: Callable[..., bytes]) -> None: + document, binary = _parse(build_glb(read_level(make_ldb()), name='demo')) + assert document['asset']['version'] == '2.0' + assert document['scenes'][0]['name'] == 'demo' + assert document['buffers'][0]['byteLength'] == len(binary) + assert document['nodes'][0]['name'] == 'demo_0' + + +def test_build_glb_writes_one_node_per_placed_mesh(make_ldb: Callable[..., bytes]) -> None: + level = read_level(make_ldb(placements=False)) + document, _ = _parse(build_glb(level)) + assert level.mesh is not None + assert len(document['nodes']) == len(level.mesh.meshes) + + +def test_build_glb_places_architecture_by_its_room(make_ldb: Callable[..., bytes]) -> None: + # A room reached through an exit is moved by that exit's transform, and so is its mesh. + document, _ = _parse( + build_glb( + read_level( + make_ldb(meshes=2, + world=((('::room::out', '::far::in', (5.0, 0.0, 0.0)), + ('::far::in', '::room::out', (-5.0, 0.0, 0.0))), + ((0, (0,), '::room'), (1, (1,), '::far'))))))) + assert document['nodes'][0]['matrix'][12:] == [0, 0, -0, 1] + assert document['nodes'][1]['matrix'][12:] == [5, 0, -0, 1] + + +def test_build_glb_mirrors_a_props_transform(make_ldb: Callable[..., bytes]) -> None: + # A prop is placed by its transform, and the depth mirror negates the Z row, column and offset. + document, _ = _parse( + build_glb(read_level(make_ldb(props=(('::room::door.DO', (7.0, 8.0, 9.0)),))))) + door = next(n for n in document['nodes'] if n['name'] == '::room::door.DO') + assert door['matrix'] == [1, 0, -0, 0, 0, 1, -0, 0, -0, -0, 1, 0, 7, 8, -9, 1] + + +def test_build_glb_groups_primitives_by_material(make_ldb: Callable[..., bytes]) -> None: + document, _ = _parse(build_glb(read_level(make_ldb(face_materials=(7, 9))))) + assert len(document['meshes'][0]['primitives']) == 2 + + +def test_build_glb_writes_the_games_texture_coordinates(make_ldb: Callable[..., bytes]) -> None: + level = read_level(make_ldb()) + document, binary = _parse(build_glb(level)) + coord = document['meshes'][0]['primitives'][0]['attributes']['TEXCOORD_0'] + assert document['accessors'][coord]['type'] == 'VEC2' + assert level.mesh is not None + raw = _accessor_bytes(document, binary, coord) + assert struct.unpack_from('<2f', raw, 0) == pytest.approx(level.mesh.corners[0].uv) + + +def test_build_glb_mirrors_positions(make_ldb: Callable[..., bytes]) -> None: + level = read_level(make_ldb()) + document, binary = _parse(build_glb(level)) + position = document['meshes'][0]['primitives'][0]['attributes']['POSITION'] + raw = _accessor_bytes(document, binary, position) + assert level.mesh is not None + original = level.mesh.meshes[0].positions[level.mesh.corners[0].position] + written = struct.unpack_from('<3f', raw, 0) + assert written == pytest.approx((original[0], original[1], -original[2])) + + +def test_build_glb_every_buffer_view_is_aligned(make_ldb: Callable[..., bytes]) -> None: + document, _ = _parse(build_glb(read_level(make_ldb()))) + assert all(view['byteOffset'] % 4 == 0 for view in document['bufferViews']) + + +def test_build_glb_embeds_a_jpeg(make_ldb: Callable[..., bytes]) -> None: + level = read_level( + make_ldb(materials=((7, 'wood', 'A.JPG'), (9, 'metal', 'B.JPG')), + textures=(('C:\\A.JPG', 4, b'\xff\xd8fake'),))) + document, _ = _parse(build_glb(level)) + assert document['images'][0]['mimeType'] == 'image/jpeg' + assert document['materials'][0]['pbrMetallicRoughness']['baseColorTexture']['index'] == 0 + + +def test_build_glb_embeds_a_png(make_ldb: Callable[..., bytes]) -> None: + level = read_level( + make_ldb(materials=((7, 'wood', 'A.PNG'), (9, 'metal', 'B.PNG')), + textures=(('C:\\A.PNG', 0, b'\x89PNGfake'),))) + document, _ = _parse(build_glb(level)) + assert document['images'][0]['mimeType'] == 'image/png' + + +def test_build_glb_converts_a_targa(make_ldb: Callable[..., bytes]) -> None: + from PIL import Image + buffer = BytesIO() + Image.new('RGB', (2, 2), (10, 20, 30)).save(buffer, format='TGA') + level = read_level( + make_ldb(materials=((7, 'wood', 'A.TGA'), (9, 'metal', 'B.TGA')), + textures=(('C:\\A.TGA', 0, buffer.getvalue()),))) + document, _ = _parse(build_glb(level)) + assert document['images'][0]['mimeType'] == 'image/png' + + +def test_build_glb_skips_an_undecodable_image(make_ldb: Callable[..., bytes]) -> None: + document, _ = _parse(build_glb(read_level(make_ldb(textures=(('C:\\A.TGA', 0, b'junk'),))))) + assert 'images' not in document + assert 'baseColorFactor' in document['materials'][0]['pbrMetallicRoughness'] + + +def test_build_glb_without_textures(make_ldb: Callable[..., bytes]) -> None: + document, _ = _parse(build_glb(read_level(make_ldb(textures=())))) + assert 'images' not in document + assert 'samplers' not in document + + +def test_build_glb_names_a_material_after_its_image(make_ldb: Callable[..., bytes]) -> None: + document, _ = _parse(build_glb(read_level(make_ldb()))) + assert document['materials'][0]['name'] == 'A.TGA' + + +def test_build_glb_skips_faces_the_engine_does_not_draw(make_ldb: Callable[..., bytes]) -> None: + hidden = read_level( + make_ldb(materials=((7, 'dummy', 'A.TGA'), (9, 'charactercollision_nodraw', 'B.TGA')), + placements=False)) + document, _ = _parse(build_glb(hidden)) + # Every face is hidden, so the mesh path adds nothing and the BSP fallback takes over. + assert len(document['nodes']) == 1 + assert 'matrix' not in document['nodes'][0] + + +def test_build_glb_writes_the_sky_flat(make_ldb: Callable[..., bytes]) -> None: + # Skybox faces close a level off where it opens to the sky, so dropping them puts a hole + # through it. They are drawn with one flat emissive colour instead of their placeholder image. + document, _ = _parse( + build_glb( + read_level( + make_ldb(materials=((7, 'skybox', 'A.TGA'), (9, 'metal', 'B.JPG')), + placements=False)))) + sky = [m for m in document['materials'] if m['name'] == 'skybox'] + assert len(sky) == 1 + assert 'baseColorTexture' not in sky[0]['pbrMetallicRoughness'] + # Written unlit so it reads the same from every angle rather than shading like a wall. + assert 'KHR_materials_unlit' in sky[0]['extensions'] + assert document['extensionsUsed'] == ['KHR_materials_unlit'] + + +def test_build_glb_falls_back_to_the_bsp_faces(make_ldb: Callable[..., bytes]) -> None: + document, _ = _parse(build_glb(read_level(make_ldb(triangles=0)))) + assert len(document['nodes']) == 1 + assert 'TEXCOORD_0' not in document['meshes'][0]['primitives'][0]['attributes'] + + +def test_build_glb_draws_a_face_whose_material_is_missing(make_ldb: Callable[..., bytes]) -> None: + # Nothing says the face is hidden, so an unknown identifier has to draw rather than vanish. + document, _ = _parse(build_glb(read_level(make_ldb(face_materials=(7, 99))))) + assert len(document['meshes'][0]['primitives']) == 2 + + +def test_build_glb_reuses_a_material_across_meshes(make_ldb: Callable[..., bytes]) -> None: + document, _ = _parse(build_glb(read_level(make_ldb(meshes=2, placements=False)))) + assert len(document['nodes']) == 2 + assert len(document['materials']) == 2 + assert (document['meshes'][0]['primitives'][0]['material'] == document['meshes'][1] + ['primitives'][0]['material']) + + +def test_build_glb_reverses_a_fan_that_faces_the_wrong_way(make_ldb: Callable[..., bytes]) -> None: + forward = _parse(build_glb(read_level(make_ldb(layout='wind_back')))) + reversed_ = _parse(build_glb(read_level(make_ldb()))) + a = _accessor_bytes(*forward, forward[0]['meshes'][0]['primitives'][0]['indices']) + b = _accessor_bytes(*reversed_, reversed_[0]['meshes'][0]['primitives'][0]['indices']) + assert struct.unpack_from('<3I', a, 0)[::-1] == struct.unpack_from('<3I', b, 0) + + +def test_build_glb_drops_a_mesh_whose_face_runs_past_the_corners( + make_ldb: Callable[..., bytes]) -> None: + document, _ = _parse(build_glb(read_level(make_ldb(corrupt='faces', placements=False)))) + # The mesh is abandoned, so only the BSP fallback node survives. + assert len(document['nodes']) == 1 + assert 'matrix' not in document['nodes'][0] + + +def test_build_glb_drops_a_mesh_whose_corner_runs_past_the_positions( + make_ldb: Callable[..., bytes]) -> None: + document, _ = _parse(build_glb(read_level(make_ldb(corrupt='corners', placements=False)))) + assert len(document['nodes']) == 1 + assert 'matrix' not in document['nodes'][0] + + +def test_build_glb_rejects_a_level_with_nothing_to_draw() -> None: + with pytest.raises(ValueError, match='no faces'): + build_glb( + Level(geometry=LevelGeometry(polygons=(), vertices=()), + materials={}, + mesh=None, + textures=())) + + +def _png(size: tuple[int, int], colour: tuple[int, int, int]) -> bytes: + from PIL import Image + buffer = BytesIO() + Image.new('RGB', size, colour).save(buffer, format='PNG') + return buffer.getvalue() + + +def _grey(size: tuple[int, int], level: int) -> bytes: + from PIL import Image + buffer = BytesIO() + Image.new('L', size, level).save(buffer, format='PNG') + return buffer.getvalue() + + +def _gradient(size: tuple[int, int]) -> bytes: + from PIL import Image + image = Image.new('L', size) + image.putdata( + [(x * 255) // max(size[0] - 1, 1) for _ in range(size[1]) for x in range(size[0])]) + buffer = BytesIO() + image.save(buffer, format='PNG') + return buffer.getvalue() + + +def _masked(**kwargs: object) -> dict[str, object]: + """Build a level naming a colour image and an alpha mask for material 7.""" + return { + 'materials': ((7, 'leaves', 'PLANT.JPG'), (9, 'metal', 'B.JPG')), + 'categories': (('leaves', (('PLANT.JPG', 'X:\\plant.png', 'X:\\plant_alpha.png'),)),), + **kwargs + } + + +def test_build_glb_cuts_out_a_masked_material(make_ldb: Callable[..., bytes]) -> None: + # A mask that is only ever black or white is a cut-out, so it alpha-tests rather than blends. + level = read_level( + make_ldb(**_masked(textures=(('X:\\plant.png', 0, _png((4, 4), (20, 200, 20))), + ('X:\\plant_alpha.png', 0, _grey((4, 4), 255)))))) + assert level.materials[7].alpha == 'X:\\plant_alpha.png' + document, _ = _parse(build_glb(level)) + material = next(m for m in document['materials'] if m['name'] == 'PLANT.JPG') + assert material['alphaMode'] == 'MASK' + assert material['alphaCutoff'] == pytest.approx(0.5) + assert material['doubleSided'] is True + assert 'baseColorTexture' in material['pbrMetallicRoughness'] + + +def test_build_glb_blends_a_gradient_mask(make_ldb: Callable[..., bytes]) -> None: + level = read_level( + make_ldb(**_masked(textures=(('X:\\plant.png', 0, _png((16, 4), (20, 200, 20))), + ('X:\\plant_alpha.png', 0, _gradient((16, 4))))))) + document, _ = _parse(build_glb(level)) + material = next(m for m in document['materials'] if m['name'] == 'PLANT.JPG') + assert material['alphaMode'] == 'BLEND' + assert 'alphaCutoff' not in material + + +def test_build_glb_resizes_a_mask_to_its_colour(make_ldb: Callable[..., bytes]) -> None: + level = read_level( + make_ldb(**_masked(textures=(('X:\\plant.png', 0, _png((8, 8), (20, 200, 20))), + ('X:\\plant_alpha.png', 0, _grey((4, 4), 255)))))) + document, _ = _parse(build_glb(level)) + assert any(image['name'].endswith('plant_alpha.png') for image in document['images']) + + +def test_build_glb_does_not_embed_a_mask_on_its_own(make_ldb: Callable[..., bytes]) -> None: + # The mask is only ever read through the material that names it. + level = read_level( + make_ldb(**_masked(textures=(('X:\\plant.png', 0, _png((4, 4), (20, 200, 20))), + ('X:\\plant_alpha.png', 0, _grey((4, 4), 255)))))) + document, _ = _parse(build_glb(level)) + assert not any(image['name'] == 'X:\\plant_alpha.png' for image in document['images']) + + +def test_build_glb_falls_back_when_a_mask_will_not_decode(make_ldb: Callable[..., bytes]) -> None: + level = read_level( + make_ldb(**_masked(textures=(('X:\\plant.png', 0, _png((4, 4), (20, 200, 20))), + ('X:\\plant_alpha.png', 0, b'junk'))))) + document, _ = _parse(build_glb(level)) + material = next(m for m in document['materials'] if m['name'] == 'PLANT.JPG') + assert 'baseColorFactor' in material['pbrMetallicRoughness'] + assert 'alphaMode' not in material + + +def test_build_glb_reuses_one_composed_texture(make_ldb: Callable[..., bytes]) -> None: + # Two materials naming the same pair must not embed the picture twice. + level = read_level( + make_ldb(materials=((7, 'leaves', 'PLANT.JPG'), (9, 'leaves', 'PLANT2.JPG')), + categories=(('leaves', (('PLANT.JPG', 'X:\\plant.png', 'X:\\plant_alpha.png'), + ('PLANT2.JPG', 'X:\\plant.png', 'X:\\plant_alpha.png'))),), + textures=(('X:\\plant.png', 0, _png( + (4, 4), (20, 200, 20))), ('X:\\plant_alpha.png', 0, _grey((4, 4), 255))))) + document, _ = _parse(build_glb(level)) + assert sum(1 for image in document['images'] if ' + ' in image['name']) == 1 + materials = [m for m in document['materials'] if m['name'].startswith('PLANT')] + assert len({m['pbrMetallicRoughness']['baseColorTexture']['index'] for m in materials}) == 1 + + +def _model(make_model: Callable[..., bytes], **kwargs: object) -> Model: + from dade.maxpane.model import read_model + from dade.maxpane.typing import TextureImage + model = read_model(make_model(**kwargs)) + return model._replace(textures=tuple( + TextureImage(data=_png((4, 4), (10, 20, 30)), kind=0, path=name) + for name in dict.fromkeys(model.materials.values()))) + + +def test_build_glb_draws_an_npc_with_its_model(make_ldb: Callable[..., bytes], + make_model: Callable[..., bytes]) -> None: + level = read_level(make_ldb()) + document, _ = _parse(build_glb(level, models={'character:transit_cop': _model(make_model)})) + node = next(n for n in document['nodes'] if n['name'].startswith('character:')) + assert 'mesh' in node + assert node['matrix'][12:] == [1, 2, -3, 1] + material = document['materials'][document['meshes'][node['mesh']]['primitives'][0]['material']] + assert material['name'] == 'Skin (skin.png)' + assert 'baseColorTexture' in material['pbrMetallicRoughness'] + + +def test_build_glb_leaves_a_placement_empty_without_a_model(make_ldb: Callable[..., bytes]) -> None: + document, _ = _parse(build_glb(read_level(make_ldb()))) + node = next(n for n in document['nodes'] if n['name'].startswith('character:')) + assert 'mesh' not in node + + +def test_build_glb_leaves_a_placement_empty_for_a_model_with_no_faces( + make_ldb: Callable[..., bytes], make_model: Callable[..., bytes]) -> None: + empty = _model(make_model, faces=(), coord_faces=(), face_materials=()) + document, _ = _parse(build_glb(read_level(make_ldb()), models={'item:ammo_ingram': empty})) + node = next(n for n in document['nodes'] if n['name'].startswith('item:')) + assert 'mesh' not in node + + +def test_build_glb_shares_one_model_material_between_placements( + make_ldb: Callable[..., bytes], make_model: Callable[..., bytes]) -> None: + shared = _model(make_model) + document, _ = _parse( + build_glb(read_level(make_ldb()), + models={ + 'character:transit_cop': shared, + 'item:ammo_ingram': shared + })) + drawn = [n for n in document['nodes'] if n['name'].startswith(('character:', 'item:'))] + indices = {document['meshes'][n['mesh']]['primitives'][0]['material'] for n in drawn} + assert len(drawn) == 2 + assert len(indices) == 1 + + +def test_build_glb_falls_back_when_a_model_names_no_image(make_ldb: Callable[..., bytes], + make_model: Callable[..., bytes]) -> None: + from dade.maxpane.model import read_model + bare = read_model(make_model()) + document, _ = _parse(build_glb(read_level(make_ldb()), models={'character:transit_cop': bare})) + node = next(n for n in document['nodes'] if n['name'].startswith('character:')) + material = document['materials'][document['meshes'][node['mesh']]['primitives'][0]['material']] + assert 'baseColorFactor' in material['pbrMetallicRoughness'] + + +def test_build_glb_names_each_mesh_of_a_multipart_model(make_ldb: Callable[..., bytes], + make_model: Callable[..., bytes]) -> None: + from dade.maxpane.model import read_model + one = read_model(make_model()) + both = one._replace(meshes=one.meshes + one.meshes) + document, _ = _parse(build_glb(read_level(make_ldb()), models={'character:transit_cop': both})) + named = [n['name'] for n in document['nodes'] if n['name'].startswith('character:')] + assert named == ['character:transit_cop ::room::e1 #0', 'character:transit_cop ::room::e1 #1'] + + +def test_build_glb_skips_a_models_empty_mesh(make_ldb: Callable[..., bytes], + make_model: Callable[..., bytes]) -> None: + from dade.maxpane.model import read_model + one = read_model(make_model()) + hollow = read_model(make_model(faces=(), coord_faces=(), face_materials=())) + mixed = one._replace(meshes=one.meshes + hollow.meshes) + document, _ = _parse(build_glb(read_level(make_ldb()), models={'character:transit_cop': mixed})) + assert len([n for n in document['nodes'] if n['name'].startswith('character:')]) == 1 + + +_SWING = ((0.0, 4.0, 0.0), (1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0), 2) +"""A clip that slides its prop four units up, in `make_ldb`'s ``motion`` shape.""" + + +def _clip(document: dict[str, Any], name: str) -> dict[str, Any]: + return next(a for a in document['animations'] if a['name'].endswith(name)) + + +def _sampler_values(document: dict[str, Any], binary: bytes, accessor: int, + size: int) -> list[tuple[float, ...]]: + raw = _accessor_bytes(document, binary, accessor) + return [struct.unpack_from(f'<{size}f', raw, at) for at in range(0, len(raw), 4 * size)] + + +def test_build_glb_animates_a_prop(make_ldb: Callable[..., bytes]) -> None: + # A prop that only slides gets no rotation channel; the level drives the two separately. + document, _ = _parse(build_glb(read_level(make_ldb(motion=_SWING)))) + clip = _clip(document, 'clip0') + assert [c['target']['path'] for c in clip['channels']] == ['translation'] + node = document['nodes'][clip['channels'][0]['target']['node']] + assert node['name'] == '::room::door.DO' + # A node carrying a matrix cannot be animated, so an animated prop is written as separate + # translation, rotation and scale. + assert 'matrix' not in node + assert node['translation'] == [7, 8, -9] + + +def test_build_glb_drives_both_channels_of_a_prop_that_slides_and_turns( + make_ldb: Callable[..., bytes], bases: dict[str, Any]) -> None: + document, _ = _parse( + build_glb(read_level(make_ldb(motion=((0.0, 4.0, 0.0), bases['half_turn'], 2))))) + clip = _clip(document, 'clip0') + assert [c['target']['path'] for c in clip['channels']] == ['translation', 'rotation'] + + +def test_build_glb_starts_a_clip_where_the_prop_rests(make_ldb: Callable[..., bytes], + bases: dict[str, Any]) -> None: + # The first keyframe has to equal the node's own placement or the prop jumps when a clip starts. + document, binary = _parse( + build_glb(read_level(make_ldb(motion=((0.0, 4.0, 0.0), bases['half_turn'], 2))))) + clip = _clip(document, 'clip0') + node = document['nodes'][clip['channels'][0]['target']['node']] + sampler = clip['samplers'][clip['channels'][0]['sampler']] + first = _sampler_values(document, binary, sampler['output'], 3)[0] + assert first == pytest.approx(node['translation']) + rotation = clip['samplers'][clip['channels'][1]['sampler']] + assert _sampler_values(document, binary, rotation['output'], + 4)[0] == pytest.approx(node['rotation']) + + +def test_build_glb_carries_a_clip_to_its_end(make_ldb: Callable[..., bytes]) -> None: + document, binary = _parse(build_glb(read_level(make_ldb(motion=_SWING)))) + clip = _clip(document, 'clip0') + sampler = clip['samplers'][clip['channels'][0]['sampler']] + values = _sampler_values(document, binary, sampler['output'], 3) + assert values[-1] == pytest.approx([7, 12, -9]) + + +def test_build_glb_times_a_clip_over_its_duration(make_ldb: Callable[..., bytes]) -> None: + document, binary = _parse(build_glb(read_level(make_ldb(motion=_SWING)))) + clip = _clip(document, 'clip0') + sampler = clip['samplers'][clip['channels'][0]['sampler']] + times = _sampler_values(document, binary, sampler['input'], 1) + assert times[0] == pytest.approx((0.0,)) + assert times[-1] == pytest.approx((1.0,)) + + +def test_build_glb_leaves_a_still_prop_on_a_matrix(make_ldb: Callable[..., bytes]) -> None: + # A clip whose two poses are the same moves nothing, so it is not worth a glTF animation. + document, _ = _parse(build_glb(read_level(make_ldb()))) + assert 'animations' not in document + node = next(n for n in document['nodes'] if n['name'] == '::room::door.DO') + assert 'matrix' in node + + +def test_build_glb_turns_a_prop_about_its_axis(make_ldb: Callable[..., bytes], + bases: dict[str, Any]) -> None: + # Half a turn is the case a quaternion cannot read straight off the matrix diagonal. + document, binary = _parse( + build_glb(read_level(make_ldb(motion=((0.0, 1.0, 0.0), bases['half_turn'], 2))))) + clip = _clip(document, 'clip0') + rotation = clip['samplers'][clip['channels'][1]['sampler']] + values = _sampler_values(document, binary, rotation['output'], 4) + assert values[0] == pytest.approx((0.0, 0.0, 0.0, 1.0)) + assert abs(values[-1][1]) == pytest.approx(1.0) + assert all(abs(sum(v * v for v in value) - 1.0) < 1e-5 for value in values) + + +def test_build_glb_walks_a_wide_turn_along_its_arc(make_ldb: Callable[..., bytes], + bases: dict[str, Any]) -> None: + # Halfway through half a turn is a quarter turn, which straight interpolation would not give. + document, binary = _parse( + build_glb(read_level(make_ldb(motion=((0.0, 1.0, 0.0), bases['half_turn'], 3))))) + clip = _clip(document, 'clip0') + rotation = clip['samplers'][clip['channels'][1]['sampler']] + middle = _sampler_values(document, binary, rotation['output'], 4)[1] + assert abs(middle[1]) == pytest.approx(0.5 ** 0.5, abs=1e-4) + + +def test_build_glb_keeps_a_reflected_prop_placed(make_ldb: Callable[..., bytes], + bases: dict[str, Any]) -> None: + # A left-handed basis is no rotation at all, so it has to survive as a negative scale. + document, _ = _parse( + build_glb(read_level(make_ldb(motion=((0.0, 1.0, 0.0), bases['reflected'], 2))))) + clip = _clip(document, 'clip0') + node = document['nodes'][clip['channels'][0]['target']['node']] + assert node['scale'][0] == pytest.approx(1.0) + + +def test_build_glb_thins_a_long_curve(make_ldb: Callable[..., bytes]) -> None: + # A shipped door carries 256 samples of a smooth ease; the motion does not need them. + document, binary = _parse(build_glb(read_level(make_ldb(motion=(_SWING[0], _SWING[1], 50))))) + clip = _clip(document, 'clip0') + times = _sampler_values(document, binary, clip['samplers'][0]['input'], 1) + assert len(times) <= 24 + assert times[-1] == pytest.approx((1.0,)) + + +def test_build_glb_walks_a_small_turn_straight(make_ldb: Callable[..., bytes], + bases: dict[str, Any]) -> None: + # Two degrees is worth a channel but far too close to take the long way round. + document, binary = _parse( + build_glb(read_level(make_ldb(motion=((0.0, 0.0, 0.0), bases['tilt'], 3))))) + clip = _clip(document, 'clip0') + assert [c['target']['path'] for c in clip['channels']] == ['rotation'] + values = _sampler_values(document, binary, clip['samplers'][0]['output'], 4) + assert all(abs(sum(v * v for v in value) - 1.0) < 1e-5 for value in values) + + +def test_build_glb_settles_a_prop_whose_clips_drive_nothing(make_ldb: Callable[..., bytes], + bases: dict[str, Any]) -> None: + # The prop ends somewhere else, but only in size, which no channel carries. + document, _ = _parse( + build_glb(read_level(make_ldb(motion=((0.0, 0.0, 0.0), bases['stretched'], 2))))) + assert 'animations' not in document + node = next(n for n in document['nodes'] if n['name'] == '::room::door.DO') + assert 'matrix' in node + assert 'translation' not in node + + +def test_build_glb_lights_a_face_with_its_atlas(make_ldb: Callable[..., bytes]) -> None: + # The atlas goes in the occlusion slot on the second coordinate set. + document, _ = _parse( + build_glb(read_level(make_ldb(lightmaps=((0, 0, _png((4, 4), (200, 200, 200))),))))) + material = next(m for m in document['materials'] if 'occlusionTexture' in m) + assert material['occlusionTexture']['texCoord'] == 1 + assert material['name'].endswith('+ lightmap 0') + assert all('TEXCOORD_1' in p['attributes'] for m in document['meshes'] for p in m['primitives']) + + +def test_build_glb_writes_the_second_coordinate_set(make_ldb: Callable[..., bytes]) -> None: + level = read_level(make_ldb(lightmaps=((0, 0, _png((4, 4), (200, 200, 200))),))) + document, binary = _parse(build_glb(level)) + coord = document['meshes'][0]['primitives'][0]['attributes']['TEXCOORD_1'] + assert level.mesh is not None + raw = _accessor_bytes(document, binary, coord) + assert struct.unpack_from('<2f', raw, 0) == pytest.approx(level.mesh.corners[0].lightmap_uv) + + +def test_build_glb_shares_one_atlas_between_materials(make_ldb: Callable[..., bytes]) -> None: + document, _ = _parse( + build_glb( + read_level( + make_ldb(face_materials=(7, 9), lightmaps=((0, 0, _png((4, 4), + (200, 200, 200))),))))) + lit = [m for m in document['materials'] if 'occlusionTexture' in m] + assert len(lit) == 2 + assert len({m['occlusionTexture']['index'] for m in lit}) == 1 + + +def test_build_glb_without_an_atlas_for_a_face(make_ldb: Callable[..., bytes]) -> None: + # The face names atlas nought but the level ships none, so nothing is attached. + document, _ = _parse(build_glb(read_level(make_ldb()))) + assert not any('occlusionTexture' in m for m in document['materials']) + + +def test_build_glb_skips_an_undecodable_atlas(make_ldb: Callable[..., bytes]) -> None: + document, _ = _parse(build_glb(read_level(make_ldb(lightmaps=((0, 0, b'junk'),))))) + assert not any('occlusionTexture' in m for m in document['materials']) + + +def test_build_glb_lifts_a_face_off_the_one_it_covers(make_ldb: Callable[..., bytes]) -> None: + # Both triangles lie in one plane over the same ground, the way a level lays graffiti on a + # wall, so the second has to come off it or a depth buffer cannot tell which is in front. + document, binary = _parse(build_glb(read_level(make_ldb(face_materials=(7,), + layout='stacked')))) + position = document['meshes'][0]['primitives'][0]['attributes']['POSITION'] + raw = _accessor_bytes(document, binary, position) + heights = [struct.unpack_from('<3f', raw, corner * 12)[1] for corner in range(6)] + assert heights[:3] == [0.0, 0.0, 0.0] + assert heights[3:] == pytest.approx([DECAL_STEP] * 3) + + +def test_build_glb_leaves_a_face_nothing_covers_alone(make_ldb: Callable[..., bytes]) -> None: + document, binary = _parse(build_glb(read_level(make_ldb(face_materials=(7,))))) + position = document['meshes'][0]['primitives'][0]['attributes']['POSITION'] + raw = _accessor_bytes(document, binary, position) + heights = [struct.unpack_from('<3f', raw, corner * 12)[1] for corner in range(6)] + assert heights == [0.0, 0.0, 0.0, 1.0, 1.0, 1.0] + + +def test_build_glb_shares_one_sky_between_materials(make_ldb: Callable[..., bytes]) -> None: + document, _ = _parse( + build_glb( + read_level( + make_ldb(materials=((7, 'skybox', 'A.TGA'), (9, 'skybox', 'B.JPG')), + face_materials=(7, 9), + placements=False)))) + assert len([m for m in document['materials'] if m['name'] == 'skybox']) == 1 + + +def test_build_glb_keeps_a_face_with_no_side_as_written(make_ldb: Callable[..., bytes]) -> None: + # Every corner is on one line, so nothing in the fan says which way the face points and the + # exporter has to leave the order alone rather than read a direction out of noise. + document, binary = _parse(build_glb(read_level(make_ldb(layout='collinear')))) + indices = document['meshes'][0]['primitives'][0]['indices'] + raw = _accessor_bytes(document, binary, indices) + assert struct.unpack_from('<3I', raw, 0) == (0, 1, 2) diff --git a/tests/maxpane/test_ldb.py b/tests/maxpane/test_ldb.py new file mode 100644 index 0000000..cc4f005 --- /dev/null +++ b/tests/maxpane/test_ldb.py @@ -0,0 +1,296 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING +import struct + +import pytest + +from dade.maxpane.ldb import InvalidLevelError, read_geometry, read_level, read_textures + +if TYPE_CHECKING: + from collections.abc import Callable + + +def test_read_geometry(make_ldb: Callable[..., bytes]) -> None: + geometry = read_geometry(make_ldb()) + assert len(geometry.vertices) == 7 + assert [p.vertex_count for p in geometry.polygons] == [4, 3] + assert [p.first_vertex for p in geometry.polygons] == [0, 4] + assert [p.mesh_index for p in geometry.polygons] == [7, 9] + assert geometry.polygons[0].normal == (0.0, 1.0, 0.0) + + +def test_read_geometry_rejects_a_missing_array_marker() -> None: + with pytest.raises(InvalidLevelError, match='Expected an array marker'): + read_geometry(b'\x14\x02') + + +def test_read_geometry_rejects_a_truncated_file() -> None: + with pytest.raises(InvalidLevelError, match='Expected an array marker'): + read_geometry(b'') + + +def test_read_geometry_rejects_too_few_corners(make_ldb: Callable[..., bytes]) -> None: + level = make_ldb(faces=(((0.0, 0.0, 0.0), (1.0, 0.0, 0.0)),), mesh_indices=(0,)) + with pytest.raises(InvalidLevelError, match='claims 2 corners'): + read_geometry(level) + + +_FACE_0 = b'\x14\x00\x14\x04\x14\x00\x14\x07' +_FACE_1 = b'\x14\x04\x14\x03\x14\x01\x14\x09' + + +def test_read_geometry_rejects_a_face_outside_the_pool(make_ldb: Callable[..., bytes]) -> None: + level = make_ldb().replace(_FACE_1, b'\x14\x7f\x14\x03\x14\x01\x14\x09') + with pytest.raises(InvalidLevelError, match='outside a pool'): + read_geometry(level) + + +def test_read_geometry_rejects_an_out_of_step_read(make_ldb: Callable[..., bytes]) -> None: + level = make_ldb().replace(_FACE_0, b'\x14\x00\x14\x03\x14\x00\x14\x07') + with pytest.raises(InvalidLevelError, match='out of step'): + read_geometry(level) + + +def test_read_textures(make_ldb: Callable[..., bytes]) -> None: + textures = read_textures( + make_ldb(textures=(('C:\\A.TGA', 0, b'\x00\x01'), ('D:\\dir\\b.jpg', 4, b'\xff\xd8')))) + assert [t.path for t in textures] == ['C:\\A.TGA', 'D:\\dir\\b.jpg'] + assert [t.kind for t in textures] == [0, 4] + assert textures[1].data == b'\xff\xd8' + + +def test_read_textures_none(make_ldb: Callable[..., bytes]) -> None: + assert read_textures(make_ldb(textures=())) == () + + +def test_read_textures_rejects_a_run_past_the_end(make_ldb: Callable[..., bytes]) -> None: + level = make_ldb(complete=False, textures=(('C:\\A.TGA', 0, b'\x00\x01'),)) + with pytest.raises(InvalidLevelError, match='but the file ends'): + read_textures(level[:-1]) + + +def test_read_textures_skips_a_populated_bsp(make_ldb: Callable[..., bytes]) -> None: + level = make_ldb(bsp=(2, 3), textures=(('C:\\A.TGA', 0, b'\x00\x01'),)) + assert [t.path for t in read_textures(level)] == ['C:\\A.TGA'] + + +def test_read_level(make_ldb: Callable[..., bytes]) -> None: + level = read_level(make_ldb()) + assert len(level.geometry.polygons) == 2 + assert level.materials[7].category == 'wood' + assert level.materials[7].texture == 'A.TGA' + assert [t.path for t in level.textures] == ['C:\\A.TGA'] + assert level.mesh is not None + assert len(level.mesh.meshes) == 1 + assert {f.material for f in level.mesh.meshes[0].faces} == {7, 9} + + +def test_read_level_reads_the_corner_array(make_ldb: Callable[..., bytes]) -> None: + level = read_level(make_ldb(triangles=40)) + assert level.mesh is not None + assert len(level.mesh.corners) == 120 + assert level.mesh.corners[0].position == 0 + assert level.mesh.corners[0].uv[1] == pytest.approx(0.5) + + +def test_read_level_reads_positions_and_normals(make_ldb: Callable[..., bytes]) -> None: + level = read_level(make_ldb(triangles=40)) + assert level.mesh is not None + mesh = level.mesh.meshes[0] + assert len(mesh.positions) == len(mesh.normals) == 120 + assert mesh.normals[0] == (0.0, 0.0, 1.0) + assert mesh.transform == (1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0) + + +def test_read_level_without_a_mesh_container(make_ldb: Callable[..., bytes]) -> None: + assert read_level(make_ldb(triangles=0)).mesh is None + + +def test_read_level_ignores_a_container_that_is_too_small(make_ldb: Callable[..., bytes]) -> None: + # Fewer corners than the reader's floor, so the candidate is rejected outright. + assert read_level(make_ldb(triangles=4)).mesh is None + + +def test_read_level_resolves_a_material_to_its_image(make_ldb: Callable[..., bytes]) -> None: + # The material's own name is not the filename; the category table is what names the picture. + level = read_level( + make_ldb(materials=((7, 'wood', 'A_128X256.JPG'), (9, 'metal', 'B.JPG')), + textures=(('X:\\art\\a_256x256.jpg', 4, b'\xff\xd8fake'),), + categories=(('wood', (('A_128X256.JPG', 'X:\\art\\a_256x256.jpg', ''),)),))) + assert level.materials[7].image == 'X:\\art\\a_256x256.jpg' + + +def test_read_level_resolves_every_material(make_ldb: Callable[..., bytes]) -> None: + level = read_level( + make_ldb(materials=((7, 'wood', 'A.TGA'),), + face_materials=(7,), + textures=(('C:\\A.TGA', 0, b'\x00\x01\x02'),))) + assert [m.image for m in level.materials.values()] == ['C:\\A.TGA'] + + +def test_read_level_reads_a_materials_alpha_mask(make_ldb: Callable[..., bytes]) -> None: + # The third category path is the alpha mask, not a second guess at the colour. + level = read_level( + make_ldb(materials=((7, 'wood', 'A.TGA'), (9, 'metal', 'B.JPG')), + textures=(('C:\\A.TGA', 0, b'\x00\x01\x02'), ('C:\\A_alpha.pcx', 3, b'\x0a\x05')), + categories=(('wood', (('A.TGA', 'C:\\A.TGA', 'C:\\A_alpha.pcx'),)),))) + assert level.materials[7].image == 'C:\\A.TGA' + assert level.materials[7].alpha == 'C:\\A_alpha.pcx' + + +def test_read_level_leaves_an_opaque_material_without_a_mask( + make_ldb: Callable[..., bytes]) -> None: + assert not read_level(make_ldb()).materials[7].alpha + + +def test_read_level_ignores_a_mask_that_is_the_colour_again(make_ldb: Callable[..., bytes]) -> None: + # Most entries repeat the colour path in both slots, which means opaque, not self-masked. + level = read_level( + make_ldb(materials=((7, 'wood', 'A.TGA'), (9, 'metal', 'B.JPG')), + textures=(('C:\\A.TGA', 0, b'\x00\x01\x02'),), + categories=(('wood', (('A.TGA', 'C:\\A.TGA', 'C:\\A.TGA'),)),))) + assert not level.materials[7].alpha + + +def test_read_level_leaves_an_unnamed_material_without_an_image( + make_ldb: Callable[..., bytes]) -> None: + # Nothing in the category table points at an embedded image, so the material draws untextured. + level = read_level(make_ldb(categories=(('wood', (('A.TGA', 'X:\\gone.jpg', ''),)),))) + assert not level.materials[7].image + assert not level.materials[9].image + + +def test_read_level_reads_the_placements(make_ldb: Callable[..., bytes]) -> None: + level = read_level(make_ldb()) + assert [(c.skin, c.placement.name) for c in level.characters] == [('transit_cop', '::room::e1')] + assert level.characters[0].placement.transform[9:] == (1.0, 2.0, 3.0) + assert [(i.item, i.placement.name) for i in level.items] == [('ammo_ingram', '::room::ammo')] + assert level.props is not None + assert level.props.names == ('::room::door.DO',) + assert level.props.meshes[0].transform[9:] == (7.0, 8.0, 9.0) + + +def test_read_level_gives_up_on_an_unwalkable_tail(make_ldb: Callable[..., bytes]) -> None: + # A count no level could hold means the walk has lost its place, so the tail is abandoned and + # the level still comes back with everything read before it. + level = read_level(make_ldb(corrupt='placements')) + assert level.mesh is not None + assert level.characters == () + assert level.items == () + assert level.props is None + + +def test_read_level_with_categories_and_lightmaps(make_ldb: Callable[..., bytes]) -> None: + level = read_level( + make_ldb(categories=(('wood', (('a', 'C:\\a', ''),)),), lightmaps=((0, 0, b'\x01\x02'),))) + assert level.mesh is not None + assert len(level.mesh.meshes) == 1 + + +def test_read_level_rejects_a_missing_map(make_ldb: Callable[..., bytes]) -> None: + level = bytearray(make_ldb()) + level[level.index(b'\x1f')] = 0x14 + with pytest.raises(InvalidLevelError, match='Expected a map marker'): + read_level(bytes(level)) + + +def test_read_level_rejects_a_missing_pair(make_ldb: Callable[..., bytes]) -> None: + level = bytearray(make_ldb()) + level[level.index(b'\x25')] = 0x14 + with pytest.raises(InvalidLevelError, match='Expected a pair marker'): + read_level(bytes(level)) + + +def test_read_level_rejects_a_missing_index_map(make_ldb: Callable[..., bytes]) -> None: + level = bytearray(make_ldb()) + second = level.index(b'\x1f', level.index(b'\x1f') + 1) + level[second] = 0x14 + with pytest.raises(InvalidLevelError, match='Expected a map marker'): + read_level(bytes(level)) + + +def test_read_level_rejects_a_missing_index_pair(make_ldb: Callable[..., bytes]) -> None: + level = bytearray(make_ldb()) + second_map = level.index(b'\x1f', level.index(b'\x1f') + 1) + level[level.index(b'\x25', second_map)] = 0x14 + with pytest.raises(InvalidLevelError, match='Expected a pair marker'): + read_level(bytes(level)) + + +def test_read_level_rejects_a_texture_past_the_end(make_ldb: Callable[..., bytes]) -> None: + with pytest.raises(InvalidLevelError, match='but the file ends'): + read_level(make_ldb(complete=False, textures=(('C:\\A.TGA', 0, b'\x00\x01'),))[:-1]) + + +def test_read_level_skips_a_candidate_with_an_implausible_count( + make_ldb: Callable[..., bytes]) -> None: + # An array marker claiming more corners than any level holds is rejected on the count alone, + # before any corner is read. + junk = b'\x1c\x02' + struct.pack(' None: + # A plausible count followed by bytes that are not corners has to fall through to the real + # container further on. + junk = b'\x1c\x13' + struct.pack(' None: + # The corners read cleanly, so this one is only rejected once the mesh count comes out absurd. + level = read_level(make_ldb(junk=make_mesh_container(40, (7,), mesh_count=-1))) + assert level.mesh is not None + assert len(level.mesh.meshes) == 1 + + +def test_read_level_ignores_an_exit_with_no_partner(make_ldb: Callable[..., bytes]) -> None: + # A one-sided exit says nothing about how two rooms meet, so it cannot place anything. + level = read_level( + make_ldb(meshes=2, + world=((('::room::out', '::gone::in', (5.0, 0.0, 0.0)),), ((0, (0,), '::room'), + (1, (1,), '::far'))))) + assert level.mesh is not None + assert level.mesh.meshes[1].transform[9:] == (0.0, 0.0, 0.0) + + +def test_read_level_places_a_room_through_its_exit(make_ldb: Callable[..., bytes]) -> None: + level = read_level( + make_ldb(meshes=2, + world=((('::room::out', '::far::in', (5.0, 1.0, 2.0)), + ('::far::in', '::room::out', (-5.0, -1.0, -2.0))), ((0, (0,), '::room'), + (1, (1,), '::far'))))) + assert level.mesh is not None + assert level.mesh.meshes[0].transform[9:] == (0.0, 0.0, 0.0) + assert level.mesh.meshes[1].transform[9:] == (5.0, 1.0, 2.0) + # The props and NPCs of a placed room move with it. + assert level.characters[0].placement.transform[9:] == (1.0, 2.0, 3.0) + + +def test_read_level_reads_the_lightmaps(make_ldb: Callable[..., bytes]) -> None: + level = read_level(make_ldb(lightmaps=((0, 0, b'\x00\x00\x02\x00'), (1, 0, b'\x00\x00\x02')))) + assert [t.path for t in level.lightmaps] == ['lightmap_0.tga', 'lightmap_1.tga'] + assert level.lightmaps[0].data == b'\x00\x00\x02\x00' + + +def test_read_level_without_lightmaps(make_ldb: Callable[..., bytes]) -> None: + assert read_level(make_ldb()).lightmaps == () + + +def test_read_level_reads_the_faces_lightmap_field(make_ldb: Callable[..., bytes]) -> None: + # A face names which of the level's atlases lights it. + level = read_level(make_ldb()) + assert level.mesh is not None + assert {f.lightmap for f in level.mesh.meshes[0].faces} == {0} + + +def test_read_level_reads_the_faces_flags(make_ldb: Callable[..., bytes]) -> None: + level = read_level(make_ldb()) + assert level.mesh is not None + assert {f.flags for f in level.mesh.meshes[0].faces} == {1} diff --git a/tests/maxpane/test_memoryfile.py b/tests/maxpane/test_memoryfile.py new file mode 100644 index 0000000..c1eb903 --- /dev/null +++ b/tests/maxpane/test_memoryfile.py @@ -0,0 +1,121 @@ +from __future__ import annotations + +import struct + +import pytest + +from dade.maxpane.memoryfile import ( + TAG_SIZES, + BasicType, + iter_values, + read_chunk_header, + read_int, + read_string, + read_vector3, +) + + +@pytest.mark.parametrize(('tag', 'payload', 'expected'), [ + (BasicType.INT8, b'\x7b', 123), + (BasicType.INT8, b'\xff', -1), + (BasicType.INT16, b'\x64\x5e', 24164), + (BasicType.INT24, b'\x0f\xd0\x00', 53263), + (BasicType.INT32, b'\x01\x02\x03\x04', 0x04030201), + (BasicType.UINT8, b'\xff', 255), + (BasicType.UINT16, b'\x00\x80', 32768), +]) +def test_read_int(tag: int, payload: bytes, expected: int) -> None: + value, end = read_int(bytes((tag,)) + payload, 0) + assert value == expected + assert end == 1 + len(payload) + + +def test_read_int_rejects_a_non_integer_tag() -> None: + with pytest.raises(ValueError, match='Not an integer tag'): + read_int(bytes((BasicType.ARRAY,)) + b'\x00', 0) + + +def test_read_vector3() -> None: + data = bytes((BasicType.VECTOR3,)) + struct.pack('<3f', 1.5, -2.5, 3.5) + assert read_vector3(data, 0) == ((1.5, -2.5, 3.5), 13) + + +def test_read_vector3_rejects_another_tag() -> None: + with pytest.raises(ValueError, match='Not a vector tag'): + read_vector3(bytes((BasicType.FLOAT,)) + bytes(4), 0) + + +def test_read_string() -> None: + data = (bytes((BasicType.STRING, BasicType.INT8, 5)) + b'hello') + assert read_string(data, 0) == ('hello', 8) + + +def test_read_string_with_a_wide_length() -> None: + text = 'x' * 300 + data = bytes((BasicType.STRING, BasicType.INT16)) + (300).to_bytes(2, 'little') + text.encode() + assert read_string(data, 0) == (text, 304) + + +def test_read_string_empty() -> None: + assert read_string(bytes((BasicType.STRING, BasicType.INT8, 0)), 0) == ('', 3) + + +def test_read_string_rejects_another_tag() -> None: + with pytest.raises(ValueError, match='Not a string tag'): + read_string(bytes((BasicType.INT8, 1)), 0) + + +def test_iter_values_follows_a_string() -> None: + data = (bytes((BasicType.STRING, BasicType.INT8, 3)) + b'abc' + bytes((BasicType.INT8, 9))) + values = list(iter_values(data)) + assert [value.tag for value in values] == [BasicType.STRING, BasicType.INT8] + assert values[0].payload == b'abc' + assert values[0].end == 6 + assert values[1].payload == b'\x09' + + +def test_iter_values_stops_on_a_truncated_string() -> None: + assert not list(iter_values(bytes((BasicType.STRING, BasicType.INT8, 40)) + b'short')) + + +def test_iter_values_stops_on_a_string_with_a_bad_length_tag() -> None: + assert not list(iter_values(bytes((BasicType.STRING, 0xAA, 1, 1)))) + + +def test_read_chunk_header() -> None: + data = bytes((BasicType.CHUNK,)) + struct.pack('<3I', 7, 2, 128) + assert read_chunk_header(data) == (7, 2, 128) + + +def test_read_chunk_header_rejects_another_tag() -> None: + with pytest.raises(ValueError, match='Not a chunk tag'): + read_chunk_header(bytes((BasicType.ARRAY,))) + + +def test_iter_values_walks_an_array() -> None: + data = (bytes((BasicType.ARRAY,)) + bytes((BasicType.INT16,)) + b'\x02\x00' + bytes( + (BasicType.VECTOR3,)) + struct.pack('<3f', 0.0, 1.0, 2.0) + bytes( + (BasicType.VECTOR3,)) + struct.pack('<3f', 3.0, 4.0, 5.0)) + values = list(iter_values(data)) + assert [value.tag for value in values] == [ + BasicType.ARRAY, BasicType.INT16, BasicType.VECTOR3, BasicType.VECTOR3 + ] + assert values[0].payload == b'' + + +def test_iter_values_stops_at_an_unknown_tag() -> None: + data = bytes((BasicType.INT8,)) + b'\x01' + b'\xaa' + b'rest' + assert [value.offset for value in iter_values(data)] == [0] + + +def test_iter_values_stops_on_a_truncated_payload() -> None: + assert not list(iter_values(bytes((BasicType.VECTOR3,)) + bytes(4))) + + +def test_iter_values_can_resume() -> None: + data = b'\xaa' + bytes((BasicType.INT8,)) + b'\x05' + assert [value.tag for value in iter_values(data, 1)] == [BasicType.INT8] + + +def test_tag_sizes_cover_every_fixed_width_tag() -> None: + assert set(TAG_SIZES) == set(BasicType) - {BasicType.STRING} diff --git a/tests/maxpane/test_model.py b/tests/maxpane/test_model.py new file mode 100644 index 0000000..92b3396 --- /dev/null +++ b/tests/maxpane/test_model.py @@ -0,0 +1,214 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING +import struct + +import pytest + +from dade.maxpane.model import InvalidModelError, read_model + +if TYPE_CHECKING: + from collections.abc import Callable + + +def test_read_model_tagged(make_model: Callable[..., bytes]) -> None: + model = read_model(make_model()) + assert model.search == ('textures', '../sharedtextures') + assert model.materials == {'Skin': 'skin.png'} + mesh = model.meshes[0] + assert mesh.name == 'body' + assert len(mesh.positions) == 3 + assert mesh.faces[0].positions == (0, 1, 2) + assert mesh.materials == ('Skin',) + + +def test_read_model_packed(make_model: Callable[..., bytes]) -> None: + # An object writes packed arrays and a flat index buffer; a skin writes tagged, face-indexed + # ones. Both have to come out the same shape. + model = read_model(make_model(packed=True)) + mesh = model.meshes[0] + assert len(mesh.positions) == 3 + assert len(mesh.normals) == 3 + assert mesh.faces[0].positions == (0, 1, 2) + + +def test_read_model_stands_the_model_up(make_model: Callable[..., bytes]) -> None: + # Models are Z-up and the game is Y-up, so the exporter's Z becomes the game's Y. + model = read_model(make_model(positions=((1.0, 2.0, 3.0), (0.0, 0.0, 0.0), (0.0, 1.0, 0.0)))) + assert model.meshes[0].positions[0] == (1.0, -3.0, 2.0) + + +def test_read_model_keeps_texture_v_as_stored(make_model: Callable[..., bytes]) -> None: + # The stored triple is (u, v, 0) with v running negative, and it is used as written: negating + # it puts a face's mouth on its forehead. + model = read_model(make_model(coords=((0.25, -0.75, 0.0), (0.0, 0.0, 0.0), (1.0, -1.0, 0.0)))) + assert model.meshes[0].coords[0] == pytest.approx((0.25, -0.75)) + + +def test_read_model_reads_the_normals_of_a_packed_mesh(make_model: Callable[..., bytes]) -> None: + assert read_model(make_model(packed=True)).meshes[0].normals[0] == (0.0, -1.0, 0.0) + + +def test_read_model_leaves_a_tagged_mesh_without_normals(make_model: Callable[..., bytes]) -> None: + assert read_model(make_model()).meshes[0].normals == () + + +def test_read_model_uses_the_per_face_material(make_model: Callable[..., bytes]) -> None: + model = read_model( + make_model(faces=((0, 1, 2), (2, 1, 0)), + coord_faces=((0, 1, 2), (2, 1, 0)), + materials=(('Skin', 'skin.png'), ('Cloth', 'cloth.png')), + face_materials=(1, 0))) + assert [f.material for f in model.meshes[0].faces] == [1, 0] + + +def test_read_model_falls_back_to_the_first_material(make_model: Callable[..., bytes]) -> None: + # A mesh that names its materials but not a per-face list draws entirely with the first. + model = read_model(make_model(face_materials=())) + assert model.meshes[0].faces[0].material == 0 + + +def test_read_model_ignores_a_material_index_past_the_end(make_model: Callable[..., bytes]) -> None: + model = read_model(make_model(face_materials=(7,))) + assert model.meshes[0].faces[0].material == 0 + + +def test_read_model_drops_a_face_indexing_a_missing_position( + make_model: Callable[..., bytes]) -> None: + model = read_model(make_model(faces=((0, 1, 9),), coord_faces=((0, 1, 2),))) + assert model.meshes[0].faces == () + + +def test_read_model_drops_a_face_that_is_not_a_triangle(make_model: Callable[..., bytes]) -> None: + model = read_model(make_model(faces=((0, 1),), coord_faces=((0, 1),))) + assert model.meshes[0].faces == () + + +def test_read_model_falls_back_when_a_coordinate_index_is_missing( + make_model: Callable[..., bytes]) -> None: + model = read_model(make_model(coord_faces=((0, 1, 9),))) + assert model.meshes[0].faces[0].coords == (0, 0, 0) + + +def test_read_model_rejects_a_file_that_is_not_chunked() -> None: + with pytest.raises(InvalidModelError, match='Not a chunked model'): + read_model(b'\x14\x02') + + +def test_read_model_rejects_an_empty_file() -> None: + with pytest.raises(InvalidModelError, match='Not a chunked model'): + read_model(b'') + + +def test_read_model_rejects_a_chunk_running_past_the_end(make_model: Callable[..., bytes]) -> None: + data = bytearray(make_model()) + struct.pack_into(' None: + positions = b'\x0c' + struct.pack('<3I', 0x00010006, 0, 18) + b'\x02' + struct.pack(' None: + # A material naming no image still has to appear, so the face it draws falls back to a colour. + data = make_model() + assert read_model(data.replace(b'Map #0', b'Map #1')).materials == {'Skin': 'skin.png'} + + +def _chunk(identifier: int, version: int, body: bytes) -> bytes: + return b'\x0c' + struct.pack('<3I', identifier, version, len(body) + 13) + body + + +def test_read_model_rejects_a_vector_that_is_not_one() -> None: + # One vector's worth of room, holding something that is not a vector. + positions = b'\x14\x01' + b'\x14' + bytes(12) + mesh = _chunk(0x00010005, 1, _chunk(0x00010006, 0, positions)) + with pytest.raises(InvalidModelError, match='Expected a vector'): + read_model(mesh) + + +def test_read_model_stops_reading_faces_at_a_foreign_chunk() -> None: + faces = b'\x14\x02' + _chunk(0x00010008, 0, b'\x14\x03\x14\x00\x14\x00\x14\x00') + faces += _chunk(0x00010000, 0, b'') + mesh = _chunk(0x00010005, 1, _chunk(0x00010007, 0, faces)) + assert read_model(mesh).meshes[0].faces == () + + +def test_read_model_ignores_a_library_chunk_that_is_not_a_material() -> None: + library = b'\x0d\x14\x00' + b'\x14\x01' + _chunk(0x00010000, 0, b'') + assert read_model(_chunk(0x0001000F, 0, library)).materials == {} + + +def test_read_model_ignores_a_material_chunk_that_is_not_a_texture() -> None: + material = b'\x0d\x14\x04Coat' + _chunk(0x00010000, 0, b'') + _chunk( + 0x00010011, 1, b'\x0d\x14\x00\x14\x00\x14\x00\x14\x01\x0d\x14\x05a.png') + library = b'\x0d\x14\x00\x14\x01' + _chunk(0x00010010, 1, material) + assert read_model(_chunk(0x0001000F, 0, library)).materials == {'Coat': 'a.png'} + + +def test_read_model_reads_a_material_naming_no_file() -> None: + texture = b'\x0d\x14\x00\x14\x00\x14\x00\x14\x00' + material = b'\x0d\x14\x04Coat' + _chunk(0x00010011, 1, texture) + library = b'\x0d\x14\x00\x14\x01' + _chunk(0x00010010, 1, material) + assert read_model(_chunk(0x0001000F, 0, library)).materials == {'Coat': ''} + + +def test_read_model_steps_over_a_string_before_a_texture_chunk() -> None: + texture = b'\x0d\x14\x00\x14\x00\x14\x00\x14\x01\x0d\x14\x05a.png' + material = b'\x0d\x14\x04Coat\x0d\x14\x04note' + _chunk(0x00010011, 1, texture) + library = b'\x0d\x14\x00\x14\x01' + _chunk(0x00010010, 1, material) + assert read_model(_chunk(0x0001000F, 0, library)).materials == {'Coat': 'a.png'} + + +def test_read_model_gives_up_on_an_unknown_tag_before_a_texture_chunk() -> None: + material = b'\x0d\x14\x04Coat\x7f' + library = b'\x0d\x14\x00\x14\x01' + _chunk(0x00010010, 1, material) + assert read_model(_chunk(0x0001000F, 0, library)).materials == {'Coat': ''} + + +def test_read_model_ignores_a_chunk_it_does_not_know(make_model: Callable[..., bytes]) -> None: + assert read_model(make_model() + _chunk(0x00010012, 5, b'\x14\x00')).meshes + + +def test_read_model_ignores_a_mesh_chunk_it_does_not_know() -> None: + mesh = _chunk(0x00010000, 1, b'\x0d\x14\x04legs\x0d\x14\x00') + mesh += _chunk(0x0001000B, 0, b'\x14\x00') + assert read_model(_chunk(0x00010005, 1, mesh)).meshes[0].name == 'legs' + + +def test_read_model_rejects_a_chunk_header_cut_short(make_model: Callable[..., bytes]) -> None: + # Enough of a header to say `chunk` and not enough to say how long, which used to raise + # `struct.error` past the reader's own error type. + with pytest.raises(InvalidModelError, match='runs past the end'): + read_model(make_model()[:5]) + + +def test_read_model_drops_a_face_indexing_backwards(make_model: Callable[..., bytes]) -> None: + # A negative index is in range for Python and would quietly pick a vertex off the far end. + model = read_model(make_model(faces=((0, 1, -1),))) + assert model.meshes[0].faces == () + + +def test_read_model_ignores_a_negative_texture_coordinate_index( + make_model: Callable[..., bytes]) -> None: + model = read_model(make_model(coord_faces=((0, 1, -2),))) + assert model.meshes[0].faces[0].coords == (0, 0, 0) + + +def test_read_model_rejects_a_run_of_vectors_past_its_chunk() -> None: + # The count says a thousand vectors; the chunk holds one. Reading on regardless takes whatever + # follows the chunk and calls it geometry. + positions = _chunk(0x00010006, 1, b'\x02' + struct.pack(' None: + faces = _chunk(0x00010007, 1, b'\x02' + struct.pack(' Path: + """Lay out a game database holding one skin and one pickup.""" + skin = root / 'skins' / 'transit_cop' + skin.mkdir(parents=True) + (skin / 'transit_cop_l0.kfs').write_bytes(model) + (skin / 'transit_cop_l1.kfs').write_bytes(model) + shared = root / 'skins' / 'sharedtextures' + shared.mkdir(parents=True) + if textures: + (shared / 'skin.png').write_bytes(image) + items = root / 'level_items' + (items / 'ammo_ingram').mkdir(parents=True) + (items / 'ammo_ingram' / 'ammo_ingram_l0.kf2').write_bytes(model) + if script is not None: + (items / 'ammo_ingram.txt').write_text(script) + return root + + +def test_load_models_finds_a_skin_and_a_pickup(tmp_path: Path, make_ldb: Callable[..., bytes], + make_model: Callable[..., bytes]) -> None: + root = _database(tmp_path, make_model(), script='[LOD]\nExportData = ammo_ingram_l0.kf2;\n') + level = read_level(make_ldb()) + models = load_models(root, level) + assert sorted(models) == ['character:transit_cop', 'item:ammo_ingram'] + assert models['character:transit_cop'].meshes[0].faces + + +def test_load_models_reads_the_images_along_the_search_path( + tmp_path: Path, make_ldb: Callable[..., bytes], make_model: Callable[..., bytes]) -> None: + # The model looks in its own `textures` first and the shared directory beside it second. + root = _database(tmp_path, make_model()) + model = load_models(root, read_level(make_ldb()))['character:transit_cop'] + assert [t.path for t in model.textures] == ['skin.png'] + + +def test_load_models_tolerates_a_missing_image(tmp_path: Path, make_ldb: Callable[..., bytes], + make_model: Callable[..., bytes]) -> None: + root = _database(tmp_path, make_model(), textures=False) + assert load_models(root, read_level(make_ldb()))['character:transit_cop'].textures == () + + +def test_load_models_prefers_the_nearest_level_of_detail(tmp_path: Path, make_ldb: Callable[..., + bytes], + make_model: Callable[..., bytes]) -> None: + root = _database(tmp_path, make_model()) + (root / 'skins' / 'transit_cop' / 'transit_cop_l0.kfs').write_bytes(make_model(name='near')) + assert load_models(root, + read_level(make_ldb()))['character:transit_cop'].meshes[0].name == 'near' + + +def test_load_models_falls_back_when_no_lod_zero_exists(tmp_path: Path, make_ldb: Callable[..., + bytes], + make_model: Callable[..., bytes]) -> None: + root = _database(tmp_path, make_model()) + (root / 'skins' / 'transit_cop' / 'transit_cop_l0.kfs').unlink() + assert 'character:transit_cop' in load_models(root, read_level(make_ldb())) + + +def test_load_models_skips_a_skin_with_no_model(tmp_path: Path, make_ldb: Callable[..., bytes], + make_model: Callable[..., bytes]) -> None: + root = _database(tmp_path, make_model()) + for path in (root / 'skins' / 'transit_cop').iterdir(): + path.unlink() + assert 'character:transit_cop' not in load_models(root, read_level(make_ldb())) + + +def test_load_models_skips_a_missing_directory(tmp_path: Path, make_ldb: Callable[..., + bytes]) -> None: + assert load_models(tmp_path, read_level(make_ldb())) == {} + + +def test_load_models_skips_a_pickup_whose_script_names_nothing( + tmp_path: Path, make_ldb: Callable[..., bytes], make_model: Callable[..., bytes]) -> None: + root = _database(tmp_path, make_model(), script='[Properties]\nTriggerRadius = 1;\n') + assert 'item:ammo_ingram' not in load_models(root, read_level(make_ldb())) + + +def test_load_models_skips_a_pickup_whose_model_is_absent(tmp_path: Path, make_ldb: Callable[..., + bytes], + make_model: Callable[..., bytes]) -> None: + root = _database(tmp_path, make_model(), script='ExportData = gone.kf2;\n') + assert 'item:ammo_ingram' not in load_models(root, read_level(make_ldb())) + + +def test_load_models_skips_a_model_that_will_not_read(tmp_path: Path, make_ldb: Callable[..., + bytes], + make_model: Callable[..., bytes]) -> None: + root = _database(tmp_path, make_model()) + (root / 'skins' / 'transit_cop' / 'transit_cop_l0.kfs').write_bytes(b'not a model') + (root / 'skins' / 'transit_cop' / 'transit_cop_l1.kfs').unlink() + assert 'character:transit_cop' not in load_models(root, read_level(make_ldb())) diff --git a/tests/maxpane/test_ras.py b/tests/maxpane/test_ras.py new file mode 100644 index 0000000..b9d1f8f --- /dev/null +++ b/tests/maxpane/test_ras.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING +import struct + +import pytest + +from dade.maxpane.ras import ( + HEADER_SIZE, + InvalidArchiveError, + is_intact, + iter_members, + member_bytes, + read_directory, + read_header, +) + +if TYPE_CHECKING: + from collections.abc import Callable + + +def test_read_header(make_ras: Callable[..., bytes]) -> None: + header = read_header(make_ras()) + assert header.file_count == 2 + assert header.directory_count == 2 + assert header.version == pytest.approx(1.2) + assert header.archiver_id == 3 + + +def test_read_header_rejects_a_foreign_file() -> None: + with pytest.raises(InvalidArchiveError, match='Not a RAS archive'): + read_header(b'MZ\x90\x00' + bytes(64)) + + +def test_read_header_rejects_an_unknown_version(make_ras: Callable[..., bytes]) -> None: + with pytest.raises(InvalidArchiveError, match='Unsupported RAS archive version'): + read_header(make_ras(version=1.1)) + + +def test_read_directory_names_members(make_ras: Callable[..., bytes]) -> None: + contents = read_directory(make_ras()) + assert [entry.path for entry in contents.entries] == ['data/a.txt', 'data/b.bin'] + assert [directory.name for directory in contents.directories] == ['\\', '\\data\\'] + + +def test_read_directory_reads_timestamps(make_ras: Callable[..., bytes]) -> None: + assert read_directory(make_ras()).entries[0].modified == '2001-07-11 17:54:28.000' + + +def test_read_directory_treats_a_zero_year_as_unset(make_ras: Callable[..., bytes]) -> None: + contents = read_directory(make_ras(modified=False)) + assert contents.entries[0].modified is None + assert contents.directories[0].modified is None + + +def test_offsets_are_cumulative(make_ras: Callable[..., bytes]) -> None: + contents = read_directory(make_ras((('a', b'0123'), ('b', b'456789')))) + first, second = contents.entries + assert second.offset == first.offset + first.stored_size + + +def test_member_bytes(make_ras: Callable[..., bytes]) -> None: + archive = make_ras() + contents = read_directory(archive) + assert member_bytes(archive, contents.entries[0]) == b'hello' + assert member_bytes(archive, contents.entries[1]) == b'world' + + +def test_member_bytes_raw_keeps_wrappers(make_ras: Callable[..., bytes], + make_lzss: Callable[[bytes], bytes]) -> None: + stream = make_lzss(b'payload') + wrapped = b'RA->' + struct.pack(' None: + assert [(entry.name, data) for entry, data in iter_members(make_ras())] == [('a.txt', b'hello'), + ('b.bin', b'world')] + + +def test_is_intact(make_ras: Callable[..., bytes]) -> None: + assert is_intact(make_ras()) + + +def test_is_intact_detects_truncation(make_ras: Callable[..., bytes]) -> None: + assert not is_intact(make_ras()[:-1]) + + +def test_read_header_reads_the_table_checksums(make_ras: Callable[..., bytes]) -> None: + # The two words after the header's own CRC are checksums of the decrypted tables, which holds + # on all five shipped archives. + import zlib + + from dade.maxpane.crypto import decrypt + archive = make_ras() + header = read_header(archive) + start = 0x2C + files = decrypt(archive[start:start + header.file_table_size], header.seed) + directories = decrypt( + archive[start + header.file_table_size:start + header.file_table_size + + header.directory_table_size], header.seed) + assert header.file_crc == zlib.crc32(files) + assert header.directory_crc == zlib.crc32(directories) + + +def test_read_header_rejects_a_truncated_archive() -> None: + # The magic is right but there is no header behind it, which used to raise `struct.error`. + with pytest.raises(InvalidArchiveError, match='at least'): + read_header(b'RAS\x00' + bytes(8)) + + +def test_read_directory_rejects_an_entry_naming_a_directory_that_is_not_there( + make_ras: Callable[..., bytes]) -> None: + with pytest.raises(InvalidArchiveError, match='names directory'): + read_directory(make_ras(directory=7)) + + +def test_read_directory_rejects_an_archive_cut_short_of_its_tables( + make_ras: Callable[..., bytes]) -> None: + # Slicing a short buffer gives back a short table rather than failing, so the walk over it used + # to die inside a name with `ValueError: subsection not found`. + with pytest.raises(InvalidArchiveError, match='The tables need'): + read_directory(make_ras()[:HEADER_SIZE + 10]) + + +def test_read_directory_reports_a_table_it_cannot_walk(make_ras: Callable[..., bytes]) -> None: + # The tables are as long as the header promised and still nonsense inside: no name ends. That + # used to surface as a bare `ValueError` from the name reader. + with pytest.raises(InvalidArchiveError, match='The tables will not read'): + read_directory(make_ras(terminate=False)) diff --git a/tests/test_cli.py b/tests/test_cli.py index 2e3141d..aba9bfa 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -20,6 +20,7 @@ 'incoming', 'jubeatplus', 'marmalade', + 'maxpane', 'misc', 'monopoly08', 'rbplus',