From aa6a9cc61c627e7a82e8c4d16706f91d3a697025 Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 16 Oct 2026 11:40:53 -0400 Subject: [PATCH] =?UTF-8?q?Adiciona=20compacta=C3=A7=C3=A3o=20mono=20opcio?= =?UTF-8?q?nal=20e=20inspe=C3=A7=C3=A3o=20de=20geometria=20AWC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Estende build_awc_bytes com compact_streams sem alterar o comportamento padrão e rejeita o uso desse modo em bancos multicanal. Inclui inspeção de spans físicos com alinhamento e orçamento explícitos e compactação de chunks opacos sem recodificar áudio. Orçamentos desconhecidos não são aprovados automaticamente. Documenta evidências, limites e responsabilidade do consumidor; adiciona fixtures sintéticas para preservação, idempotência, endianness, limites de orçamento e tabelas inválidas. Validação na base oficial: 112 testes AWC aprovados e 5 testes de integração não selecionados. --- AWC_LOADING_GEOMETRY.md | 108 +++++++++++++++++++++++ fivefury/awc/io.py | 16 +++- fivefury/awc/loading_geometry.py | 90 +++++++++++++++++++ tests/audio/awc/test_loading_geometry.py | 78 ++++++++++++++++ 4 files changed, 288 insertions(+), 4 deletions(-) create mode 100644 AWC_LOADING_GEOMETRY.md create mode 100644 fivefury/awc/loading_geometry.py create mode 100644 tests/audio/awc/test_loading_geometry.py diff --git a/AWC_LOADING_GEOMETRY.md b/AWC_LOADING_GEOMETRY.md new file mode 100644 index 0000000..4ae653e --- /dev/null +++ b/AWC_LOADING_GEOMETRY.md @@ -0,0 +1,108 @@ +# Compact mono AWC layout and loading geometry (proposal) + +## Scope + +`build_awc_bytes(awc, compact_streams=True)` places each mono stream's +chunks together. The logical chunk table order, stream IDs, payloads and +metadata are preserved. The default remains unchanged for compatibility. +Multichannel banks are deliberately rejected by this option: their shared +streaming blocks require a different loading model. + +Global chunk-type sorting can place one stream's small metadata far away +from its DATA. A loader reading the complete interval between those chunks +then needs much more space than the payload size suggests. Compact layout +removes intervening streams but does not guarantee that every individual +stream fits the application's buffer. + +`fivefury.awc.loading_geometry.inspect_loading_geometry` accepts an AWC +header/table, full file size, an explicit alignment and an optional buffer +budget. It checks table bounds and reports per-stream spans and the model +`aligned_header + stream_span`. A missing budget means unknown, not pass. +Multichannel streams do not receive mono-budget verdicts. Whole-file +encrypted headers must be decrypted before inspection. + +The caller must qualify this loading model, alignment and budget for its +application. This API does not discover runtime buffer allocation, promise +compatibility with every game version, or measure total process memory. +It never changes bitrate, pitch, channels, duration or audio content. + +## Evidence and limitations + +A private GTA V Enhanced 1.0.1158.16 test reproduced dialogue stutters and +crashes with a modified bank. Compact placement alone did not fix it. A +CodeWalker-built compact bank with two oversized entries recoded to 160 +kbit/s reduced the measured maximum requirement from 187,216 to 163,890 +bytes. The user completed the tested dialogue with all audio/subtitles, +without crash or stutter. The 175,136-byte conservative budget used there +is **not** a universal AWC limit or proof of the exact pool assigned to it. + +The Python compact writer subsequently rebuilt the approved M AWC byte for +byte (30,742,718 bytes), including all 788 entries. This establishes exact +artifact equivalence for that bank, not a new independent in-game test or +approval of other banks. Synthetic tests also exercise the geometry API. +The unchanged default writer also produces the same approved M bytes: this +bank's encrypted single-channel DATA already selects per-stream ordering. +Consequently, this experiment does not establish a default FiveFury ordering +bug as the crash cause. The explicit compact option additionally covers other +mono layouts; callers must inspect the final serialized artifact, including +any later incremental replacement, rather than infer safety from the writer. +No game assets, executables, dumps or encryption keys accompany this proposal. + +The production consumer subsequently qualified named wave-slot capacities +from the game's active DAT4 configuration against a private runtime capture. +It uses the conservative minimum for the supported speech family and the +smallest original-compatible scanner slot, not a global 175,136-byte rule. +Multichannel blocks are checked against their corresponding originals instead. +These application-specific policies belong in the consumer, not this API. +The cumulative 0.9.47 delivery structurally reviewed 1,587 banks and changed +23; that inventory is not evidence of an in-game playthrough of all banks. + +## Tests + +`python -m pytest tests/audio/awc/test_loading_geometry.py -q` + +Fixtures are synthesized PCM, not copied game content. Tests cover physical +span reduction, logical payload preservation, unknown budgets, exact budget +boundaries and malformed tables. The existing AWC unit suite must also pass. + +--- + +# Layout mono compacto e geometria de carregamento (proposta) + +A opção `compact_streams=True` reúne fisicamente os chunks de cada entrada +mono sem trocar o conteúdo lógico. A ordenação global por tipo pode separar +metadados e áudio por muitos bytes pertencentes a outras entradas, aumentando +o intervalo que o carregador precisa ler. Compactar elimina essa dispersão, +mas uma entrada individual ainda pode ultrapassar o orçamento disponível. + +O medidor recebe alinhamento e orçamento explícitos. Orçamento ausente é +desconhecido, não aprovação. Bancos multicanal não herdam o cálculo mono. +Não há redução automática de bitrate nem constante universal de memória. +A política de recodificação permanece no consumidor, com validação própria. + +O teste privado M passou no diálogo testado em GTA Enhanced 1.0.1158.16, +usando CodeWalker e duas recodificações para 160 kbit/s. O requisito máximo +medido caiu de 187.216 para 163.890 bytes. Isso não comprova o funcionamento +no jogo da nova opção Python, nem aprova todo o acervo. O orçamento de +175.136 bytes não deve ser generalizado. Posteriormente, o escritor Python +reproduziu o AWC M byte a byte (30.742.718 bytes, 788 entradas). Essa equivalência +é específica do M e não aprova automaticamente outros bancos. A contribuição usa somente fixtures +sintéticas e não distribui arquivos protegidos do jogo. + +O escritor padrão, sem a nova opção, também reproduz exatamente o M: a flag +de DATA cifrado mono desse banco já seleciona a ordenação por entrada. +Portanto, o teste não demonstra que a ordenação padrão do FiveFury causou +o crash. A opção explícita amplia a compactação para outros layouts mono; +é necessário medir o artefato final, inclusive depois de substituições +incrementais, em vez de presumir segurança pelo nome do empacotador. + +O envio upstream é restrito à API de geometria, compactação mono, documentação +e testes sintéticos; adaptações exclusivas do projeto ficam no fork. + +O consumidor de produção qualificou depois os slots nomeados da configuração +DAT4 ativa comparando-os à captura privada de execução. Usa o menor orçamento +da família Speech e o menor slot Scanner compatível com o original, não um +limite global de 175.136 bytes. Bancos multicanal são comparados aos blocos de +seus próprios originais. Essas políticas pertencem ao consumidor, não à API. +A entrega cumulativa 0.9.47 revisou estruturalmente 1.587 bancos e alterou 23; +isso não equivale a jogar e testar todas as suas falas. diff --git a/fivefury/awc/io.py b/fivefury/awc/io.py index 582ec72..0520448 100644 --- a/fivefury/awc/io.py +++ b/fivefury/awc/io.py @@ -255,10 +255,15 @@ def _layout_chunks( streams: list[AwcStream], data_offset: int, endian: str, + *, + compact_streams: bool = False, ) -> tuple[list[AwcChunk], dict[int, bytes], dict[int, AwcChunkInfo]]: all_chunks = [chunk for stream in streams for chunk in stream.chunks] owners = {id(chunk): stream for stream in streams for chunk in stream.chunks} - write_chunks = ( + # A mono loader may read the entire span between a stream's chunks. + # Global type ordering spreads tiny metadata across other streams' DATA. + # Keep table order unchanged; only physical placement changes in this mode. + write_chunks = all_chunks if compact_streams else ( sorted(all_chunks, key=lambda chunk: chunk.sort_order) if awc.multi_channel_flag or not awc.single_channel_encrypt_flag else all_chunks @@ -309,13 +314,15 @@ def _write_awc_tables( return out -def build_awc_bytes(awc: Awc) -> bytes: +def build_awc_bytes(awc: Awc, *, compact_streams: bool = False) -> bytes: from .state import serialized_state from .validation import validate_awc + if compact_streams and awc.multi_channel_flag: + raise ValueError('Compact stream layout requires a mono bank; streaming blocks use another contract') report = validate_awc(awc) if report.errors: - if (all(issue.code == "awc.codec.unsupported" for issue in report.errors) + if (not compact_streams and all(issue.code == "awc.codec.unsupported" for issue in report.errors) and awc._original_bytes is not None and awc._original_state == serialized_state(awc)): return awc._original_bytes @@ -332,7 +339,8 @@ def build_awc_bytes(awc: Awc) -> bytes: + sum(len(stream.chunks) * 8 for stream in streams) ) - write_chunks, payload_by_chunk, info_by_chunk = _layout_chunks(awc, streams, data_offset, endian) + write_chunks, payload_by_chunk, info_by_chunk = _layout_chunks( + awc, streams, data_offset, endian, compact_streams=compact_streams) out = _write_awc_tables(awc, streams, info_by_chunk, data_offset, endian) for chunk in write_chunks: info = info_by_chunk[id(chunk)] diff --git a/fivefury/awc/loading_geometry.py b/fivefury/awc/loading_geometry.py new file mode 100644 index 0000000..86edb15 --- /dev/null +++ b/fivefury/awc/loading_geometry.py @@ -0,0 +1,90 @@ +"""Inspect physical mono stream spans without decoding audio or assuming a pool. + +The caller supplies alignment and capacity for its qualified loading path. +This is not a prediction of total GPU/RAM consumption or streaming block use. +""" +import struct +from .constants import chunk_alignment + + +def compact_mono_payloads(data: bytes) -> bytes: + """Relocate opaque mono chunks without parsing/re-encoding codec payloads. + + This is useful for retail codec variants not supported by the authoring + validator. It certifies byte preservation, not codec validity or a budget. + Whole-file encryption and streaming banks are intentionally unsupported. + """ + geometry = inspect_loading_geometry(data, len(data), alignment=1) + if geometry['multichannel']: + raise ValueError('Opaque compaction supports only mono banks') + endian = '<' if data[:4] == b'ADAT' else '>' + _, _, flags, count, end = struct.unpack_from(endian+'4sHHII', data) + first = 16 + (2*count if flags & 1 else 0) + cursor = first + 4*count + result = bytearray(data[:end]) + for index in range(count): + info = struct.unpack_from(endian+'I',data,first+index*4)[0] + for _ in range(info >> 29): + packed = struct.unpack_from(endian+'Q',data,cursor)[0] + old, size, tag = packed & 0xfffffff, (packed >> 28) & 0xfffffff, packed >> 56 + alignment = chunk_alignment(tag) + if alignment: + result.extend(bytes(-len(result) % alignment)) + offset = len(result) + if offset > 0xfffffff: + raise ValueError('Compacted offset exceeds AWC field') + result.extend(data[old:old+size]) + struct.pack_into(endian+'Q',result,cursor,offset | (size << 28) | (tag << 56)) + cursor += 8 + return bytes(result) + + +def inspect_loading_geometry(header: bytes, file_size: int, *, alignment: int, + budget_bytes: int | None = None) -> dict: + if type(file_size) is not int or file_size < 16: + raise ValueError('File size must be an integer containing an AWC header') + if type(alignment) is not int or alignment <= 0: + raise ValueError('Alignment must be a positive integer') + if budget_bytes is not None and (type(budget_bytes) is not int or budget_bytes <= 0): + raise ValueError('Budget must be a positive integer or None') + if len(header) < 16: + raise ValueError('Truncated AWC header') + endian = {b'ADAT': '<', b'TADA': '>'}.get(bytes(header[:4])) + if endian is None: + raise ValueError('Encrypted or unknown header; decrypt before inspection') + _, version, flags, count, end = struct.unpack_from(endian + '4sHHII', header) + if version != 1 or not 16 <= end <= file_size or end > len(header): + raise ValueError('Invalid or incomplete header') + first = 16 + (2 * count if flags & 1 else 0) + cursor = first + count * 4 + if cursor > end: + raise ValueError('Truncated stream table') + overhead = end + (-end % alignment) + rows = [] + for index in range(count): + info = struct.unpack_from(endian + 'I', header, first + index * 4)[0] + chunks = [] + for _ in range(info >> 29): + if cursor + 8 > end: + raise ValueError('Truncated chunk table') + raw = struct.unpack_from(endian + 'Q', header, cursor)[0] + cursor += 8 + offset, size = raw & 0xfffffff, (raw >> 28) & 0xfffffff + if offset + size > file_size or (size and offset < end): + raise ValueError('Chunk outside payload') + if size: + chunks.append((offset, offset + size)) + span = max(b for _, b in chunks) - min(a for a, _ in chunks) if chunks else 0 + # Formula is intentionally explicit. Streaming banks are not judged + # against a mono-entry budget even when their tables can be parsed. + required = overhead + span if chunks and not flags & 4 else None + rows.append(dict(index=index, stream_id=f'{info & 0x1fffffff:08X}', + span_bytes=span, required_bytes=required, + exceeds_budget=(required > budget_bytes + if required is not None and budget_bytes is not None else None))) + if cursor != end: + raise ValueError('Header size differs from parsed tables') + return dict(flags=flags, multichannel=bool(flags & 4), header_bytes=end, + aligned_header_bytes=overhead, alignment=alignment, + budget_bytes=budget_bytes, streams=rows, + budget_qualified=budget_bytes is not None and not flags & 4) diff --git a/tests/audio/awc/test_loading_geometry.py b/tests/audio/awc/test_loading_geometry.py new file mode 100644 index 0000000..c4c64c1 --- /dev/null +++ b/tests/audio/awc/test_loading_geometry.py @@ -0,0 +1,78 @@ +import pytest +from fivefury.awc import Awc, AwcStream, build_awc_bytes, read_awc +from fivefury.awc.loading_geometry import inspect_loading_geometry, compact_mono_payloads + + +def fixture(): + return Awc([AwcStream.from_pcm(name, bytes(32000), sample_rate=32000) + for name in ('alpha', 'beta', 'gamma')]) + + +def test_compact_preserves_payload_and_reduces_spans(): + original = build_awc_bytes(fixture()) + compact = build_awc_bytes(fixture(), compact_streams=True) + reports = [inspect_loading_geometry(raw, len(raw), alignment=2048) + for raw in (original, compact)] + assert max(s['span_bytes'] for s in reports[1]['streams']) < max(s['span_bytes'] for s in reports[0]['streams']) + snapshots = [[(s.hash, [(c.type_value, c.to_payload()) for c in s.chunks]) + for s in read_awc(raw).streams] for raw in (original, compact)] + assert snapshots[0] == snapshots[1] + assert all(s['exceeds_budget'] is None for s in reports[1]['streams']) + + +def test_exact_budget_and_one_byte_over(): + raw = build_awc_bytes(fixture(), compact_streams=True) + r = inspect_loading_geometry(raw, len(raw), alignment=2048) + maximum = max(s['required_bytes'] for s in r['streams']) + assert not any(s['exceeds_budget'] for s in inspect_loading_geometry(raw, len(raw), alignment=2048, budget_bytes=maximum)['streams']) + assert any(s['exceeds_budget'] for s in inspect_loading_geometry(raw, len(raw), alignment=2048, budget_bytes=maximum-1)['streams']) + + +def test_truncation_rejected(): + raw = build_awc_bytes(fixture()) + with pytest.raises(ValueError): + inspect_loading_geometry(raw[:16], len(raw), alignment=2048) + with pytest.raises(ValueError): + inspect_loading_geometry(raw, 16, alignment=2048) + + +def test_opaque_compaction_preserves_every_payload(): + raw=build_awc_bytes(fixture()) + compact=compact_mono_payloads(raw) + def snap(data): + return [(s.hash,[(c.type_value,c.to_payload()) for c in s.chunks]) + for s in read_awc(data).streams] + assert snap(raw)==snap(compact) + assert compact_mono_payloads(compact)==compact + + +def test_streaming_not_qualified_as_mono(): + import struct + raw=bytearray(build_awc_bytes(fixture())) + flags=struct.unpack_from('']) +def test_encrypted_mono_default_already_compact(endian): + bank=fixture() + bank.endian=endian + bank.flags |= 2 + default=build_awc_bytes(bank) + explicit=build_awc_bytes(bank,compact_streams=True) + assert default==explicit + opaque=compact_mono_payloads(default) + assert opaque==default + assert [s.pcm_bytes() for s in read_awc(opaque).streams]==[s.pcm_bytes() for s in bank.streams] + + +@pytest.mark.parametrize('file_size',[None,True,-1,15,16.5]) +def test_invalid_file_size_rejected(file_size): + raw=build_awc_bytes(fixture()) + with pytest.raises(ValueError,match='File size'): + inspect_loading_geometry(raw,file_size,alignment=2048)