riffanim.py

#!/usr/bin/env python3
"""riffanim.py -- read the `RIFF`/`ANIM` movie container shipped by
`The Secrets of Hosea Freeman` (Tandy VIS, 1992), the platform's third
moving-picture format and the first that is neither ICOM's `.IMV` nor Video
for Windows.

Nothing in the 418-file toolbox reads it. `avi.py` raises an uncaught
`ValueError` on one (exit 1); `avicheck.py` prints `NOT AN AVI` per file and
then a full summary table of zeroes ending `files where the two disagree: 0 of
1` (exit 0), which is a confident-looking answer about a file it never opened;
`imv.py --validate` fails the file (exit 1) but three of its nine checks pass
vacuously on it. All three were run and recorded before this was written.

THE CONTAINER, derived from the bytes and not from a specification:

    'RIFF' <u32 size> 'ANIM'
    'VHDR' <u32 2>   u16          a version or magic stamp; 1234 on every file
    'RATE' <u32 2>   u16          frames per second
    'AUDH' <u32 18>  u16 + WAVEFORMATEX(16)
    then a flat sequence, no LIST and no nesting, of
      'AUD ' <u32 n>  audio payload
      'FRAM' <u32 n>  video payload
    'ENDV' <u32 0>                terminator, last chunk in the file

    Chunk headers are 8 bytes, sizes are little-endian, odd sizes pad to even.

THE FRAME, likewise derived:

    +0  u16   destination x    -- together a u32 that reads 0 on a
    +2  u16   destination y    -- full-screen frame
    +4  u16   width
    +6  u16   height
    +8  u32   restated chunk payload length
    +12 u16   palette entries, 0 meaning "keep the one already in force"
    +14 u16   unidentified
    +16 u8    one byte, 0x00 on every frame measured
    +17 ...   palette, 3 bytes per entry, **VGA DAC units, 0..63**
              then width*height 8-bit pixels, UNCOMPRESSED

    so  16 + 1 + 3*npal + width*height == the chunk payload length, exactly.

THE BYTE AT +16 IS BEFORE THE PALETTE AND NOT AFTER IT, and the arithmetic
above cannot tell you which: both readings close on 59 of 59 files, 8,040
frames and every byte of every chunk. The wrong one was found by rendering a
frame, looking at it, and comparing four pixels against the same scene's `.BID`
still, which carries the identical palette indices under an 8-bit Windows
palette. Entry 255 read (14, 45, 45) under the wrong offset and (45, 45, 45)
under the right one, and only one of those is the grey the picture needs.

The palette is 6-bit VGA DAC, not 8-bit: the largest component in the frame
measured is 45 of a possible 63, while the same scene's `.BID` palette reaches
182 of 255. Expand with `v << 2 | v >> 4`, which turns 45 into 182 exactly.
That is the strongest evidence on the disc that these films go out through
`DispDib` and segment 0xA000 rather than through GDI: they carry hardware
palette values.

There is no compression anywhere in this format and there are no delta pixels:
a frame is a raw rectangular blit at (x, y) onto a framebuffer that PERSISTS
between chunks, under a palette that also persists whenever npal is 0. That is
state carried between records, which no per-record census can check, which is
why `--render` exists and why the frames were looked at before this docstring
was written.

`--validate` is the pass that must fail on something. Give it a `.WAV` from
the same disc -- the same RIFF container with form type `WAVE` -- and it must
refuse. A reader with no demonstrated refusal path is not a measurement.

    python tools/riffanim.py FILE [FILE ...]
    python tools/riffanim.py --validate FILE [FILE ...]
    python tools/riffanim.py --census DIR
    python tools/riffanim.py --frames FILE          per-frame table
    python tools/riffanim.py --render FILE OUTDIR   PNG per frame, needs PIL
"""
import argparse
import collections
import os
import struct
import sys

MAGIC = b"RIFF"
FORM = b"ANIM"


class Refused(Exception):
    """Raised when the file is not a RIFF/ANIM. Loud on purpose."""


def chunks(d, path):
    if len(d) < 12:
        raise Refused("%s: %d bytes, too short for a RIFF header" % (path, len(d)))
    if d[:4] != MAGIC:
        raise Refused("%s: not RIFF, first four bytes %s" % (path, d[:4].hex()))
    if d[8:12] != FORM:
        raise Refused("%s: RIFF form type is %r, not %r" % (path, d[8:12], FORM))
    declared = struct.unpack("<I", d[4:8])[0]
    off = 12
    out = []
    while off + 8 <= len(d):
        fid = d[off:off + 4]
        n = struct.unpack("<I", d[off + 4:off + 8])[0]
        if off + 8 + n > len(d):
            raise Refused("%s: chunk %r at %d claims %d bytes, %d remain"
                          % (path, fid, off, n, len(d) - off - 8))
        out.append((off, fid, n))
        off += 8 + n + (n & 1)
    return declared, out, off


def read(path):
    with open(path, "rb") as f:
        d = f.read()
    declared, cs, end = chunks(d, path)
    return d, declared, cs, end


def frame_header(d, off, n):
    x, y, w, h, restated, npal, unk = struct.unpack("<HHHHIHH",
                                                    d[off + 8:off + 24])
    return x, y, w, h, restated, npal, unk


def one(path, verbose=True):
    d, declared, cs, end = read(path)
    kinds = collections.Counter(f for _, f, _ in cs)
    bykind = collections.Counter()
    for _, f, n in cs:
        bykind[f] += n
    rate = None
    vhdr = None
    fmt = None
    for off, fid, n in cs:
        if fid == b"RATE" and n == 2:
            rate = struct.unpack("<H", d[off + 8:off + 10])[0]
        elif fid == b"VHDR" and n == 2:
            vhdr = struct.unpack("<H", d[off + 8:off + 10])[0]
        elif fid == b"AUDH" and n == 18:
            fmt = struct.unpack("<HHIIHH", d[off + 10:off + 26])
    frames = [(o, n) for o, f, n in cs if f == b"FRAM"]
    geo = collections.Counter()
    keys = 0
    for o, n in frames:
        x, y, w, h, restated, npal, unk = frame_header(d, o, n)
        geo[(w, h)] += 1
        if npal:
            keys += 1
    if verbose:
        print("file             : %s" % path)
        print("size             : %d bytes" % len(d))
        print("RIFF size + 8    : %d   %s" % (declared + 8,
              "== file" if declared + 8 == len(d) else "!= file (%d)" % len(d)))
        print("chain ends at    : %d   %s" % (end,
              "== file" if end == len(d) else "!= file"))
        print("VHDR             : %s" % vhdr)
        print("RATE             : %s frames/s" % rate)
        if fmt:
            print("AUDH             : tag %d, %d channel(s), %d Hz, %d B/s, "
                  "align %d, %d bits" % fmt)
        print("chunks           : %d  %s" % (len(cs),
              " ".join("%s=%d" % (k.decode("latin1").strip(), v)
                       for k, v in kinds.most_common())))
        print("payload bytes    : %s" % " ".join(
            "%s=%d" % (k.decode("latin1").strip(), v)
            for k, v in bykind.most_common()))
        print("frame geometry   : %s" % " ".join(
            "%dx%d=%d" % (w, h, c) for (w, h), c in geo.most_common()))
        print("frames carrying a palette : %d of %d" % (keys, len(frames)))
        if rate and frames:
            print("duration, video  : %.2f s at RATE" % (len(frames) / float(rate)))
        if fmt and bykind[b"AUD "]:
            print("duration, audio  : %.2f s at %d B/s"
                  % (bykind[b"AUD "] / float(fmt[3]), fmt[3]))
    return dict(path=path, size=len(d), declared=declared, end=end, rate=rate,
                vhdr=vhdr, fmt=fmt, kinds=kinds, bykind=bykind, geo=geo,
                keys=keys, nframes=len(frames), chunks=len(cs))


def validate(paths):
    checks = collections.Counter()
    total = 0
    refused = []
    for p in paths:
        total += 1
        try:
            d, declared, cs, end = read(p)
        except Refused as e:
            refused.append(str(e))
            continue
        checks["opened"] += 1
        if declared + 8 == len(d):
            checks["RIFF size + 8 == file length"] += 1
        if end == len(d):
            checks["chunk chain ends exactly at EOF"] += 1
        ids = [f for _, f, _ in cs]
        if ids[:2] == [b"VHDR", b"RATE"]:
            checks["opens VHDR then RATE"] += 1
        if ids[:3] == [b"VHDR", b"RATE", b"AUDH"]:
            checks["AUDH third (a film with sound)"] += 1
        if ids and ids[-1] == b"ENDV":
            checks["last chunk is ENDV"] += 1
        if set(ids) <= {b"VHDR", b"RATE", b"AUDH", b"AUD ", b"FRAM", b"ENDV"}:
            checks["no chunk id outside the known six"] += 1
        ok = True
        fits = True
        restates = True
        for o, f, n in cs:
            if f != b"FRAM":
                continue
            x, y, w, h, restated, npal, unk = frame_header(d, o, n)
            if restated != n:
                restates = False
            if 16 + npal * 3 + 1 + w * h != n:
                ok = False
            if x + w > 320 or y + h > 200:
                fits = False
        if ok:
            checks["every FRAM: 16 + 3*npal + 1 + w*h == length"] += 1
        if fits:
            checks["every FRAM rectangle fits inside 320x200"] += 1
        if restates:
            checks["every FRAM restates its own length at +8"] += 1
    print("files given                                    : %d" % total)
    print("REFUSED (not RIFF/ANIM)                        : %d" % len(refused))
    for r in refused[:10]:
        print("    %s" % r)
    print("opened                                         : %d" % checks["opened"])
    for k in ("RIFF size + 8 == file length",
              "chunk chain ends exactly at EOF",
              "opens VHDR then RATE",
              "AUDH third (a film with sound)",
              "last chunk is ENDV",
              "no chunk id outside the known six",
              "every FRAM: 16 + 3*npal + 1 + w*h == length",
              "every FRAM restates its own length at +8",
              "every FRAM rectangle fits inside 320x200"):
        print("%-46s : %d / %d" % (k, checks[k], checks["opened"]))
    return 0 if checks["opened"] and not refused else 1


def census(paths):
    rows = []
    for p in paths:
        try:
            rows.append(one(p, verbose=False))
        except Refused as e:
            print("REFUSED %s" % e)
    if not rows:
        raise SystemExit("riffanim: nothing opened")
    print("files                    : %d" % len(rows))
    print("bytes                    : %d" % sum(r["size"] for r in rows))
    print("chain closes at EOF      : %d / %d"
          % (sum(1 for r in rows if r["end"] == r["size"]), len(rows)))
    print("RIFF size + 8 == length  : %d / %d"
          % (sum(1 for r in rows if r["declared"] + 8 == r["size"]), len(rows)))
    vh = collections.Counter(r["vhdr"] for r in rows)
    rt = collections.Counter(r["rate"] for r in rows)
    fm = collections.Counter(r["fmt"] for r in rows)
    geo = collections.Counter()
    for r in rows:
        geo.update(r["geo"])
    print("VHDR values              : %s" % dict(vh))
    print("RATE values              : %s" % dict(rt))
    print("AUDH formats             : %s" % dict(fm))
    print("frame geometries         : %s"
          % {"%dx%d" % k: v for k, v in geo.most_common()})
    nf = sum(r["nframes"] for r in rows)
    nk = sum(r["keys"] for r in rows)
    print("frames                   : %d   key frames %d (%.4f %%)"
          % (nf, nk, 100.0 * nk / nf if nf else 0))
    ab = sum(r["bykind"][b"AUD "] for r in rows)
    fb = sum(r["bykind"][b"FRAM"] for r in rows)
    tot = sum(r["size"] for r in rows)
    print("audio payload bytes      : %d  (%.4f %% of the files)"
          % (ab, 100.0 * ab / tot))
    print("video payload bytes      : %d  (%.4f %%)" % (fb, 100.0 * fb / tot))
    print("container overhead       : %d  (%.4f %%)"
          % (tot - ab - fb, 100.0 * (tot - ab - fb) / tot))
    # Two populations, and they are not timed by the same instrument. Films
    # with sound can be timed from their audio byte rate; silent ones can only
    # be timed from RATE. Averaging the two into one figure would be a number
    # about nothing, which is the shape of mistake this pipeline keeps making.
    snd = [r for r in rows if r["fmt"]]
    mute = [r for r in rows if not r["fmt"]]
    print("films with sound         : %d of %d, %d bytes"
          % (len(snd), len(rows), sum(r["size"] for r in snd)))
    print("films with no AUDH       : %d of %d, %d bytes"
          % (len(mute), len(rows), sum(r["size"] for r in mute)))
    rateset = set(r["fmt"][3] for r in snd)
    if len(rateset) == 1:
        br = rateset.pop()
        secs = sum(r["bykind"][b"AUD "] for r in snd) / float(br)
        sb = sum(r["size"] for r in snd)
        print("duration, sound films    : %.1f s = %.2f min at %d B/s"
              % (secs, secs / 60.0, br))
        print("mean byte rate, sound    : %.1f B/s = %.4fx of 153,600"
              % (sb / secs, (sb / secs) / 153600.0))
        per = [(r["size"] / (r["bykind"][b"AUD "] / float(br)), r["path"])
               for r in snd if r["bykind"][b"AUD "]]
        w = max(per)
        print("worst single sound film  : %s  %.1f B/s = %.4fx"
              % (os.path.basename(w[1]), w[0], w[0] / 153600.0))
        fps = [r["nframes"] / (r["bykind"][b"AUD "] / float(br))
               for r in snd if r["bykind"][b"AUD "]]
        print("real frames per second   : min %.3f  mean %.3f  max %.3f"
              % (min(fps), sum(fps) / len(fps), max(fps)))
    msecs = sum(r["nframes"] / float(r["rate"]) for r in mute if r["rate"])
    if msecs:
        mb = sum(r["size"] for r in mute)
        print("duration, silent films   : %.1f s = %.2f min, from RATE"
              % (msecs, msecs / 60.0))
        print("mean byte rate, silent   : %.1f B/s = %.4fx of 153,600"
              % (mb / msecs, (mb / msecs) / 153600.0))
    return 0


def frames_table(path):
    d, declared, cs, end = read(path)
    print("%-8s %-10s %6s %6s %6s %6s %6s" %
          ("offset", "flags", "width", "height", "npal", "unk", "bytes"))
    for o, f, n in cs:
        if f != b"FRAM":
            continue
        flags, w, h, restated, npal, unk = frame_header(d, o, n)
        print("%-8d 0x%08x %6d %6d %6d %6d %6d" % (o, flags, w, h, npal, unk, n))
    return 0


def render(path, outdir):
    from PIL import Image
    d, declared, cs, end = read(path)
    os.makedirs(outdir, exist_ok=True)
    # The framebuffer and the palette PERSIST between chunks. Rebuilding
    # either one per record is the failure that passes every per-record census
    # and draws the wrong picture anyway.
    fb = bytearray(320 * 200)
    pal = [0] * 768
    i = 0
    for o, f, n in cs:
        if f != b"FRAM":
            continue
        x, y, w, h, restated, npal, unk = frame_header(d, o, n)
        base = o + 24 + 1              # the byte at +16 comes BEFORE the palette
        if npal:
            raw = d[base:base + npal * 3]
            pal = [(v << 2) | (v >> 4) for v in raw] + [0] * (768 - npal * 3)
        px = d[base + npal * 3: base + npal * 3 + w * h]
        # Two of the 59 films place 520 rectangles past row 200, by up to 35
        # rows. Clip rather than wrap: an unclipped bytearray slice write past
        # the end silently lengthens the buffer and moves every later row.
        for row in range(h):
            if not (0 <= y + row < 200):
                continue
            dst = (y + row) * 320 + x
            n_ = min(w, 320 - x)
            if n_ <= 0:
                continue
            fb[dst:dst + n_] = px[row * w:row * w + n_]
        im = Image.frombytes("P", (320, 200), bytes(fb))
        im.putpalette(pal)
        im.convert("RGB").save(os.path.join(outdir, "f%05d.png" % i))
        i += 1
    print("%d frames -> %s" % (i, outdir))
    return 0


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("paths", nargs="+")
    ap.add_argument("--validate", action="store_true")
    ap.add_argument("--census", action="store_true")
    ap.add_argument("--frames", action="store_true")
    ap.add_argument("--render", metavar="OUTDIR")
    a = ap.parse_args()
    paths = []
    for p in a.paths:
        if os.path.isdir(p):
            for root, _, files in os.walk(p):
                for fn in sorted(files):
                    if fn.upper().endswith(".VID"):
                        paths.append(os.path.join(root, fn))
        else:
            paths.append(p)
    if a.validate:
        return validate(paths)
    if a.census:
        return census(paths)
    if a.render:
        return render(paths[0], a.render)
    if a.frames:
        return frames_table(paths[0])
    for p in paths:
        one(p)
        print()
    return 0


if __name__ == "__main__":
    try:
        sys.exit(main())
    except Refused as e:
        print("REFUSED: %s" % e, file=sys.stderr)
        sys.exit(2)