bootstrap.py

#!/usr/bin/env python3
"""
bootstrap.py — bring a fresh clone up to a running reader.

The repository commits code plus the artifacts that cannot be regenerated
(the Morpheus lemma cache, the DTM the frozen LDA is refitted from, the LSJ
glosses, the derived priors).  Everything else is cached computation and is
rebuilt here, because 206 MB of parsed work texts has no business in git when a
script reproduces it in ten minutes.

    python -u bootstrap.py                 # everything except the slow steps
    python -u bootstrap.py --with-slow     # + embedding alignment (~2h) and the
                                           #   SPhilBerta passage index (~30 min)
    python -u bootstrap.py --only-texts    # just the readable texts

Ordering matters and is enforced: texts before alignment (alignment reads them),
gold before the priors (the priors are derived from it), and the LDA refit before
the reader can project passages.

Steps are idempotent — anything already present is skipped — so this is safe to
re-run and safe to interrupt, which matters for the two-hour alignment step.
"""

# Windows consoles default to cp1252, which raises UnicodeEncodeError the moment
# this script prints a Greek lemma.  Force UTF-8 on stdout/stderr rather than
# relying on the caller to set PYTHONUTF8.
import sys as _sys
for _s in (_sys.stdout, _sys.stderr):
    try:
        if (getattr(_s, "encoding", "") or "").lower().replace("-", "") != "utf8":
            _s.reconfigure(encoding="utf-8")
    except Exception:
        pass

import os
import sys
import glob
import json
import shutil
import argparse
import subprocess
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
DATA = os.path.join(HERE, "data")
GOLD = os.path.join(HERE, "gold")

AGDT = ("https://raw.githubusercontent.com/PerseusDL/treebank_data/master/"
        "v2.1/Greek/texts/")
GORMAN = ("https://raw.githubusercontent.com/perseids-publications/gorman-trees/"
          "master/public/xml/")

# Hand-annotated gold.  Not committed: the Gorman trees are CC BY-NC-SA and
# vendoring them would impose that licence on this repository.
GOLD_FILES = [
    (GORMAN + "plato-apology.xml", "plato-apology.xml"),
    (GORMAN + "aristotle-politics-book-1-bu1.xml", "aristotle-politics-1.xml"),
    (GORMAN + "aristotle-politics-book-2-bu2.xml", "aristotle-politics-2.xml"),
    (AGDT + "tlg0059.tlg001.perseus-grc1.tb.xml", "plato-euthyphro.tb.xml"),
    (AGDT + "tlg0012.tlg001.perseus-grc1.tb.xml", "homer-iliad.tb.xml"),
    (AGDT + "tlg0012.tlg002.perseus-grc1.tb.xml", "homer-odyssey.tb.xml"),
]


def run(cmd, label):
    print(f"\n\u25b6 {label}\n  $ {' '.join(cmd)}", flush=True)
    r = subprocess.run(cmd, cwd=HERE)
    if r.returncode != 0:
        print(f"  \u2717 {label} failed (exit {r.returncode})")
        return False
    return True


def have(path, min_bytes=1000):
    p = path if os.path.isabs(path) else os.path.join(HERE, path)
    return os.path.exists(p) and os.path.getsize(p) >= min_bytes


def step_deps():
    print("\u25b6 checking dependencies")
    ok = True
    for mod, why in [("fastapi", "reader backend"), ("uvicorn", "reader backend"),
                     ("numpy", "pipeline"), ("sklearn", "frozen LDA")]:
        try:
            __import__(mod)
            print(f"  \u2713 {mod:24} ({why})")
        except ImportError:
            print(f"  \u2717 {mod:24} ({why}) — pip install -r requirements.txt")
            ok = False
    for mod, why in [("torch", "embedding alignment"),
                     ("sentence_transformers", "embedding alignment")]:
        try:
            __import__(mod)
            print(f"  \u2713 {mod:24} ({why})")
        except ImportError:
            print(f"  \u2013 {mod:24} ({why}) — optional; --with-slow needs it")
    return ok


def step_gold():
    print("\u25b6 fetching gold treebanks")
    os.makedirs(GOLD, exist_ok=True)
    for url, name in GOLD_FILES:
        dest = os.path.join(GOLD, name)
        if have(dest, 5000):
            print(f"  \u2713 {name} (present)")
            continue
        try:
            urllib.request.urlretrieve(url, dest)
            print(f"  \u2713 {name} ({os.path.getsize(dest):,} B)")
        except Exception as e:
            print(f"  \u2717 {name}: {e!r}")


def registry_slugs():
    sys.path.insert(0, os.path.join(HERE, "backend"))
    try:
        from works import ordered_slugs
        return ordered_slugs()
    except Exception as e:
        print(f"  ! works.py not importable ({e})")
        return []


def step_texts():
    slugs = registry_slugs()
    missing = [s for s in slugs
               if not (have(f"data/{s}.json") and have(f"data/{s}_en.json"))]
    if not missing:
        print(f"\u25b6 texts: all {len(slugs)} registry works present")
        return True
    print(f"\u25b6 building {len(missing)} of {len(slugs)} works "
          f"(the rest are already present)")
    return run([sys.executable, "-u", "build_texts.py", "--only", *missing],
               "build_texts.py")


def step_glosses():
    if have("data/glosses.json", 1_000_000):
        print("\u25b6 glosses: present")
        return True
    if not os.path.exists(os.path.join(HERE, "build_glosses.py")):
        print("\u25b6 glosses: MISSING and build_glosses.py is absent — the reader will"
              " run but hover glosses and lexical alignment will be empty")
        return False
    return run([sys.executable, "-u", "build_glosses.py"], "build_glosses.py")


def step_model():
    if have("data/model/lda_model.pkl", 10_000):
        print("\u25b6 frozen LDA: present")
        return True
    if not have("data/dtm_baseline.npy", 10_000):
        print("\u25b6 frozen LDA: cannot refit — data/dtm_baseline.npy is missing."
              "\n  Concept-space projection will be unavailable; everything else works.")
        return False
    return run([sys.executable, "-u", "serialize_model.py",
                "--artifacts", "data", "--out", "data/model"], "serialize_model.py")


def step_priors():
    # PROSE prior must stay prose-only: a Homer-weighted prior measured 4 points
    # WORSE on Attic prose, because Homeric ἑ drags the ὁ/ἑ frequency ratio from
    # 3047x down to 9x — just under the promotion gate.
    prose = sorted(glob.glob(os.path.join(GOLD, "plato-*.xml"))
                   + glob.glob(os.path.join(GOLD, "aristotle-*.xml")))
    verse = sorted(glob.glob(os.path.join(GOLD, "homer-*.tb.xml")))
    ok = True
    if prose and not have("data/lemma_prior.json", 5000):
        ok &= run([sys.executable, "-u", "build_lemma_prior.py",
                   "--gold", *prose, "--out", "data/lemma_prior.json"],
                  "build_lemma_prior.py (prose)")
    else:
        print("\u25b6 prose prior: present" if prose else "\u25b6 prose prior: no gold")
    if verse and not have("data/lemma_prior_verse.json", 5000):
        ok &= run([sys.executable, "-u", "build_lemma_prior.py",
                   "--gold", *verse, "--out", "data/lemma_prior_verse.json"],
                  "build_lemma_prior.py (verse)")
    else:
        print("\u25b6 verse prior: present" if verse else "\u25b6 verse prior: no gold")
    # form->lemma overlay: pooling every source is fine here, since it maps forms
    # rather than counting frequencies, so more sources only add coverage
    allg = sorted(glob.glob(os.path.join(GOLD, "*.xml")))
    if allg and not have("data/lemma_gold.json", 5000):
        ok &= run([sys.executable, "-u", "build_gold_cache.py",
                   "--gold", *allg, "--out", "data/lemma_gold.json"],
                  "build_gold_cache.py")
    else:
        print("\u25b6 gold overlay: present" if allg else "\u25b6 gold overlay: no gold")
    return ok


def step_embeddings():
    if have("data/corpus_emb/embeddings.npy", 100_000):
        print("\u25b6 SPhilBerta index: present")
        return True
    if not os.path.exists(os.path.join(HERE, "build_corpus_embeddings.py")):
        print("\u25b6 SPhilBerta index: build_corpus_embeddings.py absent — skipping")
        return False
    return run([sys.executable, "-u", "build_corpus_embeddings.py"],
               "build_corpus_embeddings.py (~30 min)")


def step_alignment():
    # resumable: build_alignment.py skips works whose edges already exist
    env = dict(os.environ)
    env.setdefault("ALIGN_ITERS", "6")
    print("\n\u25b6 embedding alignment (~2h for the full registry; resumable)")
    print("  $ ALIGN_ITERS=6 python -u build_alignment.py")
    r = subprocess.run([sys.executable, "-u", "build_alignment.py"], cwd=HERE, env=env)
    return r.returncode == 0


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--with-slow", action="store_true",
                    help="also run embedding alignment and the passage index")
    ap.add_argument("--only-texts", action="store_true")
    ap.add_argument("--skip-gold", action="store_true")
    args = ap.parse_args()

    print("=" * 66)
    print("Greek reader — bootstrap")
    print("=" * 66)
    if not step_deps():
        print("\ninstall the required packages first:  pip install -r requirements.txt")
        return 1

    if args.only_texts:
        step_texts(); return 0

    if not args.skip_gold:
        step_gold()
    step_texts()
    step_glosses()
    step_model()
    step_priors()

    if args.with_slow:
        step_embeddings()
        step_alignment()
    else:
        print("\n\u25b6 skipped (use --with-slow):")
        print("    embedding alignment    ~2h   \u2014 the amber cross-language links,")
        print("                                    and the evidence the cross-lingual")
        print("                                    disambiguation layer runs on")
        print("    SPhilBerta index       ~30m  \u2014 semantic / cross-work retrieval")

    print("\n" + "=" * 66)
    print("Next:")
    print("  1. start the reader:")
    print("     LEMMA_CACHE_PATH=data/lemma_cache.json ENABLE_EMBEDDINGS=1 \\")
    print("       uvicorn app:app --app-dir backend --port 8000")
    print("  2. open http://127.0.0.1:8000 (Chrome; Safari breaks the Colab proxy)")
    print("  3. verify:  python -u healthcheck.py")
    print("=" * 66)
    return 0


if __name__ == "__main__":
    sys.exit(main())