Makefile

# letterpress - corpus + training pipeline. Commands run packages via `python -m`.
# bash (not sh) so the train/resume recipes can `set -o pipefail`: without it the
# `... | tee log` pipelines exit with tee's status and a CRASHED run reports success.
SHELL := /bin/bash
PY     := .venv/bin/python
DATA   := data/medium
OUT    ?= data/medium_bin
PRESET ?= medium
# caffeinate holds a no-idle/no-display-sleep/no-system-sleep assertion for the
# wrapped command, so macOS does not tear down the Metal/MPS GPU context (and
# SIGTERM the process) when the screensaver / display sleep kicks in during idle.
CAFF := caffeinate -dims

.PHONY: help finalize finalize-large clean dedup index stats train-medium resume-medium train-large resume-large train-xlarge resume-xlarge train-2xlarge resume-2xlarge train-4xlarge resume-4xlarge prepare docker-train hf-publish hf-pull version test sample interact

help:
	@echo "Targets:"
	@echo "  make finalize       clean -> dedup -> index -> stats, ALL tiers (run after adding books)"
	@echo "  make finalize-large clean+dedup LARGE ONLY -> index -> stats (safe while medium trains)"
	@echo "  make clean          normalize chars / trim vocab / compress whitespace (in place, all tiers)"
	@echo "  make dedup          drop repeated long paragraphs within each tier (in place)"
	@echo "  make index          rebuild indices/<tier>.md for all five tiers"
	@echo "  make stats          corpus size vs target for all five tiers (medium 500M ... 4xlarge 8.2B)"
	@echo "  make train-medium   train the medium model into outputs/medium/ (logs in outputs/medium/logs/)"
	@echo "  make resume-medium  resume an interrupted medium run from its latest checkpoint"
	@echo "  make train-large    train the large model into outputs/large/"
	@echo "  make resume-large   resume an interrupted large run from its latest checkpoint"
	@echo "  make train-xlarge   train the ~100M xlarge model (ASPIRATIONAL: needs a big GPU + ~2B-char corpus)"
	@echo "  make resume-xlarge  resume an interrupted xlarge run from its latest checkpoint"
	@echo "  make train-2xlarge  train the ~200M 2xlarge model (ASPIRATIONAL: needs a datacenter GPU + ~4.26B-char corpus)"
	@echo "  make resume-2xlarge resume an interrupted 2xlarge run from its latest checkpoint"
	@echo "  make train-4xlarge  train the ~400M 4xlarge model (ASPIRATIONAL: needs a datacenter GPU + Chinchilla-scale corpus)"
	@echo "  make resume-4xlarge resume an interrupted 4xlarge run from its latest checkpoint"
	@echo "  make version PRESET=medium   archive the trained model into model_history/<preset>/v<N>/"
	@echo "  make test           run the pytest suite (tests/) + the model selftest"
	@echo "  make sample         generate from the medium model's best checkpoint"
	@echo "  make interact       open the REPL on the largest trained model present (2xlarge)"
	@echo "  make prepare        pre-tokenize a corpus to memmap .bin (DATA=... OUT=..._bin); train with --data-bin"
	@echo "  make docker-train   build the CUDA training image (Dockerfile.train)"
	@echo "  make hf-publish     publish the corpus to a HF dataset repo (REPO=<id> [TIERS=...] [PREPARED=xlarge]); needs hf auth"
	@echo "  make hf-pull        pull a tier from a HF dataset repo (REPO=<id> TIER=xlarge [PREPARED=1])"

# Full post-harvest pipeline, in order, over ALL tiers. clean runs before dedup so paragraph
# hashing operates on normalized text; index/stats reflect the final corpus.
finalize:
	$(PY) -m corpus clean --apply --dataset all
	$(PY) -m corpus dedup --apply --dataset all
	$(PY) -m corpus index
	$(PY) -m corpus stats

# Large-only finalize: clean + dedup touch ONLY data/large, so it is safe to run
# while a medium training run holds data/medium. index/stats only read the corpus.
finalize-large:
	$(PY) -m corpus clean --apply --dataset large
	$(PY) -m corpus dedup --apply --dataset large
	$(PY) -m corpus index
	$(PY) -m corpus stats

finalize-xlarge:
	$(PY) -m corpus clean --apply --dataset xlarge
	$(PY) -m corpus dedup --apply --dataset xlarge
	$(PY) -m corpus index
	$(PY) -m corpus stats

clean:
	$(PY) -m corpus clean --apply --dataset all

dedup:
	$(PY) -m corpus dedup --apply --dataset all

index:
	$(PY) -m corpus index

stats:
	$(PY) -m corpus stats

train-medium:
	@mkdir -p outputs/medium/logs
	set -o pipefail; $(CAFF) $(PY) -u -m training --preset medium --data $(DATA) 2>&1 | tee outputs/medium/logs/train.log

# Resume an interrupted run to max_iters from its latest checkpoint (e.g. after a
# crash/kill). Architecture + vocab come from the checkpoint; the data must match.
resume-medium:
	@mkdir -p outputs/medium/logs
	set -o pipefail; $(CAFF) $(PY) -u -m training --preset medium --data $(DATA) --resume outputs/medium/models/byob-lm.pt 2>&1 | tee -a outputs/medium/logs/train.log

train-large:
	@mkdir -p outputs/large/logs
	set -o pipefail; $(CAFF) $(PY) -u -m training --preset large --data data/large 2>&1 | tee outputs/large/logs/train.log

resume-large:
	@mkdir -p outputs/large/logs
	set -o pipefail; $(CAFF) $(PY) -u -m training --preset large --data data/large --resume outputs/large/models/byob-lm.pt 2>&1 | tee -a outputs/large/logs/train.log

# xlarge (~100M) is aspirational: it trains on data/large grown toward ~2B chars
# (XLARGE_TARGET) and wants a datacenter GPU. Same caffeinate + resume safety net.
train-xlarge:
	@mkdir -p outputs/xlarge/logs
	set -o pipefail; $(CAFF) $(PY) -u -m training --preset xlarge --data data/xlarge 2>&1 | tee outputs/xlarge/logs/train.log

resume-xlarge:
	@mkdir -p outputs/xlarge/logs
	set -o pipefail; $(CAFF) $(PY) -u -m training --preset xlarge --data data/xlarge --resume outputs/xlarge/models/byob-lm.pt 2>&1 | tee -a outputs/xlarge/logs/train.log

# 2xlarge (~200M) and 4xlarge (~400M) train on their own tiers and want a datacenter
# GPU; same caffeinate + resume safety net. Cloud recipes (pre-tokenized .bin, tuned
# params) live in cloud_training/runpod_runbook_{2xlarge,4xlarge}.md. 2xlarge v1 (val 0.8725) was
# trained this way on a rented H200.
train-2xlarge:
	@mkdir -p outputs/2xlarge/logs
	set -o pipefail; $(CAFF) $(PY) -u -m training --preset 2xlarge --data data/2xlarge 2>&1 | tee outputs/2xlarge/logs/train.log

resume-2xlarge:
	@mkdir -p outputs/2xlarge/logs
	set -o pipefail; $(CAFF) $(PY) -u -m training --preset 2xlarge --data data/2xlarge --resume outputs/2xlarge/models/byob-lm.pt 2>&1 | tee -a outputs/2xlarge/logs/train.log

train-4xlarge:
	@mkdir -p outputs/4xlarge/logs
	set -o pipefail; $(CAFF) $(PY) -u -m training --preset 4xlarge --data data/4xlarge 2>&1 | tee outputs/4xlarge/logs/train.log

resume-4xlarge:
	@mkdir -p outputs/4xlarge/logs
	set -o pipefail; $(CAFF) $(PY) -u -m training --preset 4xlarge --data data/4xlarge --resume outputs/4xlarge/models/byob-lm.pt 2>&1 | tee -a outputs/4xlarge/logs/train.log

# Archive the trained model + corpus context + metadata into model_history/<preset>/v<N>/
# so a later retrain never silently overwrites it.
version:
	$(PY) -m core.version --preset $(PRESET)

# pytest suite (shared core + corpus helpers) plus the end-to-end model selftest.
test:
	$(PY) -m pytest tests/ -q
	$(PY) -m inference.selftest

sample:
	$(PY) -m inference.sample

interact:
	$(PY) -m inference.interact

# Pre-tokenize a corpus into memmap-friendly train.bin/val.bin + meta.json (the
# cloud / large-corpus data path; train with `--data-bin $(OUT)`). The uint8 .bin
# matches from_sources byte-for-byte (vocab < 256 -> one char is one byte).
#   make prepare DATA=data/xlarge OUT=data/xlarge_bin
prepare:
	$(PY) -m core.prepare --data $(DATA) --out $(OUT)

# Build the CUDA training image (torch 2.8); push to a registry the pod can pull.
docker-train:
	docker build -f Dockerfile.train -t byob-train .

# Publish / pull the public-domain corpus to/from a Hugging Face dataset repo.
# Auth first (never commit a token):  hf auth login   (or export HF_TOKEN=...)
#   make hf-publish REPO=you/byob-pd-book-corpus PREPARED=xlarge
#   make hf-pull    REPO=you/byob-pd-book-corpus TIER=xlarge PREPARED=1
hf-publish:
	$(PY) -m corpus.hf_dataset publish --repo-id $(REPO) $(if $(TIERS),--tiers $(TIERS),) $(if $(PREPARED),--include-prepared $(PREPARED),)

hf-pull:
	$(PY) -m corpus.hf_dataset pull --repo-id $(REPO) --tier $(TIER) $(if $(PREPARED),--prepared,)

# 2xlarge finalize (the consolidated/curated slim tier); lower tiers are locked and the
# full 4xlarge master is left intact - clean/dedup/consolidation run on 2xlarge only.
finalize-2xlarge:
	$(PY) -m corpus clean --apply --dataset 2xlarge
	$(PY) -m corpus dedup --apply --dataset 2xlarge
	$(PY) -m corpus index
	$(PY) -m corpus stats