Skip to content

Commit 171be09

Browse files
gufrancogufranco
committed
feat(conformance): commit the manifest that says it is committed
The document folder carried a manifest and a fetch script, and its own README said "what is carried is the identity, which is what makes a copy checkable". Neither was carried. The folder is ignored by git, so both existed on one machine and nowhere else, and a fresh checkout had the twelve documents named in prose and no way to fetch or check any of them. They live beside the rest of the record now, where git keeps them, and they write into the folder that is not carried. Nothing about the documents changed; all twelve still verify. The script also claimed nineteen documents where the manifest holds twelve.
1 parent 71e296d commit 171be09

4 files changed

Lines changed: 623 additions & 1 deletion

File tree

‎README.md‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ Interpreters for the 65xx family, from the 6502 to the 65816, driveable from a c
44

55
[![CI](https://github.com/gufranco/mos65xx-python/actions/workflows/ci.yml/badge.svg)](https://github.com/gufranco/mos65xx-python/actions/workflows/ci.yml)
66

7-
**16** parts, **17,900,000** state cases and **17,870,080** cycle-exact cases, **0** unexplained failures, **1,206** tests, **100%** statement and branch coverage, no dependencies
7+
**16** parts, **17,900,000** state cases and **17,870,080** cycle-exact cases, **0** unexplained failures, **1,238** tests, **100%** statement and branch coverage, no dependencies
88

99
```python
1010
from mos65xx import Cpu, SparseMemory
@@ -286,6 +286,13 @@ This project ships no documents. Every claim about the hardware is traced to som
286286

287287
Every one of these is copyrighted by its publisher and not redistributable, which is why none is in this repository. Individual sentences are quoted in [`conformance/hardware.json`](conformance/hardware.json) with the page they came from.
288288

289+
Fetching them is a command rather than an exercise. [`conformance/documents.json`](conformance/documents.json) carries the full digest, the byte count and a fetchable address for each, and [`conformance/documents.py`](conformance/documents.py) brings all twelve down into `docs/`, which git ignores, and refuses anything whose digest does not match.
290+
291+
```bash
292+
python3 -m conformance.documents # fetch and verify every digest
293+
python3 -m conformance.documents --check # verify what is already here
294+
```
295+
289296
| Document | Date | Pages | SHA-256 |
290297
|:---------|:-----|------:|:--------|
291298
| [MOS Technology, Inc., *MCS6500 Microcomputer Family Hardware Manual*](https://archive.org/download/mcs-6500-family-hardware-manual-1976-01/MCS6500_family_hardware_manual_1976-01.pdf) | 1976-01 | 182 | `81ea570c9d68deff…` |

‎conformance/documents.json‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
{
2+
"note": "Every document this package is held to, by identity rather than by name. docs/documents.py brings each one down again and refuses anything whose digest does not match.",
3+
"authority": {
4+
"1": "A manufacturer document. What the maker of a part printed about it.",
5+
"2": "A recording taken off a real part, which is conformance/suites.json rather than a document, so nothing in this folder occupies that rung.",
6+
"3": "Independent research, for what no manufacturer printed."
7+
},
8+
"whyNotCarriedHere": "These are the publishers' documents, not this project's, and this repository is public. What is carried is the identity, which is what makes a copy checkable.",
9+
"readOn": "2026-08-21",
10+
"partsWithNoDocument": {
11+
"2a03": "Ricoh never published a data sheet for it. What is known about the part is that it is a 6502 with decimal mode disabled and audio hardware alongside, and the suite records it separately for exactly that reason. The MOS documents above cover the core it is built from.",
12+
"65802": "WDC's W65C802 is the 65816 core in a forty pin package with sixteen address lines brought out. None of the three W65C816S revisions here mentions it, and no separate data sheet for it was found. The 65816 documents cover the core; the packaging difference is what is undocumented here."
13+
},
14+
"documents": [
15+
{
16+
"file": "manufacturer/mcs6500-hardware-manual-1976.pdf",
17+
"rung": 1,
18+
"publisher": "MOS Technology, Inc.",
19+
"title": "MCS6500 Microcomputer Family Hardware Manual",
20+
"date": "1976-01",
21+
"retrievedFrom": "https://archive.org/download/mcs-6500-family-hardware-manual-1976-01/MCS6500_family_hardware_manual_1976-01.pdf",
22+
"covers": ["6502", "6507", "2a03"],
23+
"settles": "The NMOS parts. Appendix A gives the address bus, the data bus and the read write line for every cycle of every addressing mode, which is the only manufacturer statement of NMOS cycle behaviour this project has found.",
24+
"sha256": "81ea570c9d68deff64d67365bdf24534df93a8c62121e53036aeac6a557cea23",
25+
"bytes": 7532642,
26+
"pages": 182,
27+
"redistributable": false
28+
},
29+
{
30+
"file": "manufacturer/mcs6500-programming-manual-1976.pdf",
31+
"rung": 1,
32+
"publisher": "MOS Technology, Inc.",
33+
"title": "MCS6500 Microcomputer Family Programming Manual",
34+
"date": "1976-01",
35+
"retrievedFrom": "https://archive.org/download/6500-50A_MCS6500_Programming_Manual_1976_Jan/6500-50A_MCS6500_Programming_Manual_1976_Jan.pdf",
36+
"covers": ["6502", "6507", "2a03"],
37+
"settles": "What each documented instruction does and which flags it writes, in prose rather than in a table, which is where the decimal mode rules and the interrupt sequence are stated.",
38+
"sha256": "a2d54dd8b6557c7f2f275b4ddaa9180b5754274a762df122d79392f03264b483",
39+
"bytes": 10933392,
40+
"pages": 262,
41+
"redistributable": false
42+
},
43+
{
44+
"file": "manufacturer/mos-6500-mpu-nov-1985.pdf",
45+
"rung": 1,
46+
"publisher": "MOS Technology, Inc.",
47+
"title": "MOS 6500 Microprocessors",
48+
"date": "1985-11",
49+
"retrievedFrom": "https://6502.org/documents/datasheets/mos/mos_6500_mpu_nov_1985.pdf",
50+
"covers": ["6502", "6503", "6504", "6505", "6506", "6507", "6512", "6513", "6514", "6515"],
51+
"settles": "Which parts the NMOS family has and what separates them. Its per-model pages give each package its addressable range and the interrupt lines it brings out, which is the whole of what a program can tell apart.",
52+
"sha256": "ccd72376d1e5db1fb13777de538417f4799ab38e1f746452f9695465c422dcd7",
53+
"bytes": 6261893,
54+
"pages": 12,
55+
"redistributable": false
56+
},
57+
{
58+
"file": "manufacturer/mos-6500-mpu-mar-1980.pdf",
59+
"rung": 1,
60+
"publisher": "MOS Technology, Inc.",
61+
"title": "MOS 6500 Microprocessors",
62+
"date": "1980-03",
63+
"retrievedFrom": "https://6502.org/documents/datasheets/mos/mos_6500_mpu_mar_1980.pdf",
64+
"covers": ["6502", "6503", "6504", "6505", "6506", "6507", "6512", "6513", "6514", "6515"],
65+
"settles": "Which parts the NMOS family has and what separates them. Its per-model pages give each package its addressable range and the interrupt lines it brings out, which is the whole of what a program can tell apart.",
66+
"sha256": "a564c4de593ea178868a43eaf7e00fe7d3460016d2a7880d20eb756f2bf2d030",
67+
"bytes": 17496024,
68+
"pages": 12,
69+
"redistributable": false
70+
},
71+
{
72+
"file": "manufacturer/synertek-sy6500-microprocessors.pdf",
73+
"rung": 1,
74+
"publisher": "Synertek, Inc.",
75+
"title": "SY6500 Microprocessors",
76+
"retrievedFrom": "https://6502.org/documents/datasheets/synertek/synertek_sy6500_microprocessors.pdf",
77+
"covers": ["6502", "6503", "6504", "6505", "6506", "6512", "6513", "6514", "6515"],
78+
"settles": "That the family is nine parts rather than ten. Synertek names the same packages MOS does, in the same two groups by clock, and has no 6507 at all. So the part in the Atari is one MOS made alone, and a second source naming the other nine independently is what makes the rest of the list more than one publisher's word.",
79+
"sha256": "f20b50961df60bb0e1980f271ad49b4b788c46a143127ccc58e8d3ce84d99193",
80+
"bytes": 11958322,
81+
"pages": 6,
82+
"redistributable": false
83+
},
84+
{
85+
"file": "manufacturer/synertek-hardware-manual.pdf",
86+
"rung": 1,
87+
"publisher": "Synertek, Inc.",
88+
"title": "SY6500 Hardware Manual",
89+
"retrievedFrom": "https://6502.org/documents/datasheets/synertek/synertek_hardware_manual.pdf",
90+
"covers": ["6502"],
91+
"settles": "Nothing on its own. It is MOS's hardware manual reprinted under a second name, down to the chapter titled Bringing Up the MCS6500 and an Appendix A that matches page for page, so it is not an independent witness to anything. That is worth knowing rather than assuming: the places where MOS's appendix contradicts itself are in this printing too, which rules out a misprint in one publisher's copy.",
92+
"sha256": "9cd0991dcbb4a46bf63747528282ad175ed15e1957d08bd755ef14b978b67a1c",
93+
"bytes": 8873553,
94+
"pages": 178,
95+
"redistributable": false
96+
},
97+
{
98+
"file": "manufacturer/rockwell-r65c00-microprocessors.pdf",
99+
"rung": 1,
100+
"publisher": "Rockwell International",
101+
"title": "R65C00 Microprocessors: R65C02, R65C102 and R65C112",
102+
"date": "1987-06",
103+
"retrievedFrom": "https://6502.org/documents/datasheets/rockwell/rockwell_r65c00_microprocessors.pdf",
104+
"covers": ["r65c02"],
105+
"settles": "Rockwell's own CMOS part, which the suite records separately because it is not the same as Synertek's or WDC's: Rockwell added the bit instructions the other two do not have. Its three members, the R65C02, R65C102 and R65C112, all reach sixty four kilobytes and differ in their clock and in bus arbitration, so a program cannot tell them apart and one model covers all three.",
106+
"note": "A scan with no usable text layer. Anything taken from it was read off the rendered page image.",
107+
"sha256": "63deee76c8d0d4faee7bb8f9c50ff21662c73f2e82093ebacfb284f527faf134",
108+
"bytes": 2554392,
109+
"pages": 16,
110+
"redistributable": false
111+
},
112+
{
113+
"file": "manufacturer/w65c02s.pdf",
114+
"rung": 1,
115+
"publisher": "The Western Design Center, Inc.",
116+
"title": "W65C02S 8-bit Microprocessor Data Sheet",
117+
"date": "2022-04-08",
118+
"retrievedFrom": "https://westerndesigncenter.com/wdc/documentation/w65c02s.pdf",
119+
"covers": ["w65c02", "65c02"],
120+
"settles": "The CMOS part as WDC makes it today, including the operational enhancements table this project cites for what changed from the NMOS part and the note about the extra decimal cycle.",
121+
"sha256": "a6af3ca9da45c8a03504c94c7399cb6b08f15ce14185dd3e1e880433cdaa8cf5",
122+
"bytes": 2655196,
123+
"pages": 32,
124+
"redistributable": false
125+
},
126+
{
127+
"file": "manufacturer/w65c816s.pdf",
128+
"rung": 1,
129+
"publisher": "The Western Design Center, Inc.",
130+
"title": "W65C816S 8/16-bit Microprocessor Data Sheet",
131+
"date": "2004-06-14",
132+
"retrievedFrom": "https://datasheets.chipdb.org/Western%20Design/w65c816s.pdf",
133+
"covers": ["65816", "65802"],
134+
"settles": "The 65816 core in both modes, including the cycle by cycle bus activity of every addressing mode in Table 6-7. This is the revision every citation in this project was taken from, and the section numbers it cites are its own.",
135+
"note": "The record used to say this document is 76 pages. It is 62. That is what a citation without a digest is worth, and it is why every entry here carries one.",
136+
"sha256": "60f21a3da0331273c6b92963ba7660408ab1ba3cf9dca2e37b1e55e7950ff313",
137+
"bytes": 1156506,
138+
"pages": 62,
139+
"redistributable": false
140+
},
141+
{
142+
"file": "manufacturer/wdc-w65c816s-jul-1994.pdf",
143+
"rung": 1,
144+
"publisher": "The Western Design Center, Inc.",
145+
"title": "W65C816S 8/16-bit Microprocessor Data Sheet",
146+
"date": "1994-07",
147+
"retrievedFrom": "https://6502.org/documents/datasheets/wdc/wdc_w65c816s_jul_1994.pdf",
148+
"covers": ["65816", "65802"],
149+
"settles": "The fullest statement of which addressing modes leave the emulation stack range. Its section 8.1 names two the 2004 and 2024 revisions drop from the same sentence, and the part behaves as this revision describes.",
150+
"sha256": "823c2f286a97102fc3695596f7c03e26e2f01cff0f45023dc3bb71688e689bc7",
151+
"bytes": 41473108,
152+
"pages": 72,
153+
"redistributable": false
154+
},
155+
{
156+
"file": "manufacturer/w65c816s-2024.pdf",
157+
"rung": 1,
158+
"publisher": "The Western Design Center, Inc.",
159+
"title": "W65C816S Datasheet",
160+
"date": "2024-03-13",
161+
"retrievedFrom": "https://westerndesigncenter.com/wdc/documentation/w65c816s.pdf",
162+
"covers": ["65816", "65802"],
163+
"settles": "That the caveats table has not changed in twenty years. Every cell of its Table 7-1 matches the 2004 revision's Table 8-1, so a claim taken from that table does not depend on which revision a reader holds.",
164+
"sha256": "b9177e1b045d2c8a801d1b23619abb4e5b29b88868fa491df7296dbc0b13447e",
165+
"bytes": 1532025,
166+
"pages": 55,
167+
"redistributable": false
168+
},
169+
{
170+
"file": "manufacturer/wdc-programming-the-65816-2007.pdf",
171+
"rung": 1,
172+
"publisher": "The Western Design Center, Inc.",
173+
"title": "Programming the 65816, including the 6502, 65C02 and 65802",
174+
"authors": "David Eyes and Ron Lichty",
175+
"date": "2007",
176+
"retrievedFrom": "https://github.com/gilligan/snesdev/blob/master/docs/programming_the_65816.pdf",
177+
"covers": ["6502", "65c02", "65802", "65816"],
178+
"settles": "The manufacturer's own book, four hundred and sixty-nine pages against the data sheet's sixty-two. Its Appendix C describes each pin at length, and reading it against the data sheet showed the two address qualifiers transposed in one of them, which bears on the one qualifier question this project has open.",
179+
"sha256": "dbccafcc47d6e53436bdc1c0049b3c54ffacfc9034a4041a7d92b6e6c5d72bb5",
180+
"bytes": 1811806,
181+
"pages": 469,
182+
"redistributable": false
183+
}
184+
]
185+
}

‎conformance/documents.py‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
"""Bring the documents down again, and refuse anything that is not what was read.
2+
3+
The documents are the publishers' rather than this project's, so what the
4+
repository carries is their identity and this script, not the files. A reader who
5+
runs it ends up with the same twelve documents that every citation in this
6+
package was taken from, or with an error naming the one that differs.
7+
8+
A digest is the whole point. Two documents can share a title, a publisher and a
9+
revision number and still not be the same file, and a citation against the wrong
10+
one is worse than no citation, because it looks checked.
11+
12+
Usage:
13+
python3 -m conformance.documents [--check] [--only SUBSTRING]
14+
"""
15+
16+
from __future__ import annotations
17+
18+
import hashlib
19+
import json
20+
import subprocess
21+
import sys
22+
from collections.abc import Sequence
23+
from pathlib import Path
24+
from typing import Any
25+
26+
ROOT = Path(__file__).resolve().parent.parent / "docs"
27+
"""Where the documents go, which is not where this script lives.
28+
29+
The folder is ignored by git, so a script inside it would be ignored too and the
30+
identity of every document would exist on one machine. It lives beside the rest
31+
of the record instead, and writes into the folder that is not carried.
32+
"""
33+
34+
MANIFEST = Path(__file__).resolve().parent / "documents.json"
35+
36+
USAGE = "usage: documents.py [--check] [--only SUBSTRING]"
37+
38+
TIMEOUT = 120
39+
"""Seconds to wait on one document, none of which is larger than a few megabytes."""
40+
41+
RENDERED = "renderedAs"
42+
"""The key naming a PDF this project made from a document published as text."""
43+
44+
PRINTED = "printedFromPage"
45+
"""The key marking a document that was published as a page and printed to PDF.
46+
47+
A print is not reproducible. A different browser, a different version of the same
48+
browser, or the page having changed since all give a different file, so the digest
49+
identifies this print rather than the page it came from. Fetching the address
50+
would return the page's markup, which would overwrite the thing that was read, so
51+
these are verified and never downloaded.
52+
"""
53+
54+
55+
class Refused(Exception):
56+
"""A document arrived that is not the one recorded."""
57+
58+
59+
def documents(path: Path | str | None = None) -> list[dict[str, Any]]:
60+
with Path(path or MANIFEST).open() as handle:
61+
held: list[dict[str, Any]] = json.load(handle)["documents"]
62+
return held
63+
64+
65+
def digest(path: Path) -> str:
66+
return hashlib.sha256(path.read_bytes()).hexdigest()
67+
68+
69+
def verify(entry: dict[str, Any], where: Path) -> None:
70+
"""That the file on disk is the one the manifest names, by digest and by size."""
71+
path = where / str(entry["file"])
72+
if not path.is_file():
73+
raise Refused(f"{entry['file']} is not here")
74+
found = digest(path)
75+
if found != entry["sha256"]:
76+
raise Refused(f"{entry['file']} is a different file: {found}")
77+
if path.stat().st_size != entry["bytes"]:
78+
raise Refused(f"{entry['file']} is {path.stat().st_size} bytes, not {entry['bytes']}")
79+
80+
81+
def download(entry: dict[str, Any], where: Path) -> None:
82+
"""One document, from the publisher, into the place the manifest names.
83+
84+
Only ever called for a document the publisher serves as a file. A page that
85+
was printed to PDF is verified and left alone, because fetching its address
86+
would replace it with markup.
87+
"""
88+
path = where / str(entry["file"])
89+
path.parent.mkdir(parents=True, exist_ok=True)
90+
done = subprocess.run(
91+
["curl", "-sSL", "--fail", "--globoff", "-o", str(path), str(entry["retrievedFrom"])],
92+
capture_output=True,
93+
text=True,
94+
check=False,
95+
timeout=TIMEOUT,
96+
)
97+
if done.returncode:
98+
raise Refused(f"{entry['file']} could not be fetched\n{done.stderr}")
99+
100+
101+
def options(argv: Sequence[str]) -> tuple[bool, str | None]:
102+
check = False
103+
only: str | None = None
104+
rest = list(argv)
105+
while rest:
106+
item = rest.pop(0)
107+
if item == "--check":
108+
check = True
109+
elif item == "--only":
110+
if not rest:
111+
raise SystemExit(USAGE)
112+
only = rest.pop(0)
113+
else:
114+
raise SystemExit(USAGE)
115+
return check, only
116+
117+
118+
def wanted(held: Sequence[dict[str, Any]], only: str | None) -> list[dict[str, Any]]:
119+
if only is None:
120+
return list(held)
121+
return [entry for entry in held if only in str(entry["file"])]
122+
123+
124+
def main(argv: Sequence[str], where: Path | str | None = None) -> int:
125+
check, only = options(argv)
126+
root = Path(where or ROOT)
127+
held = wanted(documents(), only)
128+
if not held:
129+
print(f"no document matches {only}")
130+
return 1
131+
refused: list[str] = []
132+
for entry in held:
133+
printed = bool(entry.get(PRINTED))
134+
try:
135+
if not check and not printed:
136+
download(entry, root)
137+
verify(entry, root)
138+
except (Refused, subprocess.TimeoutExpired) as raised:
139+
refused.append(str(raised))
140+
continue
141+
made = entry.get(RENDERED)
142+
note = ", rendered as " + str(made) if made else ""
143+
note += ", printed from a page and not re-fetchable" if printed else ""
144+
print(f"ok rung {entry['rung']} {entry['file']}{note}")
145+
for line in refused:
146+
print(f"REFUSED {line}")
147+
print(f"{len(held) - len(refused)} of {len(held)} verified")
148+
return 1 if refused else 0
149+
150+
151+
if __name__ == "__main__":
152+
raise SystemExit(main(sys.argv[1:]))

0 commit comments

Comments
 (0)