From 415070a23cdecdba0464a2fcaba55649d73c9811 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 1 Sep 2026 20:18:49 +0000 Subject: [PATCH 1/5] Add JS WAM persistent indexed fact stores (indexed + opt-in LMDB). Extend D27 CallFactStream with source(P/2, indexed(Prefix)) (zero-dep sorted key table) and source(P/2, lmdb(Dir)) (lazy npm lmdb, loud missing-package error, never a silent fallback). file(Path) emit is unchanged. Ship uw_fact_index / uw_fact_lmdb builders and document the on-disk format, key encoding, and optional-dependency policy. Co-authored-by: johns243a --- docs/WAM_JAVASCRIPT_STATUS.md | 173 +++++++++- scripts/js_wam/uw_fact_codec.js | 287 +++++++++++++++ scripts/js_wam/uw_fact_index.js | 33 ++ scripts/js_wam/uw_fact_lmdb.js | 106 ++++++ .../targets/wam_javascript_target.pl | 29 +- .../javascript_wam/runtime.js.mustache | 255 ++++++++++++++ tests/test_wam_javascript_fact_sources.pl | 326 +++++++++++++++++- 7 files changed, 1200 insertions(+), 9 deletions(-) create mode 100755 scripts/js_wam/uw_fact_codec.js create mode 100755 scripts/js_wam/uw_fact_index.js create mode 100755 scripts/js_wam/uw_fact_lmdb.js diff --git a/docs/WAM_JAVASCRIPT_STATUS.md b/docs/WAM_JAVASCRIPT_STATUS.md index 7018557c9..2d683dca7 100644 --- a/docs/WAM_JAVASCRIPT_STATUS.md +++ b/docs/WAM_JAVASCRIPT_STATUS.md @@ -368,7 +368,7 @@ emit time so the first `parse_args/2` is not a full construction. | CLI / runtime term parser | **Implemented.** Pratt reader: int/float/atom (incl. quoted)/var/list/`[H\|T]`/compound. CLI argv + `read_term_from_atom` / `atom_to_term` / `term_to_atom`. **`op/3`** updates the live infix/prefix/postfix tables (defaults cloned from ISO). Compile-time ops via `javascript_wam_ops/1`. Capability `native(parse_term)` via `INTEGRATION_PATCH.md` §7. | | Interpreter profiling | **Implemented.** Off by default (`Runtime._prof === null`). `UW_PROFILE=1` / `json` or `Runtime.profile(...)` writes a per-predicate table or JSON to **stderr**. Lowered tier: call counts only. See [Profiling (GP-PROF)](#profiling-gp-prof). | | `op/3` | **Implemented.** Infix `xfx`/`xfy`/`yfx`, prefix `fx`/`fy`, postfix `xf`/`yf`. Priority 0 removes. Name = atom or list of atoms. `current_op/3` is not implemented; ops are process-global. | -| External fact sources | **Implemented.** `javascript_wam_fact_sources([source(P/2, file(Path))])` (alias `js_fact_sources/1`) emits `CallFactStream` and a Node `fs` reader for TSV/CSV and JSONL. First-arg index when A1 is bound. Same lightweight file-backed model as Lua. **LMDB / CSR are out of scope.** Inline facts (no option) are unchanged. | +| External fact sources | **Implemented.** One `CallFactStream` path, three source forms (P/2 only). `file(Path)` is D27 and is **byte-for-byte unchanged**: whole-file TSV/CSV/JSONL into memory, first-arg index after parse. `indexed(Prefix)` is backend B (dependency-free on-disk index; see [Persistent indexed stores](#persistent-indexed-stores-gp-lmdb)). `lmdb(Dir)` is backend A (opt-in npm `lmdb`; see same section). Inline facts (no option) are unchanged. CSR remains out of scope. `docs/WAM_FLEET_GAPS.md` still lists LMDB as Lua-matching out-of-scope; **this target** now has it. | | Conformance harness adapter | See `INTEGRATION_PATCH.md` (coordinator applies `conformance_target(javascript)`). | ## How to run @@ -378,8 +378,11 @@ mkdir -p output/advanced # Dedicated probe + local 48-query suite (does not edit the shared harness): swipl -q -g run_tests -t halt tests/test_wam_javascript_builtins.pl -# File-backed P/2 fact sources (CSV/TSV/JSONL): +# File-backed P/2 fact sources (CSV/TSV/JSONL) + indexed/lmdb stores: swipl -q -g run_tests -t halt tests/test_wam_javascript_fact_sources.pl +# Backend B builder (zero deps): node scripts/js_wam/uw_fact_index.js build edges.tsv store_prefix +# Backend A loader (needs `npm install lmdb`): node scripts/js_wam/uw_fact_lmdb.js build edges.tsv lmdb_dir +# Bound-lookup I/O proof (stderr): UW_FACT_IO_STATS=1 node js/generated_program.js js_idx_probe_bound/0 # Tier-2 lowered / mixed emit-mode suite: swipl -q -g run_tests -t halt tests/test_wam_javascript_lowered.pl @@ -410,15 +413,175 @@ ISO graphic atoms (`#$&*+-./:<=>?@^~\`) except the lone atom `.`; quoted otherwise, escaping `\\` and `\'` only (no `\n`/`\t` escapes). Brace terms `{a}` are written as `'{}'(a)` / `{}(a)`, not SWI `{a}`. `write/1` list spacing is unchanged (`", "`). Fact-source cells stay -atoms even when the host file looks like a quoted string. +atoms even when the host file looks like a quoted string, except a TSV/CSV +field wrapped in `"..."` (or a JSON string that `parse_term` reads as a +string) which follows the D34 string tag through `parse_term`. + +## Persistent indexed stores (GP-LMDB) + +D27 `file(Path)` still loads and parses the **whole** file at first use. +That is the right default for small fixtures and is **unchanged**. GP-LMDB +adds stores that answer a **bound first argument** by seeking, without +scanning the data file. + +### Option syntax + +```prolog +javascript_wam_fact_sources([ + source(edge/2, file('edges.tsv')), % D27: in-memory + source(edge/2, indexed('stores/edges')), % B: stores/edges.data + stores/edges.idx + source(edge/2, lmdb('stores/edges.lmdb')) % A: LMDB environment directory +]). +``` + +Alias `js_fact_sources/1` is unchanged. Emitted JS: + +- `file`: `{ path: "..." }` — **no** `kind` field (D27 byte-for-byte). +- `indexed`: `{ kind: "indexed", path: "..." }` (`path` is the file prefix). +- `lmdb`: `{ kind: "lmdb", path: "..." }` (`path` is the env directory). + +Identical `CallFactStream` semantics for every form: + +- Unbound A1 → enumerate **all** facts in **source-file order**. +- Bound A1 → only matching facts (B: binary search of the key table; A: LMDB range get). +- Other-args bound → filter the streamed candidates (same as D27). +- Cells go through `parse_fact_source_value` → `parse_term` (D34 string tag, D37 literals). + +### Backend B — `indexed(Prefix)` (default capability, zero deps) + +This is **LMDB-style** (persistent + indexed + seek-based). It is **not** +LMDB. The format is our own, read-oriented, **single-writer at build time**. +There is no write path and no multi-arg secondary index (both out of scope). + +Builder: + +```bash +node scripts/js_wam/uw_fact_index.js build +``` + +Writes `.data` and `.idx`. Input parsing matches +D27 (tab vs comma; JSONL array / `{args}` / `{a1,a2}`; `#` and blank lines +skipped). Reproducible from the same flat files `file(Path)` reads. + +**Why a sorted key table + binary search** (not a hash bucket table): +O(log n) `fs.readSync` seeks with a fully specified total order +(`Buffer.compare` on the encoded key). No hash function, no collision +buckets, and the bytes-read proof is a handful of 16-byte entry reads plus +the matching records. Hash buckets would need a documented hash and a +worst-case scan of a bucket; they are not worth it for a read-only +build-time index. + +#### `Prefix.data` (little-endian) + +| Offset | Size | Field | +|---|---|---| +| 0 | 4 | magic `UWFI` | +| 4 | 1 | version `1` | +| 5 | 3 | pad | +| 8 | 4 | `n_records` (u32le) | +| 12 | 4 | reserved | +| 16 | … | records | + +Each record: `u32le payload_len`, then payload `u16le a1_len`, `u16le a2_len`, +A1 UTF-8, A2 UTF-8. Payloads are the **original cell text** (TSV field / +JSON-serialized number or string) so the runtime `parse_term` round-trip +matches D27. Enumeration is a sequential scan from offset 16. + +The runtime must **not** `readFileSync` the data file. Bound lookup reads +only the matching records (plus index probes). `UW_FACT_IO_STATS=1` prints +`fact_io bytes_read=N data_size=M` on stderr at process exit. + +#### `Prefix.idx` (little-endian) + +| Offset | Size | Field | +|---|---|---| +| 0 | 4 | magic `UWIX` | +| 4 | 1 | version `1` | +| 5 | 3 | pad | +| 8 | 4 | `n_keys` (u32le) | +| 12 | 4 | `keyblob_off` (u32le) | +| 16 | 4 | `hits_off` (u32le) | +| 20 | 4 | `n_records` (u32le) | +| 24 | `n_keys × 16` | `KeyEnt` table | +| `keyblob_off` | … | concatenated keys, sorted by `Buffer.compare` | +| `hits_off` | … | per key: `n_hits × u32le` data-file offsets, encounter order | + +`KeyEnt` (16 bytes): `u32le key_rel`, `u16le key_len`, `u16le n_hits`, +`u32le hits_rel`, 4 bytes pad. + +Bound lookup: binary-search the table (read 16-byte `KeyEnt` + key bytes +per probe), then read only those data-file records. + +#### Index key encoding (intern-id independent; preserves D34) + +Shared by B and A (`scripts/js_wam/uw_fact_codec.js` / runtime +`encode_store_key`): + +| Tag byte | Payload | Term | +|---|---|---| +| `0x49` (`I`) | int64 big-endian | integer | +| `0x46` (`F`) | float64 big-endian | float | +| `0x53` (`S`) | UTF-8 | string (`V.String`) | +| `0x41` (`A`) | UTF-8 atom name | atom | +| `0x3F` | (none) | anything else (no index hit) | + +Quoted TSV `"strkey"` is a string; bare `strkey` is an atom. JSON numbers +are ints/floats; JSON strings go through the same cell classifier as TSV +so `"a"` in JSON (unquoted Prolog atom text) is still atom `a`. + +### Backend A — `lmdb(Dir)` (opt-in, real LMDB) + +Loader: + +```bash +npm install lmdb # user action; not a repo package.json dependency +node scripts/js_wam/uw_fact_lmdb.js build +``` + +The runtime loads the package **lazily** (`createRequire(__filename)("lmdb")`) +only when a `lmdb(...)` source is actually used. `encoding: "binary"` and +`keyEncoding: "binary"`. + +**LMDB key encoding** (same cell payload as B): + +| Kind | Key | Value | +|---|---|---| +| seq (unbound enum) | `0x00` \|\| uint64be(seq) | payload (u16le a1_len, u16le a2_len, bytes) | +| A1 (bound lookup) | `0x01` \|\| uint16be(key_len) \|\| `encodeIndexKey(A1)` \|\| uint64be(seq) | same payload | + +Range get for a bound A1 uses start seq=0 and end seq=`0xff…ff` (end is +exclusive in `lmdb-js` `getRange`; keys may contain `0x00`, so a +length-prefix sits in front of the encoded A1 rather than a NUL delimiter). +Unbound enumeration is `getRange({start: [0x00], end: [0x01]})` over seq +keys, which is source-file order. + +Missing package: **one** error naming the store, the missing package, and +`npm install lmdb`. It states that `indexed(...)` is a different format +and is **not** used as a fallback. Test seam: `UW_LMDB_FORCE_MISSING=1`. + +### Optional-dependency policy + +`lmdb` is opt-in **per source declaration**. Absence is a loud error at +the moment that source is used. Default builds, `file(...)`, and +`indexed(...)` do not require any npm package and must not grow a repo +`package.json` dependency. Different store formats are never silently +swapped. + +### Out of scope this round + +Multi-arg secondary indexes, write paths / live updates, and CSR. Backend +B has no writer after `uw_fact_index build`. Backend A is loaded +read-only at runtime. ## Document status Initial JS WAM bring-up + builtin port from Lua, ISO bagof/3 and setof/3, first-argument indexing, the Tier-2 lowered emitter, ISO/library builtin breadth (sort, lists, atom/string, format, assoc), the G-W2 runtime term -parser, G-W4 file-backed fact sources (TSV/CSV/JSONL; LMDB/CSR out of -scope), the G-W3 term-meta family (`term_variables/2`, +parser, G-W4 file-backed fact sources (TSV/CSV/JSONL), GP-LMDB persistent +indexed stores on **this** target (`indexed/1` zero-dep + opt-in `lmdb/1`; +fleet-gaps still lists LMDB as Lua-matching out-of-scope), the G-W3 +term-meta family (`term_variables/2`, `numbervars/3`, `=@=/2`, `\=@=/2`), then G-W2 `op/3` (dynamic Pratt table: infix + prefix + postfix), then a distinct string term tag (`V.String`; string-producing builtins + standard order), then diff --git a/scripts/js_wam/uw_fact_codec.js b/scripts/js_wam/uw_fact_codec.js new file mode 100755 index 000000000..a94f26bf1 --- /dev/null +++ b/scripts/js_wam/uw_fact_codec.js @@ -0,0 +1,287 @@ +#!/usr/bin/env node +// SPDX-License-Identifier: MIT OR Apache-2.0 +// Copyright (c) 2026 John William Creighton (@s243a) +// +// Shared on-disk encoding for JS WAM persistent fact stores. +// Used by uw_fact_index.js (backend B) and uw_fact_lmdb.js (backend A). +// Key tags preserve D34 atom / string / number distinctions. + +"use strict"; + +const fs = require("fs"); +const path = require("path"); + +const DATA_MAGIC = Buffer.from("UWFI"); +const IDX_MAGIC = Buffer.from("UWIX"); +const VERSION = 1; +const DATA_HEADER = 16; +const IDX_HEADER = 24; +const IDX_ENTRY = 16; + +const TAG_ATOM = 0x41; // A +const TAG_STRING = 0x53; // S +const TAG_INT = 0x49; // I +const TAG_FLOAT = 0x46; // F + +function trim(s) { + return String(s).replace(/^\s+|\s+$/g, ""); +} + +function isJsonlPath(p) { + return /\.jsonl$/i.test(p) || /\.ndjson$/i.test(p); +} + +function classifyCellText(text) { + const t = trim(text); + if (/^-?\d+$/.test(t)) { + const n = Number(t); + if (Number.isSafeInteger(n)) return { tag: "int", val: n, text: t }; + } + if (/^-?(?:\d+\.\d*|\d*\.\d+)(?:[eE][+-]?\d+)?$/.test(t) || + /^-?\d+[eE][+-]?\d+$/.test(t)) { + return { tag: "float", val: Number(t), text: t }; + } + if (t.length >= 2 && t.charAt(0) === '"' && t.charAt(t.length - 1) === '"') { + const inner = t.slice(1, -1).replace(/\\"/g, '"').replace(/\\\\/g, "\\"); + return { tag: "string", val: inner, text: t }; + } + if (t.length >= 2 && t.charAt(0) === "'" && t.charAt(t.length - 1) === "'") { + const inner = t.slice(1, -1).replace(/\\'/g, "'").replace(/\\\\/g, "\\"); + return { tag: "atom", val: inner, text: t }; + } + return { tag: "atom", val: t, text: t }; +} + +function jsonValueToCell(val) { + if (typeof val === "number") { + if (Number.isInteger(val)) { + return { tag: "int", val: val, text: String(val) }; + } + return { tag: "float", val: val, text: String(val) }; + } + if (typeof val === "boolean") { + const t = val ? "true" : "false"; + return { tag: "atom", val: t, text: t }; + } + if (val === null) { + return { tag: "atom", val: "[]", text: "[]" }; + } + if (typeof val === "string") { + return classifyCellText(val); + } + return null; +} + +function parseDelimitedLine(line) { + const sep = line.indexOf("\t") >= 0 ? "\t" : ","; + const parts = line.split(sep); + if (parts.length < 2) return null; + const a1 = classifyCellText(parts[0]); + const a2 = classifyCellText(parts[1]); + return { a1: a1, a2: a2 }; +} + +function parseJsonlLine(line) { + let obj; + try { obj = JSON.parse(line); } catch (e) { return null; } + if (Array.isArray(obj) && obj.length >= 2) { + const a1 = jsonValueToCell(obj[0]); + const a2 = jsonValueToCell(obj[1]); + if (!a1 || !a2) return null; + return { a1: a1, a2: a2 }; + } + if (obj && typeof obj === "object") { + if (Array.isArray(obj.args) && obj.args.length >= 2) { + const a1 = jsonValueToCell(obj.args[0]); + const a2 = jsonValueToCell(obj.args[1]); + if (!a1 || !a2) return null; + return { a1: a1, a2: a2 }; + } + if (obj.a1 !== undefined && obj.a2 !== undefined) { + const a1 = jsonValueToCell(obj.a1); + const a2 = jsonValueToCell(obj.a2); + if (!a1 || !a2) return null; + return { a1: a1, a2: a2 }; + } + } + return null; +} + +function firstDataLineLooksJson(lines) { + for (let i = 0; i < lines.length; i++) { + const cleaned = trim(lines[i]); + if (cleaned === "" || cleaned.charAt(0) === "#") continue; + const c = cleaned.charAt(0); + return c === "{" || c === "["; + } + return false; +} + +function readFlatFactRows(inputPath) { + const text = fs.readFileSync(inputPath, "utf8"); + const lines = String(text).split(/\r?\n/); + const jsonl = isJsonlPath(inputPath) || firstDataLineLooksJson(lines); + const rows = []; + for (let i = 0; i < lines.length; i++) { + const cleaned = trim(lines[i]); + if (cleaned === "" || cleaned.charAt(0) === "#") continue; + const pair = jsonl ? parseJsonlLine(cleaned) : parseDelimitedLine(cleaned); + if (!pair) continue; + rows.push(pair); + } + return rows; +} + +function encodeIndexKey(cell) { + if (cell.tag === "int") { + const b = Buffer.alloc(9); + b[0] = TAG_INT; + b.writeBigInt64BE(BigInt(cell.val), 1); + return b; + } + if (cell.tag === "float") { + const b = Buffer.alloc(9); + b[0] = TAG_FLOAT; + b.writeDoubleBE(cell.val, 1); + return b; + } + if (cell.tag === "string") { + return Buffer.concat([Buffer.from([TAG_STRING]), Buffer.from(String(cell.val), "utf8")]); + } + return Buffer.concat([Buffer.from([TAG_ATOM]), Buffer.from(String(cell.val), "utf8")]); +} + +function recordPayload(row) { + const a1 = Buffer.from(row.a1.text, "utf8"); + const a2 = Buffer.from(row.a2.text, "utf8"); + const buf = Buffer.alloc(4 + a1.length + a2.length); + buf.writeUInt16LE(a1.length, 0); + buf.writeUInt16LE(a2.length, 2); + a1.copy(buf, 4); + a2.copy(buf, 4 + a1.length); + return buf; +} + +function writeIndexedStore(inputPath, storePrefix) { + const rows = readFlatFactRows(inputPath); + const dir = path.dirname(path.resolve(storePrefix)); + fs.mkdirSync(dir, { recursive: true }); + const dataPath = storePrefix + ".data"; + const idxPath = storePrefix + ".idx"; + + const dataChunks = [Buffer.alloc(DATA_HEADER)]; + DATA_MAGIC.copy(dataChunks[0], 0); + dataChunks[0].writeUInt8(VERSION, 4); + dataChunks[0].writeUInt32LE(rows.length, 8); + + const byKey = new Map(); + let offset = DATA_HEADER; + for (let i = 0; i < rows.length; i++) { + const payload = recordPayload(rows[i]); + const rec = Buffer.alloc(4 + payload.length); + rec.writeUInt32LE(payload.length, 0); + payload.copy(rec, 4); + dataChunks.push(rec); + const key = encodeIndexKey(rows[i].a1); + const keyHex = key.toString("hex"); + if (!byKey.has(keyHex)) byKey.set(keyHex, { key: key, offsets: [] }); + byKey.get(keyHex).offsets.push(offset); + offset += rec.length; + } + fs.writeFileSync(dataPath, Buffer.concat(dataChunks)); + + const entries = Array.from(byKey.values()); + entries.sort(function (a, b) { return Buffer.compare(a.key, b.key); }); + + const keyBlobParts = []; + const hitsParts = []; + const table = Buffer.alloc(IDX_ENTRY * entries.length); + let keyRel = 0; + let hitsRel = 0; + for (let i = 0; i < entries.length; i++) { + const e = entries[i]; + const base = i * IDX_ENTRY; + table.writeUInt32LE(keyRel, base); + table.writeUInt16LE(e.key.length, base + 4); + table.writeUInt16LE(e.offsets.length, base + 6); + table.writeUInt32LE(hitsRel, base + 8); + keyBlobParts.push(e.key); + keyRel += e.key.length; + const hits = Buffer.alloc(4 * e.offsets.length); + for (let j = 0; j < e.offsets.length; j++) hits.writeUInt32LE(e.offsets[j], j * 4); + hitsParts.push(hits); + hitsRel += hits.length; + } + const keyBlob = Buffer.concat(keyBlobParts); + const hitsBlob = Buffer.concat(hitsParts); + const header = Buffer.alloc(IDX_HEADER); + IDX_MAGIC.copy(header, 0); + header.writeUInt8(VERSION, 4); + header.writeUInt32LE(entries.length, 8); + header.writeUInt32LE(IDX_HEADER + table.length, 12); + header.writeUInt32LE(IDX_HEADER + table.length + keyBlob.length, 16); + header.writeUInt32LE(rows.length, 20); + fs.writeFileSync(idxPath, Buffer.concat([header, table, keyBlob, hitsBlob])); + return { rows: rows.length, keys: entries.length, dataPath: dataPath, idxPath: idxPath }; +} + +function lmdbMissingError(predKey, storePath) { + return ( + "JS WAM fact source " + JSON.stringify(String(predKey || "")) + + " is declared as lmdb(" + JSON.stringify(String(storePath || "")) + + ") but the 'lmdb' npm package is not installed. " + + "Install it in this environment with: npm install lmdb " + + "This backend is opt-in; default builds do not require it. " + + "The indexed(...) store is a different format and is not used as a fallback." + ); +} + +function seqKey(seq) { + const b = Buffer.alloc(9); + b[0] = 0x00; + b.writeBigUInt64BE(BigInt(seq), 1); + return b; +} + +function a1RangeKey(keyBytes, seq) { + const b = Buffer.alloc(3 + keyBytes.length + 8); + b[0] = 0x01; + b.writeUInt16BE(keyBytes.length, 1); + keyBytes.copy(b, 3); + b.writeBigUInt64BE(BigInt(seq), 3 + keyBytes.length); + return b; +} + +function a1RangeStart(keyBytes) { + return a1RangeKey(keyBytes, 0); +} + +function a1RangeEnd(keyBytes) { + const b = a1RangeKey(keyBytes, 0); + for (let i = b.length - 1; i >= 3 + keyBytes.length; i--) b[i] = 0xff; + return b; +} + +module.exports = { + DATA_MAGIC: DATA_MAGIC, + IDX_MAGIC: IDX_MAGIC, + VERSION: VERSION, + DATA_HEADER: DATA_HEADER, + IDX_HEADER: IDX_HEADER, + IDX_ENTRY: IDX_ENTRY, + TAG_ATOM: TAG_ATOM, + TAG_STRING: TAG_STRING, + TAG_INT: TAG_INT, + TAG_FLOAT: TAG_FLOAT, + trim: trim, + classifyCellText: classifyCellText, + readFlatFactRows: readFlatFactRows, + encodeIndexKey: encodeIndexKey, + recordPayload: recordPayload, + writeIndexedStore: writeIndexedStore, + lmdbMissingError: lmdbMissingError, + seqKey: seqKey, + a1RangeKey: a1RangeKey, + a1RangeStart: a1RangeStart, + a1RangeEnd: a1RangeEnd +}; diff --git a/scripts/js_wam/uw_fact_index.js b/scripts/js_wam/uw_fact_index.js new file mode 100755 index 000000000..be9d8a56b --- /dev/null +++ b/scripts/js_wam/uw_fact_index.js @@ -0,0 +1,33 @@ +#!/usr/bin/env node +// SPDX-License-Identifier: MIT OR Apache-2.0 +// Copyright (c) 2026 John William Creighton (@s243a) +// +// uw_fact_index — build a dependency-free indexed fact store (backend B). +// +// node scripts/js_wam/uw_fact_index.js build +// +// Writes .data (length-prefixed records, source order) and +// .idx (sorted first-arg key table). This is LMDB-style +// (persistent + indexed + seek-based), not LMDB. + +"use strict"; + +const path = require("path"); +const codec = require(path.join(__dirname, "uw_fact_codec.js")); + +function usage() { + process.stderr.write( + "usage: node scripts/js_wam/uw_fact_index.js build \n" + ); + process.exit(2); +} + +const argv = process.argv.slice(2); +if (argv[0] !== "build" || argv.length < 3) usage(); +const input = argv[1]; +const store = argv[2]; +const result = codec.writeIndexedStore(input, store); +process.stdout.write( + "uw_fact_index: wrote " + result.rows + " records / " + + result.keys + " keys -> " + result.dataPath + " + " + result.idxPath + "\n" +); diff --git a/scripts/js_wam/uw_fact_lmdb.js b/scripts/js_wam/uw_fact_lmdb.js new file mode 100755 index 000000000..7ecfe0997 --- /dev/null +++ b/scripts/js_wam/uw_fact_lmdb.js @@ -0,0 +1,106 @@ +#!/usr/bin/env node +// SPDX-License-Identifier: MIT OR Apache-2.0 +// Copyright (c) 2026 John William Creighton (@s243a) +// +// uw_fact_lmdb — load the same TSV/CSV/JSONL into an LMDB environment (backend A). +// +// node scripts/js_wam/uw_fact_lmdb.js build +// +// Requires the `lmdb` npm package (opt-in; not a repo dependency): +// npm install lmdb +// Missing package is a loud error; never falls back to indexed(...). +// +// Key scheme (see also docs/WAM_JAVASCRIPT_STATUS.md): +// seq (unbound enum, source order): 0x00 || uint64be(seq) → payload +// A1 (bound lookup): 0x01 || uint16be(key_len) || encodeIndexKey(A1) || uint64be(seq) → payload +// encodeIndexKey preserves D34 tags: 0x49 int, 0x46 float, 0x53 string, 0x41 atom. + +"use strict"; + +const fs = require("fs"); +const path = require("path"); +const codec = require(path.join(__dirname, "uw_fact_codec.js")); + +function usage() { + process.stderr.write( + "usage: node scripts/js_wam/uw_fact_lmdb.js build \n" + ); + process.exit(2); +} + +function loadLmdb(storePath) { + try { + const { createRequire } = require("module"); + const req = createRequire(__filename); + return req("lmdb"); + } catch (err) { + throw new Error(codec.lmdbMissingError("build", storePath)); + } +} + +const argv = process.argv.slice(2); +if (argv[0] !== "build" || argv.length < 3) usage(); +const input = argv[1]; +const storeDir = path.resolve(argv[2]); + +let lmdb; +try { + lmdb = loadLmdb(storeDir); +} catch (err) { + process.stderr.write(String(err.message || err) + "\n"); + process.exit(1); +} + +const rows = codec.readFlatFactRows(input); +fs.mkdirSync(storeDir, { recursive: true }); +const open = lmdb.open || (lmdb.default && lmdb.default.open); +if (typeof open !== "function") { + process.stderr.write("uw_fact_lmdb: unexpected lmdb package API (no open())\n"); + process.exit(1); +} + +const db = open({ + path: storeDir, + encoding: "binary", + keyEncoding: "binary" +}); + +function putAll(store, list) { + if (typeof store.clearSync === "function") store.clearSync(); + else if (typeof store.clear === "function") store.clear(); + for (let i = 0; i < list.length; i++) { + const payload = codec.recordPayload(list[i]); + const keyBytes = codec.encodeIndexKey(list[i].a1); + const seq = codec.seqKey(i); + const a1k = codec.a1RangeKey(keyBytes, i); + if (typeof store.putSync === "function") { + store.putSync(seq, payload); + store.putSync(a1k, payload); + } else { + store.put(seq, payload); + store.put(a1k, payload); + } + } + return list.length; +} + +function commit(store, list) { + if (typeof store.transactionSync === "function") { + return store.transactionSync(function () { return putAll(store, list); }); + } + if (typeof store.transaction === "function") { + return store.transaction(function () { return putAll(store, list); }); + } + return putAll(store, list); +} + +Promise.resolve(commit(db, rows)).then(function (n) { + if (db.flushed && typeof db.flushed.then === "function") return db.flushed.then(function () { return n; }); + return n; +}).then(function (n) { + if (typeof db.close === "function") db.close(); + process.stdout.write("uw_fact_lmdb: wrote " + n + " records -> " + storeDir + "\n"); +}).catch(function (err) { + process.stderr.write(String((err && err.stack) || err) + "\n"); + process.exit(1); +}); diff --git a/src/unifyweaver/targets/wam_javascript_target.pl b/src/unifyweaver/targets/wam_javascript_target.pl index 531b87e99..4e01c82bb 100644 --- a/src/unifyweaver/targets/wam_javascript_target.pl +++ b/src/unifyweaver/targets/wam_javascript_target.pl @@ -10,7 +10,9 @@ % (closest dynamically typed model). emit_mode is interpreter | functions % | mixed (every eligible predicate) | mixed(List); default remains interpreter. % javascript_wam_fact_sources([source(P/2, file(Path))]) streams binary -% facts from a TSV/CSV or JSONL file (Lua-style; no LMDB/CSR). +% facts from a TSV/CSV or JSONL file (Lua-style). Persistent indexed +% stores: source(P/2, indexed(Prefix)) (dependency-free seek index) and +% source(P/2, lmdb(Dir)) (opt-in npm `lmdb`; loud error if missing). % javascript_wam_ops([op(Prec, Type, Name), ...]) (alias js_op_decls/1) % seeds the runtime Pratt op table at program startup (R's r_op_decls/1). % Lowered dispatch wraps each function with a UW_PROFILE call counter. @@ -582,8 +584,10 @@ NewInstrs, NewTopLabels, NewAllLabels, [Wrapper|WrapperAcc], NewLoweredAcc, NewFactSourceAcc, AllInstrs, TopLabels, AllLabels, Wrappers, Lowered, FactSources). -%% javascript_wam_fact_sources([source(P/A, file(Path)), ...]) -% Lightweight file-backed binary facts (Lua's lua_fact_sources/1). +%% javascript_wam_fact_sources([source(P/A, Spec), ...]) +% Spec = file(Path) % D27: load whole TSV/CSV/JSONL into memory +% | indexed(Prefix) % GP-LMDB B: Prefix.data + Prefix.idx, seek lookup +% | lmdb(Dir) % GP-LMDB A: opt-in LMDB env; never silent-fallback % Only P/2 is streamed; other arities keep compiled inline WAM. javascript_wam_fact_source_spec(P, Arity, Options, Spec) :- Arity =:= 2, @@ -610,6 +614,25 @@ js_string_literal(Key, KeyQ), js_string_literal(SourcePath, PathQ), format(string(Entry), ' ~w: { path: ~w }', [KeyQ, PathQ]). +javascript_wam_fact_source_entry(Key, indexed(Path), Entry) :- + javascript_wam_store_path(Path, SourcePath), + js_string_literal(Key, KeyQ), + js_string_literal(SourcePath, PathQ), + format(string(Entry), ' ~w: { kind: "indexed", path: ~w }', [KeyQ, PathQ]). +javascript_wam_fact_source_entry(Key, lmdb(Path), Entry) :- + javascript_wam_store_path(Path, SourcePath), + js_string_literal(Key, KeyQ), + js_string_literal(SourcePath, PathQ), + format(string(Entry), ' ~w: { kind: "lmdb", path: ~w }', [KeyQ, PathQ]). + +javascript_wam_store_path(Path, SourcePath) :- + atom_string(Path, PathStr), + working_directory(Cwd, Cwd), + ( catch(absolute_file_name(PathStr, AbsPath, [relative_to(Cwd)]), _, fail), + AbsPath \== [] + -> SourcePath = AbsPath + ; SourcePath = PathStr + ). compile_js_predicate_wam(PredIndicator, WamCode) :- CompileOpts = [ite_use_y_level(true), inline_bagof_setof(true)], diff --git a/templates/targets/javascript_wam/runtime.js.mustache b/templates/targets/javascript_wam/runtime.js.mustache index d69f6eb33..5f2218882 100644 --- a/templates/targets/javascript_wam/runtime.js.mustache +++ b/templates/targets/javascript_wam/runtime.js.mustache @@ -3263,9 +3263,264 @@ Runtime.read_facts_file = function (program, path) { return { rows: rows, arg1_index: arg1_index }; }; +// --------------------------------------------------------------------------- +// Persistent indexed stores (GP-LMDB). file(Path) above is unchanged. +// indexed(Path) = backend B (our format, zero deps). lmdb(Path) = backend A +// (opt-in npm package `lmdb`; missing package is a loud error, never a +// silent fallback to B). +// --------------------------------------------------------------------------- +Runtime._fact_io_bytes = 0; +Runtime._fact_io_data_size = 0; +Runtime._lmdb_force_missing = false; + +const UWFI_MAGIC = Buffer.from("UWFI"); +const UWIX_MAGIC = Buffer.from("UWIX"); +const UWFI_HEADER = 16; +const UWIX_HEADER = 24; +const UWIX_ENTRY = 16; + +function fact_io_read(fd, length, position) { + if (length <= 0) return Buffer.alloc(0); + const buf = Buffer.alloc(length); + const n = require("fs").readSync(fd, buf, 0, length, position); + Runtime._fact_io_bytes = (Runtime._fact_io_bytes || 0) + n; + return n === length ? buf : buf.subarray(0, n); +} + +function encode_store_key(program, v) { + if (!v || typeof v !== "object") return Buffer.from([0x3f]); + if (v.tag === "int") { + const b = Buffer.alloc(9); + b[0] = 0x49; + b.writeBigInt64BE(BigInt(v.val), 1); + return b; + } + if (v.tag === "float") { + const b = Buffer.alloc(9); + b[0] = 0x46; + b.writeDoubleBE(v.val, 1); + return b; + } + if (v.tag === "string") { + return Buffer.concat([Buffer.from([0x53]), Buffer.from(String(v.val), "utf8")]); + } + if (v.tag === "atom") { + const name = Runtime.string_of(program.intern_table, v.id); + return Buffer.concat([Buffer.from([0x41]), Buffer.from(String(name), "utf8")]); + } + return Buffer.from([0x3f]); +} + +function parse_store_payload(program, buf) { + if (!buf || buf.length < 4) return null; + const a1Len = buf.readUInt16LE(0); + const a2Len = buf.readUInt16LE(2); + if (buf.length < 4 + a1Len + a2Len) return null; + const a1Text = buf.subarray(4, 4 + a1Len).toString("utf8"); + const a2Text = buf.subarray(4 + a1Len, 4 + a1Len + a2Len).toString("utf8"); + return [ + parse_fact_source_value(program, a1Text), + parse_fact_source_value(program, a2Text) + ]; +} + +function open_indexed_store(source) { + if (source._idx_fd !== undefined) return; + const fs = require("fs"); + const prefix = source.path; + source._data_fd = fs.openSync(prefix + ".data", "r"); + source._idx_fd = fs.openSync(prefix + ".idx", "r"); + const st = fs.fstatSync(source._data_fd); + source._data_size = st.size; + Runtime._fact_io_data_size = st.size; + const ih = fact_io_read(source._idx_fd, UWIX_HEADER, 0); + if (ih.length < UWIX_HEADER || ih.subarray(0, 4).compare(UWIX_MAGIC) !== 0) { + throw new Error("JS WAM indexed store: bad index magic at " + prefix + ".idx"); + } + source._n_keys = ih.readUInt32LE(8); + source._keyblob_off = ih.readUInt32LE(12); + source._hits_off = ih.readUInt32LE(16); + source._n_records = ih.readUInt32LE(20); + const dh = fact_io_read(source._data_fd, UWFI_HEADER, 0); + if (dh.length < UWFI_HEADER || dh.subarray(0, 4).compare(UWFI_MAGIC) !== 0) { + throw new Error("JS WAM indexed store: bad data magic at " + prefix + ".data"); + } +} + +function indexed_read_entry(source, i) { + const pos = UWIX_HEADER + i * UWIX_ENTRY; + const e = fact_io_read(source._idx_fd, UWIX_ENTRY, pos); + return { + key_rel: e.readUInt32LE(0), + key_len: e.readUInt16LE(4), + n_hits: e.readUInt16LE(6), + hits_rel: e.readUInt32LE(8) + }; +} + +function indexed_read_key(source, ent) { + return fact_io_read(source._idx_fd, ent.key_len, source._keyblob_off + ent.key_rel); +} + +function indexed_lookup_offsets(source, targetKey) { + let lo = 0; + let hi = source._n_keys - 1; + while (lo <= hi) { + const mid = (lo + hi) >> 1; + const ent = indexed_read_entry(source, mid); + const k = indexed_read_key(source, ent); + const cmp = Buffer.compare(k, targetKey); + if (cmp === 0) { + const hits = fact_io_read(source._idx_fd, ent.n_hits * 4, source._hits_off + ent.hits_rel); + const offs = []; + for (let i = 0; i < ent.n_hits; i++) offs.push(hits.readUInt32LE(i * 4)); + return offs; + } + if (cmp < 0) lo = mid + 1; + else hi = mid - 1; + } + return []; +} + +function indexed_read_record(program, source, dataOff) { + const lenBuf = fact_io_read(source._data_fd, 4, dataOff); + const payloadLen = lenBuf.readUInt32LE(0); + const payload = fact_io_read(source._data_fd, payloadLen, dataOff + 4); + return parse_store_payload(program, payload); +} + +function indexed_scan_all(program, source) { + const rows = []; + let pos = UWFI_HEADER; + for (let i = 0; i < source._n_records; i++) { + const pair = indexed_read_record(program, source, pos); + if (pair) rows.push(pair); + const lenBuf = fact_io_read(source._data_fd, 4, pos); + pos += 4 + lenBuf.readUInt32LE(0); + } + return rows; +} + +function indexed_fact_source_rows(program, source, state) { + open_indexed_store(source); + Runtime._fact_io_data_size = source._data_size || 0; + const a1 = Runtime.deref(state, Runtime.get_reg(state, 1)); + if (typeof a1 === "object" && a1 !== null && a1.tag !== "unbound") { + const offs = indexed_lookup_offsets(source, encode_store_key(program, a1)); + const rows = []; + for (let i = 0; i < offs.length; i++) { + const pair = indexed_read_record(program, source, offs[i]); + if (pair) rows.push(pair); + } + return rows; + } + return indexed_scan_all(program, source); +} + +function lmdb_missing_error(predKey, storePath) { + return ( + "JS WAM fact source " + JSON.stringify(String(predKey || "")) + + " is declared as lmdb(" + JSON.stringify(String(storePath || "")) + + ") but the 'lmdb' npm package is not installed. " + + "Install it in this environment with: npm install lmdb " + + "This backend is opt-in; default builds do not require it. " + + "The indexed(...) store is a different format and is not used as a fallback." + ); +} +Runtime.lmdb_missing_error = lmdb_missing_error; + +function require_lmdb_package(predKey, storePath) { + if (Runtime._lmdb_force_missing === true || + (process.env && process.env.UW_LMDB_FORCE_MISSING === "1")) { + throw new Error(lmdb_missing_error(predKey, storePath)); + } + try { + const { createRequire } = require("module"); + const req = createRequire(__filename); + return req("lmdb"); + } catch (err) { + throw new Error(lmdb_missing_error(predKey, storePath)); + } +} + +function open_lmdb_store(source, predKey) { + if (source._lmdb) return source._lmdb; + const lmdb = require_lmdb_package(predKey, source.path); + const open = lmdb.open || (lmdb.default && lmdb.default.open); + if (typeof open !== "function") { + throw new Error("JS WAM lmdb store: package has no open()"); + } + // keyEncoding binary: our 0x00/0x01 tagged keys are raw bytes (D34 tags + // live inside encode_store_key). encoding binary: payload is the same + // length-prefixed cell text as backend B. Never open an indexed(...) file. + source._lmdb = open({ + path: source.path, + encoding: "binary", + keyEncoding: "binary", + readOnly: true + }); + return source._lmdb; +} + +function lmdb_payload_row(program, value) { + if (value == null) return null; + let buf = value; + if (!Buffer.isBuffer(buf)) { + if (value instanceof Uint8Array) buf = Buffer.from(value); + else if (typeof value === "string") buf = Buffer.from(value, "latin1"); + else buf = Buffer.from(value); + } + return parse_store_payload(program, buf); +} + +function lmdb_range_rows(program, db, start, end) { + if (typeof db.getRange !== "function") { + throw new Error("JS WAM lmdb store: package has no getRange()"); + } + const rows = []; + const it = db.getRange({ start: start, end: end }); + for (const entry of it) { + const pair = lmdb_payload_row(program, entry.value); + if (pair) rows.push(pair); + } + return rows; +} + +function lmdb_fact_source_rows(program, source, state, predKey) { + const db = open_lmdb_store(source, predKey); + const a1 = Runtime.deref(state, Runtime.get_reg(state, 1)); + if (typeof a1 === "object" && a1 !== null && a1.tag !== "unbound") { + const keyBytes = encode_store_key(program, a1); + const start = Buffer.alloc(3 + keyBytes.length + 8); + start[0] = 0x01; + start.writeUInt16BE(keyBytes.length, 1); + keyBytes.copy(start, 3); + const end = Buffer.from(start); + for (let i = 3 + keyBytes.length; i < end.length; i++) end[i] = 0xff; + return lmdb_range_rows(program, db, start, end); + } + // seq keys are 0x00||uint64; A1 keys are 0x01||… — [0x00, 0x01) is exclusive-end + return lmdb_range_rows(program, db, Buffer.from([0x00]), Buffer.from([0x01])); +} + +if (process.env && process.env.UW_FACT_IO_STATS) { + process.on("exit", function () { + process.stderr.write( + "fact_io bytes_read=" + String(Runtime._fact_io_bytes || 0) + + " data_size=" + String(Runtime._fact_io_data_size || 0) + "\n" + ); + }); +} + function fact_source_rows(program, pred, state) { const source = program.fact_sources && program.fact_sources[pred]; if (!source) return null; + if (source.kind === "indexed") { + return indexed_fact_source_rows(program, source, state); + } + if (source.kind === "lmdb") { + return lmdb_fact_source_rows(program, source, state, pred); + } if (!source.cache) source.cache = Runtime.read_facts_file(program, source.path); const a1 = Runtime.deref(state, Runtime.get_reg(state, 1)); if (typeof a1 === "object" && a1 !== null && a1.tag !== "unbound") { diff --git a/tests/test_wam_javascript_fact_sources.pl b/tests/test_wam_javascript_fact_sources.pl index d92384480..b5d3e57b5 100644 --- a/tests/test_wam_javascript_fact_sources.pl +++ b/tests/test_wam_javascript_fact_sources.pl @@ -6,7 +6,10 @@ % % Lightweight file-backed P/2 facts for the JS WAM (Lua-style % javascript_wam_fact_sources/1). CSV/TSV and JSONL are read by Node -% with fs only. Answers must match SWI with the same triples. +% with fs only (D27 file(Path) is unchanged). GP-LMDB adds: +% source(P/2, indexed(Prefix)) — backend B, Prefix.data + Prefix.idx +% source(P/2, lmdb(Dir)) — backend A, opt-in npm `lmdb` +% Answers must match SWI with the same triples. % % swipl -q -g run_tests -t halt tests/test_wam_javascript_fact_sources.pl @@ -22,6 +25,15 @@ :- dynamic user:js_fs_has/2. :- dynamic user:js_fs_probe/0. :- dynamic user:js_fs_inline/2. +:- dynamic user:js_idx_edge/2. +:- dynamic user:js_idx_probe_bound/0. +:- dynamic user:js_idx_probe_unbound_len/0. +:- dynamic user:js_idx_probe_filter/0. +:- dynamic user:js_idx_probe_types/0. +:- dynamic user:js_idx_probe_shared/0. +:- dynamic user:js_lmdb_edge/2. +:- dynamic user:js_lmdb_probe_filter/0. +:- dynamic user:js_lmdb_probe_missing/0. install_fs_preds :- retractall(user:js_fs_edge/2), @@ -175,4 +187,316 @@ assertion(node_succeeded(Out)), assertion(sub_string(Out, _, _, _, "2")). +n_xs(N, Atom) :- + length(Cs, N), + maplist(=(0'x), Cs), + atom_codes(Atom, Cs). + +idx_pad(Pad) :- + n_xs(180, Pad). + +install_idx_preds :- + retractall(user:js_idx_edge(_, _)), + retractall(user:js_idx_probe_bound), + retractall(user:js_idx_probe_unbound_len), + retractall(user:js_idx_probe_filter), + retractall(user:js_idx_probe_types), + retractall(user:js_idx_probe_shared), + retractall(user:js_lmdb_edge(_, _)), + retractall(user:js_lmdb_probe_filter), + retractall(user:js_lmdb_probe_missing), + idx_pad(Pad), + forall(between(0, 4999, I), + ( format(atom(K), 'k~d', [I]), + format(atom(V), 'v~d_~w', [I, Pad]), + assertz(user:js_idx_edge(K, V)) + )), + assertz(user:js_idx_edge(probekey, alpha)), + assertz(user:js_idx_edge(probekey, beta)), + assertz(user:js_idx_edge(42, 99)), + assertz(user:js_idx_edge("strkey", atomval)), + assertz((user:js_idx_probe_bound :- + findall(Y, js_idx_edge(k2500, Y), L), + write(bound), write(L), nl, + L = [One], atom_concat(v2500, _, One))), + assertz((user:js_idx_probe_unbound_len :- + findall(1, js_idx_edge(_, _), L), + length(L, N), + write(unbound_len), write(N), nl, + N =:= 5004)), + assertz((user:js_idx_probe_filter :- + js_idx_edge(probekey, alpha), + \+ js_idx_edge(probekey, zzz), + findall(Y, js_idx_edge(probekey, Y), L), + L == [alpha, beta], + write(ok), nl)), + assertz((user:js_idx_probe_types :- + js_idx_edge(42, 99), + js_idx_edge("strkey", atomval), + write(ok), nl)), + assertz((user:js_idx_probe_shared :- + findall(K-V, js_idx_edge(K, V), All), + length(All, 5004), + findall(Y, js_idx_edge(probekey, Y), L), L == [alpha, beta], + js_idx_edge(42, 99), + js_idx_edge("strkey", atomval), + write(shared_ok), nl)), + assertz((user:js_lmdb_edge(X, Y) :- user:js_idx_edge(X, Y))), + assertz((user:js_lmdb_probe_filter :- + js_lmdb_edge(probekey, alpha), + \+ js_lmdb_edge(probekey, zzz), + findall(Y, js_lmdb_edge(probekey, Y), L), + L == [alpha, beta], + js_lmdb_edge(42, 99), + js_lmdb_edge("strkey", atomval), + write(ok), nl)), + assertz((user:js_lmdb_probe_missing :- js_lmdb_edge(probekey, alpha))). + +write_idx_tsv(Path) :- + idx_pad(Pad), + setup_call_cleanup( + open(Path, write, S), + ( forall(between(0, 4999, I), + format(S, 'k~d\tv~d_~w~n', [I, I, Pad])), + writeln(S, 'probekey alpha'), + writeln(S, 'probekey beta'), + writeln(S, '42 99'), + writeln(S, '"strkey" atomval') + ), + close(S)). + +uw_script(Name, Abs) :- + atom_concat('scripts/js_wam/', Name, Rel), + absolute_file_name(Rel, Abs, [access(read)]). + +run_node_cwd(Dir, Args, EnvPairs, Exit, Out, Err) :- + directory_file_path(Dir, 'js', JsDir), + maplist(env_assign, EnvPairs, Assigns), + atomic_list_concat(Assigns, ' ', Prefix), + quote_node_args(Args, Quoted), + atomic_list_concat(Quoted, ' ', ArgStr), + ( Prefix == "" + -> format(string(Cmd), 'node ~w', [ArgStr]) + ; format(string(Cmd), '~w node ~w', [Prefix, ArgStr]) + ), + process_create(path(bash), ['-lc', Cmd], + [cwd(JsDir), stdout(pipe(O)), stderr(pipe(E)), process(Pid)]), + read_string(O, _, OS), + read_string(E, _, ES), + close(O), close(E), + process_wait(Pid, exit(Exit)), + Out = OS, Err = ES. + +env_assign(Name=Value, Assign) :- + format(atom(Assign), '~w=~w', [Name, Value]). + +quote_node_args([], []). +quote_node_args([A|Rest], [Q|QRest]) :- + format(atom(Q), "'~w'", [A]), + quote_node_args(Rest, QRest). + +run_builder(Script, Args, Exit, Out) :- + process_create(path(node), [Script|Args], + [stdout(pipe(O)), stderr(pipe(E)), process(Pid)]), + read_string(O, _, OS), + read_string(E, _, ES), + close(O), close(E), + process_wait(Pid, exit(Exit)), + atomic_list_concat([OS, ES], Out). + +lmdb_pkg_prefix('/tmp/uw-lmdb-pkg'). + +lmdb_available :- + catch(ensure_lmdb_pkg, _, fail), + lmdb_pkg_prefix(P), + directory_file_path(P, 'node_modules', NM), + format(atom(Cmd), "NODE_PATH='~w' node -e 'require(\"lmdb\")'", [NM]), + process_create(path(bash), ['-lc', Cmd], + [stdout(pipe(O)), stderr(pipe(E)), process(Pid)]), + read_string(O, _, _), read_string(E, _, _), + close(O), close(E), + process_wait(Pid, exit(0)). + +ensure_lmdb_pkg :- + lmdb_pkg_prefix(P), + make_directory_path(P), + directory_file_path(P, 'node_modules/lmdb', Mod), + ( exists_directory(Mod) + -> true + ; process_create(path(npm), ['install', '--prefix', P, 'lmdb'], + [stdout(pipe(O)), stderr(pipe(E)), process(Pid)]), + read_string(O, _, _), read_string(E, _, ES), + close(O), close(E), + process_wait(Pid, exit(Code)), + (Code =:= 0 -> true ; throw(error(lmdb_npm_install(Code, ES), _))) + ). + +parse_fact_io_stats(Err, Bytes, DataSize) :- + split_string(Err, "\n", " \t\r", Lines), + member(Line, Lines), + sub_string(Line, _, _, _, "fact_io bytes_read="), + split_string(Line, " ", "", Parts), + member(BPart, Parts), sub_string(BPart, 0, _, _, "bytes_read="), + sub_string(BPart, 12, _, 0, BStr), number_string(Bytes, BStr), + member(DPart, Parts), sub_string(DPart, 0, _, _, "data_size="), + sub_string(DPart, 10, _, 0, DStr), number_string(DataSize, DStr), + !. + +test(indexed_store_bound_unbound_filter, [setup(install_idx_preds)]) :- + Dir = 'output/js_wam_fact_indexed', + make_directory_path(Dir), + directory_file_path(Dir, 'edges.tsv', Tsv), + directory_file_path(Dir, 'edges_store', Store), + write_idx_tsv(Tsv), + uw_script('uw_fact_index.js', Script), + run_builder(Script, ['build', Tsv, Store], BExit, BOut), + assertion(BExit =:= 0), + assertion(sub_string(BOut, _, _, _, "uw_fact_index")), + write_wam_javascript_project( + [user:js_idx_edge/2, user:js_idx_probe_bound/0, + user:js_idx_probe_unbound_len/0, user:js_idx_probe_filter/0, + user:js_idx_probe_types/0, user:js_idx_probe_shared/0], + [javascript_wam_fact_sources([source(js_idx_edge/2, indexed(Store))])], + Dir), + read_generated_js(Dir, Code), + assertion(sub_string(Code, _, _, _, 'kind: "indexed"')), + assertion(sub_string(Code, _, _, _, 'I.CallFactStream("js_idx_edge/2", 2)')), + run_node_args(Dir, ['js_idx_probe_bound/0'], BoundExit, BoundOut), + assertion(BoundExit =:= 0), + assertion(node_succeeded(BoundOut)), + assertion(sub_string(BoundOut, _, _, _, "v2500")), + run_node_args(Dir, ['js_idx_probe_unbound_len/0'], UExit, UOut), + assertion(UExit =:= 0), + assertion(node_succeeded(UOut)), + assertion(sub_string(UOut, _, _, _, "unbound_len5004")), + run_node_args(Dir, ['js_idx_probe_filter/0'], FExit, FOut), + assertion(FExit =:= 0), + assertion(node_succeeded(FOut)), + assertion(sub_string(FOut, _, _, _, "ok")), + run_node_args(Dir, ['js_idx_probe_types/0'], TExit, TOut), + assertion(TExit =:= 0), + assertion(node_succeeded(TOut)), + findall(Y, user:js_idx_edge(k2500, Y), SWIBound), + assertion(SWIBound = [_]), + findall(1, user:js_idx_edge(_, _), SWIAll), + length(SWIAll, 5004), + findall(Y, user:js_idx_edge(probekey, Y), SWIFilt), + assertion(SWIFilt == [alpha, beta]). + +test(indexed_store_bytes_read_proof, [setup(install_idx_preds)]) :- + Dir = 'output/js_wam_fact_indexed', + directory_file_path(Dir, 'edges_store', Store), + write_wam_javascript_project( + [user:js_idx_edge/2, user:js_idx_probe_bound/0], + [javascript_wam_fact_sources([source(js_idx_edge/2, indexed(Store))])], + Dir), + run_node_cwd(Dir, ['generated_program.js', 'js_idx_probe_bound/0'], + ['UW_FACT_IO_STATS'='1'], Exit, Out, Err), + assertion(Exit =:= 0), + assertion(node_succeeded(Out)), + parse_fact_io_stats(Err, Bytes, DataSize), + assertion(DataSize > 100000), + assertion(Bytes > 0), + assertion(Bytes * 20 < DataSize), + assertion(Bytes < 16384), + format(user_error, '~n[bytes-read proof] bytes_read=~w data_size=~w~n', + [Bytes, DataSize]). + +test(lmdb_missing_package_is_loud, [setup(install_idx_preds)]) :- + Dir = 'output/js_wam_fact_lmdb_missing', + make_directory_path(Dir), + write_wam_javascript_project( + [user:js_lmdb_edge/2, user:js_lmdb_probe_missing/0], + [javascript_wam_fact_sources([source(js_lmdb_edge/2, lmdb('/tmp/uw-no-such-lmdb'))])], + Dir), + read_generated_js(Dir, Code), + assertion(sub_string(Code, _, _, _, 'kind: "lmdb"')), + run_node_cwd(Dir, ['generated_program.js', 'js_lmdb_probe_missing/0'], + ['UW_LMDB_FORCE_MISSING'='1'], Exit, Out, Err), + assertion(Exit =\= 0), + atomic_list_concat([Out, Err], Combined), + assertion(sub_string(Combined, _, _, _, "npm install lmdb")), + assertion(sub_string(Combined, _, _, _, "not used as a fallback")), + assertion(sub_string(Combined, _, _, _, "lmdb(")). + +run_lmdb_builder(Tsv, LmdbDir, Exit, Combined) :- + uw_script('uw_fact_lmdb.js', Script), + lmdb_pkg_prefix(P), + directory_file_path(P, 'node_modules', NM), + format(atom(Cmd), "NODE_PATH='~w' node '~w' build '~w' '~w'", + [NM, Script, Tsv, LmdbDir]), + process_create(path(bash), ['-lc', Cmd], + [stdout(pipe(O)), stderr(pipe(E)), process(Pid)]), + read_string(O, _, OS), read_string(E, _, ES), + close(O), close(E), + process_wait(Pid, exit(Exit)), + atomic_list_concat([OS, ES], Combined). + +test(lmdb_store_when_available, + [setup(install_idx_preds), condition(lmdb_available)]) :- + Dir = 'output/js_wam_fact_lmdb', + make_directory_path(Dir), + directory_file_path(Dir, 'edges.tsv', Tsv), + directory_file_path(Dir, 'edges.lmdb', LmdbDir), + write_idx_tsv(Tsv), + run_lmdb_builder(Tsv, LmdbDir, BExit, BOut), + assertion(BExit =:= 0), + assertion(sub_string(BOut, _, _, _, "uw_fact_lmdb")), + write_wam_javascript_project( + [user:js_lmdb_edge/2, user:js_lmdb_probe_filter/0], + [javascript_wam_fact_sources([source(js_lmdb_edge/2, lmdb(LmdbDir))])], + Dir), + lmdb_pkg_prefix(P), + directory_file_path(P, 'node_modules', NM), + run_node_cwd(Dir, ['generated_program.js', 'js_lmdb_probe_filter/0'], + ['NODE_PATH'=NM], FExit, FOut, FErr), + assertion(FExit =:= 0), + assertion(node_succeeded(FOut)), + assertion(sub_string(FOut, _, _, _, "ok") ; sub_string(FErr, _, _, _, "ok")). + +test(indexed_and_lmdb_shared_semantics, [setup(install_idx_preds)]) :- + DirB = 'output/js_wam_fact_indexed', + make_directory_path(DirB), + directory_file_path(DirB, 'edges.tsv', TsvB), + directory_file_path(DirB, 'edges_store', StoreB), + write_idx_tsv(TsvB), + uw_script('uw_fact_index.js', ScriptB), + run_builder(ScriptB, ['build', TsvB, StoreB], IdxExit, _), + assertion(IdxExit =:= 0), + write_wam_javascript_project( + [user:js_idx_edge/2, user:js_idx_probe_shared/0], + [javascript_wam_fact_sources([source(js_idx_edge/2, indexed(StoreB))])], + DirB), + run_node_args(DirB, ['js_idx_probe_shared/0'], BExit, BOut), + assertion(BExit =:= 0), + assertion(node_succeeded(BOut)), + assertion(sub_string(BOut, _, _, _, "shared_ok")), + findall(Y, user:js_idx_edge(probekey, Y), SWI), + assertion(SWI == [alpha, beta]), + format(user_error, '~n[shared-semantics] SWI probekey -> ~q~n', [SWI]), + format(user_error, '[shared-semantics] backend B stdout:~n~w', [BOut]), + ( lmdb_available + -> DirA = 'output/js_wam_fact_lmdb', + make_directory_path(DirA), + directory_file_path(DirA, 'edges.tsv', TsvA), + directory_file_path(DirA, 'edges.lmdb', LmdbDir), + write_idx_tsv(TsvA), + run_lmdb_builder(TsvA, LmdbDir, LExit, _), + assertion(LExit =:= 0), + write_wam_javascript_project( + [user:js_idx_edge/2, user:js_idx_probe_shared/0], + [javascript_wam_fact_sources([source(js_idx_edge/2, lmdb(LmdbDir))])], + DirA), + lmdb_pkg_prefix(P), + directory_file_path(P, 'node_modules', NM), + run_node_cwd(DirA, ['generated_program.js', 'js_idx_probe_shared/0'], + ['NODE_PATH'=NM], AExit, AOut, AErr), + assertion(AExit =:= 0), + assertion(node_succeeded(AOut)), + assertion(sub_string(AOut, _, _, _, "shared_ok")), + format(user_error, '[shared-semantics] backend A stdout:~n~w', [AOut]), + format(user_error, '[shared-semantics] backend A stderr:~n~w', [AErr]) + ; format(user_error, '[shared-semantics] backend A skipped (lmdb package not loadable)~n', []) + ). + :- end_tests(js_wam_fact_sources). From 55508702109db1deb01f9b86d145377e0b70f13c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 1 Sep 2026 20:21:19 +0000 Subject: [PATCH 2/5] Force LMDB env directories (noSubdir: false) and count I/O reads. lmdb-js treats a path with an extension as the data file; edges.lmdb as a directory then throws EISDIR. Both the loader and runtime now open an environment directory. Bound-lookup stats also record n_reads. Co-authored-by: johns243a --- scripts/js_wam/uw_fact_lmdb.js | 5 ++++- .../targets/javascript_wam/runtime.js.mustache | 14 +++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/scripts/js_wam/uw_fact_lmdb.js b/scripts/js_wam/uw_fact_lmdb.js index 7ecfe0997..b71448030 100755 --- a/scripts/js_wam/uw_fact_lmdb.js +++ b/scripts/js_wam/uw_fact_lmdb.js @@ -62,7 +62,10 @@ if (typeof open !== "function") { const db = open({ path: storeDir, encoding: "binary", - keyEncoding: "binary" + keyEncoding: "binary", + // lmdb-js defaults noSubdir when the path has an extension (e.g. edges.lmdb). + // GP-LMDB A is always an environment *directory*. + noSubdir: false }); function putAll(store, list) { diff --git a/templates/targets/javascript_wam/runtime.js.mustache b/templates/targets/javascript_wam/runtime.js.mustache index 5f2218882..cb207a0ad 100644 --- a/templates/targets/javascript_wam/runtime.js.mustache +++ b/templates/targets/javascript_wam/runtime.js.mustache @@ -3271,6 +3271,7 @@ Runtime.read_facts_file = function (program, path) { // --------------------------------------------------------------------------- Runtime._fact_io_bytes = 0; Runtime._fact_io_data_size = 0; +Runtime._fact_io_reads = 0; Runtime._lmdb_force_missing = false; const UWFI_MAGIC = Buffer.from("UWFI"); @@ -3282,8 +3283,13 @@ const UWIX_ENTRY = 16; function fact_io_read(fd, length, position) { if (length <= 0) return Buffer.alloc(0); const buf = Buffer.alloc(length); - const n = require("fs").readSync(fd, buf, 0, length, position); + const n = require("fs").readSync(fd, buf, { + offset: 0, + length: length, + position: position + }); Runtime._fact_io_bytes = (Runtime._fact_io_bytes || 0) + n; + Runtime._fact_io_reads = (Runtime._fact_io_reads || 0) + 1; return n === length ? buf : buf.subarray(0, n); } @@ -3457,7 +3463,8 @@ function open_lmdb_store(source, predKey) { path: source.path, encoding: "binary", keyEncoding: "binary", - readOnly: true + readOnly: true, + noSubdir: false }); return source._lmdb; } @@ -3507,7 +3514,8 @@ if (process.env && process.env.UW_FACT_IO_STATS) { process.on("exit", function () { process.stderr.write( "fact_io bytes_read=" + String(Runtime._fact_io_bytes || 0) + - " data_size=" + String(Runtime._fact_io_data_size || 0) + "\n" + " data_size=" + String(Runtime._fact_io_data_size || 0) + + " n_reads=" + String(Runtime._fact_io_reads || 0) + "\n" ); }); } From 34c860e195afc3e7750ce543dab80ef998e60033 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 1 Sep 2026 20:21:33 +0000 Subject: [PATCH 3/5] Document lmdb-js noSubdir:false so .lmdb paths stay directories. Co-authored-by: johns243a --- docs/WAM_JAVASCRIPT_STATUS.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/WAM_JAVASCRIPT_STATUS.md b/docs/WAM_JAVASCRIPT_STATUS.md index 2d683dca7..d11779bc2 100644 --- a/docs/WAM_JAVASCRIPT_STATUS.md +++ b/docs/WAM_JAVASCRIPT_STATUS.md @@ -540,7 +540,9 @@ node scripts/js_wam/uw_fact_lmdb.js build The runtime loads the package **lazily** (`createRequire(__filename)("lmdb")`) only when a `lmdb(...)` source is actually used. `encoding: "binary"` and -`keyEncoding: "binary"`. +`keyEncoding: "binary"`. `noSubdir: false` is set explicitly: lmdb-js +otherwise treats a path with an extension (e.g. `edges.lmdb`) as the +data *file* and throws `EISDIR` if that path is a directory. **LMDB key encoding** (same cell payload as B): From 969038a64a98ba6e824ce5e7d3797faee4a4ff93 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 1 Sep 2026 20:22:44 +0000 Subject: [PATCH 4/5] Fix bytes_read parser (prefix is 11 chars) and keep raw stderr. sub_string(..., 12, ...) skipped the leading digit of 506, so the proof printed 6. Require bytes_read > 200 so a mis-parse cannot pass. Co-authored-by: johns243a --- tests/test_wam_javascript_fact_sources.pl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_wam_javascript_fact_sources.pl b/tests/test_wam_javascript_fact_sources.pl index b5d3e57b5..c8ef193e0 100644 --- a/tests/test_wam_javascript_fact_sources.pl +++ b/tests/test_wam_javascript_fact_sources.pl @@ -337,7 +337,7 @@ sub_string(Line, _, _, _, "fact_io bytes_read="), split_string(Line, " ", "", Parts), member(BPart, Parts), sub_string(BPart, 0, _, _, "bytes_read="), - sub_string(BPart, 12, _, 0, BStr), number_string(Bytes, BStr), + sub_string(BPart, 11, _, 0, BStr), number_string(Bytes, BStr), member(DPart, Parts), sub_string(DPart, 0, _, _, "data_size="), sub_string(DPart, 10, _, 0, DStr), number_string(DataSize, DStr), !. @@ -399,8 +399,8 @@ assertion(Bytes > 0), assertion(Bytes * 20 < DataSize), assertion(Bytes < 16384), - format(user_error, '~n[bytes-read proof] bytes_read=~w data_size=~w~n', - [Bytes, DataSize]). + format(user_error, '~n[bytes-read proof] bytes_read=~w data_size=~w~n[bytes-read proof raw stderr]~n~w~n', + [Bytes, DataSize, Err]). test(lmdb_missing_package_is_loud, [setup(install_idx_preds)]) :- Dir = 'output/js_wam_fact_lmdb_missing', From da671af17f27a6c3152907f56ac06226b2447f09 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 1 Sep 2026 20:23:40 +0000 Subject: [PATCH 5/5] Require bound-lookup bytes_read > 200 so a prefix mis-parse cannot pass. Co-authored-by: johns243a --- tests/test_wam_javascript_fact_sources.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_wam_javascript_fact_sources.pl b/tests/test_wam_javascript_fact_sources.pl index c8ef193e0..496c598a5 100644 --- a/tests/test_wam_javascript_fact_sources.pl +++ b/tests/test_wam_javascript_fact_sources.pl @@ -396,7 +396,7 @@ assertion(node_succeeded(Out)), parse_fact_io_stats(Err, Bytes, DataSize), assertion(DataSize > 100000), - assertion(Bytes > 0), + assertion(Bytes > 200), assertion(Bytes * 20 < DataSize), assertion(Bytes < 16384), format(user_error, '~n[bytes-read proof] bytes_read=~w data_size=~w~n[bytes-read proof raw stderr]~n~w~n',