From b1598bade0f819a485c146d14b8e21616e791ddc Mon Sep 17 00:00:00 2001 From: Martin Ebner Date: Sat, 11 Jul 2026 16:47:51 +0200 Subject: [PATCH] skills: file-write results are read back from disk (add write-file-b64) write-file and append-file returned a static success atom (WRITE-FILE-SUCCESS) that certifies only that the Prolog open/write/close calls succeeded - not that the intended content is on disk. Two documented failure classes pass straight through it: content mangled on the way to disk (nested quotes, backslashes) while the write itself "succeeds", and the agent later claiming writes or byte counts with nothing in its context to check against. Read-back-after-write discipline has proven unenforceable by prompting alone. The file-writing skills now route through a new src/fileio.py (same py-call pattern as websearch/logger) and return a result built exclusively from a read-back of the file on disk: WRITE-VERIFIED file=/tmp/x.txt bytes=201 sha256=1f9a2b... head='...' tail='...' APPEND-VERIFIED file=/tmp/log.txt bytes=12 sha256=... head='line1\nline2\n' tail='' WRITE-FAILED file=/nope/x.txt: [Errno 2] No such file or directory APPEND-FAILED file=/tmp/absent.txt: file does not exist The agent can only relay these values, not compose them, and can spot a mangled write in the same cycle by comparing head/tail against what it intended. Failures return explicit *-FAILED strings; today a failed open surfaces as a generic multi-command error with no file detail. New skill write-file-b64 takes base64-encoded content and writes the decoded bytes, closing the quote/backslash-mangling class at the input side (binary-safe, tolerates line-wrapped base64; registered in helper.py command parsing and lib_omegaclaw.metta like the other py-call modules). Design notes: - append-file keeps its contract: the file must exist, a newline is added. - appendToHistory (memory.metta) keeps the Prolog-backed append under the name append-file-raw: it passes a (library ...) file-spec term that only the Prolog open/4 path resolves. Its body drops the old exists_file guard (rebase note below). - Nothing consumes the WRITE-FILE-SUCCESS/APPEND-FILE-SUCCESS atoms (definition sites only), and string results are already the norm for skills (read-file, shell, metta, websearch return strings). - Result strings are single-line: head/tail snippets are 80 bytes with backslashes, newlines and carriage returns escaped, so a literal backslash-n in content is distinguishable from a real newline; sha256 is truncated to 16 hex chars and skipped for files over 2 MB. Docs: reference-skills-io.md Returns sections updated (they said "True", which was already out of date) and a write-file-b64 entry added; README reference line extended. Tests: Autotests/unit/test_fileio_verified_writes.py - 11 in-process unit tests (module loaded by file path; no container, network, or token), wired into Autotests/run_mandatory. The existing mock/ container tests (test_create_file_mock etc.) exercise the rewired skills end-to-end. Rebased onto main after OMEGA-239 (safe history/prompt reading) merged: its appendToHistory exists-check + create collapses into a single append-file-raw call, with append-file-raw dropping the old body's leading exists_file guard so that SWI open/4 in append mode creates a missing history file (verified in PeTTa: with the guard the clause fails silently on a missing file; without it the file is created and appended). Net behavior on the harness history path matches main, including the recreate scenario of mock/test_memory_missing_history_file_mock.py. OMEGA-239's getPrompt/getHistory safe-reads are untouched. --- Autotests/run_mandatory | 1 + Autotests/unit/test_fileio_verified_writes.py | 162 ++++++++++++++++++ docs/README.md | 2 +- docs/reference-skills-io.md | 47 ++++- lib_omegaclaw.metta | 1 + src/fileio.py | 82 +++++++++ src/helper.py | 5 +- src/memory.metta | 8 +- src/skills.metta | 24 ++- 9 files changed, 310 insertions(+), 22 deletions(-) create mode 100644 Autotests/unit/test_fileio_verified_writes.py create mode 100644 src/fileio.py diff --git a/Autotests/run_mandatory b/Autotests/run_mandatory index 75075c36..8acc4dcf 100644 --- a/Autotests/run_mandatory +++ b/Autotests/run_mandatory @@ -33,3 +33,4 @@ mock/test_transition_metta_to_remember_mock.py mock/test_transition_pin_to_remember_mock.py mock_websocket/test_wschat_unit.py test_websearch_smoke.py +unit/test_fileio_verified_writes.py diff --git a/Autotests/unit/test_fileio_verified_writes.py b/Autotests/unit/test_fileio_verified_writes.py new file mode 100644 index 00000000..cf575a1b --- /dev/null +++ b/Autotests/unit/test_fileio_verified_writes.py @@ -0,0 +1,162 @@ +"""In-process unit tests for src/fileio.py (verified file writes). + +The write-file / append-file / write-file-b64 skills return a result string +built from a read-back of the file on disk (size, sha256, head/tail) — ground +truth the agent relays instead of a static success atom it could confabulate +around. Failures return explicit *-FAILED strings instead of raising. + +No container, no network, no token — same pattern as +mock_websocket/test_wschat_unit.py: the module is loaded by file path. +""" +import base64 +import hashlib +import importlib.util +import logging +import os +import sys + +import pytest + +_REPO_ROOT = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", "..")) +_FILEIO_PATH = os.path.join(_REPO_ROOT, "src", "fileio.py") + +# fileio.py does `from src.logger import get_logger` (repo-root package) at import time. +if _REPO_ROOT not in sys.path: + sys.path.insert(0, _REPO_ROOT) + + +def _load_fileio(): + spec = importlib.util.spec_from_file_location("fileio_under_test", _FILEIO_PATH) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def fileio(): + return _load_fileio() + + +def _sha16(data): + return hashlib.sha256(data).hexdigest()[:16] + + +def test_write_result_is_read_back_from_disk(fileio, tmp_path, caplog): + caplog.set_level(logging.INFO) + target = tmp_path / "out.txt" + content = "hello world" + + result = fileio.write_file(str(target), content) + + assert target.read_bytes() == b"hello world" + assert result == ( + f"WRITE-VERIFIED file={target} bytes=11 sha256={_sha16(b'hello world')} " + "head='hello world' tail=''" + ) + assert "[FILE_IO] write ok" in caplog.text + + +def test_quotes_and_backslashes_land_byte_exact(fileio, tmp_path): + target = tmp_path / "tricky.txt" + content = 'he said "hi\\n" then C:\\path\\to \'quoted\'' + + result = fileio.write_file(str(target), content) + + assert target.read_bytes() == content.encode("utf-8") + assert f"sha256={_sha16(content.encode('utf-8'))}" in result + assert result.startswith("WRITE-VERIFIED") + + +def test_bytes_count_utf8_bytes_not_chars(fileio, tmp_path): + target = tmp_path / "unicode.txt" + content = "größe €100 ✓" + + result = fileio.write_file(str(target), content) + + assert f"bytes={len(content.encode('utf-8'))}" in result + assert target.read_text(encoding="utf-8") == content + + +def test_long_content_reports_head_and_tail_snippets(fileio, tmp_path): + target = tmp_path / "long.txt" + content = "A" * 100 + "\n" + "B" * 100 + + result = fileio.write_file(str(target), content) + + assert "head='" + "A" * 80 + "'" in result + assert "tail='" + "B" * 80 + "'" in result + assert "bytes=201" in result + + +def test_snippet_distinguishes_literal_backslash_n_from_newline(fileio, tmp_path): + literal = fileio.write_file(str(tmp_path / "a.txt"), "x\\ny") # backslash + n + y + real = fileio.write_file(str(tmp_path / "b.txt"), "x\ny") # real newline + + assert "head='x\\\\ny'" in literal + assert "head='x\\ny'" in real + assert literal.split("head=")[1] != real.split("head=")[1] + + +def test_write_failure_returns_failed_string_not_exception(fileio, tmp_path, caplog): + caplog.set_level(logging.INFO) + target = tmp_path / "no_such_dir" / "out.txt" + + result = fileio.write_file(str(target), "content") + + assert result.startswith(f"WRITE-FAILED file={target}:") + assert "[FILE_IO] write failed" in caplog.text + + +def test_append_adds_newline_and_verifies_whole_file(fileio, tmp_path): + target = tmp_path / "log.txt" + target.write_bytes(b"line1\n") + + result = fileio.append_file(str(target), "line2") + + expected = b"line1\nline2\n" # no f-string expression backslashes: PEP 701 is 3.12+ + assert target.read_bytes() == expected + assert result.startswith("APPEND-VERIFIED") + assert "bytes=12" in result + assert f"sha256={_sha16(expected)}" in result + + +def test_append_to_missing_file_fails_without_creating_it(fileio, tmp_path): + target = tmp_path / "absent.txt" + + result = fileio.append_file(str(target), "line") + + assert result == f"APPEND-FAILED file={target}: file does not exist" + assert not target.exists() + + +def test_b64_roundtrips_hostile_content_byte_exact(fileio, tmp_path): + target = tmp_path / "script.sh" + content = '#!/bin/bash\necho "a\\"b" \'c\'\nprintf \'%s\\n\' "$1"\n' + encoded = base64.b64encode(content.encode("utf-8")).decode("ascii") + + result = fileio.write_file_b64(str(target), encoded) + + assert target.read_bytes() == content.encode("utf-8") + assert result.startswith("WRITE-VERIFIED") + assert f"sha256={_sha16(content.encode('utf-8'))}" in result + + +def test_b64_accepts_line_wrapped_input(fileio, tmp_path): + target = tmp_path / "wrapped.txt" + content = "x" * 200 + encoded = base64.encodebytes(content.encode("utf-8")).decode("ascii") + assert "\n" in encoded.strip() # MIME-style 76-char wrapping + + result = fileio.write_file_b64(str(target), encoded) + + assert target.read_bytes() == content.encode("utf-8") + assert result.startswith("WRITE-VERIFIED") + + +def test_b64_invalid_input_fails_without_writing(fileio, tmp_path): + target = tmp_path / "never.txt" + + result = fileio.write_file_b64(str(target), "this is !!! not base64") + + assert result.startswith(f"WRITE-FAILED file={target}: invalid base64") + assert not target.exists() diff --git a/docs/README.md b/docs/README.md index 25e5618d..f78e61c1 100644 --- a/docs/README.md +++ b/docs/README.md @@ -49,7 +49,7 @@ Numbered in suggested reading order. Each tutorial is self-contained. User-facing MeTTa skills the agent invokes. Each page follows the template **Signature → Purpose → Parameters → Returns → Examples → Notes/Limits**. - [reference-skills-memory.md](./reference-skills-memory.md) — `remember`, `query`, `episodes`, `pin` -- [reference-skills-io.md](./reference-skills-io.md) — `shell`, `read-file`, `write-file`, `append-file` +- [reference-skills-io.md](./reference-skills-io.md) — `shell`, `read-file`, `write-file`, `write-file-b64`, `append-file` - [reference-skills-communication.md](./reference-skills-communication.md) — `send`, `receive`, `websearch` - [reference-skills-reasoning.md](./reference-skills-reasoning.md) — `metta` (NAL/PLN invocation surface) - [reference-skills-remote-agents.md](./reference-skills-remote-agents.md) — `tavily-search`, `technical-analysis` diff --git a/docs/reference-skills-io.md b/docs/reference-skills-io.md index e9d37a1f..62281dac 100644 --- a/docs/reference-skills-io.md +++ b/docs/reference-skills-io.md @@ -1,6 +1,6 @@ # Reference — I/O Skills -Defined in `src/skills.metta`, with the `shell` primitive backed by `src/skills.pl`. +Defined in `src/skills.metta`; the `shell` primitive is backed by `src/skills.pl`, the write skills by `src/fileio.py`. --- @@ -74,7 +74,9 @@ Create or overwrite a file with the given contents. - `contents` — the exact bytes to write. ### Returns -`True` on success. +A verification string read back from disk after the write: +`WRITE-VERIFIED file= bytes= sha256=<16 hex chars> head='' tail=''` +— or `WRITE-FAILED file=: ` on failure (the skill returns, never raises). ### Examples ```metta @@ -83,7 +85,40 @@ Create or overwrite a file with the given contents. ### Notes / Limits - Overwrites unconditionally — there is no confirm step. -- For incremental writes, use `append-file`. +- Writes the exact bytes given — no implicit trailing newline. +- For files up to 160 bytes the `tail` snippet is empty (`head` plus `sha256` already cover the content); for files over 2 MB the hash is reported as `sha256=skipped(large)`. +- For incremental writes, use `append-file`. For content containing quotes, backslashes + or newlines, prefer `write-file-b64`. + +--- + +## `write-file-b64` + +### Signature +```metta +(write-file-b64 "path" "base64content") +``` + +### Purpose +Create or overwrite a file with base64-decoded content — byte-exact for content containing +quotes, backslashes or newlines, which the plain-text argument path can mangle. + +### Parameters +- `path` — target filesystem path. +- `base64content` — base64 encoding of the exact bytes to write, as a single line + (whitespace inside the argument is tolerated). + +### Returns +The same `WRITE-VERIFIED …` / `WRITE-FAILED …` string as `write-file`. + +### Examples +```metta +(write-file-b64 "/tmp/script.sh" "IyEvYmluL2Jhc2gKZWNobyAiYVwiYiIgJ2MnCg==") +``` + +### Notes / Limits +- Invalid base64 fails without writing anything. +- Binary-safe: the decoded bytes are written verbatim. --- @@ -102,7 +137,8 @@ Append a line to an existing file, followed by a newline. - `line` — string to append. ### Returns -`True` on success. +`APPEND-VERIFIED file= bytes= sha256=<16 hex chars> head='…' tail='…'` +read back from disk — or `APPEND-FAILED file=: ` (e.g. when the file does not exist). ### Examples ```metta @@ -110,5 +146,6 @@ Append a line to an existing file, followed by a newline. ``` ### Notes / Limits -- Fails if the file does not exist (the call checks `exists_file` first). Create it with `write-file` first if needed. +- Fails if the file does not exist (the skill checks existence first). Create it with `write-file` first if needed. +- For files up to 160 bytes the `tail` snippet is empty (`head` plus `sha256` already cover the content); for files over 2 MB the hash is reported as `sha256=skipped(large)`. - A trailing newline is always added. diff --git a/lib_omegaclaw.metta b/lib_omegaclaw.metta index 4090722f..74baad24 100644 --- a/lib_omegaclaw.metta +++ b/lib_omegaclaw.metta @@ -12,6 +12,7 @@ !(import! &self (library OmegaClaw-Core ./src/helper.py)) !(import! &self (library OmegaClaw-Core ./src/agentverse.py)) !(import! &self (library OmegaClaw-Core ./src/channels)) +!(import! &self (library OmegaClaw-Core ./src/fileio.py)) !(import! &self (library OmegaClaw-Core ./src/skills)) !(import! &self (library OmegaClaw-Core ./src/websearch.py)) !(import! &self (library OmegaClaw-Core ./src/memory)) diff --git a/src/fileio.py b/src/fileio.py new file mode 100644 index 00000000..2bb83a97 --- /dev/null +++ b/src/fileio.py @@ -0,0 +1,82 @@ +"""Verified file writes for the write-file / append-file / write-file-b64 skills. + +The string these functions return is the skill result the agent sees. It is +built exclusively from a read-back of the file on disk after the operation +(size, sha256, head/tail snippets) — ground truth the agent can only relay, +not compose. Failures return an explicit *-FAILED string instead of raising, +so the agent loop always receives a result it can act on. +""" +import base64 +import hashlib +import os +import re + +from src.logger import get_logger + +logger = get_logger(__name__) + +SNIPPET_BYTES = 80 +HASH_MAX_BYTES = 2_000_000 + + +def _snippet(raw): + # escape backslashes first so literal \n in content stays distinguishable + # from a real newline in the rendered snippet + text = raw.decode("utf-8", errors="replace") + return text.replace("\\", "\\\\").replace("\r", "\\r").replace("\n", "\\n") + + +def _readback(path): + size = os.path.getsize(path) + with open(path, "rb") as f: + head = f.read(SNIPPET_BYTES) + if size > 2 * SNIPPET_BYTES: + f.seek(-SNIPPET_BYTES, os.SEEK_END) + tail = f.read(SNIPPET_BYTES) + else: + tail = b"" + if size <= HASH_MAX_BYTES: + with open(path, "rb") as f: + digest = hashlib.sha256(f.read()).hexdigest()[:16] + else: + digest = "skipped(large)" + return size, digest, _snippet(head), _snippet(tail) + + +def _write(path, data, append): + path = str(path) + word = "APPEND" if append else "WRITE" + if append: + data = data + b"\n" + try: + with open(path, "ab" if append else "wb") as f: + f.write(data) + size, digest, head, tail = _readback(path) + logger.info(f"[FILE_IO] {word.lower()} ok file={path} bytes={size} sha256={digest}") + return (f"{word}-VERIFIED file={path} bytes={size} sha256={digest} " + f"head='{head}' tail='{tail}'") + except Exception as e: # the skill must return a result, never raise into the loop + logger.error(f"[FILE_IO] {word.lower()} failed file={path} err={e}") + return f"{word}-FAILED file={path}: {e}" + + +def write_file(path, content): + return _write(path, str(content).encode("utf-8"), append=False) + + +def append_file(path, content): + path = str(path) + if not os.path.exists(path): + logger.error(f"[FILE_IO] append failed file={path} err=file does not exist") + return f"APPEND-FAILED file={path}: file does not exist" + return _write(path, str(content).encode("utf-8"), append=True) + + +def write_file_b64(path, content_b64): + try: + # tolerate MIME-style line-wrapped base64 (embedded whitespace) + data = base64.b64decode(re.sub(r"\s+", "", str(content_b64)), validate=True) + except Exception as e: + logger.error(f"[FILE_IO] write failed file={path} err=invalid base64") + return f"WRITE-FAILED file={path}: invalid base64 ({e})" + return _write(path, data, append=False) diff --git a/src/helper.py b/src/helper.py index 31626303..91c946a3 100644 --- a/src/helper.py +++ b/src/helper.py @@ -18,6 +18,7 @@ "tavily-search", "technical-analysis", "write-file", + "write-file-b64", } @@ -135,7 +136,7 @@ def _merge_send_continuations(lines): def balance_parentheses(s): s = s.replace("_quote_", '"').replace("_newline_", "\n") sexprs = [] - special_two_arg_cmds = {"write-file", "append-file"} + special_two_arg_cmds = {"write-file", "append-file", "write-file-b64"} lines = [line.strip() for line in s.splitlines() if line.strip()] lines = _merge_send_continuations(lines) for line in lines: @@ -209,6 +210,8 @@ def normalize_string(x): def test_balance_parenthesis(): assert balance_parentheses('(write-file test.txt hello world)') == '((write-file "test.txt" "hello world"))' assert balance_parentheses('(append-file test.txt hello world)') == '((append-file "test.txt" "hello world"))' + assert balance_parentheses('(write-file-b64 test.txt aGVsbG8=)') == '((write-file-b64 "test.txt" "aGVsbG8="))' + assert balance_parentheses('write-file-b64 test.txt aGVsbG8=') == '((write-file-b64 "test.txt" "aGVsbG8="))' assert balance_parentheses('(write-file "test.txt" hello world)') == '((write-file "test.txt" "hello world"))' assert balance_parentheses('(write-file "test.txt" "hello world")') == '((write-file "test.txt" "hello world"))' assert balance_parentheses('(write-file test.txt "hello world")') == '((write-file "test.txt" "hello world"))' diff --git a/src/memory.metta b/src/memory.metta index 89007afd..8c6b9c6c 100644 --- a/src/memory.metta +++ b/src/memory.metta @@ -48,13 +48,7 @@ (useGPTEmbedding (string-safe $str)))) (= (appendToHistory $addition) - (let $history_file - (library OmegaClaw-Core ./memory/history.metta) - (progn - (if (exists-file $history_file) - () - (write-file $history_file "")) - (append-file $history_file (swrite $addition))))) + (append-file-raw (library OmegaClaw-Core ./memory/history.metta) (swrite $addition))) (= (remember $str) (progn (py-call (lib_chromadb.remember $str (embed $str) (get_time_as_string))) diff --git a/src/skills.metta b/src/skills.metta index 355284f4..3cf2f178 100644 --- a/src/skills.metta +++ b/src/skills.metta @@ -7,8 +7,9 @@ ;SHELL AND FILE I/O: "- Execute shell command without apostrophe in string, it returns the command output to you: shell string" "- Read file to string: read-file filename" - "- Write string to file: write-file filename string" - "- Append line to file: append-file filename string", + "- Write string to file, the result is read back from disk (bytes, sha256, head/tail) - relay it, never claim a write succeeded without it: write-file filename string" + "- Write base64-encoded content to file as a single line, prefer it when the content contains quotes, backslashes or multiple lines: write-file-b64 filename base64string" + "- Append line to existing file, result read back from disk like write-file: append-file filename string", ;COMMUNICATION CHANNELS: "- Send message to user: send string" "- Search the web: websearch string" @@ -35,14 +36,21 @@ $content)) (= (write-file $file $str) - (progn (translatePredicate (open $file write $Out)) - (translatePredicate (write $Out $str)) - (translatePredicate (close $Out)) - WRITE-FILE-SUCCESS)) + (py-call (fileio.write_file $file $str))) + +(= (write-file-b64 $file $b64) + (py-call (fileio.write_file_b64 $file $b64))) (= (append-file $file $str) - (progn (translatePredicate (exists_file $file)) - (translatePredicate (open $file append $Out)) + (py-call (fileio.append_file $file $str))) + +; Prolog-backed append used by the harness itself (memory.metta appendToHistory): +; it receives a (library ...) file-spec term that only the Prolog open/4 path +; resolves. Body is the previous append-file minus its exists_file guard: +; open/4 in append mode creates a missing file, so the history file +; self-heals instead of the append failing silently. +(= (append-file-raw $file $str) + (progn (translatePredicate (open $file append $Out)) (translatePredicate (write $Out $str)) (translatePredicate (nl $Out)), (translatePredicate (close $Out))