diff --git a/Autotests/run_mandatory b/Autotests/run_mandatory index 75075c36..8acc4dcf 100644 --- a/Autotests/run_mandatory +++ b/Autotests/run_mandatory @@ -33,3 +33,4 @@ mock/test_transition_metta_to_remember_mock.py mock/test_transition_pin_to_remember_mock.py mock_websocket/test_wschat_unit.py test_websearch_smoke.py +unit/test_fileio_verified_writes.py diff --git a/Autotests/unit/test_fileio_verified_writes.py b/Autotests/unit/test_fileio_verified_writes.py new file mode 100644 index 00000000..cf575a1b --- /dev/null +++ b/Autotests/unit/test_fileio_verified_writes.py @@ -0,0 +1,162 @@ +"""In-process unit tests for src/fileio.py (verified file writes). + +The write-file / append-file / write-file-b64 skills return a result string +built from a read-back of the file on disk (size, sha256, head/tail) — ground +truth the agent relays instead of a static success atom it could confabulate +around. Failures return explicit *-FAILED strings instead of raising. + +No container, no network, no token — same pattern as +mock_websocket/test_wschat_unit.py: the module is loaded by file path. +""" +import base64 +import hashlib +import importlib.util +import logging +import os +import sys + +import pytest + +_REPO_ROOT = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", "..")) +_FILEIO_PATH = os.path.join(_REPO_ROOT, "src", "fileio.py") + +# fileio.py does `from src.logger import get_logger` (repo-root package) at import time. +if _REPO_ROOT not in sys.path: + sys.path.insert(0, _REPO_ROOT) + + +def _load_fileio(): + spec = importlib.util.spec_from_file_location("fileio_under_test", _FILEIO_PATH) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def fileio(): + return _load_fileio() + + +def _sha16(data): + return hashlib.sha256(data).hexdigest()[:16] + + +def test_write_result_is_read_back_from_disk(fileio, tmp_path, caplog): + caplog.set_level(logging.INFO) + target = tmp_path / "out.txt" + content = "hello world" + + result = fileio.write_file(str(target), content) + + assert target.read_bytes() == b"hello world" + assert result == ( + f"WRITE-VERIFIED file={target} bytes=11 sha256={_sha16(b'hello world')} " + "head='hello world' tail=''" + ) + assert "[FILE_IO] write ok" in caplog.text + + +def test_quotes_and_backslashes_land_byte_exact(fileio, tmp_path): + target = tmp_path / "tricky.txt" + content = 'he said "hi\\n" then C:\\path\\to \'quoted\'' + + result = fileio.write_file(str(target), content) + + assert target.read_bytes() == content.encode("utf-8") + assert f"sha256={_sha16(content.encode('utf-8'))}" in result + assert result.startswith("WRITE-VERIFIED") + + +def test_bytes_count_utf8_bytes_not_chars(fileio, tmp_path): + target = tmp_path / "unicode.txt" + content = "größe €100 ✓" + + result = fileio.write_file(str(target), content) + + assert f"bytes={len(content.encode('utf-8'))}" in result + assert target.read_text(encoding="utf-8") == content + + +def test_long_content_reports_head_and_tail_snippets(fileio, tmp_path): + target = tmp_path / "long.txt" + content = "A" * 100 + "\n" + "B" * 100 + + result = fileio.write_file(str(target), content) + + assert "head='" + "A" * 80 + "'" in result + assert "tail='" + "B" * 80 + "'" in result + assert "bytes=201" in result + + +def test_snippet_distinguishes_literal_backslash_n_from_newline(fileio, tmp_path): + literal = fileio.write_file(str(tmp_path / "a.txt"), "x\\ny") # backslash + n + y + real = fileio.write_file(str(tmp_path / "b.txt"), "x\ny") # real newline + + assert "head='x\\\\ny'" in literal + assert "head='x\\ny'" in real + assert literal.split("head=")[1] != real.split("head=")[1] + + +def test_write_failure_returns_failed_string_not_exception(fileio, tmp_path, caplog): + caplog.set_level(logging.INFO) + target = tmp_path / "no_such_dir" / "out.txt" + + result = fileio.write_file(str(target), "content") + + assert result.startswith(f"WRITE-FAILED file={target}:") + assert "[FILE_IO] write failed" in caplog.text + + +def test_append_adds_newline_and_verifies_whole_file(fileio, tmp_path): + target = tmp_path / "log.txt" + target.write_bytes(b"line1\n") + + result = fileio.append_file(str(target), "line2") + + expected = b"line1\nline2\n" # no f-string expression backslashes: PEP 701 is 3.12+ + assert target.read_bytes() == expected + assert result.startswith("APPEND-VERIFIED") + assert "bytes=12" in result + assert f"sha256={_sha16(expected)}" in result + + +def test_append_to_missing_file_fails_without_creating_it(fileio, tmp_path): + target = tmp_path / "absent.txt" + + result = fileio.append_file(str(target), "line") + + assert result == f"APPEND-FAILED file={target}: file does not exist" + assert not target.exists() + + +def test_b64_roundtrips_hostile_content_byte_exact(fileio, tmp_path): + target = tmp_path / "script.sh" + content = '#!/bin/bash\necho "a\\"b" \'c\'\nprintf \'%s\\n\' "$1"\n' + encoded = base64.b64encode(content.encode("utf-8")).decode("ascii") + + result = fileio.write_file_b64(str(target), encoded) + + assert target.read_bytes() == content.encode("utf-8") + assert result.startswith("WRITE-VERIFIED") + assert f"sha256={_sha16(content.encode('utf-8'))}" in result + + +def test_b64_accepts_line_wrapped_input(fileio, tmp_path): + target = tmp_path / "wrapped.txt" + content = "x" * 200 + encoded = base64.encodebytes(content.encode("utf-8")).decode("ascii") + assert "\n" in encoded.strip() # MIME-style 76-char wrapping + + result = fileio.write_file_b64(str(target), encoded) + + assert target.read_bytes() == content.encode("utf-8") + assert result.startswith("WRITE-VERIFIED") + + +def test_b64_invalid_input_fails_without_writing(fileio, tmp_path): + target = tmp_path / "never.txt" + + result = fileio.write_file_b64(str(target), "this is !!! not base64") + + assert result.startswith(f"WRITE-FAILED file={target}: invalid base64") + assert not target.exists() diff --git a/docs/README.md b/docs/README.md index 25e5618d..f78e61c1 100644 --- a/docs/README.md +++ b/docs/README.md @@ -49,7 +49,7 @@ Numbered in suggested reading order. Each tutorial is self-contained. User-facing MeTTa skills the agent invokes. Each page follows the template **Signature → Purpose → Parameters → Returns → Examples → Notes/Limits**. - [reference-skills-memory.md](./reference-skills-memory.md) — `remember`, `query`, `episodes`, `pin` -- [reference-skills-io.md](./reference-skills-io.md) — `shell`, `read-file`, `write-file`, `append-file` +- [reference-skills-io.md](./reference-skills-io.md) — `shell`, `read-file`, `write-file`, `write-file-b64`, `append-file` - [reference-skills-communication.md](./reference-skills-communication.md) — `send`, `receive`, `websearch` - [reference-skills-reasoning.md](./reference-skills-reasoning.md) — `metta` (NAL/PLN invocation surface) - [reference-skills-remote-agents.md](./reference-skills-remote-agents.md) — `tavily-search`, `technical-analysis` diff --git a/docs/reference-skills-io.md b/docs/reference-skills-io.md index e9d37a1f..62281dac 100644 --- a/docs/reference-skills-io.md +++ b/docs/reference-skills-io.md @@ -1,6 +1,6 @@ # Reference — I/O Skills -Defined in `src/skills.metta`, with the `shell` primitive backed by `src/skills.pl`. +Defined in `src/skills.metta`; the `shell` primitive is backed by `src/skills.pl`, the write skills by `src/fileio.py`. --- @@ -74,7 +74,9 @@ Create or overwrite a file with the given contents. - `contents` — the exact bytes to write. ### Returns -`True` on success. +A verification string read back from disk after the write: +`WRITE-VERIFIED file= bytes= sha256=<16 hex chars> head='' tail=''` +— or `WRITE-FAILED file=: ` on failure (the skill returns, never raises). ### Examples ```metta @@ -83,7 +85,40 @@ Create or overwrite a file with the given contents. ### Notes / Limits - Overwrites unconditionally — there is no confirm step. -- For incremental writes, use `append-file`. +- Writes the exact bytes given — no implicit trailing newline. +- For files up to 160 bytes the `tail` snippet is empty (`head` plus `sha256` already cover the content); for files over 2 MB the hash is reported as `sha256=skipped(large)`. +- For incremental writes, use `append-file`. For content containing quotes, backslashes + or newlines, prefer `write-file-b64`. + +--- + +## `write-file-b64` + +### Signature +```metta +(write-file-b64 "path" "base64content") +``` + +### Purpose +Create or overwrite a file with base64-decoded content — byte-exact for content containing +quotes, backslashes or newlines, which the plain-text argument path can mangle. + +### Parameters +- `path` — target filesystem path. +- `base64content` — base64 encoding of the exact bytes to write, as a single line + (whitespace inside the argument is tolerated). + +### Returns +The same `WRITE-VERIFIED …` / `WRITE-FAILED …` string as `write-file`. + +### Examples +```metta +(write-file-b64 "/tmp/script.sh" "IyEvYmluL2Jhc2gKZWNobyAiYVwiYiIgJ2MnCg==") +``` + +### Notes / Limits +- Invalid base64 fails without writing anything. +- Binary-safe: the decoded bytes are written verbatim. --- @@ -102,7 +137,8 @@ Append a line to an existing file, followed by a newline. - `line` — string to append. ### Returns -`True` on success. +`APPEND-VERIFIED file= bytes= sha256=<16 hex chars> head='…' tail='…'` +read back from disk — or `APPEND-FAILED file=: ` (e.g. when the file does not exist). ### Examples ```metta @@ -110,5 +146,6 @@ Append a line to an existing file, followed by a newline. ``` ### Notes / Limits -- Fails if the file does not exist (the call checks `exists_file` first). Create it with `write-file` first if needed. +- Fails if the file does not exist (the skill checks existence first). Create it with `write-file` first if needed. +- For files up to 160 bytes the `tail` snippet is empty (`head` plus `sha256` already cover the content); for files over 2 MB the hash is reported as `sha256=skipped(large)`. - A trailing newline is always added. diff --git a/lib_omegaclaw.metta b/lib_omegaclaw.metta index 4090722f..74baad24 100644 --- a/lib_omegaclaw.metta +++ b/lib_omegaclaw.metta @@ -12,6 +12,7 @@ !(import! &self (library OmegaClaw-Core ./src/helper.py)) !(import! &self (library OmegaClaw-Core ./src/agentverse.py)) !(import! &self (library OmegaClaw-Core ./src/channels)) +!(import! &self (library OmegaClaw-Core ./src/fileio.py)) !(import! &self (library OmegaClaw-Core ./src/skills)) !(import! &self (library OmegaClaw-Core ./src/websearch.py)) !(import! &self (library OmegaClaw-Core ./src/memory)) diff --git a/src/fileio.py b/src/fileio.py new file mode 100644 index 00000000..2bb83a97 --- /dev/null +++ b/src/fileio.py @@ -0,0 +1,82 @@ +"""Verified file writes for the write-file / append-file / write-file-b64 skills. + +The string these functions return is the skill result the agent sees. It is +built exclusively from a read-back of the file on disk after the operation +(size, sha256, head/tail snippets) — ground truth the agent can only relay, +not compose. Failures return an explicit *-FAILED string instead of raising, +so the agent loop always receives a result it can act on. +""" +import base64 +import hashlib +import os +import re + +from src.logger import get_logger + +logger = get_logger(__name__) + +SNIPPET_BYTES = 80 +HASH_MAX_BYTES = 2_000_000 + + +def _snippet(raw): + # escape backslashes first so literal \n in content stays distinguishable + # from a real newline in the rendered snippet + text = raw.decode("utf-8", errors="replace") + return text.replace("\\", "\\\\").replace("\r", "\\r").replace("\n", "\\n") + + +def _readback(path): + size = os.path.getsize(path) + with open(path, "rb") as f: + head = f.read(SNIPPET_BYTES) + if size > 2 * SNIPPET_BYTES: + f.seek(-SNIPPET_BYTES, os.SEEK_END) + tail = f.read(SNIPPET_BYTES) + else: + tail = b"" + if size <= HASH_MAX_BYTES: + with open(path, "rb") as f: + digest = hashlib.sha256(f.read()).hexdigest()[:16] + else: + digest = "skipped(large)" + return size, digest, _snippet(head), _snippet(tail) + + +def _write(path, data, append): + path = str(path) + word = "APPEND" if append else "WRITE" + if append: + data = data + b"\n" + try: + with open(path, "ab" if append else "wb") as f: + f.write(data) + size, digest, head, tail = _readback(path) + logger.info(f"[FILE_IO] {word.lower()} ok file={path} bytes={size} sha256={digest}") + return (f"{word}-VERIFIED file={path} bytes={size} sha256={digest} " + f"head='{head}' tail='{tail}'") + except Exception as e: # the skill must return a result, never raise into the loop + logger.error(f"[FILE_IO] {word.lower()} failed file={path} err={e}") + return f"{word}-FAILED file={path}: {e}" + + +def write_file(path, content): + return _write(path, str(content).encode("utf-8"), append=False) + + +def append_file(path, content): + path = str(path) + if not os.path.exists(path): + logger.error(f"[FILE_IO] append failed file={path} err=file does not exist") + return f"APPEND-FAILED file={path}: file does not exist" + return _write(path, str(content).encode("utf-8"), append=True) + + +def write_file_b64(path, content_b64): + try: + # tolerate MIME-style line-wrapped base64 (embedded whitespace) + data = base64.b64decode(re.sub(r"\s+", "", str(content_b64)), validate=True) + except Exception as e: + logger.error(f"[FILE_IO] write failed file={path} err=invalid base64") + return f"WRITE-FAILED file={path}: invalid base64 ({e})" + return _write(path, data, append=False) diff --git a/src/helper.py b/src/helper.py index 31626303..91c946a3 100644 --- a/src/helper.py +++ b/src/helper.py @@ -18,6 +18,7 @@ "tavily-search", "technical-analysis", "write-file", + "write-file-b64", } @@ -135,7 +136,7 @@ def _merge_send_continuations(lines): def balance_parentheses(s): s = s.replace("_quote_", '"').replace("_newline_", "\n") sexprs = [] - special_two_arg_cmds = {"write-file", "append-file"} + special_two_arg_cmds = {"write-file", "append-file", "write-file-b64"} lines = [line.strip() for line in s.splitlines() if line.strip()] lines = _merge_send_continuations(lines) for line in lines: @@ -209,6 +210,8 @@ def normalize_string(x): def test_balance_parenthesis(): assert balance_parentheses('(write-file test.txt hello world)') == '((write-file "test.txt" "hello world"))' assert balance_parentheses('(append-file test.txt hello world)') == '((append-file "test.txt" "hello world"))' + assert balance_parentheses('(write-file-b64 test.txt aGVsbG8=)') == '((write-file-b64 "test.txt" "aGVsbG8="))' + assert balance_parentheses('write-file-b64 test.txt aGVsbG8=') == '((write-file-b64 "test.txt" "aGVsbG8="))' assert balance_parentheses('(write-file "test.txt" hello world)') == '((write-file "test.txt" "hello world"))' assert balance_parentheses('(write-file "test.txt" "hello world")') == '((write-file "test.txt" "hello world"))' assert balance_parentheses('(write-file test.txt "hello world")') == '((write-file "test.txt" "hello world"))' diff --git a/src/memory.metta b/src/memory.metta index 89007afd..8c6b9c6c 100644 --- a/src/memory.metta +++ b/src/memory.metta @@ -48,13 +48,7 @@ (useGPTEmbedding (string-safe $str)))) (= (appendToHistory $addition) - (let $history_file - (library OmegaClaw-Core ./memory/history.metta) - (progn - (if (exists-file $history_file) - () - (write-file $history_file "")) - (append-file $history_file (swrite $addition))))) + (append-file-raw (library OmegaClaw-Core ./memory/history.metta) (swrite $addition))) (= (remember $str) (progn (py-call (lib_chromadb.remember $str (embed $str) (get_time_as_string))) diff --git a/src/skills.metta b/src/skills.metta index 355284f4..3cf2f178 100644 --- a/src/skills.metta +++ b/src/skills.metta @@ -7,8 +7,9 @@ ;SHELL AND FILE I/O: "- Execute shell command without apostrophe in string, it returns the command output to you: shell string" "- Read file to string: read-file filename" - "- Write string to file: write-file filename string" - "- Append line to file: append-file filename string", + "- Write string to file, the result is read back from disk (bytes, sha256, head/tail) - relay it, never claim a write succeeded without it: write-file filename string" + "- Write base64-encoded content to file as a single line, prefer it when the content contains quotes, backslashes or multiple lines: write-file-b64 filename base64string" + "- Append line to existing file, result read back from disk like write-file: append-file filename string", ;COMMUNICATION CHANNELS: "- Send message to user: send string" "- Search the web: websearch string" @@ -35,14 +36,21 @@ $content)) (= (write-file $file $str) - (progn (translatePredicate (open $file write $Out)) - (translatePredicate (write $Out $str)) - (translatePredicate (close $Out)) - WRITE-FILE-SUCCESS)) + (py-call (fileio.write_file $file $str))) + +(= (write-file-b64 $file $b64) + (py-call (fileio.write_file_b64 $file $b64))) (= (append-file $file $str) - (progn (translatePredicate (exists_file $file)) - (translatePredicate (open $file append $Out)) + (py-call (fileio.append_file $file $str))) + +; Prolog-backed append used by the harness itself (memory.metta appendToHistory): +; it receives a (library ...) file-spec term that only the Prolog open/4 path +; resolves. Body is the previous append-file minus its exists_file guard: +; open/4 in append mode creates a missing file, so the history file +; self-heals instead of the append failing silently. +(= (append-file-raw $file $str) + (progn (translatePredicate (open $file append $Out)) (translatePredicate (write $Out $str)) (translatePredicate (nl $Out)), (translatePredicate (close $Out))