diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ac9590c..9e8771a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -42,12 +42,22 @@ jobs: - name: Verify exe run: | $env:CITESEAL_PORT = "8765" - $server = Start-Process -FilePath "tools/dist/citeseal_server.exe" -PassThru + $env:CITESEAL_ROOT = (Resolve-Path "tests/fixtures/accounts/example_user/tweets/2026/2026-07/20260708_180000_1234567890").Path + $originalPath = $env:PATH + $server = $null try { - Start-Sleep -Seconds 5 - (Invoke-WebRequest -UseBasicParsing "http://127.0.0.1:8765/api/health").Content + # The frozen process and its CLI child must not discover host Python. + $env:PATH = "$env:SystemRoot\System32" + $server = Start-Process -FilePath "tools/dist/citeseal_server.exe" -PassThru + $env:PATH = $originalPath + python tools/server/smoke_test_server.py ` + --root "$env:CITESEAL_ROOT" --timeout 45 + if ($LASTEXITCODE -ne 0) { throw "Windows frozen smoke test failed" } } finally { - Stop-Process -Id $server.Id -Force -ErrorAction SilentlyContinue + $env:PATH = $originalPath + if ($null -ne $server) { + Stop-Process -Id $server.Id -Force -ErrorAction SilentlyContinue + } } - name: Upload artifact uses: actions/upload-artifact@v4 @@ -71,12 +81,19 @@ jobs: python-version: "3.12" - name: Build portable tarball run: bash tools/server/build_linux.sh - - name: Smoke-test + - name: Verify and smoke-test run: | + python tools/server/verify_linux_tarball.py tools/dist/citeseal_server-linux-x64.tar.gz + rm -rf /tmp/citeseal_server-linux-x64 tar -xzf tools/dist/citeseal_server-linux-x64.tar.gz -C /tmp - CITESEAL_ROOT=$PWD/../accounts /tmp/citeseal_server-linux-x64/bin/run.sh & - sleep 5 - curl -sf http://127.0.0.1:8765/api/health + PATH=/nonexistent \ + CITESEAL_HOST=127.0.0.1 CITESEAL_PORT=8765 \ + CITESEAL_ROOT="$PWD/tests/fixtures/accounts/example_user/tweets/2026/2026-07/20260708_180000_1234567890" \ + /tmp/citeseal_server-linux-x64/bin/citeseal_server & + server_pid=$! + trap 'kill "$server_pid" 2>/dev/null || true' EXIT + python tools/server/smoke_test_server.py \ + --root "$PWD/tests/fixtures/accounts/example_user/tweets/2026/2026-07/20260708_180000_1234567890" --timeout 45 - name: Upload artifact uses: actions/upload-artifact@v4 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index cdaf291..87a0162 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,9 @@ Maintenance release restoring cross-platform demo execution and release artifact - Made the live demo's terminal output ASCII-safe and its JSON file I/O explicitly UTF-8, preventing `UnicodeEncodeError` failures on legacy Windows console encodings. - Added a `cp1252` subprocess regression test so Windows console compatibility remains covered by the full test suite. - Repaired the Windows release job by installing and invoking PyInstaller explicitly, fixing its output path, and setting the health-check port before server startup. -- Corrected the Linux tarball source layout so bundled scripts remain directly importable; added release packaging contract tests for both platforms. +- Replaced the non-relocatable Linux virtual-environment payload with a self-contained PyInstaller executable; frozen background jobs now re-execute the embedded CiteSeal CLI without searching for a host Python interpreter. +- Fixed explicit `--extract` and `--out` path handling for Markdown/PDF commands and added a ReportLab CJK fallback for minimal Linux systems without an installed CJK font. +- Hardened release verification against unsafe tar members and added real `/api/run` validation-job smoke tests on Windows and Linux; the Linux gate runs the frozen server with no usable host `PATH`. ### Added - ADR (Architecture Decision Records): `docs/adr/0001-agent-bundle-as-agent-consumption-layer.md`, `docs/adr/0002-local-first-boundary.md` diff --git a/README.md b/README.md index 95df19e..129940d 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ It is designed for builders who use social media as a research, product, or mark ![CiteSeal architecture](docs/assets/architecture.svg) -> **Status:** early-stage but functional. Playwright-based capture, structured storage with schema validation, Markdown/PDF/OCR export helpers, a FastAPI local server, GitHub Actions CI, 218 passing tests, agent bundle + provenance manifest exports, and a Flutter client skeleton for mobile/desktop review. +> **Status:** early-stage but functional. Playwright-based capture, structured storage with schema validation, Markdown/PDF/OCR export helpers, a FastAPI local server, cross-platform GitHub Actions CI, agent bundle + provenance manifest exports, and a Flutter client skeleton for mobile/desktop review. --- diff --git a/tests/server/test_frozen_runtime.py b/tests/server/test_frozen_runtime.py new file mode 100644 index 0000000..9e18056 --- /dev/null +++ b/tests/server/test_frozen_runtime.py @@ -0,0 +1,108 @@ +"""Contracts for the self-contained PyInstaller server runtime.""" +from __future__ import annotations + +import importlib +import json +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parent.parent.parent +TOOLS_DIR = ROOT / "tools" +SCRIPTS_DIR = TOOLS_DIR / "scripts" +SERVER_DIR = TOOLS_DIR / "server" +for _path in (str(SCRIPTS_DIR), str(TOOLS_DIR), str(SERVER_DIR)): + if _path not in sys.path: + sys.path.insert(0, _path) + +frozen_entry = importlib.import_module("_frozen_entry") +server_app = importlib.import_module("app") +cli = importlib.import_module("citeseal") + + +def test_frozen_server_job_reexecutes_itself_in_cli_mode(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(server_app, "FROZEN", True, raising=False) + monkeypatch.setattr(server_app, "PYTHON", "/bundle/citeseal_server") + job = server_app.Job( + id="job1", + op="validate", + args={"root": "/data/accounts", "quiet": True}, + ) + + assert server_app._job_command(job) == [ + "/bundle/citeseal_server", + "--citeseal-cli", + "validate", + "--root", + "/data/accounts", + "--quiet", + ] + + +def test_frozen_entry_dispatches_embedded_cli_before_server( + monkeypatch: pytest.MonkeyPatch, +) -> None: + assert hasattr(frozen_entry, "_run_embedded_cli") + assert hasattr(frozen_entry, "_run_server") + + calls: list[list[str]] = [] + monkeypatch.setattr( + frozen_entry, + "_run_embedded_cli", + lambda argv: calls.append(list(argv)) or 7, + ) + monkeypatch.setattr( + frozen_entry, + "_run_server", + lambda: pytest.fail("server must not start in embedded CLI mode"), + ) + monkeypatch.setattr( + sys, + "argv", + ["citeseal_server", "--citeseal-cli", "doctor"], + ) + + assert frozen_entry.main() == 7 + assert calls == [["doctor"]] + + +def test_frozen_cli_runs_bundled_script_in_process_and_restores_state( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + scripts = tmp_path / "scripts" + scripts.mkdir() + workdir = tmp_path / "work" + workdir.mkdir() + probe = scripts / "probe.py" + probe.write_text( + "import json, os, sys\n" + "from pathlib import Path\n" + "Path('probe.json').write_text(json.dumps({\n" + " 'argv': sys.argv, 'cwd': os.getcwd(), 'name': __name__\n" + "}), encoding='utf-8')\n" + "raise SystemExit(7)\n", + encoding="utf-8", + ) + + original_cwd = Path.cwd() + original_argv = list(sys.argv) + monkeypatch.setattr(cli, "_SCRIPTS_DIR", scripts) + monkeypatch.setattr(sys, "frozen", True, raising=False) + monkeypatch.setattr( + cli.subprocess, + "run", + lambda *args, **kwargs: pytest.fail( + "frozen CLI must not require a host Python subprocess" + ), + ) + + assert cli._run_script("probe.py", ["alpha", "beta"], cwd=workdir) == 7 + + result = json.loads((workdir / "probe.json").read_text(encoding="utf-8")) + assert result["argv"] == [str(probe), "alpha", "beta"] + assert result["cwd"] == str(workdir) + assert result["name"] == "__main__" + assert Path.cwd() == original_cwd + assert sys.argv == original_argv diff --git a/tests/server/test_smoke_test_server.py b/tests/server/test_smoke_test_server.py new file mode 100644 index 0000000..676557e --- /dev/null +++ b/tests/server/test_smoke_test_server.py @@ -0,0 +1,75 @@ +"""Tests for the cross-platform frozen-server smoke verifier.""" +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parent.parent.parent +SCRIPT = ROOT / "tools" / "server" / "smoke_test_server.py" +SPEC = importlib.util.spec_from_file_location("smoke_test_server", SCRIPT) +assert SPEC is not None and SPEC.loader is not None +smoke = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(smoke) + + +def test_smoke_test_waits_for_health_and_successful_validate_job( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + responses = iter( + [ + {"ok": True}, + {"job_id": "job-1", "status": "queued"}, + {"id": "job-1", "status": "running", "returncode": None}, + {"id": "job-1", "status": "done", "returncode": 0}, + ] + ) + calls: list[tuple[str, str, object]] = [] + + def fake_request(url, *, method="GET", payload=None, timeout=5.0): + calls.append((url, method, payload)) + return next(responses) + + monkeypatch.setattr(smoke, "_request_json", fake_request) + monkeypatch.setattr(smoke.time, "sleep", lambda _seconds: None) + + result = smoke.smoke_test("http://127.0.0.1:8765/", tmp_path, timeout=5) + + assert result["status"] == "done" + assert calls[1] == ( + "http://127.0.0.1:8765/api/run", + "POST", + { + "op": "validate", + "args": {"root": str(tmp_path.resolve()), "quiet": True}, + }, + ) + assert calls[-1][0].endswith("/api/jobs/job-1") + + +def test_smoke_test_fails_closed_on_failed_job( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + responses = iter( + [ + {"ok": True}, + {"job_id": "job-2", "status": "queued"}, + { + "id": "job-2", + "status": "failed", + "returncode": 2, + "stderr_tail": "boom", + }, + ] + ) + monkeypatch.setattr( + smoke, + "_request_json", + lambda *args, **kwargs: next(responses), + ) + + with pytest.raises(RuntimeError, match="validate smoke job failed"): + smoke.smoke_test("http://127.0.0.1:8765", tmp_path, timeout=5) diff --git a/tests/test_cli_smoke.py b/tests/test_cli_smoke.py index fc85e97..8cf0272 100644 --- a/tests/test_cli_smoke.py +++ b/tests/test_cli_smoke.py @@ -11,6 +11,7 @@ """ from __future__ import annotations +import json import subprocess import sys from pathlib import Path @@ -141,6 +142,46 @@ def test_doctor_runs_and_reports(self): assert "Python" in r.stdout assert "Project layout" in r.stdout + @pytest.mark.parametrize("command,suffix", [("md", ".md"), ("pdf", ".pdf")]) + def test_document_commands_accept_explicit_output_paths( + self, + command: str, + suffix: str, + tmp_path: Path, + ): + out = tmp_path / f"full{suffix}" + extract = tmp_path / "extract.json" + extract.write_text( + json.dumps( + { + "title": "Portable PDF test", + "author_handle": "example_user", + "url": "https://example.test/item", + "datetime_utc": "2026-07-08T18:01:00Z", + "blocks": [{"type": "p", "text": "Hello from CiteSeal."}], + } + ), + encoding="utf-8", + ) + r = _run( + CLI, + [ + command, + "--tweet-dir", + str(GOOD_DIR), + "--extract", + str(extract), + "--out", + str(out), + "--force", + ], + ) + assert r.returncode == 0, r.stderr + assert out.is_file() + assert out.stat().st_size > 0 + if suffix == ".pdf": + assert out.read_bytes().startswith(b"%PDF-") + # ── export-agent ──────────────────────────────────────────────────────────── diff --git a/tests/test_pdf_font.py b/tests/test_pdf_font.py new file mode 100644 index 0000000..2d52b01 --- /dev/null +++ b/tests/test_pdf_font.py @@ -0,0 +1,28 @@ +"""Tests for portable PDF font selection.""" +from __future__ import annotations + +import importlib +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parent.parent +SCRIPTS = ROOT / "tools" / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + +pdf_builder = importlib.import_module("make_article_pdf") + + +def test_cjk_font_falls_back_to_reportlab_cid_without_system_font( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("CJK_FONT_PATH", raising=False) + monkeypatch.setattr(pdf_builder.platform, "system", lambda: "Linux") + monkeypatch.setattr(pdf_builder.os.path, "exists", lambda _path: False) + + font_name = pdf_builder.register_cjk_font() + + assert font_name == "STSong-Light" + assert pdf_builder.pdfmetrics.getFont(font_name) is not None diff --git a/tests/test_release_contract.py b/tests/test_release_contract.py index a8df9b9..2d50843 100644 --- a/tests/test_release_contract.py +++ b/tests/test_release_contract.py @@ -1,15 +1,17 @@ """Release packaging contracts for the distributable server artifacts.""" from __future__ import annotations -import os +import importlib.util +import io import re -import subprocess +import tarfile from pathlib import Path import pytest ROOT = Path(__file__).resolve().parents[1] +VERIFY_LINUX_TARBALL = ROOT / "tools" / "server" / "verify_linux_tarball.py" def test_windows_release_installs_pyinstaller_before_build() -> None: @@ -32,7 +34,7 @@ def test_windows_release_installs_pyinstaller_before_build() -> None: assert "--distpath tools/dist" in after_build -def test_windows_release_sets_health_port_before_starting_server() -> None: +def test_windows_release_sets_port_before_starting_and_smoking_server() -> None: workflow = (ROOT / ".github" / "workflows" / "release.yml").read_text( encoding="utf-8" ) @@ -45,25 +47,193 @@ def test_windows_release_sets_health_port_before_starting_server() -> None: port_index = verify_step.find('$env:CITESEAL_PORT = "8765"') start_index = verify_step.find("Start-Process") - health_index = verify_step.find("http://127.0.0.1:8765/api/health") - assert min(port_index, start_index, health_index) >= 0 - assert port_index < start_index < health_index - - -@pytest.mark.skipif(os.name == "nt", reason="Linux bundle contract runs on POSIX") -def test_linux_bundle_copies_scripts_into_flat_import_path(tmp_path: Path) -> None: - build_script = ROOT / "tools" / "server" / "build_linux.sh" - copy_line = next( - line.strip() - for line in build_script.read_text(encoding="utf-8").splitlines() - if line.strip().startswith('cp -r "$TOOLS/scripts') + smoke_index = verify_step.find("smoke_test_server.py") + assert min(port_index, start_index, smoke_index) >= 0 + assert port_index < start_index < smoke_index + + +def test_pyinstaller_spec_analyzes_embedded_cli_and_script_dependencies() -> None: + spec = (ROOT / "tools" / "server" / "citeseal_server.spec").read_text( + encoding="utf-8" + ) + + assert "script_modules" in spec + assert "hiddenimports += script_modules" in spec + assert "pathex=[str(server_dir), str(tools_root), str(scripts_pkg)]" in spec + + +def test_linux_bundle_builds_frozen_server_without_runtime_venv() -> None: + script = (ROOT / "tools" / "server" / "build_linux.sh").read_text( + encoding="utf-8" + ) + + assert 'if command -v uv' in script + assert 'uv venv --seed --python python3 "$BUILD_VENV"' in script + assert 'python3 -m venv "$BUILD_VENV"' in script + assert '"$BUILD_VENV/bin/python" -m PyInstaller' in script + assert '"$BUILD_DIST/citeseal_server"' in script + assert '"$ROOT/bin/citeseal_server"' in script + assert 'exec "$HERE/citeseal_server"' in script + assert '"$ROOT/venv/bin/python"' not in script + assert 'verify_linux_tarball.py" "$TARBALL"' in script + + +def test_release_workflow_verifies_linux_tarball_before_extracting() -> None: + workflow = (ROOT / ".github" / "workflows" / "release.yml").read_text( + encoding="utf-8" + ) + linux_job = workflow.split(" linux-tarball:", 1)[1].split( + " android-apk:", 1 + )[0] + verifier_index = linux_job.find("verify_linux_tarball.py") + extract_index = linux_job.find("tar -xzf") + + assert min(verifier_index, extract_index) >= 0 + assert verifier_index < extract_index + + +def test_release_workflow_smokes_background_jobs_without_host_python() -> None: + workflow = (ROOT / ".github" / "workflows" / "release.yml").read_text( + encoding="utf-8" + ) + windows_job = workflow.split(" windows-exe:", 1)[1].split( + " linux-tarball:", 1 + )[0] + linux_job = workflow.split(" linux-tarball:", 1)[1].split( + " android-apk:", 1 + )[0] + good_fixture = ( + "tests/fixtures/accounts/example_user/tweets/2026/2026-07/" + "20260708_180000_1234567890" + ) + + assert "smoke_test_server.py" in windows_job + assert "smoke_test_server.py" in linux_job + assert good_fixture in windows_job + assert good_fixture in linux_job + assert 'CITESEAL_ROOT="$PWD/tests/fixtures/accounts"' not in linux_job + assert '$env:PATH = "$env:SystemRoot\\System32"' in windows_job + assert "PATH=/nonexistent" in linux_job + assert "/bin/citeseal_server &" in linux_job + + +def _load_tarball_verifier(): + assert VERIFY_LINUX_TARBALL.is_file(), "Linux tarball verifier is required" + spec = importlib.util.spec_from_file_location( + "verify_linux_tarball", VERIFY_LINUX_TARBALL ) - stage_root = tmp_path / "citeseal_server-linux-x64" - (stage_root / "scripts").mkdir(parents=True) - env = os.environ.copy() - env.update({"TOOLS": str(ROOT / "tools"), "ROOT": str(stage_root)}) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _write_tarball( + path: Path, + *, + extra_member: tarfile.TarInfo | None = None, + files: dict[str, bytes] | None = None, + mode: int = 0o755, +) -> None: + root = "citeseal_server-linux-x64" + if files is None: + files = { + f"{root}/bin/citeseal_server": b"binary", + f"{root}/bin/run.sh": b"#!/usr/bin/env bash\n", + } + with tarfile.open(path, "w:gz") as archive: + for name, data in files.items(): + info = tarfile.TarInfo(name) + info.size = len(data) + info.mode = mode + archive.addfile(info, io.BytesIO(data)) + if extra_member is not None: + archive.addfile(extra_member) + + +def test_linux_tarball_verifier_accepts_self_contained_layout(tmp_path: Path) -> None: + verifier = _load_tarball_verifier() + archive = tmp_path / "citeseal_server-linux-x64.tar.gz" + _write_tarball(archive) + + summary = verifier.verify_tarball(archive) + + assert summary["members"] == 2 + assert summary["links"] == 0 + + +@pytest.mark.parametrize( + "bad_member", + [ + tarfile.TarInfo("../../escape"), + tarfile.TarInfo("/absolute/path"), + ], +) +def test_linux_tarball_verifier_rejects_unsafe_paths( + tmp_path: Path, bad_member: tarfile.TarInfo +) -> None: + verifier = _load_tarball_verifier() + archive = tmp_path / "citeseal_server-linux-x64.tar.gz" + _write_tarball(archive, extra_member=bad_member) + + with pytest.raises(ValueError, match="unsafe path"): + verifier.verify_tarball(archive) + + +def test_linux_tarball_verifier_rejects_links(tmp_path: Path) -> None: + verifier = _load_tarball_verifier() + archive = tmp_path / "citeseal_server-linux-x64.tar.gz" + link = tarfile.TarInfo("citeseal_server-linux-x64/venv/bin/python3") + link.type = tarfile.SYMTYPE + link.linkname = "/opt/hostedtoolcache/Python/3.12/bin/python3" + _write_tarball(archive, extra_member=link) + + with pytest.raises(ValueError, match="links are not allowed"): + verifier.verify_tarball(archive) + + +def test_linux_tarball_verifier_rejects_duplicate_members(tmp_path: Path) -> None: + verifier = _load_tarball_verifier() + archive = tmp_path / "citeseal_server-linux-x64.tar.gz" + duplicate = tarfile.TarInfo( + "citeseal_server-linux-x64/bin/citeseal_server" + ) + _write_tarball(archive, extra_member=duplicate) + + with pytest.raises(ValueError, match="duplicate tarball member"): + verifier.verify_tarball(archive) + + +def test_linux_tarball_verifier_rejects_special_files(tmp_path: Path) -> None: + verifier = _load_tarball_verifier() + archive = tmp_path / "citeseal_server-linux-x64.tar.gz" + device = tarfile.TarInfo("citeseal_server-linux-x64/bin/device") + device.type = tarfile.CHRTYPE + _write_tarball(archive, extra_member=device) + + with pytest.raises(ValueError, match="special file is not allowed"): + verifier.verify_tarball(archive) + + +def test_linux_tarball_verifier_requires_all_launchers(tmp_path: Path) -> None: + verifier = _load_tarball_verifier() + archive = tmp_path / "citeseal_server-linux-x64.tar.gz" + root = "citeseal_server-linux-x64" + _write_tarball( + archive, + files={f"{root}/bin/run.sh": b"#!/usr/bin/env bash\n"}, + ) + + with pytest.raises(ValueError, match="required release files missing"): + verifier.verify_tarball(archive) + - subprocess.run(["bash", "-c", copy_line], env=env, check=True) +def test_linux_tarball_verifier_requires_executable_launchers( + tmp_path: Path, +) -> None: + verifier = _load_tarball_verifier() + archive = tmp_path / "citeseal_server-linux-x64.tar.gz" + _write_tarball(archive, mode=0o644) - assert (stage_root / "scripts" / "ci_common.py").is_file() - assert not (stage_root / "scripts" / "scripts").exists() + with pytest.raises(ValueError, match="release files are not executable"): + verifier.verify_tarball(archive) diff --git a/tools/citeseal.py b/tools/citeseal.py index a51e802..014e685 100644 --- a/tools/citeseal.py +++ b/tools/citeseal.py @@ -18,6 +18,8 @@ import argparse import json +import os +import runpy import shlex import subprocess import sys @@ -49,11 +51,49 @@ def _python_executable() -> str: return sys.executable or "python" +def _system_exit_code(code: object) -> int: + """Normalize ``SystemExit.code`` using Python's command-line semantics.""" + if code is None: + return 0 + if isinstance(code, int): + return code + print(code, file=sys.stderr) + return 1 + + +def _run_script_in_process( + path: Path, + args: Sequence[str], + *, + cwd: Optional[Path] = None, +) -> int: + """Execute a bundled helper inside the frozen interpreter.""" + original_argv = sys.argv[:] + original_cwd = Path.cwd() + sys.argv = [str(path), *args] + try: + if cwd is not None: + os.chdir(cwd) + try: + runpy.run_path(str(path), run_name="__main__") + except SystemExit as exc: + return _system_exit_code(exc.code) + return 0 + finally: + sys.argv = original_argv + os.chdir(original_cwd) + + def _run_script(script: str, args: Sequence[str], *, cwd: Optional[Path] = None) -> int: """Run ``tools/scripts/