diff --git a/delivery-kid/ansible/playbook.yml b/delivery-kid/ansible/playbook.yml index 08a169c..585fc93 100644 --- a/delivery-kid/ansible/playbook.yml +++ b/delivery-kid/ansible/playbook.yml @@ -136,6 +136,14 @@ - IPFS_GATEWAY_URL=https://{{ ipfs_gateway_domain }} - STAGING_DIR=/staging - COCONUT_API_KEY={{ coconut_api_key | default('') }} + # Pickipedia bot creds — used by pickipedia_client to mirror + # draft logs to ReleaseDraft:{id}/diagnostics so they survive + # a delivery-kid rebuild. Reuses the Magent@magent BotPassword + # already provisioned for the Blue Railroad import tool. When + # the password var is empty the client no-ops cleanly. + - PICKIPEDIA_URL={{ pickipedia_url | default('https://pickipedia.xyz') }} + - PICKIPEDIA_BOT_USER={{ pickipedia_bot_user | default('Magent@magent') }} + - PICKIPEDIA_BOT_PASSWORD={{ pickipedia_bot_password | default('') }} volumes: - /mnt/storage-box/staging:/staging ports: diff --git a/delivery-kid/pinning-service/app/routes/coconut.py b/delivery-kid/pinning-service/app/routes/coconut.py index 0655566..33ce54c 100644 --- a/delivery-kid/pinning-service/app/routes/coconut.py +++ b/delivery-kid/pinning-service/app/routes/coconut.py @@ -26,6 +26,7 @@ list_jobs, process_completed_job, ) +from ..services.pickipedia_client import snapshot_diagnostics_for_dict_async logger = logging.getLogger(__name__) @@ -99,6 +100,15 @@ def _update_draft_preview(staging_dir: Path, job: dict) -> None: logger.warning("[%s] Draft %s preview failed", job["id"], draft_id[:8]) data["preview_log"] = log[-PREVIEW_LOG_MAX:] draft_json.write_text(json.dumps(data, indent=2, default=str)) + + # Mirror the terminal preview state to ReleaseDraft:{id}/diagnostics + # so it survives a delivery-kid storage rebuild. ``ready`` and + # ``failed`` are both terminal — both worth snapshotting. + try: + import asyncio as _asyncio + _asyncio.create_task(snapshot_diagnostics_for_dict_async(draft_id, data)) + except RuntimeError: + logger.debug("[%s] No running loop; skipping diagnostics snapshot", job["id"]) except Exception as e: logger.error("[%s] Failed to update draft preview state: %s", job["id"], e) diff --git a/delivery-kid/pinning-service/app/routes/content.py b/delivery-kid/pinning-service/app/routes/content.py index a123434..7ebd58e 100644 --- a/delivery-kid/pinning-service/app/routes/content.py +++ b/delivery-kid/pinning-service/app/routes/content.py @@ -21,6 +21,7 @@ from ..services import analyze, ipfs, transcode from ..services.coconut import submit_to_coconut, save_job, load_job from ..services.fsutil import safe_rmtree +from ..services.pickipedia_client import snapshot_diagnostics_for_state_async logger = logging.getLogger(__name__) @@ -94,6 +95,21 @@ def _append_finalize_log(state: ContentDraftState, stage: str, message: str, state.finalize_log = state.finalize_log[-FINALIZE_LOG_MAX:] +def _fire_diagnostics_snapshot(state: ContentDraftState) -> None: + """Fire-and-forget snapshot of draft logs to ``ReleaseDraft:{id}/diagnostics``. + + Called at every terminal-state transition so the upload/finalize/preview + log trail survives a delivery-kid rebuild. Swallows scheduling errors + (e.g. no running loop) so a wiki blip never masks the underlying + upload outcome the caller is about to surface. + """ + try: + asyncio.create_task(snapshot_diagnostics_for_state_async(state)) + except RuntimeError: + logger.debug("[content:%s] No running loop; skipping diagnostics snapshot", + state.draft_id[:8]) + + @router.post("/init", response_model=ContentDraftResponse) async def init_content_draft( wallet_address: str = Depends(require_auth), @@ -219,6 +235,7 @@ def fail(http_status: int, message: str) -> HTTPException: save_draft_state(draft_dir, state) except Exception: logger.exception("[content:%s] Failed to persist upload_log on error", draft_id[:8]) + _fire_diagnostics_snapshot(state) return HTTPException(status_code=http_status, detail=message) if not files: @@ -454,6 +471,7 @@ async def _submit_preview_transcode( save_draft_state(draft_dir, state) except Exception: pass + _fire_diagnostics_snapshot(state) def _should_use_coconut(request: ContentFinalizeRequest, settings: Settings) -> bool: @@ -761,6 +779,14 @@ async def send_event(event: str, data: dict): yield await send_event("error", {"stage": "exception", "message": str(e)}) finally: + # Snapshot the final state to the wiki sub-page BEFORE we wipe + # draft.json on success. Once the rmtree below runs the in-memory + # ``state`` is the only copy of the finalize_log, so we have to + # mirror it now or never. On failure paths we also snapshot so + # the wiki has the latest log even if delivery-kid storage is + # later rebuilt. + _fire_diagnostics_snapshot(state) + # Only wipe the draft dir on a fully successful pin. On failure we # keep draft.json (with its finalize_log) so the ReleaseDraft page # can show what went wrong, and the source bytes stay on disk for diff --git a/delivery-kid/pinning-service/app/services/pickipedia_client.py b/delivery-kid/pinning-service/app/services/pickipedia_client.py new file mode 100644 index 0000000..480ef23 --- /dev/null +++ b/delivery-kid/pinning-service/app/services/pickipedia_client.py @@ -0,0 +1,146 @@ +"""Pickipedia bot client — snapshots draft diagnostics to a wiki sub-page. + +When delivery-kid's filesystem is wiped (rebuild, container churn), the +``draft.json`` for each in-flight upload goes with it. The wiki side then +loses the upload/finalize/preview log trail that was the whole point of +PR #81's diagnostics panel. To survive that, we mirror the logs onto a +``ReleaseDraft:{id}/diagnostics`` wiki sub-page at each terminal state +transition. The wiki page outlives delivery-kid's storage, and the +diagnostics-panel JS falls back to it whenever the live ``/draft-content`` +fetch returns 404. + +The bot identity reused here is the same Magent@magent BotPassword that +the Blue Railroad import tool already uses (configured via +``PICKIPEDIA_BOT_USER`` and ``PICKIPEDIA_BOT_PASSWORD`` env vars). When +those are unset (typical in dev) every snapshot call no-ops cleanly. +""" + +import asyncio +import json +import logging +import os +from datetime import datetime, timezone +from threading import Lock +from typing import Optional + +logger = logging.getLogger(__name__) + +# Lazy-initialised mwclient.Site singleton. Cached because each +# ``site.login()`` round-trips the wiki and we don't want to do that on +# every snapshot. ``_site_lock`` guards the init race when several +# webhooks fire concurrently. +_site = None +_site_lock = Lock() + + +def _parse_host(url: str) -> tuple[str, str]: + """Split a wiki URL into (host, scheme) for mwclient.Site.""" + if url.startswith("https://"): + return url[8:].rstrip("/"), "https" + if url.startswith("http://"): + return url[7:].rstrip("/"), "http" + return url.rstrip("/"), "https" + + +def _get_site(): + """Return a logged-in mwclient.Site, or None if creds aren't configured.""" + global _site + if _site is not None: + return _site + + user = os.environ.get("PICKIPEDIA_BOT_USER", "Magent@magent") + password = os.environ.get("PICKIPEDIA_BOT_PASSWORD") + if not password: + return None + + url = os.environ.get("PICKIPEDIA_URL", "https://pickipedia.xyz") + host, scheme = _parse_host(url) + + with _site_lock: + if _site is not None: + return _site + try: + import mwclient + site = mwclient.Site(host, scheme=scheme) + site.login(user, password) + _site = site + logger.info("pickipedia_client: authenticated to %s as %s", host, user) + return site + except Exception as e: + logger.error("pickipedia_client: authentication to %s failed: %s", host, e) + return None + + +def _build_snapshot_payload(state) -> dict: + """Project a ContentDraftState into the JSON payload we write to the wiki. + + Kept deliberately flat so the wiki-side renderer can consume it with + the same shape it gets from ``/draft-content``. + """ + return { + "draft_id": state.draft_id, + "snapshot_at": datetime.now(timezone.utc).isoformat(), + "status": state.status, + "preview_status": state.preview_status, + "upload_log": state.upload_log, + "finalize_log": state.finalize_log, + "preview_log": state.preview_log, + } + + +def snapshot_diagnostics(draft_id: str, payload: dict) -> bool: + """Write the diagnostics payload to ``ReleaseDraft:{id}/diagnostics``. + + Returns True if the write succeeded (or the page already had identical + content), False if creds are missing or the wiki call failed. Never + raises — terminal-state hooks call this fire-and-forget; we don't want + a wiki blip to mask the underlying upload outcome. + """ + site = _get_site() + if site is None: + return False + + title = f"ReleaseDraft:{draft_id}/diagnostics" + content = json.dumps(payload, indent=2, default=str) + summary = f"diagnostics snapshot — status={payload.get('status', 'unknown')}" + + try: + page = site.pages[title] + existing = page.text() if page.exists else None + if existing == content: + return True + page.save(content, summary=summary) + logger.info("pickipedia_client: snapshotted %s (%d bytes)", title, len(content)) + return True + except Exception as e: + logger.error("pickipedia_client: snapshot failed for %s: %s", title, e) + return False + + +async def snapshot_diagnostics_for_state_async(state) -> bool: + """Async fire-and-forget snapshot from a ContentDraftState. + + Runs the sync mwclient call in a thread pool so we don't block the + FastAPI event loop. Used by terminal-state hooks in the routes. + """ + payload = _build_snapshot_payload(state) + return await asyncio.to_thread(snapshot_diagnostics, state.draft_id, payload) + + +def snapshot_diagnostics_for_dict_async(draft_id: str, draft_data: dict): + """Async fire-and-forget snapshot from a raw draft.json dict. + + Used by the Coconut webhook path, which mutates ``draft.json`` on disk + rather than holding a ContentDraftState object. Returns the asyncio + Task so the caller can ``create_task(...)`` it without awaiting. + """ + payload = { + "draft_id": draft_id, + "snapshot_at": datetime.now(timezone.utc).isoformat(), + "status": draft_data.get("status"), + "preview_status": draft_data.get("preview_status"), + "upload_log": draft_data.get("upload_log") or [], + "finalize_log": draft_data.get("finalize_log") or [], + "preview_log": draft_data.get("preview_log") or [], + } + return asyncio.to_thread(snapshot_diagnostics, draft_id, payload) diff --git a/delivery-kid/pinning-service/requirements.txt b/delivery-kid/pinning-service/requirements.txt index cae3164..45cbed0 100644 --- a/delivery-kid/pinning-service/requirements.txt +++ b/delivery-kid/pinning-service/requirements.txt @@ -20,3 +20,7 @@ pydub>=0.25.1 # BitTorrent seeding libtorrent>=2.0.0 + +# MediaWiki client — used by pickipedia_client to snapshot draft +# diagnostics to ReleaseDraft:{id}/diagnostics on terminal state. +mwclient>=0.10.1 diff --git a/delivery-kid/pinning-service/tests/test_pickipedia_client.py b/delivery-kid/pinning-service/tests/test_pickipedia_client.py new file mode 100644 index 0000000..9b5546e --- /dev/null +++ b/delivery-kid/pinning-service/tests/test_pickipedia_client.py @@ -0,0 +1,206 @@ +"""Tests for app.services.pickipedia_client — diagnostics snapshot to wiki.""" + +import json +from datetime import datetime, timezone +from unittest.mock import MagicMock, patch + +import pytest + +from app.services import pickipedia_client + + +@pytest.fixture(autouse=True) +def _reset_site_singleton(): + """Reset the cached mwclient.Site between tests so each test sees a fresh init.""" + pickipedia_client._site = None + yield + pickipedia_client._site = None + + +def _make_state(**overrides): + """Build a minimal stand-in for ContentDraftState — only the fields snapshot reads.""" + base = MagicMock() + base.draft_id = overrides.get("draft_id", "test-draft-id") + base.status = overrides.get("status", "uploaded") + base.preview_status = overrides.get("preview_status", "none") + base.upload_log = overrides.get("upload_log", []) + base.finalize_log = overrides.get("finalize_log", []) + base.preview_log = overrides.get("preview_log", []) + return base + + +class TestParseHost: + def test_https_url(self): + assert pickipedia_client._parse_host("https://pickipedia.xyz") == ("pickipedia.xyz", "https") + + def test_http_url(self): + assert pickipedia_client._parse_host("http://localhost:8080") == ("localhost:8080", "http") + + def test_trailing_slash_stripped(self): + assert pickipedia_client._parse_host("https://pickipedia.xyz/") == ("pickipedia.xyz", "https") + + def test_bare_host_defaults_to_https(self): + assert pickipedia_client._parse_host("pickipedia.xyz") == ("pickipedia.xyz", "https") + + +class TestGetSite: + def test_no_password_returns_none(self, monkeypatch): + """When PICKIPEDIA_BOT_PASSWORD is unset, _get_site returns None — never raises.""" + monkeypatch.delenv("PICKIPEDIA_BOT_PASSWORD", raising=False) + assert pickipedia_client._get_site() is None + + def test_login_failure_returns_none(self, monkeypatch): + """If mwclient.login raises, _get_site logs and returns None.""" + monkeypatch.setenv("PICKIPEDIA_BOT_PASSWORD", "secret") + + fake_site = MagicMock() + fake_site.login.side_effect = Exception("auth failed") + with patch.object(pickipedia_client, "_site", None), \ + patch("mwclient.Site", return_value=fake_site): + assert pickipedia_client._get_site() is None + + def test_login_success_caches_site(self, monkeypatch): + """A successful login caches the Site so subsequent calls don't re-auth.""" + monkeypatch.setenv("PICKIPEDIA_BOT_PASSWORD", "secret") + fake_site = MagicMock() + + with patch("mwclient.Site", return_value=fake_site) as mock_site_cls: + first = pickipedia_client._get_site() + second = pickipedia_client._get_site() + + assert first is fake_site + assert second is fake_site + # mwclient.Site should only be constructed once across both calls + assert mock_site_cls.call_count == 1 + fake_site.login.assert_called_once_with("Magent@magent", "secret") + + def test_custom_user_env(self, monkeypatch): + """PICKIPEDIA_BOT_USER overrides the default Magent@magent username.""" + monkeypatch.setenv("PICKIPEDIA_BOT_USER", "OtherBot@slot") + monkeypatch.setenv("PICKIPEDIA_BOT_PASSWORD", "pw") + fake_site = MagicMock() + with patch("mwclient.Site", return_value=fake_site): + pickipedia_client._get_site() + fake_site.login.assert_called_once_with("OtherBot@slot", "pw") + + +class TestBuildSnapshotPayload: + def test_projects_state_fields(self): + state = _make_state( + draft_id="abc", + status="finalize_failed", + preview_status="ready", + upload_log=[{"phase": "init", "message": "go"}], + finalize_log=[{"stage": "transcode", "error": "boom"}], + preview_log=[{"message": "submitted"}], + ) + payload = pickipedia_client._build_snapshot_payload(state) + + assert payload["draft_id"] == "abc" + assert payload["status"] == "finalize_failed" + assert payload["preview_status"] == "ready" + assert payload["upload_log"] == [{"phase": "init", "message": "go"}] + assert payload["finalize_log"] == [{"stage": "transcode", "error": "boom"}] + assert payload["preview_log"] == [{"message": "submitted"}] + # snapshot_at is an ISO 8601 timestamp in UTC + assert "T" in payload["snapshot_at"] + + +class TestSnapshotDiagnostics: + def test_no_creds_returns_false(self, monkeypatch): + """Without bot creds, snapshot is a clean no-op returning False.""" + monkeypatch.delenv("PICKIPEDIA_BOT_PASSWORD", raising=False) + assert pickipedia_client.snapshot_diagnostics("draft-1", {"status": "finalized"}) is False + + def test_writes_to_subpage_title(self, monkeypatch): + """The page title is ReleaseDraft:{draft_id}/diagnostics.""" + monkeypatch.setenv("PICKIPEDIA_BOT_PASSWORD", "pw") + fake_page = MagicMock() + fake_page.exists = False + fake_page.text.return_value = "" + fake_pages = MagicMock() + fake_pages.__getitem__ = lambda self, key: fake_page + fake_site = MagicMock() + fake_site.pages = fake_pages + + with patch("mwclient.Site", return_value=fake_site): + ok = pickipedia_client.snapshot_diagnostics( + "abc-123", {"status": "finalized", "draft_id": "abc-123"} + ) + + assert ok is True + # __getitem__ should have been called with the sub-page title + # We can't easily assert against MagicMock __getitem__ so instead + # check that page.save was called with the JSON payload as content. + args, kwargs = fake_page.save.call_args + saved_content = args[0] + parsed = json.loads(saved_content) + assert parsed["status"] == "finalized" + assert parsed["draft_id"] == "abc-123" + + def test_unchanged_content_skips_save(self, monkeypatch): + """If the existing wiki content already matches, we no-op (no edit, returns True).""" + monkeypatch.setenv("PICKIPEDIA_BOT_PASSWORD", "pw") + + payload = {"status": "finalized", "draft_id": "abc"} + existing_text = json.dumps(payload, indent=2, default=str) + + fake_page = MagicMock() + fake_page.exists = True + fake_page.text.return_value = existing_text + fake_site = MagicMock() + fake_site.pages = {"ReleaseDraft:abc/diagnostics": fake_page} + + with patch("mwclient.Site", return_value=fake_site): + ok = pickipedia_client.snapshot_diagnostics("abc", payload) + + assert ok is True + fake_page.save.assert_not_called() + + def test_save_exception_returns_false(self, monkeypatch): + """A wiki save failure is logged and surfaces as False — never raises.""" + monkeypatch.setenv("PICKIPEDIA_BOT_PASSWORD", "pw") + fake_page = MagicMock() + fake_page.exists = False + fake_page.text.return_value = "" + fake_page.save.side_effect = Exception("wiki down") + fake_site = MagicMock() + fake_site.pages = {"ReleaseDraft:abc/diagnostics": fake_page} + + with patch("mwclient.Site", return_value=fake_site): + ok = pickipedia_client.snapshot_diagnostics("abc", {"status": "finalized"}) + + assert ok is False + + +class TestSnapshotForDictAsync: + @pytest.mark.asyncio + async def test_builds_payload_from_dict(self, monkeypatch): + """The dict-based variant (used by Coconut webhook) projects raw draft.json fields.""" + monkeypatch.setenv("PICKIPEDIA_BOT_PASSWORD", "pw") + + fake_page = MagicMock() + fake_page.exists = False + fake_page.text.return_value = "" + fake_site = MagicMock() + fake_site.pages = {"ReleaseDraft:abc/diagnostics": fake_page} + + with patch("mwclient.Site", return_value=fake_site): + await pickipedia_client.snapshot_diagnostics_for_dict_async( + "abc", + { + "status": "uploaded", + "preview_status": "ready", + "upload_log": [{"phase": "init"}], + "preview_log": [{"message": "done"}], + # finalize_log absent — should default to [] + }, + ) + + args, _ = fake_page.save.call_args + saved = json.loads(args[0]) + assert saved["status"] == "uploaded" + assert saved["preview_status"] == "ready" + assert saved["upload_log"] == [{"phase": "init"}] + assert saved["preview_log"] == [{"message": "done"}] + assert saved["finalize_log"] == [] # missing key projects to empty list