delivery-kid: snapshot draft diagnostics to wiki sub-page

delivery-kid/ansible/playbook.yml

@@ -136,6 +136,14 @@
                 - IPFS_GATEWAY_URL=https://{{ ipfs_gateway_domain }}
                 - STAGING_DIR=/staging
                 - COCONUT_API_KEY={{ coconut_api_key | default('') }}
+                # Pickipedia bot creds — used by pickipedia_client to mirror
+                # draft logs to ReleaseDraft:{id}/diagnostics so they survive
+                # a delivery-kid rebuild. Reuses the Magent@magent BotPassword
+                # already provisioned for the Blue Railroad import tool. When
+                # the password var is empty the client no-ops cleanly.
+                - PICKIPEDIA_URL={{ pickipedia_url | default('https://pickipedia.xyz') }}
+                - PICKIPEDIA_BOT_USER={{ pickipedia_bot_user | default('Magent@magent') }}
+                - PICKIPEDIA_BOT_PASSWORD={{ pickipedia_bot_password | default('') }}
               volumes:
                 - /mnt/storage-box/staging:/staging
               ports:

delivery-kid/pinning-service/app/routes/coconut.py

@@ -26,6 +26,7 @@
     list_jobs,
     process_completed_job,
 )
+from ..services.pickipedia_client import snapshot_diagnostics_for_dict_async
 
 logger = logging.getLogger(__name__)
 
@@ -99,6 +100,15 @@ def _update_draft_preview(staging_dir: Path, job: dict) -> None:
             logger.warning("[%s] Draft %s preview failed", job["id"], draft_id[:8])
         data["preview_log"] = log[-PREVIEW_LOG_MAX:]
         draft_json.write_text(json.dumps(data, indent=2, default=str))
+
+        # Mirror the terminal preview state to ReleaseDraft:{id}/diagnostics
+        # so it survives a delivery-kid storage rebuild. ``ready`` and
+        # ``failed`` are both terminal — both worth snapshotting.
+        try:
+            import asyncio as _asyncio
+            _asyncio.create_task(snapshot_diagnostics_for_dict_async(draft_id, data))
+        except RuntimeError:
+            logger.debug("[%s] No running loop; skipping diagnostics snapshot", job["id"])
     except Exception as e:
         logger.error("[%s] Failed to update draft preview state: %s", job["id"], e)
 

delivery-kid/pinning-service/app/routes/content.py

@@ -21,6 +21,7 @@
 from ..services import analyze, ipfs, transcode
 from ..services.coconut import submit_to_coconut, save_job, load_job
 from ..services.fsutil import safe_rmtree
+from ..services.pickipedia_client import snapshot_diagnostics_for_state_async
 
 logger = logging.getLogger(__name__)
 
@@ -94,6 +95,21 @@ def _append_finalize_log(state: ContentDraftState, stage: str, message: str,
         state.finalize_log = state.finalize_log[-FINALIZE_LOG_MAX:]
 
 
+def _fire_diagnostics_snapshot(state: ContentDraftState) -> None:
+    """Fire-and-forget snapshot of draft logs to ``ReleaseDraft:{id}/diagnostics``.
+
+    Called at every terminal-state transition so the upload/finalize/preview
+    log trail survives a delivery-kid rebuild. Swallows scheduling errors
+    (e.g. no running loop) so a wiki blip never masks the underlying
+    upload outcome the caller is about to surface.
+    """
+    try:
+        asyncio.create_task(snapshot_diagnostics_for_state_async(state))
+    except RuntimeError:
+        logger.debug("[content:%s] No running loop; skipping diagnostics snapshot",
+                     state.draft_id[:8])
+
+
 @router.post("/init", response_model=ContentDraftResponse)
 async def init_content_draft(
     wallet_address: str = Depends(require_auth),
@@ -219,6 +235,7 @@ def fail(http_status: int, message: str) -> HTTPException:
             save_draft_state(draft_dir, state)
         except Exception:
             logger.exception("[content:%s] Failed to persist upload_log on error", draft_id[:8])
+        _fire_diagnostics_snapshot(state)
         return HTTPException(status_code=http_status, detail=message)
 
     if not files:
@@ -454,6 +471,7 @@ async def _submit_preview_transcode(
             save_draft_state(draft_dir, state)
         except Exception:
             pass
+        _fire_diagnostics_snapshot(state)
 
 
 def _should_use_coconut(request: ContentFinalizeRequest, settings: Settings) -> bool:
@@ -761,6 +779,14 @@ async def send_event(event: str, data: dict):
         yield await send_event("error", {"stage": "exception", "message": str(e)})
 
     finally:
+        # Snapshot the final state to the wiki sub-page BEFORE we wipe
+        # draft.json on success. Once the rmtree below runs the in-memory
+        # ``state`` is the only copy of the finalize_log, so we have to
+        # mirror it now or never. On failure paths we also snapshot so
+        # the wiki has the latest log even if delivery-kid storage is
+        # later rebuilt.
+        _fire_diagnostics_snapshot(state)
+
         # Only wipe the draft dir on a fully successful pin. On failure we
         # keep draft.json (with its finalize_log) so the ReleaseDraft page
         # can show what went wrong, and the source bytes stay on disk for

delivery-kid/pinning-service/app/services/pickipedia_client.py

@@ -0,0 +1,146 @@
+"""Pickipedia bot client — snapshots draft diagnostics to a wiki sub-page.
+
+When delivery-kid's filesystem is wiped (rebuild, container churn), the
+``draft.json`` for each in-flight upload goes with it. The wiki side then
+loses the upload/finalize/preview log trail that was the whole point of
+PR #81's diagnostics panel. To survive that, we mirror the logs onto a
+``ReleaseDraft:{id}/diagnostics`` wiki sub-page at each terminal state
+transition. The wiki page outlives delivery-kid's storage, and the
+diagnostics-panel JS falls back to it whenever the live ``/draft-content``
+fetch returns 404.
+
+The bot identity reused here is the same Magent@magent BotPassword that
+the Blue Railroad import tool already uses (configured via
+``PICKIPEDIA_BOT_USER`` and ``PICKIPEDIA_BOT_PASSWORD`` env vars). When
+those are unset (typical in dev) every snapshot call no-ops cleanly.
+"""
+
+import asyncio
+import json
+import logging
+import os
+from datetime import datetime, timezone
+from threading import Lock
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+# Lazy-initialised mwclient.Site singleton. Cached because each
+# ``site.login()`` round-trips the wiki and we don't want to do that on
+# every snapshot. ``_site_lock`` guards the init race when several
+# webhooks fire concurrently.
+_site = None
+_site_lock = Lock()
+
+
+def _parse_host(url: str) -> tuple[str, str]:
+    """Split a wiki URL into (host, scheme) for mwclient.Site."""
+    if url.startswith("https://"):
+        return url[8:].rstrip("/"), "https"
+    if url.startswith("http://"):
+        return url[7:].rstrip("/"), "http"
+    return url.rstrip("/"), "https"
+
+
+def _get_site():
+    """Return a logged-in mwclient.Site, or None if creds aren't configured."""
+    global _site
+    if _site is not None:
+        return _site
+
+    user = os.environ.get("PICKIPEDIA_BOT_USER", "Magent@magent")
+    password = os.environ.get("PICKIPEDIA_BOT_PASSWORD")
+    if not password:
+        return None
+
+    url = os.environ.get("PICKIPEDIA_URL", "https://pickipedia.xyz")
+    host, scheme = _parse_host(url)
+
+    with _site_lock:
+        if _site is not None:
+            return _site
+        try:
+            import mwclient
+            site = mwclient.Site(host, scheme=scheme)
+            site.login(user, password)
+            _site = site
+            logger.info("pickipedia_client: authenticated to %s as %s", host, user)
+            return site
+        except Exception as e:
+            logger.error("pickipedia_client: authentication to %s failed: %s", host, e)
+            return None
+
+
+def _build_snapshot_payload(state) -> dict:
+    """Project a ContentDraftState into the JSON payload we write to the wiki.
+
+    Kept deliberately flat so the wiki-side renderer can consume it with
+    the same shape it gets from ``/draft-content``.
+    """
+    return {
+        "draft_id": state.draft_id,
+        "snapshot_at": datetime.now(timezone.utc).isoformat(),
+        "status": state.status,
+        "preview_status": state.preview_status,
+        "upload_log": state.upload_log,
+        "finalize_log": state.finalize_log,
+        "preview_log": state.preview_log,
+    }
+
+
+def snapshot_diagnostics(draft_id: str, payload: dict) -> bool:
+    """Write the diagnostics payload to ``ReleaseDraft:{id}/diagnostics``.
+
+    Returns True if the write succeeded (or the page already had identical
+    content), False if creds are missing or the wiki call failed. Never
+    raises — terminal-state hooks call this fire-and-forget; we don't want
+    a wiki blip to mask the underlying upload outcome.
+    """
+    site = _get_site()
+    if site is None:
+        return False
+
+    title = f"ReleaseDraft:{draft_id}/diagnostics"
+    content = json.dumps(payload, indent=2, default=str)
+    summary = f"diagnostics snapshot — status={payload.get('status', 'unknown')}"
+
+    try:
+        page = site.pages[title]
+        existing = page.text() if page.exists else None
+        if existing == content:
+            return True
+        page.save(content, summary=summary)
+        logger.info("pickipedia_client: snapshotted %s (%d bytes)", title, len(content))
+        return True
+    except Exception as e:
+        logger.error("pickipedia_client: snapshot failed for %s: %s", title, e)
+        return False
+
+
+async def snapshot_diagnostics_for_state_async(state) -> bool:
+    """Async fire-and-forget snapshot from a ContentDraftState.
+
+    Runs the sync mwclient call in a thread pool so we don't block the
+    FastAPI event loop. Used by terminal-state hooks in the routes.
+    """
+    payload = _build_snapshot_payload(state)
+    return await asyncio.to_thread(snapshot_diagnostics, state.draft_id, payload)
+
+
+def snapshot_diagnostics_for_dict_async(draft_id: str, draft_data: dict):
+    """Async fire-and-forget snapshot from a raw draft.json dict.
+
+    Used by the Coconut webhook path, which mutates ``draft.json`` on disk
+    rather than holding a ContentDraftState object. Returns the asyncio
+    Task so the caller can ``create_task(...)`` it without awaiting.
+    """
+    payload = {
+        "draft_id": draft_id,
+        "snapshot_at": datetime.now(timezone.utc).isoformat(),
+        "status": draft_data.get("status"),
+        "preview_status": draft_data.get("preview_status"),
+        "upload_log": draft_data.get("upload_log") or [],
+        "finalize_log": draft_data.get("finalize_log") or [],
+        "preview_log": draft_data.get("preview_log") or [],
+    }
+    return asyncio.to_thread(snapshot_diagnostics, draft_id, payload)

delivery-kid/pinning-service/requirements.txt

@@ -20,3 +20,7 @@ pydub>=0.25.1
 
 # BitTorrent seeding
 libtorrent>=2.0.0
+
+# MediaWiki client — used by pickipedia_client to snapshot draft
+# diagnostics to ReleaseDraft:{id}/diagnostics on terminal state.
+mwclient>=0.10.1