From 7d29fdc60cfeeb3deee4c0b6bb2df66022bf2de3 Mon Sep 17 00:00:00 2001 From: emranemran Date: Wed, 15 Jul 2026 13:52:36 -0700 Subject: [PATCH] =?UTF-8?q?feat(examples):=20add=20screen-agent=20?= =?UTF-8?q?=E2=80=94=20wrapping=20an=20existing=20pip=20package=20(video?= =?UTF-8?q?=20in,=20bug=20report=20out)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New example demonstrating the pattern the others don't cover: the app logic lives in an external package (screen-agent-mvp); the Dockerfile pip-installs it and runs its bundled screen-agent-runner entry point. Dynamic registration, persistent (single-shot by nature), HTTP with base64 mp4 in / report bundle out. Runs offchain with deterministic fallback engines (no GPU needed); strict GPU mode documented. Co-Authored-By: Claude Fable 5 --- README.md | 1 + screen-agent/Dockerfile | 30 +++++++++++ screen-agent/README.md | 51 +++++++++++++++++++ screen-agent/client.py | 88 +++++++++++++++++++++++++++++++++ screen-agent/demo_video.py | 60 ++++++++++++++++++++++ screen-agent/docker-compose.yml | 30 +++++++++++ screen-agent/pyproject.toml | 15 ++++++ 7 files changed, 275 insertions(+) create mode 100644 screen-agent/Dockerfile create mode 100644 screen-agent/README.md create mode 100644 screen-agent/client.py create mode 100644 screen-agent/demo_video.py create mode 100644 screen-agent/docker-compose.yml create mode 100644 screen-agent/pyproject.toml diff --git a/README.md b/README.md index ed06368..3452284 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,7 @@ Need a schema that isn't here? [Open an issue](https://github.com/livepeer/live- | [`tiles`](./tiles) | Capacity fan-out — one session per tile | dynamic | persistent (single-shot by nature) | HTTP (base64 PNG) | | [`echo`](./echo) | Realtime video, transformed and echoed back | dynamic | persistent | trickle | | [`vllm`](./vllm) | Drop-in OpenAI API; the client stays unmodified | static | persistent (single-shot by nature) | HTTP + SSE | +| [`screen-agent`](./screen-agent) | Wrapping an existing pip package — video in, bug report out | dynamic | persistent (single-shot by nature) | HTTP (base64 mp4) | Start with `hello-world` (the smallest end-to-end path); the others each layer on one new idea. More will follow, including a full example that exercises every feature. Each is self-contained and runs **offchain** (free, no wallet); most also run **on-chain** (paid) — see each README. diff --git a/screen-agent/Dockerfile b/screen-agent/Dockerfile new file mode 100644 index 0000000..3903a08 --- /dev/null +++ b/screen-agent/Dockerfile @@ -0,0 +1,30 @@ +# screen-agent example app (video → bug report). +# +# Unlike the other examples, the app logic lives in an external pip package +# (screen-agent-mvp); this image just installs it and runs its bundled +# live-runner entry point. This demonstrates wrapping an EXISTING project — +# the common real-world case — rather than an app written for the example. +FROM python:3.12-slim + +# Flush stdout/stderr immediately so output isn't block-buffered in `docker logs`. +ENV PYTHONUNBUFFERED=1 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends git \ + && rm -rf /var/lib/apt/lists/* + +# The app package, plus its live-runner glue deps (aiohttp + the SDK, which +# isn't on PyPI yet). screen-agent-mvp keeps them in a dependency group, so +# they're installed explicitly here. +RUN pip install --no-cache-dir \ + "screen-agent-mvp @ git+https://github.com/emranemran/screen-agent-mvp@main" \ + "livepeer-gateway @ git+https://github.com/livepeer/livepeer-python-gateway@ja/live-runner" \ + "aiohttp>=3.10,<4" + +EXPOSE 8989 + +# --force-fallback: deterministic heuristic/template engines, so the example +# runs anywhere with no GPU or model weights. On a GPU host with the models +# installed, drop the flag (or use --strict-models) for real VLM analysis — +# see https://github.com/emranemran/screen-agent-mvp#readme. +ENTRYPOINT ["screen-agent-runner"] diff --git a/screen-agent/README.md b/screen-agent/README.md new file mode 100644 index 0000000..083cc20 --- /dev/null +++ b/screen-agent/README.md @@ -0,0 +1,51 @@ +# screen-agent app + +A real, pip-installed application on the Livepeer network: send a screen recording, get back a structured bug report (markdown report + timeline + summary JSON). The pipeline samples keyframes, runs OCR / UI parsing / VLM reasoning over them, and writes the report — here it runs with deterministic fallback engines so the example needs no GPU; on a GPU host the same wrapper serves PaddleOCR + OmniParser + Qwen2.5-VL. + +What this example adds over `hello-world`: the app logic is an **external package** ([screen-agent-mvp](https://github.com/emranemran/screen-agent-mvp)), not code written for the example. The Dockerfile pip-installs it and runs its bundled `screen-agent-runner` entry point — the pattern for putting an *existing* project on the network: the package's [`livepeer_runner.py`](https://github.com/emranemran/screen-agent-mvp/blob/main/src/screen_agent_mvp/livepeer_runner.py) is ~150 lines of aiohttp + `register_runner()` around an unchanged pure function. + +| | | +| ------------ | ------------------------------------ | +| App id | `livepeer-example/screen-agent` | +| Runner mode | persistent (single-shot by nature) | +| Registration | dynamic (self-registers via the SDK) | +| Transport | HTTP (JSON, base64 mp4 in) | +| Port | 8989 | + +Prerequisites (Docker, `uv`, and the not-yet-released `livepeer-gateway` SDK — pinned in `pyproject.toml`) and the shared on-chain/payment setup live in the [repo README](../README.md). + +> [!NOTE] +> This app currently runs in **persistent** mode. It will switch to **single-shot** once [#5](https://github.com/livepeer/live-runner-example-apps/issues/5) ships. Keep it offchain until then — per-second billing overbills a request/response app. + +## How it's wired + +The app is **dynamically registered**: the packaged runner self-registers via `register_runner` and exposes `POST /analyze` (`{"video_b64": ..., "preset": ...}` → `{"report_markdown", "summary", "timeline"}`), reverse-proxied through the orchestrator. The client calls it with `reserve_session` → `call_runner` → `stop_runner_session` ([client.py](client.py)) — grep `# Livepeer:` for the exact calls. Two things worth stealing: + +- the runner does the analysis in `asyncio.to_thread`, so heartbeats keep flowing during a long CPU/GPU-bound call; +- the client passes `timeout=600` to `call_runner` — the SDK default is 5s, which no analysis survives. + +## Run offchain (free) + +```sh +docker compose up -d --build +curl -sk https://localhost:8935/discovery | jq '.[].runners[].app' # confirm livepeer-example/screen-agent registered +uv run demo_video.py demo.mp4 # or bring any .mp4/.webm screen recording +uv run client.py demo.mp4 --discovery https://localhost:8935/discovery +# → prints the bug report; bundle saved to screen-agent-run/ +docker compose down +``` + +The container runs with `--force-fallback` (no GPU or weights needed), so the report is the deterministic template. For real model-written reports, run the runner on a GPU host with the models installed and `--strict-models` — see the [screen-agent-mvp README](https://github.com/emranemran/screen-agent-mvp#readme). + +## Run without Docker + +Start an orchestrator built from `ja/live-runner`, then install and run the packaged runner directly: + +```sh +./livepeer -orchestrator -useLiveRunners -serviceAddr localhost:8935 -orchSecret abcdef -v 6 +pip install "screen-agent-mvp @ git+https://github.com/emranemran/screen-agent-mvp@main" \ + "livepeer-gateway @ git+https://github.com/livepeer/livepeer-python-gateway@ja/live-runner" aiohttp +screen-agent-runner --orchestrator https://localhost:8935 --orchSecret abcdef \ + --app-id livepeer-example/screen-agent --force-fallback +uv run client.py demo.mp4 +``` diff --git a/screen-agent/client.py b/screen-agent/client.py new file mode 100644 index 0000000..ec54fdc --- /dev/null +++ b/screen-agent/client.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""screen-agent client: send a screen recording, get a bug report back. + +Livepeer integration (grep `# Livepeer:`): + 1. reserve_session() — discover orchestrators advertising the app, reserve one + (to be removed once #4 lands) + 2. call_runner() — POST the video through the orchestrator + 3. stop_runner_session() — end the session (settles payment on-chain) +""" + +from __future__ import annotations + +import argparse +import asyncio +import base64 +import json +import logging +from contextlib import suppress +from pathlib import Path + +from livepeer_gateway.errors import LivepeerGatewayError +from livepeer_gateway.live_runner import call_runner, stop_runner_session +from livepeer_gateway.selection import reserve_session + +DEFAULT_DISCOVERY = "https://localhost:8935/discovery" +APP_ID = "livepeer-example/screen-agent" +# Analysis takes tens of seconds (minutes with real models) — far beyond the +# SDK's 5s default timeout. +CALL_TIMEOUT_S = 600.0 + +log = logging.getLogger("screen-agent-client") + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Run the screen-agent Live Runner demo.") + parser.add_argument("video", type=Path, help="Local .mp4/.webm screen recording") + parser.add_argument("--discovery", default=DEFAULT_DISCOVERY) + parser.add_argument( + "--preset", default="bug-report", help="bug-report | support-session | agent-eval" + ) + parser.add_argument("--out", type=Path, default=Path("screen-agent-run")) + parser.add_argument( + "--signer", default="", help="Remote signer base URL (on-chain/paid path)." + ) + return parser.parse_args() + + +async def main() -> None: + logging.basicConfig( + level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s" + ) + args = _parse_args() + signer_url = args.signer.strip() or None + session = None + try: + session = await reserve_session( # Livepeer: 1 (to be removed once #4 lands) + discovery_url=args.discovery, + app=APP_ID, + signer_url=signer_url, + ) + log.info("session_id=%s app_url=%s", session.session_id, session.app_url) + + result = await call_runner( # Livepeer: 2 + runner_url=session.app_url.rstrip("/") + "/analyze", + payload={ + "video_b64": base64.b64encode(args.video.read_bytes()).decode(), + "preset": args.preset, + }, + signer_url=signer_url, + timeout=CALL_TIMEOUT_S, + ) + data = result.data + args.out.mkdir(parents=True, exist_ok=True) + (args.out / "report.md").write_text(data["report_markdown"], encoding="utf-8") + (args.out / "summary.json").write_text(json.dumps(data["summary"], indent=2)) + (args.out / "timeline.json").write_text(json.dumps(data["timeline"], indent=2)) + print(data["report_markdown"]) + print(f"Saved bundle to {args.out}/") + except LivepeerGatewayError as exc: + raise SystemExit(f"ERROR: {exc}") from exc + finally: + if session is not None: + with suppress(Exception): + await stop_runner_session(session) # Livepeer: 3 + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/screen-agent/demo_video.py b/screen-agent/demo_video.py new file mode 100644 index 0000000..5e348e3 --- /dev/null +++ b/screen-agent/demo_video.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Generate a small synthetic screen recording to test the app with. + +Draws a fake checkout form where a "ZIP code is required" error appears — +enough visual structure for the analyzer to sample frames and build a report. +No Livepeer code here; it's just test-input generation. +""" + +from __future__ import annotations + +import argparse + +import cv2 +import numpy as np + +WIDTH, HEIGHT = 1280, 720 + + +def _frame(t: float, seconds: float) -> np.ndarray: + img = np.full((HEIGHT, WIDTH, 3), (245, 246, 248), dtype=np.uint8) + cv2.rectangle(img, (390, 120), (890, 600), (255, 255, 255), -1) + cv2.rectangle(img, (390, 120), (890, 600), (210, 210, 214), 2) + cv2.putText(img, "Acme Checkout", (430, 180), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (40, 40, 46), 2) + cv2.rectangle(img, (430, 220), (850, 270), (235, 236, 240), -1) + cv2.putText(img, "Card number 4242 4242 4242 4242", (445, 252), + cv2.FONT_HERSHEY_SIMPLEX, 0.55, (90, 90, 98), 1) + cv2.rectangle(img, (430, 300), (850, 350), (235, 236, 240), -1) + cv2.putText(img, "ZIP code", (445, 332), cv2.FONT_HERSHEY_SIMPLEX, 0.55, (150, 150, 158), 1) + # The bug appears in the second half of the recording. + if t > seconds / 2: + cv2.putText(img, "Error: ZIP code is required", (430, 420), + cv2.FONT_HERSHEY_SIMPLEX, 0.7, (40, 40, 220), 2) + button_color = (200, 200, 204) # disabled + else: + button_color = (90, 170, 90) + cv2.rectangle(img, (430, 480), (620, 540), button_color, -1) + cv2.putText(img, "Pay $49", (465, 518), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 255), 2) + return img + + +def main() -> None: + parser = argparse.ArgumentParser(description="Write a synthetic demo screen recording.") + parser.add_argument("out", nargs="?", default="demo.mp4") + parser.add_argument("--seconds", type=float, default=8.0) + parser.add_argument("--fps", type=float, default=2.0) + args = parser.parse_args() + + writer = cv2.VideoWriter( + args.out, cv2.VideoWriter_fourcc(*"mp4v"), args.fps, (WIDTH, HEIGHT) + ) + if not writer.isOpened(): + raise SystemExit(f"Could not create video: {args.out}") + for i in range(int(args.seconds * args.fps)): + writer.write(_frame(i / args.fps, args.seconds)) + writer.release() + print(args.out) + + +if __name__ == "__main__": + main() diff --git a/screen-agent/docker-compose.yml b/screen-agent/docker-compose.yml new file mode 100644 index 0000000..cca973a --- /dev/null +++ b/screen-agent/docker-compose.yml @@ -0,0 +1,30 @@ +# End-to-end offchain demo: orchestrator + screen-agent app. +# +# The orchestrator service is defined once in ../compose.orchestrator.yml and +# pulled in with `extends`; this file only adds the app. Once up, call it from +# the host with the SDK: +# docker compose up -d --build +# uv run demo_video.py demo.mp4 +# uv run client.py demo.mp4 --discovery https://localhost:8935/discovery + +services: + orchestrator: + extends: + file: ../compose.orchestrator.yml + service: orchestrator + + app: + build: . + container_name: example_apps_screen_agent + # Wait for the orchestrator's healthcheck so registration doesn't race its boot. + depends_on: + orchestrator: + condition: service_healthy + command: + - --host=0.0.0.0 + - --orchestrator=https://orchestrator:8935 + - --orchSecret=abcdef + - --runner-url=http://app:8989 + - --app-id=livepeer-example/screen-agent + # No GPU/weights in the container — deterministic fallback engines. + - --force-fallback diff --git a/screen-agent/pyproject.toml b/screen-agent/pyproject.toml new file mode 100644 index 0000000..45e84bc --- /dev/null +++ b/screen-agent/pyproject.toml @@ -0,0 +1,15 @@ +[project] +name = "livepeer-screen-agent-example" +version = "0.1.0" +description = "screen-agent example app for the Livepeer network." +requires-python = ">=3.12" +dependencies = [ + "livepeer-gateway", + # For demo_video.py only (synthetic test recording). + "numpy>=1.26,<3", + "opencv-python-headless>=4.10,<5", +] + +# livepeer-gateway is not on PyPI yet; pull it from the branch. +[tool.uv.sources] +livepeer-gateway = { git = "https://github.com/livepeer/livepeer-python-gateway", branch = "ja/live-runner" }