Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 12 additions & 3 deletions skills/ainotate/references/spec.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ Everything the renderer accepts. `SKILL.md` has the workflow; this file is for l
| `crop_pad` | margin around the marks for `"auto"` and `"tight"` (default 260 image px at 2000px width) |
| `dim` | spotlight darkness 0-1 (default 0.55) |
| `arrow_style` | `"skitch"` (default, tapered), `"straight"`, `"curved"`, or `"line"` |
| `look` | mark style: `"default"` (built in) or an installed look package such as `"neat"`; see below |
| `frame` | backdrop and window chrome, see below |
| `privacy` | `"off"` (default for images), `"auto"` (OCR the image, redact findings) or an object |

Expand All @@ -32,6 +33,14 @@ Everything the renderer accepts. `SKILL.md` has the workflow; this file is for l
| `browser` | `chromium` (default), `firefox`, `webkit` (launch mode only) |
| `timeout_ms` | navigation and wait timeout (default 30000) |

**Looks.** `"look"` picks the mark style. `"default"` is the only one built in; other packages add
looks (entry point group `ainotate.looks`, a subclass of `ainotate.looks.Look`), for example the
separate `ainotate-neat` package adds `"neat"`. Pick one per spec, with `--look` on `annotate` and
`shoot`, or with the MCP `look` argument; without one, env `AINOTATE_LOOK`, then `look` in
`~/.config/ainotate/config.toml`, then `"default"`. A look that is not installed, or fails to
load, start or draw, never breaks a render: the default look is used and a warning says why and
lists the installed looks.

`shoot` defaults `crop` to `"auto"` and `privacy` to `"auto"`. With chrome `"browser"` and no
`url` in the frame, the address bar shows the captured page's address.

Expand Down Expand Up @@ -149,9 +158,9 @@ explains it below, and can use `**bold**`, `` `code` `` and lines starting with

Final images: `<output_dir>/<prefix> <YYYY-MM-DD at HH.MM.SS> <name>.png`, plus an identical
copy in `backup_dir` if set. Never overwrites (adds " (2)"). Configuration, highest first:
env `AINOTATE_OUTPUT_DIR`, `AINOTATE_BACKUP_DIR`, `AINOTATE_PREFIX`; then
`~/.config/ainotate/config.toml` keys `output_dir`, `backup_dir`, `prefix`; then defaults
`~/Pictures/AInotate`, no backup, prefix `AInotate`.
env `AINOTATE_OUTPUT_DIR`, `AINOTATE_BACKUP_DIR`, `AINOTATE_PREFIX`, `AINOTATE_LOOK`; then
`~/.config/ainotate/config.toml` keys `output_dir`, `backup_dir`, `prefix`, `look`; then defaults
`~/Pictures/AInotate`, no backup, prefix `AInotate`, look `default`. A spec's `look` beats both.

## More patterns

Expand Down
4 changes: 4 additions & 0 deletions src/ainotate/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,8 @@ def _result_args(privacy_default):
return [A("spec", help="spec.json, inline JSON, or - for stdin"),
A("--draft", "--no-save", dest="draft", action="store_true", help="write to a temp dir"),
A("--privacy", choices=("auto", "off"), help=f"solid redaction (default: {privacy_default})"),
A("--look", help="mark style: default or an installed look such as neat "
"(default: AINOTATE_LOOK, then look in config.toml)"),
A("--copy", action="store_true", help="also put the image on the clipboard"),
A("--json", action="store_true", help="print JSON status on stdout"),
A("--debug", action="store_true", help="draw target/label boxes; implies --draft")]
Expand All @@ -147,6 +149,8 @@ def _spec_with_flags(a):
spec = _load_spec(a.spec)
if a.privacy and isinstance(spec, dict):
spec["privacy"] = a.privacy
if a.look and isinstance(spec, dict):
spec["look"] = a.look
return spec


Expand Down
191 changes: 191 additions & 0 deletions src/ainotate/looks.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,191 @@
"""Mark looks. `Look` is the built-in look and the base class for looks from other packages:
entry point group "ainotate.looks", name -> a Look subclass. The look is picked by spec "look"
(also --look and the MCP `look` argument), then env AINOTATE_LOOK, then `look` in config.toml,
then "default". A look that is not installed, fails to load, fails to start or fails while
drawing never breaks a render: the default look is used and the render gets a warning."""
from __future__ import annotations

import copy
import math
from types import SimpleNamespace

from PIL import ImageColor, ImageDraw, ImageStat

from .style import SHADOW, color_of, font_for, load_font, pill_size, rgba


def _r():
"""The render module: its drawing helpers are looked up at call time (tests patch them there)."""
from . import render
return render


class Look:
"""render() draws every mark through these methods. `ctx` holds the unmarked `page`, `marks`,
`rects` and `points` (image px, after the crop), the crop's `offset` (image px), `scale`, `u`."""
shadow = SHADOW # drop shadow under all marks: opacity, y-offset, blur (x u)
dim = 0.55 # spotlight darkness when the spec sets no "dim"
loupe_line = None # magnifier line width (None: the built-in width)

def __init__(self, ctx):
self.ctx, u = ctx, ctx.u
self.stroke = max(3, round(u / 4))
self.spot_radius = int(u)
self.font = load_font(int(1.35 * u))
self.badge_font = load_font(int(1.4 * u))

def color(self, i):
return color_of(self.ctx.marks[i].get("color"))

def label_size(self, text):
return pill_size(self.font, text, self.ctx.u)

def label(self, layer, img, i, box, text): # labels and text notes; img: the page with tints
_r().draw_pill(layer, box, text, self.font, self.color(i), self.ctx.u)

def outline(self, layer, i, g):
_r().draw_outline(layer, g, self.color(i), self.stroke, int(0.6 * self.ctx.u))

def badge(self, layer, img, i, b, n):
_r().draw_badge(layer, b, self.color(i), n, self.badge_font, max(2, round(0.12 * self.ctx.u)))

def arrow(self, layer, gray, i, path, style, badge): # badge: the mark's own step badge or None
if style == "line":
_r().draw_arrow(ImageDraw.Draw(layer), path[0], path[-1], self.color(i), self.stroke, self.ctx.u)
else:
_r().draw_arrow_skitch(layer, gray, path[0], path[-1], self.color(i), self.ctx.u, path)

def highlight(self, over, img, i, r): # lighter tint on dark regions (70 reads as a solid block there)
dark = ImageStat.Stat(img.convert("L").crop([int(v) for v in r])).mean[0] < 90
ImageDraw.Draw(over).rectangle(r, fill=rgba(self.color(i), 45 if dark else 70))

def redact_fill(self, i): # a look may change the color of a redaction, never its area
return "#1F1F1F"

def keys_font(self, i, keys):
return font_for(keys, int(1.15 * self.ctx.u))


def installed_looks() -> dict:
"""Looks from other packages: name -> entry point."""
from importlib.metadata import entry_points
return {e.name: e for e in entry_points(group="ainotate.looks")}


class LookFailed(Exception):
"""An installed look raised while drawing; render() then redraws with the default look."""


def _check(ok, what, v):
if not ok:
raise ValueError(f"{what} is {v!r}")
return v


def _num(v, lo=0.0, hi=math.inf):
return isinstance(v, (int, float)) and not isinstance(v, bool) and math.isfinite(v) and lo <= v <= hi


def _color(v):
"""The color as Pillow draws it ("info" -> "#007AFF"); ValueError for anything it cannot draw."""
c = color_of(v)
ImageColor.getrgb(c)
return c


# what render() reads from a look, checked before use: a bad value is a failure of the look
CHECKS = {
"stroke": lambda v: _check(isinstance(v, int) and _num(v, 1), "stroke", v),
"spot_radius": lambda v: _check(isinstance(v, int) and _num(v), "spot_radius", v),
"dim": lambda v: _check(_num(v, 0, 1), "dim", v),
"loupe_line": lambda v: _check(v is None or (isinstance(v, int) and _num(v, 1)), "loupe_line", v),
"shadow": lambda v: _check(isinstance(v, tuple) and len(v) == 3 and all(_num(x) for x in v), "shadow", v),
"color": _color,
"redact_fill": _color,
"label_size": lambda v: _check(isinstance(v, tuple) and len(v) == 2 and all(_num(x, 1, 1e6) for x in v),
"label_size()", v),
"keys_font": lambda v: _check(hasattr(v, "getbbox") and hasattr(v, "getmetrics"), "keys_font()", v),
}


READS_PAGE = {"label", "badge", "highlight"} # hooks given the page image as their second argument


class _Guarded:
"""An installed look whose errors and bad values become LookFailed (attribute reads too), so
they are never mistaken for core errors and render() can redraw with the default look."""

def __init__(self, look, name):
self._look, self._name = look, name

def _fail(self, e):
return LookFailed(fallback(self._name, "failed while drawing", e))

def __getattr__(self, attr):
check = CHECKS.get(attr, lambda x: x)
try:
v = getattr(self._look, attr)
if not callable(v):
return check(v)
except Exception as e: # noqa: BLE001 - any failure inside a plug-in look
raise self._fail(e) from e

def call(*a, **k):
if attr in READS_PAGE: # (layer, img, ...): drawing on its own copy of the page can never
a = (a[0], a[1].copy(), *a[2:]) # paint over a redaction
try:
return check(v(*a, **k))
except Exception as e: # noqa: BLE001
raise self._fail(e) from e
return call


def _private(ctx):
"""A copy of ctx for an installed look: nothing it changes reaches the render or the spec."""
return SimpleNamespace(page=ctx.page.copy(), marks=copy.deepcopy(ctx.marks),
rects=copy.deepcopy(ctx.rects), points=copy.deepcopy(ctx.points),
offset=ctx.offset, scale=ctx.scale, u=ctx.u)


def fallback(name, what, e=None) -> str:
why = f" ({type(e).__name__}: {e})" if e is not None else ""
return f"look {name!r} {what}{why}, so the default look was used"


def name_of(spec):
"""The look a spec asks for: its "look", else AINOTATE_LOOK, else `look` in config.toml."""
if "look" in spec:
return spec["look"]
from .output import default_look
return default_look()


def installed_name(spec):
"""The name of the installed look this spec would use, else None (never raises)."""
try:
name = name_of(spec)
return name if name != "default" and name in installed_looks() else None
except Exception: # noqa: BLE001
return None


def pick(spec, ctx, warnings):
"""The look for this render; a look that cannot be used adds a warning and gives the default."""
name = name_of(spec)
if name == "default":
return Look(ctx)
found = installed_looks()
if name not in found:
warnings.append(fallback(name, "is not installed") + "; installed looks: "
+ ", ".join(["default", *sorted(found)]))
return Look(ctx)
try:
cls = found[name].load()
except Exception as e: # noqa: BLE001
warnings.append(fallback(name, "could not be loaded", e))
return Look(ctx)
try:
return _Guarded(cls(_private(ctx)), name)
except Exception as e: # noqa: BLE001
warnings.append(fallback(name, "failed to start", e))
return Look(ctx)
10 changes: 6 additions & 4 deletions src/ainotate/marks.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,12 +92,14 @@ def _shape(d, box, shape, radius, **kw):
d.rounded_rectangle(box, radius=radius, **kw)


def draw_loupe(layer, page, sample, box, shape, color, u, path=None):
def draw_loupe(layer, page, sample, box, shape, color, u, path=None, line=None):
"""Source outline, connector and the loupe itself (zoomed page pixels, white rim, color ring).
`page` is the image before any mark is drawn (redactions and blur already applied). `path`: a
curved connector from the loupe to the sample (routed around page content), else straight."""
curved connector from the loupe to the sample (routed around page content), else straight.
`line`: width of the connector, source outline and color ring (default from u)."""
lw, lh = int(box[2] - box[0]), int(box[3] - box[1])
line = max(2, round(0.13 * u))
ring = line
line = line or max(2, round(0.13 * u))
rad_src = min(0.6 * u, (sample[3] - sample[1]) / 2)
rad_lp = min(1.3 * u, min(lw, lh) / 4)
# connector between the facing edges (drawn first: the loupe and outline cover its ends)
Expand Down Expand Up @@ -136,7 +138,7 @@ def draw_loupe(layer, page, sample, box, shape, color, u, path=None):
mask = _aa_mask((lw, lh), lambda d, S: _shape(d, [0, 0, lw * S - 1, lh * S - 1], shape, rad_lp * S, fill=255))
content.putalpha(mask)
layer.alpha_composite(content, dest=(int(box[0]), int(box[1])))
rim, ring = max(3, round(0.3 * u)), max(2, round(0.1 * u))
rim, ring = max(3, round(0.3 * u)), ring or max(2, round(0.1 * u))
inner = grow(box, -rim / 2)
aa_draw(layer, box, "white", lambda d, T, S: _shape(d, T(inner), shape, (rad_lp - rim / 2) * S,
outline="white", width=rim * S))
Expand Down
3 changes: 2 additions & 1 deletion src/ainotate/mcp_docs.py
Original file line number Diff line number Diff line change
Expand Up @@ -250,7 +250,8 @@ def _fix_for(e: BaseException) -> str:


_FALLBACK_SPEC = """Spec: {"input": image path, "scale": spec units -> image px, "crop": "auto" |
[x1,y1,x2,y2] | {x,y,w,h}, "crop_pad": 260, "dim": 0.55, "name": "file name", "marks": [...]}
[x1,y1,x2,y2] | {x,y,w,h}, "crop_pad": 260, "dim": 0.55, "name": "file name", "look": "default" or an
installed look such as "neat" (mark style), "marks": [...]}
Units: every coordinate (rect, crop, label_at, at, pad) in ONE unit. DOM rects from capture_web:
dicts {x,y,w,h} with the returned scale. locate / grid / zoom pixels: lists [x1,y1,x2,y2], scale 1.
Colors: look (orange, default), bad (red: only for wrong/bug), good (green), info (blue).
Expand Down
31 changes: 22 additions & 9 deletions src/ainotate/mcp_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,11 @@
import json
import sys
from pathlib import Path
from typing import Callable, Optional, Union
from typing import Annotated, Callable, Optional, Union

import anyio.to_thread
from mcp import types
from pydantic import Field
from PIL import Image

from . import __version__
Expand Down Expand Up @@ -94,6 +95,17 @@ def _spec(spec) -> dict:
return spec


LOOK = Annotated[Optional[str], Field(description=(
'Mark style: "default" (bold, built in) or the name of an installed look package (e.g. "neat"). '
"Omit it to use the spec's look, else the user's default (AINOTATE_LOOK, config.toml). A look "
"that is not installed or fails falls back to default with a warning listing the installed looks."))]


def _with_look(spec, look):
spec = _spec(spec)
return dict(spec, look=look) if look and isinstance(spec, dict) else spec


# ---------------------------------------------------------------- annotate

def _annotate(spec, draft: bool, debug: bool, tool: str) -> types.CallToolResult:
Expand All @@ -120,22 +132,23 @@ def _annotate(spec, draft: bool, debug: bool, tool: str) -> types.CallToolResult


@mcp.tool()
async def annotate(spec: Union[dict, str], draft: bool = True) -> types.CallToolResult:
async def annotate(spec: Union[dict, str], draft: bool = True, look: LOOK = None) -> types.CallToolResult:
"""Render an annotation spec onto its `input` image. Draft by default (temp dir): check the
returned preview, then call again with draft=false to save to the output folder.
spec: {"input": path, "scale": 1|2.., "crop": "auto"|[x1,y1,x2,y2], "name": "...",
"marks": [{"type": "step"|"box"|"arrow"|"highlight"|"spotlight"|"redact"|"text"|...,
"rect": {x,y,w,h} or [x1,y1,x2,y2], "label": "...", "n": 1, "color": "look|bad|good|info"}]}.
Every coordinate in one unit; scale converts it to image px. Full reference: ainotate://spec.
look: the mark style, "default" or an installed look such as "neat" (sets spec "look").
Returns paths, size, warnings, legend (step numbers + labels), redactions and a preview."""
return await _run("annotate", lambda: _annotate(spec, draft, False, "annotate"))
return await _run("annotate", lambda: _annotate(_with_look(spec, look), draft, False, "annotate"))


@mcp.tool()
async def preview(spec: Union[dict, str]) -> types.CallToolResult:
async def preview(spec: Union[dict, str], look: LOOK = None) -> types.CallToolResult:
"""Draft render with the debug overlay (cyan target rects, magenta label boxes); saves
nothing permanent. Use it to check placement before annotate."""
return await _run("preview", lambda: _annotate(spec, True, True, "preview"))
nothing permanent. Use it to check placement before annotate. look: as in annotate."""
return await _run("preview", lambda: _annotate(_with_look(spec, look), True, True, "preview"))


# ---------------------------------------------------------------- web
Expand Down Expand Up @@ -180,12 +193,12 @@ def work():


@mcp.tool()
async def shoot(spec: Union[dict, str], draft: bool = True) -> types.CallToolResult:
async def shoot(spec: Union[dict, str], draft: bool = True, look: LOOK = None) -> types.CallToolResult:
"""One call from URL to annotated image: spec = render spec without input/scale, plus
url (or cdp_url + page_url_contains), preset, actions, and marks with "target": {...}
(same forms as capture_web) instead of "rect". Draft by default: check the preview, then
call again with draft=false to save. Returns paths, warnings, legend, redactions (secrets
hidden automatically), the capture's resolved rects and a preview."""
hidden automatically), the capture's resolved rects and a preview. look: as in annotate."""
def work():
try:
from .shoot import shoot as _shoot
Expand All @@ -196,7 +209,7 @@ def work():
"fix": "Use capture_web(targets=...) and then annotate(spec) with the returned "
"rects and scale."}}, error=True)
raise
res = _shoot(_spec(spec), draft=draft)
res = _shoot(_with_look(spec, look), draft=draft)
paths = [str(p) for p in res.paths]
jpeg, (w, h) = preview_jpeg(paths[0])
cap = getattr(res, "capture", None)
Expand Down
8 changes: 8 additions & 0 deletions src/ainotate/output.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,14 @@ def pick(env, key, default):
return Config(Path(out).expanduser(), Path(bak).expanduser() if bak else None, prefix)


def default_look() -> str:
"""The look for a spec without "look": env AINOTATE_LOOK > config.toml `look` > "default"."""
look = os.environ.get("AINOTATE_LOOK")
if not look:
look = _read_config_file(CONFIG_PATH).get("look")
return look if isinstance(look, str) and look else "default"


def safe_component(value: str, what: str = "name") -> str:
"""`value` if it is one plain file-name component: no path separators, no '..', no control
characters, not '.'; else OutputError (a bad prefix must not write outside the folder)."""
Expand Down
Loading