From c63d9e2eab29c7cad3e954ffe1357ed143fd12b3 Mon Sep 17 00:00:00 2001 From: Aneesh Sathe Date: Fri, 9 Oct 2026 21:00:02 -0700 Subject: [PATCH 01/16] Community extensions: a submitted list, linked from the docs extensions/community.toml lists third-party extensions (none yet). The docs build renders it into a Community extensions page with a plain use-at-your-own-risk disclaimer; CI checks the file's format only. Submissions come as a pull request (template) or an issue form. Co-Authored-By: Claude Opus 5.5 --- .../ISSUE_TEMPLATE/extension-submission.yml | 82 +++++ .github/PULL_REQUEST_TEMPLATE/extension.md | 23 ++ .github/workflows/ci.yml | 5 + .github/workflows/pages.yml | 2 + .gitignore | 2 + extensions/community.toml | 41 +++ scripts/build-site.sh | 4 + scripts/community-extensions.py | 340 ++++++++++++++++++ site/src/SUMMARY.md | 1 + site/src/extensions.md | 56 +++ 10 files changed, 556 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/extension-submission.yml create mode 100644 .github/PULL_REQUEST_TEMPLATE/extension.md create mode 100644 extensions/community.toml create mode 100755 scripts/community-extensions.py diff --git a/.github/ISSUE_TEMPLATE/extension-submission.yml b/.github/ISSUE_TEMPLATE/extension-submission.yml new file mode 100644 index 0000000..31809f1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/extension-submission.yml @@ -0,0 +1,82 @@ +name: Submit a community extension +description: Ask for your Burrow extension to be listed on the Community extensions page. +title: "Community extension: " +labels: ["community-extension"] +body: + - type: markdown + attributes: + value: | + Prefer a pull request? Add an `[[extension]]` entry to `extensions/community.toml` instead (see "Submit your extension" on the Extensions page of the documentation). + + **The Burrow maintainers don't review, verify, audit or endorse community extensions.** We check that the entry is well-formed, add it, and the listing says it is used at the reader's own risk. Don't put an email address or other personal data in this form. + - type: input + id: name + attributes: + label: Name + description: The `name` in your extension.toml. Lowercase kebab-case, at most 40 characters. + placeholder: wordcount + validations: + required: true + - type: input + id: title + attributes: + label: Title + description: A short human name, at most 60 characters. + placeholder: Word count + validations: + required: true + - type: input + id: description + attributes: + label: Description + description: One line of plain text, at most 160 characters. + placeholder: Shows the word count and reading time of the open draft. + validations: + required: true + - type: input + id: repo + attributes: + label: Source repository + description: An https:// link to the public source code. + placeholder: https://github.com/example/burrow-ext-wordcount + validations: + required: true + - type: input + id: homepage + attributes: + label: Homepage (optional) + placeholder: https://example.com/wordcount + - type: input + id: author + attributes: + label: Author + description: Your GitHub handle, without the @. Not an email address. + placeholder: example + validations: + required: true + - type: textarea + id: capabilities + attributes: + label: Capabilities + description: Exactly what your manifest's `capabilities` asks for, one per line (or "none"). + placeholder: | + items.read + ui + validations: + required: true + - type: checkboxes + id: platforms + attributes: + label: Platforms + options: + - label: macOS + - label: Windows + - type: checkboxes + id: understood + attributes: + label: Before you submit + options: + - label: I wrote this extension, or I maintain it, and its source code is public. + required: true + - label: I understand the maintainers check the entry's format only, and that the listing can be removed if it's reported as malicious or misleading. + required: true diff --git a/.github/PULL_REQUEST_TEMPLATE/extension.md b/.github/PULL_REQUEST_TEMPLATE/extension.md new file mode 100644 index 0000000..0e20c03 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE/extension.md @@ -0,0 +1,23 @@ + + +## Extension + +- **Name:** +- **Source:** + +## Checklist + +- [ ] This pull request changes only `extensions/community.toml`, and only one entry. +- [ ] The entry is in alphabetical order by `name`. +- [ ] `name` matches the `name` in my `extension.toml`. +- [ ] `capabilities` lists exactly what my manifest asks for. +- [ ] `repo` links to the extension's source code, which is public. +- [ ] `author` is my GitHub handle, and there's no email address anywhere in the entry. +- [ ] `python3 scripts/community-extensions.py check` passes. +- [ ] I understand that the Burrow maintainers check the entry's format only. + They don't review, verify, audit or endorse the extension, and the listing + says so. A listing can be removed if it's reported as malicious or misleading. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a480ee8..6792915 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,6 +97,11 @@ jobs: with: persist-credentials: false + # The community extension list: format only (valid TOML, required + # fields, names, https URLs, sorted). It never fetches or runs anything. + - name: Check the community extension list + run: python3 scripts/community-extensions.py check + # The pinned mdBook; fails on mdBook errors or broken links. - name: Build the book run: scripts/build-site.sh diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 20ad3f3..d34aaff 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -15,6 +15,8 @@ on: - "CHANGELOG.md" - "scripts/build-site.sh" - "scripts/site-linkcheck.py" + - "scripts/community-extensions.py" + - "extensions/community.toml" - ".github/workflows/pages.yml" workflow_dispatch: diff --git a/.gitignore b/.gitignore index aaee0d0..0bc099b 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,5 @@ # The documentation site build (mdbook build site) /site/book/ +# Made from extensions/community.toml by scripts/build-site.sh +/site/src/generated/community-extensions.md diff --git a/extensions/community.toml b/extensions/community.toml new file mode 100644 index 0000000..f4a2a5a --- /dev/null +++ b/extensions/community.toml @@ -0,0 +1,41 @@ +# Community extensions for Burrow. +# +# Every entry here is listed on the documentation site's "Community extensions" +# page (site/src/generated/community-extensions.md, made by +# scripts/build-site.sh). The Burrow maintainers do NOT review, verify, audit +# or endorse these extensions. A listing only means the entry below passed +# scripts/community-extensions.py check, which looks at its format, nothing +# else. +# +# To list your extension, open a pull request that adds one [[extension]] +# table, kept in alphabetical order by `name`. See "Submit your extension" on +# the Extensions page of the documentation site. +# +# Fields (all strings unless noted; plain text, no Markdown or HTML): +# +# name required. The `name` in your extension.toml: lowercase +# kebab-case ([a-z0-9]+(-[a-z0-9]+)*), at most 40 characters, +# not a bundled extension's name, not already listed. +# title required. A short human name, at most 60 characters. +# description required. One line, at most 160 characters. +# repo required. https:// URL of the source code. +# homepage optional. https:// URL of a page about it. +# author required. Your GitHub handle, without "@". Never an email. +# capabilities required, a list (may be empty). Exactly what your +# manifest's `capabilities` asks for, e.g. "items.read", "ui", +# "net", "fs:~/Notes", "browser.automate:https://example.com". +# platforms required, a non-empty list: "macos", "windows". +# added required, a TOML date (no quotes): the day it was listed. +# +# Example (commented out; it is not a real extension): +# +# [[extension]] +# name = "wordcount" +# title = "Word count" +# description = "Shows the word count and reading time of the open draft." +# repo = "https://github.com/example/burrow-ext-wordcount" +# homepage = "https://example.com/wordcount" +# author = "example" +# capabilities = ["items.read", "ui"] +# platforms = ["macos", "windows"] +# added = 2026-10-09 diff --git a/scripts/build-site.sh b/scripts/build-site.sh index d3103dc..293368b 100755 --- a/scripts/build-site.sh +++ b/scripts/build-site.sh @@ -44,6 +44,10 @@ if [ -z "${MDBOOK:-}" ]; then fi fi +# The Community extensions page is made from extensions/community.toml (and +# fails the build when an entry is malformed); it isn't committed. +python3 scripts/community-extensions.py render + log="$(mktemp)" trap 'rm -f "$log"' EXIT "$MDBOOK" build site 2>&1 | tee "$log" diff --git a/scripts/community-extensions.py b/scripts/community-extensions.py new file mode 100755 index 0000000..1eb7f87 --- /dev/null +++ b/scripts/community-extensions.py @@ -0,0 +1,340 @@ +#!/usr/bin/env python3 +"""Check the community extension list, and render it as a documentation page. + + scripts/community-extensions.py check [extensions/community.toml] + scripts/community-extensions.py render [extensions/community.toml] [OUT.md] + +`check` exits 1 and prints every problem when an entry is malformed. `render` +checks first (and fails the same way), then writes the "Community extensions" +page (default: site/src/generated/community-extensions.md). +scripts/build-site.sh runs `render`; CI runs `check` on every pull request. + +The checks are about format only: they never fetch, run or review an +extension. Python 3.11+ standard library only (tomllib). +""" +import datetime +import os +import re +import sys +import tomllib +from urllib.parse import urlsplit + +ROOT = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) +DATA = os.path.join(ROOT, "extensions", "community.toml") +OUT = os.path.join(ROOT, "site", "src", "generated", "community-extensions.md") + +NAME_MAX = 40 # crates/blyg-ext/src/manifest.rs NAME_MAX +TITLE_MAX = 60 +DESCRIPTION_MAX = 160 +PLATFORMS = ("macos", "windows") +PLATFORM_WORDS = {"macos": "macOS", "windows": "Windows"} +REQUIRED = ("name", "title", "description", "repo", "author", "capabilities", "platforms", "added") +OPTIONAL = ("homepage",) + +NAME_RE = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$") +# GitHub's rule: 1-39 alphanumerics or single hyphens, not starting or ending with one. +HANDLE_RE = re.compile(r"^[A-Za-z0-9](?:[A-Za-z0-9]|-(?=[A-Za-z0-9])){0,38}$") +EMAIL_RE = re.compile(r"[^\s@<>()]+@[^\s@<>()]+\.[A-Za-z]{2,}") + +# What each capability lets an extension do, in the words of site/src/extensions.md. +CAPABILITY_WORDS = { + "items.read": "read your posts, drafts and scratch notes", + "items.write": "create drafts and scratch notes, and edit their text", + "reading.read": "read the posts held from your subscriptions", + "blyg.identity": "know your blyg's address", + "ui": "show messages, and select a post in the list", + "hooks:itemPublished": "be told when you publish a post, with its text", + "hooks:itemSaved": "be told when you edit a post, with its text", + "hooks:itemCreated": "be told when you create a post, with its text", + "browser.capture": "read the page open in the browser pane", + "net": "use the network (declared, not enforced)", +} +FS_PREFIX = "fs:" +AUTOMATE_PREFIX = "browser.automate:" + + +def read_rust_strings(path, pattern): + try: + with open(path, encoding="utf-8") as fh: + return re.findall(pattern, fh.read()) + except OSError: + return [] + + +def known_capabilities(): + """EXTENSION_CAPABILITIES from the config key table, so the two can't drift.""" + keys = os.path.join(ROOT, "crates", "blyg-core", "src", "config", "keys.rs") + try: + with open(keys, encoding="utf-8") as fh: + m = re.search(r"EXTENSION_CAPABILITIES: &\[&str\] = &\[(.*?)\];", fh.read(), re.S) + except OSError: + m = None + caps = re.findall(r'"([^"]+)"', m.group(1)) if m else [] + return caps or list(CAPABILITY_WORDS) + + +def bundled_names(): + """The `NAME` of every bundled extension crate (crates/blyg-ext-*).""" + names = {"markdown-notes", "cross-post"} + crates = os.path.join(ROOT, "crates") + if os.path.isdir(crates): + for d in sorted(os.listdir(crates)): + if d.startswith("blyg-ext-"): + lib = os.path.join(crates, d, "src", "lib.rs") + names.update(read_rust_strings(lib, r'pub const NAME: &str = "([^"]+)";')) + return names + + +def check_url(value): + if not isinstance(value, str): + return "must be a string" + if re.search(r"[\s<>\"'`]", value): + return "must not contain spaces, quotes or angle brackets" + parts = urlsplit(value) + if parts.scheme != "https": + return "must be an https:// URL" + if not parts.hostname or "." not in parts.hostname: + return "must name a host" + if parts.username or parts.password: + return "must not contain a user name or password" + return None + + +def check_plain(value, limit): + if not isinstance(value, str): + return "must be a string" + if not value.strip(): + return "must not be empty" + if value != value.strip(): + return "must not start or end with spaces" + if any(c in value for c in "\r\n\t"): + return "must be one line" + if re.search(r"[<>`]", value): + return "must be plain text (no HTML or Markdown code)" + if len(value) > limit: + return f"is {len(value)} characters; at most {limit}" + return None + + +def check_capability(cap, known): + if not isinstance(cap, str): + return "must be a string" + if cap.startswith(FS_PREFIX): + path = cap[len(FS_PREFIX):] + if not path.strip() or path != path.strip() or re.search(r"[<>`\n]", path): + return "an fs: capability needs a folder, e.g. fs:~/Notes" + return None + if cap.startswith(AUTOMATE_PREFIX): + origin = cap[len(AUTOMATE_PREFIX):] + parts = urlsplit(origin) + if parts.scheme not in ("https", "http") or not parts.hostname or parts.path not in ("", "/") \ + or parts.query or parts.fragment or parts.username: + return "browser.automate: needs an origin, e.g. browser.automate:https://example.com" + return None + if cap not in known: + return f"isn't a capability Burrow knows (one of {', '.join(known)}, fs: or browser.automate:)" + return None + + +def validate(path): + """Return (entries, problems).""" + try: + with open(path, "rb") as fh: + data = tomllib.load(fh) + except OSError as e: + return [], [f"can't read it: {e}"] + except tomllib.TOMLDecodeError as e: + return [], [f"isn't valid TOML: {e}"] + + problems = [] + extra = sorted(set(data) - {"extension"}) + if extra: + problems.append(f"unknown top-level keys {extra}: entries are [[extension]] tables") + entries = data.get("extension", []) + if not isinstance(entries, list): + return [], problems + ["`extension` must be an array of tables ([[extension]])"] + + known = known_capabilities() + bundled = bundled_names() + seen_names, seen_repos = {}, {} + today = datetime.date.today() + + for i, e in enumerate(entries, 1): + label = f"entry {i}" + if isinstance(e, dict) and isinstance(e.get("name"), str): + label += f" ({e['name']})" + + def bad(field, msg): + problems.append(f"{label}: `{field}` {msg}") + + if not isinstance(e, dict): + problems.append(f"{label}: must be a table") + continue + for f in REQUIRED: + if f not in e: + bad(f, "is required") + for f in sorted(set(e) - set(REQUIRED) - set(OPTIONAL)): + bad(f, "isn't a known field") + + name = e.get("name") + if "name" in e: + if not isinstance(name, str) or not NAME_RE.match(name): + bad("name", "must be lowercase kebab-case, like word-count") + elif len(name) > NAME_MAX: + bad("name", f"is longer than {NAME_MAX} characters") + elif name in bundled: + bad("name", "is the name of an extension bundled with Burrow") + elif name in seen_names: + bad("name", f"is already listed (entry {seen_names[name]})") + else: + seen_names[name] = i + + for f, limit in (("title", TITLE_MAX), ("description", DESCRIPTION_MAX)): + if f in e and (msg := check_plain(e[f], limit)): + bad(f, msg) + + for f in ("repo", "homepage"): + if f in e and (msg := check_url(e[f])): + bad(f, msg) + repo = e.get("repo") + if isinstance(repo, str): + key = repo.rstrip("/").lower() + if key in seen_repos: + bad("repo", f"is already listed (entry {seen_repos[key]})") + seen_repos[key] = i + + if "author" in e: + a = e["author"] + if not isinstance(a, str) or not HANDLE_RE.match(a): + bad("author", "must be a GitHub handle, like octocat (no @, no email)") + + if "capabilities" in e: + caps = e["capabilities"] + if not isinstance(caps, list): + bad("capabilities", "must be a list, e.g. [\"items.read\", \"ui\"] (or [])") + else: + for c in caps: + if msg := check_capability(c, known): + bad("capabilities", f"{c!r} {msg}") + if len(set(map(str, caps))) != len(caps): + bad("capabilities", "lists a capability twice") + + if "platforms" in e: + ps = e["platforms"] + if not isinstance(ps, list) or not ps: + bad("platforms", f"must be a non-empty list of {', '.join(PLATFORMS)}") + else: + for p in ps: + if p not in PLATFORMS: + bad("platforms", f"{p!r} isn't one of {', '.join(PLATFORMS)}") + if len(set(map(str, ps))) != len(ps): + bad("platforms", "lists a platform twice") + + if "added" in e: + d = e["added"] + if not isinstance(d, datetime.date) or isinstance(d, datetime.datetime): + bad("added", "must be a TOML date without quotes, like 2026-10-09") + elif d > today + datetime.timedelta(days=1): + bad("added", "is in the future") + + for f, v in e.items(): + if isinstance(v, str) and EMAIL_RE.search(v): + bad(f, "looks like it contains an email address; don't list one") + + names = [e.get("name") for e in entries if isinstance(e, dict) and isinstance(e.get("name"), str)] + if names != sorted(names): + problems.append(f"entries must be sorted by name: {', '.join(sorted(names))}") + + return entries, problems + + +DISCLAIMER = """\ +> **Use at your own risk.** These are third-party extensions, written and +> published by other people. The Burrow maintainers do not review, verify, +> audit or endorse them: a listing only means its entry is in the right +> format. An extension is a program that runs with your user's rights. +> Burrow enforces most of the permissions you grant it, but `fs:` and `net` +> are only the extension's own declarations, so it can read and change your +> files and use the network whatever it declares. Read the source before +> you install one, and install only what you trust. If a listing is +> malicious or misleading, +> [open an issue](https://github.com/aneeshsathe/blygger-desktop/issues/new) +> and it will be removed.""" + + +def md(text): + """Escape plain text for Markdown (it was already checked for HTML).""" + return re.sub(r"([\\`*_\[\]{}#|!~&])", r"\\\1", text) + + +def capability_words(cap): + if cap.startswith(FS_PREFIX): + return f"read and write files in `{cap[len(FS_PREFIX):]}` (declared, not enforced)" + if cap.startswith(AUTOMATE_PREFIX): + return f"fill in and, when you confirm, post on {md(cap[len(AUTOMATE_PREFIX):])}, signed in as you" + return CAPABILITY_WORDS.get(cap, md(cap)) + + +def render(entries): + out = [ + "", + "", + "# Community extensions", + "", + DISCLAIMER, + "", + "Extensions other people have written for Burrow. To install one, follow", + "[Installing someone else's extension](../extensions.md#installing-someone-elses-extension);", + "to list yours here, see [Submit your extension](../extensions.md#submit-your-extension).", + "", + ] + if not entries: + out += ["*No community extensions yet — be the first.*", ""] + return "\n".join(out) + for e in entries: + caps = e["capabilities"] + out += [f"## {md(e['title'])}", "", md(e["description"]), ""] + out.append(f"- **Name:** `{e['name']}`") + out.append(f"- **Source:** [{md(e['repo'])}]({e['repo']})") + if e.get("homepage"): + out.append(f"- **Homepage:** [{md(e['homepage'])}]({e['homepage']})") + out.append(f"- **Author:** [{md(e['author'])}](https://github.com/{e['author']})") + if caps: + out.append("- **Asks to:**") + out += [f" - `{c}`: {capability_words(c)}" for c in caps] + else: + out.append("- **Asks to:** nothing (no capabilities)") + out.append("- **Platforms:** " + ", ".join(PLATFORM_WORDS[p] for p in e["platforms"])) + out.append(f"- **Listed:** {e['added'].isoformat()}") + out.append("") + return "\n".join(out) + + +def main(argv): + if len(argv) < 2 or argv[1] not in ("check", "render"): + print(__doc__.strip(), file=sys.stderr) + return 2 + data = argv[2] if len(argv) > 2 else DATA + inside = os.path.abspath(data).startswith(ROOT + os.sep) + shown = os.path.relpath(data, ROOT) if inside else data + entries, problems = validate(data) + for p in problems: + print(f"{shown}: {p}", file=sys.stderr) + if problems: + print(f"{len(problems)} problem(s); see \"Submit your extension\" in site/src/extensions.md", + file=sys.stderr) + return 1 + if argv[1] == "check": + print(f"{shown}: {len(entries)} extension(s), all well-formed") + return 0 + out = argv[3] if len(argv) > 3 else OUT + os.makedirs(os.path.dirname(out), exist_ok=True) + with open(out, "w", encoding="utf-8") as fh: + fh.write(render(entries)) + print(f"wrote {os.path.relpath(out, ROOT)} ({len(entries)} extension(s))") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/site/src/SUMMARY.md b/site/src/SUMMARY.md index 015544b..d9257bc 100644 --- a/site/src/SUMMARY.md +++ b/site/src/SUMMARY.md @@ -12,6 +12,7 @@ - [AI helpers](ai.md) - [Themes](themes.md) - [Extensions](extensions.md) + - [Community extensions](generated/community-extensions.md) # Reference diff --git a/site/src/extensions.md b/site/src/extensions.md index 2211c33..15cdf9d 100644 --- a/site/src/extensions.md +++ b/site/src/extensions.md @@ -207,3 +207,59 @@ repository. For an extension in Rust, the `blyg-ext` crate has the protocol types and a `serve` loop, and markdown-notes (`crates/blyg-ext-notes`) is a worked example. + +## Submit your extension + +Wrote an extension? List it on the +[Community extensions](generated/community-extensions.md) page. The page is +made from one file in the repository, `extensions/community.toml`, every time +the documentation site is built, so a merged entry shows up on its own. + +> **Community extensions are used at your own risk.** The Burrow maintainers +> do not review, verify, audit or endorse them. An extension is a program that +> runs with your user's rights, and `fs:` and `net` are its own declarations, +> not something Burrow can enforce. Read the source before you install one. To +> report a malicious or misleading listing, +> [open an issue](https://github.com/aneeshsathe/blygger-desktop/issues/new) +> and it will be removed. + +To submit one, open a pull request that adds an `[[extension]]` table to +`extensions/community.toml`, in alphabetical order by `name`: + +```toml +[[extension]] +name = "wordcount" # the name in your extension.toml +title = "Word count" +description = "Shows the word count and reading time of the open draft." +repo = "https://github.com/example/burrow-ext-wordcount" +homepage = "https://example.com/wordcount" # optional +author = "example" # your GitHub handle, not an email +capabilities = ["items.read", "ui"] # exactly what your manifest asks for +platforms = ["macos", "windows"] +added = 2026-10-09 # a date, without quotes +``` + +If you'd rather not open a pull request, fill in the +[extension submission form](https://github.com/aneeshsathe/blygger-desktop/issues/new?template=extension-submission.yml) +and a maintainer will add the entry for you. + +CI runs `scripts/community-extensions.py check` on the pull request (you can +run it yourself first). It checks that: + +- the file is valid TOML, and every entry has all the fields above and no + others (only `homepage` is optional); +- `name` is lowercase kebab-case (`word-count`), at most 40 characters, not + already listed, and not the name of an extension that comes with Burrow; +- `title` (at most 60 characters) and `description` (one line, at most 160) + are plain text, with no HTML or Markdown code; +- `repo` and `homepage` are `https://` addresses; +- `author` is a GitHub handle, and no field holds an email address; +- each capability is one Burrow knows (see + [Permissions and consent](#permissions-and-consent)), and `platforms` is + `macos`, `windows` or both; +- the entries are sorted by `name`. + +A maintainer merges an entry after a light look at its format, and nothing +more: the source code isn't read, run or tested. List the source code's +repository, keep `capabilities` in step with your manifest, and open another +pull request when the details change, or to take the entry down. From 9ebca3602479627ab6fcf5046ac64af208fb62bc Mon Sep 17 00:00:00 2001 From: Aneesh Sathe Date: Fri, 9 Oct 2026 21:00:51 -0700 Subject: [PATCH 02/16] Building docs: render the community extensions page before mdbook serve Co-Authored-By: Claude Opus 5.5 --- site/src/dev/building.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/site/src/dev/building.md b/site/src/dev/building.md index 80d1c39..c559175 100644 --- a/site/src/dev/building.md +++ b/site/src/dev/building.md @@ -163,7 +163,8 @@ It downloads the pinned mdBook release into `target/` the first time (checksum-verified), and fails on an mdBook error (such as a missing `\{{#include}}` file) or a broken internal link. CI runs it on every pull request. With mdBook installed yourself, `mdbook serve site` serves the site at `http://localhost:3000` and -rebuilds it as you edit. +rebuilds it as you edit. Run `python3 scripts/community-extensions.py render` +once first: the Community extensions page is generated, not committed. `.github/workflows/pages.yml` publishes the site to GitHub Pages on every push to `main` that touches the site, `docs/`, the config keys, the keymap or the From ebe05afef1f9e071e422ade0fcbb7ddf832a5a92 Mon Sep 17 00:00:00 2001 From: Aneesh Sathe Date: Fri, 9 Oct 2026 21:34:50 -0700 Subject: [PATCH 03/16] Read state needs the reading:state scope (studio 0.39) Studio 0.39 moved markRead, markUnread and both batch read-state ops from owner:manage to a new, provisionally named scope, reading:state. - oauth: READ_STATE_SCOPE is the one place the name is spelled (a macro feeds SCOPE, OWNER_SCOPES and the manual-token wording). The browser sign-in asks only for the scopes the studio lists in scopes_supported, so an older studio isn't sent a scope it would refuse. - sync: a 403 on a read-state write no longer drops the ops. They stay queued and are skipped for the rest of the session, new marks keep queueing, and the rest of the outbox syncs. One notice per session names the missing scope and says to sign in again (or make a new token). The next sign-in sends what waited. There's no retry loop. - e2e: the harness mints tokens with reading:state (falling back to the four scopes on an older studio). New tests: a four-scope token is refused, its read stays queued, and a later credential sends it; and signing in again with a kept 0.10 client grants reading:state. - Docs, sign-in copy and the getting-started page name the scope and note that the name is provisional. Co-Authored-By: Claude Opus 5.5 --- crates/blyg-app/src/app.rs | 5 +- crates/blyg-core/src/api/oauth.rs | 114 ++++++++++- crates/blyg-core/src/sync/mod.rs | 88 +++++--- crates/blyg-core/src/sync/read_held_tests.rs | 140 +++++++++++++ crates/blyg-core/tests/e2e.rs | 203 ++++++++++++++++++- docs/SERVER.md | 24 ++- docs/SPEC.md | 5 +- scripts/e2e-wrangler.sh | 14 +- site/src/getting-started.md | 5 +- 9 files changed, 550 insertions(+), 48 deletions(-) create mode 100644 crates/blyg-core/src/sync/read_held_tests.rs diff --git a/crates/blyg-app/src/app.rs b/crates/blyg-app/src/app.rs index 4d38311..5a58cfa 100644 --- a/crates/blyg-app/src/app.rs +++ b/crates/blyg-app/src/app.rs @@ -3089,12 +3089,13 @@ impl MainView { let explain = match way { ConnectWay::Browser => { "Burrow opens your blyg's studio in your browser. Sign in there and \ - allow Burrow; it gets renewable access with all four permissions, \ + allow Burrow; it gets renewable access with every permission, \ kept in your macOS Keychain. Needs blygger-studio 0.28 or later." } ConnectWay::Token => { "Make a token in your blyg's Studio → More → Client access: choose REST \ - API and all four permissions (read, draft, publish, manage). It lasts \ + API and every permission (read, draft, publish, manage, and reading:state \ + where offered). It lasts \ 30 days. A server with Burrow's extensions can use its owner token \ here instead. Either is kept in your macOS Keychain." } diff --git a/crates/blyg-core/src/api/oauth.rs b/crates/blyg-core/src/api/oauth.rs index 1d68281..607a586 100644 --- a/crates/blyg-core/src/api/oauth.rs +++ b/crates/blyg-core/src/api/oauth.rs @@ -14,7 +14,8 @@ //! owner signs in to the studio there and allows Burrow. The code comes //! back to a listener on 127.0.0.1 ([`Loopback`]). //! 4. **Exchange** the code for a one-hour access token and a rotating -//! refresh token, bound to the API resource, with all four owner scopes. +//! refresh token, bound to the API resource, with every owner scope the +//! server knows ([`SCOPE`]). //! //! The grant ([`OAuthGrant`]) is kept as JSON under ` oauth`. An //! [`OAuthSession`] renews it shortly before it expires and after a 401, @@ -41,10 +42,34 @@ use sha2::{Digest, Sha256}; use crate::backend::{CoreError, Result}; use crate::config::{TokenStore, token_account}; -/// The owner scopes (upstream `src/permissions.ts` `OWNER_SCOPES`). -pub const OWNER_SCOPES: [&str; 4] = ["owner:read", "owner:draft", "owner:publish", "owner:manage"]; -/// What Burrow asks for: every owner scope, and a refresh token. -pub const SCOPE: &str = "owner:read owner:draft owner:publish owner:manage offline_access"; +/// The read-state scope's name, the one place it's spelled (`READ_STATE_SCOPE`, +/// `SCOPE` and `MANUAL_TOKEN_PERMISSIONS` all expand it). Studio 0.39 moved +/// markRead, markUnread and both batch ops from `owner:manage` to it. +/// PROVISIONAL: upstream may rename it; change it here. +macro_rules! read_state_scope { + () => { + "reading:state" + }; +} +/// The scope read-state writes need (studio 0.39+). Provisional name. +pub const READ_STATE_SCOPE: &str = read_state_scope!(); +/// The owner scopes (upstream `src/auth-scopes.ts` `OWNER_SCOPES`, in its +/// order, which is also the consent page's). +pub const OWNER_SCOPES: [&str; 5] = [ + "owner:read", + "owner:draft", + "owner:publish", + "owner:manage", + READ_STATE_SCOPE, +]; +/// What Burrow asks for: every owner scope, and a refresh token. A studio +/// that doesn't know one of them (`reading:state` before 0.39) refuses the +/// lot, so `discover` keeps only those the server lists (`Discovery::scope`). +pub const SCOPE: &str = concat!( + "owner:read owner:draft owner:publish owner:manage ", + read_state_scope!(), + " offline_access" +); /// The name the studio's consent page shows (self-asserted). pub const CLIENT_NAME: &str = "Burrow"; /// The loopback callback's path. @@ -151,6 +176,14 @@ fn auth_param(header: &str, name: &str) -> Option { None } +/// The permissions a manual token needs, as Studio → More → Client access +/// words them (`reading:state` is studio 0.39+; an older studio doesn't offer it). +pub const MANUAL_TOKEN_PERMISSIONS: &str = concat!( + "every permission (read, draft, publish, manage, and ", + read_state_scope!(), + " where offered)" +); + /// What to say when the blyg answered 403 "insufficient scope". `granted` /// is the grant's scopes when known (browser sign-in); a manual token's /// aren't, so it names everything the call needed. @@ -171,10 +204,10 @@ pub fn missing_scope_message(required: &[String], granted: Option<&str>) -> Stri }; match granted { Some(_) => format!( - "The blyg refused: Burrow's sign-in lacks {named}. Sign in with the browser again and allow all four permissions." + "The blyg refused: Burrow's sign-in lacks {named}. Sign in with the browser again and allow every permission." ), None => format!( - "The blyg refused: this token lacks {named}. Make a new one in Studio → More → Client access, for the REST API, with all four permissions (read, draft, publish, manage)." + "The blyg refused: this token lacks {named}. Make a new one in Studio → More → Client access, for the REST API, with {MANUAL_TOKEN_PERMISSIONS}." ), } } @@ -191,6 +224,22 @@ pub struct Discovery { pub revocation_endpoint: Option, /// The REST API's resource identifier (`{origin}/api`). pub resource: String, + /// The `scope` Burrow asks this server for: [`SCOPE`], less any scope its + /// `scopes_supported` leaves out (an unknown one is `invalid_scope`). + pub scope: String, +} + +/// [`SCOPE`] less what a server's `scopes_supported` leaves out; all of it +/// when the server doesn't list them. +pub fn scope_for(supported: Option<&[Value]>) -> String { + match supported { + None => SCOPE.to_string(), + Some(list) => SCOPE + .split(' ') + .filter(|s| list.iter().any(|x| x == s)) + .collect::>() + .join(" "), + } } /// `{blyg-url}/studio/auth/.well-known/openid-configuration`. @@ -240,6 +289,11 @@ pub fn discover(base_url: &str) -> Result> { .map(str::to_string) }) .unwrap_or_else(|| format!("{}/api", super::api_root(base_url))); + let scope = scope_for( + doc.get("scopes_supported") + .and_then(Value::as_array) + .map(Vec::as_slice), + ); Ok(Some(Discovery { issuer, authorization_endpoint, @@ -247,6 +301,7 @@ pub fn discover(base_url: &str) -> Result> { registration_endpoint: registration, revocation_endpoint: s("revocation_endpoint"), resource, + scope, })) } @@ -270,7 +325,7 @@ pub fn register(d: &Discovery) -> Result { "token_endpoint_auth_method": "none", "grant_types": ["authorization_code", "refresh_token"], "response_types": ["code"], - "scope": SCOPE, + "scope": d.scope, }); let v: Value = match agent().post(&d.registration_endpoint).send_json(body) { Ok(r) => r @@ -365,7 +420,7 @@ pub fn authorize_url( q.append_pair("response_type", "code") .append_pair("client_id", client_id) .append_pair("redirect_uri", redirect) - .append_pair("scope", SCOPE) + .append_pair("scope", &d.scope) .append_pair("state", state) .append_pair("code_challenge", challenge) .append_pair("code_challenge_method", "S256") @@ -902,6 +957,7 @@ mod tests { registration_endpoint: "https://blyg.example.com/studio/auth/oauth2/register".into(), revocation_endpoint: None, resource: "https://blyg.example.com/api".into(), + scope: SCOPE.into(), }; let u = authorize_url(&d, "cid", &redirect_uri(47811), "chal", "st", None).unwrap(); let u = url::Url::parse(&u).unwrap(); @@ -916,7 +972,7 @@ mod tests { assert_eq!(q["resource"], "https://blyg.example.com/api"); assert_eq!( q["scope"], - "owner:read owner:draft owner:publish owner:manage offline_access" + "owner:read owner:draft owner:publish owner:manage reading:state offline_access" ); assert!(!q.contains_key("prompt")); assert_eq!( @@ -925,6 +981,35 @@ mod tests { ); } + #[test] + fn scope_lists_every_owner_scope_once() { + assert_eq!(READ_STATE_SCOPE, "reading:state"); + assert_eq!(SCOPE, format!("{} offline_access", OWNER_SCOPES.join(" "))); + assert!(MANUAL_TOKEN_PERMISSIONS.contains(READ_STATE_SCOPE)); + } + + #[test] + fn asks_only_for_the_scopes_a_server_knows() { + // Studio 0.39+ lists reading:state; an older one doesn't, and would + // refuse an authorize or registration that asks for it. + let new: Vec = ["openid", "offline_access"] + .iter() + .chain(OWNER_SCOPES.iter()) + .map(|s| json!(s)) + .collect(); + assert_eq!(scope_for(Some(&new)), SCOPE); + let old: Vec = new + .iter() + .filter(|v| *v != READ_STATE_SCOPE) + .cloned() + .collect(); + assert_eq!( + scope_for(Some(&old)), + "owner:read owner:draft owner:publish owner:manage offline_access" + ); + assert_eq!(scope_for(None), SCOPE); + } + #[test] fn reads_an_insufficient_scope_challenge() { let h = r#"Bearer resource_metadata="https://blyg.example.com/studio/auth/resources/api", scope="owner:draft owner:publish", error="insufficient_scope""#; @@ -945,6 +1030,15 @@ mod tests { let m = missing_scope_message(&["owner:publish".into()], None); assert!(m.contains("Studio → More → Client access"), "{m}"); assert!(m.contains("owner:publish"), "{m}"); + // An older browser sign-in, without the read-state scope. + let m = missing_scope_message( + &[READ_STATE_SCOPE.into()], + Some("owner:read owner:draft owner:publish owner:manage offline_access"), + ); + assert!(m.contains("lacks the reading:state permission"), "{m}"); + assert!(m.contains("Sign in with the browser again"), "{m}"); + let m = missing_scope_message(&[READ_STATE_SCOPE.into()], None); + assert!(m.contains("reading:state where offered"), "{m}"); } #[test] diff --git a/crates/blyg-core/src/sync/mod.rs b/crates/blyg-core/src/sync/mod.rs index aebdb90..d114a54 100644 --- a/crates/blyg-core/src/sync/mod.rs +++ b/crates/blyg-core/src/sync/mod.rs @@ -160,12 +160,29 @@ struct NetState { pushed: HashMap, /// The earliest time a save held back by `SAVE_GAP` may go. paced_until: Option, - /// --- read/unread --- The server refused a read-state write with 403 - /// (a scoped token on a Worker that keeps those writes to the owner): - /// read state stays on this Mac for the rest of the session. + /// --- read/unread --- The server refused a read-state write with 403: + /// a credential without `reading:state` (studio 0.39+; a sign-in made + /// before it), or a scoped token on a Worker that keeps those writes to + /// the owner. Read ops stay queued but aren't sent for the rest of the + /// session (a new sign-in is a new session), and the rest of the outbox + /// goes on without them. read_denied: bool, } +/// What to say, once a session, when the blyg refuses a read-state write. +/// A studio 0.39+ names the scope (`reading:state`) in its challenge, and +/// `e` carries `oauth::missing_scope_message` (sign in again / a new token). +fn read_denied_message(e: &CoreError) -> String { + let why = match e { + CoreError::Rejected { message, .. } if !message.is_empty() => message.as_str(), + _ => "The blyg refused to save read state for this sign-in.", + }; + format!( + "Read state isn't syncing. {why} Until then your read marks are kept on this \ + Mac and wait to be sent; everything else syncs as usual." + ) +} + /// The time allowed per pull for fetching item documents (lineage). const LINEAGE_BUDGET: Duration = Duration::from_secs(8); @@ -451,6 +468,10 @@ impl Engine { let ran = self.run_op(&op); self.set_pushing(false); match ran { + // A read op held for the scope (`read_denied`): not a success, + // so no other 403's notice is reset; the loop goes on to the + // rest of the outbox, which skips the held reads. + Ok(()) if self.read_held(&op) => {} Ok(()) => { if op.kind == OpKind::Save { self.state() @@ -541,6 +562,11 @@ impl Engine { } // Ops for one item run strictly in order: a later save waits for // the create that assigns its server id. + // Read state the credential can't write waits, queued, for one + // that can; it holds nothing else up. + if self.read_held_locked(&st, &op) { + continue; + } if conflicted.contains(&op.local_id) || op.in_flight || (!force && op.not_before > now) { blocked.insert(op.local_id.clone()); @@ -1131,22 +1157,29 @@ impl Engine { // ------------------------------------------------------------ read state /// The server syncs read state (extension 5, or upstream's read-state - /// routes), as of the last reading pull. - /// False for the rest of the session once a write was refused with 403 - /// (`read_denied`): the marks stay here, unsent, and go up from a later - /// session that may write (each keeps its `read_at` until confirmed). + /// routes), as of the last reading pull. Still true once a write was + /// refused with 403 (`read_denied`): marks keep queueing, and wait. pub fn read_sync_on(&self) -> bool { - self.store.meta(READ_SYNC).as_deref() == Some("1") && !self.state().read_denied + self.store.meta(READ_SYNC).as_deref() == Some("1") + } + + /// `op` is a read-state op this session's credential may not send. + fn read_held(&self, op: &Op) -> bool { + self.read_held_locked(&self.state(), op) + } + + fn read_held_locked(&self, st: &NetState, op: &Op) -> bool { + st.read_denied && matches!(op.kind, OpKind::Read | OpKind::Unread) } - /// A read-state write answered 403: stop sending them this session, and - /// say so once. The local marks stay. - fn read_denied(&self) -> Result<()> { + /// A read-state write answered 403 (`e`, which names the missing scope, + /// `reading:state`, when the server says): hold read ops for the rest + /// of the session, queued, and say once how to get them sent. No retry + /// until a new sign-in (a new session) or the next launch. + fn read_denied(&self, e: &CoreError) -> Result<()> { let first = !std::mem::replace(&mut self.state().read_denied, true); if first { - self.emit(CoreEvent::Error( - "Read state stays on this Mac: this sign-in can't save it to your blyg".into(), - )); + self.emit(CoreEvent::Error(read_denied_message(e))); } Ok(()) } @@ -1194,6 +1227,9 @@ impl Engine { fn reconcile_reads(&self, pulled: &[ReadingItem]) -> Result { let clear = self.read_clear_on(); if self.store.meta(READ_SYNC_UPLOADED).is_none() { + if self.state().read_denied { + return Ok(false); + } let marks = self.store.read_marks(clear)?; for chunk in marks.chunks(crate::api::wire::READ_BATCH_MAX) { match self.api.put_reads(chunk) { @@ -1202,8 +1238,9 @@ impl Engine { self.set_read_sync(false)?; return Ok(false); } - Err(CoreError::Rejected { status: 403, .. }) => { - self.read_denied()?; + Err(e @ CoreError::Rejected { status: 403, .. }) => { + // Sent whole by a later session that may write. + self.read_denied(&e)?; return Ok(false); } // Refused outright (a row it doesn't like): the rest @@ -1264,11 +1301,13 @@ impl Engine { } Ok(()) }; - let denied = |this: &Self| -> Result<()> { - // Unsent this session; the rows keep their `read_at` / floor, - // so a later session that may write sends them. - done(this)?; - this.read_denied() + let denied = |this: &Self, e: &CoreError| -> Result<()> { + // Held, not dropped: the batch stays queued for a credential + // that may write it (`read_denied`). + for o in &batch { + this.store.set_in_flight(o.seq, false)?; + } + this.read_denied(e) }; let gone = |this: &Self| -> Result<()> { // The endpoint is gone (a downgraded server). @@ -1295,7 +1334,7 @@ impl Engine { done(self) } Err(CoreError::NotFound) => gone(self), - Err(CoreError::Rejected { status: 403, .. }) => denied(self), + Err(e @ CoreError::Rejected { status: 403, .. }) => denied(self, &e), Err(e) => { for o in &batch[1..] { self.store.set_in_flight(o.seq, false)?; @@ -1340,7 +1379,7 @@ impl Engine { Ok(()) } Err(CoreError::NotFound) => gone(self), - Err(CoreError::Rejected { status: 403, .. }) => denied(self), + Err(e @ CoreError::Rejected { status: 403, .. }) => denied(self, &e), // `read_at` refused (a strict body on a server that said it // could clear): stop sending it and retry without. Err(CoreError::Rejected { @@ -1605,3 +1644,6 @@ impl Engine { self.pull() } } + +#[cfg(test)] +mod read_held_tests; // --- read/unread --- diff --git a/crates/blyg-core/src/sync/read_held_tests.rs b/crates/blyg-core/src/sync/read_held_tests.rs new file mode 100644 index 0000000..8d43436 --- /dev/null +++ b/crates/blyg-core/src/sync/read_held_tests.rs @@ -0,0 +1,140 @@ +//! --- read/unread --- Read state held for the scope (`read_denied`), on +//! real SQLite: the ops stay queued, new marks still queue, and the rest of +//! the outbox goes on without them. (The 403 itself is tested against a +//! real studio: `tests/e2e.rs`, `read_state_needs_its_own_scope`.) + +use std::collections::HashMap; + +use super::*; +use crate::api::auth::Credential; + +fn row(rid: &str) -> ReadingItem { + ReadingItem { + subscription_id: "s1".into(), + remote_id: rid.into(), + subscription_title: "s1".into(), + origin: "https://a.example/".into(), + kind: Kind::Fragment, + state: "current".into(), + version: 1, + created: None, + updated: None, + observed_at: "2026-01-01T00:00:00Z".into(), + content_md: rid.into(), + content_html: String::new(), + author: None, + page: Some(format!("https://a.example/f/{rid}")), + thumb: None, + hoppers: vec![], + pinned_version_retained: None, + read_version: None, + stub_of: None, + forked_from: None, + transclusions: vec![], + lineage_known: false, + } +} + +/// An engine on an in-memory store with read sync on, two reading rows, +/// and a read op queued ahead of a draft's create. Nothing here touches the +/// network (the API points at a closed port). +fn engine() -> Engine { + let store = Store::open_in_memory().unwrap(); + let kinds: HashMap = + [("s1".to_string(), SubscriptionKind::Blyg)].into(); + store + .merge_reading(&[row("A"), row("B")], true, &kinds) + .unwrap(); + store.set_meta(READ_SYNC, "1").unwrap(); + assert_eq!(store.mark_read_rows("s1", "A", true).unwrap().len(), 1); + store.create_draft(Kind::Fragment, "a draft", 0).unwrap(); + let api = Api::new("http://127.0.0.1:9", Credential::Token("t".into())); + Engine::new(store, api, SyncOptions::default()) +} + +fn kinds_queued(e: &Engine) -> Vec { + e.store.ops().unwrap().into_iter().map(|o| o.kind).collect() +} + +#[test] +fn a_held_read_stays_queued_and_lets_the_rest_through() { + let e = engine(); + assert_eq!(kinds_queued(&e), vec![OpKind::Read, OpKind::Create]); + // Before a refusal, the read goes first. + assert_eq!(e.next_op(None, true).unwrap().unwrap().kind, OpKind::Read); + + let refused = CoreError::Rejected { + status: 403, + message: crate::api::oauth::missing_scope_message( + &[crate::api::oauth::READ_STATE_SCOPE.into()], + Some("owner:read owner:draft owner:publish owner:manage offline_access"), + ), + details: vec![], + }; + e.read_denied(&refused).unwrap(); + // The create goes; the read waits, still queued, and isn't picked again. + let next = e.next_op(None, true).unwrap().unwrap(); + assert_eq!(next.kind, OpKind::Create); + assert_eq!(kinds_queued(&e), vec![OpKind::Read, OpKind::Create]); + let read = &e.store.ops().unwrap()[0]; + assert!(e.read_held(read) && !e.read_held(&next)); + e.store.drop_op(next.seq).unwrap(); + assert!(e.next_op(None, true).unwrap().is_none(), "no retry loop"); + + // Marks made meanwhile still queue, for a later sign-in to send. + assert!(e.read_sync_on()); + e.store.mark_read_rows("s1", "B", true).unwrap(); + assert_eq!(kinds_queued(&e), vec![OpKind::Read, OpKind::Read]); + assert!(e.next_op(None, true).unwrap().is_none()); + + // A new session (a new sign-in) sends them. + let again = Engine::new( + Store::open_in_memory().unwrap(), + Api::new("http://127.0.0.1:9", Credential::Token("t".into())), + SyncOptions::default(), + ); + assert!(!again.state().read_denied); +} + +#[test] +fn the_notice_says_to_sign_in_again_and_names_the_scope() { + let refused = CoreError::Rejected { + status: 403, + message: crate::api::oauth::missing_scope_message( + &[crate::api::oauth::READ_STATE_SCOPE.into()], + Some("owner:read owner:draft owner:publish owner:manage offline_access"), + ), + details: vec![], + }; + let m = read_denied_message(&refused); + assert!(m.starts_with("Read state isn't syncing."), "{m}"); + assert!(m.contains("reading:state"), "{m}"); + assert!(m.contains("Sign in with the browser again"), "{m}"); + assert!(m.contains("kept on this Mac"), "{m}"); + // A Worker's plain 403 (no challenge) still reads sensibly. + let m = read_denied_message(&CoreError::Rejected { + status: 403, + message: String::new(), + details: vec![], + }); + assert!(m.contains("refused to save read state"), "{m}"); +} + +#[test] +fn the_notice_is_sent_once_a_session() { + let e = engine(); + let (tx, rx) = std::sync::mpsc::channel(); + e.set_sink(Arc::new(move |ev| { + if let CoreEvent::Error(m) = ev { + let _ = tx.send(m); + } + })); + let refused = CoreError::Rejected { + status: 403, + message: "no".into(), + details: vec![], + }; + e.read_denied(&refused).unwrap(); + e.read_denied(&refused).unwrap(); + assert_eq!(rx.try_iter().count(), 1); +} diff --git a/crates/blyg-core/tests/e2e.rs b/crates/blyg-core/tests/e2e.rs index c03e25c..8c30f24 100644 --- a/crates/blyg-core/tests/e2e.rs +++ b/crates/blyg-core/tests/e2e.rs @@ -5,7 +5,7 @@ //! //! - `BLYG_E2E_URL` the Worker under test (e.g. `http://127.0.0.1:18787`) //! - `BLYG_E2E_TOKEN` its `BLYG_OWNER_TOKEN`, or on studio 0.28+ a manual -//! token minted with all four owner scopes +//! token minted with every owner scope (`reading:state` too, on 0.39+) //! - `BLYG_E2E_TOKEN_B` the same for `BLYG_E2E_URL_B` (studio 0.28+ only; //! without it, `b` takes `a`'s credential) //! - `BLYG_E2E_OWNER_PASSWORD` its studio password, always: the browser @@ -2105,7 +2105,7 @@ fn owner_allows(base: &str, authorize_url: &str, password: &str) -> String { assert_eq!( scopes, blyg_core::api::oauth::OWNER_SCOPES, - "all four offered" + "every owner scope offered (reading:state: studio 0.39+)" ); let mut form: Vec<(&str, &str)> = vec![("handle", &handle), ("decision", "allow")]; form.extend(scopes.iter().map(|s| ("scope", s.as_str()))); @@ -2393,6 +2393,205 @@ fn a_token_without_a_scope_says_which() { assert_eq!(server_item(&sid)["content_md"], text); } +/// A manual token with `scope`, minted as Studio → More → Client access +/// does; `None` (skipped) on a studio without manual tokens. +fn minted_token(scope: &[&str]) -> Option { + let e = e2e(); + let password = owner_password()?; + let session = auth::login(&e.url, &password).expect("studio sign-in"); + let minted = match agent() + .post(&format!("{}/api/authorizations", e.url)) + .set("cookie", &format!("{}={session}", auth::SESSION_COOKIE)) + .send_json(json!({"name": "burrow-e2e-scoped", "scope": scope, "resource": "api"})) + { + Ok(r) => r.into_json::().unwrap(), + Err(ureq::Error::Status(404, _)) => return None, + Err(e) => panic!("minting a token: {e}"), + }; + Some( + minted["access_token"] + .as_str() + .expect("a token") + .to_string(), + ) +} + +/// The read/unread ops waiting in a closed backend's outbox. +fn queued_reads(dir: &Path) -> usize { + let store = blyg_core::store::Store::open(&dir.join(blyg_core::live::DB_FILE)).unwrap(); + store + .ops() + .unwrap() + .iter() + .filter(|o| { + matches!( + o.kind, + blyg_core::store::OpKind::Read | blyg_core::store::OpKind::Unread + ) + }) + .count() +} + +#[test] +#[ignore = "needs a local Worker: scripts/e2e-local.sh"] +fn read_state_needs_its_own_scope() { + // Studio 0.39 moved the read-state writes from owner:manage to + // reading:state (provisional name). A sign-in from before then holds + // the four owner scopes only: its reads are refused, stay queued, and + // go once a credential with the scope arrives; the rest still syncs. + use blyg_core::api::oauth::READ_STATE_SCOPE; + let e = e2e(); + let api = Api::new(&e.url, e.cred.clone()); + let Some(page) = api.stock_reading(0).unwrap() else { + eprintln!("SKIP: no GET /api/reading"); + return; + }; + if !e.stock || !page.read_sync() { + eprintln!("SKIP: needs a stock studio that keeps read state (0.39+)"); + return; + } + let old_four = ["owner:read", "owner:draft", "owner:publish", "owner:manage"]; + let Some(old) = minted_token(&old_four) else { + eprintln!("SKIP: no manual tokens here"); + return; + }; + let src_url = url_b(); + let (d1, d2, d3) = ( + tempfile::tempdir().unwrap(), + tempfile::tempdir().unwrap(), + tempfile::tempdir().unwrap(), + ); + let source = backend_at(d3.path(), &src_url, cred_b(), false); + let t = tag("scope"); + let (_, a) = published(&source, Kind::Fragment, &format!("{t} A"), None); + let owner_mac = manual(d2.path()); + let sub = owner_mac.subscribe(&src_url, Some("Scope")).unwrap(); + let _unsub = Unsub(&owner_mac, sub.id.clone()); + sync_until_held(&owner_mac, &[&a]); + + // The API names the missing scope, never the token. + let scoped = Api::new(&e.url, Credential::Token(old.clone())); + match scoped.put_read(&sub.id, &a, 1, None) { + Err(CoreError::Rejected { + status: 403, + message, + .. + }) => { + assert!(message.contains(READ_STATE_SCOPE), "{message}"); + assert!(!message.contains(&old), "never the token"); + } + other => panic!("expected a 403 naming {READ_STATE_SCOPE}, got {other:?}"), + } + assert_eq!(server_read_version(&a), Some(Value::Null)); + + // The app on that token: the read is held, the draft still goes. + let b = backend_at(d1.path(), &e.url, Credential::Token(old), false); + let (tx, rx) = std::sync::mpsc::channel(); + b.set_event_sink(Box::new(move |ev| { + if let CoreEvent::Error(m) = ev { + let _ = tx.send(m); + } + })); + sync_until_held(&b, &[&a]); + assert_eq!(b.set_read(&[(sub.id.clone(), a.clone())], true).unwrap(), 1); + let text = tag("goes-anyway"); + let id = b.create_draft(Kind::Fragment, &text).unwrap(); + b.sync_now().expect("other sync keeps working"); + assert_eq!(server_item(&sid_of(&b, &id))["content_md"], text); + assert_eq!(server_read_version(&a), Some(Value::Null), "refused"); + assert!(!reading_item(&b, &a).unwrap().is_unread(), "read here"); + b.sync_now().unwrap(); + b.sync_now().unwrap(); + let notices: Vec = rx.try_iter().collect(); + let about_reads: Vec<&String> = notices + .iter() + .filter(|m| m.contains("Read state")) + .collect(); + assert_eq!(about_reads.len(), 1, "said once: {notices:?}"); + assert!( + about_reads[0].contains(READ_STATE_SCOPE), + "{}", + about_reads[0] + ); + assert!( + about_reads[0].contains("Client access"), + "a manual token: make a new one: {}", + about_reads[0] + ); + drop(b); + assert_eq!(queued_reads(d1.path()), 1, "the read is still queued"); + + // A credential with reading:state (a new sign-in) sends it. + let b = manual(d1.path()); + b.sync_now().unwrap(); + assert_eq!(server_read_version(&a), Some(json!(1))); + drop(b); + assert_eq!(queued_reads(d1.path()), 0); +} + +#[test] +#[ignore = "needs a local Worker: scripts/e2e-local.sh"] +fn signing_in_again_with_a_kept_client_adds_reading_state() { + // An existing user's fix for held read state: sign in again. Burrow 0.10 + // registered (and kept) its client asking for the four owner scopes; + // the studio doesn't hold a client to its registered scope, so the kept + // client is reused and the new grant has reading:state. + use blyg_core::api::oauth::{self, READ_STATE_SCOPE, client_key}; + let e = e2e(); + let Some(d) = oauth::discover(&e.url).unwrap() else { + eprintln!("skipped: this studio has no browser sign-in (older than 0.28)"); + return; + }; + if !d.scope.split(' ').any(|s| s == READ_STATE_SCOPE) { + eprintln!("skipped: this studio has no {READ_STATE_SCOPE} (older than 0.39)"); + return; + } + let r: Value = agent() + .post(&d.registration_endpoint) + .send_json(json!({ + "client_name": oauth::CLIENT_NAME, + "application_type": "native", + "redirect_uris": oauth::registered_redirect_uris(), + "token_endpoint_auth_method": "none", + "grant_types": ["authorization_code", "refresh_token"], + "response_types": ["code"], + "scope": "owner:read owner:draft owner:publish owner:manage offline_access", + })) + .expect("register a 0.10-style client") + .into_json() + .unwrap(); + let old_id = r["client_id"].as_str().unwrap().to_string(); + let mem = config::MemoryTokenStore::default(); + let store: &dyn config::TokenStore = &mem; + store + .set( + &client_key(&e.url), + &json!({"issuer": d.issuer, "client_id": old_id, + "redirect_uris": oauth::registered_redirect_uris()}) + .to_string(), + ) + .unwrap(); + let Some(grant) = signed_in_with_browser(store) else { + return; + }; + assert_eq!(grant.client_id, old_id, "the kept client is reused"); + assert!( + grant.scope.split(' ').any(|s| s == READ_STATE_SCOPE), + "{}", + grant.scope + ); + // And it may write read state (refused for a reading row it doesn't + // know only as `stored: false`, never 403). + let api = Api::new( + &e.url, + Credential::OAuth(oauth::OAuthSession::new(grant.clone())), + ); + match api.put_read("no-such-sub", "no-such-item", 1, None) { + Ok(_) => {} + Err(e) => panic!("a reading:state grant may write read state: {e:?}"), + } +} + // ------------------------------------------------- studio 0.30–0.33 reading /// The source blyg's own name for itself, as a subscribe preview gives it. diff --git a/docs/SERVER.md b/docs/SERVER.md index 605d780..0f2ca8f 100644 --- a/docs/SERVER.md +++ b/docs/SERVER.md @@ -11,8 +11,8 @@ checked against `openapi.json` in its tests (`crates/blyg-core/tests/fixtures/`) studio's consent page, and keeps the grant (a one-hour access token, renewed with a rotating refresh token until the grant ends, at most 30 days after your studio sign-in) in your Keychain. Signing out revokes it. Or make an **API token** in Studio → -More → Client access (REST API, all four permissions: read, draft, publish, manage; -it lasts 30 days) and paste it. A token missing a permission gets a 403 that names +More → Client access (REST API, every permission: read, draft, publish, manage, and +on studio 0.39+ `reading:state`; it lasts 30 days) and paste it. A token missing a permission gets a 403 that names it. On an older studio, or as a fallback, sign in with the **studio password**: Burrow signs in at `{blyg-url}/studio/login` like the browser does, keeps the session cookie in memory only (the password is in your Keychain), and signs in again when the @@ -91,7 +91,16 @@ is deleted. Upstream's change triggers also advance the `reading` revision of The app reads the read version only when `read_state` is `true`. It checks the flags on every pull and remembers the last answer. -**Writes** (owner auth, like every `/api` call): +**Writes** (owner auth, like every `/api` call). **Scope:** since blygger-studio 0.39 +(migration `0026_read_state.sql`) these four writes (`markRead`, `markUnread`, +`markReadBatch`, `markUnreadBatch`) need the bearer scope **`reading:state`**, and +`owner:manage` alone is refused with `403 insufficient_scope`. The name is +**provisional** (upstream may rename it); the app spells it in one place, +`READ_STATE_SCOPE` in `crates/blyg-core/src/api/oauth.rs`. The browser sign-in asks for +it when the studio lists it in `scopes_supported` (an older studio would refuse the whole +request for a scope it doesn't know). A client registered before (Burrow 0.10) is reused; +the studio grants it the new scope. The wire shapes are unchanged: `version` must be ≥ 1 there, and `read_at` is still +accepted (for offline queues), though the studio's own client no longer sends it. | Call | Body | Success | Errors | |---|---|---|---| @@ -136,8 +145,13 @@ endpoints as "this server doesn't keep read state". server later gains `read_state_clear`, those local unreads are sent then. - The first time a database sees the capability, the app batch-uploads every read version it holds (500 per call) and records that in its `meta` table. -- A `403` on a write (a Worker that keeps these writes to the owner's own token) keeps read - state on this Mac for the rest of the session, says so once, and doesn't retry. +- A `403` on a write (a sign-in or token without `reading:state`, such as one made before + studio 0.39, or a Worker that keeps these writes to the owner's own token) holds read + state for the rest of the session: the ops stay queued (nothing is dropped), new marks + keep queueing, and the rest of the outbox syncs as usual. The app says so once, with the + server's reason: sign in with the browser again (or make a new token with + `reading:state`). It doesn't retry until the next sign-in or launch, which sends what + waited. - **Without the capability**, read state stays on this Mac. The app makes no requests to these endpoints and shows no errors. A `404` from them turns sync off and drops anything queued. diff --git a/docs/SPEC.md b/docs/SPEC.md index af0eab4..c039117 100644 --- a/docs/SPEC.md +++ b/docs/SPEC.md @@ -199,7 +199,8 @@ uses are in `docs/SERVER.md`. The key points: `read_state: true` and a per-item `read_version: number|null`; `PUT /api/reading/:sub/:remoteId/read {version}` and `POST /api/reading/read {items: [{sub, remote_id, version}]}` (≤ 500) store -`max(existing, version)`. +`max(existing, version)`. On blygger-studio 0.39+ the read-state writes need the scope +`reading:state` (a provisional name; `oauth::READ_STATE_SCOPE`), not `owner:manage`. Transclusion syntax is `![[<26-char id>]]` alone on its own line, and is only valid in threads. See upstream `worker/src/transclusion.ts` for which ids resolve @@ -228,7 +229,7 @@ valid in threads. See upstream `worker/src/transclusion.ts` for which ids resolv **Posts are dated by their author, not by the import.** The list sorts newest first by the post's own `updated` (else `created`), so an author's edit moves the post up; a row says "edited 3d ago" when `updated` is more than ten minutes after `created`, else shows when it was published. `observed_at` (when your blyg imported it) is only the fallback for a post with no usable date. Dates parse as RFC 3339 or RFC 2822 (`ReadingItem::post_time`); the store keeps the normalized sort key in `reading.sort_at` (indexed). -**Read state syncs through your blyg when the server supports it.** Read state is a number per reading row: the highest version you've read. Marking a post read is always instant and local, and it marks every duplicate of the same post. When the blyg advertises read-state sync (extension 5, `docs/SERVER.md`), the same mark also queues a `read` op per row in the outbox, which is sent like any other change (offline-safe, retried, coalesced to the highest version per row). A pull merges `max(local, server)`, so read state never goes backwards. The first time a database sees the capability, it uploads everything it has read in batches, once, and records that in `meta`. So a post read on one Mac reads as read on your others, and a fresh install isn't all unread. Without the extension, read state stays on this Mac, and nothing is sent. Nothing new is shown in the UI. +**Read state syncs through your blyg when the server supports it.** Read state is a number per reading row: the highest version you've read. Marking a post read is always instant and local, and it marks every duplicate of the same post. When the blyg advertises read-state sync (extension 5, `docs/SERVER.md`), the same mark also queues a `read` op per row in the outbox, which is sent like any other change (offline-safe, retried, coalesced to the highest version per row). A pull merges `max(local, server)`, so read state never goes backwards. The first time a database sees the capability, it uploads everything it has read in batches, once, and records that in `meta`. So a post read on one Mac reads as read on your others, and a fresh install isn't all unread. Without the extension, read state stays on this Mac, and nothing is sent. Nothing new is shown in the UI. The writes need the bearer scope `reading:state` (studio 0.39+; the name is provisional, kept in `oauth::READ_STATE_SCOPE`), which the browser sign-in asks for. A sign-in or token without it (one made before 0.39) gets a 403: the read ops stay queued, the rest of the outbox syncs, and Burrow says once that read state is waiting and to sign in again; the next sign-in sends what waited. **Search, Notational Velocity style.** A search field sits above the reading list (the Posts omnibar isn't on this screen). ⌘F or `/` (with the list focused) puts the caret in it; typing filters the list live: a case-insensitive substring over the title, the author and blyg (subscription, origin) names, and the post's text (`content_md`, or the published HTML's text when an item has no Markdown), over posts **already held locally**. Nothing is fetched to search. Title matches are highlighted as in the Posts list. ↑/↓ move through the matches from the field, ⏎ goes to the list (opening the first match when nothing is open), and esc clears the search (esc on an empty field returns to the list; esc in the list with a search clears it before it leaves the screen). An empty result shows "No posts match “…”". The open post stays open while it still matches; otherwise the selection and the reader clear, so typing never opens (or marks read) anything. diff --git a/scripts/e2e-wrangler.sh b/scripts/e2e-wrangler.sh index aa2df2b..2f1c434 100755 --- a/scripts/e2e-wrangler.sh +++ b/scripts/e2e-wrangler.sh @@ -6,7 +6,8 @@ # e2e-wrangler.sh setup derive the scratch config + secrets # e2e-wrangler.sh token | password print a scratch dev secret # e2e-wrangler.sh mint-token sign in once and mint a manual API -# token with all four owner scopes +# token with every owner scope, and +# reading:state on studio 0.39+ # (studio 0.28+); prints it, or nothing # on an older studio # e2e-wrangler.sh start [KEY=value…] @@ -110,9 +111,18 @@ mint_token() { jar="$BLYG_E2E_SCRATCH/cookies.$port" curl -fsS -o /dev/null -c "$jar" --data-urlencode "password=$(password)" "$base/studio/login" \ || die "signing in to :$port failed" + # reading:state (studio 0.39+; the name is provisional, see READ_STATE_SCOPE + # in crates/blyg-core/src/api/oauth.rs): an older studio refuses a scope it + # doesn't know with a 400, so it gets the four owner scopes. + local four='"owner:read","owner:draft","owner:publish","owner:manage"' out="$(curl -sS -b "$jar" -H 'content-type: application/json' -w '\n%{http_code}' \ - -d '{"name":"burrow-e2e","scope":["owner:read","owner:draft","owner:publish","owner:manage"],"resource":"api"}' \ + -d "{\"name\":\"burrow-e2e\",\"scope\":[$four,\"reading:state\"],\"resource\":\"api\"}" \ "$base/api/authorizations")" + if [ "${out##*$'\n'}" = 400 ]; then + out="$(curl -sS -b "$jar" -H 'content-type: application/json' -w '\n%{http_code}' \ + -d "{\"name\":\"burrow-e2e\",\"scope\":[$four],\"resource\":\"api\"}" \ + "$base/api/authorizations")" + fi rm -f "$jar" case "${out##*$'\n'}" in 200) printf '%s' "${out%$'\n'*}" | node -e 'let s="";process.stdin.on("data",d=>s+=d).on("end",()=>process.stdout.write(JSON.parse(s).access_token))' ;; diff --git a/site/src/getting-started.md b/site/src/getting-started.md index 8abc879..357363f 100644 --- a/site/src/getting-started.md +++ b/site/src/getting-started.md @@ -41,8 +41,9 @@ in: if you revoke it in the studio (**More › Client access**), it asks you to sign in again. - **API token**: make one in your studio under **More › Client access** - (REST API, with all four permissions) and paste it. A server with Burrow's - extensions also takes its `BLYG_OWNER_TOKEN` here. + (REST API, with every permission, including reading:state on studio 0.39+) + and paste it. A server with Burrow's extensions also takes its + `BLYG_OWNER_TOKEN` here. - **Studio password**, for a studio older than 0.28. It checks them with the server before saving anything, and says plainly what's From 16a40d36ab6ea1d4dd792c77d7de0820820c0cc0 Mon Sep 17 00:00:00 2001 From: Aneesh Sathe Date: Fri, 9 Oct 2026 21:55:06 -0700 Subject: [PATCH 04/16] markdown-notes: several vaults, no default folder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each vault is its own library: vault= stays the first (existing configs read as before), vault-