Label grammar: six edge cases from 0.47.0 (#70)

src/authoring.ts

@@ -71,9 +71,21 @@ export function scopeSummaries(scopes: TkScope[]): { index: number; instruction:
  * and lets the caller render the remaining markdown (plain, or via
  * previewTransclusions for threads).
  */
-export function annotateTkPreview(contentMd: string): { scopes: TkScope[]; text: string; blocks: Map<string, string>; finish: (renderedHtml: string) => string } {
+export function annotateTkPreview(contentMd: string): {
+  scopes: TkScope[];
+  text: string;
+  blockReplacements: Map<string, string>;
+  blockSources: Map<string, string>;
+  finish: (renderedHtml: string) => string;
+} {
   const { scopes } = parseScopes(contentMd);
   const { text, spans } = previewStrip(contentMd, scopes);
   const annotated = annotateGenerated(text, spans, spans.map(() => true));
-  return { scopes, text: annotated.text, blocks: annotated.blockReplacements, finish: (renderedHtml) => applyGeneratedWrappers(renderedHtml, annotated) };
+  return {
+    scopes,
+    text: annotated.text,
+    blockReplacements: annotated.blockReplacements,
+    blockSources: annotated.blockSources,
+    finish: (renderedHtml) => applyGeneratedWrappers(renderedHtml, annotated),
+  };
 }

src/code-ranges.ts

@@ -45,11 +45,6 @@ export function codeRanges(text: string): Range[] {
   return ranges;
 }
 
-/** Code regions of already-rendered HTML: every `<code>…</code>` (inline, or inside `<pre>`). */
-export function htmlCodeRanges(html: string): Range[] {
-  return [...html.matchAll(/<code[\s>][\s\S]*?<\/code>/gi)].map((m): Range => [m.index!, m.index! + m[0].length]);
-}
-
 export function inRanges(ranges: Range[], pos: number): boolean {
   return ranges.some(([s, e]) => pos >= s && pos < e);
 }

src/grammar.ts

@@ -42,8 +42,12 @@ export const linkInline = () => new RegExp(LINK_INLINE.source, "g");
 /** A directive anywhere in a line (TK source refs): id in group 1, raw label in group 2. */
 export const directiveInline = () => new RegExp(`!\\[\\[${BODY}`, "g");
 
-/** A directive alone on its line, multiline (TK output): id in group 1, raw label in group 2. */
-export const directiveOwnLine = () => new RegExp(`^[ \\t]*!\\[\\[${BODY}[ \\t]*$`, "gm");
+/**
+ * A directive alone on its line, multiline (TK output): id in group 1, raw label in group 2.
+ * `[^\S\n]` is `DIRECTIVE_LINE`'s `\s` less the newline that ends a line, so
+ * every line publish transcludes is one this finds (decision #60).
+ */
+export const directiveOwnLine = () => new RegExp(`^[^\\S\\n]*!\\[\\[${BODY}[^\\S\\n]*$`, "gm");
 
 /** The label a match carries, trimmed; undefined when absent or empty. */
 export function label(raw: string | undefined): string | undefined {
@@ -53,10 +57,12 @@ export function label(raw: string | undefined): string | undefined {
 
 /**
  * Text made safe to write as a label: one line, no `]]` (the grammar has no
- * escape, so §10.1 says a label that would contain one is rewritten).
+ * escape, so §10.1 says a label that would contain one is rewritten). Every
+ * `]` followed by another, or by the `]]` that will close the link, takes a
+ * space after it; one pass over `]]` would leave `a]]]b` reading `a] ]]`.
  */
 export function labelText(text: string): string {
-  return text.replace(/\s+/g, " ").replace(/\]\]/g, "] ]").trim();
+  return text.replace(/\s+/g, " ").trim().replace(/\](?=\]|$)/g, "] ");
 }
 
 /**

src/model.ts

@@ -5,7 +5,6 @@ import { parseCustomTheme } from "./themes.ts";
 // Data access + publish-flow semantics — v0.1-plan §3.1.
 
 import { renderMarkdown } from "./markdown.ts";
-import { htmlCodeRanges } from "./code-ranges.ts";
 import { annotateGenerated, applyGeneratedWrappers, parseScopes, stripToOutput, TkPublishError, unresolvedScopes } from "./tk.ts";
 import { recordPublishInteractions } from "./interactions.ts";
 import { applyVersionAgreement, composeStubCite, composeTransclusionCite, parseStoredStub } from "./stub.ts";
@@ -392,16 +391,15 @@ export async function publish(db: D1Database, item: ItemRow, note: string | null
   //
   // Run over the body *and* over each independently-rendered generated block,
   // because annotateGenerated has already lifted those blocks out of the body
-  // text: a link inside one would otherwise stay literal. Their raw text is
-  // gone by now, but the rendered HTML still contains the un-substituted
-  // `[[id]]` verbatim — no character of the grammar is HTML-special — so the
-  // same resolver works on either input. Token sequence is global, so the maps
-  // merge.
+  // text: a link inside one would otherwise stay literal. Each block is
+  // resolved over its own markdown and rendered again, never over its HTML,
+  // where a label has already been through markdown and escaping once.
+  // Token sequence is global, so the maps merge.
   const linkErrors: TransclusionRefError[] = [];
   const links = await resolveInternalLinks(db, annotated.text, normalizedOrigin(origin));
   linkErrors.push(...links.errors);
-  const { docs: blockLinks, errors: blockErrors } = await resolveBlockLinks(annotated.blockReplacements, (html) =>
-    resolveInternalLinks(db, html, normalizedOrigin(origin), undefined, htmlCodeRanges),
+  const { docs: blockLinks, errors: blockErrors } = await resolveBlockLinks(annotated, (md) =>
+    resolveInternalLinks(db, md, normalizedOrigin(origin)),
   );
   linkErrors.push(...blockErrors);
   if (linkErrors.length) throw new TransclusionResolveError(linkErrors);

src/read-api.ts

@@ -179,7 +179,7 @@ readApi.openapi(routes.preview, async (c) => {
   const origin = siteOrigin(await getSettings(c.env.DB), c.req.url, mount);
   const links = await previewInternalLinks(c.env.DB, tk.text, origin);
   // Links inside generated blocks resolve here too, as at publish (studio#14).
-  const blocks = await resolveBlockLinks(tk.blocks, (html) => previewInternalLinks(c.env.DB, html, origin, true));
+  const blocks = await resolveBlockLinks(tk, (md) => previewInternalLinks(c.env.DB, md, origin));
   const html = [links, ...blocks.docs].reduce((acc, doc) => applyInternalLinks(acc, doc), tk.finish(renderMarkdown(links.text)));
   return c.json({ html, scopes: await scopeSummariesWithSources(c.env.DB, tk.scopes), link_errors: [...links.errors, ...blocks.errors] });
 });
@@ -285,7 +285,7 @@ async function threadPreview(c: Context<{ Bindings: Env }>) {
   const mount = normalizeMount(c.env.MOUNT);
   const origin = siteOrigin(await getSettings(c.env.DB), c.req.url, mount);
   const links = await previewInternalLinks(c.env.DB, tk.text, origin);
-  const blocks = await resolveBlockLinks(tk.blocks, (html) => previewInternalLinks(c.env.DB, html, origin, true));
+  const blocks = await resolveBlockLinks(tk, (md) => previewInternalLinks(c.env.DB, md, origin));
   const resolved = await previewTransclusions(c.env.DB, links.text, body.item_id);
   return c.json({
     html: [links, ...blocks.docs].reduce((acc, doc) => applyInternalLinks(acc, doc), tk.finish(resolved.html)),

src/tk.ts

@@ -292,6 +292,12 @@ export interface AnnotatedDocument {
   text: string;
   /** Unique per-span token -> its independently-rendered `<div class="blyg-tk-gen">` replacement. */
   blockReplacements: Map<string, string>;
+  /**
+   * The same tokens -> each block's markdown, so `[[id]]` inside a block can be
+   * resolved over the text the author wrote rather than over its HTML, where
+   * markdown has already been applied to a label (`*em*` is `<em>` by then).
+   */
+  blockSources: Map<string, string>;
   hasInline: boolean;
 }
 
@@ -305,6 +311,7 @@ export function annotateGenerated(strippedMd: string, spans: GeneratedSpan[], ha
   let out = "";
   let last = 0;
   const blockReplacements = new Map<string, string>();
+  const blockSources = new Map<string, string>();
   let hasInline = false;
   spans.forEach((span, i) => {
     out += strippedMd.slice(last, span.start);
@@ -313,7 +320,8 @@ export function annotateGenerated(strippedMd: string, spans: GeneratedSpan[], ha
       out += text;
     } else if (span.block) {
       const token = `${BLOCK_SENTINEL}${i}${BLOCK_SENTINEL}`;
-      blockReplacements.set(token, `<div class="blyg-tk-gen">${renderMarkdown(text)}</div>`);
+      blockReplacements.set(token, renderGeneratedBlock(text));
+      blockSources.set(token, text);
       out += token;
     } else {
       hasInline = true;
@@ -322,7 +330,12 @@ export function annotateGenerated(strippedMd: string, spans: GeneratedSpan[], ha
     last = span.end;
   });
   out += strippedMd.slice(last);
-  return { text: out, blockReplacements, hasInline };
+  return { text: out, blockReplacements, blockSources, hasInline };
+}
+
+/** One generated block span rendered on its own, in its disclosure wrapper. */
+export function renderGeneratedBlock(md: string): string {
+  return `<div class="blyg-tk-gen">${renderMarkdown(md)}</div>`;
 }
 
 /** Splice annotateGenerated()'s placeholders/sentinels into already-rendered HTML. */

src/transclusion.ts

@@ -2,14 +2,15 @@
 // locked protocol surface: do not modify without Fable + Venkat.
 
 import { attachedQuote } from "./directives.ts";
-import { codeRanges, htmlCodeRanges, inRanges, lineOffsets, type Range } from "./code-ranges.ts";
+import { codeRanges, inRanges, lineOffsets } from "./code-ranges.ts";
 import { sanitizeHtml } from "./importer/sanitize.ts";
 import { blygItemUrl } from "./importer/util.ts";
 import { renderMarkdown, selectionText } from "./markdown.ts";
 import type { ImportedItemRow, ItemRow, TextQuoteSelector, Transclusion, VersionRow } from "./types.ts";
 import { absolutizeHtml, escapeHtml } from "./util.ts";
 import { DIRECTIVE_LINE, label, LINK_INLINE, RESERVED_LINE } from "./grammar.ts";
 import { linkLabel } from "./link-label.ts";
+import { renderGeneratedBlock } from "./tk.ts";
 
 /**
  * `[[id]]` — a plain internal link (§10.1, decision #32). The same bracket
@@ -430,24 +431,39 @@ function anchorText(target: ResolvedTarget): string {
  *
  * `ourOrigin` must be absolute: `content_html` travels to subscribers, where a
  * relative href would resolve against *their* origin and point at nothing.
+ *
+ * `contentMd` is always markdown, never rendered HTML: a label is the author's
+ * plain text, and HTML has already had markdown applied to it (`*em*` is
+ * `<em>` by then) and its `<`/`&` escaped. Generated blocks are resolved over
+ * their own markdown for that reason — see `resolveBlockLinks`.
  */
 export async function resolveInternalLinks(
   db: D1Database,
   contentMd: string,
   ourOrigin: string,
   onError: (err: TransclusionRefError) => string | null = () => null,
-  inert: (text: string) => Range[] = codeRanges,
 ): Promise<InternalLinkDocument> {
   const errors: TransclusionRefError[] = [];
   const replacements = new Map<string, string>();
   const labels = new Map<string, { text: string; literal: string }>();
   let out = "";
   let last = 0;
 
-  const code = inert(contentMd);
+  // A label is text, not markdown, so a backtick in one must not open a code
+  // span that swallows a later link. Each live link's label is blanked (same
+  // length, so offsets hold) and code found again. Left to right, so a span
+  // opened *before* a link still makes it inert, as markdown's own precedence
+  // has it (§10.1, #54).
+  let scanned = contentMd;
+  let code = codeRanges(scanned);
   for (const m of contentMd.matchAll(LINK_INLINE)) {
     const at = m.index ?? 0;
     if (inRanges(code, at)) continue; // inert inside code (§10.1, #54)
+    if (m[2]?.includes("`")) {
+      const end = at + m[0].length - 2; // the label runs up to the closing `]]`
+      scanned = scanned.slice(0, end - m[2].length) + " ".repeat(m[2].length) + scanned.slice(end);
+      code = codeRanges(scanned);
+    }
     out += contentMd.slice(last, at);
     last = at + m[0].length;
     const id = m[1];
@@ -498,10 +514,13 @@ const ENCODED_TOKEN = new RegExp(`${ENCODED_SENTINEL}(\\d+)${ENCODED_SENTINEL}`,
  */
 export function applyInternalLinks(html: string, doc: InternalLinkDocument): string {
   const label = (seq: string) => doc.labels.get(`${LINK_SENTINEL}${seq}${LINK_SENTINEL}`);
-  // 1. Tokens percent-encoded into attribute values: the author's literal, encoded the way the URL around it was.
+  // 1. Tokens percent-encoded into attribute values: the author's literal,
+  // encoded the way the URL around it was, and escaped like any attribute
+  // value — `encodeURI` keeps `&`, which a browser would read as an entity.
+  const inHref = (literal: string) => escapeHtml(encodeURI(literal));
   let out = html.replace(ENCODED_TOKEN, (whole, seq: string) => {
     const l = label(seq);
-    return l ? encodeURI(l.literal) : whole;
+    return l ? inHref(l.literal) : whole;
   });
   // 2. Walk tags and text, tracking whether we are inside an anchor or code.
   const parts = out.split(/(<[^>]*>)/);
@@ -523,7 +542,7 @@ export function applyInternalLinks(html: string, doc: InternalLinkDocument): str
       if (isTag) return escapeHtml(l.literal);
       if (inCode) return escapeHtml(l.literal);
       // An autolink's text is its URL: keep it reading as the URL it links to.
-      if (inAnchor) return anchorTag.includes(encodeURI(l.literal)) ? escapeHtml(l.literal) : l.text;
+      if (inAnchor) return anchorTag.includes(inHref(l.literal)) ? escapeHtml(l.literal) : l.text;
       return doc.replacements.get(token) ?? whole;
     });
     if (isTag && /^<a[\s>]/i.test(part)) { inAnchor++; anchorTag = parts[i]; }
@@ -536,18 +555,22 @@ export function applyInternalLinks(html: string, doc: InternalLinkDocument): str
  * block span is lifted out of the body and rendered on its own, so the body's
  * pass never sees it). Publish and preview both call this, so the preview shows
  * the links readers will get, and an unresolvable one is an error in both
- * places (studio#14). The input is rendered HTML, so code is `<code>`.
+ * places (studio#14).
+ *
+ * Resolved over each block's markdown, as the body is, and the block rendered
+ * again from the substituted text: resolving over its HTML would take a label
+ * that markdown had already rendered and escaped, and escape it a second time.
  */
 export async function resolveBlockLinks(
-  blocks: Map<string, string>,
-  resolve: (html: string) => Promise<InternalLinkDocument>,
+  blocks: { blockReplacements: Map<string, string>; blockSources: Map<string, string> },
+  resolve: (md: string) => Promise<InternalLinkDocument>,
 ): Promise<{ docs: InternalLinkDocument[]; errors: TransclusionRefError[] }> {
   const docs: InternalLinkDocument[] = [];
   const errors: TransclusionRefError[] = [];
-  for (const [token, blockHtml] of blocks) {
-    const doc = await resolve(blockHtml);
+  for (const [token, blockMd] of blocks.blockSources) {
+    const doc = await resolve(blockMd);
     if (doc.replacements.size || doc.errors.length) {
-      blocks.set(token, doc.text);
+      blocks.blockReplacements.set(token, renderGeneratedBlock(doc.text));
       errors.push(...doc.errors);
       docs.push(doc);
     }
@@ -560,12 +583,11 @@ export async function resolveBlockLinks(
  * the composer shows what publish will reject before the author hits publish —
  * the same contract as `previewTransclusions`.
  */
-export async function previewInternalLinks(db: D1Database, contentMd: string, ourOrigin: string, html = false): Promise<InternalLinkDocument> {
+export async function previewInternalLinks(db: D1Database, contentMd: string, ourOrigin: string): Promise<InternalLinkDocument> {
   return resolveInternalLinks(
     db,
     contentMd,
     ourOrigin,
     (err) => `<span class="blyg-link-unresolved" style="color:#b3412b">⚠ ${escapeHtml(err.reason)}</span>`,
-    html ? htmlCodeRanges : codeRanges,
   );
 }

src/ui/stub-quote.ts

@@ -15,7 +15,11 @@ import { attachedQuote } from '../directives.ts';
 import { normalizeSelection, quoteLines } from '../markdown.ts';
 
 
-/** Where the stub's quote of `id` is: its directive line and the line after its attached quote. */
+/**
+ * Where the stub's quote of `id` is: its directive line (index, and the line
+ * as written, so a rewrite keeps its label — decision #70) and the line after
+ * its attached quote.
+ */
 function findQuote(md: string, id: string) {
   const lines = md.split('\n');
   const inert = codeRanges(md);
@@ -24,7 +28,7 @@ function findQuote(md: string, id: string) {
     if (inRanges(inert, starts[i])) continue;
     if (DIRECTIVE_LINE.exec(lines[i])?.[1] !== id) continue;
     const { quoteMd, next } = attachedQuote(lines, i);
-    return { lines, at: i, next, partial: quoteMd !== null };
+    return { lines, at: i, directive: lines[i], next, partial: quoteMd !== null };
   }
   return null;
 }
@@ -78,8 +82,9 @@ export function addStubQuote(md: string, id: string, passage: string, at: number
  */
 export function withStubQuote(md: string, id: string, passage: string | null): string {
   const selection = passage === null ? '' : normalizeSelection(passage);
-  const block = [`![[${id}]]`, ...(selection ? [quoteLines(selection)] : [])].join('\n');
   const found = findQuote(md, id);
+  const directive = found?.directive ?? `![[${id}]]`;
+  const block = [directive, ...(selection ? [quoteLines(selection)] : [])].join('\n');
   if (!found) return `${block}\n\n${md.replace(/^\n+/, '')}`;
   const { lines, at, next } = found;
   return [...lines.slice(0, at), block, ...lines.slice(next)].join('\n');