From b6e07df8101f47daee374cab5d17f71b99f95162 Mon Sep 17 00:00:00 2001 From: EdwardLien <73685079+EdwardLien0426@users.noreply.github.com> Date: Thu, 16 Jul 2026 15:10:39 -0700 Subject: [PATCH 1/9] docs: add robots.txt allowing AI crawlers Added robots.txt to allow various AI crawlers and declare sitemap. --- packages/docs/static/robots.txt | 37 +++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 packages/docs/static/robots.txt diff --git a/packages/docs/static/robots.txt b/packages/docs/static/robots.txt new file mode 100644 index 000000000..007115b20 --- /dev/null +++ b/packages/docs/static/robots.txt @@ -0,0 +1,37 @@ +# robots.txt for docs.rocketride.org +# Explicitly welcomes AI crawlers + declares sitemap. + +User-agent: GPTBot +Allow: / + +User-agent: OAI-SearchBot +Allow: / + +User-agent: ChatGPT-User +Allow: / + +User-agent: ClaudeBot +Allow: / + +User-agent: Claude-User +Allow: / + +User-agent: PerplexityBot +Allow: / + +User-agent: Perplexity-User +Allow: / + +User-agent: Google-Extended +Allow: / + +User-agent: CCBot +Allow: / + +User-agent: Bingbot +Allow: / + +User-agent: * +Allow: / + +Sitemap: https://docs.rocketride.org/sitemap.xml From bc392160eaf6e53b1a7dce8cf4376bbdcf587b5a Mon Sep 17 00:00:00 2001 From: EdwardLien <73685079+EdwardLien0426@users.noreply.github.com> Date: Thu, 16 Jul 2026 15:11:33 -0700 Subject: [PATCH 2/9] docs: keep committed robots.txt out of static/ gitignore Updated .gitignore to include robots.txt in committed assets. --- packages/docs/.gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/docs/.gitignore b/packages/docs/.gitignore index c6f7fe146..52e2f6ba2 100644 --- a/packages/docs/.gitignore +++ b/packages/docs/.gitignore @@ -6,10 +6,11 @@ /content # Generated static artifacts (.md siblings, llms.txt, llms-full.txt). -# Keep committed assets (img/) and the placeholder. +# Keep committed assets (img/, robots.txt) and the placeholder. /static/* !/static/.gitkeep !/static/img/ +!/static/robots.txt node_modules *.log From 2b765c0647d1579bcd048fe6060139f0d06ba198 Mon Sep 17 00:00:00 2001 From: EdwardLien <73685079+EdwardLien0426@users.noreply.github.com> Date: Thu, 16 Jul 2026 15:15:46 -0700 Subject: [PATCH 3/9] docs: emit sitemap lastmod (showLastUpdateTime + sitemap options) --- packages/docs/docusaurus.config.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/docs/docusaurus.config.ts b/packages/docs/docusaurus.config.ts index ee5167bd6..ddc3bd96b 100644 --- a/packages/docs/docusaurus.config.ts +++ b/packages/docs/docusaurus.config.ts @@ -61,8 +61,14 @@ const config: Config = { path: contentPath, routeBasePath: '/', sidebarPath: './sidebars.ts', + showLastUpdateTime: true, }, blog: false, + sitemap: { + lastmod: 'date', + changefreq: null, + priority: null, + }, theme: { customCss: './src/css/custom.css', }, From 9e2bafe3e6b82efd97d77337fb623e5a3a455575 Mon Sep 17 00:00:00 2001 From: EdwardLien <73685079+EdwardLien0426@users.noreply.github.com> Date: Thu, 16 Jul 2026 15:18:59 -0700 Subject: [PATCH 4/9] docs: stamp last_update from source git dates in gather Updated the clearGeneratedStatic function to keep committed files and added functionality to stamp last update dates on staged pages. --- packages/docs/scripts/lib/gather.js | 63 ++++++++++++++++++++++++++--- 1 file changed, 57 insertions(+), 6 deletions(-) diff --git a/packages/docs/scripts/lib/gather.js b/packages/docs/scripts/lib/gather.js index 1d66aa816..515404803 100644 --- a/packages/docs/scripts/lib/gather.js +++ b/packages/docs/scripts/lib/gather.js @@ -261,15 +261,64 @@ function discoverContributors() { return contributors; } -/** Remove generated artifacts from static/ (keep .gitkeep and img/). */ +/** Remove generated artifacts from static/ (keep .gitkeep, img/, and committed files). */ async function clearGeneratedStatic(staticDir) { if (!(await exists(staticDir))) return; + const KEEP = new Set(['.gitkeep', 'img', 'robots.txt']); for (const name of await readDir(staticDir)) { - if (name === '.gitkeep' || name === 'img') continue; + if (KEEP.has(name)) continue; await rm(path.join(staticDir, name)); } } +// --- last_update stamping ---------------------------------------------------- +// The assembled content tree under BUILD_ROOT is not git-tracked, so Docusaurus +// cannot infer page dates there (`showLastUpdateTime` finds nothing and the +// sitemap emits no ). Stamp each staged page's front matter with the +// SOURCE file's last git commit date instead — Docusaurus prefers a +// `last_update` front-matter entry over git when present. +const { execFileSync } = require('node:child_process'); +const _gitDateCache = new Map(); + +/** + * Last commit date (YYYY-MM-DD) of a source file, or null outside a git + * checkout / for untracked files. Cached per path. Requires git history at + * build time — a shallow CI clone collapses every date to the clone day, so + * the docs job should use fetch-depth: 0. + * @param {string} srcAbs - absolute path of the git-tracked source file. + * @return {string|null} + */ +function gitLastUpdate(srcAbs) { + if (_gitDateCache.has(srcAbs)) return _gitDateCache.get(srcAbs); + let date = null; + try { + date = execFileSync('git', ['log', '-1', '--format=%cs', '--', srcAbs], { + cwd: path.dirname(srcAbs), encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], + }).trim() || null; + } catch { + /* not a git checkout — leave null; the page simply carries no date */ + } + _gitDateCache.set(srcAbs, date); + return date; +} + +/** + * Merge `last_update: {date}` into a page's front matter (creating the block + * when absent). No-op when date is null or the page already declares one. + * @param {string} content - staged page content (md/mdx). + * @param {string|null} date - YYYY-MM-DD from gitLastUpdate(). + * @return {string} + */ +function stampLastUpdate(content, date) { + if (!date || /(^|\n)last_update\s*:/.test(content)) return content; + if (content.startsWith('---\n')) { + const end = content.indexOf('\n---', 4); + if (end !== -1) return `${content.slice(0, end)}\nlast_update:\n date: ${date}${content.slice(end)}`; + return content; + } + return `---\nlast_update:\n date: ${date}\n---\n\n${content}`; +} + /** * Stage one doc file into the assembled content tree and write its raw pre-MDX * sibling for the LLM surface. @@ -291,7 +340,9 @@ async function stageFile({ srcAbs, destAbs, siblingAbs, content, mode }) { await copyFile(srcAbs, destAbs); // Windows symlink may need privilege } } else { - await copyFile(srcAbs, destAbs); + // Real write instead of a byte copy so the staged page carries the source + // file's git date (the assembled content tree itself is not git-tracked). + await writeFileEnsure(destAbs, stampLastUpdate(content, gitLastUpdate(srcAbs))); } // Raw pre-MDX sibling for the LLM surface. await writeFileEnsure(siblingAbs, content); @@ -393,7 +444,7 @@ async function gather({ projectRoot, contentStaticDir, contentDir, staticDir, mo // index so the parent label is clickable. const folderRel = toPosix(path.join(NODES_DIR, cat.slug, name)); await writeFileEnsure(path.join(contentDir, folderRel, '_category_.json'), JSON.stringify({ label, collapsed: true, link: { type: 'doc', id: `${folderRel}/index` } }, null, 2)); - await writeFileEnsure(path.join(contentDir, folderRel, 'index.md'), stageNodeMarkdown(content, { slug: `/${route}`, title: existingTitle ? null : label })); + await writeFileEnsure(path.join(contentDir, folderRel, 'index.md'), stampLastUpdate(stageNodeMarkdown(content, { slug: `/${route}`, title: existingTitle ? null : label }), gitLastUpdate(srcAbs))); await writeFileEnsure(path.join(staticDir, `${route}.md`), content); manifest.push({ id: route, route: `/${route}`, title: label, mdSibling: `/${route}.md`, source: srcAbs, node: name, category: cat.label, categoryOrder: cat.position, description }); const serviceDescriptions = await readServiceDescriptions(nodeDir); @@ -406,7 +457,7 @@ async function gather({ projectRoot, contentStaticDir, contentDir, staticDir, mo const vExistingTitle = frontMatterTitle(vcontent); const vlabel = vExistingTitle || nodeLabel(variant, ''); const vdescription = variantDescription(variant, serviceDescriptions); - await writeFileEnsure(path.join(contentDir, folderRel, `${variant}.md`), stageNodeMarkdown(vcontent, { slug: `/${vroute}`, title: vExistingTitle ? null : vlabel })); + await writeFileEnsure(path.join(contentDir, folderRel, `${variant}.md`), stampLastUpdate(stageNodeMarkdown(vcontent, { slug: `/${vroute}`, title: vExistingTitle ? null : vlabel }), gitLastUpdate(vsrcAbs))); await writeFileEnsure(path.join(staticDir, `${vroute}.md`), vcontent); manifest.push({ id: vroute, route: `/${vroute}`, title: vlabel, mdSibling: `/${vroute}.md`, source: vsrcAbs, node: name, variant, category: cat.label, categoryOrder: cat.position, description: vdescription }); } @@ -416,7 +467,7 @@ async function gather({ projectRoot, contentStaticDir, contentDir, staticDir, mo // fills in when missing, and the body H1 is dropped to avoid a duplicate // heading. Always a real write — a symlink can't carry the edits. const destAbs = path.join(contentDir, NODES_DIR, cat.slug, `${name}.md`); - await writeFileEnsure(destAbs, stageNodeMarkdown(content, { slug: `/${route}`, title: existingTitle ? null : label })); + await writeFileEnsure(destAbs, stampLastUpdate(stageNodeMarkdown(content, { slug: `/${route}`, title: existingTitle ? null : label }), gitLastUpdate(srcAbs))); await writeFileEnsure(path.join(staticDir, `${route}.md`), content); manifest.push({ id: route, route: `/${route}`, title: label, mdSibling: `/${route}.md`, source: srcAbs, node: name, category: cat.label, categoryOrder: cat.position, description }); } From a097b6893fb7a2a59e486fa639813c7f4c4519ae Mon Sep 17 00:00:00 2001 From: EdwardLien0426 Date: Thu, 16 Jul 2026 18:16:27 -0700 Subject: [PATCH 5/9] docs: warn in robots.txt that named groups shadow the wildcard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per RFC 9309 a crawler obeys only the ONE most specific User-agent group matching it and ignores all others, including `*`. Every bot named in this file therefore reads its own two lines and never sees the `*` group, so a `Disallow:` added to `*` alone would bind Googlebot but not GPTBot, ClaudeBot or any other named bot — silently, as robots.txt has no linting. Verified with urllib.robotparser: adding `Disallow: /internal` to `*` leaves GPTBot and ClaudeBot still able to fetch it while Googlebot is blocked. The groups are kept as the issue specifies them, and because they state the intent in a form a human reads. Record the trap next to them instead, so whoever first adds a rule here sees it. --- packages/docs/static/robots.txt | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/packages/docs/static/robots.txt b/packages/docs/static/robots.txt index 007115b20..8b0405c19 100644 --- a/packages/docs/static/robots.txt +++ b/packages/docs/static/robots.txt @@ -1,5 +1,18 @@ # robots.txt for docs.rocketride.org # Explicitly welcomes AI crawlers + declares sitemap. +# +# WARNING, if you are about to add a rule here: +# Per RFC 9309 a crawler obeys only the ONE most specific User-agent group +# that matches it, and ignores every other group -- including `*`. Each bot +# named below therefore reads its own two lines and never sees the `*` group +# at all. A `Disallow:` added to `*` alone would bind Googlebot but NOT +# GPTBot, ClaudeBot, PerplexityBot, or anything else named here. That failure +# is silent: robots.txt has no linting and no CI check. Any rule intended for +# everyone must be repeated in every group below. +# +# The named groups are kept because they state the intent -- we welcome AI +# crawlers -- in a form a human can read. Behaviourally they are identical +# today to the `*` group on its own, which already allows everything. User-agent: GPTBot Allow: / From 34faa1fb2e6911024fc29154f656ecb6a5d4765a Mon Sep 17 00:00:00 2001 From: EdwardLien0426 Date: Thu, 16 Jul 2026 18:16:27 -0700 Subject: [PATCH 6/9] ci(docs): fetch full history so sitemap lastmod is real MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit docs:gather stamps every page with its source file's last commit date, and that date becomes the page's in the sitemap. actions/checkout defaults to fetch-depth: 1, so the docs build saw a single-commit history and every one of the 162 pages would have dated to the clone day — the sitemap would claim the whole site changed on each deploy. Google ignores lastmod it finds untrustworthy, so shipping it that way is worse than shipping no lastmod at all. --- .github/workflows/docs.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 8fbda8177..b60df5c54 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -32,6 +32,10 @@ jobs: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: persist-credentials: false + # docs:gather stamps each page with its source file's last commit + # date, which feeds the sitemap's . A shallow clone has a + # single commit, so every page would date to the clone day. + fetch-depth: 0 - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: From 341f789a152d28c3313f43d45b7fe6dd051ec439 Mon Sep 17 00:00:00 2001 From: EdwardLien0426 Date: Thu, 16 Jul 2026 18:16:27 -0700 Subject: [PATCH 7/9] fix(docs): tolerate CRLF front matter when stamping last_update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit stampLastUpdate() detected front matter with startsWith('---\\n'), which is false on a CRLF checkout. Rather than merging into the existing block it prepended a second one, dropping title (and every other key) out of the front matter and into the body as literal text. .gitattributes sets '* text=auto' and pins eol=lf only for .patch/.sh/.py, so .md files are CRLF in a Windows working tree. CI builds the docs on ubuntu, so deploys were unaffected — this corrupted local Windows builds. --- packages/docs/scripts/lib/gather.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/docs/scripts/lib/gather.js b/packages/docs/scripts/lib/gather.js index 515404803..48fc31cad 100644 --- a/packages/docs/scripts/lib/gather.js +++ b/packages/docs/scripts/lib/gather.js @@ -311,7 +311,7 @@ function gitLastUpdate(srcAbs) { */ function stampLastUpdate(content, date) { if (!date || /(^|\n)last_update\s*:/.test(content)) return content; - if (content.startsWith('---\n')) { + if (/^---\r?\n/.test(content)) { const end = content.indexOf('\n---', 4); if (end !== -1) return `${content.slice(0, end)}\nlast_update:\n date: ${date}${content.slice(end)}`; return content; From 3ad62069a0a9f6ddb397703ec42d4b29dbbac173 Mon Sep 17 00:00:00 2001 From: EdwardLien0426 Date: Thu, 16 Jul 2026 18:16:27 -0700 Subject: [PATCH 8/9] feat(docs): give llms.txt absolute URLs and per-link descriptions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit llms.txt listed every page as a bare relative link, so an agent had to fetch all 161 pages to find out what any of them covered. The gather manifest already carried a description for node pages — it fed the /nodes catalog table but never llms.txt — and doc pages carried none at all. - Add pageDescription(): front-matter 'description:' when declared, else the first prose sentence, skipping MDX imports, JSX, headings, tables and fenced blocks. Mirrors what Docusaurus derives for its meta description. - Attach it to the spine and module-doc manifest entries. - Emit '- [title](url): description', and make the URL absolute — the llms.txt format specifies URLs and an agent that fetched the file on its own has no base to resolve against. - extractDescription() joined the services*.json description array bare, producing 'a node.Can be invoked' and defeating its own '. ' sentence split, so node blurbs ran on untruncated. Join with a space. Verified end to end (registry discover -> gather -> index): 161 links, 161 absolute, 158 described; the 3 without are generated or dateless pages. Unit-covers pageDescription and stampLastUpdate, both pure. The fenced-block case is there because the first draft returned 'code();' as a description. --- packages/docs/scripts/lib/gather.js | 57 +++++++++++++++++++-- packages/docs/scripts/lib/llms.js | 6 ++- packages/docs/src/lib/gather.test.mjs | 71 +++++++++++++++++++++++++++ packages/docs/src/lib/llms.test.mjs | 8 ++- 4 files changed, 134 insertions(+), 8 deletions(-) create mode 100644 packages/docs/src/lib/gather.test.mjs diff --git a/packages/docs/scripts/lib/gather.js b/packages/docs/scripts/lib/gather.js index 48fc31cad..0235d8ce3 100644 --- a/packages/docs/scripts/lib/gather.js +++ b/packages/docs/scripts/lib/gather.js @@ -91,12 +91,17 @@ async function readNodeMeta(nodeDir) { return { classType, title, description: extractDescription(text) }; } -/** Extract a first-sentence description from a raw services*.json text string. */ +/** + * Extract a first-sentence description from a raw services*.json text string. + * The JSON `description` is an array of complete lines, so they join with a + * space — joining bare runs them together ("a node.Can be invoked"), which also + * defeats the '. ' sentence split below and returns the whole blob. + */ function extractDescription(text) { const m = /"description"\s*:\s*\[([^\]]*)\]/.exec(text); if (!m) return ''; const parts = m[1].match(/"((?:[^"\\]|\\.)*)"/g) || []; - const full = parts.map((s) => s.slice(1, -1)).join('').trim(); + const full = parts.map((s) => s.slice(1, -1)).join(' ').replace(/\s+/g, ' ').trim(); const dot = full.indexOf('. '); return dot >= 0 ? full.slice(0, dot + 1) : full; } @@ -213,6 +218,48 @@ function frontMatterTitle(content) { return t ? t[2].replace(/^['"]|['"]$/g, '') : null; } +/** + * First-sentence description for a markdown/MDX page, mirroring what Docusaurus + * derives for its meta description. A front-matter `description:` wins when the + * page declares one; otherwise the first prose paragraph is used, skipping MDX + * imports, JSX, headings, code fences, tables and admonitions. Node pages get + * their description from services*.json instead (see extractDescription). + * @param {string} content - raw page content (md/mdx). + * @return {string} description, or '' when the page has no leading prose. + */ +function pageDescription(content) { + const fm = /^---\r?\n([\s\S]*?)\r?\n---/.exec(content); + if (fm) { + const d = /(^|\n)description:\s*(.+?)\s*(\n|$)/.exec(fm[1]); + if (d) return d[2].replace(/^['"]|['"]$/g, ''); + } + const body = content.replace(/^---\r?\n[\s\S]*?\r?\n---\r?\n/, ''); + const para = []; + let inFence = false; + for (const line of body.split(/\r?\n/)) { + const t = line.trim(); + // Skip fenced blocks wholesale — the fence delimiters AND their contents. + if (/^(```|~~~)/.test(t)) { + if (para.length) break; + inFence = !inFence; + continue; + } + if (inFence) continue; + if (!t) { + if (para.length) break; + continue; + } + if (/^(#{1,6}\s|:::|