feat: copy markdown integration

deno.json

@@ -10,11 +10,14 @@
     "juno": "juno"
   },
   "imports": {
+    "@astrojs/mdx": "npm:@astrojs/mdx@^4.3.11",
     "@astrojs/react": "npm:@astrojs/react@^4.3.0",
+    "@joplin/turndown-plugin-gfm": "npm:@joplin/turndown-plugin-gfm@^1.0.64",
     "@junobuild/cli": "npm:@junobuild/cli@0.10.0",
     "@pagefind/default-ui": "npm:@pagefind/default-ui@^1.4.0",
     "@radix-ui/react-navigation-menu": "npm:@radix-ui/react-navigation-menu@^1.2.14",
     "@types/react": "npm:@types/react@^19.1.12",
+    "@types/turndown": "npm:@types/turndown@^5.0.6",
     "astro-integration-kit": "npm:astro-integration-kit@^0.19.0",
     "astro-mermaid": "npm:astro-mermaid@^1.1.0",
     "fs": "node:fs",
@@ -23,6 +26,7 @@
     "pagefind": "npm:pagefind@^1.4.0",
     "react": "npm:react@^19.1.1",
     "react-dom": "npm:react-dom@^19.1.1",
+    "turndown": "npm:turndown@^7.2.2",
     "unist-util-visit": "npm:unist-util-visit@^5.0.0",
     "url": "node:url",
     "util": "node:util",

docs/astro.config.mjs

@@ -5,6 +5,7 @@ import starlight from "@astrojs/starlight";
 import mermaid from "astro-mermaid";
 import { multiSidebarPlugin } from "./plugins/multi-sidebar/index.ts";
 import { markdownUrlsPlugin } from "./plugins/markdown-urls/index.ts";
+import { copyPagePlugin } from "./plugins/copy-page/index.ts";
 import { matomo } from "./integrations/matomo/index.ts";
 import { getProjectsConfig, getSidebarsFromProjects } from "./projects.ts";
 
@@ -50,6 +51,7 @@ export default defineConfig({
           sidebars: getSidebarsFromProjects(projectsConfig),
         }),
         markdownUrlsPlugin(),
+        copyPagePlugin(),
       ],
     }),
     matomo({

docs/plugins/copy-page/index.ts

@@ -0,0 +1,18 @@
+import type { StarlightPlugin } from "@astrojs/starlight/types";
+import { copyPageIntegration } from "./integration.ts";
+
+export function copyPagePlugin(): StarlightPlugin {
+  return {
+    name: "starlight-copy-page-plugin",
+    hooks: {
+      "config:setup": (ctx) => {
+        ctx.addIntegration(
+          copyPageIntegration({
+            siteUrl: ctx.astroConfig.site!, // we assume this is set
+            logger: ctx.logger,
+          }),
+        );
+      },
+    },
+  };
+}

docs/plugins/copy-page/integration.ts

@@ -0,0 +1,102 @@
+import type { AstroIntegrationLogger } from "astro";
+import { z } from "astro/zod";
+import { defineIntegration } from "astro-integration-kit";
+import { htmlToMarkdown } from "./to-markdown.ts";
+import { readdir, readFile, writeFile } from "node:fs/promises";
+import { join } from "path";
+import { fileURLToPath } from "url";
+import { type Dirent } from "fs";
+
+const copyPageConfigSchema = z.object({
+  siteUrl: z.string(),
+  logger: z.custom<AstroIntegrationLogger>(),
+});
+
+export const copyPageIntegration = defineIntegration({
+  name: "starlight-copy-page-integration",
+  optionsSchema: copyPageConfigSchema,
+  setup({ options }) {
+    const { siteUrl, logger } = options;
+
+    return {
+      hooks: {
+        "astro:build:done": async ({ dir }) => {
+          logger.info("Generating markdown files from HTML pages...");
+
+          try {
+            const distPath = fileURLToPath(dir);
+            const htmlFiles = await findHtmlFiles(distPath);
+            let successCount = 0;
+            let errorCount = 0;
+
+            for (const htmlPath of htmlFiles) {
+              try {
+                // Read the HTML file
+                const html = await readFile(htmlPath, "utf-8");
+
+                // Convert to markdown
+                const markdown = htmlToMarkdown(html, siteUrl);
+
+                // Create markdown file path (replace index.html with index.md)
+                const mdPath = htmlPath.replace(/index\.html$/, "index.md");
+
+                // Write markdown file
+                await writeFile(mdPath, markdown, "utf-8");
+
+                // Get relative path for logging
+                const relativePath = mdPath.replace(distPath, "");
+                logger.debug(`Generated: ${relativePath}`);
+                successCount++;
+              } catch (error) {
+                logger.warn(
+                  `Failed to generate markdown for ${htmlPath}: ${error}`,
+                );
+                errorCount++;
+              }
+            }
+
+            logger.info(
+              `Markdown generation complete: ${successCount} files generated, ${errorCount} errors`,
+            );
+          } catch (error) {
+            logger.error(`Failed to generate markdown files: ${error}`);
+          }
+        },
+      },
+    };
+  },
+});
+
+async function findHtmlFiles(dir: string): Promise<string[]> {
+  const files: string[] = [];
+
+  async function walk(currentPath: string) {
+    const entries = await readdir(currentPath, { withFileTypes: true });
+
+    for (const entry of entries) {
+      const fullPath = join(currentPath, entry.name);
+
+      if (shouldWalk(entry)) {
+        await walk(fullPath);
+      } else if (entry.isFile() && entry.name === "index.html") {
+        files.push(fullPath);
+      }
+    }
+  }
+
+  // Start by walking subdirectories only, skipping root-level files
+  const rootEntries = await readdir(dir, { withFileTypes: true });
+  for (const entry of rootEntries) {
+    if (shouldWalk(entry)) {
+      const subDirPath = join(dir, entry.name);
+      await walk(subDirPath);
+    }
+  }
+
+  return files;
+}
+
+function shouldWalk(entry: Dirent): boolean {
+  return entry.isDirectory() && !entry.name.startsWith("_") &&
+    entry.name !== "pagefind";
+}

docs/plugins/copy-page/rules.ts

@@ -0,0 +1,110 @@
+import TurndownService from "turndown";
+
+export const HTML_TAGS_TO_REMOVE: Array<keyof HTMLElementTagNameMap> = [
+  "script",
+  "style",
+  "starlight-toc" as keyof HTMLElementTagNameMap, // Mobile table of contents
+  "mobile-starlight-toc" as keyof HTMLElementTagNameMap, // Mobile TOC variant
+];
+
+const CLASSES_TO_REMOVE = [
+  "sidebar",
+  "pagination-links",
+  "tablist-wrapper",
+  "sr-only",
+];
+
+/**
+ * Converts card titles to h4 headings
+ */
+export const cardTitlesRule: TurndownService.Rule = {
+  filter: (node) => {
+    return (
+      node.nodeName === "P" &&
+      node.classList.contains("title")
+    );
+  },
+  replacement: (content) => {
+    // Clean up content and convert to h4
+    const cleanContent = content.replace(/\s+/g, " ").trim();
+    return "\n\n#### " + cleanContent + "\n\n";
+  },
+};
+
+/**
+ * Cleans up link text to remove extra whitespace and newlines
+ */
+export const cleanLinksRule: TurndownService.Rule = {
+  filter: "a",
+  replacement: (content, node) => {
+    const href = node.getAttribute("href");
+    if (!href) {
+      return content;
+    }
+
+    // Remove extra whitespace and newlines
+    const cleanContent = content.replace(/\s+/g, " ").trim();
+    const title = node.title ? ` "${node.title}"` : "";
+
+    return `[${cleanContent}](${href}${title})`;
+  },
+};
+
+/**
+ * Parse Fenced Code Blocks with language from data-language or class attribute
+ */
+export const fencedCodeBlockRule: TurndownService.Rule = {
+  filter: (node, options) => {
+    return (
+      options.codeBlockStyle === "fenced" &&
+      node.nodeName === "PRE" &&
+      Boolean(node.firstChild) &&
+      (node.firstChild as HTMLElement).nodeName === "CODE"
+    );
+  },
+  replacement: (
+    _content,
+    node,
+    options,
+  ) => {
+    const codeNode = node.firstChild as HTMLElement;
+
+    // Try to get language from data-language attribute first, then from class
+    const language = node.getAttribute("data-language") ||
+      codeNode.getAttribute("data-language") || "";
+
+    // Extract code preserving line structure
+    // Some syntax highlighters wrap each line in divs (e.g., expressive-code)
+    let code = "";
+    const lines = codeNode.querySelectorAll(".ec-line, .line");
+
+    if (lines.length > 0) {
+      // If we found line elements, extract text from each
+      code = Array.from(lines)
+        .map((line) => line.textContent || "")
+        .join("\n");
+    } else {
+      // Fall back to plain textContent
+      code = codeNode.textContent || "";
+    }
+
+    // Trim trailing whitespace but preserve the code structure
+    code = code.trimEnd();
+
+    return (
+      "\n\n" + options.fence + language + "\n" +
+      code +
+      "\n" + options.fence + "\n\n"
+    );
+  },
+};
+
+export function shouldRemoveElement(node: HTMLElement): boolean {
+  if (!node.classList) {
+    return false;
+  }
+
+  return CLASSES_TO_REMOVE.some((className) =>
+    node.classList.contains(className)
+  );
+}

docs/plugins/copy-page/to-markdown.ts

@@ -0,0 +1,35 @@
+import TurndownService from "turndown";
+import { gfm } from "@joplin/turndown-plugin-gfm";
+import { htmlToContent, transformLinks } from "./transforms.ts";
+import {
+  cardTitlesRule,
+  cleanLinksRule,
+  fencedCodeBlockRule,
+  HTML_TAGS_TO_REMOVE,
+  shouldRemoveElement,
+} from "./rules.ts";
+
+export function htmlToMarkdown(html: string, siteUrl: string): string {
+  const mainContent = htmlToContent(html);
+
+  const turndownService = new TurndownService({
+    codeBlockStyle: "fenced",
+    headingStyle: "atx",
+    hr: "---",
+  });
+
+  turndownService.remove(HTML_TAGS_TO_REMOVE);
+  turndownService.remove(shouldRemoveElement);
+
+  turndownService.addRule("cardTitles", cardTitlesRule);
+  turndownService.addRule("cleanLinks", cleanLinksRule);
+  turndownService.addRule("fencedCodeBlock", fencedCodeBlockRule);
+
+  // Use Github flavored markdown
+  turndownService.use(gfm);
+
+  let markdown = turndownService.turndown(mainContent);
+  markdown = transformLinks(markdown, siteUrl);
+
+  return markdown;
+}

docs/plugins/copy-page/transforms.ts

@@ -0,0 +1,64 @@
+/**
+ * Extracts the main content from Starlight HTML page
+ */
+export function htmlToContent(html: string): string {
+  const mainMatch = html.match(/<main[^>]*>([\s\S]*?)<\/main>/);
+  if (mainMatch && mainMatch[1]) {
+    let content = mainMatch[1];
+
+    // Remove all img tags - they should not be in the markdown output
+    content = content.replace(/<img[^>]*>/g, "");
+
+    // Remove heading anchor links (sl-anchor-link) - these cause empty links in output
+    content = content.replace(
+      /<a[^>]*class="[^"]*sl-anchor-link[^"]*"[^>]*>[\s\S]*?<\/a>/g,
+      "",
+    );
+
+    // Remove "hidden" attribute from tab panels so all content is shown
+    content = content.replace(
+      /(<div[^>]*role="tabpanel"[^>]*)hidden([^>]*>)/g,
+      "$1$2",
+    );
+
+    return content;
+  }
+
+  return html;
+}
+
+/**
+ * Transforms internal links to absolute URLs pointing to markdown files
+ */
+export function transformLinks(markdown: string, siteUrl: string): string {
+  // Ensure siteUrl doesn't end with a slash to avoid double slashes
+  const baseUrl = siteUrl.endsWith("/") ? siteUrl.slice(0, -1) : siteUrl;
+
+  // Match markdown links: [text](url)
+  return markdown.replace(/\[([^\]]+)\]\(([^)]+)\)/g, (match, text, url) => {
+    // Only transform internal links (starting with /)
+    if (url.startsWith("/")) {
+      // Remove any hash fragments
+      const [path, hash] = url.split("#");
+
+      // Construct the full URL with .md extension
+      let fullUrl: string;
+      if (path.endsWith(".md")) {
+        fullUrl = `${baseUrl}${path}`;
+      } else {
+        // Add /index.md to directory paths
+        fullUrl = `${baseUrl}${path}${path.endsWith("/") ? "" : "/"}index.md`;
+      }
+
+      // Re-add hash if it existed
+      if (hash) {
+        fullUrl += `#${hash}`;
+      }
+
+      return `[${text}](${fullUrl})`;
+    }
+
+    // Return external links unchanged
+    return match;
+  });
+}