feat(site): harden the crawler configuration

Safety and correctness settings that were missing from the first pass:
saveBackup, and a safety check that refuses to publish an index which
lost more than 10% of its records, so a crawl that breaks cannot empty
search. url_without_anchor and type join attributesToRetrieve; the
DocSearch client needs both to build a result link and group hits.

startUrls goes back to the site root. /docs/ does not exist on the
released site yet, so the crawler had no reachable entry point until the
refactor is promoted. A section is inferred from the URL for the same
reason, as a fallback for pages that do not carry the frontmatter yet.

Code blocks are copied into crawler-only paragraphs so a search can find
an exact Taskfile key or flag. Measured at +10% indexed content, and the
level weighting still puts headings ahead of code.

The two deprecation actions collapse into one, with pageRank computed
from the path.

The frontmatter field is renamed docsearch:doc_type to match what the
extractor reads.

Author: vmaerten

website/.vitepress/config.ts

@@ -165,7 +165,7 @@ export default defineConfig({
     if (pageData.frontmatter.docType) {
       head.push([
         'meta',
-        { name: 'docsearch:type', content: pageData.frontmatter.docType }
+        { name: 'docsearch:doc_type', content: pageData.frontmatter.docType }
       ])
     }
 

website/docsearch.config.js

@@ -7,22 +7,36 @@
 // When you change the configuration in the dashboard, change it here too.
 //
 // The API key below is the crawler's *write* key and is deliberately not in
-// this repository. Set it in the dashboard, or in the crawler's environment.
+// this repository. Keep the existing key when pasting this file into the
+// dashboard; never replace this placeholder in Git.
 //
 // Every selector here was checked against the generated HTML, not assumed.
 
 new Crawler({
   appId: '7IZIJ13AI7',
-  apiKey: process.env.ALGOLIA_CRAWLER_WRITE_KEY,
+  apiKey: '<ALGOLIA_CRAWLER_WRITE_KEY>',
   indexPrefix: '',
   rateLimit: 8,
   maxDepth: 10,
-  schedule: 'every 1 day',
+  schedule: 'at 9:50 AM on Thursday',
+  ignoreCanonicalTo: true,
+  saveBackup: true,
+
+  safetyChecks: {
+    beforeIndexPublishing: {
+      maxLostRecordsPercentage: 10
+    },
+    // The dashboard's current config.d.ts exposes this at the safetyChecks
+    // level, rather than inside beforeIndexPublishing.
+    maxFailedUrls: 5
+  },
 
   // Only the released site. next.taskfile.dev serves the same URLs from the
   // upcoming release, and both sites share the single `taskfile` index, so
   // crawling it as well would give every page a duplicate record.
-  startUrls: ['https://taskfile.dev/docs/'],
+  // The root exists both before and after the documentation refactor. The old
+  // production site has no /docs/ landing page yet.
+  startUrls: ['https://taskfile.dev/'],
   sitemaps: ['https://taskfile.dev/sitemap.xml'],
   // Only /docs is indexed, so there is no reason to fetch the blog, the
   // homepage or /adopters on every crawl.
@@ -40,11 +54,68 @@ new Crawler({
     {
       indexName: 'taskfile',
       pathsToMatch: ['https://taskfile.dev/docs/**'],
-      recordExtractor: ({ $, helpers }) => {
+      recordExtractor: ({ $, helpers, url }) => {
         // The banner the llms plugin injects sits inside .vp-doc, ahead of the
         // h1. It is display:none for readers and must not become content.
         $('[data-nosnippet]').remove();
 
+        // DocSearch expects content selectors to target paragraphs or list
+        // items. Copy code blocks into crawler-only paragraphs so experienced
+        // users can search for exact Taskfile keys and command syntax without
+        // changing the page rendered to readers.
+        $('.vp-doc pre code').each((_, element) => {
+          const code = $(element).text().trim();
+          if (!code) return;
+          const paragraph = $('<p></p>').addClass('docsearch-code').text(code);
+          $(element).closest('pre').after(paragraph);
+        });
+
+        // Frontmatter metadata is available after the refactor. Infer the same
+        // values from the URL while the old monolithic guide is still live, so
+        // this configuration can be installed before the website PR merges.
+        // Remove this URL inference once the refactored documentation is live
+        // and every indexed page exposes the DocSearch metadata.
+        const pathname = url.pathname.replace(/\/+$/, '') || '/';
+        const inferredSection = (() => {
+          if (pathname === '/docs') return 'Overview';
+          if (
+            /^\/docs\/(installation|getting-started|integrations)$/.test(
+              pathname
+            )
+          ) {
+            return 'Getting Started';
+          }
+          if (/^\/docs\/reference\//.test(pathname)) return 'Reference';
+          if (/^\/docs\/(contributing|releasing|styleguide)$/.test(pathname)) {
+            return 'Contributing';
+          }
+          if (
+            /^\/docs\/(experiments|deprecations|security)(\/|$)/.test(
+              pathname
+            ) ||
+            /^\/docs\/(changelog|faq|taskfile-versions|community)$/.test(
+              pathname
+            )
+          ) {
+            return 'Project';
+          }
+          return 'Guide';
+        })();
+        const section =
+          $('meta[name="docsearch:section"]').attr('content') ||
+          inferredSection;
+        const docType =
+          $('meta[name="docsearch:doc_type"]').attr('content') ||
+          ({
+            Overview: 'overview',
+            Reference: 'reference',
+            Contributing: 'contributing',
+            Project: 'project',
+            Guide: 'guide',
+            'Getting Started': 'guide'
+          }[section] ??
+            'guide');
+
         return helpers.docsearch({
           recordProps: {
             // Not a heading on the page: the section the page belongs to,
@@ -53,8 +124,10 @@ new Crawler({
             // DOM instead, which ties the index to the theme's markup and
             // breaks silently when that markup changes.
             lvl0: {
-              selectors: 'meta[name="docsearch:section"]',
-              defaultValue: 'Documentation'
+              // Algolia documents an empty selector as the way to provide a
+              // raw, dynamically computed lvl0 through defaultValue.
+              selectors: '',
+              defaultValue: section
             },
             // Everything below is scoped to .vp-doc. VitePress renders the
             // sidebar's section labels as <h2 class="text"> inside
@@ -64,55 +137,44 @@ new Crawler({
             lvl2: '.vp-doc h2',
             lvl3: '.vp-doc h3',
             lvl4: '.vp-doc h4',
-            content: '.vp-doc p, .vp-doc li, .vp-doc td',
-            // Faceted so a search can be narrowed to reference pages, or
-            // weighted differently later.
-            docType: {
-              selectors: 'meta[name="docsearch:type"]',
-              defaultValue: 'guide'
-            }
-          },
-          aggregateContent: true,
-          recordVersion: 'v3'
-        });
-      }
-    },
-    {
-      // Deprecation notices describe what to stop doing. They should be
-      // findable by name, but never ahead of the page documenting the
-      // replacement.
-      indexName: 'taskfile',
-      pathsToMatch: ['https://taskfile.dev/docs/deprecations/**'],
-      pageRank: -10,
-      recordExtractor: ({ $, helpers }) => {
-        $('[data-nosnippet]').remove();
-        return helpers.docsearch({
-          recordProps: {
-            lvl0: {
-              selectors: 'meta[name="docsearch:section"]',
-              defaultValue: 'Deprecations'
+            lvl5: '.vp-doc h5',
+            lvl6: '.vp-doc h6',
+            content: '.vp-doc p, .vp-doc li, .vp-doc td, .vp-doc th',
+            section: { defaultValue: section },
+            doc_type: { defaultValue: docType },
+            lang: {
+              defaultValue: $('html').attr('lang') || 'en-US'
             },
-            lvl1: '.vp-doc h1',
-            lvl2: '.vp-doc h2',
-            lvl3: '.vp-doc h3',
-            content: '.vp-doc p, .vp-doc li, .vp-doc td'
+            // Deprecation notices remain findable, but don't outrank the page
+            // that documents the supported replacement.
+            pageRank: pathname.startsWith('/docs/deprecations/') ? '-10' : '0'
           },
+          indexHeadings: true,
           aggregateContent: true,
           recordVersion: 'v3'
         });
       }
     }
   ],
 
+  // These settings initialize a new index. Algolia doesn't apply them to an
+  // existing index, so import the same values in the taskfile index settings
+  // when this configuration changes them.
   initialIndexSettings: {
     taskfile: {
-      attributesForFaceting: ['type', 'lang', 'docType'],
+      hitsPerPage: 20,
+      maxValuesPerFacet: 100,
+      attributesForFaceting: ['type', 'lang', 'section', 'doc_type'],
       attributesToRetrieve: [
         'hierarchy',
         'content',
         'anchor',
         'url',
-        'docType'
+        'url_without_anchor',
+        'type',
+        'lang',
+        'section',
+        'doc_type'
       ],
       attributesToHighlight: ['hierarchy', 'content'],
       attributesToSnippet: ['content:10'],
@@ -123,6 +185,8 @@ new Crawler({
         'unordered(hierarchy.lvl2)',
         'unordered(hierarchy.lvl3)',
         'unordered(hierarchy.lvl4)',
+        'unordered(hierarchy.lvl5)',
+        'unordered(hierarchy.lvl6)',
         'content'
       ],
       distinct: true,
@@ -150,7 +214,13 @@ new Crawler({
       ignorePlurals: true,
       advancedSyntax: true,
       attributeCriteriaComputedByMinProximity: true,
-      removeWordsIfNoResults: 'allOptional'
+      removeWordsIfNoResults: 'allOptional',
+      separatorsToIndex: '_',
+      paginationLimitedTo: 1000,
+      exactOnSingleWordQuery: 'attribute',
+      queryType: 'prefixLast',
+      snippetEllipsisText: '',
+      alternativesAsExact: ['ignorePlurals', 'singleWordSynonym']
     }
   }
 });

website/src/next/docs/contributing.md

@@ -132,7 +132,8 @@ Search is provided by Algolia DocSearch. The crawler runs on Algolia's side and
 is configured through their dashboard, but `website/docsearch.config.js` holds
 the same configuration in the repository so it can be read and reviewed. If you
 change one, change the other. A page's `section:` frontmatter is what the
-crawler shows as the breadcrumb on a search result, so a new page needs one.
+crawler shows as the breadcrumb on a search result, while `docType:` powers its
+search facet, so a new page needs both.
 
 When making a change, consider whether a change to the [Usage
 Guide][usage-guide] is necessary. This document contains descriptions and