Add note: Disallow AI bots in robots.txt

Fixes #975

Author: maiertech

package.json

@@ -18,7 +18,7 @@
 		"@eslint/compat": "^1.3.1",
 		"@eslint/js": "^9.32.0",
 		"@iconify/svelte": "^5.0.1",
-		"@lucide/svelte": "^0.527.0",
+		"@lucide/svelte": "^0.528.0",
 		"@maiertech/sveltekit-helpers": "^0.28.0",
 		"@playwright/test": "^1.54.1",
 		"@sindresorhus/slugify": "^2.2.1",

pnpm-lock.yaml

@@ -1,4 +1,5 @@
 import noteAmIStillRelevantAsADeveloper from '$notes/(2025)/am-i-still-relevant-as-a-developer/meta';
+import noteDisallowingAiBotsInRobotsTxt from '$notes/(2025)/disallowing-ai-bots-in-robots-txt/meta';
 import noteEnvVarsInCode from '$notes/(2025)/env-vars-in-vscode/meta';
 import noteHostingInEurope from '$notes/(2025)/hosting-in-europe-beyond-big-tech/meta';
 import noteHowToWriteExceptionalDocs from '$notes/(2025)/how-to-write-exceptional-docs/meta';
@@ -14,6 +15,7 @@ export const prerender = true;
 export const GET: RequestHandler = async () => {
 	// Sort order: latest first.
 	return json([
+		noteDisallowingAiBotsInRobotsTxt,
 		notePenpotsMissingSvgTextToPathFeature,
 		notePrerenderingServerRoutes,
 		noteHostingInEurope,

src/routes/api/notes/2025/+server.ts

@@ -0,0 +1,12 @@
+import { resolveNote } from '$lib/server/resolvers';
+import type { PageServerLoad } from './$types';
+import meta from './meta';
+
+export const load: PageServerLoad = async () => {
+	const note = await resolveNote(meta);
+	const { title, description, ogImageUrl } = note;
+	return {
+		note,
+		seo: { title, description, ogImageUrl }
+	};
+};

src/routes/notes/(2025)/disallowing-ai-bots-in-robots-txt/+page.server.ts

@@ -0,0 +1,14 @@
+If you have not checked the HTTP logs of your website lately, I recommend doing so. I was stunned to
+see big-name AI bots crawling my website daily, hunting for new training data.
+
+Even though my personal website is a non-commercial side project, I do not like the idea of my
+content being dissolved into some AI overview with zero credit. I am trying to give you an
+incentive, useful content, to come back to my website regularly, either directly, via my RSS feeds,
+or via my socials. Even though your clicks do not make me money, they do motivate me to post more.
+
+That is why I have updated my `robots.txt` to disallow the bots from the link above and a few more I
+found in my logs. I have also explicitly allowed a couple of bots in the classic search engine
+category, some of which I had never heard of before.
+
+Keep in mind that `robots.txt` is a convention and not a technical enforcement. I count on the big
+players to comply with my `robots.txt`, but there is no guarantee that they will.

src/routes/notes/(2025)/disallowing-ai-bots-in-robots-txt/+page.svx

@@ -0,0 +1,10 @@
+import type { NoteMeta } from '@maiertech/sveltekit-helpers';
+
+export default {
+	title: 'Disallowing AI Bots in robots.txt',
+	description:
+		'How and why I updated my robots.txt to block AI bots from crawling my personal website, while allowing search engines.',
+	publishedDate: '2025-07-30',
+	link: 'https://github.com/ai-robots-txt/ai.robots.txt',
+	path: '/notes/disallowing-ai-bots-in-robots-txt'
+} satisfies NoteMeta;