feat: add robots.txt and logo asset to public directory while updating gitignore for static assets

Author: joaltoroc

.gitignore

@@ -44,5 +44,5 @@ design-system/
 skills-lock.json
 
 # Astro compiled assets in root
-public/
+/public/
 

frontend/public/logo.png

@@ -0,0 +1,50 @@
+# General rules for all search crawlers
+User-agent: *
+Allow: /
+Disallow: /api/
+
+# Block AI & LLM training crawlers completely
+User-agent: GPTBot
+Disallow: /
+
+User-agent: ClaudeBot
+Disallow: /
+
+User-agent: Google-Extended
+Disallow: /
+
+User-agent: Applebot-Extended
+Disallow: /
+
+User-agent: CCBot
+Disallow: /
+
+User-agent: Meta-ExternalAgent
+Disallow: /
+
+User-agent: Amazonbot
+Disallow: /
+
+User-agent: CohereBot
+Disallow: /
+
+# Block AI Search bots from scraping to avoid rendering resource abuse
+User-agent: PerplexityBot
+Disallow: /
+
+User-agent: OAI-SearchBot
+Disallow: /
+
+User-agent: Claude-SearchBot
+Disallow: /
+
+# Block social media preview scrapers from executing expensive API renders
+User-agent: facebookexternalhit
+Disallow: /api/
+Disallow: /private/
+
+User-agent: Twitterbot
+Disallow: /api/
+Disallow: /private/
+
+Sitemap: https://github-helpers.creativecode.com.co/sitemap.xml