Serve sitemap via static robots.txt; drop dead Django robots view (#1252)

Follow-up to the sitemap work. On the servers Apache serves the static
top-level ./robots.txt from the project checkout and never forwards
/robots.txt to Django, so the robots_txt view added earlier was dead code
in every server environment (it only ran under local runserver, which
diverges from prod). Confirmed via the test deploy: /sitemap.xml is served
by Django (WSGIServer) while /robots.txt comes from Apache as a static file,
and the served content matches this tracked ./robots.txt.

- robots.txt: advertise the sitemap (Sitemap: line); keep allow-all crawling.
- Remove website/views/robots.py, its re-export, and the /robots.txt route;
  add a comment in urls.py pointing future edits at the static file.
- Drop the now-irrelevant robots view tests; sitemap tests unchanged.

The dynamic sitemap (django.contrib.sitemaps) is unchanged and already
verified on the test server.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: jonfroehlich

makeabilitylab/urls.py

@@ -26,18 +26,21 @@
 from django.views.static import serve
 from django.conf import settings
 
-from website import views
 from website.sitemaps import sitemaps
 
 urlpatterns = [
 
     re_path(r'^admin/', admin.site.urls),
 
-    # SEO endpoints (issue #1252). Declared before the website.urls include so
-    # the app's patterns can't shadow them. The sitemap is generated from our
-    # querysets (see website/sitemaps.py); robots.txt is environment-aware.
+    # Dynamic sitemap (issue #1252), generated from our querysets (see
+    # website/sitemaps.py). Declared before the website.urls include so the
+    # app's patterns can't shadow it.
+    #
+    # NOTE: robots.txt is intentionally NOT routed here. On the servers Apache
+    # serves the static ./robots.txt from the project checkout and never
+    # forwards /robots.txt to Django, so a view here would be dead code. Edit
+    # the top-level ./robots.txt to change crawler rules or the Sitemap line.
     path('sitemap.xml', sitemap, {'sitemaps': sitemaps}, name='django.contrib.sitemaps.views.sitemap'),
-    path('robots.txt', views.robots_txt, name='robots_txt'),
 
     #Info on how to route root to website was found here http://stackoverflow.com/questions/7580220/django-urls-howto-map-root-to-app
     re_path(r'', include('website.urls')),

robots.txt

@@ -1,2 +1,4 @@
-User-agent: *
-Disallow:
+User-agent: *
+Disallow:
+
+Sitemap: https://makeabilitylab.cs.washington.edu/sitemap.xml

website/tests/test_sitemap.py

@@ -1,14 +1,14 @@
 """
-Regression tests for the dynamic sitemap and robots.txt (issue #1252).
+Regression tests for the dynamic sitemap (issue #1252).
 
-Both endpoints are exercised through the real URL/view stack so a routing or
-queryset regression is caught. See website/sitemaps.py and
-website/views/robots.py.
+The sitemap is exercised through the real URL/view stack so a routing or
+queryset regression is caught. See website/sitemaps.py.
+
+Note: robots.txt is a static file (./robots.txt) served by Apache on the
+servers, not a Django view, so it isn't covered here.
 """
 
-import os
 from datetime import date
-from unittest import mock
 
 from website.tests.base import DatabaseTestCase
 
@@ -58,23 +58,3 @@ def test_sitemap_includes_news_item(self):
         news = self.make_news_item(title="Big Lab News")
         body = self.client.get("/sitemap.xml").content.decode()
         self.assertIn(f"/news/{news.slug}/", body)
-
-
-class RobotsTxtTests(DatabaseTestCase):
-    def test_robots_is_plain_text(self):
-        resp = self.client.get("/robots.txt")
-        self.assertEqual(resp.status_code, 200)
-        self.assertEqual(resp["Content-Type"], "text/plain")
-
-    @mock.patch.dict(os.environ, {"DJANGO_ENV": "PROD"})
-    def test_robots_prod_allows_and_advertises_sitemap(self):
-        body = self.client.get("/robots.txt").content.decode()
-        self.assertIn("Allow: /", body)
-        self.assertIn("Sitemap:", body)
-        self.assertIn("/sitemap.xml", body)
-
-    @mock.patch.dict(os.environ, {"DJANGO_ENV": "TEST"})
-    def test_robots_non_prod_disallows_all(self):
-        body = self.client.get("/robots.txt").content.decode()
-        self.assertIn("Disallow: /", body)
-        self.assertNotIn("Allow: /", body)

website/views/__init__.py

@@ -9,5 +9,4 @@
 from .view_project_people import *
 from .serve_pdf import *
 from .awards import awards
-from .robots import robots_txt