From 2be25e1f01416f97cc121e41cc3fdbb7958b9b2c Mon Sep 17 00:00:00 2001 From: Jon Froehlich Date: Mon, 15 Jun 2026 15:06:46 -0700 Subject: [PATCH] Serve sitemap via static robots.txt; drop dead Django robots view (#1252) Follow-up to the sitemap work. On the servers Apache serves the static top-level ./robots.txt from the project checkout and never forwards /robots.txt to Django, so the robots_txt view added earlier was dead code in every server environment (it only ran under local runserver, which diverges from prod). Confirmed via the test deploy: /sitemap.xml is served by Django (WSGIServer) while /robots.txt comes from Apache as a static file, and the served content matches this tracked ./robots.txt. - robots.txt: advertise the sitemap (Sitemap: line); keep allow-all crawling. - Remove website/views/robots.py, its re-export, and the /robots.txt route; add a comment in urls.py pointing future edits at the static file. - Drop the now-irrelevant robots view tests; sitemap tests unchanged. The dynamic sitemap (django.contrib.sitemaps) is unchanged and already verified on the test server. Co-Authored-By: Claude Opus 4.8 (1M context) --- makeabilitylab/urls.py | 13 ++++++----- robots.txt | 6 +++-- website/tests/test_sitemap.py | 32 +++++---------------------- website/views/__init__.py | 1 - website/views/robots.py | 41 ----------------------------------- 5 files changed, 18 insertions(+), 75 deletions(-) delete mode 100644 website/views/robots.py diff --git a/makeabilitylab/urls.py b/makeabilitylab/urls.py index 0c62e369..fb505a91 100644 --- a/makeabilitylab/urls.py +++ b/makeabilitylab/urls.py @@ -26,18 +26,21 @@ from django.views.static import serve from django.conf import settings -from website import views from website.sitemaps import sitemaps urlpatterns = [ re_path(r'^admin/', admin.site.urls), - # SEO endpoints (issue #1252). Declared before the website.urls include so - # the app's patterns can't shadow them. The sitemap is generated from our - # querysets (see website/sitemaps.py); robots.txt is environment-aware. + # Dynamic sitemap (issue #1252), generated from our querysets (see + # website/sitemaps.py). Declared before the website.urls include so the + # app's patterns can't shadow it. + # + # NOTE: robots.txt is intentionally NOT routed here. On the servers Apache + # serves the static ./robots.txt from the project checkout and never + # forwards /robots.txt to Django, so a view here would be dead code. Edit + # the top-level ./robots.txt to change crawler rules or the Sitemap line. path('sitemap.xml', sitemap, {'sitemaps': sitemaps}, name='django.contrib.sitemaps.views.sitemap'), - path('robots.txt', views.robots_txt, name='robots_txt'), #Info on how to route root to website was found here http://stackoverflow.com/questions/7580220/django-urls-howto-map-root-to-app re_path(r'', include('website.urls')), diff --git a/robots.txt b/robots.txt index 8093c87b..d3785243 100644 --- a/robots.txt +++ b/robots.txt @@ -1,2 +1,4 @@ -User-agent: * -Disallow: \ No newline at end of file +User-agent: * +Disallow: + +Sitemap: https://makeabilitylab.cs.washington.edu/sitemap.xml diff --git a/website/tests/test_sitemap.py b/website/tests/test_sitemap.py index a374363d..9698b57a 100644 --- a/website/tests/test_sitemap.py +++ b/website/tests/test_sitemap.py @@ -1,14 +1,14 @@ """ -Regression tests for the dynamic sitemap and robots.txt (issue #1252). +Regression tests for the dynamic sitemap (issue #1252). -Both endpoints are exercised through the real URL/view stack so a routing or -queryset regression is caught. See website/sitemaps.py and -website/views/robots.py. +The sitemap is exercised through the real URL/view stack so a routing or +queryset regression is caught. See website/sitemaps.py. + +Note: robots.txt is a static file (./robots.txt) served by Apache on the +servers, not a Django view, so it isn't covered here. """ -import os from datetime import date -from unittest import mock from website.tests.base import DatabaseTestCase @@ -58,23 +58,3 @@ def test_sitemap_includes_news_item(self): news = self.make_news_item(title="Big Lab News") body = self.client.get("/sitemap.xml").content.decode() self.assertIn(f"/news/{news.slug}/", body) - - -class RobotsTxtTests(DatabaseTestCase): - def test_robots_is_plain_text(self): - resp = self.client.get("/robots.txt") - self.assertEqual(resp.status_code, 200) - self.assertEqual(resp["Content-Type"], "text/plain") - - @mock.patch.dict(os.environ, {"DJANGO_ENV": "PROD"}) - def test_robots_prod_allows_and_advertises_sitemap(self): - body = self.client.get("/robots.txt").content.decode() - self.assertIn("Allow: /", body) - self.assertIn("Sitemap:", body) - self.assertIn("/sitemap.xml", body) - - @mock.patch.dict(os.environ, {"DJANGO_ENV": "TEST"}) - def test_robots_non_prod_disallows_all(self): - body = self.client.get("/robots.txt").content.decode() - self.assertIn("Disallow: /", body) - self.assertNotIn("Allow: /", body) diff --git a/website/views/__init__.py b/website/views/__init__.py index 84a3ee9d..8e5915f9 100644 --- a/website/views/__init__.py +++ b/website/views/__init__.py @@ -9,5 +9,4 @@ from .view_project_people import * from .serve_pdf import * from .awards import awards -from .robots import robots_txt diff --git a/website/views/robots.py b/website/views/robots.py deleted file mode 100644 index 765ad5ae..00000000 --- a/website/views/robots.py +++ /dev/null @@ -1,41 +0,0 @@ -""" -Serves /robots.txt dynamically (issue #1252). - -We serve this from Django (not as a static file at the web-server root) because -the maintainer has no web-server access — routing it through a view is the only -control we have, and it lets us vary behavior by environment. - -Two behaviors: - - PROD: allow crawling and advertise the sitemap. - - Everything else (notably the TEST server, DJANGO_ENV=TEST): disallow all - crawling so the test site is never indexed and can't compete with the - production site in search results. - -The sitemap URL is built from the request host, so it points at whatever domain -the request came in on (prod / test / localhost). -""" - -import os - -from django.http import HttpResponse - - -def robots_txt(request): - """Return an environment-appropriate robots.txt as text/plain.""" - sitemap_url = request.build_absolute_uri("/sitemap.xml") - - if os.environ.get("DJANGO_ENV") == "PROD": - lines = [ - "User-agent: *", - "Allow: /", - "", - f"Sitemap: {sitemap_url}", - ] - else: - # Test / dev: keep the whole site out of search indexes. - lines = [ - "User-agent: *", - "Disallow: /", - ] - - return HttpResponse("\n".join(lines) + "\n", content_type="text/plain")