From 3bdd716321416f58f3112b71174ce9ac057867cf Mon Sep 17 00:00:00 2001 From: Jon Froehlich Date: Mon, 15 Jun 2026 14:46:55 -0700 Subject: [PATCH] Add dynamic sitemap.xml and robots.txt (#1252) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generates /sitemap.xml from our querysets via django.contrib.sitemaps so the sitemap stays current with zero maintenance, and serves an environment-aware /robots.txt from a Django view (no web-server access needed). - settings: add 'django.contrib.sitemaps' to INSTALLED_APPS. Deliberately NOT installing django.contrib.sites — without it the framework falls back to RequestSite, deriving the domain from the request host, so the sitemap emits the correct domain across local/test/prod with no per-env config and no extra DB migration. - website/sitemaps.py: StaticViewSitemap (home/people/publications/projects/ awards/news), ProjectSitemap (is_visible only), PersonSitemap (people with a Position, excluding the placeholder url_name), NewsSitemap. lastmod from existing date fields; explicit ordering for stable pagination. Publications have no detail page, so they're covered by the static listing entry. - website/views/robots.py: PROD allows crawling and advertises the sitemap; every other environment (notably the TEST server, DJANGO_ENV=TEST) returns Disallow: / so the test site isn't indexed. - urls: wire sitemap.xml and robots.txt above the website.urls catch-all. - tests: 9 regression tests covering both endpoints. Co-Authored-By: Claude Opus 4.8 (1M context) --- makeabilitylab/settings.py | 8 +++ makeabilitylab/urls.py | 12 +++- website/sitemaps.py | 120 ++++++++++++++++++++++++++++++++++ website/tests/test_sitemap.py | 80 +++++++++++++++++++++++ website/views/__init__.py | 1 + website/views/robots.py | 41 ++++++++++++ 6 files changed, 261 insertions(+), 1 deletion(-) create mode 100644 website/sitemaps.py create mode 100644 website/tests/test_sitemap.py create mode 100644 website/views/robots.py diff --git a/makeabilitylab/settings.py b/makeabilitylab/settings.py index a468816e..f1a1f5ce 100644 --- a/makeabilitylab/settings.py +++ b/makeabilitylab/settings.py @@ -181,6 +181,14 @@ 'django.contrib.humanize', # for humanizing numbers in templates: https://docs.djangoproject.com/en/4.2/ref/contrib/humanize/ + # Generates a dynamic /sitemap.xml from our querysets for SEO (issue #1252). + # NOTE: we deliberately do NOT install django.contrib.sites — without it, + # the sitemap framework falls back to RequestSite, deriving the domain from + # the incoming request host. That makes the sitemap emit the correct domain + # across all three environments (local / test / prod) with no per-env config + # and no extra DB migration. See website/sitemaps.py. + 'django.contrib.sitemaps', + # Image handling = two cooperating pieces: easy-thumbnails resizes/scales, # while image_cropping lets editors pick the crop box. easy-thumbnails then # renders that box at any size on demand (see crop_corners in diff --git a/makeabilitylab/urls.py b/makeabilitylab/urls.py index b1ccfb42..0c62e369 100644 --- a/makeabilitylab/urls.py +++ b/makeabilitylab/urls.py @@ -21,14 +21,24 @@ from django.urls import include, re_path, path from django.contrib import admin +from django.contrib.sitemaps.views import sitemap from django.conf.urls.static import static from django.views.static import serve from django.conf import settings +from website import views +from website.sitemaps import sitemaps + urlpatterns = [ - + re_path(r'^admin/', admin.site.urls), + # SEO endpoints (issue #1252). Declared before the website.urls include so + # the app's patterns can't shadow them. The sitemap is generated from our + # querysets (see website/sitemaps.py); robots.txt is environment-aware. + path('sitemap.xml', sitemap, {'sitemaps': sitemaps}, name='django.contrib.sitemaps.views.sitemap'), + path('robots.txt', views.robots_txt, name='robots_txt'), + #Info on how to route root to website was found here http://stackoverflow.com/questions/7580220/django-urls-howto-map-root-to-app re_path(r'', include('website.urls')), # re_path(r'^admin/', admin.site.urls), diff --git a/website/sitemaps.py b/website/sitemaps.py new file mode 100644 index 00000000..7cb92455 --- /dev/null +++ b/website/sitemaps.py @@ -0,0 +1,120 @@ +""" +Sitemap classes that power the dynamic ``/sitemap.xml`` (issue #1252). + +Django's ``django.contrib.sitemaps`` framework builds the XML on every request +straight from our querysets, so the sitemap is always current — no static file +to regenerate when content changes. + +Domain handling: we do NOT use ``django.contrib.sites``. When it isn't +installed, the framework falls back to ``RequestSite``, which takes the domain +from the incoming request host. That means the same code emits +``makeabilitylab.cs.washington.edu`` in prod, ``makeabilitylab-test...`` on the +test server, and ``localhost`` in dev — no per-environment configuration. + +We map only the pages that have real, indexable URLs: + - static listing pages (home, people, publications, projects, awards, news) + - one entry per visible Project -> /project// + - one entry per public Person -> /member// + - one entry per News item -> /news// + +Publications have no per-publication detail page (only the ``/publications/`` +listing), so they are covered by the static sitemap and not enumerated here. +""" + +from django.contrib.sitemaps import Sitemap +from django.urls import reverse + +from website.models import Project, Person, News + + +class StaticViewSitemap(Sitemap): + """Top-level listing pages that aren't tied to a single model instance.""" + + changefreq = "weekly" + priority = 0.8 + + def items(self): + # URL names (in the ``website`` namespace) for the public landing pages. + return [ + "website:index", + "website:people", + "website:publications", + "website:projects", + "website:awards", + "website:news_listing", + ] + + def location(self, item): + return reverse(item) + + +class ProjectSitemap(Sitemap): + """Public project pages: /project//.""" + + changefreq = "weekly" + priority = 0.7 + + def items(self): + # Mirror the project view's visibility rule: only is_visible projects + # are reachable by the public. Explicit ordering keeps sitemap + # pagination stable (avoids UnorderedObjectListWarning). + return Project.objects.filter(is_visible=True).order_by("short_name") + + def location(self, obj): + # The project view resolves short_name (see website/views/project.py). + return reverse("website:project", args=[obj.short_name]) + + def lastmod(self, obj): + # auto_now DateField, updated on every save. + return obj.updated + + +class PersonSitemap(Sitemap): + """Public people pages: /member//.""" + + changefreq = "monthly" + priority = 0.6 + + def items(self): + # Anyone who has held a position appears on the /people/ page and has a + # public member page. Exclude the 'placeholder' default url_name (people + # whose url_name was never generated) since those won't resolve. + return ( + Person.objects.filter(position__isnull=False) + .exclude(url_name="placeholder") + .order_by("url_name") + .distinct() + ) + + def location(self, obj): + return reverse("website:member_by_name", args=[obj.url_name]) + + def lastmod(self, obj): + # May be None for people whose bio was never edited; the framework + # simply omits in that case. + return obj.bio_datetime_modified + + +class NewsSitemap(Sitemap): + """News item pages: /news//.""" + + changefreq = "monthly" + priority = 0.5 + + def items(self): + return News.objects.exclude(slug__isnull=True) + + def location(self, obj): + return reverse("website:news_item_by_slug", args=[obj.slug]) + + def lastmod(self, obj): + return obj.date + + +# Registry passed to django.contrib.sitemaps.views.sitemap in the root URLconf. +sitemaps = { + "static": StaticViewSitemap, + "projects": ProjectSitemap, + "people": PersonSitemap, + "news": NewsSitemap, +} diff --git a/website/tests/test_sitemap.py b/website/tests/test_sitemap.py new file mode 100644 index 00000000..a374363d --- /dev/null +++ b/website/tests/test_sitemap.py @@ -0,0 +1,80 @@ +""" +Regression tests for the dynamic sitemap and robots.txt (issue #1252). + +Both endpoints are exercised through the real URL/view stack so a routing or +queryset regression is caught. See website/sitemaps.py and +website/views/robots.py. +""" + +import os +from datetime import date +from unittest import mock + +from website.tests.base import DatabaseTestCase + + +class SitemapTests(DatabaseTestCase): + def _make_position(self, person): + """Give a Person a Position so it appears in the people sitemap.""" + from website.models import Position + from website.models.position import Title + return Position.objects.create( + person=person, start_date=date(2020, 1, 1), title=Title.PHD_STUDENT + ) + + def test_sitemap_returns_xml(self): + resp = self.client.get("/sitemap.xml") + self.assertEqual(resp.status_code, 200) + self.assertIn("xml", resp["Content-Type"]) + + def test_sitemap_includes_static_pages(self): + body = self.client.get("/sitemap.xml").content.decode() + # Listing pages should always be present. + self.assertIn("/publications/", body) + self.assertIn("/people/", body) + + def test_sitemap_includes_visible_project_excludes_private(self): + self.make_project(name="Visible Proj", short_name="visibleproj", + is_visible=True) + self.make_project(name="Private Proj", short_name="privateproj", + is_visible=False) + body = self.client.get("/sitemap.xml").content.decode() + self.assertIn("/project/visibleproj/", body) + self.assertNotIn("/project/privateproj/", body) + + def test_sitemap_includes_person_with_position(self): + person = self.make_person(first_name="Ada", last_name="Lovelace") + self._make_position(person) + body = self.client.get("/sitemap.xml").content.decode() + self.assertIn(f"/member/{person.url_name}/", body) + + def test_sitemap_excludes_person_without_position(self): + # No position => not on the public people page => not in the sitemap. + person = self.make_person(first_name="Grace", last_name="Hopper") + body = self.client.get("/sitemap.xml").content.decode() + self.assertNotIn(f"/member/{person.url_name}/", body) + + def test_sitemap_includes_news_item(self): + news = self.make_news_item(title="Big Lab News") + body = self.client.get("/sitemap.xml").content.decode() + self.assertIn(f"/news/{news.slug}/", body) + + +class RobotsTxtTests(DatabaseTestCase): + def test_robots_is_plain_text(self): + resp = self.client.get("/robots.txt") + self.assertEqual(resp.status_code, 200) + self.assertEqual(resp["Content-Type"], "text/plain") + + @mock.patch.dict(os.environ, {"DJANGO_ENV": "PROD"}) + def test_robots_prod_allows_and_advertises_sitemap(self): + body = self.client.get("/robots.txt").content.decode() + self.assertIn("Allow: /", body) + self.assertIn("Sitemap:", body) + self.assertIn("/sitemap.xml", body) + + @mock.patch.dict(os.environ, {"DJANGO_ENV": "TEST"}) + def test_robots_non_prod_disallows_all(self): + body = self.client.get("/robots.txt").content.decode() + self.assertIn("Disallow: /", body) + self.assertNotIn("Allow: /", body) diff --git a/website/views/__init__.py b/website/views/__init__.py index 8e5915f9..84a3ee9d 100644 --- a/website/views/__init__.py +++ b/website/views/__init__.py @@ -9,4 +9,5 @@ from .view_project_people import * from .serve_pdf import * from .awards import awards +from .robots import robots_txt diff --git a/website/views/robots.py b/website/views/robots.py new file mode 100644 index 00000000..765ad5ae --- /dev/null +++ b/website/views/robots.py @@ -0,0 +1,41 @@ +""" +Serves /robots.txt dynamically (issue #1252). + +We serve this from Django (not as a static file at the web-server root) because +the maintainer has no web-server access — routing it through a view is the only +control we have, and it lets us vary behavior by environment. + +Two behaviors: + - PROD: allow crawling and advertise the sitemap. + - Everything else (notably the TEST server, DJANGO_ENV=TEST): disallow all + crawling so the test site is never indexed and can't compete with the + production site in search results. + +The sitemap URL is built from the request host, so it points at whatever domain +the request came in on (prod / test / localhost). +""" + +import os + +from django.http import HttpResponse + + +def robots_txt(request): + """Return an environment-appropriate robots.txt as text/plain.""" + sitemap_url = request.build_absolute_uri("/sitemap.xml") + + if os.environ.get("DJANGO_ENV") == "PROD": + lines = [ + "User-agent: *", + "Allow: /", + "", + f"Sitemap: {sitemap_url}", + ] + else: + # Test / dev: keep the whole site out of search indexes. + lines = [ + "User-agent: *", + "Disallow: /", + ] + + return HttpResponse("\n".join(lines) + "\n", content_type="text/plain")