Add dynamic sitemap.xml and robots.txt (#1252)

makeabilitylab/settings.py

@@ -181,6 +181,14 @@
 
     'django.contrib.humanize', # for humanizing numbers in templates: https://docs.djangoproject.com/en/4.2/ref/contrib/humanize/
 
+    # Generates a dynamic /sitemap.xml from our querysets for SEO (issue #1252).
+    # NOTE: we deliberately do NOT install django.contrib.sites — without it,
+    # the sitemap framework falls back to RequestSite, deriving the domain from
+    # the incoming request host. That makes the sitemap emit the correct domain
+    # across all three environments (local / test / prod) with no per-env config
+    # and no extra DB migration. See website/sitemaps.py.
+    'django.contrib.sitemaps',
+
     # Image handling = two cooperating pieces: easy-thumbnails resizes/scales,
     # while image_cropping lets editors pick the crop box. easy-thumbnails then
     # renders that box at any size on demand (see crop_corners in

makeabilitylab/urls.py

@@ -21,14 +21,24 @@
 
 from django.urls import include, re_path, path
 from django.contrib import admin
+from django.contrib.sitemaps.views import sitemap
 from django.conf.urls.static import static
 from django.views.static import serve
 from django.conf import settings
 
+from website import views
+from website.sitemaps import sitemaps
+
 urlpatterns = [
-    
+
     re_path(r'^admin/', admin.site.urls),
 
+    # SEO endpoints (issue #1252). Declared before the website.urls include so
+    # the app's patterns can't shadow them. The sitemap is generated from our
+    # querysets (see website/sitemaps.py); robots.txt is environment-aware.
+    path('sitemap.xml', sitemap, {'sitemaps': sitemaps}, name='django.contrib.sitemaps.views.sitemap'),
+    path('robots.txt', views.robots_txt, name='robots_txt'),
+
     #Info on how to route root to website was found here http://stackoverflow.com/questions/7580220/django-urls-howto-map-root-to-app
     re_path(r'', include('website.urls')),
     # re_path(r'^admin/', admin.site.urls),

website/sitemaps.py

@@ -0,0 +1,120 @@
+"""
+Sitemap classes that power the dynamic ``/sitemap.xml`` (issue #1252).
+
+Django's ``django.contrib.sitemaps`` framework builds the XML on every request
+straight from our querysets, so the sitemap is always current — no static file
+to regenerate when content changes.
+
+Domain handling: we do NOT use ``django.contrib.sites``. When it isn't
+installed, the framework falls back to ``RequestSite``, which takes the domain
+from the incoming request host. That means the same code emits
+``makeabilitylab.cs.washington.edu`` in prod, ``makeabilitylab-test...`` on the
+test server, and ``localhost`` in dev — no per-environment configuration.
+
+We map only the pages that have real, indexable URLs:
+  - static listing pages (home, people, publications, projects, awards, news)
+  - one entry per visible Project        -> /project/<short_name>/
+  - one entry per public Person          -> /member/<url_name>/
+  - one entry per News item              -> /news/<slug>/
+
+Publications have no per-publication detail page (only the ``/publications/``
+listing), so they are covered by the static sitemap and not enumerated here.
+"""
+
+from django.contrib.sitemaps import Sitemap
+from django.urls import reverse
+
+from website.models import Project, Person, News
+
+
+class StaticViewSitemap(Sitemap):
+    """Top-level listing pages that aren't tied to a single model instance."""
+
+    changefreq = "weekly"
+    priority = 0.8
+
+    def items(self):
+        # URL names (in the ``website`` namespace) for the public landing pages.
+        return [
+            "website:index",
+            "website:people",
+            "website:publications",
+            "website:projects",
+            "website:awards",
+            "website:news_listing",
+        ]
+
+    def location(self, item):
+        return reverse(item)
+
+
+class ProjectSitemap(Sitemap):
+    """Public project pages: /project/<short_name>/."""
+
+    changefreq = "weekly"
+    priority = 0.7
+
+    def items(self):
+        # Mirror the project view's visibility rule: only is_visible projects
+        # are reachable by the public. Explicit ordering keeps sitemap
+        # pagination stable (avoids UnorderedObjectListWarning).
+        return Project.objects.filter(is_visible=True).order_by("short_name")
+
+    def location(self, obj):
+        # The project view resolves short_name (see website/views/project.py).
+        return reverse("website:project", args=[obj.short_name])
+
+    def lastmod(self, obj):
+        # auto_now DateField, updated on every save.
+        return obj.updated
+
+
+class PersonSitemap(Sitemap):
+    """Public people pages: /member/<url_name>/."""
+
+    changefreq = "monthly"
+    priority = 0.6
+
+    def items(self):
+        # Anyone who has held a position appears on the /people/ page and has a
+        # public member page. Exclude the 'placeholder' default url_name (people
+        # whose url_name was never generated) since those won't resolve.
+        return (
+            Person.objects.filter(position__isnull=False)
+            .exclude(url_name="placeholder")
+            .order_by("url_name")
+            .distinct()
+        )
+
+    def location(self, obj):
+        return reverse("website:member_by_name", args=[obj.url_name])
+
+    def lastmod(self, obj):
+        # May be None for people whose bio was never edited; the framework
+        # simply omits <lastmod> in that case.
+        return obj.bio_datetime_modified
+
+
+class NewsSitemap(Sitemap):
+    """News item pages: /news/<slug>/."""
+
+    changefreq = "monthly"
+    priority = 0.5
+
+    def items(self):
+        return News.objects.exclude(slug__isnull=True)
+
+    def location(self, obj):
+        return reverse("website:news_item_by_slug", args=[obj.slug])
+
+    def lastmod(self, obj):
+        return obj.date
+
+
+# Registry passed to django.contrib.sitemaps.views.sitemap in the root URLconf.
+sitemaps = {
+    "static": StaticViewSitemap,
+    "projects": ProjectSitemap,
+    "people": PersonSitemap,
+    "news": NewsSitemap,
+}

website/tests/test_sitemap.py

@@ -0,0 +1,80 @@
+"""
+Regression tests for the dynamic sitemap and robots.txt (issue #1252).
+
+Both endpoints are exercised through the real URL/view stack so a routing or
+queryset regression is caught. See website/sitemaps.py and
+website/views/robots.py.
+"""
+
+import os
+from datetime import date
+from unittest import mock
+
+from website.tests.base import DatabaseTestCase
+
+
+class SitemapTests(DatabaseTestCase):
+    def _make_position(self, person):
+        """Give a Person a Position so it appears in the people sitemap."""
+        from website.models import Position
+        from website.models.position import Title
+        return Position.objects.create(
+            person=person, start_date=date(2020, 1, 1), title=Title.PHD_STUDENT
+        )
+
+    def test_sitemap_returns_xml(self):
+        resp = self.client.get("/sitemap.xml")
+        self.assertEqual(resp.status_code, 200)
+        self.assertIn("xml", resp["Content-Type"])
+
+    def test_sitemap_includes_static_pages(self):
+        body = self.client.get("/sitemap.xml").content.decode()
+        # Listing pages should always be present.
+        self.assertIn("/publications/", body)
+        self.assertIn("/people/", body)
+
+    def test_sitemap_includes_visible_project_excludes_private(self):
+        self.make_project(name="Visible Proj", short_name="visibleproj",
+                          is_visible=True)
+        self.make_project(name="Private Proj", short_name="privateproj",
+                          is_visible=False)
+        body = self.client.get("/sitemap.xml").content.decode()
+        self.assertIn("/project/visibleproj/", body)
+        self.assertNotIn("/project/privateproj/", body)
+
+    def test_sitemap_includes_person_with_position(self):
+        person = self.make_person(first_name="Ada", last_name="Lovelace")
+        self._make_position(person)
+        body = self.client.get("/sitemap.xml").content.decode()
+        self.assertIn(f"/member/{person.url_name}/", body)
+
+    def test_sitemap_excludes_person_without_position(self):
+        # No position => not on the public people page => not in the sitemap.
+        person = self.make_person(first_name="Grace", last_name="Hopper")
+        body = self.client.get("/sitemap.xml").content.decode()
+        self.assertNotIn(f"/member/{person.url_name}/", body)
+
+    def test_sitemap_includes_news_item(self):
+        news = self.make_news_item(title="Big Lab News")
+        body = self.client.get("/sitemap.xml").content.decode()
+        self.assertIn(f"/news/{news.slug}/", body)
+
+
+class RobotsTxtTests(DatabaseTestCase):
+    def test_robots_is_plain_text(self):
+        resp = self.client.get("/robots.txt")
+        self.assertEqual(resp.status_code, 200)
+        self.assertEqual(resp["Content-Type"], "text/plain")
+
+    @mock.patch.dict(os.environ, {"DJANGO_ENV": "PROD"})
+    def test_robots_prod_allows_and_advertises_sitemap(self):
+        body = self.client.get("/robots.txt").content.decode()
+        self.assertIn("Allow: /", body)
+        self.assertIn("Sitemap:", body)
+        self.assertIn("/sitemap.xml", body)
+
+    @mock.patch.dict(os.environ, {"DJANGO_ENV": "TEST"})
+    def test_robots_non_prod_disallows_all(self):
+        body = self.client.get("/robots.txt").content.decode()
+        self.assertIn("Disallow: /", body)
+        self.assertNotIn("Allow: /", body)

website/views/__init__.py

@@ -9,4 +9,5 @@
 from .view_project_people import *
 from .serve_pdf import *
 from .awards import awards
+from .robots import robots_txt
 

website/views/robots.py

@@ -0,0 +1,41 @@
+"""
+Serves /robots.txt dynamically (issue #1252).
+
+We serve this from Django (not as a static file at the web-server root) because
+the maintainer has no web-server access — routing it through a view is the only
+control we have, and it lets us vary behavior by environment.
+
+Two behaviors:
+  - PROD: allow crawling and advertise the sitemap.
+  - Everything else (notably the TEST server, DJANGO_ENV=TEST): disallow all
+    crawling so the test site is never indexed and can't compete with the
+    production site in search results.
+
+The sitemap URL is built from the request host, so it points at whatever domain
+the request came in on (prod / test / localhost).
+"""
+
+import os
+
+from django.http import HttpResponse
+
+
+def robots_txt(request):
+    """Return an environment-appropriate robots.txt as text/plain."""
+    sitemap_url = request.build_absolute_uri("/sitemap.xml")
+
+    if os.environ.get("DJANGO_ENV") == "PROD":
+        lines = [
+            "User-agent: *",
+            "Allow: /",
+            "",
+            f"Sitemap: {sitemap_url}",
+        ]
+    else:
+        # Test / dev: keep the whole site out of search indexes.
+        lines = [
+            "User-agent: *",
+            "Disallow: /",
+        ]
+
+    return HttpResponse("\n".join(lines) + "\n", content_type="text/plain")