From e2564b1b99f6b25a405b2f48f31dfe899d8de05e Mon Sep 17 00:00:00 2001 From: Jon Froehlich Date: Mon, 15 Jun 2026 15:17:14 -0700 Subject: [PATCH] Pin sitemap URLs to https; bump to 2.8.2 (#1252) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Test-server validation of the sitemap surfaced that every was an http:// URL that 302-redirects to https. Cause: Apache terminates TLS and proxies to Django over plain HTTP, so RequestSite uses the http request scheme. Crawling all 189 URLs confirmed they resolve 200 after the redirect (no dead links) — but advertising non-canonical http URLs with an extra hop is bad for SEO. - sitemaps: add an _HttpsSitemap base (protocol = "https") and have all four sitemaps extend it, so URLs are canonical https. - tests: add a regression test asserting every uses https. - settings: bump 2.8.1 -> 2.8.2 and note the https fix + the 189/189 200 validation in the release description. Co-Authored-By: Claude Opus 4.8 (1M context) --- makeabilitylab/settings.py | 4 ++-- website/sitemaps.py | 23 +++++++++++++++++++---- website/tests/test_sitemap.py | 15 +++++++++++++++ 3 files changed, 36 insertions(+), 6 deletions(-) diff --git a/makeabilitylab/settings.py b/makeabilitylab/settings.py index 8e8905a1..6089dd7d 100644 --- a/makeabilitylab/settings.py +++ b/makeabilitylab/settings.py @@ -72,8 +72,8 @@ ALLOWED_HOSTS = ['*'] # Makeability Lab Global Variables, including Makeability Lab version -ML_WEBSITE_VERSION = "2.8.1" # Keep this updated with each release and also change the short description below -ML_WEBSITE_VERSION_DESCRIPTION = "SEO: add a dynamic sitemap.xml and advertise it in robots.txt (#1252). /sitemap.xml is generated on each request from our querysets via django.contrib.sitemaps — static listing pages (home, people, publications, projects, awards, news), visible projects, people with a position, and news items — so it stays current with no maintenance. It emits the correct per-environment domain via RequestSite, so no django.contrib.sites and no DB migration are needed. The top-level static robots.txt (served directly by Apache, not Django) now points crawlers at the sitemap. Remaining one-time step: submit the sitemap in Google Search Console once it is live on production." +ML_WEBSITE_VERSION = "2.8.2" # Keep this updated with each release and also change the short description below +ML_WEBSITE_VERSION_DESCRIPTION = "SEO: add a dynamic sitemap.xml and advertise it in robots.txt (#1252). /sitemap.xml is generated on each request from our querysets via django.contrib.sitemaps — static listing pages (home, people, publications, projects, awards, news), visible projects, people with a position, and news items — so it stays current with no maintenance. It emits the correct per-environment domain via RequestSite (no django.contrib.sites, no DB migration) and pins URLs to https so the sitemap lists canonical links rather than http URLs that 302-redirect. Validated on the test server: all 189 sitemap URLs return 200. The top-level static robots.txt (served directly by Apache, not Django) now points crawlers at the sitemap. Remaining one-time step: submit the sitemap in Google Search Console once it is live on production." DATE_MAKEABILITYLAB_FORMED = datetime.date(2012, 1, 1) # Date Makeability Lab was formed MAX_BANNERS = 7 # Maximum number of banners on a page diff --git a/website/sitemaps.py b/website/sitemaps.py index 7cb92455..58a893f5 100644 --- a/website/sitemaps.py +++ b/website/sitemaps.py @@ -27,7 +27,22 @@ from website.models import Project, Person, News -class StaticViewSitemap(Sitemap): +class _HttpsSitemap(Sitemap): + """ + Base sitemap that pins generated URLs to the https scheme. + + Apache terminates TLS and proxies to Django over plain HTTP, so the + request scheme Django sees is ``http``. Without this, RequestSite would + emit ``http://`` URLs that only 302-redirect to https — making the + sitemap advertise non-canonical URLs with an extra hop. Pinning the + protocol here makes every sitemap list the canonical https URLs directly. + (Cosmetic only in local dev, where the site is served over http.) + """ + + protocol = "https" + + +class StaticViewSitemap(_HttpsSitemap): """Top-level listing pages that aren't tied to a single model instance.""" changefreq = "weekly" @@ -48,7 +63,7 @@ def location(self, item): return reverse(item) -class ProjectSitemap(Sitemap): +class ProjectSitemap(_HttpsSitemap): """Public project pages: /project//.""" changefreq = "weekly" @@ -69,7 +84,7 @@ def lastmod(self, obj): return obj.updated -class PersonSitemap(Sitemap): +class PersonSitemap(_HttpsSitemap): """Public people pages: /member//.""" changefreq = "monthly" @@ -95,7 +110,7 @@ def lastmod(self, obj): return obj.bio_datetime_modified -class NewsSitemap(Sitemap): +class NewsSitemap(_HttpsSitemap): """News item pages: /news//.""" changefreq = "monthly" diff --git a/website/tests/test_sitemap.py b/website/tests/test_sitemap.py index 9698b57a..b1949f47 100644 --- a/website/tests/test_sitemap.py +++ b/website/tests/test_sitemap.py @@ -8,6 +8,7 @@ servers, not a Django view, so it isn't covered here. """ +import re from datetime import date from website.tests.base import DatabaseTestCase @@ -58,3 +59,17 @@ def test_sitemap_includes_news_item(self): news = self.make_news_item(title="Big Lab News") body = self.client.get("/sitemap.xml").content.decode() self.assertIn(f"/news/{news.slug}/", body) + + def test_sitemap_uses_https_scheme(self): + # Apache proxies to Django over plain HTTP, so without a pinned + # protocol the URLs would be http:// and only 302-redirect to + # https. Every must be canonical https. See _HttpsSitemap. + self.make_project(name="Scheme Proj", short_name="schemeproj", + is_visible=True) + body = self.client.get("/sitemap.xml").content.decode() + locs = re.findall(r"(.*?)", body) + self.assertTrue(locs) # guard against an empty sitemap passing vacuously + self.assertFalse( + [loc for loc in locs if not loc.startswith("https://")], + "all sitemap URLs should use the https scheme", + )