Agent fiendly 404s feedback

apps/core/models/home.py

@@ -8,7 +8,6 @@
 from apps.llms_txt.mixins import MarkdownRouteMixin
 
 from ..blocks import HOME_BLOCKS
-from ..views import Custom404
 
 
 class HomePage(MarkdownRouteMixin, Page):
@@ -33,14 +32,6 @@ class HomePage(MarkdownRouteMixin, Page):
         APIField("sections", writable=True),
     ]
 
-    def route(self, request, path_components):
-        try:
-            return super().route(request, path_components)
-        except Http404:
-            raise Custom404(
-                fallback_pages=self.get_fallback_pages(request, path_components)
-            )
-
     def get_fallback_pages(self, request, path_components):
         # No fallback for main locale.
         if self.locale.language_code == settings.LANGUAGE_CODE:

apps/core/tests/test_404.py

@@ -1,8 +1,9 @@
 from http import HTTPStatus
 
 from django.test import TestCase
+from wagtail.models import Site
 
-from apps.core.factories import HomePageFactory
+from apps.core.factories import ContentPageFactory, HomePageFactory, LocaleFactory
 
 # Unprefixed, so the request is first redirected to the `/en/` URL.
 MISSING_PATH = "/some-path-that-does-not-exist"
@@ -26,4 +27,35 @@ def test_markdown_when_requested(self):
         self.assertEqual(response["Content-Type"], "text/markdown;charset=utf-8")
         self.assertIn("Accept", response["Vary"])
         self.assertIn("private", response["Cache-Control"])
-        self.assertIn("[llms.txt](/llms.txt)", response.content.decode())
+        root_url = Site.objects.get().root_url
+        self.assertIn(f"[llms.txt]({root_url}/llms.txt)", response.content.decode())
+
+
+def create_english_only_page():
+    home_en = HomePageFactory(locale=LocaleFactory(language_code="en"))
+    page_en = ContentPageFactory(parent=home_en, slug="only-in-english")
+    home_de = home_en.copy_for_translation(LocaleFactory(language_code="de"))
+    home_de.save_revision().publish()
+    return page_en
+
+
+class TestPageNotFoundFallbackPages(TestCase):
+    def setUp(self):
+        self.page_en = create_english_only_page()
+
+    def test_html_lists_fallback_pages(self):
+        response = self.client.get("/de/only-in-english/")
+        self.assertEqual(response.status_code, HTTPStatus.NOT_FOUND)
+        self.assertContains(
+            response, self.page_en.url, status_code=HTTPStatus.NOT_FOUND
+        )
+
+    def test_markdown_lists_fallback_pages(self):
+        response = self.client.get(
+            "/de/only-in-english/", headers={"Accept": "text/markdown"}
+        )
+        self.assertEqual(response.status_code, HTTPStatus.NOT_FOUND)
+        self.assertIn(
+            f"[{self.page_en.title} - {self.page_en.locale}]({self.page_en.full_url})",
+            response.content.decode(),
+        )

apps/core/views.py

@@ -1,6 +1,6 @@
 from django.http import HttpResponseNotFound
-from django.http.response import Http404
 from django.template import loader
+from wagtail.models import Site
 
 from apps.llms_txt.negotiation import (
     MARKDOWN_CONTENT_TYPE,
@@ -9,21 +9,57 @@
 )
 
 
-class Custom404(Http404):
-    def __init__(self, *args, fallback_pages=None, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.fallback_pages = fallback_pages
+def get_fallback_pages(request, path):
+    """
+    Pages at `path` in other locales.
 
+    Wagtail swallows exceptions raised while routing and raises a plain
+    `Http404`, so these are looked up here rather than attached to it.
+    """
+    site = Site.find_for_request(request)
+    if site is None:
+        return []
 
-def page_not_found(request, exception):
+    root_page = site.root_page.localized.specific
+    if not hasattr(root_page, "get_fallback_pages"):
+        return []
+
+    path_components = [component for component in path.split("/") if component]
+    return root_page.get_fallback_pages(request, path_components)
+
+
+def get_page_path(request):
+    """
+    The page path for 404s raised from Wagtail's serve view, else `None`.
+    """
+    match = request.resolver_match
+    if match is None or match.url_name != "wagtail_serve":
+        return None
+    # Wagtail's serve URL pattern captures the page path as its only argument.
+    return match.args[0]
+
+
+def render_page_not_found(request, path=None):
+    fallback_pages = get_fallback_pages(request, path) if path is not None else []
+    context = {"fallback_pages": fallback_pages}
     is_markdown = prefers_markdown(request)
     if is_markdown:
-        body = loader.get_template("llms_txt/404.md.jinja").render({}, request)
+        body = loader.get_template("llms_txt/404.md.jinja").render(context, request)
         response = HttpResponseNotFound(body, content_type=MARKDOWN_CONTENT_TYPE)
     else:
-        context = {"fallback_pages": getattr(exception, "fallback_pages", None)}
         body = loader.get_template("404.html").render(context, request)
         response = HttpResponseNotFound(body)
 
     patch_negotiated_response(response, is_markdown)
     return response
+
+
+def page_not_found(request, exception):
+    return render_page_not_found(request, get_page_path(request))
+
+
+def preview_page_not_found(request, path=""):
+    """
+    Render the 404 handler as if `path` was not found, for local development.
+    """
+    return render_page_not_found(request, path)

apps/guide/urls.py

@@ -1,7 +1,7 @@
 from django.conf import settings
 from django.conf.urls.i18n import i18n_patterns
 from django.contrib import admin
-from django.urls import include, path
+from django.urls import include, path, re_path
 from django.views.generic import TemplateView
 from django.views.i18n import JavaScriptCatalog
 from wagtail import urls as wagtail_urls
@@ -10,6 +10,7 @@
 from wagtail.contrib.sitemaps.views import sitemap
 from wagtail.documents import urls as wagtaildocs_urls
 
+from apps.core.views import preview_page_not_found
 from apps.guide.api import api_router
 from apps.llms_txt import views as llms_txt_views
 from apps.search import views as search_views
@@ -45,17 +46,22 @@
     urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
 
     urlpatterns += [
-        # Add views for testing 404 and 500 templates
-        path(
-            "test404/",
-            TemplateView.as_view(template_name="404.html"),
-        ),
+        # Add a view for testing the 500 template
         path(
             "test500/",
             TemplateView.as_view(template_name="500.html"),
         ),
     ]
 
+    # DEBUG shows Django's technical 404 page instead of the 404 handler, so
+    # render the handler directly. Language-prefixed like real 404s, so it gets
+    # the same locale and content negotiation, e.g. `Accept: text/markdown`.
+    # The rest of the URL is the missing page path, e.g.
+    # `/de/test404/some-english-only-page/` lists fallback pages.
+    urlpatterns += i18n_patterns(
+        re_path(r"^test404/(?P<path>.*)$", preview_page_not_found),
+    )
+
 urlpatterns += i18n_patterns(
     path("search/", search_views.search, name="search"),
     path("search_json/", search_views.search_json, name="search_json"),

apps/llms_txt/jinja2/llms_txt/404.md.jinja

@@ -1,5 +1,12 @@
+{%- set root_url = wagtail_site().root_url -%}
 # 404 Page not found
 
 The page you requested does not exist or has moved.
+{% if fallback_pages %}
+It is available in other languages:
 
-Browse the documentation via [llms.txt](/llms.txt).
+{% for page in fallback_pages -%}
+- [{{ page.title }} - {{ page.locale }}]({{ page.full_url }})
+{% endfor -%}
+{% endif %}
+Browse the documentation via [llms.txt]({{ root_url }}{{ url("llms_txt") }}).