Avoid N+1 queries when rendering llms-full.txt

apps/llms_txt/tests/test_views.py

@@ -1,5 +1,9 @@
+import json
+
 from django.core.cache import cache
+from django.db import connection
 from django.test import TestCase
+from django.test.utils import CaptureQueriesContext
 from django.urls import reverse
 from wagtail.models import Site
 
@@ -68,6 +72,31 @@ def test_llms_full_txt_renders_pages(self):
         self.assertEqual(response["Content-Type"], "text/markdown;charset=utf-8")
         self.assertIn(self.content_page.title.encode(), response.content)
 
+    def test_llms_full_txt_does_not_fetch_body_per_page(self):
+        names = ("Second", "Third", "Fourth")
+        for name in names:
+            ContentPageFactory(
+                parent=self.home_page,
+                title=f"{name} page",
+                body=json.dumps(
+                    [{"type": "text", "value": f"<p>{name} page body content.</p>"}]
+                ),
+            )
+        cache.clear()
+
+        with CaptureQueriesContext(connection) as ctx:
+            response = self.client.get("/llms-full.txt")
+
+        self.assertEqual(response.status_code, 200)
+        body_queries = [
+            query
+            for query in ctx.captured_queries
+            if 'FROM "core_contentpage"' in query["sql"] and '"body"' in query["sql"]
+        ]
+        self.assertEqual(len(body_queries), 1)
+        for name in names:
+            self.assertIn(f"{name} page body content.".encode(), response.content)
+
     def test_responses_are_cached(self):
         for path, template_name in (
             ("/llms.txt", LLMS_TXT_TEMPLATE),

apps/llms_txt/views.py

@@ -42,15 +42,31 @@ def invalidate_cache():
             cache.delete(get_cache_key(site_pk, template_name))
 
 
-def _render_llms_txt(request, template_name):
-    sitemap = Sitemap(request)
+def _get_pages(site, defer_streamfields):
+    """Same pages as Sitemap.items(), with optional StreamField deferral.
+
+    llms-full.txt renders every page body, so deferring StreamFields there
+    would fetch each body with its own query.
+    """
+    pages = (
+        site.root_page.get_descendants(inclusive=True).live().public().order_by("path")
+    )
+    if defer_streamfields:
+        pages = pages.defer_streamfields()
+    return pages.specific()
+
+
+def _render_llms_txt(request, template_name, defer_streamfields=True):
     # Same site resolution as Sitemap.items(), so the cache key is in sync
     # with the pages being rendered.
-    site = sitemap.get_wagtail_site()
+    site = Sitemap(request).get_wagtail_site()
     key = get_cache_key(site.pk, template_name)
     content = cache.get(key)
     if content is None:
-        context = {"pages": sitemap.items(), "skill_name": SKILL_NAME}
+        context = {
+            "pages": _get_pages(site, defer_streamfields),
+            "skill_name": SKILL_NAME,
+        }
         content = loader.get_template(template_name).render(context, request)
         cache.set(key, content, timeout=CACHE_TIMEOUT)
     return HttpResponse(content, content_type=RESPONSE_CONTENT_TYPE)
@@ -98,4 +114,4 @@ def llms_txt_view(request):
 
 @cache_control(max_age=3600)
 def llms_full_txt_view(request):
-    return _render_llms_txt(request, LLMS_FULL_TXT_TEMPLATE)
+    return _render_llms_txt(request, LLMS_FULL_TXT_TEMPLATE, defer_streamfields=False)