diff --git a/apps/llms_txt/tests/test_views.py b/apps/llms_txt/tests/test_views.py index 59d798ee..0fbd1829 100644 --- a/apps/llms_txt/tests/test_views.py +++ b/apps/llms_txt/tests/test_views.py @@ -1,5 +1,9 @@ +import json + from django.core.cache import cache +from django.db import connection from django.test import TestCase +from django.test.utils import CaptureQueriesContext from django.urls import reverse from wagtail.models import Site @@ -68,6 +72,31 @@ def test_llms_full_txt_renders_pages(self): self.assertEqual(response["Content-Type"], "text/markdown;charset=utf-8") self.assertIn(self.content_page.title.encode(), response.content) + def test_llms_full_txt_does_not_fetch_body_per_page(self): + names = ("Second", "Third", "Fourth") + for name in names: + ContentPageFactory( + parent=self.home_page, + title=f"{name} page", + body=json.dumps( + [{"type": "text", "value": f"
{name} page body content.
"}] + ), + ) + cache.clear() + + with CaptureQueriesContext(connection) as ctx: + response = self.client.get("/llms-full.txt") + + self.assertEqual(response.status_code, 200) + body_queries = [ + query + for query in ctx.captured_queries + if 'FROM "core_contentpage"' in query["sql"] and '"body"' in query["sql"] + ] + self.assertEqual(len(body_queries), 1) + for name in names: + self.assertIn(f"{name} page body content.".encode(), response.content) + def test_responses_are_cached(self): for path, template_name in ( ("/llms.txt", LLMS_TXT_TEMPLATE), diff --git a/apps/llms_txt/views.py b/apps/llms_txt/views.py index 4031d634..da006046 100644 --- a/apps/llms_txt/views.py +++ b/apps/llms_txt/views.py @@ -42,15 +42,31 @@ def invalidate_cache(): cache.delete(get_cache_key(site_pk, template_name)) -def _render_llms_txt(request, template_name): - sitemap = Sitemap(request) +def _get_pages(site, defer_streamfields): + """Same pages as Sitemap.items(), with optional StreamField deferral. + + llms-full.txt renders every page body, so deferring StreamFields there + would fetch each body with its own query. + """ + pages = ( + site.root_page.get_descendants(inclusive=True).live().public().order_by("path") + ) + if defer_streamfields: + pages = pages.defer_streamfields() + return pages.specific() + + +def _render_llms_txt(request, template_name, defer_streamfields=True): # Same site resolution as Sitemap.items(), so the cache key is in sync # with the pages being rendered. - site = sitemap.get_wagtail_site() + site = Sitemap(request).get_wagtail_site() key = get_cache_key(site.pk, template_name) content = cache.get(key) if content is None: - context = {"pages": sitemap.items(), "skill_name": SKILL_NAME} + context = { + "pages": _get_pages(site, defer_streamfields), + "skill_name": SKILL_NAME, + } content = loader.get_template(template_name).render(context, request) cache.set(key, content, timeout=CACHE_TIMEOUT) return HttpResponse(content, content_type=RESPONSE_CONTENT_TYPE) @@ -98,4 +114,4 @@ def llms_txt_view(request): @cache_control(max_age=3600) def llms_full_txt_view(request): - return _render_llms_txt(request, LLMS_FULL_TXT_TEMPLATE) + return _render_llms_txt(request, LLMS_FULL_TXT_TEMPLATE, defer_streamfields=False)