Snippets let you reuse content.
"}] + ), + ) + query = self._run_and_capture_query(page) + + self.assertIn("Manage snippets", query) + self.assertIn("Reusable content elements.", query) + self.assertIn("Snippets let you reuse content.", query) + + def test_falls_back_to_content_when_current_page_not_found(self): + captured = [] + + class RecordingIndex(FakeIndex): + def search_sources(self, query, *, locale=None, **kwargs): + captured.append(query) + return FakeQuerySet([]) + + with mock.patch( + "apps.core.agents.index_registry.get", + return_value=lambda: RecordingIndex([]), + ): + result = self.agent.execute( + vector_index="PageIndex", + exclude_pks=["999999"], + content="Fallback preview content", + chunk_size=100, + ) + + self.assertEqual(captured[0], "Fallback preview content") + self.assertEqual(result, []) + + def test_excludes_pages_in_exclude_pks(self): + page = self._make_page(title="Source page") + already_related = self._make_page(title="Already added") + + fake_index = FakeIndex([already_related]) + with mock.patch( + "apps.core.agents.index_registry.get", return_value=lambda: fake_index + ): + result = self.agent.execute( + vector_index="PageIndex", + exclude_pks=[str(page.pk), str(already_related.pk)], + content="ignored", + ) + + self.assertEqual(result, []) + + def test_respects_limit(self): + page = self._make_page(title="Source page") + related = [self._make_page(title=f"Match {i}") for i in range(5)] + + result = self._run(page, related, limit=2) + + self.assertEqual(len(result), 2) + + def test_preserves_order(self): + page = self._make_page(title="Source page") + first = self._make_page(title="First match") + second = self._make_page(title="Second match") + + result = self._run(page, [first, second]) + + self.assertEqual([r["title"] for r in result], ["First match", "Second match"]) + + def test_derives_locale_from_current_page_and_filters_by_it(self): + page = self._make_page(title="Source page") + filter_calls = [] + + class SpyingQuerySet(FakeQuerySet): + def filter(self, **kwargs): + filter_calls.append(kwargs) + return self + + class SpyingIndex(FakeIndex): + def search_sources(self, query, *, locale=None, **kwargs): + qs = SpyingQuerySet(self._pages) + if locale: + qs = qs.filter(locale=locale) + return qs + + with mock.patch( + "apps.core.agents.index_registry.get", return_value=lambda: SpyingIndex([]) + ): + self.agent.execute( + vector_index="PageIndex", + exclude_pks=[str(page.pk)], + content="ignored", + ) + + self.assertEqual(filter_calls, [{"locale": page.locale.language_code}]) diff --git a/apps/core/tests/test_related_pages.py b/apps/core/tests/test_related_pages.py new file mode 100644 index 00000000..76e948fd --- /dev/null +++ b/apps/core/tests/test_related_pages.py @@ -0,0 +1,74 @@ +from django.test import TestCase +from wagtail.models import PageViewRestriction + +from apps.core.factories import ContentPageFactory, HomePageFactory +from apps.core.models import RelatedPage + + +class TestRelatedPages(TestCase): + def setUp(self): + self.home = HomePageFactory() + self.source = ContentPageFactory(parent=self.home, title="Getting started") + self.related_1 = ContentPageFactory(parent=self.home, title="How-to guides") + self.related_2 = ContentPageFactory(parent=self.home, title="Concepts") + + RelatedPage.objects.create( + source_page=self.source, related_page=self.related_2, sort_order=0 + ) + RelatedPage.objects.create( + source_page=self.source, related_page=self.related_1, sort_order=1 + ) + + def _get_related_pages_html(self, response): + content = response.content.decode() + start = content.index('class="related-pages"') + end = content.index("", start) + return content[start:end] + + def test_related_pages_render_in_curated_order_with_correct_urls(self): + html = self._get_related_pages_html(self.client.get(self.source.url)) + + self.assertLess( + html.index(self.related_2.title), html.index(self.related_1.title) + ) + self.assertIn(f'href="{self.related_1.url}"', html) + self.assertIn(f'href="{self.related_2.url}"', html) + + def test_no_related_pages_section_when_empty(self): + other_page = ContentPageFactory(parent=self.home, title="Lonely page") + content = self.client.get(other_page.url).content.decode() + + self.assertNotIn("related-pages", content) + + def test_draft_and_private_pages_are_excluded(self): + draft = ContentPageFactory(parent=self.home, title="Draft", live=False) + private = ContentPageFactory(parent=self.home, title="Private") + PageViewRestriction.objects.create( + page=private, restriction_type=PageViewRestriction.LOGIN + ) + RelatedPage.objects.create( + source_page=self.source, related_page=draft, sort_order=2 + ) + RelatedPage.objects.create( + source_page=self.source, related_page=private, sort_order=3 + ) + + content = self.client.get(self.source.url).content.decode() + + self.assertNotIn("Draft", content) + self.assertNotIn("Private", content) + + def test_related_pages_appear_in_markdown_export(self): + markdown = self.source.to_markdown() + + self.assertIn("## Related pages", markdown) + self.assertIn(f"[{self.related_1.title}]({self.related_1.full_url})", markdown) + + def test_home_page_related_pages_render(self): + RelatedPage.objects.create( + source_page=self.home, related_page=self.related_1, sort_order=0 + ) + + html = self._get_related_pages_html(self.client.get(self.home.url)) + + self.assertIn(self.related_1.title, html) diff --git a/apps/core/tests/test_vector_indexes.py b/apps/core/tests/test_vector_indexes.py new file mode 100644 index 00000000..711e1ec5 --- /dev/null +++ b/apps/core/tests/test_vector_indexes.py @@ -0,0 +1,100 @@ +import json +from unittest import mock + +from django.test import TestCase + +from apps.core.factories import ContentPageFactory, HomePageFactory +from apps.core.vector_indexes import PageIndex, PageSource, render_first_paragraph + + +class TestRenderFirstParagraph(TestCase): + def _page_with_body(self, blocks): + return ContentPageFactory(body=json.dumps(blocks)) + + def test_skips_alert_and_returns_first_text_block(self): + page = self._page_with_body( + [ + { + "type": "alert", + "value": { + "alert_type": "Note", + "alert_body": "An alert callout.
", + }, + }, + {"type": "text", "value": "The real content.
"}, + ] + ) + self.assertEqual(render_first_paragraph(page), "The real content.") + + def test_returns_text_annotated_block_content(self): + page = self._page_with_body( + [ + { + "type": "text_annotated", + "value": {"content": "Annotated paragraph.
"}, + } + ] + ) + self.assertEqual(render_first_paragraph(page), "Annotated paragraph.") + + +class TestPageSource(TestCase): + def setUp(self): + self.home = HomePageFactory() + + def _make_source(self): + from apps.core.models.content import ContentPage + + return PageSource(queryset=ContentPage.objects.live().public()) + + def test_get_content_includes_title_description_and_body(self): + page = ContentPageFactory( + parent=self.home, + title="Managing snippets", + search_description="A short summary.", + body=json.dumps([{"type": "text", "value": "Snippet content.
"}]), + ) + content = self._make_source().get_content(page) + + self.assertIn("Managing snippets", content) + self.assertIn("A short summary.", content) + self.assertIn("Snippet content.", content) + + def test_get_content_strips_html_tags(self): + page = ContentPageFactory( + parent=self.home, + body=json.dumps( + [{"type": "text", "value": "Text with bold.
"}] + ), + ) + content = self._make_source().get_content(page) + + self.assertNotIn("", content) + self.assertIn("Text with bold.", content) + + def test_get_metadata_includes_locale(self): + page = ContentPageFactory(parent=self.home, title="Managing snippets") + metadata = self._make_source().get_metadata(page) + + self.assertEqual(metadata["locale"], page.locale.language_code) + + +class TestPageIndex(TestCase): + def test_search_sources_applies_locale_filter(self): + with mock.patch( + "apps.core.vector_indexes.get_llm_service" + ) as mock_get_llm_service: + mock_get_llm_service.return_value = mock.Mock() + index = PageIndex() + + fake_queryset = mock.Mock() + fake_queryset.filter.return_value = "filtered" + + with mock.patch( + "django_ai_core.contrib.index.base.VectorIndex.search_sources", + return_value=fake_queryset, + ): + result = index.search_sources("query", locale="en") + + fake_queryset.filter.assert_called_once_with(locale="en") + self.assertEqual(result, "filtered") diff --git a/apps/core/vector_indexes.py b/apps/core/vector_indexes.py new file mode 100644 index 00000000..b6c24044 --- /dev/null +++ b/apps/core/vector_indexes.py @@ -0,0 +1,85 @@ +from bs4 import BeautifulSoup +from django.template import Context, Template +from django_ai_core.contrib.index import VectorIndex, registry +from django_ai_core.contrib.index.chunking import ParagraphChunkTransformer +from django_ai_core.contrib.index.embedding import CoreEmbeddingTransformer +from django_ai_core.contrib.index.source import ModelSource +from django_ai_core.contrib.index.storage.pgvector import PgVectorProvider +from wagtail_ai.agents.base import get_llm_service + +from apps.core.models.content import ContentPage + + +def render_body_text(obj) -> str: + """Render a page's StreamField body to plain text (HTML tags stripped).""" + template = Template("{% load wagtailcore_tags %}{% include_block body %}") + html = template.render(Context({"body": obj.body})) + return BeautifulSoup(html, "lxml").get_text() + + +def render_first_paragraph(obj) -> str: + """ + Render the first substantive text block of a page's StreamField body, + skipping non-text blocks such as "alert" callouts, which tend to + restate the search description rather than add distinct content. + """ + for block in obj.body: + if block.block_type == "text": + html = str(block.value) + elif block.block_type == "text_annotated": + html = str(block.value["content"]) + else: + continue + text = BeautifulSoup(html, "lxml").get_text().strip() + if text: + return text + return "" + + +class PageSource(ModelSource): + """ModelSource that renders StreamField body to plain text for embeddings.""" + + def __init__(self, queryset=None, **kwargs): + super().__init__(queryset=queryset, **kwargs) + self.chunk_transformer = ParagraphChunkTransformer( + min_chunk_size=100, + max_chunk_size=1500, + ) + + def get_content(self, obj): + text = render_body_text(obj) + parts = [obj.title] + if obj.search_description: + parts.append(obj.search_description) + if obj.seo_title and obj.seo_title != obj.title: + parts.append(obj.seo_title) + parts.append(text) + return "\n\n".join(parts) + + def get_metadata(self, obj): + metadata = super().get_metadata(obj) + if obj.locale: + metadata["locale"] = obj.locale.language_code + return metadata + + +@registry.register() +class PageIndex(VectorIndex): + storage_provider = PgVectorProvider() + + def __init__(self): + self.sources = [ + PageSource( + queryset=ContentPage.objects.live().public(), + ) + ] + self.embedding_transformer = CoreEmbeddingTransformer( + get_llm_service("embedding") + ) + super().__init__() + + def search_sources(self, query, *, locale=None, **kwargs): + qs = super().search_sources(query, **kwargs) + if locale: + qs = qs.filter(locale=locale) + return qs diff --git a/apps/core/wagtail_hooks.py b/apps/core/wagtail_hooks.py index 49801f0c..9ddb1c68 100644 --- a/apps/core/wagtail_hooks.py +++ b/apps/core/wagtail_hooks.py @@ -9,8 +9,11 @@ from wagtail.models import Page from wagtail.snippets.models import register_snippet from wagtail.snippets.views.snippets import SnippetViewSet +from wagtail_ai.agents.suggested_content import SuggestedContentAgent # noqa: F401 from wagtail_ai.panels import AIDescriptionFieldPanel +from apps.core import vector_indexes # noqa: F401 +from apps.core.agents import LocalizedSuggestedContentAgent # noqa: F401 from apps.core.models.feedback import Feedback STYLE_GUIDE_URL = "https://github.com/wagtail/guide/blob/main/docs/style-guide.md" diff --git a/apps/frontend/static_src/scss/components/related-pages.scss b/apps/frontend/static_src/scss/components/related-pages.scss new file mode 100644 index 00000000..a41ad9e1 --- /dev/null +++ b/apps/frontend/static_src/scss/components/related-pages.scss @@ -0,0 +1,46 @@ +.related-pages { + margin-bottom: ($gutter * 3); + + &__heading { + @include fs(m); + font-weight: 700; + margin-bottom: ($gutter * 1.5); + } + + &__list { + list-style-type: none; + padding: 0; + margin: 0; + display: grid; + grid-template-columns: 1fr; + gap: $gutter; + + @include media-query(medium) { + grid-template-columns: repeat(2, 1fr); + } + } + + &__item { + margin: 0; + } + + &__link { + display: block; + padding: ($gutter * 0.75) ($gutter * 1.25); + border-radius: $border-radius--lg; + background: light-dark($color--light-grey, $color--off-black); + text-decoration: none; + transition: background $transition, transform $transition; + border-inline-start: 3px solid + light-dark($color--teal, $color--light-teal); + + @include fs(xs); + font-weight: 700; + color: light-dark($color--blue, $color--white); + + &:hover { + background: light-dark($color--extra-light-teal, $color--grey); + transform: translateY(-2px); + } + } +} diff --git a/apps/frontend/static_src/scss/main.scss b/apps/frontend/static_src/scss/main.scss index d9aa4964..4ffb0fa3 100644 --- a/apps/frontend/static_src/scss/main.scss +++ b/apps/frontend/static_src/scss/main.scss @@ -35,6 +35,7 @@ @import './components/link-block'; @import './components/nav-buttons'; @import './components/navigation'; +@import './components/related-pages'; @import './components/page-tools'; @import './components/primary-nav'; @import './components/rich-text'; diff --git a/apps/guide/settings/base.py b/apps/guide/settings/base.py index b6ef5eb1..ad8a1009 100644 --- a/apps/guide/settings/base.py +++ b/apps/guide/settings/base.py @@ -58,6 +58,9 @@ "wagtail", "modelcluster", "taggit", + "django_ai_core", + "django_ai_core.contrib.index", + "django_ai_core.contrib.index.storage.pgvector", "django.contrib.admin", "django.contrib.auth", "django.contrib.contenttypes", @@ -672,6 +675,14 @@ "api_key": os.environ.get("WAGTAIL_AI_VISION_API_KEY"), "api_base": os.environ.get("WAGTAIL_AI_VISION_API_BASE"), }, + "embedding": { + "provider": os.environ.get("WAGTAIL_AI_EMBEDDING_PROVIDER", "openai"), + "model": os.environ.get( + "WAGTAIL_AI_EMBEDDING_MODEL", "openai/text-embedding-3-large" + ), + "api_key": os.environ.get("WAGTAIL_AI_EMBEDDING_API_KEY"), + "api_base": os.environ.get("WAGTAIL_AI_EMBEDDING_API_BASE"), + }, }, "IMAGE_DESCRIPTION_PROVIDER": "vision", # Legacy configuration still in use for rich text integration. diff --git a/apps/llms_txt/jinja2/llms_txt/page.md.jinja b/apps/llms_txt/jinja2/llms_txt/page.md.jinja index 65625e3d..c9e08745 100644 --- a/apps/llms_txt/jinja2/llms_txt/page.md.jinja +++ b/apps/llms_txt/jinja2/llms_txt/page.md.jinja @@ -27,3 +27,13 @@ Page URL: {{ page.full_url }} {% endfor %} {%- endif -%} + +{%- if page.visible_related_pages %} + +## Related pages + +{% for related in page.visible_related_pages -%} +- [{{ related.title }}]({{ related.full_url }}) +{% endfor -%} +{%- endif %} + diff --git a/docker-compose.yml b/docker-compose.yml index 4668ebaf..a1eaf10e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,9 +26,9 @@ services: db: condition: service_healthy db: - image: postgres:17 - expose: - - 5432 + image: pgvector/pgvector:pg17 + ports: + - '5432:5432' environment: # Location of DB data folder PGDATA: /var/lib/postgresql/data/pgdata diff --git a/pyproject.toml b/pyproject.toml index 09e43f8f..ae3afd31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ dependencies = [ "draftjs-exporter>=7.1.0,<8.0", "jinja2>=3.1.6,<4.0.0", "wagtail-ai>=3.1.1,<4.0.0", + "pgvector>=0.3.6,<0.6.0", ] [dependency-groups] diff --git a/uv.lock b/uv.lock index 5e33c5b5..5b6b9b0c 100644 --- a/uv.lock +++ b/uv.lock @@ -642,6 +642,7 @@ dependencies = [ { name = "draftjs-exporter" }, { name = "jinja2" }, { name = "lxml" }, + { name = "pgvector" }, { name = "psycopg" }, { name = "sentry-sdk", extra = ["django"] }, { name = "wagtail" }, @@ -675,6 +676,7 @@ requires-dist = [ { name = "draftjs-exporter", specifier = ">=7.1.0,<8.0" }, { name = "jinja2", specifier = ">=3.1.6,<4.0.0" }, { name = "lxml", specifier = ">=4.9,<7" }, + { name = "pgvector", specifier = ">=0.3.6,<0.6.0" }, { name = "psycopg", specifier = ">=3.2.10,<3.3" }, { name = "sentry-sdk", extras = ["django"], specifier = ">=2.45.0,<3.0.0" }, { name = "wagtail", specifier = "==8.0rc1" }, @@ -1046,6 +1048,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, ] +[[package]] +name = "pgvector" +version = "0.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a7/ec/6eb80aebc728200f95229219882994c1b0585b956ca47da5edb9d062627a/pgvector-0.5.0.tar.gz", hash = "sha256:07a9dcf735696879406983afc6eba9a787cef7c0cf6c367ca1a5779f036dee74", size = 35170, upload-time = "2026-07-06T18:27:27.767Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/e4/a5573f2c579ca9ad133293bfb624148ba0893674ca4a6eeec85ced9a6a09/pgvector-0.5.0-py3-none-any.whl", hash = "sha256:fedc9800894e6da2be51358d7b7c574bf34f247ca741a5a09513622135f5964f", size = 30958, upload-time = "2026-07-06T18:27:26.797Z" }, +] + [[package]] name = "pillow" version = "12.3.0"