Vector search performance enhancements

frontends/main/src/app/(site)/search/page.test.tsx

@@ -0,0 +1,77 @@
+import { factories, makeRequest, setMockResponse, urls } from "api/test-utils"
+import Page from "./page"
+
+jest.mock("@/app/getQueryClient", () => {
+  const { makeBrowserQueryClient } = jest.requireActual("@/app/getQueryClient")
+  return { getQueryClient: () => makeBrowserQueryClient({ maxRetries: 0 }) }
+})
+
+const SEARCH_RESPONSE = {
+  count: 0,
+  next: null,
+  previous: null,
+  results: [],
+  metadata: {
+    aggregations: {},
+    suggestions: [],
+  },
+}
+
+beforeEach(() => {
+  setMockResponse.get(
+    urls.offerors.list(),
+    factories.learningResources.offerors({ count: 0 }),
+  )
+  setMockResponse.get(expect.stringContaining(urls.search.resources()), {
+    ...SEARCH_RESPONSE,
+  })
+  setMockResponse.get(expect.stringContaining(urls.search.vectorResources()), {
+    ...SEARCH_RESPONSE,
+  })
+})
+
+test("prefetches OpenSearch results by default", async () => {
+  await Page({
+    params: Promise.resolve({}),
+    searchParams: Promise.resolve({ q: "test" }),
+  })
+
+  expect(
+    makeRequest.mock.calls.some(([args]) =>
+      args.url.startsWith(urls.search.resources()),
+    ),
+  ).toBe(true)
+  expect(
+    makeRequest.mock.calls.some(([args]) =>
+      args.url.startsWith(urls.search.vectorResources()),
+    ),
+  ).toBe(false)
+})
+
+test("prefetches vector results when vector_search is enabled", async () => {
+  await Page({
+    params: Promise.resolve({}),
+    searchParams: Promise.resolve({
+      q: "test",
+      vector_search: "true",
+      topic: "Physics",
+    }),
+  })
+
+  const vectorCall = makeRequest.mock.calls.find(([args]) =>
+    args.url.startsWith(urls.search.vectorResources()),
+  )
+  expect(vectorCall).toBeDefined()
+  expect(
+    makeRequest.mock.calls.some(([args]) =>
+      args.url.startsWith(urls.search.resources()),
+    ),
+  ).toBe(false)
+
+  const searchParams = new URL(vectorCall?.[0].url ?? "").searchParams
+  expect(searchParams.get("hybrid_search")).toBe("true")
+  expect(searchParams.get("q")).toBe("test")
+  expect(searchParams.has("topic")).toBe(false)
+  expect(searchParams.has("limit")).toBe(false)
+  expect(searchParams.has("offset")).toBe(false)
+})

frontends/main/src/app/(site)/search/page.tsx

@@ -11,6 +11,10 @@ import {
   getExtraFacetNames,
 } from "@/app-pages/SearchPage/searchRequests"
 import getSearchParams from "@/page-components/SearchDisplay/getSearchParams"
+import {
+  toUnfacetedVectorSearchParams,
+  toVectorSearchParams,
+} from "@/page-components/SearchDisplay/vectorSearchParams"
 import validateRequestParams from "@/page-components/SearchDisplay/validateRequestParams"
 import type { ResourceSearchRequest } from "@/page-components/SearchDisplay/validateRequestParams"
 import { LearningResourcesSearchApiLearningResourcesSearchRetrieveRequest as LRSearchRequest } from "api"
@@ -50,13 +54,28 @@ const Page: React.FC<PageProps<"/search">> = async ({ searchParams }) => {
   })
 
   const queryClient = getQueryClient()
+  const isVectorSearch = urlParams.get("vector_search") === "true"
+  const hasSearchTerm = typeof params.q === "string" && params.q.trim() !== ""
 
-  await Promise.all([
-    queryClient.prefetchQuery(offerorQueries.list({})),
-    queryClient.prefetchQuery(
-      learningResourceQueries.search(params as LRSearchRequest),
-    ),
-  ])
+  if (isVectorSearch) {
+    await Promise.all([
+      queryClient.prefetchQuery(offerorQueries.list({})),
+      queryClient.prefetchQuery(
+        learningResourceQueries.vectorSearch(
+          hasSearchTerm
+            ? toUnfacetedVectorSearchParams(params)
+            : toVectorSearchParams(params),
+        ),
+      ),
+    ])
+  } else {
+    await Promise.all([
+      queryClient.prefetchQuery(offerorQueries.list({})),
+      queryClient.prefetchQuery(
+        learningResourceQueries.search(params as LRSearchRequest),
+      ),
+    ])
+  }
 
   return (
     <HydrationBoundary state={dehydrate(queryClient)}>

frontends/main/src/page-components/SearchDisplay/HybridSearchDisplay.tsx

@@ -1,107 +1,14 @@
 import React, { useMemo } from "react"
 import { learningResourceQueries } from "api/hooks/learningResources"
 import type { LearningResource } from "api"
-import type { Facets, BooleanFacets } from "@mitodl/course-search-utils"
-import type {
-  LearningResourcesVectorSearchResponse,
-  VectorLearningResourcesSearchApiVectorLearningResourcesSearchRetrieveRequest as VectorSearchRequest,
-} from "api/v0"
+import type { LearningResourcesVectorSearchResponse } from "api/v0"
 import getSearchParams from "./getSearchParams"
 import SearchDisplay, { SearchDisplayProps } from "./SearchDisplay"
-
-const mapVectorSortby = (
-  sortby?: string,
-): VectorSearchRequest["sortby"] | undefined => {
-  switch (sortby) {
-    case "-views":
-    case "popular":
-      return "-views"
-    case "upcoming":
-      return "next_start_date"
-    case "new":
-      return "-created_on"
-    default:
-      return undefined
-  }
-}
-
-/**
- * Extracts only the fields supported by the vector search API from a broader
- * search params object, dropping admin-only params (e.g., content_file_score_weight)
- * that the vector endpoint does not accept.
- *
- * The `as` casts for enum arrays are safe because the v0 and v1 generated
- * clients define separate (but structurally identical) enum types for the same
- * string-literal values (e.g., delivery: 'online' | 'hybrid' | ...).
- */
-const toVectorSearchParams = (
-  params: ReturnType<typeof getSearchParams> & { sortby?: string },
-  cutoffScore?: number,
-): VectorSearchRequest => ({
-  aggregations: params.aggregations as VectorSearchRequest["aggregations"],
-  certification: params.certification,
-  certification_type:
-    params.certification_type as VectorSearchRequest["certification_type"],
-  course_feature: params.course_feature,
-  delivery: params.delivery as VectorSearchRequest["delivery"],
-  department: params.department as VectorSearchRequest["department"],
-  free: params.free,
-  level: params.level as VectorSearchRequest["level"],
-  limit: params.limit,
-  ocw_topic: params.ocw_topic,
-  offered_by: params.offered_by as VectorSearchRequest["offered_by"],
-  offset: params.offset,
-  platform: params.platform as VectorSearchRequest["platform"],
-  professional: params.professional,
-  q: params.q,
-  resource_category:
-    params.resource_category as VectorSearchRequest["resource_category"],
-  resource_type: params.resource_type as VectorSearchRequest["resource_type"],
-  resource_type_group:
-    params.resource_type_group as VectorSearchRequest["resource_type_group"],
-  score_cutoff: cutoffScore,
-  sortby: mapVectorSortby(params.sortby),
-  topic: params.topic,
-  hybrid_search: true,
-})
-
-const VECTOR_CLIENT_FILTER_FACETS = [
-  "resource_type",
-  "certification_type",
-  "delivery",
-  "department",
-  "topic",
-  "offered_by",
-  "free",
-  "professional",
-  "resource_category",
-  "resource_type_group",
-  "level",
-  "platform",
-  "course_feature",
-] as const
-
-type VectorClientFilterFacet = (typeof VECTOR_CLIENT_FILTER_FACETS)[number]
-
-const toUnfacetedVectorSearchParams = (
-  params: ReturnType<typeof getSearchParams> & { sortby?: string },
-  constantSearchParams: Facets & BooleanFacets = {},
-  cutoffScore?: number,
-): VectorSearchRequest => {
-  const {
-    offset: _offset,
-    limit: _limit,
-    ...vectorParams
-  } = toVectorSearchParams(params, cutoffScore)
-
-  return Object.fromEntries(
-    Object.entries(vectorParams).filter(
-      ([key]) =>
-        !VECTOR_CLIENT_FILTER_FACETS.includes(key as VectorClientFilterFacet) ||
-        key in constantSearchParams,
-    ),
-  ) as VectorSearchRequest
-}
+import {
+  VECTOR_CLIENT_FILTER_FACETS,
+  toUnfacetedVectorSearchParams,
+  toVectorSearchParams,
+} from "./vectorSearchParams"
 
 const normalizeParamValues = (value: unknown): string[] => {
   if (Array.isArray(value)) {

frontends/main/src/page-components/SearchDisplay/vectorSearchParams.ts

@@ -0,0 +1,97 @@
+import type { Facets, BooleanFacets } from "@mitodl/course-search-utils"
+import type { VectorLearningResourcesSearchApiVectorLearningResourcesSearchRetrieveRequest as VectorSearchRequest } from "api/v0"
+import getSearchParams from "./getSearchParams"
+
+const mapVectorSortby = (
+  sortby?: string,
+): VectorSearchRequest["sortby"] | undefined => {
+  switch (sortby) {
+    case "-views":
+    case "popular":
+      return "-views"
+    case "upcoming":
+      return "next_start_date"
+    case "new":
+      return "-created_on"
+    default:
+      return undefined
+  }
+}
+
+/**
+ * Extracts only the fields supported by the vector search API from a broader
+ * search params object, dropping admin-only params (e.g., content_file_score_weight)
+ * that the vector endpoint does not accept.
+ *
+ * The `as` casts for enum arrays are safe because the v0 and v1 generated
+ * clients define separate (but structurally identical) enum types for the same
+ * string-literal values (e.g., delivery: 'online' | 'hybrid' | ...).
+ */
+export const toVectorSearchParams = (
+  params: ReturnType<typeof getSearchParams> & { sortby?: string },
+  cutoffScore?: number,
+): VectorSearchRequest => ({
+  aggregations: params.aggregations as VectorSearchRequest["aggregations"],
+  certification: params.certification,
+  certification_type:
+    params.certification_type as VectorSearchRequest["certification_type"],
+  course_feature: params.course_feature,
+  delivery: params.delivery as VectorSearchRequest["delivery"],
+  department: params.department as VectorSearchRequest["department"],
+  free: params.free,
+  level: params.level as VectorSearchRequest["level"],
+  limit: params.limit,
+  ocw_topic: params.ocw_topic,
+  offered_by: params.offered_by as VectorSearchRequest["offered_by"],
+  offset: params.offset,
+  platform: params.platform as VectorSearchRequest["platform"],
+  professional: params.professional,
+  q: params.q,
+  resource_category:
+    params.resource_category as VectorSearchRequest["resource_category"],
+  resource_type: params.resource_type as VectorSearchRequest["resource_type"],
+  resource_type_group:
+    params.resource_type_group as VectorSearchRequest["resource_type_group"],
+  score_cutoff: cutoffScore,
+  sortby: mapVectorSortby(params.sortby),
+  topic: params.topic,
+  hybrid_search: true,
+})
+
+export const VECTOR_CLIENT_FILTER_FACETS = [
+  "resource_type",
+  "certification_type",
+  "delivery",
+  "department",
+  "topic",
+  "offered_by",
+  "free",
+  "professional",
+  "resource_category",
+  "resource_type_group",
+  "level",
+  "platform",
+  "course_feature",
+] as const
+
+type VectorClientFilterFacet = (typeof VECTOR_CLIENT_FILTER_FACETS)[number]
+
+export const toUnfacetedVectorSearchParams = (
+  params: ReturnType<typeof getSearchParams> & { sortby?: string },
+  constantSearchParams: Facets & BooleanFacets = {},
+  cutoffScore?: number,
+): VectorSearchRequest => {
+  const {
+    offset: _offset,
+    limit: _limit,
+    ...vectorParams
+  } = toVectorSearchParams(params, cutoffScore)
+
+  return Object.fromEntries(
+    Object.entries(vectorParams).filter(
+      ([key]) =>
+        !VECTOR_CLIENT_FILTER_FACETS.includes(key as VectorClientFilterFacet) ||
+        key in constantSearchParams,
+    ),
+  ) as VectorSearchRequest
+}

main/settings.py

@@ -829,6 +829,13 @@ def get_all_config_keys():
 # hard limit for special cases where we need to return all results without pagination
 VECTOR_SEARCH_PAGE_MAX_LIMIT = get_int("VECTOR_SEARCH_PAGE_MAX_LIMIT", 200)
 
+# serve learning resource search hits from the Qdrant payload instead of
+# re-hydrating them from the database. Set to False to fall back to database
+# hydration without a deploy.
+VECTOR_SEARCH_RESOURCES_FROM_PAYLOAD = get_bool(
+    name="VECTOR_SEARCH_RESOURCES_FROM_PAYLOAD", default=True
+)
+
 # toggle to use requests (default for local) or webdriver which renders js elements
 EMBEDDINGS_EXTERNAL_FETCH_USE_WEBDRIVER = get_bool(
     "EMBEDDINGS_EXTERNAL_FETCH_USE_WEBDRIVER", default=False

vector_search/constants.py

@@ -149,6 +149,33 @@
 CONTENT_FILES_RETRIEVE_PAYLOAD = True
 RESOURCES_RETRIEVE_PAYLOAD = ["readable_id", "platform"]
 
+# Payload keys dropped when resource hits are served straight from the Qdrant
+# payload (VECTOR_SEARCH_RESOURCES_FROM_PAYLOAD): what the indexing serializer
+# adds on top of the LearningResourceSerializer shape the API returns, plus
+# video.transcript, which the response never renders.
+#
+# content_files is NOT excluded. Document and video responses declare it
+# (NestedContentFileSerializer), and search cards fall back to
+# content_files[0].image_src for the thumbnail when the resource has no image.
+# The indexing serializer re-serializes it with the *full* ContentFileSerializer,
+# so its large text fields are trimmed in Python instead -- see
+# _trim_indexing_only_list_fields.
+RESOURCES_PAYLOAD_EXCLUDE = [
+    "_id",
+    "resource_relations",
+    "is_learning_material",
+    "resource_age_date",
+    "featured_rank",
+    "is_incomplete_or_stale",
+    "vector_embedding",
+    "video.transcript",
+]
+
+# Qdrant payload selectors descend into objects but not into lists of objects,
+# so the extra fields SearchCourseNumberSerializer puts on each course number
+# cannot be named in RESOURCES_PAYLOAD_EXCLUDE and are trimmed in Python.
+COURSE_NUMBER_INDEXING_ONLY_FIELDS = frozenset({"sort_coursenum", "primary"})
+
 
 COLLECTION_PARAM_MAP = {
     RESOURCES_COLLECTION_NAME: QDRANT_RESOURCE_PARAM_MAP,