refactor(catalog): drop unused scryfall_printing_search FTS table

Search never queried the FTS5 table; it only cost disk space and a
full rebuild on every Scryfall catalog import. Remove the table, its
import row generation, and its backup entry. The migration is
reversible and repopulates the table from printings and cards on
rollback.

Author: cfbender

lib/manavault/backup.ex

@@ -13,7 +13,6 @@ defmodule Manavault.Backup do
   @manifest_name "manifest.json"
   @default_local_paths []
   @catalog_tables ~w(
-    scryfall_printing_search
     scryfall_syncs
     scryfall_printings
     scryfall_cards

lib/manavault/catalog/scryfall/import.ex

@@ -120,7 +120,6 @@ defmodule Manavault.Catalog.Scryfall.Import do
       fn ->
         insert_card_rows(rows.cards, replace_oracle_tag_fields?)
         insert_printing_rows(rows.printings)
-        refresh_printing_search_rows(rows.search_rows)
         {:ok, :imported}
       end,
       timeout: :infinity
@@ -132,7 +131,6 @@ defmodule Manavault.Catalog.Scryfall.Import do
       source_count: 0,
       cards_count: 0,
       printings_count: 0,
-      search_rows_count: 0,
       next_progress: @progress_source_card_interval
     }
   end
@@ -142,8 +140,7 @@ defmodule Manavault.Catalog.Scryfall.Import do
       counts
       | source_count: counts.source_count + source_count,
         cards_count: counts.cards_count + length(rows.cards),
-        printings_count: counts.printings_count + length(rows.printings),
-        search_rows_count: counts.search_rows_count + length(rows.search_rows)
+        printings_count: counts.printings_count + length(rows.printings)
     }
   end
 
@@ -212,8 +209,7 @@ defmodule Manavault.Catalog.Scryfall.Import do
        when processed >= next or processed == source_count do
     Logger.info(
       "Scryfall catalog import progress source_cards=#{processed}/#{source_count} " <>
-        "cards=#{counts.cards_count} printings=#{counts.printings_count} " <>
-        "search_rows=#{counts.search_rows_count}"
+        "cards=#{counts.cards_count} printings=#{counts.printings_count}"
     )
 
     %{counts | next_progress: next_progress_after(processed)}
@@ -254,67 +250,6 @@ defmodule Manavault.Catalog.Scryfall.Import do
     |> Enum.each(fn batch -> Repo.insert_all(schema, batch, opts) end)
   end
 
-  defp refresh_printing_search_rows([]), do: :ok
-
-  defp refresh_printing_search_rows(rows) do
-    rows
-    |> Enum.map(& &1.scryfall_id)
-    |> Enum.chunk_every(@batch_size)
-    |> Enum.each(fn ids ->
-      placeholders = Enum.map_join(ids, ",", fn _ -> "?" end)
-
-      Repo.query!(
-        "DELETE FROM scryfall_printing_search WHERE scryfall_id IN (#{placeholders})",
-        ids
-      )
-    end)
-
-    rows
-    |> Enum.chunk_every(@batch_size)
-    |> Enum.each(fn batch ->
-      values = Enum.map_join(batch, ",", fn _ -> "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" end)
-
-      params =
-        Enum.flat_map(batch, fn row ->
-          [
-            row.scryfall_id,
-            row.name,
-            row.compact_name,
-            row.flavor_name,
-            row.compact_flavor_name,
-            row.flavor_text,
-            row.compact_flavor_text,
-            row.type_line,
-            row.oracle_text,
-            row.compact_oracle_text,
-            row.set_code,
-            row.collector_number
-          ]
-        end)
-
-      Repo.query!(
-        """
-        INSERT INTO scryfall_printing_search (
-          scryfall_id,
-          name,
-          compact_name,
-          flavor_name,
-          compact_flavor_name,
-          flavor_text,
-          compact_flavor_text,
-          type_line,
-          oracle_text,
-          compact_oracle_text,
-          set_code,
-          collector_number
-        )
-        VALUES #{values}
-        """,
-        params
-      )
-    end)
-  end
-
   defp maybe_reconcile_printings(false, _imported_at), do: :ok
 
   defp maybe_reconcile_printings(true, imported_at) do
@@ -410,8 +345,6 @@ defmodule Manavault.Catalog.Scryfall.Import do
         |> Enum.chunk_every(@batch_size)
         |> Enum.each(&clear_trade_wants/1)
 
-        delete_printing_search_rows(stale_ids)
-
         Enum.each(Enum.chunk_every(stale_ids, @batch_size), fn ids ->
           Repo.delete_all(from printing in Printing, where: printing.scryfall_id in ^ids)
         end)
@@ -541,19 +474,6 @@ defmodule Manavault.Catalog.Scryfall.Import do
     end)
   end
 
-  defp delete_printing_search_rows([]), do: :ok
-
-  defp delete_printing_search_rows(ids) do
-    Enum.each(Enum.chunk_every(ids, @batch_size), fn batch ->
-      placeholders = Enum.map_join(batch, ",", fn _ -> "?" end)
-
-      Repo.query!(
-        "DELETE FROM scryfall_printing_search WHERE scryfall_id IN (#{placeholders})",
-        batch
-      )
-    end)
-  end
-
   defp import_timestamp(false), do: utc_now()
 
   defp import_timestamp(true) do

lib/manavault/catalog/scryfall/import_rows.ex

@@ -5,19 +5,17 @@ defmodule Manavault.Catalog.Scryfall.ImportRows do
   alias Manavault.Catalog.Search.NameMatch
 
   def rows(cards, now, oracle_tag_index) when is_list(cards) do
-    {card_rows, printing_rows, search_rows} =
-      Enum.reduce(cards, {[], [], []}, fn card, {card_rows, printing_rows, search_rows} ->
+    {card_rows, printing_rows} =
+      Enum.reduce(cards, {[], []}, fn card, {card_rows, printing_rows} ->
         {
           prepend_rows(card_row(card, now, oracle_tag_index), card_rows),
-          prepend_rows(printing_row(card, now), printing_rows),
-          prepend_rows(printing_search_row(card), search_rows)
+          prepend_rows(printing_row(card, now), printing_rows)
         }
       end)
 
     %{
       cards: :lists.reverse(card_rows),
-      printings: :lists.reverse(printing_rows),
-      search_rows: :lists.reverse(search_rows)
+      printings: :lists.reverse(printing_rows)
     }
   end
 
@@ -29,10 +27,6 @@ defmodule Manavault.Catalog.Scryfall.ImportRows do
     Enum.flat_map(cards, &printing_row(&1, now))
   end
 
-  def printing_search_rows(cards) when is_list(cards) do
-    Enum.flat_map(cards, &printing_search_row/1)
-  end
-
   defp prepend_rows(rows, acc) do
     Enum.reduce(rows, acc, fn row, rows -> [row | rows] end)
   end
@@ -96,43 +90,6 @@ defmodule Manavault.Catalog.Scryfall.ImportRows do
 
   defp printing_row(_card, _now), do: []
 
-  defp printing_search_row(%{"id" => scryfall_id, "name" => name} = card)
-       when is_binary(scryfall_id) and is_binary(name) do
-    oracle_text = oracle_text(card) || ""
-
-    [
-      %{
-        scryfall_id: scryfall_id,
-        name: normalize_search_text(name),
-        compact_name: compact_search_text(name),
-        flavor_name: normalize_search_text(flavor_name(card) || ""),
-        compact_flavor_name: compact_search_text(flavor_name(card) || ""),
-        flavor_text: normalize_search_text(flavor_text(card) || ""),
-        compact_flavor_text: compact_search_text(flavor_text(card) || ""),
-        type_line: normalize_search_text(card["type_line"] || ""),
-        oracle_text: normalize_search_text(oracle_text),
-        compact_oracle_text: compact_search_text(oracle_text),
-        set_code: normalize_search_text(card["set"] || ""),
-        collector_number: normalize_search_text(card["collector_number"] || "")
-      }
-    ]
-  end
-
-  defp printing_search_row(_card), do: []
-
-  defp normalize_search_text(value) when is_binary(value) do
-    value
-    |> String.downcase()
-    |> String.replace(~r/[^a-z0-9]+/u, " ")
-    |> String.trim()
-  end
-
-  defp compact_search_text(value) when is_binary(value) do
-    value
-    |> String.downcase()
-    |> String.replace(~r/[^a-z0-9]+/u, "")
-  end
-
   defp colors(%{"colors" => colors}) when is_list(colors), do: colors
 
   defp colors(%{"card_faces" => [%{"colors" => colors} | _faces]}) when is_list(colors),

priv/repo/migrations/20260908000001_drop_scryfall_printing_search.exs

@@ -0,0 +1,62 @@
+defmodule Manavault.Repo.Migrations.DropScryfallPrintingSearch do
+  use Ecto.Migration
+
+  # The FTS5 table was maintained on every Scryfall import (~100 MB for the
+  # full catalog) but no query ever read it. Card and collection search run
+  # against the covering indexes on scryfall_cards / scryfall_printings.
+
+  def up do
+    execute("DROP TABLE IF EXISTS scryfall_printing_search")
+  end
+
+  def down do
+    execute("""
+    CREATE VIRTUAL TABLE scryfall_printing_search USING fts5(
+      scryfall_id UNINDEXED,
+      name,
+      compact_name,
+      flavor_name,
+      compact_flavor_name,
+      flavor_text,
+      compact_flavor_text,
+      type_line,
+      oracle_text,
+      compact_oracle_text,
+      set_code,
+      collector_number
+    )
+    """)
+
+    execute("""
+    INSERT INTO scryfall_printing_search (
+      scryfall_id,
+      name,
+      compact_name,
+      flavor_name,
+      compact_flavor_name,
+      flavor_text,
+      compact_flavor_text,
+      type_line,
+      oracle_text,
+      compact_oracle_text,
+      set_code,
+      collector_number
+    )
+    SELECT
+      p.scryfall_id,
+      lower(c.name),
+      lower(replace(replace(replace(replace(replace(replace(c.name, ' ', ''), ',', ''), '''', ''), '’', ''), '-', ''), '/', '')),
+      lower(coalesce(p.flavor_name, '')),
+      lower(replace(replace(replace(replace(replace(replace(coalesce(p.flavor_name, ''), ' ', ''), ',', ''), '''', ''), '’', ''), '-', ''), '/', '')),
+      lower(coalesce(p.flavor_text, '')),
+      lower(replace(replace(replace(replace(replace(replace(coalesce(p.flavor_text, ''), ' ', ''), ',', ''), '''', ''), '’', ''), '-', ''), '/', '')),
+      lower(coalesce(c.type_line, '')),
+      lower(coalesce(c.oracle_text, '')),
+      lower(replace(replace(replace(replace(replace(replace(coalesce(c.oracle_text, ''), ' ', ''), ',', ''), '''', ''), '’', ''), '-', ''), '/', '')),
+      lower(p.set_code),
+      lower(p.collector_number)
+    FROM scryfall_printings p
+    JOIN scryfall_cards c ON c.oracle_id = p.oracle_id
+    """)
+  end
+end

test/manavault/backup/create_test.exs

@@ -17,7 +17,6 @@ defmodule Manavault.Backup.CreateTest do
   alias Manavault.Backup.CreateTestRepo
 
   @catalog_tables ~w[
-    scryfall_printing_search
     scryfall_syncs
     scryfall_printings
     scryfall_cards
@@ -97,11 +96,6 @@ defmodule Manavault.Backup.CreateTest do
         collector_number TEXT NOT NULL
       );
 
-      CREATE VIRTUAL TABLE scryfall_printing_search USING fts5(
-        scryfall_id UNINDEXED,
-        name
-      );
-
       CREATE TABLE scryfall_syncs (
         id INTEGER PRIMARY KEY,
         status TEXT NOT NULL,
@@ -120,9 +114,6 @@ defmodule Manavault.Backup.CreateTest do
       INSERT INTO scryfall_printings (scryfall_id, oracle_id, set_code, collector_number)
       VALUES ('printing-1', 'oracle-1', 'lea', '232');
 
-      INSERT INTO scryfall_printing_search (scryfall_id, name)
-      VALUES ('printing-1', 'black lotus');
-
       INSERT INTO scryfall_syncs (status, bulk_type)
       VALUES ('completed', 'default_cards');
 

test/manavault/catalog/import_test.exs

@@ -82,15 +82,9 @@ defmodule Manavault.Catalog.ImportTest do
     refute Repo.get(Printing, memorabilia["id"])
     refute Repo.get(Printing, token["id"])
     refute Repo.get(Card, token["oracle_id"])
-
-    assert %{rows: []} =
-             Repo.query!(
-               "SELECT scryfall_id FROM scryfall_printing_search WHERE scryfall_id IN (?, ?)",
-               [memorabilia["id"], token["id"]]
-             )
   end
 
-  test "import_cards releases the write lock between batches without dropping search rows" do
+  test "import_cards releases the write lock between batches" do
     test_pid = self()
     handler_id = {__MODULE__, make_ref()}
 

test/manavault/catalog/sync_test.exs

@@ -250,12 +250,6 @@ defmodule Manavault.Catalog.SyncTest do
     assert Repo.get!(Printing, @black_lotus["id"])
     refute Repo.get(Printing, memorabilia["id"])
     refute Repo.get(Printing, token["id"])
-
-    assert %{rows: []} =
-             Repo.query!(
-               "SELECT scryfall_id FROM scryfall_printing_search WHERE scryfall_id IN (?, ?)",
-               [memorabilia["id"], token["id"]]
-             )
   end
 
   test "sync_scryfall only runs the paper printing reconciliation once" do
@@ -323,8 +317,7 @@ defmodule Manavault.Catalog.SyncTest do
     assert log =~ "Scryfall catalog sync downloaded default-cards bulk"
     assert log =~ "Scryfall catalog sync decoded default-cards bulk"
 
-    assert log =~
-             "Scryfall catalog import progress source_cards=1/1 cards=1 printings=1 search_rows=1"
+    assert log =~ "Scryfall catalog import progress source_cards=1/1 cards=1 printings=1"
 
     assert log =~ "Scryfall catalog import completed source_cards=1 cards=1 printings=1"
     assert log =~ "Scryfall catalog sync succeeded"