feat(artbox-exemplar-index): self-referential artbox-phash exemplar index (issue #508 phase 1)

MPCAutofill/cardpicker/management/commands/backfill_artbox_phash_exemplars.py

@@ -0,0 +1,106 @@
+from typing import Any
+
+from django.core.management.base import BaseCommand
+
+from cardpicker.artbox_exemplar_backfill import (
+    DEFAULT_BACKFILL_BATCH_SIZE,
+    dry_run_candidate_exemplar_hashes,
+    measure_unresolved_coverage,
+    run_artbox_exemplar_backfill,
+)
+from cardpicker.models import ArtboxPhashExemplar
+
+
+class Command(BaseCommand):
+    help = (
+        "Issue #508 phase 1: seeds ArtboxPhashExemplar from our own DB - every current-evidence "
+        "card with a human-backed printing resolution, plus every card carrying a join-key "
+        "machine vote at or above the confidence floor (see artbox_exemplar_backfill.py's own "
+        "JOIN_KEY_SEED_CONFIDENCE_FLOOR comment). Never fetches Scryfall images. Idempotent and "
+        "resumable by construction (filters out cards that already have an exemplar row, so a "
+        "plain re-invocation after a kill just picks up where it left off) - no separate "
+        "--resume flag needed. Also reports the phase-2 coverage estimate: of every currently-"
+        "UNRESOLVED card carrying a current artbox_phash, how many would match the exemplar "
+        "index at d=0 and at d<=2 - computed read-only, every invocation, dry-run or not."
+    )
+
+    def add_arguments(self, parser: Any) -> None:
+        parser.add_argument(
+            "--dry-run",
+            action="store_true",
+            default=False,
+            help="Compute and report every counter, including the coverage estimate, without "
+            "writing any ArtboxPhashExemplar row.",
+        )
+        parser.add_argument(
+            "--batch-size",
+            type=int,
+            default=DEFAULT_BACKFILL_BATCH_SIZE,
+            help=f"Rows persisted per bulk_create flush, and rows per join-key evidence-lookup "
+            f"chunk. Default: {DEFAULT_BACKFILL_BATCH_SIZE}.",
+        )
+        parser.add_argument(
+            "--limit",
+            type=int,
+            default=None,
+            help="Only create this many exemplar rows total, across both seed passes combined "
+            "(for testing/sampling). Default: no limit, process the entire backlog.",
+        )
+        parser.add_argument(
+            "--run-id",
+            type=str,
+            default=None,
+            help="Stamped onto every row this invocation creates, for later retraction via "
+            "retract_artbox_phash_exemplars --run-id. Default: none stamped.",
+        )
+        parser.add_argument(
+            "--skip-coverage",
+            action="store_true",
+            default=False,
+            help="Skip the phase-2 coverage measurement pass (unresolved-card matching) - "
+            "useful for a fast seeding-only run; the coverage pass is a full scan of every "
+            "UNRESOLVED card's current artbox_phash and is the more expensive half of this "
+            "command.",
+        )
+        # --skip-checks is deliberately NOT defined here - Django's BaseCommand already adds it
+        # natively (see local_backfill_canonical_hash.py's own matching comment).
+
+    def handle(self, *args: Any, **kwargs: Any) -> None:
+        dry_run = kwargs["dry_run"]
+        batch_size = kwargs["batch_size"]
+        limit = kwargs["limit"]
+        run_id = kwargs["run_id"]
+        skip_coverage = kwargs["skip_coverage"]
+
+        mode = "DRY RUN" if dry_run else "WRITE"
+        self.stdout.write(
+            f"[{mode}] backfill_artbox_phash_exemplars --batch-size={batch_size} " f"--limit={limit} --run-id={run_id}"
+        )
+
+        result = run_artbox_exemplar_backfill(dry_run=dry_run, batch_size=batch_size, limit=limit, run_id=run_id)
+
+        self.stdout.write(
+            f"Seeded human_backed={result.human_backed_seeded} machine={result.machine_seeded} "
+            f"(machine_skipped_stale_or_missing_evidence="
+            f"{result.machine_skipped_stale_or_missing_evidence}), "
+            f"distinct_illustration_ids={result.distinct_illustration_ids}."
+        )
+        self.stdout.write(f"Elapsed {result.elapsed_seconds:.1f}s.")
+        if dry_run:
+            self.stdout.write("Dry run - nothing written.")
+
+        if skip_coverage:
+            self.stdout.write("Coverage measurement skipped (--skip-coverage).")
+            return
+
+        if dry_run:
+            exemplar_hashes = dry_run_candidate_exemplar_hashes(batch_size)
+        else:
+            exemplar_hashes = list(ArtboxPhashExemplar.objects.values_list("artbox_phash", flat=True))
+
+        coverage = measure_unresolved_coverage(exemplar_hashes)
+        self.stdout.write(
+            f"COVERAGE (phase-2 estimate): unresolved_candidates_considered="
+            f"{coverage.unresolved_candidates_considered} matches_at_d0={coverage.matches_at_d0} "
+            f"matches_at_d_le_2={coverage.matches_at_d_le_2}."
+        )

MPCAutofill/cardpicker/management/commands/retract_artbox_phash_exemplars.py

@@ -0,0 +1,93 @@
+from typing import Any
+
+from django.core.management.base import BaseCommand, CommandError
+
+from cardpicker.models import ArtboxPhashExemplar
+
+
+class Command(BaseCommand):
+    help = (
+        "Issue #508 phase 1: retracts ArtboxPhashExemplar rows. Nothing downstream reads this "
+        "table yet (no matching calculator, no vote), so retraction here is a plain filtered "
+        "delete - no consensus resync, no safety gate against a live consensus outcome (compare "
+        "retract_stage_d_by_run_id, which needs both because its target table casts votes). "
+        "--seed-group-key is the primary path (ArtboxPhashExemplar's own docstring): every "
+        "exemplar traced to the SAME source resolution or the SAME source join-key vote shares "
+        "one seed_group_key, so retracting a bad seed 'together with everything it seeded' is "
+        "exactly this filter. --card-id/--illustration-id/--run-id/--source-vote-id are narrower "
+        "or broader alternatives for a caller who already knows one of those instead. Exactly one "
+        "selector is required per invocation; combining more than one is a CommandError, not a "
+        "silent AND, since a caller reaching for two selectors at once is very likely trying to "
+        "express something this command doesn't support rather than a genuine narrowing. Dry-run "
+        "by default - --write required to actually delete anything."
+    )
+
+    def add_arguments(self, parser: Any) -> None:
+        parser.add_argument(
+            "--seed-group-key", type=str, default=None, help="Retract every exemplar sharing this seed_group_key."
+        )
+        parser.add_argument(
+            "--card-id", type=int, default=None, help="Retract the single exemplar seeded from this card."
+        )
+        parser.add_argument(
+            "--illustration-id",
+            type=str,
+            default=None,
+            help="Retract every exemplar pointing at this illustration_id (UUID).",
+        )
+        parser.add_argument(
+            "--run-id",
+            type=str,
+            default=None,
+            help="Retract every exemplar stamped with this backfill run_id "
+            "(backfill_artbox_phash_exemplars --run-id).",
+        )
+        parser.add_argument(
+            "--source-vote-id",
+            type=int,
+            default=None,
+            help="Retract the single exemplar seeded from this CardPrintingTag vote (JOIN_KEY_MACHINE seeds only).",
+        )
+        parser.add_argument(
+            "--write",
+            action="store_true",
+            default=False,
+            help="Actually delete the matched rows. Default is dry-run: report the count and "
+            "the affected illustration_ids without deleting anything.",
+        )
+
+    def handle(self, *args: Any, **kwargs: Any) -> None:
+        selectors = {
+            "seed_group_key": kwargs["seed_group_key"],
+            "card_id": kwargs["card_id"],
+            "illustration_id": kwargs["illustration_id"],
+            "run_id": kwargs["run_id"],
+            "source_vote_id": kwargs["source_vote_id"],
+        }
+        given = {key: value for key, value in selectors.items() if value is not None}
+        if len(given) != 1:
+            raise CommandError(
+                "Exactly one of --seed-group-key/--card-id/--illustration-id/--run-id/"
+                f"--source-vote-id is required (got {len(given)}: {sorted(given)})."
+            )
+
+        write = kwargs["write"]
+        mode = "WRITE" if write else "DRY RUN"
+
+        queryset = ArtboxPhashExemplar.objects.filter(**given)
+        count = queryset.count()
+        illustration_ids = sorted({str(value) for value in queryset.values_list("illustration_id", flat=True)})
+
+        self.stdout.write(f"[{mode}] retract_artbox_phash_exemplars {given}")
+        self.stdout.write(
+            f"Matched {count} exemplar row(s) across {len(illustration_ids)} distinct illustration_id(s)"
+            f"{': ' + ', '.join(illustration_ids[:20]) if illustration_ids else ''}"
+            f"{' (truncated)' if len(illustration_ids) > 20 else ''}."
+        )
+
+        if not write:
+            self.stdout.write("Dry run - nothing deleted.")
+            return
+
+        deleted_count, _ = queryset.delete()
+        self.stdout.write(f"Deleted {deleted_count} row(s).")

MPCAutofill/cardpicker/migrations/0102_artbox_phash_exemplar.py

@@ -0,0 +1,75 @@
+# Generated by Django 4.2.30 on 2026-08-05 02:13
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("cardpicker", "0101_delete_printingtagvote"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="ArtboxPhashExemplar",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("illustration_id", models.UUIDField(db_index=True)),
+                ("artbox_phash", models.BigIntegerField(db_index=True)),
+                (
+                    "seed_kind",
+                    models.CharField(
+                        choices=[
+                            ("human_resolution", "Human-backed printing resolution"),
+                            ("join_key_machine", "High-confidence join-key vote"),
+                        ],
+                        max_length=32,
+                    ),
+                ),
+                ("is_human_backed", models.BooleanField()),
+                ("confidence", models.FloatField(blank=True, null=True)),
+                ("seed_group_key", models.CharField(db_index=True, max_length=128)),
+                ("content_hash", models.BigIntegerField()),
+                ("run_id", models.CharField(blank=True, db_index=True, max_length=64, null=True)),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                (
+                    "card",
+                    models.OneToOneField(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="artbox_phash_exemplar",
+                        to="cardpicker.card",
+                    ),
+                ),
+                (
+                    "printing",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="artbox_phash_exemplars",
+                        to="cardpicker.canonicalcard",
+                    ),
+                ),
+                (
+                    "source_vote",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        related_name="+",
+                        to="cardpicker.cardprintingtag",
+                    ),
+                ),
+            ],
+        ),
+        migrations.AddConstraint(
+            model_name="artboxphashexemplar",
+            constraint=models.CheckConstraint(
+                check=models.Q(
+                    models.Q(("is_human_backed", True), ("seed_kind", "human_resolution")),
+                    models.Q(("is_human_backed", False), ("seed_kind", "join_key_machine")),
+                    _connector="OR",
+                ),
+                name="artboxphashexemplar_seed_kind_matches_human_backed",
+            ),
+        ),
+    ]

MPCAutofill/cardpicker/models.py

@@ -1390,6 +1390,115 @@ def __str__(self) -> str:
         return f"[{self.source}] {self.card.name} -> illustration {outcome}"
 
 
+class ArtboxPhashExemplarSeedKind(models.TextChoices):
+    """
+    How an `ArtboxPhashExemplar` row was seeded (issue #508 phase 1) - the provenance distinction
+    the owner made mandatory at seeding time (2026-08-05): a machine-derived seed and a
+    human-backed one must stay distinguishable forever, so a later decision to trust only the
+    latter is a query over this field, not a migration.
+    """
+
+    HUMAN_RESOLUTION = "human_resolution", gettext_lazy("Human-backed printing resolution")
+    JOIN_KEY_MACHINE = "join_key_machine", gettext_lazy("High-confidence join-key vote")
+
+
+class ArtboxPhashExemplar(models.Model):
+    """
+    Second illustration-deduction path's reference index (issue #508 phase 1, "self-referential
+    exemplar index" - owner-shaped 2026-07-28, seeding extended by the owner 2026-08-05 to include
+    machine seeds). An exemplar is a labelled association: a card's own CURRENT `artbox_phash`
+    (`ImageEvidence.artbox_phash` - see that field's own docstring, issue #480) -> the
+    `illustration_id` of the printing that scan was identified as (via
+    `CanonicalPrintingMetadata.illustration_id`, reached through the resolved/matched
+    `CanonicalCard`).
+
+    NEVER SOURCED FROM SCRYFALL IMAGES (binding, #508's design section). phash comparability
+    requires identical crop geometry/preprocessing - our own `artbox_phash` extractor is
+    self-consistent; Scryfall's `art_crop` framing differs and would make cross-source Hamming
+    distances unreliable (this is also why PR #694 was closed and deferred to #697 - do not
+    reintroduce a Scryfall fetch anywhere a seed for this table is computed).
+
+    SEED SOURCES (owner decision 2026-08-05, extending #508's original human-only spec, which
+    would have left this index dormant - only 12 human-backed resolutions exist catalogue-wide
+    at seeding time):
+
+    - `HUMAN_RESOLUTION`: `card.printing_tag_status == RESOLVED`. Resolution ALWAYS requires a
+      human-backed vote (`vote_consensus.resolve_weighted_consensus`'s non-machine-alone gate,
+      untouched by this work) - so every RESOLVED card is human-backed by construction, and no
+      per-vote inspection is needed to classify one as such.
+    - `JOIN_KEY_MACHINE`: an individual `CardPrintingTag` vote cast by the join-key calculator
+      (`local_calculate_verdicts.JOIN_KEY_ANONYMOUS_ID`) at or above
+      `artbox_exemplar_backfill.JOIN_KEY_SEED_CONFIDENCE_FLOOR` - see that constant's own comment
+      for why the floor excludes the artist-disagreement confidence tier (0.65) along with the
+      no-match tier (0.6, which is not an identification at all and can never seed regardless of
+      any floor).
+
+    `is_human_backed` is a plain denormalised copy of `seed_kind`'s own implication (never
+    `HUMAN_RESOLUTION` with `is_human_backed=False` or vice versa - enforced by the CheckConstraint
+    below), kept as its own column so a reader who only needs the human/machine split never has to
+    know the seed-kind vocabulary.
+
+    RETRACTION (owner directive 2026-08-05: "a bad seed must be retractable together with
+    everything it seeded"). `seed_group_key` is the stable identity of the SOURCE EVENT that
+    produced this row, not of the row itself: every exemplar traceable to the same md5-identity-
+    group resolution, or to the same source `CardPrintingTag` vote, shares one key, so retracting
+    a bad seed is `ArtboxPhashExemplar.objects.filter(seed_group_key=...).delete()` - one query, no
+    per-row reasoning about what else that source touched. See `artbox_exemplar_backfill.
+    human_resolution_seed_group_key`/`join_key_seed_group_key` for the exact format (the human-
+    resolution case mirrors `printing_consensus.md5_group_key`'s own group identity, so retracting
+    "this resolved identity group" here means the same set of cards `printing_consensus` itself
+    would call one group). `source_vote` is `SET_NULL` on the vote's own deletion (a purge doesn't
+    orphan this row's retractability - `seed_group_key` carries it independently of the FK's
+    referential integrity).
+
+    INDEX-NOT-STORE (CLAUDE.md's governing premise): this table holds a hash and a UUID, nothing
+    fetched or decodable back into pixels - `content_hash` records the source card's own
+    `content_phash` AT SEED TIME purely as a staleness audit trail (so a later reader can tell
+    whether the source card's image has since changed), never a second copy of anything
+    image-shaped.
+
+    PHASE 1 SCOPE: this table is read by nothing yet. No matching calculator, no vote, no
+    consensus, no change to `resolve_weighted_consensus`/the human-backed gate - see
+    `docs/identification-pipeline.md`'s "Parallel detectors" section for what this deliberately
+    does NOT do.
+    """
+
+    illustration_id = models.UUIDField(db_index=True)
+    artbox_phash = models.BigIntegerField(db_index=True)
+    card = models.OneToOneField(to=Card, on_delete=models.CASCADE, related_name="artbox_phash_exemplar")
+    printing = models.ForeignKey(to=CanonicalCard, on_delete=models.CASCADE, related_name="artbox_phash_exemplars")
+    seed_kind = models.CharField(max_length=32, choices=ArtboxPhashExemplarSeedKind.choices)
+    is_human_backed = models.BooleanField()
+    # SET_NULL, not CASCADE - see class docstring's RETRACTION section for why losing this FK
+    # on the source vote's own deletion is fine (seed_group_key carries retractability instead).
+    source_vote = models.ForeignKey(
+        to=CardPrintingTag, on_delete=models.SET_NULL, null=True, blank=True, related_name="+"
+    )
+    # Purely informational, mirroring `CardPrintingTag.confidence`'s own "not read by any
+    # resolution math" convention (`JOIN_KEY_CONFIDENCE_BOTH`'s comment in
+    # local_calculate_verdicts.py makes the identical point for that field). Null for
+    # HUMAN_RESOLUTION seeds - a resolution is a consensus outcome, not a single confidence value.
+    confidence = models.FloatField(null=True, blank=True)
+    seed_group_key = models.CharField(max_length=128, db_index=True)
+    content_hash = models.BigIntegerField()
+    run_id = models.CharField(max_length=64, null=True, blank=True, db_index=True)
+    created_at = models.DateTimeField(auto_now_add=True)
+
+    class Meta:
+        constraints = [
+            models.CheckConstraint(
+                check=(
+                    models.Q(seed_kind=ArtboxPhashExemplarSeedKind.HUMAN_RESOLUTION, is_human_backed=True)
+                    | models.Q(seed_kind=ArtboxPhashExemplarSeedKind.JOIN_KEY_MACHINE, is_human_backed=False)
+                ),
+                name="artboxphashexemplar_seed_kind_matches_human_backed",
+            ),
+        ]
+
+    def __str__(self) -> str:
+        return f"[{self.seed_kind}] card={self.card_id} -> illustration {self.illustration_id}"
+
+
 class TagModerationClass(models.TextChoices):
     """
     Whether consensus on this tag resolves like any other (STANDARD) or requires a privileged