Scope known_external_ids per-user in pipeline (bd-m6g)

src/flatpilot/pipeline.py

@@ -28,7 +28,7 @@ def run_pipeline_once(
 
     console.rule("scrape")
     try:
-        run_pipeline_scrape(profile, console)
+        run_pipeline_scrape(profile, console, user_id=user_id)
     except Exception as exc:
         console.print(f"[red]scrape failed: {exc.__class__.__name__}: {exc}[/red]")
         failures += 1
@@ -70,15 +70,20 @@ def _ensure_scrapers_registered() -> None:
     import flatpilot.scrapers.wg_gesucht  # noqa: F401 — triggers @register
 
 
-def run_pipeline_scrape(profile: Profile, console) -> None:
+def run_pipeline_scrape(
+    profile: Profile,
+    console,
+    *,
+    user_id: int = DEFAULT_USER_ID,
+) -> None:
     from flatpilot.scrapers import all_scrapers
 
     _ensure_scrapers_registered()
     scrapers = [cls() for cls in all_scrapers()]
     if not scrapers:
         console.print("[yellow]no scrapers registered[/yellow]")
         return
-    run_scrape_pass(scrapers, profile, console)
+    run_scrape_pass(scrapers, profile, console, user_id=user_id)
 
 
 def run_pipeline_match(console, *, user_id: int = DEFAULT_USER_ID) -> None:
@@ -131,7 +136,13 @@ def run_pipeline_notify(
     console.print(" · ".join(parts))
 
 
-def run_scrape_pass(scrapers: list, profile: Profile, console) -> None:
+def run_scrape_pass(
+    scrapers: list,
+    profile: Profile,
+    console,
+    *,
+    user_id: int = DEFAULT_USER_ID,
+) -> None:
     from flatpilot.database import get_conn
     from flatpilot.scrapers import backoff, supports_city
     from flatpilot.scrapers.session import ChallengeDetectedError, RateLimitedError
@@ -152,11 +163,18 @@ def run_scrape_pass(scrapers: list, profile: Profile, console) -> None:
                 f"[dim]{plat}: cooling off for {remaining:.0f}s more — skipping[/dim]"
             )
             continue
+        # bd-m6g: scope "seen" to flats this user has already had a match
+        # decision on. A brand-new user starts with an empty set so flats
+        # other users scraped still get evaluated under this user's profile.
+        # Paginating scrapers may walk more pages on a new user's first
+        # pass — accepted trade-off for multi-user correctness.
         known_external_ids = frozenset(
             row[0]
             for row in conn.execute(
-                "SELECT external_id FROM flats WHERE platform = ?",
-                (plat,),
+                "SELECT f.external_id FROM flats f "
+                "INNER JOIN matches m ON m.flat_id = f.id "
+                "WHERE f.platform = ? AND m.user_id = ?",
+                (plat, user_id),
             )
         )
         try:

tests/test_pipeline_known_ids.py

@@ -8,17 +8,42 @@
 from rich.console import Console
 
 
+def _seed_flat(conn, platform: str, ext_id: str, now: str) -> int:
+    conn.execute(
+        "INSERT OR IGNORE INTO flats "
+        "(platform, external_id, listing_url, title, scraped_at, first_seen_at) "
+        "VALUES (?, ?, ?, ?, ?, ?)",
+        (platform, ext_id, f"https://example.test/{ext_id}", "t", now, now),
+    )
+    row = conn.execute(
+        "SELECT id FROM flats WHERE platform = ? AND external_id = ?",
+        (platform, ext_id),
+    ).fetchone()
+    return int(row[0])
+
+
+def _seed_match(conn, *, user_id: int, flat_id: int, now: str) -> None:
+    conn.execute(
+        "INSERT INTO matches "
+        "(user_id, flat_id, profile_version_hash, decision, decided_at) "
+        "VALUES (?, ?, 'phash', 'match', ?)",
+        (user_id, flat_id, now),
+    )
+
+
 def test_run_scrape_pass_passes_known_ids_from_db(
     tmp_db, monkeypatch: pytest.MonkeyPatch
 ) -> None:
-    """The pipeline pre-loads (platform, external_id) pairs from `flats`
-    and passes the per-platform set into each scraper's fetch_new as
-    `known_external_ids` (kw-only, frozenset).
-
-    Setup: insert two flats for inberlinwohnen, one for wg_gesucht.
-    Run the pipeline pass with both scrapers monkeypatched. Assert each
-    scraper received exactly the set of external_ids matching its
-    platform.
+    """The pipeline pre-loads external_ids from flats the current user has
+    already decided on (matches.user_id = current_user) and passes the
+    per-platform set into each scraper's fetch_new as ``known_external_ids``
+    (kw-only, frozenset). bd-m6g moved the scoping from per-platform to
+    per-(platform, user) so a new user does not silently inherit another
+    user's "seen" set.
+
+    Setup: insert two flats for inberlinwohnen, one for wg_gesucht — and
+    a match row for DEFAULT_USER_ID against each. Run the pipeline pass
+    and assert each scraper received exactly the platform-scoped subset.
     """
     from datetime import UTC, datetime
 
@@ -27,21 +52,17 @@ def test_run_scrape_pass_passes_known_ids_from_db(
     from flatpilot.profile import Profile
     from flatpilot.scrapers import inberlinwohnen as ib
     from flatpilot.scrapers import wg_gesucht as wg
+    from flatpilot.users import DEFAULT_USER_ID
 
-    # Seed the DB with three flats.
     conn = database.get_conn()
     now = datetime.now(UTC).isoformat()
     for platform, ext_id in [
         ("inberlinwohnen", "16344"),
         ("inberlinwohnen", "16343"),
         ("wg-gesucht", "9999"),
     ]:
-        conn.execute(
-            "INSERT OR IGNORE INTO flats "
-            "(platform, external_id, listing_url, title, scraped_at, first_seen_at) "
-            "VALUES (?, ?, ?, ?, ?, ?)",
-            (platform, ext_id, f"https://example.test/{ext_id}", "t", now, now),
-        )
+        flat_id = _seed_flat(conn, platform, ext_id, now)
+        _seed_match(conn, user_id=DEFAULT_USER_ID, flat_id=flat_id, now=now)
     conn.commit()
 
     captured: dict[str, frozenset[str]] = {}
@@ -67,3 +88,49 @@ def _capture_wg(self: Any, profile: Any, **kwargs: Any) -> Any:
     # Type contract: it must be a frozenset, not a list/set/tuple.
     assert isinstance(captured["inberlinwohnen"], frozenset)
     assert isinstance(captured["wg_gesucht"], frozenset)
+
+
+def test_run_scrape_pass_excludes_flats_other_users_decided(
+    tmp_db, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """bd-m6g: a flat scraped under user A's run must not show up in user
+    B's known_external_ids — otherwise B's paginating scraper could
+    terminate early on flats B has never had a decision made on."""
+    from datetime import UTC, datetime
+
+    from flatpilot import database
+    from flatpilot.pipeline import run_scrape_pass
+    from flatpilot.profile import Profile
+    from flatpilot.scrapers import wg_gesucht as wg
+
+    conn = database.get_conn()
+    now = datetime.now(UTC).isoformat()
+    # Both users exist in the seed schema's users table — create user 2.
+    conn.execute(
+        "INSERT INTO users (id, email, created_at) VALUES (2, 'b@example', ?)",
+        (now,),
+    )
+    user_a_flat = _seed_flat(conn, "wg-gesucht", "shared-1", now)
+    user_b_only = _seed_flat(conn, "wg-gesucht", "b-only-1", now)
+    _seed_match(conn, user_id=1, flat_id=user_a_flat, now=now)  # A has decided
+    _seed_match(conn, user_id=2, flat_id=user_b_only, now=now)  # B has decided
+    conn.commit()
+
+    captured: dict[str, frozenset[str]] = {}
+
+    def _capture_wg(self: Any, profile: Any, **kwargs: Any) -> Any:
+        captured["wg_gesucht"] = kwargs.get("known_external_ids")
+        return iter([])
+
+    monkeypatch.setattr(wg.WGGesuchtScraper, "fetch_new", _capture_wg)
+
+    profile = Profile.load_example().model_copy(update={"city": "Berlin"})
+    console = Console(record=True)
+    scrapers = [wg.WGGesuchtScraper()]
+    run_scrape_pass(scrapers, profile, console, user_id=2)
+
+    # user 2 sees only the flat they themselves had a decision on.
+    assert captured["wg_gesucht"] == frozenset({"b-only-1"})
+    # And not the flat user 1 had a decision on, even though it's in the
+    # shared flats table.
+    assert "shared-1" not in captured["wg_gesucht"]