bd-ko1 Part B: detail-page enrichment for min_contract_months

src/flatpilot/cli.py

@@ -386,6 +386,32 @@ def dedup(
     console.print(f"rebuilt [bold]{total}[/bold] flats → [bold]{clusters}[/bold] clusters")
 
 
+@app.command()
+def enrich(
+    budget: int = typer.Option(
+        30, "--budget", help="Max detail pages to fetch this pass."
+    ),
+) -> None:
+    """Fetch + parse detail pages for flats missing min_contract_months."""
+    from rich.console import Console
+
+    from flatpilot.errors import ProfileMissingError
+    from flatpilot.profile import load_profile
+    from flatpilot.scrapers.detail import enrich_pending_flats
+
+    console = Console()
+    try:
+        profile = load_profile()
+        if profile is None:
+            raise ProfileMissingError(
+                "No profile at ~/.flatpilot/profile.json — run `flatpilot init` first."
+            )
+        enrich_pending_flats(profile, console, budget=budget)
+    except ProfileMissingError as exc:
+        console.print(f"[red]{exc}[/red]")
+        raise typer.Exit(1) from exc
+
+
 @app.command()
 def match() -> None:
     """Apply the matcher to unmatched listings and write matches."""

src/flatpilot/matcher/filters.py

@@ -158,6 +158,25 @@ def filter_contract(flat: Mapping[str, Any], profile: Profile) -> FilterResult:
 def filter_short_term(flat: Mapping[str, Any], profile: Profile) -> FilterResult:
     if not profile.exclude_short_term:
         return True, None
+
+    # Structured-field path (bd-ko1 Part B): a populated available_from /
+    # available_until pair from the detail-page enricher is more reliable
+    # than the title/description heuristic. Falls back to text matching
+    # below when one or both fields are absent.
+    if profile.min_contract_months is not None:
+        af = flat.get("available_from")
+        au = flat.get("available_until")
+        if af and au:
+            try:
+                start = date.fromisoformat(str(af))
+                end = date.fromisoformat(str(au))
+            except ValueError:
+                start = end = None  # type: ignore[assignment]
+            if start and end:
+                span_days = (end - start).days
+                if 0 < span_days < profile.min_contract_months * 30:
+                    return False, "short_term_listing"
+
     text = " ".join(
         str(flat.get(f) or "") for f in ("title", "description")
     ).lower()

src/flatpilot/pipeline.py

@@ -33,6 +33,13 @@ def run_pipeline_once(
         console.print(f"[red]scrape failed: {exc.__class__.__name__}: {exc}[/red]")
         failures += 1
 
+    console.rule("enrich")
+    try:
+        run_pipeline_enrich(profile, console, user_id=user_id)
+    except Exception as exc:
+        console.print(f"[red]enrich failed: {exc.__class__.__name__}: {exc}[/red]")
+        failures += 1
+
     console.rule("match")
     try:
         run_pipeline_match(console, user_id=user_id)
@@ -86,6 +93,18 @@ def run_pipeline_scrape(
     run_scrape_pass(scrapers, profile, console, user_id=user_id)
 
 
+def run_pipeline_enrich(
+    profile: Profile,
+    console,
+    *,
+    budget: int = 30,
+    user_id: int = DEFAULT_USER_ID,
+) -> None:
+    from flatpilot.scrapers.detail import enrich_pending_flats
+
+    enrich_pending_flats(profile, console, budget=budget)
+
+
 def run_pipeline_match(console, *, user_id: int = DEFAULT_USER_ID) -> None:
     from flatpilot.matcher.runner import run_match
 

src/flatpilot/schemas.py

@@ -145,6 +145,13 @@
     "triggered_by_saved_search": "TEXT",
 }
 
+# bd-ko1 Part B: detail-page enrichment exposes the end-date of a fixed-term
+# lease. Legacy DBs created before this column predate it, so add via
+# ALTER TABLE here rather than touching FLATS_CREATE_SQL.
+COLUMNS["flats"] = {
+    "available_until": "TEXT",
+}
+
 APPLICATIONS_METHOD_APPLIED_AT_INDEX_SQL = """
 CREATE INDEX IF NOT EXISTS idx_applications_method_applied_at
     ON applications(method, applied_at)

src/flatpilot/scrapers/base.py

@@ -45,6 +45,7 @@ class Flat(TypedDict, total=False):
     lng: float
     online_since: str
     available_from: str
+    available_until: str
     requires_wbs: bool
     wbs_size_category: int
     wbs_income_category: int

src/flatpilot/scrapers/detail.py

@@ -0,0 +1,127 @@
+"""Detail-page enrichment orchestrator (bd-ko1 Part B).
+
+Runs between scrape and match. For each platform whose scraper exposes
+``parse_detail``, selects up to ``budget`` canonical flats that are
+missing ``min_contract_months`` and that the user's profile would
+require it for. Fetches the detail page via the per-platform
+``fetch_detail`` (real network) or an injected callable (tests), then
+UPDATEs the flat row.
+
+Network failures are logged and counted; the flat row is left alone so
+the cheap title/description heuristic from bd-33h remains the safety
+net for fixed-term listings whose detail page we could not reach.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from collections.abc import Callable
+from typing import TypedDict
+
+from flatpilot.database import get_conn, init_db
+from flatpilot.profile import Profile
+
+logger = logging.getLogger(__name__)
+
+# Politeness gap between detail fetches within one pass. Conservative —
+# detail pages are 1 request each and almost always cached client-side
+# by the time the user visits, but a small gap keeps the platform's
+# anti-bot heuristics from seeing a burst.
+DETAIL_FETCH_DELAY_SEC: float = 2.0
+
+
+class EnrichSummary(TypedDict):
+    candidates: int
+    fetched: int
+    failed: int
+
+
+Fetcher = Callable[[str], str]
+
+
+def _default_fetchers() -> dict[str, Fetcher]:
+    from flatpilot.scrapers.kleinanzeigen import fetch_detail as fetch_ka
+    from flatpilot.scrapers.wg_gesucht import fetch_detail as fetch_wg
+
+    return {"wg-gesucht": fetch_wg, "kleinanzeigen": fetch_ka}
+
+
+def _parsers() -> dict[str, Callable[[str], dict]]:
+    from flatpilot.scrapers.kleinanzeigen import parse_detail as parse_ka
+    from flatpilot.scrapers.wg_gesucht import parse_detail as parse_wg
+
+    return {"wg-gesucht": parse_wg, "kleinanzeigen": parse_ka}
+
+
+def enrich_pending_flats(
+    profile: Profile,
+    console,
+    *,
+    budget: int = 30,
+    fetchers: dict[str, Fetcher] | None = None,
+) -> EnrichSummary:
+    """Fetch + parse detail pages for pending canonical flats.
+
+    Returns an :class:`EnrichSummary`. The ``fetchers`` injection point
+    lets tests pass deterministic stubs; production callers omit it and
+    get the real Playwright-backed helpers.
+    """
+    summary: EnrichSummary = {"candidates": 0, "fetched": 0, "failed": 0}
+    if profile.min_contract_months is None:
+        return summary
+
+    init_db()
+    conn = get_conn()
+    fetchers = fetchers or _default_fetchers()
+    parsers = _parsers()
+
+    rows = conn.execute(
+        """
+        SELECT id, platform, listing_url
+        FROM flats
+        WHERE canonical_flat_id IS NULL
+          AND min_contract_months IS NULL
+          AND platform IN ('wg-gesucht', 'kleinanzeigen')
+        ORDER BY id DESC
+        LIMIT ?
+        """,
+        (budget,),
+    ).fetchall()
+    summary["candidates"] = len(rows)
+
+    for row in rows:
+        platform = row["platform"]
+        fetcher = fetchers.get(platform)
+        parser = parsers.get(platform)
+        if fetcher is None or parser is None:
+            continue
+        try:
+            html = fetcher(row["listing_url"])
+        except Exception as exc:
+            logger.warning(
+                "%s: detail fetch failed for flat %s (%s: %s)",
+                platform, row["id"], exc.__class__.__name__, exc,
+            )
+            summary["failed"] += 1
+            continue
+
+        fields = parser(html) if html else {}
+        if fields:
+            _update_flat(conn, row["id"], fields)
+            summary["fetched"] += 1
+        time.sleep(DETAIL_FETCH_DELAY_SEC)
+
+    console.print(
+        f"enrich: [bold]{summary['candidates']}[/bold] candidates, "
+        f"[green]{summary['fetched']}[/green] enriched"
+        + (f", [red]{summary['failed']} failed[/red]" if summary["failed"] else "")
+    )
+    return summary
+
+
+def _update_flat(conn, flat_id: int, fields: dict) -> None:
+    assignments = ", ".join(f"{k} = :{k}" for k in fields)
+    params = dict(fields)
+    params["id"] = flat_id
+    conn.execute(f"UPDATE flats SET {assignments} WHERE id = :id", params)

src/flatpilot/scrapers/kleinanzeigen.py

@@ -265,6 +265,35 @@ def _parse_card(card: Any) -> Flat | None:
     return flat
 
 
+# Kleinanzeigen exposes "Mindestmietdauer" on detail pages; some sublet
+# variants show "Mietdauer" instead. Accept both, case-insensitive.
+_MIN_CONTRACT_RE = re.compile(
+    r"(?:Mindest)?Mietdauer[^0-9]{0,20}(\d{1,3})\s*Monate?", re.IGNORECASE
+)
+
+
+def parse_detail(html: str) -> dict[str, Any]:
+    """Extract ``min_contract_months`` from a Kleinanzeigen detail page.
+
+    Returns a dict with at most ``min_contract_months``. The cheap
+    ``filter_short_term`` keyword filter (bd-33h) remains the fallback
+    for fixed-term listings without an explicit Mindestmietdauer.
+    """
+    if not html:
+        return {}
+    from bs4 import BeautifulSoup
+
+    soup = BeautifulSoup(html, "html.parser")
+    text = soup.get_text(" ", strip=True)
+    out: dict[str, Any] = {}
+
+    m = _MIN_CONTRACT_RE.search(text)
+    if m:
+        out["min_contract_months"] = int(m.group(1))
+
+    return out
+
+
 def _clean(text: str) -> str:
     return " ".join(text.split())
 
@@ -281,3 +310,27 @@ def _first_float(pattern: re.Pattern[str], text: str) -> float | None:
         return float(raw)
     except ValueError:
         return None
+
+
+def fetch_detail(listing_url: str) -> str:
+    """Return the raw HTML of a Kleinanzeigen detail page.
+
+    Uses the UA pool + stealth config so a per-listing burst still
+    looks like organic traffic. The orchestrator caps the burst at
+    ``budget`` flats per pipeline pass.
+    """
+    config = SessionConfig(
+        platform=KleinanzeigenScraper.platform,
+        user_agent=pin_user_agent(KleinanzeigenScraper.platform),
+        warmup_url=None,
+        consent_selectors=CONSENT_SELECTORS,
+        stealth=True,
+    )
+    with polite_session(config) as context, session_page(context) as pg:
+        response = pg.goto(listing_url, wait_until="domcontentloaded")
+        if response is None:
+            return ""
+        check_rate_limit(response.status, KleinanzeigenScraper.platform)
+        if response.status >= 400:
+            return ""
+        return _handle_response(pg, city="")

src/flatpilot/scrapers/wg_gesucht.py

@@ -239,6 +239,41 @@ def _parse_card(card: Any) -> Flat | None:
     return flat
 
 
+_MIN_CONTRACT_RE = re.compile(
+    r"Mindestmietdauer[^0-9]{0,20}(\d{1,3})\s*Monate?", re.IGNORECASE
+)
+_AVAILABLE_UNTIL_RE = re.compile(
+    r"Verf[uü]gbar\s+bis[^0-9]{0,20}(\d{2})\.(\d{2})\.(\d{4})", re.IGNORECASE
+)
+
+
+def parse_detail(html: str) -> dict[str, Any]:
+    """Extract long-term-contract fields from a WG-Gesucht detail page.
+
+    Returns a dict with at most ``min_contract_months`` and
+    ``available_until`` (ISO date). Anything not present in the HTML is
+    omitted so the caller can ``UPDATE … SET k=v`` without overwriting
+    real data with ``None``.
+    """
+    if not html:
+        return {}
+    from bs4 import BeautifulSoup
+
+    soup = BeautifulSoup(html, "html.parser")
+    text = soup.get_text(" ", strip=True)
+    out: dict[str, Any] = {}
+
+    m = _MIN_CONTRACT_RE.search(text)
+    if m:
+        out["min_contract_months"] = int(m.group(1))
+
+    m = _AVAILABLE_UNTIL_RE.search(text)
+    if m:
+        out["available_until"] = f"{m.group(3)}-{m.group(2)}-{m.group(1)}"
+
+    return out
+
+
 def _district_from_url(href: str) -> str | None:
     # href looks like "/wohnungen-in-Berlin-Neukoelln.12345.html"
     m = re.search(r"/wohnungen-in-[^-/.]+-([^./]+)\.\d+\.", href)
@@ -266,3 +301,27 @@ def _first_date(text: str) -> str | None:
         return None
     day, month, year = m.group(1), m.group(2), m.group(3)
     return f"{year}-{month}-{day}"
+
+
+def fetch_detail(listing_url: str) -> str:
+    """Return the raw HTML of a WG-Gesucht detail page.
+
+    Each call opens a polite_session, navigates once, and closes —
+    cheap relative to the cost of being detected because polite_session
+    persists cookies across runs. The orchestrator caps how many of
+    these run per pipeline pass.
+    """
+    config = SessionConfig(
+        platform=WGGesuchtScraper.platform,
+        user_agent=WGGesuchtScraper.user_agent,
+        warmup_url=None,
+        consent_selectors=CONSENT_SELECTORS,
+    )
+    with polite_session(config) as context, session_page(context) as pg:
+        response = pg.goto(listing_url, wait_until="domcontentloaded")
+        if response is None:
+            return ""
+        check_rate_limit(response.status, WGGesuchtScraper.platform)
+        if response.status >= 400:
+            return ""
+        return pg.content()

tests/fixtures/kleinanzeigen/detail_no_min_contract.html

@@ -0,0 +1,14 @@
+<!doctype html>
+<html lang="de"><body>
+<ul id="viewad-details" class="addetailslist">
+  <li class="addetailslist--detail">
+    Vertragsart
+    <span class="addetailslist--detail--value">Unbefristet</span>
+  </li>
+  <li class="addetailslist--detail">
+    Verfügbar ab
+    <span class="addetailslist--detail--value">01.07.2026</span>
+  </li>
+</ul>
+<p id="viewad-description-text">Unbefristeter Mietvertrag.</p>
+</body></html>