From 01f471585043469d5909680e063a372c8348d63b Mon Sep 17 00:00:00 2001 From: Mukhammad Ibrokhimov Date: Wed, 20 May 2026 20:03:57 +0200 Subject: [PATCH 01/13] bd-ko1: add flats.available_until forward-migration column --- .beads/issues.jsonl | 12 ++++++------ src/flatpilot/schemas.py | 7 +++++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 243bff7..1e43c66 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -75,13 +75,13 @@ {"_type":"issue","id":"FlatPilot-l27z","title":"Test","description":"test","status":"closed","priority":2,"issue_type":"task","created_at":"2026-04-20T15:48:12Z","updated_at":"2026-04-20T15:48:23Z","closed_at":"2026-04-20T15:48:23Z","close_reason":"test cleanup","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"FlatPilot-997r","title":"Test","description":"test","status":"closed","priority":2,"issue_type":"task","created_at":"2026-04-20T15:47:25Z","updated_at":"2026-04-20T15:48:05Z","closed_at":"2026-04-20T15:48:05Z","close_reason":"test cleanup","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"FlatPilot-569","title":"ic1 Phase 2: iterate kleinanzeigen contact form and fill structured fields from profile.contact_details","description":"Follow-up to bd-ic1 Phase 1 (PR shipping the schema + wizard + Filler protocol thread-through). Phase 2 is the actual filler change.\n\nScope:\n- After _reveal_contact_form, iterate every visible input/select/textarea under form#viewad-contact-form.\n- For each field, read its name / id / associated \u003clabel for=...\u003e text.\n- Match against a configurable label-keyword map (Anrede, Vorname, Nachname, Telefon, Straße, PLZ, SCHUFA, Haushalt, ...) → profile.contact_details attribute.\n- Fill the field if it's empty and the profile defines a value. Skip otherwise. For \u003cselect\u003e, use select_option().\n- Add fixture HTML for the 15-field variant and the message-only variant (capture from a real listing).\n- Tests assert each variant is filled correctly and that fields not in the profile stay untouched.\n\nPrerequisite: empirical capture of real kleinanzeigen contact-form HTML for both variants. Without that, the label keywords and selector strategy are guesses.","status":"open","priority":3,"issue_type":"feature","owner":"ibrohimovmuhammad2020@gmail.com","created_at":"2026-05-19T13:42:03Z","created_by":"Mukhammad Ibrokhimov","updated_at":"2026-05-19T13:42:03Z","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"FlatPilot-k1v","title":"ko1 Part B: WG-Gesucht/Kleinanzeigen detail-page enrichment for min_contract_months","description":"Follow-up to bd-ko1 Part A (URL gating shipped via PR #63). Part B was scoped out of that PR to keep the diff small.\n\nWhat's still needed (verbatim from the original bd-ko1):\n\nFor flats that pass all cheap filters, fetch the detail page (politely, with the existing session machinery) and parse the explicit fields:\n- WG-Gesucht: Mindestmietdauer, Befristet: Ja/Nein, Verfügbar bis\n- Kleinanzeigen: Mietdauer, the 'Befristet' toggle if present\n\nPopulate flat['min_contract_months'] so the existing filter_contract (filters.py:119-127) actually fires. Cap the per-run detail-fetch budget (e.g. 30 flats/run) to keep latency and politeness in check.\n\nAcceptance:\n- At least one new test asserts that a detail page with 'Mindestmietdauer: 24 Monate' populates min_contract_months=24 on the flat row.\n- New flat field available_until populated when present (cross-checks the date-range short-term heuristic from bd-33h).\n- Detail fetch failure must not poison the flat — fall back to the bd-33h keyword filter.","status":"open","priority":3,"issue_type":"feature","owner":"ibrohimovmuhammad2020@gmail.com","created_at":"2026-05-19T10:03:16Z","created_by":"Mukhammad Ibrokhimov","updated_at":"2026-05-19T10:03:16Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"FlatPilot-k1v","title":"ko1 Part B: WG-Gesucht/Kleinanzeigen detail-page enrichment for min_contract_months","description":"Follow-up to bd-ko1 Part A (URL gating shipped via PR #63). Part B was scoped out of that PR to keep the diff small.\n\nWhat's still needed (verbatim from the original bd-ko1):\n\nFor flats that pass all cheap filters, fetch the detail page (politely, with the existing session machinery) and parse the explicit fields:\n- WG-Gesucht: Mindestmietdauer, Befristet: Ja/Nein, Verfügbar bis\n- Kleinanzeigen: Mietdauer, the 'Befristet' toggle if present\n\nPopulate flat['min_contract_months'] so the existing filter_contract (filters.py:119-127) actually fires. Cap the per-run detail-fetch budget (e.g. 30 flats/run) to keep latency and politeness in check.\n\nAcceptance:\n- At least one new test asserts that a detail page with 'Mindestmietdauer: 24 Monate' populates min_contract_months=24 on the flat row.\n- New flat field available_until populated when present (cross-checks the date-range short-term heuristic from bd-33h).\n- Detail fetch failure must not poison the flat — fall back to the bd-33h keyword filter.","status":"in_progress","priority":3,"issue_type":"feature","assignee":"Mukhammad Ibrokhimov","owner":"ibrohimovmuhammad2020@gmail.com","created_at":"2026-05-19T10:03:16Z","created_by":"Mukhammad Ibrokhimov","updated_at":"2026-05-20T17:44:50Z","started_at":"2026-05-20T17:44:50Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"FlatPilot-ko1","title":"Definitive long-term gating: WG-Gesucht category URL + detail-page enrichment","description":"Follow-up to the title-keyword short-term filter (cheap heuristic). For a definitive signal, fetch detail-page data so the existing `filter_contract` (filters.py:119-127) actually has `min_contract_months` to compare against.\n\nTwo parts:\n\nPART A — WG-Gesucht URL-level category gating (low risk, no detail fetch)\n- WG-Gesucht search URL is `/wohnungen-in-\u003cslug\u003e.\u003ccity_id\u003e.2.1.0.html` (src/flatpilot/scrapers/wg_gesucht.py:146).\n- The `2` is the \"Wohnung / full flat\" category. WG-Gesucht has a separate URL space for \"Befristete WG / Zwischenmiete\". Verify in `scripts/wg_probe.py` whether the current URL already excludes Zwischenmiete or if there's a finer-grained flag (e.g. `rent_type=long_term`) we should add.\n- If the current URL already filters this out: document it in the scraper docstring and close part A.\n- If not: add the appropriate query param or path segment, gated by `profile.exclude_short_term`.\n\nPART B — detail-page enrichment for matched flats only\n- For flats that pass all cheap filters, fetch the detail page (politely, with the existing session machinery).\n- Parse the explicit fields:\n - WG-Gesucht: `Mindestmietdauer`, `Befristet: Ja/Nein`, `Verfügbar bis`\n - Kleinanzeigen: `Mietdauer`, the \"Befristet\" toggle if present\n- Populate `flat[\"min_contract_months\"]` so the existing `filter_contract` filter actually fires.\n- Cap the per-run detail-fetch budget (e.g. 30 flats/run) to keep latency and politeness in check.\n- This is the same enrichment the wg_gesucht.py:8 docstring already flags as a future pass.\n\nAcceptance criteria:\n- Part A: a probe run confirms long-term-only URL behavior OR a URL change is shipped + tested.\n- Part B: at least one new test asserts that a detail page with \"Mindestmietdauer: 24 Monate\" populates `min_contract_months=24` on the flat row.\n- New flat field `available_until` populated when present (for the date-range short-term heuristic in the keyword-filter issue to cross-check).\n- Detail fetch failure must not poison the flat — fall back to the keyword filter from the sibling issue.","status":"closed","priority":3,"issue_type":"feature","assignee":"Mukhammad Ibrokhimov","owner":"ibrohimovmuhammad2020@gmail.com","created_at":"2026-05-15T18:20:21Z","created_by":"Mukhammad Ibrokhimov","updated_at":"2026-05-19T10:03:28Z","started_at":"2026-05-19T09:59:45Z","closed_at":"2026-05-19T10:03:28Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"FlatPilot-ykg","title":"Log matcher reject reasons at DEBUG with external_id for audit","description":"Matcher rejections are persisted to `matches.decision_reasons_json` but never logged. `flatpilot doctor` shows no per-flat reason; users have to query SQLite to figure out why a specific flat was dropped.\n\nEvidence:\n- src/flatpilot/matcher/filters.py — every reject reason (rent_too_high, rooms_unknown, district_unknown, location_unknown, no_pets_allowed, move_in_too_late, etc.) returns a string but no log line.\n- src/flatpilot/matcher/runner.py:76-80 — the decision is computed and persisted but not surfaced.\n\nAcceptance criteria:\n- Single `logger.debug` line per flat with `external_id`, `platform`, decision, and `reasons`.\n- Goes through the existing `flatpilot.log` config — `--verbose` or `FLATPILOT_LOG_LEVEL=DEBUG` switches it on.\n- No new dependency, no perf hit at INFO level.\n- Optional follow-up: `flatpilot doctor --explain \u003cexternal_id\u003e` to print the same info from DB without needing logs replayed.","status":"closed","priority":3,"issue_type":"feature","assignee":"Mukhammad Ibrokhimov","owner":"ibrohimovmuhammad2020@gmail.com","created_at":"2026-05-15T18:03:18Z","created_by":"Mukhammad Ibrokhimov","updated_at":"2026-05-19T09:47:49Z","started_at":"2026-05-16T20:20:09Z","closed_at":"2026-05-19T09:47:49Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"FlatPilot-gaj","title":"Continue saved-search loop in _try_flat when first match is auto_skipped","description":"`_try_flat()` returns on the first saved-search match — including when the result is `auto_skipped:` (filler missing / template error / attachment error). A different saved-search overlay on the same flat could have been completely fine, but never gets evaluated.\n\nEvidence:\n- src/flatpilot/auto_apply.py:427 — `for name in candidate_names:` loop over saved searches.\n- src/flatpilot/auto_apply.py:503 — completeness fail returns out of the whole `_try_flat()`.\n- src/flatpilot/auto_apply.py:518 — successful submit returns (correct).\n- src/flatpilot/auto_apply.py:523 — FillError returns (debatable; safer to also stop here).\n\nAcceptance criteria:\n- Distinguish \"terminal outcome for this flat\" (submitted, FillError on real submit) from \"this saved-search wasn't compatible\" (auto_skipped) and only return on the former.\n- Add a test: flat that auto_skips for saved-search A but is fully applicable under saved-search B → submits exactly once under B.\n- Verify cap and cooldown logic still work correctly when the loop continues (the cap check happens per-iteration, which is fine).","status":"closed","priority":3,"issue_type":"feature","assignee":"Mukhammad Ibrokhimov","owner":"ibrohimovmuhammad2020@gmail.com","created_at":"2026-05-15T18:03:12Z","created_by":"Mukhammad Ibrokhimov","updated_at":"2026-05-19T09:51:33Z","started_at":"2026-05-19T09:48:09Z","closed_at":"2026-05-19T09:51:33Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"FlatPilot-m6g","title":"Scope known_external_ids per-user in pipeline (multi-user safety)","description":"Today the \"seen\" set queried in pipeline.py is per-platform, not per-user. In single-user mode this is fine. In multi-user mode (Phase 5+ web UI), user A's scrape can mark a flat as seen via `INSERT OR IGNORE` and user B will never get a match row evaluated for it.\n\nEvidence:\n- src/flatpilot/pipeline.py:155-161:\n `SELECT external_id FROM flats WHERE platform = ?`\n- src/flatpilot/schemas.py:57 — `UNIQUE (platform, external_id)` is platform-scoped, not user-scoped (correct for the flat table itself — flat content is shared).\n- Matcher decisions ARE per-user (matches table has `user_id`), so the fix is at the scraper-skip layer only.\n\nAcceptance criteria:\n- `known_external_ids` for a given user/profile = external_ids the user *has already had a decision made on* (matches.user_id = ?), not just every external_id in flats.\n- This lets a brand-new user re-evaluate flats already scraped by others.\n- Tests: with two users, ensure a flat scraped under user A's run produces a match row for user B after their next pass.\n- Performance: stay O(N_flats_for_user) — index on `matches(user_id, external_id)` if needed.","status":"closed","priority":3,"issue_type":"bug","assignee":"Mukhammad Ibrokhimov","owner":"ibrohimovmuhammad2020@gmail.com","created_at":"2026-05-15T18:03:02Z","created_by":"Mukhammad Ibrokhimov","updated_at":"2026-05-19T11:30:47Z","started_at":"2026-05-19T10:03:48Z","closed_at":"2026-05-19T11:30:47Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"FlatPilot-ja7","title":"Surface auto_skipped flats in doctor/dashboard so user understands why apply 'stops'","description":"Flats recorded with `notes LIKE 'auto_skipped:%'` (filler not registered, template error, attachment error, listing_expired) are correctly excluded from cap counting and lifetime-deduped — but they sit silently in the DB. The user perceives this as \"auto-apply stops before cap\" when actually a large portion of matched flats can never be applied to.\n\nEvidence:\n- src/flatpilot/auto_apply.py:300-310 records the `auto_skipped:` rows.\n- Commit 5f556fa added lifetime TTL exclusion so these are processed once.\n- Currently no UI/CLI affordance to see the totals.\n\nAcceptance criteria:\n- `flatpilot doctor` (or `flatpilot status`) prints a per-reason breakdown:\n - `auto_skipped: filler not registered` (per platform)\n - `auto_skipped: template`\n - `auto_skipped: attachment`\n - `auto_skipped: listing_expired`\n- Same totals surface on the HTML dashboard (src/flatpilot/view.py).\n- Numbers update whenever the user closes a gap (e.g. adds a missing filler) — verify a follow-up run reduces the count.","status":"closed","priority":3,"issue_type":"feature","assignee":"Mukhammad Ibrokhimov","owner":"ibrohimovmuhammad2020@gmail.com","created_at":"2026-05-15T18:02:54Z","created_by":"Mukhammad Ibrokhimov","updated_at":"2026-05-19T09:58:05Z","started_at":"2026-05-19T09:51:48Z","closed_at":"2026-05-19T09:58:05Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"FlatPilot-ic1","title":"kleinanzeigen filler: fill landlord-required structured fields (Anrede, Vorname, etc.)","description":"The kleinanzeigen contact form has up to 15 fields (Anrede, Vorname, Nachname, Telefon, Straße/Nr., PLZ, SCHUFA-Auskunft, Haushaltstyp, Arbeitsverhältnis, …). Each landlord configures which fields they want filled. The filler currently fills only the message; on listings that mark structured fields as required, the submit either silently ignores us or fails inline validation in a way that fires neither the success nor the error banner.\n\nSymptom: 'kleinanzeigen: neither success nor error indicator appeared within 7000ms after submit' (now 15s, see FlatPilot-8kt's followup PR). Observed on flat 590 'Terrassenwohnung Berlin-Dahlem'. Sister flat 20 succeeded with the same filler — different landlords request different fields.\n\n## Plan\n\n1. Extend Profile schema with the kleinanzeigen-specific fields:\n - anrede: Literal['frau', 'herr', 'divers', 'keine_angabe'] (or use existing 'gender' if added)\n - given_name, surname (split from a single 'name' OR use 'household_contact_name')\n - phone (optional)\n - address: street, plz, city (already have city; add street + plz)\n - schufa_status: enum\n - household_type: enum\n - employment_status (already have 'status', map kleinanzeigen-specific labels to it)\n\n2. Wizard: prompt for these on `flatpilot init` (and `flatpilot init --reset`).\n\n3. Filler: after _reveal_contact_form, iterate visible fields under form#viewad-contact-form and fill each from the profile if a match exists. Use Playwright's locator(label='Anrede') / select_option for dropdowns. Skip optional fields the profile doesn't define.\n\n4. Tests: fixture HTML for the 15-field variant; fixture HTML for the message-only variant; assert each is filled correctly.\n\n## Out of scope\n\n- wg-gesucht does not use structured fields — this is kleinanzeigen-only for now.\n- inberlinwohnen has no filler at all (FlatPilot-mrt — separate decision).\n\n## Why deferred\n\nThe followup PR after FlatPilot-8kt adds failure-screenshot capture + bumps the success-detection timeout to 15s. That gives diagnostic reach AND fixes the failure case where kleinanzeigen XHR was slow but the submit DID complete. If the timeout fix alone makes flat 590-style failures go away, this bead drops priority. If failures persist with the longer timeout AND the screenshot shows visible inline-validation errors on un-filled fields, this becomes the next blocking work.","status":"in_progress","priority":3,"issue_type":"feature","assignee":"Mukhammad Ibrokhimov","owner":"ibrohimovmuhammad2020@gmail.com","created_at":"2026-05-09T22:18:34Z","created_by":"Mukhammad Ibrokhimov","updated_at":"2026-05-19T12:04:33Z","started_at":"2026-05-19T12:04:33Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"FlatPilot-ic1","title":"kleinanzeigen filler: fill landlord-required structured fields (Anrede, Vorname, etc.)","description":"The kleinanzeigen contact form has up to 15 fields (Anrede, Vorname, Nachname, Telefon, Straße/Nr., PLZ, SCHUFA-Auskunft, Haushaltstyp, Arbeitsverhältnis, …). Each landlord configures which fields they want filled. The filler currently fills only the message; on listings that mark structured fields as required, the submit either silently ignores us or fails inline validation in a way that fires neither the success nor the error banner.\n\nSymptom: 'kleinanzeigen: neither success nor error indicator appeared within 7000ms after submit' (now 15s, see FlatPilot-8kt's followup PR). Observed on flat 590 'Terrassenwohnung Berlin-Dahlem'. Sister flat 20 succeeded with the same filler — different landlords request different fields.\n\n## Plan\n\n1. Extend Profile schema with the kleinanzeigen-specific fields:\n - anrede: Literal['frau', 'herr', 'divers', 'keine_angabe'] (or use existing 'gender' if added)\n - given_name, surname (split from a single 'name' OR use 'household_contact_name')\n - phone (optional)\n - address: street, plz, city (already have city; add street + plz)\n - schufa_status: enum\n - household_type: enum\n - employment_status (already have 'status', map kleinanzeigen-specific labels to it)\n\n2. Wizard: prompt for these on `flatpilot init` (and `flatpilot init --reset`).\n\n3. Filler: after _reveal_contact_form, iterate visible fields under form#viewad-contact-form and fill each from the profile if a match exists. Use Playwright's locator(label='Anrede') / select_option for dropdowns. Skip optional fields the profile doesn't define.\n\n4. Tests: fixture HTML for the 15-field variant; fixture HTML for the message-only variant; assert each is filled correctly.\n\n## Out of scope\n\n- wg-gesucht does not use structured fields — this is kleinanzeigen-only for now.\n- inberlinwohnen has no filler at all (FlatPilot-mrt — separate decision).\n\n## Why deferred\n\nThe followup PR after FlatPilot-8kt adds failure-screenshot capture + bumps the success-detection timeout to 15s. That gives diagnostic reach AND fixes the failure case where kleinanzeigen XHR was slow but the submit DID complete. If the timeout fix alone makes flat 590-style failures go away, this bead drops priority. If failures persist with the longer timeout AND the screenshot shows visible inline-validation errors on un-filled fields, this becomes the next blocking work.","status":"closed","priority":3,"issue_type":"feature","assignee":"Mukhammad Ibrokhimov","owner":"ibrohimovmuhammad2020@gmail.com","created_at":"2026-05-09T22:18:34Z","created_by":"Mukhammad Ibrokhimov","updated_at":"2026-05-19T13:44:45Z","started_at":"2026-05-19T12:04:33Z","closed_at":"2026-05-19T13:44:45Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"FlatPilot-pns","title":"wg-gesucht filler: classify 'no contact CTA' as listing_expired (already-messaged listings)","description":"WG-Gesucht hides the 'Nachricht senden' contact CTA on a listing once the logged-in user has messaged it. The wg-gesucht filler currently raises a non-classified FormNotFoundError ('no contact CTA matching ...') in that case, which apply_to_flat records as status='failed' rather than status='failed' WITH notes='auto_skipped: listing_expired'.\n\nEffect:\n- The flat keeps getting retried by run_pipeline_apply on every pass until max_failures_per_flat (default 3) blocks it. Each retry costs a Playwright session, a polite_session ramp-up, and a row in the applications table — pure waste.\n- max_failures_per_flat is meant for genuine filler regressions (selector breakage, auth loops); using it to gate already-applied listings is a misuse that hides real failures.\n- Cooldown still fires after each 'failure', wasting cap headroom by spacing out retries that will never succeed.\n\n## Plan\n\nIn flatpilot/fillers/wg_gesucht.py, _reveal_contact_form (or wherever 'no contact CTA' is raised), wrap the FormNotFoundError raise so callers can detect 'CTA missing on a logged-in session' as ListingExpiredError instead. The existing FlatPilot-tgw machinery then:\n- Writes notes='auto_skipped: listing_expired (...)'\n- Excludes the flat from the apply queue for LISTING_EXPIRED_TTL (7 days)\n- Doesn't count toward max_failures_per_flat\n\n## Acceptance\n\n1. WGGesuchtFiller raises ListingExpiredError when the listing page renders without a 'Nachricht senden' CTA AND the page does not show the login wall (LOGIN_URL_FRAGMENTS guard already handles that case).\n2. Unit test in tests/test_filler_submit.py covers both branches: CTA missing + logged in → ListingExpiredError; CTA missing + login wall → NotAuthenticatedError (current behaviour).\n3. Pipeline integration test verifies the 'auto_skipped: listing_expired' note is written and the flat is excluded from subsequent queue scans for 7 days.\n\n## Related\n- FlatPilot-8kt — the success-detection fix exposes this gap because real successes now correctly mark flats submitted=True; legacy false-fail rows from the pre-fix era keep poisoning the queue, so the 'auto-apply only finds dead listings' symptom is most visible right now.\n- FlatPilot-tgw — original listing-expired classification + TTL.","notes":"Closed 2026-05-14: already implemented by FlatPilot-tgw (commit 3fd5f08). src/flatpilot/fillers/wg_gesucht.py:267 raises ListingExpiredError when contact CTA is absent; unit test tests/test_filler_submit.py:363 covers the logged-in branch. The acceptance's 'CTA missing + login wall' sub-branch is structurally unreachable — listing URLs don't redirect to /login for logged-out users; only the contact-form URL does, caught by the second _guard_login() at wg_gesucht.py:279 before any CTA check runs.","status":"closed","priority":3,"issue_type":"task","assignee":"Mukhammad Ibrokhimov","owner":"ibrohimovmuhammad2020@gmail.com","created_at":"2026-05-09T21:41:40Z","created_by":"Mukhammad Ibrokhimov","updated_at":"2026-05-14T10:30:16Z","started_at":"2026-05-14T10:24:50Z","closed_at":"2026-05-14T10:30:16Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"FlatPilot-iwu","title":"flatpilot status: surface auto-apply runtime state (pause, caps, cooldowns)","description":"flatpilot status today reports DB counts and last-run info. It does not surface the runtime state of the auto-apply subsystem, so a user has no quick way to answer:\n\n- Is auto-apply paused? (~/.flatpilot/PAUSE present?)\n- How many of today's daily caps are used per platform? (daily_cap_per_platform vs. submitted today)\n- Is a per-platform cooldown active right now, and how many seconds remain?\n- How many flats are currently above max_failures_per_flat and being skipped?\n\nAll of these are already computable from helpers in flatpilot/auto_apply.py (is_paused, daily_cap_remaining, cooldown_remaining_sec, failures_for_flat) and the applications table — flatpilot status just doesn't display them.\n\n## Proposed surface\n\nExtend the `flatpilot status` Rich output with a new \"Auto-apply\" panel:\n\n Auto-apply\n State: ACTIVE (or \"PAUSED — ~/.flatpilot/PAUSE\")\n wg-gesucht 12 / 20 today · cooldown 0s\n kleinanzeigen 3 / 20 today · cooldown 47s remaining\n inberlinwohnen — (no filler registered)\n Flats over max_failures: 2\n\n## Acceptance\n\n1. `flatpilot status` shows the panel above with live values pulled from the same helpers used by the pipeline.\n2. PAUSED state is visually distinct (yellow) so it cannot be missed.\n3. Platforms without a registered filler are listed but marked as not auto-applying.\n4. New unit test in tests/ covers the panel rendering with a populated DB fixture (paused vs. active, cap exhausted, cooldown active, max-failure flats).\n\n## Out of scope\n\n- No new auto-apply behavior; this is a read-only display.\n- No web UI surface — CLI only (UI work is paused, see project notes).\n\n## Why now\n\nPhase 4 (FlatPilot-f2fa) shipped the engine; without this panel the user has to read SQLite to understand what the engine is currently doing. Closing the observability gap is a natural follow-up.","status":"closed","priority":3,"issue_type":"feature","owner":"ibrohimovmuhammad2020@gmail.com","created_at":"2026-05-08T09:51:19Z","created_by":"Mukhammad Ibrokhimov","updated_at":"2026-05-08T20:46:36Z","closed_at":"2026-05-08T20:46:36Z","close_reason":"Added Auto-apply panel to flatpilot status (cli.py _print_auto_apply_panel) showing pause state (ACTIVE/PAUSED), per-platform usage and cooldown countdown, platforms-without-filler marker, and flats-over-max-failures count. New flats_over_max_failures helper in auto_apply.py with unit tests. 9 new tests in test_auto_apply_status_panel.py; full suite 505/505 green.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"FlatPilot-5mf","title":"Add ImmoScout24 login recipe (or formally drop)","description":"FlatPilot-8jx's UI mockup shows an 'ImmoScout24 [Connect →]' button on the Connected Accounts page, but src/flatpilot/login.py:_LOGIN_SITES has no immoscout24 entry — the underlying CLI cannot drive that flow. Today the ImmoScout24 integration is RSS-only (flatpilot.scrapers.immoscout24_rss reads URLs the user pastes from their account); there is no filler and arguably no need to seed cookies.\n\nTwo ways to resolve:\n1. Add a login recipe pointing at https://www.immobilienscout24.de/anmelden.html so the Connect button has a backend, even though no current code path consumes the cookies. Keeps the UI promise consistent.\n2. Decide RSS-only is the supported integration, drop the ImmoScout24 button from FlatPilot-8jx's scope, and close this as won't-do.\n\nResolution should pick one of those, not leave the contradiction.\n\nBlocks the ImmoScout24 part of FlatPilot-8jx.","status":"closed","priority":3,"issue_type":"task","assignee":"Mukhammad Ibrokhimov","owner":"ibrohimovmuhammad2020@gmail.com","created_at":"2026-05-05T20:35:10Z","created_by":"Mukhammad Ibrokhimov","updated_at":"2026-05-05T22:11:31Z","started_at":"2026-05-05T22:02:03Z","closed_at":"2026-05-05T22:11:31Z","close_reason":"Added login recipe for ImmoScout24 (anmelden.html) so 'flatpilot login immoscout24' and the Web UI Connect button (FlatPilot-8jx) have a backend, even though no current code path consumes the seeded cookies. Branch: feat/immoscout24-login-recipe.","dependency_count":0,"dependent_count":0,"comment_count":0} @@ -148,12 +148,12 @@ {"_type":"issue","id":"FlatPilot-u5gh","title":"O4. Docker-compose + deployment docs","description":"One-command local stack; deployment instructions for a VPS.","status":"open","priority":4,"issue_type":"task","created_at":"2026-04-20T15:52:57Z","updated_at":"2026-04-20T15:52:57Z","dependencies":[{"issue_id":"FlatPilot-u5gh","depends_on_id":"FlatPilot-x0pq","type":"blocks","created_at":"2026-04-20T17:52:57Z","created_by":"Mukhammad Ibrokhimov","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"FlatPilot-x0pq","title":"O1. Architecture decision record","description":"Decide: FastAPI + Next.js; magic-link auth; SQLite -\u003e Postgres migration path; deployment story (docker-compose starter).","status":"open","priority":4,"issue_type":"task","assignee":"Mukhammad Ibrokhimov","created_at":"2026-04-20T15:52:57Z","updated_at":"2026-05-05T21:08:19Z","started_at":"2026-05-03T15:38:59Z","dependency_count":0,"dependent_count":3,"comment_count":0} {"_type":"issue","id":"FlatPilot-z3me","title":"O2. Per-user isolation in data model","description":"All tables gain user_id; row-level scoping; migration from single-user state.","status":"open","priority":4,"issue_type":"task","created_at":"2026-04-20T15:52:57Z","updated_at":"2026-04-20T15:52:57Z","dependencies":[{"issue_id":"FlatPilot-z3me","depends_on_id":"FlatPilot-x0pq","type":"blocks","created_at":"2026-04-20T17:52:57Z","created_by":"Mukhammad Ibrokhimov","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} -{"_type":"memory","key":"commit-author-for-this-repo-mukhammad-ibrokhimov-ibrohimovmu","value":"Commit author for this repo: Mukhammad Ibrokhimov \u003cibrohimovmuhammad2020@gmail.com\u003e. In a fresh clone: git config user.email ibrohimovmuhammad2020@gmail.com \u0026\u0026 git config user.name 'Mukhammad Ibrokhimov'. Any other email on the machine is NOT for commits here."} -{"_type":"memory","key":"flatpilot-no-credential-storage","value":"Never store user passwords for rental platforms (WG-Gesucht, Kleinanzeigen, ImmoScout24, future) anywhere in FlatPilot — not in profile.json, .env, or any other file. Reasons: (1) Docker container can't reach host OS keyring, leaving only file storage which is unsafe for credentials; (2) programmatic credential login is a strong anti-bot signal that risks getting accounts flagged; (3) 2FA / captchas still require human intervention even with stored passwords; (4) ToS posture is much worse than cookie reuse from a real human session. Solution: cookies-only auth. CLI version (FlatPilot-5b8) uses headed Playwright + 'press Enter when logged in' UX. Future multi-user Web UI (FlatPilot-8jx) uses browser extension or bookmarklet to capture cookies from the user's normal browser session — never asks for or stores passwords. None of WG-Gesucht / Kleinanzeigen / ImmoScout24 publish OAuth endpoints, so a literal 'Sign in with Platform' redirect-and-callback flow is structurally impossible. Decided in PR #18 session 2026-04-22."} {"_type":"memory","key":"clean-room-policy-flatpilot-is-independent-mit-licensed","value":"Clean-room policy: FlatPilot is independent MIT-licensed. Never copy code from any other project (including any job-hunting agent that may have inspired the high-level architecture). Read outside code only as a design reference; always rewrite from scratch."} -{"_type":"memory","key":"after-a-flatpilot-pr-merges-do-not-delete","value":"After a FlatPilot PR merges, DO NOT delete the local or remote feature branch, and DO NOT drop stashes, unless the user explicitly asks. Why: user's standing preference to preserve merged branches for historical reference (e.g. feat/l2-anschreiben-composer was called out as hands-off in the handoff). How to apply: after merge, only sync main (git checkout main \u0026\u0026 git pull --ff-only) and stop. Do not run 'git branch -d', 'git push --delete', 'git stash drop', or any similar cleanup without an explicit request. Treat 'cleanup' as scope the user opts into, not a default step."} {"_type":"memory","key":"flatpilot-product-direction","value":"FlatPilot is currently single-user CLI (Phase 3). Long-term direction: convert to small multi-user web app — 'Option C' scope — for friends-of-friends, no public marketing, no payments. Strict sequencing: complete Phase 3 CLI for the developer's own use FIRST, then validate by using it personally on real WG-Gesucht searches for 1-2 weeks, THEN start Epic O (Web UI / FlatPilot-0wfb). Do NOT start FastAPI / React / Postgres work until Phase 3 is shipped end-to-end and personally validated. The CLI work is not throwaway — every Python module becomes a backend module the FastAPI app calls later. Phase 5 will need: Hetzner/Fly hosting, Postgres migration, FastAPI backend, React frontend, magic-link email auth, browser extension OR bookmarklet for per-user platform cookie capture. Decided in PR #18 session 2026-04-22."} +{"_type":"memory","key":"flatpilot-no-credential-storage","value":"Never store user passwords for rental platforms (WG-Gesucht, Kleinanzeigen, ImmoScout24, future) anywhere in FlatPilot — not in profile.json, .env, or any other file. Reasons: (1) Docker container can't reach host OS keyring, leaving only file storage which is unsafe for credentials; (2) programmatic credential login is a strong anti-bot signal that risks getting accounts flagged; (3) 2FA / captchas still require human intervention even with stored passwords; (4) ToS posture is much worse than cookie reuse from a real human session. Solution: cookies-only auth. CLI version (FlatPilot-5b8) uses headed Playwright + 'press Enter when logged in' UX. Future multi-user Web UI (FlatPilot-8jx) uses browser extension or bookmarklet to capture cookies from the user's normal browser session — never asks for or stores passwords. None of WG-Gesucht / Kleinanzeigen / ImmoScout24 publish OAuth endpoints, so a literal 'Sign in with Platform' redirect-and-callback flow is structurally impossible. Decided in PR #18 session 2026-04-22."} +{"_type":"memory","key":"commit-author-for-this-repo-mukhammad-ibrokhimov-ibrohimovmu","value":"Commit author for this repo: Mukhammad Ibrokhimov \u003cibrohimovmuhammad2020@gmail.com\u003e. In a fresh clone: git config user.email ibrohimovmuhammad2020@gmail.com \u0026\u0026 git config user.name 'Mukhammad Ibrokhimov'. Any other email on the machine is NOT for commits here."} {"_type":"memory","key":"flatpilot-prompt-ride-along-checklist","value":"When the user's prompt has a 'Possible ride-alongs' section that explicitly lists candidates for inline inclusion, treat each listed item as a TASK ITEM, not just background reading. Add it to the working checklist before starting implementation, and verify each one is either (a) included in the PR or (b) explicitly justified as skipped before declaring the work done. Why: in PR #30 (FlatPilot-etu) the user explicitly suggested 'A README note about what flatpilot scrape --platform inberlinwohnen now ingests on fresh install → ride along inline' and I dropped it; only caught it when the user asked 'do we have any more related PR tasks'. How to apply: when reading any FlatPilot prompt, scan for 'ride-along' / 'inline' / 'fix inline' suggestions, list them in your initial pitch back to the user, and tick them off as you go."} -{"_type":"memory","key":"for-flatpilot-batch-several-small-beads-tasks-into","value":"For FlatPilot: one PR per epic. If an epic has \u003e10 tasks, split into multiple PRs (thematic split, topologically ordered). Otherwise the whole epic lands in one PR — one commit per task, each commit message referencing its beads ID. Small leftover tasks from completed sub-areas can ride along if they fit the theme. Why: the user set this rule after observing that per-task PRs were noise and mixed-theme mega-PRs were hard to review. How to apply: before claiming the first task in an epic, count the epic's open tasks with 'bd list' filtered to that epic; if \u003e10, decide on a thematic split (e.g. 'matcher core' vs 'matcher CLI') and declare it in the first PR description. Each PR closes every beads task it contains."} {"_type":"memory","key":"never-push-directly-to-main-every-change-goes","value":"Never push directly to main. Every change goes through a PR: git fetch origin; git checkout -b \u003ctype\u003e/\u003cslug\u003e origin/main; commit; git push -u origin \u003cbranch\u003e; gh pr create --base main --head \u003cbranch\u003e --fill. Return the PR URL and stop. Force-push allowed only on personal feature branches that nobody else has pulled (--force-with-lease)."} +{"_type":"memory","key":"after-a-flatpilot-pr-merges-do-not-delete","value":"After a FlatPilot PR merges, DO NOT delete the local or remote feature branch, and DO NOT drop stashes, unless the user explicitly asks. Why: user's standing preference to preserve merged branches for historical reference (e.g. feat/l2-anschreiben-composer was called out as hands-off in the handoff). How to apply: after merge, only sync main (git checkout main \u0026\u0026 git pull --ff-only) and stop. Do not run 'git branch -d', 'git push --delete', 'git stash drop', or any similar cleanup without an explicit request. Treat 'cleanup' as scope the user opts into, not a default step."} +{"_type":"memory","key":"for-flatpilot-batch-several-small-beads-tasks-into","value":"For FlatPilot: one PR per epic. If an epic has \u003e10 tasks, split into multiple PRs (thematic split, topologically ordered). Otherwise the whole epic lands in one PR — one commit per task, each commit message referencing its beads ID. Small leftover tasks from completed sub-areas can ride along if they fit the theme. Why: the user set this rule after observing that per-task PRs were noise and mixed-theme mega-PRs were hard to review. How to apply: before claiming the first task in an epic, count the epic's open tasks with 'bd list' filtered to that epic; if \u003e10, decide on a thematic split (e.g. 'matcher core' vs 'matcher CLI') and declare it in the first PR description. Each PR closes every beads task it contains."} {"_type":"memory","key":"no-ai-co-author-trailers-in-commits-no","value":"No AI co-author trailers in commits. No 'Co-Authored-By: Claude ...', no '🤖 Generated with Claude Code'. Commits are authored by the human only."} diff --git a/src/flatpilot/schemas.py b/src/flatpilot/schemas.py index 4dc6973..6095645 100644 --- a/src/flatpilot/schemas.py +++ b/src/flatpilot/schemas.py @@ -145,6 +145,13 @@ "triggered_by_saved_search": "TEXT", } +# bd-ko1 Part B: detail-page enrichment exposes the end-date of a fixed-term +# lease. Legacy DBs created before this column predate it, so add via +# ALTER TABLE here rather than touching FLATS_CREATE_SQL. +COLUMNS["flats"] = { + "available_until": "TEXT", +} + APPLICATIONS_METHOD_APPLIED_AT_INDEX_SQL = """ CREATE INDEX IF NOT EXISTS idx_applications_method_applied_at ON applications(method, applied_at) From 54e5adeca0c1eb01ea6e8d1c9512a69cca6a24a1 Mon Sep 17 00:00:00 2001 From: Mukhammad Ibrokhimov Date: Wed, 20 May 2026 20:04:02 +0200 Subject: [PATCH 02/13] bd-ko1: thread available_until through the Flat TypedDict --- src/flatpilot/scrapers/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/flatpilot/scrapers/base.py b/src/flatpilot/scrapers/base.py index ff47759..91d30e2 100644 --- a/src/flatpilot/scrapers/base.py +++ b/src/flatpilot/scrapers/base.py @@ -45,6 +45,7 @@ class Flat(TypedDict, total=False): lng: float online_since: str available_from: str + available_until: str requires_wbs: bool wbs_size_category: int wbs_income_category: int From c756d1d0e242ab918eaac73ffb0112acef91c187 Mon Sep 17 00:00:00 2001 From: Mukhammad Ibrokhimov Date: Wed, 20 May 2026 20:07:06 +0200 Subject: [PATCH 03/13] bd-ko1: add WG-Gesucht detail-page fixtures --- tests/fixtures/wg_gesucht/detail_unbefristet.html | 10 ++++++++++ .../wg_gesucht/detail_with_min_contract.html | 12 ++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 tests/fixtures/wg_gesucht/detail_unbefristet.html create mode 100644 tests/fixtures/wg_gesucht/detail_with_min_contract.html diff --git a/tests/fixtures/wg_gesucht/detail_unbefristet.html b/tests/fixtures/wg_gesucht/detail_unbefristet.html new file mode 100644 index 0000000..a87bde7 --- /dev/null +++ b/tests/fixtures/wg_gesucht/detail_unbefristet.html @@ -0,0 +1,10 @@ + + +
+
    +
  • Befristet: Nein
    • +
  • Verfügbar ab: 01.07.2026
    • +
+
+

Unbefristeter Mietvertrag, freie Bezug.

+ diff --git a/tests/fixtures/wg_gesucht/detail_with_min_contract.html b/tests/fixtures/wg_gesucht/detail_with_min_contract.html new file mode 100644 index 0000000..a6938b2 --- /dev/null +++ b/tests/fixtures/wg_gesucht/detail_with_min_contract.html @@ -0,0 +1,12 @@ + + +
+
    +
  • Mindestmietdauer: 24 Monate
    • +
  • Befristet: Ja
    • +
  • Verfügbar ab: 01.06.2026
    • +
  • Verfügbar bis: 31.05.2028
    • +
+
+

Schöne 2-Zimmer-Wohnung in Mitte. Mindestmietdauer 24 Monate.

+ From e0a58b82d83d4c783403267ee7601647fb5cd3e6 Mon Sep 17 00:00:00 2001 From: Mukhammad Ibrokhimov Date: Wed, 20 May 2026 20:07:40 +0200 Subject: [PATCH 04/13] bd-ko1: add WG-Gesucht parse_detail for min_contract_months + available_until --- src/flatpilot/scrapers/wg_gesucht.py | 35 ++++++++++++++++++++++++++++ tests/test_wg_gesucht_scraper.py | 26 +++++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/src/flatpilot/scrapers/wg_gesucht.py b/src/flatpilot/scrapers/wg_gesucht.py index eafe769..d33cd92 100644 --- a/src/flatpilot/scrapers/wg_gesucht.py +++ b/src/flatpilot/scrapers/wg_gesucht.py @@ -239,6 +239,41 @@ def _parse_card(card: Any) -> Flat | None: return flat +_MIN_CONTRACT_RE = re.compile( + r"Mindestmietdauer[^0-9]{0,20}(\d{1,3})\s*Monate?", re.IGNORECASE +) +_AVAILABLE_UNTIL_RE = re.compile( + r"Verf[uü]gbar\s+bis[^0-9]{0,20}(\d{2})\.(\d{2})\.(\d{4})", re.IGNORECASE +) + + +def parse_detail(html: str) -> dict[str, Any]: + """Extract long-term-contract fields from a WG-Gesucht detail page. + + Returns a dict with at most ``min_contract_months`` and + ``available_until`` (ISO date). Anything not present in the HTML is + omitted so the caller can ``UPDATE … SET k=v`` without overwriting + real data with ``None``. + """ + if not html: + return {} + from bs4 import BeautifulSoup + + soup = BeautifulSoup(html, "html.parser") + text = soup.get_text(" ", strip=True) + out: dict[str, Any] = {} + + m = _MIN_CONTRACT_RE.search(text) + if m: + out["min_contract_months"] = int(m.group(1)) + + m = _AVAILABLE_UNTIL_RE.search(text) + if m: + out["available_until"] = f"{m.group(3)}-{m.group(2)}-{m.group(1)}" + + return out + + def _district_from_url(href: str) -> str | None: # href looks like "/wohnungen-in-Berlin-Neukoelln.12345.html" m = re.search(r"/wohnungen-in-[^-/.]+-([^./]+)\.\d+\.", href) diff --git a/tests/test_wg_gesucht_scraper.py b/tests/test_wg_gesucht_scraper.py index 4660eee..febba4a 100644 --- a/tests/test_wg_gesucht_scraper.py +++ b/tests/test_wg_gesucht_scraper.py @@ -102,3 +102,29 @@ def test_fetch_new_raises_for_unsupported_city(): with pytest.raises(UnknownCityError): # Generator must be drained for the body to execute. list(scraper.fetch_new(profile, known_external_ids=frozenset())) + + +DETAIL_WITH = Path(__file__).parent / "fixtures" / "wg_gesucht" / "detail_with_min_contract.html" +DETAIL_UNBEFRISTET = Path(__file__).parent / "fixtures" / "wg_gesucht" / "detail_unbefristet.html" + + +def test_parse_detail_extracts_min_contract_and_until(): + from flatpilot.scrapers.wg_gesucht import parse_detail + + fields = parse_detail(DETAIL_WITH.read_text()) + assert fields["min_contract_months"] == 24 + assert fields["available_until"] == "2028-05-31" + + +def test_parse_detail_unbefristet_yields_no_min_contract(): + from flatpilot.scrapers.wg_gesucht import parse_detail + + fields = parse_detail(DETAIL_UNBEFRISTET.read_text()) + assert "min_contract_months" not in fields + assert "available_until" not in fields + + +def test_parse_detail_handles_empty_html(): + from flatpilot.scrapers.wg_gesucht import parse_detail + + assert parse_detail("") == {} From 7bccc9c136e769256469909bad03c070dd1325da Mon Sep 17 00:00:00 2001 From: Mukhammad Ibrokhimov Date: Wed, 20 May 2026 20:11:51 +0200 Subject: [PATCH 05/13] bd-ko1: add Kleinanzeigen detail-page fixtures --- .../kleinanzeigen/detail_no_min_contract.html | 14 ++++++++++++++ .../detail_with_min_contract.html | 18 ++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 tests/fixtures/kleinanzeigen/detail_no_min_contract.html create mode 100644 tests/fixtures/kleinanzeigen/detail_with_min_contract.html diff --git a/tests/fixtures/kleinanzeigen/detail_no_min_contract.html b/tests/fixtures/kleinanzeigen/detail_no_min_contract.html new file mode 100644 index 0000000..449d130 --- /dev/null +++ b/tests/fixtures/kleinanzeigen/detail_no_min_contract.html @@ -0,0 +1,14 @@ + + +
    +
  • + Vertragsart + Unbefristet +
    • +
  • + Verfügbar ab + 01.07.2026 +
    • +
+

Unbefristeter Mietvertrag.

+ diff --git a/tests/fixtures/kleinanzeigen/detail_with_min_contract.html b/tests/fixtures/kleinanzeigen/detail_with_min_contract.html new file mode 100644 index 0000000..0e00e8b --- /dev/null +++ b/tests/fixtures/kleinanzeigen/detail_with_min_contract.html @@ -0,0 +1,18 @@ + + +
    +
  • + Vertragsart + Befristet +
    • +
  • + Mindestmietdauer + 24 Monate +
    • +
  • + Verfügbar ab + 01.06.2026 +
    • +
+

Schöne Wohnung, Mindestmietdauer 24 Monate.

+ From 865d5cab93b5a93fe17202a48aba9e1cf43901f5 Mon Sep 17 00:00:00 2001 From: Mukhammad Ibrokhimov Date: Wed, 20 May 2026 20:12:26 +0200 Subject: [PATCH 06/13] bd-ko1: add Kleinanzeigen parse_detail for min_contract_months --- src/flatpilot/scrapers/kleinanzeigen.py | 29 +++++++++++++++++++++++++ tests/test_kleinanzeigen_scraper.py | 24 ++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/src/flatpilot/scrapers/kleinanzeigen.py b/src/flatpilot/scrapers/kleinanzeigen.py index ef735b8..e4eb0c4 100644 --- a/src/flatpilot/scrapers/kleinanzeigen.py +++ b/src/flatpilot/scrapers/kleinanzeigen.py @@ -265,6 +265,35 @@ def _parse_card(card: Any) -> Flat | None: return flat +# Kleinanzeigen exposes "Mindestmietdauer" on detail pages; some sublet +# variants show "Mietdauer" instead. Accept both, case-insensitive. +_MIN_CONTRACT_RE = re.compile( + r"(?:Mindest)?Mietdauer[^0-9]{0,20}(\d{1,3})\s*Monate?", re.IGNORECASE +) + + +def parse_detail(html: str) -> dict[str, Any]: + """Extract ``min_contract_months`` from a Kleinanzeigen detail page. + + Returns a dict with at most ``min_contract_months``. The cheap + ``filter_short_term`` keyword filter (bd-33h) remains the fallback + for fixed-term listings without an explicit Mindestmietdauer. + """ + if not html: + return {} + from bs4 import BeautifulSoup + + soup = BeautifulSoup(html, "html.parser") + text = soup.get_text(" ", strip=True) + out: dict[str, Any] = {} + + m = _MIN_CONTRACT_RE.search(text) + if m: + out["min_contract_months"] = int(m.group(1)) + + return out + + def _clean(text: str) -> str: return " ".join(text.split()) diff --git a/tests/test_kleinanzeigen_scraper.py b/tests/test_kleinanzeigen_scraper.py index 05f822f..4295756 100644 --- a/tests/test_kleinanzeigen_scraper.py +++ b/tests/test_kleinanzeigen_scraper.py @@ -125,3 +125,27 @@ def test_fetch_new_uses_pinned_ua_and_stealth( assert cfg.platform == "kleinanzeigen" assert cfg.user_agent == pinned assert cfg.stealth is True + + +DETAIL_WITH = Path(__file__).parent / "fixtures" / "kleinanzeigen" / "detail_with_min_contract.html" +DETAIL_NO_MIN = Path(__file__).parent / "fixtures" / "kleinanzeigen" / "detail_no_min_contract.html" + + +def test_parse_detail_extracts_min_contract(): + from flatpilot.scrapers.kleinanzeigen import parse_detail + + fields = parse_detail(DETAIL_WITH.read_text()) + assert fields["min_contract_months"] == 24 + + +def test_parse_detail_no_min_contract_returns_empty_for_that_field(): + from flatpilot.scrapers.kleinanzeigen import parse_detail + + fields = parse_detail(DETAIL_NO_MIN.read_text()) + assert "min_contract_months" not in fields + + +def test_parse_detail_empty_html_returns_empty_dict(): + from flatpilot.scrapers.kleinanzeigen import parse_detail + + assert parse_detail("") == {} From 0e44abbb97954718029eb75ee7c07f4134d71dd4 Mon Sep 17 00:00:00 2001 From: Mukhammad Ibrokhimov Date: Wed, 20 May 2026 20:30:23 +0200 Subject: [PATCH 07/13] bd-ko1: add network fetch_detail helpers for both scrapers --- src/flatpilot/scrapers/kleinanzeigen.py | 24 ++++++++++++++++++++++++ src/flatpilot/scrapers/wg_gesucht.py | 24 ++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/src/flatpilot/scrapers/kleinanzeigen.py b/src/flatpilot/scrapers/kleinanzeigen.py index e4eb0c4..27d33ff 100644 --- a/src/flatpilot/scrapers/kleinanzeigen.py +++ b/src/flatpilot/scrapers/kleinanzeigen.py @@ -310,3 +310,27 @@ def _first_float(pattern: re.Pattern[str], text: str) -> float | None: return float(raw) except ValueError: return None + + +def fetch_detail(listing_url: str) -> str: + """Return the raw HTML of a Kleinanzeigen detail page. + + Uses the UA pool + stealth config so a per-listing burst still + looks like organic traffic. The orchestrator caps the burst at + ``budget`` flats per pipeline pass. + """ + config = SessionConfig( + platform=KleinanzeigenScraper.platform, + user_agent=pin_user_agent(KleinanzeigenScraper.platform), + warmup_url=None, + consent_selectors=CONSENT_SELECTORS, + stealth=True, + ) + with polite_session(config) as context, session_page(context) as pg: + response = pg.goto(listing_url, wait_until="domcontentloaded") + if response is None: + return "" + check_rate_limit(response.status, KleinanzeigenScraper.platform) + if response.status >= 400: + return "" + return _handle_response(pg, city="") diff --git a/src/flatpilot/scrapers/wg_gesucht.py b/src/flatpilot/scrapers/wg_gesucht.py index d33cd92..9ad51a8 100644 --- a/src/flatpilot/scrapers/wg_gesucht.py +++ b/src/flatpilot/scrapers/wg_gesucht.py @@ -301,3 +301,27 @@ def _first_date(text: str) -> str | None: return None day, month, year = m.group(1), m.group(2), m.group(3) return f"{year}-{month}-{day}" + + +def fetch_detail(listing_url: str) -> str: + """Return the raw HTML of a WG-Gesucht detail page. + + Each call opens a polite_session, navigates once, and closes — + cheap relative to the cost of being detected because polite_session + persists cookies across runs. The orchestrator caps how many of + these run per pipeline pass. + """ + config = SessionConfig( + platform=WGGesuchtScraper.platform, + user_agent=WGGesuchtScraper.user_agent, + warmup_url=None, + consent_selectors=CONSENT_SELECTORS, + ) + with polite_session(config) as context, session_page(context) as pg: + response = pg.goto(listing_url, wait_until="domcontentloaded") + if response is None: + return "" + check_rate_limit(response.status, WGGesuchtScraper.platform) + if response.status >= 400: + return "" + return pg.content() From 8bb85867b8066cff16175e14848cc27ed28aa8cb Mon Sep 17 00:00:00 2001 From: Mukhammad Ibrokhimov Date: Wed, 20 May 2026 20:44:31 +0200 Subject: [PATCH 08/13] bd-ko1: add detail enrichment orchestrator with budget and failure fallback --- src/flatpilot/scrapers/detail.py | 127 +++++++++++++++++++ tests/test_pipeline_enrich.py | 211 +++++++++++++++++++++++++++++++ 2 files changed, 338 insertions(+) create mode 100644 src/flatpilot/scrapers/detail.py create mode 100644 tests/test_pipeline_enrich.py diff --git a/src/flatpilot/scrapers/detail.py b/src/flatpilot/scrapers/detail.py new file mode 100644 index 0000000..aa60e08 --- /dev/null +++ b/src/flatpilot/scrapers/detail.py @@ -0,0 +1,127 @@ +"""Detail-page enrichment orchestrator (bd-ko1 Part B). + +Runs between scrape and match. For each platform whose scraper exposes +``parse_detail``, selects up to ``budget`` canonical flats that are +missing ``min_contract_months`` and that the user's profile would +require it for. Fetches the detail page via the per-platform +``fetch_detail`` (real network) or an injected callable (tests), then +UPDATEs the flat row. + +Network failures are logged and counted; the flat row is left alone so +the cheap title/description heuristic from bd-33h remains the safety +net for fixed-term listings whose detail page we could not reach. +""" + +from __future__ import annotations + +import logging +import time +from collections.abc import Callable +from typing import TypedDict + +from flatpilot.database import get_conn, init_db +from flatpilot.profile import Profile + +logger = logging.getLogger(__name__) + +# Politeness gap between detail fetches within one pass. Conservative — +# detail pages are 1 request each and almost always cached client-side +# by the time the user visits, but a small gap keeps the platform's +# anti-bot heuristics from seeing a burst. +DETAIL_FETCH_DELAY_SEC: float = 2.0 + + +class EnrichSummary(TypedDict): + candidates: int + fetched: int + failed: int + + +Fetcher = Callable[[str], str] + + +def _default_fetchers() -> dict[str, Fetcher]: + from flatpilot.scrapers.kleinanzeigen import fetch_detail as fetch_ka + from flatpilot.scrapers.wg_gesucht import fetch_detail as fetch_wg + + return {"wg-gesucht": fetch_wg, "kleinanzeigen": fetch_ka} + + +def _parsers() -> dict[str, Callable[[str], dict]]: + from flatpilot.scrapers.kleinanzeigen import parse_detail as parse_ka + from flatpilot.scrapers.wg_gesucht import parse_detail as parse_wg + + return {"wg-gesucht": parse_wg, "kleinanzeigen": parse_ka} + + +def enrich_pending_flats( + profile: Profile, + console, + *, + budget: int = 30, + fetchers: dict[str, Fetcher] | None = None, +) -> EnrichSummary: + """Fetch + parse detail pages for pending canonical flats. + + Returns an :class:`EnrichSummary`. The ``fetchers`` injection point + lets tests pass deterministic stubs; production callers omit it and + get the real Playwright-backed helpers. + """ + summary: EnrichSummary = {"candidates": 0, "fetched": 0, "failed": 0} + if profile.min_contract_months is None: + return summary + + init_db() + conn = get_conn() + fetchers = fetchers or _default_fetchers() + parsers = _parsers() + + rows = conn.execute( + """ + SELECT id, platform, listing_url + FROM flats + WHERE canonical_flat_id IS NULL + AND min_contract_months IS NULL + AND platform IN ('wg-gesucht', 'kleinanzeigen') + ORDER BY id DESC + LIMIT ? + """, + (budget,), + ).fetchall() + summary["candidates"] = len(rows) + + for row in rows: + platform = row["platform"] + fetcher = fetchers.get(platform) + parser = parsers.get(platform) + if fetcher is None or parser is None: + continue + try: + html = fetcher(row["listing_url"]) + except Exception as exc: + logger.warning( + "%s: detail fetch failed for flat %s (%s: %s)", + platform, row["id"], exc.__class__.__name__, exc, + ) + summary["failed"] += 1 + continue + + fields = parser(html) if html else {} + if fields: + _update_flat(conn, row["id"], fields) + summary["fetched"] += 1 + time.sleep(DETAIL_FETCH_DELAY_SEC) + + console.print( + f"enrich: [bold]{summary['candidates']}[/bold] candidates, " + f"[green]{summary['fetched']}[/green] enriched" + + (f", [red]{summary['failed']} failed[/red]" if summary["failed"] else "") + ) + return summary + + +def _update_flat(conn, flat_id: int, fields: dict) -> None: + assignments = ", ".join(f"{k} = :{k}" for k in fields) + params = dict(fields) + params["id"] = flat_id + conn.execute(f"UPDATE flats SET {assignments} WHERE id = :id", params) diff --git a/tests/test_pipeline_enrich.py b/tests/test_pipeline_enrich.py new file mode 100644 index 0000000..d41db86 --- /dev/null +++ b/tests/test_pipeline_enrich.py @@ -0,0 +1,211 @@ +"""Detail-enrichment orchestrator: budget, gating, failure-fallback.""" +from __future__ import annotations + +from rich.console import Console + +import flatpilot.scrapers.detail as _detail_mod + +_detail_mod.DETAIL_FETCH_DELAY_SEC = 0.0 + +from flatpilot.database import get_conn, init_db +from flatpilot.scrapers.detail import enrich_pending_flats + + +def _profile(**overrides): + from flatpilot.profile import Profile + return Profile.load_example().model_copy(update=overrides) + + +def _insert(conn, **flat): + flat.setdefault("scraped_at", "2026-05-20T00:00:00+00:00") + flat.setdefault("first_seen_at", "2026-05-20T00:00:00+00:00") + cols = list(flat.keys()) + sql = f"INSERT INTO flats ({', '.join(cols)}) VALUES ({', '.join(f':{c}' for c in cols)})" + conn.execute(sql, flat) + return conn.execute("SELECT last_insert_rowid() AS id").fetchone()["id"] + + +def test_enrich_populates_min_contract_and_available_until(tmp_db): + init_db() + conn = get_conn() + flat_id = _insert( + conn, + external_id="111", + platform="wg-gesucht", + listing_url="https://www.wg-gesucht.de/wohnungen-in-Berlin.111.html", + title="2 Zimmer Berlin", + rent_warm_eur=900, + rooms=2, + ) + + def fake_fetch_wg(url): + return ( + "" + "
Mindestmietdauer: 24 Monate
" + "
Verfügbar bis: 31.12.2027
" + "" + ) + + profile = _profile(city="Berlin", min_contract_months=12) + summary = enrich_pending_flats( + profile, Console(), budget=10, + fetchers={"wg-gesucht": fake_fetch_wg}, + ) + assert summary["fetched"] == 1 + row = conn.execute( + "SELECT min_contract_months, available_until FROM flats WHERE id=?", + (flat_id,), + ).fetchone() + assert row["min_contract_months"] == 24 + assert row["available_until"] == "2027-12-31" + + +def test_enrich_respects_budget(tmp_db): + init_db() + conn = get_conn() + for i in range(5): + _insert( + conn, + external_id=f"id{i}", + platform="wg-gesucht", + listing_url=f"https://www.wg-gesucht.de/{i}.html", + title=f"Listing {i}", + rent_warm_eur=900, + rooms=2, + ) + + calls = [] + + def fake_fetch_wg(url): + calls.append(url) + return "Mindestmietdauer: 24 Monate" + + profile = _profile(city="Berlin", min_contract_months=12) + summary = enrich_pending_flats( + profile, Console(), budget=2, + fetchers={"wg-gesucht": fake_fetch_wg}, + ) + assert summary["fetched"] == 2 + assert len(calls) == 2 + + +def test_enrich_skips_when_profile_has_no_min_contract(tmp_db): + init_db() + conn = get_conn() + _insert( + conn, + external_id="111", + platform="wg-gesucht", + listing_url="https://www.wg-gesucht.de/111.html", + title="t", + rent_warm_eur=900, + rooms=2, + ) + calls = [] + + def fake_fetch_wg(url): + calls.append(url) + return "" + + profile = _profile(city="Berlin", min_contract_months=None) + summary = enrich_pending_flats( + profile, Console(), budget=10, + fetchers={"wg-gesucht": fake_fetch_wg}, + ) + assert summary["fetched"] == 0 + assert calls == [] + + +def test_enrich_skips_canonical_dupes(tmp_db): + init_db() + conn = get_conn() + root_id = _insert( + conn, + external_id="root", + platform="wg-gesucht", + listing_url="https://www.wg-gesucht.de/root.html", + title="t", + rent_warm_eur=900, + rooms=2, + ) + _insert( + conn, + external_id="dupe", + platform="wg-gesucht", + listing_url="https://www.wg-gesucht.de/dupe.html", + title="t", + rent_warm_eur=900, + rooms=2, + canonical_flat_id=root_id, + ) + + fetched = [] + + def fake_fetch_wg(url): + fetched.append(url) + return "Mindestmietdauer: 24 Monate" + + profile = _profile(city="Berlin", min_contract_months=12) + summary = enrich_pending_flats( + profile, Console(), budget=10, + fetchers={"wg-gesucht": fake_fetch_wg}, + ) + assert summary["fetched"] == 1 + assert len(fetched) == 1 + assert fetched[0].endswith("root.html") + + +def test_enrich_skips_flats_already_enriched(tmp_db): + init_db() + conn = get_conn() + _insert( + conn, + external_id="done", + platform="wg-gesucht", + listing_url="https://www.wg-gesucht.de/done.html", + title="t", + rent_warm_eur=900, + rooms=2, + min_contract_months=12, + ) + + def fake_fetch_wg(url): + raise AssertionError("should not be called for already-enriched flat") + + profile = _profile(city="Berlin", min_contract_months=12) + summary = enrich_pending_flats( + profile, Console(), budget=10, + fetchers={"wg-gesucht": fake_fetch_wg}, + ) + assert summary["fetched"] == 0 + + +def test_enrich_fetch_failure_does_not_poison_flat(tmp_db): + init_db() + conn = get_conn() + flat_id = _insert( + conn, + external_id="x", + platform="wg-gesucht", + listing_url="https://www.wg-gesucht.de/x.html", + title="t", + rent_warm_eur=900, + rooms=2, + ) + + def boom(url): + raise RuntimeError("transient network failure") + + profile = _profile(city="Berlin", min_contract_months=12) + summary = enrich_pending_flats( + profile, Console(), budget=10, + fetchers={"wg-gesucht": boom}, + ) + assert summary["fetched"] == 0 + assert summary["failed"] == 1 + row = conn.execute( + "SELECT min_contract_months, available_until FROM flats WHERE id=?", + (flat_id,), + ).fetchone() + assert row["min_contract_months"] is None + assert row["available_until"] is None From a5217f23ca47b6c1abb0dbf28d417a51821fbfe3 Mon Sep 17 00:00:00 2001 From: Mukhammad Ibrokhimov Date: Wed, 20 May 2026 22:04:31 +0200 Subject: [PATCH 09/13] bd-ko1: wire enrich stage between scrape and match in run_pipeline_once --- src/flatpilot/pipeline.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/flatpilot/pipeline.py b/src/flatpilot/pipeline.py index 8fe5824..8e38580 100644 --- a/src/flatpilot/pipeline.py +++ b/src/flatpilot/pipeline.py @@ -33,6 +33,13 @@ def run_pipeline_once( console.print(f"[red]scrape failed: {exc.__class__.__name__}: {exc}[/red]") failures += 1 + console.rule("enrich") + try: + run_pipeline_enrich(profile, console, user_id=user_id) + except Exception as exc: + console.print(f"[red]enrich failed: {exc.__class__.__name__}: {exc}[/red]") + failures += 1 + console.rule("match") try: run_pipeline_match(console, user_id=user_id) @@ -86,6 +93,18 @@ def run_pipeline_scrape( run_scrape_pass(scrapers, profile, console, user_id=user_id) +def run_pipeline_enrich( + profile: Profile, + console, + *, + budget: int = 30, + user_id: int = DEFAULT_USER_ID, +) -> None: + from flatpilot.scrapers.detail import enrich_pending_flats + + enrich_pending_flats(profile, console, budget=budget) + + def run_pipeline_match(console, *, user_id: int = DEFAULT_USER_ID) -> None: from flatpilot.matcher.runner import run_match From 305a4d2da79016dcf5d96bc3502ec8d51f6c3f80 Mon Sep 17 00:00:00 2001 From: Mukhammad Ibrokhimov Date: Wed, 20 May 2026 22:04:34 +0200 Subject: [PATCH 10/13] bd-ko1: add 'flatpilot enrich' standalone command --- src/flatpilot/cli.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/flatpilot/cli.py b/src/flatpilot/cli.py index 4fe9cf3..55a5944 100644 --- a/src/flatpilot/cli.py +++ b/src/flatpilot/cli.py @@ -386,6 +386,28 @@ def dedup( console.print(f"rebuilt [bold]{total}[/bold] flats → [bold]{clusters}[/bold] clusters") +@app.command() +def enrich( + budget: int = typer.Option( + 30, "--budget", help="Max detail pages to fetch this pass." + ), +) -> None: + """Fetch + parse detail pages for flats missing min_contract_months.""" + from rich.console import Console + + from flatpilot.errors import ProfileMissingError + from flatpilot.profile import load_profile + from flatpilot.scrapers.detail import enrich_pending_flats + + console = Console() + profile = load_profile() + if profile is None: + raise ProfileMissingError( + "No profile at ~/.flatpilot/profile.json — run `flatpilot init` first." + ) + enrich_pending_flats(profile, console, budget=budget) + + @app.command() def match() -> None: """Apply the matcher to unmatched listings and write matches.""" From dd708422e6bb4dc747eff09dec69236534557618 Mon Sep 17 00:00:00 2001 From: Mukhammad Ibrokhimov Date: Wed, 20 May 2026 22:26:09 +0200 Subject: [PATCH 11/13] bd-ko1: catch ProfileMissingError in enrich CLI for consistency with match --- src/flatpilot/cli.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/flatpilot/cli.py b/src/flatpilot/cli.py index 55a5944..a5ce05a 100644 --- a/src/flatpilot/cli.py +++ b/src/flatpilot/cli.py @@ -400,12 +400,16 @@ def enrich( from flatpilot.scrapers.detail import enrich_pending_flats console = Console() - profile = load_profile() - if profile is None: - raise ProfileMissingError( - "No profile at ~/.flatpilot/profile.json — run `flatpilot init` first." - ) - enrich_pending_flats(profile, console, budget=budget) + try: + profile = load_profile() + if profile is None: + raise ProfileMissingError( + "No profile at ~/.flatpilot/profile.json — run `flatpilot init` first." + ) + enrich_pending_flats(profile, console, budget=budget) + except ProfileMissingError as exc: + console.print(f"[red]{exc}[/red]") + raise typer.Exit(1) from exc @app.command() From 2ef78e1ebee27280a7bb728dc09a4cacc3ada26c Mon Sep 17 00:00:00 2001 From: Mukhammad Ibrokhimov Date: Wed, 20 May 2026 22:27:46 +0200 Subject: [PATCH 12/13] bd-ko1: extend filter_short_term to use structured available_until --- src/flatpilot/matcher/filters.py | 19 +++++++++++++++++++ tests/test_matcher_filters.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/src/flatpilot/matcher/filters.py b/src/flatpilot/matcher/filters.py index 09d4f96..40b4c0c 100644 --- a/src/flatpilot/matcher/filters.py +++ b/src/flatpilot/matcher/filters.py @@ -158,6 +158,25 @@ def filter_contract(flat: Mapping[str, Any], profile: Profile) -> FilterResult: def filter_short_term(flat: Mapping[str, Any], profile: Profile) -> FilterResult: if not profile.exclude_short_term: return True, None + + # Structured-field path (bd-ko1 Part B): a populated available_from / + # available_until pair from the detail-page enricher is more reliable + # than the title/description heuristic. Falls back to text matching + # below when one or both fields are absent. + if profile.min_contract_months is not None: + af = flat.get("available_from") + au = flat.get("available_until") + if af and au: + try: + start = date.fromisoformat(str(af)) + end = date.fromisoformat(str(au)) + except ValueError: + start = end = None # type: ignore[assignment] + if start and end: + span_days = (end - start).days + if 0 < span_days < profile.min_contract_months * 30: + return False, "short_term_listing" + text = " ".join( str(flat.get(f) or "") for f in ("title", "description") ).lower() diff --git a/tests/test_matcher_filters.py b/tests/test_matcher_filters.py index 1b6129b..a59b321 100644 --- a/tests/test_matcher_filters.py +++ b/tests/test_matcher_filters.py @@ -391,6 +391,35 @@ def test_short_term_passes_when_title_and_description_empty(): assert ok is True and reason is None +def test_short_term_rejects_when_structured_date_range_too_short(): + profile = _profile( + exclude_short_term=True, min_contract_months=12 + ) + flat = { + "title": "Nice flat", + "description": "unbefristet", + "available_from": "2026-06-01", + "available_until": "2026-09-01", # 92-day span, well below 12 mo + } + ok, reason = filter_short_term(flat, profile) + assert ok is False + assert reason == "short_term_listing" + + +def test_short_term_passes_when_structured_date_range_is_long_enough(): + profile = _profile( + exclude_short_term=True, min_contract_months=12 + ) + flat = { + "title": "Nice flat", + "description": "unbefristet", + "available_from": "2026-06-01", + "available_until": "2028-06-01", # 2 years + } + ok, reason = filter_short_term(flat, profile) + assert ok is True and reason is None + + # --- filter_radius ------------------------------------------------------ def test_radius_passes_when_no_home_coords(): From 98a5c81af582c985b8f5a8770601bb2bb3587d8b Mon Sep 17 00:00:00 2001 From: Mukhammad Ibrokhimov Date: Wed, 20 May 2026 22:32:38 +0200 Subject: [PATCH 13/13] =?UTF-8?q?bd-ko1:=20ruff=20cleanup=20=E2=80=94=20mo?= =?UTF-8?q?ve=20import-order=20E402=20to=20satisfy=20linter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_pipeline_enrich.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_pipeline_enrich.py b/tests/test_pipeline_enrich.py index d41db86..306807b 100644 --- a/tests/test_pipeline_enrich.py +++ b/tests/test_pipeline_enrich.py @@ -4,12 +4,11 @@ from rich.console import Console import flatpilot.scrapers.detail as _detail_mod - -_detail_mod.DETAIL_FETCH_DELAY_SEC = 0.0 - from flatpilot.database import get_conn, init_db from flatpilot.scrapers.detail import enrich_pending_flats +_detail_mod.DETAIL_FETCH_DELAY_SEC = 0.0 + def _profile(**overrides): from flatpilot.profile import Profile