diff --git a/db/schema.ts b/db/schema.ts index 00357730c..8eb774cbf 100644 --- a/db/schema.ts +++ b/db/schema.ts @@ -242,6 +242,15 @@ export const propertyClusters = pgTable( councilTaxAuthorityCode: text("council_tax_authority_code"), /** Human-readable billing-authority name, for display alongside the estimate. */ councilTaxAuthorityName: text("council_tax_authority_name"), + /** + * User-supplied full address for this building, entered manually when + * the portals only exposed a street ("Brownlow Road, N11"). The user + * reads the listing photos against Google Maps to pin the exact door, + * which lets `enrich-epc` resolve the precise certificate instead of a + * postcode-level estimate. NULL = no override; the scraped address + + * reverse-geocoded postcode are used. See `setClusterAddress`. + */ + userAddress: text("user_address"), createdAt: timestamp("created_at").defaultNow().notNull(), updatedAt: timestamp("updated_at") .$onUpdate(() => new Date()) diff --git a/drizzle/0014_cuddly_lily_hollister.sql b/drizzle/0014_cuddly_lily_hollister.sql new file mode 100644 index 000000000..3c51b4e45 --- /dev/null +++ b/drizzle/0014_cuddly_lily_hollister.sql @@ -0,0 +1 @@ +ALTER TABLE "property_clusters" ADD COLUMN "user_address" text; \ No newline at end of file diff --git a/drizzle/meta/0014_snapshot.json b/drizzle/meta/0014_snapshot.json new file mode 100644 index 000000000..117e8108b --- /dev/null +++ b/drizzle/meta/0014_snapshot.json @@ -0,0 +1,2004 @@ +{ + "id": "0b609a14-5a7f-471d-94f8-f05b62442eaa", + "prevId": "fd3191e8-9fb9-4ce8-86d9-b7b59cedb3d7", + "version": "7", + "dialect": "postgresql", + "tables": { + "better_auth.account": { + "name": "account", + "schema": "better_auth", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "schemaTo": "better_auth", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.ai_runs": { + "name": "ai_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "listing_id": { + "name": "listing_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "prompt_version": { + "name": "prompt_version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "job_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "input_tokens": { + "name": "input_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "output_tokens": { + "name": "output_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "cost_usd": { + "name": "cost_usd", + "type": "numeric", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "ai_runs_listing_id_listings_id_fk": { + "name": "ai_runs_listing_id_listings_id_fk", + "tableFrom": "ai_runs", + "tableTo": "listings", + "columnsFrom": [ + "listing_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.council_tax_rates": { + "name": "council_tax_rates", + "schema": "", + "columns": { + "authority_code": { + "name": "authority_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "authority_name": { + "name": "authority_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tax_year": { + "name": "tax_year", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "band_d_pence": { + "name": "band_d_pence", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "fetched_at": { + "name": "fetched_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "council_tax_rates_tax_year_idx": { + "name": "council_tax_rates_tax_year_idx", + "columns": [ + { + "expression": "tax_year", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "council_tax_rates_authority_code_tax_year_pk": { + "name": "council_tax_rates_authority_code_tax_year_pk", + "columns": [ + "authority_code", + "tax_year" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.enrichments": { + "name": "enrichments", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "listing_id": { + "name": "listing_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "prompt_version": { + "name": "prompt_version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "features": { + "name": "features", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "epc": { + "name": "epc", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "commute_minutes": { + "name": "commute_minutes", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "broadband": { + "name": "broadband", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "crime": { + "name": "crime", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "flood": { + "name": "flood", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "amenities": { + "name": "amenities", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "ai_run_id": { + "name": "ai_run_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "enrichments_listing_prompt_version_uniq": { + "name": "enrichments_listing_prompt_version_uniq", + "columns": [ + { + "expression": "listing_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "prompt_version", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "enrichments_listing_id_listings_id_fk": { + "name": "enrichments_listing_id_listings_id_fk", + "tableFrom": "enrichments", + "tableTo": "listings", + "columnsFrom": [ + "listing_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "cascade" + }, + "enrichments_ai_run_id_ai_runs_id_fk": { + "name": "enrichments_ai_run_id_ai_runs_id_fk", + "tableFrom": "enrichments", + "tableTo": "ai_runs", + "columnsFrom": [ + "ai_run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.household_members": { + "name": "household_members", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "household_id": { + "name": "household_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "household_role", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "household_members_household_user_uniq": { + "name": "household_members_household_user_uniq", + "columns": [ + { + "expression": "household_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "household_members_user_id_idx": { + "name": "household_members_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "household_members_household_id_households_id_fk": { + "name": "household_members_household_id_households_id_fk", + "tableFrom": "household_members", + "tableTo": "households", + "columnsFrom": [ + "household_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "cascade" + }, + "household_members_user_id_user_id_fk": { + "name": "household_members_user_id_user_id_fk", + "tableFrom": "household_members", + "tableTo": "user", + "schemaTo": "better_auth", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.households": { + "name": "households", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.listing_photos": { + "name": "listing_photos", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "listing_id": { + "name": "listing_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "r2_key": { + "name": "r2_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "position": { + "name": "position", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "listing_photos_listing_id_idx": { + "name": "listing_photos_listing_id_idx", + "columns": [ + { + "expression": "listing_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "listing_photos_listing_id_listings_id_fk": { + "name": "listing_photos_listing_id_listings_id_fk", + "tableFrom": "listing_photos", + "tableTo": "listings", + "columnsFrom": [ + "listing_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.listings": { + "name": "listings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "portal": { + "name": "portal", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "portal_listing_id": { + "name": "portal_listing_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "cluster_id": { + "name": "cluster_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "search_id": { + "name": "search_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "address_raw": { + "name": "address_raw", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "postcode": { + "name": "postcode", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "bedrooms": { + "name": "bedrooms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "bathrooms": { + "name": "bathrooms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "price_monthly": { + "name": "price_monthly", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "property_type": { + "name": "property_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lat": { + "name": "lat", + "type": "numeric(9, 6)", + "primaryKey": false, + "notNull": false + }, + "lng": { + "name": "lng", + "type": "numeric(10, 6)", + "primaryKey": false, + "notNull": false + }, + "first_seen_at": { + "name": "first_seen_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "available_from": { + "name": "available_from", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "listing_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "size_sq_ft": { + "name": "size_sq_ft", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "council_tax_band": { + "name": "council_tax_band", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_at": { + "name": "published_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "pets_accepted": { + "name": "pets_accepted", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "raw_json": { + "name": "raw_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "listings_search_portal_listing_id_uniq": { + "name": "listings_search_portal_listing_id_uniq", + "columns": [ + { + "expression": "search_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "portal", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "portal_listing_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "listings_portal_listing_id_idx": { + "name": "listings_portal_listing_id_idx", + "columns": [ + { + "expression": "portal", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "portal_listing_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "listings_cluster_id_idx": { + "name": "listings_cluster_id_idx", + "columns": [ + { + "expression": "cluster_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "listings_search_id_idx": { + "name": "listings_search_id_idx", + "columns": [ + { + "expression": "search_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "listings_status_idx": { + "name": "listings_status_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "listings_cluster_id_property_clusters_id_fk": { + "name": "listings_cluster_id_property_clusters_id_fk", + "tableFrom": "listings", + "tableTo": "property_clusters", + "columnsFrom": [ + "cluster_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "cascade" + }, + "listings_search_id_searches_id_fk": { + "name": "listings_search_id_searches_id_fk", + "tableFrom": "listings", + "tableTo": "searches", + "columnsFrom": [ + "search_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.property_clusters": { + "name": "property_clusters", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "normalised_address": { + "name": "normalised_address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "postcode": { + "name": "postcode", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lat": { + "name": "lat", + "type": "numeric(9, 6)", + "primaryKey": false, + "notNull": false + }, + "lng": { + "name": "lng", + "type": "numeric(10, 6)", + "primaryKey": false, + "notNull": false + }, + "council_tax_authority_code": { + "name": "council_tax_authority_code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "council_tax_authority_name": { + "name": "council_tax_authority_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_address": { + "name": "user_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "property_clusters_normalised_address_uniq": { + "name": "property_clusters_normalised_address_uniq", + "columns": [ + { + "expression": "normalised_address", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.scrape_runs": { + "name": "scrape_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "search_id": { + "name": "search_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "portal": { + "name": "portal", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "job_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "listings_found": { + "name": "listings_found", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "new_listings": { + "name": "new_listings", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cost_usd": { + "name": "cost_usd", + "type": "numeric", + "primaryKey": false, + "notNull": false + }, + "raw_key": { + "name": "raw_key", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "scrape_runs_search_id_searches_id_fk": { + "name": "scrape_runs_search_id_searches_id_fk", + "tableFrom": "scrape_runs", + "tableTo": "searches", + "columnsFrom": [ + "search_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.searches": { + "name": "searches", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "household_id": { + "name": "household_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "portals": { + "name": "portals", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "location": { + "name": "location", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "min_bedrooms": { + "name": "min_bedrooms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "max_bedrooms": { + "name": "max_bedrooms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "min_bathrooms": { + "name": "min_bathrooms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "max_bathrooms": { + "name": "max_bathrooms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "min_price": { + "name": "min_price", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "max_price": { + "name": "max_price", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "radius_miles": { + "name": "radius_miles", + "type": "numeric(5, 2)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "property_types": { + "name": "property_types", + "type": "text[]", + "primaryKey": false, + "notNull": true + }, + "exclude_locations": { + "name": "exclude_locations", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "furnished": { + "name": "furnished", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "must_haves": { + "name": "must_haves", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'::text[]" + }, + "exclusions": { + "name": "exclusions", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'::text[]" + }, + "commute_targets": { + "name": "commute_targets", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "transport_targets": { + "name": "transport_targets", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "active": { + "name": "active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "searches_household_id_idx": { + "name": "searches_household_id_idx", + "columns": [ + { + "expression": "household_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "searches_household_id_households_id_fk": { + "name": "searches_household_id_households_id_fk", + "tableFrom": "searches", + "tableTo": "households", + "columnsFrom": [ + "household_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "better_auth.session": { + "name": "session", + "schema": "better_auth", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "schemaTo": "better_auth", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.shortlist_pipeline": { + "name": "shortlist_pipeline", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "household_id": { + "name": "household_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "cluster_id": { + "name": "cluster_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "pipeline_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "archived_reason": { + "name": "archived_reason", + "type": "pipeline_archived_reason", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_moved_at": { + "name": "last_moved_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "last_moved_by_user_id": { + "name": "last_moved_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "shortlist_pipeline_household_cluster_uniq": { + "name": "shortlist_pipeline_household_cluster_uniq", + "columns": [ + { + "expression": "household_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cluster_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "shortlist_pipeline_household_status_idx": { + "name": "shortlist_pipeline_household_status_idx", + "columns": [ + { + "expression": "household_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "shortlist_pipeline_household_id_households_id_fk": { + "name": "shortlist_pipeline_household_id_households_id_fk", + "tableFrom": "shortlist_pipeline", + "tableTo": "households", + "columnsFrom": [ + "household_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "cascade" + }, + "shortlist_pipeline_cluster_id_property_clusters_id_fk": { + "name": "shortlist_pipeline_cluster_id_property_clusters_id_fk", + "tableFrom": "shortlist_pipeline", + "tableTo": "property_clusters", + "columnsFrom": [ + "cluster_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "cascade" + }, + "shortlist_pipeline_last_moved_by_user_id_user_id_fk": { + "name": "shortlist_pipeline_last_moved_by_user_id_user_id_fk", + "tableFrom": "shortlist_pipeline", + "tableTo": "user", + "schemaTo": "better_auth", + "columnsFrom": [ + "last_moved_by_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.swipes": { + "name": "swipes", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "cluster_id": { + "name": "cluster_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "search_id": { + "name": "search_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "outcome": { + "name": "outcome", + "type": "swipe_outcome", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "swipes_user_cluster_search_uniq": { + "name": "swipes_user_cluster_search_uniq", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cluster_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "search_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "swipes_cluster_search_idx": { + "name": "swipes_cluster_search_idx", + "columns": [ + { + "expression": "cluster_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "search_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "swipes_user_id_idx": { + "name": "swipes_user_id_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "swipes_user_id_user_id_fk": { + "name": "swipes_user_id_user_id_fk", + "tableFrom": "swipes", + "tableTo": "user", + "schemaTo": "better_auth", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "cascade" + }, + "swipes_cluster_id_property_clusters_id_fk": { + "name": "swipes_cluster_id_property_clusters_id_fk", + "tableFrom": "swipes", + "tableTo": "property_clusters", + "columnsFrom": [ + "cluster_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "cascade" + }, + "swipes_search_id_searches_id_fk": { + "name": "swipes_search_id_searches_id_fk", + "tableFrom": "swipes", + "tableTo": "searches", + "columnsFrom": [ + "search_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "better_auth.user": { + "name": "user", + "schema": "better_auth", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_state": { + "name": "user_state", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "last_seen_matches": { + "name": "last_seen_matches", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "user_state_user_id_user_id_fk": { + "name": "user_state_user_id_user_id_fk", + "tableFrom": "user_state", + "tableTo": "user", + "schemaTo": "better_auth", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "cascade" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "better_auth.verification": { + "name": "verification", + "schema": "better_auth", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.household_role": { + "name": "household_role", + "schema": "public", + "values": [ + "owner", + "member" + ] + }, + "public.job_status": { + "name": "job_status", + "schema": "public", + "values": [ + "running", + "success", + "failure" + ] + }, + "public.listing_status": { + "name": "listing_status", + "schema": "public", + "values": [ + "active", + "let_agreed", + "removed" + ] + }, + "public.pipeline_archived_reason": { + "name": "pipeline_archived_reason", + "schema": "public", + "values": [ + "accepted", + "passed", + "let_to_someone_else", + "withdrawn", + "other" + ] + }, + "public.pipeline_status": { + "name": "pipeline_status", + "schema": "public", + "values": [ + "shortlisted", + "contacted", + "viewing_booked", + "offer_made", + "archived" + ] + }, + "public.swipe_outcome": { + "name": "swipe_outcome", + "schema": "public", + "values": [ + "keep", + "skip", + "shortlist" + ] + } + }, + "schemas": { + "better_auth": "better_auth" + }, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": { + "public.v_mutual_matches": { + "columns": { + "cluster_id": { + "name": "cluster_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "search_id": { + "name": "search_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "household_id": { + "name": "household_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "matched_at": { + "name": "matched_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "definition": "\n SELECT a.cluster_id, a.search_id, a.household_id, a.matched_at\n FROM (\n SELECT\n sw.cluster_id,\n sw.search_id,\n s.household_id,\n COUNT(DISTINCT sw.user_id) AS agree_count,\n MAX(sw.created_at) AS matched_at\n FROM swipes sw\n JOIN searches s ON s.id = sw.search_id\n WHERE sw.outcome IN ('keep','shortlist')\n GROUP BY sw.cluster_id, sw.search_id, s.household_id\n ) a\n JOIN (\n SELECT household_id, COUNT(*) AS member_count\n FROM household_members\n GROUP BY household_id\n ) m ON m.household_id = a.household_id\n WHERE a.agree_count = m.member_count\n", + "name": "v_mutual_matches", + "schema": "public", + "isExisting": false, + "materialized": false + } + }, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 429181b26..7adcffed7 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -99,6 +99,13 @@ "when": 1779669045483, "tag": "0013_lowly_miss_america", "breakpoints": true + }, + { + "idx": 14, + "version": "7", + "when": 1779904400857, + "tag": "0014_cuddly_lily_hollister", + "breakpoints": true } ] } \ No newline at end of file diff --git a/scripts/backfill-listing-coords.ts b/scripts/backfill-listing-coords.ts new file mode 100644 index 000000000..8bb2463ed --- /dev/null +++ b/scripts/backfill-listing-coords.ts @@ -0,0 +1,179 @@ +#!/usr/bin/env bun +/** + * Listing coordinate backfill. + * + * Zoopla/OpenRent listings carry true per-property lat/lng — but only on + * the detail page, which `scrape-detail` parses into the rich `rawJson` + * blob (top-level `rawJson.lat`/`rawJson.lng`). Column promotion of those + * coords into `listings.lat`/`lng` only shipped in 5c7fd543 ("write + * detail columns"); listings first clustered under an earlier Worker kept + * their rawJson coords but landed a NULL column, and `scrape-detail` + * never re-runs for an already-clustered listing, so they can't + * self-heal. The NULL columns starve every per-cluster geo enrichment + * (`enrich-crime`/`enrich-flood`/`enrich-amenities` no-op without lat/lng, + * and `enrich-council-tax` loses its precise arm). See [[enrichment-coords-bug]]. + * + * This promotes the stranded coords — `rawJson` → `listings.lat/lng` → + * `property_clusters.lat/lng` (a cluster is one building, so any located + * listing in it locates the cluster) — then re-fires the four + * lat/lng-gated enrichments for the newly-located clusters. Pure SQL + + * Trigger fan-out; no external geocoding (the coords are already ours, + * and the scraped postcode is only an outcode anyway). + * + * Usage: + * doppler run --project gaff --config prd --scope ~/.t-stack/orgs/timothygithinji \ + * -- bun scripts/backfill-listing-coords.ts [--limit N] [--dry-run] + * + * Flags: + * --dry-run Report how many listings/clusters would be touched; write + * nothing, trigger nothing. + * --limit N Cap the enrichment re-fan-out (the SQL promotion always + * runs in full — it's a single statement). + */ + +import { tasks } from "@trigger.dev/sdk"; +import { and, eq, isNull, sql } from "drizzle-orm"; +import { getDb } from "../db"; +import * as schema from "../db/schema"; + +// The lat/lng-gated enrichments to re-fire once a cluster is located. +// EPC + broadband key off the postcode directly and aren't gated on +// coordinates, so they're left alone here. +const GEO_TASK_IDS = [ + "enrich-crime", + "enrich-flood", + "enrich-amenities", + "enrich-council-tax", +] as const; + +const TRIGGER_BATCH_SIZE = 50; + +// A lenient numeric guard for the rawJson text value — enough to keep the +// `::numeric` cast from throwing on a stray non-number. `.` is literal +// inside the character class, so no backslash escaping to get wrong. +const latText = sql`(${schema.listings.rawJson} ->> 'lat')`; +const lngText = sql`(${schema.listings.rawJson} ->> 'lng')`; +const hasNumericCoords = and( + sql`${latText} ~ '^-?[0-9.]+$'`, + sql`${lngText} ~ '^-?[0-9.]+$'` +); + +type Args = { dryRun: boolean; limit: number | undefined }; + +function parseArgs(argv: string[]): Args { + const out: Args = { dryRun: false, limit: undefined }; + // biome-ignore lint/style/useForOf: index-based parser consuming `--flag value` pairs via lookahead (argv[++i]). + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === "--dry-run") { + out.dryRun = true; + } else if (a === "--limit") { + const next = argv[++i]; + if (!next) { + throw new Error("--limit needs a value"); + } + out.limit = Number.parseInt(next, 10); + } + } + return out; +} + +async function main(): Promise { + const args = parseArgs(process.argv.slice(2)); + if (!(args.dryRun || process.env.TRIGGER_SECRET_KEY)) { + throw new Error( + "TRIGGER_SECRET_KEY not set — run via `doppler run ... -- bun scripts/backfill-listing-coords.ts` (or pass --dry-run)" + ); + } + + const db = getDb(); + + // How many coordinate-less listings can we recover from rawJson? + const [listingScan] = await db + .select({ n: sql`count(*)::int` }) + .from(schema.listings) + .where(and(isNull(schema.listings.lat), hasNumericCoords)); + const recoverableListings = listingScan?.n ?? 0; + + // Which coordinate-less clusters have at least one such listing? These + // are the ones we'll locate and re-enrich. + const targetClusters = await db + .selectDistinct({ id: schema.propertyClusters.id }) + .from(schema.propertyClusters) + .innerJoin( + schema.listings, + eq(schema.listings.clusterId, schema.propertyClusters.id) + ) + .where(and(isNull(schema.propertyClusters.lat), hasNumericCoords)); + const targetClusterIds = targetClusters.map((c) => c.id); + + console.log( + `Recoverable: ${recoverableListings} listing(s) with stranded rawJson coords; ${targetClusterIds.length} coordinate-less cluster(s) can be located.` + ); + + if (args.dryRun) { + console.log("\n--dry-run: writing nothing, triggering nothing."); + return; + } + if (recoverableListings === 0) { + console.log("Nothing to promote."); + return; + } + + // 1. Promote rawJson coords → listings columns (single statement). + await db.execute(sql` + UPDATE ${schema.listings} + SET lat = (${schema.listings.rawJson} ->> 'lat')::numeric, + lng = (${schema.listings.rawJson} ->> 'lng')::numeric + WHERE ${schema.listings.lat} IS NULL + AND (${schema.listings.rawJson} ->> 'lat') ~ '^-?[0-9.]+$' + AND (${schema.listings.rawJson} ->> 'lng') ~ '^-?[0-9.]+$' + `); + console.log(`Promoted coords onto ${recoverableListings} listing(s).`); + + // 2. Propagate to coordinate-less clusters — a cluster is one building, + // so any located listing in it locates the cluster. Cheapest listing + // wins the tiebreak, deterministically. + await db.execute(sql` + UPDATE ${schema.propertyClusters} AS c + SET lat = src.lat, lng = src.lng + FROM ( + SELECT DISTINCT ON (cluster_id) cluster_id, lat, lng + FROM ${schema.listings} + WHERE cluster_id IS NOT NULL AND lat IS NOT NULL + ORDER BY cluster_id, price_monthly ASC NULLS LAST + ) AS src + WHERE c.id = src.cluster_id AND c.lat IS NULL + `); + console.log(`Located ${targetClusterIds.length} cluster(s).`); + + // 3. Re-fire the geo enrichments for the now-located clusters. + const ids = args.limit + ? targetClusterIds.slice(0, args.limit) + : targetClusterIds; + if (ids.length === 0) { + console.log("No clusters to re-enrich."); + return; + } + for (const taskId of GEO_TASK_IDS) { + let triggered = 0; + for (let i = 0; i < ids.length; i += TRIGGER_BATCH_SIZE) { + const slice = ids.slice(i, i + TRIGGER_BATCH_SIZE); + await tasks.batchTrigger( + taskId, + slice.map((id) => ({ payload: { clusterId: id } })) + ); + triggered += slice.length; + } + console.log(` re-triggered ${taskId}: ${triggered} runs`); + } + + console.log( + "\nDone. Watch the Trigger.dev dashboard; each run stamps its cluster's crime/flood/amenities/council-tax." + ); +} + +main().catch((err: unknown) => { + console.error(err instanceof Error ? err.message : err); + process.exit(1); +}); diff --git a/src/components/listing-detail/desktop-listing-detail.tsx b/src/components/listing-detail/desktop-listing-detail.tsx index 927c9da71..12b64e0fc 100644 --- a/src/components/listing-detail/desktop-listing-detail.tsx +++ b/src/components/listing-detail/desktop-listing-detail.tsx @@ -63,6 +63,8 @@ type Props = { disabled?: boolean; from?: ListingFromOrigin; onShortlist: () => void; + /** Open the manual address-override dialog (owned by the route). */ + onEditAddress?: () => void; pendingAction?: ListingDetailPendingAction; }; @@ -71,6 +73,7 @@ export function DesktopListingDetail({ disabled, from, onShortlist, + onEditAddress, pendingAction = null, }: Props) { return ( @@ -88,7 +91,7 @@ export function DesktopListingDetail({ />
- +
); @@ -687,10 +690,16 @@ function LocationCard({ /* ---------------- Info column ---------------- */ -function InfoColumn({ data }: { data: ListingDetailPayload }) { +function InfoColumn({ + data, + onEditAddress, +}: { + data: ListingDetailPayload; + onEditAddress?: () => void; +}) { return (
- + void; +}) { const { headline, portalSpread, cluster } = data; const title = shortAddressTitle(headline.addressRaw); const subtitle = subtitleFor(headline.postcode, cluster.postcode); @@ -726,6 +741,15 @@ function PriceCard({ data }: { data: ListingDetailPayload }) { {subtitle ? (

{subtitle}

) : null} + {onEditAddress ? ( + + ) : null}
{portalSpread.map((row, i) => ( @@ -1054,6 +1078,18 @@ function epcRecordRow(epc: ListingDetailPayload["epc"]): RecordRow | null { if (!epc) { return null; } + // Estimate: the listing had no house number to pin an exact certificate, + // so this is the typical rating for the full postcode. Mark it (~) and + // show the spread so it doesn't read as fake-precise. + if (epc.source === "estimate") { + const spread = epc.range ? ` · range ${epc.range.min}–${epc.range.max}` : ""; + const homes = epc.sampleSize ? ` · ${epc.sampleSize} homes` : ""; + return { + label: "EPC · area estimate", + value: `~${epc.rating}`, + meta: `Typical for this postcode${spread}${homes}`, + }; + } return { label: "EPC rating", value: epc.rating, diff --git a/src/components/review/info-row.tsx b/src/components/review/info-row.tsx index 1b0a0f309..7916e2650 100644 --- a/src/components/review/info-row.tsx +++ b/src/components/review/info-row.tsx @@ -25,6 +25,8 @@ type Props = { walkMinutes: number | null; stationName?: string | null; epcRating: string | null; + /** Postcode-level estimate (no exact certificate match) — shown as "~C est". */ + epcIsEstimate?: boolean; broadbandMbps: number | null; }; @@ -33,6 +35,7 @@ export function InfoRow({ walkMinutes, stationName, epcRating, + epcIsEstimate, broadbandMbps, }: Props) { return ( @@ -49,7 +52,12 @@ export function InfoRow({ value={walkMinutes === null ? "—" : `${walkMinutes}`} unit={walkMinutes === null ? "" : "min walk"} /> - + { + if (!(Number.isFinite(lat) && Number.isFinite(lng))) { + return null; + } + const fetchImpl = options.fetch ?? fetch; + const url = `${POSTCODES_BASE_URL}/postcodes?lon=${lng}&lat=${lat}&limit=1`; + const res = await fetchImpl(url, { headers: { Accept: "application/json" } }); + if (!res.ok) { + return null; + } + const body = (await res.json()) as { + result?: Array<{ postcode?: unknown }> | null; + }; + const nearest = Array.isArray(body.result) ? body.result[0] : null; + return typeof nearest?.postcode === "string" ? nearest.postcode : null; +} diff --git a/src/routes/index.tsx b/src/routes/index.tsx index dbec84364..b1305363e 100644 --- a/src/routes/index.tsx +++ b/src/routes/index.tsx @@ -860,6 +860,7 @@ function buildHeroData(card: ReviewCard): DesktopReviewData["hero"] { portalsAlsoOn, features, epcRating, + epcIsEstimate, commuteMinutes, } = card; const photos = @@ -882,7 +883,7 @@ function buildHeroData(card: ReviewCard): DesktopReviewData["hero"] { bedrooms: headlineListing.bedrooms, bathrooms: headlineListing.bathrooms, sizeSqFt: headlineListing.sizeSqFt, - epc: epcRating ?? null, + epc: epcRating ? (epcIsEstimate ? `~${epcRating}` : epcRating) : null, commuteMinutes, }), verdicts: buildVerdicts(features), diff --git a/src/routes/listings/$clusterId.tsx b/src/routes/listings/$clusterId.tsx index e9ff51dc0..e7ff4a871 100644 --- a/src/routes/listings/$clusterId.tsx +++ b/src/routes/listings/$clusterId.tsx @@ -45,6 +45,16 @@ import { PortalCrossList } from "../../components/listing-detail/portal-cross-li import { PublicRecords } from "../../components/listing-detail/public-records"; import { SmallPrint } from "../../components/listing-detail/small-print"; import { WhereItSits } from "../../components/listing-detail/where-it-sits"; +import { Button } from "../../components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "../../components/ui/dialog"; +import { Input } from "../../components/ui/input"; import { Skeleton } from "../../components/ui/skeleton"; import { requireSession } from "../../lib/auth-guard"; import { useHousehold } from "../../lib/household-context"; @@ -56,6 +66,7 @@ import { queryKeys } from "../../lib/query-keys"; import { type ListingDetailPayload, getListingDetail, + setClusterAddress, } from "../../server/functions/listing-detail"; import { recordSwipe } from "../../server/functions/review"; @@ -201,6 +212,23 @@ function ListingDetailPage() { }, }); + // Manual address override — lets the user pin the exact door (read off + // the photos against Google Maps) so EPC resolves the precise cert + // instead of a postcode estimate. The server re-fires enrich-epc. + const [addrOpen, setAddrOpen] = useState(false); + const [addrValue, setAddrValue] = useState(""); + const setAddress = useMutation({ + mutationFn: (address: string) => + setClusterAddress({ data: { clusterId, address } }), + onSuccess: () => { + setAddrOpen(false); + // enrich-epc re-fires server-side; its result lands async, so a + // plain invalidate now picks up the refreshed EPC on the next read. + qc.invalidateQueries({ queryKey: queryOpts.queryKey }); + }, + onError: (e: Error) => setError(e.message ?? "Couldn't save address"), + }); + if (!data) { return ; } @@ -223,6 +251,11 @@ function ListingDetailPage() { googleMapsApiKey, } = data; + const openAddressDialog = () => { + setAddrValue(cluster.userAddress ?? headline.addressRaw); + setAddrOpen(true); + }; + const portalsTrackingLabel = `${portalSpread.length} portal${portalSpread.length === 1 ? "" : "s"} tracking`; const title = shortAddressTitle(headline.addressRaw); const locality = localityFromPostcode(headline.postcode ?? cluster.postcode); @@ -247,6 +280,7 @@ function ListingDetailPage() { data={data} disabled={swipe.isPending} from={from} + onEditAddress={openAddressDialog} onShortlist={() => swipe.mutate({ outcome: "shortlist" })} pendingAction={pendingAction} /> @@ -344,6 +378,15 @@ function ListingDetailPage() { {locality ? (

{locality}

) : null} +
@@ -385,6 +428,53 @@ function ListingDetailPage() { pendingAction={pendingAction} /> + + + + + Pin the exact address + + Portals usually hide the door number. Read it off the photos + against Google Maps and enter the full address (with postcode) so + we can pull this building's exact EPC instead of a postcode-level + estimate. + + +
{ + e.preventDefault(); + if (addrValue.trim().length > 0) { + setAddress.mutate(addrValue.trim()); + } + }} + > + setAddrValue(e.target.value)} + placeholder="e.g. Flat 2, 14 Brownlow Road, London N11 2BP" + value={addrValue} + /> + + {cluster.userAddress ? ( + + ) : null} + + +
+ + ); } diff --git a/src/server/functions/listing-detail.ts b/src/server/functions/listing-detail.ts index 55fd0d7f5..b0228cb7b 100644 --- a/src/server/functions/listing-detail.ts +++ b/src/server/functions/listing-detail.ts @@ -25,6 +25,7 @@ * to a search owned by the caller's household. Anything else 404s. */ import { createServerFn } from "@tanstack/react-start"; +import { tasks } from "@trigger.dev/sdk"; import { and, desc, eq, inArray } from "drizzle-orm"; import { z } from "zod"; import { getDb } from "../../../db"; @@ -34,6 +35,7 @@ import { householdMembers, listingPhotos, listings, + propertyClusters, searches, swipes, user, @@ -64,6 +66,8 @@ export type ListingDetailCluster = { postcode: string | null; lat: string | null; lng: string | null; + /** Manual full-address override (see `setClusterAddress`), or null. */ + userAddress: string | null; }; export type ListingDetailHeadline = { @@ -99,6 +103,17 @@ export type ListingDetailEpc = { rating: string; potential?: string; expiresOn?: string; + /** + * How the rating was derived. "exact" = matched this building's + * certificate; "estimate" = the typical rating for the property's full + * postcode (the listing didn't expose a house number to match on). The + * UI labels the latter as an approximation. + */ + source?: "exact" | "estimate"; + /** Estimate only: the spread of ratings across the postcode, e.g. C…E. */ + range?: { min: string; max: string }; + /** Estimate only: how many certificates the typical rating drew from. */ + sampleSize?: number; }; /** @@ -286,15 +301,28 @@ function asEpc(value: unknown): ListingDetailEpc | undefined { currentRating?: unknown; potentialRating?: unknown; expiresOn?: unknown; + source?: unknown; + range?: unknown; + sampleSize?: unknown; }; if (typeof obj.currentRating !== "string") { return; } + const range = + obj.range && + typeof obj.range === "object" && + typeof (obj.range as { min?: unknown }).min === "string" && + typeof (obj.range as { max?: unknown }).max === "string" + ? (obj.range as { min: string; max: string }) + : undefined; return { rating: obj.currentRating, potential: typeof obj.potentialRating === "string" ? obj.potentialRating : undefined, expiresOn: typeof obj.expiresOn === "string" ? obj.expiresOn : undefined, + source: obj.source === "estimate" || obj.source === "exact" ? obj.source : undefined, + ...(range ? { range } : {}), + ...(typeof obj.sampleSize === "number" ? { sampleSize: obj.sampleSize } : {}), }; } @@ -685,6 +713,7 @@ export const getListingDetail = createServerFn({ method: "GET" }) postcode: cluster.postcode, lat: cluster.lat, lng: cluster.lng, + userAddress: cluster.userAddress ?? null, }, headline: { listingId: headlineListing.id, @@ -715,3 +744,55 @@ export const getListingDetail = createServerFn({ method: "GET" }) googleMapsApiKey: parsedEnv().GOOGLE_MAPS_API_KEY, }; }); + +const setClusterAddressSchema = z.object({ + clusterId: z.string().trim().min(1), + // The full address the user pinned from the photos + Google Maps. An + // empty string clears the override (back to the scraped address). + address: z.string().trim().max(300), +}); + +/** + * Manually set a building's full address so EPC can resolve the exact + * certificate instead of a postcode-level estimate (see `enrich-epc`). + * The portals usually withhold the door number; the user reads it off the + * listing photos against Google Maps and pins it here. Writes the override + * onto the cluster and re-fires `enrich-epc` to re-resolve with it. + * + * Authz: the cluster must have a listing in one of the caller's + * household's searches — you can only correct a building you can see. + */ +export const setClusterAddress = createServerFn({ method: "POST" }) + .inputValidator(setClusterAddressSchema) + .handler(async ({ data }): Promise<{ ok: true }> => { + const { householdId } = await requireHouseholdScope(); + const db = getDb(); + + const owned = await db + .select({ id: listings.id }) + .from(listings) + .innerJoin(searches, eq(listings.searchId, searches.id)) + .where( + and( + eq(listings.clusterId, data.clusterId), + eq(searches.householdId, householdId) + ) + ) + .limit(1); + if (owned.length === 0) { + throw new Error("cluster_not_found"); + } + + const userAddress = data.address.length > 0 ? data.address : null; + await db + .update(propertyClusters) + .set({ userAddress }) + .where(eq(propertyClusters.id, data.clusterId)); + + // Re-resolve EPC with the corrected address. Fire-and-forget — the + // listing-detail query is invalidated client-side and will pick up the + // refreshed enrichment on its next read. + await tasks.trigger("enrich-epc", { clusterId: data.clusterId }); + + return { ok: true }; + }); diff --git a/src/server/functions/review.ts b/src/server/functions/review.ts index 7379d9906..6810bf586 100644 --- a/src/server/functions/review.ts +++ b/src/server/functions/review.ts @@ -141,6 +141,12 @@ export type ReviewCard = { portalsAlsoOn: ReviewCardAlsoOn[]; features?: Features; epcRating?: string; + /** + * True when `epcRating` is a postcode-level estimate rather than this + * building's own certificate (the listing exposed no house number to + * match on). The card marks it with a "~" so it doesn't read as exact. + */ + epcIsEstimate?: boolean; /** Soonest commute target, in minutes, when enriched. */ commuteMinutes: number | null; /** Closest scraped station (with derived walk minutes). */ @@ -328,6 +334,15 @@ function asEpcRating(value: unknown): string | undefined { return; } +/** True when the stored EPC blob is a postcode-level estimate. */ +function isEpcEstimate(value: unknown): boolean { + return ( + !!value && + typeof value === "object" && + (value as { source?: unknown }).source === "estimate" + ); +} + /** * SQL predicate (requires `searches` joined on `listings.searchId`): the * listing's monthly price sits within its own search's band. Bounds are @@ -628,6 +643,7 @@ export const getNextReviewCard = createServerFn({ method: "GET" }) portalsAlsoOn, features: asFeatures(enrichment?.features), epcRating: asEpcRating(enrichment?.epc), + epcIsEstimate: isEpcEstimate(enrichment?.epc), commuteMinutes: pickSoonestCommute(enrichment?.commuteMinutes), nearestStation: pickNearestStation(headline.rawJson), broadband: asBroadband(enrichment?.broadband), diff --git a/src/trigger/enrich-epc.test.ts b/src/trigger/enrich-epc.test.ts index 28ffdd503..5b982b94f 100644 --- a/src/trigger/enrich-epc.test.ts +++ b/src/trigger/enrich-epc.test.ts @@ -1,16 +1,22 @@ /** - * Unit tests for the EPC enrichment mapping. + * Unit tests for EPC certificate resolution. * - * Two real bugs lived here: the EPC search endpoint returns - * `{ "column-names": [...], "rows": [...] }` rather than the bare array the - * generated spec claims, and the rows use kebab-case keys - * (`current-energy-rating`) while the app reads camelCase (`currentRating`). - * These golden-value checks pin both the row extraction and the - * normalisation so EPC can't silently stop populating again. + * Real bugs pinned here: the search endpoint returns + * `{ "column-names": [...], "rows": [...] }` (not the bare array the spec + * claims), rows use kebab-case keys, EPC rows carry NO coordinates (so we + * can't pick "the nearest"), and ~74% of listings are street-only (so we + * must NOT assert an exact match without a house number). These checks lock + * the row extraction, the conservative exact match, and the postcode-level + * estimate fallback. */ import { describe, expect, it } from "vitest"; -import { extractCertRows, normaliseEpcCert } from "./enrich-epc"; +import { + estimateBlob, + exactBlob, + extractCertRows, + pickExactCert, +} from "./enrich-epc"; describe("extractCertRows", () => { it("pulls rows out of the wrapped search response", () => { @@ -34,41 +40,100 @@ describe("extractCertRows", () => { }); }); -describe("normaliseEpcCert", () => { - it("maps kebab-case fields and derives a 10-year expiry", () => { +describe("pickExactCert", () => { + // The actual N11 3PR certificate set shape. + const certs = [ + { address: "21 Hampton Close", "current-energy-rating": "C" }, + { address: "13 Cannon Hill", "current-energy-rating": "D" }, + { address: "Flat 7, Bethell Lodge, 31 Springfield Road", "current-energy-rating": "D" }, + ]; + + it("matches on shared house number + street word", () => { + const hit = pickExactCert(certs, "13 Cannon Hill, N14"); + expect(hit?.address).toBe("13 Cannon Hill"); + }); + + it("returns null for a street-only address (no number to anchor on)", () => { + expect(pickExactCert(certs, "Hampton Close, New Southgate, N11")).toBeNull(); + }); + + it("does not match on a shared number alone when no street word overlaps", () => { + // "13" appears, but "elsewhere" shares no street word with any cert. + expect(pickExactCert(certs, "13 Elsewhere Gardens")).toBeNull(); + }); + + it("does not treat a bedroom count as a house number", () => { + // "2 Bedroom Flat … Brunswick Park Road" must NOT exact-match the cert + // for "2 Brunswick Park Road" — the 2 is a bed count, not a door number. + const bedroomCerts = [ + { address: "2 Brunswick Park Road, New Southgate", "current-energy-rating": "D" }, + ]; + expect( + pickExactCert(bedroomCerts, "2 bedroom flat brunswick park road n11") + ).toBeNull(); + }); + + it("returns null when there are no certs", () => { + expect(pickExactCert([], "13 Cannon Hill")).toBeNull(); + }); +}); + +describe("exactBlob", () => { + it("maps kebab-case fields, derives a 10-year expiry, and tags source", () => { expect( - normaliseEpcCert({ - "current-energy-rating": "D", - "potential-energy-rating": "C", - "lodgement-date": "2019-04-24", - }) + exactBlob( + { + "current-energy-rating": "d", + "potential-energy-rating": "C", + "lodgement-date": "2019-04-24", + address: "13 Cannon Hill", + }, + "N14 6LP", + "13 Cannon Hill" + ) ).toEqual({ currentRating: "D", potentialRating: "C", expiresOn: "2029-04-24", + source: "exact", + matchedAddress: "13 Cannon Hill", + postcode: "N14 6LP", }); }); - it("omits optional fields when absent", () => { - expect(normaliseEpcCert({ "current-energy-rating": "E" })).toEqual({ - currentRating: "E", - }); + it("returns null when the cert has no usable rating", () => { + expect(exactBlob({ address: "x" }, "N14 6LP", "x")).toBeNull(); }); +}); - it("trims whitespace and treats blank ratings as no certificate", () => { - expect(normaliseEpcCert({ "current-energy-rating": " B " })).toEqual({ - currentRating: "B", +describe("estimateBlob", () => { + it("summarises a postcode into a modal rating + best..worst range", () => { + const certs = [ + { "current-energy-rating": "C" }, + { "current-energy-rating": "C" }, + { "current-energy-rating": "D" }, + { "current-energy-rating": "E" }, + ]; + expect(estimateBlob(certs, "N11 3PR")).toEqual({ + currentRating: "C", + source: "estimate", + postcode: "N11 3PR", + sampleSize: 4, + range: { min: "C", max: "E" }, }); - expect(normaliseEpcCert({ "current-energy-rating": " " })).toBeNull(); - expect(normaliseEpcCert({})).toBeNull(); }); - it("ignores an unparseable lodgement date", () => { - expect( - normaliseEpcCert({ - "current-energy-rating": "C", - "lodgement-date": "not-a-date", - }) - ).toEqual({ currentRating: "C" }); + it("breaks a modal tie toward the better band", () => { + const certs = [ + { "current-energy-rating": "C" }, + { "current-energy-rating": "E" }, + ]; + const blob = estimateBlob(certs, "N11 3PR"); + expect(blob?.currentRating).toBe("C"); + expect(blob?.range).toEqual({ min: "C", max: "E" }); + }); + + it("ignores unparseable ratings and returns null when none are usable", () => { + expect(estimateBlob([{ "current-energy-rating": "?" }, {}], "N11 3PR")).toBeNull(); }); }); diff --git a/src/trigger/enrich-epc.ts b/src/trigger/enrich-epc.ts index 9da922d74..a74f6ec58 100644 --- a/src/trigger/enrich-epc.ts +++ b/src/trigger/enrich-epc.ts @@ -2,31 +2,33 @@ * Per-cluster EPC (Energy Performance Certificate) enrichment. * * Fires fire-and-forget from `clusterTask.onSuccess` for every newly - * created cluster. EPC data is keyed on UK postcode, so one query per - * cluster — not per listing — keeps API usage proportional to the - * unique buildings we see rather than the cross-portal duplicates. + * created cluster (and is re-triggered by backfills). EPC is keyed on a + * UK address, so one resolution per cluster — not per listing. * - * Flow per cluster: + * The hard problem: the EPC API needs an *exact* address, but the portals + * mostly give a street + outcode ("Friern Barnet Road, N11"), no house + * number, and EPC certificate rows carry NO coordinates — so we can't + * just take "the nearest cert". We do the best honest thing the data + * allows, in this order: * - * 1. Load `property_clusters` row. If `postcode` is null, skip — the - * EPC search endpoint requires at least a postcode. - * 2. Call the typed EPC client's `/domestic/search?postcode=...` - * endpoint. If zero results, skip silently (the postcode is in a - * building the EPC dataset hasn't got a record for, e.g. very new - * build or commercial unit). - * 3. If multiple results, pick the one whose lat/lng is closest to - * the cluster's lat/lng. Falls back to the first row when either - * side has no coordinates. - * 4. For every listing in the cluster, UPSERT an `enrichments` row: - * - row exists at `(listing_id, current promptVersion)` → UPDATE - * the `epc` column only, leaving `features` untouched. - * - row doesn't exist → INSERT with `features={}` (a valid empty - * object) and the EPC blob populated. The AI enrichment task - * will fill in `features` later via ON CONFLICT DO NOTHING. + * 1. Resolve a FULL postcode. The scraped `postcode` is usually only an + * outcode, so we snap the listing's true lat/lng (promoted from + * rawJson) to its nearest unit postcode via postcodes.io. This + * narrows the EPC search from a whole district to one street/block. + * 2. EXACT match — when the listing address carries a house/flat number, + * match the cert whose address shares that number + street. Stored + * with `source: "exact"`. + * 3. ESTIMATE fallback — when the address is street-only (the common + * case), summarise the full postcode's certificates into a typical + * rating + range. Stored with `source: "estimate"` so the UI can + * label it as an approximation rather than fake-precise. + * 4. Otherwise blank — better than the old behaviour, which stamped one + * arbitrary district cert onto every cluster. * - * No AI cost — this path doesn't count against the daily budget. The - * EPC API is free (with attribution); the only consumption is a single - * external request per new cluster. + * A future manual address override (user pins the door from the photos) + * feeds this same engine by supplying a precise address/postcode. + * + * No AI cost. The EPC API and postcodes.io are both free (with attribution). */ import { logger, task } from "@trigger.dev/sdk"; @@ -37,6 +39,7 @@ import { getDomesticSearch, } from "../lib/api-clients/epc"; import { env } from "../lib/env"; +import { reverseGeocodePostcode } from "../lib/geocode"; import { parseNumeric, upsertEnrichmentForCluster } from "./enrich-helpers"; import { scrapeQueue } from "./queues"; @@ -46,60 +49,40 @@ export type EnrichEpcPayload = { export type EnrichEpcOutput = { clusterId: string; - certificateFound: boolean; + /** "exact" | "estimate" | "none" — which arm produced the result. */ + source: "exact" | "estimate" | "none"; listingsTouched: number; }; -/** - * Pick the certificate whose lat/lng is closest to the cluster's - * lat/lng. Uses a flat Euclidean distance on raw degrees — accurate - * enough at UK latitudes to pick between certificates within a single - * postcode. Falls back to the first certificate if either side lacks - * coordinates. - * - * The EPC dataset's lat/lng fields aren't part of the strongly-typed - * shape (`Certificate` is `{ [key: string]: unknown }` in the - * generated SDK), so we sniff the well-known field names defensively. - */ -function pickClosestCertificate( - certs: Certificate[], - clusterLat: number | null, - clusterLng: number | null -): Certificate | null { - if (certs.length === 0) { - return null; - } - if (certs.length === 1) { - return certs[0] ?? null; - } - if (clusterLat == null || clusterLng == null) { - return certs[0] ?? null; - } +// A complete UK postcode (with the inward part). Mirrors the guard used in +// `council-tax.ts` / geocoding: an outcode can't pin a building. +const FULL_POSTCODE_RE = /^[A-Z]{1,2}\d[A-Z\d]?\d[A-Z]{2}$/; - let best: Certificate | null = null; - let bestDist = Number.POSITIVE_INFINITY; - for (const cert of certs) { - const lat = Number(cert.latitude ?? cert.lat); - const lng = Number(cert.longitude ?? cert.lng); - if (!Number.isFinite(lat) || !Number.isFinite(lng)) { - continue; - } - const dLat = lat - clusterLat; - const dLng = lng - clusterLng; - const dist = dLat * dLat + dLng * dLng; - if (dist < bestDist) { - bestDist = dist; - best = cert; - } - } - return best ?? certs[0] ?? null; -} +// Domestic EPC bands, best → worst. Used to order an estimate's range. +const RATING_ORDER = ["A", "B", "C", "D", "E", "F", "G"] as const; + +/** Stored EPC blob — `currentRating` stays populated for back-compat with + * the review/listing-detail readers; the rest is additive. */ +export type NormalisedEpc = { + currentRating: string; + potentialRating?: string; + expiresOn?: string; + /** How we arrived at the rating — the UI labels "estimate" distinctly. */ + source: "exact" | "estimate"; + /** Exact: the EPC certificate address we matched against. */ + matchedAddress?: string; + /** The full postcode the search/estimate was scoped to. */ + postcode?: string; + /** Estimate: how many certificates the typical rating was drawn from. */ + sampleSize?: number; + /** Estimate: the spread of ratings across the postcode, e.g. C…E. */ + range?: { min: string; max: string }; +}; /** * The search endpoint returns `{ "column-names": [...], "rows": [...] }`, NOT - * the bare array the generated spec claims (the OpenAPI doc models the 200 - * body as `Certificates`, but the live API wraps the rows). Pull the rows out - * defensively, tolerating either shape. + * the bare array the generated spec claims. Pull the rows out defensively, + * tolerating either shape. */ export function extractCertRows(data: unknown): Certificate[] { if (Array.isArray(data)) { @@ -109,52 +92,265 @@ export function extractCertRows(data: unknown): Certificate[] { return Array.isArray(rows) ? (rows as Certificate[]) : []; } -/** Stored EPC blob shape — what `asEpcRating` (review) and `asEpc` - * (listing-detail) read back. */ -type NormalisedEpc = { - currentRating: string; - potentialRating?: string; - expiresOn?: string; -}; +function epcString(value: unknown): string | undefined { + if (typeof value !== "string") { + return; + } + const trimmed = value.trim(); + return trimmed.length > 0 ? trimmed : undefined; +} + +/** EPCs are valid for 10 years from lodgement; the API exposes no expiry. */ +function tenYearsAfter(isoDate: string): string | undefined { + const d = new Date(isoDate); + if (Number.isNaN(d.getTime())) { + return; + } + d.setFullYear(d.getFullYear() + 10); + return d.toISOString().slice(0, 10); +} + +function normaliseRating(value: unknown): string | undefined { + const s = epcString(value)?.toUpperCase(); + return s && (RATING_ORDER as readonly string[]).includes(s) ? s : undefined; +} + +const INT_TOKEN_RE = /\b\d+\b/g; +// "2 Bedroom Flat", "1 bed" — the leading number is a bedroom count, not a +// house number, and must not be mistaken for one (it would false-match a +// same-numbered house on the street). Stripped before number extraction. +const BEDROOM_COUNT_RE = /\b\d+\s*(?:bed|beds|bedroom|bedrooms)\b/gi; +const NON_ALPHA_RE = /[^a-z\s]/g; +const WHITESPACE_RE = /\s+/g; +// Non-anchored: find a full postcode embedded in a user-typed address, +// tolerating the usual single space before the inward code. +const EMBEDDED_POSTCODE_RE = /[A-Z]{1,2}\d[A-Z\d]?\s?\d[A-Z]{2}/i; + +/** Pull a full postcode out of free-text (a manual address override), or null. */ +function extractFullPostcode(text: string): string | null { + const m = text.match(EMBEDDED_POSTCODE_RE); + return m ? m[0].toUpperCase().replace(WHITESPACE_RE, "") : null; +} +// Street words worth matching on — alphabetic, length ≥ 4, minus filler. +const ADDRESS_STOPWORDS = new Set([ + "flat", + "apartment", + "room", + "shared", + "house", + "london", + "road", + "street", + "avenue", + "close", + "court", + "lane", + "drive", + "place", +]); + +function numberTokens(addr: string): Set { + return new Set(addr.match(INT_TOKEN_RE) ?? []); +} + +function streetTokens(addr: string): Set { + const words = addr + .toLowerCase() + .replace(NON_ALPHA_RE, " ") + .split(WHITESPACE_RE) + .filter((w) => w.length >= 4 && !ADDRESS_STOPWORDS.has(w)); + return new Set(words); +} /** - * Map a raw EPC certificate row into the stored blob the app reads. The - * EPC Open Data API uses kebab-case keys (`current-energy-rating`, …) — not - * the camelCase the readers expect — and exposes a lodgement date rather - * than an expiry, so we derive `expiresOn` from the 10-year EPC validity - * window. Returns null when the row carries no current rating. + * Match the certificate for the cluster's exact building. Requires the + * cluster address to carry a number (house/flat) AND a cert whose address + * shares that number plus a street word — conservative on purpose, so we + * never assert "exact" on a street-only address. Returns null when no + * confident match exists. */ -export function normaliseEpcCert(cert: Certificate): NormalisedEpc | null { - const currentRating = epcString(cert["current-energy-rating"]); +export function pickExactCert( + certs: Certificate[], + clusterAddress: string +): { cert: Certificate; address: string } | null { + const cleaned = clusterAddress.replace(BEDROOM_COUNT_RE, " "); + const clusterNums = numberTokens(cleaned); + if (clusterNums.size === 0) { + return null; + } + const clusterStreet = streetTokens(cleaned); + + let best: { cert: Certificate; address: string } | null = null; + let bestScore = 0; + for (const cert of certs) { + const address = epcString(cert.address); + if (!address) { + continue; + } + const sharesNumber = [...numberTokens(address)].some((n) => + clusterNums.has(n) + ); + if (!sharesNumber) { + continue; + } + const sharedStreet = [...streetTokens(address)].filter((t) => + clusterStreet.has(t) + ).length; + if (sharedStreet === 0) { + continue; + } + if (sharedStreet > bestScore) { + bestScore = sharedStreet; + best = { cert, address }; + } + } + return best; +} + +/** Build the stored blob from a single, exactly-matched certificate. */ +export function exactBlob( + cert: Certificate, + postcode: string, + matchedAddress: string +): NormalisedEpc | null { + const currentRating = normaliseRating(cert["current-energy-rating"]); if (!currentRating) { return null; } - const potentialRating = epcString(cert["potential-energy-rating"]); + const potentialRating = normaliseRating(cert["potential-energy-rating"]); const lodgement = epcString(cert["lodgement-date"]); const expiresOn = lodgement ? tenYearsAfter(lodgement) : undefined; return { currentRating, ...(potentialRating ? { potentialRating } : {}), ...(expiresOn ? { expiresOn } : {}), + source: "exact", + matchedAddress, + postcode, }; } -function epcString(value: unknown): string | undefined { - if (typeof value !== "string") { - return; +/** + * Summarise a full postcode's certificates into a typical rating + range. + * `currentRating` is the modal band (ties broken toward the better band); + * `range` is the best…worst spread. Returns null when no cert carries a + * usable rating. + */ +export function estimateBlob( + certs: Certificate[], + postcode: string +): NormalisedEpc | null { + const ratings = certs + .map((c) => normaliseRating(c["current-energy-rating"])) + .filter((r): r is string => r !== undefined); + if (ratings.length === 0) { + return null; } - const trimmed = value.trim(); - return trimmed.length > 0 ? trimmed : undefined; + + const counts = new Map(); + for (const r of ratings) { + counts.set(r, (counts.get(r) ?? 0) + 1); + } + // Modal band; on a tie pick the better (earlier in RATING_ORDER). + let modal = ratings[0] as string; + let modalCount = 0; + for (const band of RATING_ORDER) { + const n = counts.get(band) ?? 0; + if (n > modalCount) { + modalCount = n; + modal = band; + } + } + + const present = RATING_ORDER.filter((b) => counts.has(b)); + const min = present[0] as string; + const max = present.at(-1) as string; + + return { + currentRating: modal, + source: "estimate", + postcode, + sampleSize: ratings.length, + range: { min, max }, + }; } -/** EPCs are valid for 10 years from lodgement; the API exposes no expiry. */ -function tenYearsAfter(isoDate: string): string | undefined { - const d = new Date(isoDate); - if (Number.isNaN(d.getTime())) { - return; +/** + * Resolve a FULL postcode for the EPC search: prefer an already-full + * scraped postcode, else snap the cluster's true coords to their nearest + * unit postcode. Returns null when neither is available (only an outcode, + * no coords) — the caller then has no precise search key. + */ +async function resolveFullPostcode( + postcode: string | null, + lat: number | null, + lng: number | null +): Promise { + const scraped = postcode?.trim().toUpperCase().replace(WHITESPACE_RE, ""); + if (scraped && FULL_POSTCODE_RE.test(scraped)) { + return scraped; } - d.setFullYear(d.getFullYear() + 10); - return d.toISOString().slice(0, 10); + if (lat != null && lng != null) { + return await reverseGeocodePostcode(lat, lng); + } + return null; +} + +type ClusterLocation = { + postcode: string | null; + lat: string | null; + lng: string | null; + normalisedAddress: string; + userAddress: string | null; +}; + +/** + * Decide which postcode to search EPC by and which address to match + * certificates against. A manual override (user pinned the door) wins for + * both; otherwise we search the reverse-geocoded full postcode and match + * the scraped address. `searchPostcode` falls back to the bare outcode as + * a last resort (exact-only — no estimate from an outcode). + */ +async function resolveEpcContext(cluster: ClusterLocation): Promise<{ + searchPostcode: string | null; + fullPostcode: string | null; + matchAddress: string; +}> { + const override = cluster.userAddress?.trim() || null; + const matchAddress = override ?? cluster.normalisedAddress; + const fullPostcode = + (override ? extractFullPostcode(override) : null) ?? + (await resolveFullPostcode( + cluster.postcode, + parseNumeric(cluster.lat), + parseNumeric(cluster.lng) + )); + return { + searchPostcode: fullPostcode ?? cluster.postcode, + fullPostcode, + matchAddress, + }; +} + +/** Run the EPC domestic search and return the (defensively-extracted) rows. */ +async function searchEpcCerts( + token: string, + postcode: string +): Promise { + const search = await getDomesticSearch({ + client: createEpcClient({ token }), + query: { postcode, size: 100 }, + }); + if (search.error) { + const message = + typeof search.error === "object" && + search.error !== null && + "message" in search.error + ? String((search.error as { message: unknown }).message) + : JSON.stringify(search.error); + throw new Error(`enrich-epc: EPC search failed: ${message}`); + } + return extractCertRows(search.data); } export const enrichEpcTask = task({ @@ -166,6 +362,11 @@ export const enrichEpcTask = task({ const db = getDb(); const { clusterId } = payload; const { EPC_OPENDATA_TOKEN } = env(); + const none = { + clusterId, + source: "none" as const, + listingsTouched: 0, + }; const cluster = await db.query.propertyClusters.findFirst({ where: (c, { eq: eqOp }) => eqOp(c.id, clusterId), @@ -173,62 +374,55 @@ export const enrichEpcTask = task({ if (!cluster) { throw new Error(`enrich-epc: cluster ${clusterId} not found`); } - if (!cluster.postcode) { - logger.warn("enrich-epc: cluster has no postcode, skipping", { + + // Decide the search postcode + the address to match certs against (a + // manual override wins for both). + const { searchPostcode, fullPostcode, matchAddress } = + await resolveEpcContext(cluster); + if (!searchPostcode) { + logger.warn("enrich-epc: no postcode or coords, skipping", { clusterId }); + return none; + } + + const certs = await searchEpcCerts(EPC_OPENDATA_TOKEN, searchPostcode); + if (certs.length === 0) { + logger.warn("enrich-epc: no certificates for postcode", { clusterId, + postcode: searchPostcode, }); - return { clusterId, certificateFound: false, listingsTouched: 0 }; + return none; } - const epc = createEpcClient({ token: EPC_OPENDATA_TOKEN }); - const search = await getDomesticSearch({ - client: epc, - query: { postcode: cluster.postcode, size: 25 }, - }); - if (search.error) { - const message = - typeof search.error === "object" && - search.error !== null && - "message" in search.error - ? String((search.error as { message: unknown }).message) - : JSON.stringify(search.error); - throw new Error(`enrich-epc: EPC search failed: ${message}`); + // Exact match by address (only when the address carries a number — + // most likely once the user has supplied a manual override). + const exact = pickExactCert(certs, matchAddress); + let blob = exact + ? exactBlob(exact.cert, searchPostcode, exact.address) + : null; + + // 3. Estimate fallback — only with a FULL postcode, so the sample is a + // single street/block, not a whole district. + if (!blob && fullPostcode) { + blob = estimateBlob(certs, fullPostcode); } - // The search endpoint returns `{ "column-names": [...], "rows": [...] }`, - // NOT the bare array the generated spec claims (the OpenAPI doc models the - // 200 body as `Certificates`, but the live API wraps the rows). Pull the - // rows out defensively so we iterate the actual certificates. - const certs = extractCertRows(search.data); - - const clusterLat = parseNumeric(cluster.lat); - const clusterLng = parseNumeric(cluster.lng); - const cert = pickClosestCertificate(certs, clusterLat, clusterLng); - - // Normalise the kebab-case EPC row into the camelCase blob the app reads. - const epcBlob = cert ? normaliseEpcCert(cert) : null; - if (!epcBlob) { - logger.warn("enrich-epc: no usable certificate for postcode", { + if (!blob) { + logger.log("enrich-epc: no confident match and no full postcode", { clusterId, - postcode: cluster.postcode, + postcode: searchPostcode, }); - return { clusterId, certificateFound: false, listingsTouched: 0 }; + return none; } const touched = await upsertEnrichmentForCluster(db, clusterId, { - epc: epcBlob, + epc: blob, }); - logger.log("enrich-epc: done", { clusterId, - postcode: cluster.postcode, - certificateFound: true, + postcode: searchPostcode, + source: blob.source, + currentRating: blob.currentRating, listingsTouched: touched, }); - - return { - clusterId, - certificateFound: true, - listingsTouched: touched, - }; + return { clusterId, source: blob.source, listingsTouched: touched }; }, });