utils.py

"""Utility functions for Logic-Immo scraper."""

import re
import logging
from typing import Optional, List, Dict, Any
from bs4 import BeautifulSoup

logger = logging.getLogger(__name__)


def parse_price(price_text: str) -> Optional[int]:
    """Extract numeric price from text.

    Args:
        price_text: Text containing price (e.g., "807 500 €")

    Returns:
        Integer price or None if parsing fails
    """
    if not price_text:
        return None

    # Remove currency symbol, spaces, and non-breaking spaces
    cleaned = re.sub(r"[€\s\u00a0]", "", price_text)
    # Extract digits only
    digits = re.sub(r"[^\d]", "", cleaned)

    try:
        return int(digits) if digits else None
    except ValueError:
        return None


def parse_price_per_sqm(text: str) -> Optional[float]:
    """Extract price per square meter from text.

    Args:
        text: Text containing price per sqm (e.g., "10 767 €/m²")

    Returns:
        Float price per sqm or None if parsing fails
    """
    if not text:
        return None

    # Match pattern like "10 767 €/m²" or "10767€/m²"
    match = re.search(r"([\d\s\u00a0]+)\s*€\s*/\s*m²", text)
    if match:
        value = re.sub(r"[\s\u00a0]", "", match.group(1))
        try:
            return float(value)
        except ValueError:
            return None
    return None


def parse_area(text: str) -> Optional[float]:
    """Extract living area from text.

    Args:
        text: Text containing area (e.g., "75 m²", "79,4 m²")

    Returns:
        Float area or None if parsing fails
    """
    if not text:
        return None

    # Match pattern like "75 m²" or "79,4 m²"
    match = re.search(r"([\d,.\s]+)\s*m²", text)
    if match:
        value = match.group(1).replace(",", ".").replace(" ", "")
        try:
            return float(value)
        except ValueError:
            return None
    return None


def parse_rooms(text: str) -> Optional[int]:
    """Extract number of rooms from text.

    Args:
        text: Text containing rooms (e.g., "3 pièces", "1 pièce")

    Returns:
        Integer number of rooms or None
    """
    if not text:
        return None

    match = re.search(r"(\d+)\s*pièces?", text)
    if match:
        return int(match.group(1))
    return None


def parse_bedrooms(text: str) -> Optional[int]:
    """Extract number of bedrooms from text.

    Args:
        text: Text containing bedrooms (e.g., "2 chambres", "1 chambre")

    Returns:
        Integer number of bedrooms or None
    """
    if not text:
        return None

    match = re.search(r"(\d+)\s*chambres?", text)
    if match:
        return int(match.group(1))
    return None


def parse_floor(text: str) -> str:
    """Extract floor information from text.

    Args:
        text: Text containing floor (e.g., "3ème étage", "RDC", "Étage 9/9")

    Returns:
        Floor string or empty string
    """
    if not text:
        return ""

    # Match patterns like "3ème étage", "Étage 9/9", "RDC"
    patterns = [
        r"(\d+)(?:er|ème|e)?\s*étage",
        r"Étage\s*(\d+)/\d+",
        r"RDC|rez-de-chaussée",
    ]

    for pattern in patterns:
        match = re.search(pattern, text, re.IGNORECASE)
        if match:
            if "RDC" in text.upper() or "rez-de-chaussée" in text.lower():
                return "0"
            return match.group(1) if match.lastindex else match.group(0)
    return ""


def parse_postal_code(text: str) -> str:
    """Extract postal code from location text.

    Args:
        text: Location text (e.g., "Paris 20ème (75020)")

    Returns:
        Postal code or empty string
    """
    if not text:
        return ""

    match = re.search(r"\((\d{5})\)", text)
    return match.group(1) if match else ""


def parse_energy_rating(text: str) -> str:
    """Extract energy rating (DPE) from text.

    Args:
        text: Text containing energy rating (A-G)

    Returns:
        Energy rating letter or empty string
    """
    if not text:
        return ""

    match = re.search(r"[A-G]", text.upper())
    return match.group(0) if match else ""


def parse_total_count(html: str) -> int:
    """Extract total listing count from page HTML.

    Args:
        html: HTML content of search page

    Returns:
        Total number of listings
    """
    soup = BeautifulSoup(html, "lxml")

    # Try to find count in page title
    title = soup.find("title")
    if title:
        match = re.search(r"(\d[\d\s]*)\s*annonces?", title.get_text())
        if match:
            count_str = match.group(1).replace(" ", "")
            try:
                return int(count_str)
            except ValueError:
                pass

    # Try to find count in the page content
    count_elem = soup.select_one('[data-testid="result-count"]')
    if count_elem:
        match = re.search(r"(\d[\d\s]*)", count_elem.get_text())
        if match:
            count_str = match.group(1).replace(" ", "")
            try:
                return int(count_str)
            except ValueError:
                pass

    # Try alternative selector - look for "X annonces" text
    text_content = soup.get_text()
    match = re.search(r"(\d[\d\s]*)\s*annonces?", text_content)
    if match:
        count_str = match.group(1).replace(" ", "")
        try:
            return int(count_str)
        except ValueError:
            pass

    return 0


def parse_listings_from_page(html: str) -> List[Dict[str, Any]]:
    """Parse property listings from search results page.

    Args:
        html: HTML content of search page

    Returns:
        List of property dictionaries with basic info
    """
    soup = BeautifulSoup(html, "lxml")
    properties = []

    # Find all listing cards using data-testid pattern
    listing_cards = soup.select('[data-testid^="classified-card-mfe-"]')

    if not listing_cards:
        # Alternative: find cards by structure
        listing_cards = soup.select('button[data-testid="card-mfe-covering-link-testid"]')
        if listing_cards:
            listing_cards = [card.parent.parent for card in listing_cards if card.parent and card.parent.parent]

    for card in listing_cards:
        try:
            prop = parse_listing_card(card)
            if prop and prop.get("url"):
                properties.append(prop)
        except Exception as e:
            logger.warning(f"Error parsing listing card: {e}")
            continue

    return properties


def parse_listing_card(card) -> Optional[Dict[str, Any]]:
    """Parse a single listing card from search results.

    Args:
        card: BeautifulSoup element for the card

    Returns:
        Property dictionary or None
    """
    prop = {}

    # Get card text which contains all main info
    card_text = card.get_text(separator=" ", strip=True)

    # Extract listing ID from data-testid
    testid = card.get("data-testid", "")
    match = re.search(r"classified-card-mfe-([A-Z0-9]+)", testid)
    if match:
        prop["listing_id"] = match.group(1)

    # Find the detail link
    link = card.select_one('a[href*="/detail-"]')
    if link:
        prop["url"] = link.get("href", "")
        if prop["url"] and not prop["url"].startswith("http"):
            prop["url"] = f"https://www.logic-immo.com{prop['url']}"

        # Extract listing ID from URL if not found in testid
        if not prop.get("listing_id"):
            url_match = re.search(r"detail-(?:vente|location)-(\d+)", prop["url"])
            if url_match:
                prop["listing_id"] = url_match.group(1)
    else:
        # Try finding clickable button and extract listing ID
        button = card.select_one('button[data-testid="card-mfe-covering-link-testid"]')
        if button:
            onclick = button.get("onclick", "")
            match = re.search(r"detail-(?:vente|location)-(\d+)", onclick)
            if match:
                prop["listing_id"] = match.group(1)
                prop["url"] = f"https://www.logic-immo.com/detail-vente-{match.group(1)}.htm"

    if not prop.get("url") and prop.get("listing_id"):
        prop["url"] = f"https://www.logic-immo.com/detail-vente-{prop['listing_id']}.htm"

    # Parse property type
    type_patterns = [
        "Appartement", "Maison", "Duplex", "Triplex", "Loft",
        "Parking", "Terrain", "Local commercial", "Bureau", "Immeuble"
    ]
    for ptype in type_patterns:
        if ptype.lower() in card_text.lower():
            prop["property_type"] = ptype
            break

    # Check if new construction
    prop["is_new"] = "neuf" in card_text.lower()

    # Check if exclusive
    prop["is_exclusive"] = "exclusivité" in card_text.lower()

    # Parse price
    price_match = re.search(r"([\d\s\u00a0]+)\s*€(?!\s*/\s*m)", card_text)
    if price_match:
        prop["price"] = parse_price(price_match.group(1))

    # Parse price per sqm
    prop["price_per_sqm"] = parse_price_per_sqm(card_text)

    # Parse rooms
    prop["rooms"] = parse_rooms(card_text)

    # Parse bedrooms
    prop["bedrooms"] = parse_bedrooms(card_text)

    # Parse area
    prop["living_area"] = parse_area(card_text)

    # Parse floor
    prop["floor"] = parse_floor(card_text)

    # Parse location
    location_match = re.search(r"(?:,\s*)?([^,]+)\s+\((\d{5})\)", card_text)
    if location_match:
        prop["full_address"] = location_match.group(0).strip()
        prop["city"] = location_match.group(1).strip()
        prop["postal_code"] = location_match.group(2)

    # Parse energy rating from DPE badge
    energy_elem = card.select_one('[class*="energy"], [class*="dpe"]')
    if energy_elem:
        prop["energy_rating"] = parse_energy_rating(energy_elem.get_text())
    else:
        # Try to find energy rating in text (single letter A-G)
        energy_match = re.search(r"\b([A-G])\b(?:\s|$)", card_text)
        if energy_match and len(energy_match.group(1)) == 1:
            prop["energy_rating"] = energy_match.group(1)

    # Parse agency name
    agency_elem = card.select_one('[class*="agency"], [class*="advertiser"]')
    if agency_elem:
        prop["agency_name"] = agency_elem.get_text(strip=True)

    return prop if prop.get("listing_id") or prop.get("url") else None


def parse_property_detail(html: str, url: str = "") -> Optional[Dict[str, Any]]:
    """Parse property details from detail page.

    Args:
        html: HTML content of property detail page
        url: Property URL

    Returns:
        Property dictionary or None
    """
    soup = BeautifulSoup(html, "lxml")
    prop = {"url": url}

    # Extract listing ID from URL
    if url:
        match = re.search(r"detail-(?:vente|location)-(\d+)", url)
        if match:
            prop["listing_id"] = match.group(1)

    # Parse title
    title = soup.find("h1")
    if title:
        title_text = title.get_text(strip=True)
        prop["title"] = title_text

        # Parse property type from title
        type_patterns = [
            "Appartement", "Maison", "Duplex", "Triplex", "Loft",
            "Parking", "Terrain", "Local commercial", "Bureau", "Immeuble"
        ]
        for ptype in type_patterns:
            if ptype.lower() in title_text.lower():
                prop["property_type"] = ptype
                break

        # Check if new
        prop["is_new"] = "neuf" in title_text.lower()

    # Parse price
    price_elem = soup.select_one('[class*="price"], h1')
    if price_elem:
        price_text = price_elem.get_text()
        price_match = re.search(r"([\d\s\u00a0]+)\s*€(?!\s*/\s*m)", price_text)
        if price_match:
            prop["price"] = parse_price(price_match.group(1))
        prop["price_per_sqm"] = parse_price_per_sqm(price_text)

    # Get main content text for parsing
    main_content = soup.find("main")
    if main_content:
        content_text = main_content.get_text(separator=" ", strip=True)
    else:
        content_text = soup.get_text(separator=" ", strip=True)

    # Parse rooms
    prop["rooms"] = parse_rooms(content_text)

    # Parse bedrooms
    prop["bedrooms"] = parse_bedrooms(content_text)

    # Parse area
    prop["living_area"] = parse_area(content_text)

    # Parse floor
    prop["floor"] = parse_floor(content_text)

    # Parse location
    location_match = re.search(r"(?:,\s*)?([^,]+)\s+\((\d{5})\)", content_text)
    if location_match:
        prop["full_address"] = location_match.group(0).strip()
        prop["city"] = location_match.group(1).strip()
        prop["postal_code"] = location_match.group(2)

    # Parse features from characteristics list
    features_section = soup.select('[class*="characteristics"], [class*="features"], ul')
    for section in features_section:
        text = section.get_text(separator=" ", strip=True).lower()
        if "ascenseur" in text:
            prop["has_elevator"] = True
        if "cave" in text:
            prop["has_cellar"] = "pas de cave" not in text
        if "parking" in text:
            prop["has_parking"] = True
        if "balcon" in text:
            prop["has_balcony"] = True
        if "terrasse" in text:
            prop["has_terrace"] = True
        if "jardin" in text:
            prop["has_garden"] = True
        if "meublé" in text:
            prop["is_furnished"] = "non meublé" not in text

    # Parse bathrooms
    bath_match = re.search(r"(\d+)\s*(?:salle[s]?\s*(?:de\s*)?bain|sdb)", content_text, re.IGNORECASE)
    if bath_match:
        prop["bathrooms"] = int(bath_match.group(1))

    # Parse year built
    year_match = re.search(r"(?:construction|année|baujahr)[:\s]+(\d{4})", content_text, re.IGNORECASE)
    if year_match:
        prop["year_built"] = year_match.group(1)
    else:
        year_match = re.search(r"(\d{4})(?:\s*/\s*\w+)?", content_text)
        if year_match and 1800 <= int(year_match.group(1)) <= 2030:
            prop["year_built"] = year_match.group(1)

    # Parse energy rating
    energy_section = soup.select_one('[class*="energy"], [class*="dpe"]')
    if energy_section:
        prop["energy_rating"] = parse_energy_rating(energy_section.get_text())

    # Parse description
    desc_elem = soup.select_one('[class*="description"], p')
    if desc_elem:
        prop["description"] = desc_elem.get_text(strip=True)[:1000]

    # Parse agency info
    agency_section = soup.select_one('[class*="agency"], [class*="advertiser"]')
    if agency_section:
        agency_text = agency_section.get_text(separator=" ", strip=True)
        prop["agency_name"] = agency_text[:100]

    # Parse reference
    ref_match = re.search(r"(?:réf(?:érence)?|ref)[:\s]+([A-Za-z0-9]+)", content_text, re.IGNORECASE)
    if ref_match:
        prop["reference"] = ref_match.group(1)

    # Parse listing ID from page if not already set
    id_match = re.search(r"(?:identifiant|id)[:\s]+(\d+)", content_text, re.IGNORECASE)
    if id_match and not prop.get("listing_id"):
        prop["listing_id"] = id_match.group(1)

    return prop