Update xpaths for changes to Rightmove HTML structure

rightmove_webscraper/scraper.py

@@ -97,7 +97,7 @@ def summary(self, by: str = None):
             by (str): valid column name from `get_results` DataFrame attribute.
         """
         if not by:
-            by = "type" if "commercial" in self.rent_or_sale else "number_bedrooms"
+            by = "type" if self.is_commercial else "number_bedrooms"
         assert by in self.get_results.columns, f"Column not found in `get_results`: {by}"
         df = self.get_results.dropna(axis=0, subset=["price"])
         groupers = {"price": ["count", "mean"]}
@@ -112,17 +112,12 @@ def summary(self, by: str = None):
         return df.reset_index(drop=True)
 
     @property
-    def rent_or_sale(self):
-        """String specifying if the search is for properties for rent or sale.
-        Required because Xpaths are different for the target elements."""
-        if "/property-for-sale/" in self.url or "/new-homes-for-sale/" in self.url:
-            return "sale"
-        elif "/property-to-rent/" in self.url:
-            return "rent"
-        elif "/commercial-property-for-sale/" in self.url:
-            return "sale-commercial"
-        elif "/commercial-property-to-let/" in self.url:
-            return "rent-commercial"
+    def is_commercial(self):
+        """Boolean specifying if the search is for commercial properties."""
+        if "/property-for-sale/" in self.url or "/new-homes-for-sale/" in self.url or "/property-to-rent/" in self.url:
+            return False
+        elif "/commercial-property-for-sale/" in self.url or "/commercial-property-to-let/" in self.url:
+            return True
         else:
             raise ValueError(f"Invalid rightmove URL:\n\n\t{self.url}")
 
@@ -132,7 +127,7 @@ def results_count_display(self):
         the first page of results. Note that not all listings are available to
         scrape because rightmove limits the number of accessible pages."""
         tree = html.fromstring(self._first_page)
-        xpath = """//span[@class="searchHeader-resultCount"]/text()"""
+        xpath = """//div[contains(@class,"ResultsCount_resultsCount__")]//p//span/text()"""
         return int(tree.xpath(xpath)[0].replace(",", ""))
 
     @property
@@ -155,31 +150,39 @@ def _get_page(self, request_content: str, get_floorplans: bool = False):
         # Process the html:
         tree = html.fromstring(request_content)
 
-        # Set xpath for price:
-        if "rent" in self.rent_or_sale:
-            xp_prices = """//span[@class="propertyCard-priceValue"]/text()"""
-        elif "sale" in self.rent_or_sale:
-            xp_prices = """//div[@class="propertyCard-priceValue"]/text()"""
-        else:
-            raise ValueError("Invalid URL format.")
-
-        # Set xpaths for listing title, property address, URL, and agent URL:
-        xp_titles = """//div[@class="propertyCard-details"]\
-        //a[@class="propertyCard-link"]\
-        //h2[@class="propertyCard-title"]/text()"""
-        xp_addresses = """//address[@class="propertyCard-address"]//span/text()"""
-        xp_weblinks = """//div[@class="propertyCard-details"]//a[@class="propertyCard-link"]/@href"""
-        xp_agent_urls = """//div[@class="propertyCard-contactsItem"]\
-        //div[@class="propertyCard-branchLogo"]\
-        //a[@class="propertyCard-branchLogo-link"]/@href"""
-
-        # Create data lists from xpaths:
-        price_pcm = tree.xpath(xp_prices)
-        titles = tree.xpath(xp_titles)
-        addresses = tree.xpath(xp_addresses)
+        # Find all property card containers and extract data per-card.
+        # This ensures fields like bedroom count (which may be missing) stay aligned with the correct data row.
+        cards = tree.xpath("""//div[contains(@class, "propertyCard-details")]""")
+
+        price_pcm = []
+        types = []
+        addresses = []
+        weblinks = []
+        agent_urls = []
+        number_bedrooms = []
         base = "http://www.rightmove.co.uk"
-        weblinks = [f"{base}{tree.xpath(xp_weblinks)[w]}" for w in range(len(tree.xpath(xp_weblinks)))]
-        agent_urls = [f"{base}{tree.xpath(xp_agent_urls)[a]}" for a in range(len(tree.xpath(xp_agent_urls)))]
+
+        for card in cards:
+            price = card.xpath('.//div[contains(@class, "PropertyPrice_price__")]/text()')
+            price_pcm.append(price[0] if price else None)
+
+            prop_type = card.xpath('.//span[contains(@class, "PropertyInformation_propertyType__")]/text()')
+            types.append(prop_type[0] if prop_type else None)
+
+            address = card.xpath('.//address[contains(@class, "PropertyAddress_address__")]/text()')
+            addresses.append(address[0] if address else None)
+
+            link = card.xpath('.//a[@class="propertyCard-link"]/@href')
+            weblinks.append(f"{base}{link[0]}" if link else None)
+
+            agent_url = card.xpath('.//div[contains(@class,"PropertyCardActions_estateAgent__")]//a/@href')
+            agent_urls.append(f"{base}{agent_url[0]}" if agent_url else None)
+
+            bedrooms = card.xpath('.//span[contains(@class, "PropertyInformation_bedroomsCount__")]/text()')
+            number_bedrooms.append(bedrooms[0] if bedrooms else None)
+
+        print(f"Scraped {len(cards)} listings from page.")
+        print(f"price_pcm: {len(price_pcm)}, types: {len(types)}, addresses: {len(addresses)}, weblinks: {len(weblinks)}, agent_urls: {len(agent_urls)}, number_bedrooms: {len(number_bedrooms)}")
 
         # Optionally get floorplan links from property urls (longer runtime):
         floorplan_urls = list() if get_floorplans else np.nan
@@ -197,11 +200,11 @@ def _get_page(self, request_content: str, get_floorplans: bool = False):
                     floorplan_urls.append(np.nan)
 
         # Store the data in a Pandas DataFrame:
-        data = [price_pcm, titles, addresses, weblinks, agent_urls]
+        data = [price_pcm, types, addresses, weblinks, agent_urls, number_bedrooms]
         data = data + [floorplan_urls] if get_floorplans else data
         temp_df = pd.DataFrame(data)
         temp_df = temp_df.transpose()
-        columns = ["price", "type", "address", "url", "agent_url"]
+        columns = ["price", "type", "address", "url", "agent_url", "number_bedrooms"]
         columns = columns + ["floorplan_url"] if get_floorplans else columns
         temp_df.columns = columns
 
@@ -253,11 +256,8 @@ def _clean_results(results: pd.DataFrame):
         pat = r"([A-Za-z][A-Za-z]?[0-9][0-9]?[A-Za-z]?[0-9]?\s[0-9]?[A-Za-z][A-Za-z])"
         results["full_postcode"] = results["address"].astype(str).str.extract(pat, expand=True)[0]
 
-        # Extract number of bedrooms from `type` to a separate column:
-        pat = r"\b([\d][\d]?)\b"
-        results["number_bedrooms"] = results["type"].astype(str).str.extract(pat, expand=True)[0]
-        results.loc[results["type"].str.contains("studio", case=False), "number_bedrooms"] = 0
-        results["number_bedrooms"] = pd.to_numeric(results["number_bedrooms"])
+        # Record 'studio' properties as having 0 bedrooms:
+        results.loc[results["type"].str.contains("studio", case=False), "number_bedrooms"] = "0"
 
         # Clean up annoying white spaces and newlines in `type` column:
         results["type"] = results["type"].str.strip("\n").str.strip()

test_rm.py

@@ -18,7 +18,7 @@ def test_sale_residential():
     assert required_columns.issubset(set(rm.get_results.columns))
     assert len(rm.get_results) > 0
     assert isinstance(rm.page_count, int)
-    assert rm.rent_or_sale == "sale"
+    assert rm.is_commercial == False
     assert isinstance(rm.results_count, int)
     assert isinstance(rm.results_count_display, int)
     assert url == rm.url
@@ -42,7 +42,7 @@ def test_rent_residential():
     assert required_columns.issubset(set(rm.get_results.columns))
     assert len(rm.get_results) > 0
     assert isinstance(rm.page_count, int)
-    assert rm.rent_or_sale == "rent"
+    assert rm.is_commercial == False
     assert isinstance(rm.results_count, int)
     assert isinstance(rm.results_count_display, int)
     assert url == rm.url
@@ -66,7 +66,7 @@ def test_sale_commercial():
     assert required_columns.issubset(set(rm.get_results.columns))
     assert len(rm.get_results) > 0
     assert isinstance(rm.page_count, int)
-    assert rm.rent_or_sale == "sale-commercial"
+    assert rm.is_commercial == True
     assert isinstance(rm.results_count, int)
     assert isinstance(rm.results_count_display, int)
     assert url == rm.url
@@ -92,7 +92,7 @@ def test_rent_commercial():
     assert required_columns.issubset(set(rm.get_results.columns))
     assert len(rm.get_results) > 0
     assert isinstance(rm.page_count, int)
-    assert rm.rent_or_sale == "rent-commercial"
+    assert rm.is_commercial == True
     assert isinstance(rm.results_count, int)
     assert isinstance(rm.results_count_display, int)
     assert url == rm.url