From 79326c4fa221e4edbb075db5a392d487adc73b1f Mon Sep 17 00:00:00 2001 From: David Mears Date: Sat, 21 Feb 2026 18:51:52 +0000 Subject: [PATCH 1/4] Update xpaths Relatedly, listing 'titles' are not a thing implied by the html structure any more, but I think the usage of titles in the codebase implies they correspond to what the new html structure calls 'propertyType'. Also relatedly, the method 'rent_or_sale' is no longer required for choosing xpaths, but it was used elsewhere to check if the search was for commercial properties, so I simplified it to do that only. --- rightmove_webscraper/scraper.py | 48 +++++++++++---------------------- 1 file changed, 16 insertions(+), 32 deletions(-) diff --git a/rightmove_webscraper/scraper.py b/rightmove_webscraper/scraper.py index 12acc0b..1d59a50 100644 --- a/rightmove_webscraper/scraper.py +++ b/rightmove_webscraper/scraper.py @@ -97,7 +97,7 @@ def summary(self, by: str = None): by (str): valid column name from `get_results` DataFrame attribute. """ if not by: - by = "type" if "commercial" in self.rent_or_sale else "number_bedrooms" + by = "type" if self.is_commercial() else "number_bedrooms" assert by in self.get_results.columns, f"Column not found in `get_results`: {by}" df = self.get_results.dropna(axis=0, subset=["price"]) groupers = {"price": ["count", "mean"]} @@ -112,17 +112,12 @@ def summary(self, by: str = None): return df.reset_index(drop=True) @property - def rent_or_sale(self): - """String specifying if the search is for properties for rent or sale. - Required because Xpaths are different for the target elements.""" - if "/property-for-sale/" in self.url or "/new-homes-for-sale/" in self.url: - return "sale" - elif "/property-to-rent/" in self.url: - return "rent" - elif "/commercial-property-for-sale/" in self.url: - return "sale-commercial" - elif "/commercial-property-to-let/" in self.url: - return "rent-commercial" + def is_commercial(self): + """Boolean specifying if the search is for commercial properties.""" + if "/property-for-sale/" in self.url or "/new-homes-for-sale/" in self.url or "/property-to-rent/" in self.url: + return False + elif "/commercial-property-for-sale/" in self.url or "/commercial-property-to-let/" in self.url: + return True else: raise ValueError(f"Invalid rightmove URL:\n\n\t{self.url}") @@ -132,7 +127,7 @@ def results_count_display(self): the first page of results. Note that not all listings are available to scrape because rightmove limits the number of accessible pages.""" tree = html.fromstring(self._first_page) - xpath = """//span[@class="searchHeader-resultCount"]/text()""" + xpath = """//div[contains(@class,"ResultsCount_resultsCount__")]//p//span/text()""" return int(tree.xpath(xpath)[0].replace(",", "")) @property @@ -155,27 +150,16 @@ def _get_page(self, request_content: str, get_floorplans: bool = False): # Process the html: tree = html.fromstring(request_content) - # Set xpath for price: - if "rent" in self.rent_or_sale: - xp_prices = """//span[@class="propertyCard-priceValue"]/text()""" - elif "sale" in self.rent_or_sale: - xp_prices = """//div[@class="propertyCard-priceValue"]/text()""" - else: - raise ValueError("Invalid URL format.") - - # Set xpaths for listing title, property address, URL, and agent URL: - xp_titles = """//div[@class="propertyCard-details"]\ - //a[@class="propertyCard-link"]\ - //h2[@class="propertyCard-title"]/text()""" - xp_addresses = """//address[@class="propertyCard-address"]//span/text()""" - xp_weblinks = """//div[@class="propertyCard-details"]//a[@class="propertyCard-link"]/@href""" - xp_agent_urls = """//div[@class="propertyCard-contactsItem"]\ - //div[@class="propertyCard-branchLogo"]\ - //a[@class="propertyCard-branchLogo-link"]/@href""" + # Set xpaths for listing price, type, property address, URL, and agent URL: + xp_prices = """//div[contains(@class, "PropertyPrice_price")]/text()""" + xp_types = """//span[contains(@class, "PropertyInformation_propertyType")]/text()""" + xp_addresses = """//address[contains(@class, "PropertyAddress_address")]/text()""" + xp_weblinks = """//div[contains(@class, "propertyCard-details")]//a[contains(@class, "propertyCard-link")]/@href""" + xp_agent_urls = """//div[contains(@class,"PropertyCardActions_estateAgent")]//a/@href""" # Create data lists from xpaths: price_pcm = tree.xpath(xp_prices) - titles = tree.xpath(xp_titles) + types = tree.xpath(xp_types) addresses = tree.xpath(xp_addresses) base = "http://www.rightmove.co.uk" weblinks = [f"{base}{tree.xpath(xp_weblinks)[w]}" for w in range(len(tree.xpath(xp_weblinks)))] @@ -197,7 +181,7 @@ def _get_page(self, request_content: str, get_floorplans: bool = False): floorplan_urls.append(np.nan) # Store the data in a Pandas DataFrame: - data = [price_pcm, titles, addresses, weblinks, agent_urls] + data = [price_pcm, types, addresses, weblinks, agent_urls] data = data + [floorplan_urls] if get_floorplans else data temp_df = pd.DataFrame(data) temp_df = temp_df.transpose() From f4f38733e33a6b1f50580d290c6cb1c1573b5f50 Mon Sep 17 00:00:00 2001 From: David Mears Date: Sat, 21 Feb 2026 21:29:41 +0000 Subject: [PATCH 2/4] Fix bad reference to property --- rightmove_webscraper/scraper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rightmove_webscraper/scraper.py b/rightmove_webscraper/scraper.py index 1d59a50..aa4a620 100644 --- a/rightmove_webscraper/scraper.py +++ b/rightmove_webscraper/scraper.py @@ -97,7 +97,7 @@ def summary(self, by: str = None): by (str): valid column name from `get_results` DataFrame attribute. """ if not by: - by = "type" if self.is_commercial() else "number_bedrooms" + by = "type" if self.is_commercial else "number_bedrooms" assert by in self.get_results.columns, f"Column not found in `get_results`: {by}" df = self.get_results.dropna(axis=0, subset=["price"]) groupers = {"price": ["count", "mean"]} From 7078f859a3ed4be1d4cc3bf2445cf385c3e6dd49 Mon Sep 17 00:00:00 2001 From: David Mears Date: Sat, 21 Feb 2026 21:32:31 +0000 Subject: [PATCH 3/4] Fix buggy xpaths The new xpaths didn't always find the same number of elements, because of similar class names that were caught as false positives, e.g. PropertyPrice_price matched PropertyPrice_priceLink. So we needed more specific matchers like PropertyPrice_price__. Also, the number of bedrooms is now a new html element, not part of the 'type' string, so I added an xpath lookup for that element. Since this data does not exist on every result card, we need to iterate over every card to prevent bedroom counts being applied to the wrong data rows. --- rightmove_webscraper/scraper.py | 56 +++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 20 deletions(-) diff --git a/rightmove_webscraper/scraper.py b/rightmove_webscraper/scraper.py index aa4a620..d00fd9d 100644 --- a/rightmove_webscraper/scraper.py +++ b/rightmove_webscraper/scraper.py @@ -150,20 +150,39 @@ def _get_page(self, request_content: str, get_floorplans: bool = False): # Process the html: tree = html.fromstring(request_content) - # Set xpaths for listing price, type, property address, URL, and agent URL: - xp_prices = """//div[contains(@class, "PropertyPrice_price")]/text()""" - xp_types = """//span[contains(@class, "PropertyInformation_propertyType")]/text()""" - xp_addresses = """//address[contains(@class, "PropertyAddress_address")]/text()""" - xp_weblinks = """//div[contains(@class, "propertyCard-details")]//a[contains(@class, "propertyCard-link")]/@href""" - xp_agent_urls = """//div[contains(@class,"PropertyCardActions_estateAgent")]//a/@href""" - - # Create data lists from xpaths: - price_pcm = tree.xpath(xp_prices) - types = tree.xpath(xp_types) - addresses = tree.xpath(xp_addresses) + # Find all property card containers and extract data per-card. + # This ensures fields like bedroom count (which may be missing) stay aligned with the correct data row. + cards = tree.xpath("""//div[contains(@class, "propertyCard-details")]""") + + price_pcm = [] + types = [] + addresses = [] + weblinks = [] + agent_urls = [] + number_bedrooms = [] base = "http://www.rightmove.co.uk" - weblinks = [f"{base}{tree.xpath(xp_weblinks)[w]}" for w in range(len(tree.xpath(xp_weblinks)))] - agent_urls = [f"{base}{tree.xpath(xp_agent_urls)[a]}" for a in range(len(tree.xpath(xp_agent_urls)))] + + for card in cards: + price = card.xpath('.//div[contains(@class, "PropertyPrice_price__")]/text()') + price_pcm.append(price[0] if price else None) + + prop_type = card.xpath('.//span[contains(@class, "PropertyInformation_propertyType__")]/text()') + types.append(prop_type[0] if prop_type else None) + + address = card.xpath('.//address[contains(@class, "PropertyAddress_address__")]/text()') + addresses.append(address[0] if address else None) + + link = card.xpath('.//a[@class="propertyCard-link"]/@href') + weblinks.append(f"{base}{link[0]}" if link else None) + + agent_url = card.xpath('.//div[contains(@class,"PropertyCardActions_estateAgent__")]//a/@href') + agent_urls.append(f"{base}{agent_url[0]}" if agent_url else None) + + bedrooms = card.xpath('.//span[contains(@class, "PropertyInformation_bedroomsCount__")]/text()') + number_bedrooms.append(bedrooms[0] if bedrooms else None) + + print(f"Scraped {len(cards)} listings from page.") + print(f"price_pcm: {len(price_pcm)}, types: {len(types)}, addresses: {len(addresses)}, weblinks: {len(weblinks)}, agent_urls: {len(agent_urls)}, number_bedrooms: {len(number_bedrooms)}") # Optionally get floorplan links from property urls (longer runtime): floorplan_urls = list() if get_floorplans else np.nan @@ -181,11 +200,11 @@ def _get_page(self, request_content: str, get_floorplans: bool = False): floorplan_urls.append(np.nan) # Store the data in a Pandas DataFrame: - data = [price_pcm, types, addresses, weblinks, agent_urls] + data = [price_pcm, types, addresses, weblinks, agent_urls, number_bedrooms] data = data + [floorplan_urls] if get_floorplans else data temp_df = pd.DataFrame(data) temp_df = temp_df.transpose() - columns = ["price", "type", "address", "url", "agent_url"] + columns = ["price", "type", "address", "url", "agent_url", "number_bedrooms"] columns = columns + ["floorplan_url"] if get_floorplans else columns temp_df.columns = columns @@ -237,11 +256,8 @@ def _clean_results(results: pd.DataFrame): pat = r"([A-Za-z][A-Za-z]?[0-9][0-9]?[A-Za-z]?[0-9]?\s[0-9]?[A-Za-z][A-Za-z])" results["full_postcode"] = results["address"].astype(str).str.extract(pat, expand=True)[0] - # Extract number of bedrooms from `type` to a separate column: - pat = r"\b([\d][\d]?)\b" - results["number_bedrooms"] = results["type"].astype(str).str.extract(pat, expand=True)[0] - results.loc[results["type"].str.contains("studio", case=False), "number_bedrooms"] = 0 - results["number_bedrooms"] = pd.to_numeric(results["number_bedrooms"]) + # Record 'studio' properties as having 0 bedrooms: + results.loc[results["type"].str.contains("studio", case=False), "number_bedrooms"] = "0" # Clean up annoying white spaces and newlines in `type` column: results["type"] = results["type"].str.strip("\n").str.strip() From 437845ae1d7abccdedd12f1966acb5a8acb43a76 Mon Sep 17 00:00:00 2001 From: David Mears Date: Sat, 21 Feb 2026 22:34:39 +0000 Subject: [PATCH 4/4] Update tests --- test_rm.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test_rm.py b/test_rm.py index 7e593a9..26cee75 100644 --- a/test_rm.py +++ b/test_rm.py @@ -18,7 +18,7 @@ def test_sale_residential(): assert required_columns.issubset(set(rm.get_results.columns)) assert len(rm.get_results) > 0 assert isinstance(rm.page_count, int) - assert rm.rent_or_sale == "sale" + assert rm.is_commercial == False assert isinstance(rm.results_count, int) assert isinstance(rm.results_count_display, int) assert url == rm.url @@ -42,7 +42,7 @@ def test_rent_residential(): assert required_columns.issubset(set(rm.get_results.columns)) assert len(rm.get_results) > 0 assert isinstance(rm.page_count, int) - assert rm.rent_or_sale == "rent" + assert rm.is_commercial == False assert isinstance(rm.results_count, int) assert isinstance(rm.results_count_display, int) assert url == rm.url @@ -66,7 +66,7 @@ def test_sale_commercial(): assert required_columns.issubset(set(rm.get_results.columns)) assert len(rm.get_results) > 0 assert isinstance(rm.page_count, int) - assert rm.rent_or_sale == "sale-commercial" + assert rm.is_commercial == True assert isinstance(rm.results_count, int) assert isinstance(rm.results_count_display, int) assert url == rm.url @@ -92,7 +92,7 @@ def test_rent_commercial(): assert required_columns.issubset(set(rm.get_results.columns)) assert len(rm.get_results) > 0 assert isinstance(rm.page_count, int) - assert rm.rent_or_sale == "rent-commercial" + assert rm.is_commercial == True assert isinstance(rm.results_count, int) assert isinstance(rm.results_count_display, int) assert url == rm.url