diff --git a/.gitignore b/.gitignore index eac93d3e..6a28f7a5 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,9 @@ __pycache__/ *.pyc result.csv output.csv -playwright/.cache/ \ No newline at end of file +playwright/.cache/ +venv/ +ngrok.exe +*.log +page_html.txt +panel_html.txt \ No newline at end of file diff --git a/README.md b/README.md index 17da5b8d..2c46012b 100644 --- a/README.md +++ b/README.md @@ -1,103 +1,30 @@ -# Google-Maps-Scrapper -This Python script utilizes the Playwright library to perform web scraping and data extraction from Google Maps. It is particularly designed for obtaining information about businesses, including their name, address, website, phone number, reviews, and more. +# Google Maps Scraper -## Read Prerequistes -Latest python was not used and is not suggested +## Descricao +Web scraper para extracao de dados do Google Maps. Ideal para coleta de informacoes de negocios, enderecos e contatos. -
-To do a custom web scraping project you can find me on Upwork or on Linkedin

+## Funcionalidades +- Extracao de dados de negocios do Google Maps +- Coleta de enderecos, telefones e websites +- Exportacao para CSV/JSON +- Coleta em lote para multiple locais - -medium - - - -linkedin - - - -## Table of Contents -- [Prerequisites](#prerequisites) -- [Multiple Branches](#multiple-branches) -- [Key Features](#key-features) -- [Installation](#installation) -- [Usage](#usage) -- [Example](#example) -- [Notes](#notes) -- [Video Example](#video-example) - -## Prerequisites -- Python 3.8 or 3.9 (Python 3.10+ may not be compatible with some dependencies) -- Google Chrome or Chromium browser installed (for Playwright) - -## Multiple Branches -The repo currently has 3 branches -- Main -- Latest Libraries (The one that works with latest libraries, can cause issues. Prefer Main) -- Linux ( Linux Support if main branch does not work correctly) - - -## Key Features -- Data Scraping: The script scrapes data from Google Maps listings, extracting valuable information about businesses, such as their name, address, website, and contact details. - -- Review Analysis: It extracts review counts and average ratings, providing insights into businesses' online reputation. - -- Business Type Detection: The script identifies whether a business offers in-store shopping, in-store pickup, or delivery services. - -- Operating Hours: It extracts information about the business's operating hours. - -- Introduction Extraction: The script also scrapes introductory information about the businesses when available. - -- Data Cleansing: It cleanses and organizes the scraped data, removing redundant or unnecessary columns. - -- CSV Export: The cleaned data is exported to a CSV file for further analysis or integration with other tools. - -## Installation - -1. Clone this repository: - ```bash - git clone https://github.com/zohaibbashir/Google-Maps-Scrapper.git - cd google-maps-scraper - ``` -2. Install Python dependencies: - ```bash - pip install -r requirements.txt - ``` -3. Install Playwright browsers: - ```bash - playwright install - ``` - -## Usage - -Run the script with your desired search term and number of results: +## Tecnologias +- Python +- BeautifulSoup/Selenium +- Pandas +## Instalacao ```bash -python main.py -s "Turkish Restaurants in Toronto Canada" -t 20 +git clone https://github.com/Wsanbey/Google-Maps-Scrapper.git +cd Google-Maps-Scrapper +pip install -r requirements.txt ``` -- `-s` or `--search`: Search query for Google Maps (default: "turkish stores in toronto Canada") -- `-t` or `--total`: Number of results to scrape (default: 1) -- `-o` or `--output`: Output CSV file path (default: result.csv) -- `--append`: Append results to the output file instead of overwriting (default: off) - -## Example - -Append new results to an existing CSV file: -```bash -python main.py -s "Turkish Restaurants in Toronto Canada" -t 20 -o toronto_turkish_restaurants.csv --append +## Uso +```python +python scraper.py --location "Recife, PE" --query "restaurantes" ``` -The script will launch a browser, perform the search, and start scraping information. Progress will be displayed in the terminal, and results will be saved to the specified CSV file. If `--append` is used, new results will be added to the end of the file without removing previous data. - -## Notes -- The script opens a visible browser window (not headless) for scraping. -- Google Maps DOM may change, which can break the script. If you encounter issues, update the XPaths in `main.py`. -- Avoid running too many scrapes in a short period to prevent being blocked by Google. - -## Video Example - -https://www.linkedin.com/posts/zohaibbashir_python-data-webscraping-activity-7093920891411062784-flEQ - -## License +## Licenca MIT diff --git a/api.py b/api.py new file mode 100644 index 00000000..e9e41ed4 --- /dev/null +++ b/api.py @@ -0,0 +1,62 @@ +import os +# Redireciona a pasta temporária para a pasta do projeto, evitando erros do Windows na pasta Temp padrão +project_path = os.path.abspath(os.path.dirname(__file__)) +os.environ['TEMP'] = project_path +os.environ['TMP'] = project_path + +from fastapi import FastAPI, HTTPException +from pydantic import BaseModel, Field +from typing import List +from main import scrape_places, Place +from dataclasses import asdict + +app = FastAPI( + title="Google Maps Scraper API", + description="API local para extrair dados do Google Maps usando Playwright", + version="1.0.0" +) + +class ScrapeRequest(BaseModel): + search: str = Field(..., description="Termo de busca no Google Maps") + total: int = Field(10, description="Quantidade de resultados a extrair") + +@app.post("/scrape") +def run_scraper(request: ScrapeRequest): + try: + search_query = request.search.strip() + if not search_query: + raise HTTPException(status_code=400, detail="O termo de busca não pode ser vazio.") + + places = scrape_places(search_query, request.total) + # Converte as dataclasses para dicionários para que o FastAPI serialize como JSON + return [asdict(place) for place in places] + except Exception as e: + raise HTTPException(status_code=500, detail=f"Erro durante o scraping: {str(e)}") + +if __name__ == "__main__": + import uvicorn + import os + from pyngrok import ngrok, conf + + # Define o caminho do binário do Ngrok na pasta do projeto para evitar erros de permissão na pasta Temp do Windows + project_dir = os.path.abspath(os.path.dirname(__file__)) + conf.get_default().ngrok_path = os.path.join(project_dir, "ngrok.exe") + + # Tenta abrir o túnel do Ngrok se houver token configurado + try: + port = 8000 + public_url = ngrok.connect(port).public_url + print("\n========================================================", flush=True) + print(" NGROK TÚNEL ATIVADO COM SUCESSO!", flush=True) + print(f" API Pública no endereço: {public_url}", flush=True) + print(f" Exemplo de requisição: POST {public_url}/scrape", flush=True) + print("========================================================\n", flush=True) + except Exception as e: + print("\n[Aviso] Não foi possível iniciar o túnel Ngrok.", flush=True) + print("Certifique-se de ter configurado o token rodando o 'configurar_token.bat'", flush=True) + print(f"Erro detalhado: {e}\n", flush=True) + print("Iniciando a API apenas no modo local (localhost)...", flush=True) + print("========================================================\n", flush=True) + + # Roda o servidor localmente na porta 8000 + uvicorn.run(app, host="127.0.0.1", port=8000) diff --git a/configurar_token.bat b/configurar_token.bat new file mode 100644 index 00000000..24e4d9c6 --- /dev/null +++ b/configurar_token.bat @@ -0,0 +1,32 @@ +@echo off +chcp 65001 > nul +title Configurar Token do Ngrok + +cls +echo ======================================================== +echo CONFIGURAR TOKEN DO NGROK +echo ======================================================== +echo. +echo Para usar o Ngrok, voce precisa de um Authtoken gratuito. +echo Se voce ainda nao tem um, crie sua conta e copie o token em: +echo https://dashboard.ngrok.com/get-started/your-authtoken +echo. +set /p token="Cole o seu Authtoken do Ngrok aqui: " + +if "%token%"=="" ( + echo. + echo [Erro] Token invalido ou vazio! + pause + exit /b +) + +echo. +echo Configurando o token usando o executavel local do Ngrok... +.\ngrok.exe config add-authtoken "%token%" +echo. +echo ======================================================== +echo Token configurado com sucesso! +echo Agora voce ja pode usar a API com acesso publico. +echo ======================================================== +echo. +pause diff --git a/iniciar.bat b/iniciar.bat new file mode 100644 index 00000000..caad5a57 --- /dev/null +++ b/iniciar.bat @@ -0,0 +1,43 @@ +@echo off +:: Configura o console para UTF-8 de modo a exibir acentos corretamente no Windows +chcp 65001 > nul +title Google Maps Scraper - Inicializador + +cls +echo ======================================================== +echo GOOGLE MAPS SCRAPER - INICIALIZADOR +echo ======================================================== +echo. +echo Este script irá rodar o buscador no ambiente virtual (venv). +echo Os resultados serão salvos por padrão no arquivo 'result.csv'. +echo. +echo ======================================================== +echo. + +set /p busca="Digite o termo de busca (Ex: restaurantes em Pinheiros SP): " +if "%busca%"=="" ( + echo. + echo [Erro] Você precisa digitar um termo de busca! + pause + exit /b +) + +set /p total="Digite a quantidade total de locais a extrair (Padrão: 10): " +if "%total%"=="" set total=10 + +echo. +echo ======================================================== +echo Iniciando a busca para: "%busca%" +echo Quantidade máxima: %total% +echo ======================================================== +echo. + +:: Executa o script python no venv +.\venv\Scripts\python.exe main.py -s "%busca%" -t %total% + +echo. +echo ======================================================== +echo Busca Concluída! Os dados foram salvos no arquivo 'result.csv'. +echo ======================================================== +echo. +pause diff --git a/iniciar_api.bat b/iniciar_api.bat new file mode 100644 index 00000000..feeb7777 --- /dev/null +++ b/iniciar_api.bat @@ -0,0 +1,21 @@ +@echo off +chcp 65001 > nul +title Google Maps Scraper API - Servidor + +cls +echo ======================================================== +echo GOOGLE MAPS SCRAPER - SERVIDOR API +echo ======================================================== +echo. +echo Iniciando o servidor local da API... +echo O endereco de acesso (POST) sera: http://127.0.0.1:8000/scrape +echo. +echo Exemplo de corpo da requisicao (JSON): +echo { "search": "restaurantes recife", "total": 5 } +echo. +echo Para fechar a API, basta fechar esta janela ou pressionar Ctrl+C. +echo ======================================================== +echo. + +.\venv\Scripts\python.exe api.py +pause diff --git a/main.py b/main.py index 36335cd8..1d5c839c 100644 --- a/main.py +++ b/main.py @@ -7,6 +7,7 @@ import platform import time import os +import re @dataclass class Place: @@ -21,6 +22,7 @@ class Place: store_delivery: str = "No" place_type: str = "" opens_at: str = "" + closes_at: str = "" introduction: str = "" def setup_logging(): @@ -65,7 +67,7 @@ def extract_place(page: Page) -> Place: reviews_count_raw = extract_text(page, reviews_count_xpath) if reviews_count_raw: try: - temp = reviews_count_raw.replace('\xa0', '').replace('(','').replace(')','').replace(',','') + temp = re.sub(r'\D', '', reviews_count_raw) place.reviews_count = int(temp) except Exception as e: logging.warning(f"Failed to parse reviews count: {e}") @@ -90,22 +92,68 @@ def extract_place(page: Page) -> Place: place.in_store_pickup = "Yes" if 'delivery' in check: place.store_delivery = "Yes" - # Opens At - opens_at_raw = extract_text(page, opens_at_xpath) - if opens_at_raw: - opens = opens_at_raw.split('⋅') - if len(opens) > 1: - place.opens_at = opens[1].replace("\u202f","") - else: - place.opens_at = opens_at_raw.replace("\u202f","") - else: - opens_at2_raw = extract_text(page, opens_at_xpath2) - if opens_at2_raw: - opens = opens_at2_raw.split('⋅') - if len(opens) > 1: - place.opens_at = opens[1].replace("\u202f","") - else: - place.opens_at = opens_at2_raw.replace("\u202f","") + # Opens At & Closes At + opens_at_btn = page.locator('//div[@class="MkV9"]') + if opens_at_btn.count() == 0: + opens_at_btn = page.locator('//button[contains(@data-item-id, "oh")]') + if opens_at_btn.count() > 0: + try: + # Tenta clicar no botão de horários para abrir a tabela semanal + opens_at_btn.first.click() + page.wait_for_timeout(1000) + + # Procura a tabela de horários no DOM + table_locator = page.locator('//table') + table_text = "" + for i in range(table_locator.count()): + text = table_locator.nth(i).inner_text() or "" + # A tabela de horários deve conter dias da semana em português ou inglês + if any(day in text.lower() for day in ["segunda", "terça", "quarta", "quinta", "sexta", "sábado", "domingo", "monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday"]): + table_text = text + break + if table_text: + import datetime + weekday = datetime.datetime.now().weekday() + days_pt = ["segunda-feira", "terça-feira", "quarta-feira", "quinta-feira", "sexta-feira", "sábado", "domingo"] + days_en = ["monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday"] + + target_day_pt = days_pt[weekday] + target_day_en = days_en[weekday] + + lines = [line.strip() for line in table_text.split('\n') if line.strip()] + target_idx = -1 + for idx, line in enumerate(lines): + line_lower = line.lower() + if target_day_pt in line_lower or target_day_en in line_lower: + target_idx = idx + break + + if target_idx != -1 and target_idx + 1 < len(lines): + hours_line = lines[target_idx + 1] + times = re.findall(r'\d{1,2}:\d{2}', hours_line) + if len(times) >= 2: + place.opens_at = times[0] + place.closes_at = times[-1] + elif len(times) == 1: + place.opens_at = times[0] + place.closes_at = times[0] + elif "24 horas" in hours_line.lower() or "24 hours" in hours_line.lower(): + place.opens_at = "00:00" + place.closes_at = "24:00" + elif "fechado" in hours_line.lower() or "closed" in hours_line.lower() or "fechado" in lines[target_idx].lower() or "closed" in lines[target_idx].lower(): + place.opens_at = "Fechado" + place.closes_at = "Fechado" + except Exception as e: + logging.warning(f"Failed to extract hours via clicking table: {e}") + + # Caso não tenha conseguido obter pela tabela (ou não tenha horário cadastrado), usa o fallback do texto visível + if not place.opens_at: + opens_at_raw = extract_text(page, opens_at_xpath) or extract_text(page, opens_at_xpath2) + if opens_at_raw: + place.opens_at = opens_at_raw.replace("\u202f", "").strip() + times = re.findall(r'\d{1,2}:\d{2}', opens_at_raw) + if len(times) >= 1: + place.closes_at = times[-1] return place def scrape_places(search_for: str, total: int) -> List[Place]: @@ -159,9 +207,11 @@ def scrape_places(search_for: str, total: int) -> List[Place]: def save_places_to_csv(places: List[Place], output_path: str = "result.csv", append: bool = False): df = pd.DataFrame([asdict(place) for place in places]) if not df.empty: - for column in df.columns: - if df[column].nunique() == 1: - df.drop(column, axis=1, inplace=True) + # Keep all columns, do not drop columns with only 1 unique value + # for column in df.columns: + # # if df[column].nunique() == 1: + # # df.drop(column, axis=1, inplace=True) + # pass file_exists = os.path.isfile(output_path) mode = "a" if append else "w" header = not (append and file_exists) diff --git a/result.csv b/result.csv deleted file mode 100644 index 82d5976e..00000000 --- a/result.csv +++ /dev/null @@ -1 +0,0 @@ -name,address,website,phone_number,reviews_count,reviews_average,place_type,opens_at \ No newline at end of file