From bbd114e4f5d9f396af4ebb3cf33b9040428df9d5 Mon Sep 17 00:00:00 2001 From: Greg Brandt Date: Sat, 14 Feb 2026 16:51:50 -0800 Subject: [PATCH 1/4] Improve robustness and fetch tool - Restart SeleniumBase driver after errors - More robust handling for bytes vs. string seed URLs - Add FetchTool for downloading assets via requests (e.g. images) --- .github/workflows/check.yml | 4 + .github/workflows/release.yml | 4 + .gitignore | 1 + poetry.lock | 2 +- pyproject.toml | 1 + rubbernecker/__main__.py | 2 + rubbernecker/crawl/tool.py | 71 +++++++++++++- rubbernecker/fetch/__init__.py | 5 + rubbernecker/fetch/tool.py | 166 +++++++++++++++++++++++++++++++++ tests/test_fetch_tool.py | 65 +++++++++++++ 10 files changed, 317 insertions(+), 4 deletions(-) create mode 100644 rubbernecker/fetch/__init__.py create mode 100644 rubbernecker/fetch/tool.py create mode 100644 tests/test_fetch_tool.py diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index a93cd08..9224390 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2026 Greg Brandt +# +# SPDX-License-Identifier: Apache-2.0 + name: Check on: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5e7c123..405f3cc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: 2026 Greg Brandt +# +# SPDX-License-Identifier: Apache-2.0 + name: Publish to PyPI on: diff --git a/.gitignore b/.gitignore index 0f33147..70cf86f 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ downloaded_files/ env tags tmp +latest_logs/ diff --git a/poetry.lock b/poetry.lock index b75af0d..6135aac 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2809,4 +2809,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = "^3.12" -content-hash = "05627b1a20efd14c5b342d979ae5754042c7e566cfbf64d125e3e7c19f7b7180" +content-hash = "6aa97f21e8a01a3a104284d2bfdf420ce8e2b93243f67a7c1b6d5d44988f72ab" diff --git a/pyproject.toml b/pyproject.toml index 59d9e52..97bc3b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ python = "^3.12" avrokit = "^0.0.3" beautifulsoup4 = "^4.13.4" seleniumbase = "^4.46.4" +requests = "^2.32.5" [tool.poetry.group.dev.dependencies] pytest = "^9.0.2" diff --git a/rubbernecker/__main__.py b/rubbernecker/__main__.py index 86f258c..c305208 100644 --- a/rubbernecker/__main__.py +++ b/rubbernecker/__main__.py @@ -10,11 +10,13 @@ from .parse import ParseTool from .crawl import CrawlTool from .browser import ProxyTool +from .fetch import FetchTool TOOLS: list[Tool] = [ CrawlTool(), ParseTool(), ProxyTool(), + FetchTool(), ] diff --git a/rubbernecker/crawl/tool.py b/rubbernecker/crawl/tool.py index e578776..57cd31c 100644 --- a/rubbernecker/crawl/tool.py +++ b/rubbernecker/crawl/tool.py @@ -92,6 +92,37 @@ def bloom_filter_key(self, url: str) -> str: parsed_url = urlparse(url.lower()) return f"{parsed_url.netloc}:{parsed_url.path}:{parsed_url.query}" + def is_driver_alive(self, driver) -> bool: + """ + Check if the Selenium driver is still alive. + + :param driver: The Selenium WebDriver instance. + :return: True if the driver is alive, False otherwise. + """ + try: + driver.current_url + return True + except Exception: + return False + + def restart_browser(self, sb, headless: bool, proxy_server: str | None = None): + """ + Restart the Selenium browser. + + :param sb: The SeleniumBase instance. + :param headless: Whether to run in headless mode. + :param proxy_server: Optional proxy server URL. + """ + sb.driver.quit() + sb.__init__( + uc=True, + test=True, + incognito=True, + headless=headless, + proxy=proxy_server, + locale="en-US", + ) + def load_bloom_filter(self, output_url: URL) -> BloomFilter | None: """ Load a Bloom filter from the output URL. @@ -144,6 +175,8 @@ def load_requests( with input_url.with_mode("r") as file: for line in file: url = line.strip() + if isinstance(url, bytes): + url = url.decode("utf-8") if not bloom_filter or not bloom_filter.check( self.bloom_filter_key(url) ): @@ -154,6 +187,8 @@ def load_requests( for line in file: data = json.loads(line) url = data["url"] + if isinstance(url, bytes): + url = url.decode("utf-8") if not bloom_filter or not bloom_filter.check( self.bloom_filter_key(url) ): @@ -164,6 +199,8 @@ def load_requests( for record in reader: if isinstance(record, dict) and "url" in record: url = record["url"] + if isinstance(url, bytes): + url = url.decode("utf-8") if not bloom_filter or not bloom_filter.check( self.bloom_filter_key(url) ): @@ -190,8 +227,11 @@ def crawl( # Load the Bloom filter if needed bloom_filter: BloomFilter | None = None if use_bloom_filter: - bloom_filter = self.load_bloom_filter(base_output_url) - logger.debug("Loaded Bloom filter: %s", bloom_filter) + try: + bloom_filter = self.load_bloom_filter(base_output_url) + logger.debug("Loaded Bloom filter: %s", bloom_filter) + except Exception as e: + logger.warning("Failed to load Bloom filter: %s", e) stats = CrawlToolStats() first_request = True @@ -204,6 +244,7 @@ def crawl( proxy=proxy_server, locale="en-US", ) as sb: + driver = sb.driver # Iterate over the input and output URLs for input_url, output_url in create_url_mapping( base_input_url, base_output_url @@ -230,6 +271,14 @@ def crawl( while True: try: + # Check if browser is still alive + if not self.is_driver_alive(sb.driver): + logger.warning( + "Browser died, restarting for URL: %s", url + ) + first_request = True + self.restart_browser(sb, headless, proxy_server) + logger.info( "Crawling URL: %s (depth=%d, retries=%s)", url, @@ -251,6 +300,9 @@ def crawl( "Page.navigate", {"url": url} ) + # Wait for page to be fully loaded before capturing + sb.wait_for_ready_state_complete() + # Perform load actions if provided if load_actions: for plan in load_actions: @@ -277,9 +329,12 @@ def crawl( time.sleep(sleep_success) # Save the crawled data to the output URL + current_url = sb.get_current_url() + if isinstance(current_url, bytes): + current_url = current_url.decode("utf-8") writer.append( { - "url": sb.get_current_url(), + "url": current_url, "timestamp": timestamp, "body": sb.get_page_source(), } @@ -309,6 +364,16 @@ def crawl( # No crawl actions provided, just break break except Exception as e: + # Check if browser died + if not self.is_driver_alive(sb.driver): + logger.warning( + "Browser died during crawl, restarting: %s", e + ) + first_request = True + self.restart_browser(sb, headless, proxy_server) + # Retry the same URL + continue + # Sleep on error if not in interactive mode if not interactive: time.sleep(sleep_error) diff --git a/rubbernecker/fetch/__init__.py b/rubbernecker/fetch/__init__.py new file mode 100644 index 0000000..6f53eb5 --- /dev/null +++ b/rubbernecker/fetch/__init__.py @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2026 Greg Brandt +# +# SPDX-License-Identifier: Apache-2.0 + +from .tool import FetchTool diff --git a/rubbernecker/fetch/tool.py b/rubbernecker/fetch/tool.py new file mode 100644 index 0000000..0ebb6de --- /dev/null +++ b/rubbernecker/fetch/tool.py @@ -0,0 +1,166 @@ +# SPDX-FileCopyrightText: 2026 Greg Brandt +# +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import logging +import requests +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass +from urllib.parse import urlparse +from avrokit import URL, parse_url +from rubbernecker.crawl.bloomfilter import BloomFilter + +logger = logging.getLogger("fetchtool") + + +@dataclass +class FetchToolStats: + count_input: int = 0 + count_output: int = 0 + count_error: int = 0 + count_skipped: int = 0 + + +class FetchTool: + def name(self) -> str: + return "fetch" + + def configure(self, subparsers: argparse._SubParsersAction) -> None: + parser = subparsers.add_parser(self.name(), help="Fetch assets from URLs") + parser.add_argument( + "input_url", + help="URL to the text file containing URLs to fetch", + ) + parser.add_argument( + "output_url", + help="URL to the output directory", + ) + parser.add_argument( + "--parallelism", + type=int, + default=1, + help="Number of concurrent fetches (default: 1)", + ) + parser.add_argument( + "--force", + action="store_true", + help="Redownload even if file already exists (default: skip existing)", + ) + + def load_bloom_filter(self, output_url: URL) -> BloomFilter | None: + if output_url.exists(): + bloom_filter = BloomFilter() + count = 0 + base_path = output_url.url.rstrip("/") + "/" + for url in output_url.expand(): + if url.url.startswith(base_path): + path = url.url[len(base_path) :] + else: + parsed = urlparse(url.url) + path = parsed.path.lstrip("/") + if path: + bloom_filter.add(path) + count += 1 + if count > 0: + logger.debug( + "Loaded %d paths into Bloom filter from %s", count, output_url + ) + return bloom_filter + return None + + def fetch_url( + self, + url: str, + output_base_url: URL, + force: bool, + bloom_filter: BloomFilter | None, + ) -> tuple[bool, bool]: + try: + response = requests.get(url, timeout=30) + response.raise_for_status() + content = response.content + + parsed = urlparse(url) + path = parsed.path.lstrip("/") + + if not path: + logger.warning("URL %s has no path, skipping", url) + return False, False + + if not force and bloom_filter and bloom_filter.check(path): + logger.debug("File already exists (bloom filter), skipping: %s", path) + return True, True + + output_url = parse_url(output_base_url.url + "/" + path) + with output_url.with_mode("wb") as f: + f.write(content) + + if bloom_filter: + bloom_filter.add(path) + + logger.debug("Fetched %s -> %s", url, output_url) + return True, False + + except Exception as e: + logger.error("Error fetching %s: %s", url, e) + return False, False + + def fetch( + self, input_url: URL, output_url: URL, parallelism: int, force: bool = False + ) -> FetchToolStats: + stats = FetchToolStats() + + bloom_filter: BloomFilter | None = None + if not force: + try: + bloom_filter = self.load_bloom_filter(output_url) + except Exception as e: + logger.warning("Failed to load bloom filter: %s", e) + + with input_url.with_mode("r") as f: + urls = [line.strip() for line in f if line.strip()] + + logger.info("Fetching %d URLs with parallelism=%d", len(urls), parallelism) + + if bloom_filter: + logger.info("Bloom filter loaded, will skip existing files") + + with ThreadPoolExecutor(max_workers=parallelism) as executor: + futures = { + executor.submit( + self.fetch_url, url, output_url, force, bloom_filter + ): url + for url in urls + } + + for future in as_completed(futures): + url = futures[future] + stats.count_input += 1 + try: + success, skipped = future.result() + if success: + if skipped: + stats.count_skipped += 1 + else: + stats.count_output += 1 + else: + stats.count_error += 1 + except Exception as e: + logger.error("Error processing %s: %s", url, e) + stats.count_error += 1 + + if logging.DEBUG != logger.getEffectiveLevel(): + if stats.count_output > 0 and stats.count_output % 100 == 0: + logger.info("%s", stats) + + return stats + + def run(self, args: argparse.Namespace) -> None: + stats = self.fetch( + parse_url(args.input_url), + parse_url(args.output_url), + args.parallelism, + args.force, + ) + logger.info("%s (done)", stats) diff --git a/tests/test_fetch_tool.py b/tests/test_fetch_tool.py new file mode 100644 index 0000000..4b87d89 --- /dev/null +++ b/tests/test_fetch_tool.py @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: 2026 Greg Brandt +# +# SPDX-License-Identifier: Apache-2.0 + +import os +import tempfile +import pytest +from avrokit import parse_url +from rubbernecker.fetch import FetchTool + + +@pytest.mark.integration +def test_fetch_tool(): + with tempfile.TemporaryDirectory() as tmpdir: + input_url = parse_url(os.path.join(tmpdir, "urls.txt")) + output_url = parse_url(os.path.join(tmpdir, "output")) + + with input_url.with_mode("w") as f: + f.write("https://httpbin.org/image/png\n") + + tool = FetchTool() + stats = tool.fetch(input_url, output_url, parallelism=1) + + assert stats.count_input == 1 + assert stats.count_output == 1 + assert stats.count_error == 0 + + image_path = os.path.join(tmpdir, "output", "image", "png") + assert os.path.exists(image_path) + + with open(image_path, "rb") as f: + content = f.read() + assert len(content) > 0 + assert content[:4] == b"\x89PNG" + + +@pytest.mark.integration +def test_fetch_tool_parallel(): + with tempfile.TemporaryDirectory() as tmpdir: + input_url = parse_url(os.path.join(tmpdir, "urls.txt")) + output_url = parse_url(os.path.join(tmpdir, "output")) + + with input_url.with_mode("w") as f: + f.write("https://httpbin.org/image/png\n") + f.write("https://httpbin.org/json\n") + + tool = FetchTool() + stats = tool.fetch(input_url, output_url, parallelism=2) + + assert stats.count_input == 2 + assert stats.count_output == 2 + assert stats.count_error == 0 + + png_path = os.path.join(tmpdir, "output", "image", "png") + json_path = os.path.join(tmpdir, "output", "json") + + assert os.path.exists(png_path) + assert os.path.exists(json_path) + + with open(png_path, "rb") as f: + assert f.read()[:4] == b"\x89PNG" + + with open(json_path, "r") as f: + content = f.read() + assert '"slideshow"' in content From a24628ee82ec76aabb37c67319795eedf7cdd764 Mon Sep 17 00:00:00 2001 From: Greg Brandt Date: Sat, 14 Feb 2026 19:57:49 -0800 Subject: [PATCH 2/4] Fix linting errors --- .flake8 | 1 + rubbernecker/crawl/tool.py | 1 - rubbernecker/fetch/__init__.py | 2 ++ 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.flake8 b/.flake8 index 6dd9b15..3a5b36a 100644 --- a/.flake8 +++ b/.flake8 @@ -1,2 +1,3 @@ [flake8] ignore = E501,E203,W503,E704 +exclude = .git,__pycache__,docs,build,dist,.venv,.mypy_cache,.pytest_cache diff --git a/rubbernecker/crawl/tool.py b/rubbernecker/crawl/tool.py index 57cd31c..1cdd845 100644 --- a/rubbernecker/crawl/tool.py +++ b/rubbernecker/crawl/tool.py @@ -244,7 +244,6 @@ def crawl( proxy=proxy_server, locale="en-US", ) as sb: - driver = sb.driver # Iterate over the input and output URLs for input_url, output_url in create_url_mapping( base_input_url, base_output_url diff --git a/rubbernecker/fetch/__init__.py b/rubbernecker/fetch/__init__.py index 6f53eb5..959f3ed 100644 --- a/rubbernecker/fetch/__init__.py +++ b/rubbernecker/fetch/__init__.py @@ -3,3 +3,5 @@ # SPDX-License-Identifier: Apache-2.0 from .tool import FetchTool + +__all__ = ["FetchTool"] From fd1921f0333637c0ca988249a650f5e6bf69365b Mon Sep 17 00:00:00 2001 From: Greg Brandt Date: Sat, 14 Feb 2026 19:58:38 -0800 Subject: [PATCH 3/4] Upgrade avrokit --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 6135aac..186cc43 100644 --- a/poetry.lock +++ b/poetry.lock @@ -42,14 +42,14 @@ zstandard = ["zstandard"] [[package]] name = "avrokit" -version = "0.0.3" +version = "0.0.4" description = "Python utilities for working with Avro data files" optional = false python-versions = "<4.0,>=3.12" groups = ["main"] files = [ - {file = "avrokit-0.0.3-py3-none-any.whl", hash = "sha256:44a43e8edc33befe1329417324b924d8ff8bf84ffca57f916c19869ba71af230"}, - {file = "avrokit-0.0.3.tar.gz", hash = "sha256:805109002ecf4f04b39fab44658042ca6926df9660dd49248ca068bead2b1964"}, + {file = "avrokit-0.0.4-py3-none-any.whl", hash = "sha256:41f8d3d12b9e5ff83a2b3b9f0e351e5eeb6ad7442645e8a08e6dfe21a6f1e7de"}, + {file = "avrokit-0.0.4.tar.gz", hash = "sha256:eafbaf482148c11c814bab8dead7b0f5e822e116bbe1cdad1315f64d0e298b56"}, ] [package.dependencies] @@ -2809,4 +2809,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = "^3.12" -content-hash = "6aa97f21e8a01a3a104284d2bfdf420ce8e2b93243f67a7c1b6d5d44988f72ab" +content-hash = "2a3c99bb37f994c2ab7b849fd92c25c6afe62355934aaab1c58d42c62e0b2a95" diff --git a/pyproject.toml b/pyproject.toml index 97bc3b5..c8981fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ classifiers = [ [tool.poetry.dependencies] python = "^3.12" -avrokit = "^0.0.3" +avrokit = "^0.0.4" beautifulsoup4 = "^4.13.4" seleniumbase = "^4.46.4" requests = "^2.32.5" From 005b6a579d0c463f3e03769a892258dac0e3960d Mon Sep 17 00:00:00 2001 From: Greg Brandt Date: Sat, 14 Feb 2026 20:00:20 -0800 Subject: [PATCH 4/4] Bump version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c8981fd..ca54a41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ [tool.poetry] name = "rubbernecker" -version = "0.0.3" +version = "0.0.4" description = "Web scraping engine" authors = ["Greg Brandt "] license = "Apache-2.0"