From 88294ad68bf310745752b395a55dc82b96ea39a1 Mon Sep 17 00:00:00 2001 From: Christian-Bermejo Date: Thu, 18 Jun 2026 09:47:24 +0800 Subject: [PATCH 1/2] Replace stderr prints with the logging module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Progress/warning output went through print(..., file=sys.stderr). Switch to a module logger configured (INFO -> stderr) in main(), so callers/importers control verbosity and handlers. Dry-run output stays on stdout via print() — it is the program's actual result, not progress chatter. Updates the unknown-key warn test from capsys to caplog. 59 tests green. --- scraper/tests/test_config.py | 10 +++++----- scraper/tweet_scraper.py | 34 ++++++++++++++++------------------ 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/scraper/tests/test_config.py b/scraper/tests/test_config.py index acc3aff..030134e 100644 --- a/scraper/tests/test_config.py +++ b/scraper/tests/test_config.py @@ -34,11 +34,11 @@ def test_validate_config_rejects_non_list_search_terms(): validate_config({"searchTerms": "cats"}) # list expected -def test_validate_config_warns_on_unknown_key_but_does_not_raise(capsys): - validate_config({"searchTermz": ["typo"]}) # unknown key -> warn, not error - err = capsys.readouterr().err - assert "searchTermz" in err - assert "unknown" in err.lower() +def test_validate_config_warns_on_unknown_key_but_does_not_raise(caplog): + with caplog.at_level("WARNING"): + validate_config({"searchTermz": ["typo"]}) # unknown key -> warn, not error + assert "searchTermz" in caplog.text + assert "unknown" in caplog.text.lower() def test_load_config_merges_defaults(tmp_path): diff --git a/scraper/tweet_scraper.py b/scraper/tweet_scraper.py index 0cd5aca..ab18dc5 100644 --- a/scraper/tweet_scraper.py +++ b/scraper/tweet_scraper.py @@ -52,6 +52,7 @@ import argparse import csv import json +import logging import os import sys import time @@ -62,6 +63,8 @@ import requests +log = logging.getLogger("tweet_scraper") + API_BASE = "https://api.twitterapi.io" ADVANCED_SEARCH_URL = f"{API_BASE}/twitter/tweet/advanced_search" USER_LAST_TWEETS_URL = f"{API_BASE}/twitter/user/last_tweets" @@ -160,7 +163,7 @@ def validate_config(cfg: dict[str, Any]) -> None: """ for key in cfg: if key not in _CONFIG_TYPES: - print(f"[warn] unknown config key '{key}' — ignored.", file=sys.stderr) + log.warning("unknown config key '%s' — ignored.", key) for key, types in _CONFIG_TYPES.items(): if key in cfg and not isinstance(cfg[key], types): expected = " or ".join(t.__name__ for t in types) @@ -527,6 +530,10 @@ def main() -> int: ) args = parser.parse_args() + # Progress/warnings go to stderr via logging; the dry-run report stays on + # stdout (it is the program's actual output, not progress chatter). + logging.basicConfig(level=logging.INFO, format="%(message)s", stream=sys.stderr) + if not args.config.exists(): sys.exit(f"Config not found: {args.config}") @@ -536,10 +543,7 @@ def main() -> int: until_dt = parse_config_date(cfg.get("end"), end_of_day=True) if cfg.get("customMapFunction"): - print( - "[warn] customMapFunction is a JS Apify feature — ignored in this script.", - file=sys.stderr, - ) + log.warning("customMapFunction is a JS Apify feature — ignored in this script.") if since_dt and until_dt and since_dt > until_dt: sys.exit(f"Invalid range: start ({cfg['start']}) is after end ({cfg['end']}).") @@ -583,7 +587,7 @@ def main() -> int: writer.writeheader() for term, q in search_queries: - print(f"[search] {q}", file=sys.stderr) + log.info("[search] %s", q) count = _stream_query( writer, seen, @@ -593,10 +597,10 @@ def main() -> int: reply_targets=reply_targets, ) total += count - print(f" -> {count} new tweets (total: {total})", file=sys.stderr) + log.info(" -> %d new tweets (total: %d)", count, total) for h in sorted(handles): - print(f"[user ] @{h}", file=sys.stderr) + log.info("[user ] @%s", h) count = _stream_query( writer, seen, @@ -606,23 +610,17 @@ def main() -> int: reply_targets=reply_targets, ) total += count - print(f" -> {count} new tweets (total: {total})", file=sys.stderr) + log.info(" -> %d new tweets (total: %d)", count, total) if reply_targets: - print( - "[replies] fetching replies for tweets with replyCount > 0...", - file=sys.stderr, - ) + log.info("[replies] fetching replies for tweets with replyCount > 0...") for tid in reply_targets: written = _stream_replies(writer, seen, tid, key, max_items) total += written if written: - print( - f" [replies] tweet {tid} -> {written} replies (total: {total})", - file=sys.stderr, - ) + log.info(" [replies] tweet %s -> %d replies (total: %d)", tid, written, total) - print(f"\nWrote {total} unique tweets -> {args.out}", file=sys.stderr) + log.info("Wrote %d unique tweets -> %s", total, args.out) return 0 From a4b0826a6280b33dd4f5f9bfc62c3835fb933143 Mon Sep 17 00:00:00 2001 From: Christian-Bermejo Date: Thu, 18 Jun 2026 09:52:13 +0800 Subject: [PATCH 2/2] Add [project] table + tweet-scraper console entry point pyproject.toml was tooling-config only, with a comment deferring packaging. Add a [project] table (name/version/deps/requires-python), a setuptools build-system, and a console_scripts entry point so 'pip install -e .' exposes a 'tweet-scraper' command. Verified the CLI installs and the entry point runs --dry-run. Validation loop still green (59 tests). --- scraper/pyproject.toml | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/scraper/pyproject.toml b/scraper/pyproject.toml index 8d56d5a..e9a649c 100644 --- a/scraper/pyproject.toml +++ b/scraper/pyproject.toml @@ -1,6 +1,20 @@ -# Tooling configuration ONLY — this is intentionally NOT a packaging manifest. -# There is no [project] table yet; turning this repo into an installable -# package (entry points, versioning) is deferred — see README "Roadmap". +[build-system] +requires = ["setuptools>=61"] +build-backend = "setuptools.build_meta" + +[project] +name = "tweet-scraper" +version = "0.1.0" +description = "Pay-per-use X (Twitter) tweet scraper via twitterapi.io" +requires-python = ">=3.11" +license = { text = "MIT" } +dependencies = ["requests>=2.31.0"] + +[project.scripts] +tweet-scraper = "tweet_scraper:main" + +[tool.setuptools] +py-modules = ["tweet_scraper"] [tool.ruff] line-length = 100