diff --git a/scraper/tests/test_config.py b/scraper/tests/test_config.py new file mode 100644 index 0000000..acc3aff --- /dev/null +++ b/scraper/tests/test_config.py @@ -0,0 +1,64 @@ +"""Config validation: warn on unknown keys, reject wrong types early. + +Unknown keys only warn because the config schema mirrors the Apify Tweet Scraper +V2 input (a superset of what this script implements), so a real Apify vars.json +must still load unchanged. +""" + +import json + +import pytest + +from tweet_scraper import DEFAULTS, load_config, validate_config + + +def test_validate_config_accepts_a_valid_config(): + validate_config({"searchTerms": ["cats"], "maxItems": 100, "sort": "Latest"}) + + +def test_validate_config_allows_null_for_nullable_fields(): + validate_config({"author": None, "start": None, "end": None, "tweetLanguage": None}) + + +def test_validate_config_allows_bools_and_lists(): + validate_config({"onlyVideo": True, "fetchReplies": False, "twitterHandles": ["a"]}) + + +def test_validate_config_rejects_wrong_type_for_known_key(): + with pytest.raises(SystemExit): + validate_config({"maxItems": "100"}) # int expected, not str + + +def test_validate_config_rejects_non_list_search_terms(): + with pytest.raises(SystemExit): + validate_config({"searchTerms": "cats"}) # list expected + + +def test_validate_config_warns_on_unknown_key_but_does_not_raise(capsys): + validate_config({"searchTermz": ["typo"]}) # unknown key -> warn, not error + err = capsys.readouterr().err + assert "searchTermz" in err + assert "unknown" in err.lower() + + +def test_load_config_merges_defaults(tmp_path): + p = tmp_path / "vars.json" + p.write_text(json.dumps({"searchTerms": ["cats"]}), encoding="utf-8") + cfg = load_config(p) + assert cfg["searchTerms"] == ["cats"] + assert cfg["maxItems"] == DEFAULTS["maxItems"] + assert cfg["sort"] == DEFAULTS["sort"] + + +def test_load_config_rejects_non_object_top_level(tmp_path): + p = tmp_path / "vars.json" + p.write_text(json.dumps(["not", "an", "object"]), encoding="utf-8") + with pytest.raises(SystemExit): + load_config(p) + + +def test_load_config_rejects_bad_type(tmp_path): + p = tmp_path / "vars.json" + p.write_text(json.dumps({"maxItems": "lots"}), encoding="utf-8") + with pytest.raises(SystemExit): + load_config(p) diff --git a/scraper/tweet_scraper.py b/scraper/tweet_scraper.py index d570b4d..7ed26b2 100644 --- a/scraper/tweet_scraper.py +++ b/scraper/tweet_scraper.py @@ -126,9 +126,54 @@ def tweet_created_at(t: dict[str, Any]) -> datetime | None: # ---------- config ---------------------------------------------------------- +# Accepted types per known config key. `type(None)` marks a nullable field. +# bool is intentionally allowed for maxItems too (it is an int subclass) — not +# worth special-casing for a config validator. +_CONFIG_TYPES: dict[str, tuple[type, ...]] = { + "author": (str, type(None)), + "customMapFunction": (str, type(None)), + "end": (str, type(None)), + "fetchReplies": (bool,), + "includeSearchTerms": (bool,), + "maxItems": (int, float), + "onlyImage": (bool,), + "onlyQuote": (bool,), + "onlyTwitterBlue": (bool,), + "onlyVerifiedUsers": (bool,), + "onlyVideo": (bool,), + "searchTerms": (list,), + "sort": (str,), + "start": (str, type(None)), + "startUrls": (list,), + "tweetLanguage": (str, type(None)), + "twitterHandles": (list,), +} + + +def validate_config(cfg: dict[str, Any]) -> None: + """Warn on unknown keys and reject known keys with the wrong type. + + Unknown keys only warn: the config schema mirrors the Apify Tweet Scraper V2 + input (a superset of what this script implements), so an existing Apify + vars.json must still load. A type mismatch on a known key is a hard error + because it would otherwise crash or misbehave deep in the run. + """ + for key in cfg: + if key not in _CONFIG_TYPES: + print(f"[warn] unknown config key '{key}' — ignored.", file=sys.stderr) + for key, types in _CONFIG_TYPES.items(): + if key in cfg and not isinstance(cfg[key], types): + expected = " or ".join(t.__name__ for t in types) + got = type(cfg[key]).__name__ + sys.exit(f"Config error: '{key}' must be {expected}, got {got}.") + + def load_config(path: Path) -> dict[str, Any]: with path.open("r", encoding="utf-8") as f: cfg = json.load(f) + if not isinstance(cfg, dict): + sys.exit("Config error: the top-level JSON in the config file must be an object.") + validate_config(cfg) return {**DEFAULTS, **cfg}