Validate scraper config (warn on unknown keys, reject bad types)

scraper/tests/test_config.py

@@ -0,0 +1,64 @@
+"""Config validation: warn on unknown keys, reject wrong types early.
+
+Unknown keys only warn because the config schema mirrors the Apify Tweet Scraper
+V2 input (a superset of what this script implements), so a real Apify vars.json
+must still load unchanged.
+"""
+
+import json
+
+import pytest
+
+from tweet_scraper import DEFAULTS, load_config, validate_config
+
+
+def test_validate_config_accepts_a_valid_config():
+    validate_config({"searchTerms": ["cats"], "maxItems": 100, "sort": "Latest"})
+
+
+def test_validate_config_allows_null_for_nullable_fields():
+    validate_config({"author": None, "start": None, "end": None, "tweetLanguage": None})
+
+
+def test_validate_config_allows_bools_and_lists():
+    validate_config({"onlyVideo": True, "fetchReplies": False, "twitterHandles": ["a"]})
+
+
+def test_validate_config_rejects_wrong_type_for_known_key():
+    with pytest.raises(SystemExit):
+        validate_config({"maxItems": "100"})  # int expected, not str
+
+
+def test_validate_config_rejects_non_list_search_terms():
+    with pytest.raises(SystemExit):
+        validate_config({"searchTerms": "cats"})  # list expected
+
+
+def test_validate_config_warns_on_unknown_key_but_does_not_raise(capsys):
+    validate_config({"searchTermz": ["typo"]})  # unknown key -> warn, not error
+    err = capsys.readouterr().err
+    assert "searchTermz" in err
+    assert "unknown" in err.lower()
+
+
+def test_load_config_merges_defaults(tmp_path):
+    p = tmp_path / "vars.json"
+    p.write_text(json.dumps({"searchTerms": ["cats"]}), encoding="utf-8")
+    cfg = load_config(p)
+    assert cfg["searchTerms"] == ["cats"]
+    assert cfg["maxItems"] == DEFAULTS["maxItems"]
+    assert cfg["sort"] == DEFAULTS["sort"]
+
+
+def test_load_config_rejects_non_object_top_level(tmp_path):
+    p = tmp_path / "vars.json"
+    p.write_text(json.dumps(["not", "an", "object"]), encoding="utf-8")
+    with pytest.raises(SystemExit):
+        load_config(p)
+
+
+def test_load_config_rejects_bad_type(tmp_path):
+    p = tmp_path / "vars.json"
+    p.write_text(json.dumps({"maxItems": "lots"}), encoding="utf-8")
+    with pytest.raises(SystemExit):
+        load_config(p)

scraper/tweet_scraper.py

@@ -126,9 +126,54 @@ def tweet_created_at(t: dict[str, Any]) -> datetime | None:
 # ---------- config ----------------------------------------------------------
 
 
+# Accepted types per known config key. `type(None)` marks a nullable field.
+# bool is intentionally allowed for maxItems too (it is an int subclass) — not
+# worth special-casing for a config validator.
+_CONFIG_TYPES: dict[str, tuple[type, ...]] = {
+    "author": (str, type(None)),
+    "customMapFunction": (str, type(None)),
+    "end": (str, type(None)),
+    "fetchReplies": (bool,),
+    "includeSearchTerms": (bool,),
+    "maxItems": (int, float),
+    "onlyImage": (bool,),
+    "onlyQuote": (bool,),
+    "onlyTwitterBlue": (bool,),
+    "onlyVerifiedUsers": (bool,),
+    "onlyVideo": (bool,),
+    "searchTerms": (list,),
+    "sort": (str,),
+    "start": (str, type(None)),
+    "startUrls": (list,),
+    "tweetLanguage": (str, type(None)),
+    "twitterHandles": (list,),
+}
+
+
+def validate_config(cfg: dict[str, Any]) -> None:
+    """Warn on unknown keys and reject known keys with the wrong type.
+
+    Unknown keys only warn: the config schema mirrors the Apify Tweet Scraper V2
+    input (a superset of what this script implements), so an existing Apify
+    vars.json must still load. A type mismatch on a known key is a hard error
+    because it would otherwise crash or misbehave deep in the run.
+    """
+    for key in cfg:
+        if key not in _CONFIG_TYPES:
+            print(f"[warn] unknown config key '{key}' — ignored.", file=sys.stderr)
+    for key, types in _CONFIG_TYPES.items():
+        if key in cfg and not isinstance(cfg[key], types):
+            expected = " or ".join(t.__name__ for t in types)
+            got = type(cfg[key]).__name__
+            sys.exit(f"Config error: '{key}' must be {expected}, got {got}.")
+
+
 def load_config(path: Path) -> dict[str, Any]:
     with path.open("r", encoding="utf-8") as f:
         cfg = json.load(f)
+    if not isinstance(cfg, dict):
+        sys.exit("Config error: the top-level JSON in the config file must be an object.")
+    validate_config(cfg)
     return {**DEFAULTS, **cfg}