thread_to_csv.py

"""Flatten a Reddit thread's nested comment tree into one CSV row per comment.

    export CHOCODATA_API_KEY="your_key"
    python reddit_post_scraper_api_codes/thread_to_csv.py \
        reddit_post_scraper_api_codes/urls.txt thread.csv

Reads a file of post URLs (one per line, `#` comments allowed), fetches each
thread, and writes a flat CSV: post_id, comment_id, parent_id, depth, score,
author, created, permalink, body. Nesting survives the flattening because every
row keeps its `parent_id` and `depth`.

One API request per post per sort. A post that fails is reported and the run
carries on.
"""
from __future__ import annotations

import csv
import os
import sys

import requests

ENDPOINT = "https://api.chocodata.com/api/v1/reddit/post"
TIMEOUT = 90
COLUMNS = ["post_id", "post_title", "comment_id", "parent_id", "depth", "score",
           "author", "created", "permalink", "body"]

# Comment bodies Reddit fills in for content that is no longer there. They are
# real strings, not nulls, so anything doing text analysis has to drop them.
PLACEHOLDERS = {"Comment removed by moderator", "Comment deleted by user"}


def walk(comments: list[dict], depth: int = 0):
    for c in comments:
        yield depth, c
        yield from walk(c.get("replies", []), depth + 1)


def fetch(url: str, sort: str = "top") -> dict | None:
    key = os.environ.get("CHOCODATA_API_KEY", "your_key")
    r = requests.get(ENDPOINT,
                     params={"api_key": key, "url": url, "sort": sort},
                     timeout=TIMEOUT)
    if r.status_code == 401:
        sys.exit("401 INVALID_API_KEY: key missing or not recognised. "
                 "Get one at chocodata.com")
    if r.status_code == 402:
        sys.exit("402 INSUFFICIENT_CREDITS: balance exhausted. "
                 "Top up or upgrade at chocodata.com")
    if r.status_code == 429:
        sys.exit("429 RATE_LIMITED: over your plan's concurrency. Back off and retry.")
    if r.status_code >= 400:
        print(f"  {r.status_code} skipped: {r.text[:120]}")
        return None
    return r.json()


def main(argv: list[str]) -> int:
    if len(argv) < 3:
        print(__doc__)
        return 64
    urls = [ln.strip() for ln in open(argv[1], encoding="utf-8")
            if ln.strip() and not ln.startswith("#")]
    out = argv[2]
    sort = argv[3] if len(argv) > 3 else "top"

    rows, ok, failed, dropped = [], 0, 0, 0
    for url in urls:
        print(f"GET {url}")
        data = fetch(url, sort)
        if not data:
            failed += 1
            continue
        post = data["post"]
        if post.get("title") is None:
            print(f"  stub post object (_meta.source={data['_meta']['source']!r}), "
                  "skipping")
            failed += 1
            continue
        ok += 1
        for depth, c in walk(data["comments"]):
            body = c["body"] or ""
            if body in PLACEHOLDERS:
                dropped += 1
                continue
            rows.append({
                "post_id": post["id"],
                "post_title": post["title"],
                "comment_id": c["id"],
                "parent_id": c["parent_id"] or "",
                "depth": c["depth"],
                "score": "" if c["score"] is None else c["score"],
                "author": c["author"]["username"] or "",
                "created": c["created"] or "",
                "permalink": c["permalink"] or "",
                "body": body.replace("\r", " ").replace("\n", " "),
            })
        print(f"  {post['title'][:70]}")
        print(f"  {data['comments_returned']} comments, "
              f"truncated={data['_meta']['truncated']}")

    with open(out, "w", encoding="utf-8", newline="") as fh:
        w = csv.DictWriter(fh, fieldnames=COLUMNS)
        w.writeheader()
        w.writerows(rows)

    print(f"\n{len(rows)} comment rows from {ok} threads -> {out}")
    print(f"{dropped} removed/deleted placeholder comments dropped, "
          f"{failed} threads skipped")
    return 0 if ok else 1


if __name__ == "__main__":
    raise SystemExit(main(sys.argv))