loader.py

import os
from cleaner import clean

def load_docs(folder="data"):
    docs = {}
    if not os.path.exists(folder):
        os.makedirs(folder)
        print(f"[loader] created '{folder}/'; place .txt files there.")
        return docs

    for root, dirs, files in os.walk(folder):
        for fname in sorted(files):
            if fname.endswith(".txt"):
                path = os.path.join(root, fname)
                # e.g. sub/file.txt
                rel_key = os.path.relpath(path, folder)
                with open(path, "r", encoding="utf-8", errors="ignore") as f:
                    docs[rel_key] = clean(f.read())

    if not docs:
        print(f"[loader] no .txt files found in '{folder}/'.")
    return docs