feat: duplicate log entries into dedicated files per thread and base urls

config.py

@@ -13,6 +13,8 @@
 from typing import Any
 from urllib import parse
 
+from src import logs
+
 
 class Config:
     """A global config class used throughout CWAC.
@@ -66,23 +68,18 @@ def __init__(self) -> None:
 
         # Configure logging
         log_filename = self.config["audit_name"]
-        log_format = (
-            "[{%(asctime)s} %(levelname)-7s %(filename)10s : %(lineno)-4s] %(funcName)30s %(message)s %(threadName)s"
-        )
 
         # Create the results folder
         folder_path = "./results/" + log_filename + "/"
         os.makedirs(folder_path, exist_ok=True)
 
-        # Log timestamp format (ISO 8601)
-        log_date_format = "%Y-%m-%dT%H:%M:%S%z"
-
         logging.basicConfig(
-            filename=f"./{folder_path}/{log_filename}.log",
-            format=log_format,
-            filemode="w",
             level=logging.INFO,
-            datefmt=log_date_format,
+            handlers=[
+                logs.create_file_log_handler(
+                    f"./{folder_path}/{log_filename}.log",
+                )
+            ],
         )
 
         # Write self.config to the results folder for reference

cwac.py

@@ -13,6 +13,7 @@
 from typing import cast
 from urllib.parse import urlparse, urlunparse
 
+import src.logs
 import src.verify
 from config import config
 from src.analytics import Analytics
@@ -36,10 +37,11 @@ def thread(self, thread_id: int) -> None:
         Args:
             thread_id (int): identifier for the thread
         """
-        browser = Browser(thread_id)
-        crawl = Crawler(browser=browser, url_queue=CWAC.url_queue, analytics=CWAC.analytics)
-        crawl.iterate_through_base_urls()
-        browser.close()
+        with src.logs.group_by_thread(f"./results/{config.audit_name}/logs/threads", f"{config.audit_name}_"):
+            browser = Browser(thread_id)
+            crawl = Crawler(browser=browser, url_queue=CWAC.url_queue, analytics=CWAC.analytics)
+            crawl.iterate_through_base_urls()
+            browser.close()
 
     def spawn_threads(self) -> None:
         """Create a number of threads to speed up execution.

src/crawler.py

@@ -23,6 +23,7 @@
 import src.filters
 import src.output
 from config import config
+from src import logs
 from src.analytics import Analytics
 from src.audit_manager import AuditManager
 from src.browser import Browser
@@ -66,10 +67,11 @@ def iterate_through_base_urls(self) -> None:
             with config.lock:
                 site_data = self.url_queue.get()
 
-            logging.info("Starting test %s", site_data["url"])
+            with logs.group_by_base_url(f"./results/{config.audit_name}/logs/urls", site_data["url"]):
+                logging.info("Starting test %s", site_data["url"])
 
-            # Crawl the url (the crawler also initiates tests)
-            self.crawl(site_data, site_data["url"])
+                # Crawl the url (the crawler also initiates tests)
+                self.crawl(site_data, site_data["url"])
 
             # Restart the browser between each website
             self.browser.safe_restart()

src/logs.py

@@ -0,0 +1,98 @@
+"""Logging helpers."""
+
+import logging
+import os
+import re
+import threading
+from contextlib import contextmanager
+from typing import Any, Callable, Generator
+
+
+def __register_thread_based_file_handler(directory: str, prefix: str) -> logging.Handler:
+    thread_name = threading.current_thread().name
+    log_file = f"{directory}/{prefix}{thread_name}.log"
+
+    log_handler = create_file_log_handler(log_file)
+    log_handler.addFilter(lambda record: record.threadName == thread_name)
+
+    logging.getLogger().addHandler(log_handler)
+
+    return log_handler
+
+
+def __create_and_register_selective_file_handler(
+    log_file: str, selector: Callable[[logging.LogRecord], bool | logging.LogRecord]
+) -> logging.Handler:
+    log_handler = create_file_log_handler(log_file)
+    log_handler.addFilter(selector)
+
+    logging.getLogger().addHandler(log_handler)
+
+    return log_handler
+
+
+def create_file_log_handler(log_file: str) -> logging.FileHandler:
+    """Create a file-based logging handler."""
+    log_handler = logging.FileHandler(log_file)
+    log_handler.setLevel(logging.INFO)
+
+    log_handler.setFormatter(
+        logging.Formatter(
+            "[{%(asctime)s} %(levelname)-7s %(filename)10s : %(lineno)-4s] %(funcName)30s %(message)s %(threadName)s",
+            # Log timestamp format (ISO 8601)
+            "%Y-%m-%dT%H:%M:%S%z",
+        )
+    )
+
+    return log_handler
+
+
+@contextmanager
+def group_by_thread(directory: str, prefix: str) -> Generator[None, Any, None]:
+    """Group logs made by the current thread into a secondary file."""
+    os.makedirs(directory, exist_ok=True)
+    log_handler = __register_thread_based_file_handler(directory, prefix)
+    try:
+        yield
+    finally:
+        logging.getLogger().removeHandler(log_handler)
+        log_handler.close()
+
+
+# todo: who knows if this is thread safe enough...
+matchups = {}
+
+
+def sanitise_string(string: str) -> str:
+    """Sanitise a string for use in a folder/filename.
+
+    Args:
+        string (str): the string to sanitise
+
+    Returns:
+        str: a sanitised string
+    """
+    temp_str = string.strip()
+    temp_str = re.sub(r"[^a-zA-Z0-9_\-.]", "_", temp_str)
+    temp_str = re.sub(r"_+", "_", temp_str)
+    temp_str = temp_str[:50]
+    return temp_str
+
+
+@contextmanager
+def group_by_base_url(directory: str, base_url: str) -> Generator[None, Any, None]:
+    """Group logs for the given base_url into a dedicated secondary file."""
+    os.makedirs(directory, exist_ok=True)
+    safe_base_url = sanitise_string(base_url)
+
+    matchups[threading.current_thread().ident] = base_url
+    log_handler = __create_and_register_selective_file_handler(
+        f"{directory}/{safe_base_url}.log",
+        lambda record: matchups.get(record.thread, "") == base_url,
+    )
+
+    try:
+        yield
+    finally:
+        logging.getLogger().removeHandler(log_handler)
+        log_handler.close()