diff --git a/config.py b/config.py index f49b2f27..59959b09 100644 --- a/config.py +++ b/config.py @@ -13,6 +13,8 @@ from typing import Any from urllib import parse +from src import logs + class Config: """A global config class used throughout CWAC. @@ -66,23 +68,18 @@ def __init__(self) -> None: # Configure logging log_filename = self.config["audit_name"] - log_format = ( - "[{%(asctime)s} %(levelname)-7s %(filename)10s : %(lineno)-4s] %(funcName)30s %(message)s %(threadName)s" - ) # Create the results folder folder_path = "./results/" + log_filename + "/" os.makedirs(folder_path, exist_ok=True) - # Log timestamp format (ISO 8601) - log_date_format = "%Y-%m-%dT%H:%M:%S%z" - logging.basicConfig( - filename=f"./{folder_path}/{log_filename}.log", - format=log_format, - filemode="w", level=logging.INFO, - datefmt=log_date_format, + handlers=[ + logs.create_file_log_handler( + f"./{folder_path}/{log_filename}.log", + ) + ], ) # Write self.config to the results folder for reference diff --git a/cwac.py b/cwac.py index d9ff6e9e..c488120b 100644 --- a/cwac.py +++ b/cwac.py @@ -13,6 +13,7 @@ from typing import cast from urllib.parse import urlparse, urlunparse +import src.logs import src.verify from config import config from src.analytics import Analytics @@ -36,10 +37,11 @@ def thread(self, thread_id: int) -> None: Args: thread_id (int): identifier for the thread """ - browser = Browser(thread_id) - crawl = Crawler(browser=browser, url_queue=CWAC.url_queue, analytics=CWAC.analytics) - crawl.iterate_through_base_urls() - browser.close() + with src.logs.group_by_thread(f"./results/{config.audit_name}/logs/threads", f"{config.audit_name}_"): + browser = Browser(thread_id) + crawl = Crawler(browser=browser, url_queue=CWAC.url_queue, analytics=CWAC.analytics) + crawl.iterate_through_base_urls() + browser.close() def spawn_threads(self) -> None: """Create a number of threads to speed up execution. diff --git a/src/crawler.py b/src/crawler.py index 5e86182d..369c46a7 100644 --- a/src/crawler.py +++ b/src/crawler.py @@ -23,6 +23,7 @@ import src.filters import src.output from config import config +from src import logs from src.analytics import Analytics from src.audit_manager import AuditManager from src.browser import Browser @@ -66,10 +67,11 @@ def iterate_through_base_urls(self) -> None: with config.lock: site_data = self.url_queue.get() - logging.info("Starting test %s", site_data["url"]) + with logs.group_by_base_url(f"./results/{config.audit_name}/logs/urls", site_data["url"]): + logging.info("Starting test %s", site_data["url"]) - # Crawl the url (the crawler also initiates tests) - self.crawl(site_data, site_data["url"]) + # Crawl the url (the crawler also initiates tests) + self.crawl(site_data, site_data["url"]) # Restart the browser between each website self.browser.safe_restart() diff --git a/src/logs.py b/src/logs.py new file mode 100644 index 00000000..6a68facc --- /dev/null +++ b/src/logs.py @@ -0,0 +1,98 @@ +"""Logging helpers.""" + +import logging +import os +import re +import threading +from contextlib import contextmanager +from typing import Any, Callable, Generator + + +def __register_thread_based_file_handler(directory: str, prefix: str) -> logging.Handler: + thread_name = threading.current_thread().name + log_file = f"{directory}/{prefix}{thread_name}.log" + + log_handler = create_file_log_handler(log_file) + log_handler.addFilter(lambda record: record.threadName == thread_name) + + logging.getLogger().addHandler(log_handler) + + return log_handler + + +def __create_and_register_selective_file_handler( + log_file: str, selector: Callable[[logging.LogRecord], bool | logging.LogRecord] +) -> logging.Handler: + log_handler = create_file_log_handler(log_file) + log_handler.addFilter(selector) + + logging.getLogger().addHandler(log_handler) + + return log_handler + + +def create_file_log_handler(log_file: str) -> logging.FileHandler: + """Create a file-based logging handler.""" + log_handler = logging.FileHandler(log_file) + log_handler.setLevel(logging.INFO) + + log_handler.setFormatter( + logging.Formatter( + "[{%(asctime)s} %(levelname)-7s %(filename)10s : %(lineno)-4s] %(funcName)30s %(message)s %(threadName)s", + # Log timestamp format (ISO 8601) + "%Y-%m-%dT%H:%M:%S%z", + ) + ) + + return log_handler + + +@contextmanager +def group_by_thread(directory: str, prefix: str) -> Generator[None, Any, None]: + """Group logs made by the current thread into a secondary file.""" + os.makedirs(directory, exist_ok=True) + log_handler = __register_thread_based_file_handler(directory, prefix) + try: + yield + finally: + logging.getLogger().removeHandler(log_handler) + log_handler.close() + + +# todo: who knows if this is thread safe enough... +matchups = {} + + +def sanitise_string(string: str) -> str: + """Sanitise a string for use in a folder/filename. + + Args: + string (str): the string to sanitise + + Returns: + str: a sanitised string + """ + temp_str = string.strip() + temp_str = re.sub(r"[^a-zA-Z0-9_\-.]", "_", temp_str) + temp_str = re.sub(r"_+", "_", temp_str) + temp_str = temp_str[:50] + return temp_str + + +@contextmanager +def group_by_base_url(directory: str, base_url: str) -> Generator[None, Any, None]: + """Group logs for the given base_url into a dedicated secondary file.""" + os.makedirs(directory, exist_ok=True) + safe_base_url = sanitise_string(base_url) + + matchups[threading.current_thread().ident] = base_url + log_handler = __create_and_register_selective_file_handler( + f"{directory}/{safe_base_url}.log", + lambda record: matchups.get(record.thread, "") == base_url, + ) + + try: + yield + finally: + logging.getLogger().removeHandler(log_handler) + log_handler.close()