From ef87b088443ba68899fecb83fb6b2ee4298a5ee6 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Sat, 25 Jul 2026 08:37:47 +0200 Subject: [PATCH 01/36] Add design spec for IMAP backend support Documents extracting the shared fetch-store-train-predict-move loop into base/mail.py so a new imap/ package can reuse it alongside the existing google/ backend, plus CLI and CI integration test plans. --- .../specs/2026-07-25-imap-support-design.md | 227 ++++++++++++++++++ 1 file changed, 227 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-25-imap-support-design.md diff --git a/docs/superpowers/specs/2026-07-25-imap-support-design.md b/docs/superpowers/specs/2026-07-25-imap-support-design.md new file mode 100644 index 0000000..56c209e --- /dev/null +++ b/docs/superpowers/specs/2026-07-25-imap-support-design.md @@ -0,0 +1,227 @@ +# IMAP support for gmailsorter + +## Problem + +`gmailsorter` currently only talks to Gmail through the Gmail API. This limits it to +Google Mail accounts and means the machine-learned sorting logic can never be exercised +in CI without live Google credentials. We want to add a second backend that speaks IMAP +(username + password auth), so that: + +* Users with any IMAP-capable mailbox (self-hosted, Dovecot, etc.) can use gmailsorter. +* CI can run a real, end-to-end integration test against a disposable IMAP server + (GreenMail), the way [jan-janssen/testing-imap](https://github.com/jan-janssen/testing-imap) + demonstrates. + +## Scope + +In scope: + +* A new `gmailsorter/imap/` package (`authentication.py`, `message.py`, `mail.py`) + parallel to `gmailsorter/google/`. +* A new `Imap` class in `gmailsorter/local.py`, parallel to `Gmail`. +* A new `gmailsorter-imap` CLI entry point, parallel to `gmailsorter`/`gmailsorter-daemon`. +* Refactoring the fetch-store-train-predict-move loop currently living in + `GoogleMailBase` into a shared abstract base class, so both backends reuse it instead + of duplicating it. +* Unit tests (mocked `imaplib`) and a real integration test against a `greenmail` + container in GitHub Actions. + +Out of scope (explicitly deferred): + +* The Flask webapp / gmailsorter.com login flow — stays Gmail-OAuth-only. +* `gmailsorter-daemon` — stays Gmail-only for now. +* OAuth2/XOAUTH2 for IMAP (e.g. Outlook, Gmail-via-IMAP) — only plain username/password + login (`IMAP4_SSL`/`IMAP4` `LOGIN`) is implemented. The authentication module should + not need reworking to add this later, but implementing it is not part of this change. +* Custom IMAP `SEARCH` queries (the `query_string` parameter that already exists but is + never actually used anywhere in the current codebase) — the IMAP backend only needs to + support `SEARCH ALL` for v1. + +## Architecture + +### Extracting the shared loop + +`GoogleMailBase` (`gmailsorter/google/mail.py`) currently mixes two concerns: the +backend-agnostic fetch→store→train→predict→move loop, and Gmail-API-specific calls +(`service.users().messages()...`). Adding IMAP as a second backend without extracting +the shared part would mean copy-pasting roughly 150 lines of loop/business logic +(`download_emails_for_label`, `filter_messages_from_server`, +`fit_machine_learning_model_to_database`, `get_all_emails_in_database`, +`update_database`, `_download_messages_to_dataframe`, `_store_emails_in_database`, +`_get_labels_for_email(s)`, `_move_emails`) into a new `imap/mail.py`. Instead, this +logic moves into a new class: + +``` +gmailsorter/base/mail.py + class AbstractMailBox(ABC): + # concrete, shared: + labels (property) + download_emails_for_label(label) + filter_messages_from_server(label, recommendation_ratio=0.9) + fit_machine_learning_model_to_database(...) + get_all_emails_in_database(include_deleted=False) + update_database(quick=False, label_lst=None, email_format=None) + _download_messages_to_dataframe(message_id_lst, email_format=None) + _get_labels_for_email(message_id) + _get_labels_for_emails(message_id_lst) + _move_emails(move_email_dict, label_to_ignore) + _store_emails_in_database(message_id_lst, email_format=None) + + # abstract, backend-specific: + _search_email_on_server(query_string="", label_lst=None, only_message_ids=False) + _get_message_detail(message_id, email_format=None, metadata_headers=None) + _get_label_translate_dict() + _modify_message_labels(message_id, label_id_remove_lst=None, label_id_add_lst=None) + _parse_message(message) -> dict # via each backend's AbstractMessage subclass +``` + +This mirrors the existing `base/` vs `google/` split already used for `message.py` +(`AbstractMessage`) and `database.py` (`DatabaseTemplate`/`DatabaseInterface`). + +`GoogleMailBase(AbstractMailBox)` keeps its **exact current public constructor +signature** (`google_mail_service`, `database_email`, `database_ml`, `database_token`, +`user_id`, `db_user_id`, `email_download_format`) so existing callers and tests +(`tests/test_google_integration_units.py`) are unaffected. `database_token` is confirmed +unused outside of `__init__` (grepped the codebase — it's stored on `self` but never +read again), so it stays a `GoogleMailBase`-only attribute rather than being threaded +into the shared base class. + +The small `MLStripper` HTML-to-text helper currently in `gmailsorter/google/message.py` +is generic (not Gmail-specific), so it moves to `gmailsorter/base/message.py` and both +`google/message.py` and the new `imap/message.py` reuse it from there. + +### `gmailsorter/imap/authentication.py` + +```python +def create_service(host, port, username, password, use_ssl=True): + """Open and log in to an IMAP4_SSL/IMAP4 connection. Raises on failure.""" +``` + +No token database, no refresh flow — the password is supplied directly each time a +connection is created (matches how `Gmail`'s `client_config` is supplied directly, just +without the OAuth indirection). If the connection drops, callers reconnect by calling +`create_service` again. + +### `gmailsorter/imap/message.py` + +`Message(AbstractMessage)` parses a raw `email.message.Message` (as returned by +`email.message_from_bytes` after an IMAP `FETCH ... (RFC822)`), plus the folder name it +was fetched from: + +* `get_email_id()` → composite `f"{folder}\x1f{uid}"`. IMAP UIDs are only unique/stable + *within one mailbox* (a `MOVE` to another folder assigns a new UID at the + destination), so the folder is baked into the id used as the primary key in the local + database. +* `get_thread_id()` → first `References` header entry, else `In-Reply-To`, else the + message's own `Message-ID` (so a thread-starting message is its own thread root). +* `get_label_ids()` → `[folder]` — a single-item list, since one IMAP mailbox = one + label. This fits the existing multi-label list contract in `ml/encoding.py` unchanged. +* `get_from`/`get_to`/`get_cc`/`get_subject`/`get_date` → parsed from the standard email + headers (`email.utils.parseaddr`/`getaddresses`, `email.utils.parsedate_to_datetime`). +* `get_content()` → walks MIME parts for `text/plain`, falling back to `text/html` + stripped via the shared `MLStripper`. + +### `gmailsorter/imap/mail.py` + +`ImapMailBase(AbstractMailBox)` implements the abstract hooks: + +* `_get_label_translate_dict()` — `IMAP LIST` all mailboxes, skipping ones flagged + `\Noselect`, returned as `{name: name}` (IMAP has no separate id vs. display name). +* `_search_email_on_server(query_string="", label_lst=None, only_message_ids=False)` — + * If `label_lst` is non-empty: `SELECT` each named folder and `UID SEARCH ALL`. + * If `label_lst` is empty (the case `update_database()` always uses in practice — + verified `__main__.py` and `daemon/daemon.py` both call it with no `label_lst`, + exactly mirroring how Gmail's own `label_ids=[]` means "no filter, whole account"): + iterate over **every** folder from `_get_label_translate_dict()` and aggregate. + * A non-empty `query_string` raises `NotImplementedError` (not silently ignored), + since custom IMAP `SEARCH` syntax isn't implemented in v1 and it's better to fail + loudly than search the wrong thing. +* `_get_message_detail(message_id, ...)` — splits the composite id into + `(folder, uid)`, `SELECT`s the folder, `UID FETCH ... (RFC822)`. +* `_modify_message_labels(message_id, label_id_remove_lst, label_id_add_lst)` — treated + as "move `message_id` from `label_id_remove_lst[0]` to `label_id_add_lst[0]`" (IMAP + only has one folder per message, unlike Gmail's multi-label add/remove). Issues IMAP + `MOVE` if the server advertises the `MOVE` capability, otherwise falls back to `COPY` + + `STORE +FLAGS (\Deleted)` + `EXPUNGE`. +* `_create_databases(connection_str)` — creates only `database_email` and `database_ml` + (no token database, since there's no OAuth token to persist). + +### `gmailsorter/local.py` + +```python +class Imap(ImapMailBase): + def __init__(self, host, port, username, password, connection_str, + db_user_id=1, use_ssl=True, email_download_format="metadata"): + ... +``` + +Parallel to the existing `Gmail` class: builds the two databases, opens the IMAP +connection via `imap.authentication.create_service`, and calls `super().__init__(...)`. + +### CLI: `gmailsorter-imap` + +A new console-script entry point in `pyproject.toml` +(`gmailsorter-imap = "gmailsorter.imap.__main__:command_line_parser"`), parallel to the +existing `gmailsorter`/`gmailsorter-daemon`/`gmailsorter-app` scripts (a new top-level +CLI rather than overloading the existing `gmailsorter` parser with two unrelated +credential schemes). Flags: + +* `--host`, `--port` (default `993`), `--username` +* `--password-env` (name of an environment variable holding the password; default + `IMAP_PASSWORD`) — the password is never accepted as a literal CLI argument, so it + never ends up in shell history or `ps` output. +* `--database`, `--update`, `--label`, `--identification` — same meaning as the + existing `gmailsorter` CLI. + +## Testing + +* `tests/test_imap_message.py` — mirrors `tests/test_google_message.py`: constructs a + raw `email.message.Message`, asserts each `get_*` method and `to_dict()`. +* `tests/test_imap_integration_units.py` — mirrors + `tests/test_google_integration_units.py`: mocks `imaplib.IMAP4_SSL` and asserts + `ImapMailBase`'s hook methods issue the right IMAP commands (`SELECT`, `UID SEARCH`, + `UID FETCH`, `MOVE`/`COPY`+`STORE`+`EXPUNGE`), plus `Imap` wiring in `local.py`. +* `tests/test_mail_base.py` — new tests for the extracted `AbstractMailBox` loop logic + itself (currently only exercised indirectly through `GoogleMailBase` in + `test_google_integration_units.py`), using a minimal concrete stub subclass. +* Existing `tests/test_google_integration_units.py` continues to pass unmodified, + proving the refactor didn't change `GoogleMailBase`'s observable behavior. +* `tests/test_imap_service_integration.py` — a real end-to-end test (not mocked) that: + 1. Connects to a live `greenmail/standalone` container via `smtplib` (send) and + `imaplib` (fetch), following the pattern in + [jan-janssen/testing-imap](https://github.com/jan-janssen/testing-imap)'s + `tests/test_imap_service.py`. + 2. Drives it through `gmailsorter.local.Imap` — updates a SQLite database from the + live GreenMail mailbox, verifies stored content, and exercises a folder move. + 3. Reads connection details from environment variables + (`TEST_IMAP_HOST`/`TEST_IMAP_PORT`/`TEST_IMAP_USERNAME`/`TEST_EMAIL_PASSWORD`/ + `TEST_SMTP_HOST`/`TEST_SMTP_PORT`), matching the testing-imap repo's convention, so + the exact same environment variable names configure both. +* `.github/workflows/unittest.yml` gets a `greenmail` entry under `services:` (image + `greenmail/standalone:2.1.11`, same `GREENMAIL_OPTS`/ports as the testing-imap repo) + and the matching env vars, so `tests/test_imap_service_integration.py` runs on every + push/PR alongside the rest of the unit test suite. Mocked tests keep running on all + three OSes/Python versions in the existing matrix; the GreenMail-backed integration + test only needs to run once (GitHub Actions service containers are Linux-only), so it + runs as an additional step gated to the `ubuntu-latest` job. + +## Documentation + +* `docs/source/developer.md` — add an "IMAP" section parallel to the existing Python + Interface section, showing `Imap(...)` construction and noting it shares the exact + same `update_database`/`get_all_emails_in_database`/`filter_messages_from_server` API + as `Gmail`. +* `docs/source/architecture.md` — update "the source of truth" bullet to mention IMAP as + an alternative to the Gmail API, and note that IMAP folders play the role Gmail labels + play elsewhere in the document. +* `README.md` — mention IMAP support alongside the existing Gmail description, if it + currently states Gmail-only. + +## Non-goals / known limitations carried into v1 + +* No OAuth2/XOAUTH2 (Outlook, Gmail-via-IMAP) — plain `LOGIN` only. +* No custom IMAP `SEARCH` query support. +* A message's database identity changes when it's moved between folders (old id is + marked deleted, a new id is created at the destination) — this is a direct, accepted + consequence of IMAP's per-mailbox UID model, not a bug to fix here. +* Webapp and daemon remain Gmail-only. From af5e118b3e9c5fdbc901f0a7e447319c4953cc3a Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Sat, 25 Jul 2026 08:50:37 +0200 Subject: [PATCH 02/36] Add implementation plan for IMAP backend support Nine-task TDD plan: extract AbstractMailBox from GoogleMailBase, add gmailsorter/imap/ (message, authentication, mail), wire up Imap in local.py, add the gmailsorter-imap CLI, and add a GreenMail-backed CI integration test plus docs. --- .../plans/2026-07-25-imap-support.md | 2330 +++++++++++++++++ 1 file changed, 2330 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-25-imap-support.md diff --git a/docs/superpowers/plans/2026-07-25-imap-support.md b/docs/superpowers/plans/2026-07-25-imap-support.md new file mode 100644 index 0000000..09c5899 --- /dev/null +++ b/docs/superpowers/plans/2026-07-25-imap-support.md @@ -0,0 +1,2330 @@ +# IMAP Backend Support Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a second, IMAP-based backend to gmailsorter (username/password auth, one IMAP folder = one Gmail-style label) that reuses the existing fetch-store-train-predict-move loop, ships a `gmailsorter-imap` CLI, and is exercised in CI against a real GreenMail IMAP/SMTP server. + +**Architecture:** Extract the backend-agnostic loop currently living in `GoogleMailBase` into a new `gmailsorter.base.mail.AbstractMailBox` (mirroring the existing `base/` vs `google/` split for `message.py`/`database.py`), then add a parallel `gmailsorter/imap/` package (`authentication.py`, `message.py`, `mail.py`) plus an `Imap` class in `local.py` and a `gmailsorter-imap` console script. + +**Tech Stack:** Python stdlib `imaplib`/`email` (no new dependencies), existing `sqlalchemy`/`pandas`/`scikit-learn` stack, `unittest` + `unittest.mock`, GitHub Actions `services:` container (`greenmail/standalone:2.1.11`). + +## Global Constraints + +- Target Python: `>=3.10` (repo classifiers test 3.11–3.14) — avoid syntax newer than that. +- Lint: `ruff` with rules `E, F, UP, B, SIM, I, C4, ERA, PL` (ignoring `E501`, `PLR0913`) via `.pre-commit-config.yaml`, applied to files under `gmailsorter/`. Keep new code consistent with this (no unused imports, no commented-out code, etc). +- No new runtime dependencies: `imaplib` and `email` are stdlib; do not add packages to `pyproject.toml` `dependencies`. +- Follow existing docstring style (Google-style `Args:`/`Returns:`) used throughout `gmailsorter/`. +- Existing public API (`gmailsorter.Gmail`, `gmailsorter.load_client_secrets_file`, `GoogleMailBase.__init__` signature) must not change — `tests/test_google_integration_units.py` must keep passing with, at most, its `@patch(...)` target strings updated to follow code that moved (no assertion or behavior changes). +- Test runner: `coverage run --omit gmailsorter/_version.py -m unittest discover tests` (see `.github/workflows/unittest.yml`) — every new test file must be discoverable by `unittest discover tests` (class extends `unittest.TestCase`, file name starts with `test_`). +- IMAP auth is username/password only for this plan (no OAuth2/XOAUTH2). Passwords must never be accepted as a literal CLI argument. +- Webapp (`gmailsorter/webapp/`) and daemon (`gmailsorter/daemon/`) are explicitly out of scope — do not modify them. + +--- + +### Task 1: Shared HTML-to-text helper in `base/message.py` + +**Files:** +- Modify: `gmailsorter/base/message.py` +- Modify: `gmailsorter/google/message.py` +- Test: `tests/test_message.py` + +**Interfaces:** +- Produces: `gmailsorter.base.message.strip_html_tags(html: str) -> str`, used by both `gmailsorter/google/message.py` (Task 1) and `gmailsorter/imap/message.py` (Task 3). + +- [ ] **Step 1: Write the failing test** + +Add to `tests/test_message.py` (append inside the existing `MessageTest` class, and add the import at the top): + +```python +from gmailsorter.base.message import email_date_converter, strip_html_tags +``` + +```python + def test_strip_html_tags(self): + self.assertEqual( + strip_html_tags("

Hello World

"), + "Hello World", + ) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python -m unittest tests.test_message -v` +Expected: FAIL with `ImportError: cannot import name 'strip_html_tags'` + +- [ ] **Step 3: Move `MLStripper` into `base/message.py` as `strip_html_tags`** + +In `gmailsorter/base/message.py`, add these imports at the top (alongside the existing `abc`/`datetime` imports): + +```python +from html.parser import HTMLParser +from io import StringIO +``` + +Then add, after the `_DATE_HYPHEN_COUNT` constant and before `email_date_converter`: + +```python +# https://stackoverflow.com/questions/753052/strip-html-from-strings-in-python +class _MLStripper(HTMLParser): + def __init__(self): + super().__init__() + self.reset() + self.strict = False + self.convert_charrefs = True + self.text = StringIO() + + def handle_data(self, d): + self.text.write(d) + + def get_data(self): + return self.text.getvalue() + + +def strip_html_tags(html): + stripper = _MLStripper() + stripper.feed(html) + return stripper.get_data() +``` + +- [ ] **Step 4: Update `gmailsorter/google/message.py` to use the shared helper** + +Replace the top of `gmailsorter/google/message.py` — delete the `MLStripper` class and its imports, and import `strip_html_tags` instead: + +```python +import base64 + +from gmailsorter.base.message import AbstractMessage, email_date_converter, strip_html_tags +``` + +(This replaces the old `import base64` / `from html.parser import HTMLParser` / `from io import StringIO` / `from gmailsorter.base.message import ...` block, and removes the `MLStripper` class definition that followed it.) + +In the `Message` class, change `_get_parts_content` to call the shared function instead of `self._strip_tags`: + +```python + def _get_parts_content(self, message_parts): + content_types = [p["mimeType"] for p in message_parts if "mimeType" in p] + if "text/plain" in content_types: + return self._get_email_body( + message_parts=message_parts[content_types.index("text/plain")] + ) + elif "text/html" in content_types: + return strip_html_tags( + html=self._get_email_body( + message_parts=message_parts[content_types.index("text/html")] + ) + ) + elif "multipart/alternative" in content_types: + multi_part_content = message_parts[ + content_types.index("multipart/alternative") + ] + if "parts" in multi_part_content: + return self._get_parts_content( + message_parts=multi_part_content["parts"] + ) + else: + return None + else: + return None +``` + +Delete the now-unused `_strip_tags` staticmethod entirely (it was right after `_get_email_body`). + +- [ ] **Step 5: Run test to verify it passes** + +Run: `python -m unittest tests.test_message -v` +Expected: PASS + +- [ ] **Step 6: Run the full existing suite to confirm no regression** + +Run: `python -m unittest discover tests -v` +Expected: All tests PASS (in particular `tests/test_google_message.py`, unaffected since `Message._get_parts_content`'s observable behavior is unchanged). + +- [ ] **Step 7: Commit** + +```bash +git add gmailsorter/base/message.py gmailsorter/google/message.py tests/test_message.py +git commit -m "refactor: move HTML-to-text stripping into base/message.py so it can be reused by imap/message.py" +``` + +--- + +### Task 2: Extract `AbstractMailBox` shared loop; refactor `GoogleMailBase` + +**Files:** +- Create: `gmailsorter/base/mail.py` +- Modify: `gmailsorter/google/mail.py` (full rewrite) +- Modify: `tests/test_google_integration_units.py` (patch targets only) +- Test: `tests/test_mail_base.py` + +**Interfaces:** +- Produces: `gmailsorter.base.mail.AbstractMailBox(ABC)` with constructor + `__init__(self, mail_service, database_email=None, database_ml=None, user_id="me", db_user_id=1, email_download_format="metadata")`, + concrete methods `labels` (property), `download_emails_for_label(label)`, + `filter_messages_from_server(label, recommendation_ratio=0.9)`, + `fit_machine_learning_model_to_database(n_estimators=100, max_features=400, random_state=42, bootstrap=True, include_deleted=False)`, + `get_all_emails_in_database(include_deleted=False)`, + `update_database(quick=False, label_lst=None, email_format=None)`, + and abstract hooks `_search_email_on_server(query_string="", label_lst=None, only_message_ids=False)`, + `_get_message_detail(message_id, email_format=None, metadata_headers=None)`, + `_get_label_translate_dict()`, + `_modify_message_labels(message_id, label_id_remove_lst=None, label_id_add_lst=None)`, + `_get_labels_for_email(message_id)`, `_parse_message(message)`. +- Consumed by: Task 5 (`ImapMailBase(AbstractMailBox)`). + +This is a **behavior-preserving refactor** of already-tested code, not new functionality, so the TDD cycle here is: move the code, then prove the full existing test suite (plus a new isolation-focused test file) still passes — rather than writing a new failing test first. + +- [ ] **Step 1: Create `gmailsorter/base/mail.py`** + +```python +from abc import ABC, abstractmethod + +import pandas +from tqdm import tqdm + +from gmailsorter.ml import ( + encode_df_for_machine_learning, + fit_machine_learning_models, + get_predictions_from_machine_learning_models, +) + + +class AbstractMailBox(ABC): + def __init__( + self, + mail_service, + database_email=None, + database_ml=None, + user_id="me", + db_user_id=1, + email_download_format="metadata", + ): + """ + Shared fetch-store-train-predict-move loop for a mailbox backend, independent of + whether the backend is the Gmail API or a plain IMAP connection. + + Args: + mail_service: backend-specific connection object (Gmail API service resource, + imaplib connection, ...) + database_email (gmailsorter.base.database.DatabaseInterface): SQLalchemy interface for email database + database_ml (gmailsorter.ml.database.DatabaseInterface): SQLalchemy interface for machine learning database + user_id (str): backend-specific user identifier + db_user_id (int): Default 1 - set a user id when sharing a database with multiple users + email_download_format (str): backend-specific download format hint + """ + self._service = mail_service + self._db_email = database_email + self._db_ml = database_ml + self._db_user_id = db_user_id + self._userid = user_id + self._email_download_format = email_download_format + self._label_dict = self._get_label_translate_dict() + self._label_dict_inverse = {v: k for k, v in self._label_dict.items()} + + @property + def labels(self): + return list(self._label_dict.keys()) + + def download_emails_for_label(self, label): + """ + Download emails for a specific label + + Args: + label (str): label to download emails for + + Returns: + pandas.DataFrame: Email content for the downloaded emails + """ + return self._download_messages_to_dataframe( + message_id_lst=self._search_email_on_server( + label_lst=[label], only_message_ids=True + ) + ) + + def filter_messages_from_server( + self, + label, + recommendation_ratio=0.9, + ): + """ + Filter new emails based on machine learning model recommendations. + + Args: + label (str): Email label to filter for + recommendation_ratio (float): Only accept recommendation above this ratio (0 0: + model_reload_dict, feature_reload_lst = self._db_ml.load_models() + df_partial_features = encode_df_for_machine_learning( + df=df_partial, + feature_lst=feature_reload_lst, + label_lst=list(model_reload_dict.keys()), + return_labels=False, + ) + df_partial_features = df_partial_features.reindex( + sorted(df_partial_features.columns), axis=1 + ) + model_recommendation_dict = get_predictions_from_machine_learning_models( + df_features=df_partial_features, + model_dict=model_reload_dict, + recommendation_ratio=recommendation_ratio, + ) + self._move_emails( + move_email_dict=model_recommendation_dict, label_to_ignore=label + ) + + def fit_machine_learning_model_to_database( + self, + n_estimators=100, + max_features=400, + random_state=42, + bootstrap=True, + include_deleted=False, + ): + """ + Fit machine learning models to emails stored in database and afterwards store machine learning models in + database. + + Args: + n_estimators (int): Number of estimators + max_features (int): Number of features + random_state (int): Random state + bootstrap (boolean): Whether bootstrap samples are used when building trees. If False, the whole dataset is + used to build each tree. (default: true) + include_deleted (bool): Flag to include deleted emails - default False + """ + df_all = self.get_all_emails_in_database(include_deleted=include_deleted) + df_all_features, df_all_labels = encode_df_for_machine_learning( + df=df_all, feature_lst=[], label_lst=[], return_labels=True + ) + df_all_features = df_all_features.loc[ + :, ~df_all_features.columns.duplicated() + ].copy() + df_all_features = df_all_features.reindex( + sorted(df_all_features.columns), axis=1 + ) + model_dict = fit_machine_learning_models( + df_all_features=df_all_features, + df_all_labels=df_all_labels, + n_estimators=n_estimators, + max_features=max_features, + random_state=random_state, + bootstrap=bootstrap, + ) + self._db_ml.store_models( + model_dict=model_dict, + feature_lst=df_all_features.columns.values.tolist(), + user_id=self._db_user_id, + commit=True, + ) + + def get_all_emails_in_database(self, include_deleted=False): + """ + Get all emails stored in the local database + + Args: + include_deleted (bool): Flag to include deleted emails - default False + + Returns: + pandas.DataFrame: With all emails and the corresponding information + """ + return self._db_email.get_all_emails( + include_deleted=include_deleted, user_id=self._db_user_id + ) + + def update_database(self, quick=False, label_lst=None, email_format=None): + """ + Update local email database + + Args: + quick (boolean): Only add new emails, do not update existing labels - by default: False + label_lst (list): list of labels to be searched + email_format (str/None): Email format to download + """ + if label_lst is None: + label_lst = [] + if self._db_email is not None: + message_id_lst = self._search_email_on_server( + label_lst=label_lst, only_message_ids=True + ) + ( + new_messages_lst, + message_label_updates_lst, + deleted_messages_lst, + ) = self._db_email.get_labels_to_update( + message_id_lst=message_id_lst, user_id=self._db_user_id + ) + if not quick: + self._db_email.mark_emails_as_deleted( + message_id_lst=deleted_messages_lst, user_id=self._db_user_id + ) + self._db_email.update_labels( + message_id_lst=message_label_updates_lst, + message_meta_lst=self._get_labels_for_emails( + message_id_lst=message_label_updates_lst + ), + user_id=self._db_user_id, + ) + self._store_emails_in_database( + message_id_lst=new_messages_lst, email_format=email_format + ) + + def _download_messages_to_dataframe(self, message_id_lst, email_format=None): + """ + Download a list of messages based on their email IDs and store the content in a pandas.DataFrame. + + Args: + message_id_lst (list): list of emails IDs + email_format (str): Email format to download - default: "full" + + Returns: + pandas.DataFrame: pandas.DataFrame which contains the rendered emails + """ + return pandas.DataFrame( + [ + message + for message in [ + self._parse_message( + message=self._get_message_detail( + message_id=message_id, + email_format=email_format, + metadata_headers=[], + ) + ) + for message_id in tqdm( + iterable=message_id_lst, desc="Download messages to DataFrame" + ) + ] + if message is not None + ] + ) + + def _get_labels_for_emails(self, message_id_lst): + """ + Get labels for a list of emails + + Args: + message_id_lst (list): list of emails IDs + + Returns: + list: Nested list of email labels for each email + """ + return [ + self._get_labels_for_email(message_id=message_id) + for message_id in tqdm( + iterable=message_id_lst, desc="Get labels for emails" + ) + ] + + def _move_emails(self, move_email_dict, label_to_ignore): + label_existing = self._label_dict[label_to_ignore] + for message_id, label_add in tqdm( + iterable=move_email_dict.items(), desc="Move emails" + ): + if label_add is not None and label_add != label_existing: + self._modify_message_labels( + message_id=message_id, + label_id_remove_lst=[label_existing], + label_id_add_lst=[label_add], + ) + + def _store_emails_in_database(self, message_id_lst, email_format=None): + df = self._download_messages_to_dataframe( + message_id_lst=message_id_lst, email_format=email_format + ) + if len(df) > 0: + self._db_email.store_dataframe(df=df, user_id=self._db_user_id) + + @abstractmethod + def _search_email_on_server( + self, query_string="", label_lst=None, only_message_ids=False + ): + """ + Search emails either by a specific query or optionally limit your search to a list of labels + + Args: + query_string (str): query string to search for + label_lst (list): list of labels to be searched + only_message_ids (bool): return only the email IDs not the thread IDs - default: false + + Returns: + list: list of message ids (or backend-specific list items) matching the search + """ + + @abstractmethod + def _get_message_detail(self, message_id, email_format=None, metadata_headers=None): + """ + Get the raw, backend-specific representation of a single email message. + + Args: + message_id (str): id used by this backend to uniquely identify the email + email_format (str/None): backend-specific format hint + metadata_headers (list): backend-specific list of metadata headers + + Returns: + The backend-specific raw message representation, passed on to `_parse_message`. + """ + + @abstractmethod + def _get_label_translate_dict(self): + """ + Returns: + dict: mapping of label/folder display name to the backend-specific label/folder id + """ + + @abstractmethod + def _modify_message_labels( + self, message_id, label_id_remove_lst=None, label_id_add_lst=None + ): + """ + Apply a label/folder change to a single email message. + """ + + @abstractmethod + def _get_labels_for_email(self, message_id): + """ + Args: + message_id (str): id used by this backend to uniquely identify the email + + Returns: + list: list of labels/folders currently assigned to the email + """ + + @abstractmethod + def _parse_message(self, message): + """ + Args: + message: the backend-specific raw message representation returned by `_get_message_detail` + + Returns: + dict/None: the common gmailsorter email dict (see `gmailsorter.base.message.AbstractMessage.to_dict`), + or None if the message could not be parsed + """ +``` + +- [ ] **Step 2: Rewrite `gmailsorter/google/mail.py`** + +Replace the entire file content with: + +```python +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from gmailsorter.base import get_email_database +from gmailsorter.base.mail import AbstractMailBox +from gmailsorter.google.database import get_token_database +from gmailsorter.google.message import get_email_dict +from gmailsorter.ml import get_machine_learning_database + + +class GoogleMailBase(AbstractMailBox): + def __init__( + self, + google_mail_service, + database_email=None, + database_ml=None, + database_token=None, + user_id="me", + db_user_id=1, + email_download_format="metadata", + ): + """ + Gmail class to manage Emails via the Gmail API directly from Python + + Args: + google_mail_service: A Resource object with methods for interacting with the service. + database_email (gmailsorter.base.database.DatabaseInterface): SQLalchemy interface for email database + database_ml (gmailsorter.ml.database.DatabaseInterface): SQLalchemy interface for machine learning database + database_token (gmailsorter.google.database.DatabaseInterface): SQLalchemy interface for google database + user_id (str): in most cases this should be simply "me" + db_user_id (int): Default 1 - set a user id when sharing a database with multiple users + email_download_format (str): API response format [full, metadata] + """ + self._db_token = database_token + super().__init__( + mail_service=google_mail_service, + database_email=database_email, + database_ml=database_ml, + user_id=user_id, + db_user_id=db_user_id, + email_download_format=email_download_format, + ) + + def _get_label_translate_dict(self): + results = self._service.users().labels().list(userId=self._userid).execute() + labels = results.get("labels", []) + return {label["name"]: label["id"] for label in labels} + + def _get_message_detail(self, message_id, email_format=None, metadata_headers=None): + """ + Get details of a specific email message based on its email ID + + Args: + message_id (str): email IDs used by Google Mail to uniquely identify emails + email_format (str/None): API response format [raw, minimal, full, metadata] + metadata_headers (list): list of meta data headers + + Returns: + dict: details of the email as python dictionary + """ + if email_format is None: + email_format = self._email_download_format + if metadata_headers is None: + metadata_headers = [] + return ( + self._service.users() + .messages() + .get( + userId=self._userid, + id=message_id, + format=email_format, + metadataHeaders=metadata_headers, + ) + .execute() + ) + + def _get_messages_page(self, label_ids, query_string, next_page_token=None): + message_list_response = ( + self._service.users() + .messages() + .list( + userId=self._userid, + labelIds=label_ids, + q=query_string, + pageToken=next_page_token, + ) + .execute() + ) + + return [ + message_list_response.get("messages", []), + message_list_response.get("nextPageToken"), + ] + + def _get_messages(self, query_string="", label_ids=None): + if label_ids is None: + label_ids = [] + message_items_lst, next_page_token = self._get_messages_page( + label_ids=label_ids, query_string=query_string, next_page_token=None + ) + + while next_page_token: + message_items, next_page_token = self._get_messages_page( + label_ids=label_ids, + query_string=query_string, + next_page_token=next_page_token, + ) + message_items_lst.extend(message_items) + + return message_items_lst + + def _modify_message_labels( + self, message_id, label_id_remove_lst=None, label_id_add_lst=None + ): + if label_id_remove_lst is None: + label_id_remove_lst = [] + if label_id_add_lst is None: + label_id_add_lst = [] + body_dict = {} + if len(label_id_remove_lst) > 0: + body_dict["removeLabelIds"] = label_id_remove_lst + if len(label_id_add_lst) > 0: + body_dict["addLabelIds"] = label_id_add_lst + if len(body_dict) > 0: + self._service.users().messages().modify( + userId=self._userid, id=message_id, body=body_dict + ).execute() + + def _search_email_on_server( + self, query_string="", label_lst=None, only_message_ids=False + ): + """ + Search emails either by a specific query or optionally limit your search to a list of labels + + Args: + query_string (str): query string to search for + label_lst (list): list of labels to be searched + only_message_ids (bool): return only the email IDs not the thread IDs - default: false + + Returns: + list: list with email IDs and thread IDs of the messages which match the search + """ + if label_lst is None: + label_lst = [] + label_ids = [self._label_dict[label] for label in label_lst] + message_id_lst = self._get_messages( + query_string=query_string, label_ids=label_ids + ) + if not only_message_ids: + return message_id_lst + else: + return [d["id"] for d in message_id_lst] + + def _get_labels_for_email(self, message_id): + """ + Get labels for email + + Args: + message_id (str): email ID + + Returns: + list: List of email labels + """ + message_dict = self._get_message_detail( + message_id=message_id, + email_format="metadata", + metadata_headers=["labelIds"], + ) + if "labelIds" in message_dict: + return message_dict["labelIds"] + else: + return [] + + def _parse_message(self, message): + return get_email_dict(message=message) + + @staticmethod + def _create_databases(connection_str): + engine = create_engine(connection_str) + session = sessionmaker(bind=engine)() + db_email = get_email_database(engine=engine, session=session) + db_ml = get_machine_learning_database(engine=engine, session=session) + db_token = get_token_database(engine=engine, session=session) + return db_email, db_ml, db_token + + @staticmethod + def _get_message_ids(message_lst): + return [d["id"] for d in message_lst] +``` + +- [ ] **Step 3: Update patch targets in `tests/test_google_integration_units.py`** + +`encode_df_for_machine_learning`, `fit_machine_learning_models`, and `get_predictions_from_machine_learning_models` now execute from `gmailsorter.base.mail`, not `gmailsorter.google.mail`, so the two tests that patch them must point at the new location. In the `test_filter_messages_from_server` method: + +```python + @patch("gmailsorter.base.mail.get_predictions_from_machine_learning_models") + @patch("gmailsorter.base.mail.encode_df_for_machine_learning") + def test_filter_messages_from_server(self, encode_mock, predict_mock): +``` + +(was `@patch("gmailsorter.google.mail.get_predictions_from_machine_learning_models")` / `@patch("gmailsorter.google.mail.encode_df_for_machine_learning")`) + +In the `test_fit_machine_learning_model_to_database` method: + +```python + @patch("gmailsorter.base.mail.fit_machine_learning_models") + @patch("gmailsorter.base.mail.encode_df_for_machine_learning") + def test_fit_machine_learning_model_to_database(self, encode_mock, fit_mock): +``` + +(was `@patch("gmailsorter.google.mail.fit_machine_learning_models")` / `@patch("gmailsorter.google.mail.encode_df_for_machine_learning")`) + +No other lines in this file change — every assertion stays exactly as-is. + +- [ ] **Step 4: Run the full existing suite to confirm no regression** + +Run: `python -m unittest discover tests -v` +Expected: All tests PASS, including every test in `tests/test_google_integration_units.py` with unchanged assertions. + +- [ ] **Step 5: Create `tests/test_mail_base.py` to test the extracted loop in isolation** + +```python +from unittest import TestCase +from unittest.mock import MagicMock, patch + +import pandas as pd + +from gmailsorter.base.mail import AbstractMailBox + + +class _StubMailBox(AbstractMailBox): + """Minimal concrete AbstractMailBox used to test the shared loop in isolation.""" + + def __init__(self, label_dict_fixture=None, **kwargs): + self.label_dict_fixture = label_dict_fixture or {"Inbox": "Inbox", "Spam": "Spam"} + self.search_result = [] + self.message_detail_dict = {} + self.modify_calls = [] + self.labels_for_email_dict = {} + super().__init__(mail_service=MagicMock(), **kwargs) + + def _search_email_on_server(self, query_string="", label_lst=None, only_message_ids=False): + return self.search_result + + def _get_message_detail(self, message_id, email_format=None, metadata_headers=None): + return self.message_detail_dict.get(message_id) + + def _get_label_translate_dict(self): + return self.label_dict_fixture + + def _modify_message_labels(self, message_id, label_id_remove_lst=None, label_id_add_lst=None): + self.modify_calls.append((message_id, label_id_remove_lst, label_id_add_lst)) + + def _get_labels_for_email(self, message_id): + return self.labels_for_email_dict.get(message_id, []) + + def _parse_message(self, message): + return message + + +class AbstractMailBoxTest(TestCase): + def test_labels_property(self): + mailbox = _StubMailBox() + self.assertEqual(sorted(mailbox.labels), ["Inbox", "Spam"]) + + def test_download_emails_for_label(self): + mailbox = _StubMailBox() + mailbox.search_result = ["id1", "id2"] + mailbox.message_detail_dict = { + "id1": { + "id": "id1", + "threads": "t1", + "labels": [], + "to": [], + "from": None, + "cc": [], + "subject": "s1", + "content": "c1", + "date": None, + }, + "id2": None, + } + + df = mailbox.download_emails_for_label(label="Inbox") + + self.assertEqual(df["id"].tolist(), ["id1"]) + + def test_move_emails_skips_matching_or_none_labels(self): + mailbox = _StubMailBox() + + mailbox._move_emails( + move_email_dict={"id1": None, "id2": "Inbox", "id3": "Spam"}, + label_to_ignore="Inbox", + ) + + self.assertEqual(mailbox.modify_calls, [("id3", ["Inbox"], ["Spam"])]) + + def test_update_database_marks_missing_as_deleted(self): + db_email = MagicMock() + db_email.get_labels_to_update.return_value = (["new"], [], ["deleted"]) + mailbox = _StubMailBox(database_email=db_email) + mailbox.search_result = ["new"] + mailbox.message_detail_dict = { + "new": { + "id": "new", + "threads": "t", + "labels": [], + "to": [], + "from": None, + "cc": [], + "subject": "s", + "content": "c", + "date": None, + } + } + + mailbox.update_database(quick=False) + + db_email.mark_emails_as_deleted.assert_called_once_with( + message_id_lst=["deleted"], user_id=1 + ) + db_email.store_dataframe.assert_called_once() + + @patch("gmailsorter.base.mail.fit_machine_learning_models") + @patch("gmailsorter.base.mail.encode_df_for_machine_learning") + def test_fit_machine_learning_model_to_database(self, encode_mock, fit_mock): + db_email = MagicMock() + db_email.get_all_emails.return_value = pd.DataFrame( + [{"id": "x", "from": "a@b.com", "to": [], "cc": [], "labels": [], "threads": "t"}] + ) + db_ml = MagicMock() + mailbox = _StubMailBox(database_email=db_email, database_ml=db_ml) + features = pd.DataFrame([{"email_id": "x", "f1": 1}]) + labels = pd.DataFrame([{"labels_Inbox": 1}]) + encode_mock.return_value = (features, labels) + fit_mock.return_value = {"Inbox": MagicMock()} + + mailbox.fit_machine_learning_model_to_database(n_estimators=5, max_features=2) + + db_ml.store_models.assert_called_once() +``` + +- [ ] **Step 6: Run the new test to verify it passes** + +Run: `python -m unittest tests.test_mail_base -v` +Expected: PASS (5 tests) + +- [ ] **Step 7: Commit** + +```bash +git add gmailsorter/base/mail.py gmailsorter/google/mail.py tests/test_google_integration_units.py tests/test_mail_base.py +git commit -m "refactor: extract AbstractMailBox loop from GoogleMailBase into base/mail.py" +``` + +--- + +### Task 3: `gmailsorter/imap/message.py` + +**Files:** +- Create: `gmailsorter/imap/__init__.py` (empty package marker for now — populated in Task 6) +- Create: `gmailsorter/imap/message.py` +- Test: `tests/test_imap_message.py` + +**Interfaces:** +- Consumes: `gmailsorter.base.message.AbstractMessage`, `gmailsorter.base.message.strip_html_tags` (Task 1). +- Produces: `gmailsorter.imap.message.Message(AbstractMessage)` with constructor `Message(message, folder, uid)`, and `gmailsorter.imap.message.get_email_dict(message, folder, uid) -> dict | None`. Consumed by Task 5 (`ImapMailBase._parse_message`). + +- [ ] **Step 1: Create the package marker** + +Create `gmailsorter/imap/__init__.py` with just: + +```python +``` + +(empty file — populated with real exports in Task 6, once `authentication.py` and `mail.py` exist) + +- [ ] **Step 2: Write the failing test** + +Create `tests/test_imap_message.py`: + +```python +from datetime import datetime +from email.message import EmailMessage +from unittest import TestCase + +from gmailsorter.imap.message import Message, get_email_dict + + +class MessageTest(TestCase): + @classmethod + def setUpClass(cls) -> None: + msg = EmailMessage() + msg["Subject"] = "Test Email Subject" + msg["From"] = "sender@server.net" + msg["To"] = "me@mail.com, friend@provider.org" + msg["Date"] = "Fri, 11 Feb 2022 18:08:46 +0100" + msg["Message-ID"] = "" + msg.set_content("Hello world") + cls._message = msg + cls.message = Message(message=msg, folder="INBOX", uid="42") + + def test_subject(self): + self.assertEqual(self.message.get_subject(), "Test Email Subject") + + def test_from(self): + self.assertEqual(self.message.get_from(), "sender@server.net") + + def test_to(self): + self.assertEqual( + self.message.get_to(), ["me@mail.com", "friend@provider.org"] + ) + + def test_cc_empty(self): + self.assertEqual(self.message.get_cc(), []) + + def test_email_id(self): + self.assertEqual(self.message.get_email_id(), "INBOX\x1f42") + + def test_thread_id_falls_back_to_message_id(self): + self.assertEqual(self.message.get_thread_id(), "") + + def test_label_ids(self): + self.assertEqual(self.message.get_label_ids(), ["INBOX"]) + + def test_get_date(self): + self.assertEqual( + self.message.get_date(), + datetime.strptime( + "Fri, 11 Feb 2022 18:08:46 +0100", "%a, %d %b %Y %H:%M:%S %z" + ), + ) + + def test_get_content(self): + self.assertEqual(self.message.get_content().strip(), "Hello world") + + def test_get_content_html_fallback(self): + html_msg = EmailMessage() + html_msg["Subject"] = "HTML" + html_msg["From"] = "sender@server.net" + html_msg["To"] = "me@mail.com" + html_msg["Date"] = "Fri, 11 Feb 2022 18:08:46 +0100" + html_msg.set_content("

Hello World

", subtype="html") + message = Message(message=html_msg, folder="INBOX", uid="43") + + self.assertEqual(message.get_content().strip(), "Hello World") + + def test_thread_id_uses_references_header(self): + msg = EmailMessage() + msg["Subject"] = "Re: Test" + msg["References"] = " " + msg["Message-ID"] = "" + message = Message(message=msg, folder="INBOX", uid="44") + + self.assertEqual(message.get_thread_id(), "") + + def test_from_with_multiple_addresses_is_none(self): + msg = EmailMessage() + msg["From"] = "a@server.net, b@server.net" + message = Message(message=msg, folder="INBOX", uid="45") + + self.assertIsNone(message.get_from()) + + def test_get_email_dict(self): + result = get_email_dict(self._message, folder="INBOX", uid="42") + content = result.pop("content") + + self.assertEqual(content.strip(), "Hello world") + self.assertEqual( + result, + { + "cc": [], + "date": datetime.strptime( + "Fri, 11 Feb 2022 18:08:46 +0100", "%a, %d %b %Y %H:%M:%S %z" + ), + "from": "sender@server.net", + "id": "INBOX\x1f42", + "labels": ["INBOX"], + "subject": "Test Email Subject", + "threads": "", + "to": ["me@mail.com", "friend@provider.org"], + }, + ) +``` + +- [ ] **Step 3: Run test to verify it fails** + +Run: `python -m unittest tests.test_imap_message -v` +Expected: FAIL with `ModuleNotFoundError: No module named 'gmailsorter.imap.message'` + +- [ ] **Step 4: Implement `gmailsorter/imap/message.py`** + +```python +import email.utils + +from gmailsorter.base.message import AbstractMessage, strip_html_tags + + +def get_email_dict(message, folder, uid): + try: + return Message(message=message, folder=folder, uid=uid).to_dict() + except (ValueError, KeyError) as e: + print(message, str(e)) + return None + + +class Message(AbstractMessage): + def __init__(self, message, folder, uid): + """ + Message class to parse a raw email.message.Message (as produced by + email.message_from_bytes() after an IMAP FETCH) into the common gmailsorter + email representation. + + Args: + message (email.message.Message): parsed RFC822 message + folder (str): IMAP mailbox/folder the message was fetched from + uid (str): IMAP UID of the message within `folder` + """ + self._message = message + self._folder = folder + self._uid = str(uid) + + def get_from(self): + from_header = self._message.get("From") + if from_header is None: + return None + addresses = [ + address + for _, address in email.utils.getaddresses([from_header]) + if address + ] + if len(addresses) == 1: + return addresses[0].lower() + return None + + def get_to(self): + return self._split_addresses(self._message.get_all("To")) + + def get_cc(self): + return self._split_addresses(self._message.get_all("Cc")) + + def get_label_ids(self): + return [self._folder] + + def get_subject(self): + return self._message.get("Subject") + + def get_date(self): + date_header = self._message.get("Date") + if date_header is None: + return None + return email.utils.parsedate_to_datetime(date_header) + + def get_content(self): + text_plain, text_html = None, None + if self._message.is_multipart(): + for part in self._message.walk(): + if part.get_content_maintype() == "multipart": + continue + if part.get_content_type() == "text/plain" and text_plain is None: + text_plain = self._decode_part(part) + elif part.get_content_type() == "text/html" and text_html is None: + text_html = self._decode_part(part) + elif self._message.get_content_type() == "text/plain": + text_plain = self._decode_part(self._message) + elif self._message.get_content_type() == "text/html": + text_html = self._decode_part(self._message) + if text_plain is not None: + return text_plain + elif text_html is not None: + return strip_html_tags(text_html) + else: + return None + + def get_thread_id(self): + references = self._message.get("References") + if references: + return references.split()[0] + in_reply_to = self._message.get("In-Reply-To") + if in_reply_to: + return in_reply_to.strip() + message_id = self._message.get("Message-ID") + if message_id: + return message_id.strip() + return self.get_email_id() + + def get_email_id(self): + return f"{self._folder}\x1f{self._uid}" + + @staticmethod + def _decode_part(part): + payload = part.get_payload(decode=True) + if payload is None: + return "" + charset = part.get_content_charset() or "utf-8" + return payload.decode(charset, errors="replace") + + @staticmethod + def _split_addresses(header_values): + if not header_values: + return [] + return [ + address.lower() + for _, address in email.utils.getaddresses(header_values) + if address + ] +``` + +- [ ] **Step 5: Run test to verify it passes** + +Run: `python -m unittest tests.test_imap_message -v` +Expected: PASS (13 tests) + +- [ ] **Step 6: Commit** + +```bash +git add gmailsorter/imap/__init__.py gmailsorter/imap/message.py tests/test_imap_message.py +git commit -m "feat: add IMAP message parsing (gmailsorter.imap.message)" +``` + +--- + +### Task 4: `gmailsorter/imap/authentication.py` + +**Files:** +- Create: `gmailsorter/imap/authentication.py` +- Test: `tests/test_imap_integration_units.py` (new file — also extended in Tasks 5 and 6) + +**Interfaces:** +- Produces: `gmailsorter.imap.authentication.create_service(host, port, username, password, use_ssl=True) -> imaplib.IMAP4`. Consumed by Task 6 (`Imap.__init__`). + +- [ ] **Step 1: Write the failing test** + +Create `tests/test_imap_integration_units.py`: + +```python +from unittest import TestCase +from unittest.mock import patch + +from gmailsorter.imap.authentication import create_service + + +class TestImapAuthentication(TestCase): + @patch("gmailsorter.imap.authentication.IMAP4_SSL") + def test_create_service_uses_ssl_by_default(self, imap_ssl_cls): + connection = imap_ssl_cls.return_value + + result = create_service( + host="localhost", port=993, username="user", password="secret" + ) + + imap_ssl_cls.assert_called_once_with("localhost", 993) + connection.login.assert_called_once_with("user", "secret") + self.assertIs(result, connection) + + @patch("gmailsorter.imap.authentication.IMAP4") + def test_create_service_without_ssl(self, imap_cls): + connection = imap_cls.return_value + + result = create_service( + host="localhost", + port=143, + username="user", + password="secret", + use_ssl=False, + ) + + imap_cls.assert_called_once_with("localhost", 143) + connection.login.assert_called_once_with("user", "secret") + self.assertIs(result, connection) + + +if __name__ == "__main__": + import unittest + + unittest.main() +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python -m unittest tests.test_imap_integration_units -v` +Expected: FAIL with `ModuleNotFoundError: No module named 'gmailsorter.imap.authentication'` + +- [ ] **Step 3: Implement `gmailsorter/imap/authentication.py`** + +```python +from imaplib import IMAP4, IMAP4_SSL + + +def create_service(host, port, username, password, use_ssl=True): + """ + Open and log in to an IMAP connection. + + Args: + host (str): IMAP server hostname + port (int): IMAP server port + username (str): IMAP account username + password (str): IMAP account password + use_ssl (bool): connect via IMAP4_SSL (default) or plain IMAP4 + + Returns: + imaplib.IMAP4: logged-in IMAP connection + """ + connection_cls = IMAP4_SSL if use_ssl else IMAP4 + connection = connection_cls(host, port) + connection.login(username, password) + return connection +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `python -m unittest tests.test_imap_integration_units -v` +Expected: PASS (2 tests) + +- [ ] **Step 5: Commit** + +```bash +git add gmailsorter/imap/authentication.py tests/test_imap_integration_units.py +git commit -m "feat: add IMAP username/password authentication (gmailsorter.imap.authentication)" +``` + +--- + +### Task 5: `gmailsorter/imap/mail.py` + +**Files:** +- Create: `gmailsorter/imap/mail.py` +- Modify: `tests/test_imap_integration_units.py` (append) + +**Interfaces:** +- Consumes: `gmailsorter.base.mail.AbstractMailBox` (Task 2), `gmailsorter.imap.message.get_email_dict` (Task 3). +- Produces: `gmailsorter.imap.mail.ImapMailBase(AbstractMailBox)` (no custom `__init__` — inherits `AbstractMailBox.__init__`), plus `ImapMailBase._create_databases(connection_str) -> (database_email, database_ml)`. Consumed by Task 6 (`Imap` class, `gmailsorter/imap/__init__.py`). +- Composite message id format: `f"{folder}\x1f{uid}"` (matches `gmailsorter.imap.message.Message.get_email_id`). + +- [ ] **Step 1: Write the failing tests** + +Append to `tests/test_imap_integration_units.py` (add these imports at the top, alongside the existing ones): + +```python +from unittest.mock import MagicMock +``` + +```python +from gmailsorter.imap.mail import ImapMailBase +``` + +Then add this test class at the end of the file (before the `if __name__ == "__main__":` block): + +```python +class TestImapMailBase(TestCase): + def _create_mock_service_with_folders(self, folders=None): + service = MagicMock() + service.capabilities = ["IMAP4rev1", "MOVE"] + service.list.return_value = ( + "OK", + folders + if folders is not None + else [ + b'(\\HasNoChildren) "/" "INBOX"', + b'(\\HasNoChildren) "/" "MailSortInbox"', + b'(\\Noselect \\HasChildren) "/" "[Gmail]"', + ], + ) + return service + + def test_get_label_translate_dict_skips_noselect(self): + service = self._create_mock_service_with_folders() + mail = ImapMailBase(mail_service=service) + + self.assertEqual(sorted(mail.labels), ["INBOX", "MailSortInbox"]) + + def test_search_email_on_server_single_folder(self): + service = self._create_mock_service_with_folders() + service.select.return_value = ("OK", [b"1"]) + service.uid.return_value = ("OK", [b"1 2"]) + mail = ImapMailBase(mail_service=service) + + ids = mail._search_email_on_server(label_lst=["INBOX"], only_message_ids=True) + + service.select.assert_called_once_with('"INBOX"') + service.uid.assert_called_once_with("search", None, "ALL") + self.assertEqual(ids, ["INBOX\x1f1", "INBOX\x1f2"]) + + def test_search_email_on_server_all_folders_when_no_label(self): + service = self._create_mock_service_with_folders() + service.select.return_value = ("OK", [b"1"]) + service.uid.return_value = ("OK", [b"5"]) + mail = ImapMailBase(mail_service=service) + + ids = mail._search_email_on_server(only_message_ids=True) + + self.assertEqual( + service.select.call_args_list, + [(('"INBOX"',),), (('"MailSortInbox"',),)], + ) + self.assertEqual(ids, ["INBOX\x1f5", "MailSortInbox\x1f5"]) + + def test_search_email_on_server_rejects_query_string(self): + service = self._create_mock_service_with_folders() + mail = ImapMailBase(mail_service=service) + + with self.assertRaises(NotImplementedError): + mail._search_email_on_server(query_string="SUBJECT foo") + + def test_get_message_detail_selects_and_fetches(self): + service = self._create_mock_service_with_folders() + raw_message = b"Subject: hi\r\nFrom: a@b.com\r\nTo: c@d.com\r\n\r\nbody" + service.select.return_value = ("OK", [b"1"]) + service.uid.return_value = ("OK", [(b"1 (RFC822 {10}", raw_message)]) + mail = ImapMailBase(mail_service=service) + + folder, uid, message = mail._get_message_detail(message_id="INBOX\x1f7") + + service.select.assert_called_once_with('"INBOX"') + service.uid.assert_called_once_with("fetch", "7", "(RFC822)") + self.assertEqual(folder, "INBOX") + self.assertEqual(uid, "7") + self.assertEqual(message["Subject"], "hi") + + def test_get_labels_for_email_from_composite_id(self): + service = self._create_mock_service_with_folders() + mail = ImapMailBase(mail_service=service) + + self.assertEqual(mail._get_labels_for_email("INBOX\x1f7"), ["INBOX"]) + + def test_modify_message_labels_uses_move_when_supported(self): + service = self._create_mock_service_with_folders() + service.select.return_value = ("OK", [b"1"]) + service.uid.return_value = ("OK", [b"1"]) + mail = ImapMailBase(mail_service=service) + + mail._modify_message_labels( + message_id="INBOX\x1f7", + label_id_remove_lst=["INBOX"], + label_id_add_lst=["MailSortInbox"], + ) + + service.select.assert_called_once_with('"INBOX"') + service.uid.assert_called_once_with("move", "7", '"MailSortInbox"') + service.expunge.assert_not_called() + + def test_modify_message_labels_falls_back_to_copy_delete(self): + service = self._create_mock_service_with_folders() + service.capabilities = ["IMAP4rev1"] + service.select.return_value = ("OK", [b"1"]) + service.uid.return_value = ("OK", [b"1"]) + mail = ImapMailBase(mail_service=service) + + mail._modify_message_labels( + message_id="INBOX\x1f7", + label_id_remove_lst=["INBOX"], + label_id_add_lst=["MailSortInbox"], + ) + + self.assertEqual( + service.uid.call_args_list, + [ + (("copy", "7", '"MailSortInbox"'),), + (("store", "7", "+FLAGS", r"(\Deleted)"),), + ], + ) + service.expunge.assert_called_once() + + def test_modify_message_labels_noop_without_target(self): + service = self._create_mock_service_with_folders() + mail = ImapMailBase(mail_service=service) + + mail._modify_message_labels(message_id="INBOX\x1f7") + + service.select.assert_not_called() + + @patch("gmailsorter.imap.mail.get_email_dict") + def test_parse_message_delegates_to_get_email_dict(self, get_email_dict_mock): + service = self._create_mock_service_with_folders() + mail = ImapMailBase(mail_service=service) + get_email_dict_mock.return_value = {"id": "INBOX\x1f7"} + + result = mail._parse_message(("INBOX", "7", "raw")) + + get_email_dict_mock.assert_called_once_with( + message="raw", folder="INBOX", uid="7" + ) + self.assertEqual(result, {"id": "INBOX\x1f7"}) + + def test_create_databases(self): + with ( + patch("gmailsorter.imap.mail.create_engine") as create_engine_mock, + patch("gmailsorter.imap.mail.sessionmaker") as sessionmaker_mock, + patch("gmailsorter.imap.mail.get_email_database") as get_email_db_mock, + patch( + "gmailsorter.imap.mail.get_machine_learning_database" + ) as get_ml_db_mock, + ): + engine = MagicMock() + session = MagicMock() + create_engine_mock.return_value = engine + sessionmaker_mock.return_value.return_value = session + get_email_db_mock.return_value = "EMAIL_DB" + get_ml_db_mock.return_value = "ML_DB" + + dbs = ImapMailBase._create_databases("sqlite:///file.db") + + self.assertEqual(dbs, ("EMAIL_DB", "ML_DB")) +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `python -m unittest tests.test_imap_integration_units -v` +Expected: FAIL with `ModuleNotFoundError: No module named 'gmailsorter.imap.mail'` + +- [ ] **Step 3: Implement `gmailsorter/imap/mail.py`** + +```python +import email +import re + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from gmailsorter.base import get_email_database +from gmailsorter.base.mail import AbstractMailBox +from gmailsorter.imap.message import get_email_dict +from gmailsorter.ml import get_machine_learning_database + +_LIST_ENTRY_PATTERN = re.compile( + r'\((?P[^)]*)\)\s+"(?P.*)"\s+(?P.+)' +) + + +class ImapMailBase(AbstractMailBox): + def _get_label_translate_dict(self): + status, mailbox_lst = self._service.list() + if status != "OK" or mailbox_lst is None: + return {} + label_dict = {} + for entry in mailbox_lst: + flags, _delimiter, name = self._parse_list_entry(entry) + if "\\Noselect" in flags: + continue + label_dict[name] = name + return label_dict + + def _search_email_on_server( + self, query_string="", label_lst=None, only_message_ids=False + ): + """ + Search emails either by a specific query or optionally limit your search to a list of labels + + Args: + query_string (str): not supported yet - must be empty + label_lst (list): list of IMAP folders to search; if empty, every folder is searched + only_message_ids (bool): return only the composite email IDs - default: false + + Returns: + list: list of composite "{folder}\\x1f{uid}" ids matching the search + """ + if query_string: + raise NotImplementedError( + "Custom IMAP search queries are not supported yet, only label_lst filtering." + ) + if label_lst is None: + label_lst = [] + folder_lst = label_lst if len(label_lst) > 0 else list(self._label_dict.keys()) + message_id_lst = [ + f"{folder}\x1f{uid}" + for folder in folder_lst + for uid in self._search_folder(folder=folder) + ] + if only_message_ids: + return message_id_lst + else: + return [{"id": message_id} for message_id in message_id_lst] + + def _search_folder(self, folder): + status, _ = self._service.select(f'"{folder}"') + if status != "OK": + return [] + status, data = self._service.uid("search", None, "ALL") + if status != "OK" or data[0] is None: + return [] + return [ + uid.decode() if isinstance(uid, bytes) else uid for uid in data[0].split() + ] + + def _get_message_detail(self, message_id, email_format=None, metadata_headers=None): + """ + Fetch the raw RFC822 message for a composite "{folder}\\x1f{uid}" id. + + Returns: + tuple: (folder, uid, email.message.Message) + """ + folder, uid = message_id.split("\x1f", 1) + status, _ = self._service.select(f'"{folder}"') + if status != "OK": + raise RuntimeError(f"Could not select IMAP folder {folder!r}") + status, data = self._service.uid("fetch", uid, "(RFC822)") + if status != "OK" or not data or data[0] is None: + raise RuntimeError(f"Could not fetch IMAP message {message_id!r}") + raw_message = data[0][1] + parsed_message = email.message_from_bytes(raw_message) + return folder, uid, parsed_message + + def _modify_message_labels( + self, message_id, label_id_remove_lst=None, label_id_add_lst=None + ): + if not label_id_add_lst: + return + folder, uid = message_id.split("\x1f", 1) + target_folder = label_id_add_lst[0] + status, _ = self._service.select(f'"{folder}"') + if status != "OK": + raise RuntimeError(f"Could not select IMAP folder {folder!r}") + if "MOVE" in self._service.capabilities: + status, _ = self._service.uid("move", uid, f'"{target_folder}"') + if status != "OK": + raise RuntimeError( + f"Could not move IMAP message {message_id!r} to {target_folder!r}" + ) + else: + status, _ = self._service.uid("copy", uid, f'"{target_folder}"') + if status != "OK": + raise RuntimeError( + f"Could not copy IMAP message {message_id!r} to {target_folder!r}" + ) + self._service.uid("store", uid, "+FLAGS", r"(\Deleted)") + self._service.expunge() + + def _get_labels_for_email(self, message_id): + folder, _uid = message_id.split("\x1f", 1) + return [folder] + + def _parse_message(self, message): + folder, uid, parsed_message = message + return get_email_dict(message=parsed_message, folder=folder, uid=uid) + + @staticmethod + def _parse_list_entry(entry): + decoded = entry.decode() if isinstance(entry, bytes) else entry + match = _LIST_ENTRY_PATTERN.match(decoded) + flags = match.group("flags").split() + delimiter = match.group("delimiter") + name = match.group("name").strip('"') + return flags, delimiter, name + + @staticmethod + def _create_databases(connection_str): + engine = create_engine(connection_str) + session = sessionmaker(bind=engine)() + db_email = get_email_database(engine=engine, session=session) + db_ml = get_machine_learning_database(engine=engine, session=session) + return db_email, db_ml +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `python -m unittest tests.test_imap_integration_units -v` +Expected: PASS (all tests in the file, including the `TestImapAuthentication` tests from Task 4) + +- [ ] **Step 5: Commit** + +```bash +git add gmailsorter/imap/mail.py tests/test_imap_integration_units.py +git commit -m "feat: add ImapMailBase (folders-as-labels, MOVE/COPY+EXPUNGE)" +``` + +--- + +### Task 6: `Imap` class in `local.py`, IMAP package exports, top-level export + +**Files:** +- Modify: `gmailsorter/imap/__init__.py` +- Modify: `gmailsorter/local.py` +- Modify: `gmailsorter/__init__.py` +- Modify: `tests/test_imap_integration_units.py` (append) + +**Interfaces:** +- Produces: `gmailsorter.imap.create_service`, `gmailsorter.imap.ImapMailBase` (re-exports), `gmailsorter.local.Imap(host, port, username, password, connection_str, db_user_id=1, use_ssl=True, email_download_format="metadata")`, `gmailsorter.Imap`. Consumed by Task 7 (CLI). + +- [ ] **Step 1: Write the failing test** + +Append to `tests/test_imap_integration_units.py` (add these imports at the top, alongside the existing ones): + +```python +from gmailsorter.local import Imap +``` + +Then add this test class at the end of the file (before the `if __name__ == "__main__":` block): + +```python +class TestImapLocalHelpers(TestCase): + @patch("gmailsorter.local.ImapMailBase.__init__", return_value=None) + @patch("gmailsorter.local.create_imap_service") + @patch("gmailsorter.local.Imap._create_databases") + def test_imap_initialization_wiring( + self, create_databases_mock, create_service_mock, base_init_mock + ): + db_email, db_ml = MagicMock(), MagicMock() + create_databases_mock.return_value = (db_email, db_ml) + connection = MagicMock() + create_service_mock.return_value = connection + + Imap( + host="localhost", + port=993, + username="user", + password="secret", + connection_str="sqlite:///:memory:", + db_user_id=4, + ) + + create_databases_mock.assert_called_once_with( + connection_str="sqlite:///:memory:" + ) + create_service_mock.assert_called_once_with( + host="localhost", + port=993, + username="user", + password="secret", + use_ssl=True, + ) + base_init_mock.assert_called_once_with( + mail_service=connection, + database_email=db_email, + database_ml=db_ml, + user_id="user", + db_user_id=4, + email_download_format="metadata", + ) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python -m unittest tests.test_imap_integration_units -v` +Expected: FAIL with `ImportError: cannot import name 'Imap' from 'gmailsorter.local'` + +- [ ] **Step 3: Populate `gmailsorter/imap/__init__.py`** + +```python +from gmailsorter.imap.authentication import create_service +from gmailsorter.imap.mail import ImapMailBase + +__all__ = ["create_service", "ImapMailBase"] +``` + +- [ ] **Step 4: Add `Imap` to `gmailsorter/local.py`** + +Add these imports at the top of `gmailsorter/local.py` (alongside the existing ones): + +```python +from gmailsorter.imap import ImapMailBase +from gmailsorter.imap import create_service as create_imap_service +``` + +Then append the `Imap` class at the end of the file, after `load_client_secrets_file`: + +```python +class Imap(ImapMailBase): + def __init__( + self, + host, + port, + username, + password, + connection_str, + db_user_id=1, + use_ssl=True, + email_download_format="metadata", + ): + """ + Imap class to manage Emails via a plain IMAP connection directly from Python + + Args: + host (str): IMAP server hostname + port (int): IMAP server port, typically 993 for IMAP4_SSL or 143 for IMAP4 + username (str): IMAP account username + password (str): IMAP account password + connection_str (str): SQLalchemy compatible connection string to connect to the SQL database + db_user_id (int): Default 1 - set a user id when sharing a database with multiple users + use_ssl (bool): connect via IMAP4_SSL (default) or plain IMAP4 + email_download_format (str): unused for IMAP, kept for interface parity with Gmail + """ + self._connection_str = connection_str + + database_email, database_ml = self._create_databases( + connection_str=self._connection_str + ) + + imap_connection = create_imap_service( + host=host, + port=port, + username=username, + password=password, + use_ssl=use_ssl, + ) + + super().__init__( + mail_service=imap_connection, + database_email=database_email, + database_ml=database_ml, + user_id=username, + db_user_id=db_user_id, + email_download_format=email_download_format, + ) +``` + +- [ ] **Step 5: Export `Imap` from `gmailsorter/__init__.py`** + +Replace the file content with: + +```python +from gmailsorter.local import Gmail, Imap, load_client_secrets_file + +from . import _version + +__version__: str = _version.__version__ +__all__ = ["Gmail", "Imap", "load_client_secrets_file"] +``` + +- [ ] **Step 6: Run test to verify it passes** + +Run: `python -m unittest tests.test_imap_integration_units -v` +Expected: PASS (all tests) + +- [ ] **Step 7: Run the full suite** + +Run: `python -m unittest discover tests -v` +Expected: All tests PASS + +- [ ] **Step 8: Commit** + +```bash +git add gmailsorter/imap/__init__.py gmailsorter/local.py gmailsorter/__init__.py tests/test_imap_integration_units.py +git commit -m "feat: add Imap convenience class and gmailsorter.Imap export" +``` + +--- + +### Task 7: `gmailsorter-imap` CLI + +**Files:** +- Create: `gmailsorter/imap/__main__.py` +- Modify: `pyproject.toml` +- Test: `tests/test_imap_cli.py` + +**Interfaces:** +- Consumes: `gmailsorter.Imap` (Task 6). +- Produces: `gmailsorter.imap.__main__.command_line_parser()`, console script `gmailsorter-imap`. + +- [ ] **Step 1: Write the failing tests** + +Create `tests/test_imap_cli.py`: + +```python +import os +from unittest import TestCase +from unittest.mock import patch + +from gmailsorter.imap.__main__ import command_line_parser + + +class ImapCliTest(TestCase): + @patch("gmailsorter.imap.__main__.Imap") + def test_update_wires_imap_and_triggers_update(self, imap_cls): + imap_instance = imap_cls.return_value + os.environ["IMAP_PASSWORD"] = "secret" + try: + with patch( + "sys.argv", + [ + "gmailsorter-imap", + "--host", + "localhost", + "--port", + "993", + "--username", + "user", + "-d", + "sqlite:///:memory:", + "-u", + ], + ): + command_line_parser() + finally: + del os.environ["IMAP_PASSWORD"] + + imap_cls.assert_called_once_with( + host="localhost", + port=993, + username="user", + password="secret", + connection_str="sqlite:///:memory:", + db_user_id=1, + use_ssl=True, + email_download_format="metadata", + ) + imap_instance.update_database.assert_called_once_with(quick=False) + imap_instance.fit_machine_learning_model_to_database.assert_called_once_with( + n_estimators=100, + max_features=400, + random_state=42, + bootstrap=True, + include_deleted=False, + ) + + @patch("gmailsorter.imap.__main__.Imap") + def test_label_wires_imap_and_triggers_filter(self, imap_cls): + imap_instance = imap_cls.return_value + os.environ["IMAP_PASSWORD"] = "secret" + try: + with patch( + "sys.argv", + [ + "gmailsorter-imap", + "--host", + "localhost", + "--username", + "user", + "-d", + "sqlite:///:memory:", + "-l", + "MailSortInbox", + ], + ): + command_line_parser() + finally: + del os.environ["IMAP_PASSWORD"] + + imap_instance.filter_messages_from_server.assert_called_once_with( + label="MailSortInbox", recommendation_ratio=0.9 + ) + + @patch("gmailsorter.imap.__main__.Imap") + def test_missing_password_env_skips_wiring(self, imap_cls): + os.environ.pop("IMAP_PASSWORD", None) + with patch( + "sys.argv", + ["gmailsorter-imap", "--host", "localhost", "--username", "user"], + ): + command_line_parser() + + imap_cls.assert_not_called() + + @patch("gmailsorter.imap.__main__.Imap") + def test_missing_host_skips_wiring(self, imap_cls): + with patch("sys.argv", ["gmailsorter-imap", "--username", "user"]): + command_line_parser() + + imap_cls.assert_not_called() + + +if __name__ == "__main__": + import unittest + + unittest.main() +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `python -m unittest tests.test_imap_cli -v` +Expected: FAIL with `ModuleNotFoundError: No module named 'gmailsorter.imap.__main__'` + +- [ ] **Step 3: Implement `gmailsorter/imap/__main__.py`** + +```python +import argparse +import os + +from gmailsorter import Imap + + +def command_line_parser(): + """ + Main function primarily used for the command line interface of the IMAP backend + """ + parser = argparse.ArgumentParser(prog="gmailsorter-imap") + parser.add_argument( + "--host", + help="IMAP server hostname e.g. imap.example.com .", + ) + parser.add_argument( + "--port", + type=int, + default=993, + help="IMAP server port - default: 993 .", + ) + parser.add_argument( + "--username", + help="IMAP account username.", + ) + parser.add_argument( + "--password-env", + default="IMAP_PASSWORD", + help=( + "Name of the environment variable holding the IMAP account password - " + "default: IMAP_PASSWORD ." + ), + ) + parser.add_argument( + "--no-ssl", + action="store_true", + help="Connect without SSL (IMAP4 instead of IMAP4_SSL).", + ) + parser.add_argument( + "-d", + "--database", + help="Connection string to connect to database e.g. sqlite:///email.db .", + ) + parser.add_argument( + "-u", + "--update", + action="store_true", + help="Update local database and retrain machine learning model.", + ) + parser.add_argument( + "-i", + "--identification", + help="User ID of the database user e.g. 1 .", + ) + parser.add_argument( + "-l", + "--label", + help="Email label (IMAP folder) to be filtered with machine learning.", + ) + args = parser.parse_args() + db_user_id = int(args.identification) if args.identification else 1 + password = os.environ.get(args.password_env) + if not args.host or not args.username: + print("Please provide --host and --username.") + elif not password: + print( + f"Please set the {args.password_env} environment variable to your IMAP password." + ) + else: + database = args.database or "sqlite:///email.db" + imap = Imap( + host=args.host, + port=args.port, + username=args.username, + password=password, + connection_str=database, + db_user_id=db_user_id, + use_ssl=not args.no_ssl, + email_download_format="metadata", + ) + if args.update: + imap.update_database(quick=False) + imap.fit_machine_learning_model_to_database( + n_estimators=100, + max_features=400, + random_state=42, + bootstrap=True, + include_deleted=False, + ) + elif args.label: + imap.filter_messages_from_server(label=args.label, recommendation_ratio=0.9) + else: + parser.print_help() + + +if __name__ == "__main__": + command_line_parser() +``` + +- [ ] **Step 4: Register the console script in `pyproject.toml`** + +In the `[project.scripts]` section, add a fourth line: + +```toml +[project.scripts] +gmailsorter = "gmailsorter.__main__:command_line_parser" +gmailsorter-daemon = "gmailsorter.daemon.__main__:command_line_parser" +gmailsorter-app = "gmailsorter.webapp.app:run_app" +gmailsorter-imap = "gmailsorter.imap.__main__:command_line_parser" +``` + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `python -m unittest tests.test_imap_cli -v` +Expected: PASS (4 tests) + +- [ ] **Step 6: Run the full suite** + +Run: `python -m unittest discover tests -v` +Expected: All tests PASS + +- [ ] **Step 7: Commit** + +```bash +git add gmailsorter/imap/__main__.py pyproject.toml tests/test_imap_cli.py +git commit -m "feat: add gmailsorter-imap CLI entry point" +``` + +--- + +### Task 8: GreenMail integration test + CI job + +**Files:** +- Create: `tests/test_imap_service_integration.py` +- Modify: `.github/workflows/unittest.yml` + +**Interfaces:** +- Consumes: `gmailsorter.local.Imap` (Task 6). +- Environment variables (matching the [testing-imap](https://github.com/jan-janssen/testing-imap) convention): `TEST_SMTP_HOST`, `TEST_SMTP_PORT`, `TEST_IMAP_HOST`, `TEST_IMAP_PORT`, `TEST_IMAP_USERNAME`, `TEST_EMAIL`, `TEST_EMAIL_PASSWORD`. + +This test talks to a **real** GreenMail server, so it cannot be driven through a plain RED/GREEN cycle without one running. It's written to skip cleanly (not fail) when no server is reachable, so `python -m unittest discover tests` stays green for contributors without Docker; CI (Step 4 below) is what proves it actually passes. + +- [ ] **Step 1: Create `tests/test_imap_service_integration.py`** + +```python +import os +import smtplib +import time +import unittest +import uuid +from email.message import EmailMessage +from imaplib import IMAP4 + +from gmailsorter.local import Imap + + +class TestImapServiceIntegration(unittest.TestCase): + smtp_host = os.environ.get("TEST_SMTP_HOST", "localhost") + smtp_port = int(os.environ.get("TEST_SMTP_PORT", "3025")) + imap_host = os.environ.get("TEST_IMAP_HOST", "localhost") + imap_port = int(os.environ.get("TEST_IMAP_PORT", "3143")) + username = os.environ.get("TEST_IMAP_USERNAME", "testuser") + recipient = os.environ.get("TEST_EMAIL", "testuser@example.test") + password = os.environ.get("TEST_EMAIL_PASSWORD", "secret") + + @classmethod + def setUpClass(cls): + if not cls._imap_server_available(): + raise unittest.SkipTest( + "No IMAP test server reachable at " + f"{cls.imap_host}:{cls.imap_port} - start the greenmail container " + "described in https://github.com/jan-janssen/testing-imap to run this test." + ) + + @classmethod + def _imap_server_available(cls, timeout=2.0): + try: + with IMAP4(cls.imap_host, cls.imap_port, timeout=timeout) as client: + status, _ = client.noop() + return status == "OK" + except OSError: + return False + + def setUp(self): + with IMAP4(self.imap_host, self.imap_port, timeout=10) as client: + client.login(self.username, self.password) + client.select("INBOX") + status, data = client.search(None, "ALL") + for message_id in data[0].split(): + client.store(message_id, "+FLAGS", r"(\Deleted)") + client.expunge() + for folder in ("MailSortInbox", "Sorted"): + client.create(folder) + + def _send_message(self, subject, body): + message_id = f"<{uuid.uuid4()}@example.test>" + message = EmailMessage() + message["From"] = "sender@example.test" + message["To"] = self.recipient + message["Subject"] = subject + message["Message-ID"] = message_id + message.set_content(body) + with smtplib.SMTP(self.smtp_host, self.smtp_port, timeout=10) as smtp: + smtp.send_message(message) + return message_id + + def _wait_for_message_in_inbox(self, message_id, timeout=10.0): + deadline = time.monotonic() + timeout + with IMAP4(self.imap_host, self.imap_port, timeout=10) as client: + client.login(self.username, self.password) + client.select("INBOX") + while time.monotonic() < deadline: + status, data = client.search( + None, "HEADER", "Message-ID", f'"{message_id}"' + ) + self.assertEqual(status, "OK") + if data[0].split(): + return + time.sleep(0.2) + self.fail(f"Message {message_id!r} was not delivered to INBOX") + + def test_update_database_and_move_round_trip(self): + message_id = self._send_message( + subject="Integration test message", + body="Body from gmailsorter IMAP test.", + ) + self._wait_for_message_in_inbox(message_id) + + imap = Imap( + host=self.imap_host, + port=self.imap_port, + username=self.username, + password=self.password, + connection_str="sqlite:///:memory:", + use_ssl=False, + ) + + imap.update_database(quick=False) + df = imap.get_all_emails_in_database() + + self.assertIn("Integration test message", df["subject"].tolist()) + stored_id = df.loc[ + df["subject"] == "Integration test message", "id" + ].iloc[0] + self.assertTrue(stored_id.startswith("INBOX\x1f")) + + imap._modify_message_labels( + message_id=stored_id, + label_id_remove_lst=["INBOX"], + label_id_add_lst=["MailSortInbox"], + ) + + imap.update_database(quick=False) + df_after_move = imap.get_all_emails_in_database() + moved_row = df_after_move.loc[ + df_after_move["subject"] == "Integration test message" + ] + self.assertEqual(len(moved_row), 1) + self.assertTrue(moved_row.iloc[0]["id"].startswith("MailSortInbox\x1f")) + + +if __name__ == "__main__": + unittest.main() +``` + +- [ ] **Step 2: Run it locally (expected to skip without Docker)** + +Run: `python -m unittest tests.test_imap_service_integration -v` +Expected: `skipped 'No IMAP test server reachable at localhost:3143 - ...'` + +- [ ] **Step 3: (Optional local verification) Run it against a real GreenMail container** + +If Docker is available locally: + +```bash +docker run -d --rm --name greenmail-test \ + -p 3025:3025 -p 3143:3143 \ + -e GREENMAIL_OPTS='-Dgreenmail.setup.test.smtp -Dgreenmail.setup.test.imap -Dgreenmail.hostname=0.0.0.0 -Dgreenmail.users=testuser:secret@example.test' \ + greenmail/standalone:2.1.11 +python -m unittest tests.test_imap_service_integration -v +docker stop greenmail-test +``` + +Expected: PASS (1 test). Skip this step if Docker isn't available — Step 4 (CI) is the authoritative check. + +- [ ] **Step 4: Add a `imap-integration` job to `.github/workflows/unittest.yml`** + +Append a second top-level job under `jobs:` (as a sibling of the existing `build` job), so the full file reads: + +```yaml +# This workflow is used to run the unittest of pyiron + +name: Unittests + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + + runs-on: ${{ matrix.operating-system }} + strategy: + matrix: + operating-system: [ubuntu-latest, windows-latest, macos-latest] + python-version: ['3.14'] + include: + - operating-system: ubuntu-latest + python-version: '3.11' + - operating-system: ubuntu-latest + python-version: '3.12' + - operating-system: ubuntu-latest + python-version: '3.13' + + steps: + - uses: actions/checkout@v4 + - name: Conda config + shell: bash -l {0} + run: echo -e "channels:\n - conda-forge\n" > .condarc + - uses: conda-incubator/setup-miniconda@v3 + with: + python-version: ${{ matrix.python-version }} + miniforge-version: latest + condarc-file: .condarc + environment-file: .ci_support/environment.yml + - name: Test + shell: bash -l {0} + timeout-minutes: 30 + run: | + pip install --no-deps . + coverage run --omit gmailsorter/_version.py -m unittest discover tests + + imap-integration: + runs-on: ubuntu-latest + + services: + greenmail: + image: greenmail/standalone:2.1.11 + env: + GREENMAIL_OPTS: >- + -Dgreenmail.setup.test.smtp + -Dgreenmail.setup.test.imap + -Dgreenmail.hostname=0.0.0.0 + -Dgreenmail.users=testuser:secret@example.test + ports: + - 3025:3025 + - 3143:3143 + + env: + TEST_SMTP_HOST: localhost + TEST_SMTP_PORT: "3025" + TEST_IMAP_HOST: localhost + TEST_IMAP_PORT: "3143" + TEST_IMAP_USERNAME: testuser + TEST_EMAIL: testuser@example.test + TEST_EMAIL_PASSWORD: secret + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install package + run: pip install . + - name: Run IMAP integration test + run: python -m unittest tests.test_imap_service_integration -v +``` + +(Note: this is a separate job, not an extra step in `build` — GitHub Actions `services:` containers only run on Linux-hosted runners, and `build` mixes `ubuntu-latest`/`windows-latest`/`macos-latest` in one matrix, so the GreenMail-backed test needs its own Linux-only job.) + +- [ ] **Step 5: Commit** + +```bash +git add tests/test_imap_service_integration.py .github/workflows/unittest.yml +git commit -m "test: add GreenMail-backed IMAP integration test and CI job" +``` + +--- + +### Task 9: Documentation + +**Files:** +- Modify: `docs/source/developer.md` +- Modify: `docs/source/architecture.md` + +**Interfaces:** None (documentation only). + +- [ ] **Step 1: Add an IMAP section to `docs/source/developer.md`** + +After the existing `### Filter emails using machine learning` subsection and before `## Future directions`, insert: + +```markdown +## IMAP accounts +`gmailsorter` also supports plain IMAP accounts (username and password, e.g. an app +password), for mail servers other than Google Mail. Import the `Imap` class instead of +`Gmail`: +``` +from gmailsorter import Imap +``` +``` +imap = Imap( + host="imap.example.com", + port=993, + username="user@example.com", + password="app-password", + connection_str="sqlite:////absolute/path/to/email.db", +) +``` +`Imap` exposes the exact same `update_database()`, `get_all_emails_in_database()` and +`filter_messages_from_server()` methods as `Gmail` - the only difference is that IMAP +folders play the role Gmail labels play elsewhere in this document: each folder is +treated as one label, and moving an email means moving it from one IMAP folder to +another. A command line interface is also available as `gmailsorter-imap`, reading the +account password from an environment variable (`IMAP_PASSWORD` by default) rather than +accepting it as a command line argument: +``` +export IMAP_PASSWORD=app-password +gmailsorter-imap --host imap.example.com --username user@example.com -d sqlite:///email.db -u +``` +``` + +- [ ] **Step 2: Mention IMAP in `docs/source/architecture.md`** + +In the "The big picture" section, change: + +```markdown +* **Your Google Mail account** - the source of truth for your emails and labels, accessed exclusively through the + official [Gmail API](https://developers.google.com/gmail/api/guides). `gmailsorter` never reads your mailbox + through any other channel and never stores your Google password. +``` + +to: + +```markdown +* **Your email account** - the source of truth for your emails and labels, accessed either through the official + [Gmail API](https://developers.google.com/gmail/api/guides) or, for any other IMAP-capable provider, through a + plain IMAP connection. `gmailsorter` never stores your Google password, and for IMAP accounts the password you + provide is used only to log in - it is not persisted anywhere. When talking to a plain IMAP server, each mailbox + folder plays the role a Gmail label plays throughout the rest of this page - "moving" an email between labels + means moving it between IMAP folders. +``` + +- [ ] **Step 3: Commit** + +```bash +git add docs/source/developer.md docs/source/architecture.md +git commit -m "docs: document the Imap class and CLI" +``` + +--- + +## Final verification (after all tasks) + +- [ ] Run the full suite one more time: `coverage run --omit gmailsorter/_version.py -m unittest discover tests -v` — expect all tests PASS (GreenMail test SKIPPED unless Docker is running locally). +- [ ] Run `ruff check gmailsorter/` and `ruff format --check gmailsorter/` (or `pre-commit run --all-files` if available) — expect no lint errors. +- [ ] Push the branch and confirm both the `build` matrix and the new `imap-integration` job go green in GitHub Actions before opening the PR. From 92c20a8cdffa62177ad630ba3565d943aa1321ac Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Sat, 25 Jul 2026 08:59:54 +0200 Subject: [PATCH 03/36] Add .gitignore for local virtualenv and build artifacts Needed a project-local .venv to install and test gmailsorter in isolation rather than the shared base conda environment. --- .gitignore | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f99bad2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +.venv/ +__pycache__/ +*.pyc +*.egg-info/ +build/ +dist/ +.coverage From 4cd05c3a8804eab503d6e0b99d0bda4e4f10a20a Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Sat, 25 Jul 2026 09:02:50 +0200 Subject: [PATCH 04/36] refactor: move HTML-to-text stripping into base/message.py so it can be reused by imap/message.py --- gmailsorter/base/message.py | 24 ++++++++++++++++++++++++ gmailsorter/google/message.py | 28 ++-------------------------- tests/test_message.py | 8 +++++++- 3 files changed, 33 insertions(+), 27 deletions(-) diff --git a/gmailsorter/base/message.py b/gmailsorter/base/message.py index c303cfe..8190a81 100644 --- a/gmailsorter/base/message.py +++ b/gmailsorter/base/message.py @@ -1,10 +1,34 @@ from abc import ABC, abstractmethod from datetime import datetime +from html.parser import HTMLParser +from io import StringIO _MAX_DATE_COMMAS = 2 _DATE_HYPHEN_COUNT = 2 +# https://stackoverflow.com/questions/753052/strip-html-from-strings-in-python +class _MLStripper(HTMLParser): + def __init__(self): + super().__init__() + self.reset() + self.strict = False + self.convert_charrefs = True + self.text = StringIO() + + def handle_data(self, d): + self.text.write(d) + + def get_data(self): + return self.text.getvalue() + + +def strip_html_tags(html): + stripper = _MLStripper() + stripper.feed(html) + return stripper.get_data() + + def email_date_converter(email_date): if not isinstance(email_date, str): return None diff --git a/gmailsorter/google/message.py b/gmailsorter/google/message.py index 51bc1b4..4b8b30f 100644 --- a/gmailsorter/google/message.py +++ b/gmailsorter/google/message.py @@ -1,24 +1,6 @@ import base64 -from html.parser import HTMLParser -from io import StringIO -from gmailsorter.base.message import AbstractMessage, email_date_converter - - -# https://stackoverflow.com/questions/753052/strip-html-from-strings-in-python -class MLStripper(HTMLParser): - def __init__(self): - super().__init__() - self.reset() - self.strict = False - self.convert_charrefs = True - self.text = StringIO() - - def handle_data(self, d): - self.text.write(d) - - def get_data(self): - return self.text.getvalue() +from gmailsorter.base.message import AbstractMessage, email_date_converter, strip_html_tags def get_email_dict(message): @@ -100,7 +82,7 @@ def _get_parts_content(self, message_parts): message_parts=message_parts[content_types.index("text/plain")] ) elif "text/html" in content_types: - return self._strip_tags( + return strip_html_tags( html=self._get_email_body( message_parts=message_parts[content_types.index("text/html")] ) @@ -138,12 +120,6 @@ def _get_email_body(message_parts): else: return "" - @staticmethod - def _strip_tags(html): - s = MLStripper() - s.feed(html) - return s.get_data() - @staticmethod def _get_email_address(email): email_split = email.split("<") diff --git a/tests/test_message.py b/tests/test_message.py index 3e220ef..53d6c85 100644 --- a/tests/test_message.py +++ b/tests/test_message.py @@ -1,6 +1,6 @@ from unittest import TestCase from datetime import datetime -from gmailsorter.base.message import email_date_converter, AbstractMessage +from gmailsorter.base.message import email_date_converter, AbstractMessage, strip_html_tags class MessageTest(TestCase): @@ -76,3 +76,9 @@ def test_email_date_converter(self): datetime.strptime("24-01-2022", "%d-%m-%Y"), ) self.assertEqual(email_date_converter(None), None) + + def test_strip_html_tags(self): + self.assertEqual( + strip_html_tags("

Hello World

"), + "Hello World", + ) From f34fdad32fc52de7a32663d230104ecc92c115dc Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Sat, 25 Jul 2026 09:07:38 +0200 Subject: [PATCH 05/36] refactor: extract AbstractMailBox loop from GoogleMailBase into base/mail.py --- gmailsorter/base/mail.py | 324 +++++++++++++++++++++++++ gmailsorter/google/mail.py | 280 +++------------------ tests/test_google_integration_units.py | 8 +- tests/test_mail_base.py | 118 +++++++++ 4 files changed, 477 insertions(+), 253 deletions(-) create mode 100644 gmailsorter/base/mail.py create mode 100644 tests/test_mail_base.py diff --git a/gmailsorter/base/mail.py b/gmailsorter/base/mail.py new file mode 100644 index 0000000..1477bce --- /dev/null +++ b/gmailsorter/base/mail.py @@ -0,0 +1,324 @@ +from abc import ABC, abstractmethod + +import pandas +from tqdm import tqdm + +from gmailsorter.ml import ( + encode_df_for_machine_learning, + fit_machine_learning_models, + get_predictions_from_machine_learning_models, +) + + +class AbstractMailBox(ABC): + def __init__( + self, + mail_service, + database_email=None, + database_ml=None, + user_id="me", + db_user_id=1, + email_download_format="metadata", + ): + """ + Shared fetch-store-train-predict-move loop for a mailbox backend, independent of + whether the backend is the Gmail API or a plain IMAP connection. + + Args: + mail_service: backend-specific connection object (Gmail API service resource, + imaplib connection, ...) + database_email (gmailsorter.base.database.DatabaseInterface): SQLalchemy interface for email database + database_ml (gmailsorter.ml.database.DatabaseInterface): SQLalchemy interface for machine learning database + user_id (str): backend-specific user identifier + db_user_id (int): Default 1 - set a user id when sharing a database with multiple users + email_download_format (str): backend-specific download format hint + """ + self._service = mail_service + self._db_email = database_email + self._db_ml = database_ml + self._db_user_id = db_user_id + self._userid = user_id + self._email_download_format = email_download_format + self._label_dict = self._get_label_translate_dict() + self._label_dict_inverse = {v: k for k, v in self._label_dict.items()} + + @property + def labels(self): + return list(self._label_dict.keys()) + + def download_emails_for_label(self, label): + """ + Download emails for a specific label + + Args: + label (str): label to download emails for + + Returns: + pandas.DataFrame: Email content for the downloaded emails + """ + return self._download_messages_to_dataframe( + message_id_lst=self._search_email_on_server( + label_lst=[label], only_message_ids=True + ) + ) + + def filter_messages_from_server( + self, + label, + recommendation_ratio=0.9, + ): + """ + Filter new emails based on machine learning model recommendations. + + Args: + label (str): Email label to filter for + recommendation_ratio (float): Only accept recommendation above this ratio (0 0: + model_reload_dict, feature_reload_lst = self._db_ml.load_models() + df_partial_features = encode_df_for_machine_learning( + df=df_partial, + feature_lst=feature_reload_lst, + label_lst=list(model_reload_dict.keys()), + return_labels=False, + ) + df_partial_features = df_partial_features.reindex( + sorted(df_partial_features.columns), axis=1 + ) + model_recommendation_dict = get_predictions_from_machine_learning_models( + df_features=df_partial_features, + model_dict=model_reload_dict, + recommendation_ratio=recommendation_ratio, + ) + self._move_emails( + move_email_dict=model_recommendation_dict, label_to_ignore=label + ) + + def fit_machine_learning_model_to_database( + self, + n_estimators=100, + max_features=400, + random_state=42, + bootstrap=True, + include_deleted=False, + ): + """ + Fit machine learning models to emails stored in database and afterwards store machine learning models in + database. + + Args: + n_estimators (int): Number of estimators + max_features (int): Number of features + random_state (int): Random state + bootstrap (boolean): Whether bootstrap samples are used when building trees. If False, the whole dataset is + used to build each tree. (default: true) + include_deleted (bool): Flag to include deleted emails - default False + """ + df_all = self.get_all_emails_in_database(include_deleted=include_deleted) + df_all_features, df_all_labels = encode_df_for_machine_learning( + df=df_all, feature_lst=[], label_lst=[], return_labels=True + ) + df_all_features = df_all_features.loc[ + :, ~df_all_features.columns.duplicated() + ].copy() + df_all_features = df_all_features.reindex( + sorted(df_all_features.columns), axis=1 + ) + model_dict = fit_machine_learning_models( + df_all_features=df_all_features, + df_all_labels=df_all_labels, + n_estimators=n_estimators, + max_features=max_features, + random_state=random_state, + bootstrap=bootstrap, + ) + self._db_ml.store_models( + model_dict=model_dict, + feature_lst=df_all_features.columns.values.tolist(), + user_id=self._db_user_id, + commit=True, + ) + + def get_all_emails_in_database(self, include_deleted=False): + """ + Get all emails stored in the local database + + Args: + include_deleted (bool): Flag to include deleted emails - default False + + Returns: + pandas.DataFrame: With all emails and the corresponding information + """ + return self._db_email.get_all_emails( + include_deleted=include_deleted, user_id=self._db_user_id + ) + + def update_database(self, quick=False, label_lst=None, email_format=None): + """ + Update local email database + + Args: + quick (boolean): Only add new emails, do not update existing labels - by default: False + label_lst (list): list of labels to be searched + email_format (str/None): Email format to download + """ + if label_lst is None: + label_lst = [] + if self._db_email is not None: + message_id_lst = self._search_email_on_server( + label_lst=label_lst, only_message_ids=True + ) + ( + new_messages_lst, + message_label_updates_lst, + deleted_messages_lst, + ) = self._db_email.get_labels_to_update( + message_id_lst=message_id_lst, user_id=self._db_user_id + ) + if not quick: + self._db_email.mark_emails_as_deleted( + message_id_lst=deleted_messages_lst, user_id=self._db_user_id + ) + self._db_email.update_labels( + message_id_lst=message_label_updates_lst, + message_meta_lst=self._get_labels_for_emails( + message_id_lst=message_label_updates_lst + ), + user_id=self._db_user_id, + ) + self._store_emails_in_database( + message_id_lst=new_messages_lst, email_format=email_format + ) + + def _download_messages_to_dataframe(self, message_id_lst, email_format=None): + """ + Download a list of messages based on their email IDs and store the content in a pandas.DataFrame. + + Args: + message_id_lst (list): list of emails IDs + email_format (str): Email format to download - default: "full" + + Returns: + pandas.DataFrame: pandas.DataFrame which contains the rendered emails + """ + return pandas.DataFrame( + [ + message + for message in [ + self._parse_message( + message=self._get_message_detail( + message_id=message_id, + email_format=email_format, + metadata_headers=[], + ) + ) + for message_id in tqdm( + iterable=message_id_lst, desc="Download messages to DataFrame" + ) + ] + if message is not None + ] + ) + + def _get_labels_for_emails(self, message_id_lst): + """ + Get labels for a list of emails + + Args: + message_id_lst (list): list of emails IDs + + Returns: + list: Nested list of email labels for each email + """ + return [ + self._get_labels_for_email(message_id=message_id) + for message_id in tqdm( + iterable=message_id_lst, desc="Get labels for emails" + ) + ] + + def _move_emails(self, move_email_dict, label_to_ignore): + label_existing = self._label_dict[label_to_ignore] + for message_id, label_add in tqdm( + iterable=move_email_dict.items(), desc="Move emails" + ): + if label_add is not None and label_add != label_existing: + self._modify_message_labels( + message_id=message_id, + label_id_remove_lst=[label_existing], + label_id_add_lst=[label_add], + ) + + def _store_emails_in_database(self, message_id_lst, email_format=None): + df = self._download_messages_to_dataframe( + message_id_lst=message_id_lst, email_format=email_format + ) + if len(df) > 0: + self._db_email.store_dataframe(df=df, user_id=self._db_user_id) + + @abstractmethod + def _search_email_on_server( + self, query_string="", label_lst=None, only_message_ids=False + ): + """ + Search emails either by a specific query or optionally limit your search to a list of labels + + Args: + query_string (str): query string to search for + label_lst (list): list of labels to be searched + only_message_ids (bool): return only the email IDs not the thread IDs - default: false + + Returns: + list: list of message ids (or backend-specific list items) matching the search + """ + + @abstractmethod + def _get_message_detail(self, message_id, email_format=None, metadata_headers=None): + """ + Get the raw, backend-specific representation of a single email message. + + Args: + message_id (str): id used by this backend to uniquely identify the email + email_format (str/None): backend-specific format hint + metadata_headers (list): backend-specific list of metadata headers + + Returns: + The backend-specific raw message representation, passed on to `_parse_message`. + """ + + @abstractmethod + def _get_label_translate_dict(self): + """ + Returns: + dict: mapping of label/folder display name to the backend-specific label/folder id + """ + + @abstractmethod + def _modify_message_labels( + self, message_id, label_id_remove_lst=None, label_id_add_lst=None + ): + """ + Apply a label/folder change to a single email message. + """ + + @abstractmethod + def _get_labels_for_email(self, message_id): + """ + Args: + message_id (str): id used by this backend to uniquely identify the email + + Returns: + list: list of labels/folders currently assigned to the email + """ + + @abstractmethod + def _parse_message(self, message): + """ + Args: + message: the backend-specific raw message representation returned by `_get_message_detail` + + Returns: + dict/None: the common gmailsorter email dict (see `gmailsorter.base.message.AbstractMessage.to_dict`), + or None if the message could not be parsed + """ diff --git a/gmailsorter/google/mail.py b/gmailsorter/google/mail.py index 471e855..7b0897b 100644 --- a/gmailsorter/google/mail.py +++ b/gmailsorter/google/mail.py @@ -1,20 +1,14 @@ -import pandas from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker -from tqdm import tqdm from gmailsorter.base import get_email_database +from gmailsorter.base.mail import AbstractMailBox from gmailsorter.google.database import get_token_database from gmailsorter.google.message import get_email_dict -from gmailsorter.ml import ( - encode_df_for_machine_learning, - fit_machine_learning_models, - get_machine_learning_database, - get_predictions_from_machine_learning_models, -) +from gmailsorter.ml import get_machine_learning_database -class GoogleMailBase: +class GoogleMailBase(AbstractMailBox): def __init__( self, google_mail_service, @@ -37,232 +31,16 @@ def __init__( db_user_id (int): Default 1 - set a user id when sharing a database with multiple users email_download_format (str): API response format [full, metadata] """ - self._service = google_mail_service - self._db_email = database_email - self._db_ml = database_ml self._db_token = database_token - self._db_user_id = db_user_id - self._userid = user_id - self._email_download_format = email_download_format - self._label_dict = self._get_label_translate_dict() - self._label_dict_inverse = {v: k for k, v in self._label_dict.items()} - - @property - def labels(self): - return list(self._label_dict.keys()) - - def download_emails_for_label(self, label): - """ - Download emails for a specific label - - Args: - label (str): label to download emails for - - Returns: - pandas.DataFrame: Email content for the downloaded emails - """ - return self._download_messages_to_dataframe( - message_id_lst=self._search_email_on_server( - label_lst=[label], only_message_ids=True - ) - ) - - def filter_messages_from_server( - self, - label, - recommendation_ratio=0.9, - ): - """ - Filter new emails based on machine learning model recommendations. - - Args: - label (str): Email label to filter for - recommendation_ratio (float): Only accept recommendation above this ratio (0 0: - model_reload_dict, feature_reload_lst = self._db_ml.load_models() - df_partial_features = encode_df_for_machine_learning( - df=df_partial, - feature_lst=feature_reload_lst, - label_lst=list(model_reload_dict.keys()), - return_labels=False, - ) - df_partial_features = df_partial_features.reindex( - sorted(df_partial_features.columns), axis=1 - ) - model_recommendation_dict = get_predictions_from_machine_learning_models( - df_features=df_partial_features, - model_dict=model_reload_dict, - recommendation_ratio=recommendation_ratio, - ) - self._move_emails( - move_email_dict=model_recommendation_dict, label_to_ignore=label - ) - - def fit_machine_learning_model_to_database( - self, - n_estimators=100, - max_features=400, - random_state=42, - bootstrap=True, - include_deleted=False, - ): - """ - Fit machine learning models to emails stored in database and afterwards store machine learning models in - database. - - Args: - n_estimators (int): Number of estimators - max_features (int): Number of features - random_state (int): Random state - bootstrap (boolean): Whether bootstrap samples are used when building trees. If False, the whole dataset is - used to build each tree. (default: true) - include_deleted (bool): Flag to include deleted emails - default False - """ - df_all = self.get_all_emails_in_database(include_deleted=include_deleted) - df_all_features, df_all_labels = encode_df_for_machine_learning( - df=df_all, feature_lst=[], label_lst=[], return_labels=True - ) - df_all_features = df_all_features.loc[ - :, ~df_all_features.columns.duplicated() - ].copy() - df_all_features = df_all_features.reindex( - sorted(df_all_features.columns), axis=1 - ) - model_dict = fit_machine_learning_models( - df_all_features=df_all_features, - df_all_labels=df_all_labels, - n_estimators=n_estimators, - max_features=max_features, - random_state=random_state, - bootstrap=bootstrap, - ) - self._db_ml.store_models( - model_dict=model_dict, - feature_lst=df_all_features.columns.values.tolist(), - user_id=self._db_user_id, - commit=True, - ) - - def get_all_emails_in_database(self, include_deleted=False): - """ - Get all emails stored in the local database - - Args: - include_deleted (bool): Flag to include deleted emails - default False - - Returns: - pandas.DataFrame: With all emails and the corresponding information - """ - return self._db_email.get_all_emails( - include_deleted=include_deleted, user_id=self._db_user_id - ) - - def update_database(self, quick=False, label_lst=None, email_format=None): - """ - Update local email database - - Args: - quick (boolean): Only add new emails, do not update existing labels - by default: False - label_lst (list): list of labels to be searched - email_format (str/None): Email format to download - """ - if label_lst is None: - label_lst = [] - if self._db_email is not None: - message_id_lst = self._search_email_on_server( - label_lst=label_lst, only_message_ids=True - ) - ( - new_messages_lst, - message_label_updates_lst, - deleted_messages_lst, - ) = self._db_email.get_labels_to_update( - message_id_lst=message_id_lst, user_id=self._db_user_id - ) - if not quick: - self._db_email.mark_emails_as_deleted( - message_id_lst=deleted_messages_lst, user_id=self._db_user_id - ) - self._db_email.update_labels( - message_id_lst=message_label_updates_lst, - message_meta_lst=self._get_labels_for_emails( - message_id_lst=message_label_updates_lst - ), - user_id=self._db_user_id, - ) - self._store_emails_in_database( - message_id_lst=new_messages_lst, email_format=email_format - ) - - def _download_messages_to_dataframe(self, message_id_lst, email_format=None): - """ - Download a list of messages based on their email IDs and store the content in a pandas.DataFrame. - - Args: - message_id_lst (list): list of emails IDs - email_format (str): Email format to download - default: "full" - - Returns: - pandas.DataFrame: pandas.DataFrame which contains the rendered emails - """ - return pandas.DataFrame( - [ - message - for message in [ - get_email_dict( - message=self._get_message_detail( - message_id=message_id, - email_format=email_format, - metadata_headers=[], - ) - ) - for message_id in tqdm( - iterable=message_id_lst, desc="Download messages to DataFrame" - ) - ] - if message is not None - ] + super().__init__( + mail_service=google_mail_service, + database_email=database_email, + database_ml=database_ml, + user_id=user_id, + db_user_id=db_user_id, + email_download_format=email_download_format, ) - def _get_labels_for_email(self, message_id): - """ - Get labels for email - - Args: - message_id (str): email ID - - Returns: - list: List of email labels - """ - message_dict = self._get_message_detail( - message_id=message_id, - email_format="metadata", - metadata_headers=["labelIds"], - ) - if "labelIds" in message_dict: - return message_dict["labelIds"] - else: - return [] - - def _get_labels_for_emails(self, message_id_lst): - """ - Get labels for a list of emails - - Args: - message_id_lst (list): list of emails IDs - - Returns: - list: Nested list of email labels for each email - """ - return [ - self._get_labels_for_email(message_id=message_id) - for message_id in tqdm( - iterable=message_id_lst, desc="Get labels for emails" - ) - ] - def _get_label_translate_dict(self): results = self._service.users().labels().list(userId=self._userid).execute() labels = results.get("labels", []) @@ -348,18 +126,6 @@ def _modify_message_labels( userId=self._userid, id=message_id, body=body_dict ).execute() - def _move_emails(self, move_email_dict, label_to_ignore): - label_existing = self._label_dict[label_to_ignore] - for message_id, label_add in tqdm( - iterable=move_email_dict.items(), desc="Move emails" - ): - if label_add is not None and label_add != label_existing: - self._modify_message_labels( - message_id=message_id, - label_id_remove_lst=[label_existing], - label_id_add_lst=[label_add], - ) - def _search_email_on_server( self, query_string="", label_lst=None, only_message_ids=False ): @@ -385,12 +151,28 @@ def _search_email_on_server( else: return [d["id"] for d in message_id_lst] - def _store_emails_in_database(self, message_id_lst, email_format=None): - df = self._download_messages_to_dataframe( - message_id_lst=message_id_lst, email_format=email_format + def _get_labels_for_email(self, message_id): + """ + Get labels for email + + Args: + message_id (str): email ID + + Returns: + list: List of email labels + """ + message_dict = self._get_message_detail( + message_id=message_id, + email_format="metadata", + metadata_headers=["labelIds"], ) - if len(df) > 0: - self._db_email.store_dataframe(df=df, user_id=self._db_user_id) + if "labelIds" in message_dict: + return message_dict["labelIds"] + else: + return [] + + def _parse_message(self, message): + return get_email_dict(message=message) @staticmethod def _create_databases(connection_str): diff --git a/tests/test_google_integration_units.py b/tests/test_google_integration_units.py index ed32c31..fb62368 100644 --- a/tests/test_google_integration_units.py +++ b/tests/test_google_integration_units.py @@ -361,8 +361,8 @@ def test_update_database_quick_and_full_paths(self): db_email.update_labels.assert_not_called() store_mock.assert_called_once_with(message_id_lst=["new2"], email_format=None) - @patch("gmailsorter.google.mail.get_predictions_from_machine_learning_models") - @patch("gmailsorter.google.mail.encode_df_for_machine_learning") + @patch("gmailsorter.base.mail.get_predictions_from_machine_learning_models") + @patch("gmailsorter.base.mail.encode_df_for_machine_learning") def test_filter_messages_from_server(self, encode_mock, predict_mock): service = self._create_mock_service_with_labels() db_ml = MagicMock() @@ -395,8 +395,8 @@ def test_filter_messages_from_server(self, encode_mock, predict_mock): mail.filter_messages_from_server("Inbox") encode_mock.assert_not_called() - @patch("gmailsorter.google.mail.fit_machine_learning_models") - @patch("gmailsorter.google.mail.encode_df_for_machine_learning") + @patch("gmailsorter.base.mail.fit_machine_learning_models") + @patch("gmailsorter.base.mail.encode_df_for_machine_learning") def test_fit_machine_learning_model_to_database(self, encode_mock, fit_mock): service = self._create_mock_service_with_labels() db_ml = MagicMock() diff --git a/tests/test_mail_base.py b/tests/test_mail_base.py new file mode 100644 index 0000000..c68f558 --- /dev/null +++ b/tests/test_mail_base.py @@ -0,0 +1,118 @@ +from unittest import TestCase +from unittest.mock import MagicMock, patch + +import pandas as pd + +from gmailsorter.base.mail import AbstractMailBox + + +class _StubMailBox(AbstractMailBox): + """Minimal concrete AbstractMailBox used to test the shared loop in isolation.""" + + def __init__(self, label_dict_fixture=None, **kwargs): + self.label_dict_fixture = label_dict_fixture or {"Inbox": "Inbox", "Spam": "Spam"} + self.search_result = [] + self.message_detail_dict = {} + self.modify_calls = [] + self.labels_for_email_dict = {} + super().__init__(mail_service=MagicMock(), **kwargs) + + def _search_email_on_server(self, query_string="", label_lst=None, only_message_ids=False): + return self.search_result + + def _get_message_detail(self, message_id, email_format=None, metadata_headers=None): + return self.message_detail_dict.get(message_id) + + def _get_label_translate_dict(self): + return self.label_dict_fixture + + def _modify_message_labels(self, message_id, label_id_remove_lst=None, label_id_add_lst=None): + self.modify_calls.append((message_id, label_id_remove_lst, label_id_add_lst)) + + def _get_labels_for_email(self, message_id): + return self.labels_for_email_dict.get(message_id, []) + + def _parse_message(self, message): + return message + + +class AbstractMailBoxTest(TestCase): + def test_labels_property(self): + mailbox = _StubMailBox() + self.assertEqual(sorted(mailbox.labels), ["Inbox", "Spam"]) + + def test_download_emails_for_label(self): + mailbox = _StubMailBox() + mailbox.search_result = ["id1", "id2"] + mailbox.message_detail_dict = { + "id1": { + "id": "id1", + "threads": "t1", + "labels": [], + "to": [], + "from": None, + "cc": [], + "subject": "s1", + "content": "c1", + "date": None, + }, + "id2": None, + } + + df = mailbox.download_emails_for_label(label="Inbox") + + self.assertEqual(df["id"].tolist(), ["id1"]) + + def test_move_emails_skips_matching_or_none_labels(self): + mailbox = _StubMailBox() + + mailbox._move_emails( + move_email_dict={"id1": None, "id2": "Inbox", "id3": "Spam"}, + label_to_ignore="Inbox", + ) + + self.assertEqual(mailbox.modify_calls, [("id3", ["Inbox"], ["Spam"])]) + + def test_update_database_marks_missing_as_deleted(self): + db_email = MagicMock() + db_email.get_labels_to_update.return_value = (["new"], [], ["deleted"]) + mailbox = _StubMailBox(database_email=db_email) + mailbox.search_result = ["new"] + mailbox.message_detail_dict = { + "new": { + "id": "new", + "threads": "t", + "labels": [], + "to": [], + "from": None, + "cc": [], + "subject": "s", + "content": "c", + "date": None, + } + } + + mailbox.update_database(quick=False) + + db_email.mark_emails_as_deleted.assert_called_once_with( + message_id_lst=["deleted"], user_id=1 + ) + db_email.store_dataframe.assert_called_once() + + @patch("gmailsorter.base.mail.fit_machine_learning_models") + @patch("gmailsorter.base.mail.encode_df_for_machine_learning") + def test_fit_machine_learning_model_to_database(self, encode_mock, fit_mock): + db_email = MagicMock() + db_email.get_all_emails.return_value = pd.DataFrame( + [{"id": "x", "from": "a@b.com", "to": [], "cc": [], "labels": [], "threads": "t"}] + ) + db_ml = MagicMock() + mailbox = _StubMailBox(database_email=db_email, database_ml=db_ml) + features = pd.DataFrame([{"email_id": "x", "f1": 1}]) + labels = pd.DataFrame([{"labels_Inbox": 1}]) + encode_mock.return_value = (features, labels) + fit_mock.return_value = {"Inbox": MagicMock()} + + mailbox.fit_machine_learning_model_to_database(n_estimators=5, max_features=2) + + db_ml.store_models.assert_called_once() From 04feeff2fde7c771d2138d8f2b7c68b7f2e00b76 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Sat, 25 Jul 2026 09:13:12 +0200 Subject: [PATCH 06/36] feat: add IMAP message parsing (gmailsorter.imap.message) --- gmailsorter/imap/__init__.py | 0 gmailsorter/imap/message.py | 113 +++++++++++++++++++++++++++++++++++ tests/test_imap_message.py | 101 +++++++++++++++++++++++++++++++ 3 files changed, 214 insertions(+) create mode 100644 gmailsorter/imap/__init__.py create mode 100644 gmailsorter/imap/message.py create mode 100644 tests/test_imap_message.py diff --git a/gmailsorter/imap/__init__.py b/gmailsorter/imap/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gmailsorter/imap/message.py b/gmailsorter/imap/message.py new file mode 100644 index 0000000..f51be47 --- /dev/null +++ b/gmailsorter/imap/message.py @@ -0,0 +1,113 @@ +import email.utils + +from gmailsorter.base.message import AbstractMessage, strip_html_tags + + +def get_email_dict(message, folder, uid): + try: + return Message(message=message, folder=folder, uid=uid).to_dict() + except (ValueError, KeyError) as e: + print(message, str(e)) + return None + + +class Message(AbstractMessage): + def __init__(self, message, folder, uid): + """ + Message class to parse a raw email.message.Message (as produced by + email.message_from_bytes() after an IMAP FETCH) into the common gmailsorter + email representation. + + Args: + message (email.message.Message): parsed RFC822 message + folder (str): IMAP mailbox/folder the message was fetched from + uid (str): IMAP UID of the message within `folder` + """ + self._message = message + self._folder = folder + self._uid = str(uid) + + def get_from(self): + from_header = self._message.get("From") + if from_header is None: + return None + addresses = [ + address + for _, address in email.utils.getaddresses([from_header]) + if address + ] + if len(addresses) == 1: + return addresses[0].lower() + return None + + def get_to(self): + return self._split_addresses(self._message.get_all("To")) + + def get_cc(self): + return self._split_addresses(self._message.get_all("Cc")) + + def get_label_ids(self): + return [self._folder] + + def get_subject(self): + return self._message.get("Subject") + + def get_date(self): + date_header = self._message.get("Date") + if date_header is None: + return None + return email.utils.parsedate_to_datetime(date_header) + + def get_content(self): + text_plain, text_html = None, None + if self._message.is_multipart(): + for part in self._message.walk(): + if part.get_content_maintype() == "multipart": + continue + if part.get_content_type() == "text/plain" and text_plain is None: + text_plain = self._decode_part(part) + elif part.get_content_type() == "text/html" and text_html is None: + text_html = self._decode_part(part) + elif self._message.get_content_type() == "text/plain": + text_plain = self._decode_part(self._message) + elif self._message.get_content_type() == "text/html": + text_html = self._decode_part(self._message) + if text_plain is not None: + return text_plain + elif text_html is not None: + return strip_html_tags(text_html) + else: + return None + + def get_thread_id(self): + references = self._message.get("References") + if references: + return references.split()[0] + in_reply_to = self._message.get("In-Reply-To") + if in_reply_to: + return in_reply_to.strip() + message_id = self._message.get("Message-ID") + if message_id: + return message_id.strip() + return self.get_email_id() + + def get_email_id(self): + return f"{self._folder}\x1f{self._uid}" + + @staticmethod + def _decode_part(part): + payload = part.get_payload(decode=True) + if payload is None: + return "" + charset = part.get_content_charset() or "utf-8" + return payload.decode(charset, errors="replace") + + @staticmethod + def _split_addresses(header_values): + if not header_values: + return [] + return [ + address.lower() + for _, address in email.utils.getaddresses(header_values) + if address + ] diff --git a/tests/test_imap_message.py b/tests/test_imap_message.py new file mode 100644 index 0000000..3c5c270 --- /dev/null +++ b/tests/test_imap_message.py @@ -0,0 +1,101 @@ +from datetime import datetime +from email.message import EmailMessage +from unittest import TestCase + +from gmailsorter.imap.message import Message, get_email_dict + + +class MessageTest(TestCase): + @classmethod + def setUpClass(cls) -> None: + msg = EmailMessage() + msg["Subject"] = "Test Email Subject" + msg["From"] = "sender@server.net" + msg["To"] = "me@mail.com, friend@provider.org" + msg["Date"] = "Fri, 11 Feb 2022 18:08:46 +0100" + msg["Message-ID"] = "" + msg.set_content("Hello world") + cls._message = msg + cls.message = Message(message=msg, folder="INBOX", uid="42") + + def test_subject(self): + self.assertEqual(self.message.get_subject(), "Test Email Subject") + + def test_from(self): + self.assertEqual(self.message.get_from(), "sender@server.net") + + def test_to(self): + self.assertEqual( + self.message.get_to(), ["me@mail.com", "friend@provider.org"] + ) + + def test_cc_empty(self): + self.assertEqual(self.message.get_cc(), []) + + def test_email_id(self): + self.assertEqual(self.message.get_email_id(), "INBOX\x1f42") + + def test_thread_id_falls_back_to_message_id(self): + self.assertEqual(self.message.get_thread_id(), "") + + def test_label_ids(self): + self.assertEqual(self.message.get_label_ids(), ["INBOX"]) + + def test_get_date(self): + self.assertEqual( + self.message.get_date(), + datetime.strptime( + "Fri, 11 Feb 2022 18:08:46 +0100", "%a, %d %b %Y %H:%M:%S %z" + ), + ) + + def test_get_content(self): + self.assertEqual(self.message.get_content().strip(), "Hello world") + + def test_get_content_html_fallback(self): + html_msg = EmailMessage() + html_msg["Subject"] = "HTML" + html_msg["From"] = "sender@server.net" + html_msg["To"] = "me@mail.com" + html_msg["Date"] = "Fri, 11 Feb 2022 18:08:46 +0100" + html_msg.set_content("

Hello World

", subtype="html") + message = Message(message=html_msg, folder="INBOX", uid="43") + + self.assertEqual(message.get_content().strip(), "Hello World") + + def test_thread_id_uses_references_header(self): + msg = EmailMessage() + msg["Subject"] = "Re: Test" + msg["References"] = " " + msg["Message-ID"] = "" + message = Message(message=msg, folder="INBOX", uid="44") + + self.assertEqual(message.get_thread_id(), "") + + def test_from_with_multiple_addresses_is_none(self): + msg = EmailMessage() + msg["From"] = "a@server.net, b@server.net" + message = Message(message=msg, folder="INBOX", uid="45") + + self.assertIsNone(message.get_from()) + + def test_get_email_dict(self): + result = get_email_dict(self._message, folder="INBOX", uid="42") + content = result.pop("content") + + self.assertEqual(content.strip(), "Hello world") + self.assertEqual( + result, + { + "cc": [], + "date": datetime.strptime( + "Fri, 11 Feb 2022 18:08:46 +0100", "%a, %d %b %Y %H:%M:%S %z" + ), + "from": "sender@server.net", + "id": "INBOX\x1f42", + "labels": ["INBOX"], + "subject": "Test Email Subject", + "threads": "", + "to": ["me@mail.com", "friend@provider.org"], + }, + ) From 84f1274064340294e62259c2a80952b456c99212 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Sat, 25 Jul 2026 09:17:02 +0200 Subject: [PATCH 07/36] feat: add IMAP username/password authentication (gmailsorter.imap.authentication) --- gmailsorter/imap/authentication.py | 21 +++++++++++++++ tests/test_imap_integration_units.py | 40 ++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 gmailsorter/imap/authentication.py create mode 100644 tests/test_imap_integration_units.py diff --git a/gmailsorter/imap/authentication.py b/gmailsorter/imap/authentication.py new file mode 100644 index 0000000..bfa3200 --- /dev/null +++ b/gmailsorter/imap/authentication.py @@ -0,0 +1,21 @@ +from imaplib import IMAP4, IMAP4_SSL + + +def create_service(host, port, username, password, use_ssl=True): + """ + Open and log in to an IMAP connection. + + Args: + host (str): IMAP server hostname + port (int): IMAP server port + username (str): IMAP account username + password (str): IMAP account password + use_ssl (bool): connect via IMAP4_SSL (default) or plain IMAP4 + + Returns: + imaplib.IMAP4: logged-in IMAP connection + """ + connection_cls = IMAP4_SSL if use_ssl else IMAP4 + connection = connection_cls(host, port) + connection.login(username, password) + return connection diff --git a/tests/test_imap_integration_units.py b/tests/test_imap_integration_units.py new file mode 100644 index 0000000..5f67125 --- /dev/null +++ b/tests/test_imap_integration_units.py @@ -0,0 +1,40 @@ +from unittest import TestCase +from unittest.mock import patch + +from gmailsorter.imap.authentication import create_service + + +class TestImapAuthentication(TestCase): + @patch("gmailsorter.imap.authentication.IMAP4_SSL") + def test_create_service_uses_ssl_by_default(self, imap_ssl_cls): + connection = imap_ssl_cls.return_value + + result = create_service( + host="localhost", port=993, username="user", password="secret" + ) + + imap_ssl_cls.assert_called_once_with("localhost", 993) + connection.login.assert_called_once_with("user", "secret") + self.assertIs(result, connection) + + @patch("gmailsorter.imap.authentication.IMAP4") + def test_create_service_without_ssl(self, imap_cls): + connection = imap_cls.return_value + + result = create_service( + host="localhost", + port=143, + username="user", + password="secret", + use_ssl=False, + ) + + imap_cls.assert_called_once_with("localhost", 143) + connection.login.assert_called_once_with("user", "secret") + self.assertIs(result, connection) + + +if __name__ == "__main__": + import unittest + + unittest.main() From d109dcba46f2e0bb418899f4080b243ebc843da8 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Sat, 25 Jul 2026 09:20:16 +0200 Subject: [PATCH 08/36] feat: add ImapMailBase (folders-as-labels, MOVE/COPY+EXPUNGE) --- gmailsorter/imap/mail.py | 138 +++++++++++++++++++++++ tests/test_imap_integration_units.py | 159 ++++++++++++++++++++++++++- 2 files changed, 296 insertions(+), 1 deletion(-) create mode 100644 gmailsorter/imap/mail.py diff --git a/gmailsorter/imap/mail.py b/gmailsorter/imap/mail.py new file mode 100644 index 0000000..6c14d36 --- /dev/null +++ b/gmailsorter/imap/mail.py @@ -0,0 +1,138 @@ +import email +import re + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from gmailsorter.base import get_email_database +from gmailsorter.base.mail import AbstractMailBox +from gmailsorter.imap.message import get_email_dict +from gmailsorter.ml import get_machine_learning_database + +_LIST_ENTRY_PATTERN = re.compile( + r'\((?P[^)]*)\)\s+"(?P.*)"\s+(?P.+)' +) + + +class ImapMailBase(AbstractMailBox): + def _get_label_translate_dict(self): + status, mailbox_lst = self._service.list() + if status != "OK" or mailbox_lst is None: + return {} + label_dict = {} + for entry in mailbox_lst: + flags, _delimiter, name = self._parse_list_entry(entry) + if "\\Noselect" in flags: + continue + label_dict[name] = name + return label_dict + + def _search_email_on_server( + self, query_string="", label_lst=None, only_message_ids=False + ): + """ + Search emails either by a specific query or optionally limit your search to a list of labels + + Args: + query_string (str): not supported yet - must be empty + label_lst (list): list of IMAP folders to search; if empty, every folder is searched + only_message_ids (bool): return only the composite email IDs - default: false + + Returns: + list: list of composite "{folder}\\x1f{uid}" ids matching the search + """ + if query_string: + raise NotImplementedError( + "Custom IMAP search queries are not supported yet, only label_lst filtering." + ) + if label_lst is None: + label_lst = [] + folder_lst = label_lst if len(label_lst) > 0 else list(self._label_dict.keys()) + message_id_lst = [ + f"{folder}\x1f{uid}" + for folder in folder_lst + for uid in self._search_folder(folder=folder) + ] + if only_message_ids: + return message_id_lst + else: + return [{"id": message_id} for message_id in message_id_lst] + + def _search_folder(self, folder): + status, _ = self._service.select(f'"{folder}"') + if status != "OK": + return [] + status, data = self._service.uid("search", None, "ALL") + if status != "OK" or data[0] is None: + return [] + return [ + uid.decode() if isinstance(uid, bytes) else uid for uid in data[0].split() + ] + + def _get_message_detail(self, message_id, email_format=None, metadata_headers=None): + """ + Fetch the raw RFC822 message for a composite "{folder}\\x1f{uid}" id. + + Returns: + tuple: (folder, uid, email.message.Message) + """ + folder, uid = message_id.split("\x1f", 1) + status, _ = self._service.select(f'"{folder}"') + if status != "OK": + raise RuntimeError(f"Could not select IMAP folder {folder!r}") + status, data = self._service.uid("fetch", uid, "(RFC822)") + if status != "OK" or not data or data[0] is None: + raise RuntimeError(f"Could not fetch IMAP message {message_id!r}") + raw_message = data[0][1] + parsed_message = email.message_from_bytes(raw_message) + return folder, uid, parsed_message + + def _modify_message_labels( + self, message_id, label_id_remove_lst=None, label_id_add_lst=None + ): + if not label_id_add_lst: + return + folder, uid = message_id.split("\x1f", 1) + target_folder = label_id_add_lst[0] + status, _ = self._service.select(f'"{folder}"') + if status != "OK": + raise RuntimeError(f"Could not select IMAP folder {folder!r}") + if "MOVE" in self._service.capabilities: + status, _ = self._service.uid("move", uid, f'"{target_folder}"') + if status != "OK": + raise RuntimeError( + f"Could not move IMAP message {message_id!r} to {target_folder!r}" + ) + else: + status, _ = self._service.uid("copy", uid, f'"{target_folder}"') + if status != "OK": + raise RuntimeError( + f"Could not copy IMAP message {message_id!r} to {target_folder!r}" + ) + self._service.uid("store", uid, "+FLAGS", r"(\Deleted)") + self._service.expunge() + + def _get_labels_for_email(self, message_id): + folder, _uid = message_id.split("\x1f", 1) + return [folder] + + def _parse_message(self, message): + folder, uid, parsed_message = message + return get_email_dict(message=parsed_message, folder=folder, uid=uid) + + @staticmethod + def _parse_list_entry(entry): + decoded = entry.decode() if isinstance(entry, bytes) else entry + match = _LIST_ENTRY_PATTERN.match(decoded) + flags = match.group("flags").split() + delimiter = match.group("delimiter") + name = match.group("name").strip('"') + return flags, delimiter, name + + @staticmethod + def _create_databases(connection_str): + engine = create_engine(connection_str) + session = sessionmaker(bind=engine)() + db_email = get_email_database(engine=engine, session=session) + db_ml = get_machine_learning_database(engine=engine, session=session) + return db_email, db_ml diff --git a/tests/test_imap_integration_units.py b/tests/test_imap_integration_units.py index 5f67125..9b4cfe9 100644 --- a/tests/test_imap_integration_units.py +++ b/tests/test_imap_integration_units.py @@ -1,7 +1,8 @@ from unittest import TestCase -from unittest.mock import patch +from unittest.mock import MagicMock, patch from gmailsorter.imap.authentication import create_service +from gmailsorter.imap.mail import ImapMailBase class TestImapAuthentication(TestCase): @@ -34,6 +35,162 @@ def test_create_service_without_ssl(self, imap_cls): self.assertIs(result, connection) +class TestImapMailBase(TestCase): + def _create_mock_service_with_folders(self, folders=None): + service = MagicMock() + service.capabilities = ["IMAP4rev1", "MOVE"] + service.list.return_value = ( + "OK", + folders + if folders is not None + else [ + b'(\\HasNoChildren) "/" "INBOX"', + b'(\\HasNoChildren) "/" "MailSortInbox"', + b'(\\Noselect \\HasChildren) "/" "[Gmail]"', + ], + ) + return service + + def test_get_label_translate_dict_skips_noselect(self): + service = self._create_mock_service_with_folders() + mail = ImapMailBase(mail_service=service) + + self.assertEqual(sorted(mail.labels), ["INBOX", "MailSortInbox"]) + + def test_search_email_on_server_single_folder(self): + service = self._create_mock_service_with_folders() + service.select.return_value = ("OK", [b"1"]) + service.uid.return_value = ("OK", [b"1 2"]) + mail = ImapMailBase(mail_service=service) + + ids = mail._search_email_on_server(label_lst=["INBOX"], only_message_ids=True) + + service.select.assert_called_once_with('"INBOX"') + service.uid.assert_called_once_with("search", None, "ALL") + self.assertEqual(ids, ["INBOX\x1f1", "INBOX\x1f2"]) + + def test_search_email_on_server_all_folders_when_no_label(self): + service = self._create_mock_service_with_folders() + service.select.return_value = ("OK", [b"1"]) + service.uid.return_value = ("OK", [b"5"]) + mail = ImapMailBase(mail_service=service) + + ids = mail._search_email_on_server(only_message_ids=True) + + self.assertEqual( + service.select.call_args_list, + [(('"INBOX"',),), (('"MailSortInbox"',),)], + ) + self.assertEqual(ids, ["INBOX\x1f5", "MailSortInbox\x1f5"]) + + def test_search_email_on_server_rejects_query_string(self): + service = self._create_mock_service_with_folders() + mail = ImapMailBase(mail_service=service) + + with self.assertRaises(NotImplementedError): + mail._search_email_on_server(query_string="SUBJECT foo") + + def test_get_message_detail_selects_and_fetches(self): + service = self._create_mock_service_with_folders() + raw_message = b"Subject: hi\r\nFrom: a@b.com\r\nTo: c@d.com\r\n\r\nbody" + service.select.return_value = ("OK", [b"1"]) + service.uid.return_value = ("OK", [(b"1 (RFC822 {10}", raw_message)]) + mail = ImapMailBase(mail_service=service) + + folder, uid, message = mail._get_message_detail(message_id="INBOX\x1f7") + + service.select.assert_called_once_with('"INBOX"') + service.uid.assert_called_once_with("fetch", "7", "(RFC822)") + self.assertEqual(folder, "INBOX") + self.assertEqual(uid, "7") + self.assertEqual(message["Subject"], "hi") + + def test_get_labels_for_email_from_composite_id(self): + service = self._create_mock_service_with_folders() + mail = ImapMailBase(mail_service=service) + + self.assertEqual(mail._get_labels_for_email("INBOX\x1f7"), ["INBOX"]) + + def test_modify_message_labels_uses_move_when_supported(self): + service = self._create_mock_service_with_folders() + service.select.return_value = ("OK", [b"1"]) + service.uid.return_value = ("OK", [b"1"]) + mail = ImapMailBase(mail_service=service) + + mail._modify_message_labels( + message_id="INBOX\x1f7", + label_id_remove_lst=["INBOX"], + label_id_add_lst=["MailSortInbox"], + ) + + service.select.assert_called_once_with('"INBOX"') + service.uid.assert_called_once_with("move", "7", '"MailSortInbox"') + service.expunge.assert_not_called() + + def test_modify_message_labels_falls_back_to_copy_delete(self): + service = self._create_mock_service_with_folders() + service.capabilities = ["IMAP4rev1"] + service.select.return_value = ("OK", [b"1"]) + service.uid.return_value = ("OK", [b"1"]) + mail = ImapMailBase(mail_service=service) + + mail._modify_message_labels( + message_id="INBOX\x1f7", + label_id_remove_lst=["INBOX"], + label_id_add_lst=["MailSortInbox"], + ) + + self.assertEqual( + service.uid.call_args_list, + [ + (("copy", "7", '"MailSortInbox"'),), + (("store", "7", "+FLAGS", r"(\Deleted)"),), + ], + ) + service.expunge.assert_called_once() + + def test_modify_message_labels_noop_without_target(self): + service = self._create_mock_service_with_folders() + mail = ImapMailBase(mail_service=service) + + mail._modify_message_labels(message_id="INBOX\x1f7") + + service.select.assert_not_called() + + @patch("gmailsorter.imap.mail.get_email_dict") + def test_parse_message_delegates_to_get_email_dict(self, get_email_dict_mock): + service = self._create_mock_service_with_folders() + mail = ImapMailBase(mail_service=service) + get_email_dict_mock.return_value = {"id": "INBOX\x1f7"} + + result = mail._parse_message(("INBOX", "7", "raw")) + + get_email_dict_mock.assert_called_once_with( + message="raw", folder="INBOX", uid="7" + ) + self.assertEqual(result, {"id": "INBOX\x1f7"}) + + def test_create_databases(self): + with ( + patch("gmailsorter.imap.mail.create_engine") as create_engine_mock, + patch("gmailsorter.imap.mail.sessionmaker") as sessionmaker_mock, + patch("gmailsorter.imap.mail.get_email_database") as get_email_db_mock, + patch( + "gmailsorter.imap.mail.get_machine_learning_database" + ) as get_ml_db_mock, + ): + engine = MagicMock() + session = MagicMock() + create_engine_mock.return_value = engine + sessionmaker_mock.return_value.return_value = session + get_email_db_mock.return_value = "EMAIL_DB" + get_ml_db_mock.return_value = "ML_DB" + + dbs = ImapMailBase._create_databases("sqlite:///file.db") + + self.assertEqual(dbs, ("EMAIL_DB", "ML_DB")) + + if __name__ == "__main__": import unittest From ab24538eb205a7edf6e3563b381fd5c658e4ef67 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Sat, 25 Jul 2026 09:28:05 +0200 Subject: [PATCH 09/36] feat: add Imap convenience class and gmailsorter.Imap export Co-Authored-By: Claude Sonnet 5 --- gmailsorter/__init__.py | 4 +-- gmailsorter/imap/__init__.py | 4 +++ gmailsorter/local.py | 51 ++++++++++++++++++++++++++++ tests/test_imap_integration_units.py | 42 +++++++++++++++++++++++ 4 files changed, 99 insertions(+), 2 deletions(-) diff --git a/gmailsorter/__init__.py b/gmailsorter/__init__.py index 49301a6..a9f2fba 100644 --- a/gmailsorter/__init__.py +++ b/gmailsorter/__init__.py @@ -1,6 +1,6 @@ -from gmailsorter.local import Gmail, load_client_secrets_file +from gmailsorter.local import Gmail, Imap, load_client_secrets_file from . import _version __version__: str = _version.__version__ -__all__ = ["Gmail", "load_client_secrets_file"] +__all__ = ["Gmail", "Imap", "load_client_secrets_file"] diff --git a/gmailsorter/imap/__init__.py b/gmailsorter/imap/__init__.py index e69de29..4bb407f 100644 --- a/gmailsorter/imap/__init__.py +++ b/gmailsorter/imap/__init__.py @@ -0,0 +1,4 @@ +from gmailsorter.imap.authentication import create_service +from gmailsorter.imap.mail import ImapMailBase + +__all__ = ["create_service", "ImapMailBase"] diff --git a/gmailsorter/local.py b/gmailsorter/local.py index 9d06495..a5b3965 100644 --- a/gmailsorter/local.py +++ b/gmailsorter/local.py @@ -1,6 +1,8 @@ import json from gmailsorter.google import GoogleMailBase, create_service +from gmailsorter.imap import ImapMailBase +from gmailsorter.imap import create_service as create_imap_service class Gmail(GoogleMailBase): @@ -64,3 +66,52 @@ def __init__( def load_client_secrets_file(client_secrets_file): with open(client_secrets_file) as json_file: return json.load(json_file) + + +class Imap(ImapMailBase): + def __init__( + self, + host, + port, + username, + password, + connection_str, + db_user_id=1, + use_ssl=True, + email_download_format="metadata", + ): + """ + Imap class to manage Emails via a plain IMAP connection directly from Python + + Args: + host (str): IMAP server hostname + port (int): IMAP server port, typically 993 for IMAP4_SSL or 143 for IMAP4 + username (str): IMAP account username + password (str): IMAP account password + connection_str (str): SQLalchemy compatible connection string to connect to the SQL database + db_user_id (int): Default 1 - set a user id when sharing a database with multiple users + use_ssl (bool): connect via IMAP4_SSL (default) or plain IMAP4 + email_download_format (str): unused for IMAP, kept for interface parity with Gmail + """ + self._connection_str = connection_str + + database_email, database_ml = self._create_databases( + connection_str=self._connection_str + ) + + imap_connection = create_imap_service( + host=host, + port=port, + username=username, + password=password, + use_ssl=use_ssl, + ) + + super().__init__( + mail_service=imap_connection, + database_email=database_email, + database_ml=database_ml, + user_id=username, + db_user_id=db_user_id, + email_download_format=email_download_format, + ) diff --git a/tests/test_imap_integration_units.py b/tests/test_imap_integration_units.py index 9b4cfe9..3f17518 100644 --- a/tests/test_imap_integration_units.py +++ b/tests/test_imap_integration_units.py @@ -3,6 +3,7 @@ from gmailsorter.imap.authentication import create_service from gmailsorter.imap.mail import ImapMailBase +from gmailsorter.local import Imap class TestImapAuthentication(TestCase): @@ -191,6 +192,47 @@ def test_create_databases(self): self.assertEqual(dbs, ("EMAIL_DB", "ML_DB")) +class TestImapLocalHelpers(TestCase): + @patch("gmailsorter.local.ImapMailBase.__init__", return_value=None) + @patch("gmailsorter.local.create_imap_service") + @patch("gmailsorter.local.Imap._create_databases") + def test_imap_initialization_wiring( + self, create_databases_mock, create_service_mock, base_init_mock + ): + db_email, db_ml = MagicMock(), MagicMock() + create_databases_mock.return_value = (db_email, db_ml) + connection = MagicMock() + create_service_mock.return_value = connection + + Imap( + host="localhost", + port=993, + username="user", + password="secret", + connection_str="sqlite:///:memory:", + db_user_id=4, + ) + + create_databases_mock.assert_called_once_with( + connection_str="sqlite:///:memory:" + ) + create_service_mock.assert_called_once_with( + host="localhost", + port=993, + username="user", + password="secret", + use_ssl=True, + ) + base_init_mock.assert_called_once_with( + mail_service=connection, + database_email=db_email, + database_ml=db_ml, + user_id="user", + db_user_id=4, + email_download_format="metadata", + ) + + if __name__ == "__main__": import unittest From 709ee1dd6c4a93bdb150c1c040fb41acafa474c1 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Sat, 25 Jul 2026 09:31:58 +0200 Subject: [PATCH 10/36] feat: add gmailsorter-imap CLI entry point --- gmailsorter/imap/__main__.py | 97 +++++++++++++++++++++++++++++++++ pyproject.toml | 1 + tests/test_imap_cli.py | 101 +++++++++++++++++++++++++++++++++++ 3 files changed, 199 insertions(+) create mode 100644 gmailsorter/imap/__main__.py create mode 100644 tests/test_imap_cli.py diff --git a/gmailsorter/imap/__main__.py b/gmailsorter/imap/__main__.py new file mode 100644 index 0000000..28bae79 --- /dev/null +++ b/gmailsorter/imap/__main__.py @@ -0,0 +1,97 @@ +import argparse +import os + +from gmailsorter import Imap + + +def command_line_parser(): + """ + Main function primarily used for the command line interface of the IMAP backend + """ + parser = argparse.ArgumentParser(prog="gmailsorter-imap") + parser.add_argument( + "--host", + help="IMAP server hostname e.g. imap.example.com .", + ) + parser.add_argument( + "--port", + type=int, + default=993, + help="IMAP server port - default: 993 .", + ) + parser.add_argument( + "--username", + help="IMAP account username.", + ) + parser.add_argument( + "--password-env", + default="IMAP_PASSWORD", + help=( + "Name of the environment variable holding the IMAP account password - " + "default: IMAP_PASSWORD ." + ), + ) + parser.add_argument( + "--no-ssl", + action="store_true", + help="Connect without SSL (IMAP4 instead of IMAP4_SSL).", + ) + parser.add_argument( + "-d", + "--database", + help="Connection string to connect to database e.g. sqlite:///email.db .", + ) + parser.add_argument( + "-u", + "--update", + action="store_true", + help="Update local database and retrain machine learning model.", + ) + parser.add_argument( + "-i", + "--identification", + help="User ID of the database user e.g. 1 .", + ) + parser.add_argument( + "-l", + "--label", + help="Email label (IMAP folder) to be filtered with machine learning.", + ) + args = parser.parse_args() + db_user_id = int(args.identification) if args.identification else 1 + password = os.environ.get(args.password_env) + if not args.host or not args.username: + print("Please provide --host and --username.") + elif not password: + print( + f"Please set the {args.password_env} environment variable to your IMAP password." + ) + else: + database = args.database or "sqlite:///email.db" + imap = Imap( + host=args.host, + port=args.port, + username=args.username, + password=password, + connection_str=database, + db_user_id=db_user_id, + use_ssl=not args.no_ssl, + email_download_format="metadata", + ) + if args.update: + imap.update_database(quick=False) + imap.fit_machine_learning_model_to_database( + n_estimators=100, + max_features=400, + random_state=42, + bootstrap=True, + include_deleted=False, + ) + elif args.label: + imap.filter_messages_from_server(label=args.label, recommendation_ratio=0.9) + else: + parser.print_help() + + +if __name__ == "__main__": + command_line_parser() diff --git a/pyproject.toml b/pyproject.toml index 29bfac4..f6686f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ Homepage = "https://github.com/jan-janssen/gmailsorter" gmailsorter = "gmailsorter.__main__:command_line_parser" gmailsorter-daemon = "gmailsorter.daemon.__main__:command_line_parser" gmailsorter-app = "gmailsorter.webapp.app:run_app" +gmailsorter-imap = "gmailsorter.imap.__main__:command_line_parser" [tool.ruff.lint] select = [ diff --git a/tests/test_imap_cli.py b/tests/test_imap_cli.py new file mode 100644 index 0000000..b57c988 --- /dev/null +++ b/tests/test_imap_cli.py @@ -0,0 +1,101 @@ +import os +from unittest import TestCase +from unittest.mock import patch + +from gmailsorter.imap.__main__ import command_line_parser + + +class ImapCliTest(TestCase): + @patch("gmailsorter.imap.__main__.Imap") + def test_update_wires_imap_and_triggers_update(self, imap_cls): + imap_instance = imap_cls.return_value + os.environ["IMAP_PASSWORD"] = "secret" + try: + with patch( + "sys.argv", + [ + "gmailsorter-imap", + "--host", + "localhost", + "--port", + "993", + "--username", + "user", + "-d", + "sqlite:///:memory:", + "-u", + ], + ): + command_line_parser() + finally: + del os.environ["IMAP_PASSWORD"] + + imap_cls.assert_called_once_with( + host="localhost", + port=993, + username="user", + password="secret", + connection_str="sqlite:///:memory:", + db_user_id=1, + use_ssl=True, + email_download_format="metadata", + ) + imap_instance.update_database.assert_called_once_with(quick=False) + imap_instance.fit_machine_learning_model_to_database.assert_called_once_with( + n_estimators=100, + max_features=400, + random_state=42, + bootstrap=True, + include_deleted=False, + ) + + @patch("gmailsorter.imap.__main__.Imap") + def test_label_wires_imap_and_triggers_filter(self, imap_cls): + imap_instance = imap_cls.return_value + os.environ["IMAP_PASSWORD"] = "secret" + try: + with patch( + "sys.argv", + [ + "gmailsorter-imap", + "--host", + "localhost", + "--username", + "user", + "-d", + "sqlite:///:memory:", + "-l", + "MailSortInbox", + ], + ): + command_line_parser() + finally: + del os.environ["IMAP_PASSWORD"] + + imap_instance.filter_messages_from_server.assert_called_once_with( + label="MailSortInbox", recommendation_ratio=0.9 + ) + + @patch("gmailsorter.imap.__main__.Imap") + def test_missing_password_env_skips_wiring(self, imap_cls): + os.environ.pop("IMAP_PASSWORD", None) + with patch( + "sys.argv", + ["gmailsorter-imap", "--host", "localhost", "--username", "user"], + ): + command_line_parser() + + imap_cls.assert_not_called() + + @patch("gmailsorter.imap.__main__.Imap") + def test_missing_host_skips_wiring(self, imap_cls): + with patch("sys.argv", ["gmailsorter-imap", "--username", "user"]): + command_line_parser() + + imap_cls.assert_not_called() + + +if __name__ == "__main__": + import unittest + + unittest.main() From 8b6d8af93b06ebcedd376001068752bbddf51062 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Sat, 25 Jul 2026 09:35:54 +0200 Subject: [PATCH 11/36] test: add GreenMail-backed IMAP integration test and CI job --- .github/workflows/unittest.yml | 35 ++++++++ tests/test_imap_service_integration.py | 118 +++++++++++++++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 tests/test_imap_service_integration.py diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 3dc84ea..55a2052 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -41,3 +41,38 @@ jobs: run: | pip install --no-deps . coverage run --omit gmailsorter/_version.py -m unittest discover tests + + imap-integration: + runs-on: ubuntu-latest + + services: + greenmail: + image: greenmail/standalone:2.1.11 + env: + GREENMAIL_OPTS: >- + -Dgreenmail.setup.test.smtp + -Dgreenmail.setup.test.imap + -Dgreenmail.hostname=0.0.0.0 + -Dgreenmail.users=testuser:secret@example.test + ports: + - 3025:3025 + - 3143:3143 + + env: + TEST_SMTP_HOST: localhost + TEST_SMTP_PORT: "3025" + TEST_IMAP_HOST: localhost + TEST_IMAP_PORT: "3143" + TEST_IMAP_USERNAME: testuser + TEST_EMAIL: testuser@example.test + TEST_EMAIL_PASSWORD: secret + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install package + run: pip install . + - name: Run IMAP integration test + run: python -m unittest tests.test_imap_service_integration -v diff --git a/tests/test_imap_service_integration.py b/tests/test_imap_service_integration.py new file mode 100644 index 0000000..086addc --- /dev/null +++ b/tests/test_imap_service_integration.py @@ -0,0 +1,118 @@ +import os +import smtplib +import time +import unittest +import uuid +from email.message import EmailMessage +from imaplib import IMAP4 + +from gmailsorter.local import Imap + + +class TestImapServiceIntegration(unittest.TestCase): + smtp_host = os.environ.get("TEST_SMTP_HOST", "localhost") + smtp_port = int(os.environ.get("TEST_SMTP_PORT", "3025")) + imap_host = os.environ.get("TEST_IMAP_HOST", "localhost") + imap_port = int(os.environ.get("TEST_IMAP_PORT", "3143")) + username = os.environ.get("TEST_IMAP_USERNAME", "testuser") + recipient = os.environ.get("TEST_EMAIL", "testuser@example.test") + password = os.environ.get("TEST_EMAIL_PASSWORD", "secret") + + @classmethod + def setUpClass(cls): + if not cls._imap_server_available(): + raise unittest.SkipTest( + "No IMAP test server reachable at " + f"{cls.imap_host}:{cls.imap_port} - start the greenmail container " + "described in https://github.com/jan-janssen/testing-imap to run this test." + ) + + @classmethod + def _imap_server_available(cls, timeout=2.0): + try: + with IMAP4(cls.imap_host, cls.imap_port, timeout=timeout) as client: + status, _ = client.noop() + return status == "OK" + except OSError: + return False + + def setUp(self): + with IMAP4(self.imap_host, self.imap_port, timeout=10) as client: + client.login(self.username, self.password) + client.select("INBOX") + status, data = client.search(None, "ALL") + for message_id in data[0].split(): + client.store(message_id, "+FLAGS", r"(\Deleted)") + client.expunge() + for folder in ("MailSortInbox", "Sorted"): + client.create(folder) + + def _send_message(self, subject, body): + message_id = f"<{uuid.uuid4()}@example.test>" + message = EmailMessage() + message["From"] = "sender@example.test" + message["To"] = self.recipient + message["Subject"] = subject + message["Message-ID"] = message_id + message.set_content(body) + with smtplib.SMTP(self.smtp_host, self.smtp_port, timeout=10) as smtp: + smtp.send_message(message) + return message_id + + def _wait_for_message_in_inbox(self, message_id, timeout=10.0): + deadline = time.monotonic() + timeout + with IMAP4(self.imap_host, self.imap_port, timeout=10) as client: + client.login(self.username, self.password) + client.select("INBOX") + while time.monotonic() < deadline: + status, data = client.search( + None, "HEADER", "Message-ID", f'"{message_id}"' + ) + self.assertEqual(status, "OK") + if data[0].split(): + return + time.sleep(0.2) + self.fail(f"Message {message_id!r} was not delivered to INBOX") + + def test_update_database_and_move_round_trip(self): + message_id = self._send_message( + subject="Integration test message", + body="Body from gmailsorter IMAP test.", + ) + self._wait_for_message_in_inbox(message_id) + + imap = Imap( + host=self.imap_host, + port=self.imap_port, + username=self.username, + password=self.password, + connection_str="sqlite:///:memory:", + use_ssl=False, + ) + + imap.update_database(quick=False) + df = imap.get_all_emails_in_database() + + self.assertIn("Integration test message", df["subject"].tolist()) + stored_id = df.loc[ + df["subject"] == "Integration test message", "id" + ].iloc[0] + self.assertTrue(stored_id.startswith("INBOX\x1f")) + + imap._modify_message_labels( + message_id=stored_id, + label_id_remove_lst=["INBOX"], + label_id_add_lst=["MailSortInbox"], + ) + + imap.update_database(quick=False) + df_after_move = imap.get_all_emails_in_database() + moved_row = df_after_move.loc[ + df_after_move["subject"] == "Integration test message" + ] + self.assertEqual(len(moved_row), 1) + self.assertTrue(moved_row.iloc[0]["id"].startswith("MailSortInbox\x1f")) + + +if __name__ == "__main__": + unittest.main() From f86b463eb2f21f99f47385f91094fffe24420ae9 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Sat, 25 Jul 2026 09:41:04 +0200 Subject: [PATCH 12/36] test: retry IMAP reachability probe so CI doesn't skip on GreenMail startup lag GitHub Actions services: containers are only guaranteed to be running, not to have finished binding their ports, so a single 2s connection attempt in _imap_server_available() could cause the integration test to silently skip in CI while GreenMail is still starting up. Retry up to 5 times with a 1.5s delay between attempts, keeping the worst-case wait for a genuinely absent server well under 30s. Co-Authored-By: Claude Sonnet 5 --- tests/test_imap_service_integration.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/test_imap_service_integration.py b/tests/test_imap_service_integration.py index 086addc..3f55c51 100644 --- a/tests/test_imap_service_integration.py +++ b/tests/test_imap_service_integration.py @@ -28,13 +28,18 @@ def setUpClass(cls): ) @classmethod - def _imap_server_available(cls, timeout=2.0): - try: - with IMAP4(cls.imap_host, cls.imap_port, timeout=timeout) as client: - status, _ = client.noop() - return status == "OK" - except OSError: - return False + def _imap_server_available(cls, timeout=2.0, attempts=5, delay=1.5): + for attempt in range(attempts): + try: + with IMAP4(cls.imap_host, cls.imap_port, timeout=timeout) as client: + status, _ = client.noop() + if status == "OK": + return True + except OSError: + pass + if attempt < attempts - 1: + time.sleep(delay) + return False def setUp(self): with IMAP4(self.imap_host, self.imap_port, timeout=10) as client: From 24aa0024e860fd88048e2c3736a04a3ae2fab4cc Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Sat, 25 Jul 2026 09:43:41 +0200 Subject: [PATCH 13/36] docs: document the Imap class and CLI --- docs/source/architecture.md | 9 ++++++--- docs/source/developer.md | 28 ++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/docs/source/architecture.md b/docs/source/architecture.md index d797a14..93214ca 100644 --- a/docs/source/architecture.md +++ b/docs/source/architecture.md @@ -9,9 +9,12 @@ machine learning knowledge required. If you are looking for setup instructions i Regardless of whether you use the hosted [gmailsorter.com](https://gmailsorter.com) service, the Docker container or the plain Python package, `gmailsorter` is built from the same three building blocks: -* **Your Google Mail account** - the source of truth for your emails and labels, accessed exclusively through the - official [Gmail API](https://developers.google.com/gmail/api/guides). `gmailsorter` never reads your mailbox - through any other channel and never stores your Google password. +* **Your email account** - the source of truth for your emails and labels, accessed either through the official + [Gmail API](https://developers.google.com/gmail/api/guides) or, for any other IMAP-capable provider, through a + plain IMAP connection. `gmailsorter` never stores your Google password, and for IMAP accounts the password you + provide is used only to log in - it is not persisted anywhere. When talking to a plain IMAP server, each mailbox + folder plays the role a Gmail label plays throughout the rest of this page - "moving" an email between labels + means moving it between IMAP folders. * **A local database** - a SQL database (SQLite by default, though any database supported by [SQLAlchemy](https://www.sqlalchemy.org/) works) that keeps a private copy of your email metadata, your login token and your trained models. In the Docker container and the plain Python package this database lives entirely diff --git a/docs/source/developer.md b/docs/source/developer.md index 7a35901..77b6b19 100644 --- a/docs/source/developer.md +++ b/docs/source/developer.md @@ -60,6 +60,34 @@ a selected label `"MyLabel"`. Then reloads the machine learning model from the l the correct labels for these emails. The `recommendation_ratio` defines the level of certainty required to actually move the email, with `0.9` equalling a certainty of 90%. +## IMAP accounts +`gmailsorter` also supports plain IMAP accounts (username and password, e.g. an app +password), for mail servers other than Google Mail. Import the `Imap` class instead of +`Gmail`: +``` +from gmailsorter import Imap +``` +``` +imap = Imap( + host="imap.example.com", + port=993, + username="user@example.com", + password="app-password", + connection_str="sqlite:////absolute/path/to/email.db", +) +``` +`Imap` exposes the exact same `update_database()`, `get_all_emails_in_database()` and +`filter_messages_from_server()` methods as `Gmail` - the only difference is that IMAP +folders play the role Gmail labels play elsewhere in this document: each folder is +treated as one label, and moving an email means moving it from one IMAP folder to +another. A command line interface is also available as `gmailsorter-imap`, reading the +account password from an environment variable (`IMAP_PASSWORD` by default) rather than +accepting it as a command line argument: +``` +export IMAP_PASSWORD=app-password +gmailsorter-imap --host imap.example.com --username user@example.com -d sqlite:///email.db -u +``` + ## Future directions The current machine learning model is limited in the precision and memory usage. So there is a great interest to replace it with a computationally more efficient model. All suggestions and feedback are welcome. Beyond the optimization of the From d0dee2815a81f3a6a1e1a20b18e9bfbabcbc9757 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Sat, 25 Jul 2026 10:28:00 +0200 Subject: [PATCH 14/36] style: apply ruff format and black to gmailsorter and tests Fixes the black --check CI failure on gmailsorter/google/message.py and gmailsorter/imap/message.py, the ruff import-sort error in gmailsorter/google/message.py, and ruff-format drift across several test files. The mock-folder helper in tests/test_imap_integration_units.py is restructured to an early-return default so that black and ruff format agree on it. Co-Authored-By: Claude Sonnet 5 --- gmailsorter/google/message.py | 6 +++++- gmailsorter/imap/message.py | 4 +--- tests/test_imap_integration_units.py | 11 ++++------- tests/test_imap_message.py | 4 +--- tests/test_imap_service_integration.py | 4 +--- tests/test_mail_base.py | 24 ++++++++++++++++++++---- tests/test_message.py | 6 +++++- 7 files changed, 37 insertions(+), 22 deletions(-) diff --git a/gmailsorter/google/message.py b/gmailsorter/google/message.py index 4b8b30f..b368b91 100644 --- a/gmailsorter/google/message.py +++ b/gmailsorter/google/message.py @@ -1,6 +1,10 @@ import base64 -from gmailsorter.base.message import AbstractMessage, email_date_converter, strip_html_tags +from gmailsorter.base.message import ( + AbstractMessage, + email_date_converter, + strip_html_tags, +) def get_email_dict(message): diff --git a/gmailsorter/imap/message.py b/gmailsorter/imap/message.py index f51be47..607e4b2 100644 --- a/gmailsorter/imap/message.py +++ b/gmailsorter/imap/message.py @@ -32,9 +32,7 @@ def get_from(self): if from_header is None: return None addresses = [ - address - for _, address in email.utils.getaddresses([from_header]) - if address + address for _, address in email.utils.getaddresses([from_header]) if address ] if len(addresses) == 1: return addresses[0].lower() diff --git a/tests/test_imap_integration_units.py b/tests/test_imap_integration_units.py index 3f17518..73fe75d 100644 --- a/tests/test_imap_integration_units.py +++ b/tests/test_imap_integration_units.py @@ -40,16 +40,13 @@ class TestImapMailBase(TestCase): def _create_mock_service_with_folders(self, folders=None): service = MagicMock() service.capabilities = ["IMAP4rev1", "MOVE"] - service.list.return_value = ( - "OK", - folders - if folders is not None - else [ + if folders is None: + folders = [ b'(\\HasNoChildren) "/" "INBOX"', b'(\\HasNoChildren) "/" "MailSortInbox"', b'(\\Noselect \\HasChildren) "/" "[Gmail]"', - ], - ) + ] + service.list.return_value = ("OK", folders) return service def test_get_label_translate_dict_skips_noselect(self): diff --git a/tests/test_imap_message.py b/tests/test_imap_message.py index 3c5c270..0a9513a 100644 --- a/tests/test_imap_message.py +++ b/tests/test_imap_message.py @@ -25,9 +25,7 @@ def test_from(self): self.assertEqual(self.message.get_from(), "sender@server.net") def test_to(self): - self.assertEqual( - self.message.get_to(), ["me@mail.com", "friend@provider.org"] - ) + self.assertEqual(self.message.get_to(), ["me@mail.com", "friend@provider.org"]) def test_cc_empty(self): self.assertEqual(self.message.get_cc(), []) diff --git a/tests/test_imap_service_integration.py b/tests/test_imap_service_integration.py index 3f55c51..c0e645a 100644 --- a/tests/test_imap_service_integration.py +++ b/tests/test_imap_service_integration.py @@ -99,9 +99,7 @@ def test_update_database_and_move_round_trip(self): df = imap.get_all_emails_in_database() self.assertIn("Integration test message", df["subject"].tolist()) - stored_id = df.loc[ - df["subject"] == "Integration test message", "id" - ].iloc[0] + stored_id = df.loc[df["subject"] == "Integration test message", "id"].iloc[0] self.assertTrue(stored_id.startswith("INBOX\x1f")) imap._modify_message_labels( diff --git a/tests/test_mail_base.py b/tests/test_mail_base.py index c68f558..63cd0a9 100644 --- a/tests/test_mail_base.py +++ b/tests/test_mail_base.py @@ -10,14 +10,19 @@ class _StubMailBox(AbstractMailBox): """Minimal concrete AbstractMailBox used to test the shared loop in isolation.""" def __init__(self, label_dict_fixture=None, **kwargs): - self.label_dict_fixture = label_dict_fixture or {"Inbox": "Inbox", "Spam": "Spam"} + self.label_dict_fixture = label_dict_fixture or { + "Inbox": "Inbox", + "Spam": "Spam", + } self.search_result = [] self.message_detail_dict = {} self.modify_calls = [] self.labels_for_email_dict = {} super().__init__(mail_service=MagicMock(), **kwargs) - def _search_email_on_server(self, query_string="", label_lst=None, only_message_ids=False): + def _search_email_on_server( + self, query_string="", label_lst=None, only_message_ids=False + ): return self.search_result def _get_message_detail(self, message_id, email_format=None, metadata_headers=None): @@ -26,7 +31,9 @@ def _get_message_detail(self, message_id, email_format=None, metadata_headers=No def _get_label_translate_dict(self): return self.label_dict_fixture - def _modify_message_labels(self, message_id, label_id_remove_lst=None, label_id_add_lst=None): + def _modify_message_labels( + self, message_id, label_id_remove_lst=None, label_id_add_lst=None + ): self.modify_calls.append((message_id, label_id_remove_lst, label_id_add_lst)) def _get_labels_for_email(self, message_id): @@ -104,7 +111,16 @@ def test_update_database_marks_missing_as_deleted(self): def test_fit_machine_learning_model_to_database(self, encode_mock, fit_mock): db_email = MagicMock() db_email.get_all_emails.return_value = pd.DataFrame( - [{"id": "x", "from": "a@b.com", "to": [], "cc": [], "labels": [], "threads": "t"}] + [ + { + "id": "x", + "from": "a@b.com", + "to": [], + "cc": [], + "labels": [], + "threads": "t", + } + ] ) db_ml = MagicMock() mailbox = _StubMailBox(database_email=db_email, database_ml=db_ml) diff --git a/tests/test_message.py b/tests/test_message.py index 53d6c85..3d4ce85 100644 --- a/tests/test_message.py +++ b/tests/test_message.py @@ -1,6 +1,10 @@ from unittest import TestCase from datetime import datetime -from gmailsorter.base.message import email_date_converter, AbstractMessage, strip_html_tags +from gmailsorter.base.message import ( + email_date_converter, + AbstractMessage, + strip_html_tags, +) class MessageTest(TestCase): From 1717d9fbba4e37dbd5db19d05fc9ab8c8a704ae7 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Sat, 25 Jul 2026 10:28:24 +0200 Subject: [PATCH 15/36] fix: use BODY.PEEK[] for IMAP fetch to avoid marking messages as read RFC822 is equivalent to BODY[], which implicitly sets the \Seen flag on every fetched message (RFC 3501). update_database() fetches every message in every folder, so it silently marked the user's entire mailbox as read. BODY.PEEK[] returns the identical raw message without touching \Seen. Co-Authored-By: Claude Sonnet 5 --- gmailsorter/imap/mail.py | 8 ++++++-- tests/test_imap_integration_units.py | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/gmailsorter/imap/mail.py b/gmailsorter/imap/mail.py index 6c14d36..5b7d722 100644 --- a/gmailsorter/imap/mail.py +++ b/gmailsorter/imap/mail.py @@ -71,7 +71,11 @@ def _search_folder(self, folder): def _get_message_detail(self, message_id, email_format=None, metadata_headers=None): """ - Fetch the raw RFC822 message for a composite "{folder}\\x1f{uid}" id. + Fetch the raw message for a composite "{folder}\\x1f{uid}" id. + + BODY.PEEK[] is used rather than RFC822/BODY[] because the latter implicitly + set the \\Seen flag (RFC 3501), which would mark the whole mailbox as read + on every update_database() run. Returns: tuple: (folder, uid, email.message.Message) @@ -80,7 +84,7 @@ def _get_message_detail(self, message_id, email_format=None, metadata_headers=No status, _ = self._service.select(f'"{folder}"') if status != "OK": raise RuntimeError(f"Could not select IMAP folder {folder!r}") - status, data = self._service.uid("fetch", uid, "(RFC822)") + status, data = self._service.uid("fetch", uid, "(BODY.PEEK[])") if status != "OK" or not data or data[0] is None: raise RuntimeError(f"Could not fetch IMAP message {message_id!r}") raw_message = data[0][1] diff --git a/tests/test_imap_integration_units.py b/tests/test_imap_integration_units.py index 73fe75d..9eb6c02 100644 --- a/tests/test_imap_integration_units.py +++ b/tests/test_imap_integration_units.py @@ -92,13 +92,13 @@ def test_get_message_detail_selects_and_fetches(self): service = self._create_mock_service_with_folders() raw_message = b"Subject: hi\r\nFrom: a@b.com\r\nTo: c@d.com\r\n\r\nbody" service.select.return_value = ("OK", [b"1"]) - service.uid.return_value = ("OK", [(b"1 (RFC822 {10}", raw_message)]) + service.uid.return_value = ("OK", [(b"1 (BODY[] {10}", raw_message)]) mail = ImapMailBase(mail_service=service) folder, uid, message = mail._get_message_detail(message_id="INBOX\x1f7") service.select.assert_called_once_with('"INBOX"') - service.uid.assert_called_once_with("fetch", "7", "(RFC822)") + service.uid.assert_called_once_with("fetch", "7", "(BODY.PEEK[])") self.assertEqual(folder, "INBOX") self.assertEqual(uid, "7") self.assertEqual(message["Subject"], "hi") From 2768c5321cf882fabb23bf489f1d83849afefe2e Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Sat, 25 Jul 2026 10:30:05 +0200 Subject: [PATCH 16/36] fix: make the IMAP LIST-response parser tolerant of legal responses _parse_list_entry crashed with an unguarded match.group() AttributeError on three legal LIST responses: an unquoted NIL hierarchy delimiter, a mailbox name sent as an IMAP literal (which imaplib returns as a tuple rather than bytes), and imaplib returning [None] for an empty mailbox list. Because _get_label_translate_dict runs from AbstractMailBox.__init__, any of these turned constructing Imap into an opaque failure. Unparseable entries are now skipped instead of raising. Co-Authored-By: Claude Sonnet 5 --- gmailsorter/imap/mail.py | 64 +++++++++++++++++++++++++--- tests/test_imap_integration_units.py | 36 ++++++++++++++++ 2 files changed, 95 insertions(+), 5 deletions(-) diff --git a/gmailsorter/imap/mail.py b/gmailsorter/imap/mail.py index 5b7d722..22bcbe3 100644 --- a/gmailsorter/imap/mail.py +++ b/gmailsorter/imap/mail.py @@ -10,18 +10,39 @@ from gmailsorter.ml import get_machine_learning_database _LIST_ENTRY_PATTERN = re.compile( - r'\((?P[^)]*)\)\s+"(?P.*)"\s+(?P.+)' + r'\((?P[^)]*)\)\s+(?:"(?P[^"]*)"|NIL)\s*(?P.*)' ) +def _decode_imap_bytes(value): + """ + Decode a bytes/str fragment of an IMAP response, returning None for anything else. + + Mailbox names are usually pure ASCII (modified UTF-7), but servers may send raw + UTF-8 literals - latin-1 is used as a lossless fallback so that a surprising + encoding never raises out of the LIST parser. + """ + if isinstance(value, str): + return value + if not isinstance(value, bytes): + return None + try: + return value.decode() + except UnicodeDecodeError: + return value.decode("latin-1") + + class ImapMailBase(AbstractMailBox): def _get_label_translate_dict(self): status, mailbox_lst = self._service.list() - if status != "OK" or mailbox_lst is None: + if status != "OK" or not mailbox_lst: return {} label_dict = {} for entry in mailbox_lst: - flags, _delimiter, name = self._parse_list_entry(entry) + parsed_entry = self._parse_list_entry(entry) + if parsed_entry is None: + continue + flags, _delimiter, name = parsed_entry if "\\Noselect" in flags: continue label_dict[name] = name @@ -126,11 +147,44 @@ def _parse_message(self, message): @staticmethod def _parse_list_entry(entry): - decoded = entry.decode() if isinstance(entry, bytes) else entry + """ + Parse a single entry of an IMAP LIST response. + + Handles the plain bytes/str form, the ``(header, literal_name)`` tuple form + imaplib returns when the server encodes the mailbox name as an IMAP literal, + and the unquoted ``NIL`` hierarchy delimiter which is legal per RFC 3501 for + servers without a folder hierarchy. + + Args: + entry (bytes/str/tuple/None): one element of ``imaplib.IMAP4.list()`` data + + Returns: + tuple/None: (flags, delimiter, name) or None if the entry is unparseable. + Unparseable entries are skipped rather than raised on, because + this runs from ``AbstractMailBox.__init__``. + """ + literal_name = None + if isinstance(entry, tuple): + try: + entry, literal_name = entry[0], _decode_imap_bytes(entry[1]) + except IndexError: + return None + if literal_name is None: + return None + decoded = _decode_imap_bytes(entry) + if decoded is None: + return None match = _LIST_ENTRY_PATTERN.match(decoded) + if match is None: + return None flags = match.group("flags").split() delimiter = match.group("delimiter") - name = match.group("name").strip('"') + if literal_name is not None: + name = literal_name + else: + name = match.group("name").strip().strip('"') + if not name: + return None return flags, delimiter, name @staticmethod diff --git a/tests/test_imap_integration_units.py b/tests/test_imap_integration_units.py index 9eb6c02..ff9e87c 100644 --- a/tests/test_imap_integration_units.py +++ b/tests/test_imap_integration_units.py @@ -55,6 +55,42 @@ def test_get_label_translate_dict_skips_noselect(self): self.assertEqual(sorted(mail.labels), ["INBOX", "MailSortInbox"]) + def test_get_label_translate_dict_accepts_nil_delimiter(self): + service = self._create_mock_service_with_folders( + folders=[b"(\\HasNoChildren) NIL INBOX"] + ) + mail = ImapMailBase(mail_service=service) + + self.assertEqual(mail.labels, ["INBOX"]) + + def test_get_label_translate_dict_accepts_literal_name_tuple(self): + service = self._create_mock_service_with_folders( + folders=[(b'(\\HasNoChildren) "/" {11}', b"MailSortBox"), b")"] + ) + mail = ImapMailBase(mail_service=service) + + self.assertEqual(mail.labels, ["MailSortBox"]) + + def test_get_label_translate_dict_handles_empty_mailbox_list(self): + service = self._create_mock_service_with_folders(folders=[None]) + mail = ImapMailBase(mail_service=service) + + self.assertEqual(mail.labels, []) + + def test_get_label_translate_dict_skips_unparseable_entries(self): + service = self._create_mock_service_with_folders( + folders=[b"total garbage", b'(\\HasNoChildren) "/" "MailSortInbox"'] + ) + mail = ImapMailBase(mail_service=service) + + self.assertEqual(mail.labels, ["MailSortInbox"]) + + def test_parse_list_entry_returns_none_for_unparseable_input(self): + self.assertIsNone(ImapMailBase._parse_list_entry(None)) + self.assertIsNone(ImapMailBase._parse_list_entry(b"not a list response")) + self.assertIsNone(ImapMailBase._parse_list_entry((b'(\\Noselect) "/" {3}',))) + self.assertIsNone(ImapMailBase._parse_list_entry(b'(\\HasNoChildren) "/" ')) + def test_search_email_on_server_single_folder(self): service = self._create_mock_service_with_folders() service.select.return_value = ("OK", [b"1"]) From 7f9cfc49cd11b5caf460d598a0a9b5c2465f901f Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Sat, 25 Jul 2026 10:31:15 +0200 Subject: [PATCH 17/36] fix: exclude Trash/Junk/Sent/Drafts folders from IMAP labels update_database() scans every folder and filter_messages_from_server can recommend moving mail into any label the model was trained on, so treating every selectable folder as a sorting label let the classifier learn to file mail into Trash, Spam, Sent or Drafts. Special folders are now detected via the RFC 6154 special-use attributes and, for servers which do not advertise them, a short list of unambiguous folder names (matched exactly and case-insensitively, also against the leaf of a nested name). Co-Authored-By: Claude Sonnet 5 --- gmailsorter/imap/mail.py | 65 +++++++++++++++++++++++++++- tests/test_imap_integration_units.py | 58 +++++++++++++++++++++++++ 2 files changed, 121 insertions(+), 2 deletions(-) diff --git a/gmailsorter/imap/mail.py b/gmailsorter/imap/mail.py index 22bcbe3..345b184 100644 --- a/gmailsorter/imap/mail.py +++ b/gmailsorter/imap/mail.py @@ -13,6 +13,45 @@ r'\((?P[^)]*)\)\s+(?:"(?P[^"]*)"|NIL)\s*(?P.*)' ) +# Folder attributes which mark a folder as not usable as a sorting target: \Noselect +# (RFC 3501) plus the special-use attributes of RFC 6154. Without this the machine +# learning model can be trained on - and therefore recommend moving mail into - +# Trash, Spam, Sent or Drafts. +_SKIP_FOLDER_ATTRIBUTES = frozenset( + { + "\\noselect", + "\\all", + "\\archive", + "\\drafts", + "\\flagged", + "\\junk", + "\\sent", + "\\trash", + } +) + +# Many servers (GreenMail among them) do not advertise the RFC 6154 special-use +# attributes at all, so a deliberately short list of unambiguous special folder names +# is used as a fallback. Only exact (case-insensitive) matches are excluded, so a +# custom sorting folder such as "Sorted" or "MailSortInbox" is unaffected. +_SKIP_FOLDER_NAMES = frozenset( + { + "all mail", + "archive", + "deleted items", + "deleted messages", + "drafts", + "junk", + "junk e-mail", + "junk email", + "sent", + "sent items", + "sent messages", + "spam", + "trash", + } +) + def _decode_imap_bytes(value): """ @@ -42,12 +81,34 @@ def _get_label_translate_dict(self): parsed_entry = self._parse_list_entry(entry) if parsed_entry is None: continue - flags, _delimiter, name = parsed_entry - if "\\Noselect" in flags: + flags, delimiter, name = parsed_entry + if self._is_special_folder(flags=flags, delimiter=delimiter, name=name): continue label_dict[name] = name return label_dict + @staticmethod + def _is_special_folder(flags, delimiter, name): + """ + Check whether an IMAP folder is a special-purpose folder rather than a folder + emails may be sorted into. + + Args: + flags (list): folder attributes from the LIST response + delimiter (str/None): hierarchy delimiter from the LIST response + name (str): full folder name + + Returns: + bool: True if the folder must not be offered as a sorting label + """ + if any(flag.lower() in _SKIP_FOLDER_ATTRIBUTES for flag in flags): + return True + name_lst = [name.strip().lower()] + if delimiter: + # also check the leaf of a nested name such as "[Gmail]/Trash" + name_lst.append(name.rsplit(delimiter, 1)[-1].strip().lower()) + return any(candidate in _SKIP_FOLDER_NAMES for candidate in name_lst) + def _search_email_on_server( self, query_string="", label_lst=None, only_message_ids=False ): diff --git a/tests/test_imap_integration_units.py b/tests/test_imap_integration_units.py index ff9e87c..74cf882 100644 --- a/tests/test_imap_integration_units.py +++ b/tests/test_imap_integration_units.py @@ -55,6 +55,64 @@ def test_get_label_translate_dict_skips_noselect(self): self.assertEqual(sorted(mail.labels), ["INBOX", "MailSortInbox"]) + def test_get_label_translate_dict_skips_special_use_attributes(self): + service = self._create_mock_service_with_folders( + folders=[ + b'(\\HasNoChildren \\Trash) "/" "Papierkorb"', + b'(\\HasNoChildren \\Junk) "/" "Unerwuenscht"', + b'(\\HasNoChildren \\Sent) "/" "Gesendet"', + b'(\\HasNoChildren \\Drafts) "/" "Entwuerfe"', + b'(\\HasNoChildren \\Archive) "/" "Ablage"', + b'(\\HasNoChildren \\All) "/" "Alle"', + b'(\\HasNoChildren \\Flagged) "/" "Markiert"', + b'(\\HasNoChildren) "/" "MailSortInbox"', + ] + ) + mail = ImapMailBase(mail_service=service) + + self.assertEqual(mail.labels, ["MailSortInbox"]) + + def test_get_label_translate_dict_skips_special_names_without_attributes(self): + service = self._create_mock_service_with_folders( + folders=[ + b'(\\HasNoChildren) "/" "Trash"', + b'(\\HasNoChildren) "/" "junk e-mail"', + b'(\\HasNoChildren) "/" "Deleted Items"', + b'(\\HasNoChildren) "/" "SPAM"', + b'(\\HasNoChildren) "/" "Sent Items"', + b'(\\HasNoChildren) "/" "Drafts"', + b'(\\HasNoChildren) "/" "All Mail"', + b'(\\HasNoChildren) "/" "[Gmail]/Trash"', + b'(\\HasNoChildren) "/" "MailSortInbox"', + ] + ) + mail = ImapMailBase(mail_service=service) + + self.assertEqual(mail.labels, ["MailSortInbox"]) + + def test_get_label_translate_dict_keeps_custom_folders(self): + service = self._create_mock_service_with_folders( + folders=[ + b'(\\HasNoChildren) "/" "MailSortInbox"', + b'(\\HasNoChildren) "/" "Sorted"', + b'(\\HasNoChildren) "/" "Archived Projects"', + b'(\\HasNoChildren) "/" "Trashcan Design"', + b'(\\HasNoChildren) "/" "INBOX"', + ] + ) + mail = ImapMailBase(mail_service=service) + + self.assertEqual( + sorted(mail.labels), + [ + "Archived Projects", + "INBOX", + "MailSortInbox", + "Sorted", + "Trashcan Design", + ], + ) + def test_get_label_translate_dict_accepts_nil_delimiter(self): service = self._create_mock_service_with_folders( folders=[b"(\\HasNoChildren) NIL INBOX"] From dfbbe1455d6c0997a5c1a0e9a410d76b3a935ba0 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Sat, 25 Jul 2026 10:31:55 +0200 Subject: [PATCH 18/36] fix: use UID EXPUNGE in the IMAP COPY fallback when UIDPLUS is available The COPY+STORE fallback used for servers without MOVE finished with a bare EXPUNGE, which permanently removes every \Deleted-flagged message in the folder - including messages the user's own mail client had flagged but not yet expunged. When the server advertises UIDPLUS the RFC 4315 UID EXPUNGE command is now used to expunge only the message being moved. Co-Authored-By: Claude Sonnet 5 --- gmailsorter/imap/mail.py | 10 +++++++++- tests/test_imap_integration_units.py | 23 +++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/gmailsorter/imap/mail.py b/gmailsorter/imap/mail.py index 345b184..b8f21b3 100644 --- a/gmailsorter/imap/mail.py +++ b/gmailsorter/imap/mail.py @@ -196,7 +196,15 @@ def _modify_message_labels( f"Could not copy IMAP message {message_id!r} to {target_folder!r}" ) self._service.uid("store", uid, "+FLAGS", r"(\Deleted)") - self._service.expunge() + if "UIDPLUS" in self._service.capabilities: + # RFC 4315 UID EXPUNGE - expunges only the message just copied + self._service.uid("expunge", uid) + else: + # Without UIDPLUS a bare EXPUNGE is the only option, and it also + # permanently removes any other message in this folder which is + # already flagged as \Deleted - an unavoidable limitation of + # servers supporting neither MOVE nor UIDPLUS. + self._service.expunge() def _get_labels_for_email(self, message_id): folder, _uid = message_id.split("\x1f", 1) diff --git a/tests/test_imap_integration_units.py b/tests/test_imap_integration_units.py index 74cf882..14b44d0 100644 --- a/tests/test_imap_integration_units.py +++ b/tests/test_imap_integration_units.py @@ -241,6 +241,29 @@ def test_modify_message_labels_falls_back_to_copy_delete(self): ) service.expunge.assert_called_once() + def test_modify_message_labels_uses_uid_expunge_with_uidplus(self): + service = self._create_mock_service_with_folders() + service.capabilities = ["IMAP4rev1", "UIDPLUS"] + service.select.return_value = ("OK", [b"1"]) + service.uid.return_value = ("OK", [b"1"]) + mail = ImapMailBase(mail_service=service) + + mail._modify_message_labels( + message_id="INBOX\x1f7", + label_id_remove_lst=["INBOX"], + label_id_add_lst=["MailSortInbox"], + ) + + self.assertEqual( + service.uid.call_args_list, + [ + (("copy", "7", '"MailSortInbox"'),), + (("store", "7", "+FLAGS", r"(\Deleted)"),), + (("expunge", "7"),), + ], + ) + service.expunge.assert_not_called() + def test_modify_message_labels_noop_without_target(self): service = self._create_mock_service_with_folders() mail = ImapMailBase(mail_service=service) From 9ffd65c6866c52f9d3a0fd36f18a90cffa694b1d Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Sat, 25 Jul 2026 10:33:05 +0200 Subject: [PATCH 19/36] test: fail the IMAP integration job when no IMAP server is reachable setUpClass skipped whenever no IMAP server answered, which is right for a local contributor without Docker but meant the imap-integration CI job reported green even if GreenMail never came up. With IMAP_INTEGRATION_REQUIRED set - which the workflow now does - an unreachable server raises instead of skipping. Without it the previous clean-skip behaviour is unchanged. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/unittest.yml | 2 + tests/test_imap_service_integration.py | 68 ++++++++++++++++++++++++-- 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 55a2052..eae0c34 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -66,6 +66,8 @@ jobs: TEST_IMAP_USERNAME: testuser TEST_EMAIL: testuser@example.test TEST_EMAIL_PASSWORD: secret + # fail this job rather than skip it if the greenmail service never came up + IMAP_INTEGRATION_REQUIRED: "true" steps: - uses: actions/checkout@v4 diff --git a/tests/test_imap_service_integration.py b/tests/test_imap_service_integration.py index c0e645a..5d91814 100644 --- a/tests/test_imap_service_integration.py +++ b/tests/test_imap_service_integration.py @@ -5,9 +5,24 @@ import uuid from email.message import EmailMessage from imaplib import IMAP4 +from unittest.mock import patch from gmailsorter.local import Imap +#: When this environment variable is set to any non-empty value, an unreachable IMAP +#: server is a test failure rather than a skip. CI sets it so the imap-integration job +#: cannot report green when the GreenMail service never came up; local runs leave it +#: unset and keep the clean skip. +IMAP_INTEGRATION_REQUIRED = "IMAP_INTEGRATION_REQUIRED" + + +def imap_integration_required(): + """ + Returns: + bool: True if a missing IMAP test server must fail instead of skip + """ + return bool(os.environ.get(IMAP_INTEGRATION_REQUIRED, "").strip()) + class TestImapServiceIntegration(unittest.TestCase): smtp_host = os.environ.get("TEST_SMTP_HOST", "localhost") @@ -20,12 +35,19 @@ class TestImapServiceIntegration(unittest.TestCase): @classmethod def setUpClass(cls): - if not cls._imap_server_available(): - raise unittest.SkipTest( - "No IMAP test server reachable at " - f"{cls.imap_host}:{cls.imap_port} - start the greenmail container " - "described in https://github.com/jan-janssen/testing-imap to run this test." + if cls._imap_server_available(): + return + reason = ( + "No IMAP test server reachable at " + f"{cls.imap_host}:{cls.imap_port} - start the greenmail container " + "described in https://github.com/jan-janssen/testing-imap to run this test." + ) + if imap_integration_required(): + raise AssertionError( + f"{IMAP_INTEGRATION_REQUIRED} is set, so this test must actually run " + f"rather than be skipped. {reason}" ) + raise unittest.SkipTest(reason) @classmethod def _imap_server_available(cls, timeout=2.0, attempts=5, delay=1.5): @@ -117,5 +139,41 @@ def test_update_database_and_move_round_trip(self): self.assertTrue(moved_row.iloc[0]["id"].startswith("MailSortInbox\x1f")) +class TestImapIntegrationRequiredGate(unittest.TestCase): + """Covers the env-var gate itself, which needs no IMAP server.""" + + def test_not_required_by_default(self): + with patch.dict(os.environ, {}, clear=True): + self.assertFalse(imap_integration_required()) + + def test_not_required_for_empty_value(self): + with patch.dict(os.environ, {IMAP_INTEGRATION_REQUIRED: " "}): + self.assertFalse(imap_integration_required()) + + def test_required_for_truthy_value(self): + with patch.dict(os.environ, {IMAP_INTEGRATION_REQUIRED: "true"}): + self.assertTrue(imap_integration_required()) + + def test_set_up_class_skips_without_server_by_default(self): + with ( + patch.dict(os.environ, {}, clear=True), + patch.object( + TestImapServiceIntegration, "_imap_server_available", return_value=False + ), + self.assertRaises(unittest.SkipTest), + ): + TestImapServiceIntegration.setUpClass() + + def test_set_up_class_fails_without_server_when_required(self): + with ( + patch.dict(os.environ, {IMAP_INTEGRATION_REQUIRED: "true"}), + patch.object( + TestImapServiceIntegration, "_imap_server_available", return_value=False + ), + self.assertRaises(AssertionError), + ): + TestImapServiceIntegration.setUpClass() + + if __name__ == "__main__": unittest.main() From db2d31590dcb56ef2602445950ada27d51f26f10 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Sat, 25 Jul 2026 10:36:11 +0200 Subject: [PATCH 20/36] feat: add IMAP connection lifecycle management and reconnect-on-drop Nothing ever logged out of the IMAP connection and a dropped connection left the Imap instance permanently unusable, which matters because IMAP servers commonly time out idle connections and update_database() can keep one connection busy for a long time on a large mailbox. ImapMailBase now offers close() (tolerating an already dead connection) and works as a context manager. The four network-calling hooks run through _run_with_reconnect, which retries the operation exactly once after calling the overridable _reconnect() hook; only Imap, which knows the connection details, implements it, and the original IMAP4.abort is re-raised otherwise. Gmail's behaviour is unchanged - none of this is on AbstractMailBox. Co-Authored-By: Claude Sonnet 5 --- gmailsorter/imap/mail.py | 90 +++++++++++++++++ gmailsorter/local.py | 37 +++++-- tests/test_imap_integration_units.py | 130 +++++++++++++++++++++++++ tests/test_imap_service_integration.py | 6 +- 4 files changed, 254 insertions(+), 9 deletions(-) diff --git a/gmailsorter/imap/mail.py b/gmailsorter/imap/mail.py index b8f21b3..15b1842 100644 --- a/gmailsorter/imap/mail.py +++ b/gmailsorter/imap/mail.py @@ -1,4 +1,6 @@ +import contextlib import email +import imaplib import re from sqlalchemy import create_engine @@ -72,7 +74,68 @@ def _decode_imap_bytes(value): class ImapMailBase(AbstractMailBox): + def close(self): + """ + Log out and close the IMAP connection. + + Errors raised while logging out are ignored on purpose - the connection is + being discarded anyway, and a connection which the server already dropped + must not turn closing it into a failure. + """ + with contextlib.suppress(imaplib.IMAP4.error, OSError): + self._service.logout() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.close() + return False + + def _reconnect(self): + """ + Re-establish the IMAP connection and store it in self._service. + + Only a subclass which knows the connection details can do this, so the base + class signals that no reconnect is possible and the original connection error + is re-raised to the caller. + """ + raise NotImplementedError( + "ImapMailBase cannot reconnect because it does not know the connection " + "details - use gmailsorter.Imap for automatic reconnects." + ) + + def _run_with_reconnect(self, operation): + """ + Run an IMAP operation, retrying it exactly once on a dropped connection. + + IMAP servers commonly drop idle connections after 20-30 minutes and + update_database() can keep a single connection busy for far longer than that + on a large mailbox. imaplib signals a dropped connection with IMAP4.abort, + which would otherwise leave this instance permanently unusable. The operation + is re-run from the start after reconnecting, so a modification the server had + already applied before dropping the connection may report a failure on the + retry rather than being applied twice. + + Args: + operation (callable): zero-argument callable performing the IMAP calls + + Returns: + the return value of `operation` + """ + try: + return operation() + except imaplib.IMAP4.abort as error: + try: + self._reconnect() + except NotImplementedError: + raise error from None + return operation() + def _get_label_translate_dict(self): + return self._run_with_reconnect(self._get_label_translate_dict_impl) + + def _get_label_translate_dict_impl(self): status, mailbox_lst = self._service.list() if status != "OK" or not mailbox_lst: return {} @@ -123,6 +186,17 @@ def _search_email_on_server( Returns: list: list of composite "{folder}\\x1f{uid}" ids matching the search """ + return self._run_with_reconnect( + lambda: self._search_email_on_server_impl( + query_string=query_string, + label_lst=label_lst, + only_message_ids=only_message_ids, + ) + ) + + def _search_email_on_server_impl( + self, query_string="", label_lst=None, only_message_ids=False + ): if query_string: raise NotImplementedError( "Custom IMAP search queries are not supported yet, only label_lst filtering." @@ -162,6 +236,11 @@ def _get_message_detail(self, message_id, email_format=None, metadata_headers=No Returns: tuple: (folder, uid, email.message.Message) """ + return self._run_with_reconnect( + lambda: self._get_message_detail_impl(message_id=message_id) + ) + + def _get_message_detail_impl(self, message_id): folder, uid = message_id.split("\x1f", 1) status, _ = self._service.select(f'"{folder}"') if status != "OK": @@ -175,6 +254,17 @@ def _get_message_detail(self, message_id, email_format=None, metadata_headers=No def _modify_message_labels( self, message_id, label_id_remove_lst=None, label_id_add_lst=None + ): + return self._run_with_reconnect( + lambda: self._modify_message_labels_impl( + message_id=message_id, + label_id_remove_lst=label_id_remove_lst, + label_id_add_lst=label_id_add_lst, + ) + ) + + def _modify_message_labels_impl( + self, message_id, label_id_remove_lst=None, label_id_add_lst=None ): if not label_id_add_lst: return diff --git a/gmailsorter/local.py b/gmailsorter/local.py index a5b3965..1671025 100644 --- a/gmailsorter/local.py +++ b/gmailsorter/local.py @@ -83,6 +83,12 @@ def __init__( """ Imap class to manage Emails via a plain IMAP connection directly from Python + The IMAP connection is kept open for the lifetime of the object. Call close() + when done, or use the object as a context manager: + + >>> with Imap(...) as imap: + ... imap.update_database() + Args: host (str): IMAP server hostname port (int): IMAP server port, typically 993 for IMAP4_SSL or 143 for IMAP4 @@ -94,18 +100,18 @@ def __init__( email_download_format (str): unused for IMAP, kept for interface parity with Gmail """ self._connection_str = connection_str + # kept so the connection can be re-established when the server drops it + self._host = host + self._port = port + self._username = username + self._password = password + self._use_ssl = use_ssl database_email, database_ml = self._create_databases( connection_str=self._connection_str ) - imap_connection = create_imap_service( - host=host, - port=port, - username=username, - password=password, - use_ssl=use_ssl, - ) + imap_connection = self._connect() super().__init__( mail_service=imap_connection, @@ -115,3 +121,20 @@ def __init__( db_user_id=db_user_id, email_download_format=email_download_format, ) + + def _connect(self): + return create_imap_service( + host=self._host, + port=self._port, + username=self._username, + password=self._password, + use_ssl=self._use_ssl, + ) + + def _reconnect(self): + """ + Re-establish the IMAP connection after the server dropped it, discarding the + old connection first. + """ + self.close() + self._service = self._connect() diff --git a/tests/test_imap_integration_units.py b/tests/test_imap_integration_units.py index 14b44d0..54b08a2 100644 --- a/tests/test_imap_integration_units.py +++ b/tests/test_imap_integration_units.py @@ -1,3 +1,4 @@ +import imaplib from unittest import TestCase from unittest.mock import MagicMock, patch @@ -6,6 +7,24 @@ from gmailsorter.local import Imap +class ReconnectingImapMailBase(ImapMailBase): + """ + Test double which reconnects by swapping in the next prepared mock connection. + + This stands in for gmailsorter.local.Imap, which is the class that actually knows + the connection details, without requiring a real IMAP server. + """ + + def __init__(self, service_lst, **kwargs): + self.reconnect_count = 0 + self._service_lst = list(service_lst) + super().__init__(mail_service=self._service_lst.pop(0), **kwargs) + + def _reconnect(self): + self.reconnect_count += 1 + self._service = self._service_lst.pop(0) + + class TestImapAuthentication(TestCase): @patch("gmailsorter.imap.authentication.IMAP4_SSL") def test_create_service_uses_ssl_by_default(self, imap_ssl_cls): @@ -305,6 +324,84 @@ def test_create_databases(self): self.assertEqual(dbs, ("EMAIL_DB", "ML_DB")) + def test_close_logs_out(self): + service = self._create_mock_service_with_folders() + mail = ImapMailBase(mail_service=service) + + mail.close() + + service.logout.assert_called_once_with() + + def test_close_ignores_logout_failure(self): + for error in (imaplib.IMAP4.abort("connection lost"), OSError("socket gone")): + with self.subTest(error=type(error).__name__): + service = self._create_mock_service_with_folders() + service.logout.side_effect = error + mail = ImapMailBase(mail_service=service) + + mail.close() + + service.logout.assert_called_once_with() + + def test_context_manager_closes_connection(self): + service = self._create_mock_service_with_folders() + + with ImapMailBase(mail_service=service) as mail: + self.assertEqual(sorted(mail.labels), ["INBOX", "MailSortInbox"]) + service.logout.assert_not_called() + + service.logout.assert_called_once_with() + + def test_abort_propagates_without_reconnect_support(self): + service = self._create_mock_service_with_folders() + mail = ImapMailBase(mail_service=service) + service.select.side_effect = imaplib.IMAP4.abort("connection lost") + + with self.assertRaises(imaplib.IMAP4.abort): + mail._search_email_on_server(label_lst=["INBOX"], only_message_ids=True) + + def test_search_reconnects_and_retries_once_after_abort(self): + dead_service = self._create_mock_service_with_folders() + dead_service.select.side_effect = imaplib.IMAP4.abort("connection lost") + fresh_service = self._create_mock_service_with_folders() + fresh_service.select.return_value = ("OK", [b"1"]) + fresh_service.uid.return_value = ("OK", [b"3"]) + mail = ReconnectingImapMailBase(service_lst=[dead_service, fresh_service]) + + ids = mail._search_email_on_server(label_lst=["INBOX"], only_message_ids=True) + + self.assertEqual(mail.reconnect_count, 1) + self.assertEqual(ids, ["INBOX\x1f3"]) + + def test_get_message_detail_reconnects_and_retries_once_after_abort(self): + raw_message = b"Subject: hi\r\nFrom: a@b.com\r\nTo: c@d.com\r\n\r\nbody" + dead_service = self._create_mock_service_with_folders() + dead_service.select.side_effect = imaplib.IMAP4.abort("connection lost") + fresh_service = self._create_mock_service_with_folders() + fresh_service.select.return_value = ("OK", [b"1"]) + fresh_service.uid.return_value = ("OK", [(b"1 (BODY[] {10}", raw_message)]) + mail = ReconnectingImapMailBase(service_lst=[dead_service, fresh_service]) + + folder, uid, message = mail._get_message_detail(message_id="INBOX\x1f7") + + self.assertEqual(mail.reconnect_count, 1) + self.assertEqual((folder, uid), ("INBOX", "7")) + self.assertEqual(message["Subject"], "hi") + + def test_retry_is_attempted_only_once(self): + dead_service = self._create_mock_service_with_folders() + dead_service.select.side_effect = imaplib.IMAP4.abort("connection lost") + still_dead_service = self._create_mock_service_with_folders() + still_dead_service.select.side_effect = imaplib.IMAP4.abort("connection lost") + mail = ReconnectingImapMailBase( + service_lst=[dead_service, still_dead_service], + ) + + with self.assertRaises(imaplib.IMAP4.abort): + mail._search_email_on_server(label_lst=["INBOX"], only_message_ids=True) + + self.assertEqual(mail.reconnect_count, 1) + class TestImapLocalHelpers(TestCase): @patch("gmailsorter.local.ImapMailBase.__init__", return_value=None) @@ -346,6 +443,39 @@ def test_imap_initialization_wiring( email_download_format="metadata", ) + @patch("gmailsorter.local.ImapMailBase.__init__", return_value=None) + @patch("gmailsorter.local.create_imap_service") + @patch("gmailsorter.local.Imap._create_databases") + def test_imap_reconnect_replaces_the_connection( + self, create_databases_mock, create_service_mock, base_init_mock + ): + create_databases_mock.return_value = (MagicMock(), MagicMock()) + dead_connection, fresh_connection = MagicMock(), MagicMock() + create_service_mock.side_effect = [dead_connection, fresh_connection] + + imap = Imap( + host="mail.example.test", + port=143, + username="user", + password="secret", + connection_str="sqlite:///:memory:", + use_ssl=False, + ) + # ImapMailBase.__init__ is mocked out above, so _service is set by hand here + imap._service = dead_connection + + imap._reconnect() + + dead_connection.logout.assert_called_once_with() + self.assertIs(imap._service, fresh_connection) + create_service_mock.assert_called_with( + host="mail.example.test", + port=143, + username="user", + password="secret", + use_ssl=False, + ) + if __name__ == "__main__": import unittest diff --git a/tests/test_imap_service_integration.py b/tests/test_imap_service_integration.py index 5d91814..2664597 100644 --- a/tests/test_imap_service_integration.py +++ b/tests/test_imap_service_integration.py @@ -108,15 +108,17 @@ def test_update_database_and_move_round_trip(self): ) self._wait_for_message_in_inbox(message_id) - imap = Imap( + with Imap( host=self.imap_host, port=self.imap_port, username=self.username, password=self.password, connection_str="sqlite:///:memory:", use_ssl=False, - ) + ) as imap: + self._assert_move_round_trip(imap=imap) + def _assert_move_round_trip(self, imap): imap.update_database(quick=False) df = imap.get_all_emails_in_database() From d5068133f139a80ecaeebd759e5faee550313587 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Sat, 25 Jul 2026 10:41:00 +0200 Subject: [PATCH 21/36] Delete docs/superpowers/plans/2026-07-25-imap-support.md --- .../plans/2026-07-25-imap-support.md | 2330 ----------------- 1 file changed, 2330 deletions(-) delete mode 100644 docs/superpowers/plans/2026-07-25-imap-support.md diff --git a/docs/superpowers/plans/2026-07-25-imap-support.md b/docs/superpowers/plans/2026-07-25-imap-support.md deleted file mode 100644 index 09c5899..0000000 --- a/docs/superpowers/plans/2026-07-25-imap-support.md +++ /dev/null @@ -1,2330 +0,0 @@ -# IMAP Backend Support Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add a second, IMAP-based backend to gmailsorter (username/password auth, one IMAP folder = one Gmail-style label) that reuses the existing fetch-store-train-predict-move loop, ships a `gmailsorter-imap` CLI, and is exercised in CI against a real GreenMail IMAP/SMTP server. - -**Architecture:** Extract the backend-agnostic loop currently living in `GoogleMailBase` into a new `gmailsorter.base.mail.AbstractMailBox` (mirroring the existing `base/` vs `google/` split for `message.py`/`database.py`), then add a parallel `gmailsorter/imap/` package (`authentication.py`, `message.py`, `mail.py`) plus an `Imap` class in `local.py` and a `gmailsorter-imap` console script. - -**Tech Stack:** Python stdlib `imaplib`/`email` (no new dependencies), existing `sqlalchemy`/`pandas`/`scikit-learn` stack, `unittest` + `unittest.mock`, GitHub Actions `services:` container (`greenmail/standalone:2.1.11`). - -## Global Constraints - -- Target Python: `>=3.10` (repo classifiers test 3.11–3.14) — avoid syntax newer than that. -- Lint: `ruff` with rules `E, F, UP, B, SIM, I, C4, ERA, PL` (ignoring `E501`, `PLR0913`) via `.pre-commit-config.yaml`, applied to files under `gmailsorter/`. Keep new code consistent with this (no unused imports, no commented-out code, etc). -- No new runtime dependencies: `imaplib` and `email` are stdlib; do not add packages to `pyproject.toml` `dependencies`. -- Follow existing docstring style (Google-style `Args:`/`Returns:`) used throughout `gmailsorter/`. -- Existing public API (`gmailsorter.Gmail`, `gmailsorter.load_client_secrets_file`, `GoogleMailBase.__init__` signature) must not change — `tests/test_google_integration_units.py` must keep passing with, at most, its `@patch(...)` target strings updated to follow code that moved (no assertion or behavior changes). -- Test runner: `coverage run --omit gmailsorter/_version.py -m unittest discover tests` (see `.github/workflows/unittest.yml`) — every new test file must be discoverable by `unittest discover tests` (class extends `unittest.TestCase`, file name starts with `test_`). -- IMAP auth is username/password only for this plan (no OAuth2/XOAUTH2). Passwords must never be accepted as a literal CLI argument. -- Webapp (`gmailsorter/webapp/`) and daemon (`gmailsorter/daemon/`) are explicitly out of scope — do not modify them. - ---- - -### Task 1: Shared HTML-to-text helper in `base/message.py` - -**Files:** -- Modify: `gmailsorter/base/message.py` -- Modify: `gmailsorter/google/message.py` -- Test: `tests/test_message.py` - -**Interfaces:** -- Produces: `gmailsorter.base.message.strip_html_tags(html: str) -> str`, used by both `gmailsorter/google/message.py` (Task 1) and `gmailsorter/imap/message.py` (Task 3). - -- [ ] **Step 1: Write the failing test** - -Add to `tests/test_message.py` (append inside the existing `MessageTest` class, and add the import at the top): - -```python -from gmailsorter.base.message import email_date_converter, strip_html_tags -``` - -```python - def test_strip_html_tags(self): - self.assertEqual( - strip_html_tags("

Hello World

"), - "Hello World", - ) -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `python -m unittest tests.test_message -v` -Expected: FAIL with `ImportError: cannot import name 'strip_html_tags'` - -- [ ] **Step 3: Move `MLStripper` into `base/message.py` as `strip_html_tags`** - -In `gmailsorter/base/message.py`, add these imports at the top (alongside the existing `abc`/`datetime` imports): - -```python -from html.parser import HTMLParser -from io import StringIO -``` - -Then add, after the `_DATE_HYPHEN_COUNT` constant and before `email_date_converter`: - -```python -# https://stackoverflow.com/questions/753052/strip-html-from-strings-in-python -class _MLStripper(HTMLParser): - def __init__(self): - super().__init__() - self.reset() - self.strict = False - self.convert_charrefs = True - self.text = StringIO() - - def handle_data(self, d): - self.text.write(d) - - def get_data(self): - return self.text.getvalue() - - -def strip_html_tags(html): - stripper = _MLStripper() - stripper.feed(html) - return stripper.get_data() -``` - -- [ ] **Step 4: Update `gmailsorter/google/message.py` to use the shared helper** - -Replace the top of `gmailsorter/google/message.py` — delete the `MLStripper` class and its imports, and import `strip_html_tags` instead: - -```python -import base64 - -from gmailsorter.base.message import AbstractMessage, email_date_converter, strip_html_tags -``` - -(This replaces the old `import base64` / `from html.parser import HTMLParser` / `from io import StringIO` / `from gmailsorter.base.message import ...` block, and removes the `MLStripper` class definition that followed it.) - -In the `Message` class, change `_get_parts_content` to call the shared function instead of `self._strip_tags`: - -```python - def _get_parts_content(self, message_parts): - content_types = [p["mimeType"] for p in message_parts if "mimeType" in p] - if "text/plain" in content_types: - return self._get_email_body( - message_parts=message_parts[content_types.index("text/plain")] - ) - elif "text/html" in content_types: - return strip_html_tags( - html=self._get_email_body( - message_parts=message_parts[content_types.index("text/html")] - ) - ) - elif "multipart/alternative" in content_types: - multi_part_content = message_parts[ - content_types.index("multipart/alternative") - ] - if "parts" in multi_part_content: - return self._get_parts_content( - message_parts=multi_part_content["parts"] - ) - else: - return None - else: - return None -``` - -Delete the now-unused `_strip_tags` staticmethod entirely (it was right after `_get_email_body`). - -- [ ] **Step 5: Run test to verify it passes** - -Run: `python -m unittest tests.test_message -v` -Expected: PASS - -- [ ] **Step 6: Run the full existing suite to confirm no regression** - -Run: `python -m unittest discover tests -v` -Expected: All tests PASS (in particular `tests/test_google_message.py`, unaffected since `Message._get_parts_content`'s observable behavior is unchanged). - -- [ ] **Step 7: Commit** - -```bash -git add gmailsorter/base/message.py gmailsorter/google/message.py tests/test_message.py -git commit -m "refactor: move HTML-to-text stripping into base/message.py so it can be reused by imap/message.py" -``` - ---- - -### Task 2: Extract `AbstractMailBox` shared loop; refactor `GoogleMailBase` - -**Files:** -- Create: `gmailsorter/base/mail.py` -- Modify: `gmailsorter/google/mail.py` (full rewrite) -- Modify: `tests/test_google_integration_units.py` (patch targets only) -- Test: `tests/test_mail_base.py` - -**Interfaces:** -- Produces: `gmailsorter.base.mail.AbstractMailBox(ABC)` with constructor - `__init__(self, mail_service, database_email=None, database_ml=None, user_id="me", db_user_id=1, email_download_format="metadata")`, - concrete methods `labels` (property), `download_emails_for_label(label)`, - `filter_messages_from_server(label, recommendation_ratio=0.9)`, - `fit_machine_learning_model_to_database(n_estimators=100, max_features=400, random_state=42, bootstrap=True, include_deleted=False)`, - `get_all_emails_in_database(include_deleted=False)`, - `update_database(quick=False, label_lst=None, email_format=None)`, - and abstract hooks `_search_email_on_server(query_string="", label_lst=None, only_message_ids=False)`, - `_get_message_detail(message_id, email_format=None, metadata_headers=None)`, - `_get_label_translate_dict()`, - `_modify_message_labels(message_id, label_id_remove_lst=None, label_id_add_lst=None)`, - `_get_labels_for_email(message_id)`, `_parse_message(message)`. -- Consumed by: Task 5 (`ImapMailBase(AbstractMailBox)`). - -This is a **behavior-preserving refactor** of already-tested code, not new functionality, so the TDD cycle here is: move the code, then prove the full existing test suite (plus a new isolation-focused test file) still passes — rather than writing a new failing test first. - -- [ ] **Step 1: Create `gmailsorter/base/mail.py`** - -```python -from abc import ABC, abstractmethod - -import pandas -from tqdm import tqdm - -from gmailsorter.ml import ( - encode_df_for_machine_learning, - fit_machine_learning_models, - get_predictions_from_machine_learning_models, -) - - -class AbstractMailBox(ABC): - def __init__( - self, - mail_service, - database_email=None, - database_ml=None, - user_id="me", - db_user_id=1, - email_download_format="metadata", - ): - """ - Shared fetch-store-train-predict-move loop for a mailbox backend, independent of - whether the backend is the Gmail API or a plain IMAP connection. - - Args: - mail_service: backend-specific connection object (Gmail API service resource, - imaplib connection, ...) - database_email (gmailsorter.base.database.DatabaseInterface): SQLalchemy interface for email database - database_ml (gmailsorter.ml.database.DatabaseInterface): SQLalchemy interface for machine learning database - user_id (str): backend-specific user identifier - db_user_id (int): Default 1 - set a user id when sharing a database with multiple users - email_download_format (str): backend-specific download format hint - """ - self._service = mail_service - self._db_email = database_email - self._db_ml = database_ml - self._db_user_id = db_user_id - self._userid = user_id - self._email_download_format = email_download_format - self._label_dict = self._get_label_translate_dict() - self._label_dict_inverse = {v: k for k, v in self._label_dict.items()} - - @property - def labels(self): - return list(self._label_dict.keys()) - - def download_emails_for_label(self, label): - """ - Download emails for a specific label - - Args: - label (str): label to download emails for - - Returns: - pandas.DataFrame: Email content for the downloaded emails - """ - return self._download_messages_to_dataframe( - message_id_lst=self._search_email_on_server( - label_lst=[label], only_message_ids=True - ) - ) - - def filter_messages_from_server( - self, - label, - recommendation_ratio=0.9, - ): - """ - Filter new emails based on machine learning model recommendations. - - Args: - label (str): Email label to filter for - recommendation_ratio (float): Only accept recommendation above this ratio (0 0: - model_reload_dict, feature_reload_lst = self._db_ml.load_models() - df_partial_features = encode_df_for_machine_learning( - df=df_partial, - feature_lst=feature_reload_lst, - label_lst=list(model_reload_dict.keys()), - return_labels=False, - ) - df_partial_features = df_partial_features.reindex( - sorted(df_partial_features.columns), axis=1 - ) - model_recommendation_dict = get_predictions_from_machine_learning_models( - df_features=df_partial_features, - model_dict=model_reload_dict, - recommendation_ratio=recommendation_ratio, - ) - self._move_emails( - move_email_dict=model_recommendation_dict, label_to_ignore=label - ) - - def fit_machine_learning_model_to_database( - self, - n_estimators=100, - max_features=400, - random_state=42, - bootstrap=True, - include_deleted=False, - ): - """ - Fit machine learning models to emails stored in database and afterwards store machine learning models in - database. - - Args: - n_estimators (int): Number of estimators - max_features (int): Number of features - random_state (int): Random state - bootstrap (boolean): Whether bootstrap samples are used when building trees. If False, the whole dataset is - used to build each tree. (default: true) - include_deleted (bool): Flag to include deleted emails - default False - """ - df_all = self.get_all_emails_in_database(include_deleted=include_deleted) - df_all_features, df_all_labels = encode_df_for_machine_learning( - df=df_all, feature_lst=[], label_lst=[], return_labels=True - ) - df_all_features = df_all_features.loc[ - :, ~df_all_features.columns.duplicated() - ].copy() - df_all_features = df_all_features.reindex( - sorted(df_all_features.columns), axis=1 - ) - model_dict = fit_machine_learning_models( - df_all_features=df_all_features, - df_all_labels=df_all_labels, - n_estimators=n_estimators, - max_features=max_features, - random_state=random_state, - bootstrap=bootstrap, - ) - self._db_ml.store_models( - model_dict=model_dict, - feature_lst=df_all_features.columns.values.tolist(), - user_id=self._db_user_id, - commit=True, - ) - - def get_all_emails_in_database(self, include_deleted=False): - """ - Get all emails stored in the local database - - Args: - include_deleted (bool): Flag to include deleted emails - default False - - Returns: - pandas.DataFrame: With all emails and the corresponding information - """ - return self._db_email.get_all_emails( - include_deleted=include_deleted, user_id=self._db_user_id - ) - - def update_database(self, quick=False, label_lst=None, email_format=None): - """ - Update local email database - - Args: - quick (boolean): Only add new emails, do not update existing labels - by default: False - label_lst (list): list of labels to be searched - email_format (str/None): Email format to download - """ - if label_lst is None: - label_lst = [] - if self._db_email is not None: - message_id_lst = self._search_email_on_server( - label_lst=label_lst, only_message_ids=True - ) - ( - new_messages_lst, - message_label_updates_lst, - deleted_messages_lst, - ) = self._db_email.get_labels_to_update( - message_id_lst=message_id_lst, user_id=self._db_user_id - ) - if not quick: - self._db_email.mark_emails_as_deleted( - message_id_lst=deleted_messages_lst, user_id=self._db_user_id - ) - self._db_email.update_labels( - message_id_lst=message_label_updates_lst, - message_meta_lst=self._get_labels_for_emails( - message_id_lst=message_label_updates_lst - ), - user_id=self._db_user_id, - ) - self._store_emails_in_database( - message_id_lst=new_messages_lst, email_format=email_format - ) - - def _download_messages_to_dataframe(self, message_id_lst, email_format=None): - """ - Download a list of messages based on their email IDs and store the content in a pandas.DataFrame. - - Args: - message_id_lst (list): list of emails IDs - email_format (str): Email format to download - default: "full" - - Returns: - pandas.DataFrame: pandas.DataFrame which contains the rendered emails - """ - return pandas.DataFrame( - [ - message - for message in [ - self._parse_message( - message=self._get_message_detail( - message_id=message_id, - email_format=email_format, - metadata_headers=[], - ) - ) - for message_id in tqdm( - iterable=message_id_lst, desc="Download messages to DataFrame" - ) - ] - if message is not None - ] - ) - - def _get_labels_for_emails(self, message_id_lst): - """ - Get labels for a list of emails - - Args: - message_id_lst (list): list of emails IDs - - Returns: - list: Nested list of email labels for each email - """ - return [ - self._get_labels_for_email(message_id=message_id) - for message_id in tqdm( - iterable=message_id_lst, desc="Get labels for emails" - ) - ] - - def _move_emails(self, move_email_dict, label_to_ignore): - label_existing = self._label_dict[label_to_ignore] - for message_id, label_add in tqdm( - iterable=move_email_dict.items(), desc="Move emails" - ): - if label_add is not None and label_add != label_existing: - self._modify_message_labels( - message_id=message_id, - label_id_remove_lst=[label_existing], - label_id_add_lst=[label_add], - ) - - def _store_emails_in_database(self, message_id_lst, email_format=None): - df = self._download_messages_to_dataframe( - message_id_lst=message_id_lst, email_format=email_format - ) - if len(df) > 0: - self._db_email.store_dataframe(df=df, user_id=self._db_user_id) - - @abstractmethod - def _search_email_on_server( - self, query_string="", label_lst=None, only_message_ids=False - ): - """ - Search emails either by a specific query or optionally limit your search to a list of labels - - Args: - query_string (str): query string to search for - label_lst (list): list of labels to be searched - only_message_ids (bool): return only the email IDs not the thread IDs - default: false - - Returns: - list: list of message ids (or backend-specific list items) matching the search - """ - - @abstractmethod - def _get_message_detail(self, message_id, email_format=None, metadata_headers=None): - """ - Get the raw, backend-specific representation of a single email message. - - Args: - message_id (str): id used by this backend to uniquely identify the email - email_format (str/None): backend-specific format hint - metadata_headers (list): backend-specific list of metadata headers - - Returns: - The backend-specific raw message representation, passed on to `_parse_message`. - """ - - @abstractmethod - def _get_label_translate_dict(self): - """ - Returns: - dict: mapping of label/folder display name to the backend-specific label/folder id - """ - - @abstractmethod - def _modify_message_labels( - self, message_id, label_id_remove_lst=None, label_id_add_lst=None - ): - """ - Apply a label/folder change to a single email message. - """ - - @abstractmethod - def _get_labels_for_email(self, message_id): - """ - Args: - message_id (str): id used by this backend to uniquely identify the email - - Returns: - list: list of labels/folders currently assigned to the email - """ - - @abstractmethod - def _parse_message(self, message): - """ - Args: - message: the backend-specific raw message representation returned by `_get_message_detail` - - Returns: - dict/None: the common gmailsorter email dict (see `gmailsorter.base.message.AbstractMessage.to_dict`), - or None if the message could not be parsed - """ -``` - -- [ ] **Step 2: Rewrite `gmailsorter/google/mail.py`** - -Replace the entire file content with: - -```python -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker - -from gmailsorter.base import get_email_database -from gmailsorter.base.mail import AbstractMailBox -from gmailsorter.google.database import get_token_database -from gmailsorter.google.message import get_email_dict -from gmailsorter.ml import get_machine_learning_database - - -class GoogleMailBase(AbstractMailBox): - def __init__( - self, - google_mail_service, - database_email=None, - database_ml=None, - database_token=None, - user_id="me", - db_user_id=1, - email_download_format="metadata", - ): - """ - Gmail class to manage Emails via the Gmail API directly from Python - - Args: - google_mail_service: A Resource object with methods for interacting with the service. - database_email (gmailsorter.base.database.DatabaseInterface): SQLalchemy interface for email database - database_ml (gmailsorter.ml.database.DatabaseInterface): SQLalchemy interface for machine learning database - database_token (gmailsorter.google.database.DatabaseInterface): SQLalchemy interface for google database - user_id (str): in most cases this should be simply "me" - db_user_id (int): Default 1 - set a user id when sharing a database with multiple users - email_download_format (str): API response format [full, metadata] - """ - self._db_token = database_token - super().__init__( - mail_service=google_mail_service, - database_email=database_email, - database_ml=database_ml, - user_id=user_id, - db_user_id=db_user_id, - email_download_format=email_download_format, - ) - - def _get_label_translate_dict(self): - results = self._service.users().labels().list(userId=self._userid).execute() - labels = results.get("labels", []) - return {label["name"]: label["id"] for label in labels} - - def _get_message_detail(self, message_id, email_format=None, metadata_headers=None): - """ - Get details of a specific email message based on its email ID - - Args: - message_id (str): email IDs used by Google Mail to uniquely identify emails - email_format (str/None): API response format [raw, minimal, full, metadata] - metadata_headers (list): list of meta data headers - - Returns: - dict: details of the email as python dictionary - """ - if email_format is None: - email_format = self._email_download_format - if metadata_headers is None: - metadata_headers = [] - return ( - self._service.users() - .messages() - .get( - userId=self._userid, - id=message_id, - format=email_format, - metadataHeaders=metadata_headers, - ) - .execute() - ) - - def _get_messages_page(self, label_ids, query_string, next_page_token=None): - message_list_response = ( - self._service.users() - .messages() - .list( - userId=self._userid, - labelIds=label_ids, - q=query_string, - pageToken=next_page_token, - ) - .execute() - ) - - return [ - message_list_response.get("messages", []), - message_list_response.get("nextPageToken"), - ] - - def _get_messages(self, query_string="", label_ids=None): - if label_ids is None: - label_ids = [] - message_items_lst, next_page_token = self._get_messages_page( - label_ids=label_ids, query_string=query_string, next_page_token=None - ) - - while next_page_token: - message_items, next_page_token = self._get_messages_page( - label_ids=label_ids, - query_string=query_string, - next_page_token=next_page_token, - ) - message_items_lst.extend(message_items) - - return message_items_lst - - def _modify_message_labels( - self, message_id, label_id_remove_lst=None, label_id_add_lst=None - ): - if label_id_remove_lst is None: - label_id_remove_lst = [] - if label_id_add_lst is None: - label_id_add_lst = [] - body_dict = {} - if len(label_id_remove_lst) > 0: - body_dict["removeLabelIds"] = label_id_remove_lst - if len(label_id_add_lst) > 0: - body_dict["addLabelIds"] = label_id_add_lst - if len(body_dict) > 0: - self._service.users().messages().modify( - userId=self._userid, id=message_id, body=body_dict - ).execute() - - def _search_email_on_server( - self, query_string="", label_lst=None, only_message_ids=False - ): - """ - Search emails either by a specific query or optionally limit your search to a list of labels - - Args: - query_string (str): query string to search for - label_lst (list): list of labels to be searched - only_message_ids (bool): return only the email IDs not the thread IDs - default: false - - Returns: - list: list with email IDs and thread IDs of the messages which match the search - """ - if label_lst is None: - label_lst = [] - label_ids = [self._label_dict[label] for label in label_lst] - message_id_lst = self._get_messages( - query_string=query_string, label_ids=label_ids - ) - if not only_message_ids: - return message_id_lst - else: - return [d["id"] for d in message_id_lst] - - def _get_labels_for_email(self, message_id): - """ - Get labels for email - - Args: - message_id (str): email ID - - Returns: - list: List of email labels - """ - message_dict = self._get_message_detail( - message_id=message_id, - email_format="metadata", - metadata_headers=["labelIds"], - ) - if "labelIds" in message_dict: - return message_dict["labelIds"] - else: - return [] - - def _parse_message(self, message): - return get_email_dict(message=message) - - @staticmethod - def _create_databases(connection_str): - engine = create_engine(connection_str) - session = sessionmaker(bind=engine)() - db_email = get_email_database(engine=engine, session=session) - db_ml = get_machine_learning_database(engine=engine, session=session) - db_token = get_token_database(engine=engine, session=session) - return db_email, db_ml, db_token - - @staticmethod - def _get_message_ids(message_lst): - return [d["id"] for d in message_lst] -``` - -- [ ] **Step 3: Update patch targets in `tests/test_google_integration_units.py`** - -`encode_df_for_machine_learning`, `fit_machine_learning_models`, and `get_predictions_from_machine_learning_models` now execute from `gmailsorter.base.mail`, not `gmailsorter.google.mail`, so the two tests that patch them must point at the new location. In the `test_filter_messages_from_server` method: - -```python - @patch("gmailsorter.base.mail.get_predictions_from_machine_learning_models") - @patch("gmailsorter.base.mail.encode_df_for_machine_learning") - def test_filter_messages_from_server(self, encode_mock, predict_mock): -``` - -(was `@patch("gmailsorter.google.mail.get_predictions_from_machine_learning_models")` / `@patch("gmailsorter.google.mail.encode_df_for_machine_learning")`) - -In the `test_fit_machine_learning_model_to_database` method: - -```python - @patch("gmailsorter.base.mail.fit_machine_learning_models") - @patch("gmailsorter.base.mail.encode_df_for_machine_learning") - def test_fit_machine_learning_model_to_database(self, encode_mock, fit_mock): -``` - -(was `@patch("gmailsorter.google.mail.fit_machine_learning_models")` / `@patch("gmailsorter.google.mail.encode_df_for_machine_learning")`) - -No other lines in this file change — every assertion stays exactly as-is. - -- [ ] **Step 4: Run the full existing suite to confirm no regression** - -Run: `python -m unittest discover tests -v` -Expected: All tests PASS, including every test in `tests/test_google_integration_units.py` with unchanged assertions. - -- [ ] **Step 5: Create `tests/test_mail_base.py` to test the extracted loop in isolation** - -```python -from unittest import TestCase -from unittest.mock import MagicMock, patch - -import pandas as pd - -from gmailsorter.base.mail import AbstractMailBox - - -class _StubMailBox(AbstractMailBox): - """Minimal concrete AbstractMailBox used to test the shared loop in isolation.""" - - def __init__(self, label_dict_fixture=None, **kwargs): - self.label_dict_fixture = label_dict_fixture or {"Inbox": "Inbox", "Spam": "Spam"} - self.search_result = [] - self.message_detail_dict = {} - self.modify_calls = [] - self.labels_for_email_dict = {} - super().__init__(mail_service=MagicMock(), **kwargs) - - def _search_email_on_server(self, query_string="", label_lst=None, only_message_ids=False): - return self.search_result - - def _get_message_detail(self, message_id, email_format=None, metadata_headers=None): - return self.message_detail_dict.get(message_id) - - def _get_label_translate_dict(self): - return self.label_dict_fixture - - def _modify_message_labels(self, message_id, label_id_remove_lst=None, label_id_add_lst=None): - self.modify_calls.append((message_id, label_id_remove_lst, label_id_add_lst)) - - def _get_labels_for_email(self, message_id): - return self.labels_for_email_dict.get(message_id, []) - - def _parse_message(self, message): - return message - - -class AbstractMailBoxTest(TestCase): - def test_labels_property(self): - mailbox = _StubMailBox() - self.assertEqual(sorted(mailbox.labels), ["Inbox", "Spam"]) - - def test_download_emails_for_label(self): - mailbox = _StubMailBox() - mailbox.search_result = ["id1", "id2"] - mailbox.message_detail_dict = { - "id1": { - "id": "id1", - "threads": "t1", - "labels": [], - "to": [], - "from": None, - "cc": [], - "subject": "s1", - "content": "c1", - "date": None, - }, - "id2": None, - } - - df = mailbox.download_emails_for_label(label="Inbox") - - self.assertEqual(df["id"].tolist(), ["id1"]) - - def test_move_emails_skips_matching_or_none_labels(self): - mailbox = _StubMailBox() - - mailbox._move_emails( - move_email_dict={"id1": None, "id2": "Inbox", "id3": "Spam"}, - label_to_ignore="Inbox", - ) - - self.assertEqual(mailbox.modify_calls, [("id3", ["Inbox"], ["Spam"])]) - - def test_update_database_marks_missing_as_deleted(self): - db_email = MagicMock() - db_email.get_labels_to_update.return_value = (["new"], [], ["deleted"]) - mailbox = _StubMailBox(database_email=db_email) - mailbox.search_result = ["new"] - mailbox.message_detail_dict = { - "new": { - "id": "new", - "threads": "t", - "labels": [], - "to": [], - "from": None, - "cc": [], - "subject": "s", - "content": "c", - "date": None, - } - } - - mailbox.update_database(quick=False) - - db_email.mark_emails_as_deleted.assert_called_once_with( - message_id_lst=["deleted"], user_id=1 - ) - db_email.store_dataframe.assert_called_once() - - @patch("gmailsorter.base.mail.fit_machine_learning_models") - @patch("gmailsorter.base.mail.encode_df_for_machine_learning") - def test_fit_machine_learning_model_to_database(self, encode_mock, fit_mock): - db_email = MagicMock() - db_email.get_all_emails.return_value = pd.DataFrame( - [{"id": "x", "from": "a@b.com", "to": [], "cc": [], "labels": [], "threads": "t"}] - ) - db_ml = MagicMock() - mailbox = _StubMailBox(database_email=db_email, database_ml=db_ml) - features = pd.DataFrame([{"email_id": "x", "f1": 1}]) - labels = pd.DataFrame([{"labels_Inbox": 1}]) - encode_mock.return_value = (features, labels) - fit_mock.return_value = {"Inbox": MagicMock()} - - mailbox.fit_machine_learning_model_to_database(n_estimators=5, max_features=2) - - db_ml.store_models.assert_called_once() -``` - -- [ ] **Step 6: Run the new test to verify it passes** - -Run: `python -m unittest tests.test_mail_base -v` -Expected: PASS (5 tests) - -- [ ] **Step 7: Commit** - -```bash -git add gmailsorter/base/mail.py gmailsorter/google/mail.py tests/test_google_integration_units.py tests/test_mail_base.py -git commit -m "refactor: extract AbstractMailBox loop from GoogleMailBase into base/mail.py" -``` - ---- - -### Task 3: `gmailsorter/imap/message.py` - -**Files:** -- Create: `gmailsorter/imap/__init__.py` (empty package marker for now — populated in Task 6) -- Create: `gmailsorter/imap/message.py` -- Test: `tests/test_imap_message.py` - -**Interfaces:** -- Consumes: `gmailsorter.base.message.AbstractMessage`, `gmailsorter.base.message.strip_html_tags` (Task 1). -- Produces: `gmailsorter.imap.message.Message(AbstractMessage)` with constructor `Message(message, folder, uid)`, and `gmailsorter.imap.message.get_email_dict(message, folder, uid) -> dict | None`. Consumed by Task 5 (`ImapMailBase._parse_message`). - -- [ ] **Step 1: Create the package marker** - -Create `gmailsorter/imap/__init__.py` with just: - -```python -``` - -(empty file — populated with real exports in Task 6, once `authentication.py` and `mail.py` exist) - -- [ ] **Step 2: Write the failing test** - -Create `tests/test_imap_message.py`: - -```python -from datetime import datetime -from email.message import EmailMessage -from unittest import TestCase - -from gmailsorter.imap.message import Message, get_email_dict - - -class MessageTest(TestCase): - @classmethod - def setUpClass(cls) -> None: - msg = EmailMessage() - msg["Subject"] = "Test Email Subject" - msg["From"] = "sender@server.net" - msg["To"] = "me@mail.com, friend@provider.org" - msg["Date"] = "Fri, 11 Feb 2022 18:08:46 +0100" - msg["Message-ID"] = "" - msg.set_content("Hello world") - cls._message = msg - cls.message = Message(message=msg, folder="INBOX", uid="42") - - def test_subject(self): - self.assertEqual(self.message.get_subject(), "Test Email Subject") - - def test_from(self): - self.assertEqual(self.message.get_from(), "sender@server.net") - - def test_to(self): - self.assertEqual( - self.message.get_to(), ["me@mail.com", "friend@provider.org"] - ) - - def test_cc_empty(self): - self.assertEqual(self.message.get_cc(), []) - - def test_email_id(self): - self.assertEqual(self.message.get_email_id(), "INBOX\x1f42") - - def test_thread_id_falls_back_to_message_id(self): - self.assertEqual(self.message.get_thread_id(), "") - - def test_label_ids(self): - self.assertEqual(self.message.get_label_ids(), ["INBOX"]) - - def test_get_date(self): - self.assertEqual( - self.message.get_date(), - datetime.strptime( - "Fri, 11 Feb 2022 18:08:46 +0100", "%a, %d %b %Y %H:%M:%S %z" - ), - ) - - def test_get_content(self): - self.assertEqual(self.message.get_content().strip(), "Hello world") - - def test_get_content_html_fallback(self): - html_msg = EmailMessage() - html_msg["Subject"] = "HTML" - html_msg["From"] = "sender@server.net" - html_msg["To"] = "me@mail.com" - html_msg["Date"] = "Fri, 11 Feb 2022 18:08:46 +0100" - html_msg.set_content("

Hello World

", subtype="html") - message = Message(message=html_msg, folder="INBOX", uid="43") - - self.assertEqual(message.get_content().strip(), "Hello World") - - def test_thread_id_uses_references_header(self): - msg = EmailMessage() - msg["Subject"] = "Re: Test" - msg["References"] = " " - msg["Message-ID"] = "" - message = Message(message=msg, folder="INBOX", uid="44") - - self.assertEqual(message.get_thread_id(), "") - - def test_from_with_multiple_addresses_is_none(self): - msg = EmailMessage() - msg["From"] = "a@server.net, b@server.net" - message = Message(message=msg, folder="INBOX", uid="45") - - self.assertIsNone(message.get_from()) - - def test_get_email_dict(self): - result = get_email_dict(self._message, folder="INBOX", uid="42") - content = result.pop("content") - - self.assertEqual(content.strip(), "Hello world") - self.assertEqual( - result, - { - "cc": [], - "date": datetime.strptime( - "Fri, 11 Feb 2022 18:08:46 +0100", "%a, %d %b %Y %H:%M:%S %z" - ), - "from": "sender@server.net", - "id": "INBOX\x1f42", - "labels": ["INBOX"], - "subject": "Test Email Subject", - "threads": "", - "to": ["me@mail.com", "friend@provider.org"], - }, - ) -``` - -- [ ] **Step 3: Run test to verify it fails** - -Run: `python -m unittest tests.test_imap_message -v` -Expected: FAIL with `ModuleNotFoundError: No module named 'gmailsorter.imap.message'` - -- [ ] **Step 4: Implement `gmailsorter/imap/message.py`** - -```python -import email.utils - -from gmailsorter.base.message import AbstractMessage, strip_html_tags - - -def get_email_dict(message, folder, uid): - try: - return Message(message=message, folder=folder, uid=uid).to_dict() - except (ValueError, KeyError) as e: - print(message, str(e)) - return None - - -class Message(AbstractMessage): - def __init__(self, message, folder, uid): - """ - Message class to parse a raw email.message.Message (as produced by - email.message_from_bytes() after an IMAP FETCH) into the common gmailsorter - email representation. - - Args: - message (email.message.Message): parsed RFC822 message - folder (str): IMAP mailbox/folder the message was fetched from - uid (str): IMAP UID of the message within `folder` - """ - self._message = message - self._folder = folder - self._uid = str(uid) - - def get_from(self): - from_header = self._message.get("From") - if from_header is None: - return None - addresses = [ - address - for _, address in email.utils.getaddresses([from_header]) - if address - ] - if len(addresses) == 1: - return addresses[0].lower() - return None - - def get_to(self): - return self._split_addresses(self._message.get_all("To")) - - def get_cc(self): - return self._split_addresses(self._message.get_all("Cc")) - - def get_label_ids(self): - return [self._folder] - - def get_subject(self): - return self._message.get("Subject") - - def get_date(self): - date_header = self._message.get("Date") - if date_header is None: - return None - return email.utils.parsedate_to_datetime(date_header) - - def get_content(self): - text_plain, text_html = None, None - if self._message.is_multipart(): - for part in self._message.walk(): - if part.get_content_maintype() == "multipart": - continue - if part.get_content_type() == "text/plain" and text_plain is None: - text_plain = self._decode_part(part) - elif part.get_content_type() == "text/html" and text_html is None: - text_html = self._decode_part(part) - elif self._message.get_content_type() == "text/plain": - text_plain = self._decode_part(self._message) - elif self._message.get_content_type() == "text/html": - text_html = self._decode_part(self._message) - if text_plain is not None: - return text_plain - elif text_html is not None: - return strip_html_tags(text_html) - else: - return None - - def get_thread_id(self): - references = self._message.get("References") - if references: - return references.split()[0] - in_reply_to = self._message.get("In-Reply-To") - if in_reply_to: - return in_reply_to.strip() - message_id = self._message.get("Message-ID") - if message_id: - return message_id.strip() - return self.get_email_id() - - def get_email_id(self): - return f"{self._folder}\x1f{self._uid}" - - @staticmethod - def _decode_part(part): - payload = part.get_payload(decode=True) - if payload is None: - return "" - charset = part.get_content_charset() or "utf-8" - return payload.decode(charset, errors="replace") - - @staticmethod - def _split_addresses(header_values): - if not header_values: - return [] - return [ - address.lower() - for _, address in email.utils.getaddresses(header_values) - if address - ] -``` - -- [ ] **Step 5: Run test to verify it passes** - -Run: `python -m unittest tests.test_imap_message -v` -Expected: PASS (13 tests) - -- [ ] **Step 6: Commit** - -```bash -git add gmailsorter/imap/__init__.py gmailsorter/imap/message.py tests/test_imap_message.py -git commit -m "feat: add IMAP message parsing (gmailsorter.imap.message)" -``` - ---- - -### Task 4: `gmailsorter/imap/authentication.py` - -**Files:** -- Create: `gmailsorter/imap/authentication.py` -- Test: `tests/test_imap_integration_units.py` (new file — also extended in Tasks 5 and 6) - -**Interfaces:** -- Produces: `gmailsorter.imap.authentication.create_service(host, port, username, password, use_ssl=True) -> imaplib.IMAP4`. Consumed by Task 6 (`Imap.__init__`). - -- [ ] **Step 1: Write the failing test** - -Create `tests/test_imap_integration_units.py`: - -```python -from unittest import TestCase -from unittest.mock import patch - -from gmailsorter.imap.authentication import create_service - - -class TestImapAuthentication(TestCase): - @patch("gmailsorter.imap.authentication.IMAP4_SSL") - def test_create_service_uses_ssl_by_default(self, imap_ssl_cls): - connection = imap_ssl_cls.return_value - - result = create_service( - host="localhost", port=993, username="user", password="secret" - ) - - imap_ssl_cls.assert_called_once_with("localhost", 993) - connection.login.assert_called_once_with("user", "secret") - self.assertIs(result, connection) - - @patch("gmailsorter.imap.authentication.IMAP4") - def test_create_service_without_ssl(self, imap_cls): - connection = imap_cls.return_value - - result = create_service( - host="localhost", - port=143, - username="user", - password="secret", - use_ssl=False, - ) - - imap_cls.assert_called_once_with("localhost", 143) - connection.login.assert_called_once_with("user", "secret") - self.assertIs(result, connection) - - -if __name__ == "__main__": - import unittest - - unittest.main() -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `python -m unittest tests.test_imap_integration_units -v` -Expected: FAIL with `ModuleNotFoundError: No module named 'gmailsorter.imap.authentication'` - -- [ ] **Step 3: Implement `gmailsorter/imap/authentication.py`** - -```python -from imaplib import IMAP4, IMAP4_SSL - - -def create_service(host, port, username, password, use_ssl=True): - """ - Open and log in to an IMAP connection. - - Args: - host (str): IMAP server hostname - port (int): IMAP server port - username (str): IMAP account username - password (str): IMAP account password - use_ssl (bool): connect via IMAP4_SSL (default) or plain IMAP4 - - Returns: - imaplib.IMAP4: logged-in IMAP connection - """ - connection_cls = IMAP4_SSL if use_ssl else IMAP4 - connection = connection_cls(host, port) - connection.login(username, password) - return connection -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `python -m unittest tests.test_imap_integration_units -v` -Expected: PASS (2 tests) - -- [ ] **Step 5: Commit** - -```bash -git add gmailsorter/imap/authentication.py tests/test_imap_integration_units.py -git commit -m "feat: add IMAP username/password authentication (gmailsorter.imap.authentication)" -``` - ---- - -### Task 5: `gmailsorter/imap/mail.py` - -**Files:** -- Create: `gmailsorter/imap/mail.py` -- Modify: `tests/test_imap_integration_units.py` (append) - -**Interfaces:** -- Consumes: `gmailsorter.base.mail.AbstractMailBox` (Task 2), `gmailsorter.imap.message.get_email_dict` (Task 3). -- Produces: `gmailsorter.imap.mail.ImapMailBase(AbstractMailBox)` (no custom `__init__` — inherits `AbstractMailBox.__init__`), plus `ImapMailBase._create_databases(connection_str) -> (database_email, database_ml)`. Consumed by Task 6 (`Imap` class, `gmailsorter/imap/__init__.py`). -- Composite message id format: `f"{folder}\x1f{uid}"` (matches `gmailsorter.imap.message.Message.get_email_id`). - -- [ ] **Step 1: Write the failing tests** - -Append to `tests/test_imap_integration_units.py` (add these imports at the top, alongside the existing ones): - -```python -from unittest.mock import MagicMock -``` - -```python -from gmailsorter.imap.mail import ImapMailBase -``` - -Then add this test class at the end of the file (before the `if __name__ == "__main__":` block): - -```python -class TestImapMailBase(TestCase): - def _create_mock_service_with_folders(self, folders=None): - service = MagicMock() - service.capabilities = ["IMAP4rev1", "MOVE"] - service.list.return_value = ( - "OK", - folders - if folders is not None - else [ - b'(\\HasNoChildren) "/" "INBOX"', - b'(\\HasNoChildren) "/" "MailSortInbox"', - b'(\\Noselect \\HasChildren) "/" "[Gmail]"', - ], - ) - return service - - def test_get_label_translate_dict_skips_noselect(self): - service = self._create_mock_service_with_folders() - mail = ImapMailBase(mail_service=service) - - self.assertEqual(sorted(mail.labels), ["INBOX", "MailSortInbox"]) - - def test_search_email_on_server_single_folder(self): - service = self._create_mock_service_with_folders() - service.select.return_value = ("OK", [b"1"]) - service.uid.return_value = ("OK", [b"1 2"]) - mail = ImapMailBase(mail_service=service) - - ids = mail._search_email_on_server(label_lst=["INBOX"], only_message_ids=True) - - service.select.assert_called_once_with('"INBOX"') - service.uid.assert_called_once_with("search", None, "ALL") - self.assertEqual(ids, ["INBOX\x1f1", "INBOX\x1f2"]) - - def test_search_email_on_server_all_folders_when_no_label(self): - service = self._create_mock_service_with_folders() - service.select.return_value = ("OK", [b"1"]) - service.uid.return_value = ("OK", [b"5"]) - mail = ImapMailBase(mail_service=service) - - ids = mail._search_email_on_server(only_message_ids=True) - - self.assertEqual( - service.select.call_args_list, - [(('"INBOX"',),), (('"MailSortInbox"',),)], - ) - self.assertEqual(ids, ["INBOX\x1f5", "MailSortInbox\x1f5"]) - - def test_search_email_on_server_rejects_query_string(self): - service = self._create_mock_service_with_folders() - mail = ImapMailBase(mail_service=service) - - with self.assertRaises(NotImplementedError): - mail._search_email_on_server(query_string="SUBJECT foo") - - def test_get_message_detail_selects_and_fetches(self): - service = self._create_mock_service_with_folders() - raw_message = b"Subject: hi\r\nFrom: a@b.com\r\nTo: c@d.com\r\n\r\nbody" - service.select.return_value = ("OK", [b"1"]) - service.uid.return_value = ("OK", [(b"1 (RFC822 {10}", raw_message)]) - mail = ImapMailBase(mail_service=service) - - folder, uid, message = mail._get_message_detail(message_id="INBOX\x1f7") - - service.select.assert_called_once_with('"INBOX"') - service.uid.assert_called_once_with("fetch", "7", "(RFC822)") - self.assertEqual(folder, "INBOX") - self.assertEqual(uid, "7") - self.assertEqual(message["Subject"], "hi") - - def test_get_labels_for_email_from_composite_id(self): - service = self._create_mock_service_with_folders() - mail = ImapMailBase(mail_service=service) - - self.assertEqual(mail._get_labels_for_email("INBOX\x1f7"), ["INBOX"]) - - def test_modify_message_labels_uses_move_when_supported(self): - service = self._create_mock_service_with_folders() - service.select.return_value = ("OK", [b"1"]) - service.uid.return_value = ("OK", [b"1"]) - mail = ImapMailBase(mail_service=service) - - mail._modify_message_labels( - message_id="INBOX\x1f7", - label_id_remove_lst=["INBOX"], - label_id_add_lst=["MailSortInbox"], - ) - - service.select.assert_called_once_with('"INBOX"') - service.uid.assert_called_once_with("move", "7", '"MailSortInbox"') - service.expunge.assert_not_called() - - def test_modify_message_labels_falls_back_to_copy_delete(self): - service = self._create_mock_service_with_folders() - service.capabilities = ["IMAP4rev1"] - service.select.return_value = ("OK", [b"1"]) - service.uid.return_value = ("OK", [b"1"]) - mail = ImapMailBase(mail_service=service) - - mail._modify_message_labels( - message_id="INBOX\x1f7", - label_id_remove_lst=["INBOX"], - label_id_add_lst=["MailSortInbox"], - ) - - self.assertEqual( - service.uid.call_args_list, - [ - (("copy", "7", '"MailSortInbox"'),), - (("store", "7", "+FLAGS", r"(\Deleted)"),), - ], - ) - service.expunge.assert_called_once() - - def test_modify_message_labels_noop_without_target(self): - service = self._create_mock_service_with_folders() - mail = ImapMailBase(mail_service=service) - - mail._modify_message_labels(message_id="INBOX\x1f7") - - service.select.assert_not_called() - - @patch("gmailsorter.imap.mail.get_email_dict") - def test_parse_message_delegates_to_get_email_dict(self, get_email_dict_mock): - service = self._create_mock_service_with_folders() - mail = ImapMailBase(mail_service=service) - get_email_dict_mock.return_value = {"id": "INBOX\x1f7"} - - result = mail._parse_message(("INBOX", "7", "raw")) - - get_email_dict_mock.assert_called_once_with( - message="raw", folder="INBOX", uid="7" - ) - self.assertEqual(result, {"id": "INBOX\x1f7"}) - - def test_create_databases(self): - with ( - patch("gmailsorter.imap.mail.create_engine") as create_engine_mock, - patch("gmailsorter.imap.mail.sessionmaker") as sessionmaker_mock, - patch("gmailsorter.imap.mail.get_email_database") as get_email_db_mock, - patch( - "gmailsorter.imap.mail.get_machine_learning_database" - ) as get_ml_db_mock, - ): - engine = MagicMock() - session = MagicMock() - create_engine_mock.return_value = engine - sessionmaker_mock.return_value.return_value = session - get_email_db_mock.return_value = "EMAIL_DB" - get_ml_db_mock.return_value = "ML_DB" - - dbs = ImapMailBase._create_databases("sqlite:///file.db") - - self.assertEqual(dbs, ("EMAIL_DB", "ML_DB")) -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `python -m unittest tests.test_imap_integration_units -v` -Expected: FAIL with `ModuleNotFoundError: No module named 'gmailsorter.imap.mail'` - -- [ ] **Step 3: Implement `gmailsorter/imap/mail.py`** - -```python -import email -import re - -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker - -from gmailsorter.base import get_email_database -from gmailsorter.base.mail import AbstractMailBox -from gmailsorter.imap.message import get_email_dict -from gmailsorter.ml import get_machine_learning_database - -_LIST_ENTRY_PATTERN = re.compile( - r'\((?P[^)]*)\)\s+"(?P.*)"\s+(?P.+)' -) - - -class ImapMailBase(AbstractMailBox): - def _get_label_translate_dict(self): - status, mailbox_lst = self._service.list() - if status != "OK" or mailbox_lst is None: - return {} - label_dict = {} - for entry in mailbox_lst: - flags, _delimiter, name = self._parse_list_entry(entry) - if "\\Noselect" in flags: - continue - label_dict[name] = name - return label_dict - - def _search_email_on_server( - self, query_string="", label_lst=None, only_message_ids=False - ): - """ - Search emails either by a specific query or optionally limit your search to a list of labels - - Args: - query_string (str): not supported yet - must be empty - label_lst (list): list of IMAP folders to search; if empty, every folder is searched - only_message_ids (bool): return only the composite email IDs - default: false - - Returns: - list: list of composite "{folder}\\x1f{uid}" ids matching the search - """ - if query_string: - raise NotImplementedError( - "Custom IMAP search queries are not supported yet, only label_lst filtering." - ) - if label_lst is None: - label_lst = [] - folder_lst = label_lst if len(label_lst) > 0 else list(self._label_dict.keys()) - message_id_lst = [ - f"{folder}\x1f{uid}" - for folder in folder_lst - for uid in self._search_folder(folder=folder) - ] - if only_message_ids: - return message_id_lst - else: - return [{"id": message_id} for message_id in message_id_lst] - - def _search_folder(self, folder): - status, _ = self._service.select(f'"{folder}"') - if status != "OK": - return [] - status, data = self._service.uid("search", None, "ALL") - if status != "OK" or data[0] is None: - return [] - return [ - uid.decode() if isinstance(uid, bytes) else uid for uid in data[0].split() - ] - - def _get_message_detail(self, message_id, email_format=None, metadata_headers=None): - """ - Fetch the raw RFC822 message for a composite "{folder}\\x1f{uid}" id. - - Returns: - tuple: (folder, uid, email.message.Message) - """ - folder, uid = message_id.split("\x1f", 1) - status, _ = self._service.select(f'"{folder}"') - if status != "OK": - raise RuntimeError(f"Could not select IMAP folder {folder!r}") - status, data = self._service.uid("fetch", uid, "(RFC822)") - if status != "OK" or not data or data[0] is None: - raise RuntimeError(f"Could not fetch IMAP message {message_id!r}") - raw_message = data[0][1] - parsed_message = email.message_from_bytes(raw_message) - return folder, uid, parsed_message - - def _modify_message_labels( - self, message_id, label_id_remove_lst=None, label_id_add_lst=None - ): - if not label_id_add_lst: - return - folder, uid = message_id.split("\x1f", 1) - target_folder = label_id_add_lst[0] - status, _ = self._service.select(f'"{folder}"') - if status != "OK": - raise RuntimeError(f"Could not select IMAP folder {folder!r}") - if "MOVE" in self._service.capabilities: - status, _ = self._service.uid("move", uid, f'"{target_folder}"') - if status != "OK": - raise RuntimeError( - f"Could not move IMAP message {message_id!r} to {target_folder!r}" - ) - else: - status, _ = self._service.uid("copy", uid, f'"{target_folder}"') - if status != "OK": - raise RuntimeError( - f"Could not copy IMAP message {message_id!r} to {target_folder!r}" - ) - self._service.uid("store", uid, "+FLAGS", r"(\Deleted)") - self._service.expunge() - - def _get_labels_for_email(self, message_id): - folder, _uid = message_id.split("\x1f", 1) - return [folder] - - def _parse_message(self, message): - folder, uid, parsed_message = message - return get_email_dict(message=parsed_message, folder=folder, uid=uid) - - @staticmethod - def _parse_list_entry(entry): - decoded = entry.decode() if isinstance(entry, bytes) else entry - match = _LIST_ENTRY_PATTERN.match(decoded) - flags = match.group("flags").split() - delimiter = match.group("delimiter") - name = match.group("name").strip('"') - return flags, delimiter, name - - @staticmethod - def _create_databases(connection_str): - engine = create_engine(connection_str) - session = sessionmaker(bind=engine)() - db_email = get_email_database(engine=engine, session=session) - db_ml = get_machine_learning_database(engine=engine, session=session) - return db_email, db_ml -``` - -- [ ] **Step 4: Run tests to verify they pass** - -Run: `python -m unittest tests.test_imap_integration_units -v` -Expected: PASS (all tests in the file, including the `TestImapAuthentication` tests from Task 4) - -- [ ] **Step 5: Commit** - -```bash -git add gmailsorter/imap/mail.py tests/test_imap_integration_units.py -git commit -m "feat: add ImapMailBase (folders-as-labels, MOVE/COPY+EXPUNGE)" -``` - ---- - -### Task 6: `Imap` class in `local.py`, IMAP package exports, top-level export - -**Files:** -- Modify: `gmailsorter/imap/__init__.py` -- Modify: `gmailsorter/local.py` -- Modify: `gmailsorter/__init__.py` -- Modify: `tests/test_imap_integration_units.py` (append) - -**Interfaces:** -- Produces: `gmailsorter.imap.create_service`, `gmailsorter.imap.ImapMailBase` (re-exports), `gmailsorter.local.Imap(host, port, username, password, connection_str, db_user_id=1, use_ssl=True, email_download_format="metadata")`, `gmailsorter.Imap`. Consumed by Task 7 (CLI). - -- [ ] **Step 1: Write the failing test** - -Append to `tests/test_imap_integration_units.py` (add these imports at the top, alongside the existing ones): - -```python -from gmailsorter.local import Imap -``` - -Then add this test class at the end of the file (before the `if __name__ == "__main__":` block): - -```python -class TestImapLocalHelpers(TestCase): - @patch("gmailsorter.local.ImapMailBase.__init__", return_value=None) - @patch("gmailsorter.local.create_imap_service") - @patch("gmailsorter.local.Imap._create_databases") - def test_imap_initialization_wiring( - self, create_databases_mock, create_service_mock, base_init_mock - ): - db_email, db_ml = MagicMock(), MagicMock() - create_databases_mock.return_value = (db_email, db_ml) - connection = MagicMock() - create_service_mock.return_value = connection - - Imap( - host="localhost", - port=993, - username="user", - password="secret", - connection_str="sqlite:///:memory:", - db_user_id=4, - ) - - create_databases_mock.assert_called_once_with( - connection_str="sqlite:///:memory:" - ) - create_service_mock.assert_called_once_with( - host="localhost", - port=993, - username="user", - password="secret", - use_ssl=True, - ) - base_init_mock.assert_called_once_with( - mail_service=connection, - database_email=db_email, - database_ml=db_ml, - user_id="user", - db_user_id=4, - email_download_format="metadata", - ) -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `python -m unittest tests.test_imap_integration_units -v` -Expected: FAIL with `ImportError: cannot import name 'Imap' from 'gmailsorter.local'` - -- [ ] **Step 3: Populate `gmailsorter/imap/__init__.py`** - -```python -from gmailsorter.imap.authentication import create_service -from gmailsorter.imap.mail import ImapMailBase - -__all__ = ["create_service", "ImapMailBase"] -``` - -- [ ] **Step 4: Add `Imap` to `gmailsorter/local.py`** - -Add these imports at the top of `gmailsorter/local.py` (alongside the existing ones): - -```python -from gmailsorter.imap import ImapMailBase -from gmailsorter.imap import create_service as create_imap_service -``` - -Then append the `Imap` class at the end of the file, after `load_client_secrets_file`: - -```python -class Imap(ImapMailBase): - def __init__( - self, - host, - port, - username, - password, - connection_str, - db_user_id=1, - use_ssl=True, - email_download_format="metadata", - ): - """ - Imap class to manage Emails via a plain IMAP connection directly from Python - - Args: - host (str): IMAP server hostname - port (int): IMAP server port, typically 993 for IMAP4_SSL or 143 for IMAP4 - username (str): IMAP account username - password (str): IMAP account password - connection_str (str): SQLalchemy compatible connection string to connect to the SQL database - db_user_id (int): Default 1 - set a user id when sharing a database with multiple users - use_ssl (bool): connect via IMAP4_SSL (default) or plain IMAP4 - email_download_format (str): unused for IMAP, kept for interface parity with Gmail - """ - self._connection_str = connection_str - - database_email, database_ml = self._create_databases( - connection_str=self._connection_str - ) - - imap_connection = create_imap_service( - host=host, - port=port, - username=username, - password=password, - use_ssl=use_ssl, - ) - - super().__init__( - mail_service=imap_connection, - database_email=database_email, - database_ml=database_ml, - user_id=username, - db_user_id=db_user_id, - email_download_format=email_download_format, - ) -``` - -- [ ] **Step 5: Export `Imap` from `gmailsorter/__init__.py`** - -Replace the file content with: - -```python -from gmailsorter.local import Gmail, Imap, load_client_secrets_file - -from . import _version - -__version__: str = _version.__version__ -__all__ = ["Gmail", "Imap", "load_client_secrets_file"] -``` - -- [ ] **Step 6: Run test to verify it passes** - -Run: `python -m unittest tests.test_imap_integration_units -v` -Expected: PASS (all tests) - -- [ ] **Step 7: Run the full suite** - -Run: `python -m unittest discover tests -v` -Expected: All tests PASS - -- [ ] **Step 8: Commit** - -```bash -git add gmailsorter/imap/__init__.py gmailsorter/local.py gmailsorter/__init__.py tests/test_imap_integration_units.py -git commit -m "feat: add Imap convenience class and gmailsorter.Imap export" -``` - ---- - -### Task 7: `gmailsorter-imap` CLI - -**Files:** -- Create: `gmailsorter/imap/__main__.py` -- Modify: `pyproject.toml` -- Test: `tests/test_imap_cli.py` - -**Interfaces:** -- Consumes: `gmailsorter.Imap` (Task 6). -- Produces: `gmailsorter.imap.__main__.command_line_parser()`, console script `gmailsorter-imap`. - -- [ ] **Step 1: Write the failing tests** - -Create `tests/test_imap_cli.py`: - -```python -import os -from unittest import TestCase -from unittest.mock import patch - -from gmailsorter.imap.__main__ import command_line_parser - - -class ImapCliTest(TestCase): - @patch("gmailsorter.imap.__main__.Imap") - def test_update_wires_imap_and_triggers_update(self, imap_cls): - imap_instance = imap_cls.return_value - os.environ["IMAP_PASSWORD"] = "secret" - try: - with patch( - "sys.argv", - [ - "gmailsorter-imap", - "--host", - "localhost", - "--port", - "993", - "--username", - "user", - "-d", - "sqlite:///:memory:", - "-u", - ], - ): - command_line_parser() - finally: - del os.environ["IMAP_PASSWORD"] - - imap_cls.assert_called_once_with( - host="localhost", - port=993, - username="user", - password="secret", - connection_str="sqlite:///:memory:", - db_user_id=1, - use_ssl=True, - email_download_format="metadata", - ) - imap_instance.update_database.assert_called_once_with(quick=False) - imap_instance.fit_machine_learning_model_to_database.assert_called_once_with( - n_estimators=100, - max_features=400, - random_state=42, - bootstrap=True, - include_deleted=False, - ) - - @patch("gmailsorter.imap.__main__.Imap") - def test_label_wires_imap_and_triggers_filter(self, imap_cls): - imap_instance = imap_cls.return_value - os.environ["IMAP_PASSWORD"] = "secret" - try: - with patch( - "sys.argv", - [ - "gmailsorter-imap", - "--host", - "localhost", - "--username", - "user", - "-d", - "sqlite:///:memory:", - "-l", - "MailSortInbox", - ], - ): - command_line_parser() - finally: - del os.environ["IMAP_PASSWORD"] - - imap_instance.filter_messages_from_server.assert_called_once_with( - label="MailSortInbox", recommendation_ratio=0.9 - ) - - @patch("gmailsorter.imap.__main__.Imap") - def test_missing_password_env_skips_wiring(self, imap_cls): - os.environ.pop("IMAP_PASSWORD", None) - with patch( - "sys.argv", - ["gmailsorter-imap", "--host", "localhost", "--username", "user"], - ): - command_line_parser() - - imap_cls.assert_not_called() - - @patch("gmailsorter.imap.__main__.Imap") - def test_missing_host_skips_wiring(self, imap_cls): - with patch("sys.argv", ["gmailsorter-imap", "--username", "user"]): - command_line_parser() - - imap_cls.assert_not_called() - - -if __name__ == "__main__": - import unittest - - unittest.main() -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `python -m unittest tests.test_imap_cli -v` -Expected: FAIL with `ModuleNotFoundError: No module named 'gmailsorter.imap.__main__'` - -- [ ] **Step 3: Implement `gmailsorter/imap/__main__.py`** - -```python -import argparse -import os - -from gmailsorter import Imap - - -def command_line_parser(): - """ - Main function primarily used for the command line interface of the IMAP backend - """ - parser = argparse.ArgumentParser(prog="gmailsorter-imap") - parser.add_argument( - "--host", - help="IMAP server hostname e.g. imap.example.com .", - ) - parser.add_argument( - "--port", - type=int, - default=993, - help="IMAP server port - default: 993 .", - ) - parser.add_argument( - "--username", - help="IMAP account username.", - ) - parser.add_argument( - "--password-env", - default="IMAP_PASSWORD", - help=( - "Name of the environment variable holding the IMAP account password - " - "default: IMAP_PASSWORD ." - ), - ) - parser.add_argument( - "--no-ssl", - action="store_true", - help="Connect without SSL (IMAP4 instead of IMAP4_SSL).", - ) - parser.add_argument( - "-d", - "--database", - help="Connection string to connect to database e.g. sqlite:///email.db .", - ) - parser.add_argument( - "-u", - "--update", - action="store_true", - help="Update local database and retrain machine learning model.", - ) - parser.add_argument( - "-i", - "--identification", - help="User ID of the database user e.g. 1 .", - ) - parser.add_argument( - "-l", - "--label", - help="Email label (IMAP folder) to be filtered with machine learning.", - ) - args = parser.parse_args() - db_user_id = int(args.identification) if args.identification else 1 - password = os.environ.get(args.password_env) - if not args.host or not args.username: - print("Please provide --host and --username.") - elif not password: - print( - f"Please set the {args.password_env} environment variable to your IMAP password." - ) - else: - database = args.database or "sqlite:///email.db" - imap = Imap( - host=args.host, - port=args.port, - username=args.username, - password=password, - connection_str=database, - db_user_id=db_user_id, - use_ssl=not args.no_ssl, - email_download_format="metadata", - ) - if args.update: - imap.update_database(quick=False) - imap.fit_machine_learning_model_to_database( - n_estimators=100, - max_features=400, - random_state=42, - bootstrap=True, - include_deleted=False, - ) - elif args.label: - imap.filter_messages_from_server(label=args.label, recommendation_ratio=0.9) - else: - parser.print_help() - - -if __name__ == "__main__": - command_line_parser() -``` - -- [ ] **Step 4: Register the console script in `pyproject.toml`** - -In the `[project.scripts]` section, add a fourth line: - -```toml -[project.scripts] -gmailsorter = "gmailsorter.__main__:command_line_parser" -gmailsorter-daemon = "gmailsorter.daemon.__main__:command_line_parser" -gmailsorter-app = "gmailsorter.webapp.app:run_app" -gmailsorter-imap = "gmailsorter.imap.__main__:command_line_parser" -``` - -- [ ] **Step 5: Run tests to verify they pass** - -Run: `python -m unittest tests.test_imap_cli -v` -Expected: PASS (4 tests) - -- [ ] **Step 6: Run the full suite** - -Run: `python -m unittest discover tests -v` -Expected: All tests PASS - -- [ ] **Step 7: Commit** - -```bash -git add gmailsorter/imap/__main__.py pyproject.toml tests/test_imap_cli.py -git commit -m "feat: add gmailsorter-imap CLI entry point" -``` - ---- - -### Task 8: GreenMail integration test + CI job - -**Files:** -- Create: `tests/test_imap_service_integration.py` -- Modify: `.github/workflows/unittest.yml` - -**Interfaces:** -- Consumes: `gmailsorter.local.Imap` (Task 6). -- Environment variables (matching the [testing-imap](https://github.com/jan-janssen/testing-imap) convention): `TEST_SMTP_HOST`, `TEST_SMTP_PORT`, `TEST_IMAP_HOST`, `TEST_IMAP_PORT`, `TEST_IMAP_USERNAME`, `TEST_EMAIL`, `TEST_EMAIL_PASSWORD`. - -This test talks to a **real** GreenMail server, so it cannot be driven through a plain RED/GREEN cycle without one running. It's written to skip cleanly (not fail) when no server is reachable, so `python -m unittest discover tests` stays green for contributors without Docker; CI (Step 4 below) is what proves it actually passes. - -- [ ] **Step 1: Create `tests/test_imap_service_integration.py`** - -```python -import os -import smtplib -import time -import unittest -import uuid -from email.message import EmailMessage -from imaplib import IMAP4 - -from gmailsorter.local import Imap - - -class TestImapServiceIntegration(unittest.TestCase): - smtp_host = os.environ.get("TEST_SMTP_HOST", "localhost") - smtp_port = int(os.environ.get("TEST_SMTP_PORT", "3025")) - imap_host = os.environ.get("TEST_IMAP_HOST", "localhost") - imap_port = int(os.environ.get("TEST_IMAP_PORT", "3143")) - username = os.environ.get("TEST_IMAP_USERNAME", "testuser") - recipient = os.environ.get("TEST_EMAIL", "testuser@example.test") - password = os.environ.get("TEST_EMAIL_PASSWORD", "secret") - - @classmethod - def setUpClass(cls): - if not cls._imap_server_available(): - raise unittest.SkipTest( - "No IMAP test server reachable at " - f"{cls.imap_host}:{cls.imap_port} - start the greenmail container " - "described in https://github.com/jan-janssen/testing-imap to run this test." - ) - - @classmethod - def _imap_server_available(cls, timeout=2.0): - try: - with IMAP4(cls.imap_host, cls.imap_port, timeout=timeout) as client: - status, _ = client.noop() - return status == "OK" - except OSError: - return False - - def setUp(self): - with IMAP4(self.imap_host, self.imap_port, timeout=10) as client: - client.login(self.username, self.password) - client.select("INBOX") - status, data = client.search(None, "ALL") - for message_id in data[0].split(): - client.store(message_id, "+FLAGS", r"(\Deleted)") - client.expunge() - for folder in ("MailSortInbox", "Sorted"): - client.create(folder) - - def _send_message(self, subject, body): - message_id = f"<{uuid.uuid4()}@example.test>" - message = EmailMessage() - message["From"] = "sender@example.test" - message["To"] = self.recipient - message["Subject"] = subject - message["Message-ID"] = message_id - message.set_content(body) - with smtplib.SMTP(self.smtp_host, self.smtp_port, timeout=10) as smtp: - smtp.send_message(message) - return message_id - - def _wait_for_message_in_inbox(self, message_id, timeout=10.0): - deadline = time.monotonic() + timeout - with IMAP4(self.imap_host, self.imap_port, timeout=10) as client: - client.login(self.username, self.password) - client.select("INBOX") - while time.monotonic() < deadline: - status, data = client.search( - None, "HEADER", "Message-ID", f'"{message_id}"' - ) - self.assertEqual(status, "OK") - if data[0].split(): - return - time.sleep(0.2) - self.fail(f"Message {message_id!r} was not delivered to INBOX") - - def test_update_database_and_move_round_trip(self): - message_id = self._send_message( - subject="Integration test message", - body="Body from gmailsorter IMAP test.", - ) - self._wait_for_message_in_inbox(message_id) - - imap = Imap( - host=self.imap_host, - port=self.imap_port, - username=self.username, - password=self.password, - connection_str="sqlite:///:memory:", - use_ssl=False, - ) - - imap.update_database(quick=False) - df = imap.get_all_emails_in_database() - - self.assertIn("Integration test message", df["subject"].tolist()) - stored_id = df.loc[ - df["subject"] == "Integration test message", "id" - ].iloc[0] - self.assertTrue(stored_id.startswith("INBOX\x1f")) - - imap._modify_message_labels( - message_id=stored_id, - label_id_remove_lst=["INBOX"], - label_id_add_lst=["MailSortInbox"], - ) - - imap.update_database(quick=False) - df_after_move = imap.get_all_emails_in_database() - moved_row = df_after_move.loc[ - df_after_move["subject"] == "Integration test message" - ] - self.assertEqual(len(moved_row), 1) - self.assertTrue(moved_row.iloc[0]["id"].startswith("MailSortInbox\x1f")) - - -if __name__ == "__main__": - unittest.main() -``` - -- [ ] **Step 2: Run it locally (expected to skip without Docker)** - -Run: `python -m unittest tests.test_imap_service_integration -v` -Expected: `skipped 'No IMAP test server reachable at localhost:3143 - ...'` - -- [ ] **Step 3: (Optional local verification) Run it against a real GreenMail container** - -If Docker is available locally: - -```bash -docker run -d --rm --name greenmail-test \ - -p 3025:3025 -p 3143:3143 \ - -e GREENMAIL_OPTS='-Dgreenmail.setup.test.smtp -Dgreenmail.setup.test.imap -Dgreenmail.hostname=0.0.0.0 -Dgreenmail.users=testuser:secret@example.test' \ - greenmail/standalone:2.1.11 -python -m unittest tests.test_imap_service_integration -v -docker stop greenmail-test -``` - -Expected: PASS (1 test). Skip this step if Docker isn't available — Step 4 (CI) is the authoritative check. - -- [ ] **Step 4: Add a `imap-integration` job to `.github/workflows/unittest.yml`** - -Append a second top-level job under `jobs:` (as a sibling of the existing `build` job), so the full file reads: - -```yaml -# This workflow is used to run the unittest of pyiron - -name: Unittests - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - build: - - runs-on: ${{ matrix.operating-system }} - strategy: - matrix: - operating-system: [ubuntu-latest, windows-latest, macos-latest] - python-version: ['3.14'] - include: - - operating-system: ubuntu-latest - python-version: '3.11' - - operating-system: ubuntu-latest - python-version: '3.12' - - operating-system: ubuntu-latest - python-version: '3.13' - - steps: - - uses: actions/checkout@v4 - - name: Conda config - shell: bash -l {0} - run: echo -e "channels:\n - conda-forge\n" > .condarc - - uses: conda-incubator/setup-miniconda@v3 - with: - python-version: ${{ matrix.python-version }} - miniforge-version: latest - condarc-file: .condarc - environment-file: .ci_support/environment.yml - - name: Test - shell: bash -l {0} - timeout-minutes: 30 - run: | - pip install --no-deps . - coverage run --omit gmailsorter/_version.py -m unittest discover tests - - imap-integration: - runs-on: ubuntu-latest - - services: - greenmail: - image: greenmail/standalone:2.1.11 - env: - GREENMAIL_OPTS: >- - -Dgreenmail.setup.test.smtp - -Dgreenmail.setup.test.imap - -Dgreenmail.hostname=0.0.0.0 - -Dgreenmail.users=testuser:secret@example.test - ports: - - 3025:3025 - - 3143:3143 - - env: - TEST_SMTP_HOST: localhost - TEST_SMTP_PORT: "3025" - TEST_IMAP_HOST: localhost - TEST_IMAP_PORT: "3143" - TEST_IMAP_USERNAME: testuser - TEST_EMAIL: testuser@example.test - TEST_EMAIL_PASSWORD: secret - - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Install package - run: pip install . - - name: Run IMAP integration test - run: python -m unittest tests.test_imap_service_integration -v -``` - -(Note: this is a separate job, not an extra step in `build` — GitHub Actions `services:` containers only run on Linux-hosted runners, and `build` mixes `ubuntu-latest`/`windows-latest`/`macos-latest` in one matrix, so the GreenMail-backed test needs its own Linux-only job.) - -- [ ] **Step 5: Commit** - -```bash -git add tests/test_imap_service_integration.py .github/workflows/unittest.yml -git commit -m "test: add GreenMail-backed IMAP integration test and CI job" -``` - ---- - -### Task 9: Documentation - -**Files:** -- Modify: `docs/source/developer.md` -- Modify: `docs/source/architecture.md` - -**Interfaces:** None (documentation only). - -- [ ] **Step 1: Add an IMAP section to `docs/source/developer.md`** - -After the existing `### Filter emails using machine learning` subsection and before `## Future directions`, insert: - -```markdown -## IMAP accounts -`gmailsorter` also supports plain IMAP accounts (username and password, e.g. an app -password), for mail servers other than Google Mail. Import the `Imap` class instead of -`Gmail`: -``` -from gmailsorter import Imap -``` -``` -imap = Imap( - host="imap.example.com", - port=993, - username="user@example.com", - password="app-password", - connection_str="sqlite:////absolute/path/to/email.db", -) -``` -`Imap` exposes the exact same `update_database()`, `get_all_emails_in_database()` and -`filter_messages_from_server()` methods as `Gmail` - the only difference is that IMAP -folders play the role Gmail labels play elsewhere in this document: each folder is -treated as one label, and moving an email means moving it from one IMAP folder to -another. A command line interface is also available as `gmailsorter-imap`, reading the -account password from an environment variable (`IMAP_PASSWORD` by default) rather than -accepting it as a command line argument: -``` -export IMAP_PASSWORD=app-password -gmailsorter-imap --host imap.example.com --username user@example.com -d sqlite:///email.db -u -``` -``` - -- [ ] **Step 2: Mention IMAP in `docs/source/architecture.md`** - -In the "The big picture" section, change: - -```markdown -* **Your Google Mail account** - the source of truth for your emails and labels, accessed exclusively through the - official [Gmail API](https://developers.google.com/gmail/api/guides). `gmailsorter` never reads your mailbox - through any other channel and never stores your Google password. -``` - -to: - -```markdown -* **Your email account** - the source of truth for your emails and labels, accessed either through the official - [Gmail API](https://developers.google.com/gmail/api/guides) or, for any other IMAP-capable provider, through a - plain IMAP connection. `gmailsorter` never stores your Google password, and for IMAP accounts the password you - provide is used only to log in - it is not persisted anywhere. When talking to a plain IMAP server, each mailbox - folder plays the role a Gmail label plays throughout the rest of this page - "moving" an email between labels - means moving it between IMAP folders. -``` - -- [ ] **Step 3: Commit** - -```bash -git add docs/source/developer.md docs/source/architecture.md -git commit -m "docs: document the Imap class and CLI" -``` - ---- - -## Final verification (after all tasks) - -- [ ] Run the full suite one more time: `coverage run --omit gmailsorter/_version.py -m unittest discover tests -v` — expect all tests PASS (GreenMail test SKIPPED unless Docker is running locally). -- [ ] Run `ruff check gmailsorter/` and `ruff format --check gmailsorter/` (or `pre-commit run --all-files` if available) — expect no lint errors. -- [ ] Push the branch and confirm both the `build` matrix and the new `imap-integration` job go green in GitHub Actions before opening the PR. From 2092854988da5a12b38bda915a09087f42ab2cd0 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Sat, 25 Jul 2026 10:41:11 +0200 Subject: [PATCH 22/36] Delete docs/superpowers/specs/2026-07-25-imap-support-design.md --- .../specs/2026-07-25-imap-support-design.md | 227 ------------------ 1 file changed, 227 deletions(-) delete mode 100644 docs/superpowers/specs/2026-07-25-imap-support-design.md diff --git a/docs/superpowers/specs/2026-07-25-imap-support-design.md b/docs/superpowers/specs/2026-07-25-imap-support-design.md deleted file mode 100644 index 56c209e..0000000 --- a/docs/superpowers/specs/2026-07-25-imap-support-design.md +++ /dev/null @@ -1,227 +0,0 @@ -# IMAP support for gmailsorter - -## Problem - -`gmailsorter` currently only talks to Gmail through the Gmail API. This limits it to -Google Mail accounts and means the machine-learned sorting logic can never be exercised -in CI without live Google credentials. We want to add a second backend that speaks IMAP -(username + password auth), so that: - -* Users with any IMAP-capable mailbox (self-hosted, Dovecot, etc.) can use gmailsorter. -* CI can run a real, end-to-end integration test against a disposable IMAP server - (GreenMail), the way [jan-janssen/testing-imap](https://github.com/jan-janssen/testing-imap) - demonstrates. - -## Scope - -In scope: - -* A new `gmailsorter/imap/` package (`authentication.py`, `message.py`, `mail.py`) - parallel to `gmailsorter/google/`. -* A new `Imap` class in `gmailsorter/local.py`, parallel to `Gmail`. -* A new `gmailsorter-imap` CLI entry point, parallel to `gmailsorter`/`gmailsorter-daemon`. -* Refactoring the fetch-store-train-predict-move loop currently living in - `GoogleMailBase` into a shared abstract base class, so both backends reuse it instead - of duplicating it. -* Unit tests (mocked `imaplib`) and a real integration test against a `greenmail` - container in GitHub Actions. - -Out of scope (explicitly deferred): - -* The Flask webapp / gmailsorter.com login flow — stays Gmail-OAuth-only. -* `gmailsorter-daemon` — stays Gmail-only for now. -* OAuth2/XOAUTH2 for IMAP (e.g. Outlook, Gmail-via-IMAP) — only plain username/password - login (`IMAP4_SSL`/`IMAP4` `LOGIN`) is implemented. The authentication module should - not need reworking to add this later, but implementing it is not part of this change. -* Custom IMAP `SEARCH` queries (the `query_string` parameter that already exists but is - never actually used anywhere in the current codebase) — the IMAP backend only needs to - support `SEARCH ALL` for v1. - -## Architecture - -### Extracting the shared loop - -`GoogleMailBase` (`gmailsorter/google/mail.py`) currently mixes two concerns: the -backend-agnostic fetch→store→train→predict→move loop, and Gmail-API-specific calls -(`service.users().messages()...`). Adding IMAP as a second backend without extracting -the shared part would mean copy-pasting roughly 150 lines of loop/business logic -(`download_emails_for_label`, `filter_messages_from_server`, -`fit_machine_learning_model_to_database`, `get_all_emails_in_database`, -`update_database`, `_download_messages_to_dataframe`, `_store_emails_in_database`, -`_get_labels_for_email(s)`, `_move_emails`) into a new `imap/mail.py`. Instead, this -logic moves into a new class: - -``` -gmailsorter/base/mail.py - class AbstractMailBox(ABC): - # concrete, shared: - labels (property) - download_emails_for_label(label) - filter_messages_from_server(label, recommendation_ratio=0.9) - fit_machine_learning_model_to_database(...) - get_all_emails_in_database(include_deleted=False) - update_database(quick=False, label_lst=None, email_format=None) - _download_messages_to_dataframe(message_id_lst, email_format=None) - _get_labels_for_email(message_id) - _get_labels_for_emails(message_id_lst) - _move_emails(move_email_dict, label_to_ignore) - _store_emails_in_database(message_id_lst, email_format=None) - - # abstract, backend-specific: - _search_email_on_server(query_string="", label_lst=None, only_message_ids=False) - _get_message_detail(message_id, email_format=None, metadata_headers=None) - _get_label_translate_dict() - _modify_message_labels(message_id, label_id_remove_lst=None, label_id_add_lst=None) - _parse_message(message) -> dict # via each backend's AbstractMessage subclass -``` - -This mirrors the existing `base/` vs `google/` split already used for `message.py` -(`AbstractMessage`) and `database.py` (`DatabaseTemplate`/`DatabaseInterface`). - -`GoogleMailBase(AbstractMailBox)` keeps its **exact current public constructor -signature** (`google_mail_service`, `database_email`, `database_ml`, `database_token`, -`user_id`, `db_user_id`, `email_download_format`) so existing callers and tests -(`tests/test_google_integration_units.py`) are unaffected. `database_token` is confirmed -unused outside of `__init__` (grepped the codebase — it's stored on `self` but never -read again), so it stays a `GoogleMailBase`-only attribute rather than being threaded -into the shared base class. - -The small `MLStripper` HTML-to-text helper currently in `gmailsorter/google/message.py` -is generic (not Gmail-specific), so it moves to `gmailsorter/base/message.py` and both -`google/message.py` and the new `imap/message.py` reuse it from there. - -### `gmailsorter/imap/authentication.py` - -```python -def create_service(host, port, username, password, use_ssl=True): - """Open and log in to an IMAP4_SSL/IMAP4 connection. Raises on failure.""" -``` - -No token database, no refresh flow — the password is supplied directly each time a -connection is created (matches how `Gmail`'s `client_config` is supplied directly, just -without the OAuth indirection). If the connection drops, callers reconnect by calling -`create_service` again. - -### `gmailsorter/imap/message.py` - -`Message(AbstractMessage)` parses a raw `email.message.Message` (as returned by -`email.message_from_bytes` after an IMAP `FETCH ... (RFC822)`), plus the folder name it -was fetched from: - -* `get_email_id()` → composite `f"{folder}\x1f{uid}"`. IMAP UIDs are only unique/stable - *within one mailbox* (a `MOVE` to another folder assigns a new UID at the - destination), so the folder is baked into the id used as the primary key in the local - database. -* `get_thread_id()` → first `References` header entry, else `In-Reply-To`, else the - message's own `Message-ID` (so a thread-starting message is its own thread root). -* `get_label_ids()` → `[folder]` — a single-item list, since one IMAP mailbox = one - label. This fits the existing multi-label list contract in `ml/encoding.py` unchanged. -* `get_from`/`get_to`/`get_cc`/`get_subject`/`get_date` → parsed from the standard email - headers (`email.utils.parseaddr`/`getaddresses`, `email.utils.parsedate_to_datetime`). -* `get_content()` → walks MIME parts for `text/plain`, falling back to `text/html` - stripped via the shared `MLStripper`. - -### `gmailsorter/imap/mail.py` - -`ImapMailBase(AbstractMailBox)` implements the abstract hooks: - -* `_get_label_translate_dict()` — `IMAP LIST` all mailboxes, skipping ones flagged - `\Noselect`, returned as `{name: name}` (IMAP has no separate id vs. display name). -* `_search_email_on_server(query_string="", label_lst=None, only_message_ids=False)` — - * If `label_lst` is non-empty: `SELECT` each named folder and `UID SEARCH ALL`. - * If `label_lst` is empty (the case `update_database()` always uses in practice — - verified `__main__.py` and `daemon/daemon.py` both call it with no `label_lst`, - exactly mirroring how Gmail's own `label_ids=[]` means "no filter, whole account"): - iterate over **every** folder from `_get_label_translate_dict()` and aggregate. - * A non-empty `query_string` raises `NotImplementedError` (not silently ignored), - since custom IMAP `SEARCH` syntax isn't implemented in v1 and it's better to fail - loudly than search the wrong thing. -* `_get_message_detail(message_id, ...)` — splits the composite id into - `(folder, uid)`, `SELECT`s the folder, `UID FETCH ... (RFC822)`. -* `_modify_message_labels(message_id, label_id_remove_lst, label_id_add_lst)` — treated - as "move `message_id` from `label_id_remove_lst[0]` to `label_id_add_lst[0]`" (IMAP - only has one folder per message, unlike Gmail's multi-label add/remove). Issues IMAP - `MOVE` if the server advertises the `MOVE` capability, otherwise falls back to `COPY` + - `STORE +FLAGS (\Deleted)` + `EXPUNGE`. -* `_create_databases(connection_str)` — creates only `database_email` and `database_ml` - (no token database, since there's no OAuth token to persist). - -### `gmailsorter/local.py` - -```python -class Imap(ImapMailBase): - def __init__(self, host, port, username, password, connection_str, - db_user_id=1, use_ssl=True, email_download_format="metadata"): - ... -``` - -Parallel to the existing `Gmail` class: builds the two databases, opens the IMAP -connection via `imap.authentication.create_service`, and calls `super().__init__(...)`. - -### CLI: `gmailsorter-imap` - -A new console-script entry point in `pyproject.toml` -(`gmailsorter-imap = "gmailsorter.imap.__main__:command_line_parser"`), parallel to the -existing `gmailsorter`/`gmailsorter-daemon`/`gmailsorter-app` scripts (a new top-level -CLI rather than overloading the existing `gmailsorter` parser with two unrelated -credential schemes). Flags: - -* `--host`, `--port` (default `993`), `--username` -* `--password-env` (name of an environment variable holding the password; default - `IMAP_PASSWORD`) — the password is never accepted as a literal CLI argument, so it - never ends up in shell history or `ps` output. -* `--database`, `--update`, `--label`, `--identification` — same meaning as the - existing `gmailsorter` CLI. - -## Testing - -* `tests/test_imap_message.py` — mirrors `tests/test_google_message.py`: constructs a - raw `email.message.Message`, asserts each `get_*` method and `to_dict()`. -* `tests/test_imap_integration_units.py` — mirrors - `tests/test_google_integration_units.py`: mocks `imaplib.IMAP4_SSL` and asserts - `ImapMailBase`'s hook methods issue the right IMAP commands (`SELECT`, `UID SEARCH`, - `UID FETCH`, `MOVE`/`COPY`+`STORE`+`EXPUNGE`), plus `Imap` wiring in `local.py`. -* `tests/test_mail_base.py` — new tests for the extracted `AbstractMailBox` loop logic - itself (currently only exercised indirectly through `GoogleMailBase` in - `test_google_integration_units.py`), using a minimal concrete stub subclass. -* Existing `tests/test_google_integration_units.py` continues to pass unmodified, - proving the refactor didn't change `GoogleMailBase`'s observable behavior. -* `tests/test_imap_service_integration.py` — a real end-to-end test (not mocked) that: - 1. Connects to a live `greenmail/standalone` container via `smtplib` (send) and - `imaplib` (fetch), following the pattern in - [jan-janssen/testing-imap](https://github.com/jan-janssen/testing-imap)'s - `tests/test_imap_service.py`. - 2. Drives it through `gmailsorter.local.Imap` — updates a SQLite database from the - live GreenMail mailbox, verifies stored content, and exercises a folder move. - 3. Reads connection details from environment variables - (`TEST_IMAP_HOST`/`TEST_IMAP_PORT`/`TEST_IMAP_USERNAME`/`TEST_EMAIL_PASSWORD`/ - `TEST_SMTP_HOST`/`TEST_SMTP_PORT`), matching the testing-imap repo's convention, so - the exact same environment variable names configure both. -* `.github/workflows/unittest.yml` gets a `greenmail` entry under `services:` (image - `greenmail/standalone:2.1.11`, same `GREENMAIL_OPTS`/ports as the testing-imap repo) - and the matching env vars, so `tests/test_imap_service_integration.py` runs on every - push/PR alongside the rest of the unit test suite. Mocked tests keep running on all - three OSes/Python versions in the existing matrix; the GreenMail-backed integration - test only needs to run once (GitHub Actions service containers are Linux-only), so it - runs as an additional step gated to the `ubuntu-latest` job. - -## Documentation - -* `docs/source/developer.md` — add an "IMAP" section parallel to the existing Python - Interface section, showing `Imap(...)` construction and noting it shares the exact - same `update_database`/`get_all_emails_in_database`/`filter_messages_from_server` API - as `Gmail`. -* `docs/source/architecture.md` — update "the source of truth" bullet to mention IMAP as - an alternative to the Gmail API, and note that IMAP folders play the role Gmail labels - play elsewhere in the document. -* `README.md` — mention IMAP support alongside the existing Gmail description, if it - currently states Gmail-only. - -## Non-goals / known limitations carried into v1 - -* No OAuth2/XOAUTH2 (Outlook, Gmail-via-IMAP) — plain `LOGIN` only. -* No custom IMAP `SEARCH` query support. -* A message's database identity changes when it's moved between folders (old id is - marked deleted, a new id is created at the destination) — this is a direct, accepted - consequence of IMAP's per-mailbox UID model, not a bug to fix here. -* Webapp and daemon remain Gmail-only. From 6e22011e3328f82ad406dfa55ffc4458df915f40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sun, 26 Jul 2026 18:03:39 +0200 Subject: [PATCH 23/36] Fix test_google_message import of removed MLStripper MLStripper was refactored into base/message.py as private _MLStripper, exposed via strip_html_tags(). Update the test to match. Co-Authored-By: Claude Sonnet 5 --- tests/test_google_message.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_google_message.py b/tests/test_google_message.py index 934e331..89314b3 100644 --- a/tests/test_google_message.py +++ b/tests/test_google_message.py @@ -2,7 +2,8 @@ from unittest import TestCase from datetime import datetime from datetime import datetime, timezone, timedelta -from gmailsorter.google.message import Message, MLStripper, get_email_dict +from gmailsorter.base.message import strip_html_tags +from gmailsorter.google.message import Message, get_email_dict class MessageTest(TestCase): @@ -198,9 +199,9 @@ def test_get_content_missing_body_data_returns_empty_string(self): self.assertEqual(message.get_content(), "") def test_mlstripper_removes_tags(self): - stripper = MLStripper() - stripper.feed("
Hello World
") - self.assertEqual(stripper.get_data(), "Hello World") + self.assertEqual( + strip_html_tags("
Hello World
"), "Hello World" + ) def test_get_email_dict_catches_value_error_and_returns_none(self): message_dict = { From 7ad0da0ded9b923fc2160a161904abd2aa849afb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sun, 26 Jul 2026 18:17:35 +0200 Subject: [PATCH 24/36] Extend IMAP test coverage for mail.py, message.py, and __main__.py Covers previously-missing branches: get_from/get_date with absent headers, multipart content parsing, thread-id fallback chain, the get_email_dict exception handler, IMAP LIST entry edge cases (str entries, latin-1 fallback, undecodable literal names), label-dict failure on a non-OK LIST, search/fetch/move/copy failure paths that raise RuntimeError or return empty results, and the CLI's print_help branch. Co-Authored-By: Claude Sonnet 5 --- tests/test_imap_cli.py | 24 ++++++ tests/test_imap_integration_units.py | 112 +++++++++++++++++++++++++++ tests/test_imap_message.py | 56 ++++++++++++++ 3 files changed, 192 insertions(+) diff --git a/tests/test_imap_cli.py b/tests/test_imap_cli.py index b57c988..f49715f 100644 --- a/tests/test_imap_cli.py +++ b/tests/test_imap_cli.py @@ -76,6 +76,30 @@ def test_label_wires_imap_and_triggers_filter(self, imap_cls): label="MailSortInbox", recommendation_ratio=0.9 ) + @patch("gmailsorter.imap.__main__.Imap") + def test_no_update_or_label_prints_help(self, imap_cls): + imap_instance = imap_cls.return_value + os.environ["IMAP_PASSWORD"] = "secret" + try: + with patch( + "sys.argv", + [ + "gmailsorter-imap", + "--host", + "localhost", + "--username", + "user", + "-d", + "sqlite:///:memory:", + ], + ): + command_line_parser() + finally: + del os.environ["IMAP_PASSWORD"] + + imap_instance.update_database.assert_not_called() + imap_instance.filter_messages_from_server.assert_not_called() + @patch("gmailsorter.imap.__main__.Imap") def test_missing_password_env_skips_wiring(self, imap_cls): os.environ.pop("IMAP_PASSWORD", None) diff --git a/tests/test_imap_integration_units.py b/tests/test_imap_integration_units.py index 54b08a2..4aaa68b 100644 --- a/tests/test_imap_integration_units.py +++ b/tests/test_imap_integration_units.py @@ -154,6 +154,13 @@ def test_get_label_translate_dict_handles_empty_mailbox_list(self): self.assertEqual(mail.labels, []) + def test_get_label_translate_dict_returns_empty_on_list_failure(self): + service = self._create_mock_service_with_folders() + service.list.return_value = ("NO", None) + mail = ImapMailBase(mail_service=service) + + self.assertEqual(mail.labels, []) + def test_get_label_translate_dict_skips_unparseable_entries(self): service = self._create_mock_service_with_folders( folders=[b"total garbage", b'(\\HasNoChildren) "/" "MailSortInbox"'] @@ -168,6 +175,21 @@ def test_parse_list_entry_returns_none_for_unparseable_input(self): self.assertIsNone(ImapMailBase._parse_list_entry((b'(\\Noselect) "/" {3}',))) self.assertIsNone(ImapMailBase._parse_list_entry(b'(\\HasNoChildren) "/" ')) + def test_parse_list_entry_returns_none_when_literal_name_undecodable(self): + self.assertIsNone( + ImapMailBase._parse_list_entry((b'(\\HasNoChildren) "/" {3}', None)) + ) + + def test_parse_list_entry_accepts_plain_str_entry(self): + result = ImapMailBase._parse_list_entry('(\\HasNoChildren) "/" "INBOX"') + + self.assertEqual(result, (["\\HasNoChildren"], "/", "INBOX")) + + def test_parse_list_entry_falls_back_to_latin1_on_invalid_utf8(self): + result = ImapMailBase._parse_list_entry(b'(\\HasNoChildren) "/" "Caf\xe9"') + + self.assertEqual(result, (["\\HasNoChildren"], "/", "Caf\xe9")) + def test_search_email_on_server_single_folder(self): service = self._create_mock_service_with_folders() service.select.return_value = ("OK", [b"1"]) @@ -201,6 +223,46 @@ def test_search_email_on_server_rejects_query_string(self): with self.assertRaises(NotImplementedError): mail._search_email_on_server(query_string="SUBJECT foo") + def test_search_email_on_server_returns_dicts_by_default(self): + service = self._create_mock_service_with_folders() + service.select.return_value = ("OK", [b"1"]) + service.uid.return_value = ("OK", [b"1 2"]) + mail = ImapMailBase(mail_service=service) + + result = mail._search_email_on_server(label_lst=["INBOX"]) + + self.assertEqual(result, [{"id": "INBOX\x1f1"}, {"id": "INBOX\x1f2"}]) + + def test_search_folder_returns_empty_list_when_select_fails(self): + service = self._create_mock_service_with_folders() + service.select.return_value = ("NO", [b"failed"]) + mail = ImapMailBase(mail_service=service) + + ids = mail._search_email_on_server(label_lst=["INBOX"], only_message_ids=True) + + service.uid.assert_not_called() + self.assertEqual(ids, []) + + def test_search_folder_returns_empty_list_when_search_fails(self): + service = self._create_mock_service_with_folders() + service.select.return_value = ("OK", [b"1"]) + service.uid.return_value = ("NO", [None]) + mail = ImapMailBase(mail_service=service) + + ids = mail._search_email_on_server(label_lst=["INBOX"], only_message_ids=True) + + self.assertEqual(ids, []) + + def test_search_folder_returns_empty_list_when_search_data_is_none(self): + service = self._create_mock_service_with_folders() + service.select.return_value = ("OK", [b"1"]) + service.uid.return_value = ("OK", [None]) + mail = ImapMailBase(mail_service=service) + + ids = mail._search_email_on_server(label_lst=["INBOX"], only_message_ids=True) + + self.assertEqual(ids, []) + def test_get_message_detail_selects_and_fetches(self): service = self._create_mock_service_with_folders() raw_message = b"Subject: hi\r\nFrom: a@b.com\r\nTo: c@d.com\r\n\r\nbody" @@ -216,6 +278,23 @@ def test_get_message_detail_selects_and_fetches(self): self.assertEqual(uid, "7") self.assertEqual(message["Subject"], "hi") + def test_get_message_detail_raises_when_select_fails(self): + service = self._create_mock_service_with_folders() + service.select.return_value = ("NO", [b"failed"]) + mail = ImapMailBase(mail_service=service) + + with self.assertRaises(RuntimeError): + mail._get_message_detail(message_id="INBOX\x1f7") + + def test_get_message_detail_raises_when_fetch_fails(self): + service = self._create_mock_service_with_folders() + service.select.return_value = ("OK", [b"1"]) + service.uid.return_value = ("NO", [None]) + mail = ImapMailBase(mail_service=service) + + with self.assertRaises(RuntimeError): + mail._get_message_detail(message_id="INBOX\x1f7") + def test_get_labels_for_email_from_composite_id(self): service = self._create_mock_service_with_folders() mail = ImapMailBase(mail_service=service) @@ -283,6 +362,39 @@ def test_modify_message_labels_uses_uid_expunge_with_uidplus(self): ) service.expunge.assert_not_called() + def test_modify_message_labels_raises_when_select_fails(self): + service = self._create_mock_service_with_folders() + service.select.return_value = ("NO", [b"failed"]) + mail = ImapMailBase(mail_service=service) + + with self.assertRaises(RuntimeError): + mail._modify_message_labels( + message_id="INBOX\x1f7", label_id_add_lst=["MailSortInbox"] + ) + + def test_modify_message_labels_raises_when_move_fails(self): + service = self._create_mock_service_with_folders() + service.select.return_value = ("OK", [b"1"]) + service.uid.return_value = ("NO", [None]) + mail = ImapMailBase(mail_service=service) + + with self.assertRaises(RuntimeError): + mail._modify_message_labels( + message_id="INBOX\x1f7", label_id_add_lst=["MailSortInbox"] + ) + + def test_modify_message_labels_raises_when_copy_fails(self): + service = self._create_mock_service_with_folders() + service.capabilities = ["IMAP4rev1"] + service.select.return_value = ("OK", [b"1"]) + service.uid.return_value = ("NO", [None]) + mail = ImapMailBase(mail_service=service) + + with self.assertRaises(RuntimeError): + mail._modify_message_labels( + message_id="INBOX\x1f7", label_id_add_lst=["MailSortInbox"] + ) + def test_modify_message_labels_noop_without_target(self): service = self._create_mock_service_with_folders() mail = ImapMailBase(mail_service=service) diff --git a/tests/test_imap_message.py b/tests/test_imap_message.py index 0a9513a..8e08d96 100644 --- a/tests/test_imap_message.py +++ b/tests/test_imap_message.py @@ -1,5 +1,7 @@ from datetime import datetime from email.message import EmailMessage +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText from unittest import TestCase from gmailsorter.imap.message import Message, get_email_dict @@ -77,6 +79,60 @@ def test_from_with_multiple_addresses_is_none(self): self.assertIsNone(message.get_from()) + def test_get_from_missing_header_returns_none(self): + msg = EmailMessage() + message = Message(message=msg, folder="INBOX", uid="46") + + self.assertIsNone(message.get_from()) + + def test_get_date_missing_header_returns_none(self): + msg = EmailMessage() + message = Message(message=msg, folder="INBOX", uid="47") + + self.assertIsNone(message.get_date()) + + def test_get_content_multipart_prefers_plain_over_html(self): + outer = MIMEMultipart("mixed") + inner = MIMEMultipart("alternative") + inner.attach(MIMEText("Hello world", "plain")) + inner.attach(MIMEText("

Hello World

", "html")) + outer.attach(inner) + message = Message(message=outer, folder="INBOX", uid="48") + + self.assertEqual(message.get_content().strip(), "Hello world") + + def test_get_content_returns_none_for_unknown_mimetype(self): + msg = EmailMessage() + msg.set_content(b"\x00\x01", maintype="application", subtype="octet-stream") + message = Message(message=msg, folder="INBOX", uid="49") + + self.assertIsNone(message.get_content()) + + def test_thread_id_uses_in_reply_to_when_no_references(self): + msg = EmailMessage() + msg["In-Reply-To"] = " " + msg["Message-ID"] = "" + message = Message(message=msg, folder="INBOX", uid="50") + + self.assertEqual(message.get_thread_id(), "") + + def test_thread_id_falls_back_to_email_id_without_any_headers(self): + msg = EmailMessage() + message = Message(message=msg, folder="INBOX", uid="51") + + self.assertEqual(message.get_thread_id(), "INBOX\x1f51") + + def test_decode_part_returns_empty_string_when_payload_is_none(self): + multipart_msg = MIMEMultipart("mixed") + + self.assertEqual(Message._decode_part(multipart_msg), "") + + def test_get_email_dict_catches_value_error_and_returns_none(self): + msg = EmailMessage() + msg["Date"] = "Mon, 32 Jan 2024 25:99:99 +0000" + + self.assertIsNone(get_email_dict(msg, folder="INBOX", uid="52")) + def test_get_email_dict(self): result = get_email_dict(self._message, folder="INBOX", uid="42") content = result.pop("content") From 8b301550521eba610f2c1a10806a7f32f25efb50 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 10 Aug 2026 09:58:46 +0000 Subject: [PATCH 25/36] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- gmailsorter/google/mail.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/gmailsorter/google/mail.py b/gmailsorter/google/mail.py index 89e9375..174090a 100644 --- a/gmailsorter/google/mail.py +++ b/gmailsorter/google/mail.py @@ -1,21 +1,17 @@ from typing import Any -import pandas from googleapiclient.discovery import Resource from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from gmailsorter.base import get_email_database -from gmailsorter.base.mail import AbstractMailBox from gmailsorter.base.database import DatabaseInterface as EmailDatabaseInterface +from gmailsorter.base.mail import AbstractMailBox from gmailsorter.google.database import DatabaseInterface as TokenDatabaseInterface from gmailsorter.google.database import get_token_database from gmailsorter.google.message import get_email_dict from gmailsorter.ml import ( - encode_df_for_machine_learning, - fit_machine_learning_models, get_machine_learning_database, - get_predictions_from_machine_learning_models, ) from gmailsorter.ml.database import MachineLearningDatabase From d0f6b67c6298017d59c045d8015f9e32e91e565e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Mon, 10 Aug 2026 12:02:35 +0200 Subject: [PATCH 26/36] fix datetime import --- gmailsorter/google/message.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gmailsorter/google/message.py b/gmailsorter/google/message.py index b8b3e48..b17e498 100644 --- a/gmailsorter/google/message.py +++ b/gmailsorter/google/message.py @@ -1,4 +1,5 @@ import base64 +from datetime import datetime from typing import Any from gmailsorter.base.message import ( From b3426036cfc3cf1283a124bffae4196591c7e615 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Mon, 10 Aug 2026 12:10:02 +0200 Subject: [PATCH 27/36] fix a couple of type hints --- gmailsorter/google/mail.py | 4 ++-- gmailsorter/google/message.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gmailsorter/google/mail.py b/gmailsorter/google/mail.py index 174090a..997f8ca 100644 --- a/gmailsorter/google/mail.py +++ b/gmailsorter/google/mail.py @@ -53,7 +53,7 @@ def __init__( email_download_format=email_download_format, ) - def _get_label_translate_dict(self): + def _get_label_translate_dict(self) -> dict[str, str]: results = self._service.users().labels().list(userId=self._userid).execute() labels = results.get("labels", []) return {label["name"]: label["id"] for label in labels} @@ -181,7 +181,7 @@ def _search_email_on_server( else: return [d["id"] for d in message_id_lst] - def _get_labels_for_email(self, message_id): + def _get_labels_for_email(self, message_id: str) -> list[str]: """ Get labels for email diff --git a/gmailsorter/google/message.py b/gmailsorter/google/message.py index b17e498..a259827 100644 --- a/gmailsorter/google/message.py +++ b/gmailsorter/google/message.py @@ -127,7 +127,7 @@ def _get_email_body(message_parts: dict[str, Any]) -> str: return "" @staticmethod - def _get_email_address(email): + def _get_email_address(email: str) -> str: email_split = email.split("<") if len(email_split) == 1: return email.lower() From 3d59520f4fd09b59909a04255d0c96070beb6fd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Mon, 10 Aug 2026 12:12:57 +0200 Subject: [PATCH 28/36] fixes --- gmailsorter/base/mail.py | 3 +++ tests/test_google_integration_units.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/gmailsorter/base/mail.py b/gmailsorter/base/mail.py index 1477bce..400867d 100644 --- a/gmailsorter/base/mail.py +++ b/gmailsorter/base/mail.py @@ -102,6 +102,7 @@ def fit_machine_learning_model_to_database( random_state=42, bootstrap=True, include_deleted=False, + max_workers=None, ): """ Fit machine learning models to emails stored in database and afterwards store machine learning models in @@ -114,6 +115,7 @@ def fit_machine_learning_model_to_database( bootstrap (boolean): Whether bootstrap samples are used when building trees. If False, the whole dataset is used to build each tree. (default: true) include_deleted (bool): Flag to include deleted emails - default False + max_workers (int): maximum number of workers for the machine learning models """ df_all = self.get_all_emails_in_database(include_deleted=include_deleted) df_all_features, df_all_labels = encode_df_for_machine_learning( @@ -132,6 +134,7 @@ def fit_machine_learning_model_to_database( max_features=max_features, random_state=random_state, bootstrap=bootstrap, + max_workers=max_workers, ) self._db_ml.store_models( model_dict=model_dict, diff --git a/tests/test_google_integration_units.py b/tests/test_google_integration_units.py index 98ec674..1f1f5f0 100644 --- a/tests/test_google_integration_units.py +++ b/tests/test_google_integration_units.py @@ -440,8 +440,8 @@ def test_fit_machine_learning_model_to_database(self, encode_mock, fit_mock): db_ml.store_models.assert_called_once() self.assertEqual(mail.get_all_emails_in_database().iloc[0]["id"], "x") - @patch("gmailsorter.google.mail.fit_machine_learning_models") - @patch("gmailsorter.google.mail.encode_df_for_machine_learning") + @patch("gmailsorter.base.mail.fit_machine_learning_models") + @patch("gmailsorter.base.mail.encode_df_for_machine_learning") def test_fit_machine_learning_model_to_database_forwards_max_workers( self, encode_mock, fit_mock ): From bf6db61ef11b34beed8f3e849f8cd1738786e670 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Mon, 10 Aug 2026 19:52:51 +0200 Subject: [PATCH 29/36] fixes --- gmailsorter/imap/message.py | 27 ++++++++++++++++++++++++++- tests/test_imap_message.py | 22 +++++++++++++++++++++- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/gmailsorter/imap/message.py b/gmailsorter/imap/message.py index 607e4b2..b7887b5 100644 --- a/gmailsorter/imap/message.py +++ b/gmailsorter/imap/message.py @@ -1,3 +1,4 @@ +import email.header import email.utils from gmailsorter.base.message import AbstractMessage, strip_html_tags @@ -48,7 +49,7 @@ def get_label_ids(self): return [self._folder] def get_subject(self): - return self._message.get("Subject") + return self._decode_header(self._message.get("Subject")) def get_date(self): date_header = self._message.get("Date") @@ -92,6 +93,30 @@ def get_thread_id(self): def get_email_id(self): return f"{self._folder}\x1f{self._uid}" + @staticmethod + def _decode_header(header_value): + """ + Decode an email header into a plain str. + + Under the default compat32 policy, email.message.Message.get() normally + returns a str, but a header containing raw non-ASCII bytes that are not + valid RFC 2047 encoded-words (seen from some IMAP servers) is returned as + an email.header.Header instead. That object is not a str subclass, so it + fails SQLAlchemy's parameter binding - str() it first, then run RFC 2047 + decoding to resolve any encoded-words into text. + """ + if header_value is None: + return None + if isinstance(header_value, email.header.Header): + header_value = str(header_value) + decoded_chunks = email.header.decode_header(header_value) + return "".join( + chunk.decode(charset or "utf-8", errors="replace") + if isinstance(chunk, bytes) + else chunk + for chunk, charset in decoded_chunks + ) + @staticmethod def _decode_part(part): payload = part.get_payload(decode=True) diff --git a/tests/test_imap_message.py b/tests/test_imap_message.py index 8e08d96..24ce732 100644 --- a/tests/test_imap_message.py +++ b/tests/test_imap_message.py @@ -1,5 +1,6 @@ from datetime import datetime -from email.message import EmailMessage +from email.header import Header +from email.message import EmailMessage, Message as EmailLibMessage from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from unittest import TestCase @@ -23,6 +24,25 @@ def setUpClass(cls) -> None: def test_subject(self): self.assertEqual(self.message.get_subject(), "Test Email Subject") + def test_subject_encoded_word(self): + msg = EmailMessage() + msg["Subject"] = "Exclusieve Nieuwsbrief • Binobet" + message = Message(message=msg, folder="INBOX", uid="1") + self.assertEqual( + message.get_subject(), "Exclusieve Nieuwsbrief • Binobet" + ) + + def test_subject_header_object_is_coerced_to_str(self): + # Some servers/Python versions cause Message.get() to return an + # email.header.Header instance instead of a str, which used to crash + # SQLAlchemy's parameter binding when inserted into the database. + msg = EmailLibMessage() + msg["Subject"] = Header("Exclusieve Nieuwsbrief • Binobet", "utf-8") + message = Message(message=msg, folder="INBOX", uid="1") + subject = message.get_subject() + self.assertIsInstance(subject, str) + self.assertEqual(subject, "Exclusieve Nieuwsbrief • Binobet") + def test_from(self): self.assertEqual(self.message.get_from(), "sender@server.net") From a1cb148b1c488644321f6a31b9c321526cde9c8c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 10 Aug 2026 17:53:06 +0000 Subject: [PATCH 30/36] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_imap_message.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_imap_message.py b/tests/test_imap_message.py index 24ce732..ccac012 100644 --- a/tests/test_imap_message.py +++ b/tests/test_imap_message.py @@ -28,9 +28,7 @@ def test_subject_encoded_word(self): msg = EmailMessage() msg["Subject"] = "Exclusieve Nieuwsbrief • Binobet" message = Message(message=msg, folder="INBOX", uid="1") - self.assertEqual( - message.get_subject(), "Exclusieve Nieuwsbrief • Binobet" - ) + self.assertEqual(message.get_subject(), "Exclusieve Nieuwsbrief • Binobet") def test_subject_header_object_is_coerced_to_str(self): # Some servers/Python versions cause Message.get() to return an From ab44dba0e16e33666d8add84fe95bf3b8b0cce0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Mon, 10 Aug 2026 20:08:12 +0200 Subject: [PATCH 31/36] black fixes --- gmailsorter/imap/message.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/gmailsorter/imap/message.py b/gmailsorter/imap/message.py index b7887b5..71ecc2b 100644 --- a/gmailsorter/imap/message.py +++ b/gmailsorter/imap/message.py @@ -111,9 +111,11 @@ def _decode_header(header_value): header_value = str(header_value) decoded_chunks = email.header.decode_header(header_value) return "".join( - chunk.decode(charset or "utf-8", errors="replace") - if isinstance(chunk, bytes) - else chunk + ( + chunk.decode(charset or "utf-8", errors="replace") + if isinstance(chunk, bytes) + else chunk + ) for chunk, charset in decoded_chunks ) From 5c0871d5151eb268f53d5db01c39a7361f24cb9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sun, 23 Aug 2026 21:18:17 +0200 Subject: [PATCH 32/36] Merge changes from main --- gmailsorter/__main__.py | 2 +- gmailsorter/base/mail.py | 2 ++ gmailsorter/daemon/daemon.py | 1 + gmailsorter/imap/__main__.py | 2 +- notebooks/demo.ipynb | 4 ++-- 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/gmailsorter/__main__.py b/gmailsorter/__main__.py index 36615a8..d8ea634 100644 --- a/gmailsorter/__main__.py +++ b/gmailsorter/__main__.py @@ -79,7 +79,7 @@ def command_line_parser() -> None: ) elif args.label: gmail.filter_messages_from_server( - label=args.label, recommendation_ratio=0.9 + label=args.label, recommendation_ratio=0.9, label_prefix="labels_Label_", ) else: parser.print_help() diff --git a/gmailsorter/base/mail.py b/gmailsorter/base/mail.py index 400867d..32de96a 100644 --- a/gmailsorter/base/mail.py +++ b/gmailsorter/base/mail.py @@ -66,6 +66,7 @@ def filter_messages_from_server( self, label, recommendation_ratio=0.9, + label_prefix: str="labels_", ): """ Filter new emails based on machine learning model recommendations. @@ -82,6 +83,7 @@ def filter_messages_from_server( feature_lst=feature_reload_lst, label_lst=list(model_reload_dict.keys()), return_labels=False, + label_prefix=label_prefix, ) df_partial_features = df_partial_features.reindex( sorted(df_partial_features.columns), axis=1 diff --git a/gmailsorter/daemon/daemon.py b/gmailsorter/daemon/daemon.py index 006e11b..ad67d09 100644 --- a/gmailsorter/daemon/daemon.py +++ b/gmailsorter/daemon/daemon.py @@ -134,6 +134,7 @@ def iterate_over_users( gmail.filter_messages_from_server( label=MAILSORT_LABEL, recommendation_ratio=recommendation_ratio, + label_prefix="labels_Label_", ) except HttpError: update_task_status( diff --git a/gmailsorter/imap/__main__.py b/gmailsorter/imap/__main__.py index 28bae79..d3f7350 100644 --- a/gmailsorter/imap/__main__.py +++ b/gmailsorter/imap/__main__.py @@ -88,7 +88,7 @@ def command_line_parser(): include_deleted=False, ) elif args.label: - imap.filter_messages_from_server(label=args.label, recommendation_ratio=0.9) + imap.filter_messages_from_server(label=args.label, recommendation_ratio=0.9, label_prefix="labels_") else: parser.print_help() diff --git a/notebooks/demo.ipynb b/notebooks/demo.ipynb index 29d0a84..2fe4d0f 100644 --- a/notebooks/demo.ipynb +++ b/notebooks/demo.ipynb @@ -127,7 +127,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "907c830a", "metadata": {}, "outputs": [ @@ -141,7 +141,7 @@ } ], "source": [ - "gmail.filter_messages_from_server(label=\"9-emails-to-sort\", recommendation_ratio=0.9)" + "gmail.filter_messages_from_server(label=\"9-emails-to-sort\", recommendation_ratio=0.9, label_prefix=\"labels_Label_\")" ] } ], From 9f14b4e6f548ef14fda2f6f30f6617ee42413d27 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 23 Aug 2026 19:18:30 +0000 Subject: [PATCH 33/36] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- gmailsorter/__main__.py | 4 +++- gmailsorter/base/mail.py | 2 +- gmailsorter/imap/__main__.py | 4 +++- notebooks/demo.ipynb | 4 +++- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/gmailsorter/__main__.py b/gmailsorter/__main__.py index d8ea634..f68dfe6 100644 --- a/gmailsorter/__main__.py +++ b/gmailsorter/__main__.py @@ -79,7 +79,9 @@ def command_line_parser() -> None: ) elif args.label: gmail.filter_messages_from_server( - label=args.label, recommendation_ratio=0.9, label_prefix="labels_Label_", + label=args.label, + recommendation_ratio=0.9, + label_prefix="labels_Label_", ) else: parser.print_help() diff --git a/gmailsorter/base/mail.py b/gmailsorter/base/mail.py index 32de96a..9b18f20 100644 --- a/gmailsorter/base/mail.py +++ b/gmailsorter/base/mail.py @@ -66,7 +66,7 @@ def filter_messages_from_server( self, label, recommendation_ratio=0.9, - label_prefix: str="labels_", + label_prefix: str = "labels_", ): """ Filter new emails based on machine learning model recommendations. diff --git a/gmailsorter/imap/__main__.py b/gmailsorter/imap/__main__.py index d3f7350..96bbac1 100644 --- a/gmailsorter/imap/__main__.py +++ b/gmailsorter/imap/__main__.py @@ -88,7 +88,9 @@ def command_line_parser(): include_deleted=False, ) elif args.label: - imap.filter_messages_from_server(label=args.label, recommendation_ratio=0.9, label_prefix="labels_") + imap.filter_messages_from_server( + label=args.label, recommendation_ratio=0.9, label_prefix="labels_" + ) else: parser.print_help() diff --git a/notebooks/demo.ipynb b/notebooks/demo.ipynb index 2fe4d0f..0e5767f 100644 --- a/notebooks/demo.ipynb +++ b/notebooks/demo.ipynb @@ -141,7 +141,9 @@ } ], "source": [ - "gmail.filter_messages_from_server(label=\"9-emails-to-sort\", recommendation_ratio=0.9, label_prefix=\"labels_Label_\")" + "gmail.filter_messages_from_server(\n", + " label=\"9-emails-to-sort\", recommendation_ratio=0.9, label_prefix=\"labels_Label_\"\n", + ")" ] } ], From dd41079e1d9abeaa585e464c6b53c8fc9aa1aa94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sun, 23 Aug 2026 21:25:32 +0200 Subject: [PATCH 34/36] fix tests --- tests/test_imap_cli.py | 2 +- tests/test_main.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_imap_cli.py b/tests/test_imap_cli.py index f49715f..84896f7 100644 --- a/tests/test_imap_cli.py +++ b/tests/test_imap_cli.py @@ -73,7 +73,7 @@ def test_label_wires_imap_and_triggers_filter(self, imap_cls): del os.environ["IMAP_PASSWORD"] imap_instance.filter_messages_from_server.assert_called_once_with( - label="MailSortInbox", recommendation_ratio=0.9 + label="MailSortInbox", recommendation_ratio=0.9, label_prefix='labels_', ) @patch("gmailsorter.imap.__main__.Imap") diff --git a/tests/test_main.py b/tests/test_main.py index 2e44c04..fca2340 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -105,7 +105,7 @@ def test_label_flag_triggers_filter(self, load_secrets_mock, gmail_cls): email_download_format="metadata", ) gmail_instance.filter_messages_from_server.assert_called_once_with( - label="Inbox", recommendation_ratio=0.9 + label="Inbox", recommendation_ratio=0.9, label_prefix='labels_', ) gmail_instance.update_database.assert_not_called() From 415efb8f5a212aaef77f5848a99dffd09369b881 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 23 Aug 2026 19:25:48 +0000 Subject: [PATCH 35/36] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_imap_cli.py | 4 +++- tests/test_main.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_imap_cli.py b/tests/test_imap_cli.py index 84896f7..04fcf69 100644 --- a/tests/test_imap_cli.py +++ b/tests/test_imap_cli.py @@ -73,7 +73,9 @@ def test_label_wires_imap_and_triggers_filter(self, imap_cls): del os.environ["IMAP_PASSWORD"] imap_instance.filter_messages_from_server.assert_called_once_with( - label="MailSortInbox", recommendation_ratio=0.9, label_prefix='labels_', + label="MailSortInbox", + recommendation_ratio=0.9, + label_prefix="labels_", ) @patch("gmailsorter.imap.__main__.Imap") diff --git a/tests/test_main.py b/tests/test_main.py index fca2340..d40863f 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -105,7 +105,9 @@ def test_label_flag_triggers_filter(self, load_secrets_mock, gmail_cls): email_download_format="metadata", ) gmail_instance.filter_messages_from_server.assert_called_once_with( - label="Inbox", recommendation_ratio=0.9, label_prefix='labels_', + label="Inbox", + recommendation_ratio=0.9, + label_prefix="labels_", ) gmail_instance.update_database.assert_not_called() From 5fee1798724ac26e0b547764390a9b4f0b40f611 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sun, 23 Aug 2026 21:30:00 +0200 Subject: [PATCH 36/36] fixes --- tests/test_main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_main.py b/tests/test_main.py index d40863f..5c719dd 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -107,7 +107,7 @@ def test_label_flag_triggers_filter(self, load_secrets_mock, gmail_cls): gmail_instance.filter_messages_from_server.assert_called_once_with( label="Inbox", recommendation_ratio=0.9, - label_prefix="labels_", + label_prefix="labels_Label_", ) gmail_instance.update_database.assert_not_called()