diff --git a/CHANGELOG.md b/CHANGELOG.md index bd581db..892742c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## [0.8.0] - 2026-07-15 +### Added +- Added import support for encrypted Threema Android data-backup ZIP files, including conversations, participants, messages, reactions, polls, and optional media extraction. +- Added the Threema Android backup source and password field to the Windows GUI and CLI. + ## [0.7.0] - 2026-04-22 ### Added - Added `export_summary.txt`, `manifest.json`, and `log.txt` for every export run. diff --git a/Examples/README.md b/Examples/README.md index c31fc4e..de45bce 100644 --- a/Examples/README.md +++ b/Examples/README.md @@ -4,6 +4,7 @@ This folder contains synthetic example data for local testing and demo purposes. Included in the repository: - `ThreemaSynthetic/`: synthetic Threema SQLite example with one direct chat and one group chat +- `ThreemaAndroidBackupSynthetic/`: synthetic encrypted Threema Android data backup with direct, group, and distribution-list messages - `WhatsAppSynthetic/`: synthetic WhatsApp ZIP examples for direct and group chats - `run_examples.ipynb`: notebook to run the example exports diff --git a/Examples/ThreemaAndroidBackupSynthetic/README.txt b/Examples/ThreemaAndroidBackupSynthetic/README.txt new file mode 100644 index 0000000..2251e87 --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/README.txt @@ -0,0 +1,47 @@ +Synthetic Threema Android Data Backup +===================================== + +This fixture contains only invented data and media generated by build_fixture.py. +It follows the current Threema Android CSV backup format version 27. + +Encrypted backup: + threema-backup_synthetic.zip + threema-backup_synthetic_no_media.zip + +Password: + testpassword + +Expected own Threema ID: + 0ZAHXXHB + +Coverage: + - direct chats, including an empty chat + - one group chat + - one distribution list + - incoming and outgoing messages + - text, quote, edit, deletion, location, poll, and call/status messages + - contact and group reactions + - image, audio, video, PDF, and text attachments + - thumbnails, contact avatars, profile pictures, and a group avatar + - one deliberately unavailable attachment payload + - a second valid backup variant created without message media or thumbnails + - CSV quoting, commas, backslashes, line breaks, Unicode, and emoji + +The source/ directory contains the plaintext archive members for transparent +inspection. expected.json contains the counts and identifiers expected from an +importer. The identity entry uses the public legacy test vector from the +official Threema Android source and does not contain a real identity or key. + +Regeneration requirements: + - Python project dependencies + - 7-Zip on PATH + - FFmpeg on PATH + +Regenerate with: + python .\Examples\ThreemaAndroidBackupSynthetic\build_fixture.py + +Run the importer regression tests with: + python -m unittest discover -v + +Run a manual export with: + python -m chat_export --source threema_android_backup --input-path .\Examples\ThreemaAndroidBackupSynthetic\threema-backup_synthetic.zip --backup-password testpassword --out-dir .\Examples\Output\threema_android_backup diff --git a/Examples/ThreemaAndroidBackupSynthetic/build_fixture.py b/Examples/ThreemaAndroidBackupSynthetic/build_fixture.py new file mode 100644 index 0000000..93025a1 --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/build_fixture.py @@ -0,0 +1,803 @@ +"""Build a fully synthetic Threema Android data-backup fixture.""" + +from __future__ import annotations + +import csv +import json +import math +import shutil +import struct +import subprocess +import wave +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Iterable + +from PIL import Image, ImageDraw +from reportlab.pdfgen import canvas + + +FIXTURE_DIR = Path(__file__).resolve().parent +SOURCE_DIR = FIXTURE_DIR / "source" +ARCHIVE_PATH = FIXTURE_DIR / "threema-backup_synthetic.zip" +NO_MEDIA_ARCHIVE_PATH = FIXTURE_DIR / "threema-backup_synthetic_no_media.zip" +EXPECTED_PATH = FIXTURE_DIR / "expected.json" +PASSWORD = "testpassword" +SELF_IDENTITY = "0ZAHXXHB" + +# Public test vector from the official Threema identity-backup tests. +IDENTITY_BACKUP = ( + "4K4M-5Q6T-KFUH-KHL5-2VCJ-ZM57-NL7R-WJTA-V45L-NJAM-" + "WLEU-5DS4-XF7S-OPH4-CTCL-N2CF-3C4C-HPB7-YZWW-U3S6" +) + +CONTACT_HEADER = [ + "identity", + "publickey", + "verification", + "acid", + "firstname", + "lastname", + "nick_name", + "last_update", + "hidden", + "archived", + "identity_id", +] +MESSAGE_HEADER = [ + "apiid", + "uid", + "isoutbox", + "isread", + "issaved", + "messagestae", + "posted_at", + "created_at", + "modified_at", + "type", + "body", + "isstatusmessage", + "caption", + "quoted_message_apiid", + "delivered_at", + "read_at", + "g_msg_states", + "display_tags", + "edited_at", + "deleted_at", +] +GROUP_MESSAGE_HEADER = MESSAGE_HEADER[:2] + ["identity"] + MESSAGE_HEADER[2:] +DISTRIBUTION_MESSAGE_HEADER = [ + "apiid", + "uid", + "identity", + "isoutbox", + "isread", + "issaved", + "messagestae", + "posted_at", + "created_at", + "modified_at", + "type", + "body", + "isstatusmessage", + "caption", + "quoted_message_apiid", + "delivered_at", + "read_at", +] + +BASE_TIME = datetime(2026, 1, 15, 9, 0, tzinfo=timezone.utc) + + +def epoch_ms(offset_minutes: int) -> str: + """Return a deterministic Unix timestamp in milliseconds.""" + value = BASE_TIME + timedelta(minutes=offset_minutes) + return str(int(value.timestamp() * 1000)) + + +def uid(number: int) -> str: + """Return a deterministic UUID-shaped message identifier.""" + return f"00000000-0000-4000-8000-{number:012d}" + + +def api_id(number: int) -> str: + """Return a deterministic 16-character API message identifier.""" + return f"{number:016x}" + + +def csv_value(value: Any) -> str: + """Serialize values like Threema's CSVRow helper.""" + if value is None: + return "" + if isinstance(value, bool): + return "1" if value else "0" + if isinstance(value, (list, tuple)): + value = ";".join(str(item) for item in value) + return str(value).replace("\\", "\\\\") + + +def write_csv(path: Path, header: list[str], rows: Iterable[dict[str, Any]]) -> None: + """Write an OpenCSV-compatible, fully quoted UTF-8 file.""" + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.writer(handle, quoting=csv.QUOTE_ALL, lineterminator="\n") + writer.writerow(header) + for row in rows: + writer.writerow([csv_value(row.get(column)) for column in header]) + + +def message( + number: int, + *, + offset: int, + message_type: str, + body: str, + outbox: bool, + identity: str | None = None, + state: str | None = None, + caption: str | None = None, + quote: str | None = None, + edited_at: str | None = None, + deleted_at: str | None = None, + group_states: str | None = None, +) -> dict[str, Any]: + """Build one version-27 message row.""" + timestamp = epoch_ms(offset) + row: dict[str, Any] = { + "apiid": api_id(number), + "uid": uid(number), + "isoutbox": outbox, + "isread": True, + "issaved": True, + "messagestae": state or ("READ" if outbox else ""), + "posted_at": timestamp, + "created_at": timestamp, + "modified_at": edited_at or timestamp, + "type": message_type, + "body": body, + "isstatusmessage": False, + "caption": caption, + "quoted_message_apiid": quote, + "delivered_at": epoch_ms(offset + 1) if outbox else None, + "read_at": epoch_ms(offset + 2), + "g_msg_states": group_states, + "display_tags": 0, + "edited_at": edited_at, + "deleted_at": deleted_at, + } + if identity is not None: + row["identity"] = identity + return row + + +def file_body( + *, + mime_type: str, + size: int, + filename: str, + rendering_type: int, + downloaded: bool, + caption: str | None = None, + thumbnail_mime_type: str | None = None, + metadata: dict[str, Any] | None = None, +) -> str: + """Serialize a current FileDataModel JSON body.""" + return json.dumps( + [ + "00112233445566778899aabbccddeeff", + "00" * 32, + mime_type, + size, + filename, + rendering_type, + downloaded, + caption, + thumbnail_mime_type, + metadata or {}, + ], + ensure_ascii=False, + separators=(",", ":"), + ) + + +def save_pattern_image(path: Path, *, size: tuple[int, int], colors: tuple[str, str], image_format: str) -> None: + """Create a deterministic synthetic image.""" + image = Image.new("RGB", size, colors[0]) + draw = ImageDraw.Draw(image) + width, height = size + draw.rectangle((width // 2, 0, width, height), fill=colors[1]) + draw.line((0, 0, width, height), fill="white", width=max(1, width // 16)) + draw.line((0, height, width, 0), fill="white", width=max(1, width // 16)) + image.save(path, format=image_format) + + +def save_wave(path: Path) -> None: + """Create a short synthetic mono WAV attachment.""" + sample_rate = 8000 + duration_seconds = 0.4 + frames = int(sample_rate * duration_seconds) + with wave.open(str(path), "wb") as output: + output.setnchannels(1) + output.setsampwidth(2) + output.setframerate(sample_rate) + for index in range(frames): + sample = int(9000 * math.sin(2 * math.pi * 440 * index / sample_rate)) + output.writeframesraw(struct.pack(" None: + """Create a one-page synthetic PDF attachment.""" + document = canvas.Canvas(str(path), pagesize=(240, 160)) + document.setTitle("Synthetic attachment") + document.drawString(24, 112, "Synthetic Threema attachment") + document.drawString(24, 88, "No real case or personal data") + document.save() + + +def save_video(path: Path) -> None: + """Create a one-second synthetic MP4 attachment with FFmpeg.""" + ffmpeg = shutil.which("ffmpeg") + if not ffmpeg: + raise RuntimeError("FFmpeg is required to regenerate the synthetic MP4") + subprocess.run( + [ + ffmpeg, + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "color=c=0x2f7d8c:s=96x64:d=1", + "-c:v", + "mpeg4", + "-pix_fmt", + "yuv420p", + "-f", + "mp4", + "-y", + str(path), + ], + check=True, + ) + + +def create_media() -> dict[str, Path]: + """Create attachment payloads, thumbnails, and avatars.""" + media = { + "direct_image": SOURCE_DIR / f"message_media_{uid(5)}", + "direct_audio": SOURCE_DIR / f"message_media_{uid(6)}", + "direct_pdf": SOURCE_DIR / f"message_media_{uid(7)}", + "group_image": SOURCE_DIR / f"group_message_media_{uid(106)}", + "group_video": SOURCE_DIR / f"group_message_media_{uid(107)}", + "distribution_text": SOURCE_DIR / f"distribution_list_message_media_{uid(203)}", + } + save_pattern_image( + media["direct_image"], + size=(96, 64), + colors=("#22577a", "#80ed99"), + image_format="PNG", + ) + save_wave(media["direct_audio"]) + save_pdf(media["direct_pdf"]) + save_pattern_image( + media["group_image"], + size=(80, 80), + colors=("#9b2226", "#e9d8a6"), + image_format="JPEG", + ) + save_video(media["group_video"]) + media["distribution_text"].write_text( + "Synthetic distribution-list attachment.\n", + encoding="utf-8", + ) + + save_pattern_image( + SOURCE_DIR / f"message_thumbnail_{uid(5)}", + size=(32, 24), + colors=("#22577a", "#80ed99"), + image_format="JPEG", + ) + save_pattern_image( + SOURCE_DIR / f"group_message_thumbnail_{uid(106)}", + size=(32, 32), + colors=("#9b2226", "#e9d8a6"), + image_format="JPEG", + ) + save_pattern_image( + SOURCE_DIR / f"group_message_thumbnail_{uid(107)}", + size=(32, 24), + colors=("#2f7d8c", "#f4a261"), + image_format="JPEG", + ) + + avatar_specs = { + "contact_avatar_me": ("#264653", "#2a9d8f"), + "contact_avatar_0000000001": ("#6d597a", "#e56b6f"), + "contact_profile_pic_0000000001": ("#355070", "#eaac8b"), + "group_avatar_1000000001": ("#005f73", "#ee9b00"), + } + for name, colors in avatar_specs.items(): + save_pattern_image( + SOURCE_DIR / name, + size=(48, 48), + colors=colors, + image_format="PNG", + ) + return media + + +def create_structured_files(media: dict[str, Path]) -> dict[str, Any]: + """Create all version-27 CSV and metadata archive members.""" + write_csv(SOURCE_DIR / "settings", ["version", "27"], []) + (SOURCE_DIR / "identity").write_text(IDENTITY_BACKUP, encoding="ascii") + + contacts = [ + { + "identity": "ALICE001", + "publickey": "11" * 32, + "verification": "FULLY_VERIFIED", + "firstname": "Alice", + "lastname": "Example", + "nick_name": "Ali", + "last_update": epoch_ms(0), + "hidden": False, + "archived": False, + "identity_id": "0000000001", + }, + { + "identity": "BOB00002", + "publickey": "22" * 32, + "verification": "SERVER_VERIFIED", + "firstname": "Bob", + "lastname": "Sample", + "hidden": False, + "archived": False, + "identity_id": "0000000002", + }, + { + "identity": "CAROL003", + "publickey": "33" * 32, + "verification": "UNVERIFIED", + "firstname": "Carol", + "lastname": "Test", + "hidden": True, + "archived": True, + "identity_id": "0000000003", + }, + ] + write_csv(SOURCE_DIR / "contacts.csv", CONTACT_HEADER, contacts) + + alice_messages = [ + message(1, offset=1, message_type="TEXT", body="Hello from Alice.", outbox=False), + message( + 2, + offset=2, + message_type="TEXT", + body="Quoted reply from the synthetic owner.", + outbox=True, + quote=api_id(1), + ), + message( + 3, + offset=3, + message_type="TEXT", + body="Current edited message text.", + outbox=True, + edited_at=epoch_ms(4), + ), + message( + 4, + offset=5, + message_type="TEXT", + body="", + outbox=False, + deleted_at=epoch_ms(6), + ), + message( + 5, + offset=7, + message_type="FILE", + body=file_body( + mime_type="image/png", + size=media["direct_image"].stat().st_size, + filename="synthetic-image.png", + rendering_type=1, + downloaded=True, + caption="Synthetic image caption", + thumbnail_mime_type="image/jpeg", + metadata={"w": 96, "h": 64}, + ), + outbox=True, + caption="Synthetic image caption", + ), + message( + 6, + offset=8, + message_type="FILE", + body=file_body( + mime_type="audio/wav", + size=media["direct_audio"].stat().st_size, + filename="synthetic-audio.wav", + rendering_type=1, + downloaded=True, + metadata={"d": 0.4}, + ), + outbox=False, + ), + message( + 7, + offset=9, + message_type="FILE", + body=file_body( + mime_type="application/pdf", + size=media["direct_pdf"].stat().st_size, + filename="synthetic-document.pdf", + rendering_type=0, + downloaded=True, + ), + outbox=True, + ), + message( + 8, + offset=10, + message_type="LOCATION", + body=json.dumps( + [46.123456, 8.654321, 5.0, "Example Street 1", "Test Location"], + separators=(",", ":"), + ), + outbox=False, + ), + message( + 9, + offset=11, + message_type="VOIP_STATUS", + body='[1,{"status":2,"callId":123456789,"duration":42}]', + outbox=True, + ), + message( + 10, + offset=12, + message_type="TEXT", + body='CSV test: comma, "quote", backslash C:\\Temp, line\nbreak and emoji 🙂', + outbox=False, + ), + ] + write_csv(SOURCE_DIR / "message_0000000001.csv", MESSAGE_HEADER, alice_messages) + + missing_uid = uid(12) + bob_messages = [ + message(11, offset=20, message_type="TEXT", body="Short Bob test chat.", outbox=False), + message( + 12, + offset=21, + message_type="FILE", + body=file_body( + mime_type="application/octet-stream", + size=2048, + filename="unavailable-synthetic.bin", + rendering_type=0, + downloaded=False, + ), + outbox=False, + ), + ] + write_csv(SOURCE_DIR / "message_0000000002.csv", MESSAGE_HEADER, bob_messages) + write_csv(SOURCE_DIR / "message_0000000003.csv", MESSAGE_HEADER, []) + + group_uid = "1000000001" + group_id = "0011223344556677" + group_rows = [ + { + "id": group_id, + "creator": "ALICE001", + "groupname": "Project Aurora", + "created_at": epoch_ms(30), + "last_update": epoch_ms(50), + "members": [SELF_IDENTITY, "ALICE001", "BOB00002", "CAROL003"], + "archived": False, + "groupDesc": "Synthetic group for importer tests", + "groupDescTimestamp": epoch_ms(31), + "group_uid": group_uid, + "user_state": 0, + } + ] + write_csv( + SOURCE_DIR / "groups.csv", + [ + "id", + "creator", + "groupname", + "created_at", + "last_update", + "members", + "archived", + "groupDesc", + "groupDescTimestamp", + "group_uid", + "user_state", + ], + group_rows, + ) + + group_messages = [ + message(101, offset=32, message_type="TEXT", body="Group message from Alice.", outbox=False, identity="ALICE001"), + message( + 102, + offset=33, + message_type="TEXT", + body="Synthetic group reply.", + outbox=True, + identity=SELF_IDENTITY, + quote=api_id(101), + group_states='{"ALICE001":"READ","BOB00002":"DELIVERED"}', + ), + message( + 103, + offset=34, + message_type="GROUP_STATUS", + body='[4,{"status":3,"identity":"CAROL003"}]', + outbox=False, + identity="ALICE001", + ), + message( + 104, + offset=35, + message_type="GROUP_CALL_STATUS", + body='[2,{"status":1,"callId":"synthetic-call","callerIdentity":"BOB00002","groupId":1}]', + outbox=False, + identity="BOB00002", + ), + message( + 105, + offset=36, + message_type="BALLOT", + body="[1,1]", + outbox=True, + identity=SELF_IDENTITY, + ), + message( + 106, + offset=37, + message_type="FILE", + body=file_body( + mime_type="image/jpeg", + size=media["group_image"].stat().st_size, + filename="group-image.jpg", + rendering_type=1, + downloaded=True, + thumbnail_mime_type="image/jpeg", + metadata={"w": 80, "h": 80}, + ), + outbox=False, + identity="CAROL003", + ), + message( + 107, + offset=38, + message_type="FILE", + body=file_body( + mime_type="video/mp4", + size=media["group_video"].stat().st_size, + filename="group-video.mp4", + rendering_type=1, + downloaded=True, + thumbnail_mime_type="image/jpeg", + metadata={"d": 1, "w": 96, "h": 64}, + ), + outbox=True, + identity=SELF_IDENTITY, + ), + message( + 108, + offset=39, + message_type="TEXT", + body="", + outbox=False, + identity="BOB00002", + deleted_at=epoch_ms(40), + ), + ] + write_csv(SOURCE_DIR / f"group_message_{group_uid}.csv", GROUP_MESSAGE_HEADER, group_messages) + + write_csv( + SOURCE_DIR / "distribution_list.csv", + ["id", "distribution_list_name", "created_at", "last_update", "distribution_members", "archived"], + [ + { + "id": 1, + "distribution_list_name": "Synthetic Announcements", + "created_at": epoch_ms(60), + "last_update": epoch_ms(63), + "distribution_members": ["ALICE001", "BOB00002"], + "archived": False, + } + ], + ) + distribution_messages = [ + message(201, offset=61, message_type="TEXT", body="Announcement for Alice.", outbox=True, identity="ALICE001"), + message(202, offset=62, message_type="TEXT", body="Announcement for Bob.", outbox=True, identity="BOB00002"), + message( + 203, + offset=63, + message_type="FILE", + body=file_body( + mime_type="text/plain", + size=media["distribution_text"].stat().st_size, + filename="announcement.txt", + rendering_type=0, + downloaded=True, + ), + outbox=True, + identity="ALICE001", + ), + ] + write_csv( + SOURCE_DIR / "distribution_list_message_1.csv", + DISTRIBUTION_MESSAGE_HEADER, + distribution_messages, + ) + + write_csv( + SOURCE_DIR / "ballot.csv", + ["id", "aid", "creator", "ref", "ref_id", "name", "state", "assessment", "type", "choice_type", "last_viewed_at", "created_at", "modified_at"], + [ + { + "id": 1, + "aid": "a1b2c3d4e5f60708", + "creator": SELF_IDENTITY, + "ref": "GroupBallotModel", + "ref_id": group_uid, + "name": "Choose a synthetic option", + "state": "OPEN", + "assessment": "SINGLE", + "type": "INTERMEDIATE", + "choice_type": "TEXT", + "last_viewed_at": epoch_ms(41), + "created_at": epoch_ms(36), + "modified_at": epoch_ms(41), + } + ], + ) + write_csv( + SOURCE_DIR / "ballot_choice.csv", + ["id", "ballot", "aid", "type", "name", "vote_count", "order", "created_at", "modified_at"], + [ + {"id": 1, "ballot": "1:1", "aid": 1, "type": "TEXT", "name": "Option Alpha", "vote_count": 1, "order": 1, "created_at": epoch_ms(36), "modified_at": epoch_ms(41)}, + {"id": 2, "ballot": "1:1", "aid": 2, "type": "TEXT", "name": "Option Beta", "vote_count": 1, "order": 2, "created_at": epoch_ms(36), "modified_at": epoch_ms(41)}, + ], + ) + write_csv( + SOURCE_DIR / "ballot_vote.csv", + ["id", "ballot_uid", "choice_uid", "identity", "choice", "created_at", "modified_at"], + [ + {"id": 1, "ballot_uid": "1:1", "choice_uid": "1:1:1", "identity": "ALICE001", "choice": 1, "created_at": epoch_ms(40), "modified_at": epoch_ms(40)}, + {"id": 2, "ballot_uid": "1:1", "choice_uid": "1:1:2", "identity": SELF_IDENTITY, "choice": 1, "created_at": epoch_ms(41), "modified_at": epoch_ms(41)}, + ], + ) + + write_csv( + SOURCE_DIR / "contact_reactions.csv", + ["identity", "api_message_id", "sender_identity", "emoji_sequence", "reacted_at"], + [ + {"identity": "ALICE001", "api_message_id": api_id(1), "sender_identity": SELF_IDENTITY, "emoji_sequence": "👍", "reacted_at": epoch_ms(13)}, + {"identity": "ALICE001", "api_message_id": api_id(2), "sender_identity": "ALICE001", "emoji_sequence": "🙂", "reacted_at": epoch_ms(14)}, + ], + ) + write_csv( + SOURCE_DIR / "group_reactions.csv", + ["api_group_id", "group_creator_identity", "api_message_id", "sender_identity", "emoji_sequence", "reacted_at"], + [ + {"api_group_id": group_id, "group_creator_identity": "ALICE001", "api_message_id": api_id(101), "sender_identity": "BOB00002", "emoji_sequence": "✅", "reacted_at": epoch_ms(42)}, + {"api_group_id": group_id, "group_creator_identity": "ALICE001", "api_message_id": api_id(102), "sender_identity": SELF_IDENTITY, "emoji_sequence": "👍", "reacted_at": epoch_ms(43)}, + ], + ) + write_csv( + SOURCE_DIR / "reaction_counts.csv", + ["contactReactions", "groupReactions"], + [{"contactReactions": 2, "groupReactions": 2}], + ) + + write_csv(SOURCE_DIR / "nonces.csv", ["nonces"], [{"nonces": "aa" * 32}, {"nonces": "bb" * 32}]) + write_csv(SOURCE_DIR / "nonces_d2d.csv", ["nonces"], [{"nonces": "cc" * 32}]) + write_csv(SOURCE_DIR / "nonce_counts.csv", ["csp", "d2d"], [{"csp": 2, "d2d": 1}]) + + expected = { + "backup_version": 27, + "self_identity": SELF_IDENTITY, + "contacts": 3, + "groups": 1, + "distribution_lists": 1, + "archive_message_files": {"direct": 3, "group": 1, "distribution_list": 1}, + "exported_conversations": { + "direct": 2, + "group": 1, + "distribution_list": 1, + "total": 4, + }, + "skipped_empty_conversations": 1, + "messages": {"direct": 12, "group": 8, "distribution_list": 3, "total": 23}, + "attachment_references": 7, + "media_payloads": 6, + "missing_media_payloads": 1, + "thumbnails": 3, + "contact_reactions": 2, + "group_reactions": 2, + "polls": 1, + "poll_choices": 2, + "poll_votes": 2, + "empty_direct_chat_identity": "CAROL003", + "missing_media_message_uid": missing_uid, + "archive_variants": { + "threema-backup_synthetic.zip": { + "media_payloads": 6, + "thumbnails": 3, + }, + "threema-backup_synthetic_no_media.zip": { + "media_payloads": 0, + "thumbnails": 0, + }, + }, + } + return expected + + +def is_message_media(path: Path) -> bool: + """Return whether one archive member is message media or a thumbnail.""" + return path.name.startswith( + ( + "message_media_", + "message_thumbnail_", + "group_message_media_", + "group_message_thumbnail_", + "distribution_list_message_media_", + "distribution_list_thumbnail_", + ) + ) + + +def build_archive(path: Path, members: list[Path]) -> None: + """Create one AES-256 encrypted ZIP from selected source members.""" + seven_zip = shutil.which("7z") + if not seven_zip: + raise RuntimeError("7-Zip is required to build the encrypted fixture") + path.unlink(missing_ok=True) + subprocess.run( + [ + seven_zip, + "a", + "-tzip", + str(path), + *(member.name for member in members), + f"-p{PASSWORD}", + "-mem=AES256", + "-mx=9", + ], + cwd=SOURCE_DIR, + check=True, + ) + + +def main() -> None: + """Regenerate the plaintext source, expectations, and encrypted archive.""" + if SOURCE_DIR.exists(): + shutil.rmtree(SOURCE_DIR) + SOURCE_DIR.mkdir(parents=True) + media = create_media() + expected = create_structured_files(media) + EXPECTED_PATH.write_text( + json.dumps(expected, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + members = sorted(path for path in SOURCE_DIR.iterdir() if path.is_file()) + build_archive(ARCHIVE_PATH, members) + build_archive( + NO_MEDIA_ARCHIVE_PATH, + [path for path in members if not is_message_media(path)], + ) + print(f"Created {ARCHIVE_PATH}") + print(f"Created {NO_MEDIA_ARCHIVE_PATH}") + + +if __name__ == "__main__": + main() diff --git a/Examples/ThreemaAndroidBackupSynthetic/expected.json b/Examples/ThreemaAndroidBackupSynthetic/expected.json new file mode 100644 index 0000000..ee6906d --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/expected.json @@ -0,0 +1,46 @@ +{ + "backup_version": 27, + "self_identity": "0ZAHXXHB", + "contacts": 3, + "groups": 1, + "distribution_lists": 1, + "archive_message_files": { + "direct": 3, + "group": 1, + "distribution_list": 1 + }, + "exported_conversations": { + "direct": 2, + "group": 1, + "distribution_list": 1, + "total": 4 + }, + "skipped_empty_conversations": 1, + "messages": { + "direct": 12, + "group": 8, + "distribution_list": 3, + "total": 23 + }, + "attachment_references": 7, + "media_payloads": 6, + "missing_media_payloads": 1, + "thumbnails": 3, + "contact_reactions": 2, + "group_reactions": 2, + "polls": 1, + "poll_choices": 2, + "poll_votes": 2, + "empty_direct_chat_identity": "CAROL003", + "missing_media_message_uid": "00000000-0000-4000-8000-000000000012", + "archive_variants": { + "threema-backup_synthetic.zip": { + "media_payloads": 6, + "thumbnails": 3 + }, + "threema-backup_synthetic_no_media.zip": { + "media_payloads": 0, + "thumbnails": 0 + } + } +} diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/ballot.csv b/Examples/ThreemaAndroidBackupSynthetic/source/ballot.csv new file mode 100644 index 0000000..9fc88e9 --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/ballot.csv @@ -0,0 +1,2 @@ +"id","aid","creator","ref","ref_id","name","state","assessment","type","choice_type","last_viewed_at","created_at","modified_at" +"1","a1b2c3d4e5f60708","0ZAHXXHB","GroupBallotModel","1000000001","Choose a synthetic option","OPEN","SINGLE","INTERMEDIATE","TEXT","1768470060000","1768469760000","1768470060000" diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/ballot_choice.csv b/Examples/ThreemaAndroidBackupSynthetic/source/ballot_choice.csv new file mode 100644 index 0000000..a78c86a --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/ballot_choice.csv @@ -0,0 +1,3 @@ +"id","ballot","aid","type","name","vote_count","order","created_at","modified_at" +"1","1:1","1","TEXT","Option Alpha","1","1","1768469760000","1768470060000" +"2","1:1","2","TEXT","Option Beta","1","2","1768469760000","1768470060000" diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/ballot_vote.csv b/Examples/ThreemaAndroidBackupSynthetic/source/ballot_vote.csv new file mode 100644 index 0000000..b0f3b77 --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/ballot_vote.csv @@ -0,0 +1,3 @@ +"id","ballot_uid","choice_uid","identity","choice","created_at","modified_at" +"1","1:1","1:1:1","ALICE001","1","1768470000000","1768470000000" +"2","1:1","1:1:2","0ZAHXXHB","1","1768470060000","1768470060000" diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/contact_avatar_0000000001 b/Examples/ThreemaAndroidBackupSynthetic/source/contact_avatar_0000000001 new file mode 100644 index 0000000..26bffe0 Binary files /dev/null and b/Examples/ThreemaAndroidBackupSynthetic/source/contact_avatar_0000000001 differ diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/contact_avatar_me b/Examples/ThreemaAndroidBackupSynthetic/source/contact_avatar_me new file mode 100644 index 0000000..9c1df77 Binary files /dev/null and b/Examples/ThreemaAndroidBackupSynthetic/source/contact_avatar_me differ diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/contact_profile_pic_0000000001 b/Examples/ThreemaAndroidBackupSynthetic/source/contact_profile_pic_0000000001 new file mode 100644 index 0000000..e2444fc Binary files /dev/null and b/Examples/ThreemaAndroidBackupSynthetic/source/contact_profile_pic_0000000001 differ diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/contact_reactions.csv b/Examples/ThreemaAndroidBackupSynthetic/source/contact_reactions.csv new file mode 100644 index 0000000..d31ac3c --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/contact_reactions.csv @@ -0,0 +1,3 @@ +"identity","api_message_id","sender_identity","emoji_sequence","reacted_at" +"ALICE001","0000000000000001","0ZAHXXHB","👍","1768468380000" +"ALICE001","0000000000000002","ALICE001","🙂","1768468440000" diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/contacts.csv b/Examples/ThreemaAndroidBackupSynthetic/source/contacts.csv new file mode 100644 index 0000000..4dfbb4f --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/contacts.csv @@ -0,0 +1,4 @@ +"identity","publickey","verification","acid","firstname","lastname","nick_name","last_update","hidden","archived","identity_id" +"ALICE001","1111111111111111111111111111111111111111111111111111111111111111","FULLY_VERIFIED","","Alice","Example","Ali","1768467600000","0","0","0000000001" +"BOB00002","2222222222222222222222222222222222222222222222222222222222222222","SERVER_VERIFIED","","Bob","Sample","","","0","0","0000000002" +"CAROL003","3333333333333333333333333333333333333333333333333333333333333333","UNVERIFIED","","Carol","Test","","","1","1","0000000003" diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/distribution_list.csv b/Examples/ThreemaAndroidBackupSynthetic/source/distribution_list.csv new file mode 100644 index 0000000..ac1c7be --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/distribution_list.csv @@ -0,0 +1,2 @@ +"id","distribution_list_name","created_at","last_update","distribution_members","archived" +"1","Synthetic Announcements","1768471200000","1768471380000","ALICE001;BOB00002","0" diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/distribution_list_message_1.csv b/Examples/ThreemaAndroidBackupSynthetic/source/distribution_list_message_1.csv new file mode 100644 index 0000000..4f94d8d --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/distribution_list_message_1.csv @@ -0,0 +1,4 @@ +"apiid","uid","identity","isoutbox","isread","issaved","messagestae","posted_at","created_at","modified_at","type","body","isstatusmessage","caption","quoted_message_apiid","delivered_at","read_at" +"00000000000000c9","00000000-0000-4000-8000-000000000201","ALICE001","1","1","1","READ","1768471260000","1768471260000","1768471260000","TEXT","Announcement for Alice.","0","","","1768471320000","1768471380000" +"00000000000000ca","00000000-0000-4000-8000-000000000202","BOB00002","1","1","1","READ","1768471320000","1768471320000","1768471320000","TEXT","Announcement for Bob.","0","","","1768471380000","1768471440000" +"00000000000000cb","00000000-0000-4000-8000-000000000203","ALICE001","1","1","1","READ","1768471380000","1768471380000","1768471380000","FILE","[""00112233445566778899aabbccddeeff"",""0000000000000000000000000000000000000000000000000000000000000000"",""text/plain"",41,""announcement.txt"",0,true,null,null,{}]","0","","","1768471440000","1768471500000" diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/distribution_list_message_media_00000000-0000-4000-8000-000000000203 b/Examples/ThreemaAndroidBackupSynthetic/source/distribution_list_message_media_00000000-0000-4000-8000-000000000203 new file mode 100644 index 0000000..a468d78 --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/distribution_list_message_media_00000000-0000-4000-8000-000000000203 @@ -0,0 +1 @@ +Synthetic distribution-list attachment. diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/group_avatar_1000000001 b/Examples/ThreemaAndroidBackupSynthetic/source/group_avatar_1000000001 new file mode 100644 index 0000000..c2b7f57 Binary files /dev/null and b/Examples/ThreemaAndroidBackupSynthetic/source/group_avatar_1000000001 differ diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/group_message_1000000001.csv b/Examples/ThreemaAndroidBackupSynthetic/source/group_message_1000000001.csv new file mode 100644 index 0000000..7541c41 --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/group_message_1000000001.csv @@ -0,0 +1,9 @@ +"apiid","uid","identity","isoutbox","isread","issaved","messagestae","posted_at","created_at","modified_at","type","body","isstatusmessage","caption","quoted_message_apiid","delivered_at","read_at","g_msg_states","display_tags","edited_at","deleted_at" +"0000000000000065","00000000-0000-4000-8000-000000000101","ALICE001","0","1","1","","1768469520000","1768469520000","1768469520000","TEXT","Group message from Alice.","0","","","","1768469640000","","0","","" +"0000000000000066","00000000-0000-4000-8000-000000000102","0ZAHXXHB","1","1","1","READ","1768469580000","1768469580000","1768469580000","TEXT","Synthetic group reply.","0","","0000000000000065","1768469640000","1768469700000","{""ALICE001"":""READ"",""BOB00002"":""DELIVERED""}","0","","" +"0000000000000067","00000000-0000-4000-8000-000000000103","ALICE001","0","1","1","","1768469640000","1768469640000","1768469640000","GROUP_STATUS","[4,{""status"":3,""identity"":""CAROL003""}]","0","","","","1768469760000","","0","","" +"0000000000000068","00000000-0000-4000-8000-000000000104","BOB00002","0","1","1","","1768469700000","1768469700000","1768469700000","GROUP_CALL_STATUS","[2,{""status"":1,""callId"":""synthetic-call"",""callerIdentity"":""BOB00002"",""groupId"":1}]","0","","","","1768469820000","","0","","" +"0000000000000069","00000000-0000-4000-8000-000000000105","0ZAHXXHB","1","1","1","READ","1768469760000","1768469760000","1768469760000","BALLOT","[1,1]","0","","","1768469820000","1768469880000","","0","","" +"000000000000006a","00000000-0000-4000-8000-000000000106","CAROL003","0","1","1","","1768469820000","1768469820000","1768469820000","FILE","[""00112233445566778899aabbccddeeff"",""0000000000000000000000000000000000000000000000000000000000000000"",""image/jpeg"",1866,""group-image.jpg"",1,true,null,""image/jpeg"",{""w"":80,""h"":80}]","0","","","","1768469940000","","0","","" +"000000000000006b","00000000-0000-4000-8000-000000000107","0ZAHXXHB","1","1","1","READ","1768469880000","1768469880000","1768469880000","FILE","[""00112233445566778899aabbccddeeff"",""0000000000000000000000000000000000000000000000000000000000000000"",""video/mp4"",1826,""group-video.mp4"",1,true,null,""image/jpeg"",{""d"":1,""w"":96,""h"":64}]","0","","","1768469940000","1768470000000","","0","","" +"000000000000006c","00000000-0000-4000-8000-000000000108","BOB00002","0","1","1","","1768469940000","1768469940000","1768469940000","TEXT","","0","","","","1768470060000","","0","","1768470000000" diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/group_message_media_00000000-0000-4000-8000-000000000106 b/Examples/ThreemaAndroidBackupSynthetic/source/group_message_media_00000000-0000-4000-8000-000000000106 new file mode 100644 index 0000000..4b411d5 Binary files /dev/null and b/Examples/ThreemaAndroidBackupSynthetic/source/group_message_media_00000000-0000-4000-8000-000000000106 differ diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/group_message_media_00000000-0000-4000-8000-000000000107 b/Examples/ThreemaAndroidBackupSynthetic/source/group_message_media_00000000-0000-4000-8000-000000000107 new file mode 100644 index 0000000..aba1adb Binary files /dev/null and b/Examples/ThreemaAndroidBackupSynthetic/source/group_message_media_00000000-0000-4000-8000-000000000107 differ diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/group_message_thumbnail_00000000-0000-4000-8000-000000000106 b/Examples/ThreemaAndroidBackupSynthetic/source/group_message_thumbnail_00000000-0000-4000-8000-000000000106 new file mode 100644 index 0000000..024990d Binary files /dev/null and b/Examples/ThreemaAndroidBackupSynthetic/source/group_message_thumbnail_00000000-0000-4000-8000-000000000106 differ diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/group_message_thumbnail_00000000-0000-4000-8000-000000000107 b/Examples/ThreemaAndroidBackupSynthetic/source/group_message_thumbnail_00000000-0000-4000-8000-000000000107 new file mode 100644 index 0000000..7b9938c Binary files /dev/null and b/Examples/ThreemaAndroidBackupSynthetic/source/group_message_thumbnail_00000000-0000-4000-8000-000000000107 differ diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/group_reactions.csv b/Examples/ThreemaAndroidBackupSynthetic/source/group_reactions.csv new file mode 100644 index 0000000..38c8bbf --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/group_reactions.csv @@ -0,0 +1,3 @@ +"api_group_id","group_creator_identity","api_message_id","sender_identity","emoji_sequence","reacted_at" +"0011223344556677","ALICE001","0000000000000065","BOB00002","✅","1768470120000" +"0011223344556677","ALICE001","0000000000000066","0ZAHXXHB","👍","1768470180000" diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/groups.csv b/Examples/ThreemaAndroidBackupSynthetic/source/groups.csv new file mode 100644 index 0000000..7840462 --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/groups.csv @@ -0,0 +1,2 @@ +"id","creator","groupname","created_at","last_update","members","archived","groupDesc","groupDescTimestamp","group_uid","user_state" +"0011223344556677","ALICE001","Project Aurora","1768469400000","1768470600000","0ZAHXXHB;ALICE001;BOB00002;CAROL003","0","Synthetic group for importer tests","1768469460000","1000000001","0" diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/identity b/Examples/ThreemaAndroidBackupSynthetic/source/identity new file mode 100644 index 0000000..3eab8f1 --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/identity @@ -0,0 +1 @@ +4K4M-5Q6T-KFUH-KHL5-2VCJ-ZM57-NL7R-WJTA-V45L-NJAM-WLEU-5DS4-XF7S-OPH4-CTCL-N2CF-3C4C-HPB7-YZWW-U3S6 \ No newline at end of file diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/message_0000000001.csv b/Examples/ThreemaAndroidBackupSynthetic/source/message_0000000001.csv new file mode 100644 index 0000000..24d2283 --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/message_0000000001.csv @@ -0,0 +1,12 @@ +"apiid","uid","isoutbox","isread","issaved","messagestae","posted_at","created_at","modified_at","type","body","isstatusmessage","caption","quoted_message_apiid","delivered_at","read_at","g_msg_states","display_tags","edited_at","deleted_at" +"0000000000000001","00000000-0000-4000-8000-000000000001","0","1","1","","1768467660000","1768467660000","1768467660000","TEXT","Hello from Alice.","0","","","","1768467780000","","0","","" +"0000000000000002","00000000-0000-4000-8000-000000000002","1","1","1","READ","1768467720000","1768467720000","1768467720000","TEXT","Quoted reply from the synthetic owner.","0","","0000000000000001","1768467780000","1768467840000","","0","","" +"0000000000000003","00000000-0000-4000-8000-000000000003","1","1","1","READ","1768467780000","1768467780000","1768467840000","TEXT","Current edited message text.","0","","","1768467840000","1768467900000","","0","1768467840000","" +"0000000000000004","00000000-0000-4000-8000-000000000004","0","1","1","","1768467900000","1768467900000","1768467900000","TEXT","","0","","","","1768468020000","","0","","1768467960000" +"0000000000000005","00000000-0000-4000-8000-000000000005","1","1","1","READ","1768468020000","1768468020000","1768468020000","FILE","[""00112233445566778899aabbccddeeff"",""0000000000000000000000000000000000000000000000000000000000000000"",""image/png"",766,""synthetic-image.png"",1,true,""Synthetic image caption"",""image/jpeg"",{""w"":96,""h"":64}]","0","Synthetic image caption","","1768468080000","1768468140000","","0","","" +"0000000000000006","00000000-0000-4000-8000-000000000006","0","1","1","","1768468080000","1768468080000","1768468080000","FILE","[""00112233445566778899aabbccddeeff"",""0000000000000000000000000000000000000000000000000000000000000000"",""audio/wav"",6444,""synthetic-audio.wav"",1,true,null,null,{""d"":0.4}]","0","","","","1768468200000","","0","","" +"0000000000000007","00000000-0000-4000-8000-000000000007","1","1","1","READ","1768468140000","1768468140000","1768468140000","FILE","[""00112233445566778899aabbccddeeff"",""0000000000000000000000000000000000000000000000000000000000000000"",""application/pdf"",1410,""synthetic-document.pdf"",0,true,null,null,{}]","0","","","1768468200000","1768468260000","","0","","" +"0000000000000008","00000000-0000-4000-8000-000000000008","0","1","1","","1768468200000","1768468200000","1768468200000","LOCATION","[46.123456,8.654321,5.0,""Example Street 1"",""Test Location""]","0","","","","1768468320000","","0","","" +"0000000000000009","00000000-0000-4000-8000-000000000009","1","1","1","READ","1768468260000","1768468260000","1768468260000","VOIP_STATUS","[1,{""status"":2,""callId"":123456789,""duration"":42}]","0","","","1768468320000","1768468380000","","0","","" +"000000000000000a","00000000-0000-4000-8000-000000000010","0","1","1","","1768468320000","1768468320000","1768468320000","TEXT","CSV test: comma, ""quote"", backslash C:\\Temp, line +break and emoji 🙂","0","","","","1768468440000","","0","","" diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/message_0000000002.csv b/Examples/ThreemaAndroidBackupSynthetic/source/message_0000000002.csv new file mode 100644 index 0000000..2588511 --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/message_0000000002.csv @@ -0,0 +1,3 @@ +"apiid","uid","isoutbox","isread","issaved","messagestae","posted_at","created_at","modified_at","type","body","isstatusmessage","caption","quoted_message_apiid","delivered_at","read_at","g_msg_states","display_tags","edited_at","deleted_at" +"000000000000000b","00000000-0000-4000-8000-000000000011","0","1","1","","1768468800000","1768468800000","1768468800000","TEXT","Short Bob test chat.","0","","","","1768468920000","","0","","" +"000000000000000c","00000000-0000-4000-8000-000000000012","0","1","1","","1768468860000","1768468860000","1768468860000","FILE","[""00112233445566778899aabbccddeeff"",""0000000000000000000000000000000000000000000000000000000000000000"",""application/octet-stream"",2048,""unavailable-synthetic.bin"",0,false,null,null,{}]","0","","","","1768468980000","","0","","" diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/message_0000000003.csv b/Examples/ThreemaAndroidBackupSynthetic/source/message_0000000003.csv new file mode 100644 index 0000000..2468121 --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/message_0000000003.csv @@ -0,0 +1 @@ +"apiid","uid","isoutbox","isread","issaved","messagestae","posted_at","created_at","modified_at","type","body","isstatusmessage","caption","quoted_message_apiid","delivered_at","read_at","g_msg_states","display_tags","edited_at","deleted_at" diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/message_media_00000000-0000-4000-8000-000000000005 b/Examples/ThreemaAndroidBackupSynthetic/source/message_media_00000000-0000-4000-8000-000000000005 new file mode 100644 index 0000000..b71285f Binary files /dev/null and b/Examples/ThreemaAndroidBackupSynthetic/source/message_media_00000000-0000-4000-8000-000000000005 differ diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/message_media_00000000-0000-4000-8000-000000000006 b/Examples/ThreemaAndroidBackupSynthetic/source/message_media_00000000-0000-4000-8000-000000000006 new file mode 100644 index 0000000..f3fb7dd Binary files /dev/null and b/Examples/ThreemaAndroidBackupSynthetic/source/message_media_00000000-0000-4000-8000-000000000006 differ diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/message_media_00000000-0000-4000-8000-000000000007 b/Examples/ThreemaAndroidBackupSynthetic/source/message_media_00000000-0000-4000-8000-000000000007 new file mode 100644 index 0000000..0632743 --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/message_media_00000000-0000-4000-8000-000000000007 @@ -0,0 +1,68 @@ +%PDF-1.3 +% ReportLab Generated PDF document (opensource) +1 0 obj +<< +/F1 2 0 R +>> +endobj +2 0 obj +<< +/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font +>> +endobj +3 0 obj +<< +/Contents 7 0 R /MediaBox [ 0 0 240 160 ] /Parent 6 0 R /Resources << +/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] +>> /Rotate 0 /Trans << + +>> + /Type /Page +>> +endobj +4 0 obj +<< +/PageMode /UseNone /Pages 6 0 R /Type /Catalog +>> +endobj +5 0 obj +<< +/Author (anonymous) /CreationDate (D:20260715085538+02'00') /Creator (anonymous) /Keywords () /ModDate (D:20260715085538+02'00') /Producer (ReportLab PDF Library - \(opensource\)) + /Subject (unspecified) /Title (Synthetic attachment) /Trapped /False +>> +endobj +6 0 obj +<< +/Count 1 /Kids [ 3 0 R ] /Type /Pages +>> +endobj +7 0 obj +<< +/Filter [ /ASCII85Decode /FlateDecode ] /Length 157 +>> +stream +Gar?'YmS?5&4HDC`K\aoEPiNj"g,]iqZG$PE_>ap0&-0q?@mkR"&lk#7l]#Feij.*7Xlc3!5=(jkJ.n(\>c"4f>3$h4eeM(ab&6aK0rLq!\k?1U*eWGTA][_4H>nXbpY%t]I\qFnC"W'<5D:&IUk8Wi]Sd4~>endstream +endobj +xref +0 8 +0000000000 65535 f +0000000061 00000 n +0000000092 00000 n +0000000199 00000 n +0000000392 00000 n +0000000460 00000 n +0000000733 00000 n +0000000792 00000 n +trailer +<< +/ID +[] +% ReportLab generated PDF document -- digest (opensource) + +/Info 5 0 R +/Root 4 0 R +/Size 8 +>> +startxref +1039 +%%EOF diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/message_thumbnail_00000000-0000-4000-8000-000000000005 b/Examples/ThreemaAndroidBackupSynthetic/source/message_thumbnail_00000000-0000-4000-8000-000000000005 new file mode 100644 index 0000000..c1b55bc Binary files /dev/null and b/Examples/ThreemaAndroidBackupSynthetic/source/message_thumbnail_00000000-0000-4000-8000-000000000005 differ diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/nonce_counts.csv b/Examples/ThreemaAndroidBackupSynthetic/source/nonce_counts.csv new file mode 100644 index 0000000..554804d --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/nonce_counts.csv @@ -0,0 +1,2 @@ +"csp","d2d" +"2","1" diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/nonces.csv b/Examples/ThreemaAndroidBackupSynthetic/source/nonces.csv new file mode 100644 index 0000000..c894246 --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/nonces.csv @@ -0,0 +1,3 @@ +"nonces" +"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/nonces_d2d.csv b/Examples/ThreemaAndroidBackupSynthetic/source/nonces_d2d.csv new file mode 100644 index 0000000..c2295ed --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/nonces_d2d.csv @@ -0,0 +1,2 @@ +"nonces" +"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/reaction_counts.csv b/Examples/ThreemaAndroidBackupSynthetic/source/reaction_counts.csv new file mode 100644 index 0000000..78fefdb --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/reaction_counts.csv @@ -0,0 +1,2 @@ +"contactReactions","groupReactions" +"2","2" diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/settings b/Examples/ThreemaAndroidBackupSynthetic/source/settings new file mode 100644 index 0000000..97fd763 --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/settings @@ -0,0 +1 @@ +"version","27" diff --git a/Examples/ThreemaAndroidBackupSynthetic/threema-backup_synthetic.zip b/Examples/ThreemaAndroidBackupSynthetic/threema-backup_synthetic.zip new file mode 100644 index 0000000..72b2d2e Binary files /dev/null and b/Examples/ThreemaAndroidBackupSynthetic/threema-backup_synthetic.zip differ diff --git a/Examples/ThreemaAndroidBackupSynthetic/threema-backup_synthetic_no_media.zip b/Examples/ThreemaAndroidBackupSynthetic/threema-backup_synthetic_no_media.zip new file mode 100644 index 0000000..f7ac864 Binary files /dev/null and b/Examples/ThreemaAndroidBackupSynthetic/threema-backup_synthetic_no_media.zip differ diff --git a/README.md b/README.md index 1488a94..410ba9d 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,8 @@ ChatExportPDF exports chat data from messenger sources into PDF reports. Current sources: -- `threema` (iOS only) +- `threema` (iOS database) +- `threema_android_backup` (encrypted Android data-backup ZIP) - `whatsapp` For each conversation, the exporter creates: @@ -11,7 +12,7 @@ For each conversation, the exporter creates: - a `*_TECH.pdf` report - optionally, an Excel workbook with structured conversation data -The main report is generic. The TECH report can be importer-specific. Threema has its own dedicated TECH report. WhatsApp currently uses the generic fallback TECH report. +The main report is generic. The TECH report can be importer-specific. The Threema iOS database importer has its own dedicated TECH report. Threema Android backups and WhatsApp use the generic fallback TECH report. ## Desktop GUI @@ -20,8 +21,9 @@ For Windows, use the standalone desktop app from the [Releases page](https://git 1. Download `ChatExportPDF.exe`. 2. Start the EXE. No Python installation is required. 3. Select the source: - - `threema`: choose `ThreemaData.sqlite`; optionally choose `_EXTERNAL_DATA`. - - `whatsapp`: choose the exported WhatsApp `.zip`. + - `Threema iOS database`: choose `ThreemaData.sqlite`; optionally choose `_EXTERNAL_DATA`. + - `Threema Android backup`: choose the encrypted data-backup `.zip` and enter its password. + - `WhatsApp export`: choose the exported WhatsApp `.zip`. 4. Choose or keep the suggested output folder. 5. Start the export. @@ -50,6 +52,14 @@ Typical iOS paths: - `/private/var/mobile/Containers/Shared/AppGroup//ThreemaData.sqlite` - `/private/var/mobile/Containers/Shared/AppGroup//.ThreemaData_SUPPORT/_EXTERNAL_DATA` +### Threema Android Data Backup + +- encrypted data-backup `.zip` created by the Threema Android app +- backup password selected when the backup was created +- current backup format version `27` +- direct chats, groups, distribution lists, participants, messages, reactions, polls, and attachment references +- media payloads are extracted when they were included in the backup and media export is enabled + ### WhatsApp - exported `.zip` file @@ -110,6 +120,16 @@ Recommended with external attachment folder: python -m chat_export --source threema --input-path "./ThreemaData.sqlite" --out-dir "./export" --external-folder "./_EXTERNAL_DATA" ``` +### Threema Android Data Backup + +The CLI prompts for the backup password without displaying it: + +```bash +python -m chat_export --source threema_android_backup --input-path "./threema-backup.zip" --out-dir "./export" +``` + +For non-interactive use, the password can be supplied with `--backup-password`. Be aware that command-line arguments can remain in shell history. + ### WhatsApp WhatsApp export: @@ -143,12 +163,13 @@ The preferred CLI entry points are: ### Source Selection -- `--source {threema,whatsapp}` +- `--source {threema,threema_android_backup,whatsapp}` Selects the importer. Default: `threema`. - `--input-path PATH` Generic input path for the selected source. - for `threema`: path to `ThreemaData.sqlite` + - for `threema_android_backup`: path to the encrypted Android data-backup `.zip` - for `whatsapp`: path to the exported `.zip` - `--db-path PATH` @@ -157,10 +178,13 @@ The preferred CLI entry points are: - `--chat-text-name NAME` Only relevant for WhatsApp. Forces a specific chat text file inside the ZIP when multiple plausible `.txt` files exist. +- `--backup-password PASSWORD` + Only relevant for Threema Android data backups. If omitted in an interactive terminal, the CLI prompts for it securely. + ### Optional - `--external-folder PATH` - Threema only. Path to `_EXTERNAL_DATA`. Recommended for complete attachment export. + Threema iOS database only. Path to `_EXTERNAL_DATA`. Recommended for complete attachment export. - `--tz TIMEZONE` Timezone for rendered timestamps. Default: `Europe/Zurich`. diff --git a/packaging/windows/ChatExportPDF.spec b/packaging/windows/ChatExportPDF.spec index 09834ff..8fe03dd 100644 --- a/packaging/windows/ChatExportPDF.spec +++ b/packaging/windows/ChatExportPDF.spec @@ -20,7 +20,14 @@ datas = [] binaries = [] hiddenimports = [] -for package in ("reportlab", "PIL", "xlsxwriter", "emoji", "tzdata"): +for package in ( + "reportlab", + "PIL", + "xlsxwriter", + "emoji", + "pyzipper", + "tzdata", +): package_datas, package_binaries, package_hiddenimports = _collect_optional(package) datas += package_datas binaries += package_binaries diff --git a/pyproject.toml b/pyproject.toml index 55536ac..d938e39 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "chat-export-pdf" -version = "0.7.0" +version = "0.8.0" description = "Export chat conversations from multiple sources to PDFs with optional media extraction" readme = "README.md" requires-python = ">=3.10" @@ -15,6 +15,7 @@ dependencies = [ "reportlab>=4.0.0", "XlsxWriter>=3.2.0", "emoji>=2.0.0", + "pyzipper>=0.3.6", "tzdata>=2023.3; platform_system=='Windows'", ] diff --git a/src/chat_export/__init__.py b/src/chat_export/__init__.py index bf48c60..a53c8bb 100644 --- a/src/chat_export/__init__.py +++ b/src/chat_export/__init__.py @@ -1,4 +1,4 @@ """Top-level package metadata for the chat export application.""" __all__ = ["__version__"] -__version__ = "0.7.0" +__version__ = "0.8.0" diff --git a/src/chat_export/cli.py b/src/chat_export/cli.py index 6fe8198..1187179 100644 --- a/src/chat_export/cli.py +++ b/src/chat_export/cli.py @@ -4,11 +4,17 @@ import argparse import logging import sys +from getpass import getpass from typing import Optional from .common.logging_setup import sanitize_local_paths, setup_logging from .config_factory import build_export_config -from .constants import DEFAULT_SOURCE_APP, DEFAULT_TIMEZONE, SOURCE_APPS +from .constants import ( + DEFAULT_SOURCE_APP, + DEFAULT_TIMEZONE, + SOURCE_APPS, + SOURCE_APP_THREEMA_ANDROID_BACKUP, +) from .orchestrator import export_all_conversations @@ -26,7 +32,7 @@ def build_parser() -> argparse.ArgumentParser: "--source", default=DEFAULT_SOURCE_APP, choices=SOURCE_APPS, - help="Source app / importer to use (currently supported: threema, whatsapp)", + help="Source app / importer to use", ) p.add_argument( "--input-path", @@ -43,6 +49,11 @@ def build_parser() -> argparse.ArgumentParser: default=None, help="For WhatsApp ZIPs with multiple plausible .txt files: exact chat text filename to use", ) + p.add_argument( + "--backup-password", + default=None, + help="Password for a Threema Android data-backup ZIP; omit for a secure prompt", + ) p.add_argument("--out-dir", required=True, help="Output directory") p.add_argument( "--external-folder", @@ -123,6 +134,17 @@ def main(argv: Optional[list[str]] = None) -> int: """ parser = build_parser() args = parser.parse_args(argv) + backup_password = args.backup_password + if ( + args.source == SOURCE_APP_THREEMA_ANDROID_BACKUP + and not backup_password + and sys.stdin.isatty() + ): + try: + backup_password = getpass("Threema Android backup password: ") + except (EOFError, KeyboardInterrupt): + print("Backup password input cancelled.", file=sys.stderr) + return 1 try: cfg = build_export_config( @@ -131,6 +153,7 @@ def main(argv: Optional[list[str]] = None) -> int: input_path=args.input_path, db_path=args.db_path, chat_text_name=args.chat_text_name, + backup_password=backup_password, external_folder=args.external_folder, tz_name=args.tz, export_media=not args.no_media, @@ -165,7 +188,12 @@ def main(argv: Optional[list[str]] = None) -> int: return 1 log = logging.getLogger("chat_export") - log.debug("Parsed CLI args: %s", vars(args)) + log.debug( + "Parsed CLI options source=%s input_configured=%s output_configured=%s", + args.source, + bool(args.input_path or args.db_path), + bool(args.out_dir), + ) log.info("Starting export source=%s tz=%s", args.source, args.tz) log.debug( "Export options source=%s media=%s image_previews=%s excel=%s max_media_bytes=%s limit_conversations=%s limit_messages=%s chat_text_name=%s", @@ -190,7 +218,8 @@ def main(argv: Optional[list[str]] = None) -> int: try: res = export_all_conversations(cfg) except Exception as e: - log.exception("Export failed: %s", e) + log.error("Export failed: %s", e) + log.debug("Export failure details", exc_info=True) return 1 log.info( diff --git a/src/chat_export/config.py b/src/chat_export/config.py index 4b7b1c2..6c00ff7 100644 --- a/src/chat_export/config.py +++ b/src/chat_export/config.py @@ -1,11 +1,15 @@ """Define export configuration and validation.""" import logging -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from typing import Optional -from .constants import DEFAULT_SOURCE_APP, DEFAULT_TIMEZONE +from .constants import ( + DEFAULT_SOURCE_APP, + DEFAULT_TIMEZONE, + SOURCE_APP_THREEMA_ANDROID_BACKUP, +) log = logging.getLogger(__name__) @@ -20,6 +24,7 @@ class ExportConfig: input_path (Optional[str]): Generic source input path. db_path (Optional[str]): Threema SQLite database path. chat_text_name (Optional[str]): Explicit WhatsApp chat text file name. + backup_password (Optional[str]): Password for an encrypted source backup. external_folder (Optional[str]): Threema external media directory path. tz_name (str): IANA timezone name for rendered timestamps. export_media (bool): Enable attachment export. @@ -40,6 +45,7 @@ class ExportConfig: input_path: Optional[str] = None db_path: Optional[str] = None chat_text_name: Optional[str] = None + backup_password: Optional[str] = field(default=None, repr=False) external_folder: Optional[str] = None tz_name: str = DEFAULT_TIMEZONE @@ -94,6 +100,11 @@ def validate(self) -> None: raise ValueError("source_app must not be empty") if not self.out_dir or not self.out_dir.strip(): raise ValueError("out_dir must not be empty") + if ( + self.source_app == SOURCE_APP_THREEMA_ANDROID_BACKUP + and not self.backup_password + ): + raise ValueError("backup_password is required for Threema Android backups") out = Path(self.out_dir) out.mkdir(parents=True, exist_ok=True) diff --git a/src/chat_export/config_factory.py b/src/chat_export/config_factory.py index 3a34f31..0419912 100644 --- a/src/chat_export/config_factory.py +++ b/src/chat_export/config_factory.py @@ -5,7 +5,12 @@ from typing import Optional from .config import ExportConfig -from .constants import DEFAULT_TIMEZONE, SOURCE_APP_THREEMA +from .constants import ( + DEFAULT_TIMEZONE, + SOURCE_APP_THREEMA, + SOURCE_APP_THREEMA_ANDROID_BACKUP, + SOURCE_APP_WHATSAPP, +) from .export_summary import default_log_file @@ -49,6 +54,7 @@ def build_export_config( input_path: Optional[str], db_path: Optional[str] = None, chat_text_name: Optional[str] = None, + backup_password: Optional[str] = None, external_folder: Optional[str] = None, tz_name: Optional[str] = None, export_media: bool = True, @@ -70,6 +76,7 @@ def build_export_config( input_path (Optional[str]): Generic source input path. db_path (Optional[str]): Explicit database path override. chat_text_name (Optional[str]): Explicit WhatsApp chat text file name. + backup_password (Optional[str]): Password for an encrypted source backup. external_folder (Optional[str]): Optional Threema external folder path. tz_name (Optional[str]): Optional timezone override. export_media (bool): Enable media export. @@ -89,6 +96,7 @@ def build_export_config( effective_input_path = _optional_text(input_path) effective_source = source_app.strip() effective_chat_text_name = _optional_text(chat_text_name) + effective_backup_password = backup_password if backup_password else None effective_external_folder = _optional_text(external_folder) effective_tz_name = (tz_name or "").strip() or DEFAULT_TIMEZONE effective_db_path = _optional_text(db_path) @@ -102,6 +110,17 @@ def build_export_config( if not effective_input_path: raise ValueError("input_path must not be empty for non-Threema sources.") + if effective_source != SOURCE_APP_THREEMA_ANDROID_BACKUP: + effective_backup_password = None + if effective_source != SOURCE_APP_WHATSAPP: + effective_chat_text_name = None + + if ( + effective_source == SOURCE_APP_THREEMA_ANDROID_BACKUP + and not effective_backup_password + ): + raise ValueError("backup_password is required for Threema Android backups.") + effective_out_dir = out_dir.strip() if not effective_out_dir: raise ValueError("out_dir must not be empty or whitespace-only.") @@ -113,6 +132,7 @@ def build_export_config( input_path=effective_input_path, db_path=effective_db_path, chat_text_name=effective_chat_text_name, + backup_password=effective_backup_password, external_folder=effective_external_folder, tz_name=effective_tz_name, export_media=export_media, diff --git a/src/chat_export/constants.py b/src/chat_export/constants.py index 422fad3..357582b 100644 --- a/src/chat_export/constants.py +++ b/src/chat_export/constants.py @@ -1,10 +1,12 @@ """Define shared application constants.""" SOURCE_APP_THREEMA = "threema" +SOURCE_APP_THREEMA_ANDROID_BACKUP = "threema_android_backup" SOURCE_APP_WHATSAPP = "whatsapp" SOURCE_APPS = ( SOURCE_APP_THREEMA, + SOURCE_APP_THREEMA_ANDROID_BACKUP, SOURCE_APP_WHATSAPP, ) diff --git a/src/chat_export/export_summary.py b/src/chat_export/export_summary.py index ceb1c37..90d344c 100644 --- a/src/chat_export/export_summary.py +++ b/src/chat_export/export_summary.py @@ -11,6 +11,7 @@ from . import __version__ from .common.logging_setup import sanitize_local_paths from .config import ExportConfig +from .constants import SOURCE_APP_THREEMA, SOURCE_APP_WHATSAPP TOOL_NAME = "ChatExportPDF" REPOSITORY_URL = "https://github.com/JeromeMaag/ChatExportPDF" @@ -351,6 +352,8 @@ def _settings(cfg: ExportConfig, results: dict[str, Any] | None) -> dict[str, An "selected_source": cfg.source_app, "timezone": cfg.tz_name, "time_mode": (results or {}).get("time_mode", "unknown"), + "backup_version": (results or {}).get("backup_version"), + "source_owner_identity": (results or {}).get("self_identity"), "media_export_enabled": cfg.export_media, "image_previews_enabled": cfg.export_image_previews, "excel_export_enabled": cfg.export_excel, @@ -360,8 +363,16 @@ def _settings(cfg: ExportConfig, results: dict[str, Any] | None) -> dict[str, An "log_file": LOG_FILENAME, "console_and_gui_log_level": "INFO", "log_file_level": "DEBUG", - "chat_text_name": cfg.chat_text_name or "auto", - "external_folder_provided": bool(cfg.external_folder), + "chat_text_name": ( + cfg.chat_text_name or "auto" + if cfg.source_app == SOURCE_APP_WHATSAPP + else None + ), + "external_folder_provided": ( + bool(cfg.external_folder) + if cfg.source_app == SOURCE_APP_THREEMA + else None + ), } @@ -471,6 +482,8 @@ def build_summary_text(manifest: dict[str, Any]) -> str: _line("Selected source", settings["selected_source"]), _line("Timezone", settings["timezone"]), _line("Time mode", settings["time_mode"]), + _line("Backup version", settings["backup_version"]), + _line("Source owner identity", settings["source_owner_identity"]), _line("Media export enabled", settings["media_export_enabled"]), _line("Image previews enabled", settings["image_previews_enabled"]), _line("Excel export enabled", settings["excel_export_enabled"]), diff --git a/src/chat_export/gui.py b/src/chat_export/gui.py index ebf0cba..bbe2813 100644 --- a/src/chat_export/gui.py +++ b/src/chat_export/gui.py @@ -21,10 +21,20 @@ DEFAULT_TIMEZONE, SOURCE_APPS, SOURCE_APP_THREEMA, + SOURCE_APP_THREEMA_ANDROID_BACKUP, + SOURCE_APP_WHATSAPP, ) from .orchestrator import export_all_conversations +SOURCE_LABELS = { + SOURCE_APP_THREEMA: "Threema iOS database", + SOURCE_APP_THREEMA_ANDROID_BACKUP: "Threema Android backup", + SOURCE_APP_WHATSAPP: "WhatsApp export", +} +SOURCE_KEYS_BY_LABEL = {label: key for key, label in SOURCE_LABELS.items()} + + class QueueLogHandler(logging.Handler): """Forward formatted log records to a GUI queue.""" @@ -99,11 +109,12 @@ def __init__(self, root: tk.Tk) -> None: def _build_variables(self) -> None: """Create Tkinter state variables.""" - self.source_var = tk.StringVar(value=DEFAULT_SOURCE_APP) + self.source_var = tk.StringVar(value=SOURCE_LABELS[DEFAULT_SOURCE_APP]) self.input_path_var = tk.StringVar() self.out_dir_var = tk.StringVar() self.external_folder_var = tk.StringVar() self.chat_text_name_var = tk.StringVar() + self.backup_password_var = tk.StringVar() self.tz_var = tk.StringVar(value=DEFAULT_TIMEZONE) self.export_media_var = tk.BooleanVar(value=True) self.export_image_previews_var = tk.BooleanVar(value=True) @@ -137,7 +148,7 @@ def _build_layout(self) -> None: self.source_combo = ttk.Combobox( basic, textvariable=self.source_var, - values=SOURCE_APPS, + values=tuple(SOURCE_LABELS[source] for source in SOURCE_APPS), state="readonly", ) self.source_combo.grid(row=0, column=1, sticky="ew", padx=(0, 8), pady=(8, 6)) @@ -163,6 +174,13 @@ def _build_layout(self) -> None: self.external_entry = ttk.Entry(basic, textvariable=self.external_folder_var) self.external_button = ttk.Button(basic, text="Browse...", command=self._browse_external_folder) + self.backup_password_label = ttk.Label(basic, text="Backup password") + self.backup_password_entry = ttk.Entry( + basic, + textvariable=self.backup_password_var, + show="*", + ) + options_row = ttk.Frame(basic) options_row.grid(row=4, column=0, columnspan=3, sticky="w", padx=8, pady=(2, 8)) self.export_media_check = ttk.Checkbutton( @@ -370,13 +388,40 @@ def _on_out_dir_changed(self, *_args: object) -> None: def _refresh_source_fields(self) -> None: """Update source-specific labels and field visibility.""" - source = self.source_var.get() + source = self._selected_source() if source == SOURCE_APP_THREEMA: self.input_label.configure(text="Threema SQLite DB") self.source_hint_var.set("Threema: select ThreemaData.sqlite. _EXTERNAL_DATA is optional but recommended.") self.external_label.grid(row=3, column=0, sticky="w", padx=(8, 8), pady=6) self.external_entry.grid(row=3, column=1, sticky="ew", padx=(0, 8), pady=6) self.external_button.grid(row=3, column=2, sticky="ew", padx=(0, 8), pady=6) + self.backup_password_label.grid_remove() + self.backup_password_entry.grid_remove() + self.chat_text_label.grid_remove() + self.chat_text_entry.grid_remove() + elif source == SOURCE_APP_THREEMA_ANDROID_BACKUP: + self.input_label.configure(text="Threema Android backup") + self.source_hint_var.set( + "Threema Android: select an encrypted data-backup ZIP and enter its password." + ) + self.external_label.grid_remove() + self.external_entry.grid_remove() + self.external_button.grid_remove() + self.backup_password_label.grid( + row=3, + column=0, + sticky="w", + padx=(8, 8), + pady=6, + ) + self.backup_password_entry.grid( + row=3, + column=1, + columnspan=2, + sticky="ew", + padx=(0, 8), + pady=6, + ) self.chat_text_label.grid_remove() self.chat_text_entry.grid_remove() else: @@ -385,6 +430,8 @@ def _refresh_source_fields(self) -> None: self.external_label.grid_remove() self.external_entry.grid_remove() self.external_button.grid_remove() + self.backup_password_label.grid_remove() + self.backup_password_entry.grid_remove() self.chat_text_label.grid() self.chat_text_entry.grid() @@ -402,7 +449,7 @@ def _toggle_advanced(self) -> None: def _browse_input_path(self) -> None: """Open the source-specific file picker.""" - source = self.source_var.get() + source = self._selected_source() if source == SOURCE_APP_THREEMA: filetypes = [ ("SQLite database", "*.sqlite"), @@ -553,7 +600,7 @@ def _build_config(self) -> ExportConfig: Raises: ValueError: If required fields are missing or invalid. """ - source = self.source_var.get().strip() + source = self._selected_source() input_path = self.input_path_var.get().strip() out_dir = self.out_dir_var.get().strip() if not source: @@ -568,6 +615,7 @@ def _build_config(self) -> ExportConfig: source_app=source, input_path=input_path, chat_text_name=self.chat_text_name_var.get(), + backup_password=self.backup_password_var.get(), external_folder=self.external_folder_var.get(), tz_name=self.tz_var.get(), export_media=self.export_media_var.get(), @@ -591,6 +639,11 @@ def _build_config(self) -> ExportConfig: case_description=self.case_description_var.get(), ) + def _selected_source(self) -> str: + """Return the importer key selected in the source combobox.""" + selected = self.source_var.get().strip() + return SOURCE_KEYS_BY_LABEL.get(selected, selected) + def _set_running(self, running: bool) -> None: """Enable or disable controls during export execution. @@ -607,6 +660,7 @@ def _set_running(self, running: bool) -> None: self.output_browse_button, self.external_entry, self.external_button, + self.backup_password_entry, self.chat_text_entry, self.case_number_entry, self.examiner_entry, @@ -644,7 +698,9 @@ def _run_export_worker(self, cfg: ExportConfig) -> None: ) result = export_all_conversations(cfg) except Exception as exc: - logging.getLogger("chat_export.gui").exception("GUI export failed") + gui_log = logging.getLogger("chat_export.gui") + gui_log.error("GUI export failed: %s", exc) + gui_log.debug("GUI export failure details", exc_info=True) self._result_queue.put( GuiResult( ok=False, diff --git a/src/chat_export/orchestrator.py b/src/chat_export/orchestrator.py index 5d74c5b..4ee6a77 100644 --- a/src/chat_export/orchestrator.py +++ b/src/chat_export/orchestrator.py @@ -9,7 +9,11 @@ from .common.logging_setup import sanitize_local_paths from .common.util import ensure_dir, safe_filename from .config import ExportConfig -from .constants import SOURCE_APP_THREEMA, SOURCE_APP_WHATSAPP +from .constants import ( + SOURCE_APP_THREEMA, + SOURCE_APP_THREEMA_ANDROID_BACKUP, + SOURCE_APP_WHATSAPP, +) from .export_summary import ( EXPORT_SUMMARY_FILENAME, MANIFEST_FILENAME, @@ -21,6 +25,7 @@ from .render.pdf_builder import build_conversation_pdf, build_fallback_tech_pdf from .threema.importer import ThreemaImporter from .threema.tech_pdf import build_threema_tech_pdf +from .threema_android.importer import ThreemaAndroidBackupImporter from .whatsapp.importer import WhatsAppImporter log = logging.getLogger(__name__) @@ -28,6 +33,7 @@ IMPORTERS: Dict[str, ConversationImporter] = { SOURCE_APP_THREEMA: ThreemaImporter(), + SOURCE_APP_THREEMA_ANDROID_BACKUP: ThreemaAndroidBackupImporter(), SOURCE_APP_WHATSAPP: WhatsAppImporter(), } diff --git a/src/chat_export/render/pdf_builder.py b/src/chat_export/render/pdf_builder.py index 4fd51dd..7573118 100644 --- a/src/chat_export/render/pdf_builder.py +++ b/src/chat_export/render/pdf_builder.py @@ -200,7 +200,7 @@ def _case_summary(conversation: NormalizedConversation) -> dict[str, int]: if message.message_type == "text": counts["text"] += 1 for attachment in message.attachments: - if attachment.absolute_path and attachment.kind in counts: + if attachment.kind in counts: counts[attachment.kind] += 1 return counts @@ -421,25 +421,35 @@ def _attachment_filename(attachment: NormalizedAttachment) -> str: return os.path.basename(attachment.absolute_path) return "attachment" + def _attachment_status(attachment: NormalizedAttachment) -> str: + metadata = attachment.metadata + if metadata.get("skipped_due_to_limit"): + return "not exported (size limit)" + if metadata.get("missing_from_backup") or metadata.get("missing_from_zip"): + return "not available in source" + return "not exported" + def _attachment_link( attachment: NormalizedAttachment, *, style=normal, - ) -> Paragraph | None: - """Build one attachment link paragraph. + ) -> Paragraph: + """Build one attachment paragraph. Args: attachment (NormalizedAttachment): Normalized attachment. style: ReportLab paragraph style. Returns: - Paragraph | None: Link paragraph if the attachment has a file path. + Paragraph: Linked filename or an unavailable-file status. """ - if not attachment.absolute_path: - return None - return link( - f"Attachment ({attachment.kind}) [{_attachment_filename(attachment)}]", - rel(attachment.absolute_path), + label = ( + f"Attachment ({attachment.kind}) [{_attachment_filename(attachment)}]" + ) + if attachment.absolute_path: + return link(label, rel(attachment.absolute_path), style) + return p( + f"{esc_xml(label)} - {_attachment_status(attachment)}", style, ) @@ -568,25 +578,25 @@ def attachment_index_table(): p("Sender"), p("Type"), p("Filename"), - p("Path"), + p("Path / status"), p("SHA256"), ] ] for message in conversation.messages: for attachment in message.attachments: - if not attachment.absolute_path: - continue - filename = attachment.filename or os.path.basename( - attachment.absolute_path + path_or_status = ( + link("open", rel(attachment.absolute_path), normal) + if attachment.absolute_path + else p(f"{_attachment_status(attachment)}", normal) ) data.append( [ p(esc_xml(message.timestamp or "NULL")), p(esc_xml(message.sender_display)), p(esc_xml(attachment.kind)), - p(esc_xml(filename)), - link("open", rel(attachment.absolute_path), normal), + p(esc_xml(_attachment_filename(attachment))), + path_or_status, p(esc_xml(attachment.sha256 or "NULL"), mono), ] ) @@ -665,8 +675,6 @@ def append_message_linear(story: list[object], message: NormalizedMessage): for attachment in message.attachments: attachment_link = _attachment_link(attachment, style=normal) - if attachment_link is None: - continue story.append(attachment_link) preview = _image_preview_or_none(message, attachment) if preview is not None: @@ -852,8 +860,7 @@ def append_message_bubble(story: list[object], message: NormalizedMessage): if preview is not None: bubble_rows.append([preview]) attachment_link = _attachment_link(attachment, style=bubble_aux) - if attachment_link is not None: - attachment_links.append(attachment_link) + attachment_links.append(attachment_link) reaction_summary = _reaction_summary(message) if reaction_summary: diff --git a/src/chat_export/threema_android/__init__.py b/src/chat_export/threema_android/__init__.py new file mode 100644 index 0000000..8ab48b6 --- /dev/null +++ b/src/chat_export/threema_android/__init__.py @@ -0,0 +1 @@ +"""Import encrypted Threema Android data backups.""" diff --git a/src/chat_export/threema_android/archive.py b/src/chat_export/threema_android/archive.py new file mode 100644 index 0000000..ff44293 --- /dev/null +++ b/src/chat_export/threema_android/archive.py @@ -0,0 +1,177 @@ +"""Access encrypted Threema Android backup members without extracting them.""" + +from __future__ import annotations + +import csv +import io +import zipfile +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import PurePosixPath +from typing import BinaryIO, Iterator + +import pyzipper + + +SUPPORTED_BACKUP_VERSION = 27 + + +class BackupFormatError(ValueError): + """Raised when an archive is not a supported Threema data backup.""" + + +@dataclass(slots=True) +class CsvRows: + rows: list[dict[str, str]] + unparseable_count: int = 0 + + +def _decode_backup_value(value: str | None) -> str: + if value is None: + return "" + return value.replace("\\\\", "\\") + + +class ThreemaAndroidArchive: + """Provide password-protected, streaming access to one data backup.""" + + def __init__(self, path: str, password: str) -> None: + self.path = path + self._password = password.encode("utf-8") + self._archive: pyzipper.AESZipFile | None = None + self._members: dict[str, zipfile.ZipInfo] = {} + self.version = 0 + + def __enter__(self) -> "ThreemaAndroidArchive": + try: + self._archive = pyzipper.AESZipFile(self.path, "r") + self._archive.setpassword(self._password) + for info in self._archive.infolist(): + name = info.filename.replace("\\", "/") + if info.is_dir(): + continue + if PurePosixPath(name).name != name: + raise BackupFormatError( + "Threema Android backups must contain flat archive members" + ) + if name in self._members: + raise BackupFormatError(f"Duplicate archive member: {name}") + self._members[name] = info + self.version = self._read_version() + except BackupFormatError as exc: + self.close() + if str(exc).startswith("Unable to read archive member"): + raise BackupFormatError( + "Unable to open the Threema Android backup. " + "Check the backup file and password." + ) from exc + raise + except (OSError, RuntimeError, ValueError, zipfile.BadZipFile) as exc: + self.close() + raise BackupFormatError( + "Unable to open the Threema Android backup. " + "Check the backup file and password." + ) from exc + return self + + def __exit__(self, *_args: object) -> None: + self.close() + + def close(self) -> None: + if self._archive is not None: + self._archive.close() + self._archive = None + + @property + def member_names(self) -> tuple[str, ...]: + return tuple(self._members) + + def has_member(self, name: str) -> bool: + return name in self._members + + def member_size(self, name: str) -> int | None: + info = self._members.get(name) + return info.file_size if info is not None else None + + @contextmanager + def open_member(self, name: str) -> Iterator[BinaryIO]: + if self._archive is None: + raise RuntimeError("Backup archive is not open") + info = self._members.get(name) + if info is None: + raise BackupFormatError(f"Required archive member is missing: {name}") + try: + stream = self._archive.open(info, "r", pwd=self._password) + except (OSError, RuntimeError, ValueError, zipfile.BadZipFile) as exc: + raise BackupFormatError( + f"Unable to read archive member: {name}" + ) from exc + try: + yield stream + except (OSError, RuntimeError, ValueError, zipfile.BadZipFile) as exc: + raise BackupFormatError( + f"Unable to read archive member: {name}" + ) from exc + finally: + stream.close() + + def read_text(self, name: str) -> str: + with self.open_member(name) as stream: + try: + return stream.read().decode("utf-8-sig") + except UnicodeDecodeError as exc: + raise BackupFormatError( + f"Archive member is not valid UTF-8 text: {name}" + ) from exc + + def read_csv( + self, + name: str, + *, + required_fields: tuple[str, ...] = (), + ) -> CsvRows: + rows: list[dict[str, str]] = [] + unparseable_count = 0 + with self.open_member(name) as binary_stream: + text_stream = io.TextIOWrapper(binary_stream, encoding="utf-8-sig", newline="") + reader = csv.DictReader(text_stream, strict=True) + fields = tuple(reader.fieldnames or ()) + missing_fields = [field for field in required_fields if field not in fields] + if missing_fields: + raise BackupFormatError( + f"Archive member {name} is missing CSV fields: " + + ", ".join(missing_fields) + ) + try: + for row in reader: + if None in row or any(value is None for value in row.values()): + unparseable_count += 1 + continue + decoded = { + str(key): _decode_backup_value(value) + for key, value in row.items() + } + if not any(decoded.values()): + continue + rows.append(decoded) + except (csv.Error, UnicodeDecodeError): + unparseable_count += 1 + return CsvRows(rows=rows, unparseable_count=unparseable_count) + + def _read_version(self) -> int: + if "settings" not in self._members: + raise BackupFormatError("Required archive member is missing: settings") + settings = self.read_text("settings") + try: + for row in csv.reader(io.StringIO(settings), strict=True): + if len(row) == 2 and row[0] == "version": + version = int(row[1]) + if version != SUPPORTED_BACKUP_VERSION: + raise BackupFormatError( + "Unsupported Threema Android backup version: " + f"{version}. Supported version: {SUPPORTED_BACKUP_VERSION}" + ) + return version + except (csv.Error, ValueError) as exc: + raise BackupFormatError("Invalid Threema backup settings") from exc + raise BackupFormatError("Threema backup version is missing") diff --git a/src/chat_export/threema_android/identity.py b/src/chat_export/threema_android/identity.py new file mode 100644 index 0000000..6a621cb --- /dev/null +++ b/src/chat_export/threema_android/identity.py @@ -0,0 +1,138 @@ +"""Read the owner identity from a Threema identity backup.""" + +from __future__ import annotations + +import base64 +import hashlib +import hmac +import struct + + +class IdentityBackupError(ValueError): + """Raised when a Threema identity backup cannot be decoded.""" + + +def _rotate_left(value: int, count: int) -> int: + return ((value << count) | (value >> (32 - count))) & 0xFFFFFFFF + + +def _salsa20_rounds(state: list[int]) -> list[int]: + x = state.copy() + for _ in range(10): + x[4] ^= _rotate_left((x[0] + x[12]) & 0xFFFFFFFF, 7) + x[8] ^= _rotate_left((x[4] + x[0]) & 0xFFFFFFFF, 9) + x[12] ^= _rotate_left((x[8] + x[4]) & 0xFFFFFFFF, 13) + x[0] ^= _rotate_left((x[12] + x[8]) & 0xFFFFFFFF, 18) + x[9] ^= _rotate_left((x[5] + x[1]) & 0xFFFFFFFF, 7) + x[13] ^= _rotate_left((x[9] + x[5]) & 0xFFFFFFFF, 9) + x[1] ^= _rotate_left((x[13] + x[9]) & 0xFFFFFFFF, 13) + x[5] ^= _rotate_left((x[1] + x[13]) & 0xFFFFFFFF, 18) + x[14] ^= _rotate_left((x[10] + x[6]) & 0xFFFFFFFF, 7) + x[2] ^= _rotate_left((x[14] + x[10]) & 0xFFFFFFFF, 9) + x[6] ^= _rotate_left((x[2] + x[14]) & 0xFFFFFFFF, 13) + x[10] ^= _rotate_left((x[6] + x[2]) & 0xFFFFFFFF, 18) + x[3] ^= _rotate_left((x[15] + x[11]) & 0xFFFFFFFF, 7) + x[7] ^= _rotate_left((x[3] + x[15]) & 0xFFFFFFFF, 9) + x[11] ^= _rotate_left((x[7] + x[3]) & 0xFFFFFFFF, 13) + x[15] ^= _rotate_left((x[11] + x[7]) & 0xFFFFFFFF, 18) + + x[1] ^= _rotate_left((x[0] + x[3]) & 0xFFFFFFFF, 7) + x[2] ^= _rotate_left((x[1] + x[0]) & 0xFFFFFFFF, 9) + x[3] ^= _rotate_left((x[2] + x[1]) & 0xFFFFFFFF, 13) + x[0] ^= _rotate_left((x[3] + x[2]) & 0xFFFFFFFF, 18) + x[6] ^= _rotate_left((x[5] + x[4]) & 0xFFFFFFFF, 7) + x[7] ^= _rotate_left((x[6] + x[5]) & 0xFFFFFFFF, 9) + x[4] ^= _rotate_left((x[7] + x[6]) & 0xFFFFFFFF, 13) + x[5] ^= _rotate_left((x[4] + x[7]) & 0xFFFFFFFF, 18) + x[11] ^= _rotate_left((x[10] + x[9]) & 0xFFFFFFFF, 7) + x[8] ^= _rotate_left((x[11] + x[10]) & 0xFFFFFFFF, 9) + x[9] ^= _rotate_left((x[8] + x[11]) & 0xFFFFFFFF, 13) + x[10] ^= _rotate_left((x[9] + x[8]) & 0xFFFFFFFF, 18) + x[12] ^= _rotate_left((x[15] + x[14]) & 0xFFFFFFFF, 7) + x[13] ^= _rotate_left((x[12] + x[15]) & 0xFFFFFFFF, 9) + x[14] ^= _rotate_left((x[13] + x[12]) & 0xFFFFFFFF, 13) + x[15] ^= _rotate_left((x[14] + x[13]) & 0xFFFFFFFF, 18) + return x + + +def _salsa_state(key: bytes, nonce: bytes, counter: int = 0) -> list[int]: + constants = struct.unpack("<4I", b"expand 32-byte k") + key_words = struct.unpack("<8I", key) + nonce_words = struct.unpack("<2I", nonce) + return [ + constants[0], + *key_words[:4], + constants[1], + *nonce_words, + counter & 0xFFFFFFFF, + (counter >> 32) & 0xFFFFFFFF, + constants[2], + *key_words[4:], + constants[3], + ] + + +def _hsalsa20(key: bytes, nonce: bytes) -> bytes: + constants = struct.unpack("<4I", b"expand 32-byte k") + key_words = struct.unpack("<8I", key) + nonce_words = struct.unpack("<4I", nonce) + state = [ + constants[0], + *key_words[:4], + constants[1], + *nonce_words, + constants[2], + *key_words[4:], + constants[3], + ] + x = _salsa20_rounds(state) + return struct.pack("<8I", x[0], x[5], x[10], x[15], x[6], x[7], x[8], x[9]) + + +def _xsalsa20_xor(data: bytes, key: bytes, nonce: bytes) -> bytes: + subkey = _hsalsa20(key, nonce[:16]) + tail_nonce = nonce[16:] + output = bytearray(len(data)) + for offset in range(0, len(data), 64): + state = _salsa_state(subkey, tail_nonce, offset // 64) + rounded = _salsa20_rounds(state) + block = struct.pack( + "<16I", + *((rounded[index] + state[index]) & 0xFFFFFFFF for index in range(16)), + ) + chunk = data[offset : offset + 64] + for index, value in enumerate(chunk): + output[offset + index] = value ^ block[index] + return bytes(output) + + +def decrypt_owner_identity(encrypted_backup: str, password: str) -> str: + """Decrypt the legacy identity entry and return only the Threema ID.""" + compact = "".join(encrypted_backup.split()).replace("-", "") + try: + encoded = base64.b32decode(compact, casefold=True) + except Exception as exc: + raise IdentityBackupError("Invalid identity backup encoding") from exc + if len(encoded) != 50: + raise IdentityBackupError("Unsupported identity backup format") + + salt, ciphertext = encoded[:8], encoded[8:] + key = hashlib.pbkdf2_hmac( + "sha256", + password.encode("utf-8"), + salt, + 100_000, + dklen=32, + ) + plaintext = _xsalsa20_xor(ciphertext, key, bytes(24)) + backup_data, stored_digest = plaintext[:40], plaintext[40:] + expected_digest = hashlib.sha256(backup_data).digest()[:2] + if not hmac.compare_digest(stored_digest, expected_digest): + raise IdentityBackupError("Identity backup decryption failed") + try: + identity = backup_data[:8].decode("ascii") + except UnicodeDecodeError as exc: + raise IdentityBackupError("Invalid Threema ID in identity backup") from exc + if len(identity) != 8 or not identity.isalnum(): + raise IdentityBackupError("Invalid Threema ID in identity backup") + return identity diff --git a/src/chat_export/threema_android/importer.py b/src/chat_export/threema_android/importer.py new file mode 100644 index 0000000..a6f83dd --- /dev/null +++ b/src/chat_export/threema_android/importer.py @@ -0,0 +1,631 @@ +"""Import encrypted Threema Android data-backup ZIP files.""" + +from __future__ import annotations + +import logging +import os +import re +from typing import Any + +from ..common.timeutil import TimeMode +from ..common.util import safe_filename +from ..config import ExportConfig +from ..constants import SOURCE_APP_THREEMA_ANDROID_BACKUP +from ..importers.base import ImportRun, ImportedConversation +from .archive import BackupFormatError, CsvRows, ThreemaAndroidArchive +from .identity import IdentityBackupError, decrypt_owner_identity +from .media import AttachmentOutcome, prepare_attachment +from .models import ( + AndroidContact, + AndroidConversationSpec, + AndroidDistributionList, + AndroidGroup, + parse_bool, +) +from .normalize import normalize_conversation + +log = logging.getLogger(__name__) + +DIRECT_MESSAGE_RE = re.compile(r"^message_(.+)\.csv$") +GROUP_MESSAGE_RE = re.compile(r"^group_message_(.+)\.csv$") +DISTRIBUTION_MESSAGE_RE = re.compile(r"^distribution_list_message_(.+)\.csv$") +FILE_MESSAGE_TYPES = {"FILE", "IMAGE", "VIDEO", "VOICEMESSAGE"} + + +def _optional_csv( + archive: ThreemaAndroidArchive, + name: str, + *, + required_fields: tuple[str, ...] = (), +) -> CsvRows: + if not archive.has_member(name): + return CsvRows([]) + return archive.read_csv(name, required_fields=required_fields) + + +def _load_contacts( + archive: ThreemaAndroidArchive, +) -> tuple[list[AndroidContact], int]: + data = archive.read_csv( + "contacts.csv", + required_fields=("identity", "firstname", "lastname"), + ) + contacts = [AndroidContact.from_row(row) for row in data.rows] + return contacts, data.unparseable_count + + +def _load_groups( + archive: ThreemaAndroidArchive, +) -> tuple[list[AndroidGroup], int]: + data = _optional_csv( + archive, + "groups.csv", + required_fields=("id", "creator", "groupname", "members"), + ) + groups = [ + AndroidGroup.from_row(row) + for row in data.rows + if not parse_bool(row.get("deleted")) + ] + return groups, data.unparseable_count + + +def _load_distribution_lists( + archive: ThreemaAndroidArchive, +) -> tuple[list[AndroidDistributionList], int]: + data = _optional_csv( + archive, + "distribution_list.csv", + required_fields=("id", "distribution_list_name", "distribution_members"), + ) + return ( + [AndroidDistributionList.from_row(row) for row in data.rows], + data.unparseable_count, + ) + + +def _build_specs( + archive: ThreemaAndroidArchive, + contacts: list[AndroidContact], + groups: list[AndroidGroup], + distribution_lists: list[AndroidDistributionList], + self_identity: str, +) -> list[AndroidConversationSpec]: + names = set(archive.member_names) + specs: list[AndroidConversationSpec] = [] + used_members: set[str] = set() + + contacts_by_backup_id = {contact.backup_id: contact for contact in contacts} + contacts_by_identity = {contact.identity: contact for contact in contacts} + for contact in contacts: + candidates = ( + f"message_{contact.backup_id}.csv", + f"message_{contact.identity}.csv", + ) + member = next((name for name in candidates if name in names), None) + if member is None: + continue + specs.append( + AndroidConversationSpec( + conversation_id=f"threema-android:direct:{contact.identity}", + title=contact.display_name(), + conversation_type="direct", + message_member=member, + media_prefix="message_media_", + thumbnail_prefix="message_thumbnail_", + member_identities=(contact.identity, self_identity), + direct_contact=contact, + metadata={ + "contact_identity": contact.identity, + "contact_backup_id": contact.backup_id, + }, + ) + ) + used_members.add(member) + + for group in groups: + fallback_uid = f"{group.api_group_id}-{group.creator_identity}" + group_uid = group.group_uid or fallback_uid + candidates = ( + f"group_message_{group_uid}.csv", + f"group_message_{fallback_uid}.csv", + ) + member = next((name for name in candidates if name in names), None) + if member is None: + continue + title = group.name.strip() or f"Group {group.api_group_id}" + specs.append( + AndroidConversationSpec( + conversation_id=( + f"threema-android:group:{group.creator_identity}:" + f"{group.api_group_id}" + ), + title=title, + conversation_type="group", + message_member=member, + media_prefix="group_message_media_", + thumbnail_prefix="group_message_thumbnail_", + member_identities=tuple( + dict.fromkeys((*group.members, group.creator_identity)) + ), + group=group, + metadata={ + "api_group_id": group.api_group_id, + "group_creator_identity": group.creator_identity, + "group_backup_id": group_uid, + "created_at_raw": group.created_at or None, + "last_update_raw": group.last_update or None, + "archived": group.archived, + "description": group.description or None, + "description_timestamp_raw": ( + group.description_timestamp or None + ), + "user_state": group.user_state, + }, + ) + ) + used_members.add(member) + + for distribution_list in distribution_lists: + member = f"distribution_list_message_{distribution_list.list_id}.csv" + if member not in names: + continue + title = distribution_list.name.strip() or ( + f"Distribution list {distribution_list.list_id}" + ) + specs.append( + AndroidConversationSpec( + conversation_id=( + f"threema-android:distribution-list:{distribution_list.list_id}" + ), + title=title, + conversation_type="distribution_list", + message_member=member, + media_prefix="distribution_list_message_media_", + thumbnail_prefix="distribution_list_thumbnail_", + member_identities=distribution_list.members, + distribution_list=distribution_list, + metadata={ + "distribution_list_id": distribution_list.list_id, + "created_at_raw": distribution_list.created_at or None, + "last_update_raw": distribution_list.last_update or None, + "archived": distribution_list.archived, + }, + ) + ) + used_members.add(member) + + for name in sorted(names - used_members): + direct_match = DIRECT_MESSAGE_RE.match(name) + group_match = GROUP_MESSAGE_RE.match(name) + distribution_match = DISTRIBUTION_MESSAGE_RE.match(name) + if group_match or distribution_match or not direct_match: + continue + backup_id = direct_match.group(1) + contact = contacts_by_backup_id.get(backup_id) or contacts_by_identity.get( + backup_id + ) + identity = contact.identity if contact else backup_id + specs.append( + AndroidConversationSpec( + conversation_id=f"threema-android:direct:{identity}", + title=contact.display_name() if contact else identity, + conversation_type="direct", + message_member=name, + media_prefix="message_media_", + thumbnail_prefix="message_thumbnail_", + member_identities=(identity, self_identity), + direct_contact=contact, + metadata={ + "contact_identity": identity, + "contact_backup_id": backup_id, + }, + ) + ) + return specs + + +def _load_polls( + archive: ThreemaAndroidArchive, +) -> tuple[dict[str, dict[str, Any]], int]: + poll_data = _optional_csv(archive, "ballot.csv", required_fields=("id", "name")) + choice_data = _optional_csv( + archive, + "ballot_choice.csv", + required_fields=("ballot", "name"), + ) + vote_data = _optional_csv( + archive, + "ballot_vote.csv", + required_fields=("ballot_uid", "identity"), + ) + polls: dict[str, dict[str, Any]] = {} + poll_uids: dict[str, str] = {} + for row in poll_data.rows: + poll_id = row.get("id", "") + poll = { + "id": poll_id, + "api_id": row.get("aid") or None, + "creator_identity": row.get("creator") or None, + "name": row.get("name") or None, + "state": row.get("state") or None, + "assessment": row.get("assessment") or None, + "type": row.get("type") or None, + "choice_type": row.get("choice_type") or None, + "reference_type": row.get("ref") or None, + "reference_id": row.get("ref_id") or None, + "created_at_raw": row.get("created_at") or None, + "modified_at_raw": row.get("modified_at") or None, + "choices": [], + "votes": [], + } + polls[poll_id] = poll + poll_uids[f"{row.get('aid', '')}-{row.get('creator', '')}"] = poll_id + + def resolve_poll_id(uid: str) -> str | None: + if uid in poll_uids: + return poll_uids[uid] + if uid in polls: + return uid + if len(polls) == 1: + return next(iter(polls)) + return None + + for row in choice_data.rows: + poll_id = resolve_poll_id(row.get("ballot", "")) + if poll_id: + polls[poll_id]["choices"].append( + { + "api_id": row.get("aid") or None, + "name": row.get("name") or None, + "vote_count": row.get("vote_count") or None, + "order": row.get("order") or None, + "created_at_raw": row.get("created_at") or None, + "modified_at_raw": row.get("modified_at") or None, + } + ) + for row in vote_data.rows: + poll_id = resolve_poll_id(row.get("ballot_uid", "")) + if poll_id: + polls[poll_id]["votes"].append( + { + "choice_uid": row.get("choice_uid") or None, + "identity": row.get("identity") or None, + "choice": row.get("choice") or None, + "created_at_raw": row.get("created_at") or None, + "modified_at_raw": row.get("modified_at") or None, + } + ) + unparseable = ( + poll_data.unparseable_count + + choice_data.unparseable_count + + vote_data.unparseable_count + ) + return polls, unparseable + + +def _load_reactions( + archive: ThreemaAndroidArchive, +) -> tuple[ + dict[tuple[str, str], list[dict[str, str]]], + dict[tuple[str, str, str], list[dict[str, str]]], + int, +]: + contact_data = _optional_csv( + archive, + "contact_reactions.csv", + required_fields=("identity", "api_message_id", "sender_identity"), + ) + group_data = _optional_csv( + archive, + "group_reactions.csv", + required_fields=( + "api_group_id", + "group_creator_identity", + "api_message_id", + "sender_identity", + ), + ) + contacts: dict[tuple[str, str], list[dict[str, str]]] = {} + groups: dict[tuple[str, str, str], list[dict[str, str]]] = {} + for row in contact_data.rows: + key = (row.get("identity", ""), row.get("api_message_id", "")) + contacts.setdefault(key, []).append(row) + for row in group_data.rows: + key = ( + row.get("api_group_id", ""), + row.get("group_creator_identity", ""), + row.get("api_message_id", ""), + ) + groups.setdefault(key, []).append(row) + return ( + contacts, + groups, + contact_data.unparseable_count + group_data.unparseable_count, + ) + + +def _conversation_reactions( + spec: AndroidConversationSpec, + rows: list[dict[str, str]], + contact_reactions: dict[tuple[str, str], list[dict[str, str]]], + group_reactions: dict[tuple[str, str, str], list[dict[str, str]]], +) -> dict[str, list[dict[str, str]]]: + result: dict[str, list[dict[str, str]]] = {} + for row in rows: + api_id = row.get("apiid", "") + if not api_id: + continue + if spec.direct_contact: + values = contact_reactions.get((spec.direct_contact.identity, api_id), []) + elif spec.group: + values = group_reactions.get( + ( + spec.group.api_group_id, + spec.group.creator_identity, + api_id, + ), + [], + ) + else: + values = [] + if values: + result[api_id] = values + return result + + +class ThreemaAndroidBackupImporter: + """Import current Threema Android CSV data backups.""" + + source_app = SOURCE_APP_THREEMA_ANDROID_BACKUP + + def load_conversations(self, cfg: ExportConfig) -> ImportRun: + input_path = cfg.resolved_input_path() + if not cfg.backup_password: + raise ValueError("A backup password is required") + + log.info("Loading Threema Android data backup") + log.debug( + "Loading Threema Android data backup archive=%s", + os.path.basename(input_path), + ) + with ThreemaAndroidArchive(input_path, cfg.backup_password) as archive: + if not archive.has_member("identity"): + raise BackupFormatError("Required archive member is missing: identity") + if not archive.has_member("contacts.csv"): + raise BackupFormatError("Required archive member is missing: contacts.csv") + try: + self_identity = decrypt_owner_identity( + archive.read_text("identity"), cfg.backup_password + ) + except IdentityBackupError as exc: + raise BackupFormatError( + "Unable to determine the owner identity from the backup" + ) from exc + + contacts, contact_unparseable = _load_contacts(archive) + groups, group_unparseable = _load_groups(archive) + distribution_lists, distribution_unparseable = ( + _load_distribution_lists(archive) + ) + polls, poll_unparseable = _load_polls(archive) + contact_reactions, group_reactions, reaction_unparseable = ( + _load_reactions(archive) + ) + unparseable_count = ( + contact_unparseable + + group_unparseable + + distribution_unparseable + + poll_unparseable + + reaction_unparseable + ) + contacts_by_identity = {contact.identity: contact for contact in contacts} + specs = _build_specs( + archive, + contacts, + groups, + distribution_lists, + self_identity, + ) + log.debug( + "Loaded Threema Android backup metadata version=%s contacts=%s " + "groups=%s distribution_lists=%s conversation_files=%s", + archive.version, + len(contacts), + len(groups), + len(distribution_lists), + len(specs), + ) + + imported: list[ImportedConversation] = [] + total_missing = 0 + total_skipped = 0 + skipped_empty = 0 + malformed_file_messages = 0 + media_root = os.path.join(os.path.abspath(cfg.out_dir), "media") + + for spec in specs: + if cfg.limit_conversations and len(imported) >= cfg.limit_conversations: + break + message_data = archive.read_csv( + spec.message_member, + required_fields=( + "apiid", + "uid", + "isoutbox", + "posted_at", + "type", + "body", + ), + ) + unparseable_count += message_data.unparseable_count + if not message_data.rows: + skipped_empty += 1 + log.debug( + "Skipping empty Threema Android conversation member=%s", + spec.message_member, + ) + continue + rows = message_data.rows + if cfg.limit_messages: + rows = rows[: cfg.limit_messages] + media_dir = ( + os.path.join( + media_root, + safe_filename(spec.conversation_id, max_len=70) + + "_" + + safe_filename(spec.title, max_len=60), + ) + if cfg.export_media + else None + ) + attachment_outcomes: dict[str, AttachmentOutcome] = {} + for row in rows: + if (row.get("type") or "").upper() not in FILE_MESSAGE_TYPES: + continue + outcome = prepare_attachment( + archive, + spec, + row, + cfg, + media_dir, + ) + uid = ( + row.get("uid") + or row.get("apiid") + or str(len(attachment_outcomes)) + ) + attachment_outcomes[uid] = outcome + + missing_count = sum( + 1 for outcome in attachment_outcomes.values() if outcome.missing + ) + skipped_count = sum( + 1 for outcome in attachment_outcomes.values() if outcome.skipped + ) + malformed_count = sum( + 1 for outcome in attachment_outcomes.values() if outcome.malformed + ) + total_missing += missing_count + total_skipped += skipped_count + malformed_file_messages += malformed_count + if missing_count: + log.warning( + "Threema Android conversation has attachment payloads " + "missing from the backup count=%s conversation_id=%s", + missing_count, + spec.conversation_id, + ) + if skipped_count: + log.warning( + "Skipped Threema Android attachments due to size limit " + "count=%s conversation_id=%s max_media_bytes=%s", + skipped_count, + spec.conversation_id, + cfg.max_media_bytes, + ) + if malformed_count: + log.warning( + "Unable to parse Threema Android file-message metadata " + "count=%s conversation_id=%s", + malformed_count, + spec.conversation_id, + ) + + reactions = _conversation_reactions( + spec, + rows, + contact_reactions, + group_reactions, + ) + conversation = normalize_conversation( + spec=spec, + rows=rows, + contacts=contacts_by_identity, + self_identity=self_identity, + tz_name=cfg.tz_name, + backup_version=archive.version, + attachments=attachment_outcomes, + reactions_by_api_id=reactions, + polls=polls, + ) + imported.append( + ImportedConversation( + conversation=conversation, + tech_payload=None, + tech_renderer=None, + metadata={ + "message_count": len(rows), + "media_dir": ( + os.path.abspath(media_dir) if media_dir else None + ), + "missing_media_count": missing_count, + "skipped_media_count": skipped_count, + "unparseable_line_count": message_data.unparseable_count, + }, + ) + ) + log.info( + "Prepared Threema Android conversation conversation_id=%s " + "messages=%s participants=%s attachments=%s", + conversation.conversation_id, + len(conversation.messages), + len(conversation.participants), + len(attachment_outcomes), + ) + + if unparseable_count: + log.warning( + "Threema Android backup contains unparseable CSV rows count=%s", + unparseable_count, + ) + + media_payload_count = sum( + 1 + for name in archive.member_names + if name.startswith( + ( + "message_media_", + "group_message_media_", + "distribution_list_message_media_", + ) + ) + ) + thumbnail_count = sum( + 1 + for name in archive.member_names + if name.startswith( + ( + "message_thumbnail_", + "group_message_thumbnail_", + "distribution_list_thumbnail_", + ) + ) + ) + metadata = { + "time_mode": TimeMode.UNIX_MS, + "timezone": cfg.tz_name, + "backup_version": archive.version, + "self_identity": self_identity, + "contact_count": len(contacts), + "group_count": len(groups), + "distribution_list_count": len(distribution_lists), + "media_payload_count": media_payload_count, + "thumbnail_count": thumbnail_count, + "skipped_empty_conversation_count": skipped_empty, + "missing_media_count": total_missing, + "skipped_media_count": total_skipped, + "unparseable_line_count": unparseable_count, + "malformed_file_message_count": malformed_file_messages, + } + + log.info( + "Completed Threema Android backup import conversations=%s backup_version=%s", + len(imported), + metadata["backup_version"], + ) + return ImportRun( + source_app=self.source_app, + conversations=imported, + metadata=metadata, + ) diff --git a/src/chat_export/threema_android/media.py b/src/chat_export/threema_android/media.py new file mode 100644 index 0000000..80c234d --- /dev/null +++ b/src/chat_export/threema_android/media.py @@ -0,0 +1,192 @@ +"""Parse and export message attachments from an Android backup.""" + +from __future__ import annotations + +import hashlib +import json +import mimetypes +import os +from dataclasses import dataclass +from typing import Any + +from ..common.util import ensure_dir, safe_filename +from ..config import ExportConfig +from ..normalized.models import NormalizedAttachment +from .archive import ThreemaAndroidArchive +from .models import AndroidConversationSpec, parse_bool, parse_int + + +@dataclass(frozen=True, slots=True) +class FileData: + blob_id: str | None + mime_type: str | None + declared_size: int | None + filename: str | None + rendering_type: int | None + downloaded: bool + caption: str | None + thumbnail_mime_type: str | None + metadata: dict[str, Any] + + +@dataclass(frozen=True, slots=True) +class AttachmentOutcome: + attachment: NormalizedAttachment + missing: bool + skipped: bool + malformed: bool + + +def _display_filename(filename: str | None, uid: str, mime_type: str | None) -> str: + name = (filename or "").replace("\\", "/").rsplit("/", 1)[-1].strip() + if name: + return name + extension = mimetypes.guess_extension(mime_type or "") or "" + return f"attachment_{uid}{extension}" + + +def parse_file_data(body: str) -> FileData | None: + try: + values = json.loads(body) + except (TypeError, ValueError, json.JSONDecodeError): + return None + if not isinstance(values, list) or len(values) < 5: + return None + + metadata = values[9] if len(values) > 9 and isinstance(values[9], dict) else {} + declared_size = parse_int(str(values[3]), -1) if len(values) > 3 else -1 + return FileData( + blob_id=str(values[0]) if values[0] is not None else None, + mime_type=str(values[2]) if len(values) > 2 and values[2] else None, + declared_size=declared_size if declared_size >= 0 else None, + filename=str(values[4]) if len(values) > 4 and values[4] else None, + rendering_type=( + parse_int(str(values[5])) if len(values) > 5 and values[5] is not None else None + ), + downloaded=( + values[6] + if len(values) > 6 and isinstance(values[6], bool) + else parse_bool(str(values[6])) if len(values) > 6 else False + ), + caption=str(values[7]) if len(values) > 7 and values[7] else None, + thumbnail_mime_type=( + str(values[8]) if len(values) > 8 and values[8] else None + ), + metadata=metadata, + ) + + +def _attachment_kind(mime_type: str | None, source_type: str) -> str: + mime = (mime_type or "").lower() + if mime.startswith("image/") or source_type == "IMAGE": + return "image" + if mime.startswith("audio/") or source_type == "VOICEMESSAGE": + return "audio" + if mime.startswith("video/") or source_type == "VIDEO": + return "video" + return "file" + + +def prepare_attachment( + archive: ThreemaAndroidArchive, + spec: AndroidConversationSpec, + row: dict[str, str], + cfg: ExportConfig, + media_dir: str | None, +) -> AttachmentOutcome: + uid = row.get("uid") or row.get("apiid") or "unknown" + source_type = (row.get("type") or "FILE").upper() + file_data = parse_file_data(row.get("body", "")) + malformed = file_data is None + if file_data is None: + file_data = FileData( + blob_id=None, + mime_type=None, + declared_size=None, + filename=None, + rendering_type=None, + downloaded=False, + caption=row.get("caption") or None, + thumbnail_mime_type=None, + metadata={}, + ) + + filename = _display_filename(file_data.filename, uid, file_data.mime_type) + mime_type = file_data.mime_type or mimetypes.guess_type(filename)[0] + media_member = f"{spec.media_prefix}{uid}" + thumbnail_member = f"{spec.thumbnail_prefix}{uid}" + archive_size = archive.member_size(media_member) + missing = archive_size is None + skipped = bool( + archive_size is not None + and cfg.max_media_bytes + and archive_size > cfg.max_media_bytes + ) + + absolute_path = None + relative_path = None + sha256 = None + exported_size = archive_size + if archive_size is not None and not skipped and cfg.export_media and media_dir: + ensure_dir(media_dir) + target_name = safe_filename(f"{uid}_{filename}", max_len=140) + target_path = os.path.join(media_dir, target_name) + digest = hashlib.sha256() + written = 0 + with archive.open_member(media_member) as source, open(target_path, "wb") as target: + while True: + chunk = source.read(1024 * 1024) + if not chunk: + break + target.write(chunk) + digest.update(chunk) + written += len(chunk) + absolute_path = os.path.abspath(target_path) + relative_path = os.path.relpath(absolute_path, os.path.abspath(cfg.out_dir)) + sha256 = digest.hexdigest() + exported_size = written + + metadata = { + "source_type": source_type, + "archive_member": media_member if archive_size is not None else None, + "thumbnail_archive_member": ( + thumbnail_member if archive.has_member(thumbnail_member) else None + ), + "blob_id": file_data.blob_id, + "declared_size": file_data.declared_size, + "archive_size": archive_size, + "declared_size_mismatch": ( + archive_size is not None + and file_data.declared_size is not None + and archive_size != file_data.declared_size + ), + "rendering_type": file_data.rendering_type, + "downloaded": file_data.downloaded, + "thumbnail_mime_type": file_data.thumbnail_mime_type, + "file_metadata": file_data.metadata, + "missing_from_backup": missing, + "skipped_due_to_limit": skipped, + "skip_reason": ( + f"size={archive_size} exceeds max_media_bytes={cfg.max_media_bytes}" + if skipped + else None + ), + "file_data_parse_error": malformed, + } + attachment = NormalizedAttachment( + attachment_id=f"threema-android-attachment:{uid}", + kind=_attachment_kind(mime_type, source_type), + filename=filename, + absolute_path=absolute_path, + relative_path=relative_path, + mime_type=mime_type, + size=exported_size if archive_size is not None else file_data.declared_size, + sha256=sha256, + metadata=metadata, + ) + return AttachmentOutcome( + attachment=attachment, + missing=missing, + skipped=skipped, + malformed=malformed, + ) diff --git a/src/chat_export/threema_android/models.py b/src/chat_export/threema_android/models.py new file mode 100644 index 0000000..8c1e48c --- /dev/null +++ b/src/chat_export/threema_android/models.py @@ -0,0 +1,144 @@ +"""Represent contacts and conversations from an Android data backup.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +def parse_bool(value: str | None) -> bool: + return (value or "").strip().lower() in {"1", "true", "yes"} + + +def parse_int(value: str | None, default: int = 0) -> int: + try: + return int(value or default) + except (TypeError, ValueError): + return default + + +def parse_identities(value: str | None) -> tuple[str, ...]: + return tuple( + dict.fromkeys( + identity.strip() + for identity in (value or "").split(";") + if identity.strip() + ) + ) + + +@dataclass(frozen=True, slots=True) +class AndroidContact: + identity: str + first_name: str + last_name: str + nickname: str + public_key: str + verification: str + android_contact_id: str + last_update: str + hidden: bool + archived: bool + backup_id: str + + @classmethod + def from_row(cls, row: dict[str, str]) -> "AndroidContact": + identity = row.get("identity", "") + return cls( + identity=identity, + first_name=row.get("firstname", ""), + last_name=row.get("lastname", ""), + nickname=row.get("nick_name", ""), + public_key=row.get("publickey", ""), + verification=row.get("verification", ""), + android_contact_id=row.get("acid", ""), + last_update=row.get("last_update", ""), + hidden=parse_bool(row.get("hidden")), + archived=parse_bool(row.get("archived")), + backup_id=row.get("identity_id", "") or identity, + ) + + def display_name(self) -> str: + full_name = " ".join( + value.strip() for value in (self.first_name, self.last_name) if value.strip() + ) + return full_name or self.nickname.strip() or self.identity or "Unknown contact" + + def metadata(self) -> dict[str, Any]: + return { + "public_key": self.public_key or None, + "verification": self.verification or None, + "android_contact_id": self.android_contact_id or None, + "last_update_raw": self.last_update or None, + "nickname": self.nickname or None, + "hidden": self.hidden, + "archived": self.archived, + "backup_contact_id": self.backup_id, + } + + +@dataclass(frozen=True, slots=True) +class AndroidGroup: + api_group_id: str + creator_identity: str + name: str + members: tuple[str, ...] + group_uid: str + created_at: str + last_update: str + archived: bool + description: str + description_timestamp: str + user_state: int + + @classmethod + def from_row(cls, row: dict[str, str]) -> "AndroidGroup": + return cls( + api_group_id=row.get("id", ""), + creator_identity=row.get("creator", ""), + name=row.get("groupname", ""), + members=parse_identities(row.get("members")), + group_uid=row.get("group_uid", ""), + created_at=row.get("created_at", ""), + last_update=row.get("last_update", ""), + archived=parse_bool(row.get("archived")), + description=row.get("groupDesc", ""), + description_timestamp=row.get("groupDescTimestamp", ""), + user_state=parse_int(row.get("user_state")), + ) + + +@dataclass(frozen=True, slots=True) +class AndroidDistributionList: + list_id: str + name: str + members: tuple[str, ...] + created_at: str + last_update: str + archived: bool + + @classmethod + def from_row(cls, row: dict[str, str]) -> "AndroidDistributionList": + return cls( + list_id=row.get("id", ""), + name=row.get("distribution_list_name", ""), + members=parse_identities(row.get("distribution_members")), + created_at=row.get("created_at", ""), + last_update=row.get("last_update", ""), + archived=parse_bool(row.get("archived")), + ) + + +@dataclass(frozen=True, slots=True) +class AndroidConversationSpec: + conversation_id: str + title: str + conversation_type: str + message_member: str + media_prefix: str + thumbnail_prefix: str + member_identities: tuple[str, ...] + direct_contact: AndroidContact | None = None + group: AndroidGroup | None = None + distribution_list: AndroidDistributionList | None = None + metadata: dict[str, Any] = field(default_factory=dict) diff --git a/src/chat_export/threema_android/normalize.py b/src/chat_export/threema_android/normalize.py new file mode 100644 index 0000000..32576fb --- /dev/null +++ b/src/chat_export/threema_android/normalize.py @@ -0,0 +1,457 @@ +"""Normalize Threema Android backup rows for the shared renderers.""" + +from __future__ import annotations + +import json +from typing import Any + +from ..common.timeutil import TimeMode, format_dt +from ..constants import SOURCE_APP_THREEMA_ANDROID_BACKUP +from ..normalized.models import ( + NormalizedConversation, + NormalizedEdit, + NormalizedMessage, + NormalizedParticipant, + NormalizedReaction, +) +from .media import AttachmentOutcome, parse_file_data +from .models import AndroidContact, AndroidConversationSpec, parse_bool, parse_int + + +GROUP_STATUS_LABELS = { + 0: "Group created", + 1: "Group renamed", + 2: "Group picture updated", + 3: "Group member added", + 4: "Group member left", + 5: "Group member removed", + 6: "Group changed to notes group", + 7: "Group changed to people group", + 8: "First poll vote received", + 9: "Poll vote changed", + 10: "Anonymous poll vote received", + 11: "Poll voting completed", + 12: "Group description changed", + 13: "Group creator left", +} +CALL_STATUS_LABELS = { + 1: "Missed call", + 2: "Finished call", + 3: "Rejected call", + 4: "Aborted call", +} +GROUP_CALL_STATUS_LABELS = {1: "Group call started", 2: "Group call ended"} +POLL_EVENT_LABELS = {1: "Poll created", 2: "Poll modified", 3: "Poll closed"} + + +def _participant_id(identity: str) -> str: + return f"threema-id:{identity}" + + +def _display_name( + identity: str, + contacts: dict[str, AndroidContact], + self_identity: str, +) -> str: + if identity == self_identity: + return "Me" + contact = contacts.get(identity) + return contact.display_name() if contact else identity or "Unknown" + + +def _parse_json(value: str) -> Any | None: + try: + return json.loads(value) + except (TypeError, ValueError, json.JSONDecodeError): + return None + + +def _sender_identity( + row: dict[str, str], + spec: AndroidConversationSpec, + self_identity: str, +) -> str: + if parse_bool(row.get("isoutbox")): + return self_identity + if spec.direct_contact is not None: + return spec.direct_contact.identity + return row.get("identity", "") + + +def _build_participants( + spec: AndroidConversationSpec, + rows: list[dict[str, str]], + contacts: dict[str, AndroidContact], + self_identity: str, +) -> list[NormalizedParticipant]: + identities = list(spec.member_identities) + for row in rows: + identity = row.get("identity", "") + if identity and identity not in identities: + identities.append(identity) + if self_identity and self_identity not in identities: + identities.append(self_identity) + + participants: list[NormalizedParticipant] = [] + for identity in identities: + if not identity: + continue + contact = contacts.get(identity) + role = "member" + if identity == self_identity: + role = "me" + elif spec.direct_contact and identity == spec.direct_contact.identity: + role = "chat_partner" + elif spec.distribution_list is not None: + role = "recipient" + elif spec.group and identity == spec.group.creator_identity: + role = "admin" + metadata = contact.metadata() if contact else {} + if spec.group: + metadata["group_creator"] = identity == spec.group.creator_identity + participants.append( + NormalizedParticipant( + participant_id=_participant_id(identity), + display_name=_display_name(identity, contacts, self_identity), + identity=identity, + role=role, + metadata=metadata, + ) + ) + return participants + + +def _message_status(row: dict[str, str]) -> str: + state = (row.get("messagestae") or "").strip().lower() + if state: + return state + if row.get("read_at") or parse_bool(row.get("isread")): + return "read" + if row.get("delivered_at"): + return "delivered" + return "unknown" + + +def _status_event( + source_type: str, + body: str, + contacts: dict[str, AndroidContact], + self_identity: str, +) -> tuple[str, Any | None]: + decoded = _parse_json(body) + params: dict[str, Any] = {} + if isinstance(decoded, list) and len(decoded) > 1 and isinstance(decoded[1], dict): + params = decoded[1] + + if source_type == "VOIP_STATUS": + label = CALL_STATUS_LABELS.get(parse_int(str(params.get("status"))), "Call event") + if params.get("duration") is not None: + label += f" ({params['duration']} seconds)" + return label, decoded + if source_type == "GROUP_CALL_STATUS": + label = GROUP_CALL_STATUS_LABELS.get( + parse_int(str(params.get("status"))), "Group call event" + ) + caller = str(params.get("callerIdentity") or "") + if caller: + label += f" by {_display_name(caller, contacts, self_identity)}" + return label, decoded + if source_type == "GROUP_STATUS": + label = GROUP_STATUS_LABELS.get( + parse_int(str(params.get("status"))), "Group event" + ) + identity = str(params.get("identity") or "") + if identity: + label += f": {_display_name(identity, contacts, self_identity)}" + detail = params.get("newGroupName") or params.get("ballotName") + if detail: + label += f" ({detail})" + return label, decoded + return body or "Status event", decoded + + +def _location_text(body: str) -> tuple[str, Any | None]: + decoded = _parse_json(body) + if not isinstance(decoded, list) or len(decoded) < 2: + return body or "Location", decoded + latitude, longitude = decoded[0], decoded[1] + parts = [f"Location: {latitude}, {longitude}"] + if len(decoded) > 4 and decoded[4]: + parts.append(str(decoded[4])) + if len(decoded) > 3 and decoded[3]: + parts.append(str(decoded[3])) + if len(decoded) > 2 and decoded[2] is not None: + parts.append(f"Accuracy: {decoded[2]} m") + return "\n".join(parts), decoded + + +def _poll_text(body: str, polls: dict[str, dict[str, Any]]) -> tuple[str, dict[str, Any] | None]: + decoded = _parse_json(body) + event_id = None + poll_id = None + if isinstance(decoded, list): + if decoded: + event_id = parse_int(str(decoded[0]), -1) + if len(decoded) > 1: + poll_id = str(decoded[1]) + poll = polls.get(poll_id or "") + label = POLL_EVENT_LABELS.get(event_id or -1, "Poll") + if poll is None: + return label, None + lines = [f"{label}: {poll.get('name') or 'Unnamed poll'}"] + choices = poll.get("choices") or [] + for choice in sorted(choices, key=lambda item: parse_int(item.get("order"))): + name = choice.get("name") or "Unnamed option" + vote_count = choice.get("vote_count") + lines.append(f"- {name}: {vote_count or '0'} vote(s)") + return "\n".join(lines), poll + + +def _reaction_rows( + row: dict[str, str], + spec: AndroidConversationSpec, + source_rows: list[dict[str, str]], + contacts: dict[str, AndroidContact], + self_identity: str, + tz_name: str, +) -> list[NormalizedReaction]: + reactions: list[NormalizedReaction] = [] + seen: set[tuple[str, str, str]] = set() + + def add(identity: str, sequence: str, timestamp: str, source: str) -> None: + key = (identity, sequence, timestamp) + if not identity or not sequence or key in seen: + return + seen.add(key) + reactions.append( + NormalizedReaction( + creator_display=_display_name(identity, contacts, self_identity), + reaction=sequence, + timestamp=format_dt(timestamp, TimeMode.UNIX_MS, tz_name), + metadata={"sender_identity": identity, "source": source}, + ) + ) + + for reaction in source_rows: + add( + reaction.get("sender_identity", ""), + reaction.get("emoji_sequence", ""), + reaction.get("reacted_at", ""), + "reaction_csv", + ) + + state = (row.get("messagestae") or "").upper() + if state in {"USERACK", "USERDEC"}: + if not parse_bool(row.get("isoutbox")): + creator = self_identity + elif spec.direct_contact is not None: + creator = spec.direct_contact.identity + else: + creator = row.get("identity", "") + add( + creator, + "\U0001f44d" if state == "USERACK" else "\U0001f44e", + row.get("modified_at") or row.get("created_at") or "", + "legacy_message_state", + ) + + group_states = _parse_json(row.get("g_msg_states", "")) + if isinstance(group_states, dict): + for identity, group_state in group_states.items(): + normalized_state = str(group_state).upper() + if normalized_state in {"USERACK", "USERDEC"}: + add( + str(identity), + "\U0001f44d" if normalized_state == "USERACK" else "\U0001f44e", + row.get("modified_at") or row.get("created_at") or "", + "legacy_group_message_state", + ) + return reactions + + +def _message_preview(row: dict[str, str]) -> str: + source_type = (row.get("type") or "TEXT").upper() + if source_type == "FILE": + file_data = parse_file_data(row.get("body", "")) + return file_data.filename if file_data and file_data.filename else "Attachment" + text = row.get("body") or source_type.title() + return text.replace("\r", " ").replace("\n", " ")[:120] + + +def normalize_conversation( + *, + spec: AndroidConversationSpec, + rows: list[dict[str, str]], + contacts: dict[str, AndroidContact], + self_identity: str, + tz_name: str, + backup_version: int, + attachments: dict[str, AttachmentOutcome], + reactions_by_api_id: dict[str, list[dict[str, str]]], + polls: dict[str, dict[str, Any]], +) -> NormalizedConversation: + participants = _build_participants(spec, rows, contacts, self_identity) + api_index = { + row.get("apiid", ""): row for row in rows if row.get("apiid") + } + normalized_messages: list[NormalizedMessage] = [] + + for index, row in enumerate(rows, start=1): + source_type = (row.get("type") or "TEXT").upper() + uid = row.get("uid") or row.get("apiid") or str(index) + api_id = row.get("apiid", "") + sender_identity = _sender_identity(row, spec, self_identity) + is_system = parse_bool(row.get("isstatusmessage")) or source_type in { + "STATUS", + "GROUP_STATUS", + "GROUP_CALL_STATUS", + } + direction = ( + "system" + if is_system + else "outgoing" if parse_bool(row.get("isoutbox")) else "incoming" + ) + + attachment_outcome = attachments.get(uid) + message_attachments = ( + [attachment_outcome.attachment] if attachment_outcome is not None else [] + ) + message_type = source_type.lower() + text: str | None = row.get("body") or None + caption = row.get("caption") or None + decoded_content: Any | None = None + poll_details: dict[str, Any] | None = None + + if row.get("deleted_at"): + text = "Message deleted" + elif source_type in {"FILE", "IMAGE", "VIDEO", "VOICEMESSAGE"}: + text = None + if attachment_outcome is not None: + message_type = attachment_outcome.attachment.kind + file_data = parse_file_data(row.get("body", "")) + if file_data and not caption: + caption = file_data.caption + elif source_type == "LOCATION": + text, decoded_content = _location_text(row.get("body", "")) + message_type = "location" + elif source_type == "BALLOT": + text, poll_details = _poll_text(row.get("body", ""), polls) + message_type = "poll" + elif source_type in {"STATUS", "GROUP_STATUS", "GROUP_CALL_STATUS", "VOIP_STATUS"}: + text, decoded_content = _status_event( + source_type, + row.get("body", ""), + contacts, + self_identity, + ) + message_type = "system" if is_system else "call" + + quoted_id = row.get("quoted_message_apiid") or None + quoted_preview = None + if quoted_id: + quoted_row = api_index.get(quoted_id) + if quoted_row is None: + quoted_preview = "Reply to: unknown message" + else: + quoted_sender = _sender_identity(quoted_row, spec, self_identity) + quoted_preview = ( + "Reply to: " + + _display_name(quoted_sender, contacts, self_identity) + + " - " + + _message_preview(quoted_row) + ) + + edits = [] + if row.get("edited_at"): + edits.append( + NormalizedEdit( + timestamp=format_dt( + row.get("edited_at"), TimeMode.UNIX_MS, tz_name + ), + text=None, + metadata={ + "note": ( + "The backup marks this message as edited; previous " + "text is not included" + ) + }, + ) + ) + + metadata = { + "source_type": source_type, + "api_message_id": api_id or None, + "uid": uid, + "is_outbox": parse_bool(row.get("isoutbox")), + "is_read": parse_bool(row.get("isread")), + "is_saved": parse_bool(row.get("issaved")), + "is_status_message": parse_bool(row.get("isstatusmessage")), + "posted_at_raw": row.get("posted_at") or None, + "created_at_raw": row.get("created_at") or None, + "modified_at_raw": row.get("modified_at") or None, + "delivered_at_raw": row.get("delivered_at") or None, + "read_at_raw": row.get("read_at") or None, + "edited_at_raw": row.get("edited_at") or None, + "deleted_at_raw": row.get("deleted_at") or None, + "message_state": row.get("messagestae") or None, + "display_tags": row.get("display_tags") or None, + "recipient_identity": ( + row.get("identity") or None + if spec.distribution_list is not None + else None + ), + "decoded_content": decoded_content, + "poll": poll_details, + } + normalized_messages.append( + NormalizedMessage( + message_id=f"threema-android-message:{uid}", + timestamp=format_dt( + row.get("posted_at") or row.get("created_at"), + TimeMode.UNIX_MS, + tz_name, + ), + sender_id=_participant_id(sender_identity) if sender_identity else None, + sender_display=_display_name( + sender_identity, contacts, self_identity + ), + direction=direction, + message_type=message_type, + text=text, + caption=caption, + quoted_message_ref=( + f"threema-android-api:{quoted_id}" if quoted_id else None + ), + quoted_preview=quoted_preview, + status=_message_status(row), + attachments=message_attachments, + reactions=_reaction_rows( + row, + spec, + reactions_by_api_id.get(api_id, []), + contacts, + self_identity, + tz_name, + ), + edits=edits, + metadata=metadata, + ) + ) + + conversation_metadata = { + "backup_version": backup_version, + "archive_message_member": spec.message_member, + **spec.metadata, + } + return NormalizedConversation( + source_app=SOURCE_APP_THREEMA_ANDROID_BACKUP, + conversation_id=spec.conversation_id, + title=spec.title, + conversation_type=spec.conversation_type, + participants=participants, + messages=normalized_messages, + timezone=tz_name, + time_mode=TimeMode.UNIX_MS, + self_participant_id=_participant_id(self_identity) if self_identity else None, + metadata=conversation_metadata, + ) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..85dc1d6 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Project regression tests.""" diff --git a/tests/test_threema_android_backup.py b/tests/test_threema_android_backup.py new file mode 100644 index 0000000..0ab65dd --- /dev/null +++ b/tests/test_threema_android_backup.py @@ -0,0 +1,265 @@ +from __future__ import annotations + +import hashlib +import json +import shutil +import unittest +import uuid +from pathlib import Path + +from chat_export.common.logging_setup import setup_logging +from chat_export.config_factory import build_export_config +from chat_export.constants import SOURCE_APP_THREEMA_ANDROID_BACKUP +from chat_export.orchestrator import export_all_conversations +from chat_export.render.pdf_builder import _case_summary +from chat_export.threema_android.archive import ( + BackupFormatError, + ThreemaAndroidArchive, +) +from chat_export.threema_android.identity import ( + IdentityBackupError, + decrypt_owner_identity, +) +from chat_export.threema_android.importer import ThreemaAndroidBackupImporter + + +ROOT = Path(__file__).resolve().parents[1] +FIXTURE_DIR = ROOT / "Examples" / "ThreemaAndroidBackupSynthetic" +BACKUP_PATH = FIXTURE_DIR / "threema-backup_synthetic.zip" +NO_MEDIA_BACKUP_PATH = FIXTURE_DIR / "threema-backup_synthetic_no_media.zip" +PASSWORD = "testpassword" +IDENTITY_BACKUP = ( + "4K4M-5Q6T-KFUH-KHL5-2VCJ-ZM57-NL7R-WJTA-V45L-NJAM-" + "WLEU-5DS4-XF7S-OPH4-CTCL-N2CF-3C4C-HPB7-YZWW-U3S6" +) + + +def _config( + input_path: Path, + out_dir: str, + *, + export_media: bool = False, + export_excel: bool = False, + max_media_bytes: int = 0, +): + return build_export_config( + out_dir=out_dir, + source_app=SOURCE_APP_THREEMA_ANDROID_BACKUP, + input_path=str(input_path), + backup_password=PASSWORD, + export_media=export_media, + export_excel=export_excel, + max_media_bytes=max_media_bytes, + ) + + +class IdentityBackupTests(unittest.TestCase): + def test_decrypts_official_identity_backup_vector(self) -> None: + self.assertEqual( + decrypt_owner_identity(IDENTITY_BACKUP, PASSWORD), + "0ZAHXXHB", + ) + + def test_rejects_wrong_identity_password(self) -> None: + with self.assertRaises(IdentityBackupError): + decrypt_owner_identity(IDENTITY_BACKUP, "wrong-password") + + +class ArchiveTests(unittest.TestCase): + def test_reads_encrypted_backup_metadata(self) -> None: + with ThreemaAndroidArchive(str(BACKUP_PATH), PASSWORD) as archive: + self.assertEqual(archive.version, 27) + self.assertTrue(archive.has_member("contacts.csv")) + self.assertTrue(archive.has_member("identity")) + self.assertEqual(len(archive.read_csv("contacts.csv").rows), 3) + + def test_rejects_wrong_archive_password(self) -> None: + with self.assertRaisesRegex( + BackupFormatError, + "Check the backup file and password", + ): + with ThreemaAndroidArchive(str(BACKUP_PATH), "wrong-password"): + pass + + +class ImporterTests(unittest.TestCase): + def test_preserves_backup_password_whitespace(self) -> None: + cfg = build_export_config( + out_dir=str(FIXTURE_DIR), + source_app=SOURCE_APP_THREEMA_ANDROID_BACKUP, + input_path=str(BACKUP_PATH), + backup_password=" password with spaces ", + export_media=False, + ) + + self.assertEqual(cfg.backup_password, " password with spaces ") + + def test_imports_current_backup_structure(self) -> None: + cfg = _config(BACKUP_PATH, str(FIXTURE_DIR)) + run = ThreemaAndroidBackupImporter().load_conversations(cfg) + + self.assertEqual(run.source_app, SOURCE_APP_THREEMA_ANDROID_BACKUP) + self.assertEqual(len(run.conversations), 4) + self.assertEqual( + sum(len(item.conversation.messages) for item in run.conversations), + 23, + ) + self.assertEqual( + sum( + len(message.attachments) + for item in run.conversations + for message in item.conversation.messages + ), + 7, + ) + self.assertEqual( + sum( + len(message.reactions) + for item in run.conversations + for message in item.conversation.messages + ), + 4, + ) + self.assertEqual(run.metadata["backup_version"], 27) + self.assertEqual(run.metadata["self_identity"], "0ZAHXXHB") + self.assertEqual(run.metadata["missing_media_count"], 1) + self.assertEqual(run.metadata["skipped_empty_conversation_count"], 1) + self.assertNotIn("input_path", run.metadata) + for item in run.conversations: + self.assertEqual( + item.conversation.self_participant_id, + "threema-id:0ZAHXXHB", + ) + + def test_normalizes_current_backup_semantics(self) -> None: + cfg = _config(BACKUP_PATH, str(FIXTURE_DIR)) + run = ThreemaAndroidBackupImporter().load_conversations(cfg) + conversations = { + item.conversation.conversation_id: item.conversation + for item in run.conversations + } + + direct = conversations["threema-android:direct:ALICE001"] + self.assertEqual( + [(participant.identity, participant.role) for participant in direct.participants], + [("ALICE001", "chat_partner"), ("0ZAHXXHB", "me")], + ) + self.assertEqual(direct.messages[0].direction, "incoming") + self.assertEqual(direct.messages[1].direction, "outgoing") + self.assertIn("Reply to: Alice Example", direct.messages[1].quoted_preview) + self.assertEqual(len(direct.messages[2].edits), 1) + self.assertEqual(direct.messages[4].attachments[0].filename, "synthetic-image.png") + + group = conversations["threema-android:group:ALICE001:0011223344556677"] + self.assertEqual(group.messages[2].message_type, "system") + self.assertEqual(group.messages[4].message_type, "poll") + self.assertIn("Option Alpha: 1 vote(s)", group.messages[4].text) + self.assertEqual(group.messages[0].reactions[0].creator_display, "Bob Sample") + + missing = conversations["threema-android:direct:BOB00002"].messages[1] + self.assertTrue(missing.attachments[0].metadata["missing_from_backup"]) + + distribution = conversations["threema-android:distribution-list:1"] + self.assertEqual(distribution.conversation_type, "distribution_list") + self.assertTrue( + all(message.direction == "outgoing" for message in distribution.messages) + ) + + def test_counts_missing_payloads_in_backup_without_media(self) -> None: + cfg = _config(NO_MEDIA_BACKUP_PATH, str(FIXTURE_DIR)) + run = ThreemaAndroidBackupImporter().load_conversations(cfg) + + self.assertEqual(run.metadata["media_payload_count"], 0) + self.assertEqual(run.metadata["missing_media_count"], 7) + attachment_count = sum( + sum( + _case_summary(item.conversation)[kind] + for kind in ("image", "audio", "video", "file") + ) + for item in run.conversations + ) + self.assertEqual(attachment_count, 7) + + def test_applies_media_limit_without_reading_payloads(self) -> None: + cfg = _config( + BACKUP_PATH, + str(FIXTURE_DIR), + max_media_bytes=1, + ) + run = ThreemaAndroidBackupImporter().load_conversations(cfg) + + self.assertEqual(run.metadata["missing_media_count"], 1) + self.assertEqual(run.metadata["skipped_media_count"], 6) + + +class EndToEndTests(unittest.TestCase): + def test_exports_all_artifacts_without_exposing_password(self) -> None: + temp_path = ROOT / "Examples" / "Output" / f"android-test-{uuid.uuid4()}" + temp_path.mkdir(parents=True) + self.addCleanup(shutil.rmtree, temp_path, True) + temp_dir = str(temp_path) + try: + cfg = _config( + BACKUP_PATH, + temp_dir, + export_media=True, + export_excel=True, + ) + setup_logging(cfg.log_file, console=False) + result = export_all_conversations(cfg) + + self.assertEqual(result["status"], "Completed with warnings") + self.assertEqual(len(result["exported"]), 4) + self.assertEqual(len(list(Path(temp_dir, "conversations").glob("*.pdf"))), 8) + self.assertEqual(len(list(Path(temp_dir, "excel").glob("*.xlsx"))), 4) + exported_media = [ + path for path in Path(temp_dir, "media").rglob("*") if path.is_file() + ] + self.assertEqual(len(exported_media), 6) + source_media = [ + path + for path in (FIXTURE_DIR / "source").iterdir() + if path.name.startswith( + ( + "message_media_", + "group_message_media_", + "distribution_list_message_media_", + ) + ) + ] + self.assertEqual( + sorted(hashlib.sha256(path.read_bytes()).hexdigest() for path in exported_media), + sorted(hashlib.sha256(path.read_bytes()).hexdigest() for path in source_media), + ) + + summary_path = Path(temp_dir, "export_summary.txt") + manifest_path = Path(temp_dir, "manifest.json") + self.assertTrue(summary_path.is_file()) + self.assertTrue(manifest_path.is_file()) + manifest_text = manifest_path.read_text(encoding="utf-8") + manifest = json.loads(manifest_text) + self.assertEqual(manifest["settings"]["backup_version"], 27) + self.assertEqual( + manifest["settings"]["source_owner_identity"], + "0ZAHXXHB", + ) + self.assertEqual(manifest["results"]["message_count"], 23) + self.assertEqual(manifest["results"]["attachment_count"], 7) + traceability_text = "\n".join( + ( + manifest_text, + summary_path.read_text(encoding="utf-8"), + Path(temp_dir, "log.txt").read_text(encoding="utf-8"), + ) + ) + self.assertNotIn(PASSWORD, traceability_text) + self.assertNotIn(str(BACKUP_PATH.resolve()), traceability_text) + self.assertNotIn(str(BACKUP_PATH.resolve()).replace("\\", "/"), traceability_text) + self.assertNotIn(PASSWORD, repr(cfg)) + finally: + setup_logging(console=False) + shutil.rmtree(temp_path, ignore_errors=True) + + +if __name__ == "__main__": + unittest.main()