From eaff79c955ed09b0b3ad4fb8d579a16d8a43c7a9 Mon Sep 17 00:00:00 2001 From: Jerome Maag Date: Wed, 15 Jul 2026 09:34:00 +0200 Subject: [PATCH 1/3] Add synthetic Threema backup files and data for testing --- Examples/README.md | 1 + .../ThreemaAndroidBackupSynthetic/README.txt | 41 + .../build_fixture.py | 803 ++++++++++++++++++ .../expected.json | 46 + .../source/ballot.csv | 2 + .../source/ballot_choice.csv | 3 + .../source/ballot_vote.csv | 3 + .../source/contact_avatar_0000000001 | Bin 0 -> 398 bytes .../source/contact_avatar_me | Bin 0 -> 408 bytes .../source/contact_profile_pic_0000000001 | Bin 0 -> 400 bytes .../source/contact_reactions.csv | 3 + .../source/contacts.csv | 4 + .../source/distribution_list.csv | 2 + .../source/distribution_list_message_1.csv | 4 + ...media_00000000-0000-4000-8000-000000000203 | 1 + .../source/group_avatar_1000000001 | Bin 0 -> 392 bytes .../source/group_message_1000000001.csv | 9 + ...media_00000000-0000-4000-8000-000000000106 | Bin 0 -> 1866 bytes ...media_00000000-0000-4000-8000-000000000107 | Bin 0 -> 1826 bytes ...bnail_00000000-0000-4000-8000-000000000106 | Bin 0 -> 954 bytes ...bnail_00000000-0000-4000-8000-000000000107 | Bin 0 -> 957 bytes .../source/group_reactions.csv | 3 + .../source/groups.csv | 2 + .../source/identity | 1 + .../source/message_0000000001.csv | 12 + .../source/message_0000000002.csv | 3 + .../source/message_0000000003.csv | 1 + ...media_00000000-0000-4000-8000-000000000005 | Bin 0 -> 766 bytes ...media_00000000-0000-4000-8000-000000000006 | Bin 0 -> 6444 bytes ...media_00000000-0000-4000-8000-000000000007 | 68 ++ ...bnail_00000000-0000-4000-8000-000000000005 | Bin 0 -> 945 bytes .../source/nonce_counts.csv | 2 + .../source/nonces.csv | 3 + .../source/nonces_d2d.csv | 2 + .../source/reaction_counts.csv | 2 + .../source/settings | 1 + .../threema-backup_synthetic.zip | Bin 0 -> 18918 bytes .../threema-backup_synthetic_no_media.zip | Bin 0 -> 9878 bytes 38 files changed, 1022 insertions(+) create mode 100644 Examples/ThreemaAndroidBackupSynthetic/README.txt create mode 100644 Examples/ThreemaAndroidBackupSynthetic/build_fixture.py create mode 100644 Examples/ThreemaAndroidBackupSynthetic/expected.json create mode 100644 Examples/ThreemaAndroidBackupSynthetic/source/ballot.csv create mode 100644 Examples/ThreemaAndroidBackupSynthetic/source/ballot_choice.csv create mode 100644 Examples/ThreemaAndroidBackupSynthetic/source/ballot_vote.csv create mode 100644 Examples/ThreemaAndroidBackupSynthetic/source/contact_avatar_0000000001 create mode 100644 Examples/ThreemaAndroidBackupSynthetic/source/contact_avatar_me create mode 100644 Examples/ThreemaAndroidBackupSynthetic/source/contact_profile_pic_0000000001 create mode 100644 Examples/ThreemaAndroidBackupSynthetic/source/contact_reactions.csv create mode 100644 Examples/ThreemaAndroidBackupSynthetic/source/contacts.csv create mode 100644 Examples/ThreemaAndroidBackupSynthetic/source/distribution_list.csv create mode 100644 Examples/ThreemaAndroidBackupSynthetic/source/distribution_list_message_1.csv create mode 100644 Examples/ThreemaAndroidBackupSynthetic/source/distribution_list_message_media_00000000-0000-4000-8000-000000000203 create mode 100644 Examples/ThreemaAndroidBackupSynthetic/source/group_avatar_1000000001 create mode 100644 Examples/ThreemaAndroidBackupSynthetic/source/group_message_1000000001.csv create mode 100644 Examples/ThreemaAndroidBackupSynthetic/source/group_message_media_00000000-0000-4000-8000-000000000106 create mode 100644 Examples/ThreemaAndroidBackupSynthetic/source/group_message_media_00000000-0000-4000-8000-000000000107 create mode 100644 Examples/ThreemaAndroidBackupSynthetic/source/group_message_thumbnail_00000000-0000-4000-8000-000000000106 create mode 100644 Examples/ThreemaAndroidBackupSynthetic/source/group_message_thumbnail_00000000-0000-4000-8000-000000000107 create mode 100644 Examples/ThreemaAndroidBackupSynthetic/source/group_reactions.csv create mode 100644 Examples/ThreemaAndroidBackupSynthetic/source/groups.csv create mode 100644 Examples/ThreemaAndroidBackupSynthetic/source/identity create mode 100644 Examples/ThreemaAndroidBackupSynthetic/source/message_0000000001.csv create mode 100644 Examples/ThreemaAndroidBackupSynthetic/source/message_0000000002.csv create mode 100644 Examples/ThreemaAndroidBackupSynthetic/source/message_0000000003.csv create mode 100644 Examples/ThreemaAndroidBackupSynthetic/source/message_media_00000000-0000-4000-8000-000000000005 create mode 100644 Examples/ThreemaAndroidBackupSynthetic/source/message_media_00000000-0000-4000-8000-000000000006 create mode 100644 Examples/ThreemaAndroidBackupSynthetic/source/message_media_00000000-0000-4000-8000-000000000007 create mode 100644 Examples/ThreemaAndroidBackupSynthetic/source/message_thumbnail_00000000-0000-4000-8000-000000000005 create mode 100644 Examples/ThreemaAndroidBackupSynthetic/source/nonce_counts.csv create mode 100644 Examples/ThreemaAndroidBackupSynthetic/source/nonces.csv create mode 100644 Examples/ThreemaAndroidBackupSynthetic/source/nonces_d2d.csv create mode 100644 Examples/ThreemaAndroidBackupSynthetic/source/reaction_counts.csv create mode 100644 Examples/ThreemaAndroidBackupSynthetic/source/settings create mode 100644 Examples/ThreemaAndroidBackupSynthetic/threema-backup_synthetic.zip create mode 100644 Examples/ThreemaAndroidBackupSynthetic/threema-backup_synthetic_no_media.zip diff --git a/Examples/README.md b/Examples/README.md index c31fc4e..de45bce 100644 --- a/Examples/README.md +++ b/Examples/README.md @@ -4,6 +4,7 @@ This folder contains synthetic example data for local testing and demo purposes. Included in the repository: - `ThreemaSynthetic/`: synthetic Threema SQLite example with one direct chat and one group chat +- `ThreemaAndroidBackupSynthetic/`: synthetic encrypted Threema Android data backup with direct, group, and distribution-list messages - `WhatsAppSynthetic/`: synthetic WhatsApp ZIP examples for direct and group chats - `run_examples.ipynb`: notebook to run the example exports diff --git a/Examples/ThreemaAndroidBackupSynthetic/README.txt b/Examples/ThreemaAndroidBackupSynthetic/README.txt new file mode 100644 index 0000000..c555882 --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/README.txt @@ -0,0 +1,41 @@ +Synthetic Threema Android Data Backup +===================================== + +This fixture contains only invented data and media generated by build_fixture.py. +It follows the current Threema Android CSV backup format version 27. + +Encrypted backup: + threema-backup_synthetic.zip + threema-backup_synthetic_no_media.zip + +Password: + testpassword + +Expected own Threema ID: + 0ZAHXXHB + +Coverage: + - direct chats, including an empty chat + - one group chat + - one distribution list + - incoming and outgoing messages + - text, quote, edit, deletion, location, poll, and call/status messages + - contact and group reactions + - image, audio, video, PDF, and text attachments + - thumbnails, contact avatars, profile pictures, and a group avatar + - one deliberately unavailable attachment payload + - a second valid backup variant created without message media or thumbnails + - CSV quoting, commas, backslashes, line breaks, Unicode, and emoji + +The source/ directory contains the plaintext archive members for transparent +inspection. expected.json contains the counts and identifiers expected from an +importer. The identity entry uses the public legacy test vector from the +official Threema Android source and does not contain a real identity or key. + +Regeneration requirements: + - Python project dependencies + - 7-Zip on PATH + - FFmpeg on PATH + +Regenerate with: + python .\Examples\ThreemaAndroidBackupSynthetic\build_fixture.py diff --git a/Examples/ThreemaAndroidBackupSynthetic/build_fixture.py b/Examples/ThreemaAndroidBackupSynthetic/build_fixture.py new file mode 100644 index 0000000..93025a1 --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/build_fixture.py @@ -0,0 +1,803 @@ +"""Build a fully synthetic Threema Android data-backup fixture.""" + +from __future__ import annotations + +import csv +import json +import math +import shutil +import struct +import subprocess +import wave +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Iterable + +from PIL import Image, ImageDraw +from reportlab.pdfgen import canvas + + +FIXTURE_DIR = Path(__file__).resolve().parent +SOURCE_DIR = FIXTURE_DIR / "source" +ARCHIVE_PATH = FIXTURE_DIR / "threema-backup_synthetic.zip" +NO_MEDIA_ARCHIVE_PATH = FIXTURE_DIR / "threema-backup_synthetic_no_media.zip" +EXPECTED_PATH = FIXTURE_DIR / "expected.json" +PASSWORD = "testpassword" +SELF_IDENTITY = "0ZAHXXHB" + +# Public test vector from the official Threema identity-backup tests. +IDENTITY_BACKUP = ( + "4K4M-5Q6T-KFUH-KHL5-2VCJ-ZM57-NL7R-WJTA-V45L-NJAM-" + "WLEU-5DS4-XF7S-OPH4-CTCL-N2CF-3C4C-HPB7-YZWW-U3S6" +) + +CONTACT_HEADER = [ + "identity", + "publickey", + "verification", + "acid", + "firstname", + "lastname", + "nick_name", + "last_update", + "hidden", + "archived", + "identity_id", +] +MESSAGE_HEADER = [ + "apiid", + "uid", + "isoutbox", + "isread", + "issaved", + "messagestae", + "posted_at", + "created_at", + "modified_at", + "type", + "body", + "isstatusmessage", + "caption", + "quoted_message_apiid", + "delivered_at", + "read_at", + "g_msg_states", + "display_tags", + "edited_at", + "deleted_at", +] +GROUP_MESSAGE_HEADER = MESSAGE_HEADER[:2] + ["identity"] + MESSAGE_HEADER[2:] +DISTRIBUTION_MESSAGE_HEADER = [ + "apiid", + "uid", + "identity", + "isoutbox", + "isread", + "issaved", + "messagestae", + "posted_at", + "created_at", + "modified_at", + "type", + "body", + "isstatusmessage", + "caption", + "quoted_message_apiid", + "delivered_at", + "read_at", +] + +BASE_TIME = datetime(2026, 1, 15, 9, 0, tzinfo=timezone.utc) + + +def epoch_ms(offset_minutes: int) -> str: + """Return a deterministic Unix timestamp in milliseconds.""" + value = BASE_TIME + timedelta(minutes=offset_minutes) + return str(int(value.timestamp() * 1000)) + + +def uid(number: int) -> str: + """Return a deterministic UUID-shaped message identifier.""" + return f"00000000-0000-4000-8000-{number:012d}" + + +def api_id(number: int) -> str: + """Return a deterministic 16-character API message identifier.""" + return f"{number:016x}" + + +def csv_value(value: Any) -> str: + """Serialize values like Threema's CSVRow helper.""" + if value is None: + return "" + if isinstance(value, bool): + return "1" if value else "0" + if isinstance(value, (list, tuple)): + value = ";".join(str(item) for item in value) + return str(value).replace("\\", "\\\\") + + +def write_csv(path: Path, header: list[str], rows: Iterable[dict[str, Any]]) -> None: + """Write an OpenCSV-compatible, fully quoted UTF-8 file.""" + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.writer(handle, quoting=csv.QUOTE_ALL, lineterminator="\n") + writer.writerow(header) + for row in rows: + writer.writerow([csv_value(row.get(column)) for column in header]) + + +def message( + number: int, + *, + offset: int, + message_type: str, + body: str, + outbox: bool, + identity: str | None = None, + state: str | None = None, + caption: str | None = None, + quote: str | None = None, + edited_at: str | None = None, + deleted_at: str | None = None, + group_states: str | None = None, +) -> dict[str, Any]: + """Build one version-27 message row.""" + timestamp = epoch_ms(offset) + row: dict[str, Any] = { + "apiid": api_id(number), + "uid": uid(number), + "isoutbox": outbox, + "isread": True, + "issaved": True, + "messagestae": state or ("READ" if outbox else ""), + "posted_at": timestamp, + "created_at": timestamp, + "modified_at": edited_at or timestamp, + "type": message_type, + "body": body, + "isstatusmessage": False, + "caption": caption, + "quoted_message_apiid": quote, + "delivered_at": epoch_ms(offset + 1) if outbox else None, + "read_at": epoch_ms(offset + 2), + "g_msg_states": group_states, + "display_tags": 0, + "edited_at": edited_at, + "deleted_at": deleted_at, + } + if identity is not None: + row["identity"] = identity + return row + + +def file_body( + *, + mime_type: str, + size: int, + filename: str, + rendering_type: int, + downloaded: bool, + caption: str | None = None, + thumbnail_mime_type: str | None = None, + metadata: dict[str, Any] | None = None, +) -> str: + """Serialize a current FileDataModel JSON body.""" + return json.dumps( + [ + "00112233445566778899aabbccddeeff", + "00" * 32, + mime_type, + size, + filename, + rendering_type, + downloaded, + caption, + thumbnail_mime_type, + metadata or {}, + ], + ensure_ascii=False, + separators=(",", ":"), + ) + + +def save_pattern_image(path: Path, *, size: tuple[int, int], colors: tuple[str, str], image_format: str) -> None: + """Create a deterministic synthetic image.""" + image = Image.new("RGB", size, colors[0]) + draw = ImageDraw.Draw(image) + width, height = size + draw.rectangle((width // 2, 0, width, height), fill=colors[1]) + draw.line((0, 0, width, height), fill="white", width=max(1, width // 16)) + draw.line((0, height, width, 0), fill="white", width=max(1, width // 16)) + image.save(path, format=image_format) + + +def save_wave(path: Path) -> None: + """Create a short synthetic mono WAV attachment.""" + sample_rate = 8000 + duration_seconds = 0.4 + frames = int(sample_rate * duration_seconds) + with wave.open(str(path), "wb") as output: + output.setnchannels(1) + output.setsampwidth(2) + output.setframerate(sample_rate) + for index in range(frames): + sample = int(9000 * math.sin(2 * math.pi * 440 * index / sample_rate)) + output.writeframesraw(struct.pack(" None: + """Create a one-page synthetic PDF attachment.""" + document = canvas.Canvas(str(path), pagesize=(240, 160)) + document.setTitle("Synthetic attachment") + document.drawString(24, 112, "Synthetic Threema attachment") + document.drawString(24, 88, "No real case or personal data") + document.save() + + +def save_video(path: Path) -> None: + """Create a one-second synthetic MP4 attachment with FFmpeg.""" + ffmpeg = shutil.which("ffmpeg") + if not ffmpeg: + raise RuntimeError("FFmpeg is required to regenerate the synthetic MP4") + subprocess.run( + [ + ffmpeg, + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "color=c=0x2f7d8c:s=96x64:d=1", + "-c:v", + "mpeg4", + "-pix_fmt", + "yuv420p", + "-f", + "mp4", + "-y", + str(path), + ], + check=True, + ) + + +def create_media() -> dict[str, Path]: + """Create attachment payloads, thumbnails, and avatars.""" + media = { + "direct_image": SOURCE_DIR / f"message_media_{uid(5)}", + "direct_audio": SOURCE_DIR / f"message_media_{uid(6)}", + "direct_pdf": SOURCE_DIR / f"message_media_{uid(7)}", + "group_image": SOURCE_DIR / f"group_message_media_{uid(106)}", + "group_video": SOURCE_DIR / f"group_message_media_{uid(107)}", + "distribution_text": SOURCE_DIR / f"distribution_list_message_media_{uid(203)}", + } + save_pattern_image( + media["direct_image"], + size=(96, 64), + colors=("#22577a", "#80ed99"), + image_format="PNG", + ) + save_wave(media["direct_audio"]) + save_pdf(media["direct_pdf"]) + save_pattern_image( + media["group_image"], + size=(80, 80), + colors=("#9b2226", "#e9d8a6"), + image_format="JPEG", + ) + save_video(media["group_video"]) + media["distribution_text"].write_text( + "Synthetic distribution-list attachment.\n", + encoding="utf-8", + ) + + save_pattern_image( + SOURCE_DIR / f"message_thumbnail_{uid(5)}", + size=(32, 24), + colors=("#22577a", "#80ed99"), + image_format="JPEG", + ) + save_pattern_image( + SOURCE_DIR / f"group_message_thumbnail_{uid(106)}", + size=(32, 32), + colors=("#9b2226", "#e9d8a6"), + image_format="JPEG", + ) + save_pattern_image( + SOURCE_DIR / f"group_message_thumbnail_{uid(107)}", + size=(32, 24), + colors=("#2f7d8c", "#f4a261"), + image_format="JPEG", + ) + + avatar_specs = { + "contact_avatar_me": ("#264653", "#2a9d8f"), + "contact_avatar_0000000001": ("#6d597a", "#e56b6f"), + "contact_profile_pic_0000000001": ("#355070", "#eaac8b"), + "group_avatar_1000000001": ("#005f73", "#ee9b00"), + } + for name, colors in avatar_specs.items(): + save_pattern_image( + SOURCE_DIR / name, + size=(48, 48), + colors=colors, + image_format="PNG", + ) + return media + + +def create_structured_files(media: dict[str, Path]) -> dict[str, Any]: + """Create all version-27 CSV and metadata archive members.""" + write_csv(SOURCE_DIR / "settings", ["version", "27"], []) + (SOURCE_DIR / "identity").write_text(IDENTITY_BACKUP, encoding="ascii") + + contacts = [ + { + "identity": "ALICE001", + "publickey": "11" * 32, + "verification": "FULLY_VERIFIED", + "firstname": "Alice", + "lastname": "Example", + "nick_name": "Ali", + "last_update": epoch_ms(0), + "hidden": False, + "archived": False, + "identity_id": "0000000001", + }, + { + "identity": "BOB00002", + "publickey": "22" * 32, + "verification": "SERVER_VERIFIED", + "firstname": "Bob", + "lastname": "Sample", + "hidden": False, + "archived": False, + "identity_id": "0000000002", + }, + { + "identity": "CAROL003", + "publickey": "33" * 32, + "verification": "UNVERIFIED", + "firstname": "Carol", + "lastname": "Test", + "hidden": True, + "archived": True, + "identity_id": "0000000003", + }, + ] + write_csv(SOURCE_DIR / "contacts.csv", CONTACT_HEADER, contacts) + + alice_messages = [ + message(1, offset=1, message_type="TEXT", body="Hello from Alice.", outbox=False), + message( + 2, + offset=2, + message_type="TEXT", + body="Quoted reply from the synthetic owner.", + outbox=True, + quote=api_id(1), + ), + message( + 3, + offset=3, + message_type="TEXT", + body="Current edited message text.", + outbox=True, + edited_at=epoch_ms(4), + ), + message( + 4, + offset=5, + message_type="TEXT", + body="", + outbox=False, + deleted_at=epoch_ms(6), + ), + message( + 5, + offset=7, + message_type="FILE", + body=file_body( + mime_type="image/png", + size=media["direct_image"].stat().st_size, + filename="synthetic-image.png", + rendering_type=1, + downloaded=True, + caption="Synthetic image caption", + thumbnail_mime_type="image/jpeg", + metadata={"w": 96, "h": 64}, + ), + outbox=True, + caption="Synthetic image caption", + ), + message( + 6, + offset=8, + message_type="FILE", + body=file_body( + mime_type="audio/wav", + size=media["direct_audio"].stat().st_size, + filename="synthetic-audio.wav", + rendering_type=1, + downloaded=True, + metadata={"d": 0.4}, + ), + outbox=False, + ), + message( + 7, + offset=9, + message_type="FILE", + body=file_body( + mime_type="application/pdf", + size=media["direct_pdf"].stat().st_size, + filename="synthetic-document.pdf", + rendering_type=0, + downloaded=True, + ), + outbox=True, + ), + message( + 8, + offset=10, + message_type="LOCATION", + body=json.dumps( + [46.123456, 8.654321, 5.0, "Example Street 1", "Test Location"], + separators=(",", ":"), + ), + outbox=False, + ), + message( + 9, + offset=11, + message_type="VOIP_STATUS", + body='[1,{"status":2,"callId":123456789,"duration":42}]', + outbox=True, + ), + message( + 10, + offset=12, + message_type="TEXT", + body='CSV test: comma, "quote", backslash C:\\Temp, line\nbreak and emoji 🙂', + outbox=False, + ), + ] + write_csv(SOURCE_DIR / "message_0000000001.csv", MESSAGE_HEADER, alice_messages) + + missing_uid = uid(12) + bob_messages = [ + message(11, offset=20, message_type="TEXT", body="Short Bob test chat.", outbox=False), + message( + 12, + offset=21, + message_type="FILE", + body=file_body( + mime_type="application/octet-stream", + size=2048, + filename="unavailable-synthetic.bin", + rendering_type=0, + downloaded=False, + ), + outbox=False, + ), + ] + write_csv(SOURCE_DIR / "message_0000000002.csv", MESSAGE_HEADER, bob_messages) + write_csv(SOURCE_DIR / "message_0000000003.csv", MESSAGE_HEADER, []) + + group_uid = "1000000001" + group_id = "0011223344556677" + group_rows = [ + { + "id": group_id, + "creator": "ALICE001", + "groupname": "Project Aurora", + "created_at": epoch_ms(30), + "last_update": epoch_ms(50), + "members": [SELF_IDENTITY, "ALICE001", "BOB00002", "CAROL003"], + "archived": False, + "groupDesc": "Synthetic group for importer tests", + "groupDescTimestamp": epoch_ms(31), + "group_uid": group_uid, + "user_state": 0, + } + ] + write_csv( + SOURCE_DIR / "groups.csv", + [ + "id", + "creator", + "groupname", + "created_at", + "last_update", + "members", + "archived", + "groupDesc", + "groupDescTimestamp", + "group_uid", + "user_state", + ], + group_rows, + ) + + group_messages = [ + message(101, offset=32, message_type="TEXT", body="Group message from Alice.", outbox=False, identity="ALICE001"), + message( + 102, + offset=33, + message_type="TEXT", + body="Synthetic group reply.", + outbox=True, + identity=SELF_IDENTITY, + quote=api_id(101), + group_states='{"ALICE001":"READ","BOB00002":"DELIVERED"}', + ), + message( + 103, + offset=34, + message_type="GROUP_STATUS", + body='[4,{"status":3,"identity":"CAROL003"}]', + outbox=False, + identity="ALICE001", + ), + message( + 104, + offset=35, + message_type="GROUP_CALL_STATUS", + body='[2,{"status":1,"callId":"synthetic-call","callerIdentity":"BOB00002","groupId":1}]', + outbox=False, + identity="BOB00002", + ), + message( + 105, + offset=36, + message_type="BALLOT", + body="[1,1]", + outbox=True, + identity=SELF_IDENTITY, + ), + message( + 106, + offset=37, + message_type="FILE", + body=file_body( + mime_type="image/jpeg", + size=media["group_image"].stat().st_size, + filename="group-image.jpg", + rendering_type=1, + downloaded=True, + thumbnail_mime_type="image/jpeg", + metadata={"w": 80, "h": 80}, + ), + outbox=False, + identity="CAROL003", + ), + message( + 107, + offset=38, + message_type="FILE", + body=file_body( + mime_type="video/mp4", + size=media["group_video"].stat().st_size, + filename="group-video.mp4", + rendering_type=1, + downloaded=True, + thumbnail_mime_type="image/jpeg", + metadata={"d": 1, "w": 96, "h": 64}, + ), + outbox=True, + identity=SELF_IDENTITY, + ), + message( + 108, + offset=39, + message_type="TEXT", + body="", + outbox=False, + identity="BOB00002", + deleted_at=epoch_ms(40), + ), + ] + write_csv(SOURCE_DIR / f"group_message_{group_uid}.csv", GROUP_MESSAGE_HEADER, group_messages) + + write_csv( + SOURCE_DIR / "distribution_list.csv", + ["id", "distribution_list_name", "created_at", "last_update", "distribution_members", "archived"], + [ + { + "id": 1, + "distribution_list_name": "Synthetic Announcements", + "created_at": epoch_ms(60), + "last_update": epoch_ms(63), + "distribution_members": ["ALICE001", "BOB00002"], + "archived": False, + } + ], + ) + distribution_messages = [ + message(201, offset=61, message_type="TEXT", body="Announcement for Alice.", outbox=True, identity="ALICE001"), + message(202, offset=62, message_type="TEXT", body="Announcement for Bob.", outbox=True, identity="BOB00002"), + message( + 203, + offset=63, + message_type="FILE", + body=file_body( + mime_type="text/plain", + size=media["distribution_text"].stat().st_size, + filename="announcement.txt", + rendering_type=0, + downloaded=True, + ), + outbox=True, + identity="ALICE001", + ), + ] + write_csv( + SOURCE_DIR / "distribution_list_message_1.csv", + DISTRIBUTION_MESSAGE_HEADER, + distribution_messages, + ) + + write_csv( + SOURCE_DIR / "ballot.csv", + ["id", "aid", "creator", "ref", "ref_id", "name", "state", "assessment", "type", "choice_type", "last_viewed_at", "created_at", "modified_at"], + [ + { + "id": 1, + "aid": "a1b2c3d4e5f60708", + "creator": SELF_IDENTITY, + "ref": "GroupBallotModel", + "ref_id": group_uid, + "name": "Choose a synthetic option", + "state": "OPEN", + "assessment": "SINGLE", + "type": "INTERMEDIATE", + "choice_type": "TEXT", + "last_viewed_at": epoch_ms(41), + "created_at": epoch_ms(36), + "modified_at": epoch_ms(41), + } + ], + ) + write_csv( + SOURCE_DIR / "ballot_choice.csv", + ["id", "ballot", "aid", "type", "name", "vote_count", "order", "created_at", "modified_at"], + [ + {"id": 1, "ballot": "1:1", "aid": 1, "type": "TEXT", "name": "Option Alpha", "vote_count": 1, "order": 1, "created_at": epoch_ms(36), "modified_at": epoch_ms(41)}, + {"id": 2, "ballot": "1:1", "aid": 2, "type": "TEXT", "name": "Option Beta", "vote_count": 1, "order": 2, "created_at": epoch_ms(36), "modified_at": epoch_ms(41)}, + ], + ) + write_csv( + SOURCE_DIR / "ballot_vote.csv", + ["id", "ballot_uid", "choice_uid", "identity", "choice", "created_at", "modified_at"], + [ + {"id": 1, "ballot_uid": "1:1", "choice_uid": "1:1:1", "identity": "ALICE001", "choice": 1, "created_at": epoch_ms(40), "modified_at": epoch_ms(40)}, + {"id": 2, "ballot_uid": "1:1", "choice_uid": "1:1:2", "identity": SELF_IDENTITY, "choice": 1, "created_at": epoch_ms(41), "modified_at": epoch_ms(41)}, + ], + ) + + write_csv( + SOURCE_DIR / "contact_reactions.csv", + ["identity", "api_message_id", "sender_identity", "emoji_sequence", "reacted_at"], + [ + {"identity": "ALICE001", "api_message_id": api_id(1), "sender_identity": SELF_IDENTITY, "emoji_sequence": "👍", "reacted_at": epoch_ms(13)}, + {"identity": "ALICE001", "api_message_id": api_id(2), "sender_identity": "ALICE001", "emoji_sequence": "🙂", "reacted_at": epoch_ms(14)}, + ], + ) + write_csv( + SOURCE_DIR / "group_reactions.csv", + ["api_group_id", "group_creator_identity", "api_message_id", "sender_identity", "emoji_sequence", "reacted_at"], + [ + {"api_group_id": group_id, "group_creator_identity": "ALICE001", "api_message_id": api_id(101), "sender_identity": "BOB00002", "emoji_sequence": "✅", "reacted_at": epoch_ms(42)}, + {"api_group_id": group_id, "group_creator_identity": "ALICE001", "api_message_id": api_id(102), "sender_identity": SELF_IDENTITY, "emoji_sequence": "👍", "reacted_at": epoch_ms(43)}, + ], + ) + write_csv( + SOURCE_DIR / "reaction_counts.csv", + ["contactReactions", "groupReactions"], + [{"contactReactions": 2, "groupReactions": 2}], + ) + + write_csv(SOURCE_DIR / "nonces.csv", ["nonces"], [{"nonces": "aa" * 32}, {"nonces": "bb" * 32}]) + write_csv(SOURCE_DIR / "nonces_d2d.csv", ["nonces"], [{"nonces": "cc" * 32}]) + write_csv(SOURCE_DIR / "nonce_counts.csv", ["csp", "d2d"], [{"csp": 2, "d2d": 1}]) + + expected = { + "backup_version": 27, + "self_identity": SELF_IDENTITY, + "contacts": 3, + "groups": 1, + "distribution_lists": 1, + "archive_message_files": {"direct": 3, "group": 1, "distribution_list": 1}, + "exported_conversations": { + "direct": 2, + "group": 1, + "distribution_list": 1, + "total": 4, + }, + "skipped_empty_conversations": 1, + "messages": {"direct": 12, "group": 8, "distribution_list": 3, "total": 23}, + "attachment_references": 7, + "media_payloads": 6, + "missing_media_payloads": 1, + "thumbnails": 3, + "contact_reactions": 2, + "group_reactions": 2, + "polls": 1, + "poll_choices": 2, + "poll_votes": 2, + "empty_direct_chat_identity": "CAROL003", + "missing_media_message_uid": missing_uid, + "archive_variants": { + "threema-backup_synthetic.zip": { + "media_payloads": 6, + "thumbnails": 3, + }, + "threema-backup_synthetic_no_media.zip": { + "media_payloads": 0, + "thumbnails": 0, + }, + }, + } + return expected + + +def is_message_media(path: Path) -> bool: + """Return whether one archive member is message media or a thumbnail.""" + return path.name.startswith( + ( + "message_media_", + "message_thumbnail_", + "group_message_media_", + "group_message_thumbnail_", + "distribution_list_message_media_", + "distribution_list_thumbnail_", + ) + ) + + +def build_archive(path: Path, members: list[Path]) -> None: + """Create one AES-256 encrypted ZIP from selected source members.""" + seven_zip = shutil.which("7z") + if not seven_zip: + raise RuntimeError("7-Zip is required to build the encrypted fixture") + path.unlink(missing_ok=True) + subprocess.run( + [ + seven_zip, + "a", + "-tzip", + str(path), + *(member.name for member in members), + f"-p{PASSWORD}", + "-mem=AES256", + "-mx=9", + ], + cwd=SOURCE_DIR, + check=True, + ) + + +def main() -> None: + """Regenerate the plaintext source, expectations, and encrypted archive.""" + if SOURCE_DIR.exists(): + shutil.rmtree(SOURCE_DIR) + SOURCE_DIR.mkdir(parents=True) + media = create_media() + expected = create_structured_files(media) + EXPECTED_PATH.write_text( + json.dumps(expected, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + members = sorted(path for path in SOURCE_DIR.iterdir() if path.is_file()) + build_archive(ARCHIVE_PATH, members) + build_archive( + NO_MEDIA_ARCHIVE_PATH, + [path for path in members if not is_message_media(path)], + ) + print(f"Created {ARCHIVE_PATH}") + print(f"Created {NO_MEDIA_ARCHIVE_PATH}") + + +if __name__ == "__main__": + main() diff --git a/Examples/ThreemaAndroidBackupSynthetic/expected.json b/Examples/ThreemaAndroidBackupSynthetic/expected.json new file mode 100644 index 0000000..ee6906d --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/expected.json @@ -0,0 +1,46 @@ +{ + "backup_version": 27, + "self_identity": "0ZAHXXHB", + "contacts": 3, + "groups": 1, + "distribution_lists": 1, + "archive_message_files": { + "direct": 3, + "group": 1, + "distribution_list": 1 + }, + "exported_conversations": { + "direct": 2, + "group": 1, + "distribution_list": 1, + "total": 4 + }, + "skipped_empty_conversations": 1, + "messages": { + "direct": 12, + "group": 8, + "distribution_list": 3, + "total": 23 + }, + "attachment_references": 7, + "media_payloads": 6, + "missing_media_payloads": 1, + "thumbnails": 3, + "contact_reactions": 2, + "group_reactions": 2, + "polls": 1, + "poll_choices": 2, + "poll_votes": 2, + "empty_direct_chat_identity": "CAROL003", + "missing_media_message_uid": "00000000-0000-4000-8000-000000000012", + "archive_variants": { + "threema-backup_synthetic.zip": { + "media_payloads": 6, + "thumbnails": 3 + }, + "threema-backup_synthetic_no_media.zip": { + "media_payloads": 0, + "thumbnails": 0 + } + } +} diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/ballot.csv b/Examples/ThreemaAndroidBackupSynthetic/source/ballot.csv new file mode 100644 index 0000000..9fc88e9 --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/ballot.csv @@ -0,0 +1,2 @@ +"id","aid","creator","ref","ref_id","name","state","assessment","type","choice_type","last_viewed_at","created_at","modified_at" +"1","a1b2c3d4e5f60708","0ZAHXXHB","GroupBallotModel","1000000001","Choose a synthetic option","OPEN","SINGLE","INTERMEDIATE","TEXT","1768470060000","1768469760000","1768470060000" diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/ballot_choice.csv b/Examples/ThreemaAndroidBackupSynthetic/source/ballot_choice.csv new file mode 100644 index 0000000..a78c86a --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/ballot_choice.csv @@ -0,0 +1,3 @@ +"id","ballot","aid","type","name","vote_count","order","created_at","modified_at" +"1","1:1","1","TEXT","Option Alpha","1","1","1768469760000","1768470060000" +"2","1:1","2","TEXT","Option Beta","1","2","1768469760000","1768470060000" diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/ballot_vote.csv b/Examples/ThreemaAndroidBackupSynthetic/source/ballot_vote.csv new file mode 100644 index 0000000..b0f3b77 --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/ballot_vote.csv @@ -0,0 +1,3 @@ +"id","ballot_uid","choice_uid","identity","choice","created_at","modified_at" +"1","1:1","1:1:1","ALICE001","1","1768470000000","1768470000000" +"2","1:1","1:1:2","0ZAHXXHB","1","1768470060000","1768470060000" diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/contact_avatar_0000000001 b/Examples/ThreemaAndroidBackupSynthetic/source/contact_avatar_0000000001 new file mode 100644 index 0000000000000000000000000000000000000000..26bffe0b2d936edfd46a40b0e5575333d6850928 GIT binary patch literal 398 zcmV;90df9`P)Y-p0r!?E94l{8^vjxSNIIl^7dYibYNrU>CxTeX1dYD+CDS>=VEYnm#UM9Jw4uaX# zq!EE&Ftuu=AgD|VO#=vxCY`1Y1hGl2X$C=HvS>6Q6ir5r5`>J&uF-?=W^&P(K*;}` z)EX-YDU(Ix0GevIEEF`!-k~7SwAO4Yr8bd^=5X&M7ZZ7D9$`0;qsGNVz8brU+%-lM z_0U*M)JdZ@Q9q5&L|rus6ZO`#nkYciWTF_2+(cm-p@||j5)%b$swRrplua~5Q!vpe sP0qA?O@oY@lVKW^?xEn^EMu9cUzgqpp}cl1E&u=k07*qoM6N<$g1JPkFaQ7m literal 0 HcmV?d00001 diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/contact_avatar_me b/Examples/ThreemaAndroidBackupSynthetic/source/contact_avatar_me new file mode 100644 index 0000000000000000000000000000000000000000..9c1df77b90ff8f7009e8825eb31fff598bb4fd9f GIT binary patch literal 408 zcmV;J0cZY+P)gLXSe8|ioga_w@%WZQMxX3Aks?pa zvfxt3=T{i&@b+YR9RR<70HE9ReEv3`XxD5&15Ny|`33DJ6>s8|=GviwCf;fuzdo-c z8)D)+%_c*^CQfPgp$;=~MzaUSm^iOVfO?y_s7ZtRnYgCOf_j)(peccTO)S$?Kwc)f zrVfJH)T9xCU@*06q#&qF3QYqDjV7I@4Fs`Ct!V~9V6td5AQVkTjS_^6$*ut~J)B;2 zCKPk6)95=?{&P}mtRSRJ7L5aFuHCXw&@B571%c+RW;ZFdiCi?tOOjkn=y)s!^Dzx2DxZ0h%Tg#c1Rv3eyNp6seJzC|FZ9 zQM{&Xq9K}siAHI1rs=i~GHMQnZBRNx!M$16x=%kE1qg>-+V61y0000SyAbCJX9eVu7Xv@-?wcQvrFIb?!1cJfTs*!@AGAT3-AT*kEnl=!`Cbgy+1cAw-(ST4i88u1}GA6r555k+tMPmXX z|8r7ntRSRJ7L5aFs@<|s&?I|@f?U&8 z7){heV=+-DjoL*0G&&P?)hJBVThnTy08NvLVl;9Sg=vH)iquF<6s)P5C|*-G(GX3+ uM58o0)9y75GHNb{X;8X{f^)NsWuC4zDF!IPQ?ix-0000fl literal 0 HcmV?d00001 diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/contact_reactions.csv b/Examples/ThreemaAndroidBackupSynthetic/source/contact_reactions.csv new file mode 100644 index 0000000..d31ac3c --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/contact_reactions.csv @@ -0,0 +1,3 @@ +"identity","api_message_id","sender_identity","emoji_sequence","reacted_at" +"ALICE001","0000000000000001","0ZAHXXHB","👍","1768468380000" +"ALICE001","0000000000000002","ALICE001","🙂","1768468440000" diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/contacts.csv b/Examples/ThreemaAndroidBackupSynthetic/source/contacts.csv new file mode 100644 index 0000000..4dfbb4f --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/contacts.csv @@ -0,0 +1,4 @@ +"identity","publickey","verification","acid","firstname","lastname","nick_name","last_update","hidden","archived","identity_id" +"ALICE001","1111111111111111111111111111111111111111111111111111111111111111","FULLY_VERIFIED","","Alice","Example","Ali","1768467600000","0","0","0000000001" +"BOB00002","2222222222222222222222222222222222222222222222222222222222222222","SERVER_VERIFIED","","Bob","Sample","","","0","0","0000000002" +"CAROL003","3333333333333333333333333333333333333333333333333333333333333333","UNVERIFIED","","Carol","Test","","","1","1","0000000003" diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/distribution_list.csv b/Examples/ThreemaAndroidBackupSynthetic/source/distribution_list.csv new file mode 100644 index 0000000..ac1c7be --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/distribution_list.csv @@ -0,0 +1,2 @@ +"id","distribution_list_name","created_at","last_update","distribution_members","archived" +"1","Synthetic Announcements","1768471200000","1768471380000","ALICE001;BOB00002","0" diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/distribution_list_message_1.csv b/Examples/ThreemaAndroidBackupSynthetic/source/distribution_list_message_1.csv new file mode 100644 index 0000000..4f94d8d --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/distribution_list_message_1.csv @@ -0,0 +1,4 @@ +"apiid","uid","identity","isoutbox","isread","issaved","messagestae","posted_at","created_at","modified_at","type","body","isstatusmessage","caption","quoted_message_apiid","delivered_at","read_at" +"00000000000000c9","00000000-0000-4000-8000-000000000201","ALICE001","1","1","1","READ","1768471260000","1768471260000","1768471260000","TEXT","Announcement for Alice.","0","","","1768471320000","1768471380000" +"00000000000000ca","00000000-0000-4000-8000-000000000202","BOB00002","1","1","1","READ","1768471320000","1768471320000","1768471320000","TEXT","Announcement for Bob.","0","","","1768471380000","1768471440000" +"00000000000000cb","00000000-0000-4000-8000-000000000203","ALICE001","1","1","1","READ","1768471380000","1768471380000","1768471380000","FILE","[""00112233445566778899aabbccddeeff"",""0000000000000000000000000000000000000000000000000000000000000000"",""text/plain"",41,""announcement.txt"",0,true,null,null,{}]","0","","","1768471440000","1768471500000" diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/distribution_list_message_media_00000000-0000-4000-8000-000000000203 b/Examples/ThreemaAndroidBackupSynthetic/source/distribution_list_message_media_00000000-0000-4000-8000-000000000203 new file mode 100644 index 0000000..a468d78 --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/distribution_list_message_media_00000000-0000-4000-8000-000000000203 @@ -0,0 +1 @@ +Synthetic distribution-list attachment. diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/group_avatar_1000000001 b/Examples/ThreemaAndroidBackupSynthetic/source/group_avatar_1000000001 new file mode 100644 index 0000000000000000000000000000000000000000..c2b7f5733fbac53ee3d12537eb23f801c9a0870a GIT binary patch literal 392 zcmV;30eAk1P)fDn$7namArqz}P$f;H_g?rgwlThO0R3beB;=HHrbndnCmG^) z`nGH*BL28b@LK2Vm+?fqW(Qhm;(yH@bf{Fki8q>Oh8CK5ulc#8XUUeB_)W9xpe6GjT<81jU%Rt|@?Ko4BbdgXWpIr>TNwm{_1`fqYFY({w;yCb^~$BC}~oBLb1Z zG^&w;NM%xJ20%1u(rLy(BsQrv!ypovEE)}nnkJ)038IS0uF-?&&E%pnfvEm-QfsUr zDw!-A2T<=>GYeBAb3J3K(cBOS8jZ$cuqtwLDn@EVHKYGrl3IX60Vm)DgOC9f2Z7-r zl6s&j-4g-%RNzZNpfET>28lw;Vx$e_^8geAgF)di1Og71c1KBd0LLK|)eNm<<~#c! z$-D7JF{xQ7b?f3fC6~4_im`7MM=$IkZn?{|+qnf02C!`JyxEqrPS*=T;7(kgXAs&m}f%`!SWei-g&rVsL@bH|R;mqY> zaTKeOoM=H;rSG6}-*NJUlM%uzO(UM1b{p1qc;72JCe~*R+${BBg$`S~v6h*~PQG(GFBFywW>?Dvr`$G?fHVoHHQyrzKCk~qudGT~#Mw0GDrZr+ z)YX^AQD#aLzaowRMMoezknfjC?IiGL0kj^VH4h>JZSecUehGUVkn-Hykv}SC;+=T5C$HDo2CVzlOOsY|u8j$y6=DDWLq$ z`>2(z&Nle?IS4^^T11+PUxY?#Y&JQqnR^hBR)w)$a5fY^#o*qD-_#K+Y&0@c0HB|p ze4uXM@r&}%p$$CfhV7*V>kn+%>3R7 z%`U2towlwH;#^!#|3g9Fc*fGQby>sPMG4jn`M5RyMJZa3X1fX$Hea_AvlL#7_#}df f*2{R3RTe2Hc*x$THL>GcdFqoFc6WzZlJ36%nA$`m literal 0 HcmV?d00001 diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/group_message_media_00000000-0000-4000-8000-000000000107 b/Examples/ThreemaAndroidBackupSynthetic/source/group_message_media_00000000-0000-4000-8000-000000000107 new file mode 100644 index 0000000000000000000000000000000000000000..aba1adba09597d1aab7e7c479f40ff9eacfff5c5 GIT binary patch literal 1826 zcmcIl&ubG=5T4Ce?ZHT-!L~F)r1fTNvy|dRs0fmSQ2V=0J!F&J*wDQ0vbzoH!S<*^ z!J~hI-il&87NJTm`UjXp4_=C1^d>0QZ{8+$6JHMs9WrlbXTF(vJM;F1h*G8Co?G@E zK@=mFU_-cL86xT~dA3cYUlo=a5NR(daSEcA$WJ{UFrSRvXlx9KpW^km+TOdy1EQzd zxAE+9eTFfVmpwAxf&ZC0bF)#zz8 zdPa?&QKM(o=vg&-PK};ZqFY@)ckErBSJtmBsCT=dMlY(-i)!?e8oi`O=hWz&8ojJW zFDuclE~k#U%laGPI8}U16;WNWAS0yjD~y8*L7 zbGtHbc*UxCQo~X$Tex%%X&%o`)?BkHHe+IEM0u?QC0P|RX>{8|R_IGvp3N90_Mcw& zD(O(yKKMbQ0{yce_^p93<|A%B9u!kFXK2aewX;x}nY5mpX0PqHaPL3x|KO8;P;{C#c^{X+g>@@1A#=i(lHC`g#|cN! kGhLS_{)dLk;hDYoJa9PSdHla|5>86!a|v_J;5Tag1{DX-Q~&?~ literal 0 HcmV?d00001 diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/group_message_thumbnail_00000000-0000-4000-8000-000000000106 b/Examples/ThreemaAndroidBackupSynthetic/source/group_message_thumbnail_00000000-0000-4000-8000-000000000106 new file mode 100644 index 0000000000000000000000000000000000000000..024990d4c26b49691f3ab1a5760cd013ec52228b GIT binary patch literal 954 zcmex=^(PF6}rMnOeST|r4lSw=>~TvNxu(8R<c1}I=;VrF4wW9Q)H;sz?% zD!{d!pzFb!U9xX3zTPI5o8roG<0MW4oqZMDikqloVbuf*=gfJ(V&YTRE(2~ znmD<{#3dx9RMpfqG__1j&CD$#Lrq|0jpOAYiRA5@KKd?E3>0)Jj6c%Mz+fPH zK;DP{-QROk5lfoV+cNH1CMT`yTVk}~K@-2TOVCM{lO{$sMaK?Kyx5VNctWACZTI;v z`JHCbN5KmQrNuCm_J7kg5tYI4kf2DbI}Cw`vqzVLI~ z?$g}d@r)79YhD|u{Q7cwzyIZwmke)B7H|9#w^MGXMnpu^53WP9>5+}<*O$fqs++Cu z!JfHPwtvyr6JJmI8U6eBI%NN>Z{N!2Jxjau$vmij!5fALjE%AK{42lY?s|9n;h#U# zrQOY@)l^Oq*Fda?+(vq&*&IkGfavV`8B9^iMnWbq3X`N#k2X4KT} OpZ`brd7ZrT|C<1gv2$?% literal 0 HcmV?d00001 diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/group_message_thumbnail_00000000-0000-4000-8000-000000000107 b/Examples/ThreemaAndroidBackupSynthetic/source/group_message_thumbnail_00000000-0000-4000-8000-000000000107 new file mode 100644 index 0000000000000000000000000000000000000000..7b9938ce28e9b96b6b3a6e39d6a8b469d1e6e0c8 GIT binary patch literal 957 zcmex=^(PF6}rMnOeST|r4lSw=>~TvNxu(8R<c1}I=;VrF4wW9Q)H;sz?% zD!{d!pzFb!U9xX3zTPI5o8roG<0MW4oqZMDikqloVbuf*=gfJ(V&YTRE(2~ znmD<{#3dx9RMpfqG__1j&CD$#!==98o*#8rN5AUMU-oK6r|i}@89|y%4GUEQ z5B+D@1B|H)dVFH*m&9dM^-MFr_-1M)OVPT$GyMzde;uFT8o2KMp-=a-ZuuTMcQGZS06PdsOI z&8lDY;{NAzuLHur8y@XHt^QMfx$)iU8!zvF6#Ab*Zq0#{C6iK5zFFHNdi<;0atqOq zpS=3I!|(sufBxOO+Aq;-S6?jq&tU)Q_ivfI)7y2ooVM5+FFbSY0;9KG2ZM$0FkWwR zyDC#uU6ty2{@Nbb(lw75Cf<7a(K174LBPi2^A||kR@$CCX56K*enb1-_r?!aSNBh{ zmVLVP%{QN8YSJ-@7u!OQFO+}2Hh$W<`@xZUB@co^8RuQ-t36fKpZ{;eqOKpmA2F`?kK&YU2H0cmCf50ID2;_W%F@ literal 0 HcmV?d00001 diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/group_reactions.csv b/Examples/ThreemaAndroidBackupSynthetic/source/group_reactions.csv new file mode 100644 index 0000000..38c8bbf --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/group_reactions.csv @@ -0,0 +1,3 @@ +"api_group_id","group_creator_identity","api_message_id","sender_identity","emoji_sequence","reacted_at" +"0011223344556677","ALICE001","0000000000000065","BOB00002","✅","1768470120000" +"0011223344556677","ALICE001","0000000000000066","0ZAHXXHB","👍","1768470180000" diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/groups.csv b/Examples/ThreemaAndroidBackupSynthetic/source/groups.csv new file mode 100644 index 0000000..7840462 --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/groups.csv @@ -0,0 +1,2 @@ +"id","creator","groupname","created_at","last_update","members","archived","groupDesc","groupDescTimestamp","group_uid","user_state" +"0011223344556677","ALICE001","Project Aurora","1768469400000","1768470600000","0ZAHXXHB;ALICE001;BOB00002;CAROL003","0","Synthetic group for importer tests","1768469460000","1000000001","0" diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/identity b/Examples/ThreemaAndroidBackupSynthetic/source/identity new file mode 100644 index 0000000..3eab8f1 --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/identity @@ -0,0 +1 @@ +4K4M-5Q6T-KFUH-KHL5-2VCJ-ZM57-NL7R-WJTA-V45L-NJAM-WLEU-5DS4-XF7S-OPH4-CTCL-N2CF-3C4C-HPB7-YZWW-U3S6 \ No newline at end of file diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/message_0000000001.csv b/Examples/ThreemaAndroidBackupSynthetic/source/message_0000000001.csv new file mode 100644 index 0000000..24d2283 --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/message_0000000001.csv @@ -0,0 +1,12 @@ +"apiid","uid","isoutbox","isread","issaved","messagestae","posted_at","created_at","modified_at","type","body","isstatusmessage","caption","quoted_message_apiid","delivered_at","read_at","g_msg_states","display_tags","edited_at","deleted_at" +"0000000000000001","00000000-0000-4000-8000-000000000001","0","1","1","","1768467660000","1768467660000","1768467660000","TEXT","Hello from Alice.","0","","","","1768467780000","","0","","" +"0000000000000002","00000000-0000-4000-8000-000000000002","1","1","1","READ","1768467720000","1768467720000","1768467720000","TEXT","Quoted reply from the synthetic owner.","0","","0000000000000001","1768467780000","1768467840000","","0","","" +"0000000000000003","00000000-0000-4000-8000-000000000003","1","1","1","READ","1768467780000","1768467780000","1768467840000","TEXT","Current edited message text.","0","","","1768467840000","1768467900000","","0","1768467840000","" +"0000000000000004","00000000-0000-4000-8000-000000000004","0","1","1","","1768467900000","1768467900000","1768467900000","TEXT","","0","","","","1768468020000","","0","","1768467960000" +"0000000000000005","00000000-0000-4000-8000-000000000005","1","1","1","READ","1768468020000","1768468020000","1768468020000","FILE","[""00112233445566778899aabbccddeeff"",""0000000000000000000000000000000000000000000000000000000000000000"",""image/png"",766,""synthetic-image.png"",1,true,""Synthetic image caption"",""image/jpeg"",{""w"":96,""h"":64}]","0","Synthetic image caption","","1768468080000","1768468140000","","0","","" +"0000000000000006","00000000-0000-4000-8000-000000000006","0","1","1","","1768468080000","1768468080000","1768468080000","FILE","[""00112233445566778899aabbccddeeff"",""0000000000000000000000000000000000000000000000000000000000000000"",""audio/wav"",6444,""synthetic-audio.wav"",1,true,null,null,{""d"":0.4}]","0","","","","1768468200000","","0","","" +"0000000000000007","00000000-0000-4000-8000-000000000007","1","1","1","READ","1768468140000","1768468140000","1768468140000","FILE","[""00112233445566778899aabbccddeeff"",""0000000000000000000000000000000000000000000000000000000000000000"",""application/pdf"",1410,""synthetic-document.pdf"",0,true,null,null,{}]","0","","","1768468200000","1768468260000","","0","","" +"0000000000000008","00000000-0000-4000-8000-000000000008","0","1","1","","1768468200000","1768468200000","1768468200000","LOCATION","[46.123456,8.654321,5.0,""Example Street 1"",""Test Location""]","0","","","","1768468320000","","0","","" +"0000000000000009","00000000-0000-4000-8000-000000000009","1","1","1","READ","1768468260000","1768468260000","1768468260000","VOIP_STATUS","[1,{""status"":2,""callId"":123456789,""duration"":42}]","0","","","1768468320000","1768468380000","","0","","" +"000000000000000a","00000000-0000-4000-8000-000000000010","0","1","1","","1768468320000","1768468320000","1768468320000","TEXT","CSV test: comma, ""quote"", backslash C:\\Temp, line +break and emoji 🙂","0","","","","1768468440000","","0","","" diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/message_0000000002.csv b/Examples/ThreemaAndroidBackupSynthetic/source/message_0000000002.csv new file mode 100644 index 0000000..2588511 --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/message_0000000002.csv @@ -0,0 +1,3 @@ +"apiid","uid","isoutbox","isread","issaved","messagestae","posted_at","created_at","modified_at","type","body","isstatusmessage","caption","quoted_message_apiid","delivered_at","read_at","g_msg_states","display_tags","edited_at","deleted_at" +"000000000000000b","00000000-0000-4000-8000-000000000011","0","1","1","","1768468800000","1768468800000","1768468800000","TEXT","Short Bob test chat.","0","","","","1768468920000","","0","","" +"000000000000000c","00000000-0000-4000-8000-000000000012","0","1","1","","1768468860000","1768468860000","1768468860000","FILE","[""00112233445566778899aabbccddeeff"",""0000000000000000000000000000000000000000000000000000000000000000"",""application/octet-stream"",2048,""unavailable-synthetic.bin"",0,false,null,null,{}]","0","","","","1768468980000","","0","","" diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/message_0000000003.csv b/Examples/ThreemaAndroidBackupSynthetic/source/message_0000000003.csv new file mode 100644 index 0000000..2468121 --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/message_0000000003.csv @@ -0,0 +1 @@ +"apiid","uid","isoutbox","isread","issaved","messagestae","posted_at","created_at","modified_at","type","body","isstatusmessage","caption","quoted_message_apiid","delivered_at","read_at","g_msg_states","display_tags","edited_at","deleted_at" diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/message_media_00000000-0000-4000-8000-000000000005 b/Examples/ThreemaAndroidBackupSynthetic/source/message_media_00000000-0000-4000-8000-000000000005 new file mode 100644 index 0000000000000000000000000000000000000000..b71285fd03a4b03a05ad707565675383130aa9eb GIT binary patch literal 766 zcmV&-UHVsKQ+*`Nd(0K3Ef2hdmvOoJCAPe2JeyTg8baRsOcm4XYPtT^Y_ zQ$Tc;f;}LKI2k}hm4YQ8xj1=1ER}*8pdxXkfZ(o@1)vIX3II8)qzfT~qW0P+;a2@uLv zx(Uc%96vy%D$M~=#PJ3Mu}YVKh~f|c2~|1+;)+89gq%vd!@dV`n1HH}ZJp7KwF0Sz z!hI}-4$~mHN-Q*FgZ@g90Y$-+DE-W~uKPo}OM~LFIEg0?MWv1wiunF%8yAZGEQMSF zMZu0txYclD#NVvqdQCLhtkPBT02E8QLZ$5vYeuw&85S1POARxYN``=9RcE@obV9Fq^sK_jc6b5heV>Zq z1yhw;0Y&X8A&c>s7DgrmRA`u?Y8jxiP^l77G@Dd39l@($q&dV1jX6%0Ax=e=$^b?C zX=$G^U^yd&L2{_hlIsqVx2q%q6de;A&QXLcqxvulZ1OBdVU}u@Qh*{OHO7iVEioz% z)WJAYXC0_3Rp|yGZgMrSBbgIM(oiqFa=mV$UZG0AfO?2Lqo-K-$mj&92XTluFQiqv wUrmv_n&ReP>36{R*W%Co{Bo1~^*FewFP~q|e7c&idjJ3c07*qoM6N<$f^|Sjng9R* literal 0 HcmV?d00001 diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/message_media_00000000-0000-4000-8000-000000000006 b/Examples/ThreemaAndroidBackupSynthetic/source/message_media_00000000-0000-4000-8000-000000000006 new file mode 100644 index 0000000000000000000000000000000000000000..f3fb7dd238d159a84b8c5c90ec2642b322653fdd GIT binary patch literal 6444 zcmeI0yGtWc6vl7jD<;NF933Z_O=jla6&5=K7B+D=vdT7!5wo2rEUO5M2r2AbI}2g4 z5$(iMQPIxAfN3m**vW!(Cz)(sj*dx;iSfzif3Q8A_HdfRIp6pFg5L%QKe`b5{w4JF z=dZBDAcRb{`Ya;!+mDbLG3duscnZ~e5X#wRxPW&VAK{p{zq8olHj#2z4<-+&NOu#3 zto82fHP*b{&U)+^k#q5queJ`?1s?o2fByMZT{(_VW^9Fx=bfgDwo_qFCS+flaOauD zhS^8qR!+ei!lw8 literal 0 HcmV?d00001 diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/message_media_00000000-0000-4000-8000-000000000007 b/Examples/ThreemaAndroidBackupSynthetic/source/message_media_00000000-0000-4000-8000-000000000007 new file mode 100644 index 0000000..0632743 --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/message_media_00000000-0000-4000-8000-000000000007 @@ -0,0 +1,68 @@ +%PDF-1.3 +% ReportLab Generated PDF document (opensource) +1 0 obj +<< +/F1 2 0 R +>> +endobj +2 0 obj +<< +/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font +>> +endobj +3 0 obj +<< +/Contents 7 0 R /MediaBox [ 0 0 240 160 ] /Parent 6 0 R /Resources << +/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] +>> /Rotate 0 /Trans << + +>> + /Type /Page +>> +endobj +4 0 obj +<< +/PageMode /UseNone /Pages 6 0 R /Type /Catalog +>> +endobj +5 0 obj +<< +/Author (anonymous) /CreationDate (D:20260715085538+02'00') /Creator (anonymous) /Keywords () /ModDate (D:20260715085538+02'00') /Producer (ReportLab PDF Library - \(opensource\)) + /Subject (unspecified) /Title (Synthetic attachment) /Trapped /False +>> +endobj +6 0 obj +<< +/Count 1 /Kids [ 3 0 R ] /Type /Pages +>> +endobj +7 0 obj +<< +/Filter [ /ASCII85Decode /FlateDecode ] /Length 157 +>> +stream +Gar?'YmS?5&4HDC`K\aoEPiNj"g,]iqZG$PE_>ap0&-0q?@mkR"&lk#7l]#Feij.*7Xlc3!5=(jkJ.n(\>c"4f>3$h4eeM(ab&6aK0rLq!\k?1U*eWGTA][_4H>nXbpY%t]I\qFnC"W'<5D:&IUk8Wi]Sd4~>endstream +endobj +xref +0 8 +0000000000 65535 f +0000000061 00000 n +0000000092 00000 n +0000000199 00000 n +0000000392 00000 n +0000000460 00000 n +0000000733 00000 n +0000000792 00000 n +trailer +<< +/ID +[] +% ReportLab generated PDF document -- digest (opensource) + +/Info 5 0 R +/Root 4 0 R +/Size 8 +>> +startxref +1039 +%%EOF diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/message_thumbnail_00000000-0000-4000-8000-000000000005 b/Examples/ThreemaAndroidBackupSynthetic/source/message_thumbnail_00000000-0000-4000-8000-000000000005 new file mode 100644 index 0000000000000000000000000000000000000000..c1b55bcc6b8c73b94c868f298b3944c615a592ca GIT binary patch literal 945 zcmex=^(PF6}rMnOeST|r4lSw=>~TvNxu(8R<c1}I=;VrF4wW9Q)H;sz?% zD!{d!pzFb!U9xX3zTPI5o8roG<0MW4oqZMDikqloVbuf*=gfJ(V&YTRE(2~ znmD<{#3dx9RMpfqG__1j&CD$#Lrzp*_oLHh+koNVbxpa?;;sH8rwOM-SUI)& z#Q%O^VBl+Xuz9zAQG7*p$YlR3N;i#)B#Jiarv=}U|H#fF7M4!ByPk0y_l2siKB+vK#J!@^hX7w-I{|sx-z0#PM8D?>1^Tq>A zTkJ1>+5avkZuRds{qocNKmD3_WUKfc?cU$B0f`UYFG{|sM`uUu30bNjoh zIYwT6-I3)lKfQnbz1-&B;X~VYx%OC{@1FJO+RbIR`Q~hV^ymfu-h;|XeMbH&RjHn* zum1_GT=#an;L-GDu8P&w+$Eeck-r#se6_T_xv}R_m-dU7zwdpvepS8im2KaxchO!?fzefy0o8b{ysnTO~HllPgb7JKYii< FO#uDOcIp5C literal 0 HcmV?d00001 diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/nonce_counts.csv b/Examples/ThreemaAndroidBackupSynthetic/source/nonce_counts.csv new file mode 100644 index 0000000..554804d --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/nonce_counts.csv @@ -0,0 +1,2 @@ +"csp","d2d" +"2","1" diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/nonces.csv b/Examples/ThreemaAndroidBackupSynthetic/source/nonces.csv new file mode 100644 index 0000000..c894246 --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/nonces.csv @@ -0,0 +1,3 @@ +"nonces" +"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/nonces_d2d.csv b/Examples/ThreemaAndroidBackupSynthetic/source/nonces_d2d.csv new file mode 100644 index 0000000..c2295ed --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/nonces_d2d.csv @@ -0,0 +1,2 @@ +"nonces" +"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/reaction_counts.csv b/Examples/ThreemaAndroidBackupSynthetic/source/reaction_counts.csv new file mode 100644 index 0000000..78fefdb --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/reaction_counts.csv @@ -0,0 +1,2 @@ +"contactReactions","groupReactions" +"2","2" diff --git a/Examples/ThreemaAndroidBackupSynthetic/source/settings b/Examples/ThreemaAndroidBackupSynthetic/source/settings new file mode 100644 index 0000000..97fd763 --- /dev/null +++ b/Examples/ThreemaAndroidBackupSynthetic/source/settings @@ -0,0 +1 @@ +"version","27" diff --git a/Examples/ThreemaAndroidBackupSynthetic/threema-backup_synthetic.zip b/Examples/ThreemaAndroidBackupSynthetic/threema-backup_synthetic.zip new file mode 100644 index 0000000000000000000000000000000000000000..72b2d2e12e1e82f0b16ce376cf69e88fc6dae00f GIT binary patch literal 18918 zcmb_@byOT$l6T{7!QCxLkYE9VI|O%k3GVLh8Z5XIJh;0Bw-DSRxVwD)W;ZWv@^)tS zebeU@=QMxxuj-cFs=6g30S3+t00HO&&V`RO0Km^KkXS$f09XJVKu_1)+{&I_-_8*v z843Uj;1dFa1~{@&G@*(ezOI28#~Bh@!Vlo*K~W|?t-gb~9VodYBxvhND6f1(Mr>L& zO!SpqD-*K(;HE0AD zr$)t`t^QiNZE39jLtm6!+CdKQC>~gl>_tm~GsDc%&B7$^8TEMCn2vN0;usv;IZ@?q zI%lDx8f+6zL84F$0%+fT8!?U3d`GNXldTP;$Y-zhyfCD#?rT+Xez^O%xKjll1&wI1 znyR9LO(+XWcjDA;s@P(-s&O`w*+Q8f_w&o)cKz)WhAPc5ncog49e6n3frRqs;b`ld zSefb@{&GgUN5;_x?V{0)d)4Q@D8!bnYZ!FhC#oEqeSQjsnJ7xLA)v3m%Fkeb2%YaV z!^IT$$D?YwJmKDT@j~VKHel%LmZK;*cw8L-K}dlh;OgCnt}SMj&|JI8v4imGfa(V2 z{WnA=jGj*a@~@({s0z7XEsy3|3*Pk<5ojEX59q zfzm?5MMh4GI5@$B3%a3>6?NAM9;TPIGD}lfoeN{Qc8@6I#XOcL&Vk#XKaQP%0xnCA zpgFyfq~ZgA=l=@G`)g1D03OJ{zLllDuD-q9FDm)O@z|(o^#1erdR+o3(U^FONiw%! zow$Y=Oc}DFTs8EJQH}SJs43igw*!(mPLxn6JrUI^@WOA&0zJ~wz-!HV4vxNUPFPHr z&qkq1bju)!Vw9VJH`wl4M7;77w=3@L$By$9WT^7pWWdX0TPeJ(Cg7YOYbY2dQ4qgAEo(BtZ%$+L`=5@AG;KII{|iCYgaBreYB9 zfseh#>=|Xcc=hs7`&LFNCKV=YGf|y9yI|FNa$w;*YUwD=fdtR*(`=k1Qeq{b4}#Fv z^DEe2>@$xSBKjD%7n1Yx4Qxq%KdWM(b4LOR`{!9{>pJS%>)L8F{P-~b>F)r58OmEG zheD?y<<@tpPtl{v7cvDKCDR}Es9DAE@bx;73&5rJ{lPKynyg(sTQ>UnLqJ-tm$Zs| zV_0+xrLP6&roTF6rwo;m=Ljw&QZ#sGJP`%3f08@3lkMSh>Eus8EtZV115hR@!OywC zU?21Gu^ri-r&V-Gk^;1a&(pYZp2&aHYTICpKs@! zz`jLsK~5)Kf6ifsF&s%LGW(8mhveWCt4Uoq0E=zfW|fBICabq_2Jy9l847hWwcdrR z&E}keDjC?L$Lqp-sIz5o*}cfsH)v3i`4@zy6&qgr1 zo>-38wrLG7V$$g(fSW7nT{~HZg2(kz)>W+#%n*UGYC6I2qJ(+EhqySs?6o${B^0Z_4HfB|&EJxLJJ^0$raFw3ki(2n9^>u^vwT+vdTmo$ z941=1qL=oz8|@MIorL1TMNxc?htMZ*WicDvX>JB`yvm)xf(ir43YfS;=yJlAde(U# zOzdIFlKiZi-*sIL)N>M${)kuq?H50EZDIIK$5XV`n}RWTkBD(;>@)rXzC+4NE8>qD z9WH9K)ev`^(OP!|gr0BMv5IP56+@tzpWVdI6w(J!kKW$g23v@KB3Mk8YP7`(`FQ|A# zWqBnbpT3Gq4wONO+7WL{^~@Ls+vkyE;dTJY$Ql%`;pry`3Q$aBno;Wa0I@OY61dJ_ zi2@*F$8fMgWSL$Wqv6jeO6sr>vb5>QceePamm_(W>B=6m^;d-gPgLI&sitq8^gSd zBxc+xwH_~h$DjOO_8Y>hPm=F{ z*LDd|)6qa8c&2S@TPq_|b3<)wQ~keoZy_%jW%bGi!xvn$A737LK3pkDNyS(z$&;EZ zEWB&0d=pfCDjha2mh;6ZnR+2~`+%4(zJmYUMkq>q)_yS10QX>y+m|vGJkSnR8j+IV zMbYI%w=0!c#nin)X(ro={`p61$f=w3Rt^nJqu$12-|F>t$E&lQLnYOUm+xO`A za{D`iw!D>O^R#G*Ur9UxfwROUqY(#2U8f0pi~d{KJ4rN(>$Wr2-kv1+UTXAks6ejo z3A+{Z()NvUw(8spqH^b}2NOm_uV!ubB?n^y9KzqiidB+18@9(*-sm%*MZ@)=!l0y> z@Z2Zao4GF-^a(bF9n9X!G1p=DqT_I>T9c$=y}{(-UVwHId^^Jz+>dcm>~6~ zK_;zx*=TcUpO8S(zCSmKzER;H`njgUyX`(|lW-!6moH^kRR&aB?xy>rUu4eM2|<*B z$pS>yw>ur-w%iUnX;0HYz_RCS_5*G^!s5A%uhQkbJMMb`m%P>ZV|**w7qBUEqOYyn zq9y%3lR_YUe-|qgC{pEri)CPHXK!n&=kSL}+UCG7za;isTgvd~yGVWK6?c>et)Ue6Dy1C=Tqof6O8kW`4mc zSgp^yDrw?-VuxOmAh}u)NYlfVDUgk zcV)15Dxgk z_T8fr!Ci;c&KRu_T5-dXktu*K=iXCzg8@D?(R?sXpW8>L&Y<#s-@vE^GGMrj6@QSV z(?N}qAL&~m!IfT9PpJo6?(aGi0$L+AkOcp!&MXWKOm+W^vvhx4r|4e)@rV77KYx7v z$A^i5`C0M>C1y3s(ra*}h1_lq&TIFs@0;%pWt<^oYU_cBTJxWtEx+*Ah^$&dl9h zrLqW^N~xi!oykWnS}Af@tOdO>=}uTkQe8se1z;;{7tyVLg}S%xvzfTyfs)}44TG*G z!AW}|MXN1=NZP%`{_4;^4iD!73Ugk~yY0P*j?hU=_IT9KQ<`nHKIz@I_G*o4*BY5o z3CHL6(#jzNLK{ZpPnoeOW}(%{JFf47i%fU7H$Ee1;v6I3$ak+1`8vNf^elZWe(%Cj zAOQ1bdz_*~tYiz7kWVScnaq|7Y6g{~VB$Lg2=tMk1Zpj5thN_d>>@vy;ZYVuz;p#k65`i+$om~qso#OI@IfH(p$cf<u`kg2J}9bJ-T z@gr!=6VRNbFZSERl3%5`D2fpbIU(fOrgYVuPnW3c$VRHB_L5w_8PSc-6lHxJ zdHwOo9{^vang^3S;^%V)a6HP&q)^+jTUx~u8{IWBq_<#d6BE5X36e3$!<_~-cv+1h zOJrYFq9gE7%R&o}lq|bb5=S2qQ?_p^n2e;gF-E9F#`9spxlrBoMd=H;k@4p{yuXh* z{P;0aOqwMoFHj>p-JOxsbLs?)Bd*fMdyT#P-o4_cp|-zlhtM)i?2N4rFqp?KtE4ce zz}?xY3pyT1d{4BGmc9YFzvFM63*>|68+C`W$!6hVo%g274Pz_4_kV|h@#H&>ha59{ z8JmjJe1>@21+ARh^0n{|rPlSp6~?M8)C$TP-;wxGCi9T&F#WZmdk5{w{Rb)IBy;Hs zk@Ypr?hG-+H(v2ut1@K=_RQA8$&LmQkUpWN^9B!_y6Gr!)2l?wRvWLSsmtTJRP%q= zaTc&B%zyy^@cz{CpW*dSCI1pE$b+}+`{W^kF;+c7&77LH?T$shmp^V( znU8`N3;T6*V`&{e&9@u%6Xg3zZ`Cjpv{{jOZjR6RAY@s$9iasuMN096e4@^M!u%=U zElAbn%_xT?eL@xLrWdr$mKuCnS8X>oq1r*~;;e1d%(M#%;%0N1WLw{ILZIRuTKNJM z6in+)EQLf#g+V9vcU|`Mm>O%r>zv?oqtRW}3FX6_5cbP^yP>KB9TOLVG2WOZ_$J2N zH_8lFO|c?1IqD!V`moJ?D{tCKM>|kT=?NVbK5ON$mGWt!eu7&lh?svhzjcr7@U*RA z_I>HGBc$SZ`|$Hku;x@J2cy0ToChGVZ;bE#(5)MSS&dkdS3ZWDH!U(cHp93DyXK%? zHTj9Gt^a+VB5SaWib@=3&u@nL#3)09!1q>S&|a&p;~9M~;o16O#d4#u}Y_ids4%ew2#?1>`@JR1qUk`Z0xx&mxmceY|QXM1~M zm(|FgQzq6Dk?La04QF3z9NghRqc7>ZLqT-vu0^YIAf+B*?Q}5wTAX8_0Z8YGu1D$Q zNhk>O+ctIcURt8}+^Jj{qfk49Uw^l9K1cvS90~wn|INz(&4k3r!1`xGf(C%Ap7lg% zESd6nKhC`@lHAj(RK@HsqKQ&x`sI5#d9`J!Y>qaT)m$JLlcH@fS#Y z@$dGXK}l0{n(&s8PBZa$$tnrpV=wBY{6n@y zm2##F`S)$#_M)~4Sv5$_CF7Bv!dOJExQc9{4BC*W`Yd1sr4P-5$%%eN4(M-}@W|Js zX9+$dz%CF{w6({PZk!F!vEPfZf5AJPhl}1dvMh=~(J|1+hRM)6?Sly|ruCF{u`w-f ze0n;a+OTC?e*%cdgmppo)-xq4g_qK42|#a6h9B~BGo_C;cC8%VbJlswxz@nsF($aa z^noM)`-#Jb?8;DvQVYE1&0KnujTT1sRI^|69g+uf&r1qdo12KtoCSP7;)NE<^VQ)i zjwc*Qo`=$h!I&9Acx#`D(XD)f^V{0RtTB`|?1+wTL$Qf8LKt6PJ9&E0Dg8GTagy{D z;o`3=YfRxM$g!_|yB}yw$J{g7UcKm6D=CKW@XdAG|F(*6$`JF={UpruVV`JibqTw9 znLit69P*Un=~0}DJTYs~3s|Wjwyo464JY&#aQ8OJ)Y|3Bg@H-i{Wbr)&o-98og5 zlDtQcfD5ZhDKNgwd4rD8^79?CJ5JK&c;aqkp*2C_Wo2c3I!ZOS1HLGW?8|BQw1{?d zzBz_dp}?37^~3EPoJO|*@tztgHW6=e)RV!50HoPx^S~ z=}`tz^15H0W6|}hAKouD0bK$$tOLzkw`&2pdWP_3O&xBYsn`*t zVzQ5(rrE6SHgM>ZyD1iQBgiK#Q4j*7fGbHo-Y`N07e>KZs0V4Am)+9xY;T zk3w73#bvIkY(WOv)C@xtUkmVUR@w$ruGdH~Gi%|ZoGXyVWx@H+!|FwHb7>KGjN^pN zPYzUGu2Ka-zkupIc?VEVP4(*c{S>AQcAkHE1!%;C`ofF+3bxMYW0?7+j!bh2vM4}b zgY_u+G0)bY`p)8Q4tII?nP!&;TSAYLp~08b9chxLbFYMAe6l5ogleWytt~ z;W`YGZ>^x#G4$Q%OkI$;bOqlJpGZ2P{bK5~Nt9B+;S*!komN)4 zrW-C*T)!V?u}a~u^tR#64-ZDn0CV+rfspxFT2{|noU6C;n%Nh75<^XW-i0g-WE*Qe zxrCZ=Z-cJYxhmeNFGNZGp6?}JN%hC|g;agf&xu?SV|Mc$p9eH!ik>D@)Z?yc0)6Ws zUR$2Uet^Q{uuYMq3lB$lXBU=ZQ-)Y$yhd*$qJUS%`2OLc1F~Hshr9k9)xl8IH*dPHk{X$mMPT% zIgZNt#ohb{ks(f$SuH)31EeuW$H1>6n$E<2t0K^k_UJ)~7_P>$x`-bB44d*hIEo&@ z5kAQ{X?SQEnB!KHAk(BZTSZ(4y1;8*o}3Aq$pf*i0i8r%#n=0g#NsbM`(X$6 zIZ>2vNf|b(%?D?Z!TJOUk!5lS|BNB|)47x8>W9O^vwZe#!yGR?MU9Kv-!qgg@E%6; z|CgcIo@FT72P_#yGy($2Y9yX_LVUSSR5ntQF-AB3pIQ2q?AdHKXec$&aM8uih3E zmqEO`S&HA%Zt3znE3L!BHx9x0F!X^sZnPybs!&!Q;MDn_c-t9p(+z;3`!yl zvJ^$mLh=ya*WVt^ragu3o+1_wu&g}ah01(oSM>_!8Jxg6v^MBn5_@5UqPfXA&2FVA3*3C4X zLZWN(crGih)GKq!>=dzE7!pDGVAqPDEC-w$lf@!g2=@jux_AR@YdJn>;8d^cKhZf89<^ zfN<3DH(O(xsstna;0el4!Ep?BG3T2-F_iDCi3CrZHhAW$zQWzC;~&30j-EAW~#S zRc`o8^Ba=S@siY$emv;UiQ@Z6o-ms(*Vu{Rb}pfKE_o-o9vmBol?CaSjVSRFZlZST zo?FNkaE=`Q#0}lzs3umTsfAmYpJ`tC0y`o1%CfYde>Lsnqj#Oxi8B0^USOyhz$EHbHa~C79l|47O5wlWB9tY|BuB;p0 zK+)x(7q{`hKD2TRc`TL0Knq)tHc6KjTmG#7MO6J!b>Fu1&UiMVteGsuJU(dSOB`8~} zaX@+CaCKy6a7rWl-3iHn4%qVl?}X2Di%P8!DTHRmG~K2e=|!G5M!MSWH(kysdxM7( z3Wb-bS-4ZhMhry*H1Mc8VF(|Fhm|CqwLg)J)@*zfw2^I?`BH_e$a2QiXsjr zw)J`HaWnXR30iI|<+X}n`~_tA$n}l4lVGoTp4vl*Vr_2jE^ei1a zpgmvcL|K%roFsX%b9Pa@-Jo6!dM|-s@Jc_tM;A=xwwjdyS<I{4TV>CIPz}OrmdLipN^@j z&`RnVa&9E+`_EEF^ww*AoKdujZ}*dYQQBFa zjGAUQSMq6LyPZm#<&OzuSKs0%MxAyDs2=pxpkD`TP1>bdP6;K`fMAJ4rrLk_?RY| zy!#;P%T4Hzgt*Tvt@+L17vu!Iz{qKsNi$-zHZUJwe!rthtJF86R|4RwPnoe1<6Jl{{8D z7a7G9;SFPDVqK+rv(#|##YDhpmV&>k)TA8r64bDKc7=*iIV$teOY;3wJV2ez|64(o zXdE_10s;IAc#>)bk@>@1K;FDK+ogtT*iGN3Cmg+(G<{+P#rIsSY7Q{$rrhkQ&E+oD zYtDgBFK|?A6PwZD^m6RgwkUZN>}aflhB3P{dw0U96rx!Po0X@?Z&a zQ0OFVLrjWOM|9zf}s<>B66~althqcGF!Pa+ zJp$PR1=j}>G*EC;14B!DQ+t65L zT#(`iSl(GXF&egHN@mh2l1;R{eTbE5rR3P( zfu{OYV`S=1J6YB$Ja2c5*Iv<(P{Bj)bqCd&pH}6r%$T5JwN{Dl!^Ii7+}QeXYqYnrq!*=e&&_gwj3dLB)xIvfkYHNk+Nak13+ez9_77m-(+F06Fzj@DZleOuX30CKR<^jVBW(JZ?8F5|>2 z+3@sh!wg9vEgT>sZaX_?mFspQ^z;H$TThW8M`APS;&3r z@pyFRFUPWwEKp4!*G31;7!EISBwTmBmtVnCn++1q=883QpYA1p+yB~9wvk2KL4(;r zCnKOV@sJ|w)6gMS19}^zCA?b+se&mksd>UGl(PbLkDU)CsP6m+`1&B9cHmQ+S40)> z;v^`(uw)|^Ox87UH=;YRcBFmgQb) z+jqpiCC+>iZ2g5;HeTQEuyUN;Ipy_94=cA!OW?P$1o1ENYyEqXrNIfUJ0GgYNVxgR zOd?EN-$g9VExLPwVum+;nMu9LZ2d5BASW)7E10Ltz4IAzYT|>NE4{AAFoR0Q23eiO ze9cgcE$QeZg!5YEbnx1_TxcJ&vTx0*#wQyU#9{2 zlZ8%-xyUsSV7lnXSSl<^hG)waJWku+J_eXCRimfnNWmAg+je|$2s-oc4BM0|tYqwM zuL9J?dIxiqs{!6Qe-~HT;Xd-rv1*~xr3%(6gKX7Tq`3oM9@B>S=y8yNS@3AEM$)5) zS?Q%dh<^$&3amWi6slSP=lE<0h+GBL(ZglFzr=l#a8mePLw`KkJO$EUYKZw44K3lQ z%*|YZkU6sJTw^~dO5l+?fNoaYPvr3UREqdxShQx}sJ>V11@$JA6y zT(|s+?%~!iR}~Uk#}A%d8)LNVFjWQ@BBF)vyW$>$v+cDM92tMohbmoe&o9niweU4{ z$KQ#oTO!f7QYFW;mz2I}mgR-lHwSeBV96Kgt*d0&IT3@jZZDlem-n$Qn(>{l9#lDD ze-{o1XnK#p)(X?J$o_A({23Tn{!CNAAu$IlYP~SCA0>!=@&yQ%q;fzuy8`lY8)uex z8tuJI!uEXir}Uk+D8d*vfi};tQk`E5@ajS`KVFP}twx-xPyrJD#{? z=+w>Z#>Yb*7*)|O==w}gm=W(tMx*c!AUo+?FT7QXoEbs8Y2$k9Zxn5hz1 z{WWgrNkwBkF1v)+o=uvNr8G6-s1NPE^Z;owJvU?_DDu(umMB=Gwwt^qN{qO-C0yGf zzHkb~6D3k`^^(y&i4eSrP;CGpKsKQN0jd9T4$?N83mq-RPIxa*a{Z%NP)b0R0p^Wc z1QV1ly};153GLD=_VC*CMB@%{9L+L^Z;}*#7Y51b+$ZlYAoF#bZCkU^Cbp~-=M=g> z_se#c9h)(6FjrugzLg2B>d;t4M?*~!$?97cvK-l`kU_$D-;b$*4qe)8$FL?*`cTz{yH};-yuY0mx8u5927xJM);kBkeC2%Ag~?I;uG&Dj#l$Ib2~dJ< z=jiJ6O?G2OL-Oj>&s_6e=1O~>T%80UvL}MqmNk#|{JOL+4&O^|KEXNq)Hj->gGif< z$-R~K*B}b0w5k-BUNKu&u)3K?{DxH>>L@)`b#0oLB+E?N%*y*FtoEhDs$u>@#%dT2 zek+BQTv*!6ggsd5Zr?e7{?Xy%EzM|OR!Ctk2Xa`@C`6l0Td*96Hy;m}hTS7uP(WxG z8ux7zy5N)|t1(`ybjj-lQXFCO1|c zl~7E2eK4zPJ!cx5ob6zaMFfKbNp>#sYX3yo`LIdgMMcH~U9?iPp)j!obKiQA99eiT<;M|@-b{RkYMthsS7a4x>F?dz05=Q)bLb6D~f{&ldP}<kapx8HDGWm)+)0lVMgXCn{&a<0chM;X7Kv0`3+UX7y)%A?#61DxAu(!Z zh<4x^nFR+Yr^mN3-~O1@7l=d9N}o|3^(77M;vwKphaSW)EL&GvZ6Q)Z&&3RU{S581 zc%Y2CYWF48F8Ru|?+xT`h_Jmf7MUz;-+mf45^`GkT23wc1xKiZVt8)Wcs-jhI?iw> z4F}i+LrRyxNTcl>2+mTy6{(RYzulB%O7hi^RZAKAp69o&HcffKRTD&mGUn1Ze1kex zQ{EekId}OS?y7#ERZW48Eic+H*!@J&eEknbeMq(Kd()2uau-nF?9e2dWbSTz+s|JS zmU6a;okj`EY$(d`l}i5Zffzs!tOOG4vy0aMq0RsNvIPxz(Q-WSO~Mk+Z>3AR;dmI& zi;jK`8nf8S08K>`Vc+2GCkfnQyxWeMYGN%e5ziNf4b*!tJ9Cpc;3Fy+2Tw3^_{{{I znedtf>N!Tkr8hz7Fec}c*5-Yu$`38zgZ7&^I^f>EoyKcUbGD7Yp`l?dU0s z;k;|Kii)pQ+Y%1}%IPD{#+*_J%HbYic_Mby%oRleuQk0Kp+aKh-x5LjmhEHfg@3o) z8tRRBI$Z(7_MFlu^DhUqoW795r8-XWWq5y>I+09&H%xz;yA3<55hmWgzvMfl8?s4q zjzdCH-n;=x@7bQJpzx(q`M8ek($=+cLaYHs^P@}cTy2q< z>LgU1jP4dOtG5-b;Dew&(G33ez^-X$LzvJzJqHiC4_7^CE2^#@E9GYt2>~Ng=?kr9 z%JA(uFB{0&L=?4Up{XP?n1!gTDey@<{oRglN|N+tGcMCJSCu(+%Y_?02nNFBZO`-3xXt9VK6j~ZnG(rC<8=Nzv} zAwa(@=PhUqol=H{ACFoxH*ze(jG?a~n*nlMFcS{5hVb)qyPj@?AQ?9kCpNQK1&y(Y z={P-!eam{mwOudl59g?Od>lqqp$VaaT4Y?em~OzNi6Kvq#p(!sBZ{&iOzl0l2W3R%VJl?B(&YLbNMQZ zZv~5M3PM*-3dvlrbWLWlr1*yzMPgtdjg3Tq$6x^dJ{pjae_&|qTRB+%vZ4k6SQ^(y z)9g=_KJlkAPV-*#Mbfcti7{Sy?n>QuO@Nk(AM~HeqSwPext%A|VqjuA{SL$f1fd1e zA6?iV{nx+3f(C34d-LG4ZRpJLY!(kc2qhqkLY}z?>SM@tI4Go)ws<$|bbMl>P_F zHP<~Uj4n8%71ZaMYlo%G^1Ea*Cbw-`v9snr+W<=ZJsPax#G)^k2I;&k+M)0l)9A{1G*uBSM4zEnEG$AM+eh zA2`AI7uzz=vHxZS__^ot92*nipJD%-9XH_C$8!ervWNb;fBm(k|6J>M&fox$!JkVW z&td+i4t_32Jcrpu{BM}wS0$cfl4|@d+y41j|2gIe@(;`(v#X!`8UG=J`}uzQIdC=V zufYGNqJfWko}b43`=)=c=)cZIKcCD!XG#g@Z!^XAoGD%8FMo%rpO2oNGewR6D^vf6 z+US?l`j>9;*QfRKA;@#a%835I82cY5_BR>(IT(M=7#rCS#{Sdc`sY0QIUWN!;Lqvw zb2IpxN&e?V`8miW5afrC{><*b`sm+;{5ffR4oMH($@>lRA9J_=fRsN+N6#V0Uj7B- zKL$zv0mz>No#&9Hz{~XyDgVtF1nf9GXQ#fX@t?c4 zU$FDnPX4nG`kbA3#=pVN|8VWU8KHkRg`Ts6!Sa)x|3eu3MPYw4+Wah;pR*Lk4lFMJ zETEs;g=X*Hx}p6nnxDfQ{|WP>g#Hge{j8mzLn(0lw?xmX+y4{TU)%f7`uRCn3lQu_ vnEqez{^s5KXVLr|kNpkc&l38%Q2*Ao3Iezb_D6M01mHpf0OEl^1pxd%fv;J% literal 0 HcmV?d00001 diff --git a/Examples/ThreemaAndroidBackupSynthetic/threema-backup_synthetic_no_media.zip b/Examples/ThreemaAndroidBackupSynthetic/threema-backup_synthetic_no_media.zip new file mode 100644 index 0000000000000000000000000000000000000000..f7ac864c280069efa268086c99544fe64c6c142f GIT binary patch literal 9878 zcmb`MbySt#w)Zy;o9+~ml$4N0>F#b2Hr*|aASEFp-6dVpAf1A+l@{rS4Jr*%@@~&L z%6`Q&?!AuR!|Tx=X};&-!-R-JOUyY01f~G97^pN0HDtn)VSdQ03ZMzU}|D# z=jg@;a`k{qK>@%6L?sbW0gy&#$6dpbN*@38`b?=SU>Db#JQ{cLx?~FqlE6iQEI>hG z7haEm+kmZvIg!UR^Hvy|L{S!~!x&M4%Dm7Tb)pNFP)RI26aTI6BTN94V1%^n84(aw9mI(BMFz zhWjfx#vm(4YmoVGF`_x_(*K$y9&8PU^sp6R=~0$GsvPHov|XeuEzY?rMw#*{?T}|Z zgDkYn3)w1k>IUzQhM~RC;mlfyWENHHse3JuzljxlCOEVDlDNwWn+CyqDNA|=`B~A@ zu%_HQ;y}l#*H-CGpJ~ZW{WCX^kQ6Na0NusNFG$G{ecE;YC8Q9A?J;>j6l5J z%HZd5Zl&ktYmP-{X7(}GZ7+^?cu>f$Z%EnJEf1?h`3&srCK>Dj$x}*v8K7- z&B(#<%9mM1Qu@ZLt&=uD*rwJbI^U+IiXr3l#9pr?Kw;QC?&#H=UZ|qRJz0l6&~4TBs@+hnO+;#B$vsb3 zt$c$o-AxM_7N(cdM=NSpyO9XIM4!=4MM`hRssy^ZpF@rGI#$Le9wu%kF2?LXAC6!9 z9RTp!OGuSj)blUx5gOhvnM6b|0!Op%)oU&4i7+c031SSQ=BTq+%vBvHFg6P|H@tsX z9z-iG#ojA^4^*x2;7Sji@k$Mq$8GR{> zeQMECPD573@XV~qLjM5s+i)GDtV;G87N+Gtq?g9`XM z>4KPzv{mOIfYCm^Nk(zIF|n6xtBcq$jt{F zx#+w#Zb)V%iLx4XVVfPUh)mt?7 z^vgS|VwDcopKLA5j^~Zko;^Lo(C;;>Z)Ow2@mg0Un7C)rg}TRl2W#w}`QsV0gLTxj z334?2_pIa^{s6x{<@CvgJ-G{`8WY8qAde5xceixjxC>Jz+|~DdMvo?|!VLGU4~twu z?ihJ>WQ};)1BO+Fr*5xVOv<2Oa5~#l|PS45SM4=v}8n49?0% zm#E%Xnk>i1adyK0Sgj=9mr0C$kwFq~bU$HKqjfZs`qTL=FQe+Q)#6-eK!+shusx?0 z)6S=i(6OlilDC_TN7@!b-OIixPsHE1(@rdSJyKjXb|s)njif-z@yowYbO>>~Gn_3t zlz65^m{e82U%wP7=}z z>0m0+X431x37#y-x0npWllT_#(j~K%x)u;7Qb9>3WbqWiPRmSKwMHf-j0iimXFPCM z%6D#0a$kV%X12?qnSKE^N|kMB|yPv9js$YlcSi zqQ3Zw3qHABU&qJYr@hZH&LwlvpAYMA`Ai=V>U1k)Dzlz>ELcreczY4jBx3fMn5=Dn z(y36p@(*fj9$4(YWfNKe?AI|4?Gucx#vTFghYS!-$o0%fmiS4Ev4#$1?5gaCH#u87 z_-PcTPnHRp@QmyZe(=7S4!BUOodYt5U!ErPj1q)X@ptk|sPq|qRGgvJF-il{2n0&tdtZY z`h^BErT2ZtBW@bpHrl42iVIDV7O}64Z>g%rO)YFa!1fUKa!%uGFMAVNsGMIOe@a3& zJ+kz~GF7`SWv&B3VncE>Ur=eR?6$rh{qx1=s?we5pdHe0#l|n^A2U6dwwYWsO5yZW z=OItyl^&nO_XIH9zmONk(yElTBUll9qIu=@tEor^YaWmy9 z(1ULXYJ@N;cQJ<^f7Xr;zn_3PoK;i_Rs9KLdB-)4$Q*(AmaL;nKCvlxU3e{615R2) zr*LHA#fu|$mNSJta;py%8=k7IYJSC>KzrwoftM&~wN&q>vdJCvTvMHdyY#G#)`sd5 zw@Y~OeDc{$TAqFw?Uq-yb28k^+mc9`&99>J^@=w6T57cr3N17qymdzPAqM)My>c_+ zmFnAHt#W$3y%#v7r1yv^$oi&R*-)3N{^ORJwX2(pwW<3LmyGS8fBbf^Q+&8vfn{vO z@lp(__v79{%B3V`mBS!wWSl`DX2mrusXwq(YXKZio>S|(XZp`tUpDmg!8|*FQF-_;0lDCUQjLZ&gV!$L%WFztt zFcVXRAfamP$ostrzF2iyrQDY@%O>q2So<|(Z0EYe$Eufls-r(8Zl} zBLZumGoazD)M-_YHn<$~S>M`Df6rr9qZ-gtqgY5M z;P_h74{h^wP{r+AhQHcRY}*v3TUb!fgKx$2wvsXk+{^0~m`Q0!cDR4$*4Pk3PdN=J z=4tz$+tWDd%cjxuwrz|?<_xzwwO&P}KC0km&b~swMhR%%eq4(zT^!w=e&1cQ9#+-v z{@fFUZn-3u2!V-oHPm<1$>_8PO?we#L*Sp40rX$u6Qq zgob(yD4*bz=gtq`;S6EfkXtq z>zO)qe{rt_M`NASEq!CY7KyzG95Nrfo#@kFcQ$#4vw_e`IOo$DA1-cj-hr$1C(pA* z)asA$vlm>YVv2&T2l(f@3a81p^s6eu);l)LX986r{D;!CYCUpiY3}f@=X39SrEkS; z6SSIpJq0Yk^sE9Lq<$_|R+<^4N~?Gl)_HJMCWAdL8O-~Ac$$EuL0I<5+CVJOZH=4d zf!la93H%}bi|y);w(sRZ|bNLo-@`i zm0`4(rZ!V)Q@~8heIS z4iv{Z2MK14g*to&o_bW`kf=)9p70GuS}mgq1y%YlVCND%DsA)7DO}pm%0QdnMK@uq zyqV)1=oKpq4FI6{mE&K<>Q|C~t6CJuU5t;?*YV%vj6e6m?6B0;rt}&U)>Tz;#^0*Q zX1{w7w0FuKKsD(i0;(~KF&`fWeTPiQ@{Fy^V+Ck`e8b%JroV*9GpcZ(yDd74h9anF zR4|)x-ZjSX;dF4}^hxB^O6rMVlS*gY0%m5DWz_qvuGvm1-oh$fjWQVla8HLXDWQP; zXe6OqNkF8E2)^SbhGI#^6AnDp=e}?c`~0ox6tLb+?Me;S?c~LA6fu72?1N7=@{WWQ z&Ja2!c`=w5t^k+T7I0fksW!S-%6F?~L+Zj-Dw3EBhEx~Hdx^D3%|>WK7eyf-CN5%X zlgJks0apk)vkr?ZGtO$l`?D)d5@C^1$M_?nZtwBnb?VUzKdA@5B*4J4bt~3SpJ{&< zDG;js1eL<|WY6#l4pNEtb#+Ufn2#UZnQCZ=a^gK^JwuCLG8y43wy)JT*-uqnY};<5 zPrsfIunu+%P|ySm_cf*KnZi?{OlkB#`WkHy)}UnHH3jmj1pzRv5J6(MjKsK{(C$qkZXsALRCJWy> zKvUXQ3a&`t$LUcySxl;4v`4e|ISLqVe&wyuQjZ0#Wca_n^8co&VkA*F773RZmJA7U zcd*_i3>x24po-7-yWb)nUXryKc!qA&UDh1Y1(wte)}lt`@+;RS%NTkvURrI4SgV+c z*>|i7eAK0%TdW>5We~n5i=Qr2%Ilyi)dP5*uKv}|k(?a_6(GCyt*fDC zv)1F)MHd3v(US{I&=z4fZoOYVF~>-|I!dz8R1z~Ur~h5Lt$ohd%QGhID7Y3KMz05o zHbxsUR6Qx)=Ll1HS-xSXHutgV6QeH2tBEV3`9BQ>e84SRUKa@UrcLP z$n41SiI54=0!^6Jv)FIU4@V>V-q+Mf{&tKk{f|gzE`r(*=~E~5*jXk#t^~!za%w}e)$<-VfabR)BQA&?OPj|djn6o0h5eN- zKX{O;(LQEHIO+Qc=k=VY31m+KQJ`UCdfzrdYw+27NNWFHBrWNH`c%FE`$t?pQj*%$ zq>am6^)Fx+3Indbs~3kKCGLU6?%W3E*(!@ohw%U}$Xtvx$RGE-!VOj_Wrz5O-5<7( z4-L*DIZEf0aUy&h85+$fD@!`@Xn8!tpO~!uTtIbzUWxdQq5;9g(Z@XPQed7O0$ZA& z2T6&+*)tljoeN*iCwR1XrH0=*r07h~X(fkDOeN(jnEY(MPF|TyUkK81(qD#@{4@dXqp497Sde6k!30w_l7MMW> z_t(j{e9KS1Pk%FPgvbwsA;oK72Fz9^X0JgJ~k^m&kEeh*EgFN3tAd#~YbY zxi8|Xg2HY(8TI5UmDlVxnicUvE4t!IWOLh*$#eCq^X{DQF4-hwk z!eocQ{)!qdvPZ|YKyJ|vBB!QmE`4}JX4Ul%J-0)VdFVvTF=JWzegRe(jJefZ`G8HlDflstU!}=s7s-rMV_G5G~t|W!A3n;Fq+wb z&}HuhX6=jF8;DT!RM$X92^gsjx+~7MyPVWd#Ku79X9NhII#YhcbkFXV@@sbCmGp4` z6ZK2*PE${P4UbGoV(8dNzQ{>=G1~?uK@%qkq}uLA8f_4SE&7DRoppf2b{xkrVvi*OM`pE08*c> zDI#TLzbWy|-pXwg-tJgJRoT3`YDko)iRisIj4B&sh>W&bm@mzlI+%_VA4%PY;E1ql zqj(ai9pB8*kLklc)c%+uuHP~=C!*lkzu;&hLk38 zU$lo+s`ScxF}eB!4jLtvT8+Bsds%v;n#wZ;zMNM$Z{TIt+0i!U5+J&~$x{*=!b0!! z1Gf{$+I(J>IlR3qxP74*i8?}9(aR)d7X4yh+bT6yG!S=f_j%u1cG?zE z#@;?Vki-bLeWIGcHsQMcB~9w~li{3sY-bCz`=u+cX9>=&^&J|jgEt|pP=Er|uzo>| z%{a||iyS6kh#;$5J9nx4^D72=ii({A&h$A#jTn(RCa3l{xmsqdc{FR%+h%8e@J{+S z8SK#a6F?2;C&L)z=8fJsZ$x z_=tSbhN$<_>772jbZ}AlO^`4Y!UDA)I_0NA`4bBjVB^?Gi|iaIRV{#FVzVd9vH9WB zQc8f29SnDF$zo>HLB{ZB@;qhDgep|6pk_=k%_w z)%UzmVipy7ICx>ee;YyGgED?RRR8)7fEh>v|MNdM014ne@B2M^f;njY`v0F_D?$bQ z33ENngT>r}zUJR^Pngl(KR~-4vBA=gu4z99Zm_J~$A9J7^*9WcRR_Jj!4Ah@LFz%^ zzX@Rpbou|DCBcH0pr^=h(}rt}21`Xm`X8u(KPK*assT#{Bi*3doBso}>#jO1tr_K- z_U9fuEIO9uuN=MZ?!uxgpf~WJ;r+Sq|5YquiGTU!WJ7n*zZ7khnah2@I(L2T zgoR{6kNKZ*ye@FR$MG+b*K!;dc}n~T7v`r5{0C6i@(&gzO8y^@Uk Date: Wed, 15 Jul 2026 14:53:16 +0200 Subject: [PATCH 2/3] feat(threema_android): implement media parsing and exporting from Android backups --- CHANGELOG.md | 5 + .../ThreemaAndroidBackupSynthetic/README.txt | 6 + README.md | 36 +- packaging/windows/ChatExportPDF.spec | 9 +- pyproject.toml | 3 +- src/chat_export/__init__.py | 2 +- src/chat_export/cli.py | 37 +- src/chat_export/config.py | 15 +- src/chat_export/config_factory.py | 22 +- src/chat_export/constants.py | 2 + src/chat_export/export_summary.py | 17 +- src/chat_export/gui.py | 68 +- src/chat_export/orchestrator.py | 8 +- src/chat_export/render/pdf_builder.py | 47 +- src/chat_export/threema_android/__init__.py | 1 + src/chat_export/threema_android/archive.py | 177 +++++ src/chat_export/threema_android/identity.py | 138 ++++ src/chat_export/threema_android/importer.py | 629 ++++++++++++++++++ src/chat_export/threema_android/media.py | 192 ++++++ src/chat_export/threema_android/models.py | 144 ++++ src/chat_export/threema_android/normalize.py | 457 +++++++++++++ tests/__init__.py | 1 + tests/test_threema_android_backup.py | 264 ++++++++ 23 files changed, 2235 insertions(+), 45 deletions(-) create mode 100644 src/chat_export/threema_android/__init__.py create mode 100644 src/chat_export/threema_android/archive.py create mode 100644 src/chat_export/threema_android/identity.py create mode 100644 src/chat_export/threema_android/importer.py create mode 100644 src/chat_export/threema_android/media.py create mode 100644 src/chat_export/threema_android/models.py create mode 100644 src/chat_export/threema_android/normalize.py create mode 100644 tests/__init__.py create mode 100644 tests/test_threema_android_backup.py diff --git a/CHANGELOG.md b/CHANGELOG.md index bd581db..892742c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## [0.8.0] - 2026-07-15 +### Added +- Added import support for encrypted Threema Android data-backup ZIP files, including conversations, participants, messages, reactions, polls, and optional media extraction. +- Added the Threema Android backup source and password field to the Windows GUI and CLI. + ## [0.7.0] - 2026-04-22 ### Added - Added `export_summary.txt`, `manifest.json`, and `log.txt` for every export run. diff --git a/Examples/ThreemaAndroidBackupSynthetic/README.txt b/Examples/ThreemaAndroidBackupSynthetic/README.txt index c555882..2251e87 100644 --- a/Examples/ThreemaAndroidBackupSynthetic/README.txt +++ b/Examples/ThreemaAndroidBackupSynthetic/README.txt @@ -39,3 +39,9 @@ Regeneration requirements: Regenerate with: python .\Examples\ThreemaAndroidBackupSynthetic\build_fixture.py + +Run the importer regression tests with: + python -m unittest discover -v + +Run a manual export with: + python -m chat_export --source threema_android_backup --input-path .\Examples\ThreemaAndroidBackupSynthetic\threema-backup_synthetic.zip --backup-password testpassword --out-dir .\Examples\Output\threema_android_backup diff --git a/README.md b/README.md index 1488a94..410ba9d 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,8 @@ ChatExportPDF exports chat data from messenger sources into PDF reports. Current sources: -- `threema` (iOS only) +- `threema` (iOS database) +- `threema_android_backup` (encrypted Android data-backup ZIP) - `whatsapp` For each conversation, the exporter creates: @@ -11,7 +12,7 @@ For each conversation, the exporter creates: - a `*_TECH.pdf` report - optionally, an Excel workbook with structured conversation data -The main report is generic. The TECH report can be importer-specific. Threema has its own dedicated TECH report. WhatsApp currently uses the generic fallback TECH report. +The main report is generic. The TECH report can be importer-specific. The Threema iOS database importer has its own dedicated TECH report. Threema Android backups and WhatsApp use the generic fallback TECH report. ## Desktop GUI @@ -20,8 +21,9 @@ For Windows, use the standalone desktop app from the [Releases page](https://git 1. Download `ChatExportPDF.exe`. 2. Start the EXE. No Python installation is required. 3. Select the source: - - `threema`: choose `ThreemaData.sqlite`; optionally choose `_EXTERNAL_DATA`. - - `whatsapp`: choose the exported WhatsApp `.zip`. + - `Threema iOS database`: choose `ThreemaData.sqlite`; optionally choose `_EXTERNAL_DATA`. + - `Threema Android backup`: choose the encrypted data-backup `.zip` and enter its password. + - `WhatsApp export`: choose the exported WhatsApp `.zip`. 4. Choose or keep the suggested output folder. 5. Start the export. @@ -50,6 +52,14 @@ Typical iOS paths: - `/private/var/mobile/Containers/Shared/AppGroup//ThreemaData.sqlite` - `/private/var/mobile/Containers/Shared/AppGroup//.ThreemaData_SUPPORT/_EXTERNAL_DATA` +### Threema Android Data Backup + +- encrypted data-backup `.zip` created by the Threema Android app +- backup password selected when the backup was created +- current backup format version `27` +- direct chats, groups, distribution lists, participants, messages, reactions, polls, and attachment references +- media payloads are extracted when they were included in the backup and media export is enabled + ### WhatsApp - exported `.zip` file @@ -110,6 +120,16 @@ Recommended with external attachment folder: python -m chat_export --source threema --input-path "./ThreemaData.sqlite" --out-dir "./export" --external-folder "./_EXTERNAL_DATA" ``` +### Threema Android Data Backup + +The CLI prompts for the backup password without displaying it: + +```bash +python -m chat_export --source threema_android_backup --input-path "./threema-backup.zip" --out-dir "./export" +``` + +For non-interactive use, the password can be supplied with `--backup-password`. Be aware that command-line arguments can remain in shell history. + ### WhatsApp WhatsApp export: @@ -143,12 +163,13 @@ The preferred CLI entry points are: ### Source Selection -- `--source {threema,whatsapp}` +- `--source {threema,threema_android_backup,whatsapp}` Selects the importer. Default: `threema`. - `--input-path PATH` Generic input path for the selected source. - for `threema`: path to `ThreemaData.sqlite` + - for `threema_android_backup`: path to the encrypted Android data-backup `.zip` - for `whatsapp`: path to the exported `.zip` - `--db-path PATH` @@ -157,10 +178,13 @@ The preferred CLI entry points are: - `--chat-text-name NAME` Only relevant for WhatsApp. Forces a specific chat text file inside the ZIP when multiple plausible `.txt` files exist. +- `--backup-password PASSWORD` + Only relevant for Threema Android data backups. If omitted in an interactive terminal, the CLI prompts for it securely. + ### Optional - `--external-folder PATH` - Threema only. Path to `_EXTERNAL_DATA`. Recommended for complete attachment export. + Threema iOS database only. Path to `_EXTERNAL_DATA`. Recommended for complete attachment export. - `--tz TIMEZONE` Timezone for rendered timestamps. Default: `Europe/Zurich`. diff --git a/packaging/windows/ChatExportPDF.spec b/packaging/windows/ChatExportPDF.spec index 09834ff..8fe03dd 100644 --- a/packaging/windows/ChatExportPDF.spec +++ b/packaging/windows/ChatExportPDF.spec @@ -20,7 +20,14 @@ datas = [] binaries = [] hiddenimports = [] -for package in ("reportlab", "PIL", "xlsxwriter", "emoji", "tzdata"): +for package in ( + "reportlab", + "PIL", + "xlsxwriter", + "emoji", + "pyzipper", + "tzdata", +): package_datas, package_binaries, package_hiddenimports = _collect_optional(package) datas += package_datas binaries += package_binaries diff --git a/pyproject.toml b/pyproject.toml index 55536ac..d938e39 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "chat-export-pdf" -version = "0.7.0" +version = "0.8.0" description = "Export chat conversations from multiple sources to PDFs with optional media extraction" readme = "README.md" requires-python = ">=3.10" @@ -15,6 +15,7 @@ dependencies = [ "reportlab>=4.0.0", "XlsxWriter>=3.2.0", "emoji>=2.0.0", + "pyzipper>=0.3.6", "tzdata>=2023.3; platform_system=='Windows'", ] diff --git a/src/chat_export/__init__.py b/src/chat_export/__init__.py index bf48c60..a53c8bb 100644 --- a/src/chat_export/__init__.py +++ b/src/chat_export/__init__.py @@ -1,4 +1,4 @@ """Top-level package metadata for the chat export application.""" __all__ = ["__version__"] -__version__ = "0.7.0" +__version__ = "0.8.0" diff --git a/src/chat_export/cli.py b/src/chat_export/cli.py index 6fe8198..1187179 100644 --- a/src/chat_export/cli.py +++ b/src/chat_export/cli.py @@ -4,11 +4,17 @@ import argparse import logging import sys +from getpass import getpass from typing import Optional from .common.logging_setup import sanitize_local_paths, setup_logging from .config_factory import build_export_config -from .constants import DEFAULT_SOURCE_APP, DEFAULT_TIMEZONE, SOURCE_APPS +from .constants import ( + DEFAULT_SOURCE_APP, + DEFAULT_TIMEZONE, + SOURCE_APPS, + SOURCE_APP_THREEMA_ANDROID_BACKUP, +) from .orchestrator import export_all_conversations @@ -26,7 +32,7 @@ def build_parser() -> argparse.ArgumentParser: "--source", default=DEFAULT_SOURCE_APP, choices=SOURCE_APPS, - help="Source app / importer to use (currently supported: threema, whatsapp)", + help="Source app / importer to use", ) p.add_argument( "--input-path", @@ -43,6 +49,11 @@ def build_parser() -> argparse.ArgumentParser: default=None, help="For WhatsApp ZIPs with multiple plausible .txt files: exact chat text filename to use", ) + p.add_argument( + "--backup-password", + default=None, + help="Password for a Threema Android data-backup ZIP; omit for a secure prompt", + ) p.add_argument("--out-dir", required=True, help="Output directory") p.add_argument( "--external-folder", @@ -123,6 +134,17 @@ def main(argv: Optional[list[str]] = None) -> int: """ parser = build_parser() args = parser.parse_args(argv) + backup_password = args.backup_password + if ( + args.source == SOURCE_APP_THREEMA_ANDROID_BACKUP + and not backup_password + and sys.stdin.isatty() + ): + try: + backup_password = getpass("Threema Android backup password: ") + except (EOFError, KeyboardInterrupt): + print("Backup password input cancelled.", file=sys.stderr) + return 1 try: cfg = build_export_config( @@ -131,6 +153,7 @@ def main(argv: Optional[list[str]] = None) -> int: input_path=args.input_path, db_path=args.db_path, chat_text_name=args.chat_text_name, + backup_password=backup_password, external_folder=args.external_folder, tz_name=args.tz, export_media=not args.no_media, @@ -165,7 +188,12 @@ def main(argv: Optional[list[str]] = None) -> int: return 1 log = logging.getLogger("chat_export") - log.debug("Parsed CLI args: %s", vars(args)) + log.debug( + "Parsed CLI options source=%s input_configured=%s output_configured=%s", + args.source, + bool(args.input_path or args.db_path), + bool(args.out_dir), + ) log.info("Starting export source=%s tz=%s", args.source, args.tz) log.debug( "Export options source=%s media=%s image_previews=%s excel=%s max_media_bytes=%s limit_conversations=%s limit_messages=%s chat_text_name=%s", @@ -190,7 +218,8 @@ def main(argv: Optional[list[str]] = None) -> int: try: res = export_all_conversations(cfg) except Exception as e: - log.exception("Export failed: %s", e) + log.error("Export failed: %s", e) + log.debug("Export failure details", exc_info=True) return 1 log.info( diff --git a/src/chat_export/config.py b/src/chat_export/config.py index 4b7b1c2..6c00ff7 100644 --- a/src/chat_export/config.py +++ b/src/chat_export/config.py @@ -1,11 +1,15 @@ """Define export configuration and validation.""" import logging -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from typing import Optional -from .constants import DEFAULT_SOURCE_APP, DEFAULT_TIMEZONE +from .constants import ( + DEFAULT_SOURCE_APP, + DEFAULT_TIMEZONE, + SOURCE_APP_THREEMA_ANDROID_BACKUP, +) log = logging.getLogger(__name__) @@ -20,6 +24,7 @@ class ExportConfig: input_path (Optional[str]): Generic source input path. db_path (Optional[str]): Threema SQLite database path. chat_text_name (Optional[str]): Explicit WhatsApp chat text file name. + backup_password (Optional[str]): Password for an encrypted source backup. external_folder (Optional[str]): Threema external media directory path. tz_name (str): IANA timezone name for rendered timestamps. export_media (bool): Enable attachment export. @@ -40,6 +45,7 @@ class ExportConfig: input_path: Optional[str] = None db_path: Optional[str] = None chat_text_name: Optional[str] = None + backup_password: Optional[str] = field(default=None, repr=False) external_folder: Optional[str] = None tz_name: str = DEFAULT_TIMEZONE @@ -94,6 +100,11 @@ def validate(self) -> None: raise ValueError("source_app must not be empty") if not self.out_dir or not self.out_dir.strip(): raise ValueError("out_dir must not be empty") + if ( + self.source_app == SOURCE_APP_THREEMA_ANDROID_BACKUP + and not self.backup_password + ): + raise ValueError("backup_password is required for Threema Android backups") out = Path(self.out_dir) out.mkdir(parents=True, exist_ok=True) diff --git a/src/chat_export/config_factory.py b/src/chat_export/config_factory.py index 3a34f31..0419912 100644 --- a/src/chat_export/config_factory.py +++ b/src/chat_export/config_factory.py @@ -5,7 +5,12 @@ from typing import Optional from .config import ExportConfig -from .constants import DEFAULT_TIMEZONE, SOURCE_APP_THREEMA +from .constants import ( + DEFAULT_TIMEZONE, + SOURCE_APP_THREEMA, + SOURCE_APP_THREEMA_ANDROID_BACKUP, + SOURCE_APP_WHATSAPP, +) from .export_summary import default_log_file @@ -49,6 +54,7 @@ def build_export_config( input_path: Optional[str], db_path: Optional[str] = None, chat_text_name: Optional[str] = None, + backup_password: Optional[str] = None, external_folder: Optional[str] = None, tz_name: Optional[str] = None, export_media: bool = True, @@ -70,6 +76,7 @@ def build_export_config( input_path (Optional[str]): Generic source input path. db_path (Optional[str]): Explicit database path override. chat_text_name (Optional[str]): Explicit WhatsApp chat text file name. + backup_password (Optional[str]): Password for an encrypted source backup. external_folder (Optional[str]): Optional Threema external folder path. tz_name (Optional[str]): Optional timezone override. export_media (bool): Enable media export. @@ -89,6 +96,7 @@ def build_export_config( effective_input_path = _optional_text(input_path) effective_source = source_app.strip() effective_chat_text_name = _optional_text(chat_text_name) + effective_backup_password = backup_password if backup_password else None effective_external_folder = _optional_text(external_folder) effective_tz_name = (tz_name or "").strip() or DEFAULT_TIMEZONE effective_db_path = _optional_text(db_path) @@ -102,6 +110,17 @@ def build_export_config( if not effective_input_path: raise ValueError("input_path must not be empty for non-Threema sources.") + if effective_source != SOURCE_APP_THREEMA_ANDROID_BACKUP: + effective_backup_password = None + if effective_source != SOURCE_APP_WHATSAPP: + effective_chat_text_name = None + + if ( + effective_source == SOURCE_APP_THREEMA_ANDROID_BACKUP + and not effective_backup_password + ): + raise ValueError("backup_password is required for Threema Android backups.") + effective_out_dir = out_dir.strip() if not effective_out_dir: raise ValueError("out_dir must not be empty or whitespace-only.") @@ -113,6 +132,7 @@ def build_export_config( input_path=effective_input_path, db_path=effective_db_path, chat_text_name=effective_chat_text_name, + backup_password=effective_backup_password, external_folder=effective_external_folder, tz_name=effective_tz_name, export_media=export_media, diff --git a/src/chat_export/constants.py b/src/chat_export/constants.py index 422fad3..357582b 100644 --- a/src/chat_export/constants.py +++ b/src/chat_export/constants.py @@ -1,10 +1,12 @@ """Define shared application constants.""" SOURCE_APP_THREEMA = "threema" +SOURCE_APP_THREEMA_ANDROID_BACKUP = "threema_android_backup" SOURCE_APP_WHATSAPP = "whatsapp" SOURCE_APPS = ( SOURCE_APP_THREEMA, + SOURCE_APP_THREEMA_ANDROID_BACKUP, SOURCE_APP_WHATSAPP, ) diff --git a/src/chat_export/export_summary.py b/src/chat_export/export_summary.py index ceb1c37..90d344c 100644 --- a/src/chat_export/export_summary.py +++ b/src/chat_export/export_summary.py @@ -11,6 +11,7 @@ from . import __version__ from .common.logging_setup import sanitize_local_paths from .config import ExportConfig +from .constants import SOURCE_APP_THREEMA, SOURCE_APP_WHATSAPP TOOL_NAME = "ChatExportPDF" REPOSITORY_URL = "https://github.com/JeromeMaag/ChatExportPDF" @@ -351,6 +352,8 @@ def _settings(cfg: ExportConfig, results: dict[str, Any] | None) -> dict[str, An "selected_source": cfg.source_app, "timezone": cfg.tz_name, "time_mode": (results or {}).get("time_mode", "unknown"), + "backup_version": (results or {}).get("backup_version"), + "source_owner_identity": (results or {}).get("self_identity"), "media_export_enabled": cfg.export_media, "image_previews_enabled": cfg.export_image_previews, "excel_export_enabled": cfg.export_excel, @@ -360,8 +363,16 @@ def _settings(cfg: ExportConfig, results: dict[str, Any] | None) -> dict[str, An "log_file": LOG_FILENAME, "console_and_gui_log_level": "INFO", "log_file_level": "DEBUG", - "chat_text_name": cfg.chat_text_name or "auto", - "external_folder_provided": bool(cfg.external_folder), + "chat_text_name": ( + cfg.chat_text_name or "auto" + if cfg.source_app == SOURCE_APP_WHATSAPP + else None + ), + "external_folder_provided": ( + bool(cfg.external_folder) + if cfg.source_app == SOURCE_APP_THREEMA + else None + ), } @@ -471,6 +482,8 @@ def build_summary_text(manifest: dict[str, Any]) -> str: _line("Selected source", settings["selected_source"]), _line("Timezone", settings["timezone"]), _line("Time mode", settings["time_mode"]), + _line("Backup version", settings["backup_version"]), + _line("Source owner identity", settings["source_owner_identity"]), _line("Media export enabled", settings["media_export_enabled"]), _line("Image previews enabled", settings["image_previews_enabled"]), _line("Excel export enabled", settings["excel_export_enabled"]), diff --git a/src/chat_export/gui.py b/src/chat_export/gui.py index ebf0cba..bbe2813 100644 --- a/src/chat_export/gui.py +++ b/src/chat_export/gui.py @@ -21,10 +21,20 @@ DEFAULT_TIMEZONE, SOURCE_APPS, SOURCE_APP_THREEMA, + SOURCE_APP_THREEMA_ANDROID_BACKUP, + SOURCE_APP_WHATSAPP, ) from .orchestrator import export_all_conversations +SOURCE_LABELS = { + SOURCE_APP_THREEMA: "Threema iOS database", + SOURCE_APP_THREEMA_ANDROID_BACKUP: "Threema Android backup", + SOURCE_APP_WHATSAPP: "WhatsApp export", +} +SOURCE_KEYS_BY_LABEL = {label: key for key, label in SOURCE_LABELS.items()} + + class QueueLogHandler(logging.Handler): """Forward formatted log records to a GUI queue.""" @@ -99,11 +109,12 @@ def __init__(self, root: tk.Tk) -> None: def _build_variables(self) -> None: """Create Tkinter state variables.""" - self.source_var = tk.StringVar(value=DEFAULT_SOURCE_APP) + self.source_var = tk.StringVar(value=SOURCE_LABELS[DEFAULT_SOURCE_APP]) self.input_path_var = tk.StringVar() self.out_dir_var = tk.StringVar() self.external_folder_var = tk.StringVar() self.chat_text_name_var = tk.StringVar() + self.backup_password_var = tk.StringVar() self.tz_var = tk.StringVar(value=DEFAULT_TIMEZONE) self.export_media_var = tk.BooleanVar(value=True) self.export_image_previews_var = tk.BooleanVar(value=True) @@ -137,7 +148,7 @@ def _build_layout(self) -> None: self.source_combo = ttk.Combobox( basic, textvariable=self.source_var, - values=SOURCE_APPS, + values=tuple(SOURCE_LABELS[source] for source in SOURCE_APPS), state="readonly", ) self.source_combo.grid(row=0, column=1, sticky="ew", padx=(0, 8), pady=(8, 6)) @@ -163,6 +174,13 @@ def _build_layout(self) -> None: self.external_entry = ttk.Entry(basic, textvariable=self.external_folder_var) self.external_button = ttk.Button(basic, text="Browse...", command=self._browse_external_folder) + self.backup_password_label = ttk.Label(basic, text="Backup password") + self.backup_password_entry = ttk.Entry( + basic, + textvariable=self.backup_password_var, + show="*", + ) + options_row = ttk.Frame(basic) options_row.grid(row=4, column=0, columnspan=3, sticky="w", padx=8, pady=(2, 8)) self.export_media_check = ttk.Checkbutton( @@ -370,13 +388,40 @@ def _on_out_dir_changed(self, *_args: object) -> None: def _refresh_source_fields(self) -> None: """Update source-specific labels and field visibility.""" - source = self.source_var.get() + source = self._selected_source() if source == SOURCE_APP_THREEMA: self.input_label.configure(text="Threema SQLite DB") self.source_hint_var.set("Threema: select ThreemaData.sqlite. _EXTERNAL_DATA is optional but recommended.") self.external_label.grid(row=3, column=0, sticky="w", padx=(8, 8), pady=6) self.external_entry.grid(row=3, column=1, sticky="ew", padx=(0, 8), pady=6) self.external_button.grid(row=3, column=2, sticky="ew", padx=(0, 8), pady=6) + self.backup_password_label.grid_remove() + self.backup_password_entry.grid_remove() + self.chat_text_label.grid_remove() + self.chat_text_entry.grid_remove() + elif source == SOURCE_APP_THREEMA_ANDROID_BACKUP: + self.input_label.configure(text="Threema Android backup") + self.source_hint_var.set( + "Threema Android: select an encrypted data-backup ZIP and enter its password." + ) + self.external_label.grid_remove() + self.external_entry.grid_remove() + self.external_button.grid_remove() + self.backup_password_label.grid( + row=3, + column=0, + sticky="w", + padx=(8, 8), + pady=6, + ) + self.backup_password_entry.grid( + row=3, + column=1, + columnspan=2, + sticky="ew", + padx=(0, 8), + pady=6, + ) self.chat_text_label.grid_remove() self.chat_text_entry.grid_remove() else: @@ -385,6 +430,8 @@ def _refresh_source_fields(self) -> None: self.external_label.grid_remove() self.external_entry.grid_remove() self.external_button.grid_remove() + self.backup_password_label.grid_remove() + self.backup_password_entry.grid_remove() self.chat_text_label.grid() self.chat_text_entry.grid() @@ -402,7 +449,7 @@ def _toggle_advanced(self) -> None: def _browse_input_path(self) -> None: """Open the source-specific file picker.""" - source = self.source_var.get() + source = self._selected_source() if source == SOURCE_APP_THREEMA: filetypes = [ ("SQLite database", "*.sqlite"), @@ -553,7 +600,7 @@ def _build_config(self) -> ExportConfig: Raises: ValueError: If required fields are missing or invalid. """ - source = self.source_var.get().strip() + source = self._selected_source() input_path = self.input_path_var.get().strip() out_dir = self.out_dir_var.get().strip() if not source: @@ -568,6 +615,7 @@ def _build_config(self) -> ExportConfig: source_app=source, input_path=input_path, chat_text_name=self.chat_text_name_var.get(), + backup_password=self.backup_password_var.get(), external_folder=self.external_folder_var.get(), tz_name=self.tz_var.get(), export_media=self.export_media_var.get(), @@ -591,6 +639,11 @@ def _build_config(self) -> ExportConfig: case_description=self.case_description_var.get(), ) + def _selected_source(self) -> str: + """Return the importer key selected in the source combobox.""" + selected = self.source_var.get().strip() + return SOURCE_KEYS_BY_LABEL.get(selected, selected) + def _set_running(self, running: bool) -> None: """Enable or disable controls during export execution. @@ -607,6 +660,7 @@ def _set_running(self, running: bool) -> None: self.output_browse_button, self.external_entry, self.external_button, + self.backup_password_entry, self.chat_text_entry, self.case_number_entry, self.examiner_entry, @@ -644,7 +698,9 @@ def _run_export_worker(self, cfg: ExportConfig) -> None: ) result = export_all_conversations(cfg) except Exception as exc: - logging.getLogger("chat_export.gui").exception("GUI export failed") + gui_log = logging.getLogger("chat_export.gui") + gui_log.error("GUI export failed: %s", exc) + gui_log.debug("GUI export failure details", exc_info=True) self._result_queue.put( GuiResult( ok=False, diff --git a/src/chat_export/orchestrator.py b/src/chat_export/orchestrator.py index 5d74c5b..4ee6a77 100644 --- a/src/chat_export/orchestrator.py +++ b/src/chat_export/orchestrator.py @@ -9,7 +9,11 @@ from .common.logging_setup import sanitize_local_paths from .common.util import ensure_dir, safe_filename from .config import ExportConfig -from .constants import SOURCE_APP_THREEMA, SOURCE_APP_WHATSAPP +from .constants import ( + SOURCE_APP_THREEMA, + SOURCE_APP_THREEMA_ANDROID_BACKUP, + SOURCE_APP_WHATSAPP, +) from .export_summary import ( EXPORT_SUMMARY_FILENAME, MANIFEST_FILENAME, @@ -21,6 +25,7 @@ from .render.pdf_builder import build_conversation_pdf, build_fallback_tech_pdf from .threema.importer import ThreemaImporter from .threema.tech_pdf import build_threema_tech_pdf +from .threema_android.importer import ThreemaAndroidBackupImporter from .whatsapp.importer import WhatsAppImporter log = logging.getLogger(__name__) @@ -28,6 +33,7 @@ IMPORTERS: Dict[str, ConversationImporter] = { SOURCE_APP_THREEMA: ThreemaImporter(), + SOURCE_APP_THREEMA_ANDROID_BACKUP: ThreemaAndroidBackupImporter(), SOURCE_APP_WHATSAPP: WhatsAppImporter(), } diff --git a/src/chat_export/render/pdf_builder.py b/src/chat_export/render/pdf_builder.py index 4fd51dd..7573118 100644 --- a/src/chat_export/render/pdf_builder.py +++ b/src/chat_export/render/pdf_builder.py @@ -200,7 +200,7 @@ def _case_summary(conversation: NormalizedConversation) -> dict[str, int]: if message.message_type == "text": counts["text"] += 1 for attachment in message.attachments: - if attachment.absolute_path and attachment.kind in counts: + if attachment.kind in counts: counts[attachment.kind] += 1 return counts @@ -421,25 +421,35 @@ def _attachment_filename(attachment: NormalizedAttachment) -> str: return os.path.basename(attachment.absolute_path) return "attachment" + def _attachment_status(attachment: NormalizedAttachment) -> str: + metadata = attachment.metadata + if metadata.get("skipped_due_to_limit"): + return "not exported (size limit)" + if metadata.get("missing_from_backup") or metadata.get("missing_from_zip"): + return "not available in source" + return "not exported" + def _attachment_link( attachment: NormalizedAttachment, *, style=normal, - ) -> Paragraph | None: - """Build one attachment link paragraph. + ) -> Paragraph: + """Build one attachment paragraph. Args: attachment (NormalizedAttachment): Normalized attachment. style: ReportLab paragraph style. Returns: - Paragraph | None: Link paragraph if the attachment has a file path. + Paragraph: Linked filename or an unavailable-file status. """ - if not attachment.absolute_path: - return None - return link( - f"Attachment ({attachment.kind}) [{_attachment_filename(attachment)}]", - rel(attachment.absolute_path), + label = ( + f"Attachment ({attachment.kind}) [{_attachment_filename(attachment)}]" + ) + if attachment.absolute_path: + return link(label, rel(attachment.absolute_path), style) + return p( + f"{esc_xml(label)} - {_attachment_status(attachment)}", style, ) @@ -568,25 +578,25 @@ def attachment_index_table(): p("Sender"), p("Type"), p("Filename"), - p("Path"), + p("Path / status"), p("SHA256"), ] ] for message in conversation.messages: for attachment in message.attachments: - if not attachment.absolute_path: - continue - filename = attachment.filename or os.path.basename( - attachment.absolute_path + path_or_status = ( + link("open", rel(attachment.absolute_path), normal) + if attachment.absolute_path + else p(f"{_attachment_status(attachment)}", normal) ) data.append( [ p(esc_xml(message.timestamp or "NULL")), p(esc_xml(message.sender_display)), p(esc_xml(attachment.kind)), - p(esc_xml(filename)), - link("open", rel(attachment.absolute_path), normal), + p(esc_xml(_attachment_filename(attachment))), + path_or_status, p(esc_xml(attachment.sha256 or "NULL"), mono), ] ) @@ -665,8 +675,6 @@ def append_message_linear(story: list[object], message: NormalizedMessage): for attachment in message.attachments: attachment_link = _attachment_link(attachment, style=normal) - if attachment_link is None: - continue story.append(attachment_link) preview = _image_preview_or_none(message, attachment) if preview is not None: @@ -852,8 +860,7 @@ def append_message_bubble(story: list[object], message: NormalizedMessage): if preview is not None: bubble_rows.append([preview]) attachment_link = _attachment_link(attachment, style=bubble_aux) - if attachment_link is not None: - attachment_links.append(attachment_link) + attachment_links.append(attachment_link) reaction_summary = _reaction_summary(message) if reaction_summary: diff --git a/src/chat_export/threema_android/__init__.py b/src/chat_export/threema_android/__init__.py new file mode 100644 index 0000000..8ab48b6 --- /dev/null +++ b/src/chat_export/threema_android/__init__.py @@ -0,0 +1 @@ +"""Import encrypted Threema Android data backups.""" diff --git a/src/chat_export/threema_android/archive.py b/src/chat_export/threema_android/archive.py new file mode 100644 index 0000000..ff44293 --- /dev/null +++ b/src/chat_export/threema_android/archive.py @@ -0,0 +1,177 @@ +"""Access encrypted Threema Android backup members without extracting them.""" + +from __future__ import annotations + +import csv +import io +import zipfile +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import PurePosixPath +from typing import BinaryIO, Iterator + +import pyzipper + + +SUPPORTED_BACKUP_VERSION = 27 + + +class BackupFormatError(ValueError): + """Raised when an archive is not a supported Threema data backup.""" + + +@dataclass(slots=True) +class CsvRows: + rows: list[dict[str, str]] + unparseable_count: int = 0 + + +def _decode_backup_value(value: str | None) -> str: + if value is None: + return "" + return value.replace("\\\\", "\\") + + +class ThreemaAndroidArchive: + """Provide password-protected, streaming access to one data backup.""" + + def __init__(self, path: str, password: str) -> None: + self.path = path + self._password = password.encode("utf-8") + self._archive: pyzipper.AESZipFile | None = None + self._members: dict[str, zipfile.ZipInfo] = {} + self.version = 0 + + def __enter__(self) -> "ThreemaAndroidArchive": + try: + self._archive = pyzipper.AESZipFile(self.path, "r") + self._archive.setpassword(self._password) + for info in self._archive.infolist(): + name = info.filename.replace("\\", "/") + if info.is_dir(): + continue + if PurePosixPath(name).name != name: + raise BackupFormatError( + "Threema Android backups must contain flat archive members" + ) + if name in self._members: + raise BackupFormatError(f"Duplicate archive member: {name}") + self._members[name] = info + self.version = self._read_version() + except BackupFormatError as exc: + self.close() + if str(exc).startswith("Unable to read archive member"): + raise BackupFormatError( + "Unable to open the Threema Android backup. " + "Check the backup file and password." + ) from exc + raise + except (OSError, RuntimeError, ValueError, zipfile.BadZipFile) as exc: + self.close() + raise BackupFormatError( + "Unable to open the Threema Android backup. " + "Check the backup file and password." + ) from exc + return self + + def __exit__(self, *_args: object) -> None: + self.close() + + def close(self) -> None: + if self._archive is not None: + self._archive.close() + self._archive = None + + @property + def member_names(self) -> tuple[str, ...]: + return tuple(self._members) + + def has_member(self, name: str) -> bool: + return name in self._members + + def member_size(self, name: str) -> int | None: + info = self._members.get(name) + return info.file_size if info is not None else None + + @contextmanager + def open_member(self, name: str) -> Iterator[BinaryIO]: + if self._archive is None: + raise RuntimeError("Backup archive is not open") + info = self._members.get(name) + if info is None: + raise BackupFormatError(f"Required archive member is missing: {name}") + try: + stream = self._archive.open(info, "r", pwd=self._password) + except (OSError, RuntimeError, ValueError, zipfile.BadZipFile) as exc: + raise BackupFormatError( + f"Unable to read archive member: {name}" + ) from exc + try: + yield stream + except (OSError, RuntimeError, ValueError, zipfile.BadZipFile) as exc: + raise BackupFormatError( + f"Unable to read archive member: {name}" + ) from exc + finally: + stream.close() + + def read_text(self, name: str) -> str: + with self.open_member(name) as stream: + try: + return stream.read().decode("utf-8-sig") + except UnicodeDecodeError as exc: + raise BackupFormatError( + f"Archive member is not valid UTF-8 text: {name}" + ) from exc + + def read_csv( + self, + name: str, + *, + required_fields: tuple[str, ...] = (), + ) -> CsvRows: + rows: list[dict[str, str]] = [] + unparseable_count = 0 + with self.open_member(name) as binary_stream: + text_stream = io.TextIOWrapper(binary_stream, encoding="utf-8-sig", newline="") + reader = csv.DictReader(text_stream, strict=True) + fields = tuple(reader.fieldnames or ()) + missing_fields = [field for field in required_fields if field not in fields] + if missing_fields: + raise BackupFormatError( + f"Archive member {name} is missing CSV fields: " + + ", ".join(missing_fields) + ) + try: + for row in reader: + if None in row or any(value is None for value in row.values()): + unparseable_count += 1 + continue + decoded = { + str(key): _decode_backup_value(value) + for key, value in row.items() + } + if not any(decoded.values()): + continue + rows.append(decoded) + except (csv.Error, UnicodeDecodeError): + unparseable_count += 1 + return CsvRows(rows=rows, unparseable_count=unparseable_count) + + def _read_version(self) -> int: + if "settings" not in self._members: + raise BackupFormatError("Required archive member is missing: settings") + settings = self.read_text("settings") + try: + for row in csv.reader(io.StringIO(settings), strict=True): + if len(row) == 2 and row[0] == "version": + version = int(row[1]) + if version != SUPPORTED_BACKUP_VERSION: + raise BackupFormatError( + "Unsupported Threema Android backup version: " + f"{version}. Supported version: {SUPPORTED_BACKUP_VERSION}" + ) + return version + except (csv.Error, ValueError) as exc: + raise BackupFormatError("Invalid Threema backup settings") from exc + raise BackupFormatError("Threema backup version is missing") diff --git a/src/chat_export/threema_android/identity.py b/src/chat_export/threema_android/identity.py new file mode 100644 index 0000000..6a621cb --- /dev/null +++ b/src/chat_export/threema_android/identity.py @@ -0,0 +1,138 @@ +"""Read the owner identity from a Threema identity backup.""" + +from __future__ import annotations + +import base64 +import hashlib +import hmac +import struct + + +class IdentityBackupError(ValueError): + """Raised when a Threema identity backup cannot be decoded.""" + + +def _rotate_left(value: int, count: int) -> int: + return ((value << count) | (value >> (32 - count))) & 0xFFFFFFFF + + +def _salsa20_rounds(state: list[int]) -> list[int]: + x = state.copy() + for _ in range(10): + x[4] ^= _rotate_left((x[0] + x[12]) & 0xFFFFFFFF, 7) + x[8] ^= _rotate_left((x[4] + x[0]) & 0xFFFFFFFF, 9) + x[12] ^= _rotate_left((x[8] + x[4]) & 0xFFFFFFFF, 13) + x[0] ^= _rotate_left((x[12] + x[8]) & 0xFFFFFFFF, 18) + x[9] ^= _rotate_left((x[5] + x[1]) & 0xFFFFFFFF, 7) + x[13] ^= _rotate_left((x[9] + x[5]) & 0xFFFFFFFF, 9) + x[1] ^= _rotate_left((x[13] + x[9]) & 0xFFFFFFFF, 13) + x[5] ^= _rotate_left((x[1] + x[13]) & 0xFFFFFFFF, 18) + x[14] ^= _rotate_left((x[10] + x[6]) & 0xFFFFFFFF, 7) + x[2] ^= _rotate_left((x[14] + x[10]) & 0xFFFFFFFF, 9) + x[6] ^= _rotate_left((x[2] + x[14]) & 0xFFFFFFFF, 13) + x[10] ^= _rotate_left((x[6] + x[2]) & 0xFFFFFFFF, 18) + x[3] ^= _rotate_left((x[15] + x[11]) & 0xFFFFFFFF, 7) + x[7] ^= _rotate_left((x[3] + x[15]) & 0xFFFFFFFF, 9) + x[11] ^= _rotate_left((x[7] + x[3]) & 0xFFFFFFFF, 13) + x[15] ^= _rotate_left((x[11] + x[7]) & 0xFFFFFFFF, 18) + + x[1] ^= _rotate_left((x[0] + x[3]) & 0xFFFFFFFF, 7) + x[2] ^= _rotate_left((x[1] + x[0]) & 0xFFFFFFFF, 9) + x[3] ^= _rotate_left((x[2] + x[1]) & 0xFFFFFFFF, 13) + x[0] ^= _rotate_left((x[3] + x[2]) & 0xFFFFFFFF, 18) + x[6] ^= _rotate_left((x[5] + x[4]) & 0xFFFFFFFF, 7) + x[7] ^= _rotate_left((x[6] + x[5]) & 0xFFFFFFFF, 9) + x[4] ^= _rotate_left((x[7] + x[6]) & 0xFFFFFFFF, 13) + x[5] ^= _rotate_left((x[4] + x[7]) & 0xFFFFFFFF, 18) + x[11] ^= _rotate_left((x[10] + x[9]) & 0xFFFFFFFF, 7) + x[8] ^= _rotate_left((x[11] + x[10]) & 0xFFFFFFFF, 9) + x[9] ^= _rotate_left((x[8] + x[11]) & 0xFFFFFFFF, 13) + x[10] ^= _rotate_left((x[9] + x[8]) & 0xFFFFFFFF, 18) + x[12] ^= _rotate_left((x[15] + x[14]) & 0xFFFFFFFF, 7) + x[13] ^= _rotate_left((x[12] + x[15]) & 0xFFFFFFFF, 9) + x[14] ^= _rotate_left((x[13] + x[12]) & 0xFFFFFFFF, 13) + x[15] ^= _rotate_left((x[14] + x[13]) & 0xFFFFFFFF, 18) + return x + + +def _salsa_state(key: bytes, nonce: bytes, counter: int = 0) -> list[int]: + constants = struct.unpack("<4I", b"expand 32-byte k") + key_words = struct.unpack("<8I", key) + nonce_words = struct.unpack("<2I", nonce) + return [ + constants[0], + *key_words[:4], + constants[1], + *nonce_words, + counter & 0xFFFFFFFF, + (counter >> 32) & 0xFFFFFFFF, + constants[2], + *key_words[4:], + constants[3], + ] + + +def _hsalsa20(key: bytes, nonce: bytes) -> bytes: + constants = struct.unpack("<4I", b"expand 32-byte k") + key_words = struct.unpack("<8I", key) + nonce_words = struct.unpack("<4I", nonce) + state = [ + constants[0], + *key_words[:4], + constants[1], + *nonce_words, + constants[2], + *key_words[4:], + constants[3], + ] + x = _salsa20_rounds(state) + return struct.pack("<8I", x[0], x[5], x[10], x[15], x[6], x[7], x[8], x[9]) + + +def _xsalsa20_xor(data: bytes, key: bytes, nonce: bytes) -> bytes: + subkey = _hsalsa20(key, nonce[:16]) + tail_nonce = nonce[16:] + output = bytearray(len(data)) + for offset in range(0, len(data), 64): + state = _salsa_state(subkey, tail_nonce, offset // 64) + rounded = _salsa20_rounds(state) + block = struct.pack( + "<16I", + *((rounded[index] + state[index]) & 0xFFFFFFFF for index in range(16)), + ) + chunk = data[offset : offset + 64] + for index, value in enumerate(chunk): + output[offset + index] = value ^ block[index] + return bytes(output) + + +def decrypt_owner_identity(encrypted_backup: str, password: str) -> str: + """Decrypt the legacy identity entry and return only the Threema ID.""" + compact = "".join(encrypted_backup.split()).replace("-", "") + try: + encoded = base64.b32decode(compact, casefold=True) + except Exception as exc: + raise IdentityBackupError("Invalid identity backup encoding") from exc + if len(encoded) != 50: + raise IdentityBackupError("Unsupported identity backup format") + + salt, ciphertext = encoded[:8], encoded[8:] + key = hashlib.pbkdf2_hmac( + "sha256", + password.encode("utf-8"), + salt, + 100_000, + dklen=32, + ) + plaintext = _xsalsa20_xor(ciphertext, key, bytes(24)) + backup_data, stored_digest = plaintext[:40], plaintext[40:] + expected_digest = hashlib.sha256(backup_data).digest()[:2] + if not hmac.compare_digest(stored_digest, expected_digest): + raise IdentityBackupError("Identity backup decryption failed") + try: + identity = backup_data[:8].decode("ascii") + except UnicodeDecodeError as exc: + raise IdentityBackupError("Invalid Threema ID in identity backup") from exc + if len(identity) != 8 or not identity.isalnum(): + raise IdentityBackupError("Invalid Threema ID in identity backup") + return identity diff --git a/src/chat_export/threema_android/importer.py b/src/chat_export/threema_android/importer.py new file mode 100644 index 0000000..eb6fc86 --- /dev/null +++ b/src/chat_export/threema_android/importer.py @@ -0,0 +1,629 @@ +"""Import encrypted Threema Android data-backup ZIP files.""" + +from __future__ import annotations + +import logging +import os +import re +from typing import Any + +from ..common.timeutil import TimeMode +from ..common.util import safe_filename +from ..config import ExportConfig +from ..constants import SOURCE_APP_THREEMA_ANDROID_BACKUP +from ..importers.base import ImportRun, ImportedConversation +from .archive import BackupFormatError, CsvRows, ThreemaAndroidArchive +from .identity import IdentityBackupError, decrypt_owner_identity +from .media import AttachmentOutcome, prepare_attachment +from .models import ( + AndroidContact, + AndroidConversationSpec, + AndroidDistributionList, + AndroidGroup, + parse_bool, +) +from .normalize import normalize_conversation + +log = logging.getLogger(__name__) + +DIRECT_MESSAGE_RE = re.compile(r"^message_(.+)\.csv$") +GROUP_MESSAGE_RE = re.compile(r"^group_message_(.+)\.csv$") +DISTRIBUTION_MESSAGE_RE = re.compile(r"^distribution_list_message_(.+)\.csv$") +FILE_MESSAGE_TYPES = {"FILE", "IMAGE", "VIDEO", "VOICEMESSAGE"} + + +def _optional_csv( + archive: ThreemaAndroidArchive, + name: str, + *, + required_fields: tuple[str, ...] = (), +) -> CsvRows: + if not archive.has_member(name): + return CsvRows([]) + return archive.read_csv(name, required_fields=required_fields) + + +def _load_contacts( + archive: ThreemaAndroidArchive, +) -> tuple[list[AndroidContact], int]: + data = archive.read_csv( + "contacts.csv", + required_fields=("identity", "firstname", "lastname"), + ) + contacts = [AndroidContact.from_row(row) for row in data.rows] + return contacts, data.unparseable_count + + +def _load_groups( + archive: ThreemaAndroidArchive, +) -> tuple[list[AndroidGroup], int]: + data = _optional_csv( + archive, + "groups.csv", + required_fields=("id", "creator", "groupname", "members"), + ) + groups = [ + AndroidGroup.from_row(row) + for row in data.rows + if not parse_bool(row.get("deleted")) + ] + return groups, data.unparseable_count + + +def _load_distribution_lists( + archive: ThreemaAndroidArchive, +) -> tuple[list[AndroidDistributionList], int]: + data = _optional_csv( + archive, + "distribution_list.csv", + required_fields=("id", "distribution_list_name", "distribution_members"), + ) + return ( + [AndroidDistributionList.from_row(row) for row in data.rows], + data.unparseable_count, + ) + + +def _build_specs( + archive: ThreemaAndroidArchive, + contacts: list[AndroidContact], + groups: list[AndroidGroup], + distribution_lists: list[AndroidDistributionList], + self_identity: str, +) -> list[AndroidConversationSpec]: + names = set(archive.member_names) + specs: list[AndroidConversationSpec] = [] + used_members: set[str] = set() + + contacts_by_backup_id = {contact.backup_id: contact for contact in contacts} + contacts_by_identity = {contact.identity: contact for contact in contacts} + for contact in contacts: + candidates = ( + f"message_{contact.backup_id}.csv", + f"message_{contact.identity}.csv", + ) + member = next((name for name in candidates if name in names), None) + if member is None: + continue + specs.append( + AndroidConversationSpec( + conversation_id=f"threema-android:direct:{contact.identity}", + title=contact.display_name(), + conversation_type="direct", + message_member=member, + media_prefix="message_media_", + thumbnail_prefix="message_thumbnail_", + member_identities=(contact.identity, self_identity), + direct_contact=contact, + metadata={ + "contact_identity": contact.identity, + "contact_backup_id": contact.backup_id, + }, + ) + ) + used_members.add(member) + + for group in groups: + fallback_uid = f"{group.api_group_id}-{group.creator_identity}" + group_uid = group.group_uid or fallback_uid + candidates = ( + f"group_message_{group_uid}.csv", + f"group_message_{fallback_uid}.csv", + ) + member = next((name for name in candidates if name in names), None) + if member is None: + continue + title = group.name.strip() or f"Group {group.api_group_id}" + specs.append( + AndroidConversationSpec( + conversation_id=( + f"threema-android:group:{group.creator_identity}:" + f"{group.api_group_id}" + ), + title=title, + conversation_type="group", + message_member=member, + media_prefix="group_message_media_", + thumbnail_prefix="group_message_thumbnail_", + member_identities=tuple( + dict.fromkeys((*group.members, group.creator_identity)) + ), + group=group, + metadata={ + "api_group_id": group.api_group_id, + "group_creator_identity": group.creator_identity, + "group_backup_id": group_uid, + "created_at_raw": group.created_at or None, + "last_update_raw": group.last_update or None, + "archived": group.archived, + "description": group.description or None, + "description_timestamp_raw": ( + group.description_timestamp or None + ), + "user_state": group.user_state, + }, + ) + ) + used_members.add(member) + + for distribution_list in distribution_lists: + member = f"distribution_list_message_{distribution_list.list_id}.csv" + if member not in names: + continue + title = distribution_list.name.strip() or ( + f"Distribution list {distribution_list.list_id}" + ) + specs.append( + AndroidConversationSpec( + conversation_id=( + f"threema-android:distribution-list:{distribution_list.list_id}" + ), + title=title, + conversation_type="distribution_list", + message_member=member, + media_prefix="distribution_list_message_media_", + thumbnail_prefix="distribution_list_thumbnail_", + member_identities=distribution_list.members, + distribution_list=distribution_list, + metadata={ + "distribution_list_id": distribution_list.list_id, + "created_at_raw": distribution_list.created_at or None, + "last_update_raw": distribution_list.last_update or None, + "archived": distribution_list.archived, + }, + ) + ) + used_members.add(member) + + for name in sorted(names - used_members): + direct_match = DIRECT_MESSAGE_RE.match(name) + group_match = GROUP_MESSAGE_RE.match(name) + distribution_match = DISTRIBUTION_MESSAGE_RE.match(name) + if group_match or distribution_match or not direct_match: + continue + backup_id = direct_match.group(1) + contact = contacts_by_backup_id.get(backup_id) or contacts_by_identity.get( + backup_id + ) + identity = contact.identity if contact else backup_id + specs.append( + AndroidConversationSpec( + conversation_id=f"threema-android:direct:{identity}", + title=contact.display_name() if contact else identity, + conversation_type="direct", + message_member=name, + media_prefix="message_media_", + thumbnail_prefix="message_thumbnail_", + member_identities=(identity, self_identity), + direct_contact=contact, + metadata={ + "contact_identity": identity, + "contact_backup_id": backup_id, + }, + ) + ) + return specs + + +def _load_polls( + archive: ThreemaAndroidArchive, +) -> tuple[dict[str, dict[str, Any]], int]: + poll_data = _optional_csv(archive, "ballot.csv", required_fields=("id", "name")) + choice_data = _optional_csv( + archive, + "ballot_choice.csv", + required_fields=("ballot", "name"), + ) + vote_data = _optional_csv( + archive, + "ballot_vote.csv", + required_fields=("ballot_uid", "identity"), + ) + polls: dict[str, dict[str, Any]] = {} + poll_uids: dict[str, str] = {} + for row in poll_data.rows: + poll_id = row.get("id", "") + poll = { + "id": poll_id, + "api_id": row.get("aid") or None, + "creator_identity": row.get("creator") or None, + "name": row.get("name") or None, + "state": row.get("state") or None, + "assessment": row.get("assessment") or None, + "type": row.get("type") or None, + "choice_type": row.get("choice_type") or None, + "reference_type": row.get("ref") or None, + "reference_id": row.get("ref_id") or None, + "created_at_raw": row.get("created_at") or None, + "modified_at_raw": row.get("modified_at") or None, + "choices": [], + "votes": [], + } + polls[poll_id] = poll + poll_uids[f"{row.get('aid', '')}-{row.get('creator', '')}"] = poll_id + + def resolve_poll_id(uid: str) -> str | None: + if uid in poll_uids: + return poll_uids[uid] + if uid in polls: + return uid + if len(polls) == 1: + return next(iter(polls)) + return None + + for row in choice_data.rows: + poll_id = resolve_poll_id(row.get("ballot", "")) + if poll_id: + polls[poll_id]["choices"].append( + { + "api_id": row.get("aid") or None, + "name": row.get("name") or None, + "vote_count": row.get("vote_count") or None, + "order": row.get("order") or None, + "created_at_raw": row.get("created_at") or None, + "modified_at_raw": row.get("modified_at") or None, + } + ) + for row in vote_data.rows: + poll_id = resolve_poll_id(row.get("ballot_uid", "")) + if poll_id: + polls[poll_id]["votes"].append( + { + "choice_uid": row.get("choice_uid") or None, + "identity": row.get("identity") or None, + "choice": row.get("choice") or None, + "created_at_raw": row.get("created_at") or None, + "modified_at_raw": row.get("modified_at") or None, + } + ) + unparseable = ( + poll_data.unparseable_count + + choice_data.unparseable_count + + vote_data.unparseable_count + ) + return polls, unparseable + + +def _load_reactions( + archive: ThreemaAndroidArchive, +) -> tuple[ + dict[tuple[str, str], list[dict[str, str]]], + dict[tuple[str, str, str], list[dict[str, str]]], + int, +]: + contact_data = _optional_csv( + archive, + "contact_reactions.csv", + required_fields=("identity", "api_message_id", "sender_identity"), + ) + group_data = _optional_csv( + archive, + "group_reactions.csv", + required_fields=( + "api_group_id", + "group_creator_identity", + "api_message_id", + "sender_identity", + ), + ) + contacts: dict[tuple[str, str], list[dict[str, str]]] = {} + groups: dict[tuple[str, str, str], list[dict[str, str]]] = {} + for row in contact_data.rows: + key = (row.get("identity", ""), row.get("api_message_id", "")) + contacts.setdefault(key, []).append(row) + for row in group_data.rows: + key = ( + row.get("api_group_id", ""), + row.get("group_creator_identity", ""), + row.get("api_message_id", ""), + ) + groups.setdefault(key, []).append(row) + return ( + contacts, + groups, + contact_data.unparseable_count + group_data.unparseable_count, + ) + + +def _conversation_reactions( + spec: AndroidConversationSpec, + rows: list[dict[str, str]], + contact_reactions: dict[tuple[str, str], list[dict[str, str]]], + group_reactions: dict[tuple[str, str, str], list[dict[str, str]]], +) -> dict[str, list[dict[str, str]]]: + result: dict[str, list[dict[str, str]]] = {} + for row in rows: + api_id = row.get("apiid", "") + if not api_id: + continue + if spec.direct_contact: + values = contact_reactions.get((spec.direct_contact.identity, api_id), []) + elif spec.group: + values = group_reactions.get( + ( + spec.group.api_group_id, + spec.group.creator_identity, + api_id, + ), + [], + ) + else: + values = [] + if values: + result[api_id] = values + return result + + +class ThreemaAndroidBackupImporter: + """Import current Threema Android CSV data backups.""" + + source_app = SOURCE_APP_THREEMA_ANDROID_BACKUP + + def load_conversations(self, cfg: ExportConfig) -> ImportRun: + input_path = cfg.resolved_input_path() + if not cfg.backup_password: + raise ValueError("A backup password is required") + + log.info("Loading Threema Android data backup") + log.debug("Loading Threema Android data backup archive=%s", input_path) + with ThreemaAndroidArchive(input_path, cfg.backup_password) as archive: + if not archive.has_member("identity"): + raise BackupFormatError("Required archive member is missing: identity") + if not archive.has_member("contacts.csv"): + raise BackupFormatError("Required archive member is missing: contacts.csv") + try: + self_identity = decrypt_owner_identity( + archive.read_text("identity"), cfg.backup_password + ) + except IdentityBackupError as exc: + raise BackupFormatError( + "Unable to determine the owner identity from the backup" + ) from exc + + contacts, contact_unparseable = _load_contacts(archive) + groups, group_unparseable = _load_groups(archive) + distribution_lists, distribution_unparseable = ( + _load_distribution_lists(archive) + ) + polls, poll_unparseable = _load_polls(archive) + contact_reactions, group_reactions, reaction_unparseable = ( + _load_reactions(archive) + ) + unparseable_count = ( + contact_unparseable + + group_unparseable + + distribution_unparseable + + poll_unparseable + + reaction_unparseable + ) + contacts_by_identity = {contact.identity: contact for contact in contacts} + specs = _build_specs( + archive, + contacts, + groups, + distribution_lists, + self_identity, + ) + log.debug( + "Loaded Threema Android backup metadata version=%s contacts=%s " + "groups=%s distribution_lists=%s conversation_files=%s", + archive.version, + len(contacts), + len(groups), + len(distribution_lists), + len(specs), + ) + + imported: list[ImportedConversation] = [] + total_missing = 0 + total_skipped = 0 + skipped_empty = 0 + malformed_file_messages = 0 + media_root = os.path.join(os.path.abspath(cfg.out_dir), "media") + + for spec in specs: + if cfg.limit_conversations and len(imported) >= cfg.limit_conversations: + break + message_data = archive.read_csv( + spec.message_member, + required_fields=( + "apiid", + "uid", + "isoutbox", + "posted_at", + "type", + "body", + ), + ) + unparseable_count += message_data.unparseable_count + if not message_data.rows: + skipped_empty += 1 + log.debug( + "Skipping empty Threema Android conversation member=%s", + spec.message_member, + ) + continue + rows = message_data.rows + if cfg.limit_messages: + rows = rows[: cfg.limit_messages] + media_dir = ( + os.path.join( + media_root, + safe_filename(spec.conversation_id, max_len=70) + + "_" + + safe_filename(spec.title, max_len=60), + ) + if cfg.export_media + else None + ) + attachment_outcomes: dict[str, AttachmentOutcome] = {} + for row in rows: + if (row.get("type") or "").upper() not in FILE_MESSAGE_TYPES: + continue + outcome = prepare_attachment( + archive, + spec, + row, + cfg, + media_dir, + ) + uid = ( + row.get("uid") + or row.get("apiid") + or str(len(attachment_outcomes)) + ) + attachment_outcomes[uid] = outcome + + missing_count = sum( + 1 for outcome in attachment_outcomes.values() if outcome.missing + ) + skipped_count = sum( + 1 for outcome in attachment_outcomes.values() if outcome.skipped + ) + malformed_count = sum( + 1 for outcome in attachment_outcomes.values() if outcome.malformed + ) + total_missing += missing_count + total_skipped += skipped_count + malformed_file_messages += malformed_count + if missing_count: + log.warning( + "Threema Android conversation has attachment payloads " + "missing from the backup count=%s conversation_id=%s", + missing_count, + spec.conversation_id, + ) + if skipped_count: + log.warning( + "Skipped Threema Android attachments due to size limit " + "count=%s conversation_id=%s max_media_bytes=%s", + skipped_count, + spec.conversation_id, + cfg.max_media_bytes, + ) + if malformed_count: + log.warning( + "Unable to parse Threema Android file-message metadata " + "count=%s conversation_id=%s", + malformed_count, + spec.conversation_id, + ) + + reactions = _conversation_reactions( + spec, + rows, + contact_reactions, + group_reactions, + ) + conversation = normalize_conversation( + spec=spec, + rows=rows, + contacts=contacts_by_identity, + self_identity=self_identity, + tz_name=cfg.tz_name, + backup_version=archive.version, + attachments=attachment_outcomes, + reactions_by_api_id=reactions, + polls=polls, + ) + imported.append( + ImportedConversation( + conversation=conversation, + tech_payload=None, + tech_renderer=None, + metadata={ + "message_count": len(rows), + "media_dir": ( + os.path.abspath(media_dir) if media_dir else None + ), + "missing_media_count": missing_count, + "skipped_media_count": skipped_count, + "unparseable_line_count": message_data.unparseable_count, + }, + ) + ) + log.info( + "Prepared Threema Android conversation conversation_id=%s " + "messages=%s participants=%s attachments=%s", + conversation.conversation_id, + len(conversation.messages), + len(conversation.participants), + len(attachment_outcomes), + ) + + if unparseable_count: + log.warning( + "Threema Android backup contains unparseable CSV rows count=%s", + unparseable_count, + ) + + media_payload_count = sum( + 1 + for name in archive.member_names + if name.startswith( + ( + "message_media_", + "group_message_media_", + "distribution_list_message_media_", + ) + ) + ) + thumbnail_count = sum( + 1 + for name in archive.member_names + if name.startswith( + ( + "message_thumbnail_", + "group_message_thumbnail_", + "distribution_list_thumbnail_", + ) + ) + ) + metadata = { + "input_path": input_path, + "time_mode": TimeMode.UNIX_MS, + "timezone": cfg.tz_name, + "backup_version": archive.version, + "self_identity": self_identity, + "contact_count": len(contacts), + "group_count": len(groups), + "distribution_list_count": len(distribution_lists), + "media_payload_count": media_payload_count, + "thumbnail_count": thumbnail_count, + "skipped_empty_conversation_count": skipped_empty, + "missing_media_count": total_missing, + "skipped_media_count": total_skipped, + "unparseable_line_count": unparseable_count, + "malformed_file_message_count": malformed_file_messages, + } + + log.info( + "Completed Threema Android backup import conversations=%s backup_version=%s", + len(imported), + metadata["backup_version"], + ) + return ImportRun( + source_app=self.source_app, + conversations=imported, + metadata=metadata, + ) diff --git a/src/chat_export/threema_android/media.py b/src/chat_export/threema_android/media.py new file mode 100644 index 0000000..80c234d --- /dev/null +++ b/src/chat_export/threema_android/media.py @@ -0,0 +1,192 @@ +"""Parse and export message attachments from an Android backup.""" + +from __future__ import annotations + +import hashlib +import json +import mimetypes +import os +from dataclasses import dataclass +from typing import Any + +from ..common.util import ensure_dir, safe_filename +from ..config import ExportConfig +from ..normalized.models import NormalizedAttachment +from .archive import ThreemaAndroidArchive +from .models import AndroidConversationSpec, parse_bool, parse_int + + +@dataclass(frozen=True, slots=True) +class FileData: + blob_id: str | None + mime_type: str | None + declared_size: int | None + filename: str | None + rendering_type: int | None + downloaded: bool + caption: str | None + thumbnail_mime_type: str | None + metadata: dict[str, Any] + + +@dataclass(frozen=True, slots=True) +class AttachmentOutcome: + attachment: NormalizedAttachment + missing: bool + skipped: bool + malformed: bool + + +def _display_filename(filename: str | None, uid: str, mime_type: str | None) -> str: + name = (filename or "").replace("\\", "/").rsplit("/", 1)[-1].strip() + if name: + return name + extension = mimetypes.guess_extension(mime_type or "") or "" + return f"attachment_{uid}{extension}" + + +def parse_file_data(body: str) -> FileData | None: + try: + values = json.loads(body) + except (TypeError, ValueError, json.JSONDecodeError): + return None + if not isinstance(values, list) or len(values) < 5: + return None + + metadata = values[9] if len(values) > 9 and isinstance(values[9], dict) else {} + declared_size = parse_int(str(values[3]), -1) if len(values) > 3 else -1 + return FileData( + blob_id=str(values[0]) if values[0] is not None else None, + mime_type=str(values[2]) if len(values) > 2 and values[2] else None, + declared_size=declared_size if declared_size >= 0 else None, + filename=str(values[4]) if len(values) > 4 and values[4] else None, + rendering_type=( + parse_int(str(values[5])) if len(values) > 5 and values[5] is not None else None + ), + downloaded=( + values[6] + if len(values) > 6 and isinstance(values[6], bool) + else parse_bool(str(values[6])) if len(values) > 6 else False + ), + caption=str(values[7]) if len(values) > 7 and values[7] else None, + thumbnail_mime_type=( + str(values[8]) if len(values) > 8 and values[8] else None + ), + metadata=metadata, + ) + + +def _attachment_kind(mime_type: str | None, source_type: str) -> str: + mime = (mime_type or "").lower() + if mime.startswith("image/") or source_type == "IMAGE": + return "image" + if mime.startswith("audio/") or source_type == "VOICEMESSAGE": + return "audio" + if mime.startswith("video/") or source_type == "VIDEO": + return "video" + return "file" + + +def prepare_attachment( + archive: ThreemaAndroidArchive, + spec: AndroidConversationSpec, + row: dict[str, str], + cfg: ExportConfig, + media_dir: str | None, +) -> AttachmentOutcome: + uid = row.get("uid") or row.get("apiid") or "unknown" + source_type = (row.get("type") or "FILE").upper() + file_data = parse_file_data(row.get("body", "")) + malformed = file_data is None + if file_data is None: + file_data = FileData( + blob_id=None, + mime_type=None, + declared_size=None, + filename=None, + rendering_type=None, + downloaded=False, + caption=row.get("caption") or None, + thumbnail_mime_type=None, + metadata={}, + ) + + filename = _display_filename(file_data.filename, uid, file_data.mime_type) + mime_type = file_data.mime_type or mimetypes.guess_type(filename)[0] + media_member = f"{spec.media_prefix}{uid}" + thumbnail_member = f"{spec.thumbnail_prefix}{uid}" + archive_size = archive.member_size(media_member) + missing = archive_size is None + skipped = bool( + archive_size is not None + and cfg.max_media_bytes + and archive_size > cfg.max_media_bytes + ) + + absolute_path = None + relative_path = None + sha256 = None + exported_size = archive_size + if archive_size is not None and not skipped and cfg.export_media and media_dir: + ensure_dir(media_dir) + target_name = safe_filename(f"{uid}_{filename}", max_len=140) + target_path = os.path.join(media_dir, target_name) + digest = hashlib.sha256() + written = 0 + with archive.open_member(media_member) as source, open(target_path, "wb") as target: + while True: + chunk = source.read(1024 * 1024) + if not chunk: + break + target.write(chunk) + digest.update(chunk) + written += len(chunk) + absolute_path = os.path.abspath(target_path) + relative_path = os.path.relpath(absolute_path, os.path.abspath(cfg.out_dir)) + sha256 = digest.hexdigest() + exported_size = written + + metadata = { + "source_type": source_type, + "archive_member": media_member if archive_size is not None else None, + "thumbnail_archive_member": ( + thumbnail_member if archive.has_member(thumbnail_member) else None + ), + "blob_id": file_data.blob_id, + "declared_size": file_data.declared_size, + "archive_size": archive_size, + "declared_size_mismatch": ( + archive_size is not None + and file_data.declared_size is not None + and archive_size != file_data.declared_size + ), + "rendering_type": file_data.rendering_type, + "downloaded": file_data.downloaded, + "thumbnail_mime_type": file_data.thumbnail_mime_type, + "file_metadata": file_data.metadata, + "missing_from_backup": missing, + "skipped_due_to_limit": skipped, + "skip_reason": ( + f"size={archive_size} exceeds max_media_bytes={cfg.max_media_bytes}" + if skipped + else None + ), + "file_data_parse_error": malformed, + } + attachment = NormalizedAttachment( + attachment_id=f"threema-android-attachment:{uid}", + kind=_attachment_kind(mime_type, source_type), + filename=filename, + absolute_path=absolute_path, + relative_path=relative_path, + mime_type=mime_type, + size=exported_size if archive_size is not None else file_data.declared_size, + sha256=sha256, + metadata=metadata, + ) + return AttachmentOutcome( + attachment=attachment, + missing=missing, + skipped=skipped, + malformed=malformed, + ) diff --git a/src/chat_export/threema_android/models.py b/src/chat_export/threema_android/models.py new file mode 100644 index 0000000..8c1e48c --- /dev/null +++ b/src/chat_export/threema_android/models.py @@ -0,0 +1,144 @@ +"""Represent contacts and conversations from an Android data backup.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +def parse_bool(value: str | None) -> bool: + return (value or "").strip().lower() in {"1", "true", "yes"} + + +def parse_int(value: str | None, default: int = 0) -> int: + try: + return int(value or default) + except (TypeError, ValueError): + return default + + +def parse_identities(value: str | None) -> tuple[str, ...]: + return tuple( + dict.fromkeys( + identity.strip() + for identity in (value or "").split(";") + if identity.strip() + ) + ) + + +@dataclass(frozen=True, slots=True) +class AndroidContact: + identity: str + first_name: str + last_name: str + nickname: str + public_key: str + verification: str + android_contact_id: str + last_update: str + hidden: bool + archived: bool + backup_id: str + + @classmethod + def from_row(cls, row: dict[str, str]) -> "AndroidContact": + identity = row.get("identity", "") + return cls( + identity=identity, + first_name=row.get("firstname", ""), + last_name=row.get("lastname", ""), + nickname=row.get("nick_name", ""), + public_key=row.get("publickey", ""), + verification=row.get("verification", ""), + android_contact_id=row.get("acid", ""), + last_update=row.get("last_update", ""), + hidden=parse_bool(row.get("hidden")), + archived=parse_bool(row.get("archived")), + backup_id=row.get("identity_id", "") or identity, + ) + + def display_name(self) -> str: + full_name = " ".join( + value.strip() for value in (self.first_name, self.last_name) if value.strip() + ) + return full_name or self.nickname.strip() or self.identity or "Unknown contact" + + def metadata(self) -> dict[str, Any]: + return { + "public_key": self.public_key or None, + "verification": self.verification or None, + "android_contact_id": self.android_contact_id or None, + "last_update_raw": self.last_update or None, + "nickname": self.nickname or None, + "hidden": self.hidden, + "archived": self.archived, + "backup_contact_id": self.backup_id, + } + + +@dataclass(frozen=True, slots=True) +class AndroidGroup: + api_group_id: str + creator_identity: str + name: str + members: tuple[str, ...] + group_uid: str + created_at: str + last_update: str + archived: bool + description: str + description_timestamp: str + user_state: int + + @classmethod + def from_row(cls, row: dict[str, str]) -> "AndroidGroup": + return cls( + api_group_id=row.get("id", ""), + creator_identity=row.get("creator", ""), + name=row.get("groupname", ""), + members=parse_identities(row.get("members")), + group_uid=row.get("group_uid", ""), + created_at=row.get("created_at", ""), + last_update=row.get("last_update", ""), + archived=parse_bool(row.get("archived")), + description=row.get("groupDesc", ""), + description_timestamp=row.get("groupDescTimestamp", ""), + user_state=parse_int(row.get("user_state")), + ) + + +@dataclass(frozen=True, slots=True) +class AndroidDistributionList: + list_id: str + name: str + members: tuple[str, ...] + created_at: str + last_update: str + archived: bool + + @classmethod + def from_row(cls, row: dict[str, str]) -> "AndroidDistributionList": + return cls( + list_id=row.get("id", ""), + name=row.get("distribution_list_name", ""), + members=parse_identities(row.get("distribution_members")), + created_at=row.get("created_at", ""), + last_update=row.get("last_update", ""), + archived=parse_bool(row.get("archived")), + ) + + +@dataclass(frozen=True, slots=True) +class AndroidConversationSpec: + conversation_id: str + title: str + conversation_type: str + message_member: str + media_prefix: str + thumbnail_prefix: str + member_identities: tuple[str, ...] + direct_contact: AndroidContact | None = None + group: AndroidGroup | None = None + distribution_list: AndroidDistributionList | None = None + metadata: dict[str, Any] = field(default_factory=dict) diff --git a/src/chat_export/threema_android/normalize.py b/src/chat_export/threema_android/normalize.py new file mode 100644 index 0000000..32576fb --- /dev/null +++ b/src/chat_export/threema_android/normalize.py @@ -0,0 +1,457 @@ +"""Normalize Threema Android backup rows for the shared renderers.""" + +from __future__ import annotations + +import json +from typing import Any + +from ..common.timeutil import TimeMode, format_dt +from ..constants import SOURCE_APP_THREEMA_ANDROID_BACKUP +from ..normalized.models import ( + NormalizedConversation, + NormalizedEdit, + NormalizedMessage, + NormalizedParticipant, + NormalizedReaction, +) +from .media import AttachmentOutcome, parse_file_data +from .models import AndroidContact, AndroidConversationSpec, parse_bool, parse_int + + +GROUP_STATUS_LABELS = { + 0: "Group created", + 1: "Group renamed", + 2: "Group picture updated", + 3: "Group member added", + 4: "Group member left", + 5: "Group member removed", + 6: "Group changed to notes group", + 7: "Group changed to people group", + 8: "First poll vote received", + 9: "Poll vote changed", + 10: "Anonymous poll vote received", + 11: "Poll voting completed", + 12: "Group description changed", + 13: "Group creator left", +} +CALL_STATUS_LABELS = { + 1: "Missed call", + 2: "Finished call", + 3: "Rejected call", + 4: "Aborted call", +} +GROUP_CALL_STATUS_LABELS = {1: "Group call started", 2: "Group call ended"} +POLL_EVENT_LABELS = {1: "Poll created", 2: "Poll modified", 3: "Poll closed"} + + +def _participant_id(identity: str) -> str: + return f"threema-id:{identity}" + + +def _display_name( + identity: str, + contacts: dict[str, AndroidContact], + self_identity: str, +) -> str: + if identity == self_identity: + return "Me" + contact = contacts.get(identity) + return contact.display_name() if contact else identity or "Unknown" + + +def _parse_json(value: str) -> Any | None: + try: + return json.loads(value) + except (TypeError, ValueError, json.JSONDecodeError): + return None + + +def _sender_identity( + row: dict[str, str], + spec: AndroidConversationSpec, + self_identity: str, +) -> str: + if parse_bool(row.get("isoutbox")): + return self_identity + if spec.direct_contact is not None: + return spec.direct_contact.identity + return row.get("identity", "") + + +def _build_participants( + spec: AndroidConversationSpec, + rows: list[dict[str, str]], + contacts: dict[str, AndroidContact], + self_identity: str, +) -> list[NormalizedParticipant]: + identities = list(spec.member_identities) + for row in rows: + identity = row.get("identity", "") + if identity and identity not in identities: + identities.append(identity) + if self_identity and self_identity not in identities: + identities.append(self_identity) + + participants: list[NormalizedParticipant] = [] + for identity in identities: + if not identity: + continue + contact = contacts.get(identity) + role = "member" + if identity == self_identity: + role = "me" + elif spec.direct_contact and identity == spec.direct_contact.identity: + role = "chat_partner" + elif spec.distribution_list is not None: + role = "recipient" + elif spec.group and identity == spec.group.creator_identity: + role = "admin" + metadata = contact.metadata() if contact else {} + if spec.group: + metadata["group_creator"] = identity == spec.group.creator_identity + participants.append( + NormalizedParticipant( + participant_id=_participant_id(identity), + display_name=_display_name(identity, contacts, self_identity), + identity=identity, + role=role, + metadata=metadata, + ) + ) + return participants + + +def _message_status(row: dict[str, str]) -> str: + state = (row.get("messagestae") or "").strip().lower() + if state: + return state + if row.get("read_at") or parse_bool(row.get("isread")): + return "read" + if row.get("delivered_at"): + return "delivered" + return "unknown" + + +def _status_event( + source_type: str, + body: str, + contacts: dict[str, AndroidContact], + self_identity: str, +) -> tuple[str, Any | None]: + decoded = _parse_json(body) + params: dict[str, Any] = {} + if isinstance(decoded, list) and len(decoded) > 1 and isinstance(decoded[1], dict): + params = decoded[1] + + if source_type == "VOIP_STATUS": + label = CALL_STATUS_LABELS.get(parse_int(str(params.get("status"))), "Call event") + if params.get("duration") is not None: + label += f" ({params['duration']} seconds)" + return label, decoded + if source_type == "GROUP_CALL_STATUS": + label = GROUP_CALL_STATUS_LABELS.get( + parse_int(str(params.get("status"))), "Group call event" + ) + caller = str(params.get("callerIdentity") or "") + if caller: + label += f" by {_display_name(caller, contacts, self_identity)}" + return label, decoded + if source_type == "GROUP_STATUS": + label = GROUP_STATUS_LABELS.get( + parse_int(str(params.get("status"))), "Group event" + ) + identity = str(params.get("identity") or "") + if identity: + label += f": {_display_name(identity, contacts, self_identity)}" + detail = params.get("newGroupName") or params.get("ballotName") + if detail: + label += f" ({detail})" + return label, decoded + return body or "Status event", decoded + + +def _location_text(body: str) -> tuple[str, Any | None]: + decoded = _parse_json(body) + if not isinstance(decoded, list) or len(decoded) < 2: + return body or "Location", decoded + latitude, longitude = decoded[0], decoded[1] + parts = [f"Location: {latitude}, {longitude}"] + if len(decoded) > 4 and decoded[4]: + parts.append(str(decoded[4])) + if len(decoded) > 3 and decoded[3]: + parts.append(str(decoded[3])) + if len(decoded) > 2 and decoded[2] is not None: + parts.append(f"Accuracy: {decoded[2]} m") + return "\n".join(parts), decoded + + +def _poll_text(body: str, polls: dict[str, dict[str, Any]]) -> tuple[str, dict[str, Any] | None]: + decoded = _parse_json(body) + event_id = None + poll_id = None + if isinstance(decoded, list): + if decoded: + event_id = parse_int(str(decoded[0]), -1) + if len(decoded) > 1: + poll_id = str(decoded[1]) + poll = polls.get(poll_id or "") + label = POLL_EVENT_LABELS.get(event_id or -1, "Poll") + if poll is None: + return label, None + lines = [f"{label}: {poll.get('name') or 'Unnamed poll'}"] + choices = poll.get("choices") or [] + for choice in sorted(choices, key=lambda item: parse_int(item.get("order"))): + name = choice.get("name") or "Unnamed option" + vote_count = choice.get("vote_count") + lines.append(f"- {name}: {vote_count or '0'} vote(s)") + return "\n".join(lines), poll + + +def _reaction_rows( + row: dict[str, str], + spec: AndroidConversationSpec, + source_rows: list[dict[str, str]], + contacts: dict[str, AndroidContact], + self_identity: str, + tz_name: str, +) -> list[NormalizedReaction]: + reactions: list[NormalizedReaction] = [] + seen: set[tuple[str, str, str]] = set() + + def add(identity: str, sequence: str, timestamp: str, source: str) -> None: + key = (identity, sequence, timestamp) + if not identity or not sequence or key in seen: + return + seen.add(key) + reactions.append( + NormalizedReaction( + creator_display=_display_name(identity, contacts, self_identity), + reaction=sequence, + timestamp=format_dt(timestamp, TimeMode.UNIX_MS, tz_name), + metadata={"sender_identity": identity, "source": source}, + ) + ) + + for reaction in source_rows: + add( + reaction.get("sender_identity", ""), + reaction.get("emoji_sequence", ""), + reaction.get("reacted_at", ""), + "reaction_csv", + ) + + state = (row.get("messagestae") or "").upper() + if state in {"USERACK", "USERDEC"}: + if not parse_bool(row.get("isoutbox")): + creator = self_identity + elif spec.direct_contact is not None: + creator = spec.direct_contact.identity + else: + creator = row.get("identity", "") + add( + creator, + "\U0001f44d" if state == "USERACK" else "\U0001f44e", + row.get("modified_at") or row.get("created_at") or "", + "legacy_message_state", + ) + + group_states = _parse_json(row.get("g_msg_states", "")) + if isinstance(group_states, dict): + for identity, group_state in group_states.items(): + normalized_state = str(group_state).upper() + if normalized_state in {"USERACK", "USERDEC"}: + add( + str(identity), + "\U0001f44d" if normalized_state == "USERACK" else "\U0001f44e", + row.get("modified_at") or row.get("created_at") or "", + "legacy_group_message_state", + ) + return reactions + + +def _message_preview(row: dict[str, str]) -> str: + source_type = (row.get("type") or "TEXT").upper() + if source_type == "FILE": + file_data = parse_file_data(row.get("body", "")) + return file_data.filename if file_data and file_data.filename else "Attachment" + text = row.get("body") or source_type.title() + return text.replace("\r", " ").replace("\n", " ")[:120] + + +def normalize_conversation( + *, + spec: AndroidConversationSpec, + rows: list[dict[str, str]], + contacts: dict[str, AndroidContact], + self_identity: str, + tz_name: str, + backup_version: int, + attachments: dict[str, AttachmentOutcome], + reactions_by_api_id: dict[str, list[dict[str, str]]], + polls: dict[str, dict[str, Any]], +) -> NormalizedConversation: + participants = _build_participants(spec, rows, contacts, self_identity) + api_index = { + row.get("apiid", ""): row for row in rows if row.get("apiid") + } + normalized_messages: list[NormalizedMessage] = [] + + for index, row in enumerate(rows, start=1): + source_type = (row.get("type") or "TEXT").upper() + uid = row.get("uid") or row.get("apiid") or str(index) + api_id = row.get("apiid", "") + sender_identity = _sender_identity(row, spec, self_identity) + is_system = parse_bool(row.get("isstatusmessage")) or source_type in { + "STATUS", + "GROUP_STATUS", + "GROUP_CALL_STATUS", + } + direction = ( + "system" + if is_system + else "outgoing" if parse_bool(row.get("isoutbox")) else "incoming" + ) + + attachment_outcome = attachments.get(uid) + message_attachments = ( + [attachment_outcome.attachment] if attachment_outcome is not None else [] + ) + message_type = source_type.lower() + text: str | None = row.get("body") or None + caption = row.get("caption") or None + decoded_content: Any | None = None + poll_details: dict[str, Any] | None = None + + if row.get("deleted_at"): + text = "Message deleted" + elif source_type in {"FILE", "IMAGE", "VIDEO", "VOICEMESSAGE"}: + text = None + if attachment_outcome is not None: + message_type = attachment_outcome.attachment.kind + file_data = parse_file_data(row.get("body", "")) + if file_data and not caption: + caption = file_data.caption + elif source_type == "LOCATION": + text, decoded_content = _location_text(row.get("body", "")) + message_type = "location" + elif source_type == "BALLOT": + text, poll_details = _poll_text(row.get("body", ""), polls) + message_type = "poll" + elif source_type in {"STATUS", "GROUP_STATUS", "GROUP_CALL_STATUS", "VOIP_STATUS"}: + text, decoded_content = _status_event( + source_type, + row.get("body", ""), + contacts, + self_identity, + ) + message_type = "system" if is_system else "call" + + quoted_id = row.get("quoted_message_apiid") or None + quoted_preview = None + if quoted_id: + quoted_row = api_index.get(quoted_id) + if quoted_row is None: + quoted_preview = "Reply to: unknown message" + else: + quoted_sender = _sender_identity(quoted_row, spec, self_identity) + quoted_preview = ( + "Reply to: " + + _display_name(quoted_sender, contacts, self_identity) + + " - " + + _message_preview(quoted_row) + ) + + edits = [] + if row.get("edited_at"): + edits.append( + NormalizedEdit( + timestamp=format_dt( + row.get("edited_at"), TimeMode.UNIX_MS, tz_name + ), + text=None, + metadata={ + "note": ( + "The backup marks this message as edited; previous " + "text is not included" + ) + }, + ) + ) + + metadata = { + "source_type": source_type, + "api_message_id": api_id or None, + "uid": uid, + "is_outbox": parse_bool(row.get("isoutbox")), + "is_read": parse_bool(row.get("isread")), + "is_saved": parse_bool(row.get("issaved")), + "is_status_message": parse_bool(row.get("isstatusmessage")), + "posted_at_raw": row.get("posted_at") or None, + "created_at_raw": row.get("created_at") or None, + "modified_at_raw": row.get("modified_at") or None, + "delivered_at_raw": row.get("delivered_at") or None, + "read_at_raw": row.get("read_at") or None, + "edited_at_raw": row.get("edited_at") or None, + "deleted_at_raw": row.get("deleted_at") or None, + "message_state": row.get("messagestae") or None, + "display_tags": row.get("display_tags") or None, + "recipient_identity": ( + row.get("identity") or None + if spec.distribution_list is not None + else None + ), + "decoded_content": decoded_content, + "poll": poll_details, + } + normalized_messages.append( + NormalizedMessage( + message_id=f"threema-android-message:{uid}", + timestamp=format_dt( + row.get("posted_at") or row.get("created_at"), + TimeMode.UNIX_MS, + tz_name, + ), + sender_id=_participant_id(sender_identity) if sender_identity else None, + sender_display=_display_name( + sender_identity, contacts, self_identity + ), + direction=direction, + message_type=message_type, + text=text, + caption=caption, + quoted_message_ref=( + f"threema-android-api:{quoted_id}" if quoted_id else None + ), + quoted_preview=quoted_preview, + status=_message_status(row), + attachments=message_attachments, + reactions=_reaction_rows( + row, + spec, + reactions_by_api_id.get(api_id, []), + contacts, + self_identity, + tz_name, + ), + edits=edits, + metadata=metadata, + ) + ) + + conversation_metadata = { + "backup_version": backup_version, + "archive_message_member": spec.message_member, + **spec.metadata, + } + return NormalizedConversation( + source_app=SOURCE_APP_THREEMA_ANDROID_BACKUP, + conversation_id=spec.conversation_id, + title=spec.title, + conversation_type=spec.conversation_type, + participants=participants, + messages=normalized_messages, + timezone=tz_name, + time_mode=TimeMode.UNIX_MS, + self_participant_id=_participant_id(self_identity) if self_identity else None, + metadata=conversation_metadata, + ) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..85dc1d6 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Project regression tests.""" diff --git a/tests/test_threema_android_backup.py b/tests/test_threema_android_backup.py new file mode 100644 index 0000000..a49abc1 --- /dev/null +++ b/tests/test_threema_android_backup.py @@ -0,0 +1,264 @@ +from __future__ import annotations + +import hashlib +import json +import shutil +import unittest +import uuid +from pathlib import Path + +from chat_export.common.logging_setup import setup_logging +from chat_export.config_factory import build_export_config +from chat_export.constants import SOURCE_APP_THREEMA_ANDROID_BACKUP +from chat_export.orchestrator import export_all_conversations +from chat_export.render.pdf_builder import _case_summary +from chat_export.threema_android.archive import ( + BackupFormatError, + ThreemaAndroidArchive, +) +from chat_export.threema_android.identity import ( + IdentityBackupError, + decrypt_owner_identity, +) +from chat_export.threema_android.importer import ThreemaAndroidBackupImporter + + +ROOT = Path(__file__).resolve().parents[1] +FIXTURE_DIR = ROOT / "Examples" / "ThreemaAndroidBackupSynthetic" +BACKUP_PATH = FIXTURE_DIR / "threema-backup_synthetic.zip" +NO_MEDIA_BACKUP_PATH = FIXTURE_DIR / "threema-backup_synthetic_no_media.zip" +PASSWORD = "testpassword" +IDENTITY_BACKUP = ( + "4K4M-5Q6T-KFUH-KHL5-2VCJ-ZM57-NL7R-WJTA-V45L-NJAM-" + "WLEU-5DS4-XF7S-OPH4-CTCL-N2CF-3C4C-HPB7-YZWW-U3S6" +) + + +def _config( + input_path: Path, + out_dir: str, + *, + export_media: bool = False, + export_excel: bool = False, + max_media_bytes: int = 0, +): + return build_export_config( + out_dir=out_dir, + source_app=SOURCE_APP_THREEMA_ANDROID_BACKUP, + input_path=str(input_path), + backup_password=PASSWORD, + export_media=export_media, + export_excel=export_excel, + max_media_bytes=max_media_bytes, + ) + + +class IdentityBackupTests(unittest.TestCase): + def test_decrypts_official_identity_backup_vector(self) -> None: + self.assertEqual( + decrypt_owner_identity(IDENTITY_BACKUP, PASSWORD), + "0ZAHXXHB", + ) + + def test_rejects_wrong_identity_password(self) -> None: + with self.assertRaises(IdentityBackupError): + decrypt_owner_identity(IDENTITY_BACKUP, "wrong-password") + + +class ArchiveTests(unittest.TestCase): + def test_reads_encrypted_backup_metadata(self) -> None: + with ThreemaAndroidArchive(str(BACKUP_PATH), PASSWORD) as archive: + self.assertEqual(archive.version, 27) + self.assertTrue(archive.has_member("contacts.csv")) + self.assertTrue(archive.has_member("identity")) + self.assertEqual(len(archive.read_csv("contacts.csv").rows), 3) + + def test_rejects_wrong_archive_password(self) -> None: + with self.assertRaisesRegex( + BackupFormatError, + "Check the backup file and password", + ): + with ThreemaAndroidArchive(str(BACKUP_PATH), "wrong-password"): + pass + + +class ImporterTests(unittest.TestCase): + def test_preserves_backup_password_whitespace(self) -> None: + cfg = build_export_config( + out_dir=str(FIXTURE_DIR), + source_app=SOURCE_APP_THREEMA_ANDROID_BACKUP, + input_path=str(BACKUP_PATH), + backup_password=" password with spaces ", + export_media=False, + ) + + self.assertEqual(cfg.backup_password, " password with spaces ") + + def test_imports_current_backup_structure(self) -> None: + cfg = _config(BACKUP_PATH, str(FIXTURE_DIR)) + run = ThreemaAndroidBackupImporter().load_conversations(cfg) + + self.assertEqual(run.source_app, SOURCE_APP_THREEMA_ANDROID_BACKUP) + self.assertEqual(len(run.conversations), 4) + self.assertEqual( + sum(len(item.conversation.messages) for item in run.conversations), + 23, + ) + self.assertEqual( + sum( + len(message.attachments) + for item in run.conversations + for message in item.conversation.messages + ), + 7, + ) + self.assertEqual( + sum( + len(message.reactions) + for item in run.conversations + for message in item.conversation.messages + ), + 4, + ) + self.assertEqual(run.metadata["backup_version"], 27) + self.assertEqual(run.metadata["self_identity"], "0ZAHXXHB") + self.assertEqual(run.metadata["missing_media_count"], 1) + self.assertEqual(run.metadata["skipped_empty_conversation_count"], 1) + for item in run.conversations: + self.assertEqual( + item.conversation.self_participant_id, + "threema-id:0ZAHXXHB", + ) + + def test_normalizes_current_backup_semantics(self) -> None: + cfg = _config(BACKUP_PATH, str(FIXTURE_DIR)) + run = ThreemaAndroidBackupImporter().load_conversations(cfg) + conversations = { + item.conversation.conversation_id: item.conversation + for item in run.conversations + } + + direct = conversations["threema-android:direct:ALICE001"] + self.assertEqual( + [(participant.identity, participant.role) for participant in direct.participants], + [("ALICE001", "chat_partner"), ("0ZAHXXHB", "me")], + ) + self.assertEqual(direct.messages[0].direction, "incoming") + self.assertEqual(direct.messages[1].direction, "outgoing") + self.assertIn("Reply to: Alice Example", direct.messages[1].quoted_preview) + self.assertEqual(len(direct.messages[2].edits), 1) + self.assertEqual(direct.messages[4].attachments[0].filename, "synthetic-image.png") + + group = conversations["threema-android:group:ALICE001:0011223344556677"] + self.assertEqual(group.messages[2].message_type, "system") + self.assertEqual(group.messages[4].message_type, "poll") + self.assertIn("Option Alpha: 1 vote(s)", group.messages[4].text) + self.assertEqual(group.messages[0].reactions[0].creator_display, "Bob Sample") + + missing = conversations["threema-android:direct:BOB00002"].messages[1] + self.assertTrue(missing.attachments[0].metadata["missing_from_backup"]) + + distribution = conversations["threema-android:distribution-list:1"] + self.assertEqual(distribution.conversation_type, "distribution_list") + self.assertTrue( + all(message.direction == "outgoing" for message in distribution.messages) + ) + + def test_counts_missing_payloads_in_backup_without_media(self) -> None: + cfg = _config(NO_MEDIA_BACKUP_PATH, str(FIXTURE_DIR)) + run = ThreemaAndroidBackupImporter().load_conversations(cfg) + + self.assertEqual(run.metadata["media_payload_count"], 0) + self.assertEqual(run.metadata["missing_media_count"], 7) + attachment_count = sum( + sum( + _case_summary(item.conversation)[kind] + for kind in ("image", "audio", "video", "file") + ) + for item in run.conversations + ) + self.assertEqual(attachment_count, 7) + + def test_applies_media_limit_without_reading_payloads(self) -> None: + cfg = _config( + BACKUP_PATH, + str(FIXTURE_DIR), + max_media_bytes=1, + ) + run = ThreemaAndroidBackupImporter().load_conversations(cfg) + + self.assertEqual(run.metadata["missing_media_count"], 1) + self.assertEqual(run.metadata["skipped_media_count"], 6) + + +class EndToEndTests(unittest.TestCase): + def test_exports_all_artifacts_without_exposing_password(self) -> None: + temp_path = ROOT / "Examples" / "Output" / f"android-test-{uuid.uuid4()}" + temp_path.mkdir(parents=True) + self.addCleanup(shutil.rmtree, temp_path, True) + temp_dir = str(temp_path) + try: + cfg = _config( + BACKUP_PATH, + temp_dir, + export_media=True, + export_excel=True, + ) + setup_logging(cfg.log_file, console=False) + result = export_all_conversations(cfg) + + self.assertEqual(result["status"], "Completed with warnings") + self.assertEqual(len(result["exported"]), 4) + self.assertEqual(len(list(Path(temp_dir, "conversations").glob("*.pdf"))), 8) + self.assertEqual(len(list(Path(temp_dir, "excel").glob("*.xlsx"))), 4) + exported_media = [ + path for path in Path(temp_dir, "media").rglob("*") if path.is_file() + ] + self.assertEqual(len(exported_media), 6) + source_media = [ + path + for path in (FIXTURE_DIR / "source").iterdir() + if path.name.startswith( + ( + "message_media_", + "group_message_media_", + "distribution_list_message_media_", + ) + ) + ] + self.assertEqual( + sorted(hashlib.sha256(path.read_bytes()).hexdigest() for path in exported_media), + sorted(hashlib.sha256(path.read_bytes()).hexdigest() for path in source_media), + ) + + summary_path = Path(temp_dir, "export_summary.txt") + manifest_path = Path(temp_dir, "manifest.json") + self.assertTrue(summary_path.is_file()) + self.assertTrue(manifest_path.is_file()) + manifest_text = manifest_path.read_text(encoding="utf-8") + manifest = json.loads(manifest_text) + self.assertEqual(manifest["settings"]["backup_version"], 27) + self.assertEqual( + manifest["settings"]["source_owner_identity"], + "0ZAHXXHB", + ) + self.assertEqual(manifest["results"]["message_count"], 23) + self.assertEqual(manifest["results"]["attachment_count"], 7) + traceability_text = "\n".join( + ( + manifest_text, + summary_path.read_text(encoding="utf-8"), + Path(temp_dir, "log.txt").read_text(encoding="utf-8"), + ) + ) + self.assertNotIn(PASSWORD, traceability_text) + self.assertNotIn(str(BACKUP_PATH.resolve()), traceability_text) + self.assertNotIn(str(BACKUP_PATH.resolve()).replace("\\", "/"), traceability_text) + self.assertNotIn(PASSWORD, repr(cfg)) + finally: + setup_logging(console=False) + shutil.rmtree(temp_path, ignore_errors=True) + + +if __name__ == "__main__": + unittest.main() From de99b2995ee307cbad8be54e1fb7addcd7122274 Mon Sep 17 00:00:00 2001 From: Jerome Maag Date: Fri, 17 Jul 2026 16:14:47 +0200 Subject: [PATCH 3/3] refactor(threema_android): improve debug logging and remove input_path from metadata --- src/chat_export/threema_android/importer.py | 6 ++++-- tests/test_threema_android_backup.py | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/chat_export/threema_android/importer.py b/src/chat_export/threema_android/importer.py index eb6fc86..a6f83dd 100644 --- a/src/chat_export/threema_android/importer.py +++ b/src/chat_export/threema_android/importer.py @@ -385,7 +385,10 @@ def load_conversations(self, cfg: ExportConfig) -> ImportRun: raise ValueError("A backup password is required") log.info("Loading Threema Android data backup") - log.debug("Loading Threema Android data backup archive=%s", input_path) + log.debug( + "Loading Threema Android data backup archive=%s", + os.path.basename(input_path), + ) with ThreemaAndroidArchive(input_path, cfg.backup_password) as archive: if not archive.has_member("identity"): raise BackupFormatError("Required archive member is missing: identity") @@ -600,7 +603,6 @@ def load_conversations(self, cfg: ExportConfig) -> ImportRun: ) ) metadata = { - "input_path": input_path, "time_mode": TimeMode.UNIX_MS, "timezone": cfg.tz_name, "backup_version": archive.version, diff --git a/tests/test_threema_android_backup.py b/tests/test_threema_android_backup.py index a49abc1..0ab65dd 100644 --- a/tests/test_threema_android_backup.py +++ b/tests/test_threema_android_backup.py @@ -124,6 +124,7 @@ def test_imports_current_backup_structure(self) -> None: self.assertEqual(run.metadata["self_identity"], "0ZAHXXHB") self.assertEqual(run.metadata["missing_media_count"], 1) self.assertEqual(run.metadata["skipped_empty_conversation_count"], 1) + self.assertNotIn("input_path", run.metadata) for item in run.conversations: self.assertEqual( item.conversation.self_participant_id,