diff --git a/CHANGELOG.md b/CHANGELOG.md index 81a0cb7..0c480df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,10 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Option to add attached files to Digital Shipments. +- API endpoint to get attached files from a shipment. + ### Changed - Updated nginx configuration. - Made Memo Label optional for Fjernpost. +- Made shipment order of letters stable to ensure proper caching. ## [0.2.0] diff --git a/src/OpenPostbud/database/document_storage.py b/src/OpenPostbud/database/document_storage.py index 950f67e..b97aa06 100644 --- a/src/OpenPostbud/database/document_storage.py +++ b/src/OpenPostbud/database/document_storage.py @@ -5,12 +5,40 @@ from pathlib import Path import shutil +from dataclasses import dataclass STORAGE_FOLDER = Path("OpenPostbud_document_storage") SHIPMENTS_FOLDER = STORAGE_FOLDER / "Shipments" LETTER_SUFFIX = ".pdf" +# Supported file types per the SF1601 documentation +ATTACHMENT_FILE_TYPES = { + '.pdf': 'application/pdf', + '.html': 'text/html', + '.txt': 'text/plain', + '.doc': 'application/msword', + '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + '.rtf': 'application/msword', + '.bmp': 'image/bmp', + '.gif': 'image/gif', + '.jpg': 'image/jpeg', + '.png': 'image/png', + '.tif': 'image/tiff', + '.xls': 'application/vnd.ms-excel', + '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + '.odt': "application/vnd.oasis.opendocument.text", + '.ods': "application/vnd.oasis.opendocument.spreadsheet", +} + + +@dataclass +class Attachment: + """A dataclass representing an attachment file.""" + name: str + data: bytes + mime_type: str | None = None + def _get_shipment_folder(shipment_id: str) -> Path: """Get the folder associated with the given shipment id.""" @@ -49,3 +77,68 @@ def get_letter_doc(shipment_id: str, letter_id: str) -> bytes | None: return letter_path.read_bytes() except FileNotFoundError: return None + + +def _get_attachments_folder(shipment_id: str) -> Path: + """Get the attachments folder for the given shipment.""" + return _get_shipment_folder(shipment_id) / "attachments" + + +def get_attachments(shipment_id: str) -> list[Attachment]: + """Get all attachments attached to the shipment.""" + folder = _get_attachments_folder(shipment_id) + + if not folder.is_dir(): + return [] + + result = [] + + for file in folder.rglob("*"): + if file.is_file(): + mime_type = ATTACHMENT_FILE_TYPES[file.suffix.lower()] + result.append(Attachment(file.name, file.read_bytes(), mime_type)) + + return result + + +def list_attachments(shipment_id: str) -> list[tuple[str, int]]: + """Return a list of names of all attachments on the shipment. + Includes the index of the attachment to avoid name collisions. + """ + folder = _get_attachments_folder(shipment_id) + + if not folder.is_dir(): + return [] + + result = [] + + for sub_folder in folder.iterdir(): + i = int(sub_folder.name) + file_name = next(sub_folder.iterdir()).name + result.append((file_name, i)) + + return result + + +def get_attachment(shipment_id: str, index: int) -> Attachment: + """Get the attachment file with the given index for the shipment.""" + folder = _get_attachments_folder(shipment_id) / str(index) + + if not folder.is_dir(): + raise ValueError(f"No attachment with index {index} exists for shipment {shipment_id}.") + + file_path = next(folder.iterdir()) + return Attachment(file_path.name, file_path.read_bytes(), ATTACHMENT_FILE_TYPES[file_path.suffix.lower()]) + + +def add_attachments(shipment_id: str, attachments: list[Attachment]): + """Add a list of attachments to the shipment. + This should only ever be called once per shipment. + Each attachment is stored in a numbered folder to avoid name + collisions. + """ + folder = _get_attachments_folder(shipment_id) + for i, attachment in enumerate(attachments): + attachment_path = folder / str(i) / attachment.name + attachment_path.parent.mkdir(parents=True, exist_ok=True) + attachment_path.write_bytes(attachment.data) diff --git a/src/OpenPostbud/routes/api/shipments.py b/src/OpenPostbud/routes/api/shipments.py index ac6d46c..7b522c8 100644 --- a/src/OpenPostbud/routes/api/shipments.py +++ b/src/OpenPostbud/routes/api/shipments.py @@ -5,9 +5,9 @@ from fastapi import APIRouter, status from fastapi.exceptions import HTTPException -from pydantic import BaseModel +from pydantic import BaseModel, Field -from OpenPostbud.database import connection +from OpenPostbud.database import connection, document_storage from OpenPostbud.database.digital_post import shipments as shipments_db from OpenPostbud.database.digital_post import letters as letters_db @@ -29,6 +29,7 @@ class ShipmentDetail(ShipmentModel): """ description: str letter_ids: list[str] + has_attachments: bool class LetterDetail(BaseModel): @@ -37,7 +38,13 @@ class LetterDetail(BaseModel): shipment_id: str recipient_id: str status: str - letter_pdf: str + letter_pdf: str = Field(description="Base64-encoded file contents.") + + +class AttachmentModel(BaseModel): + """A pydantic model representing an attachment response.""" + file_name: str + file_data: str = Field(description="Base64-encoded file contents.") @router.get("/shipments", tags=["Shipments"]) @@ -56,7 +63,7 @@ def get_shipments() -> list[ShipmentModel]: ] -@router.get("/shipment/{shipment_id}", tags=["Shipments"], response_model=ShipmentDetail) +@router.get("/shipment/{shipment_id}", tags=["Shipments"]) def get_shipment(shipment_id: str) -> ShipmentDetail: """Get a shipment by id.""" @@ -74,10 +81,23 @@ def get_shipment(shipment_id: str) -> ShipmentDetail: description=shipment.description, created_at=shipment.created_at, created_by=shipment.created_by, - letter_ids=letter_ids + letter_ids=letter_ids, + has_attachments=len(document_storage.list_attachments(shipment_id)) > 0 ) +@router.get("/shipment/{shipment_id}/attachments", tags=["Shipments"]) +def get_attachments(shipment_id: str) -> list[AttachmentModel]: + """Get all attachments for the given shipment.""" + shipment = shipments_db.get_shipment(shipment_id) + + if not shipment: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "No shipment exists with the given id") + + attachments = document_storage.get_attachments(shipment_id) + return [AttachmentModel(file_name=a.name, file_data=base64.b64encode(a.data).decode()) for a in attachments] + + @router.get("/letter/{letter_id}", tags=["Letters"]) def get_letter(letter_id: str) -> LetterDetail: """Get a letter by id. Merges and returns the final letter as a pdf diff --git a/src/OpenPostbud/routes/user/forsendelser.py b/src/OpenPostbud/routes/user/forsendelser.py index 68d63e7..4552aab 100644 --- a/src/OpenPostbud/routes/user/forsendelser.py +++ b/src/OpenPostbud/routes/user/forsendelser.py @@ -6,7 +6,7 @@ from OpenPostbud.middleware import authentication from OpenPostbud.database.digital_post import letters from OpenPostbud.database.digital_post import shipments, templates -from OpenPostbud.database import db_util +from OpenPostbud.database import db_util, document_storage SHIPMENTS_COLUMNS = [ {'name': "id", 'label': "ID", 'field': "id"}, @@ -76,6 +76,8 @@ def __init__(self, shipment_id: str) -> None: template_name = templates.get_template_name(self.shipment.template_id) + attachments = document_storage.list_attachments(shipment_id) + with ui.grid(columns="auto auto"): ui.label("Navn:").classes("text-bold") ui.label(self.shipment.name) @@ -89,6 +91,12 @@ def __init__(self, shipment_id: str) -> None: ui.label("Skabelon:").classes("text-bold") ui.link(template_name).on("click", self._download_template) + if attachments: + ui.label("Vedhæftede filer:").classes("text-bold") + with ui.column(): + for attachment in attachments: + ui.link(attachment[0]).on("click", lambda i=attachment[1]: self._download_attachment(i)) + ui.label("Oprettet den:").classes("text-bold") ui.label(self.shipment.created_at.strftime("%d/%m/%Y %H:%M:%S")) @@ -110,6 +118,10 @@ def _download_template(self): template = templates.get_template(self.shipment.template_id) ui.download(template.file_data, template.file_name) + def _download_attachment(self, index): + attachment = document_storage.get_attachment(self.shipment.id, index) + ui.download(attachment.data, attachment.name) + async def _abort_shipment(self): """Abort all waiting letters for the shipment.""" if await ui_components.question_popup("Er du sikker på du vil afbryde forsendelsen?", "Afbryd forsendelse", "Annuller"): diff --git a/src/OpenPostbud/routes/user/send_post.py b/src/OpenPostbud/routes/user/send_post.py index 6f2b134..da8dd20 100644 --- a/src/OpenPostbud/routes/user/send_post.py +++ b/src/OpenPostbud/routes/user/send_post.py @@ -3,6 +3,7 @@ from csv import DictReader from collections import Counter from collections.abc import Callable +from pathlib import Path from typing import Literal, NamedTuple import asyncio @@ -12,6 +13,7 @@ from jinja2.exceptions import TemplateSyntaxError from OpenPostbud import ui_components +from OpenPostbud.database import document_storage from OpenPostbud.middleware import authentication from OpenPostbud.database.digital_post import letters, shipments, templates from OpenPostbud.database.digital_post.letters import MemoFields @@ -51,8 +53,13 @@ def __init__(self): get_post_type=lambda: self.step1.post_type.value, ) _stepper_navigation(stepper, validate_callback=self.step2.validate) + with ui.step("Vedhæftede filer") as step: + self.step3 = AttachmentsStep() + _stepper_navigation(stepper) + # Disable entire step if selected post type is physical + step.bind_enabled_from(self.step1.post_type, 'value', backward=lambda v: v != PostType.PHYSICAL) with ui.step("Gennemgå eksempler"): - self.step3 = ExamplesStep(merge_letter=self.step2.merge_letter) + self.step4 = ExamplesStep(merge_letter=self.step2.merge_letter) _stepper_navigation(stepper) with ui.step("Send post"): ui.button("Send Post", on_click=self._send_post) @@ -64,12 +71,16 @@ def __init__(self): def _on_csv_data_changed(self, fields: list[str], rows: list[dict[str, str]] | None): """Forward csv changes from step 2 to step 3.""" - self.step3.set_data(fields, rows) + self.step4.set_data(fields, rows) - def _send_post(self): + async def _send_post(self): """Add the shipment and letters to the database and navigate to the detail page of the shipment. """ + # Read the attachments before the spinner dialog steals focus, since it + # relies on a round-trip to the client. + attachments = await self.step3.get_attachments() + with ui.dialog(value=True) as dialog: dialog.props("persistent") ui.spinner(size="5em") @@ -83,6 +94,7 @@ def _send_post(self): template_id, self.step1.post_type.value) letters.add_letters(shipment_id, self.step2.csv_data) + document_storage.add_attachments(shipment_id, attachments) ui.navigate.to(app.url_path_for("Shipment Detail", shipment_id=shipment_id)) finally: dialog.close() @@ -273,6 +285,54 @@ def refresh_messages(self): self.message_area.add_message(msg.text, type_=msg.type_) +class AttachmentsStep: + """A class representing the attachments step in the Send Post flow. + Here the user can upload extra files to be sent alongside the letter. + """ + def __init__(self): + ui.label("Her kan du vedhæfte ekstra filer til forsendelsen.") + ui.label("Vedhæftede filer sendes, som de er, og flettes derfor ikke.") + ui.label("Digital Post understøtter op til 10 vedhæftede filer og op til 74MB i alt inkl. brev.") + ui.label("Bemærk at vedhæftede filer kun understøttes i Digital Post.") + + self._attachments: dict[tuple[str, int], document_storage.Attachment] = {} + + with ui.grid(columns=1): + file_types = f"accept={','.join(document_storage.ATTACHMENT_FILE_TYPES.keys())}" + self.upload = ui.upload(multiple=True, max_files=10, auto_upload=True, on_upload=self._on_upload, on_rejected=lambda: ui.notify("Upload afvist", type="warning")).props(file_types) + self.remove_button = ui_components.DisableButton("Nulstil vedhæftninger", on_click=self._remove_attachments) + self.remove_button.disable() + + async def _on_upload(self, e: UploadEventArguments): + """Buffer each uploaded file, skipping unsupported file types.""" + suffix = Path(e.file.name).suffix.lower() + if suffix not in document_storage.ATTACHMENT_FILE_TYPES: + ui.notify(f"Filtypen '{suffix}' understøttes ikke: {e.file.name}", type="negative") + return + self._attachments[(e.file.name, e.file.size())] = document_storage.Attachment( + e.file.name, await e.file.read() + ) + self.remove_button.enable() + + def _remove_attachments(self): + """Remove all already uploaded attachments.""" + self._attachments = {} + self.remove_button.disable() + self.upload.reset() + + async def get_attachments(self) -> list[document_storage.Attachment]: + """Return the attachments still shown in the uploader. + + The buffer is reconciled against the uploader's current file list, so + files the user removed in the browser are excluded. + """ + names = await ui.run_javascript( + f"return getElement({self.upload.id}).$refs.qRef.files.map(f => [f.name, f.size])" + ) + names = [tuple(n) for n in names] + return [self._attachments[name] for name in names if name in self._attachments] + + class ExamplesStep: """A class representing the third step in the Send Post flow. Here the user can verify the uploaded data and download sample letters. diff --git a/src/OpenPostbud/workers/shipment_worker.py b/src/OpenPostbud/workers/shipment_worker.py index dca3b75..b241924 100644 --- a/src/OpenPostbud/workers/shipment_worker.py +++ b/src/OpenPostbud/workers/shipment_worker.py @@ -4,6 +4,7 @@ import base64 from datetime import datetime, timedelta +from functools import lru_cache import logging import time import uuid @@ -12,7 +13,7 @@ from sqlalchemy import select, update from python_serviceplatformen.authentication import KombitAccess from python_serviceplatformen import digital_post -from python_serviceplatformen.models.message import Message, MessageHeader, MessageBody, MainDocument, Sender, Recipient, File +from python_serviceplatformen.models.message import Message, MessageHeader, MessageBody, MainDocument, Sender, Recipient, File, AdditionalDocument from python_serviceplatformen.models.physical_mail import create_physical_mail from requests import Timeout, HTTPError @@ -20,7 +21,14 @@ from OpenPostbud.database import connection from OpenPostbud.database.digital_post.letters import Letter, MemoFields from OpenPostbud.database.digital_post import shipments +from OpenPostbud.database.digital_post.shipments import Shipment from OpenPostbud.database.common import ShipmentStatus, PostType +from OpenPostbud.database import document_storage + + +# Maximum size of files before base 64 encoding accepted by the receiving APIs. +DIGITAL_MAX_PAYLOAD_BYTES = 74 * 1024 * 1024 +PHYSICAL_MAX_PAYLOAD_BYTES = 7 * 1024 * 1024 def start_process(): @@ -64,10 +72,12 @@ def get_waiting_letter() -> Letter | None: with connection.get_session() as session: sub_q = ( select(Letter.id) + .join(Shipment, Letter.shipment_id == Shipment.id) .where( Letter.status == ShipmentStatus.WAITING, datetime.now() - timedelta(seconds=config.SHIPMENT_WORKER_DELAY) > Letter.updated_at ) + .order_by(Shipment.created_at, Letter.shipment_id) .limit(1) .scalar_subquery() ) @@ -130,6 +140,30 @@ def send_digital(letter: Letter, kombit_access: KombitAccess): id_type = "CPR" if len(letter.recipient_id) == 10 else "CVR" + attachments = _get_attachments(letter.shipment_id) + + payload_size = len(document) + sum(len(attachment.data) for attachment in attachments) + if payload_size > DIGITAL_MAX_PAYLOAD_BYTES: + letter.set_status(ShipmentStatus.FAILED, message=f"Digital Post størrelse oversteg {DIGITAL_MAX_PAYLOAD_BYTES/(1024*1024)}MB") + logging.error(f"Digital letter failed. Total document size exceeded limit of {DIGITAL_MAX_PAYLOAD_BYTES}: {payload_size}") + return + + additional_documents = [] + + for attachment in attachments: + additional_documents.append( + AdditionalDocument( + files=[ + File( + encodingFormat=attachment.mime_type, + filename=attachment.name, + language="da", + content=base64.b64encode(attachment.data).decode() + ) + ] + ) + ) + message = Message( messageHeader=MessageHeader( messageType="DIGITALPOST", @@ -156,7 +190,8 @@ def send_digital(letter: Letter, kombit_access: KombitAccess): content=b64_doc ) ] - ) + ), + additionalDocuments=additional_documents ) ) @@ -174,6 +209,11 @@ def send_physical(letter: Letter, kombit_access: KombitAccess): """ document = letter.merge_letter() + if len(document) > PHYSICAL_MAX_PAYLOAD_BYTES: + letter.set_status(ShipmentStatus.FAILED, message=f"Fjernpost størrelse oversteg {PHYSICAL_MAX_PAYLOAD_BYTES/(1024*1024)}MB") + logging.error(f"Physical letter failed. Letter size exceeded limit of {PHYSICAL_MAX_PAYLOAD_BYTES}: {len(document)}") + return + forsendelse = create_physical_mail(config.PHYSICAL_MAIL_FORSENDELSE_TYPE, document) logging.info(f"Sending physical letter {letter.id}") @@ -182,5 +222,11 @@ def send_physical(letter: Letter, kombit_access: KombitAccess): logging.info(f"Physical letter sent {letter.id} - {transaction_id=} - afsendelse={forsendelse.afsendelse_identifikator.value}") +@lru_cache(maxsize=1) +def _get_attachments(shipment_id: str) -> list[document_storage.Attachment]: + """Wrapper around document_storage.get_attachments to allow caching.""" + return document_storage.get_attachments(shipment_id) + + if __name__ == '__main__': start_process()