feat(uploads): chunked resumable image upload with JP2 ingest pipeline

.gitignore

@@ -162,6 +162,9 @@ cython_debug/
 .vscode/
 staticfiles
 uploads
+# `uploads` above targets runtime upload dirs, but being unanchored it also
+# matches the apps/uploads Django app — re-include the source code.
+!apps/uploads/
 storage
 sipi
 *.sql

Dockerfile

@@ -6,8 +6,12 @@ ENV PYTHONUNBUFFERED=true
 LABEL org.opencontainers.image.source="https://github.com/archetype-pal/backend"
 LABEL authors="ahmed.elghareeb@proton.com"
 
-# Pull in latest security patches before anything else
-RUN apt-get update && apt-get upgrade -y && rm -rf /var/lib/apt/lists/*
+# Pull in latest security patches before anything else.
+# libvips-tools provides the `vips` CLI used by the upload-ingest pipeline
+# (apps.uploads) to convert uploads to lossless JP2 before SIPI serves them.
+RUN apt-get update && apt-get upgrade -y && \
+    apt-get install -y --no-install-recommends libvips-tools && \
+    rm -rf /var/lib/apt/lists/*
 
 # Create non-root user early for improved security
 RUN groupadd -r archetype && useradd -r -g archetype archetype

apps/common/views.py

@@ -93,6 +93,7 @@ def get(self, request: Request) -> Response:
             settings.BASE_DIR / "apps/scribes/schema.yaml",
             settings.BASE_DIR / "apps/annotations/schema.yaml",
             settings.BASE_DIR / "apps/worksets/schema.yaml",
+            settings.BASE_DIR / "apps/uploads/schema.yaml",
         ]
         core_object: dict[str, Any] = self._load_schema_file(core_file)
         for supporting_file in supporting_files:

apps/manuscripts/apps.py

@@ -10,6 +10,6 @@ def ready(self) -> None:
         from apps.common.audit import register_audited_models
 
         from . import signals  # noqa: F401  (registers the Graph pre_delete receiver)
-        from .models import ImageText
+        from .models import ImageText, ItemImage
 
-        register_audited_models(ImageText)
+        register_audited_models(ImageText, ItemImage)

apps/manuscripts/migrations/0023_itemimage_checksum_sha256_itemimage_created_and_more.py

@@ -0,0 +1,66 @@
+# Generated by Django 6.0.6 on 2026-07-16 09:51
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('manuscripts', '0022_alter_historicalitem_date'),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='itemimage',
+            name='checksum_sha256',
+            field=models.CharField(blank=True, default='', max_length=64),
+        ),
+        migrations.AddField(
+            model_name='itemimage',
+            name='created',
+            field=models.DateTimeField(auto_now_add=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='itemimage',
+            name='exif',
+            field=models.JSONField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='itemimage',
+            name='height',
+            field=models.PositiveIntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='itemimage',
+            name='modified',
+            field=models.DateTimeField(auto_now=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='itemimage',
+            name='original_path',
+            field=models.CharField(blank=True, default='', max_length=255),
+        ),
+        migrations.AddField(
+            model_name='itemimage',
+            name='size_bytes',
+            field=models.BigIntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='itemimage',
+            name='source_format',
+            field=models.CharField(blank=True, default='', max_length=16),
+        ),
+        migrations.AddField(
+            model_name='itemimage',
+            name='uploaded_by',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='uploaded_images', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AddField(
+            model_name='itemimage',
+            name='width',
+            field=models.PositiveIntegerField(blank=True, null=True),
+        ),
+    ]

apps/manuscripts/migrations/0024_merge_itemimage_metadata_msdescarea.py

@@ -0,0 +1,17 @@
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    """Relink the two concurrent 0023 leaves into a single line.
+
+    A rebase left two migrations branching off 0022: this feature's ItemImage
+    upload-metadata fields and main's MsDescArea model. They touch different
+    tables, so this is a pure ordering merge with no operations.
+    """
+
+    dependencies = [
+        ("manuscripts", "0023_itemimage_checksum_sha256_itemimage_created_and_more"),
+        ("manuscripts", "0023_msdescarea"),
+    ]
+
+    operations = []

apps/manuscripts/models.py

@@ -212,6 +212,24 @@ class ItemImage(models.Model):
     locus = models.CharField(max_length=72, blank=True, default="")
     tags = tagulous.models.TagField(force_lowercase=True, blank=True)
 
+    # Technical metadata captured by the upload pipeline (apps.uploads). All
+    # nullable/blank: the migrated corpus predates these and stays untouched.
+    width = models.PositiveIntegerField(null=True, blank=True)
+    height = models.PositiveIntegerField(null=True, blank=True)
+    source_format = models.CharField(max_length=16, blank=True, default="")
+    size_bytes = models.BigIntegerField(null=True, blank=True)
+    checksum_sha256 = models.CharField(max_length=64, blank=True, default="")
+    # Relative path of the archived upload under UPLOADS_ORIGINALS_DIR (not
+    # MEDIA_ROOT — originals must never be SIPI-servable). Blank for migrated
+    # rows and for uploads whose served .jp2 IS the original bytes.
+    original_path = models.CharField(max_length=255, blank=True, default="")
+    exif = models.JSONField(null=True, blank=True)
+    uploaded_by = models.ForeignKey(
+        settings.AUTH_USER_MODEL, null=True, blank=True, on_delete=models.SET_NULL, related_name="uploaded_images"
+    )
+    created = models.DateTimeField(null=True, blank=True, auto_now_add=True)
+    modified = models.DateTimeField(null=True, blank=True, auto_now=True)
+
     class Meta:
         ordering = ["item_part", "locus"]
 

apps/manuscripts/schema.yaml

@@ -288,6 +288,77 @@ paths:
                   $ref: '#/components/schemas/ItemImageManagementItem'
       tags:
         - manuscripts-management
+    post:
+      operationId: management-item-images-create
+      security:
+        - api_key: []
+      description: >-
+        Register an ItemImage row pointing at an EXISTING media-relative path.
+        `image` accepts only a path string — uploading file bytes goes through
+        /api/v1/uploads/, which converts to JP2 and smoke-tests SIPI first.
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/ItemImageManagementItem'
+      responses:
+        201:
+          description: Created item image.
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ItemImageManagementItem'
+        400:
+          description: Invalid payload (e.g. a file instead of a path string).
+      tags:
+        - manuscripts-management
+  /api/v1/manuscripts/management/item-images/{id}/:
+    patch:
+      operationId: management-item-images-update
+      security:
+        - api_key: []
+      description: >-
+        Update locus/tags or repoint `image` at another existing media path
+        (string only; never a file upload).
+      parameters:
+        - name: id
+          in: path
+          required: true
+          schema:
+            type: integer
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/ItemImageManagementItem'
+      responses:
+        200:
+          description: Updated item image.
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ItemImageManagementItem'
+        400:
+          description: Invalid payload (e.g. a file instead of a path string).
+      tags:
+        - manuscripts-management
+    delete:
+      operationId: management-item-images-delete
+      security:
+        - api_key: []
+      parameters:
+        - name: id
+          in: path
+          required: true
+          schema:
+            type: integer
+      responses:
+        204:
+          description: Item image deleted (cascades to its texts and graphs).
+      tags:
+        - manuscripts-management
   /api/v1/manuscripts/management/image-texts/:
     get:
       operationId: management-image-texts-list
@@ -692,14 +763,51 @@ components:
           type: integer
         image:
           type: string
+          description: >-
+            Media-relative path (the SIPI IIIF identifier). Writes accept ONLY
+            a path string — file bytes must go through /api/v1/uploads/.
         locus:
           type: string
+        tags:
+          # tagulous TagField serializes to a list of tag names (empty = []),
+          # not a comma-joined string. (The write-input shape on
+          # UploadSessionCreate.tags IS a string — different direction.)
+          type: array
+          items:
+            type: string
         annotation_count:
           type: integer
         texts:
           type: array
           items:
             $ref: '#/components/schemas/ImageTextManagementItem'
+        width:
+          type: integer
+          nullable: true
+        height:
+          type: integer
+          nullable: true
+        source_format:
+          type: string
+        size_bytes:
+          type: integer
+          nullable: true
+        checksum_sha256:
+          type: string
+        original_path:
+          type: string
+          description: Archive-relative path of the preservation original; empty for migrated rows.
+        uploaded_by_username:
+          type: string
+          nullable: true
+        created:
+          type: string
+          format: date-time
+          nullable: true
+        modified:
+          type: string
+          format: date-time
+          nullable: true
     CatalogueNumberManagementItem:
       type: object
       properties:

apps/manuscripts/serializers/management.py

@@ -128,13 +128,69 @@ class Meta:
         read_only_fields = fields
 
 
+class ImagePathField(serializers.CharField):
+    """`ItemImage.image` as the media-relative path string it really is.
+
+    The model field is an ImageField subclass, so DRF's default mapping was a
+    binary file field that 400'd the backoffice's JSON path edits. This field
+    deliberately accepts ONLY strings — raw byte uploads must go through
+    `apps.uploads`, which normalizes to JP2 and smoke-tests a SIPI tile before
+    any row exists (otherwise unconverted files recreate issue #114).
+    """
+
+    def to_internal_value(self, data):
+        if not isinstance(data, str):
+            raise serializers.ValidationError(
+                "Provide a media-relative path string. File uploads go through /api/v1/uploads/."
+            )
+        value = data.strip().lstrip("/")
+        if not value:
+            raise serializers.ValidationError("Image path cannot be empty.")
+        if ".." in value.split("/"):
+            raise serializers.ValidationError("Image path may not contain '..'.")
+        return super().to_internal_value(value)
+
+    def to_representation(self, value):
+        # The FieldFile's .name is the stored relative path.
+        return str(getattr(value, "name", value) or "")
+
+
 class ItemImageManagementSerializer(serializers.ModelSerializer):
     texts = ImageTextManagementSerializer(many=True, read_only=True)
     annotation_count = serializers.IntegerField(read_only=True)
+    image = ImagePathField(max_length=200)
+    uploaded_by_username = serializers.CharField(source="uploaded_by.username", read_only=True, default=None)
 
     class Meta:
         model = ItemImage
-        fields = ["id", "item_part", "image", "locus", "tags", "texts", "annotation_count"]
+        fields = [
+            "id",
+            "item_part",
+            "image",
+            "locus",
+            "tags",
+            "texts",
+            "annotation_count",
+            "width",
+            "height",
+            "source_format",
+            "size_bytes",
+            "checksum_sha256",
+            "original_path",
+            "uploaded_by_username",
+            "created",
+            "modified",
+        ]
+        read_only_fields = [
+            "width",
+            "height",
+            "source_format",
+            "size_bytes",
+            "checksum_sha256",
+            "original_path",
+            "created",
+            "modified",
+        ]
 
 
 class CatalogueNumberManagementSerializer(serializers.ModelSerializer):

apps/manuscripts/services/__init__.py

@@ -11,7 +11,7 @@
 
 logger = logging.getLogger(__name__)
 
-_IMAGE_EXTENSIONS: tuple[str, ...] = (".png", ".jpg", ".jpeg", ".gif", ".tif")
+_IMAGE_EXTENSIONS: tuple[str, ...] = (".png", ".jpg", ".jpeg", ".gif", ".tif", ".tiff", ".jp2")
 
 
 def build_image_picker_payload(*, media_root: str, relative_path: str) -> dict[str, list[dict[str, str]]]: