diff --git a/TEKDB/Dockerfile b/TEKDB/Dockerfile index 896b7676..1ba7ac1f 100644 --- a/TEKDB/Dockerfile +++ b/TEKDB/Dockerfile @@ -22,6 +22,7 @@ WORKDIR /usr/src/app COPY requirements.txt requirements_linux.txt /usr/src/app/ # Upgrade pip and install Python dependencies +# Note: editable packages (-e) will be installed at runtime via entrypoint.sh RUN pip install --upgrade pip \ && pip install -r requirements.txt -r requirements_linux.txt diff --git a/TEKDB/TEKDB/admin.py b/TEKDB/TEKDB/admin.py index 04c5bbb5..cc551409 100644 --- a/TEKDB/TEKDB/admin.py +++ b/TEKDB/TEKDB/admin.py @@ -61,6 +61,7 @@ from TEKDB.settings import BASE_DIR from TEKDB.widgets import OpenLayers6Widget + admin.site.site_header = ADMIN_SITE_HEADER @@ -626,57 +627,69 @@ def save_model(self, request, obj, form, change): activities = form.cleaned_data.get("activities") placesresources = form.cleaned_data.get("placesresources") - for file in request.FILES.getlist("files"): - mime_type, _ = guess_type(file.name) - # if mime_type: - file_mime_type = mime_type.split("/")[0] - media_type_instance = LookupMediaType.objects.filter( - mediatype__startswith=file_mime_type - ).first() - if media_type_instance: - mediatype = media_type_instance - else: - media_type_instance = LookupMediaType.objects.filter( - mediatype__startswith="other" - ).first() + # Handle async uploaded file (comes as a list of file path strings) + uploaded_file_paths = form.cleaned_data.get("files") + + if uploaded_file_paths and isinstance(uploaded_file_paths, list): + import os + + for uploaded_file_path in uploaded_file_paths: + # Extract just the filename from the path + file_name = os.path.basename(uploaded_file_path) + + # Guess MIME type from filename + mime_type, _ = guess_type(file_name) + if mime_type: + file_mime_type = mime_type.split("/")[0] + media_type_instance = LookupMediaType.objects.filter( + mediatype__startswith=file_mime_type + ).first() + else: + media_type_instance = None + + if not media_type_instance: + media_type_instance = LookupMediaType.objects.filter( + mediatype__startswith="other" + ).first() + mediatype = media_type_instance - filename = file.name.split(".")[0] + filename = file_name.rsplit(".", 1)[0] # Remove extension - media_instance = Media( - medianame=filename, - mediadescription=f'Part of the "{obj.mediabulkname}" Media Bulk Upload that was uploaded on {obj.mediabulkdate}', - mediafile=file, - mediatype=mediatype, - ) - media_instance.save() - obj.mediabulkupload.add(media_instance) - - # Add relationships - if places: - for place in places: - PlacesMediaEvents.objects.create( - placeid=place, mediaid=media_instance - ) - if resources: - for resource in resources: - ResourcesMediaEvents.objects.create( - resourceid=resource, mediaid=media_instance - ) - if citations: - for citation in citations: - MediaCitationEvents.objects.create( - citationid=citation, mediaid=media_instance - ) - if activities: - for activity in activities: - ResourceActivityMediaEvents.objects.create( - resourceactivityid=activity, mediaid=media_instance - ) - if placesresources: - for placeresource in placesresources: - PlacesResourceMediaEvents.objects.create( - placeresourceid=placeresource, mediaid=media_instance - ) + media_instance = Media( + medianame=filename, + mediadescription=f'Part of the "{obj.mediabulkname}" Media Bulk Upload that was uploaded on {obj.mediabulkdate}', + mediafile=uploaded_file_path, + mediatype=mediatype, + ) + media_instance.save() + obj.mediabulkupload.add(media_instance) + + # Add relationships + if places: + for place in places: + PlacesMediaEvents.objects.create( + placeid=place, mediaid=media_instance + ) + if resources: + for resource in resources: + ResourcesMediaEvents.objects.create( + resourceid=resource, mediaid=media_instance + ) + if citations: + for citation in citations: + MediaCitationEvents.objects.create( + citationid=citation, mediaid=media_instance + ) + if activities: + for activity in activities: + ResourceActivityMediaEvents.objects.create( + resourceactivityid=activity, mediaid=media_instance + ) + if placesresources: + for placeresource in placesresources: + PlacesResourceMediaEvents.objects.create( + placeresourceid=placeresource, mediaid=media_instance + ) @admin.display(description="Thumbnails") def thumbnail_gallery(self, obj): diff --git a/TEKDB/TEKDB/forms.py b/TEKDB/TEKDB/forms.py index 9808785c..1be8e12a 100644 --- a/TEKDB/TEKDB/forms.py +++ b/TEKDB/TEKDB/forms.py @@ -1,5 +1,7 @@ from django import forms from django.contrib.admin.widgets import FilteredSelectMultiple +from django_resumable_async_upload.fields import FormResumableMultipleFileField +from django_resumable_async_upload.widgets import ResumableAdminWidget from .models import ( MediaBulkUpload, Places, @@ -7,26 +9,17 @@ Citations, ResourcesActivityEvents, PlacesResourceEvents, + Media, ) -from .widgets import ThumbnailFileInput - - -class MultipleFileField(forms.FileField): - def __init__(self, *args, **kwargs): - kwargs.setdefault("widget", ThumbnailFileInput) - super().__init__(*args, **kwargs) - - def clean(self, data, initial=None): - single_file_clean = super().clean - if isinstance(data, (list, tuple)): - result = [single_file_clean(d, initial) for d in data] - else: - result = [single_file_clean(data, initial)] - return result class MediaBulkUploadForm(forms.ModelForm): - files = MultipleFileField() + files = FormResumableMultipleFileField( + required=False, + # not passing max_files here because FormResumableFileField defaults to undefined (unlimited), + # which is what we want for bulk upload. + widget=ResumableAdminWidget(attrs={"model": Media, "field_name": "mediafile"}), + ) places = forms.ModelMultipleChoiceField( queryset=Places.objects.all(), required=False, diff --git a/TEKDB/TEKDB/migrations/0026_alter_media_mediafile_and_more.py b/TEKDB/TEKDB/migrations/0026_alter_media_mediafile_and_more.py new file mode 100644 index 00000000..76ca5d54 --- /dev/null +++ b/TEKDB/TEKDB/migrations/0026_alter_media_mediafile_and_more.py @@ -0,0 +1,24 @@ +# Generated by Django 4.2.27 on 2026-01-30 19:11 + +from django.db import migrations, models +import django_resumable_async_upload.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('TEKDB', '0025_alter_mediabulkupload_mediabulkname'), + ] + + operations = [ + migrations.AlterField( + model_name='media', + name='mediafile', + field=django_resumable_async_upload.models.AsyncFileField(blank=True, db_column='mediafile', max_files=1, max_length=255, null=True, upload_to='', verbose_name='file'), + ), + migrations.AlterField( + model_name='mediabulkupload', + name='mediabulkname', + field=models.CharField(blank=True, default='Bulk Upload on 2026-01-30', max_length=255, null=True, verbose_name='name'), + ), + ] diff --git a/TEKDB/TEKDB/models.py b/TEKDB/TEKDB/models.py index a4dd8d3a..f91e20fb 100644 --- a/TEKDB/TEKDB/models.py +++ b/TEKDB/TEKDB/models.py @@ -24,6 +24,8 @@ from django.conf import settings from django.contrib.gis.db.models import GeometryField from tinymce.models import HTMLField +from django_resumable_async_upload.models import AsyncFileField + # from moderation.db import ModeratedModel import os @@ -2753,12 +2755,13 @@ class Media(Reviewable, Queryable, Record, ModeratedModel): null=True, verbose_name="historic location", ) - mediafile = models.FileField( + mediafile = AsyncFileField( db_column="mediafile", max_length=255, blank=True, null=True, verbose_name="file", + max_files=1, ) limitedaccess = models.BooleanField( db_column="limitedaccess", diff --git a/TEKDB/TEKDB/settings.py b/TEKDB/TEKDB/settings.py index 1b0a3d10..888e00a6 100644 --- a/TEKDB/TEKDB/settings.py +++ b/TEKDB/TEKDB/settings.py @@ -70,6 +70,7 @@ "Relationships", "reversion", "django.contrib.sites", + "django_resumable_async_upload", # 'moderation.apps.SimpleModerationConfig', ] @@ -346,6 +347,11 @@ DEFAULT_MAXIMUM_RESULTS = 500 +# Django Resumable Async Upload settings +ADMIN_RESUMABLE_SHOW_THUMB = True +ADMIN_SIMULTANEOUS_UPLOADS = 1 +ADMIN_RESUMABLE_CHUNK_FOLDER = "resumable_chunks" + try: from TEKDB.local_settings import * # noqa: F403 except Exception: diff --git a/TEKDB/TEKDB/tests/test_admin.py b/TEKDB/TEKDB/tests/test_admin.py index 8a713b86..b433fdb4 100644 --- a/TEKDB/TEKDB/tests/test_admin.py +++ b/TEKDB/TEKDB/tests/test_admin.py @@ -1,7 +1,7 @@ # from django.conf import settings +import json from django.test import RequestFactory from unittest.mock import patch -from django.core.files.uploadedfile import SimpleUploadedFile from django.contrib.auth import get_user_model from django.contrib.admin.sites import AdminSite from django.urls import reverse @@ -165,9 +165,8 @@ def test_media_bulk_upload_admin_add(self): from TEKDB.admin import MediaBulkUploadAdmin url = reverse("admin:TEKDB_mediabulkupload_add") - test_image = SimpleUploadedFile( - "./test_image.jpg", b"\x00\x00\x00\x00", content_type="image" - ) + + test_image_paths = ["test_image.jpg", "test_image.jpg"] place = Places.objects.create(indigenousplacename="Test Place") resource = Resources.objects.create(commonname="Test Resource") @@ -180,7 +179,7 @@ def test_media_bulk_upload_admin_add(self): activity = ResourcesActivityEvents.objects.create(placeresourceid=placeresource) post_data = { - "files": [test_image, test_image], + "files": json.dumps(test_image_paths), "places": [place.pk], "resources": [resource.pk], "citations": [citation.pk], @@ -208,14 +207,6 @@ def test_media_bulk_upload_admin_add(self): PlacesResourceEvents.objects.filter(pk=placeresource.pk).exists() ) - for media in Media.objects.filter(medianame="test_image"): - self.assertTrue(os.path.exists(media.mediafile.path)) - os.remove( - media.mediafile.path - ) # Clean up the uploaded files after the test - self.assertFalse(os.path.exists(media.mediafile.path)) - media.delete() - # Clean up related objects activity.delete() placeresource.delete() @@ -227,14 +218,11 @@ def test_media_bulk_upload_admin_other_types(self): from TEKDB.admin import MediaBulkUploadAdmin url = reverse("admin:TEKDB_mediabulkupload_add") - test_other_type = SimpleUploadedFile( - "./test_thing.shp", b"\x00\x00\x00\x00", content_type="other" - ) request = self.factory.post( url, { - "files": [test_other_type, test_other_type], + "files": json.dumps(["test_thing.shp", "test_thing.shp"]), }, ) @@ -245,48 +233,25 @@ def test_media_bulk_upload_admin_other_types(self): bulk_admin.save_model( obj=MediaBulkUpload(), request=request, form=bulk_form, change=None ) - - for media in Media.objects.filter(medianame="test_thing"): - self.assertTrue(os.path.exists(media.mediafile.path)) - os.remove( - media.mediafile.path - ) # Clean up the uploaded files after the test - self.assertFalse(os.path.exists(media.mediafile.path)) - media.delete() + self.assertTrue(Media.objects.filter(medianame="test_thing").exists()) + self.assertEqual(Media.objects.filter(medianame="test_thing").count(), 2) def test_media_bulk_upload_admin_thumbnail_gallery(self): from TEKDB.admin import MediaBulkUploadAdmin from TEKDB.models import MediaBulkUpload, Media url = reverse("admin:TEKDB_mediabulkupload_add") - test_image = SimpleUploadedFile( - "./thumbnail_test_image.jpg", b"\x00\x00\x00\x00", content_type="image" - ) - test_video = SimpleUploadedFile( - "./thumbnail_test_video.mp4", b"\x00\x00\x00\x00", content_type="video" - ) - test_audio = SimpleUploadedFile( - "./thumbnail_test_audio.mp3", b"\x00\x00\x00\x00", content_type="audio" - ) - test_text = SimpleUploadedFile( - "./thumbnail_test_text.txt", b"\x00\x00\x00\x00", content_type="text" - ) - test_other = SimpleUploadedFile( - "./thumbnail_test_thing.shp", b"\x00\x00\x00\x00", content_type="other" - ) - test_unknown_type = SimpleUploadedFile( - "./thumbnail_test_unknown.xyz", b"\x00\x00\x00\x00", content_type="unknown" - ) + test_file_paths = [ + "thumbnail_test_image.jpg", + "thumbnail_test_video.mp4", + "thumbnail_test_audio.mp3", + "thumbnail_test_text.txt", + "thumbnail_test_thing.shp", + "thumbnail_test_unknown.xyz", + ] post_data = { - "files": [ - test_image, - test_video, - test_audio, - test_text, - test_other, - test_unknown_type, - ], + "files": json.dumps(test_file_paths), } request = self.factory.post(url, post_data) diff --git a/TEKDB/TEKDB/urls.py b/TEKDB/TEKDB/urls.py index ef2e60f3..eb694a67 100644 --- a/TEKDB/TEKDB/urls.py +++ b/TEKDB/TEKDB/urls.py @@ -77,6 +77,10 @@ views.ResourceActivityAutocompleteView.as_view(), name="select2_fk_resourceactivity", ), + re_path( + r"^django_resumable_async_upload/", + include("django_resumable_async_upload.urls"), + ), path("", include("explore.urls")), ] urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) diff --git a/TEKDB/TEKDB/widgets.py b/TEKDB/TEKDB/widgets.py index 200fb1f0..4187a729 100644 --- a/TEKDB/TEKDB/widgets.py +++ b/TEKDB/TEKDB/widgets.py @@ -2,8 +2,6 @@ from django.contrib.gis.geometry import json_regex from django.contrib.gis.forms.widgets import BaseGeometryWidget -from django.forms.widgets import ClearableFileInput -from django.utils.safestring import mark_safe logger = logging.getLogger("django.contrib.gis") @@ -61,23 +59,3 @@ class Media: "assets/openlayers6/ol.js", "gis/js/OL6MapPolygonWidget.js", ) - - -class ThumbnailFileInput(ClearableFileInput): - template_name = "widgets/thumbnail_file_input.html" - allow_multiple_selected = True # Enable multiple file uploads - - def format_value(self, value): - if value and hasattr(value, "url"): - return mark_safe(f'') - return super().format_value(value) - - def __init__(self, attrs=None): - if attrs is None: - attrs = {} - attrs.update( - { - "multiple": "multiple" # Allow multiple file selection - } - ) - super().__init__(attrs) diff --git a/TEKDB/requirements.txt b/TEKDB/requirements.txt index 276851e8..9a0cf228 100644 --- a/TEKDB/requirements.txt +++ b/TEKDB/requirements.txt @@ -14,8 +14,8 @@ pillow psycopg2-binary psutil django-filebrowser-no-grappelli>=4.0.0,<5.0.0 - XlsxWriter +django-resumable-async-upload #-e git+https://github.com/dominno/django-moderation.git@master#egg=moderation #DEPENDENCIES @@ -35,4 +35,4 @@ ipython # Serve static files in production containers whitenoise>=6.0.0,<7.0.0 # FORMATTING AND LINTING -ruff +ruff \ No newline at end of file