From 7a8b5ba7f7a236af450d94b701759b789dd5842a Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Wed, 7 Jan 2026 16:11:57 -0800 Subject: [PATCH 01/16] wip: use django-async-upload for media records --- TEKDB/TEKDB/models.py | 4 +++- TEKDB/TEKDB/settings.py | 1 + TEKDB/TEKDB/urls.py | 1 + TEKDB/requirements.txt | 1 + 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/TEKDB/TEKDB/models.py b/TEKDB/TEKDB/models.py index a4dd8d3a..4a31117f 100644 --- a/TEKDB/TEKDB/models.py +++ b/TEKDB/TEKDB/models.py @@ -24,6 +24,8 @@ from django.conf import settings from django.contrib.gis.db.models import GeometryField from tinymce.models import HTMLField +from admin_async_upload.models import AsyncFileField + # from moderation.db import ModeratedModel import os @@ -2753,7 +2755,7 @@ class Media(Reviewable, Queryable, Record, ModeratedModel): null=True, verbose_name="historic location", ) - mediafile = models.FileField( + mediafile = AsyncFileField( db_column="mediafile", max_length=255, blank=True, diff --git a/TEKDB/TEKDB/settings.py b/TEKDB/TEKDB/settings.py index 1b0a3d10..7be9ba58 100644 --- a/TEKDB/TEKDB/settings.py +++ b/TEKDB/TEKDB/settings.py @@ -70,6 +70,7 @@ "Relationships", "reversion", "django.contrib.sites", + "admin_async_upload", # 'moderation.apps.SimpleModerationConfig', ] diff --git a/TEKDB/TEKDB/urls.py b/TEKDB/TEKDB/urls.py index ef2e60f3..536e2d27 100644 --- a/TEKDB/TEKDB/urls.py +++ b/TEKDB/TEKDB/urls.py @@ -77,6 +77,7 @@ views.ResourceActivityAutocompleteView.as_view(), name="select2_fk_resourceactivity", ), + re_path(r"^admin_async_upload/", include("admin_async_upload.urls")), path("", include("explore.urls")), ] urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) diff --git a/TEKDB/requirements.txt b/TEKDB/requirements.txt index 276851e8..590b2934 100644 --- a/TEKDB/requirements.txt +++ b/TEKDB/requirements.txt @@ -14,6 +14,7 @@ pillow psycopg2-binary psutil django-filebrowser-no-grappelli>=4.0.0,<5.0.0 +django-async-upload XlsxWriter #-e git+https://github.com/dominno/django-moderation.git@master#egg=moderation From 619adf59d34ea12f84271bb8aada2f9e33e0e2f9 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Thu, 8 Jan 2026 15:14:40 -0800 Subject: [PATCH 02/16] use local admin async libraary during development --- TEKDB/Dockerfile | 1 + TEKDB/entrypoint.sh | 8 ++++++++ TEKDB/requirements.txt | 4 +--- docker/docker-compose.yml | 1 + 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/TEKDB/Dockerfile b/TEKDB/Dockerfile index 896b7676..1ba7ac1f 100644 --- a/TEKDB/Dockerfile +++ b/TEKDB/Dockerfile @@ -22,6 +22,7 @@ WORKDIR /usr/src/app COPY requirements.txt requirements_linux.txt /usr/src/app/ # Upgrade pip and install Python dependencies +# Note: editable packages (-e) will be installed at runtime via entrypoint.sh RUN pip install --upgrade pip \ && pip install -r requirements.txt -r requirements_linux.txt diff --git a/TEKDB/entrypoint.sh b/TEKDB/entrypoint.sh index fac43bbe..fc9e8b00 100644 --- a/TEKDB/entrypoint.sh +++ b/TEKDB/entrypoint.sh @@ -3,6 +3,14 @@ # Exit on errors set -e +# Install editable package if the directory is mounted +if [ -d "/django-resumable-async-upload" ]; then + echo "Installing django-resumable-async-upload in editable mode for local development..." + pip install -e /django-resumable-async-upload +else + echo "django-resumable-async-upload directory not found, skipping editable install" +fi + # If a SQL_HOST is provided, wait for Postgres to become available before running # migrations. This prevents race conditions when using docker-compose where the # web container starts before the DB is ready. diff --git a/TEKDB/requirements.txt b/TEKDB/requirements.txt index 590b2934..b7afcd26 100644 --- a/TEKDB/requirements.txt +++ b/TEKDB/requirements.txt @@ -14,8 +14,6 @@ pillow psycopg2-binary psutil django-filebrowser-no-grappelli>=4.0.0,<5.0.0 -django-async-upload - XlsxWriter #-e git+https://github.com/dominno/django-moderation.git@master#egg=moderation @@ -36,4 +34,4 @@ ipython # Serve static files in production containers whitenoise>=6.0.0,<7.0.0 # FORMATTING AND LINTING -ruff +ruff \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index f865e83a..0db8d15d 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -40,6 +40,7 @@ services: - "8000:8000" volumes: - ../TEKDB:/usr/src/app + - ../../django-resumable-async-upload:/django-resumable-async-upload volumes: tekdb_db_data: From 9546b7625f500475bdaf22482642c2825c51931f Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 13 Jan 2026 14:22:08 -0800 Subject: [PATCH 03/16] use AsyncFileCleanupMixin in MediaAdmin form --- TEKDB/TEKDB/admin.py | 4 +++- TEKDB/TEKDB/settings.py | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/TEKDB/TEKDB/admin.py b/TEKDB/TEKDB/admin.py index 04c5bbb5..c6eeb413 100644 --- a/TEKDB/TEKDB/admin.py +++ b/TEKDB/TEKDB/admin.py @@ -60,6 +60,8 @@ from TEKDB.settings import ADMIN_SITE_HEADER from TEKDB.settings import BASE_DIR from TEKDB.widgets import OpenLayers6Widget +from admin_async_upload.admin import AsyncFileCleanupMixin + admin.site.site_header = ADMIN_SITE_HEADER @@ -830,7 +832,7 @@ def has_add_permission(self, request): @admin.register(Media) -class MediaAdmin(RecordAdminProxy, RecordModelAdmin): +class MediaAdmin(AsyncFileCleanupMixin, RecordAdminProxy, RecordModelAdmin): readonly_fields = ( "medialink", "enteredbyname", diff --git a/TEKDB/TEKDB/settings.py b/TEKDB/TEKDB/settings.py index 7be9ba58..a003c68d 100644 --- a/TEKDB/TEKDB/settings.py +++ b/TEKDB/TEKDB/settings.py @@ -83,6 +83,7 @@ "django.contrib.auth.middleware.AuthenticationMiddleware", "django.contrib.messages.middleware.MessageMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", + "admin_async_upload.middleware.OrphanedFileCleanupMiddleware", ] ROOT_URLCONF = "TEKDB.urls" @@ -347,6 +348,8 @@ DEFAULT_MAXIMUM_RESULTS = 500 +ADMIN_RESUMABLE_SHOW_THUMB = True + try: from TEKDB.local_settings import * # noqa: F403 except Exception: From 630ae9694fb1792b106b2de18c0529b580d2f7eb Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 20 Jan 2026 11:58:25 -0800 Subject: [PATCH 04/16] support async uploads for MediaBulkUploads --- TEKDB/TEKDB/admin.py | 117 ++++++++++++++++++++++++------------------ TEKDB/TEKDB/forms.py | 12 ++++- TEKDB/TEKDB/models.py | 1 + 3 files changed, 78 insertions(+), 52 deletions(-) diff --git a/TEKDB/TEKDB/admin.py b/TEKDB/TEKDB/admin.py index c6eeb413..885cfee4 100644 --- a/TEKDB/TEKDB/admin.py +++ b/TEKDB/TEKDB/admin.py @@ -615,7 +615,7 @@ class CitationsAdmin(RecordAdminProxy, RecordModelAdmin): # * Bulk Media Upload Admin @admin.register(MediaBulkUpload) -class MediaBulkUploadAdmin(admin.ModelAdmin): +class MediaBulkUploadAdmin(AsyncFileCleanupMixin, admin.ModelAdmin): form = MediaBulkUploadForm list_display = ("mediabulkname", "mediabulkdate", "enteredbyname", "enteredbydate") @@ -628,57 +628,74 @@ def save_model(self, request, obj, form, change): activities = form.cleaned_data.get("activities") placesresources = form.cleaned_data.get("placesresources") - for file in request.FILES.getlist("files"): - mime_type, _ = guess_type(file.name) - # if mime_type: - file_mime_type = mime_type.split("/")[0] - media_type_instance = LookupMediaType.objects.filter( - mediatype__startswith=file_mime_type - ).first() - if media_type_instance: - mediatype = media_type_instance - else: - media_type_instance = LookupMediaType.objects.filter( - mediatype__startswith="other" - ).first() + # Handle async uploaded file (comes as a comma separated string of file paths) + uploaded_file_paths = form.cleaned_data.get("files") + if uploaded_file_paths: + from django.core.files.storage import default_storage + import os + + # split the comma-separated string into a list + # TODO: need to add a step in the frontend to clean up any commas in filenames + uploaded_file_paths_list = uploaded_file_paths.split(",") + + for uploaded_file_path in uploaded_file_paths_list: + # Extract just the filename from the path + file_name = os.path.basename(uploaded_file_path) + + # Guess MIME type from filename + mime_type, _ = guess_type(file_name) + if mime_type: + file_mime_type = mime_type.split("/")[0] + media_type_instance = LookupMediaType.objects.filter( + mediatype__startswith=file_mime_type + ).first() + else: + media_type_instance = None + + if not media_type_instance: + media_type_instance = LookupMediaType.objects.filter( + mediatype__startswith="other" + ).first() + mediatype = media_type_instance - filename = file.name.split(".")[0] + filename = file_name.rsplit(".", 1)[0] # Remove extension - media_instance = Media( - medianame=filename, - mediadescription=f'Part of the "{obj.mediabulkname}" Media Bulk Upload that was uploaded on {obj.mediabulkdate}', - mediafile=file, - mediatype=mediatype, - ) - media_instance.save() - obj.mediabulkupload.add(media_instance) - - # Add relationships - if places: - for place in places: - PlacesMediaEvents.objects.create( - placeid=place, mediaid=media_instance - ) - if resources: - for resource in resources: - ResourcesMediaEvents.objects.create( - resourceid=resource, mediaid=media_instance - ) - if citations: - for citation in citations: - MediaCitationEvents.objects.create( - citationid=citation, mediaid=media_instance - ) - if activities: - for activity in activities: - ResourceActivityMediaEvents.objects.create( - resourceactivityid=activity, mediaid=media_instance - ) - if placesresources: - for placeresource in placesresources: - PlacesResourceMediaEvents.objects.create( - placeresourceid=placeresource, mediaid=media_instance - ) + media_instance = Media( + medianame=filename, + mediadescription=f'Part of the "{obj.mediabulkname}" Media Bulk Upload that was uploaded on {obj.mediabulkdate}', + mediafile=uploaded_file_path, + mediatype=mediatype, + ) + print(f"[DEBUG]: Creating Media instance: {filename}") + media_instance.save() + obj.mediabulkupload.add(media_instance) + + # Add relationships + if places: + for place in places: + PlacesMediaEvents.objects.create( + placeid=place, mediaid=media_instance + ) + if resources: + for resource in resources: + ResourcesMediaEvents.objects.create( + resourceid=resource, mediaid=media_instance + ) + if citations: + for citation in citations: + MediaCitationEvents.objects.create( + citationid=citation, mediaid=media_instance + ) + if activities: + for activity in activities: + ResourceActivityMediaEvents.objects.create( + resourceactivityid=activity, mediaid=media_instance + ) + if placesresources: + for placeresource in placesresources: + PlacesResourceMediaEvents.objects.create( + placeresourceid=placeresource, mediaid=media_instance + ) @admin.display(description="Thumbnails") def thumbnail_gallery(self, obj): diff --git a/TEKDB/TEKDB/forms.py b/TEKDB/TEKDB/forms.py index 9808785c..78d52cb1 100644 --- a/TEKDB/TEKDB/forms.py +++ b/TEKDB/TEKDB/forms.py @@ -1,5 +1,7 @@ from django import forms from django.contrib.admin.widgets import FilteredSelectMultiple +from admin_async_upload.fields import FormResumableFileField +from admin_async_upload.widgets import ResumableAdminWidget from .models import ( MediaBulkUpload, Places, @@ -7,6 +9,7 @@ Citations, ResourcesActivityEvents, PlacesResourceEvents, + Media, ) from .widgets import ThumbnailFileInput @@ -26,7 +29,13 @@ def clean(self, data, initial=None): class MediaBulkUploadForm(forms.ModelForm): - files = MultipleFileField() + # files = MultipleFileField() + files = FormResumableFileField( + required=False, + # not passing max_files here because FormResumableFileField defaults to undefined (unlimited), + # which is what we want for bulk upload. + widget=ResumableAdminWidget(attrs={"model": Media, "field_name": "mediafile"}), + ) places = forms.ModelMultipleChoiceField( queryset=Places.objects.all(), required=False, @@ -59,7 +68,6 @@ class Meta: "mediabulkname", "mediabulkdescription", "mediabulkdate", - "files", "places", "resources", "citations", diff --git a/TEKDB/TEKDB/models.py b/TEKDB/TEKDB/models.py index 4a31117f..1eada9a2 100644 --- a/TEKDB/TEKDB/models.py +++ b/TEKDB/TEKDB/models.py @@ -2761,6 +2761,7 @@ class Media(Reviewable, Queryable, Record, ModeratedModel): blank=True, null=True, verbose_name="file", + max_files=1, ) limitedaccess = models.BooleanField( db_column="limitedaccess", From 2484eae951ae200a66b95177d976209c8c50dda8 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Wed, 21 Jan 2026 15:00:12 -0800 Subject: [PATCH 05/16] fix tests for media bulk upload admin --- TEKDB/TEKDB/admin.py | 2 +- TEKDB/TEKDB/tests/test_admin.py | 61 ++++++--------------------------- 2 files changed, 11 insertions(+), 52 deletions(-) diff --git a/TEKDB/TEKDB/admin.py b/TEKDB/TEKDB/admin.py index 885cfee4..95d84a1b 100644 --- a/TEKDB/TEKDB/admin.py +++ b/TEKDB/TEKDB/admin.py @@ -630,8 +630,8 @@ def save_model(self, request, obj, form, change): # Handle async uploaded file (comes as a comma separated string of file paths) uploaded_file_paths = form.cleaned_data.get("files") + if uploaded_file_paths: - from django.core.files.storage import default_storage import os # split the comma-separated string into a list diff --git a/TEKDB/TEKDB/tests/test_admin.py b/TEKDB/TEKDB/tests/test_admin.py index 8a713b86..bbf28f63 100644 --- a/TEKDB/TEKDB/tests/test_admin.py +++ b/TEKDB/TEKDB/tests/test_admin.py @@ -1,7 +1,6 @@ # from django.conf import settings from django.test import RequestFactory from unittest.mock import patch -from django.core.files.uploadedfile import SimpleUploadedFile from django.contrib.auth import get_user_model from django.contrib.admin.sites import AdminSite from django.urls import reverse @@ -165,9 +164,8 @@ def test_media_bulk_upload_admin_add(self): from TEKDB.admin import MediaBulkUploadAdmin url = reverse("admin:TEKDB_mediabulkupload_add") - test_image = SimpleUploadedFile( - "./test_image.jpg", b"\x00\x00\x00\x00", content_type="image" - ) + + test_image_paths = "test_image.jpg,test_image.jpg" place = Places.objects.create(indigenousplacename="Test Place") resource = Resources.objects.create(commonname="Test Resource") @@ -180,7 +178,7 @@ def test_media_bulk_upload_admin_add(self): activity = ResourcesActivityEvents.objects.create(placeresourceid=placeresource) post_data = { - "files": [test_image, test_image], + "files": test_image_paths, "places": [place.pk], "resources": [resource.pk], "citations": [citation.pk], @@ -208,14 +206,6 @@ def test_media_bulk_upload_admin_add(self): PlacesResourceEvents.objects.filter(pk=placeresource.pk).exists() ) - for media in Media.objects.filter(medianame="test_image"): - self.assertTrue(os.path.exists(media.mediafile.path)) - os.remove( - media.mediafile.path - ) # Clean up the uploaded files after the test - self.assertFalse(os.path.exists(media.mediafile.path)) - media.delete() - # Clean up related objects activity.delete() placeresource.delete() @@ -227,14 +217,11 @@ def test_media_bulk_upload_admin_other_types(self): from TEKDB.admin import MediaBulkUploadAdmin url = reverse("admin:TEKDB_mediabulkupload_add") - test_other_type = SimpleUploadedFile( - "./test_thing.shp", b"\x00\x00\x00\x00", content_type="other" - ) request = self.factory.post( url, { - "files": [test_other_type, test_other_type], + "files": "test_thing.shp,test_thing.shp", }, ) @@ -245,48 +232,20 @@ def test_media_bulk_upload_admin_other_types(self): bulk_admin.save_model( obj=MediaBulkUpload(), request=request, form=bulk_form, change=None ) - - for media in Media.objects.filter(medianame="test_thing"): - self.assertTrue(os.path.exists(media.mediafile.path)) - os.remove( - media.mediafile.path - ) # Clean up the uploaded files after the test - self.assertFalse(os.path.exists(media.mediafile.path)) - media.delete() + self.assertTrue(Media.objects.filter(medianame="test_thing").exists()) + self.assertEqual(Media.objects.filter(medianame="test_thing").count(), 2) def test_media_bulk_upload_admin_thumbnail_gallery(self): from TEKDB.admin import MediaBulkUploadAdmin from TEKDB.models import MediaBulkUpload, Media url = reverse("admin:TEKDB_mediabulkupload_add") - test_image = SimpleUploadedFile( - "./thumbnail_test_image.jpg", b"\x00\x00\x00\x00", content_type="image" - ) - test_video = SimpleUploadedFile( - "./thumbnail_test_video.mp4", b"\x00\x00\x00\x00", content_type="video" - ) - test_audio = SimpleUploadedFile( - "./thumbnail_test_audio.mp3", b"\x00\x00\x00\x00", content_type="audio" - ) - test_text = SimpleUploadedFile( - "./thumbnail_test_text.txt", b"\x00\x00\x00\x00", content_type="text" - ) - test_other = SimpleUploadedFile( - "./thumbnail_test_thing.shp", b"\x00\x00\x00\x00", content_type="other" - ) - test_unknown_type = SimpleUploadedFile( - "./thumbnail_test_unknown.xyz", b"\x00\x00\x00\x00", content_type="unknown" - ) + test_file_paths = [ + "thumbnail_test_image.jpg, thumbnail_test_video.mp4, thumbnail_test_audio.mp3, thumbnail_test_text.txt, thumbnail_test_thing.shp, thumbnail_test_unknown.xyz" + ] post_data = { - "files": [ - test_image, - test_video, - test_audio, - test_text, - test_other, - test_unknown_type, - ], + "files": test_file_paths, } request = self.factory.post(url, post_data) From 56004dc08799c39b6a466021ce578f20686298f6 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Fri, 23 Jan 2026 14:09:38 -0800 Subject: [PATCH 06/16] set ADMIN_SIMULTANEOUS_UPLOADS to 1 --- TEKDB/TEKDB/settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/TEKDB/TEKDB/settings.py b/TEKDB/TEKDB/settings.py index a003c68d..2b759a9f 100644 --- a/TEKDB/TEKDB/settings.py +++ b/TEKDB/TEKDB/settings.py @@ -348,7 +348,9 @@ DEFAULT_MAXIMUM_RESULTS = 500 +# Django Resumable Async Upload settings ADMIN_RESUMABLE_SHOW_THUMB = True +ADMIN_SIMULTANEOUS_UPLOADS = 1 try: from TEKDB.local_settings import * # noqa: F403 From d251595c594dd0333b93cab992e139c8336572ff Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Fri, 23 Jan 2026 16:42:17 -0800 Subject: [PATCH 07/16] use ADMIN_RESUMABLE_CHUNK_FOLDER setting --- TEKDB/TEKDB/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/TEKDB/TEKDB/settings.py b/TEKDB/TEKDB/settings.py index 2b759a9f..9fb2626a 100644 --- a/TEKDB/TEKDB/settings.py +++ b/TEKDB/TEKDB/settings.py @@ -351,6 +351,7 @@ # Django Resumable Async Upload settings ADMIN_RESUMABLE_SHOW_THUMB = True ADMIN_SIMULTANEOUS_UPLOADS = 1 +ADMIN_RESUMABLE_CHUNK_FOLDER = "resumable_chunks" try: from TEKDB.local_settings import * # noqa: F403 From 22a25afb409cd207b845d25b1943de9a6cc3f838 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Mon, 26 Jan 2026 13:45:27 -0800 Subject: [PATCH 08/16] clean up comments --- TEKDB/TEKDB/admin.py | 1 - TEKDB/TEKDB/forms.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/TEKDB/TEKDB/admin.py b/TEKDB/TEKDB/admin.py index 95d84a1b..c3307512 100644 --- a/TEKDB/TEKDB/admin.py +++ b/TEKDB/TEKDB/admin.py @@ -635,7 +635,6 @@ def save_model(self, request, obj, form, change): import os # split the comma-separated string into a list - # TODO: need to add a step in the frontend to clean up any commas in filenames uploaded_file_paths_list = uploaded_file_paths.split(",") for uploaded_file_path in uploaded_file_paths_list: diff --git a/TEKDB/TEKDB/forms.py b/TEKDB/TEKDB/forms.py index 78d52cb1..3f236fb9 100644 --- a/TEKDB/TEKDB/forms.py +++ b/TEKDB/TEKDB/forms.py @@ -29,7 +29,6 @@ def clean(self, data, initial=None): class MediaBulkUploadForm(forms.ModelForm): - # files = MultipleFileField() files = FormResumableFileField( required=False, # not passing max_files here because FormResumableFileField defaults to undefined (unlimited), @@ -68,6 +67,7 @@ class Meta: "mediabulkname", "mediabulkdescription", "mediabulkdate", + "files", "places", "resources", "citations", From a0eed17a96c6989883abc7ce7b4c06c054d13f34 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 27 Jan 2026 09:32:18 -0800 Subject: [PATCH 09/16] remove debug log --- TEKDB/TEKDB/admin.py | 1 - 1 file changed, 1 deletion(-) diff --git a/TEKDB/TEKDB/admin.py b/TEKDB/TEKDB/admin.py index c3307512..d6785deb 100644 --- a/TEKDB/TEKDB/admin.py +++ b/TEKDB/TEKDB/admin.py @@ -665,7 +665,6 @@ def save_model(self, request, obj, form, change): mediafile=uploaded_file_path, mediatype=mediatype, ) - print(f"[DEBUG]: Creating Media instance: {filename}") media_instance.save() obj.mediabulkupload.add(media_instance) From 2671ad760f37c0da15707e28bbc0619732826336 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 27 Jan 2026 15:22:36 -0800 Subject: [PATCH 10/16] remove OrphanedFileCleanupMiddleware and AsyncFileCleanupMixin --- TEKDB/TEKDB/admin.py | 5 ++--- TEKDB/TEKDB/settings.py | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/TEKDB/TEKDB/admin.py b/TEKDB/TEKDB/admin.py index d6785deb..ee17bf84 100644 --- a/TEKDB/TEKDB/admin.py +++ b/TEKDB/TEKDB/admin.py @@ -60,7 +60,6 @@ from TEKDB.settings import ADMIN_SITE_HEADER from TEKDB.settings import BASE_DIR from TEKDB.widgets import OpenLayers6Widget -from admin_async_upload.admin import AsyncFileCleanupMixin admin.site.site_header = ADMIN_SITE_HEADER @@ -615,7 +614,7 @@ class CitationsAdmin(RecordAdminProxy, RecordModelAdmin): # * Bulk Media Upload Admin @admin.register(MediaBulkUpload) -class MediaBulkUploadAdmin(AsyncFileCleanupMixin, admin.ModelAdmin): +class MediaBulkUploadAdmin(admin.ModelAdmin): form = MediaBulkUploadForm list_display = ("mediabulkname", "mediabulkdate", "enteredbyname", "enteredbydate") @@ -847,7 +846,7 @@ def has_add_permission(self, request): @admin.register(Media) -class MediaAdmin(AsyncFileCleanupMixin, RecordAdminProxy, RecordModelAdmin): +class MediaAdmin(RecordAdminProxy, RecordModelAdmin): readonly_fields = ( "medialink", "enteredbyname", diff --git a/TEKDB/TEKDB/settings.py b/TEKDB/TEKDB/settings.py index 9fb2626a..e12a2e25 100644 --- a/TEKDB/TEKDB/settings.py +++ b/TEKDB/TEKDB/settings.py @@ -83,7 +83,6 @@ "django.contrib.auth.middleware.AuthenticationMiddleware", "django.contrib.messages.middleware.MessageMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", - "admin_async_upload.middleware.OrphanedFileCleanupMiddleware", ] ROOT_URLCONF = "TEKDB.urls" From 736938caf2a54568b1905b9001af8de6a8375721 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Wed, 28 Jan 2026 11:47:15 -0800 Subject: [PATCH 11/16] refactor MediaBulkUploadForm to use FormResumableMultipleFileField --- TEKDB/TEKDB/admin.py | 9 +++------ TEKDB/TEKDB/forms.py | 4 ++-- TEKDB/TEKDB/tests/test_admin.py | 16 +++++++++++----- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/TEKDB/TEKDB/admin.py b/TEKDB/TEKDB/admin.py index ee17bf84..cc551409 100644 --- a/TEKDB/TEKDB/admin.py +++ b/TEKDB/TEKDB/admin.py @@ -627,16 +627,13 @@ def save_model(self, request, obj, form, change): activities = form.cleaned_data.get("activities") placesresources = form.cleaned_data.get("placesresources") - # Handle async uploaded file (comes as a comma separated string of file paths) + # Handle async uploaded file (comes as a list of file path strings) uploaded_file_paths = form.cleaned_data.get("files") - if uploaded_file_paths: + if uploaded_file_paths and isinstance(uploaded_file_paths, list): import os - # split the comma-separated string into a list - uploaded_file_paths_list = uploaded_file_paths.split(",") - - for uploaded_file_path in uploaded_file_paths_list: + for uploaded_file_path in uploaded_file_paths: # Extract just the filename from the path file_name = os.path.basename(uploaded_file_path) diff --git a/TEKDB/TEKDB/forms.py b/TEKDB/TEKDB/forms.py index 3f236fb9..d9319797 100644 --- a/TEKDB/TEKDB/forms.py +++ b/TEKDB/TEKDB/forms.py @@ -1,6 +1,6 @@ from django import forms from django.contrib.admin.widgets import FilteredSelectMultiple -from admin_async_upload.fields import FormResumableFileField +from admin_async_upload.fields import FormResumableMultipleFileField from admin_async_upload.widgets import ResumableAdminWidget from .models import ( MediaBulkUpload, @@ -29,7 +29,7 @@ def clean(self, data, initial=None): class MediaBulkUploadForm(forms.ModelForm): - files = FormResumableFileField( + files = FormResumableMultipleFileField( required=False, # not passing max_files here because FormResumableFileField defaults to undefined (unlimited), # which is what we want for bulk upload. diff --git a/TEKDB/TEKDB/tests/test_admin.py b/TEKDB/TEKDB/tests/test_admin.py index bbf28f63..b433fdb4 100644 --- a/TEKDB/TEKDB/tests/test_admin.py +++ b/TEKDB/TEKDB/tests/test_admin.py @@ -1,4 +1,5 @@ # from django.conf import settings +import json from django.test import RequestFactory from unittest.mock import patch from django.contrib.auth import get_user_model @@ -165,7 +166,7 @@ def test_media_bulk_upload_admin_add(self): url = reverse("admin:TEKDB_mediabulkupload_add") - test_image_paths = "test_image.jpg,test_image.jpg" + test_image_paths = ["test_image.jpg", "test_image.jpg"] place = Places.objects.create(indigenousplacename="Test Place") resource = Resources.objects.create(commonname="Test Resource") @@ -178,7 +179,7 @@ def test_media_bulk_upload_admin_add(self): activity = ResourcesActivityEvents.objects.create(placeresourceid=placeresource) post_data = { - "files": test_image_paths, + "files": json.dumps(test_image_paths), "places": [place.pk], "resources": [resource.pk], "citations": [citation.pk], @@ -221,7 +222,7 @@ def test_media_bulk_upload_admin_other_types(self): request = self.factory.post( url, { - "files": "test_thing.shp,test_thing.shp", + "files": json.dumps(["test_thing.shp", "test_thing.shp"]), }, ) @@ -241,11 +242,16 @@ def test_media_bulk_upload_admin_thumbnail_gallery(self): url = reverse("admin:TEKDB_mediabulkupload_add") test_file_paths = [ - "thumbnail_test_image.jpg, thumbnail_test_video.mp4, thumbnail_test_audio.mp3, thumbnail_test_text.txt, thumbnail_test_thing.shp, thumbnail_test_unknown.xyz" + "thumbnail_test_image.jpg", + "thumbnail_test_video.mp4", + "thumbnail_test_audio.mp3", + "thumbnail_test_text.txt", + "thumbnail_test_thing.shp", + "thumbnail_test_unknown.xyz", ] post_data = { - "files": test_file_paths, + "files": json.dumps(test_file_paths), } request = self.factory.post(url, post_data) From 0788466f7d38a654cf387fdb833641a595703cb7 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Wed, 28 Jan 2026 11:49:18 -0800 Subject: [PATCH 12/16] remove unused ThumbnailFileInput and MultipleFileField --- TEKDB/TEKDB/forms.py | 15 --------------- TEKDB/TEKDB/widgets.py | 22 ---------------------- 2 files changed, 37 deletions(-) diff --git a/TEKDB/TEKDB/forms.py b/TEKDB/TEKDB/forms.py index d9319797..be04eeb0 100644 --- a/TEKDB/TEKDB/forms.py +++ b/TEKDB/TEKDB/forms.py @@ -11,21 +11,6 @@ PlacesResourceEvents, Media, ) -from .widgets import ThumbnailFileInput - - -class MultipleFileField(forms.FileField): - def __init__(self, *args, **kwargs): - kwargs.setdefault("widget", ThumbnailFileInput) - super().__init__(*args, **kwargs) - - def clean(self, data, initial=None): - single_file_clean = super().clean - if isinstance(data, (list, tuple)): - result = [single_file_clean(d, initial) for d in data] - else: - result = [single_file_clean(data, initial)] - return result class MediaBulkUploadForm(forms.ModelForm): diff --git a/TEKDB/TEKDB/widgets.py b/TEKDB/TEKDB/widgets.py index 200fb1f0..4187a729 100644 --- a/TEKDB/TEKDB/widgets.py +++ b/TEKDB/TEKDB/widgets.py @@ -2,8 +2,6 @@ from django.contrib.gis.geometry import json_regex from django.contrib.gis.forms.widgets import BaseGeometryWidget -from django.forms.widgets import ClearableFileInput -from django.utils.safestring import mark_safe logger = logging.getLogger("django.contrib.gis") @@ -61,23 +59,3 @@ class Media: "assets/openlayers6/ol.js", "gis/js/OL6MapPolygonWidget.js", ) - - -class ThumbnailFileInput(ClearableFileInput): - template_name = "widgets/thumbnail_file_input.html" - allow_multiple_selected = True # Enable multiple file uploads - - def format_value(self, value): - if value and hasattr(value, "url"): - return mark_safe(f'') - return super().format_value(value) - - def __init__(self, attrs=None): - if attrs is None: - attrs = {} - attrs.update( - { - "multiple": "multiple" # Allow multiple file selection - } - ) - super().__init__(attrs) From 95da7bf87db6612639c2722ed32c962767a23712 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Fri, 30 Jan 2026 10:38:36 -0800 Subject: [PATCH 13/16] use django_resumable_async_upload package --- TEKDB/TEKDB/forms.py | 4 ++-- TEKDB/TEKDB/models.py | 2 +- TEKDB/TEKDB/settings.py | 2 +- TEKDB/TEKDB/urls.py | 5 ++++- TEKDB/entrypoint.sh | 8 -------- TEKDB/requirements.txt | 1 + docker/docker-compose.yml | 1 - 7 files changed, 9 insertions(+), 14 deletions(-) diff --git a/TEKDB/TEKDB/forms.py b/TEKDB/TEKDB/forms.py index be04eeb0..1be8e12a 100644 --- a/TEKDB/TEKDB/forms.py +++ b/TEKDB/TEKDB/forms.py @@ -1,7 +1,7 @@ from django import forms from django.contrib.admin.widgets import FilteredSelectMultiple -from admin_async_upload.fields import FormResumableMultipleFileField -from admin_async_upload.widgets import ResumableAdminWidget +from django_resumable_async_upload.fields import FormResumableMultipleFileField +from django_resumable_async_upload.widgets import ResumableAdminWidget from .models import ( MediaBulkUpload, Places, diff --git a/TEKDB/TEKDB/models.py b/TEKDB/TEKDB/models.py index 1eada9a2..f91e20fb 100644 --- a/TEKDB/TEKDB/models.py +++ b/TEKDB/TEKDB/models.py @@ -24,7 +24,7 @@ from django.conf import settings from django.contrib.gis.db.models import GeometryField from tinymce.models import HTMLField -from admin_async_upload.models import AsyncFileField +from django_resumable_async_upload.models import AsyncFileField # from moderation.db import ModeratedModel diff --git a/TEKDB/TEKDB/settings.py b/TEKDB/TEKDB/settings.py index e12a2e25..888e00a6 100644 --- a/TEKDB/TEKDB/settings.py +++ b/TEKDB/TEKDB/settings.py @@ -70,7 +70,7 @@ "Relationships", "reversion", "django.contrib.sites", - "admin_async_upload", + "django_resumable_async_upload", # 'moderation.apps.SimpleModerationConfig', ] diff --git a/TEKDB/TEKDB/urls.py b/TEKDB/TEKDB/urls.py index 536e2d27..eb694a67 100644 --- a/TEKDB/TEKDB/urls.py +++ b/TEKDB/TEKDB/urls.py @@ -77,7 +77,10 @@ views.ResourceActivityAutocompleteView.as_view(), name="select2_fk_resourceactivity", ), - re_path(r"^admin_async_upload/", include("admin_async_upload.urls")), + re_path( + r"^django_resumable_async_upload/", + include("django_resumable_async_upload.urls"), + ), path("", include("explore.urls")), ] urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) diff --git a/TEKDB/entrypoint.sh b/TEKDB/entrypoint.sh index fc9e8b00..fac43bbe 100644 --- a/TEKDB/entrypoint.sh +++ b/TEKDB/entrypoint.sh @@ -3,14 +3,6 @@ # Exit on errors set -e -# Install editable package if the directory is mounted -if [ -d "/django-resumable-async-upload" ]; then - echo "Installing django-resumable-async-upload in editable mode for local development..." - pip install -e /django-resumable-async-upload -else - echo "django-resumable-async-upload directory not found, skipping editable install" -fi - # If a SQL_HOST is provided, wait for Postgres to become available before running # migrations. This prevents race conditions when using docker-compose where the # web container starts before the DB is ready. diff --git a/TEKDB/requirements.txt b/TEKDB/requirements.txt index b7afcd26..9a0cf228 100644 --- a/TEKDB/requirements.txt +++ b/TEKDB/requirements.txt @@ -15,6 +15,7 @@ psycopg2-binary psutil django-filebrowser-no-grappelli>=4.0.0,<5.0.0 XlsxWriter +django-resumable-async-upload #-e git+https://github.com/dominno/django-moderation.git@master#egg=moderation #DEPENDENCIES diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 0db8d15d..f865e83a 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -40,7 +40,6 @@ services: - "8000:8000" volumes: - ../TEKDB:/usr/src/app - - ../../django-resumable-async-upload:/django-resumable-async-upload volumes: tekdb_db_data: From 295dd49cfe6c54a60a29533c22e553512ebda6fc Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Fri, 30 Jan 2026 10:56:22 -0800 Subject: [PATCH 14/16] use python 3.11 in gh action run-tests --- .github/workflows/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 83c10b7a..2b0aa017 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -32,7 +32,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 with: - python-version: '3.10' + python-version: '3.11' - name: Install system dependencies run: | From be3da99a297a4a415be9585ce1f4747534c8ba58 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Fri, 30 Jan 2026 11:14:39 -0800 Subject: [PATCH 15/16] add migration for async file fields --- .../0026_alter_media_mediafile_and_more.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 TEKDB/TEKDB/migrations/0026_alter_media_mediafile_and_more.py diff --git a/TEKDB/TEKDB/migrations/0026_alter_media_mediafile_and_more.py b/TEKDB/TEKDB/migrations/0026_alter_media_mediafile_and_more.py new file mode 100644 index 00000000..76ca5d54 --- /dev/null +++ b/TEKDB/TEKDB/migrations/0026_alter_media_mediafile_and_more.py @@ -0,0 +1,24 @@ +# Generated by Django 4.2.27 on 2026-01-30 19:11 + +from django.db import migrations, models +import django_resumable_async_upload.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('TEKDB', '0025_alter_mediabulkupload_mediabulkname'), + ] + + operations = [ + migrations.AlterField( + model_name='media', + name='mediafile', + field=django_resumable_async_upload.models.AsyncFileField(blank=True, db_column='mediafile', max_files=1, max_length=255, null=True, upload_to='', verbose_name='file'), + ), + migrations.AlterField( + model_name='mediabulkupload', + name='mediabulkname', + field=models.CharField(blank=True, default='Bulk Upload on 2026-01-30', max_length=255, null=True, verbose_name='name'), + ), + ] From 7404d034b8471fb68cb492dd79b6102a3637d9b6 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Fri, 30 Jan 2026 11:50:42 -0800 Subject: [PATCH 16/16] revert tests to run on python 3.10 --- .github/workflows/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 2b0aa017..83c10b7a 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -32,7 +32,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 with: - python-version: '3.11' + python-version: '3.10' - name: Install system dependencies run: |