diff --git a/physionet-django/console/file_utils.py b/physionet-django/console/file_utils.py new file mode 100644 index 0000000000..03f7ac7b07 --- /dev/null +++ b/physionet-django/console/file_utils.py @@ -0,0 +1,258 @@ +import os +import subprocess +import logging +from django.core.exceptions import ValidationError + +LOGGER = logging.getLogger(__name__) + + +class FileUnpacker: + """ + Utility class for unpacking compressed files in project directories + """ + + SUPPORTED_EXTENSIONS = { + '.tar.gz': 'tar', + '.tgz': 'tar', + '.tar.bz2': 'tar', + '.tar.xz': 'tar', + '.zip': 'zip', + '.gz': 'gzip', + '.bz2': 'bzip2', + '.xz': 'xz' + } + + @classmethod + def get_archive_type(cls, file_path): + """ + Determine the archive type based on file extension + """ + for ext in cls.SUPPORTED_EXTENSIONS: + if file_path.endswith(ext): + return cls.SUPPORTED_EXTENSIONS[ext] + return None + + @classmethod + def unpack_file(cls, project_root, file_path, target_directory=None, overwrite_existing=False): + """ + Unpack a compressed file in the project directory + + Args: + project_root: Root directory of the project + file_path: Relative path to the compressed file within the project + target_directory: Optional target directory for extraction + overwrite_existing: Whether to overwrite existing files + + Returns: + dict: Result information including success status and extracted files + """ + full_file_path = os.path.join(project_root, file_path) + + if not os.path.exists(full_file_path): + raise ValidationError(f"File not found: {file_path}") + + if not os.path.isfile(full_file_path): + raise ValidationError(f"Path is not a file: {file_path}") + + archive_type = cls.get_archive_type(file_path) + if not archive_type: + raise ValidationError(f"Unsupported file type: {file_path}") + + # Determine target directory + if target_directory: + extract_dir = os.path.join(project_root, target_directory) + if not os.path.exists(extract_dir): + os.makedirs(extract_dir, exist_ok=True) + else: + # Extract to the same directory as the archive + extract_dir = os.path.dirname(full_file_path) + + try: + if archive_type == 'tar': + return cls._extract_tar(full_file_path, extract_dir, overwrite_existing) + elif archive_type == 'zip': + return cls._extract_zip(full_file_path, extract_dir, overwrite_existing) + elif archive_type in ['gzip', 'bzip2', 'xz']: + return cls._extract_single_file(full_file_path, extract_dir, overwrite_existing) + else: + raise ValidationError(f"Unsupported archive type: {archive_type}") + except Exception as e: + LOGGER.error(f"Error unpacking {file_path}: {str(e)}") + raise ValidationError(f"Failed to unpack file: {str(e)}") + + @classmethod + def _extract_tar(cls, file_path, extract_dir, overwrite_existing): + """ + Extract a tar archive + """ + flags = ['-xf'] + if overwrite_existing: + flags.append('--overwrite') + + cmd = ['tar'] + flags + [file_path, '-C', extract_dir] + + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + cwd=extract_dir, + timeout=300 # 5 minute timeout + ) + + if result.returncode != 0: + raise subprocess.CalledProcessError( + result.returncode, cmd, result.stdout, result.stderr + ) + + # Get list of extracted files + extracted_files = cls._list_extracted_files(extract_dir, file_path) + + return { + 'success': True, + 'extracted_files': extracted_files, + 'extract_directory': extract_dir + } + + except subprocess.TimeoutExpired: + raise ValidationError("Extraction timed out. The archive may be very large.") + except subprocess.CalledProcessError as e: + raise ValidationError(f"Tar extraction failed: {e.stderr}") + + @classmethod + def _extract_zip(cls, file_path, extract_dir, overwrite_existing): + """ + Extract a zip archive + """ + # Quiet mode + flags = ['-q'] + if overwrite_existing: + # Overwrite without prompting + flags.append('-o') + + cmd = ['unzip'] + flags + [file_path, '-d', extract_dir] + + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + cwd=extract_dir, + timeout=300 # 5 minute timeout + ) + + if result.returncode != 0: + raise subprocess.CalledProcessError( + result.returncode, cmd, result.stdout, result.stderr + ) + + extracted_files = cls._list_extracted_files(extract_dir, file_path) + + return { + 'success': True, + 'extracted_files': extracted_files, + 'extract_directory': extract_dir + } + + except subprocess.TimeoutExpired: + raise ValidationError("Extraction timed out. The archive may be very large.") + except subprocess.CalledProcessError as e: + raise ValidationError(f"Zip extraction failed: {e.stderr}") + + @classmethod + def _extract_single_file(cls, file_path, extract_dir, overwrite_existing): + """ + Extract a single compressed file (gzip, bzip2, xz) + """ + filename = os.path.basename(file_path) + base_name = filename + + # Remove compression extensions + for ext in ['.gz', '.bz2', '.xz']: + if base_name.endswith(ext): + base_name = base_name[:-len(ext)] + break + + output_path = os.path.join(extract_dir, base_name) + + if os.path.exists(output_path) and not overwrite_existing: + raise ValidationError(f"File already exists: {base_name}") + + # Determine decompression command + if file_path.endswith('.gz'): + cmd = ['gunzip', '-c', file_path] + elif file_path.endswith('.bz2'): + cmd = ['bunzip2', '-c', file_path] + elif file_path.endswith('.xz'): + cmd = ['unxz', '-c', file_path] + else: + raise ValidationError(f"Unsupported compression type: {file_path}") + + try: + with open(output_path, 'wb') as output_file: + result = subprocess.run( + cmd, + stdout=output_file, + stderr=subprocess.PIPE, + text=True, + timeout=300 # 5 minute timeout + ) + + if result.returncode != 0: + os.remove(output_path) # Clean up failed extraction + raise subprocess.CalledProcessError( + result.returncode, cmd, stderr=result.stderr + ) + + return { + 'success': True, + 'extracted_files': [base_name], + 'extract_directory': extract_dir + } + + except subprocess.TimeoutExpired: + if os.path.exists(output_path): + os.remove(output_path) + raise ValidationError("Extraction timed out. The file may be very large.") + except subprocess.CalledProcessError as e: + if os.path.exists(output_path): + os.remove(output_path) + raise ValidationError(f"Extraction failed: {e.stderr}") + + @classmethod + def _list_extracted_files(cls, extract_dir, original_file): + """ + List files that were extracted (excluding the original archive) + """ + extracted_files = [] + original_filename = os.path.basename(original_file) + + for root, dirs, files in os.walk(extract_dir): + for file in files: + if file != original_filename: + rel_path = os.path.relpath(os.path.join(root, file), extract_dir) + extracted_files.append(rel_path) + + return sorted(extracted_files) + + @classmethod + def validate_file_path(cls, project_root, file_path): + """ + Validate that a file path is safe and exists + """ + # Prevent directory traversal + if '..' in file_path or file_path.startswith('/'): + raise ValidationError("Invalid file path") + + full_path = os.path.join(project_root, file_path) + + # Ensure the path is within the project directory + try: + full_path = os.path.realpath(full_path) + project_root = os.path.realpath(project_root) + if not full_path.startswith(project_root): + raise ValidationError("File path is outside project directory") + except (OSError, ValueError): + raise ValidationError("Invalid file path") + + return full_path diff --git a/physionet-django/console/forms.py b/physionet-django/console/forms.py index d39bfc276d..43e9df21d3 100644 --- a/physionet-django/console/forms.py +++ b/physionet-django/console/forms.py @@ -1037,3 +1037,37 @@ class Meta: model = CodeOfConduct fields = ('name', 'version', 'slug', 'html_content') labels = {'html_content': 'Content'} + + +class FileUnpackForm(forms.Form): + """ + Form for unpacking compressed files in a project + """ + file_path = forms.CharField( + max_length=500, + label='File Path', + widget=forms.TextInput(attrs={ + 'class': 'form-control', + 'placeholder': 'Enter the path to the compressed file (e.g., data/archive.tar.gz)' + }), + help_text='Enter the relative path to the compressed file within the project directory' + ) + + target_directory = forms.CharField( + max_length=500, + required=False, + label='Target Directory', + widget=forms.TextInput(attrs={ + 'class': 'form-control', + 'placeholder': 'Leave empty to extract to the same directory as the archive' + }), + help_text='Optional: Specify a target directory for extraction (relative to project root)' + ) + + overwrite_existing = forms.BooleanField( + required=False, + initial=False, + label='Overwrite Existing Files', + widget=forms.CheckboxInput(attrs={'class': 'form-check-input'}), + help_text='Check this to overwrite existing files during extraction' + ) diff --git a/physionet-django/console/templates/console/submission_info_card.html b/physionet-django/console/templates/console/submission_info_card.html index 62d1bdacfd..ff22c6c36a 100644 --- a/physionet-django/console/templates/console/submission_info_card.html +++ b/physionet-django/console/templates/console/submission_info_card.html @@ -22,6 +22,9 @@
+ This tool allows you to unpack compressed files (tar.gz, zip, gz, bz2, xz) within the project directory. + This is useful for extracting archives that authors have uploaded. +
+ +The following DOIs have been registered for the project. These may be useful diff --git a/physionet-django/console/views.py b/physionet-django/console/views.py index 4c732f19cc..7f1b1d606e 100644 --- a/physionet-django/console/views.py +++ b/physionet-django/console/views.py @@ -77,6 +77,7 @@ from physionet.enums import LogCategory from console import forms, utility, services from console.forms import ProjectFilterForm, UserFilterForm +from console.file_utils import FileUnpacker from project.cloud.s3 import ( create_s3_bucket, upload_project_to_S3, @@ -227,7 +228,8 @@ def submitted_projects(request): 'copyedit_projects': copyedit_projects, 'approval_projects': approval_projects, 'publish_projects': publish_projects, - 'yesterday': yesterday}) + 'yesterday': yesterday + }) @console_permission_required('project.change_activeproject') @@ -269,7 +271,8 @@ def editor_home(request): 'copyedit_projects': copyedit_projects, 'approval_projects': approval_projects, 'publish_projects': publish_projects, - 'yesterday': yesterday, 'editor_home': True}) + 'yesterday': yesterday, 'editor_home': True + }) def submission_info_redirect(request, project_slug): @@ -282,7 +285,8 @@ def submission_info_card_params(request, embargo_form, internal_note_form, bulk_download, - force_calculate): + force_calculate, + file_unpack_form=None): """ Parameters used across submission_info_card.html pages, including: @@ -311,6 +315,7 @@ def submission_info_card_params(request, 'embargo_form': embargo_form, 'notes': notes, 'internal_note_form': internal_note_form, + 'file_unpack_form': file_unpack_form, } @@ -328,6 +333,7 @@ def submission_info(request, project_slug): reassign_editor_form = forms.ReassignEditorForm(project=project, data=data) internal_note_form = forms.InternalNoteForm(data) embargo_form = forms.EmbargoFilesDaysForm() + file_unpack_form = forms.FileUnpackForm(data) passphrase = '' anonymous_url = project.get_anonymous_url() @@ -376,22 +382,57 @@ def submission_info(request, project_slug): else: messages.error(request, "You are not authorized to delete this note.") return redirect(f'{request.path}?tab=notes') + elif 'unpack_file' in request.POST and user == project.editor: + if file_unpack_form.is_valid(): + try: + project_root = project.file_root() + + LOGGER.info(f"Project root: {project_root}") + LOGGER.info(f"File path: {file_unpack_form.cleaned_data['file_path']}") + full_path = os.path.join(project_root, file_unpack_form.cleaned_data['file_path']) + LOGGER.info(f"Full file path: {full_path}") + + result = FileUnpacker.unpack_file( + project_root=project_root, + file_path=file_unpack_form.cleaned_data['file_path'], + target_directory=file_unpack_form.cleaned_data['target_directory'] or None, + overwrite_existing=file_unpack_form.cleaned_data['overwrite_existing'] + ) + + if result['success']: + messages.success( + request, + f"Successfully unpacked {len(result['extracted_files'])} " + f"files to {result['extract_directory']}" + ) + # Reset form after successful unpacking + file_unpack_form = forms.FileUnpackForm() + else: + messages.error(request, "File unpacking failed") + + except Exception as e: + error_msg = f"Error unpacking file: {str(e)}" + if "File not found" in str(e): + error_msg += f" (searched in: {project_root})" + messages.error(request, error_msg) + LOGGER.error(f"File unpacking error for project {project.slug}: {str(e)}") + else: + messages.error(request, 'Invalid file unpacking submission. See errors below.') return render(request, 'console/submission_info.html', {**submission_info_card_params( - request, - project, - reassign_editor_form, - embargo_form, - internal_note_form, - bulk_download=True, - force_calculate=False - ), - 'copyedit_logs': copyedit_logs, - 'passphrase': passphrase, - 'anonymous_url': anonymous_url, - } - ) + request, + project, + reassign_editor_form, + embargo_form, + internal_note_form, + bulk_download=True, + force_calculate=False, + file_unpack_form=file_unpack_form), + 'copyedit_logs': copyedit_logs, + 'passphrase': passphrase, + 'anonymous_url': anonymous_url, + 'file_unpack_form': file_unpack_form}) @handling_editor @@ -440,21 +481,18 @@ def edit_submission(request, project_slug, *args, **kwargs): edit_submission_form = forms.EditSubmissionForm( resource_type=project.resource_type, instance=edit_log) - return render(request, - 'console/edit_submission.html', + return render(request, 'console/edit_submission.html', {**submission_info_card_params( - request, - project, - reassign_editor_form, - embargo_form, - internal_note_form, - bulk_download=True, - force_calculate=False - ), - 'edit_submission_form': edit_submission_form, - 'editor_home': True, - } - ) + request, + project, + reassign_editor_form, + embargo_form, + internal_note_form, + bulk_download=True, + force_calculate=False, + file_unpack_form=None), + 'edit_submission_form': edit_submission_form, + 'editor_home': True}) @handling_editor @@ -587,46 +625,42 @@ def copyedit_submission(request, project_slug, *args, **kwargs): edit_url = reverse('edit_content_item', args=[project.slug]) - response = render( - request, - 'console/copyedit_submission.html', - {**submission_info_card_params( - request, - project, - reassign_editor_form, - embargo_form, - internal_note_form, - bulk_download=False, - force_calculate=True, - ), - 'description_form': description_form, - 'ethics_form': ethics_form, - 'individual_size_limit': readable_size(ActiveProject.INDIVIDUAL_FILE_SIZE_LIMIT), - 'access_form': access_form, - 'reference_formset': reference_formset, - 'publication_formset': publication_formset, - 'topic_formset': topic_formset, - 'storage_type': settings.STORAGE_TYPE, - 'upload_files_form': upload_files_form, - 'create_folder_form': create_folder_form, - 'rename_item_form': rename_item_form, - 'move_items_form': move_items_form, - 'delete_items_form': delete_items_form, - 'subdir': subdir, - 'display_files': display_files, - 'display_dirs': display_dirs, - 'dir_breadcrumbs': dir_breadcrumbs, - 'file_error': file_error, - 'editor_home': True, - 'is_editor': True, - 'files_editable': True, - 'copyedit_form': copyedit_form, - 'copyedit_logs': copyedit_logs, - 'add_item_url': edit_url, - 'remove_item_url': edit_url, - 'discovery_form': discovery_form, - }, - ) + response = render(request, 'console/copyedit_submission.html', + {**submission_info_card_params(request, + project, + reassign_editor_form, + embargo_form, + internal_note_form, + bulk_download=False, + force_calculate=True, + file_unpack_form=None), + 'description_form': description_form, + 'ethics_form': ethics_form, + 'individual_size_limit': readable_size(ActiveProject.INDIVIDUAL_FILE_SIZE_LIMIT), + 'access_form': access_form, + 'reference_formset': reference_formset, + 'publication_formset': publication_formset, + 'topic_formset': topic_formset, + 'storage_type': settings.STORAGE_TYPE, + 'upload_files_form': upload_files_form, + 'create_folder_form': create_folder_form, + 'rename_item_form': rename_item_form, + 'move_items_form': move_items_form, + 'delete_items_form': delete_items_form, + 'subdir': subdir, + 'display_files': display_files, + 'display_dirs': display_dirs, + 'dir_breadcrumbs': dir_breadcrumbs, + 'file_error': file_error, + 'editor_home': True, + 'is_editor': True, + 'files_editable': True, + 'copyedit_form': copyedit_form, + 'copyedit_logs': copyedit_logs, + 'add_item_url': edit_url, + 'remove_item_url': edit_url, + 'discovery_form': discovery_form}) + if description_form_saved: set_saved_fields_cookie(description_form, request.path, response) return response @@ -664,22 +698,21 @@ def awaiting_authors(request, project_slug, *args, **kwargs): yesterday = timezone.now() + timezone.timedelta(days=-1) - return render(request, - 'console/awaiting_authors.html', + return render(request, 'console/awaiting_authors.html', {**submission_info_card_params( - request, - project, - reassign_editor_form, - embargo_form, - internal_note_form, - bulk_download=True, - force_calculate=False, - ), - 'copyedit_logs': copyedit_logs, - 'outstanding_emails': outstanding_emails, - 'yesterday': yesterday, - 'editor_home': True, - 'reassign_editor_form': reassign_editor_form}) + request, + project, + reassign_editor_form, + embargo_form, + internal_note_form, + bulk_download=True, + force_calculate=False, + file_unpack_form=None), + 'copyedit_logs': copyedit_logs, + 'outstanding_emails': outstanding_emails, + 'yesterday': yesterday, + 'editor_home': True, + 'reassign_editor_form': reassign_editor_form}) @handling_editor @@ -781,24 +814,21 @@ def publish_submission(request, project_slug, *args, **kwargs): publishable = project.is_publishable() publish_form = forms.PublishForm(project=project) - return render(request, - 'console/publish_submission.html', + return render(request, 'console/publish_submission.html', {**submission_info_card_params( - request, - project, - reassign_editor_form, - embargo_form, - internal_note_form, - bulk_download=True, - force_calculate=True - ), - 'publishable': publishable, - 'copyedit_logs': copyedit_logs, - 'publish_form': publish_form, - 'max_slug_length': MAX_PROJECT_SLUG_LENGTH, - 'editor_home': True, - } - ) + request, + project, + reassign_editor_form, + embargo_form, + internal_note_form, + bulk_download=True, + force_calculate=True, + file_unpack_form=None), + 'publishable': publishable, + 'copyedit_logs': copyedit_logs, + 'publish_form': publish_form, + 'max_slug_length': MAX_PROJECT_SLUG_LENGTH, + 'editor_home': True}) @console_permission_required('project.change_storagerequest')