diff --git a/backend/core/process/builders/inputs_builder.py b/backend/core/process/builders/inputs_builder.py index dfb14f7..8a1d9a2 100644 --- a/backend/core/process/builders/inputs_builder.py +++ b/backend/core/process/builders/inputs_builder.py @@ -12,7 +12,7 @@ class InputsBuilder: "ra": "RA", "dec": "Dec", "z": "z", - "id": "id", + "id": "ID", "z_flag": "z_flag", "z_err": "z_err", "survey": "survey", @@ -76,7 +76,7 @@ def _build_columns(self, product): return columns def _get_column(self, product, alias): - matches = product.contents.filter(alias=alias) + matches = product.contents.filter(alias__iexact=alias) if matches.count() != 1: LOGGER.warning( diff --git a/backend/core/test/test_inputs_builder.py b/backend/core/test/test_inputs_builder.py new file mode 100644 index 0000000..e166577 --- /dev/null +++ b/backend/core/test/test_inputs_builder.py @@ -0,0 +1,137 @@ +from core.models import Product, ProductContent, ProductType +from core.process.builders.inputs_builder import InputsBuilder +from django.contrib.auth.models import User +from django.test import TestCase + + +class InputsBuilderTestCase(TestCase): + fixtures = [ + "core/fixtures/initial_data.yaml", + ] + + def setUp(self): + self.user = User.objects.create_user( + "john", "john@snow.com", "you_know_nothing" + ) + self.product_type = ProductType.objects.get(name="redshift_catalog") + self.product = Product.objects.create( + product_type=self.product_type, + user=self.user, + display_name="Sample Redshift Catalog", + official_product=False, + path="redshift_catalog/sample", + internal_name="sample_redshift_catalog", + ) + + def test_build_columns_uses_id_alias_from_ucd_mapping(self): + ProductContent.objects.bulk_create([ + ProductContent( + product=self.product, + column_name="object_id", + ucd="meta.id;meta.main", + alias="ID", + order=0, + ), + ProductContent( + product=self.product, + column_name="RAdeg", + ucd="pos.eq.ra;meta.main", + alias="RA", + order=1, + ), + ProductContent( + product=self.product, + column_name="DEdeg", + ucd="pos.eq.dec;meta.main", + alias="Dec", + order=2, + ), + ProductContent( + product=self.product, + column_name="zfinal", + ucd="src.redshift", + alias="z", + order=3, + ), + ]) + + columns = InputsBuilder(process=None)._build_columns(self.product) + + self.assertEqual(columns["id"], "object_id") + self.assertEqual(columns["ra"], "RAdeg") + self.assertEqual(columns["dec"], "DEdeg") + self.assertEqual(columns["z"], "zfinal") + + def test_build_columns_accepts_id_alias_case_variations(self): + ProductContent.objects.bulk_create([ + ProductContent( + product=self.product, + column_name="object_id", + ucd="meta.id;meta.main", + alias="Id", + order=0, + ), + ProductContent( + product=self.product, + column_name="RAdeg", + ucd="pos.eq.ra;meta.main", + alias="RA", + order=1, + ), + ProductContent( + product=self.product, + column_name="DEdeg", + ucd="pos.eq.dec;meta.main", + alias="Dec", + order=2, + ), + ProductContent( + product=self.product, + column_name="zfinal", + ucd="src.redshift", + alias="z", + order=3, + ), + ]) + + columns = InputsBuilder(process=None)._build_columns(self.product) + + self.assertEqual(columns["id"], "object_id") + + def test_build_columns_accepts_required_alias_case_variations(self): + ProductContent.objects.bulk_create([ + ProductContent( + product=self.product, + column_name="object_id", + ucd="meta.id;meta.main", + alias="ID", + order=0, + ), + ProductContent( + product=self.product, + column_name="RAdeg", + ucd="pos.eq.ra;meta.main", + alias="ra", + order=1, + ), + ProductContent( + product=self.product, + column_name="DEdeg", + ucd="pos.eq.dec;meta.main", + alias="DEC", + order=2, + ), + ProductContent( + product=self.product, + column_name="zfinal", + ucd="src.redshift", + alias="Z", + order=3, + ), + ]) + + columns = InputsBuilder(process=None)._build_columns(self.product) + + self.assertEqual(columns["ra"], "RAdeg") + self.assertEqual(columns["dec"], "DEdeg") + self.assertEqual(columns["z"], "zfinal") diff --git a/backend/core/test/test_product_download.py b/backend/core/test/test_product_download.py index c2e0448..72d8c4b 100644 --- a/backend/core/test/test_product_download.py +++ b/backend/core/test/test_product_download.py @@ -6,9 +6,11 @@ import yaml from core.models import ( + FileRoles, Product, ProductDownloadArchive, ProductDownloadArchiveStatus, + ProductFile, ProductType, Release, ) @@ -486,6 +488,31 @@ def create_product_source_file(self, media_root): (product_path / "data.csv").write_text("a,b\n1,2\n", encoding="utf-8") return product_path + def create_hats_main_file(self, media_root): + product_path = Path(media_root) / self.product.path + hats_path = product_path / "hats_catalog" + data_path = hats_path / "dataset" / "Norder=0" / "Dir=0" + data_path.mkdir(parents=True) + + (hats_path / "collection.properties").write_text( + "catalog_name=hats_catalog\n", + encoding="utf-8", + ) + (data_path / "Npix=0.parquet").write_bytes(b"parquet") + + ProductFile.objects.create( + product=self.product, + role=FileRoles.MAIN, + name="hats_catalog", + type="application/x-hats", + extension="", + size=0, + file=f"{self.product.path}/hats_catalog", + is_directory=True, + ) + + return hats_path + def test_download_prepare_creates_archive_and_queues_task(self): with tempfile.TemporaryDirectory() as media_root: self.create_product_source_file(media_root) @@ -854,3 +881,38 @@ def test_legacy_download_endpoint_includes_deprecation_headers(self): f"/api/products/{self.product.pk}/download/prepare/", response["Link"], ) + + def test_download_main_file_returns_zip_for_hats_directory(self): + with tempfile.TemporaryDirectory() as media_root: + self.create_hats_main_file(media_root) + + with override_settings(MEDIA_ROOT=media_root): + response = self.client.get( + f"/api/products/{self.product.pk}/download_main_file/" + ) + + content = b"".join(response.streaming_content) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response["Content-Type"], "application/zip") + self.assertIn( + 'filename="hats_catalog.zip"', + response["Content-Disposition"], + ) + + with tempfile.TemporaryDirectory() as tmpdirname: + zip_path = Path(tmpdirname) / "hats_catalog.zip" + zip_path.write_bytes(content) + + with zipfile.ZipFile(zip_path) as archive: + self.assertEqual( + sorted(archive.namelist()), + [ + "collection.properties", + "dataset/Norder=0/Dir=0/Npix=0.parquet", + ], + ) + self.assertEqual( + archive.read("collection.properties").decode("utf-8"), + "catalog_name=hats_catalog\n", + ) diff --git a/backend/core/views/product.py b/backend/core/views/product.py index 78a9c4c..9de5db2 100644 --- a/backend/core/views/product.py +++ b/backend/core/views/product.py @@ -2,6 +2,7 @@ import mimetypes import pathlib import tempfile +import zipfile from json import dumps, loads from pathlib import Path @@ -223,6 +224,32 @@ def __get_full_product(self, product): return product_full + def __build_directory_zip_response(self, directory_path): + zip_handle = tempfile.TemporaryFile() + + with zipfile.ZipFile( + zip_handle, + "w", + compression=zipfile.ZIP_DEFLATED, + compresslevel=ProductDownloadArchiveService.get_compression_level(), + ) as zip_file: + for file_path in sorted(directory_path.rglob("*")): + if not file_path.is_file(): + continue + + arcname = file_path.relative_to(directory_path).as_posix() + zip_file.write(file_path, arcname=arcname) + + size = zip_handle.tell() + zip_handle.seek(0) + + response = FileResponse(zip_handle, content_type="application/zip") + response["Content-Length"] = size + response["Content-Disposition"] = ( + f'attachment; filename="{directory_path.name}.zip"' + ) + return response + def __get_flag_translation_name(self, config): """Extract the flags translation filename from a nested config.""" if isinstance(config, dict): @@ -474,15 +501,7 @@ def download_main_file(self, request, **kwargs): ) if product_path.is_dir(): - return Response( - { - "error": ( - "Main file is a directory-based dataset. " - "Please use the full product download endpoint." - ) - }, - status=status.HTTP_400_BAD_REQUEST, - ) + return self.__build_directory_zip_response(product_path) # Abre o arquivo e envia em bites para o navegador mimetype, _ = mimetypes.guess_type(product_path) @@ -588,6 +607,7 @@ def main_file_info(self, request, **kwargs): main_file["extension"] = product_file.extension main_file["size"] = product_file.size main_file["n_rows"] = product_file.n_rows + main_file["is_directory"] = product_file.is_directory product_contents = self.__get_product_contents(product) if product_contents: diff --git a/orchestration b/orchestration index 48ea57b..145cb70 160000 --- a/orchestration +++ b/orchestration @@ -1 +1 @@ -Subproject commit 48ea57b3f79649be3239bdde7038c3fbe7b320c5 +Subproject commit 145cb70b66af8e1bdcc3e770114aca6d608dce8d