Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions backend/core/process/builders/inputs_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ class InputsBuilder:
"ra": "RA",
"dec": "Dec",
"z": "z",
"id": "id",
"id": "ID",
"z_flag": "z_flag",
"z_err": "z_err",
"survey": "survey",
Expand Down Expand Up @@ -76,7 +76,7 @@ def _build_columns(self, product):
return columns

def _get_column(self, product, alias):
matches = product.contents.filter(alias=alias)
matches = product.contents.filter(alias__iexact=alias)

if matches.count() != 1:
LOGGER.warning(
Expand Down
137 changes: 137 additions & 0 deletions backend/core/test/test_inputs_builder.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
from core.models import Product, ProductContent, ProductType
from core.process.builders.inputs_builder import InputsBuilder
from django.contrib.auth.models import User
from django.test import TestCase


class InputsBuilderTestCase(TestCase):
fixtures = [
"core/fixtures/initial_data.yaml",
]

def setUp(self):
self.user = User.objects.create_user(
"john", "john@snow.com", "you_know_nothing"
)
self.product_type = ProductType.objects.get(name="redshift_catalog")
self.product = Product.objects.create(
product_type=self.product_type,
user=self.user,
display_name="Sample Redshift Catalog",
official_product=False,
path="redshift_catalog/sample",
internal_name="sample_redshift_catalog",
)

def test_build_columns_uses_id_alias_from_ucd_mapping(self):
ProductContent.objects.bulk_create([
ProductContent(
product=self.product,
column_name="object_id",
ucd="meta.id;meta.main",
alias="ID",
order=0,
),
ProductContent(
product=self.product,
column_name="RAdeg",
ucd="pos.eq.ra;meta.main",
alias="RA",
order=1,
),
ProductContent(
product=self.product,
column_name="DEdeg",
ucd="pos.eq.dec;meta.main",
alias="Dec",
order=2,
),
ProductContent(
product=self.product,
column_name="zfinal",
ucd="src.redshift",
alias="z",
order=3,
),
])

columns = InputsBuilder(process=None)._build_columns(self.product)

self.assertEqual(columns["id"], "object_id")
self.assertEqual(columns["ra"], "RAdeg")
self.assertEqual(columns["dec"], "DEdeg")
self.assertEqual(columns["z"], "zfinal")

def test_build_columns_accepts_id_alias_case_variations(self):
ProductContent.objects.bulk_create([
ProductContent(
product=self.product,
column_name="object_id",
ucd="meta.id;meta.main",
alias="Id",
order=0,
),
ProductContent(
product=self.product,
column_name="RAdeg",
ucd="pos.eq.ra;meta.main",
alias="RA",
order=1,
),
ProductContent(
product=self.product,
column_name="DEdeg",
ucd="pos.eq.dec;meta.main",
alias="Dec",
order=2,
),
ProductContent(
product=self.product,
column_name="zfinal",
ucd="src.redshift",
alias="z",
order=3,
),
])

columns = InputsBuilder(process=None)._build_columns(self.product)

self.assertEqual(columns["id"], "object_id")

def test_build_columns_accepts_required_alias_case_variations(self):
ProductContent.objects.bulk_create([
ProductContent(
product=self.product,
column_name="object_id",
ucd="meta.id;meta.main",
alias="ID",
order=0,
),
ProductContent(
product=self.product,
column_name="RAdeg",
ucd="pos.eq.ra;meta.main",
alias="ra",
order=1,
),
ProductContent(
product=self.product,
column_name="DEdeg",
ucd="pos.eq.dec;meta.main",
alias="DEC",
order=2,
),
ProductContent(
product=self.product,
column_name="zfinal",
ucd="src.redshift",
alias="Z",
order=3,
),
])

columns = InputsBuilder(process=None)._build_columns(self.product)

self.assertEqual(columns["ra"], "RAdeg")
self.assertEqual(columns["dec"], "DEdeg")
self.assertEqual(columns["z"], "zfinal")
62 changes: 62 additions & 0 deletions backend/core/test/test_product_download.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,11 @@

import yaml
from core.models import (
FileRoles,
Product,
ProductDownloadArchive,
ProductDownloadArchiveStatus,
ProductFile,
ProductType,
Release,
)
Expand Down Expand Up @@ -486,6 +488,31 @@ def create_product_source_file(self, media_root):
(product_path / "data.csv").write_text("a,b\n1,2\n", encoding="utf-8")
return product_path

def create_hats_main_file(self, media_root):
product_path = Path(media_root) / self.product.path
hats_path = product_path / "hats_catalog"
data_path = hats_path / "dataset" / "Norder=0" / "Dir=0"
data_path.mkdir(parents=True)

(hats_path / "collection.properties").write_text(
"catalog_name=hats_catalog\n",
encoding="utf-8",
)
(data_path / "Npix=0.parquet").write_bytes(b"parquet")

ProductFile.objects.create(
product=self.product,
role=FileRoles.MAIN,
name="hats_catalog",
type="application/x-hats",
extension="",
size=0,
file=f"{self.product.path}/hats_catalog",
is_directory=True,
)

return hats_path

def test_download_prepare_creates_archive_and_queues_task(self):
with tempfile.TemporaryDirectory() as media_root:
self.create_product_source_file(media_root)
Expand Down Expand Up @@ -854,3 +881,38 @@ def test_legacy_download_endpoint_includes_deprecation_headers(self):
f"/api/products/{self.product.pk}/download/prepare/",
response["Link"],
)

def test_download_main_file_returns_zip_for_hats_directory(self):
with tempfile.TemporaryDirectory() as media_root:
self.create_hats_main_file(media_root)

with override_settings(MEDIA_ROOT=media_root):
response = self.client.get(
f"/api/products/{self.product.pk}/download_main_file/"
)

content = b"".join(response.streaming_content)

self.assertEqual(response.status_code, 200)
self.assertEqual(response["Content-Type"], "application/zip")
self.assertIn(
'filename="hats_catalog.zip"',
response["Content-Disposition"],
)

with tempfile.TemporaryDirectory() as tmpdirname:
zip_path = Path(tmpdirname) / "hats_catalog.zip"
zip_path.write_bytes(content)

with zipfile.ZipFile(zip_path) as archive:
self.assertEqual(
sorted(archive.namelist()),
[
"collection.properties",
"dataset/Norder=0/Dir=0/Npix=0.parquet",
],
)
self.assertEqual(
archive.read("collection.properties").decode("utf-8"),
"catalog_name=hats_catalog\n",
)
38 changes: 29 additions & 9 deletions backend/core/views/product.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import mimetypes
import pathlib
import tempfile
import zipfile
from json import dumps, loads
from pathlib import Path

Expand Down Expand Up @@ -223,6 +224,32 @@ def __get_full_product(self, product):

return product_full

def __build_directory_zip_response(self, directory_path):
zip_handle = tempfile.TemporaryFile()

with zipfile.ZipFile(
zip_handle,
"w",
compression=zipfile.ZIP_DEFLATED,
compresslevel=ProductDownloadArchiveService.get_compression_level(),
) as zip_file:
for file_path in sorted(directory_path.rglob("*")):
if not file_path.is_file():
continue

arcname = file_path.relative_to(directory_path).as_posix()
zip_file.write(file_path, arcname=arcname)

size = zip_handle.tell()
zip_handle.seek(0)

response = FileResponse(zip_handle, content_type="application/zip")
response["Content-Length"] = size
response["Content-Disposition"] = (
f'attachment; filename="{directory_path.name}.zip"'
)
return response

def __get_flag_translation_name(self, config):
"""Extract the flags translation filename from a nested config."""
if isinstance(config, dict):
Expand Down Expand Up @@ -474,15 +501,7 @@ def download_main_file(self, request, **kwargs):
)

if product_path.is_dir():
return Response(
{
"error": (
"Main file is a directory-based dataset. "
"Please use the full product download endpoint."
)
},
status=status.HTTP_400_BAD_REQUEST,
)
return self.__build_directory_zip_response(product_path)

# Abre o arquivo e envia em bites para o navegador
mimetype, _ = mimetypes.guess_type(product_path)
Expand Down Expand Up @@ -588,6 +607,7 @@ def main_file_info(self, request, **kwargs):
main_file["extension"] = product_file.extension
main_file["size"] = product_file.size
main_file["n_rows"] = product_file.n_rows
main_file["is_directory"] = product_file.is_directory

product_contents = self.__get_product_contents(product)
if product_contents:
Expand Down
2 changes: 1 addition & 1 deletion orchestration
Loading