From daf400b28deb74d3d6165e4953e9cb3a09b8d056 Mon Sep 17 00:00:00 2001 From: glaubervila Date: Sat, 11 Jul 2026 11:58:52 -0300 Subject: [PATCH 1/4] Add perform_cutout as the single cutout execution path for async jobs Introduce perform_cutout(job_id, task_id), a single function that reads all execution parameters from the Task row in the database, discovers the input tiles, runs the real cutout engine and records the JobResult and status transitions in the correct order. - Add cutout/service/cutout_runner.py with file discovery and engine execution duplicated from the sync path (sync flow left untouched) - Add perform_cutout_task, a thin Celery wrapper used by the async chord; finalize_job remains the callback that completes the Job - Dispatch perform_cutout_task in ImageCutoutPolicy.dispatch_async, replacing the placeholder pipeline - Remove the now unused run_cutout_for_pos and fake_image_cutout - Add unit tests with mocked locator/engine (test_cutout_runner.py), update async API tests accordingly, and add a manual e2e battery (teste_cutout_runner.py) based on the exemplos.txt scenarios Co-Authored-By: Claude Fable 5 --- cutout/service/cutout_runner.py | 200 ++++++++++++++++++++ cutout/service/policy.py | 4 +- cutout/service/tasks.py | 114 +---------- cutout/service/teste_cutout_runner.py | 135 +++++++++++++ cutout/service/tests/test_async_api.py | 35 ++++ cutout/service/tests/test_cutout_runner.py | 210 +++++++++++++++++++++ 6 files changed, 587 insertions(+), 111 deletions(-) create mode 100644 cutout/service/cutout_runner.py create mode 100644 cutout/service/teste_cutout_runner.py create mode 100644 cutout/service/tests/test_cutout_runner.py diff --git a/cutout/service/cutout_runner.py b/cutout/service/cutout_runner.py new file mode 100644 index 0000000..be4551e --- /dev/null +++ b/cutout/service/cutout_runner.py @@ -0,0 +1,200 @@ +"""Single entry point for executing one cutout unit (a Task row) end to end. + +Reads every execution parameter from the database, discovers the input files, +runs the cutout engine and records the result and status transitions. Used as +a direct call by the sync flow and wrapped in a Celery task for the async flow +(``cutout.service.tasks.perform_cutout_task``). +""" + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Any + +from django.utils import timezone + +from cutout.service.cutout_engine import create_cutout_engine +from cutout.service.discovery import DesCsvFileLocator +from cutout.service.models import Job, JobResult, Task +from cutout.service.stencils import Stencil +from cutout.service.uws.exceptions import ParameterError + +logger = logging.getLogger("cutout") + +InputFiles = list[str] | dict[str, list[str]] + + +def _parse_rgb_bands(raw: str) -> list[str]: + """Parse rgb_bands accepting 'gri', 'g,r,i' or 'g r i'.""" + if "," in raw: + return [b.strip() for b in raw.split(",") if b.strip()] + if " " in raw: + return [b.strip() for b in raw.split() if b.strip()] + return list(raw) + + +def _validate_input_files(files: InputFiles | None) -> None: + if not files: + return + + paths: list[str] = [] + if isinstance(files, dict): + for v in files.values(): + paths.extend(v or []) + else: + paths = list(files) + + missing = [f for f in paths if not Path(f).exists()] + if missing: + msg = "Input file unavailable: " + ", ".join(missing) + raise FileNotFoundError(msg) + + +def _find_existing_files(locator: DesCsvFileLocator, task: Task, stencil: Stencil, band: str) -> list[str]: + descriptors = locator.find_files(survey_id=task.survey_id, stencil=stencil, band=band) + if not descriptors: + raise ParameterError(f"No files found for band {band} in the requested region") + + candidates = [str(d.file_path) for d in descriptors if d.file_path] + existing = [p for p in candidates if Path(p).exists()] + if not existing: + raise ParameterError(f"No available files on disk for band {band} in the requested region") + return existing + + +def _discover_input_files(task: Task) -> InputFiles: + """Locate the input tiles for a task, per band when color composition is requested.""" + stencil = Stencil.from_dict(task.stencil) + locator = DesCsvFileLocator() + + if task.color: + bands = _parse_rgb_bands(task.rgb_bands or "gri") + files_map = {band: _find_existing_files(locator, task, stencil, band) for band in bands} + logger.info("[perform_cutout] task_id=%s color bands=%s files=%s", task.id, bands, files_map) + return files_map + + files = _find_existing_files(locator, task, stencil, task.band) + logger.info("[perform_cutout] task_id=%s band=%s files=%s", task.id, task.band, files) + return files + + +def _mime_type_for_format(output_format: str) -> str: + return "image/png" if str(output_format).lower() == "png" else "application/fits" + + +def perform_cutout(job_id: int | str, task_id: int | str) -> dict[str, Any]: + """Execute one cutout Task end to end, reading everything from the database. + + Loads the Job and Task rows, transitions their states, discovers the input + files, runs the cutout engine and registers the JobResult. Marking the Job + as COMPLETED is the caller's responsibility (chord callback in async mode). + """ + job_pk = int(str(job_id).strip()) + task_pk = int(str(task_id).strip()) + + job = Job.objects.get(pk=job_pk) + task = Task.objects.get(pk=task_pk) + + if task.job_id != job.pk: + raise ValueError(f"Task {task_pk} does not belong to job {job_pk}") + + if job.phase == Job.ExecutionPhase.ABORTED: + logger.info("[perform_cutout] job_id=%s is ABORTED, skipping task_id=%s", job_pk, task_pk) + return {} + + logger.info( + "[perform_cutout] START job_id=%s task_id=%s survey_id=%s stencil_type=%s band=%s " + "format=%s engine=%s color=%s rgb_bands=%s", + job_pk, + task_pk, + task.survey_id, + task.stencil_type, + task.band, + task.output_format, + task.engine, + task.color, + task.rgb_bands, + ) + + # First task to run transitions the job to EXECUTING (idempotent under concurrency). + # PENDING is accepted besides QUEUED so the function can run standalone, without dispatch. + Job.objects.filter(pk=job_pk, phase__in=(Job.ExecutionPhase.PENDING, Job.ExecutionPhase.QUEUED)).update( + phase=Job.ExecutionPhase.EXECUTING, + start_time=timezone.now(), + ) + + Task.objects.filter(pk=task_pk, status=Task.Status.PENDING).update( + status=Task.Status.EXECUTING, + start_time=timezone.now(), + ) + + try: + files = _discover_input_files(task) + _validate_input_files(files) + + engine = create_cutout_engine(task.engine) + result_path = Path( + engine.run_cutout( + source_id=task.survey_id, + stencil=task.stencil, + input_files=files, + band=task.band, + output_format=task.output_format, + output_path=task.output_path, + color=task.color, + rgb_bands=task.rgb_bands, + persist=task.persist, + ) + ) + + if not result_path.exists(): + raise FileNotFoundError(f"Engine did not produce result file {result_path}") + + result_id = result_path.stem + size = result_path.stat().st_size + + JobResult.objects.update_or_create( + job=job, + sequence=task.sequence, + defaults={ + "result_id": result_id, + "size": size, + "mime_type": _mime_type_for_format(task.output_format), + "url": f"/api/async/{job_pk}/results/{result_id}", + "file_path": str(result_path), + }, + ) + + Task.objects.filter(pk=task_pk).update( + status=Task.Status.COMPLETED, + end_time=timezone.now(), + ) + + logger.info( + "[perform_cutout] COMPLETED job_id=%s task_id=%s result_id=%s size=%s path=%s", + job_pk, + task_pk, + result_id, + size, + result_path, + ) + return { + "task_id": task_pk, + "result_id": result_id, + "file_path": str(result_path), + "size": size, + } + + except Exception as exc: + logger.exception("[perform_cutout] ERROR job_id=%s task_id=%s: %s", job_pk, task_pk, exc) + Task.objects.filter(pk=task_pk).update( + status=Task.Status.ERROR, + end_time=timezone.now(), + error_message=str(exc), + ) + Job.objects.filter(pk=job_pk).update( + phase=Job.ExecutionPhase.ERROR, + end_time=timezone.now(), + ) + raise diff --git a/cutout/service/policy.py b/cutout/service/policy.py index 8ce8238..b803a75 100644 --- a/cutout/service/policy.py +++ b/cutout/service/policy.py @@ -16,7 +16,7 @@ from cutout.service.discovery import DesCsvFileLocator from cutout.service.models import Task as SQLTask from cutout.service.policies import DesPublicAccessPolicy -from cutout.service.tasks import finalize_job, image_cutout, run_cutout_for_pos +from cutout.service.tasks import finalize_job, image_cutout, perform_cutout_task from cutout.service.uws.exceptions import MultiValuedParameterError, ParameterError, PermissionDeniedError from cutout.service.uws.models import Job, JobParameter from cutout.service.uws.policy import UWSPolicy @@ -226,7 +226,7 @@ def dispatch_async(self, job: Job, message_id: str): logger.info("[dispatch_async] job_id=%s message_id=%s", job.job_id, message_id) db_tasks = list(SQLTask.objects.filter(job_id=int(job.job_id)).order_by("sequence")) - cutout_sigs = [run_cutout_for_pos.s(job_id=job.job_id, task_id=str(task.id)) for task in db_tasks] + cutout_sigs = [perform_cutout_task.s(job_id=job.job_id, task_id=str(task.id)) for task in db_tasks] result = celery_chord(cutout_sigs)(finalize_job.s(job_id=job.job_id).set(task_id=message_id)) logger.info("[dispatch_async] chord dispatched: %d task(s), callback_id=%s", len(cutout_sigs), message_id) return result diff --git a/cutout/service/tasks.py b/cutout/service/tasks.py index 14ec2ef..684dd9b 100644 --- a/cutout/service/tasks.py +++ b/cutout/service/tasks.py @@ -1,4 +1,3 @@ -import json import logging from pathlib import Path from typing import Any @@ -8,6 +7,7 @@ from config import celery_app from cutout.lib.des_cutout import DesCutout from cutout.service.cutout_engine import create_cutout_engine +from cutout.service.cutout_runner import perform_cutout from cutout.service.models import Job, Task @@ -120,46 +120,6 @@ def image_cutout( return str(result) -def fake_image_cutout( - job_id: str, - source_id: str, - stencil: dict[str, Any], - engine: str, - band: str, - format: str, - path: str, - files: list[str] | None = None, - color: bool = False, - rgb_bands: str | None = None, - persist: bool = False, -) -> dict[str, Any]: - result_path = Path(path) - result_path.parent.mkdir(parents=True, exist_ok=True) - payload = { - "job_id": job_id, - "source_id": source_id, - "stencil": stencil, - "engine": engine, - "band": band, - "format": format, - "path": path, - "files": files or [], - "color": color, - "rgb_bands": rgb_bands, - "persist": persist, - "mode": "fake_async_result", - } - result_path.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8") - - mime_type = "image/png" if str(format).lower() == "png" else "application/fits" - return { - "result_id": result_path.stem, - "file_path": str(result_path), - "mime_type": mime_type, - "size": result_path.stat().st_size, - } - - @celery_app.task( bind=True, autoretry_for=(Job.DoesNotExist, Task.DoesNotExist), @@ -167,81 +127,17 @@ def fake_image_cutout( retry_jitter=True, retry_kwargs={"max_retries": 5}, ) -def run_cutout_for_pos(self, job_id: str, task_id: str) -> dict[str, Any]: - """Execute a single cutout unit. Reads all parameters from the Task row in DB.""" +def perform_cutout_task(self, job_id: str, task_id: str) -> dict[str, Any]: + """Celery wrapper for `perform_cutout`. All parameters are read from the Task row in DB.""" logger = logging.getLogger("cutout") - job_pk = int(str(job_id).strip()) - task_pk = int(str(task_id).strip()) - logger.info( - "[run_cutout_for_pos] celery_task_id=%s retries=%s job_id=%r task_id=%r", + "[perform_cutout_task] celery_task_id=%s retries=%s job_id=%r task_id=%r", self.request.id, self.request.retries, job_id, task_id, ) - - job = Job.objects.get(pk=job_pk) - task = Task.objects.get(pk=task_pk) - - if job.phase == Job.ExecutionPhase.ABORTED: - logger.info("[run_cutout_for_pos] Job %r is ABORTED, skipping", job_id) - return {} - - # First task to run transitions QUEUED → EXECUTING (idempotent under concurrency) - Job.objects.filter(pk=job_pk, phase=Job.ExecutionPhase.QUEUED).update( - phase=Job.ExecutionPhase.EXECUTING, - start_time=timezone.now(), - ) - - # PENDING → EXECUTING (idempotent — only first call wins if multi-band) - Task.objects.filter(pk=task_pk, status=Task.Status.PENDING).update( - status=Task.Status.EXECUTING, - start_time=timezone.now(), - ) - - try: - result = fake_image_cutout( - job_id=job_id, - source_id=task.survey_id, - stencil=task.stencil, - engine=task.engine, - band=task.band, - format=task.output_format, - path=task.output_path, - color=task.color, - rgb_bands=task.rgb_bands, - persist=task.persist, - ) - - job.results.create( - result_id=result["result_id"], - sequence=task.sequence, - size=result.get("size") or 0, - mime_type=result.get("mime_type"), - url=f"/api/async/{job_id}/results/{result['result_id']}", - file_path=result.get("file_path"), - ) - - Task.objects.filter(pk=task_pk).update( - status=Task.Status.COMPLETED, - end_time=timezone.now(), - ) - - logger.info("[run_cutout_for_pos] completed task_id=%r result_id=%s", task_id, result.get("result_id")) - return {"task_id": task_pk, "result_id": result["result_id"]} - - except Exception as exc: - Job.objects.filter(pk=job_pk).update( - phase=Job.ExecutionPhase.ERROR, - end_time=timezone.now(), - ) - Task.objects.filter(pk=task_pk).update( - status=Task.Status.ERROR, - end_time=timezone.now(), - error_message=str(exc), - ) - raise + return perform_cutout(job_id, task_id) @celery_app.task diff --git a/cutout/service/teste_cutout_runner.py b/cutout/service/teste_cutout_runner.py new file mode 100644 index 0000000..437dc99 --- /dev/null +++ b/cutout/service/teste_cutout_runner.py @@ -0,0 +1,135 @@ +"""Bateria manual de testes do `perform_cutout` baseada em exemplos.txt. + +Cenários já validados no fluxo sync, usando os FITS reais Y6A1/r4907 em +/data/tiles/des_dr2 (disponíveis no container). + +Uso (dentro do container): + + docker compose exec django python manage.py shell + + >>> from cutout.service.teste_cutout_runner import run_exemplo, run_todos + >>> run_exemplo(1) # cria Job+Task e executa perform_cutout direto + >>> run_todos() # cenários 1 a 8 + >>> job, task = run_exemplo(9, dispatch=False) # só cria; executar via worker: + >>> from cutout.service.tasks import perform_cutout_task + >>> perform_cutout_task.delay(job.id, task.id) +""" + +from __future__ import annotations + +from pathlib import Path + +from cutout.service.cutout_runner import perform_cutout +from cutout.service.uws.models import JobParameter +from cutout.service.uws.service import JobService + +EXEMPLOS = { + 1: { + "params": {"pos": "CIRCLE 0.5 0.017 0.016667", "band": "g", "format": "fits"}, + "esperado": "COMPLETED — FITS ~0.8MB, 1 tile (DES0002+0001, 1')", + }, + 2: { + "params": {"pos": "CIRCLE 1.25 -0.683 0.05", "band": "z", "format": "fits"}, + "esperado": "COMPLETED — FITS ~7.2MB (DES0005-0041, 3')", + }, + 3: { + "params": {"pos": "CIRCLE 0.75 2.867 0.033333", "format": "png", "color": "true", "rgb_bands": "gri"}, + "esperado": "COMPLETED — PNG RGB ~2.0MB (DES0003+0252, 2')", + }, + 4: { + "params": {"pos": "CIRCLE 0.5 0.017 0.116667", "band": "Y", "format": "fits"}, + "esperado": "COMPLETED — FITS ~38.9MB (DES0002+0001, 7')", + }, + 5: { + "params": {"pos": "CIRCLE 1.10 2.50 0.083333", "band": "r", "format": "fits"}, + "esperado": "COMPLETED — mosaico de 2 tiles, header com NINPUTS=2 (5')", + }, + 6: { + "params": {"pos": "CIRCLE 1.07 2.15 0.083333", "band": "r", "format": "fits"}, + "esperado": "COMPLETED — cobertura parcial, borda leste preenchida com zeros (5')", + }, + 7: { + "params": {"pos": "CIRCLE 10.0 10.0 0.016667", "band": "r", "format": "fits"}, + "esperado": "Task ERROR — 'No available files on disk...' (fora do footprint)", + }, + 8: { + "params": {"pos": "CIRCLE 0.5 1.0 0.166667", "band": "r", "format": "fits"}, + "esperado": "Task ERROR — vão entre tiles (Dec entre DES0002+0001 e DES0002+0209)", + }, + 9: { + "params": {"pos": "CIRCLE 0.5 2.15 0.25", "band": "i", "format": "fits"}, + "esperado": "COMPLETED via worker — 15', caso que estoura o timeout do sync", + }, +} + + +def _get_user(username: str | None = None): + from cutout.users.models import User + + if username: + return User.objects.get(username=username) + return User.objects.filter(username="dev").first() or User.objects.first() + + +def create_job_exemplo(n: int, user=None): + """Cria Job + Tasks (sem despachar) para o cenário `n` e retorna (job, task).""" + scenario = EXEMPLOS[n] + params = [JobParameter(parameter_id="id", value="des_dr2", is_post=True)] + params += [JobParameter(parameter_id=key, value=value, is_post=True) for key, value in scenario["params"].items()] + + job = JobService().create(user=user or _get_user(), params=params, run_id=f"teste_exemplo_{n}") + task = job.tasks.order_by("sequence").first() + return job, task + + +def _print_status(job, task) -> None: + job.refresh_from_db() + task.refresh_from_db() + + print(f" Job {job.id}: phase={job.phase} start={job.start_time} end={job.end_time}") + print(f" Task {task.id}: status={task.status} start={task.start_time} end={task.end_time}") + if task.error_message: + print(f" Task error_message: {task.error_message}") + + for result in job.results.order_by("sequence"): + exists = Path(result.file_path).exists() if result.file_path else False + print( + f" Result {result.result_id}: size={result.size} mime={result.mime_type} " + f"path={result.file_path} exists={exists}" + ) + if not job.results.exists(): + print(" Sem JobResult registrado.") + + +def run_exemplo(n: int, user=None, dispatch: bool = True): + """Executa o cenário `n` chamando perform_cutout(job_id, task_id) diretamente. + + Com dispatch=False apenas cria o Job+Task (para despachar manualmente via + perform_cutout_task.delay e observar o worker). + """ + scenario = EXEMPLOS[n] + print(f"=== Exemplo {n}: {scenario['params']}") + print(f" Esperado: {scenario['esperado']}") + + job, task = create_job_exemplo(n, user) + print(f" Criado job_id={job.id} task_id={task.id}") + + if not dispatch: + print(" Despache com: perform_cutout_task.delay(job.id, task.id)") + return job, task + + try: + result = perform_cutout(job.id, task.id) + print(f" perform_cutout retornou: {result}") + except Exception as exc: + print(f" perform_cutout levantou: {type(exc).__name__}: {exc}") + + _print_status(job, task) + return job, task + + +def run_todos(user=None) -> None: + """Roda os cenários 1 a 8 (o 9 é grande — rodar via worker com dispatch=False).""" + for n in range(1, 9): + run_exemplo(n, user=user) + print() diff --git a/cutout/service/tests/test_async_api.py b/cutout/service/tests/test_async_api.py index 15168fd..b15c1a9 100644 --- a/cutout/service/tests/test_async_api.py +++ b/cutout/service/tests/test_async_api.py @@ -4,6 +4,8 @@ from django.urls import reverse from rest_framework.test import APIClient +from cutout.service import cutout_runner +from cutout.service.discovery import FileDescriptor from cutout.service.policy import ImageCutoutPolicy from cutout.service.uws.models import JobParameter from cutout.service.uws.service import JobService @@ -20,8 +22,40 @@ def _fake_path(self, job, task_params, sequence): monkeypatch.setattr(ImageCutoutPolicy, "_build_async_result_path", _fake_path) +class _FakeLocator: + def __init__(self, input_file: Path): + self._input_file = input_file + + def find_files(self, *, survey_id, stencil, band=None): + return [ + FileDescriptor( + tile_id="DES0002+0001", + archive_path="Y6A1/r4907/DES0002+0001/p01/coadd", + file_path=self._input_file, + band=band, + ) + ] + + +class _FakeEngine: + def run_cutout(self, **kwargs): + output_path = Path(kwargs["output_path"]) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_bytes(b"fake fits data") + return output_path + + +def _patch_cutout_execution(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Replace file discovery and the cutout engine so perform_cutout runs without real tiles.""" + input_file = tmp_path / "DES0002+0001_r4907p01_g.fits.fz" + input_file.write_bytes(b"tile data") + monkeypatch.setattr(cutout_runner, "DesCsvFileLocator", lambda: _FakeLocator(input_file)) + monkeypatch.setattr(cutout_runner, "create_cutout_engine", lambda name: _FakeEngine()) + + def test_async_create_runs_job_and_persists_result(user, settings, monkeypatch, tmp_path): _patch_async_result_path(monkeypatch, tmp_path) + _patch_cutout_execution(monkeypatch, tmp_path) settings.CELERY_TASK_ALWAYS_EAGER = True settings.CELERY_TASK_EAGER_PROPAGATES = True client = APIClient() @@ -58,6 +92,7 @@ def test_async_create_runs_job_and_persists_result(user, settings, monkeypatch, def test_async_phase_run_starts_pending_job(user, settings, monkeypatch, tmp_path): _patch_async_result_path(monkeypatch, tmp_path) + _patch_cutout_execution(monkeypatch, tmp_path) settings.CELERY_TASK_ALWAYS_EAGER = True settings.CELERY_TASK_EAGER_PROPAGATES = True client = APIClient() diff --git a/cutout/service/tests/test_cutout_runner.py b/cutout/service/tests/test_cutout_runner.py new file mode 100644 index 0000000..b6513dc --- /dev/null +++ b/cutout/service/tests/test_cutout_runner.py @@ -0,0 +1,210 @@ +from pathlib import Path + +import pytest + +from cutout.service import cutout_runner +from cutout.service.cutout_runner import perform_cutout +from cutout.service.discovery import FileDescriptor +from cutout.service.models import Job, Task +from cutout.service.uws.exceptions import ParameterError + +pytestmark = pytest.mark.django_db + +CIRCLE_STENCIL = {"type": "circle", "center": {"ra": 0.5, "dec": 0.017}, "radius": 0.016667} + + +def _create_job_and_task(user, output_path, **task_overrides): + job = Job.objects.create(owner=user, phase=Job.ExecutionPhase.PENDING) + fields = { + "job": job, + "sequence": 1, + "survey_id": "des_dr2", + "stencil": CIRCLE_STENCIL, + "stencil_type": "circle", + "band": "g", + "output_format": "fits", + "engine": "astrocut", + "color": False, + "rgb_bands": "gri", + "persist": False, + "output_path": str(output_path), + } + fields.update(task_overrides) + task = Task.objects.create(**fields) + return job, task + + +class _FakeLocator: + def __init__(self, input_file: Path | None): + self._input_file = input_file + + def find_files(self, *, survey_id, stencil, band=None): + if self._input_file is None: + return [] + return [ + FileDescriptor( + tile_id="DES0002+0001", + archive_path="Y6A1/r4907/DES0002+0001/p01/coadd", + file_path=self._input_file, + band=band, + ) + ] + + +class _FakeEngine: + def __init__(self, payload: bytes = b"fake fits data"): + self.payload = payload + self.calls: list[dict] = [] + + def run_cutout(self, **kwargs): + self.calls.append(kwargs) + output_path = Path(kwargs["output_path"]) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_bytes(self.payload) + return output_path + + +class _FailingEngine: + def run_cutout(self, **kwargs): + raise RuntimeError("engine exploded") + + +def _patch_runner(monkeypatch, tmp_path, engine=None, input_file="present"): + if input_file == "present": + input_file = tmp_path / "DES0002+0001_r4907p01_g.fits.fz" + input_file.write_bytes(b"tile data") + engine = engine or _FakeEngine() + monkeypatch.setattr(cutout_runner, "DesCsvFileLocator", lambda: _FakeLocator(input_file)) + monkeypatch.setattr(cutout_runner, "create_cutout_engine", lambda name: engine) + return engine + + +def test_perform_cutout_success_records_result_and_statuses(user, monkeypatch, tmp_path): + engine = _patch_runner(monkeypatch, tmp_path) + job, task = _create_job_and_task(user, tmp_path / "out" / "job_1_g.fits") + + result = perform_cutout(job.id, task.id) + + task.refresh_from_db() + assert task.status == Task.Status.COMPLETED + assert task.start_time is not None + assert task.end_time is not None + assert task.end_time >= task.start_time + + job.refresh_from_db() + assert job.phase == Job.ExecutionPhase.EXECUTING + assert job.start_time is not None + assert job.end_time is None + + job_result = job.results.get() + assert job_result.result_id == "job_1_g" + assert job_result.sequence == task.sequence + assert job_result.size == len(engine.payload) + assert job_result.mime_type == "application/fits" + assert job_result.file_path == task.output_path + assert job_result.url == f"/api/async/{job.id}/results/job_1_g" + + assert result == { + "task_id": task.id, + "result_id": "job_1_g", + "file_path": task.output_path, + "size": job_result.size, + } + + engine_kwargs = engine.calls[0] + assert engine_kwargs["source_id"] == "des_dr2" + assert engine_kwargs["stencil"] == CIRCLE_STENCIL + assert engine_kwargs["band"] == "g" + assert engine_kwargs["output_format"] == "fits" + assert isinstance(engine_kwargs["input_files"], list) + + +def test_perform_cutout_color_passes_files_per_band(user, monkeypatch, tmp_path): + engine = _patch_runner(monkeypatch, tmp_path) + job, task = _create_job_and_task( + user, + tmp_path / "out" / "job_1_rgb.png", + color=True, + rgb_bands="gri", + output_format="png", + ) + + perform_cutout(job.id, task.id) + + engine_kwargs = engine.calls[0] + assert isinstance(engine_kwargs["input_files"], dict) + assert sorted(engine_kwargs["input_files"].keys()) == ["g", "i", "r"] + assert job.results.get().mime_type == "image/png" + + +def test_perform_cutout_is_idempotent_for_reruns(user, monkeypatch, tmp_path): + _patch_runner(monkeypatch, tmp_path) + job, task = _create_job_and_task(user, tmp_path / "out" / "job_1_g.fits") + + perform_cutout(job.id, task.id) + perform_cutout(job.id, task.id) + + assert job.results.count() == 1 + task.refresh_from_db() + assert task.status == Task.Status.COMPLETED + + +def test_perform_cutout_engine_error_marks_task_and_job(user, monkeypatch, tmp_path): + _patch_runner(monkeypatch, tmp_path, engine=_FailingEngine()) + job, task = _create_job_and_task(user, tmp_path / "out" / "job_1_g.fits") + + with pytest.raises(RuntimeError, match="engine exploded"): + perform_cutout(job.id, task.id) + + task.refresh_from_db() + assert task.status == Task.Status.ERROR + assert task.error_message == "engine exploded" + assert task.end_time is not None + + job.refresh_from_db() + assert job.phase == Job.ExecutionPhase.ERROR + assert job.end_time is not None + assert job.results.count() == 0 + + +def test_perform_cutout_no_files_marks_error(user, monkeypatch, tmp_path): + _patch_runner(monkeypatch, tmp_path, input_file=None) + job, task = _create_job_and_task(user, tmp_path / "out" / "job_1_g.fits") + + with pytest.raises(ParameterError, match="No files found"): + perform_cutout(job.id, task.id) + + task.refresh_from_db() + assert task.status == Task.Status.ERROR + assert "No files found" in task.error_message + + job.refresh_from_db() + assert job.phase == Job.ExecutionPhase.ERROR + + +def test_perform_cutout_skips_aborted_job(user, monkeypatch, tmp_path): + _patch_runner(monkeypatch, tmp_path) + job, task = _create_job_and_task(user, tmp_path / "out" / "job_1_g.fits") + job.phase = Job.ExecutionPhase.ABORTED + job.save() + + result = perform_cutout(job.id, task.id) + + assert result == {} + task.refresh_from_db() + assert task.status == Task.Status.PENDING + assert job.results.count() == 0 + + +def test_perform_cutout_rejects_task_from_another_job(user, monkeypatch, tmp_path): + _patch_runner(monkeypatch, tmp_path) + job_a, _ = _create_job_and_task(user, tmp_path / "out" / "job_a.fits") + job_b, task_b = _create_job_and_task(user, tmp_path / "out" / "job_b.fits") + + with pytest.raises(ValueError, match="does not belong"): + perform_cutout(job_a.id, task_b.id) + + task_b.refresh_from_db() + assert task_b.status == Task.Status.PENDING + job_a.refresh_from_db() + assert job_a.phase == Job.ExecutionPhase.PENDING From 1ee1c1468d4d452c53f4f2b1642bb21ef35b855c Mon Sep 17 00:00:00 2001 From: glaubervila Date: Sat, 11 Jul 2026 13:22:24 -0300 Subject: [PATCH 2/4] Migrate sync endpoint to perform_cutout and remove the legacy pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sync endpoint now runs the same database flow as async — Job, Task and JobResult rows with ordered status transitions recorded by perform_cutout — executed inline in the request and returning the result file directly. Result files are now split by execution mode (/data/results/sync/ vs /data/results/async/) via the execution_mode parameter on JobService.create / create_tasks_for_job. With no remaining callers, the legacy pipeline was removed: - JobService.start / mark_executing and ImageCutoutPolicy.dispatch - Celery tasks image_cutout and des_cutout_circle - cutout/lib (DesCutout/BaseCutout) and the DesCutoutEngine "legacy" engine option; astrocut is now the only supported engine - legacy debug scripts (scripts/) and orphaned or stale test files - UWSPolicy abstract base now declares create_tasks_for_job and dispatch_async instead of the obsolete dispatch Add sync API tests (success, discovery error recorded in the DB, and multi-task rejection) and update the async/factory tests. Also cleans up planning notes and ad-hoc root test scripts, and gitignores the new sync results directory. Co-Authored-By: Claude Fable 5 --- .gitignore | 2 + AGENTS.md | 6 - ANALISE_CUTOUT_PROTOTIPO.md | 198 ------- Anotacoes.md | 18 +- CURL_TESTES_SYNC.md | 146 ----- PLANO_ASYNC.md | 217 -------- PLANO_SYNC_IVOA_DES_DESCOBERTA.md | 521 ------------------ STATUS_IMPLEMENTACAO_SYNC.md | 355 ------------ config/api_router.py | 2 +- config/urls.py | 4 - cutout/service/api/views.py | 81 +-- cutout/service/cutout_engine/__init__.py | 3 +- cutout/service/cutout_engine/des_engine.py | 27 - cutout/service/cutout_engine/factory.py | 3 - .../cutout_engine/tests/test_des_engine.py | 32 -- .../cutout_engine/tests/test_factory.py | 8 +- cutout/service/policy.py | 134 +---- cutout/service/tasks.py | 119 ---- cutout/service/tests/test_async_api.py | 6 +- cutout/service/tests/test_sync_api.py | 137 +++++ cutout/service/tests/test_tasks.py | 2 +- cutout/service/uws/policy.py | 32 +- cutout/service/uws/service.py | 50 +- data/results/sync/.gitkeep | 0 merge_production_dotenvs_in_dotenv.py | 26 - scripts/debug_band_fits.py | 44 -- scripts/debug_cutout.py | 229 -------- scripts/test_engine_color.py | 43 -- test_async_endpoint.md | 27 + test_async_endpoint.py | 212 ------- test_celery_ping.py | 6 - test_plot_cutout_fits.py | 130 ----- test_worker_dispatch.py | 102 ---- tests/test_e2e_png.py | 50 -- ...test_merge_production_dotenvs_in_dotenv.py | 34 -- 35 files changed, 278 insertions(+), 2728 deletions(-) delete mode 100644 ANALISE_CUTOUT_PROTOTIPO.md delete mode 100644 CURL_TESTES_SYNC.md delete mode 100644 PLANO_ASYNC.md delete mode 100644 PLANO_SYNC_IVOA_DES_DESCOBERTA.md delete mode 100644 STATUS_IMPLEMENTACAO_SYNC.md delete mode 100644 cutout/service/cutout_engine/des_engine.py delete mode 100644 cutout/service/cutout_engine/tests/test_des_engine.py create mode 100644 cutout/service/tests/test_sync_api.py create mode 100644 data/results/sync/.gitkeep delete mode 100644 merge_production_dotenvs_in_dotenv.py delete mode 100644 scripts/debug_band_fits.py delete mode 100644 scripts/debug_cutout.py delete mode 100644 scripts/test_engine_color.py create mode 100644 test_async_endpoint.md delete mode 100644 test_async_endpoint.py delete mode 100644 test_celery_ping.py delete mode 100644 test_plot_cutout_fits.py delete mode 100644 test_worker_dispatch.py delete mode 100644 tests/test_e2e_png.py delete mode 100644 tests/test_merge_production_dotenvs_in_dotenv.py diff --git a/.gitignore b/.gitignore index d0b1bb2..ba173ef 100644 --- a/.gitignore +++ b/.gitignore @@ -293,3 +293,5 @@ data/original_files/*.tar.gz data/tiles/* data/results/debug/* data/results/async/* +data/results/sync/* +!data/results/sync/.gitkeep diff --git a/AGENTS.md b/AGENTS.md index e0ceb60..d3c9fb8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -95,9 +95,3 @@ Para as proximas fases de codigo: 3. Testes executados dentro do container. 4. Lint executado dentro do container. 5. Nenhuma dependencia foi instalada no host. -6. Arquivo `STATUS_IMPLEMENTACAO_SYNC.md` atualizado com: - - resumo da fase - - arquivos alterados - - regras de negocio/decisoes - - validacoes executadas - - commit associado diff --git a/ANALISE_CUTOUT_PROTOTIPO.md b/ANALISE_CUTOUT_PROTOTIPO.md deleted file mode 100644 index 6cc4bea..0000000 --- a/ANALISE_CUTOUT_PROTOTIPO.md +++ /dev/null @@ -1,198 +0,0 @@ -# Analise do prototipo do app Django cutout - -Data da analise: 2026-04-26 - -## Objetivo desta analise -Avaliar o que foi realmente implementado no prototipo, com foco em: -- API e endpoints -- parametros de entrada (estilo VO/SODA/UWS) -- identificacao de arquivos FITS envolvidos no cutout por coordenada -- worker separado, fila e distribuicao de tarefas - -Tambem foi feita uma comparacao com referencias VO encontradas no codigo. - -## Referencias VO encontradas no repositorio - -### UWS (link explicito no codigo) -- cutout/service/uws/models.py - - https://www.ivoa.net/documents/UWS/20161024/REC-UWS-1.1-20161024.html -- cutout/service/models/job.py -- cutout/service/models/job_parameter.py -- cutout/service/models/job_result.py - - todos mencionam implementacao baseada no UWS 1.1 - -### SODA/VO (mencoes no codigo e anotacoes) -- cutout/service/api/views.py - - comentario sobre IVOA/SODA/WD-SODA 3.2.2 para parametro BAND - - comentario com referencia ao projeto lsst-sqre/vo-cutouts -- Anotacoes.md - - item sobre SODA BAND - - item sobre DALI 3.2.1 (-Inf/+Inf em intervalos) - - item sobre formato de erro esperado pelo SODA - -### Verificacao externa (VO) -Foi consultado o conteudo de: -- UWS 1.1 (IVOA) -- SODA 1.0 (IVOA) - -Resumo normativo relevante para este projeto: -- SODA: recursos sync e async, parametros como ID, POS/CIRCLE/POLYGON, BAND, TIME, POL, erros em text/plain com prefixos padrao (UsageError, MultiValuedParamNotSupported, etc). -- UWS: arvore de recursos de job (lista, job, phase, parameters, results, destruction, executionduration, abort/start), fases padronizadas e operacoes REST para controle de job. - -## O que esta implementado de fato - -### 1) Endpoints expostos hoje -Arquivos: -- config/urls.py -- config/api_router.py -- cutout/service/api/views.py - -Implementado: -- GET /api/cutout - - retorna mensagem fixa "Hello, world!" -- GET /api/sync - - aceita query params - - converte params para JobParameter - - cria job e chama inicio do processamento via JobService - - resposta atual: JSON {"success": true} - -Importante: -- Nao ha endpoint async estilo UWS exposto ao cliente (ex.: /jobs, /jobs/{id}, /jobs/{id}/phase, /jobs/{id}/results). -- JobRequestViewSet existe no codigo, mas nao esta registrado no router. -- SyncCutoutView tem muito codigo comentado de uma versao anterior (retorno de FITS/PNG por FileResponse), sem estar ativo na versao atual. - -### 2) Modelo de parametros (parcialmente VO-like) -Arquivos: -- cutout/service/api/views.py -- cutout/service/cutout_parameters.py -- cutout/service/stencils.py - -Implementado: -- Parametros aceitos/documentados no schema do endpoint sync: - - id - - pos (CIRCLE/RANGE/POLYGON dentro da string) - - runid - - format - - band -- Parser de stencil: - - POS com CIRCLE, RANGE, POLYGON - - CIRCLE e POLYGON diretos tambem sao aceitos no parser (pela logica parse_stencil) -- Validacao de negocio no policy: - - apenas 1 id - - apenas 1 stencil - - RANGE rejeitado no momento - -Lacunas importantes: -- Nao ha implementacao completa dos parametros SODA padrao (TIME, POL, etc). -- BAND foi modelado como string de banda (g,r,i,z,Y), nao como intervalo em metros (como no SODA 1.0). -- Multiplicidade de parametros existe no parser/listas, mas fluxo funcional esta restrito a um unico id e um unico stencil. - -### 3) Fila, worker separado e escalabilidade -Arquivos: -- config/celery_app.py -- config/settings/base.py -- docker-compose.yml -- production.yml -- cutout/service/tasks.py -- cutout/service/policy.py -- cutout/service/uws/service.py - -Implementado: -- Infra de fila baseada em Celery + Redis esta configurada. -- Servicos separados em compose para: - - django - - redis - - celeryworker - - celerybeat - - flower -- JobService chama policy.dispatch, que envia tarefas para Celery. - -Pontos ainda incompletos/inconsistentes: -- dispatch atual usa chord(headers[0])(callback), ou seja, efetivamente nao executa grupo completo de tarefas; so usa o primeiro header. -- callback job_completed nao atualiza corretamente estado UWS no banco (funcao retorna string de teste). -- task image_cutout retorna string fixa "Resultado do cutout 1", sem metadados de resultado UWS persistidos. -- transicoes de fase estao incompletas (QUEUED parcial; EXECUTING/COMPLETED/ERROR sem fluxo robusto de ponta a ponta). - -### 4) Identificacao dos arquivos FITS por coordenada -Arquivos: -- cutout/lib/des_cutout.py -- cutout/lib/base_cutout.py -- cutout/lib/cutout.py - -Implementado (parte cientifica do recorte para DES): -- calculo de vertices do cutout a partir de RA/DEC/size -- leitura de catalogo de tiles (dr2_tiles.csv) -- identificacao dos tiles que cobrem os vertices -- derivacao de paths de arquivos FITS comprimidos (.fits.fz) -- descompressao on-demand com funpack para /data/tmp -- leitura parcial via astropy Cutout2D e montagem quando cruza 1, 2 ou 3 tiles -- geracao de saida FITS -- geracao de PNG Lupton (gri) - -Lacunas: -- logica de identificacao de tile usa matching simples por bounding box e assume que sempre encontra indice valido (risco de excecao em bordas/falhas). -- nao ha camada de abstracao para varios surveys pronta em producao (apenas des_dr2 implementado). -- RANGE/POLYGON nao estao implementados de ponta a ponta no backend de execucao atual. - -### 5) Persistencia de jobs e resultados -Arquivos: -- cutout/service/models/job.py -- cutout/service/models/job_parameter.py -- cutout/service/models/job_result.py -- cutout/service/uws/models.py -- cutout/service/uws/job.py - -Implementado: -- tabelas para Job, JobParameter, JobResult -- fases UWS no modelo de Job -- run_id, owner, creation/start/end/destruction/execution_duration/quote - -Lacunas: -- escrita efetiva de JobResult durante processamento nao esta fechada -- erro UWS (errorSummary/detail) nao esta modelado e persistido de forma completa -- endpoints de consulta/gestao de jobs nao estao expostos - -## Comparacao objetiva com o esperado (VO + requisitos do projeto) - -### Requisito: API VO-compliant para cutout -Status: PARCIAL -- Existe tentativa de alinhar com UWS/SODA em nomenclatura e parser de POS. -- Falta contrato completo de API SODA/UWS (principalmente endpoints async UWS e respostas/payloads padronizados). - -### Requisito: receber coordenada/tamanho ou lista -Status: PARCIAL -- Coordenada unica com POS=CIRCLE entra no fluxo. -- Lista/multiplos valores nao esta pronta no fluxo final (restrito por policy). - -### Requisito: identificar arquivos pela coordenada e gerar resultado -Status: PARCIALMENTE IMPLEMENTADO -- Identificacao de tiles/arquivos DES e recorte FITS existe no backend cientifico. -- Integracao robusta com retorno de resultado via API/UWS ainda incompleta. - -### Requisito: worker separado e escalavel -Status: BASE IMPLEMENTADA, FLUXO INCOMPLETO -- Infra de worker separado e fila existe (Celery + Redis + containers dedicados). -- Orquestracao completa do ciclo de vida dos jobs e resultados ainda incompleta. - -### Requisito: sistema de fila e distribuicao de tasks -Status: PARCIAL -- Fila existe. -- Distribuicao atual de tarefas em chord/group nao esta finalizada para multiplos itens. - -## Principais gaps para concluir o servico - -1. Expor API UWS/SODA completa (sync + async) com recursos de job padronizados. -2. Implementar respostas de erro no formato SODA (text/plain com prefixos corretos). -3. Finalizar ciclo de vida do job: PENDING -> QUEUED -> EXECUTING -> COMPLETED/ERROR/ABORTED. -4. Persistir resultados reais (JobResult com URL, mime_type, size) e disponibilizar download. -5. Implementar de fato multiplos inputs (lista de coordenadas e combinacoes de parametros) de forma escalavel. -6. Revisar parametro BAND para compatibilidade com SODA (ou declarar extensao custom com metadados claros). -7. Implementar endpoints/fluxos de monitoramento e gestao de jobs do usuario. -8. Fechar politicas de retencao/garbage collector para resultados e temporarios. - -## Conclusao executiva -O projeto ja tem uma base tecnica relevante: parser de stencils, modelos de job inspirados em UWS, infraestrutura Celery/Redis com worker separado e implementacao cientifica de cutout DES com identificacao de tiles por coordenada. - -Porem, no estado atual ele ainda e um prototipo incompleto no que mais importa para operacao VO-compliant: contrato de API UWS/SODA de ponta a ponta, fluxo async padronizado, ciclo de vida completo de jobs e publicacao de resultados/erros no formato esperado. - -Em resumo: o nucleo de processamento existe, a infraestrutura de fila existe, mas a camada de produto/API padrao VO ainda nao esta finalizada. \ No newline at end of file diff --git a/Anotacoes.md b/Anotacoes.md index 596c085..6d82a8d 100644 --- a/Anotacoes.md +++ b/Anotacoes.md @@ -37,14 +37,14 @@ Duvidas: - [X] Endpoint Sync para cutouts DES POS Circle Fits. - [X] Cutout Sync em background com Celery. - [x] Endpoint Sync para cutouts DES POS Circle Png com `engine=astrocut` (corrigido naming de saida). -- [] Endpoint Sync para cutouts DES POS Polygon Jpg. -- [] função para Cutout DES usando Range de posições. -- [] função para Cutout DES usando Poligono. -- [] Registrar a criação do cutout sync na listas de job do usuario??? -- [] PNGs só tem opção de gerar imagens coloridas, usando gri. (Acredito que de para ter mais opções, png para uma banda só ou outras combinações.) +- [x] Endpoint Sync para cutouts DES POS Polygon Jpg. +- [x] função para Cutout DES usando Range de posições. +- [x] função para Cutout DES usando Poligono. +- [x] Registrar a criação do cutout sync na listas de job do usuario??? +- [x] PNGs só tem opção de gerar imagens coloridas, usando gri. (Acredito que de para ter mais opções, png para uma banda só ou outras combinações.) - [] Fluxo sync `engine=legacy` + `format=png` ainda com falha (investigar stack completa e cobrir com teste de regressao). - [] Criar exemplos de uso da api no Jupyter Notebook. -- [] Remover o metodo "legacy" (código do adriano) e deixar apenas o astrocut??? +- [x] Remover o metodo "legacy" (código do adriano) e deixar apenas o astrocut??? ## Cutout Sync LSST @@ -55,15 +55,15 @@ Duvidas: ## Cutout Async DES -- [] Submeter lista de coordenadas para processamento em background com celery. +- [x] Submeter lista de coordenadas para processamento em background com celery. - [] Email com informação do job asyncrono. (opcional) - [] Definir Limit qtd/tamanho de Cutouts ## Monitoramento dos Jobs -- [] Regitrar no banco de dados cada cutout criado pelo usuario. (Incluir Cutouts Sync?) +- [x] Regitrar no banco de dados cada cutout criado pelo usuario. (Incluir Cutouts Sync?) - [] Interface com a lista de Jobs -- [] Download dos resultados do job. +- [x] Download dos resultados do job. - [] Cancelar/Deletar um Job. - [] Mostrar uso da quota? - [] Limite de Jobs? diff --git a/CURL_TESTES_SYNC.md b/CURL_TESTES_SYNC.md deleted file mode 100644 index a1945f9..0000000 --- a/CURL_TESTES_SYNC.md +++ /dev/null @@ -1,146 +0,0 @@ -# Comandos curl para testar o endpoint sync - -## 1) Teste sem autenticacao (esperado: 401) - -```bash -curl -i "http://localhost:8000/api/sync?id=des_dr2&pos=CIRCLE%2036.30911%20-10.18749%202&format=fits&band=g" -``` - -## 2) Obter token de autenticacao - -```bash -curl -s -X POST "http://localhost:8000/auth-token/" \ - -H "Content-Type: application/json" \ - -d '{"username":"SEU_USUARIO","password":"SUA_SENHA"}' -``` - -Saida esperada: - -```json -{"token":"..."} -``` - -## 3) Exportar token em variavel de ambiente - -```bash -TOKEN="SEU_TOKEN_AQUI" -``` - -## 4) Chamar endpoint sync autenticado (engine astrocut) - -```bash -curl -i -G "http://localhost:8000/api/sync" \ - -H "Authorization: Token $TOKEN" \ - --data-urlencode "id=des_dr2" \ - --data-urlencode "pos=CIRCLE 36.30911 -10.18749 2" \ - --data-urlencode "engine=astrocut" \ - --data-urlencode "format=fits" \ - --data-urlencode "band=g" -``` - -- Se houver todos os arquivos de entrada e o processamento concluir, retorna 200 com arquivo. -- Se faltar algum FITS esperado localmente, retorna 422 com `Input file unavailable`. - -## 5) Chamar endpoint sync autenticado (engine legacy) - -```bash -curl -i -G "http://localhost:8000/api/sync" \ - -H "Authorization: Token $TOKEN" \ - --data-urlencode "id=des_dr2" \ - --data-urlencode "pos=CIRCLE 36.30911 -10.18749 0.01" \ - --data-urlencode "engine=legacy" \ - --data-urlencode "format=fits" \ - --data-urlencode "band=g" -``` - -## 6) Salvar retorno binario em arquivo local (quando houver 200) - -```bash -curl -s -G "http://localhost:8000/api/sync" \ - -H "Authorization: Token $TOKEN" \ - --data-urlencode "id=des_dr2" \ - --data-urlencode "pos=CIRCLE 36.30911 -10.18749 0.01" \ - --data-urlencode "engine=astrocut" \ - --data-urlencode "format=fits" \ - --data-urlencode "band=g" \ - -o sync_result_test.fits -``` - -## 7) Exemplo para validar negacao de acesso por policy (esperado: 403) - -```bash -curl -i -G "http://localhost:8000/api/sync" \ - -H "Authorization: Token $TOKEN" \ - --data-urlencode "id=private_survey" \ - --data-urlencode "pos=CIRCLE 36.30911 -10.18749 0.01" \ - --data-urlencode "engine=astrocut" \ - --data-urlencode "format=fits" \ - --data-urlencode "band=g" -``` - -```bash -curl -i -G "http://localhost:80/api/sync" \ - -H "Authorization: Token 89cce9976177ed2c275b75782290faa915089518" \ - --data-urlencode "id=des_dr2" \ - --data-urlencode "pos=CIRCLE 36.30911 -10.18749 2" \ - --data-urlencode "engine=astrocut" \ - --data-urlencode "format=png" \ - --data-urlencode "band=gri" --o sync_result_test.png -``` - - -## 8) Fluxo completo em duas linhas (token + teste) - -```bash -TOKEN=$(curl -s -X POST "http://localhost:8000/auth-token/" -H "Content-Type: application/json" -d '{"username":"SEU_USUARIO","password":"SUA_SENHA"}' | python -c "import sys,json; print(json.load(sys.stdin)['token'])") -curl -i -G "http://localhost:8000/api/sync" -H "Authorization: Token $TOKEN" --data-urlencode "id=des_dr2" --data-urlencode "pos=CIRCLE 36.30911 -10.18749 0.01" --data-urlencode "engine=astrocut" --data-urlencode "format=fits" --data-urlencode "band=g" -``` - -## 9) Ajustar ownership de arquivo gerado no container - -Quando o arquivo de saida for criado dentro do container com owner `root:root`, ajuste para uid/gid do host: - -```bash -docker compose exec django sudo chown 1000:1000 /data/results/teste.fits -``` - -Para verificar: - -```bash -docker compose exec django ls -l /data/results/teste.fits -``` - -Coordenadas de exemplo: cutout/lib/des_cutout.py - -```python -if __name__ == "__main__": - cutouts = [ - {"ra": 36.30911, "dec": -10.18749, "size": 2.0, "band": "g", "format": "fits"}, # 1 - Tile - {"ra": 36.30911, "dec": -10.18749, "size": 2.0, "band": "gri", "format": "png"}, # 1 - Tile - {"ra": 36.15801, "dec": -10.33579, "size": 2.0, "band": "g", "format": "fits"}, # 2 - Tile - {"ra": 36.15801, "dec": -10.33579, "size": 2.0, "band": "gri", "format": "png"}, # 2 - Tile - # {"ra": 35.23676, "dec": -10.33269, "size": 10.0, "band": "g", "format": "fits"}, # 3 - Tile - ] - - dc = DesCutout() - - for c in cutouts: - if c["format"] == "fits": - filename = "{:.5f}_{:.5f}_{}.fits".format(round(c["ra"], 5), round(c["dec"], 5), c["band"]) - resultfile = Path("/data/results").joinpath(filename) - - result = dc.single_cutout_fits( - ra=c["ra"], dec=c["dec"], size_arcmin=c["size"], band=c["band"], path=resultfile - ) - print(result) - - if c["format"] == "png": - filename = "{:.5f}_{:.5f}.png".format(round(c["ra"], 5), round(c["dec"], 5)) - resultfile = Path("/data/results").joinpath(filename) - - result = dc.single_cutout_png( - ra=c["ra"], dec=c["dec"], size_arcmin=c["size"], band=c["band"], path=resultfile - ) - print(result) -``` diff --git a/PLANO_ASYNC.md b/PLANO_ASYNC.md deleted file mode 100644 index 879bdd5..0000000 --- a/PLANO_ASYNC.md +++ /dev/null @@ -1,217 +0,0 @@ -# Plano de Implementacao - API Async UWS/SODA - -Data de inicio: 2026-05-23 -Escopo: implementar a superficie async do servico de cutout com UWS/SODA, usando workers em background, sem executar ainda o cutout cientifico real. -Publico: humanos e agentes de AI. - -## Objetivo desta fase - -Implementar a camada de API async para: - -- receber os parametros corretos do pedido de cutout; -- criar jobs UWS no banco; -- enfileirar execucao em workers; -- expor endpoints para acompanhar job, parametros, fase e resultados; -- persistir resultados fake; -- manter a assinatura da funcao de worker compativel com a futura execucao real. - -## Decisoes atuais - -1. O endpoint sync existente permanece como esta. -2. A arvore async sera exposta em `/api/async`. -3. Nesta fase, o worker executa uma funcao fake, mas com assinatura compativel com o pipeline real. -4. O foco agora e fechar contrato de API, fila, transicoes de fase e retrieval de resultados. -5. A semantica final de erro SODA/DALI em `text/plain` continua adiada; vamos priorizar a estrutura UWS e o ciclo de vida dos jobs. - -## Superficie de API planejada - -### Colecao de jobs - -- `POST /api/async` - - cria um job async; - - aceita parametros de cutout; - - pode iniciar execucao imediatamente quando apropriado; - - retorna `303 See Other` com `Location` apontando para o recurso do job. - -- `GET /api/async` - - lista jobs do usuario autenticado. - -### Recurso de job - -- `GET /api/async/{job_id}` - - retorna metadados do job. - -- `DELETE /api/async/{job_id}` - - remove o job do usuario; se necessario, aborta antes. - -### Fase do job - -- `GET /api/async/{job_id}/phase` - - retorna a fase atual do job. - -- `POST /api/async/{job_id}/phase` - - aceita ao menos `PHASE=RUN` e `PHASE=ABORT`. - -### Parametros do job - -- `GET /api/async/{job_id}/parameters` - - retorna os parametros persistidos do job. - -### Resultados do job - -- `GET /api/async/{job_id}/results` - - retorna a lista de resultados do job. - -- `GET /api/async/{job_id}/results/{result_id}` - - entrega ou redireciona para o artefato do resultado. - -## Parametros da primeira iteracao - -Nesta fase, a API async vai aceitar o mesmo conjunto de parametros hoje usado no fluxo sync atual: - -- `id` -- `pos` -- `runid` -- `format` -- `band` -- `engine` -- `color` -- `rgb_bands` -- `persist` - -## Regras de ciclo de vida - -Fluxo esperado: - -1. Criar job em `PENDING`. -2. Ao disparar execucao, registrar `message_id` e mover para `QUEUED`. -3. Worker ao iniciar move para `EXECUTING`. -4. Worker fake ao concluir persiste resultado(s) e move para `COMPLETED`. -5. Em falha, mover para `ERROR`. -6. Em cancelamento, mover para `ABORTED`. - -## Worker fake - -O worker fake deve manter a assinatura funcional esperada do processamento real: - -`fake_image_cutout(job_id, source_id, stencil, engine, band, format, path, files=None, color=False, rgb_bands=None, persist=False)` - -Ele pode gerar um arquivo pequeno em disco contendo um resumo dos parametros recebidos. Esse arquivo sera usado para validar: - -- persistencia do resultado; -- endpoint de resultados; -- download do artefato. - -## Evolucoes de modelo previstas - -### Job - -Manter o modelo atual e expandir o service para suportar: - -- listagem por usuario; -- detalhe por usuario; -- start; -- abort; -- delete; -- registro de erro; -- registro de resultados. - -### JobResult - -Precisamos permitir retrieval real do resultado async. A direcao atual e adicionar campos para guardar o caminho e/ou URL do artefato persistido. - -## Arquivos principais previstos - -- `PLANO_ASYNC.md` -- `config/api_router.py` -- `cutout/service/api/views.py` -- `cutout/service/api/serializers.py` -- `cutout/service/uws/service.py` -- `cutout/service/uws/models.py` -- `cutout/service/policy.py` -- `cutout/service/tasks.py` -- `cutout/service/models/job_result.py` -- `cutout/service/migrations/*` -- testes em `cutout/service/tests/` ou pasta equivalente - -## Sequencia de implementacao - -### Fase 1 - scaffold da API async - -- registrar rotas async; -- criar views para colecao, job, phase, parameters e results; -- centralizar parse de parametros. - -### Fase 2 - service layer - -- expandir `JobService` para ciclo completo do async; -- garantir ownership; -- implementar conversao de resultados persistidos. - -### Fase 3 - fila e worker fake - -- enfileirar execucao via Celery; -- registrar `message_id`; -- criar funcao fake e task de orquestracao do job; -- persistir `JobResult`. - -### Fase 4 - resultados e download - -- listar resultados; -- expor download por `result_id`. - -### Fase 5 - testes e ajustes - -- cobrir criacao, ownership, fases, resultados e download; -- atualizar este arquivo com decisoes relevantes conforme avancarmos. - -## Status atual - -- Scaffold principal da API async implementado. -- Endpoints async registrados em `config/api_router.py`. -- Job async criado e disparado automaticamente via worker fake. -- Persistencia de resultados fake implementada em `JobResult` com `url` e `file_path`. -- Download de resultado por `result_id` implementado. -- Testes da API async adicionados e passando. - -## Criterios de aceite - -1. `POST /api/async` cria job e retorna localizacao do recurso. -2. O job pode ser iniciado e passa por `QUEUED -> EXECUTING -> COMPLETED`. -3. `GET /api/async/{job_id}` reflete a fase correta. -4. `GET /parameters` retorna os parametros persistidos. -5. `GET /results` lista resultados fake persistidos. -6. `GET /results/{result_id}` entrega um artefato real. -7. Jobs de outros usuarios nao ficam acessiveis. - -## Registro de decisoes - -### 2026-05-23 - -- O arquivo de acompanhamento da fase async sera `PLANO_ASYNC.md`. -- A implementacao inicial prioriza a arvore de recursos UWS e o ciclo de vida dos jobs. -- A funcao cientifica principal continua fora de escopo nesta etapa. -- `POST /api/async` vai iniciar o job imediatamente por padrao, tratando ausencia de `PHASE` como `RUN`. -- Os detalhes de job, parametros e resultados serao retornados em JSON nesta primeira entrega, mantendo `GET/POST` de `phase` em formato textual. -- `JobResult` passa a persistir `url` e `file_path` para permitir listagem e download real do artefato fake. -- A task async fake recebe os mesmos argumentos estruturais do cutout real e grava um artefato simples em disco para exercitar retrieval. -- A serializacao enviada ao worker async usa apenas dados JSON-safe; objetos de stencil ficam restritos ao fluxo sync. -- O lint foi explicitamente postergado nesta etapa para priorizar o fluxo funcional da API async. - -## Validacoes executadas - -- `pytest cutout/service/tests/test_async_api.py -q` -> 4 passed - -## Arquivos alterados nesta etapa - -- `PLANO_ASYNC.md` -- `config/api_router.py` -- `cutout/service/api/views.py` -- `cutout/service/api/serializers.py` -- `cutout/service/models/job_result.py` -- `cutout/service/migrations/0003_jobresult_storage_fields.py` -- `cutout/service/policy.py` -- `cutout/service/tasks.py` -- `cutout/service/tests/test_async_api.py` -- `cutout/service/uws/models.py` -- `cutout/service/uws/service.py` diff --git a/PLANO_SYNC_IVOA_DES_DESCOBERTA.md b/PLANO_SYNC_IVOA_DES_DESCOBERTA.md deleted file mode 100644 index a4377b3..0000000 --- a/PLANO_SYNC_IVOA_DES_DESCOBERTA.md +++ /dev/null @@ -1,521 +0,0 @@ -# Plano de Implementacao: Sync Cutout com Descoberta de Arquivos (IVOA) - -Data: 2026-04-26 -Escopo: endpoint sync primeiro, com modulo separado de descoberta de arquivos para DES -Publico: humanos + agentes de AI - -Atualizacao de prioridade (2026-04-26): -- Este plano foi reordenado para refletir a execucao por prioridade de negocio (1 a 6). -- A semantica de erro em `text/plain` continua no plano geral SODA, mas foi explicitamente movida para depois da fase async. - -## 1) Status de protocolos IVOA (atualizacao) - -Baseado na pagina de standards do IVOA (https://www.ivoa.net/documents/): - -- UWS: 1.1 (Recommendation, 2016-10-24) -- SODA: 1.0 (Recommendation, 2017-05-17) -- SIA: 2.0 (Recommendation, 2015-12-23) -- DataLink: 1.1 (Recommendation, 2023-12-15) -- DALI: 1.1 e 1.2 em progresso (PR no ciclo 2025) - -Conclusao pratica para este projeto: -- Continuar modelando execucao assinc com UWS 1.1. -- Continuar modelando operacao de cutout com SODA 1.0. -- Para descoberta de arquivos por coordenada, usar padrao de descoberta SIA/ObsCore (ou TAP/ObsCore) + DataLink quando necessario. - -## 2) Protocolo para "identificacao dos arquivos envolvidos" - -Pergunta: existe um protocolo IVOA especifico para "dado coordenada, retornar lista de arquivos"? - -Resposta curta: -- Nao ha um protocolo unico e isolado so para isso. -- O fluxo VO recomendado e: - 1. Descoberta de datasets por posicao com SIA 2.0 (ou TAP/ObsCore). - 2. Se um dataset tiver multiplos arquivos/derivacoes, usar DataLink para listar e qualificar links/arquivos/servicos. - 3. Depois chamar SODA para recorte (sync ou async). - -Para este projeto (DES): -- Fase inicial: descoberta via catalogo local (CSV) e retorno de lista de arquivos internos. -- Evolucao: trocar o backend de descoberta para SIA/TAP/DataLink sem mudar contrato interno do sistema. - -## 3) Objetivo tecnico imediato - -Implementar o endpoint sync com pipeline claro: -1. Usuario envia coordenada + tamanho + parametros basicos. -2. Camada de policy valida se o usuario pode acessar o survey solicitado. -3. Modulo de descoberta identifica os arquivos DES envolvidos. -4. Modulo de cutout executa em task Celery (backend atual; depois astrocut). -5. Endpoint sync devolve resultado (arquivo ou redirect para resultado), mantendo semantica SODA-like. - -Escopo funcional do endpoint sync nesta fase: -- aceitar os 3 tipos de pedido espacial: CIRCLE, RANGE e POLYGON -- para DES DR2 (publico), policy inicial retorna sempre true - -## 4) Arquitetura alvo (modular) - -## 4.1 Modulos novos (separados) - -### A) modulo de descoberta de arquivos (novo) -Responsabilidade: -- Dado stencil (CIRCLE inicialmente) e survey/release, retornar lista de arquivos candidatos para cutout. - -Contrato interno sugerido: - -```python -class FileLocator(Protocol): - def find_files( - self, - *, - survey_id: str, - stencil: dict, - band: str | None = None, - ) -> list[FileDescriptor]: - ... -``` - -`FileDescriptor` minimo: -- dataset_id (opcional no inicio) -- file_path (ou URI) -- tile_id -- band -- metadados minimos de cobertura (opcional na v1) - -Implementacao v1 (DES): -- `DesCsvFileLocator` -- Fonte: arquivo local com cobertura/tile/path - -Implementacoes futuras: -- `SiaFileLocator` (consulta SIA 2.0) -- `TapObsCoreFileLocator` (consulta TAP/ObsCore) -- `DataLinkResolver` (expande dataset em links/arquivos) - -### B) modulo de motor de cutout (abstracao) -Responsabilidade: -- Receber lista de arquivos + stencil + formato e produzir resultado. - -Contrato interno sugerido: - -```python -class CutoutEngine(Protocol): - def run_cutout( - self, - *, - files: list[FileDescriptor], - stencil: dict, - output_format: str, - band: str | None, - output_path: str, - ) -> CutoutResult: - ... -``` - -Implementacao v1: -- adapter sobre implementacao atual (DesCutout/base_cutout) - -Implementacao v2: -- `AstrocutEngine` (troca transparente) - -### C) orquestrador de job sync -Responsabilidade: -- Validar parametros -- Aplicar policy de acesso ao survey -- Chamar file locator -- Enfileirar task Celery -- Aguardar ate timeout sync -- Responder com arquivo ou redirect - -### D) camada de policy de acesso por survey (novo) -Responsabilidade: -- Centralizar decisao de autorizacao para uso de survey/release no cutout. -- Permitir evolucao para surveys privados sem mudar endpoint e sem acoplamento com regra de negocio especifica. - -Contrato interno sugerido: - -```python -class SurveyAccessPolicy(Protocol): - def can_request_cutout( - self, - *, - user_id: str, - survey_id: str, - release: str | None = None, - ) -> bool: - ... -``` - -Implementacao v1 (agora): -- `DesPublicAccessPolicy` -- Comportamento: retorna sempre `True` para DES DR2 (dados publicos de teste) - -Implementacoes futuras: -- `RoleBasedSurveyAccessPolicy` -- `ExternalAuthSurveyAccessPolicy` (LDAP/SSO/servico interno) - -## 4.2 Fluxo Sync (alto nivel) - -1. `GET/POST /api/sync` -2. Parse parametros (id/pos/format/band/runid) -3. Converter parametro espacial para stencil interno (CIRCLE, RANGE ou POLYGON) -4. `survey_access_policy.can_request_cutout(...)` -5. Se policy negar: retornar erro de autorizacao no formato padrao -6. `file_locator.find_files(...)` -7. Se vazio: retornar 204 (SODA sync sem pixels/sem match) -8. Enfileirar `run_sync_cutout_task.delay(...)` -9. Esperar resultado por janela limitada (ex: 20-30s) -10. Sucesso: -- opcao A: `FileResponse` -- opcao B: `303` para URL de resultado -11. Erro: -- resposta `text/plain` com prefixos SODA/DALI (UsageError, Error, ServiceUnavailable, MultiValuedParamNotSupported) - -## 5) Contrato API Sync (proposta inicial) - -## 5.1 Entrada (fase 1) -- `id` (ex: des_dr2) -- parametro espacial (3 formas suportadas no endpoint): - - `pos=CIRCLE ra dec radius` - - `pos=RANGE ra_min ra_max dec_min dec_max` - - `pos=POLYGON ra1 dec1 ra2 dec2 ra3 dec3 ...` -- `format` (`fits` inicialmente; `png` opcional) -- `band` (custom DES, enquanto nao houver BAND padrao em metros) -- `runid` (opcional) - -Observacao de compatibilidade VO: -- Em SODA 1.0, BAND padrao e intervalo em metros. -- Para DES no curto prazo, manter parametro custom de banda por letra, mas documentar como extensao local. - -## 5.2 Saida -- 200 + stream de arquivo (ou 303 para URL de arquivo) -- 204 sem conteudo quando nao houver sobreposicao/arquivos -- 4xx/5xx em `text/plain` com prefixo de erro padronizado -- 403 para survey sem permissao (quando policy negar) - -Exemplo de erro: -- `UsageError: invalid POS value` -- `MultiValuedParamNotSupported: only one POS in sync mode` -- `AuthorizationError: user has no access to requested survey` - -## 6) Plano de execucao em fases - -## Fase 0 - Hardening do prototipo atual -Objetivo: -- Limpar caminho minimo do sync para ficar previsivel. - -Entregaveis: -- remover codigo morto/comentado do endpoint -- padronizar parse/validacao de parametros -- padronizar suporte de parse para CIRCLE/RANGE/POLYGON no endpoint -- padronizar respostas de erro text/plain - -Criterios de aceite: -- endpoint sync com comportamento estavel para sucesso/falha -- testes de erro basicos passando - -## Fase 1 - Modulo separado de descoberta (DES CSV) -Objetivo: -- introduzir separacao formal entre API e descoberta de arquivos - -Entregaveis: -- pacote novo, ex: `cutout/service/discovery/` -- interface `FileLocator` -- implementacao `DesCsvFileLocator` -- testes unitarios para descoberta por CIRCLE/RANGE/POLYGON - -Criterios de aceite: -- dado POS CIRCLE/RANGE/POLYGON conhecido, retorna lista coerente de arquivos -- endpoint sync usa o locator (sem chamar logica de tile diretamente) - -## Fase 1.5 - Camada de policy de acesso -Objetivo: -- introduzir camada de autorizacao por survey, desacoplada da API e da descoberta. - -Entregaveis: -- pacote novo, ex: `cutout/service/policies/` -- interface `SurveyAccessPolicy` -- implementacao inicial `DesPublicAccessPolicy` (retorna true) -- integracao da policy no fluxo do endpoint sync - -Criterios de aceite: -- endpoint chama policy antes da descoberta/cutout -- para DES DR2, comportamento permanece liberado -- erro de autorizacao padronizado para surveys futuros privados - -## Fase 2 - Orquestracao sync via Celery -Objetivo: -- executar pipeline completo no worker, mantendo semantica sync no endpoint - -Entregaveis: -- task Celery `run_sync_cutout_task` -- payload de task contendo stencil + files + output spec -- retorno com caminho/URL do resultado - -Criterios de aceite: -- endpoint sync gera resultado real -- tempos de timeout tratados com erro claro -- fase de job no banco ao menos em PENDING/QUEUED/COMPLETED/ERROR para sync - -## Fase 3 - Adapter de motor de cutout -Objetivo: -- preparar troca para astrocut sem quebrar API - -Entregaveis: -- interface `CutoutEngine` -- adapter engine atual -- stub/integracao inicial `AstrocutEngine` - -Criterios de aceite: -- troca de engine por configuracao -- mesmos testes de endpoint continuam validos - -## Fase 4 - Caminho VO de descoberta (futuro) -Objetivo: -- adicionar backend de descoberta por SIA/TAP/DataLink - -Entregaveis: -- `SiaFileLocator` e/ou `TapObsCoreFileLocator` -- opcional `DataLinkResolver` - -Criterios de aceite: -- mesmo endpoint sync funcionando com backend local ou remoto - -## 7) Estrutura de codigo sugerida - -```text -cutout/service/ - api/ - views.py - policies/ - __init__.py - base.py # SurveyAccessPolicy - des_public.py # retorna True para DES DR2 - discovery/ - __init__.py - base.py # protocolos/interfaces - des_csv_locator.py # v1 - models.py # FileDescriptor - cutout_engine/ - __init__.py - base.py # CutoutEngine - des_engine.py # adapter atual - astrocut_engine.py # futuro - orchestration/ - sync_service.py # fluxo endpoint -> locator -> celery - tasks.py # tasks celery -``` - -## 8) Plano de testes (humanos + AI) - -## 8.1 Unitarios -- parse POS (CIRCLE/RANGE/POLYGON validos e invalidos) -- DES CSV locator (match unico, multiplos tiles, vazio) para os 3 stencils -- policy de acesso (DES publico -> true) -- mapeamento de erros para prefixos SODA - -## 8.2 Integracao -- endpoint sync -> celery -> resultado FITS -- caso sem overlap -> 204 -- timeout do worker -> ServiceUnavailable -- endpoint sync com CIRCLE, RANGE e POLYGON -- fluxo com policy aplicada antes da descoberta - -## 8.3 Contrato -- validacao de content-type de erro (`text/plain`) -- validacao de status codes esperados - -## 9) Decisoes abertas (precisam de definicao) - -1. No sync, retorno principal sera `200 file stream` ou `303 para URL`? -2. Limite de tempo do sync (timeout oficial)? -3. Parametro `band` custom DES sera mantido temporariamente com qual nome oficial? -4. Formatos suportados na fase 1: apenas FITS ou FITS+PNG? -5. Persistencia de resultados sync: por quanto tempo? - -## 10) Backlog pronto para execucao por agentes AI - -## Sprint A (infra de codigo) -- criar interfaces `SurveyAccessPolicy`, `FileLocator` e `CutoutEngine` -- criar `DesPublicAccessPolicy` (always true) -- criar `DesCsvFileLocator` -- adicionar testes unitarios de policy e locator - -## Sprint B (sync funcional) -- criar `SyncCutoutService` (orquestracao) -- integrar endpoint `/api/sync` ao service -- garantir suporte no endpoint para CIRCLE/RANGE/POLYGON -- criar task celery unica para sync cutout -- implementar respostas de erro padrao - -## Sprint C (qualidade) -- testes de integracao endpoint+celery -- metrica/log estruturado por runid/job -- documentacao OpenAPI atualizada - -## Definition of Done (DoD) -- endpoint sync funcionando ponta a ponta com descoberta separada -- endpoint sync suportando CIRCLE, RANGE e POLYGON -- camada de policy integrada no fluxo (DES DR2 liberado) -- descoberta DES desacoplada da API -- task em worker separado via Celery -- testes cobrindo sucesso, vazio, erro e timeout -- documento de contrato API atualizado - -## 11) Nota de compatibilidade com o estado atual do repositorio - -Este plano assume refatoracao incremental sobre o prototipo existente, sem reescrever tudo de uma vez. -Prioridade imediata: -- estabilizar sync -- extrair descoberta de arquivos para modulo proprio -- manter possibilidade de troca do motor de recorte para astrocut. - -## 12) Replanejamento priorizado (ordem de execucao) - -## 12.1 Panorama do que ja esta pronto - -- Discovery DES/CSV desacoplado com suporte de parse e descoberta para CIRCLE, RANGE e POLYGON. -- Policy de acesso por survey integrada antes do discovery. -- Pipeline sync com Celery e fases de job (PENDING/QUEUED/EXECUTING/COMPLETED/ERROR). -- Selecao de engine por parametro (`astrocut` default e `legacy` disponivel). -- Astrocut nativo atualmente operacional para `format=fits` com stencil `circle`. - -## 12.2 Prioridade 1 - Suporte a PNG (inclui PNG colorido) - -Objetivo: -- Adicionar `format=png` no pipeline sync, com suporte a PNG monocromatico e RGB. - -Decisao de compatibilidade IVOA: -- Em SODA 1.0 nao existe parametro padrao para "PNG colorido"/composicao RGB. -- SODA define parametros como ID, POS/CIRCLE/POLYGON, BAND, TIME e POL; custom parameters sao permitidos. -- Portanto, o modo colorido sera extensao local documentada no contrato. - -Contrato proposto: -- `format=png` ativa saida PNG. -- Novo parametro `color` (boolean): - - `color=false` (default): PNG de banda unica. - - `color=true`: PNG RGB composto. -- Novo parametro opcional `rgb_bands` (string): default `gri`. - - Exemplo: `rgb_bands=gri` ou `rgb_bands=riz`. -- Regra de negocio: - - Se `color=true` e `band` vier com banda unica, `rgb_bands` prevalece para composicao. - - Se `color=false`, usa apenas `band`. - -## 12.3 Depuração: plano de ação passo-a-passo (2026-04-26) - -Contexto: -- Foram observados arquivos gerados incorretos: FITS que nao abre e PNG aparentemente vazio. A implementação legacy tambem nao gerou arquivos. - -Objetivo: -- Fazer debug sistematico na cadeia de criação de cutouts, do mais simples ao composto, garantindo que o caminho FITS mono funcione antes de avançar. - -Passos: -1. Validar `single_cutout_fits` da implementação legacy (`DesCutout`) usando um script de depuração que execute um corte mono (uma banda) e escreva um FITS em `/data/results`. Verificar integridade com `astropy.io.fits.open`. -2. Validar `single_cutout_png` (mono) usando o mesmo script: geracao via `single_cutout_png` e abertura com PIL/visualizacao simples. -3. Validar composição RGB (`single_cutout_png` com `band='gri'` ou `color=true` na pipeline nova): garantir que cada banda produz um FITS parcial e que o empilhamento gera um PNG visivel. -4. Se qualquer passo falhar, acrescentar logs detalhados (shapes, min/max, nan counts, dtype) e pequenos scripts de inspeção para cada FITS temporario criado. - -Entregaveis: -- `scripts/debug_cutout.py` (script de depuracao que reproduz as chamadas comentadas em `des_cutout.py` e salva artefatos em `/data/results/debug/`). -- Atualizacao breve de `STATUS_IMPLEMENTACAO_SYNC.md` com o registro do problema, passos de depuracao e responsavel. - -Critérios de aceite para depuracao: -- `scripts/debug_cutout.py` executa dentro do container e produz um FITS mono legivel por `astropy.io.fits`. -- PNG mono criado abre com PIL e tem bytes > 0. -- Logs e artefatos temporarios gravados em `/data/results/debug/` para inspeção. - -Sequencia desejada de acao: -1. Criar `scripts/debug_cutout.py` e commitar. -2. Executar o script dentro do container e coletar logs/artefatos. -3. Corrigir a funcao que falha (se for o caso) e reexecutar. -4. Depois do FITS mono OK, avançar para PNG mono e por fim RGB. - -Tempo estimado: 1-2 horas para diagnostico e consertos menores (depende de complexidade dos erros observados). - -Implementacao esperada: -- Engine legacy: manter comportamento atual de referencia DES para PNG. -- Engine astrocut: implementar caminho de composicao RGB (3 bandas) e caminho monocromatico (1 banda). -- Task/policy: encaminhar `format`, `color` e bandas efetivas para o engine. - -Criterios de aceite: -- `format=png&color=false&band=g` retorna PNG valido. -- `format=png&color=true&rgb_bands=gri` retorna PNG RGB valido. -- Validacoes claras para combinacoes invalidas de parametros. - -## 12.3 Prioridade 2 - RANGE/POLYGON ponta a ponta - -Objetivo: -- Garantir RANGE/POLYGON no fluxo completo da engine default, nao apenas no parse/discovery. - -Escopo: -- Completar suporte do `astrocut` para RANGE e POLYGON. -- Se houver limitacao da biblioteca, implementar adaptacao geometrica controlada (ex: conversao para janela equivalente) sem quebrar contrato. - -Criterios de aceite: -- Requests sync com `POS=RANGE ...` e `POS=POLYGON ...` executam com `engine=astrocut` e retornam resultado valido. -- Testes de integracao cobrindo CIRCLE/RANGE/POLYGON com engine default. - -## 12.4 Prioridade 3 - Parametro para persistir resultado em data/results - -Objetivo: -- Permitir controlar se o arquivo final sera persistido localmente em `data/results`. - -Contrato proposto: -- Novo parametro `persist` (boolean): - - `persist=false` (default): comportamento atual de resposta sync sem compromisso de retencao longa. - - `persist=true`: manter arquivo final em `data/results`. - -Regras: -- Definir naming deterministico por job/runid e evitar sobrescrita acidental. -- Registrar caminho final quando persistido. - -Criterios de aceite: -- `persist=true` preserva arquivo em `data/results`. -- `persist=false` nao cria persistencia adicional alem do necessario para stream/resposta. - -## 12.5 Prioridade 4 - Registro em banco para todos os pedidos sync - -Objetivo: -- Registrar auditoria completa de cada pedido de cutout sync. - -Campos minimos: -- usuario solicitante -- parametros usados -- status final (gerou resultado ou nao) -- tamanho do resultado em bytes -- tempo total de execucao - -Observacao: -- Parte da estrutura ja existe via `Job`, `JobParameter`, `JobResult`, `start_time` e `end_time`, mas falta consolidar populacao obrigatoria para sync em todos os cenarios. - -Criterios de aceite: -- Cada chamada sync gera registro auditavel com os campos minimos. -- Consultas administrativas conseguem listar historico por usuario e periodo. - -## 12.6 Prioridade 5 - Atualizacao OpenAPI - -Objetivo: -- Atualizar schema do endpoint sync com novos parametros e semantica atual. - -Escopo minimo: -- Documentar `format=png`, `color`, `rgb_bands`, `persist`. -- Documentar suporte de POS (CIRCLE/RANGE/POLYGON) e limites conhecidos por engine. -- Documentar exemplos de request/response para FITS e PNG. - -Criterios de aceite: -- OpenAPI e exemplos de uso alinhados com o comportamento real da API. - -## 12.7 Prioridade 6 - Semantica de errors (postergada) - -Decisao: -- Manter erros em JSON por enquanto. -- Migracao para semantica SODA/DALI (`text/plain` com prefixos padrao) fica para depois da fase async. - -Criterios de aceite futuro: -- Contrato de erro unico para sync/async alinhado ao padrao IVOA. - -## 13) Backlog operacional imediato (proxima sprint) - -1. Implementar prioridade 1 (PNG mono + colorido) incluindo parametro `color` e `rgb_bands`. -2. Fechar prioridade 2 (RANGE/POLYGON ponta a ponta no engine default). -3. Incluir prioridade 3 (`persist`) e registro de caminho final. -4. Fechar prioridade 4 (auditoria completa sync em banco). -5. Atualizar prioridade 5 (OpenAPI e exemplos curl). -6. Manter prioridade 6 explicitamente adiada ate fase async. diff --git a/STATUS_IMPLEMENTACAO_SYNC.md b/STATUS_IMPLEMENTACAO_SYNC.md deleted file mode 100644 index 7014cf3..0000000 --- a/STATUS_IMPLEMENTACAO_SYNC.md +++ /dev/null @@ -1,355 +0,0 @@ -# Status de Implementacao - Sync Cutout - -Data de inicio: 2026-04-26 -Escopo: acompanhar implementacao incremental do endpoint sync conforme plano IVOA/DES. -Publico: humanos e agentes de AI. - -## Como usar este arquivo - -Atualize este documento a cada fase implementada, sempre antes de abrir PR ou seguir para a fase seguinte. - -Checklist de atualizacao: -1. Resumo do que foi implementado. -2. Arquivos criados/alterados. -3. Regras de negocio adicionadas ou alteradas. -4. Decisoes tecnicas tomadas. -5. Comandos de validacao executados e resultado. -6. Commit(s) associado(s). - -## Estado atual por fase - -- Fase 0 (hardening): parcial -- Fase 1 (discovery desacoplado): concluida -- Fase 1.5 (policy de acesso por survey): concluida -- Fase 2 (orquestracao sync via celery): concluida -- Fase 3 (adapter de engine): concluida -- Fase 3.1 (astrocut nativo com discovery DES): concluida -- Fase 4 (descoberta VO remota): adiada (decisao de arquitetura pendente) - -## Prioridades atuais (replanejamento 2026-04-26) - -1. PNG (saida png, incluindo composicao colorida): pendente -2. RANGE/POLYGON ponta a ponta no engine default: parcial -3. Parametro para persistir resultado em `data/results`: pendente -4. Registro completo de pedidos sync no banco: parcial -5. Atualizacao OpenAPI: parcial -6. Semantica de errors padrao SODA/DALI: adiada para pos-fase-async (manter JSON por ora) - -## Resumo consolidado do que ja foi feito - -- Pipeline sync com Celery e espera sincrona no endpoint implementado. -- Registro de job com usuario e parametros implementado (`Job` + `JobParameter`). -- Fases de execucao registradas (PENDING/QUEUED/EXECUTING/COMPLETED/ERROR). -- Discovery DES/CSV separado da API com suporte de CIRCLE/RANGE/POLYGON na descoberta. -- Policy de survey integrada antes do discovery. -- Selecao de engine operacional (`astrocut` default, `legacy` alternativo). -- Astrocut nativo implementado para CIRCLE em FITS. - -## Lacunas tecnicas relevantes para o novo plano - -- Astrocut ainda restrito a `circle` e `fits`, impactando prioridades 1 e 2. -- Falta contrato de persistencia controlada por parametro (prioridade 3). -- Falta consolidar auditoria de resultado (sucesso, tamanho e tempo total) em todos os fluxos sync (prioridade 4). -- OpenAPI ainda nao cobre novos parametros planejados de PNG colorido e persistencia (prioridade 5). - -## Regras de negocio vigentes - -1. Tipos espaciais aceitos no parse: CIRCLE, RANGE, POLYGON. -2. Discovery v1 usa intersecao por bounding box do stencil com tiles DES em CSV. -3. Survey suportado no discovery atual: des_dr2. -4. Se nenhum arquivo for encontrado para a regiao solicitada, o fluxo falha explicitamente com erro de parametro. -5. Camada de policy de survey roda antes do discovery. -6. Policy inicial libera des_dr2 e nega surveys nao reconhecidos. -7. Durante execucao da task, se o arquivo esperado nao estiver acessivel no disco local, a API retorna erro explicito de arquivo indisponivel. -8. A API aceita selecao de engine via parametro `engine`. -9. Engine default atual: `astrocut`. -10. Engine legado permanece disponivel via `engine=legacy`. -11. Discovery de arquivos permanece no backend DES/CSV nesta etapa (sem SIA/TAP por enquanto). - -## Decisoes tecnicas registradas - -1. Manter abordagem geometrica simples (bounding box) nesta etapa e deixar refinamento cientifico para revisao posterior. -2. Estruturar policy de acesso com interface separada para permitir evolucao futura para surveys privados. -3. Validar cada fase dentro de container antes de commit. -4. Trabalhar em fases pequenas: implementar, testar, validar, commit. -5. Durante fase de aprovacao, manter multiplas ferramentas de cutout ativas para comparacao controlada. -6. Adiar integracao SIA/TAP ate definicao arquitetural formal; manter estabilidade com discovery DES local. - -## Historico de implementacao - -### Entrada 2026-04-26 - Fase 1 - -Resumo: -- Criado modulo de discovery desacoplado. -- Implementado locator DES por CSV com suporte a CIRCLE, RANGE e POLYGON. -- Integrado locator no fluxo de dispatch do policy. -- Ajustada task para receber lista de arquivos descobertos. -- Adicionado erro explicito quando nao ha arquivos para a regiao. - -Arquivos criados: -- cutout/service/discovery/__init__.py -- cutout/service/discovery/base.py -- cutout/service/discovery/models.py -- cutout/service/discovery/des_csv_locator.py -- cutout/service/discovery/tests/test_des_csv_locator.py - -Arquivos alterados: -- cutout/service/policy.py -- cutout/service/tasks.py - -Validacao executada: -- docker compose exec django black --check cutout/service/discovery cutout/service/policy.py cutout/service/tasks.py -- docker compose exec django isort --check-only cutout/service/discovery cutout/service/policy.py cutout/service/tasks.py -- docker compose exec django pytest cutout/service/discovery/tests/test_des_csv_locator.py -q -- Resultado: 5 passed - -Commit: -- b2edcfb - feat(discovery): add DES CSV locator and integrate dispatch file lookup - -### Entrada 2026-04-26 - Fase 1.5 - -Resumo: -- Criada camada de policy de acesso por survey. -- Implementada policy inicial publica para DES DR2. -- Integrada validacao de acesso antes do discovery no fluxo de dispatch. -- Confirmado comportamento de negacao para survey nao permitido. - -Arquivos criados: -- cutout/service/policies/__init__.py -- cutout/service/policies/base.py -- cutout/service/policies/des_public.py -- cutout/service/policies/tests/test_des_public_policy.py - -Arquivos alterados: -- cutout/service/policy.py - -Validacao executada: -- docker compose exec django black --check cutout/service/policies cutout/service/policy.py -- docker compose exec django isort --check-only cutout/service/policies cutout/service/policy.py -- docker compose exec django pytest cutout/service/policies/tests/test_des_public_policy.py cutout/service/discovery/tests/test_des_csv_locator.py -q -- Resultado: 7 passed - -Smoke test de autorizacao: -- Requisicao com id=private_survey em /api/sync -- Resultado observado: 403 com mensagem de acesso negado - -Commit: -- b332e38 - feat(policy): add survey access layer before discovery dispatch - -## Proxima fase planejada (repriorizada) - -Fase P1/P2 - PNG + RANGE/POLYGON no engine default - -Objetivos imediatos: -1. Implementar `format=png` no fluxo sync (mono e RGB). -2. Introduzir parametro explicito para PNG colorido (`color`) e bandas RGB (`rgb_bands`, default `gri`). -3. Fechar execucao ponta a ponta de RANGE/POLYGON com `engine=astrocut`. -4. Validar comportamento com testes e curls de regressao para CIRCLE/RANGE/POLYGON em FITS e PNG. - -### Entrada 2026-04-26 - Preparacao da Fase 3 - -Resumo: -- Criado esqueleto do modulo `cutout_engine` para iniciar a separacao entre orquestracao e motor de recorte. -- Adicionada interface `CutoutEngine` e implementacoes iniciais para DES e Astrocut (stub). -- Incluido teste unitario basico para validar delegacao do adapter DES para a classe de cutout existente. - -Arquivos criados: -- cutout/service/cutout_engine/__init__.py -- cutout/service/cutout_engine/base.py -- cutout/service/cutout_engine/des_engine.py -- cutout/service/cutout_engine/astrocut_engine.py -- cutout/service/cutout_engine/tests/test_des_engine.py - -Status: -- Preparacao inicial concluida. -- Integracao do adapter no fluxo principal sera feita na execucao completa da Fase 3. - -### Entrada 2026-04-26 - Fase 2 - -Resumo: -- Endpoint sync passou a aguardar o resultado real da task Celery no fluxo sincrono. -- JobService passou a registrar transicoes de fase EXECUTING, COMPLETED e ERROR. -- Task de cutout passou a validar explicitamente a disponibilidade dos arquivos de entrada. -- Em caso de arquivo de entrada ausente/inacessivel, a API retorna erro explicito (422) com detalhe dos caminhos faltantes. -- Endpoint passou a retornar arquivo via FileResponse quando o resultado existe, com timeout configurado e erro 503 para indisponibilidade do servico. - -Arquivos criados: -- cutout/service/tests/test_tasks.py - -Arquivos alterados: -- cutout/service/api/views.py -- cutout/service/policy.py -- cutout/service/tasks.py -- cutout/service/uws/service.py -- cutout/service/uws/exceptions.py -- cutout/lib/cutout.py -- .gitignore - -Validacao executada: -- docker compose exec django black --check cutout/service/api/views.py cutout/service/policy.py cutout/service/tasks.py cutout/service/uws/service.py cutout/service/uws/exceptions.py cutout/lib/cutout.py cutout/service/tests/test_tasks.py -- docker compose exec django isort --check-only cutout/service/api/views.py cutout/service/policy.py cutout/service/tasks.py cutout/service/uws/service.py cutout/service/uws/exceptions.py cutout/lib/cutout.py cutout/service/tests/test_tasks.py -- docker compose exec django pytest cutout/service/tests/test_tasks.py cutout/service/policies/tests/test_des_public_policy.py cutout/service/discovery/tests/test_des_csv_locator.py -q -- Resultado: 10 passed - -Smoke test relevante: -- Requisicao em /api/sync com des_dr2 e coordenada valida no catalogo, mas sem arquivo FITS correspondente local. -- Resultado observado: 422 com mensagem Input file unavailable e lista de arquivos ausentes. - -Status: -- Fase 2 finalizada no codigo. -- Comportamento de erro para indisponibilidade de arquivo local validado conforme requisito de ambiente de testes parcial. - -### Entrada 2026-04-26 - Fase 3 - -Resumo: -- Integrada camada de selecao de engine no pipeline de cutout. -- Implementado factory para escolha de backend por nome de engine. -- Mantida ferramenta antiga funcional como opcao (`legacy`). -- Definido `astrocut` como default da API. -- Nesta fase, `astrocut` usa fallback controlado para engine legado para manter operacao funcional durante aprovacao. - -Arquivos criados: -- cutout/service/cutout_engine/factory.py -- cutout/service/cutout_engine/tests/test_factory.py -- cutout/service/cutout_engine/tests/test_astrocut_engine.py -- cutout/service/tests/test_cutout_parameters.py - -Arquivos alterados: -- cutout/service/api/views.py -- cutout/service/cutout_engine/__init__.py -- cutout/service/cutout_engine/astrocut_engine.py -- cutout/service/cutout_parameters.py -- cutout/service/policy.py -- cutout/service/tasks.py -- test_sync_endpoint.py -- CURL_TESTES_SYNC.md - -Validacao executada: -- docker compose exec django black --check cutout/service/api/views.py cutout/service/cutout_parameters.py cutout/service/policy.py cutout/service/tasks.py cutout/service/cutout_engine test_sync_endpoint.py -- docker compose exec django isort --check-only cutout/service/api/views.py cutout/service/cutout_parameters.py cutout/service/policy.py cutout/service/tasks.py cutout/service/cutout_engine test_sync_endpoint.py -- docker compose exec django pytest cutout/service/cutout_engine/tests/test_des_engine.py cutout/service/cutout_engine/tests/test_factory.py cutout/service/cutout_engine/tests/test_astrocut_engine.py cutout/service/tests/test_tasks.py cutout/service/tests/test_cutout_parameters.py -q -- Resultado: 10 passed - -Smoke test relevante: -- `engine=legacy` em `/api/sync`: status 200 (streaming) -- `engine=astrocut` em `/api/sync`: status 200 (streaming) - -Status: -- Fase 3 finalizada para aprovacao funcional com duas opcoes de ferramenta. -- Integracao nativa do astrocut permanece como evolucao da fase seguinte. - -### Entrada 2026-04-26 - Checkpoint de compatibilidade de dependencias - -Resumo: -- Validada compatibilidade do ambiente com novas versoes cientificas. -- Confirmado funcionamento do modo legado DES apos upgrade. - -Dependencias em runtime (container django): -- numpy 2.4.4 -- astropy 7.2.0 -- astrocut 1.2.0 - -Validacao executada: -- docker compose exec django pytest cutout/service/tests/test_tasks.py cutout/service/discovery/tests/test_des_csv_locator.py cutout/service/cutout_engine/tests/test_des_engine.py -q -- docker compose exec django python manage.py shell -c "... engine=legacy ..." -- Resultado: testes verdes e endpoint sync legacy retornando 200 (streaming, application/fits) - -### Entrada 2026-04-26 - Fase 3.1 (Astrocut nativo com discovery DES) - -Resumo: -- Removido fallback `astrocut -> legacy` no engine Astrocut. -- Integrada chamada nativa ao `astrocut.fits_cut` para pedidos `CIRCLE` em `fits`. -- Mantida descoberta de arquivos no locator DES/CSV atual (sem SIA nesta fase). -- Task passou a repassar a lista de arquivos de entrada para o engine selecionado. - -Arquivos alterados: -- cutout/service/cutout_engine/base.py -- cutout/service/cutout_engine/des_engine.py -- cutout/service/cutout_engine/astrocut_engine.py -- cutout/service/tasks.py -- cutout/service/cutout_engine/tests/test_des_engine.py -- cutout/service/cutout_engine/tests/test_astrocut_engine.py - -Validacao executada: -- docker compose exec django isort --check-only cutout/service/cutout_engine/astrocut_engine.py cutout/service/cutout_engine/tests/test_astrocut_engine.py cutout/service/cutout_engine/tests/test_des_engine.py cutout/service/tasks.py cutout/service/cutout_engine/base.py cutout/service/cutout_engine/des_engine.py -- docker compose exec django black --check cutout/service/cutout_engine/astrocut_engine.py cutout/service/cutout_engine/tests/test_astrocut_engine.py cutout/service/cutout_engine/tests/test_des_engine.py cutout/service/tasks.py cutout/service/cutout_engine/base.py cutout/service/cutout_engine/des_engine.py -- docker compose exec django pytest cutout/service/cutout_engine/tests/test_des_engine.py cutout/service/cutout_engine/tests/test_astrocut_engine.py cutout/service/cutout_engine/tests/test_factory.py cutout/service/tests/test_tasks.py cutout/service/tests/test_cutout_parameters.py cutout/service/discovery/tests/test_des_csv_locator.py -q -- docker compose exec django python manage.py shell -c "... engine=legacy e engine=astrocut ..." -- Resultado: 17 passed; `engine=legacy` e `engine=astrocut` com status 200 e streaming FITS. - -Status: -- Fase 3.1 concluida. -- Proxima etapa permanece focada em estabilizacao/expansao do engine astrocut sem alterar discovery DES. - -### Entrada 2026-04-26 - Replanejamento orientado a prioridade - -Resumo: -- Plano e status foram atualizados para ordem de execucao por prioridade de negocio (1 a 6). -- Prioridades 1 e 2 tornaram-se foco imediato de implementacao. -- Priorizacao de semantica de erro foi formalmente adiada para depois da fase async. - -Nota de compatibilidade IVOA para prioridade 1: -- Nao foi identificado parametro padrao em SODA 1.0 para requisitar "PNG colorido/RGB". -- O padrao permite parametros customizados; portanto o comportamento sera documentado como extensao local. - -Arquivos alterados nesta entrada: -- PLANO_SYNC_IVOA_DES_DESCOBERTA.md -- STATUS_IMPLEMENTACAO_SYNC.md - -Status: -- Replanejamento aplicado e pronto para execucao incremental. - -### Entrada 2026-04-26 - Bug report e plano de depuracao - -Resumo: -- Foram detectados artefatos gerados incorretos: FITS que nao abre e PNG aparentemente vazio. Também foi observado que o caminho legado (`legacy`) nao gerou arquivos como antes. - -Acao imediata: -1. Criar script de depuracao `scripts/debug_cutout.py` que reproduz as chamadas existentes em `des_cutout.py` e grava artefatos em `/data/results/debug/`. -2. Executar steps ordenados: FITS mono (legacy), FITS mono (engine astrocut), PNG mono, PNG RGB. -3. Coletar logs e propriedades de cada arquivo (tamanho, shape, min/max, nan count) e anexar ao ticket/issue. - -Arquivos criados/alterados: -- scripts/debug_cutout.py (novo) - -Validacao executada: -- executar `docker compose exec django python scripts/debug_cutout.py` dentro do container e inspecionar `/data/results/debug/`. - -Responsavel: equipe de desenvolvimento (agent). - -Proximo passo: -- executar o script dentro do container, analisar saídas e corrigir a função que estiver produzindo artefatos inválidos. - -### Entrada 2026-04-26 - Correcao de nome de saida no sync PNG (astrocut) - -Resumo: -- Corrigido o nome de arquivo de saida no dispatch do policy, removendo hardcode `teste.fits`. -- Fluxo sync agora gera path dinamico em `/data/results` com extensao aderente ao parametro `format` (`.png` ou `.fits`). -- Arquivos intermediarios do Astrocut passaram a usar nomes unicos por execucao para evitar colisao em concorrencia. -- Resultado observado em teste manual: fluxo sync com `engine=astrocut` e `format=png` voltou a gerar artefato no formato esperado. - -Arquivos alterados: -- cutout/service/policy.py -- cutout/service/cutout_engine/astrocut_engine.py - -Regras/decisoes desta entrada: -1. Diretorio de saida oficial permanece `/data/results`. -2. Nome final do resultado deve ser deterministico por job e seguro para filesystem. -3. Arquivos temporarios do pipeline PNG devem ser unicos por execucao (evitar overwrite entre jobs). - -Validacao executada: -- docker compose exec django pytest cutout/service/cutout_engine/tests/test_factory.py -q -- docker compose exec django pytest cutout/service/policies/tests/test_des_public_policy.py -q -- docker compose exec django python -m py_compile cutout/service/policy.py cutout/service/cutout_engine/astrocut_engine.py -- Resultado: 5 passed; compilacao sem erros. - -Pendencia conhecida: -- Fluxo sync `engine=legacy` com `format=png` continua com problema e requer depuracao dedicada no motor legado. - -Proximos passos planejados: -1. Reproduzir `legacy+png` com caso minimo (CIRCLE pequeno) e capturar traceback completo em worker + django. -2. Revisar caminho de conversao/serializacao PNG no engine legado e validar extensao/mimetype de saida. -3. Adicionar teste automatizado de regressao para sync `legacy+png` (esperado 200 + arquivo PNG valido). - -Commit associado: -- Este commit (correcao do naming dinamico no sync e plano para pendencia legacy PNG). diff --git a/config/api_router.py b/config/api_router.py index bc6059c..f37a04f 100644 --- a/config/api_router.py +++ b/config/api_router.py @@ -28,7 +28,7 @@ urlpatterns += [ path("cutout", CutoutView.as_view(), name="cutout"), - path("sync", SyncCutoutView.as_view(), name="sync_cutout"), + path("sync", transaction.non_atomic_requests(SyncCutoutView.as_view()), name="sync_cutout"), path("async", transaction.non_atomic_requests(AsyncCutoutView.as_view()), name="async_cutout"), path("async/", AsyncJobDetailView.as_view(), name="async_job_detail"), path("async//phase", AsyncJobPhaseView.as_view(), name="async_job_phase"), diff --git a/config/urls.py b/config/urls.py index 33655bf..96b81f0 100644 --- a/config/urls.py +++ b/config/urls.py @@ -57,7 +57,3 @@ ), path("500/", default_views.server_error), ] - if "debug_toolbar" in settings.INSTALLED_APPS: - import debug_toolbar - - urlpatterns = [path("__debug__/", include(debug_toolbar.urls))] + urlpatterns diff --git a/cutout/service/api/views.py b/cutout/service/api/views.py index 535aba9..31ffb0e 100644 --- a/cutout/service/api/views.py +++ b/cutout/service/api/views.py @@ -1,17 +1,17 @@ import logging from collections.abc import Iterable from pathlib import Path -from typing import List, Optional -from celery.exceptions import TimeoutError as CeleryTimeoutError from django.http import FileResponse, HttpResponse from django.urls import reverse from django.utils.encoding import escape_uri_path from drf_spectacular.utils import OpenApiParameter, extend_schema, extend_schema_view from rest_framework import status +from rest_framework.exceptions import APIException from rest_framework.response import Response from rest_framework.views import APIView +from cutout.service.cutout_runner import perform_cutout from cutout.service.models import Job from cutout.service.uws.exceptions import ParameterError, ServiceUnavailableError from cutout.service.uws.models import JobParameter @@ -155,7 +155,7 @@ def get(self, request, format=None): allow_blank=False, many=False, default="astrocut", - description=("Cutout backend engine. Supported values: astrocut, legacy"), + description=("Cutout backend engine. Supported values: astrocut"), ), ], ) @@ -163,54 +163,48 @@ def get(self, request, format=None): @extend_schema_view(get=cutout_schema, post=cutout_schema) class SyncCutoutView(APIView): - sync_timeout_seconds = 25 - def _mimetype_for_format(self, output_format: str) -> str: if output_format.lower() == "png": return "image/x-png" return "application/fits" def sync_cutout(self, user: User, params: list[JobParameter], run_id: str | None): - print("Entrou aqui") + """Run a cutout synchronously inside the request. + + Database flow is identical to the async pipeline (Job + Task rows, + status transitions and JobResult recorded by `perform_cutout`), but + execution happens inline and the result file is returned directly. + """ + logger = logging.getLogger("cutout") job_service = JobService() - job = job_service.create(user=user, params=params, run_id=run_id) - print("step 0") - async_result = job_service.start(user, job_id=job.id) - print("step 1") - output_format = "fits" - for p in params: - if p.parameter_id == "format": - output_format = p.value - break - try: - print("step 2") - job_service.mark_executing(job.id) - print("step 3") - result_path = async_result.get(timeout=self.sync_timeout_seconds) - except CeleryTimeoutError as exc: - print("step 4") + job = job_service.create(user=user, params=params, run_id=run_id, execution_mode="sync") + logger.info("[SyncCutoutView] created job_id=%s", job.id) + + tasks = list(job.tasks.order_by("sequence")) + if len(tasks) != 1: job_service.mark_error(job.id) - print("step 5") - raise ServiceUnavailableError("Sync cutout timed out") from exc + raise ParameterError("Only one cutout task is supported in sync mode") + task = tasks[0] + + try: + result = perform_cutout(job.id, task.id) + except APIException: + # Task and Job are already marked ERROR by perform_cutout + raise except Exception as exc: - print("step 6") - job_service.mark_error(job.id) - print("step 7") raise ParameterError(str(exc)) from exc - print("step 8") - result_file = Path(result_path) + result_file = Path(result["file_path"]) if not result_file.exists(): - print("step 9") job_service.mark_error(job.id) raise ServiceUnavailableError("Result file unavailable") - print("step 10") - job_service.mark_completed(job.id) + logger.info("[SyncCutoutView] job_id=%s completed result_id=%s", job.id, result["result_id"]) + fp = open(result_file, "rb") - response = FileResponse(fp, content_type=self._mimetype_for_format(output_format), as_attachment=True) + response = FileResponse(fp, content_type=self._mimetype_for_format(task.output_format), as_attachment=True) response["Content-Length"] = result_file.stat().st_size response["Content-Disposition"] = f"attachment; filename={escape_uri_path(result_file.name)}" return response @@ -224,14 +218,22 @@ def get(self, request, format=None): class AsyncCutoutView(APIView): def get(self, request, format=None): jobs = JobService().list_for_user(request.user) - serializer = AsyncJobSummarySerializer(jobs, many=True, context={"request": request}) + serializer = AsyncJobSummarySerializer( + jobs, + many=True, + context={"request": request}, + ) return Response({"jobs": serializer.data}) def post(self, request, format=None): logger = logging.getLogger("cutout") logger.info(f"[AsyncCutoutView.post] called with data={request.data}") - params, run_id, requested_phase = _extract_job_request(request.data or request.query_params, is_post=True) - logger.info(f"[AsyncCutoutView.post] params={params} run_id={run_id} requested_phase={requested_phase}") + + params, run_id, requested_phase = _extract_job_request( + request.data or request.query_params, + is_post=True, + ) + logger.info(f"[AsyncCutoutView.post] params={params} run_id={run_id} " f"requested_phase={requested_phase}") if not params: logger.error("[AsyncCutoutView.post] No params provided") raise ParameterError("At least one cutout parameter is required") @@ -242,8 +244,13 @@ def post(self, request, format=None): raise ParameterError("Only PHASE=RUN is supported when creating async jobs") job_service = JobService() - job = job_service.create(user=request.user, params=params, run_id=run_id) + job = job_service.create( + user=request.user, + params=params, + run_id=run_id, + ) logger.info(f"[AsyncCutoutView.post] Created job id={job.id}") + job_service.start_async(request.user, job.id) logger.info(f"[AsyncCutoutView.post] Dispatched start_async for job id={job.id}") diff --git a/cutout/service/cutout_engine/__init__.py b/cutout/service/cutout_engine/__init__.py index 34c524e..90576e2 100644 --- a/cutout/service/cutout_engine/__init__.py +++ b/cutout/service/cutout_engine/__init__.py @@ -1,6 +1,5 @@ from .astrocut_engine import AstrocutEngine from .base import CutoutEngine -from .des_engine import DesCutoutEngine from .factory import create_cutout_engine -__all__ = ["CutoutEngine", "DesCutoutEngine", "AstrocutEngine", "create_cutout_engine"] +__all__ = ["CutoutEngine", "AstrocutEngine", "create_cutout_engine"] diff --git a/cutout/service/cutout_engine/des_engine.py b/cutout/service/cutout_engine/des_engine.py deleted file mode 100644 index 2127845..0000000 --- a/cutout/service/cutout_engine/des_engine.py +++ /dev/null @@ -1,27 +0,0 @@ -from __future__ import annotations - -from pathlib import Path -from typing import Any - -from cutout.lib.cutout import Cutout - -from .base import CutoutEngine - - -class DesCutoutEngine(CutoutEngine): - def run_cutout( - self, - *, - source_id: str, - stencil: dict[str, Any], - input_files: list[str] | dict[str, list[str]] | None, - band: str, - output_format: str, - output_path: str | Path, - color: bool = False, - rgb_bands: str | None = None, - persist: bool = False, - ) -> Path: - # Legacy engine: ignore `input_files` mapping (discovery is internal to DesCutout) - cutout = Cutout(source_id=source_id, stencil=stencil, band=band, format=output_format) - return cutout.create(output_path) diff --git a/cutout/service/cutout_engine/factory.py b/cutout/service/cutout_engine/factory.py index ff94c6b..4021f9a 100644 --- a/cutout/service/cutout_engine/factory.py +++ b/cutout/service/cutout_engine/factory.py @@ -2,7 +2,6 @@ from .astrocut_engine import AstrocutEngine from .base import CutoutEngine -from .des_engine import DesCutoutEngine def create_cutout_engine(engine_name: str) -> CutoutEngine: @@ -10,7 +9,5 @@ def create_cutout_engine(engine_name: str) -> CutoutEngine: if name == "astrocut": return AstrocutEngine() - if name in ("legacy", "des"): - return DesCutoutEngine() raise ValueError(f"Unsupported cutout engine: {engine_name}") diff --git a/cutout/service/cutout_engine/tests/test_des_engine.py b/cutout/service/cutout_engine/tests/test_des_engine.py deleted file mode 100644 index 6feea40..0000000 --- a/cutout/service/cutout_engine/tests/test_des_engine.py +++ /dev/null @@ -1,32 +0,0 @@ -from pathlib import Path - -from cutout.service.cutout_engine.des_engine import DesCutoutEngine - - -class DummyCutout: - def __init__(self, source_id, stencil, band, format): - self.source_id = source_id - self.stencil = stencil - self.band = band - self.format = format - - def create(self, output_path): - return Path(output_path) - - -def test_des_cutout_engine_delegates_to_cutout(monkeypatch): - import cutout.service.cutout_engine.des_engine as des_engine_module - - monkeypatch.setattr(des_engine_module, "Cutout", DummyCutout) - - engine = DesCutoutEngine() - result = engine.run_cutout( - source_id="des_dr2", - stencil={"type": "circle", "center": {"ra": 10.0, "dec": -1.0}, "radius": 1.0}, - input_files=["/tmp/source.fits.fz"], - band="g", - output_format="fits", - output_path="/tmp/out.fits", - ) - - assert result == Path("/tmp/out.fits") diff --git a/cutout/service/cutout_engine/tests/test_factory.py b/cutout/service/cutout_engine/tests/test_factory.py index 0cdea8e..bd869a1 100644 --- a/cutout/service/cutout_engine/tests/test_factory.py +++ b/cutout/service/cutout_engine/tests/test_factory.py @@ -1,6 +1,6 @@ import pytest -from cutout.service.cutout_engine import AstrocutEngine, DesCutoutEngine, create_cutout_engine +from cutout.service.cutout_engine import AstrocutEngine, create_cutout_engine def test_factory_returns_astrocut_engine() -> None: @@ -8,9 +8,9 @@ def test_factory_returns_astrocut_engine() -> None: assert isinstance(engine, AstrocutEngine) -def test_factory_returns_legacy_engine() -> None: - engine = create_cutout_engine("legacy") - assert isinstance(engine, DesCutoutEngine) +def test_factory_rejects_legacy_engine() -> None: + with pytest.raises(ValueError, match="Unsupported cutout engine"): + create_cutout_engine("legacy") def test_factory_rejects_unknown_engine() -> None: diff --git a/cutout/service/policy.py b/cutout/service/policy.py index b803a75..e158f8c 100644 --- a/cutout/service/policy.py +++ b/cutout/service/policy.py @@ -8,15 +8,13 @@ import re from datetime import datetime from pathlib import Path -from typing import List from celery import chord as celery_chord from cutout.service.cutout_parameters import CutoutParameters -from cutout.service.discovery import DesCsvFileLocator from cutout.service.models import Task as SQLTask from cutout.service.policies import DesPublicAccessPolicy -from cutout.service.tasks import finalize_job, image_cutout, perform_cutout_task +from cutout.service.tasks import finalize_job, perform_cutout_task from cutout.service.uws.exceptions import MultiValuedParameterError, ParameterError, PermissionDeniedError from cutout.service.uws.models import Job, JobParameter from cutout.service.uws.policy import UWSPolicy @@ -49,7 +47,6 @@ class ImageCutoutPolicy(UWSPolicy): # self._logger = logger def __init__(self) -> None: self._survey_access_policy = DesPublicAccessPolicy() - self._file_locator = DesCsvFileLocator() def _safe_token(self, value: str) -> str: """Normalize token for filesystem-safe filenames.""" @@ -67,133 +64,12 @@ def _build_result_path(self, job: Job, task_params: dict) -> Path: return Path("/data/results").joinpath(filename) - def _build_async_result_path(self, job: Job, task_params: dict, sequence: int) -> Path: + def _build_task_result_path(self, job: Job, task_params: dict, sequence: int, execution_mode: str) -> Path: base_path = self._build_result_path(job, task_params) filename = f"{base_path.stem}_{sequence}{base_path.suffix or '.fits'}" - return Path("/data/results/async").joinpath(filename) + return Path("/data/results").joinpath(execution_mode, filename) - def dispatch(self, job: Job): - """Dispatch a cutout request to the backend. - - Parameters - ---------- - job - The submitted job description. - - Returns - ------- - dramatiq.Message - The dispatched message to the backend. - - Notes - ----- - Currently, only one dataset ID and only one stencil are supported. - This limitation is expected to be relaxed in a later version. - """ - print(f"[dispatch] job_id={job.job_id} dispatching to backend") - cutout_params = CutoutParameters.from_job_parameters(job.parameters) - print(f"[dispatch] cutout_params: {cutout_params}") - - print(f"[dispatch] calling convert_to_list_of_task_params with cutout_params: {cutout_params}") - tasks_params = self.convert_to_list_of_task_params(cutout_params) - print(f"[dispatch] tasks_params: {tasks_params}") - - # Celery tasks signature - tasks = [] - - for t in tasks_params: - print(f"[dispatch] checking survey access for user_id={job.owner} survey_id={t['id']}") - if not self._survey_access_policy.can_request_cutout(user_id=job.owner, survey_id=t["id"]): - raise PermissionDeniedError(f"User has no access to survey {t['id']}") - - print(f"[dispatch] building result path for job_id={job.job_id} task_params={t}") - resultfile = self._build_result_path(job, t) - print(f"[dispatch] resultfile path: {resultfile}") - - # If color composition requested, collect files per RGB band - if t.get("color"): - print(f"[dispatch] color composition requested, parsing rgb_bands for task_params={t}") - - # parse rgb_bands: accept 'gri', 'g,r,i' or 'g r i' - raw = t.get("rgb_bands", "gri") - if "," in raw: - bands = [b.strip() for b in raw.split(",") if b.strip()] - elif " " in raw: - bands = [b.strip() for b in raw.split() if b.strip()] - else: - bands = list(raw) - - files_map = {} - for b in bands: - files_b = self._file_locator.find_files(survey_id=t["id"], stencil=t["stencil_obj"], band=b) - if not files_b: - raise ParameterError(f"No files found for band {b} in the requested region") - # keep only paths that exist on the current filesystem - candidate_paths = [str(f.file_path) for f in files_b if f.file_path] - existing = [p for p in candidate_paths if Path(p).exists()] - if not existing: - raise ParameterError(f"No available files on disk for band {b} in the requested region") - files_map[b] = existing - - # Debug logging: show files_map and existence - print(f"[policy] dispatch: files_map for bands={bands}: {files_map}") - for band_name, paths in files_map.items(): - for p in paths: - print(f"[policy] file check: band={band_name} path={p} exists=True") - - tasks.append( - image_cutout.s( - job_id=job.job_id, - source_id=t["id"], - stencil=t["stencil"], - files=files_map, - engine=t["engine"], - band=t["band"], - format=t["format"], - path=str(resultfile), - color=t.get("color", False), - rgb_bands=t.get("rgb_bands"), - persist=t.get("persist", False), - ) - ) - else: - print( - f"[dispatch] single band requested, finding files for survey_id={t['id']} stencil={t['stencil_obj']} band={t['band']}" - ) - - files = self._file_locator.find_files(survey_id=t["id"], stencil=t["stencil_obj"], band=t["band"]) - print( - f"[policy] dispatch: found {len(files)} files for survey_id={t['id']} stencil={t['stencil_obj']} band={t['band']}" - ) - - if not files: - raise ParameterError("No files found for the requested region") - candidate = [str(f.file_path) for f in files if f.file_path] - existing = [p for p in candidate if Path(p).exists()] - if not existing: - raise ParameterError("No available files on disk for the requested region") - tasks.append( - image_cutout.s( - job_id=job.job_id, - source_id=t["id"], - stencil=t["stencil"], - files=existing, - engine=t["engine"], - band=t["band"], - format=t["format"], - path=str(resultfile), - color=False, - rgb_bands=t.get("rgb_bands"), - persist=t.get("persist", False), - ) - ) - - if len(tasks) == 1: - return tasks[0].apply_async() - - raise ParameterError("Only one cutout task is supported in sync mode") - - def create_tasks_for_job(self, job: Job, params: list[JobParameter]) -> list: + def create_tasks_for_job(self, job: Job, params: list[JobParameter], execution_mode: str = "async") -> list: """Create one Task row per cutout execution unit (stencil × band × format × engine).""" cutout_params = CutoutParameters.from_job_parameters(params) task_dicts = self.convert_to_list_of_task_params(cutout_params) @@ -201,7 +77,7 @@ def create_tasks_for_job(self, job: Job, params: list[JobParameter]) -> list: for sequence, t in enumerate(task_dicts, start=1): if not self._survey_access_policy.can_request_cutout(user_id=job.owner, survey_id=t["id"]): raise PermissionDeniedError(f"User has no access to survey {t['id']}") - output_path = str(self._build_async_result_path(job, t, sequence)) + output_path = str(self._build_task_result_path(job, t, sequence, execution_mode)) stencil_obj = t["stencil_obj"] stencil_dict = stencil_obj.to_dict() task = SQLTask.objects.create( diff --git a/cutout/service/tasks.py b/cutout/service/tasks.py index 684dd9b..54c6b2e 100644 --- a/cutout/service/tasks.py +++ b/cutout/service/tasks.py @@ -1,125 +1,13 @@ import logging -from pathlib import Path from typing import Any from django.utils import timezone from config import celery_app -from cutout.lib.des_cutout import DesCutout -from cutout.service.cutout_engine import create_cutout_engine from cutout.service.cutout_runner import perform_cutout from cutout.service.models import Job, Task -def _validate_input_files(files: list[str] | dict[str, list[str]] | None) -> None: - if not files: - return - - # normalize to list of paths for existence check - paths: list[str] = [] - if isinstance(files, dict): - for v in files.values(): - paths.extend(v or []) - else: - paths = list(files) - - missing = [f for f in paths if not Path(f).exists()] - if missing: - msg = "Input file unavailable: " + ", ".join(missing) - raise FileNotFoundError(msg) - - -def _ensure_unpacked( - files: list[str] | dict[str, list[str]] | None, -) -> list[str] | dict[str, list[str]] | None: - """If any input paths point to compressed ``.fz`` archives, unpack them to a tmp location and - return a structure of uncompressed paths suitable for engines that require ``.fits`` files. - - .. note:: - - ``fits_cut`` from astrocut handles ``.fz`` natively via ``.section``, so this helper - is currently **not used** by ``image_cutout``. It is kept as a legacy utility for - engines or ad-hoc scripts that need uncompressed files on disk. - """ - if not files: - return files - - dc = DesCutout() - - def _unpack_path(p: str) -> str: - pth = Path(p) - if pth.suffix == ".fz": - out_name = pth.name.rsplit(".fz", 1)[0] - out_path = dc.tmp_path.joinpath(out_name) - if not out_path.exists(): - try: - dc.funpack(pth, out_path) - except Exception: - # let downstream code fail with clearer message if unpack fails - pass - return str(out_path) - return str(p) - - if isinstance(files, dict): - out: dict[str, list[str]] = {} - for k, lst in files.items(): - out[k] = [_unpack_path(p) for p in (lst or [])] - print(f"[tasks] _ensure_unpacked: band={k} unpacked_paths={out[k]}") - return out - - return [_unpack_path(p) for p in files] - - -@celery_app.task() -def des_cutout_circle(**kwargs) -> str: - dc = DesCutout() - result = dc.cutout_circle(**kwargs) - return str(result) - - -@celery_app.task() -def image_cutout( - job_id: str, - source_id: str, - stencil: dict[str, Any], - engine: str, - band: str, - format: str, - path: str, - files: list[str] | None = None, - color: bool = False, - rgb_bands: str | None = None, - persist: bool = False, -) -> str: - print( - "[tasks] image_cutout START " - f"job_id={job_id} engine={engine} band={band} format={format} " - f"color={color} rgb_bands={rgb_bands}" - ) - print(f"[tasks] image_cutout initial files={files}") - cutout_engine = create_cutout_engine(engine) - _validate_input_files(files) - - try: - print(f"[tasks] calling engine.run_cutout engine={engine} path={path}") - result = cutout_engine.run_cutout( - source_id=source_id, - stencil=stencil, - input_files=files, - band=band, - output_format=format, - output_path=path, - color=color, - rgb_bands=rgb_bands, - persist=persist, - ) - print(f"[tasks] engine.run_cutout completed result={result}") - except Exception as e: - print(f"[tasks] engine.run_cutout raised: {type(e).__name__}: {e}") - raise - return str(result) - - @celery_app.task( bind=True, autoretry_for=(Job.DoesNotExist, Task.DoesNotExist), @@ -166,18 +54,11 @@ def finalize_job(_results: list[dict[str, Any]], job_id: str) -> None: logger.info("[finalize_job] job_id=%r marked COMPLETED", job_id) -# @celery_app.task() -# def on_success_cutout(job_id: str, ) -> str: -# return str(result) - - @celery_app.task(bind=True) -# def job_completed(job_id: str, results) -> str: def job_completed(result, **kwargs) -> str: print(result) print(kwargs) return f"TESTE: {result}" - # uws_job_completed(job_id=job_id, results=results) @celery_app.task() diff --git a/cutout/service/tests/test_async_api.py b/cutout/service/tests/test_async_api.py index b15c1a9..4f7aba2 100644 --- a/cutout/service/tests/test_async_api.py +++ b/cutout/service/tests/test_async_api.py @@ -15,11 +15,11 @@ def _patch_async_result_path(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - def _fake_path(self, job, task_params, sequence): + def _fake_path(self, job, task_params, sequence, execution_mode): extension = "png" if str(task_params.get("format", "fits")).lower() == "png" else "fits" - return tmp_path / f"job_{job.job_id}_{sequence}.{extension}" + return tmp_path / execution_mode / f"job_{job.job_id}_{sequence}.{extension}" - monkeypatch.setattr(ImageCutoutPolicy, "_build_async_result_path", _fake_path) + monkeypatch.setattr(ImageCutoutPolicy, "_build_task_result_path", _fake_path) class _FakeLocator: diff --git a/cutout/service/tests/test_sync_api.py b/cutout/service/tests/test_sync_api.py new file mode 100644 index 0000000..b452c1e --- /dev/null +++ b/cutout/service/tests/test_sync_api.py @@ -0,0 +1,137 @@ +from pathlib import Path + +import pytest +from django.urls import reverse +from rest_framework.test import APIClient + +from cutout.service import cutout_runner +from cutout.service.discovery import FileDescriptor +from cutout.service.models import Job, Task +from cutout.service.policy import ImageCutoutPolicy + +pytestmark = pytest.mark.django_db + +FAKE_PAYLOAD = b"fake fits data" + + +def _patch_async_result_path(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + def _fake_path(self, job, task_params, sequence, execution_mode): + extension = "png" if str(task_params.get("format", "fits")).lower() == "png" else "fits" + return tmp_path / execution_mode / f"job_{job.job_id}_{sequence}.{extension}" + + monkeypatch.setattr(ImageCutoutPolicy, "_build_task_result_path", _fake_path) + + +class _FakeLocator: + def __init__(self, input_file: Path | None): + self._input_file = input_file + + def find_files(self, *, survey_id, stencil, band=None): + if self._input_file is None: + return [] + return [ + FileDescriptor( + tile_id="DES0002+0001", + archive_path="Y6A1/r4907/DES0002+0001/p01/coadd", + file_path=self._input_file, + band=band, + ) + ] + + +class _FakeEngine: + def run_cutout(self, **kwargs): + output_path = Path(kwargs["output_path"]) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_bytes(FAKE_PAYLOAD) + return output_path + + +def _patch_cutout_execution(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, with_files: bool = True) -> None: + input_file = None + if with_files: + input_file = tmp_path / "DES0002+0001_r4907p01_g.fits.fz" + input_file.write_bytes(b"tile data") + monkeypatch.setattr(cutout_runner, "DesCsvFileLocator", lambda: _FakeLocator(input_file)) + monkeypatch.setattr(cutout_runner, "create_cutout_engine", lambda name: _FakeEngine()) + + +def _client(user) -> APIClient: + client = APIClient() + client.force_authenticate(user=user) + return client + + +def test_sync_get_runs_cutout_and_returns_file(user, monkeypatch, tmp_path): + _patch_async_result_path(monkeypatch, tmp_path) + _patch_cutout_execution(monkeypatch, tmp_path) + + response = _client(user).get( + reverse("api:sync_cutout"), + {"id": "des_dr2", "pos": "CIRCLE 10 0 1", "band": "g", "format": "fits"}, + ) + + assert response.status_code == 200 + assert response["Content-Disposition"].startswith("attachment;") + assert response["Content-Type"] == "application/fits" + assert int(response["Content-Length"]) == len(FAKE_PAYLOAD) + + job = Job.objects.get() + assert job.phase == Job.ExecutionPhase.COMPLETED + assert job.start_time is not None + assert job.end_time is not None + + task = job.tasks.get() + assert task.status == Task.Status.COMPLETED + assert task.start_time is not None + assert task.end_time is not None + + job_result = job.results.get() + assert job_result.size == len(FAKE_PAYLOAD) + assert job_result.mime_type == "application/fits" + assert Path(job_result.file_path).exists() + assert "/sync/" in job_result.file_path + + +# transaction=True: on error responses DRF's exception handler calls set_rollback, which would +# poison the test's wrapping atomic block (production uses non_atomic_requests on this route). +@pytest.mark.django_db(transaction=True) +def test_sync_get_no_files_marks_error_and_returns_422(user, monkeypatch, tmp_path): + _patch_async_result_path(monkeypatch, tmp_path) + _patch_cutout_execution(monkeypatch, tmp_path, with_files=False) + + response = _client(user).get( + reverse("api:sync_cutout"), + {"id": "des_dr2", "pos": "CIRCLE 10 0 1", "band": "g", "format": "fits"}, + ) + + assert response.status_code == 422 + assert "No files found" in response.json()["detail"] + + job = Job.objects.get() + assert job.phase == Job.ExecutionPhase.ERROR + assert job.end_time is not None + + task = job.tasks.get() + assert task.status == Task.Status.ERROR + assert "No files found" in task.error_message + assert job.results.count() == 0 + + +@pytest.mark.django_db(transaction=True) +def test_sync_get_rejects_multiple_tasks(user, monkeypatch, tmp_path): + _patch_async_result_path(monkeypatch, tmp_path) + _patch_cutout_execution(monkeypatch, tmp_path) + + response = _client(user).get( + reverse("api:sync_cutout"), + {"id": "des_dr2", "pos": "CIRCLE 10 0 1", "band": ["g", "r"], "format": "fits"}, + ) + + assert response.status_code == 422 + assert "Only one cutout task" in response.json()["detail"] + + job = Job.objects.get() + assert job.phase == Job.ExecutionPhase.ERROR + assert job.tasks.count() == 2 + assert all(task.status == Task.Status.PENDING for task in job.tasks.all()) diff --git a/cutout/service/tests/test_tasks.py b/cutout/service/tests/test_tasks.py index 7721ce9..b724a26 100644 --- a/cutout/service/tests/test_tasks.py +++ b/cutout/service/tests/test_tasks.py @@ -2,7 +2,7 @@ import pytest -from cutout.service.tasks import _validate_input_files +from cutout.service.cutout_runner import _validate_input_files def test_validate_input_files_accepts_none() -> None: diff --git a/cutout/service/uws/policy.py b/cutout/service/uws/policy.py index a16a361..79b4f47 100644 --- a/cutout/service/uws/policy.py +++ b/cutout/service/uws/policy.py @@ -25,22 +25,40 @@ class UWSPolicy(ABC): """ @abstractmethod - def dispatch(self, job: Job): - """Dispatch a job to a backend worker. + def create_tasks_for_job(self, job: Job, params: list[JobParameter], execution_mode: str = "async") -> list: + """Create the Task rows for a job, one per cutout execution unit. - This method is responsible for converting UWS job parameters to the - appropriate arguments for a backend job and invoking it with the - appropriate timeout. + Parameters + ---------- + job + The job the tasks belong to. + params + The job parameters. + execution_mode + "sync" or "async"; selects the results directory for the + generated files. + + Returns + ------- + list + The created Task rows. + """ + + @abstractmethod + def dispatch_async(self, job: Job, message_id: str): + """Dispatch the job's tasks to the backend workers. Parameters ---------- job The job to start. + message_id + Identifier used to track the dispatched work. Returns ------- - dramatiq.Message - The message sent to the backend worker. + celery.result.AsyncResult + The result handle of the dispatched workflow. """ @abstractmethod diff --git a/cutout/service/uws/service.py b/cutout/service/uws/service.py index 77b1adf..f2fcd67 100644 --- a/cutout/service/uws/service.py +++ b/cutout/service/uws/service.py @@ -1,6 +1,5 @@ import logging from pathlib import Path -from typing import List, Optional from celery import uuid from django.db import transaction @@ -19,11 +18,19 @@ def __init__(self) -> None: # TODO: Setup Settings, Logging self._policy = ImageCutoutPolicy() - def create(self, user: User, params: list[JobParameter], run_id: str | None = None) -> Job: - """Create a pending job. - - This does not start execution of the job. That must be done - separately with `start`.""" + def create( + self, + user: User, + params: list[JobParameter], + run_id: str | None = None, + execution_mode: str = "async", + ) -> Job: + """Create a pending job with its Task rows. + + This does not start execution of the job. Async jobs are started + separately with `start_async`; sync jobs execute their task directly + with `perform_cutout`. `execution_mode` ("sync" or "async") selects + the results directory for the generated files.""" self._policy.validate_params(params) job = Job( @@ -35,7 +42,7 @@ def create(self, user: User, params: list[JobParameter], run_id: str | None = No for p in params: job.parameters.create(parameter=p.parameter_id, value=p.value, is_post=p.is_post) - self._policy.create_tasks_for_job(_convert_job(job), params) + self._policy.create_tasks_for_job(_convert_job(job), params, execution_mode=execution_mode) return job @@ -48,25 +55,8 @@ def get_for_user(self, user: User, job_id: int) -> Job: raise PermissionDeniedError(f"Access to job {job_id} denied") return job - def start(self, user: User, job_id: int): - """Start execution of a job.""" - print(f"[JobService.start] called with user={user} job_id={job_id}") - sqljob = self.get_for_user(user, job_id) - if sqljob.phase not in (Job.ExecutionPhase.PENDING, Job.ExecutionPhase.HELD): - raise InvalidPhaseError("Cannot start job in phase {job.phase}") - - print(f"[JobService.start] sqljob.phase={sqljob.phase}") - job = _convert_job(sqljob) - print(f"[JobService.start] calling policy.dispatch with job_id={job.job_id}") - message = self._policy.dispatch(job) - print(f"[JobService.start] policy.dispatch returned: {message}") - - # TODO: Marcar o job como QUEUED - self.mark_queued(job_id, message.id) - return message - def start_async(self, user: User, job_id: int): - """Start async execution using the fake worker pipeline.""" + """Dispatch the job's tasks to the Celery workers.""" logger = logging.getLogger("cutout") logger.info(f"[JobService.start_async] called with user={user} job_id={job_id}") @@ -78,12 +68,16 @@ def start_async(self, user: User, job_id: int): raise InvalidPhaseError(f"Cannot start job in phase {sqljob.phase}") job = _convert_job(sqljob) + message_id = uuid() + logger.info(f"[JobService.start_async] mark_queued with message_id={message_id}") self.mark_queued(job_id, message_id) + logger.info( f"[JobService.start_async] calling policy.dispatch_async with job_id={job.job_id} message_id={message_id}" ) + message = self._policy.dispatch_async(job, message_id=message_id) logger.info(f"[JobService.start_async] policy.dispatch_async returned: {message}") return message @@ -98,12 +92,6 @@ def mark_queued(self, job_id: int, message_id: str) -> None: job.save() - def mark_executing(self, job_id: int) -> None: - job = Job.objects.get(pk=job_id) - job.phase = Job.ExecutionPhase.EXECUTING - job.start_time = timezone.now() - job.save() - def mark_completed(self, job_id: int) -> None: job = Job.objects.get(pk=job_id) job.phase = Job.ExecutionPhase.COMPLETED diff --git a/data/results/sync/.gitkeep b/data/results/sync/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/merge_production_dotenvs_in_dotenv.py b/merge_production_dotenvs_in_dotenv.py deleted file mode 100644 index 35139fb..0000000 --- a/merge_production_dotenvs_in_dotenv.py +++ /dev/null @@ -1,26 +0,0 @@ -import os -from collections.abc import Sequence -from pathlib import Path - -BASE_DIR = Path(__file__).parent.resolve() -PRODUCTION_DOTENVS_DIR = BASE_DIR / ".envs" / ".production" -PRODUCTION_DOTENV_FILES = [ - PRODUCTION_DOTENVS_DIR / ".django", - PRODUCTION_DOTENVS_DIR / ".postgres", -] -DOTENV_FILE = BASE_DIR / ".env" - - -def merge( - output_file: Path, - files_to_merge: Sequence[Path], -) -> None: - merged_content = "" - for merge_file in files_to_merge: - merged_content += merge_file.read_text() - merged_content += os.linesep - output_file.write_text(merged_content) - - -if __name__ == "__main__": - merge(DOTENV_FILE, PRODUCTION_DOTENV_FILES) diff --git a/scripts/debug_band_fits.py b/scripts/debug_band_fits.py deleted file mode 100644 index 1f33e82..0000000 --- a/scripts/debug_band_fits.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python3 -from cutout.lib.des_cutout import DesCutout -from astrocut import fits_cut -from astropy import units as u -from astropy.coordinates import SkyCoord -from pathlib import Path - -OUTDIR = Path('/data/results/debug') -OUTDIR.mkdir(parents=True, exist_ok=True) - -def main(): - ra = 36.30911 - dec = -10.18749 - size = 2.0 - print('Testing per-band fits_cut for coord', ra, dec) - dc = DesCutout() - verts = dc.get_cutout_verts(ra, dec, size) - for b in ['g','r','i']: - print('\n--- band', b, '---') - comp_files = dc.get_fits_files(verts, b) - print('compressed:', comp_files) - files = [] - for c in comp_files: - fits_filename = c.name.split('.fz')[0] - uncompressed = dc.tmp_path.joinpath(fits_filename) - if not uncompressed.exists(): - print(' uncompressing', c, '->', uncompressed) - try: - dc.funpack(c, uncompressed) - except Exception as e: - print(' funpack failed', e) - files.append(str(uncompressed)) - print('uncompressed files:', files) - try: - coord = SkyCoord(ra * u.deg, dec * u.deg, frame='icrs') - res = fits_cut(input_files=files, coordinates=coord, cutout_size=size * u.arcmin, single_outfile=True, cutout_prefix=f'dbg_{b}', output_dir=str(OUTDIR)) - print('fits_cut produced', res) - except Exception as e: - import traceback - print('fits_cut failed for band', b, ':', e) - traceback.print_exc() - -if __name__ == '__main__': - main() diff --git a/scripts/debug_cutout.py b/scripts/debug_cutout.py deleted file mode 100644 index b247b07..0000000 --- a/scripts/debug_cutout.py +++ /dev/null @@ -1,229 +0,0 @@ -#!/usr/bin/env python3 -"""Debug script for cutout generation. - -Creates FITS and PNG artefacts using legacy and DesCutout functions and -inspects the generated files (size, shape, min/max, nan counts). - -Run inside container: - - docker compose exec django python scripts/debug_cutout.py - -""" -from pathlib import Path -import sys -import traceback - -from PIL import Image -import numpy as np -from astropy.io import fits - -from cutout.lib.des_cutout import DesCutout -from cutout.lib.cutout import Cutout -from cutout.service.discovery.des_csv_locator import DesCsvFileLocator - - -OUT_DIR = Path("/data/results/debug") -OUT_DIR.mkdir(parents=True, exist_ok=True) - - -def inspect_fits(path: Path): - print(f"Inspecting FITS: {path}") - if not path.exists(): - print(" MISSING") - return - print(f" size_bytes: {path.stat().st_size}") - try: - with fits.open(path) as hdul: - print(" HDU list:") - hdul.info(output=sys.stdout) - # try to find first HDU with data - found = False - for i, h in enumerate(hdul): - if getattr(h, 'data', None) is not None: - data = h.data - print(f" found data in HDU {i}: dtype={data.dtype}, shape={data.shape}") - arr = np.array(data) - print(f" min: {np.nanmin(arr)}, max: {np.nanmax(arr)}, mean: {np.nanmean(arr)}") - print(f" nans: {np.isnan(arr).sum()} / {arr.size}") - found = True - break - if not found: - print(" No data array found in any HDU") - except Exception as e: - print(" ERROR reading FITS:", e) - traceback.print_exc() - - -def inspect_png(path: Path): - print(f"Inspecting PNG: {path}") - if not path.exists(): - print(" MISSING") - return - print(f" size_bytes: {path.stat().st_size}") - try: - img = Image.open(path) - print(f" mode: {img.mode}, size: {img.size}") - except Exception as e: - print(" ERROR reading PNG:", e) - traceback.print_exc() - - -def run_legacy_fits(ra, dec, size_arcmin, band): - dc = DesCutout() - filename = f"{ra:.5f}_{dec:.5f}_{band}.fits" - out = OUT_DIR.joinpath("legacy_" + filename) - print(f"Running legacy FITS: {out}") - try: - res = dc.single_cutout_fits(ra=ra, dec=dec, size_arcmin=size_arcmin, band=band, path=out) - print(" produced:", res) - inspect_fits(out) - except Exception as e: - print(" legacy FITS failed:", e) - traceback.print_exc() - - -def run_legacy_png(ra, dec, size_arcmin, band): - dc = DesCutout() - filename = f"{ra:.5f}_{dec:.5f}.png" - out = OUT_DIR.joinpath("legacy_" + filename) - print(f"Running legacy PNG: {out}") - try: - res = dc.single_cutout_png(ra=ra, dec=dec, size_arcmin=size_arcmin, band=band, path=out) - print(" produced:", res) - inspect_png(out) - except Exception as e: - print(" legacy PNG failed:", e) - traceback.print_exc() - - -def run_engine(engine_name, stencil, band, fmt, files=None): - print(f"Running engine {engine_name} format={fmt} band={band}") - from cutout.service.cutout_engine import create_cutout_engine - - engine = create_cutout_engine(engine_name) - filename = f"engine_{engine_name}_{stencil['center']['ra']:.5f}_{stencil['center']['dec']:.5f}.{fmt}" - out = OUT_DIR.joinpath(filename) - try: - # If no files provided, try to discover and uncompress via DesCutout - if not files: - try: - dc = DesCutout() - verts = dc.get_cutout_verts(stencil["center"]["ra"], stencil["center"]["dec"], stencil["radius"]) # type: ignore - # If band is multiple letters (e.g. 'gri'), build mapping per band - if isinstance(band, str) and len(band) > 1 and "," not in band and " " not in band: - bands = list(band) - else: - # split by comma or space if present - if isinstance(band, str) and "," in band: - bands = [b.strip() for b in band.split(",") if b.strip()] - elif isinstance(band, str) and " " in band: - bands = [b.strip() for b in band.split() if b.strip()] - else: - bands = [band] - - if len(bands) == 1: - comp_files = dc.get_fits_files(verts, bands[0]) - files = [] - for comp in comp_files: - fits_filename = comp.name.split(".fz")[0] - uncompressed = dc.tmp_path.joinpath(fits_filename) - if not uncompressed.exists(): - print(f" uncompressing {comp} -> {uncompressed}") - try: - dc.funpack(comp, uncompressed) - except Exception as e: - print(" funpack failed:", e) - if uncompressed.exists(): - files.append(str(uncompressed)) - else: - files_map = {} - for b in bands: - comp_files = dc.get_fits_files(verts, b) - files_map[b] = [] - for comp in comp_files: - fits_filename = comp.name.split(".fz")[0] - uncompressed = dc.tmp_path.joinpath(fits_filename) - if not uncompressed.exists(): - print(f" uncompressing {comp} -> {uncompressed}") - try: - dc.funpack(comp, uncompressed) - except Exception as e: - print(" funpack failed:", e) - if uncompressed.exists(): - files_map[b].append(str(uncompressed)) - files = files_map - except Exception: - files = files or [] - - print(' input_files passed to engine:', files) - # Build extra kwargs (e.g., color/rgb_bands) for multi-band PNG tests - extra_kwargs = {} - if engine_name == 'astrocut' and fmt == 'png' and isinstance(band, str) and len(band) > 1 and ',' not in band and ' ' not in band: - extra_kwargs['color'] = True - extra_kwargs['rgb_bands'] = band - - res = engine.run_cutout( - source_id="des_dr2", - stencil=stencil, - input_files=files, - band=band, - output_format=fmt, - output_path=out, - **extra_kwargs, - ) - print(" produced:", res) - if fmt == "fits": - inspect_fits(out) - else: - inspect_png(out) - except Exception as e: - print(f" engine {engine_name} failed:", e) - traceback.print_exc() - - -def main(): - # Coordinates from examples - cutouts = [ - {"ra": 36.30911, "dec": -10.18749, "size": 2.0, "band": "g", "format": "fits"}, - {"ra": 36.30911, "dec": -10.18749, "size": 2.0, "band": "gri", "format": "png"}, - {"ra": 36.15801, "dec": -10.33579, "size": 2.0, "band": "g", "format": "fits"}, - {"ra": 36.15801, "dec": -10.33579, "size": 2.0, "band": "gri", "format": "png"}, - ] - - for c in cutouts: - ra = c["ra"] - dec = c["dec"] - size = c["size"] - band = c["band"] - fmt = c["format"] - - print("\n=== Test case ===") - print(c) - - if fmt == "fits": - # legacy path - run_legacy_fits(ra, dec, size, band) - # engine path (des engine) - stencil = {"type": "circle", "center": {"ra": ra, "dec": dec}, "radius": size} - run_engine("legacy", stencil, band, "fits") - try: - run_engine("astrocut", stencil, band, "fits") - except Exception: - pass - - elif fmt == "png": - # legacy PNG - run_legacy_png(ra, dec, size, band) - # engine mono PNG (if band is single) - stencil = {"type": "circle", "center": {"ra": ra, "dec": dec}, "radius": size} - if len(band) == 1: - run_engine("legacy", stencil, band, "png") - run_engine("astrocut", stencil, band, "png") - else: - # multi-band: let run_engine perform discovery/uncompress for each band - run_engine("legacy", stencil, band, "png", files=None) - run_engine("astrocut", stencil, band, "png", files=None) - - -if __name__ == "__main__": - main() diff --git a/scripts/test_engine_color.py b/scripts/test_engine_color.py deleted file mode 100644 index 7230efa..0000000 --- a/scripts/test_engine_color.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python3 -from cutout.lib.des_cutout import DesCutout -from cutout.service.cutout_engine import create_cutout_engine -from pathlib import Path -from astropy import units as u -from astropy.coordinates import SkyCoord - -OUTDIR = Path('/data/results/debug') -OUTDIR.mkdir(parents=True, exist_ok=True) - -def main(): - ra = 36.30911 - dec = -10.18749 - size = 2.0 - dc = DesCutout() - verts = dc.get_cutout_verts(ra, dec, size) - bands = ['g','r','i'] - files_map = {} - for b in bands: - comp_files = dc.get_fits_files(verts, b) - files_map[b] = [] - for comp in comp_files: - fits_filename = comp.name.split('.fz')[0] - uncompressed = dc.tmp_path.joinpath(fits_filename) - if not uncompressed.exists(): - print('uncompressing', comp, '->', uncompressed) - dc.funpack(comp, uncompressed) - files_map[b].append(str(uncompressed)) - - print('files_map:', files_map) - stencil = {"type": "circle", "center": {"ra": ra, "dec": dec}, "radius": size} - engine = create_cutout_engine('astrocut') - out = OUTDIR.joinpath('engine_astrocut_test_gri.png') - try: - res = engine.run_cutout(source_id='des_dr2', stencil=stencil, input_files=files_map, band='gri', output_format='png', output_path=out, color=True, rgb_bands='gri') - print('engine produced', res) - except Exception as e: - import traceback - print('engine.run_cutout failed:', e) - traceback.print_exc() - -if __name__ == '__main__': - main() diff --git a/test_async_endpoint.md b/test_async_endpoint.md new file mode 100644 index 0000000..b37c17f --- /dev/null +++ b/test_async_endpoint.md @@ -0,0 +1,27 @@ +# 1. Obter o token com usuário/senha +TOKEN=$(curl -s -X POST "http://localhost:80/auth-token/" \ + -d "username=gverde&password=adminadmin" | python3 -c "import sys,json; print(json.load(sys.stdin)['token'])") + +# 2. Submeter o job async (PNG colorido gri, CIRCLE 0.75 2.867 0.033333) +curl -s -X POST "http://localhost:80/api/async" \ + -H "Authorization: Token ${TOKEN}" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "POS=CIRCLE 0.75 2.867 0.033333&format=png&color=true&rgb_bands=gri&id=des_dr2&phase=RUN" +# → retorna JSON com "job_id" e "phase":"QUEUED" + +# 3. Consultar a fase (repita até COMPLETED) — troque 154 pelo job_id retornado +curl -s "http://localhost:80/api/async/154/phase" -H "Authorization: Token ${TOKEN}" + +# 4. Listar os resultados +curl -s "http://localhost:80/api/async/154/results" -H "Authorization: Token ${TOKEN}" + +# 5. Baixar o PNG (result_id vem da listagem acima) +curl -s -o cutout_rgb.png \ + "http://localhost:80/api/async/154/results/job_154_des_dr2_astrocut_rgb_1" \ + -H "Authorization: Token ${TOKEN}" + +Observações: + +- No POST via form-urlencoded os espaços do POS podem ir literais (o curl codifica); só use %20 se passar na query string de um GET. +- O passo 3 devolve texto puro (QUEUED/EXECUTING/COMPLETED/ERROR). No teste que rodei o job completou em menos de 2 segundos. +- Para um exemplo de erro registrado no banco, use uma região fora do footprint (ex.: POS=CIRCLE 10.0 10.0 0.016667&band=r&format=fits) — a fase termina em ERROR e a mensagem fica no error_message da task. diff --git a/test_async_endpoint.py b/test_async_endpoint.py deleted file mode 100644 index af00c7f..0000000 --- a/test_async_endpoint.py +++ /dev/null @@ -1,212 +0,0 @@ -#!/usr/bin/env python3 -"""Smoke test for the sync cutout endpoint using token authentication. - -Examples: - - python test_async_endpoint.py --username gverde --password adminadmin --engine astrocut --pos "CIRCLE 36.30911 -10.18749 0.01" --output /tmp/async_astrocut_result.fits - - python test_async_endpoint.py \ - --username gverde \ - --password adminadmin \ - --engine astrocut \ - --pos "CIRCLE 36.30911 -10.18749 0.01" \ - --output /tmp/async_astrocut_result.fits - - python test_async_endpoint.py \ - --username gverde \ - --password adminadmin \ - --engine legacy \ - --pos "CIRCLE 36.30911 -10.18749 0.01" \ - --output /tmp/async_legacy_result.fits - - python test_async_endpoint.py \ - --username gverde \ - --password adminadmin \ - --id private_survey \ - --engine astrocut -""" - -from __future__ import annotations - -import argparse -import json -import sys -import urllib.error -import urllib.parse -import urllib.request -from pathlib import Path - - -def get_token(base_url: str, username: str, password: str) -> str: - url = f"{base_url.rstrip('/')}/auth-token/" - payload = json.dumps({"username": username, "password": password}).encode("utf-8") - req = urllib.request.Request( - url, - data=payload, - headers={"Content-Type": "application/json"}, - method="POST", - ) - with urllib.request.urlopen(req, timeout=20) as resp: - body = resp.read().decode("utf-8") - data = json.loads(body) - token = data.get("token") - if not token: - raise RuntimeError(f"Token not found in auth response: {body}") - return token - - -import time - - -def call_async_cutout( - *, - base_url: str, - token: str, - survey_id: str, - pos: str, - engine: str, - output_format: str, - band: str, - poll_interval: float = 1.0, - max_polls: int = 30, -) -> tuple[int, str, bytes]: - # 1. Cria job async (urllib segue o 303 automaticamente e entrega o job detail) - url = f"{base_url.rstrip('/')}/api/async" - payload = json.dumps( - { - "id": survey_id, - "pos": pos, - "engine": engine, - "format": output_format, - "band": band, - } - ).encode("utf-8") - - req = urllib.request.Request( - url, - data=payload, - headers={"Authorization": f"Token {token}", "Content-Type": "application/json"}, - method="POST", - ) - with urllib.request.urlopen(req, timeout=30) as resp: - job_url = resp.geturl() - body = resp.read() - - job_data = json.loads(body.decode("utf-8")) - job_id = job_data.get("job_id") - if not job_id: - raise RuntimeError(f"No job_id in response: {body[:1000].decode('utf-8', 'replace')}") - print(f"job created: id={job_id} url={job_url}") - - # 2. Poll job status - for _ in range(max_polls): - job_req = urllib.request.Request( - job_url, - headers={"Authorization": f"Token {token}"}, - method="GET", - ) - with urllib.request.urlopen(job_req, timeout=20) as job_resp: - job_data = json.loads(job_resp.read().decode("utf-8")) - phase = job_data.get("phase") - print(f" phase: {phase}") - if phase == "COMPLETED": - break - elif phase == "ERROR": - raise RuntimeError(f"Job failed: {job_data}") - time.sleep(poll_interval) - else: - raise TimeoutError("Job did not complete in time") - - # 3. Get results - results_url = job_data.get("results_url") - if not results_url: - raise RuntimeError("No results_url in job data") - results_req = urllib.request.Request( - results_url, - headers={"Authorization": f"Token {token}"}, - method="GET", - ) - with urllib.request.urlopen(results_req, timeout=20) as results_resp: - results_data = json.loads(results_resp.read().decode("utf-8")) - results = results_data.get("results", []) - if not results: - raise RuntimeError("No results found for job") - result = results[0] - download_url = result.get("download_url") - if not download_url: - raise RuntimeError("No download_url in result") - print(f"downloading from: {download_url}") - - # 4. Download result - download_req = urllib.request.Request( - download_url, - headers={"Authorization": f"Token {token}"}, - method="GET", - ) - with urllib.request.urlopen(download_req, timeout=60) as dl_resp: - dl_status = dl_resp.getcode() - content_type = dl_resp.headers.get("Content-Type", "") - body = dl_resp.read() - return dl_status, content_type, body - - -def main() -> int: - parser = argparse.ArgumentParser(description="Test /api/async endpoint") - parser.add_argument("--base-url", default="http://localhost:8000", help="API base URL") - parser.add_argument("--username", required=True, help="Django username") - parser.add_argument("--password", required=True, help="Django password") - parser.add_argument("--id", default="des_dr2", dest="survey_id", help="Survey ID") - parser.add_argument("--pos", default="CIRCLE 36.30911 -10.18749 2", help="POS value") - parser.add_argument("--engine", default="astrocut", help="Engine name: astrocut or legacy") - parser.add_argument("--format", default="fits", dest="output_format", help="Output format") - parser.add_argument("--band", default="g", help="Band") - parser.add_argument( - "--output", - default="/tmp/async_result_test.fits", - help="Output file path for successful binary response", - ) - parser.add_argument("--poll-interval", type=float, default=1.0, help="Polling interval in seconds") - parser.add_argument("--max-polls", type=int, default=30, help="Maximum polling attempts") - args = parser.parse_args() - - try: - token = get_token(args.base_url, args.username, args.password) - print("auth: ok") - - status, content_type, body = call_async_cutout( - base_url=args.base_url, - token=token, - survey_id=args.survey_id, - pos=args.pos, - engine=args.engine, - output_format=args.output_format, - band=args.band, - poll_interval=args.poll_interval, - max_polls=args.max_polls, - ) - print(f"status: {status}") - print(f"content-type: {content_type}") - - if status == 200 and "application/json" not in content_type: - output = Path(args.output) - output.parent.mkdir(parents=True, exist_ok=True) - output.write_bytes(body) - print(f"result saved to: {output}") - else: - text = body.decode("utf-8", errors="replace") - print("response body:") - print(text[:2000]) - - return 0 - except urllib.error.HTTPError as e: - body = e.read().decode("utf-8", errors="replace") - print(f"http error: {e.code}") - print(body[:2000]) - return 1 - except Exception as e: # noqa: BLE001 - print(f"error: {e}") - return 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/test_celery_ping.py b/test_celery_ping.py deleted file mode 100644 index e9458c4..0000000 --- a/test_celery_ping.py +++ /dev/null @@ -1,6 +0,0 @@ -from cutout.service.tasks_test_celery import ping - -if __name__ == "__main__": - result = ping.delay(42) - print("Task submitted, waiting result...") - print(result.get(timeout=10)) diff --git a/test_plot_cutout_fits.py b/test_plot_cutout_fits.py deleted file mode 100644 index 30d375b..0000000 --- a/test_plot_cutout_fits.py +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env python3 -"""Inspect and plot a FITS cutout file. - -Examples: - python test_plot_cutout_fits.py --file /tmp/sync_result_test.fits - - python test_plot_cutout_fits.py \ - --file /tmp/sync_result_test.fits \ - --save /tmp/sync_result_test.png \ - --no-show - - python test_plot_cutout_fits.py --file /tmp/sync_result_test.fits --wcs -""" - -from __future__ import annotations - -import argparse -from pathlib import Path - -import matplotlib.pyplot as plt -import numpy as np -from astropy.io import fits -from astropy.wcs import WCS - - -def _find_image_hdu( - hdul: fits.HDUList, preferred_ext: int | None -) -> tuple[int, fits.hdu.base.ExtensionHDU | fits.PrimaryHDU]: - if preferred_ext is not None: - hdu = hdul[preferred_ext] - if hdu.data is None: - raise ValueError(f"HDU {preferred_ext} has no data") - return preferred_ext, hdu - - for idx, hdu in enumerate(hdul): - if hdu.data is not None and getattr(hdu.data, "ndim", 0) >= 2: - return idx, hdu - - raise ValueError("No image HDU found in FITS file") - - -def _print_stats(data: np.ndarray, hdu_index: int) -> None: - finite_mask = np.isfinite(data) - finite_count = int(np.count_nonzero(finite_mask)) - total_count = int(data.size) - - print(f"HDU index: {hdu_index}") - print(f"Shape: {data.shape}") - print(f"dtype: {data.dtype}") - print(f"Finite pixels: {finite_count}/{total_count}") - - if finite_count == 0: - print("No finite pixels available for stats") - return - - finite = data[finite_mask] - print(f"Min: {float(np.min(finite)):.6g}") - print(f"Max: {float(np.max(finite)):.6g}") - print(f"Mean: {float(np.mean(finite)):.6g}") - print(f"Median: {float(np.median(finite)):.6g}") - - -def _plot_image( - data: np.ndarray, header: fits.Header, use_wcs: bool, title: str, save_path: Path | None, show: bool -) -> None: - vmin, vmax = np.nanpercentile(data, [1, 99]) - - if use_wcs: - wcs = WCS(header) - fig = plt.figure(figsize=(8, 7)) - ax = fig.add_subplot(111, projection=wcs) - ax.set_xlabel("RA") - ax.set_ylabel("Dec") - else: - fig, ax = plt.subplots(figsize=(8, 7)) - ax.set_xlabel("X pixel") - ax.set_ylabel("Y pixel") - - image = ax.imshow(data, origin="lower", cmap="gray", vmin=vmin, vmax=vmax) - ax.set_title(title) - fig.colorbar(image, ax=ax, fraction=0.046, pad=0.04, label="Flux") - fig.tight_layout() - - if save_path is not None: - save_path.parent.mkdir(parents=True, exist_ok=True) - fig.savefig(save_path, dpi=150) - print(f"Plot saved to: {save_path}") - - if show: - plt.show() - else: - plt.close(fig) - - -def main() -> int: - parser = argparse.ArgumentParser(description="Inspect and plot a FITS cutout") - parser.add_argument("--file", required=True, help="Path to FITS file") - parser.add_argument("--ext", type=int, default=None, help="HDU extension index (default: first image HDU)") - parser.add_argument("--wcs", action="store_true", help="Use WCS projection when available") - parser.add_argument("--save", default=None, help="Save plot image path (PNG, JPG, etc.)") - parser.add_argument("--no-show", action="store_true", help="Do not open interactive plot window") - args = parser.parse_args() - - fits_path = Path(args.file) - if not fits_path.exists(): - raise FileNotFoundError(f"FITS file does not exist: {fits_path}") - - with fits.open(fits_path) as hdul: - hdu_index, hdu = _find_image_hdu(hdul, args.ext) - data = np.asarray(hdu.data, dtype=float) - if data.ndim > 2: - data = np.squeeze(data) - if data.ndim != 2: - raise ValueError(f"Expected a 2D image after squeeze, got shape={data.shape}") - - _print_stats(data, hdu_index) - _plot_image( - data=data, - header=hdu.header, - use_wcs=args.wcs, - title=f"Cutout: {fits_path.name}", - save_path=Path(args.save) if args.save else None, - show=not args.no_show, - ) - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/test_worker_dispatch.py b/test_worker_dispatch.py deleted file mode 100644 index 27e3d13..0000000 --- a/test_worker_dispatch.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python3 -""" -Dispatch a cutout job directly to Celery workers, bypassing the HTTP API. -Useful for testing worker execution in isolation. - -Usage: - docker compose exec django python test_worker_dispatch.py --job-id 103 - docker compose exec django python test_worker_dispatch.py --job-id 103 --reset -""" -from __future__ import annotations - -import argparse -import os -import sys -import time - -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local") - -import django # noqa: E402 - -django.setup() - -from cutout.service.models import Job as SQLJob # noqa: E402 -from cutout.service.uws.service import JobService # noqa: E402 - - -def main() -> int: - parser = argparse.ArgumentParser(description="Dispatch a cutout job directly to Celery workers") - parser.add_argument("--job-id", required=True, type=int, help="ID of the job to dispatch") - parser.add_argument( - "--reset", - action="store_true", - help="Reset job to PENDING before dispatching (required if job already ran)", - ) - parser.add_argument("--poll-interval", type=float, default=1.0, help="Poll interval in seconds") - parser.add_argument("--max-polls", type=int, default=60, help="Maximum poll attempts before timeout") - args = parser.parse_args() - - try: - sqljob = SQLJob.objects.get(pk=args.job_id) - except SQLJob.DoesNotExist: - print(f"error: job {args.job_id} not found") - return 1 - - print(f"job found: id={sqljob.id} phase={sqljob.phase} owner={sqljob.owner}") - params = list(sqljob.parameters.order_by("id").values_list("parameter", "value")) - print(f" params: {params}") - - terminal_phases = ( - SQLJob.ExecutionPhase.COMPLETED, - SQLJob.ExecutionPhase.ERROR, - SQLJob.ExecutionPhase.ABORTED, - SQLJob.ExecutionPhase.EXECUTING, - SQLJob.ExecutionPhase.QUEUED, - ) - if sqljob.phase in terminal_phases: - if not args.reset: - print(f" job is already in phase={sqljob.phase}. Use --reset to re-dispatch.") - return 1 - sqljob.phase = SQLJob.ExecutionPhase.PENDING - sqljob.message_id = None - sqljob.start_time = None - sqljob.end_time = None - sqljob.results.all().delete() - sqljob.save() - print(" reset to PENDING") - - # Dispatch via the same service layer the API uses - job_service = JobService() - job_service.start_async(sqljob.owner, sqljob.id) - print(f" dispatched — polling every {args.poll_interval}s (max {args.max_polls} attempts)") - - # Poll until terminal state - for i in range(1, args.max_polls + 1): - time.sleep(args.poll_interval) - sqljob.refresh_from_db() - phase = sqljob.phase - print(f" [{i:2d}] phase={phase}") - if phase == SQLJob.ExecutionPhase.COMPLETED: - break - elif phase in (SQLJob.ExecutionPhase.ERROR, SQLJob.ExecutionPhase.ABORTED): - print(" job ended with failure") - return 1 - else: - print(f" timeout: job did not complete after {args.max_polls} polls") - return 1 - - # Print results - results = list(sqljob.results.order_by("sequence")) - print(f"\nresults ({len(results)}):") - for r in results: - print(f" [{r.sequence}] result_id : {r.result_id}") - print(f" file : {r.file_path}") - print(f" mime : {r.mime_type}") - print(f" size : {r.size} bytes") - print(f" url : {r.url}") - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tests/test_e2e_png.py b/tests/test_e2e_png.py deleted file mode 100644 index 4ddf919..0000000 --- a/tests/test_e2e_png.py +++ /dev/null @@ -1,50 +0,0 @@ -import pytest -from pathlib import Path - - -def _has_tiles() -> bool: - return Path("/data/tiles").exists() - - -def test_astrocut_end_to_end_png(): - if not _has_tiles(): - pytest.skip("No tiles available in /data/tiles for E2E test") - - from cutout.lib.des_cutout import DesCutout - from cutout.service.tasks import image_cutout - from PIL import Image - - ra = 36.30911 - dec = -10.18749 - size = 2.0 - dc = DesCutout() - verts = dc.get_cutout_verts(ra, dec, size) - - bands = ["g", "r", "i"] - files_map = {} - for b in bands: - comp_files = dc.get_fits_files(verts, b) - if not comp_files: - pytest.skip(f"No files found for band {b}") - files_map[b] = [str(p.file_path) if hasattr(p, 'file_path') else str(p) for p in comp_files] - - out = "/data/results/test_e2e_astrocut_gri.png" - - res = image_cutout.run( - job_id="e2e-test", - source_id="des_dr2", - stencil={"type": "circle", "center": {"ra": ra, "dec": dec}, "radius": size}, - engine="astrocut", - band="gri", - format="png", - path=out, - files=files_map, - color=True, - rgb_bands="gri", - persist=False, - ) - - assert Path(res).exists(), f"Result file missing: {res}" - im = Image.open(res) - assert im.mode in ("RGB", "RGBA") - assert im.size[0] > 0 and im.size[1] > 0 diff --git a/tests/test_merge_production_dotenvs_in_dotenv.py b/tests/test_merge_production_dotenvs_in_dotenv.py deleted file mode 100644 index c0e68f6..0000000 --- a/tests/test_merge_production_dotenvs_in_dotenv.py +++ /dev/null @@ -1,34 +0,0 @@ -from pathlib import Path - -import pytest - -from merge_production_dotenvs_in_dotenv import merge - - -@pytest.mark.parametrize( - ("input_contents", "expected_output"), - [ - ([], ""), - ([""], "\n"), - (["JANE=doe"], "JANE=doe\n"), - (["SEP=true", "AR=ator"], "SEP=true\nAR=ator\n"), - (["A=0", "B=1", "C=2"], "A=0\nB=1\nC=2\n"), - (["X=x\n", "Y=y", "Z=z\n"], "X=x\n\nY=y\nZ=z\n\n"), - ], -) -def test_merge( - tmp_path: Path, - input_contents: list[str], - expected_output: str, -): - output_file = tmp_path / ".env" - - files_to_merge = [] - for num, input_content in enumerate(input_contents, start=1): - merge_file = tmp_path / f".service{num}" - merge_file.write_text(input_content) - files_to_merge.append(merge_file) - - merge(output_file, files_to_merge) - - assert output_file.read_text() == expected_output From dce4a0432f1a89329fe2aabf7cdbf5ce23e085db Mon Sep 17 00:00:00 2001 From: glaubervila Date: Sat, 11 Jul 2026 13:31:52 -0300 Subject: [PATCH 3/4] Remove remaining prototype modules from cutout/service des_cutout_functions.py, teste_funcoes.py and teste_classe.py carried the original Cutout2D/make_lupton_rgb prototype and ad-hoc experiments, none of them referenced anywhere. The astrocut engine pipeline supersedes them all. Co-Authored-By: Claude Fable 5 --- cutout/service/des_cutout_functions.py | 219 --------------------- cutout/service/teste_classe.py | 76 -------- cutout/service/teste_funcoes.py | 260 ------------------------- 3 files changed, 555 deletions(-) delete mode 100644 cutout/service/des_cutout_functions.py delete mode 100755 cutout/service/teste_classe.py delete mode 100755 cutout/service/teste_funcoes.py diff --git a/cutout/service/des_cutout_functions.py b/cutout/service/des_cutout_functions.py deleted file mode 100644 index fddad91..0000000 --- a/cutout/service/des_cutout_functions.py +++ /dev/null @@ -1,219 +0,0 @@ -#!/usr/bin/env python3 - -import glob - -import numpy as np -from astropy import units as u -from astropy.coordinates import SkyCoord -from astropy.io import fits -from astropy.nddata.utils import Cutout2D -from astropy.visualization import make_lupton_rgb -from astropy.wcs import WCS - - -def get_fits_data(RA, DEC, size, tiles, band, path): - """Access data (image and wcs) from FITS files. - - Parameters - ---------- - RA : float - Equatorial coordinate of the center of cutout (degrees). - DEC : float - Equatorial coordinate of the center of cutout (degrees). - size : float - Size of cutout (arcmin). - tiles : list - List with the tiles where the vertices of cutout reside. - Sorted by: Upper left, Upper right, Lower right, Lower left. - band : str - Band of cutout. - path : str - Path to folder with the FITS files. - """ - - tile_un, ind = np.unique(tiles[:], return_index=True) - tile_un = tile_un[np.argsort(ind)] - - if len(tile_un) == 1: - data_, wcs_ = cutout_fits(RA, DEC, size, tiles[0], band, path) - return (data_, wcs_) - - elif len(tile_un) == 2: - data_1, wcs_1 = cutout_fits(RA, DEC, size, tile_un[0], band, path, "trim") - data_2, wcs_2 = cutout_fits(RA, DEC, size, tile_un[1], band, path, "trim") - - if np.shape(data_1)[1] < np.shape(data_1)[0]: - # side-by-side - data_1 = data_1[:, :-118] - data_2 = data_2[:, 118:] - data_ = np.concatenate((data_1, data_2), axis=1) - return (data_, wcs_1) - - else: - # top-bottom - data_1 = data_1[114:, :] - data_2 = data_2[:-114, :] - data_ = np.concatenate((data_2, data_1), axis=0) - return (data_, wcs_2) - - elif len(tile_un) == 3: - data_1, wcs_1 = cutout_fits(RA, DEC, size, tile_un[0], band, path, "trim") - data_2, wcs_2 = cutout_fits(RA, DEC, size, tile_un[1], band, path, "trim") - data_3, wcs_3 = cutout_fits(RA, DEC, size, tile_un[2], band, path, "trim") - - # Biggest at bottom: - if np.shape(data_1)[1] < np.shape(data_3)[1]: - data_12 = np.concatenate((data_1[118:, :-118], data_2[118:, 118:]), axis=1) - data_ = np.concatenate((data_3[:-118, :], data_12[:, 0 : np.shape(data_3)[1]]), axis=0) - # Biggest at top: - else: - data_23 = np.concatenate((data_2[:-118, 118:], data_3[:-118, :-118]), axis=1) - data_ = np.concatenate((data_23, data_1[118:, 0 : np.shape(data_23)[1]]), axis=0) - return (data_, wcs_3) - - -def cutout_lupton(g_data, r_data, i_data, minimum, stretch, Q, filename): - """Make RGB image and saves as png or jpg files using Lupton method. - TODO: Improve quality of image for cutout with saturated data. - - Parameters - ---------- - g_data : array - Cutout data from first band. - r_data : array - Cutout data from second band. - i_data : array - Cutout data from third band. - filename : str - Name of file to be saved. - """ - - rgb_default = make_lupton_rgb(i_data, r_data, g_data, minimum=minimum, stretch=stretch, Q=Q, filename=filename) - - -def write_cutout_file(data, wcs, filename): - """Saves cutout file. - - Parameters - ---------- - data : array - Array with image data. - wcs : astropy object - Information about world coordinate system of cutout. - filename : str - Name of file to be saved. - """ - hdu = fits.PrimaryHDU(data) - hdu.header.update(wcs) - hdu.writeto(filename, overwrite=True) - - -def cutout_fits(RA_center, DEC_center, size_arcmin, tile_name, band, path, mode="partial"): - """Return data (image array and wcs) from tile. - - Parameters - ---------- - RA_center : float - Equatorial coordinate of center of tile. - DEC_center : float - Equatorial coordinate of center of tile. - size_arcmin : float - Size of cutout in arcmin. - tile_name : str - Name of tile where total or part of the cutout image resides. - band : str - Band of image. - path : str - Path to folder where the FITS files are stored. - mode : str, optional - Mode of cutout. See: - https://docs.astropy.org/en/stable/api/astropy.nddata.Cutout2D.html - By default set to 'partial'. - - Returns - ------- - arrays - Two arrays, one with image data and other with WCS astropy object. - """ - file_name_ = glob.glob(path + "/" + tile_name + "_*_" + band + ".fits") - file_name = file_name_[0] - f = fits.open(file_name) - wcs = WCS(f[1].header) - - cutout1 = Cutout2D( - fits.getdata(file_name, ext=0), - (SkyCoord(ra=RA_center * u.degree, dec=DEC_center * u.degree, frame="icrs")), - size_arcmin * u.arcmin, - wcs=wcs, - mode=mode, - ) - - return cutout1.data, cutout1.wcs.to_header() - - -def cutout_verts(RA_center, DEC_center, size_arcmin): - """Defines the position of vertices in each cutout. - See the pos_angle where the vertices are sorted. - - Parameters - ---------- - RA_center : float - Equatorial coordinate of center of tile. - DEC_center : float - Equatorial coordinate of center of tile. - size_arcmin : float - Size (length of each side) of cutout, in arcmin. - - Returns - ------- - SkyCoord astropy object - Location of vertices of cutout. - """ - pos_angle = [45, 315, 225, 135] * u.deg - RA, DEC = [], [] - for i, j in enumerate(RA_center): - c1 = SkyCoord(RA_center[i] * u.deg, DEC_center[i] * u.deg, frame="icrs") - sep = 0.5 * np.sqrt(2.0) * size_arcmin[i] * u.arcmin - RA.append(list(c1.directional_offset_by(pos_angle, sep).ra.deg)) - DEC.append(list(c1.directional_offset_by(pos_angle, sep).dec.deg)) - return SkyCoord(RA * u.deg, DEC * u.deg, frame="icrs") - - -def tiles_from_cat(cat, file_path): - """Read information about tiles. - TODO: read more information about the vertices of tiles in - order to have a correct overlap in case cutouts are in the - edge of tiles. - - Parameters - ---------- - cat : SkyCoord astropy object - Object with information about coordinates of vertices. - file_path : str - File with tile's information. - - Returns - ------- - list - List of tiles where the vertices of cutout reside. - """ - ra_ll, dec_ll, ra_ul, dec_ul, ra_ur, dec_ur, ra_lr, dec_lr = np.loadtxt( - file_path, usecols=(9, 10, 11, 12, 13, 14, 15, 16), delimiter=",", unpack=True - ) - tile_names = np.loadtxt(file_path, usecols=(2), delimiter=",", dtype=str, unpack=True) - - ra = cat.ra.deg - dec = cat.dec.deg - - tile_match = [] - - for i in range(np.shape(ra)[0]): - idx_ = [] - for j in range(4): - idx_.append( - np.argwhere((ra_ll < ra[i][j]) & (ra_ur > ra[i][j]) & (dec_ll < dec[i][j]) & (dec_ur > dec[i][j]))[0][ - 0 - ] - ) - tile_match.append([tile_names[k] for k in idx_]) - return tile_match diff --git a/cutout/service/teste_classe.py b/cutout/service/teste_classe.py deleted file mode 100755 index c122dfc..0000000 --- a/cutout/service/teste_classe.py +++ /dev/null @@ -1,76 +0,0 @@ -from pathlib import Path - -from celery import group - -# from cutout.lib.cutout import Cutout -# from cutout.lib.des_cutout import DesCutout -# from cutout.service.policy import ImageCutoutPolicy -# from cutout.service.uws.models import JobParameter -# from cutout.service.uws.service import JobService -# from cutout.users.models import User - -if __name__ == "__main__": - from cutout.service.tasks import task_1, task_completed - - header = [task_1.s(1, 2), task_1.s(3, 4)] - - g = group(header) - gresult = g.apply_async() - print(gresult.get()) - - # params = [] - # teste_params = { - # "id": "des_dr2", - # "runid": "MeujobCutout", - # "band": "g", - # "format": "fits", - # "pos": "CIRCLE 36.30911 -10.18749 2", - # } - # for key, value in teste_params.items(): - # if key.lower() == "runid": - # run_id = value - # else: - # params.append(JobParameter(parameter_id=key.lower(), value=value, is_post=False)) - # user = User.objects.get(pk=1) - # policy = ImageCutoutPolicy() - # job_service = JobService(policy=policy) - # job = job_service.create(user=user, params=params, run_id=run_id) - # job_service.start(user, job_id=job.id) - # cutouts = [ - # { - # "id": "des_dr2", - # "stencil": {"type": "circle", "center": {"ra": 36.30911, "dec": -10.18749}, "radius": 2.0}, - # "band": "g", - # "format": "fits", - # } - # ] - # for c in cutouts: - # dc = Cutout(source_id=c["id"], stencil=c["stencil"], band=c["band"], format=c["format"]) - # # filename = "{:.5f}_{:.5f}_{}.fits".format(round(c["stencil"]["center"]["ra"], 5), round(c["stencil"]["center"]["dec"], 5), c["band"]) - # filename = "teste.fits" - # resultfile = Path("/data/results").joinpath(filename) - # dc.create(path=resultfile) - # cutouts = [ - # {"ra": 36.30911, "dec": -10.18749, "size": 2.0, "band": "g", "format": "fits"}, # 1 - Tile - # # {"ra": 36.30911, "dec": -10.18749, "size": 2.0, "band": "gri", "format": "png"}, # 1 - Tile - # # {"ra": 36.15801, "dec": -10.33579, "size": 2.0, "band": "g", "format": "fits"}, # 2 - Tile - # # {"ra": 36.15801, "dec": -10.33579, "size": 2.0, "band": "gri", "format": "png"}, # 2 - Tile - # # {"ra": 35.23676, "dec": -10.33269, "size": 10.0, "band": "g", "format": "fits"}, # 3 - Tile - # # {"ra": 35.23676, "dec": -10.33269, "size": 10.0, "band": "gri", "format": "png"}, # 3 - Tile - # ] - # dc = DesCutout() - # for c in cutouts: - # if c["format"] == "fits": - # filename = "{:.5f}_{:.5f}_{}.fits".format(round(c["ra"], 5), round(c["dec"], 5), c["band"]) - # resultfile = Path("/data/results").joinpath(filename) - # result = dc.single_cutout_fits( - # ra=c["ra"], dec=c["dec"], size_arcmin=c["size"], band=c["band"], path=resultfile - # ) - # print(result) - # if c["format"] == "png": - # filename = "{:.5f}_{:.5f}.png".format(round(c["ra"], 5), round(c["dec"], 5)) - # resultfile = Path("/data/results").joinpath(filename) - # result = dc.single_cutout_png( - # ra=c["ra"], dec=c["dec"], size_arcmin=c["size"], band=c["band"], path=resultfile - # ) - # print(result) diff --git a/cutout/service/teste_funcoes.py b/cutout/service/teste_funcoes.py deleted file mode 100755 index 751013a..0000000 --- a/cutout/service/teste_funcoes.py +++ /dev/null @@ -1,260 +0,0 @@ -from pathlib import Path - -import numpy as np -from astropy import units as u -from astropy.coordinates import SkyCoord -from astropy.io import fits -from astropy.nddata.utils import Cutout2D -from astropy.visualization import make_lupton_rgb -from astropy.wcs import WCS - - -def cutout_verts(RA_center, DEC_center, size_arcmin): - """Defines the position of vertices in each cutout. - See the pos_angle where the vertices are sorted. - - Parameters - ---------- - RA_center : float - Equatorial coordinate of center of cutout. - DEC_center : float - Equatorial coordinate of center of cutout. - size_arcmin : float - Size (length of each side) of cutout, in arcmin. - - Returns - ------- - SkyCoord astropy object - Location of vertices of cutout. - """ - pos_angle = [45, 315, 225, 135] * u.deg - c1 = SkyCoord(RA_center * u.deg, DEC_center * u.deg, frame="icrs") - sep = 0.5 * np.sqrt(2.0) * size_arcmin * u.arcmin - RA = c1.directional_offset_by(pos_angle, sep).ra.deg - DEC = c1.directional_offset_by(pos_angle, sep).dec.deg - return SkyCoord(RA * u.deg, DEC * u.deg, frame="icrs") - - -def tiles_from_cat(cat, file_path): - """Read information about tiles. - TODO: read more information about the vertices of tiles in - order to have a correct overlap in case cutouts are in the - edge of tiles. - - Parameters - ---------- - cat : SkyCoord astropy object - Object with information about coordinates of vertices. - file_path : str - File with tile's information. - - Returns - ------- - list - List of tiles where the vertices of cutout reside. - """ - ra_ll, dec_ll, ra_ul, dec_ul, ra_ur, dec_ur, ra_lr, dec_lr = np.loadtxt( - file_path, usecols=(9, 10, 11, 12, 13, 14, 15, 16), delimiter=",", unpack=True - ) - tile_names = np.loadtxt(file_path, usecols=(2), delimiter=",", dtype=str, unpack=True) - - ra = cat.ra.deg - dec = cat.dec.deg - - idx_ = [] - for j in range(4): - idx_.append(np.argwhere((ra_ll < ra[j]) & (ra_ur > ra[j]) & (dec_ll < dec[j]) & (dec_ur > dec[j]))[0][0]) - tile_match = [tile_names[k] for k in idx_] - # TODO: Remover tiles duplicadas - return tile_match - - -def cutout_fits(RA_center, DEC_center, size_arcmin, tile_name, band, path, mode="partial"): - """Return data (image array and wcs) from tile. - - Parameters - ---------- - RA_center : float - Equatorial coordinate of center of tile. - DEC_center : float - Equatorial coordinate of center of tile. - size_arcmin : float - Size of cutout in arcmin. - tile_name : str - Name of tile where total or part of the cutout image resides. - band : str - Band of image. - path : str - Path to folder where the FITS files are stored. - mode : str, optional - Mode of cutout. See: - https://docs.astropy.org/en/stable/api/astropy.nddata.Cutout2D.html - By default set to 'partial'. - - Returns - ------- - arrays - Two arrays, one with image data and other with WCS astropy object. - """ - # file_name_ = glob.glob(path + "/" + tile_name + "_*_" + band + ".fits") - # file_name = file_name_[0] - # TODO: Ter o tilename completo. - fits_filepath = path.joinpath(f"{tile_name}_r4920p01_{band}.fits") - f = fits.open(fits_filepath) - wcs = WCS(f[1].header) - print(fits_filepath) - cutout1 = Cutout2D( - fits.getdata(fits_filepath, ext=0), - (SkyCoord(ra=RA_center * u.degree, dec=DEC_center * u.degree, frame="icrs")), - size_arcmin * u.arcmin, - wcs=wcs, - mode=mode, - ) - - return cutout1.data, cutout1.wcs.to_header() - - -def get_fits_data(RA, DEC, size, tiles, band, path): - """Access data (image and wcs) from FITS files. - - Parameters - ---------- - RA : float - Equatorial coordinate of the center of cutout (degrees). - DEC : float - Equatorial coordinate of the center of cutout (degrees). - size : float - Size of cutout (arcmin). - tiles : list - List with the tiles where the vertices of cutout reside. - Sorted by: Upper left, Upper right, Lower right, Lower left. - band : str - Band of cutout. - path : str - Path to folder with the FITS files. - """ - - tile_un, ind = np.unique(tiles[:], return_index=True) - tile_un = tile_un[np.argsort(ind)] - - if len(tile_un) == 1: - data_, wcs_ = cutout_fits(RA, DEC, size, tiles[0], band, path) - return (data_, wcs_) - - elif len(tile_un) == 2: - data_1, wcs_1 = cutout_fits(RA, DEC, size, tile_un[0], band, path, "trim") - data_2, wcs_2 = cutout_fits(RA, DEC, size, tile_un[1], band, path, "trim") - - if np.shape(data_1)[1] < np.shape(data_1)[0]: - # side-by-side - data_1 = data_1[:, :-118] - data_2 = data_2[:, 118:] - data_ = np.concatenate((data_1, data_2), axis=1) - return (data_, wcs_1) - - else: - # top-bottom - data_1 = data_1[114:, :] - data_2 = data_2[:-114, :] - data_ = np.concatenate((data_2, data_1), axis=0) - return (data_, wcs_2) - - elif len(tile_un) == 3: - data_1, wcs_1 = cutout_fits(RA, DEC, size, tile_un[0], band, path, "trim") - data_2, wcs_2 = cutout_fits(RA, DEC, size, tile_un[1], band, path, "trim") - data_3, wcs_3 = cutout_fits(RA, DEC, size, tile_un[2], band, path, "trim") - - # Biggest at bottom: - if np.shape(data_1)[1] < np.shape(data_3)[1]: - data_12 = np.concatenate((data_1[118:, :-118], data_2[118:, 118:]), axis=1) - data_ = np.concatenate((data_3[:-118, :], data_12[:, 0 : np.shape(data_3)[1]]), axis=0) - # Biggest at top: - else: - data_23 = np.concatenate((data_2[:-118, 118:], data_3[:-118, :-118]), axis=1) - data_ = np.concatenate((data_23, data_1[118:, 0 : np.shape(data_23)[1]]), axis=0) - return (data_, wcs_3) - - -def write_cutout_file(data, wcs, filename): - """Saves cutout file. - - Parameters - ---------- - data : array - Array with image data. - wcs : astropy object - Information about world coordinate system of cutout. - filename : str - Name of file to be saved. - """ - hdu = fits.PrimaryHDU(data) - hdu.header.update(wcs) - hdu.writeto(filename, overwrite=True) - - -def cutout_lupton(g_data, r_data, i_data, minimum, stretch, Q, filename): - """Make RGB image and saves as png or jpg files using Lupton method. - TODO: Improve quality of image for cutout with saturated data. - - Parameters - ---------- - g_data : array - Cutout data from first band. - r_data : array - Cutout data from second band. - i_data : array - Cutout data from third band. - filename : str - Name of file to be saved. - """ - - rgb_default = make_lupton_rgb(i_data, r_data, g_data, minimum=minimum, stretch=stretch, Q=Q, filename=filename) - - -if __name__ == "__main__": - cutout_1_tile = {"ra": 36.30911, "dec": -10.18749, "size": 2.0, "band": "g"} - cutout_2_tile = {"ra": 36.15801, "dec": -10.33579, "size": 2.0, "band": "g"} - cutout_3_tiles = {"ra": 35.23676, "dec": -10.33269, "size": 10.0, "band": "g"} - - cutout = cutout_3_tiles - - ra = cutout["ra"] - dec = cutout["dec"] - size = cutout["size"] - band = cutout["band"] - # Calculates the cutout's vertices to access tiles - verts = cutout_verts(ra, dec, size) - print(verts) - # Set tiles from vertices - tile_list = Path("/app/cutout/service/coaddtiles-20121015.csv") - tiles = tiles_from_cat(verts, tile_list) - print(tiles) - - # Cutout Fits: - path_to_fits = Path("/data/tiles") - data, wcs_ = get_fits_data(ra, dec, size, tiles, band, path_to_fits) - # print(data) - - # Exemplo Cutout FITS - # result_path = Path("/data/results") - # filename = "{:.5f}_{:.5f}_{}.fits".format(round(ra, 5), round(dec, 5), band) - # filepath = result_path.joinpath(filename) - # if filepath.exists(): - # filepath.unlink() - # write_cutout_file(data, wcs_, filepath) - - # Exemplo Cutout PNG - # Usando 3 bandas, primeiro faz 3 cutouts fits em g, r, i - # Depois gera a png. - result_path = Path("/data/results") - filename = f"{round(ra, 5):.5f}_{round(dec, 5):.5f}.png" - filepath = result_path.joinpath(filename) - if filepath.exists(): - filepath.unlink() - print(filepath) - # Fits g, r, i - data_g, wcs_g = get_fits_data(ra, dec, size, tiles, "g", path_to_fits) - data_r, wcs_r = get_fits_data(ra, dec, size, tiles, "r", path_to_fits) - data_i, wcs_i = get_fits_data(ra, dec, size, tiles, "i", path_to_fits) - - cutout_lupton(data_g, data_r, data_i, 0.05, 10, 0.5, filepath) From 4861feb62d2ec21d9a924e2ac2ed869b857ebb28 Mon Sep 17 00:00:00 2001 From: glaubervila Date: Sat, 11 Jul 2026 13:34:57 -0300 Subject: [PATCH 4/4] Update stale tests to match current behavior Three tests had drifted from the code they exercise: - test_des_csv_locator: the locator builds file paths under tiles_root/{tilename}/ since ef23765, not under the archive path - test_cutout_parameters: engines defaults to ["astrocut"] when the parameter is absent - test_png_output: fits_cut is called with memory_only=True and returns in-memory HDULists since ea8eafe; mocks now follow that contract for both mono and RGB paths Full suite is green: 53 passed, 0 failed. Co-Authored-By: Claude Fable 5 --- .../discovery/tests/test_des_csv_locator.py | 2 +- .../service/tests/test_cutout_parameters.py | 4 +- tests/test_png_output.py | 40 ++++++------------- 3 files changed, 15 insertions(+), 31 deletions(-) diff --git a/cutout/service/discovery/tests/test_des_csv_locator.py b/cutout/service/discovery/tests/test_des_csv_locator.py index 07db772..3d87e31 100644 --- a/cutout/service/discovery/tests/test_des_csv_locator.py +++ b/cutout/service/discovery/tests/test_des_csv_locator.py @@ -23,7 +23,7 @@ def test_find_files_circle_returns_intersecting_tiles(tmp_path: Path) -> None: files = locator.find_files(survey_id="des_dr2", stencil=stencil, band="g") assert [f.tile_id for f in files] == ["TILE_A", "TILE_B"] - assert str(files[0].file_path).endswith("/Y6A1/r4907/TILE_A/p01/coadd/TILE_A_r4907p01_g.fits.fz") + assert str(files[0].file_path) == "/data/tiles/TILE_A/TILE_A_r4907p01_g.fits.fz" def test_find_files_range_returns_single_tile(tmp_path: Path) -> None: diff --git a/cutout/service/tests/test_cutout_parameters.py b/cutout/service/tests/test_cutout_parameters.py index ee415fb..c040a4a 100644 --- a/cutout/service/tests/test_cutout_parameters.py +++ b/cutout/service/tests/test_cutout_parameters.py @@ -17,7 +17,7 @@ def test_cutout_parameters_parses_engine() -> None: assert parsed.ids == ["des_dr2"] -def test_cutout_parameters_without_engine_keeps_empty_list() -> None: +def test_cutout_parameters_without_engine_defaults_to_astrocut() -> None: params = [ JobParameter(parameter_id="id", value="des_dr2"), JobParameter(parameter_id="pos", value="CIRCLE 10 0 1"), @@ -27,4 +27,4 @@ def test_cutout_parameters_without_engine_keeps_empty_list() -> None: parsed = CutoutParameters.from_job_parameters(params) - assert parsed.engines == [] + assert parsed.engines == ["astrocut"] diff --git a/tests/test_png_output.py b/tests/test_png_output.py index e5ffa1a..56e4070 100644 --- a/tests/test_png_output.py +++ b/tests/test_png_output.py @@ -1,25 +1,19 @@ -import numpy as np from pathlib import Path +import numpy as np from astropy.io import fits from cutout.service.cutout_engine.astrocut_engine import AstrocutEngine -def make_fits(path: Path, shape=(64, 64), value=1): - data = np.full(shape, value, dtype=float) - fits.writeto(path, data, overwrite=True) - return str(path) +def _hdulist(value: float, shape=(64, 64)) -> fits.HDUList: + data = np.full(shape, value, dtype=np.float32) + return fits.HDUList([fits.PrimaryHDU(data=data)]) def test_mono_png(tmp_path, monkeypatch): - in_fits = tmp_path / "in.fits" - make_fits(in_fits, value=42) - - def mock_fits_cut(**kwargs): - out = tmp_path / f"{kwargs.get('cutout_prefix','cut')}.fits" - make_fits(out, value=42) - return str(out) + def mock_fits_cut(input_files, coordinates, cutout_size, single_outfile=True, memory_only=True): + return [_hdulist(42)] monkeypatch.setattr("cutout.service.cutout_engine.astrocut_engine.fits_cut", mock_fits_cut) @@ -29,7 +23,7 @@ def mock_fits_cut(**kwargs): res = engine.run_cutout( source_id="des_dr2", stencil=stencil, - input_files=[str(in_fits)], + input_files=[str(tmp_path / "in.fits")], band="g", output_format="png", output_path=out_png, @@ -41,27 +35,17 @@ def mock_fits_cut(**kwargs): def test_rgb_png(tmp_path, monkeypatch): - def mock_fits_cut(input_files, coordinates, cutout_size, single_outfile, cutout_prefix, output_dir): - # produce file with value dependent on prefix suffix - val = 10 - if cutout_prefix.endswith("_g"): - val = 50 - elif cutout_prefix.endswith("_r"): - val = 100 - elif cutout_prefix.endswith("_i"): - val = 150 - out = Path(output_dir) / f"{cutout_prefix}.fits" - make_fits(out, value=val) - return str(out) + band_values = {"g1": 50.0, "r1": 100.0, "i1": 150.0} + + def mock_fits_cut(input_files, coordinates, cutout_size, single_outfile=True, memory_only=True): + stem = Path(str(input_files[0])).stem + return [_hdulist(band_values.get(stem, 10.0))] monkeypatch.setattr("cutout.service.cutout_engine.astrocut_engine.fits_cut", mock_fits_cut) engine = AstrocutEngine() stencil = {"type": "circle", "center": {"ra": 36.0, "dec": -10.0}, "radius": 1.0} - # prepare dummy original files (not used by mock but kept for clarity) in_map = {"g": [str(tmp_path / "g1.fits")], "r": [str(tmp_path / "r1.fits")], "i": [str(tmp_path / "i1.fits")]} - for v in in_map.values(): - make_fits(Path(v[0]), value=1) out_png = tmp_path / "out_rgb.png" res = engine.run_cutout(