diff --git a/.gitignore b/.gitignore index d0b1bb2..ba173ef 100644 --- a/.gitignore +++ b/.gitignore @@ -293,3 +293,5 @@ data/original_files/*.tar.gz data/tiles/* data/results/debug/* data/results/async/* +data/results/sync/* +!data/results/sync/.gitkeep diff --git a/AGENTS.md b/AGENTS.md index e0ceb60..d3c9fb8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -95,9 +95,3 @@ Para as proximas fases de codigo: 3. Testes executados dentro do container. 4. Lint executado dentro do container. 5. Nenhuma dependencia foi instalada no host. -6. Arquivo `STATUS_IMPLEMENTACAO_SYNC.md` atualizado com: - - resumo da fase - - arquivos alterados - - regras de negocio/decisoes - - validacoes executadas - - commit associado diff --git a/ANALISE_CUTOUT_PROTOTIPO.md b/ANALISE_CUTOUT_PROTOTIPO.md deleted file mode 100644 index 6cc4bea..0000000 --- a/ANALISE_CUTOUT_PROTOTIPO.md +++ /dev/null @@ -1,198 +0,0 @@ -# Analise do prototipo do app Django cutout - -Data da analise: 2026-04-26 - -## Objetivo desta analise -Avaliar o que foi realmente implementado no prototipo, com foco em: -- API e endpoints -- parametros de entrada (estilo VO/SODA/UWS) -- identificacao de arquivos FITS envolvidos no cutout por coordenada -- worker separado, fila e distribuicao de tarefas - -Tambem foi feita uma comparacao com referencias VO encontradas no codigo. - -## Referencias VO encontradas no repositorio - -### UWS (link explicito no codigo) -- cutout/service/uws/models.py - - https://www.ivoa.net/documents/UWS/20161024/REC-UWS-1.1-20161024.html -- cutout/service/models/job.py -- cutout/service/models/job_parameter.py -- cutout/service/models/job_result.py - - todos mencionam implementacao baseada no UWS 1.1 - -### SODA/VO (mencoes no codigo e anotacoes) -- cutout/service/api/views.py - - comentario sobre IVOA/SODA/WD-SODA 3.2.2 para parametro BAND - - comentario com referencia ao projeto lsst-sqre/vo-cutouts -- Anotacoes.md - - item sobre SODA BAND - - item sobre DALI 3.2.1 (-Inf/+Inf em intervalos) - - item sobre formato de erro esperado pelo SODA - -### Verificacao externa (VO) -Foi consultado o conteudo de: -- UWS 1.1 (IVOA) -- SODA 1.0 (IVOA) - -Resumo normativo relevante para este projeto: -- SODA: recursos sync e async, parametros como ID, POS/CIRCLE/POLYGON, BAND, TIME, POL, erros em text/plain com prefixos padrao (UsageError, MultiValuedParamNotSupported, etc). -- UWS: arvore de recursos de job (lista, job, phase, parameters, results, destruction, executionduration, abort/start), fases padronizadas e operacoes REST para controle de job. - -## O que esta implementado de fato - -### 1) Endpoints expostos hoje -Arquivos: -- config/urls.py -- config/api_router.py -- cutout/service/api/views.py - -Implementado: -- GET /api/cutout - - retorna mensagem fixa "Hello, world!" -- GET /api/sync - - aceita query params - - converte params para JobParameter - - cria job e chama inicio do processamento via JobService - - resposta atual: JSON {"success": true} - -Importante: -- Nao ha endpoint async estilo UWS exposto ao cliente (ex.: /jobs, /jobs/{id}, /jobs/{id}/phase, /jobs/{id}/results). -- JobRequestViewSet existe no codigo, mas nao esta registrado no router. -- SyncCutoutView tem muito codigo comentado de uma versao anterior (retorno de FITS/PNG por FileResponse), sem estar ativo na versao atual. - -### 2) Modelo de parametros (parcialmente VO-like) -Arquivos: -- cutout/service/api/views.py -- cutout/service/cutout_parameters.py -- cutout/service/stencils.py - -Implementado: -- Parametros aceitos/documentados no schema do endpoint sync: - - id - - pos (CIRCLE/RANGE/POLYGON dentro da string) - - runid - - format - - band -- Parser de stencil: - - POS com CIRCLE, RANGE, POLYGON - - CIRCLE e POLYGON diretos tambem sao aceitos no parser (pela logica parse_stencil) -- Validacao de negocio no policy: - - apenas 1 id - - apenas 1 stencil - - RANGE rejeitado no momento - -Lacunas importantes: -- Nao ha implementacao completa dos parametros SODA padrao (TIME, POL, etc). -- BAND foi modelado como string de banda (g,r,i,z,Y), nao como intervalo em metros (como no SODA 1.0). -- Multiplicidade de parametros existe no parser/listas, mas fluxo funcional esta restrito a um unico id e um unico stencil. - -### 3) Fila, worker separado e escalabilidade -Arquivos: -- config/celery_app.py -- config/settings/base.py -- docker-compose.yml -- production.yml -- cutout/service/tasks.py -- cutout/service/policy.py -- cutout/service/uws/service.py - -Implementado: -- Infra de fila baseada em Celery + Redis esta configurada. -- Servicos separados em compose para: - - django - - redis - - celeryworker - - celerybeat - - flower -- JobService chama policy.dispatch, que envia tarefas para Celery. - -Pontos ainda incompletos/inconsistentes: -- dispatch atual usa chord(headers[0])(callback), ou seja, efetivamente nao executa grupo completo de tarefas; so usa o primeiro header. -- callback job_completed nao atualiza corretamente estado UWS no banco (funcao retorna string de teste). -- task image_cutout retorna string fixa "Resultado do cutout 1", sem metadados de resultado UWS persistidos. -- transicoes de fase estao incompletas (QUEUED parcial; EXECUTING/COMPLETED/ERROR sem fluxo robusto de ponta a ponta). - -### 4) Identificacao dos arquivos FITS por coordenada -Arquivos: -- cutout/lib/des_cutout.py -- cutout/lib/base_cutout.py -- cutout/lib/cutout.py - -Implementado (parte cientifica do recorte para DES): -- calculo de vertices do cutout a partir de RA/DEC/size -- leitura de catalogo de tiles (dr2_tiles.csv) -- identificacao dos tiles que cobrem os vertices -- derivacao de paths de arquivos FITS comprimidos (.fits.fz) -- descompressao on-demand com funpack para /data/tmp -- leitura parcial via astropy Cutout2D e montagem quando cruza 1, 2 ou 3 tiles -- geracao de saida FITS -- geracao de PNG Lupton (gri) - -Lacunas: -- logica de identificacao de tile usa matching simples por bounding box e assume que sempre encontra indice valido (risco de excecao em bordas/falhas). -- nao ha camada de abstracao para varios surveys pronta em producao (apenas des_dr2 implementado). -- RANGE/POLYGON nao estao implementados de ponta a ponta no backend de execucao atual. - -### 5) Persistencia de jobs e resultados -Arquivos: -- cutout/service/models/job.py -- cutout/service/models/job_parameter.py -- cutout/service/models/job_result.py -- cutout/service/uws/models.py -- cutout/service/uws/job.py - -Implementado: -- tabelas para Job, JobParameter, JobResult -- fases UWS no modelo de Job -- run_id, owner, creation/start/end/destruction/execution_duration/quote - -Lacunas: -- escrita efetiva de JobResult durante processamento nao esta fechada -- erro UWS (errorSummary/detail) nao esta modelado e persistido de forma completa -- endpoints de consulta/gestao de jobs nao estao expostos - -## Comparacao objetiva com o esperado (VO + requisitos do projeto) - -### Requisito: API VO-compliant para cutout -Status: PARCIAL -- Existe tentativa de alinhar com UWS/SODA em nomenclatura e parser de POS. -- Falta contrato completo de API SODA/UWS (principalmente endpoints async UWS e respostas/payloads padronizados). - -### Requisito: receber coordenada/tamanho ou lista -Status: PARCIAL -- Coordenada unica com POS=CIRCLE entra no fluxo. -- Lista/multiplos valores nao esta pronta no fluxo final (restrito por policy). - -### Requisito: identificar arquivos pela coordenada e gerar resultado -Status: PARCIALMENTE IMPLEMENTADO -- Identificacao de tiles/arquivos DES e recorte FITS existe no backend cientifico. -- Integracao robusta com retorno de resultado via API/UWS ainda incompleta. - -### Requisito: worker separado e escalavel -Status: BASE IMPLEMENTADA, FLUXO INCOMPLETO -- Infra de worker separado e fila existe (Celery + Redis + containers dedicados). -- Orquestracao completa do ciclo de vida dos jobs e resultados ainda incompleta. - -### Requisito: sistema de fila e distribuicao de tasks -Status: PARCIAL -- Fila existe. -- Distribuicao atual de tarefas em chord/group nao esta finalizada para multiplos itens. - -## Principais gaps para concluir o servico - -1. Expor API UWS/SODA completa (sync + async) com recursos de job padronizados. -2. Implementar respostas de erro no formato SODA (text/plain com prefixos corretos). -3. Finalizar ciclo de vida do job: PENDING -> QUEUED -> EXECUTING -> COMPLETED/ERROR/ABORTED. -4. Persistir resultados reais (JobResult com URL, mime_type, size) e disponibilizar download. -5. Implementar de fato multiplos inputs (lista de coordenadas e combinacoes de parametros) de forma escalavel. -6. Revisar parametro BAND para compatibilidade com SODA (ou declarar extensao custom com metadados claros). -7. Implementar endpoints/fluxos de monitoramento e gestao de jobs do usuario. -8. Fechar politicas de retencao/garbage collector para resultados e temporarios. - -## Conclusao executiva -O projeto ja tem uma base tecnica relevante: parser de stencils, modelos de job inspirados em UWS, infraestrutura Celery/Redis com worker separado e implementacao cientifica de cutout DES com identificacao de tiles por coordenada. - -Porem, no estado atual ele ainda e um prototipo incompleto no que mais importa para operacao VO-compliant: contrato de API UWS/SODA de ponta a ponta, fluxo async padronizado, ciclo de vida completo de jobs e publicacao de resultados/erros no formato esperado. - -Em resumo: o nucleo de processamento existe, a infraestrutura de fila existe, mas a camada de produto/API padrao VO ainda nao esta finalizada. \ No newline at end of file diff --git a/Anotacoes.md b/Anotacoes.md index 596c085..6d82a8d 100644 --- a/Anotacoes.md +++ b/Anotacoes.md @@ -37,14 +37,14 @@ Duvidas: - [X] Endpoint Sync para cutouts DES POS Circle Fits. - [X] Cutout Sync em background com Celery. - [x] Endpoint Sync para cutouts DES POS Circle Png com `engine=astrocut` (corrigido naming de saida). -- [] Endpoint Sync para cutouts DES POS Polygon Jpg. -- [] função para Cutout DES usando Range de posições. -- [] função para Cutout DES usando Poligono. -- [] Registrar a criação do cutout sync na listas de job do usuario??? -- [] PNGs só tem opção de gerar imagens coloridas, usando gri. (Acredito que de para ter mais opções, png para uma banda só ou outras combinações.) +- [x] Endpoint Sync para cutouts DES POS Polygon Jpg. +- [x] função para Cutout DES usando Range de posições. +- [x] função para Cutout DES usando Poligono. +- [x] Registrar a criação do cutout sync na listas de job do usuario??? +- [x] PNGs só tem opção de gerar imagens coloridas, usando gri. (Acredito que de para ter mais opções, png para uma banda só ou outras combinações.) - [] Fluxo sync `engine=legacy` + `format=png` ainda com falha (investigar stack completa e cobrir com teste de regressao). - [] Criar exemplos de uso da api no Jupyter Notebook. -- [] Remover o metodo "legacy" (código do adriano) e deixar apenas o astrocut??? +- [x] Remover o metodo "legacy" (código do adriano) e deixar apenas o astrocut??? ## Cutout Sync LSST @@ -55,15 +55,15 @@ Duvidas: ## Cutout Async DES -- [] Submeter lista de coordenadas para processamento em background com celery. +- [x] Submeter lista de coordenadas para processamento em background com celery. - [] Email com informação do job asyncrono. (opcional) - [] Definir Limit qtd/tamanho de Cutouts ## Monitoramento dos Jobs -- [] Regitrar no banco de dados cada cutout criado pelo usuario. (Incluir Cutouts Sync?) +- [x] Regitrar no banco de dados cada cutout criado pelo usuario. (Incluir Cutouts Sync?) - [] Interface com a lista de Jobs -- [] Download dos resultados do job. +- [x] Download dos resultados do job. - [] Cancelar/Deletar um Job. - [] Mostrar uso da quota? - [] Limite de Jobs? diff --git a/CURL_TESTES_SYNC.md b/CURL_TESTES_SYNC.md deleted file mode 100644 index a1945f9..0000000 --- a/CURL_TESTES_SYNC.md +++ /dev/null @@ -1,146 +0,0 @@ -# Comandos curl para testar o endpoint sync - -## 1) Teste sem autenticacao (esperado: 401) - -```bash -curl -i "http://localhost:8000/api/sync?id=des_dr2&pos=CIRCLE%2036.30911%20-10.18749%202&format=fits&band=g" -``` - -## 2) Obter token de autenticacao - -```bash -curl -s -X POST "http://localhost:8000/auth-token/" \ - -H "Content-Type: application/json" \ - -d '{"username":"SEU_USUARIO","password":"SUA_SENHA"}' -``` - -Saida esperada: - -```json -{"token":"..."} -``` - -## 3) Exportar token em variavel de ambiente - -```bash -TOKEN="SEU_TOKEN_AQUI" -``` - -## 4) Chamar endpoint sync autenticado (engine astrocut) - -```bash -curl -i -G "http://localhost:8000/api/sync" \ - -H "Authorization: Token $TOKEN" \ - --data-urlencode "id=des_dr2" \ - --data-urlencode "pos=CIRCLE 36.30911 -10.18749 2" \ - --data-urlencode "engine=astrocut" \ - --data-urlencode "format=fits" \ - --data-urlencode "band=g" -``` - -- Se houver todos os arquivos de entrada e o processamento concluir, retorna 200 com arquivo. -- Se faltar algum FITS esperado localmente, retorna 422 com `Input file unavailable`. - -## 5) Chamar endpoint sync autenticado (engine legacy) - -```bash -curl -i -G "http://localhost:8000/api/sync" \ - -H "Authorization: Token $TOKEN" \ - --data-urlencode "id=des_dr2" \ - --data-urlencode "pos=CIRCLE 36.30911 -10.18749 0.01" \ - --data-urlencode "engine=legacy" \ - --data-urlencode "format=fits" \ - --data-urlencode "band=g" -``` - -## 6) Salvar retorno binario em arquivo local (quando houver 200) - -```bash -curl -s -G "http://localhost:8000/api/sync" \ - -H "Authorization: Token $TOKEN" \ - --data-urlencode "id=des_dr2" \ - --data-urlencode "pos=CIRCLE 36.30911 -10.18749 0.01" \ - --data-urlencode "engine=astrocut" \ - --data-urlencode "format=fits" \ - --data-urlencode "band=g" \ - -o sync_result_test.fits -``` - -## 7) Exemplo para validar negacao de acesso por policy (esperado: 403) - -```bash -curl -i -G "http://localhost:8000/api/sync" \ - -H "Authorization: Token $TOKEN" \ - --data-urlencode "id=private_survey" \ - --data-urlencode "pos=CIRCLE 36.30911 -10.18749 0.01" \ - --data-urlencode "engine=astrocut" \ - --data-urlencode "format=fits" \ - --data-urlencode "band=g" -``` - -```bash -curl -i -G "http://localhost:80/api/sync" \ - -H "Authorization: Token 89cce9976177ed2c275b75782290faa915089518" \ - --data-urlencode "id=des_dr2" \ - --data-urlencode "pos=CIRCLE 36.30911 -10.18749 2" \ - --data-urlencode "engine=astrocut" \ - --data-urlencode "format=png" \ - --data-urlencode "band=gri" --o sync_result_test.png -``` - - -## 8) Fluxo completo em duas linhas (token + teste) - -```bash -TOKEN=$(curl -s -X POST "http://localhost:8000/auth-token/" -H "Content-Type: application/json" -d '{"username":"SEU_USUARIO","password":"SUA_SENHA"}' | python -c "import sys,json; print(json.load(sys.stdin)['token'])") -curl -i -G "http://localhost:8000/api/sync" -H "Authorization: Token $TOKEN" --data-urlencode "id=des_dr2" --data-urlencode "pos=CIRCLE 36.30911 -10.18749 0.01" --data-urlencode "engine=astrocut" --data-urlencode "format=fits" --data-urlencode "band=g" -``` - -## 9) Ajustar ownership de arquivo gerado no container - -Quando o arquivo de saida for criado dentro do container com owner `root:root`, ajuste para uid/gid do host: - -```bash -docker compose exec django sudo chown 1000:1000 /data/results/teste.fits -``` - -Para verificar: - -```bash -docker compose exec django ls -l /data/results/teste.fits -``` - -Coordenadas de exemplo: cutout/lib/des_cutout.py - -```python -if __name__ == "__main__": - cutouts = [ - {"ra": 36.30911, "dec": -10.18749, "size": 2.0, "band": "g", "format": "fits"}, # 1 - Tile - {"ra": 36.30911, "dec": -10.18749, "size": 2.0, "band": "gri", "format": "png"}, # 1 - Tile - {"ra": 36.15801, "dec": -10.33579, "size": 2.0, "band": "g", "format": "fits"}, # 2 - Tile - {"ra": 36.15801, "dec": -10.33579, "size": 2.0, "band": "gri", "format": "png"}, # 2 - Tile - # {"ra": 35.23676, "dec": -10.33269, "size": 10.0, "band": "g", "format": "fits"}, # 3 - Tile - ] - - dc = DesCutout() - - for c in cutouts: - if c["format"] == "fits": - filename = "{:.5f}_{:.5f}_{}.fits".format(round(c["ra"], 5), round(c["dec"], 5), c["band"]) - resultfile = Path("/data/results").joinpath(filename) - - result = dc.single_cutout_fits( - ra=c["ra"], dec=c["dec"], size_arcmin=c["size"], band=c["band"], path=resultfile - ) - print(result) - - if c["format"] == "png": - filename = "{:.5f}_{:.5f}.png".format(round(c["ra"], 5), round(c["dec"], 5)) - resultfile = Path("/data/results").joinpath(filename) - - result = dc.single_cutout_png( - ra=c["ra"], dec=c["dec"], size_arcmin=c["size"], band=c["band"], path=resultfile - ) - print(result) -``` diff --git a/PLANO_ASYNC.md b/PLANO_ASYNC.md deleted file mode 100644 index 879bdd5..0000000 --- a/PLANO_ASYNC.md +++ /dev/null @@ -1,217 +0,0 @@ -# Plano de Implementacao - API Async UWS/SODA - -Data de inicio: 2026-05-23 -Escopo: implementar a superficie async do servico de cutout com UWS/SODA, usando workers em background, sem executar ainda o cutout cientifico real. -Publico: humanos e agentes de AI. - -## Objetivo desta fase - -Implementar a camada de API async para: - -- receber os parametros corretos do pedido de cutout; -- criar jobs UWS no banco; -- enfileirar execucao em workers; -- expor endpoints para acompanhar job, parametros, fase e resultados; -- persistir resultados fake; -- manter a assinatura da funcao de worker compativel com a futura execucao real. - -## Decisoes atuais - -1. O endpoint sync existente permanece como esta. -2. A arvore async sera exposta em `/api/async`. -3. Nesta fase, o worker executa uma funcao fake, mas com assinatura compativel com o pipeline real. -4. O foco agora e fechar contrato de API, fila, transicoes de fase e retrieval de resultados. -5. A semantica final de erro SODA/DALI em `text/plain` continua adiada; vamos priorizar a estrutura UWS e o ciclo de vida dos jobs. - -## Superficie de API planejada - -### Colecao de jobs - -- `POST /api/async` - - cria um job async; - - aceita parametros de cutout; - - pode iniciar execucao imediatamente quando apropriado; - - retorna `303 See Other` com `Location` apontando para o recurso do job. - -- `GET /api/async` - - lista jobs do usuario autenticado. - -### Recurso de job - -- `GET /api/async/{job_id}` - - retorna metadados do job. - -- `DELETE /api/async/{job_id}` - - remove o job do usuario; se necessario, aborta antes. - -### Fase do job - -- `GET /api/async/{job_id}/phase` - - retorna a fase atual do job. - -- `POST /api/async/{job_id}/phase` - - aceita ao menos `PHASE=RUN` e `PHASE=ABORT`. - -### Parametros do job - -- `GET /api/async/{job_id}/parameters` - - retorna os parametros persistidos do job. - -### Resultados do job - -- `GET /api/async/{job_id}/results` - - retorna a lista de resultados do job. - -- `GET /api/async/{job_id}/results/{result_id}` - - entrega ou redireciona para o artefato do resultado. - -## Parametros da primeira iteracao - -Nesta fase, a API async vai aceitar o mesmo conjunto de parametros hoje usado no fluxo sync atual: - -- `id` -- `pos` -- `runid` -- `format` -- `band` -- `engine` -- `color` -- `rgb_bands` -- `persist` - -## Regras de ciclo de vida - -Fluxo esperado: - -1. Criar job em `PENDING`. -2. Ao disparar execucao, registrar `message_id` e mover para `QUEUED`. -3. Worker ao iniciar move para `EXECUTING`. -4. Worker fake ao concluir persiste resultado(s) e move para `COMPLETED`. -5. Em falha, mover para `ERROR`. -6. Em cancelamento, mover para `ABORTED`. - -## Worker fake - -O worker fake deve manter a assinatura funcional esperada do processamento real: - -`fake_image_cutout(job_id, source_id, stencil, engine, band, format, path, files=None, color=False, rgb_bands=None, persist=False)` - -Ele pode gerar um arquivo pequeno em disco contendo um resumo dos parametros recebidos. Esse arquivo sera usado para validar: - -- persistencia do resultado; -- endpoint de resultados; -- download do artefato. - -## Evolucoes de modelo previstas - -### Job - -Manter o modelo atual e expandir o service para suportar: - -- listagem por usuario; -- detalhe por usuario; -- start; -- abort; -- delete; -- registro de erro; -- registro de resultados. - -### JobResult - -Precisamos permitir retrieval real do resultado async. A direcao atual e adicionar campos para guardar o caminho e/ou URL do artefato persistido. - -## Arquivos principais previstos - -- `PLANO_ASYNC.md` -- `config/api_router.py` -- `cutout/service/api/views.py` -- `cutout/service/api/serializers.py` -- `cutout/service/uws/service.py` -- `cutout/service/uws/models.py` -- `cutout/service/policy.py` -- `cutout/service/tasks.py` -- `cutout/service/models/job_result.py` -- `cutout/service/migrations/*` -- testes em `cutout/service/tests/` ou pasta equivalente - -## Sequencia de implementacao - -### Fase 1 - scaffold da API async - -- registrar rotas async; -- criar views para colecao, job, phase, parameters e results; -- centralizar parse de parametros. - -### Fase 2 - service layer - -- expandir `JobService` para ciclo completo do async; -- garantir ownership; -- implementar conversao de resultados persistidos. - -### Fase 3 - fila e worker fake - -- enfileirar execucao via Celery; -- registrar `message_id`; -- criar funcao fake e task de orquestracao do job; -- persistir `JobResult`. - -### Fase 4 - resultados e download - -- listar resultados; -- expor download por `result_id`. - -### Fase 5 - testes e ajustes - -- cobrir criacao, ownership, fases, resultados e download; -- atualizar este arquivo com decisoes relevantes conforme avancarmos. - -## Status atual - -- Scaffold principal da API async implementado. -- Endpoints async registrados em `config/api_router.py`. -- Job async criado e disparado automaticamente via worker fake. -- Persistencia de resultados fake implementada em `JobResult` com `url` e `file_path`. -- Download de resultado por `result_id` implementado. -- Testes da API async adicionados e passando. - -## Criterios de aceite - -1. `POST /api/async` cria job e retorna localizacao do recurso. -2. O job pode ser iniciado e passa por `QUEUED -> EXECUTING -> COMPLETED`. -3. `GET /api/async/{job_id}` reflete a fase correta. -4. `GET /parameters` retorna os parametros persistidos. -5. `GET /results` lista resultados fake persistidos. -6. `GET /results/{result_id}` entrega um artefato real. -7. Jobs de outros usuarios nao ficam acessiveis. - -## Registro de decisoes - -### 2026-05-23 - -- O arquivo de acompanhamento da fase async sera `PLANO_ASYNC.md`. -- A implementacao inicial prioriza a arvore de recursos UWS e o ciclo de vida dos jobs. -- A funcao cientifica principal continua fora de escopo nesta etapa. -- `POST /api/async` vai iniciar o job imediatamente por padrao, tratando ausencia de `PHASE` como `RUN`. -- Os detalhes de job, parametros e resultados serao retornados em JSON nesta primeira entrega, mantendo `GET/POST` de `phase` em formato textual. -- `JobResult` passa a persistir `url` e `file_path` para permitir listagem e download real do artefato fake. -- A task async fake recebe os mesmos argumentos estruturais do cutout real e grava um artefato simples em disco para exercitar retrieval. -- A serializacao enviada ao worker async usa apenas dados JSON-safe; objetos de stencil ficam restritos ao fluxo sync. -- O lint foi explicitamente postergado nesta etapa para priorizar o fluxo funcional da API async. - -## Validacoes executadas - -- `pytest cutout/service/tests/test_async_api.py -q` -> 4 passed - -## Arquivos alterados nesta etapa - -- `PLANO_ASYNC.md` -- `config/api_router.py` -- `cutout/service/api/views.py` -- `cutout/service/api/serializers.py` -- `cutout/service/models/job_result.py` -- `cutout/service/migrations/0003_jobresult_storage_fields.py` -- `cutout/service/policy.py` -- `cutout/service/tasks.py` -- `cutout/service/tests/test_async_api.py` -- `cutout/service/uws/models.py` -- `cutout/service/uws/service.py` diff --git a/PLANO_SYNC_IVOA_DES_DESCOBERTA.md b/PLANO_SYNC_IVOA_DES_DESCOBERTA.md deleted file mode 100644 index a4377b3..0000000 --- a/PLANO_SYNC_IVOA_DES_DESCOBERTA.md +++ /dev/null @@ -1,521 +0,0 @@ -# Plano de Implementacao: Sync Cutout com Descoberta de Arquivos (IVOA) - -Data: 2026-04-26 -Escopo: endpoint sync primeiro, com modulo separado de descoberta de arquivos para DES -Publico: humanos + agentes de AI - -Atualizacao de prioridade (2026-04-26): -- Este plano foi reordenado para refletir a execucao por prioridade de negocio (1 a 6). -- A semantica de erro em `text/plain` continua no plano geral SODA, mas foi explicitamente movida para depois da fase async. - -## 1) Status de protocolos IVOA (atualizacao) - -Baseado na pagina de standards do IVOA (https://www.ivoa.net/documents/): - -- UWS: 1.1 (Recommendation, 2016-10-24) -- SODA: 1.0 (Recommendation, 2017-05-17) -- SIA: 2.0 (Recommendation, 2015-12-23) -- DataLink: 1.1 (Recommendation, 2023-12-15) -- DALI: 1.1 e 1.2 em progresso (PR no ciclo 2025) - -Conclusao pratica para este projeto: -- Continuar modelando execucao assinc com UWS 1.1. -- Continuar modelando operacao de cutout com SODA 1.0. -- Para descoberta de arquivos por coordenada, usar padrao de descoberta SIA/ObsCore (ou TAP/ObsCore) + DataLink quando necessario. - -## 2) Protocolo para "identificacao dos arquivos envolvidos" - -Pergunta: existe um protocolo IVOA especifico para "dado coordenada, retornar lista de arquivos"? - -Resposta curta: -- Nao ha um protocolo unico e isolado so para isso. -- O fluxo VO recomendado e: - 1. Descoberta de datasets por posicao com SIA 2.0 (ou TAP/ObsCore). - 2. Se um dataset tiver multiplos arquivos/derivacoes, usar DataLink para listar e qualificar links/arquivos/servicos. - 3. Depois chamar SODA para recorte (sync ou async). - -Para este projeto (DES): -- Fase inicial: descoberta via catalogo local (CSV) e retorno de lista de arquivos internos. -- Evolucao: trocar o backend de descoberta para SIA/TAP/DataLink sem mudar contrato interno do sistema. - -## 3) Objetivo tecnico imediato - -Implementar o endpoint sync com pipeline claro: -1. Usuario envia coordenada + tamanho + parametros basicos. -2. Camada de policy valida se o usuario pode acessar o survey solicitado. -3. Modulo de descoberta identifica os arquivos DES envolvidos. -4. Modulo de cutout executa em task Celery (backend atual; depois astrocut). -5. Endpoint sync devolve resultado (arquivo ou redirect para resultado), mantendo semantica SODA-like. - -Escopo funcional do endpoint sync nesta fase: -- aceitar os 3 tipos de pedido espacial: CIRCLE, RANGE e POLYGON -- para DES DR2 (publico), policy inicial retorna sempre true - -## 4) Arquitetura alvo (modular) - -## 4.1 Modulos novos (separados) - -### A) modulo de descoberta de arquivos (novo) -Responsabilidade: -- Dado stencil (CIRCLE inicialmente) e survey/release, retornar lista de arquivos candidatos para cutout. - -Contrato interno sugerido: - -```python -class FileLocator(Protocol): - def find_files( - self, - *, - survey_id: str, - stencil: dict, - band: str | None = None, - ) -> list[FileDescriptor]: - ... -``` - -`FileDescriptor` minimo: -- dataset_id (opcional no inicio) -- file_path (ou URI) -- tile_id -- band -- metadados minimos de cobertura (opcional na v1) - -Implementacao v1 (DES): -- `DesCsvFileLocator` -- Fonte: arquivo local com cobertura/tile/path - -Implementacoes futuras: -- `SiaFileLocator` (consulta SIA 2.0) -- `TapObsCoreFileLocator` (consulta TAP/ObsCore) -- `DataLinkResolver` (expande dataset em links/arquivos) - -### B) modulo de motor de cutout (abstracao) -Responsabilidade: -- Receber lista de arquivos + stencil + formato e produzir resultado. - -Contrato interno sugerido: - -```python -class CutoutEngine(Protocol): - def run_cutout( - self, - *, - files: list[FileDescriptor], - stencil: dict, - output_format: str, - band: str | None, - output_path: str, - ) -> CutoutResult: - ... -``` - -Implementacao v1: -- adapter sobre implementacao atual (DesCutout/base_cutout) - -Implementacao v2: -- `AstrocutEngine` (troca transparente) - -### C) orquestrador de job sync -Responsabilidade: -- Validar parametros -- Aplicar policy de acesso ao survey -- Chamar file locator -- Enfileirar task Celery -- Aguardar ate timeout sync -- Responder com arquivo ou redirect - -### D) camada de policy de acesso por survey (novo) -Responsabilidade: -- Centralizar decisao de autorizacao para uso de survey/release no cutout. -- Permitir evolucao para surveys privados sem mudar endpoint e sem acoplamento com regra de negocio especifica. - -Contrato interno sugerido: - -```python -class SurveyAccessPolicy(Protocol): - def can_request_cutout( - self, - *, - user_id: str, - survey_id: str, - release: str | None = None, - ) -> bool: - ... -``` - -Implementacao v1 (agora): -- `DesPublicAccessPolicy` -- Comportamento: retorna sempre `True` para DES DR2 (dados publicos de teste) - -Implementacoes futuras: -- `RoleBasedSurveyAccessPolicy` -- `ExternalAuthSurveyAccessPolicy` (LDAP/SSO/servico interno) - -## 4.2 Fluxo Sync (alto nivel) - -1. `GET/POST /api/sync` -2. Parse parametros (id/pos/format/band/runid) -3. Converter parametro espacial para stencil interno (CIRCLE, RANGE ou POLYGON) -4. `survey_access_policy.can_request_cutout(...)` -5. Se policy negar: retornar erro de autorizacao no formato padrao -6. `file_locator.find_files(...)` -7. Se vazio: retornar 204 (SODA sync sem pixels/sem match) -8. Enfileirar `run_sync_cutout_task.delay(...)` -9. Esperar resultado por janela limitada (ex: 20-30s) -10. Sucesso: -- opcao A: `FileResponse` -- opcao B: `303` para URL de resultado -11. Erro: -- resposta `text/plain` com prefixos SODA/DALI (UsageError, Error, ServiceUnavailable, MultiValuedParamNotSupported) - -## 5) Contrato API Sync (proposta inicial) - -## 5.1 Entrada (fase 1) -- `id` (ex: des_dr2) -- parametro espacial (3 formas suportadas no endpoint): - - `pos=CIRCLE ra dec radius` - - `pos=RANGE ra_min ra_max dec_min dec_max` - - `pos=POLYGON ra1 dec1 ra2 dec2 ra3 dec3 ...` -- `format` (`fits` inicialmente; `png` opcional) -- `band` (custom DES, enquanto nao houver BAND padrao em metros) -- `runid` (opcional) - -Observacao de compatibilidade VO: -- Em SODA 1.0, BAND padrao e intervalo em metros. -- Para DES no curto prazo, manter parametro custom de banda por letra, mas documentar como extensao local. - -## 5.2 Saida -- 200 + stream de arquivo (ou 303 para URL de arquivo) -- 204 sem conteudo quando nao houver sobreposicao/arquivos -- 4xx/5xx em `text/plain` com prefixo de erro padronizado -- 403 para survey sem permissao (quando policy negar) - -Exemplo de erro: -- `UsageError: invalid POS value` -- `MultiValuedParamNotSupported: only one POS in sync mode` -- `AuthorizationError: user has no access to requested survey` - -## 6) Plano de execucao em fases - -## Fase 0 - Hardening do prototipo atual -Objetivo: -- Limpar caminho minimo do sync para ficar previsivel. - -Entregaveis: -- remover codigo morto/comentado do endpoint -- padronizar parse/validacao de parametros -- padronizar suporte de parse para CIRCLE/RANGE/POLYGON no endpoint -- padronizar respostas de erro text/plain - -Criterios de aceite: -- endpoint sync com comportamento estavel para sucesso/falha -- testes de erro basicos passando - -## Fase 1 - Modulo separado de descoberta (DES CSV) -Objetivo: -- introduzir separacao formal entre API e descoberta de arquivos - -Entregaveis: -- pacote novo, ex: `cutout/service/discovery/` -- interface `FileLocator` -- implementacao `DesCsvFileLocator` -- testes unitarios para descoberta por CIRCLE/RANGE/POLYGON - -Criterios de aceite: -- dado POS CIRCLE/RANGE/POLYGON conhecido, retorna lista coerente de arquivos -- endpoint sync usa o locator (sem chamar logica de tile diretamente) - -## Fase 1.5 - Camada de policy de acesso -Objetivo: -- introduzir camada de autorizacao por survey, desacoplada da API e da descoberta. - -Entregaveis: -- pacote novo, ex: `cutout/service/policies/` -- interface `SurveyAccessPolicy` -- implementacao inicial `DesPublicAccessPolicy` (retorna true) -- integracao da policy no fluxo do endpoint sync - -Criterios de aceite: -- endpoint chama policy antes da descoberta/cutout -- para DES DR2, comportamento permanece liberado -- erro de autorizacao padronizado para surveys futuros privados - -## Fase 2 - Orquestracao sync via Celery -Objetivo: -- executar pipeline completo no worker, mantendo semantica sync no endpoint - -Entregaveis: -- task Celery `run_sync_cutout_task` -- payload de task contendo stencil + files + output spec -- retorno com caminho/URL do resultado - -Criterios de aceite: -- endpoint sync gera resultado real -- tempos de timeout tratados com erro claro -- fase de job no banco ao menos em PENDING/QUEUED/COMPLETED/ERROR para sync - -## Fase 3 - Adapter de motor de cutout -Objetivo: -- preparar troca para astrocut sem quebrar API - -Entregaveis: -- interface `CutoutEngine` -- adapter engine atual -- stub/integracao inicial `AstrocutEngine` - -Criterios de aceite: -- troca de engine por configuracao -- mesmos testes de endpoint continuam validos - -## Fase 4 - Caminho VO de descoberta (futuro) -Objetivo: -- adicionar backend de descoberta por SIA/TAP/DataLink - -Entregaveis: -- `SiaFileLocator` e/ou `TapObsCoreFileLocator` -- opcional `DataLinkResolver` - -Criterios de aceite: -- mesmo endpoint sync funcionando com backend local ou remoto - -## 7) Estrutura de codigo sugerida - -```text -cutout/service/ - api/ - views.py - policies/ - __init__.py - base.py # SurveyAccessPolicy - des_public.py # retorna True para DES DR2 - discovery/ - __init__.py - base.py # protocolos/interfaces - des_csv_locator.py # v1 - models.py # FileDescriptor - cutout_engine/ - __init__.py - base.py # CutoutEngine - des_engine.py # adapter atual - astrocut_engine.py # futuro - orchestration/ - sync_service.py # fluxo endpoint -> locator -> celery - tasks.py # tasks celery -``` - -## 8) Plano de testes (humanos + AI) - -## 8.1 Unitarios -- parse POS (CIRCLE/RANGE/POLYGON validos e invalidos) -- DES CSV locator (match unico, multiplos tiles, vazio) para os 3 stencils -- policy de acesso (DES publico -> true) -- mapeamento de erros para prefixos SODA - -## 8.2 Integracao -- endpoint sync -> celery -> resultado FITS -- caso sem overlap -> 204 -- timeout do worker -> ServiceUnavailable -- endpoint sync com CIRCLE, RANGE e POLYGON -- fluxo com policy aplicada antes da descoberta - -## 8.3 Contrato -- validacao de content-type de erro (`text/plain`) -- validacao de status codes esperados - -## 9) Decisoes abertas (precisam de definicao) - -1. No sync, retorno principal sera `200 file stream` ou `303 para URL`? -2. Limite de tempo do sync (timeout oficial)? -3. Parametro `band` custom DES sera mantido temporariamente com qual nome oficial? -4. Formatos suportados na fase 1: apenas FITS ou FITS+PNG? -5. Persistencia de resultados sync: por quanto tempo? - -## 10) Backlog pronto para execucao por agentes AI - -## Sprint A (infra de codigo) -- criar interfaces `SurveyAccessPolicy`, `FileLocator` e `CutoutEngine` -- criar `DesPublicAccessPolicy` (always true) -- criar `DesCsvFileLocator` -- adicionar testes unitarios de policy e locator - -## Sprint B (sync funcional) -- criar `SyncCutoutService` (orquestracao) -- integrar endpoint `/api/sync` ao service -- garantir suporte no endpoint para CIRCLE/RANGE/POLYGON -- criar task celery unica para sync cutout -- implementar respostas de erro padrao - -## Sprint C (qualidade) -- testes de integracao endpoint+celery -- metrica/log estruturado por runid/job -- documentacao OpenAPI atualizada - -## Definition of Done (DoD) -- endpoint sync funcionando ponta a ponta com descoberta separada -- endpoint sync suportando CIRCLE, RANGE e POLYGON -- camada de policy integrada no fluxo (DES DR2 liberado) -- descoberta DES desacoplada da API -- task em worker separado via Celery -- testes cobrindo sucesso, vazio, erro e timeout -- documento de contrato API atualizado - -## 11) Nota de compatibilidade com o estado atual do repositorio - -Este plano assume refatoracao incremental sobre o prototipo existente, sem reescrever tudo de uma vez. -Prioridade imediata: -- estabilizar sync -- extrair descoberta de arquivos para modulo proprio -- manter possibilidade de troca do motor de recorte para astrocut. - -## 12) Replanejamento priorizado (ordem de execucao) - -## 12.1 Panorama do que ja esta pronto - -- Discovery DES/CSV desacoplado com suporte de parse e descoberta para CIRCLE, RANGE e POLYGON. -- Policy de acesso por survey integrada antes do discovery. -- Pipeline sync com Celery e fases de job (PENDING/QUEUED/EXECUTING/COMPLETED/ERROR). -- Selecao de engine por parametro (`astrocut` default e `legacy` disponivel). -- Astrocut nativo atualmente operacional para `format=fits` com stencil `circle`. - -## 12.2 Prioridade 1 - Suporte a PNG (inclui PNG colorido) - -Objetivo: -- Adicionar `format=png` no pipeline sync, com suporte a PNG monocromatico e RGB. - -Decisao de compatibilidade IVOA: -- Em SODA 1.0 nao existe parametro padrao para "PNG colorido"/composicao RGB. -- SODA define parametros como ID, POS/CIRCLE/POLYGON, BAND, TIME e POL; custom parameters sao permitidos. -- Portanto, o modo colorido sera extensao local documentada no contrato. - -Contrato proposto: -- `format=png` ativa saida PNG. -- Novo parametro `color` (boolean): - - `color=false` (default): PNG de banda unica. - - `color=true`: PNG RGB composto. -- Novo parametro opcional `rgb_bands` (string): default `gri`. - - Exemplo: `rgb_bands=gri` ou `rgb_bands=riz`. -- Regra de negocio: - - Se `color=true` e `band` vier com banda unica, `rgb_bands` prevalece para composicao. - - Se `color=false`, usa apenas `band`. - -## 12.3 Depuração: plano de ação passo-a-passo (2026-04-26) - -Contexto: -- Foram observados arquivos gerados incorretos: FITS que nao abre e PNG aparentemente vazio. A implementação legacy tambem nao gerou arquivos. - -Objetivo: -- Fazer debug sistematico na cadeia de criação de cutouts, do mais simples ao composto, garantindo que o caminho FITS mono funcione antes de avançar. - -Passos: -1. Validar `single_cutout_fits` da implementação legacy (`DesCutout`) usando um script de depuração que execute um corte mono (uma banda) e escreva um FITS em `/data/results`. Verificar integridade com `astropy.io.fits.open`. -2. Validar `single_cutout_png` (mono) usando o mesmo script: geracao via `single_cutout_png` e abertura com PIL/visualizacao simples. -3. Validar composição RGB (`single_cutout_png` com `band='gri'` ou `color=true` na pipeline nova): garantir que cada banda produz um FITS parcial e que o empilhamento gera um PNG visivel. -4. Se qualquer passo falhar, acrescentar logs detalhados (shapes, min/max, nan counts, dtype) e pequenos scripts de inspeção para cada FITS temporario criado. - -Entregaveis: -- `scripts/debug_cutout.py` (script de depuracao que reproduz as chamadas comentadas em `des_cutout.py` e salva artefatos em `/data/results/debug/`). -- Atualizacao breve de `STATUS_IMPLEMENTACAO_SYNC.md` com o registro do problema, passos de depuracao e responsavel. - -Critérios de aceite para depuracao: -- `scripts/debug_cutout.py` executa dentro do container e produz um FITS mono legivel por `astropy.io.fits`. -- PNG mono criado abre com PIL e tem bytes > 0. -- Logs e artefatos temporarios gravados em `/data/results/debug/` para inspeção. - -Sequencia desejada de acao: -1. Criar `scripts/debug_cutout.py` e commitar. -2. Executar o script dentro do container e coletar logs/artefatos. -3. Corrigir a funcao que falha (se for o caso) e reexecutar. -4. Depois do FITS mono OK, avançar para PNG mono e por fim RGB. - -Tempo estimado: 1-2 horas para diagnostico e consertos menores (depende de complexidade dos erros observados). - -Implementacao esperada: -- Engine legacy: manter comportamento atual de referencia DES para PNG. -- Engine astrocut: implementar caminho de composicao RGB (3 bandas) e caminho monocromatico (1 banda). -- Task/policy: encaminhar `format`, `color` e bandas efetivas para o engine. - -Criterios de aceite: -- `format=png&color=false&band=g` retorna PNG valido. -- `format=png&color=true&rgb_bands=gri` retorna PNG RGB valido. -- Validacoes claras para combinacoes invalidas de parametros. - -## 12.3 Prioridade 2 - RANGE/POLYGON ponta a ponta - -Objetivo: -- Garantir RANGE/POLYGON no fluxo completo da engine default, nao apenas no parse/discovery. - -Escopo: -- Completar suporte do `astrocut` para RANGE e POLYGON. -- Se houver limitacao da biblioteca, implementar adaptacao geometrica controlada (ex: conversao para janela equivalente) sem quebrar contrato. - -Criterios de aceite: -- Requests sync com `POS=RANGE ...` e `POS=POLYGON ...` executam com `engine=astrocut` e retornam resultado valido. -- Testes de integracao cobrindo CIRCLE/RANGE/POLYGON com engine default. - -## 12.4 Prioridade 3 - Parametro para persistir resultado em data/results - -Objetivo: -- Permitir controlar se o arquivo final sera persistido localmente em `data/results`. - -Contrato proposto: -- Novo parametro `persist` (boolean): - - `persist=false` (default): comportamento atual de resposta sync sem compromisso de retencao longa. - - `persist=true`: manter arquivo final em `data/results`. - -Regras: -- Definir naming deterministico por job/runid e evitar sobrescrita acidental. -- Registrar caminho final quando persistido. - -Criterios de aceite: -- `persist=true` preserva arquivo em `data/results`. -- `persist=false` nao cria persistencia adicional alem do necessario para stream/resposta. - -## 12.5 Prioridade 4 - Registro em banco para todos os pedidos sync - -Objetivo: -- Registrar auditoria completa de cada pedido de cutout sync. - -Campos minimos: -- usuario solicitante -- parametros usados -- status final (gerou resultado ou nao) -- tamanho do resultado em bytes -- tempo total de execucao - -Observacao: -- Parte da estrutura ja existe via `Job`, `JobParameter`, `JobResult`, `start_time` e `end_time`, mas falta consolidar populacao obrigatoria para sync em todos os cenarios. - -Criterios de aceite: -- Cada chamada sync gera registro auditavel com os campos minimos. -- Consultas administrativas conseguem listar historico por usuario e periodo. - -## 12.6 Prioridade 5 - Atualizacao OpenAPI - -Objetivo: -- Atualizar schema do endpoint sync com novos parametros e semantica atual. - -Escopo minimo: -- Documentar `format=png`, `color`, `rgb_bands`, `persist`. -- Documentar suporte de POS (CIRCLE/RANGE/POLYGON) e limites conhecidos por engine. -- Documentar exemplos de request/response para FITS e PNG. - -Criterios de aceite: -- OpenAPI e exemplos de uso alinhados com o comportamento real da API. - -## 12.7 Prioridade 6 - Semantica de errors (postergada) - -Decisao: -- Manter erros em JSON por enquanto. -- Migracao para semantica SODA/DALI (`text/plain` com prefixos padrao) fica para depois da fase async. - -Criterios de aceite futuro: -- Contrato de erro unico para sync/async alinhado ao padrao IVOA. - -## 13) Backlog operacional imediato (proxima sprint) - -1. Implementar prioridade 1 (PNG mono + colorido) incluindo parametro `color` e `rgb_bands`. -2. Fechar prioridade 2 (RANGE/POLYGON ponta a ponta no engine default). -3. Incluir prioridade 3 (`persist`) e registro de caminho final. -4. Fechar prioridade 4 (auditoria completa sync em banco). -5. Atualizar prioridade 5 (OpenAPI e exemplos curl). -6. Manter prioridade 6 explicitamente adiada ate fase async. diff --git a/STATUS_IMPLEMENTACAO_SYNC.md b/STATUS_IMPLEMENTACAO_SYNC.md deleted file mode 100644 index 7014cf3..0000000 --- a/STATUS_IMPLEMENTACAO_SYNC.md +++ /dev/null @@ -1,355 +0,0 @@ -# Status de Implementacao - Sync Cutout - -Data de inicio: 2026-04-26 -Escopo: acompanhar implementacao incremental do endpoint sync conforme plano IVOA/DES. -Publico: humanos e agentes de AI. - -## Como usar este arquivo - -Atualize este documento a cada fase implementada, sempre antes de abrir PR ou seguir para a fase seguinte. - -Checklist de atualizacao: -1. Resumo do que foi implementado. -2. Arquivos criados/alterados. -3. Regras de negocio adicionadas ou alteradas. -4. Decisoes tecnicas tomadas. -5. Comandos de validacao executados e resultado. -6. Commit(s) associado(s). - -## Estado atual por fase - -- Fase 0 (hardening): parcial -- Fase 1 (discovery desacoplado): concluida -- Fase 1.5 (policy de acesso por survey): concluida -- Fase 2 (orquestracao sync via celery): concluida -- Fase 3 (adapter de engine): concluida -- Fase 3.1 (astrocut nativo com discovery DES): concluida -- Fase 4 (descoberta VO remota): adiada (decisao de arquitetura pendente) - -## Prioridades atuais (replanejamento 2026-04-26) - -1. PNG (saida png, incluindo composicao colorida): pendente -2. RANGE/POLYGON ponta a ponta no engine default: parcial -3. Parametro para persistir resultado em `data/results`: pendente -4. Registro completo de pedidos sync no banco: parcial -5. Atualizacao OpenAPI: parcial -6. Semantica de errors padrao SODA/DALI: adiada para pos-fase-async (manter JSON por ora) - -## Resumo consolidado do que ja foi feito - -- Pipeline sync com Celery e espera sincrona no endpoint implementado. -- Registro de job com usuario e parametros implementado (`Job` + `JobParameter`). -- Fases de execucao registradas (PENDING/QUEUED/EXECUTING/COMPLETED/ERROR). -- Discovery DES/CSV separado da API com suporte de CIRCLE/RANGE/POLYGON na descoberta. -- Policy de survey integrada antes do discovery. -- Selecao de engine operacional (`astrocut` default, `legacy` alternativo). -- Astrocut nativo implementado para CIRCLE em FITS. - -## Lacunas tecnicas relevantes para o novo plano - -- Astrocut ainda restrito a `circle` e `fits`, impactando prioridades 1 e 2. -- Falta contrato de persistencia controlada por parametro (prioridade 3). -- Falta consolidar auditoria de resultado (sucesso, tamanho e tempo total) em todos os fluxos sync (prioridade 4). -- OpenAPI ainda nao cobre novos parametros planejados de PNG colorido e persistencia (prioridade 5). - -## Regras de negocio vigentes - -1. Tipos espaciais aceitos no parse: CIRCLE, RANGE, POLYGON. -2. Discovery v1 usa intersecao por bounding box do stencil com tiles DES em CSV. -3. Survey suportado no discovery atual: des_dr2. -4. Se nenhum arquivo for encontrado para a regiao solicitada, o fluxo falha explicitamente com erro de parametro. -5. Camada de policy de survey roda antes do discovery. -6. Policy inicial libera des_dr2 e nega surveys nao reconhecidos. -7. Durante execucao da task, se o arquivo esperado nao estiver acessivel no disco local, a API retorna erro explicito de arquivo indisponivel. -8. A API aceita selecao de engine via parametro `engine`. -9. Engine default atual: `astrocut`. -10. Engine legado permanece disponivel via `engine=legacy`. -11. Discovery de arquivos permanece no backend DES/CSV nesta etapa (sem SIA/TAP por enquanto). - -## Decisoes tecnicas registradas - -1. Manter abordagem geometrica simples (bounding box) nesta etapa e deixar refinamento cientifico para revisao posterior. -2. Estruturar policy de acesso com interface separada para permitir evolucao futura para surveys privados. -3. Validar cada fase dentro de container antes de commit. -4. Trabalhar em fases pequenas: implementar, testar, validar, commit. -5. Durante fase de aprovacao, manter multiplas ferramentas de cutout ativas para comparacao controlada. -6. Adiar integracao SIA/TAP ate definicao arquitetural formal; manter estabilidade com discovery DES local. - -## Historico de implementacao - -### Entrada 2026-04-26 - Fase 1 - -Resumo: -- Criado modulo de discovery desacoplado. -- Implementado locator DES por CSV com suporte a CIRCLE, RANGE e POLYGON. -- Integrado locator no fluxo de dispatch do policy. -- Ajustada task para receber lista de arquivos descobertos. -- Adicionado erro explicito quando nao ha arquivos para a regiao. - -Arquivos criados: -- cutout/service/discovery/__init__.py -- cutout/service/discovery/base.py -- cutout/service/discovery/models.py -- cutout/service/discovery/des_csv_locator.py -- cutout/service/discovery/tests/test_des_csv_locator.py - -Arquivos alterados: -- cutout/service/policy.py -- cutout/service/tasks.py - -Validacao executada: -- docker compose exec django black --check cutout/service/discovery cutout/service/policy.py cutout/service/tasks.py -- docker compose exec django isort --check-only cutout/service/discovery cutout/service/policy.py cutout/service/tasks.py -- docker compose exec django pytest cutout/service/discovery/tests/test_des_csv_locator.py -q -- Resultado: 5 passed - -Commit: -- b2edcfb - feat(discovery): add DES CSV locator and integrate dispatch file lookup - -### Entrada 2026-04-26 - Fase 1.5 - -Resumo: -- Criada camada de policy de acesso por survey. -- Implementada policy inicial publica para DES DR2. -- Integrada validacao de acesso antes do discovery no fluxo de dispatch. -- Confirmado comportamento de negacao para survey nao permitido. - -Arquivos criados: -- cutout/service/policies/__init__.py -- cutout/service/policies/base.py -- cutout/service/policies/des_public.py -- cutout/service/policies/tests/test_des_public_policy.py - -Arquivos alterados: -- cutout/service/policy.py - -Validacao executada: -- docker compose exec django black --check cutout/service/policies cutout/service/policy.py -- docker compose exec django isort --check-only cutout/service/policies cutout/service/policy.py -- docker compose exec django pytest cutout/service/policies/tests/test_des_public_policy.py cutout/service/discovery/tests/test_des_csv_locator.py -q -- Resultado: 7 passed - -Smoke test de autorizacao: -- Requisicao com id=private_survey em /api/sync -- Resultado observado: 403 com mensagem de acesso negado - -Commit: -- b332e38 - feat(policy): add survey access layer before discovery dispatch - -## Proxima fase planejada (repriorizada) - -Fase P1/P2 - PNG + RANGE/POLYGON no engine default - -Objetivos imediatos: -1. Implementar `format=png` no fluxo sync (mono e RGB). -2. Introduzir parametro explicito para PNG colorido (`color`) e bandas RGB (`rgb_bands`, default `gri`). -3. Fechar execucao ponta a ponta de RANGE/POLYGON com `engine=astrocut`. -4. Validar comportamento com testes e curls de regressao para CIRCLE/RANGE/POLYGON em FITS e PNG. - -### Entrada 2026-04-26 - Preparacao da Fase 3 - -Resumo: -- Criado esqueleto do modulo `cutout_engine` para iniciar a separacao entre orquestracao e motor de recorte. -- Adicionada interface `CutoutEngine` e implementacoes iniciais para DES e Astrocut (stub). -- Incluido teste unitario basico para validar delegacao do adapter DES para a classe de cutout existente. - -Arquivos criados: -- cutout/service/cutout_engine/__init__.py -- cutout/service/cutout_engine/base.py -- cutout/service/cutout_engine/des_engine.py -- cutout/service/cutout_engine/astrocut_engine.py -- cutout/service/cutout_engine/tests/test_des_engine.py - -Status: -- Preparacao inicial concluida. -- Integracao do adapter no fluxo principal sera feita na execucao completa da Fase 3. - -### Entrada 2026-04-26 - Fase 2 - -Resumo: -- Endpoint sync passou a aguardar o resultado real da task Celery no fluxo sincrono. -- JobService passou a registrar transicoes de fase EXECUTING, COMPLETED e ERROR. -- Task de cutout passou a validar explicitamente a disponibilidade dos arquivos de entrada. -- Em caso de arquivo de entrada ausente/inacessivel, a API retorna erro explicito (422) com detalhe dos caminhos faltantes. -- Endpoint passou a retornar arquivo via FileResponse quando o resultado existe, com timeout configurado e erro 503 para indisponibilidade do servico. - -Arquivos criados: -- cutout/service/tests/test_tasks.py - -Arquivos alterados: -- cutout/service/api/views.py -- cutout/service/policy.py -- cutout/service/tasks.py -- cutout/service/uws/service.py -- cutout/service/uws/exceptions.py -- cutout/lib/cutout.py -- .gitignore - -Validacao executada: -- docker compose exec django black --check cutout/service/api/views.py cutout/service/policy.py cutout/service/tasks.py cutout/service/uws/service.py cutout/service/uws/exceptions.py cutout/lib/cutout.py cutout/service/tests/test_tasks.py -- docker compose exec django isort --check-only cutout/service/api/views.py cutout/service/policy.py cutout/service/tasks.py cutout/service/uws/service.py cutout/service/uws/exceptions.py cutout/lib/cutout.py cutout/service/tests/test_tasks.py -- docker compose exec django pytest cutout/service/tests/test_tasks.py cutout/service/policies/tests/test_des_public_policy.py cutout/service/discovery/tests/test_des_csv_locator.py -q -- Resultado: 10 passed - -Smoke test relevante: -- Requisicao em /api/sync com des_dr2 e coordenada valida no catalogo, mas sem arquivo FITS correspondente local. -- Resultado observado: 422 com mensagem Input file unavailable e lista de arquivos ausentes. - -Status: -- Fase 2 finalizada no codigo. -- Comportamento de erro para indisponibilidade de arquivo local validado conforme requisito de ambiente de testes parcial. - -### Entrada 2026-04-26 - Fase 3 - -Resumo: -- Integrada camada de selecao de engine no pipeline de cutout. -- Implementado factory para escolha de backend por nome de engine. -- Mantida ferramenta antiga funcional como opcao (`legacy`). -- Definido `astrocut` como default da API. -- Nesta fase, `astrocut` usa fallback controlado para engine legado para manter operacao funcional durante aprovacao. - -Arquivos criados: -- cutout/service/cutout_engine/factory.py -- cutout/service/cutout_engine/tests/test_factory.py -- cutout/service/cutout_engine/tests/test_astrocut_engine.py -- cutout/service/tests/test_cutout_parameters.py - -Arquivos alterados: -- cutout/service/api/views.py -- cutout/service/cutout_engine/__init__.py -- cutout/service/cutout_engine/astrocut_engine.py -- cutout/service/cutout_parameters.py -- cutout/service/policy.py -- cutout/service/tasks.py -- test_sync_endpoint.py -- CURL_TESTES_SYNC.md - -Validacao executada: -- docker compose exec django black --check cutout/service/api/views.py cutout/service/cutout_parameters.py cutout/service/policy.py cutout/service/tasks.py cutout/service/cutout_engine test_sync_endpoint.py -- docker compose exec django isort --check-only cutout/service/api/views.py cutout/service/cutout_parameters.py cutout/service/policy.py cutout/service/tasks.py cutout/service/cutout_engine test_sync_endpoint.py -- docker compose exec django pytest cutout/service/cutout_engine/tests/test_des_engine.py cutout/service/cutout_engine/tests/test_factory.py cutout/service/cutout_engine/tests/test_astrocut_engine.py cutout/service/tests/test_tasks.py cutout/service/tests/test_cutout_parameters.py -q -- Resultado: 10 passed - -Smoke test relevante: -- `engine=legacy` em `/api/sync`: status 200 (streaming) -- `engine=astrocut` em `/api/sync`: status 200 (streaming) - -Status: -- Fase 3 finalizada para aprovacao funcional com duas opcoes de ferramenta. -- Integracao nativa do astrocut permanece como evolucao da fase seguinte. - -### Entrada 2026-04-26 - Checkpoint de compatibilidade de dependencias - -Resumo: -- Validada compatibilidade do ambiente com novas versoes cientificas. -- Confirmado funcionamento do modo legado DES apos upgrade. - -Dependencias em runtime (container django): -- numpy 2.4.4 -- astropy 7.2.0 -- astrocut 1.2.0 - -Validacao executada: -- docker compose exec django pytest cutout/service/tests/test_tasks.py cutout/service/discovery/tests/test_des_csv_locator.py cutout/service/cutout_engine/tests/test_des_engine.py -q -- docker compose exec django python manage.py shell -c "... engine=legacy ..." -- Resultado: testes verdes e endpoint sync legacy retornando 200 (streaming, application/fits) - -### Entrada 2026-04-26 - Fase 3.1 (Astrocut nativo com discovery DES) - -Resumo: -- Removido fallback `astrocut -> legacy` no engine Astrocut. -- Integrada chamada nativa ao `astrocut.fits_cut` para pedidos `CIRCLE` em `fits`. -- Mantida descoberta de arquivos no locator DES/CSV atual (sem SIA nesta fase). -- Task passou a repassar a lista de arquivos de entrada para o engine selecionado. - -Arquivos alterados: -- cutout/service/cutout_engine/base.py -- cutout/service/cutout_engine/des_engine.py -- cutout/service/cutout_engine/astrocut_engine.py -- cutout/service/tasks.py -- cutout/service/cutout_engine/tests/test_des_engine.py -- cutout/service/cutout_engine/tests/test_astrocut_engine.py - -Validacao executada: -- docker compose exec django isort --check-only cutout/service/cutout_engine/astrocut_engine.py cutout/service/cutout_engine/tests/test_astrocut_engine.py cutout/service/cutout_engine/tests/test_des_engine.py cutout/service/tasks.py cutout/service/cutout_engine/base.py cutout/service/cutout_engine/des_engine.py -- docker compose exec django black --check cutout/service/cutout_engine/astrocut_engine.py cutout/service/cutout_engine/tests/test_astrocut_engine.py cutout/service/cutout_engine/tests/test_des_engine.py cutout/service/tasks.py cutout/service/cutout_engine/base.py cutout/service/cutout_engine/des_engine.py -- docker compose exec django pytest cutout/service/cutout_engine/tests/test_des_engine.py cutout/service/cutout_engine/tests/test_astrocut_engine.py cutout/service/cutout_engine/tests/test_factory.py cutout/service/tests/test_tasks.py cutout/service/tests/test_cutout_parameters.py cutout/service/discovery/tests/test_des_csv_locator.py -q -- docker compose exec django python manage.py shell -c "... engine=legacy e engine=astrocut ..." -- Resultado: 17 passed; `engine=legacy` e `engine=astrocut` com status 200 e streaming FITS. - -Status: -- Fase 3.1 concluida. -- Proxima etapa permanece focada em estabilizacao/expansao do engine astrocut sem alterar discovery DES. - -### Entrada 2026-04-26 - Replanejamento orientado a prioridade - -Resumo: -- Plano e status foram atualizados para ordem de execucao por prioridade de negocio (1 a 6). -- Prioridades 1 e 2 tornaram-se foco imediato de implementacao. -- Priorizacao de semantica de erro foi formalmente adiada para depois da fase async. - -Nota de compatibilidade IVOA para prioridade 1: -- Nao foi identificado parametro padrao em SODA 1.0 para requisitar "PNG colorido/RGB". -- O padrao permite parametros customizados; portanto o comportamento sera documentado como extensao local. - -Arquivos alterados nesta entrada: -- PLANO_SYNC_IVOA_DES_DESCOBERTA.md -- STATUS_IMPLEMENTACAO_SYNC.md - -Status: -- Replanejamento aplicado e pronto para execucao incremental. - -### Entrada 2026-04-26 - Bug report e plano de depuracao - -Resumo: -- Foram detectados artefatos gerados incorretos: FITS que nao abre e PNG aparentemente vazio. Também foi observado que o caminho legado (`legacy`) nao gerou arquivos como antes. - -Acao imediata: -1. Criar script de depuracao `scripts/debug_cutout.py` que reproduz as chamadas existentes em `des_cutout.py` e grava artefatos em `/data/results/debug/`. -2. Executar steps ordenados: FITS mono (legacy), FITS mono (engine astrocut), PNG mono, PNG RGB. -3. Coletar logs e propriedades de cada arquivo (tamanho, shape, min/max, nan count) e anexar ao ticket/issue. - -Arquivos criados/alterados: -- scripts/debug_cutout.py (novo) - -Validacao executada: -- executar `docker compose exec django python scripts/debug_cutout.py` dentro do container e inspecionar `/data/results/debug/`. - -Responsavel: equipe de desenvolvimento (agent). - -Proximo passo: -- executar o script dentro do container, analisar saídas e corrigir a função que estiver produzindo artefatos inválidos. - -### Entrada 2026-04-26 - Correcao de nome de saida no sync PNG (astrocut) - -Resumo: -- Corrigido o nome de arquivo de saida no dispatch do policy, removendo hardcode `teste.fits`. -- Fluxo sync agora gera path dinamico em `/data/results` com extensao aderente ao parametro `format` (`.png` ou `.fits`). -- Arquivos intermediarios do Astrocut passaram a usar nomes unicos por execucao para evitar colisao em concorrencia. -- Resultado observado em teste manual: fluxo sync com `engine=astrocut` e `format=png` voltou a gerar artefato no formato esperado. - -Arquivos alterados: -- cutout/service/policy.py -- cutout/service/cutout_engine/astrocut_engine.py - -Regras/decisoes desta entrada: -1. Diretorio de saida oficial permanece `/data/results`. -2. Nome final do resultado deve ser deterministico por job e seguro para filesystem. -3. Arquivos temporarios do pipeline PNG devem ser unicos por execucao (evitar overwrite entre jobs). - -Validacao executada: -- docker compose exec django pytest cutout/service/cutout_engine/tests/test_factory.py -q -- docker compose exec django pytest cutout/service/policies/tests/test_des_public_policy.py -q -- docker compose exec django python -m py_compile cutout/service/policy.py cutout/service/cutout_engine/astrocut_engine.py -- Resultado: 5 passed; compilacao sem erros. - -Pendencia conhecida: -- Fluxo sync `engine=legacy` com `format=png` continua com problema e requer depuracao dedicada no motor legado. - -Proximos passos planejados: -1. Reproduzir `legacy+png` com caso minimo (CIRCLE pequeno) e capturar traceback completo em worker + django. -2. Revisar caminho de conversao/serializacao PNG no engine legado e validar extensao/mimetype de saida. -3. Adicionar teste automatizado de regressao para sync `legacy+png` (esperado 200 + arquivo PNG valido). - -Commit associado: -- Este commit (correcao do naming dinamico no sync e plano para pendencia legacy PNG). diff --git a/config/api_router.py b/config/api_router.py index bc6059c..f37a04f 100644 --- a/config/api_router.py +++ b/config/api_router.py @@ -28,7 +28,7 @@ urlpatterns += [ path("cutout", CutoutView.as_view(), name="cutout"), - path("sync", SyncCutoutView.as_view(), name="sync_cutout"), + path("sync", transaction.non_atomic_requests(SyncCutoutView.as_view()), name="sync_cutout"), path("async", transaction.non_atomic_requests(AsyncCutoutView.as_view()), name="async_cutout"), path("async/", AsyncJobDetailView.as_view(), name="async_job_detail"), path("async//phase", AsyncJobPhaseView.as_view(), name="async_job_phase"), diff --git a/config/urls.py b/config/urls.py index 33655bf..96b81f0 100644 --- a/config/urls.py +++ b/config/urls.py @@ -57,7 +57,3 @@ ), path("500/", default_views.server_error), ] - if "debug_toolbar" in settings.INSTALLED_APPS: - import debug_toolbar - - urlpatterns = [path("__debug__/", include(debug_toolbar.urls))] + urlpatterns diff --git a/cutout/service/api/views.py b/cutout/service/api/views.py index 535aba9..31ffb0e 100644 --- a/cutout/service/api/views.py +++ b/cutout/service/api/views.py @@ -1,17 +1,17 @@ import logging from collections.abc import Iterable from pathlib import Path -from typing import List, Optional -from celery.exceptions import TimeoutError as CeleryTimeoutError from django.http import FileResponse, HttpResponse from django.urls import reverse from django.utils.encoding import escape_uri_path from drf_spectacular.utils import OpenApiParameter, extend_schema, extend_schema_view from rest_framework import status +from rest_framework.exceptions import APIException from rest_framework.response import Response from rest_framework.views import APIView +from cutout.service.cutout_runner import perform_cutout from cutout.service.models import Job from cutout.service.uws.exceptions import ParameterError, ServiceUnavailableError from cutout.service.uws.models import JobParameter @@ -155,7 +155,7 @@ def get(self, request, format=None): allow_blank=False, many=False, default="astrocut", - description=("Cutout backend engine. Supported values: astrocut, legacy"), + description=("Cutout backend engine. Supported values: astrocut"), ), ], ) @@ -163,54 +163,48 @@ def get(self, request, format=None): @extend_schema_view(get=cutout_schema, post=cutout_schema) class SyncCutoutView(APIView): - sync_timeout_seconds = 25 - def _mimetype_for_format(self, output_format: str) -> str: if output_format.lower() == "png": return "image/x-png" return "application/fits" def sync_cutout(self, user: User, params: list[JobParameter], run_id: str | None): - print("Entrou aqui") + """Run a cutout synchronously inside the request. + + Database flow is identical to the async pipeline (Job + Task rows, + status transitions and JobResult recorded by `perform_cutout`), but + execution happens inline and the result file is returned directly. + """ + logger = logging.getLogger("cutout") job_service = JobService() - job = job_service.create(user=user, params=params, run_id=run_id) - print("step 0") - async_result = job_service.start(user, job_id=job.id) - print("step 1") - output_format = "fits" - for p in params: - if p.parameter_id == "format": - output_format = p.value - break - try: - print("step 2") - job_service.mark_executing(job.id) - print("step 3") - result_path = async_result.get(timeout=self.sync_timeout_seconds) - except CeleryTimeoutError as exc: - print("step 4") + job = job_service.create(user=user, params=params, run_id=run_id, execution_mode="sync") + logger.info("[SyncCutoutView] created job_id=%s", job.id) + + tasks = list(job.tasks.order_by("sequence")) + if len(tasks) != 1: job_service.mark_error(job.id) - print("step 5") - raise ServiceUnavailableError("Sync cutout timed out") from exc + raise ParameterError("Only one cutout task is supported in sync mode") + task = tasks[0] + + try: + result = perform_cutout(job.id, task.id) + except APIException: + # Task and Job are already marked ERROR by perform_cutout + raise except Exception as exc: - print("step 6") - job_service.mark_error(job.id) - print("step 7") raise ParameterError(str(exc)) from exc - print("step 8") - result_file = Path(result_path) + result_file = Path(result["file_path"]) if not result_file.exists(): - print("step 9") job_service.mark_error(job.id) raise ServiceUnavailableError("Result file unavailable") - print("step 10") - job_service.mark_completed(job.id) + logger.info("[SyncCutoutView] job_id=%s completed result_id=%s", job.id, result["result_id"]) + fp = open(result_file, "rb") - response = FileResponse(fp, content_type=self._mimetype_for_format(output_format), as_attachment=True) + response = FileResponse(fp, content_type=self._mimetype_for_format(task.output_format), as_attachment=True) response["Content-Length"] = result_file.stat().st_size response["Content-Disposition"] = f"attachment; filename={escape_uri_path(result_file.name)}" return response @@ -224,14 +218,22 @@ def get(self, request, format=None): class AsyncCutoutView(APIView): def get(self, request, format=None): jobs = JobService().list_for_user(request.user) - serializer = AsyncJobSummarySerializer(jobs, many=True, context={"request": request}) + serializer = AsyncJobSummarySerializer( + jobs, + many=True, + context={"request": request}, + ) return Response({"jobs": serializer.data}) def post(self, request, format=None): logger = logging.getLogger("cutout") logger.info(f"[AsyncCutoutView.post] called with data={request.data}") - params, run_id, requested_phase = _extract_job_request(request.data or request.query_params, is_post=True) - logger.info(f"[AsyncCutoutView.post] params={params} run_id={run_id} requested_phase={requested_phase}") + + params, run_id, requested_phase = _extract_job_request( + request.data or request.query_params, + is_post=True, + ) + logger.info(f"[AsyncCutoutView.post] params={params} run_id={run_id} " f"requested_phase={requested_phase}") if not params: logger.error("[AsyncCutoutView.post] No params provided") raise ParameterError("At least one cutout parameter is required") @@ -242,8 +244,13 @@ def post(self, request, format=None): raise ParameterError("Only PHASE=RUN is supported when creating async jobs") job_service = JobService() - job = job_service.create(user=request.user, params=params, run_id=run_id) + job = job_service.create( + user=request.user, + params=params, + run_id=run_id, + ) logger.info(f"[AsyncCutoutView.post] Created job id={job.id}") + job_service.start_async(request.user, job.id) logger.info(f"[AsyncCutoutView.post] Dispatched start_async for job id={job.id}") diff --git a/cutout/service/cutout_engine/__init__.py b/cutout/service/cutout_engine/__init__.py index 34c524e..90576e2 100644 --- a/cutout/service/cutout_engine/__init__.py +++ b/cutout/service/cutout_engine/__init__.py @@ -1,6 +1,5 @@ from .astrocut_engine import AstrocutEngine from .base import CutoutEngine -from .des_engine import DesCutoutEngine from .factory import create_cutout_engine -__all__ = ["CutoutEngine", "DesCutoutEngine", "AstrocutEngine", "create_cutout_engine"] +__all__ = ["CutoutEngine", "AstrocutEngine", "create_cutout_engine"] diff --git a/cutout/service/cutout_engine/des_engine.py b/cutout/service/cutout_engine/des_engine.py deleted file mode 100644 index 2127845..0000000 --- a/cutout/service/cutout_engine/des_engine.py +++ /dev/null @@ -1,27 +0,0 @@ -from __future__ import annotations - -from pathlib import Path -from typing import Any - -from cutout.lib.cutout import Cutout - -from .base import CutoutEngine - - -class DesCutoutEngine(CutoutEngine): - def run_cutout( - self, - *, - source_id: str, - stencil: dict[str, Any], - input_files: list[str] | dict[str, list[str]] | None, - band: str, - output_format: str, - output_path: str | Path, - color: bool = False, - rgb_bands: str | None = None, - persist: bool = False, - ) -> Path: - # Legacy engine: ignore `input_files` mapping (discovery is internal to DesCutout) - cutout = Cutout(source_id=source_id, stencil=stencil, band=band, format=output_format) - return cutout.create(output_path) diff --git a/cutout/service/cutout_engine/factory.py b/cutout/service/cutout_engine/factory.py index ff94c6b..4021f9a 100644 --- a/cutout/service/cutout_engine/factory.py +++ b/cutout/service/cutout_engine/factory.py @@ -2,7 +2,6 @@ from .astrocut_engine import AstrocutEngine from .base import CutoutEngine -from .des_engine import DesCutoutEngine def create_cutout_engine(engine_name: str) -> CutoutEngine: @@ -10,7 +9,5 @@ def create_cutout_engine(engine_name: str) -> CutoutEngine: if name == "astrocut": return AstrocutEngine() - if name in ("legacy", "des"): - return DesCutoutEngine() raise ValueError(f"Unsupported cutout engine: {engine_name}") diff --git a/cutout/service/cutout_engine/tests/test_des_engine.py b/cutout/service/cutout_engine/tests/test_des_engine.py deleted file mode 100644 index 6feea40..0000000 --- a/cutout/service/cutout_engine/tests/test_des_engine.py +++ /dev/null @@ -1,32 +0,0 @@ -from pathlib import Path - -from cutout.service.cutout_engine.des_engine import DesCutoutEngine - - -class DummyCutout: - def __init__(self, source_id, stencil, band, format): - self.source_id = source_id - self.stencil = stencil - self.band = band - self.format = format - - def create(self, output_path): - return Path(output_path) - - -def test_des_cutout_engine_delegates_to_cutout(monkeypatch): - import cutout.service.cutout_engine.des_engine as des_engine_module - - monkeypatch.setattr(des_engine_module, "Cutout", DummyCutout) - - engine = DesCutoutEngine() - result = engine.run_cutout( - source_id="des_dr2", - stencil={"type": "circle", "center": {"ra": 10.0, "dec": -1.0}, "radius": 1.0}, - input_files=["/tmp/source.fits.fz"], - band="g", - output_format="fits", - output_path="/tmp/out.fits", - ) - - assert result == Path("/tmp/out.fits") diff --git a/cutout/service/cutout_engine/tests/test_factory.py b/cutout/service/cutout_engine/tests/test_factory.py index 0cdea8e..bd869a1 100644 --- a/cutout/service/cutout_engine/tests/test_factory.py +++ b/cutout/service/cutout_engine/tests/test_factory.py @@ -1,6 +1,6 @@ import pytest -from cutout.service.cutout_engine import AstrocutEngine, DesCutoutEngine, create_cutout_engine +from cutout.service.cutout_engine import AstrocutEngine, create_cutout_engine def test_factory_returns_astrocut_engine() -> None: @@ -8,9 +8,9 @@ def test_factory_returns_astrocut_engine() -> None: assert isinstance(engine, AstrocutEngine) -def test_factory_returns_legacy_engine() -> None: - engine = create_cutout_engine("legacy") - assert isinstance(engine, DesCutoutEngine) +def test_factory_rejects_legacy_engine() -> None: + with pytest.raises(ValueError, match="Unsupported cutout engine"): + create_cutout_engine("legacy") def test_factory_rejects_unknown_engine() -> None: diff --git a/cutout/service/cutout_runner.py b/cutout/service/cutout_runner.py new file mode 100644 index 0000000..be4551e --- /dev/null +++ b/cutout/service/cutout_runner.py @@ -0,0 +1,200 @@ +"""Single entry point for executing one cutout unit (a Task row) end to end. + +Reads every execution parameter from the database, discovers the input files, +runs the cutout engine and records the result and status transitions. Used as +a direct call by the sync flow and wrapped in a Celery task for the async flow +(``cutout.service.tasks.perform_cutout_task``). +""" + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Any + +from django.utils import timezone + +from cutout.service.cutout_engine import create_cutout_engine +from cutout.service.discovery import DesCsvFileLocator +from cutout.service.models import Job, JobResult, Task +from cutout.service.stencils import Stencil +from cutout.service.uws.exceptions import ParameterError + +logger = logging.getLogger("cutout") + +InputFiles = list[str] | dict[str, list[str]] + + +def _parse_rgb_bands(raw: str) -> list[str]: + """Parse rgb_bands accepting 'gri', 'g,r,i' or 'g r i'.""" + if "," in raw: + return [b.strip() for b in raw.split(",") if b.strip()] + if " " in raw: + return [b.strip() for b in raw.split() if b.strip()] + return list(raw) + + +def _validate_input_files(files: InputFiles | None) -> None: + if not files: + return + + paths: list[str] = [] + if isinstance(files, dict): + for v in files.values(): + paths.extend(v or []) + else: + paths = list(files) + + missing = [f for f in paths if not Path(f).exists()] + if missing: + msg = "Input file unavailable: " + ", ".join(missing) + raise FileNotFoundError(msg) + + +def _find_existing_files(locator: DesCsvFileLocator, task: Task, stencil: Stencil, band: str) -> list[str]: + descriptors = locator.find_files(survey_id=task.survey_id, stencil=stencil, band=band) + if not descriptors: + raise ParameterError(f"No files found for band {band} in the requested region") + + candidates = [str(d.file_path) for d in descriptors if d.file_path] + existing = [p for p in candidates if Path(p).exists()] + if not existing: + raise ParameterError(f"No available files on disk for band {band} in the requested region") + return existing + + +def _discover_input_files(task: Task) -> InputFiles: + """Locate the input tiles for a task, per band when color composition is requested.""" + stencil = Stencil.from_dict(task.stencil) + locator = DesCsvFileLocator() + + if task.color: + bands = _parse_rgb_bands(task.rgb_bands or "gri") + files_map = {band: _find_existing_files(locator, task, stencil, band) for band in bands} + logger.info("[perform_cutout] task_id=%s color bands=%s files=%s", task.id, bands, files_map) + return files_map + + files = _find_existing_files(locator, task, stencil, task.band) + logger.info("[perform_cutout] task_id=%s band=%s files=%s", task.id, task.band, files) + return files + + +def _mime_type_for_format(output_format: str) -> str: + return "image/png" if str(output_format).lower() == "png" else "application/fits" + + +def perform_cutout(job_id: int | str, task_id: int | str) -> dict[str, Any]: + """Execute one cutout Task end to end, reading everything from the database. + + Loads the Job and Task rows, transitions their states, discovers the input + files, runs the cutout engine and registers the JobResult. Marking the Job + as COMPLETED is the caller's responsibility (chord callback in async mode). + """ + job_pk = int(str(job_id).strip()) + task_pk = int(str(task_id).strip()) + + job = Job.objects.get(pk=job_pk) + task = Task.objects.get(pk=task_pk) + + if task.job_id != job.pk: + raise ValueError(f"Task {task_pk} does not belong to job {job_pk}") + + if job.phase == Job.ExecutionPhase.ABORTED: + logger.info("[perform_cutout] job_id=%s is ABORTED, skipping task_id=%s", job_pk, task_pk) + return {} + + logger.info( + "[perform_cutout] START job_id=%s task_id=%s survey_id=%s stencil_type=%s band=%s " + "format=%s engine=%s color=%s rgb_bands=%s", + job_pk, + task_pk, + task.survey_id, + task.stencil_type, + task.band, + task.output_format, + task.engine, + task.color, + task.rgb_bands, + ) + + # First task to run transitions the job to EXECUTING (idempotent under concurrency). + # PENDING is accepted besides QUEUED so the function can run standalone, without dispatch. + Job.objects.filter(pk=job_pk, phase__in=(Job.ExecutionPhase.PENDING, Job.ExecutionPhase.QUEUED)).update( + phase=Job.ExecutionPhase.EXECUTING, + start_time=timezone.now(), + ) + + Task.objects.filter(pk=task_pk, status=Task.Status.PENDING).update( + status=Task.Status.EXECUTING, + start_time=timezone.now(), + ) + + try: + files = _discover_input_files(task) + _validate_input_files(files) + + engine = create_cutout_engine(task.engine) + result_path = Path( + engine.run_cutout( + source_id=task.survey_id, + stencil=task.stencil, + input_files=files, + band=task.band, + output_format=task.output_format, + output_path=task.output_path, + color=task.color, + rgb_bands=task.rgb_bands, + persist=task.persist, + ) + ) + + if not result_path.exists(): + raise FileNotFoundError(f"Engine did not produce result file {result_path}") + + result_id = result_path.stem + size = result_path.stat().st_size + + JobResult.objects.update_or_create( + job=job, + sequence=task.sequence, + defaults={ + "result_id": result_id, + "size": size, + "mime_type": _mime_type_for_format(task.output_format), + "url": f"/api/async/{job_pk}/results/{result_id}", + "file_path": str(result_path), + }, + ) + + Task.objects.filter(pk=task_pk).update( + status=Task.Status.COMPLETED, + end_time=timezone.now(), + ) + + logger.info( + "[perform_cutout] COMPLETED job_id=%s task_id=%s result_id=%s size=%s path=%s", + job_pk, + task_pk, + result_id, + size, + result_path, + ) + return { + "task_id": task_pk, + "result_id": result_id, + "file_path": str(result_path), + "size": size, + } + + except Exception as exc: + logger.exception("[perform_cutout] ERROR job_id=%s task_id=%s: %s", job_pk, task_pk, exc) + Task.objects.filter(pk=task_pk).update( + status=Task.Status.ERROR, + end_time=timezone.now(), + error_message=str(exc), + ) + Job.objects.filter(pk=job_pk).update( + phase=Job.ExecutionPhase.ERROR, + end_time=timezone.now(), + ) + raise diff --git a/cutout/service/des_cutout_functions.py b/cutout/service/des_cutout_functions.py deleted file mode 100644 index fddad91..0000000 --- a/cutout/service/des_cutout_functions.py +++ /dev/null @@ -1,219 +0,0 @@ -#!/usr/bin/env python3 - -import glob - -import numpy as np -from astropy import units as u -from astropy.coordinates import SkyCoord -from astropy.io import fits -from astropy.nddata.utils import Cutout2D -from astropy.visualization import make_lupton_rgb -from astropy.wcs import WCS - - -def get_fits_data(RA, DEC, size, tiles, band, path): - """Access data (image and wcs) from FITS files. - - Parameters - ---------- - RA : float - Equatorial coordinate of the center of cutout (degrees). - DEC : float - Equatorial coordinate of the center of cutout (degrees). - size : float - Size of cutout (arcmin). - tiles : list - List with the tiles where the vertices of cutout reside. - Sorted by: Upper left, Upper right, Lower right, Lower left. - band : str - Band of cutout. - path : str - Path to folder with the FITS files. - """ - - tile_un, ind = np.unique(tiles[:], return_index=True) - tile_un = tile_un[np.argsort(ind)] - - if len(tile_un) == 1: - data_, wcs_ = cutout_fits(RA, DEC, size, tiles[0], band, path) - return (data_, wcs_) - - elif len(tile_un) == 2: - data_1, wcs_1 = cutout_fits(RA, DEC, size, tile_un[0], band, path, "trim") - data_2, wcs_2 = cutout_fits(RA, DEC, size, tile_un[1], band, path, "trim") - - if np.shape(data_1)[1] < np.shape(data_1)[0]: - # side-by-side - data_1 = data_1[:, :-118] - data_2 = data_2[:, 118:] - data_ = np.concatenate((data_1, data_2), axis=1) - return (data_, wcs_1) - - else: - # top-bottom - data_1 = data_1[114:, :] - data_2 = data_2[:-114, :] - data_ = np.concatenate((data_2, data_1), axis=0) - return (data_, wcs_2) - - elif len(tile_un) == 3: - data_1, wcs_1 = cutout_fits(RA, DEC, size, tile_un[0], band, path, "trim") - data_2, wcs_2 = cutout_fits(RA, DEC, size, tile_un[1], band, path, "trim") - data_3, wcs_3 = cutout_fits(RA, DEC, size, tile_un[2], band, path, "trim") - - # Biggest at bottom: - if np.shape(data_1)[1] < np.shape(data_3)[1]: - data_12 = np.concatenate((data_1[118:, :-118], data_2[118:, 118:]), axis=1) - data_ = np.concatenate((data_3[:-118, :], data_12[:, 0 : np.shape(data_3)[1]]), axis=0) - # Biggest at top: - else: - data_23 = np.concatenate((data_2[:-118, 118:], data_3[:-118, :-118]), axis=1) - data_ = np.concatenate((data_23, data_1[118:, 0 : np.shape(data_23)[1]]), axis=0) - return (data_, wcs_3) - - -def cutout_lupton(g_data, r_data, i_data, minimum, stretch, Q, filename): - """Make RGB image and saves as png or jpg files using Lupton method. - TODO: Improve quality of image for cutout with saturated data. - - Parameters - ---------- - g_data : array - Cutout data from first band. - r_data : array - Cutout data from second band. - i_data : array - Cutout data from third band. - filename : str - Name of file to be saved. - """ - - rgb_default = make_lupton_rgb(i_data, r_data, g_data, minimum=minimum, stretch=stretch, Q=Q, filename=filename) - - -def write_cutout_file(data, wcs, filename): - """Saves cutout file. - - Parameters - ---------- - data : array - Array with image data. - wcs : astropy object - Information about world coordinate system of cutout. - filename : str - Name of file to be saved. - """ - hdu = fits.PrimaryHDU(data) - hdu.header.update(wcs) - hdu.writeto(filename, overwrite=True) - - -def cutout_fits(RA_center, DEC_center, size_arcmin, tile_name, band, path, mode="partial"): - """Return data (image array and wcs) from tile. - - Parameters - ---------- - RA_center : float - Equatorial coordinate of center of tile. - DEC_center : float - Equatorial coordinate of center of tile. - size_arcmin : float - Size of cutout in arcmin. - tile_name : str - Name of tile where total or part of the cutout image resides. - band : str - Band of image. - path : str - Path to folder where the FITS files are stored. - mode : str, optional - Mode of cutout. See: - https://docs.astropy.org/en/stable/api/astropy.nddata.Cutout2D.html - By default set to 'partial'. - - Returns - ------- - arrays - Two arrays, one with image data and other with WCS astropy object. - """ - file_name_ = glob.glob(path + "/" + tile_name + "_*_" + band + ".fits") - file_name = file_name_[0] - f = fits.open(file_name) - wcs = WCS(f[1].header) - - cutout1 = Cutout2D( - fits.getdata(file_name, ext=0), - (SkyCoord(ra=RA_center * u.degree, dec=DEC_center * u.degree, frame="icrs")), - size_arcmin * u.arcmin, - wcs=wcs, - mode=mode, - ) - - return cutout1.data, cutout1.wcs.to_header() - - -def cutout_verts(RA_center, DEC_center, size_arcmin): - """Defines the position of vertices in each cutout. - See the pos_angle where the vertices are sorted. - - Parameters - ---------- - RA_center : float - Equatorial coordinate of center of tile. - DEC_center : float - Equatorial coordinate of center of tile. - size_arcmin : float - Size (length of each side) of cutout, in arcmin. - - Returns - ------- - SkyCoord astropy object - Location of vertices of cutout. - """ - pos_angle = [45, 315, 225, 135] * u.deg - RA, DEC = [], [] - for i, j in enumerate(RA_center): - c1 = SkyCoord(RA_center[i] * u.deg, DEC_center[i] * u.deg, frame="icrs") - sep = 0.5 * np.sqrt(2.0) * size_arcmin[i] * u.arcmin - RA.append(list(c1.directional_offset_by(pos_angle, sep).ra.deg)) - DEC.append(list(c1.directional_offset_by(pos_angle, sep).dec.deg)) - return SkyCoord(RA * u.deg, DEC * u.deg, frame="icrs") - - -def tiles_from_cat(cat, file_path): - """Read information about tiles. - TODO: read more information about the vertices of tiles in - order to have a correct overlap in case cutouts are in the - edge of tiles. - - Parameters - ---------- - cat : SkyCoord astropy object - Object with information about coordinates of vertices. - file_path : str - File with tile's information. - - Returns - ------- - list - List of tiles where the vertices of cutout reside. - """ - ra_ll, dec_ll, ra_ul, dec_ul, ra_ur, dec_ur, ra_lr, dec_lr = np.loadtxt( - file_path, usecols=(9, 10, 11, 12, 13, 14, 15, 16), delimiter=",", unpack=True - ) - tile_names = np.loadtxt(file_path, usecols=(2), delimiter=",", dtype=str, unpack=True) - - ra = cat.ra.deg - dec = cat.dec.deg - - tile_match = [] - - for i in range(np.shape(ra)[0]): - idx_ = [] - for j in range(4): - idx_.append( - np.argwhere((ra_ll < ra[i][j]) & (ra_ur > ra[i][j]) & (dec_ll < dec[i][j]) & (dec_ur > dec[i][j]))[0][ - 0 - ] - ) - tile_match.append([tile_names[k] for k in idx_]) - return tile_match diff --git a/cutout/service/discovery/tests/test_des_csv_locator.py b/cutout/service/discovery/tests/test_des_csv_locator.py index 07db772..3d87e31 100644 --- a/cutout/service/discovery/tests/test_des_csv_locator.py +++ b/cutout/service/discovery/tests/test_des_csv_locator.py @@ -23,7 +23,7 @@ def test_find_files_circle_returns_intersecting_tiles(tmp_path: Path) -> None: files = locator.find_files(survey_id="des_dr2", stencil=stencil, band="g") assert [f.tile_id for f in files] == ["TILE_A", "TILE_B"] - assert str(files[0].file_path).endswith("/Y6A1/r4907/TILE_A/p01/coadd/TILE_A_r4907p01_g.fits.fz") + assert str(files[0].file_path) == "/data/tiles/TILE_A/TILE_A_r4907p01_g.fits.fz" def test_find_files_range_returns_single_tile(tmp_path: Path) -> None: diff --git a/cutout/service/policy.py b/cutout/service/policy.py index 8ce8238..e158f8c 100644 --- a/cutout/service/policy.py +++ b/cutout/service/policy.py @@ -8,15 +8,13 @@ import re from datetime import datetime from pathlib import Path -from typing import List from celery import chord as celery_chord from cutout.service.cutout_parameters import CutoutParameters -from cutout.service.discovery import DesCsvFileLocator from cutout.service.models import Task as SQLTask from cutout.service.policies import DesPublicAccessPolicy -from cutout.service.tasks import finalize_job, image_cutout, run_cutout_for_pos +from cutout.service.tasks import finalize_job, perform_cutout_task from cutout.service.uws.exceptions import MultiValuedParameterError, ParameterError, PermissionDeniedError from cutout.service.uws.models import Job, JobParameter from cutout.service.uws.policy import UWSPolicy @@ -49,7 +47,6 @@ class ImageCutoutPolicy(UWSPolicy): # self._logger = logger def __init__(self) -> None: self._survey_access_policy = DesPublicAccessPolicy() - self._file_locator = DesCsvFileLocator() def _safe_token(self, value: str) -> str: """Normalize token for filesystem-safe filenames.""" @@ -67,133 +64,12 @@ def _build_result_path(self, job: Job, task_params: dict) -> Path: return Path("/data/results").joinpath(filename) - def _build_async_result_path(self, job: Job, task_params: dict, sequence: int) -> Path: + def _build_task_result_path(self, job: Job, task_params: dict, sequence: int, execution_mode: str) -> Path: base_path = self._build_result_path(job, task_params) filename = f"{base_path.stem}_{sequence}{base_path.suffix or '.fits'}" - return Path("/data/results/async").joinpath(filename) + return Path("/data/results").joinpath(execution_mode, filename) - def dispatch(self, job: Job): - """Dispatch a cutout request to the backend. - - Parameters - ---------- - job - The submitted job description. - - Returns - ------- - dramatiq.Message - The dispatched message to the backend. - - Notes - ----- - Currently, only one dataset ID and only one stencil are supported. - This limitation is expected to be relaxed in a later version. - """ - print(f"[dispatch] job_id={job.job_id} dispatching to backend") - cutout_params = CutoutParameters.from_job_parameters(job.parameters) - print(f"[dispatch] cutout_params: {cutout_params}") - - print(f"[dispatch] calling convert_to_list_of_task_params with cutout_params: {cutout_params}") - tasks_params = self.convert_to_list_of_task_params(cutout_params) - print(f"[dispatch] tasks_params: {tasks_params}") - - # Celery tasks signature - tasks = [] - - for t in tasks_params: - print(f"[dispatch] checking survey access for user_id={job.owner} survey_id={t['id']}") - if not self._survey_access_policy.can_request_cutout(user_id=job.owner, survey_id=t["id"]): - raise PermissionDeniedError(f"User has no access to survey {t['id']}") - - print(f"[dispatch] building result path for job_id={job.job_id} task_params={t}") - resultfile = self._build_result_path(job, t) - print(f"[dispatch] resultfile path: {resultfile}") - - # If color composition requested, collect files per RGB band - if t.get("color"): - print(f"[dispatch] color composition requested, parsing rgb_bands for task_params={t}") - - # parse rgb_bands: accept 'gri', 'g,r,i' or 'g r i' - raw = t.get("rgb_bands", "gri") - if "," in raw: - bands = [b.strip() for b in raw.split(",") if b.strip()] - elif " " in raw: - bands = [b.strip() for b in raw.split() if b.strip()] - else: - bands = list(raw) - - files_map = {} - for b in bands: - files_b = self._file_locator.find_files(survey_id=t["id"], stencil=t["stencil_obj"], band=b) - if not files_b: - raise ParameterError(f"No files found for band {b} in the requested region") - # keep only paths that exist on the current filesystem - candidate_paths = [str(f.file_path) for f in files_b if f.file_path] - existing = [p for p in candidate_paths if Path(p).exists()] - if not existing: - raise ParameterError(f"No available files on disk for band {b} in the requested region") - files_map[b] = existing - - # Debug logging: show files_map and existence - print(f"[policy] dispatch: files_map for bands={bands}: {files_map}") - for band_name, paths in files_map.items(): - for p in paths: - print(f"[policy] file check: band={band_name} path={p} exists=True") - - tasks.append( - image_cutout.s( - job_id=job.job_id, - source_id=t["id"], - stencil=t["stencil"], - files=files_map, - engine=t["engine"], - band=t["band"], - format=t["format"], - path=str(resultfile), - color=t.get("color", False), - rgb_bands=t.get("rgb_bands"), - persist=t.get("persist", False), - ) - ) - else: - print( - f"[dispatch] single band requested, finding files for survey_id={t['id']} stencil={t['stencil_obj']} band={t['band']}" - ) - - files = self._file_locator.find_files(survey_id=t["id"], stencil=t["stencil_obj"], band=t["band"]) - print( - f"[policy] dispatch: found {len(files)} files for survey_id={t['id']} stencil={t['stencil_obj']} band={t['band']}" - ) - - if not files: - raise ParameterError("No files found for the requested region") - candidate = [str(f.file_path) for f in files if f.file_path] - existing = [p for p in candidate if Path(p).exists()] - if not existing: - raise ParameterError("No available files on disk for the requested region") - tasks.append( - image_cutout.s( - job_id=job.job_id, - source_id=t["id"], - stencil=t["stencil"], - files=existing, - engine=t["engine"], - band=t["band"], - format=t["format"], - path=str(resultfile), - color=False, - rgb_bands=t.get("rgb_bands"), - persist=t.get("persist", False), - ) - ) - - if len(tasks) == 1: - return tasks[0].apply_async() - - raise ParameterError("Only one cutout task is supported in sync mode") - - def create_tasks_for_job(self, job: Job, params: list[JobParameter]) -> list: + def create_tasks_for_job(self, job: Job, params: list[JobParameter], execution_mode: str = "async") -> list: """Create one Task row per cutout execution unit (stencil × band × format × engine).""" cutout_params = CutoutParameters.from_job_parameters(params) task_dicts = self.convert_to_list_of_task_params(cutout_params) @@ -201,7 +77,7 @@ def create_tasks_for_job(self, job: Job, params: list[JobParameter]) -> list: for sequence, t in enumerate(task_dicts, start=1): if not self._survey_access_policy.can_request_cutout(user_id=job.owner, survey_id=t["id"]): raise PermissionDeniedError(f"User has no access to survey {t['id']}") - output_path = str(self._build_async_result_path(job, t, sequence)) + output_path = str(self._build_task_result_path(job, t, sequence, execution_mode)) stencil_obj = t["stencil_obj"] stencil_dict = stencil_obj.to_dict() task = SQLTask.objects.create( @@ -226,7 +102,7 @@ def dispatch_async(self, job: Job, message_id: str): logger.info("[dispatch_async] job_id=%s message_id=%s", job.job_id, message_id) db_tasks = list(SQLTask.objects.filter(job_id=int(job.job_id)).order_by("sequence")) - cutout_sigs = [run_cutout_for_pos.s(job_id=job.job_id, task_id=str(task.id)) for task in db_tasks] + cutout_sigs = [perform_cutout_task.s(job_id=job.job_id, task_id=str(task.id)) for task in db_tasks] result = celery_chord(cutout_sigs)(finalize_job.s(job_id=job.job_id).set(task_id=message_id)) logger.info("[dispatch_async] chord dispatched: %d task(s), callback_id=%s", len(cutout_sigs), message_id) return result diff --git a/cutout/service/tasks.py b/cutout/service/tasks.py index 14ec2ef..54c6b2e 100644 --- a/cutout/service/tasks.py +++ b/cutout/service/tasks.py @@ -1,165 +1,13 @@ -import json import logging -from pathlib import Path from typing import Any from django.utils import timezone from config import celery_app -from cutout.lib.des_cutout import DesCutout -from cutout.service.cutout_engine import create_cutout_engine +from cutout.service.cutout_runner import perform_cutout from cutout.service.models import Job, Task -def _validate_input_files(files: list[str] | dict[str, list[str]] | None) -> None: - if not files: - return - - # normalize to list of paths for existence check - paths: list[str] = [] - if isinstance(files, dict): - for v in files.values(): - paths.extend(v or []) - else: - paths = list(files) - - missing = [f for f in paths if not Path(f).exists()] - if missing: - msg = "Input file unavailable: " + ", ".join(missing) - raise FileNotFoundError(msg) - - -def _ensure_unpacked( - files: list[str] | dict[str, list[str]] | None, -) -> list[str] | dict[str, list[str]] | None: - """If any input paths point to compressed ``.fz`` archives, unpack them to a tmp location and - return a structure of uncompressed paths suitable for engines that require ``.fits`` files. - - .. note:: - - ``fits_cut`` from astrocut handles ``.fz`` natively via ``.section``, so this helper - is currently **not used** by ``image_cutout``. It is kept as a legacy utility for - engines or ad-hoc scripts that need uncompressed files on disk. - """ - if not files: - return files - - dc = DesCutout() - - def _unpack_path(p: str) -> str: - pth = Path(p) - if pth.suffix == ".fz": - out_name = pth.name.rsplit(".fz", 1)[0] - out_path = dc.tmp_path.joinpath(out_name) - if not out_path.exists(): - try: - dc.funpack(pth, out_path) - except Exception: - # let downstream code fail with clearer message if unpack fails - pass - return str(out_path) - return str(p) - - if isinstance(files, dict): - out: dict[str, list[str]] = {} - for k, lst in files.items(): - out[k] = [_unpack_path(p) for p in (lst or [])] - print(f"[tasks] _ensure_unpacked: band={k} unpacked_paths={out[k]}") - return out - - return [_unpack_path(p) for p in files] - - -@celery_app.task() -def des_cutout_circle(**kwargs) -> str: - dc = DesCutout() - result = dc.cutout_circle(**kwargs) - return str(result) - - -@celery_app.task() -def image_cutout( - job_id: str, - source_id: str, - stencil: dict[str, Any], - engine: str, - band: str, - format: str, - path: str, - files: list[str] | None = None, - color: bool = False, - rgb_bands: str | None = None, - persist: bool = False, -) -> str: - print( - "[tasks] image_cutout START " - f"job_id={job_id} engine={engine} band={band} format={format} " - f"color={color} rgb_bands={rgb_bands}" - ) - print(f"[tasks] image_cutout initial files={files}") - cutout_engine = create_cutout_engine(engine) - _validate_input_files(files) - - try: - print(f"[tasks] calling engine.run_cutout engine={engine} path={path}") - result = cutout_engine.run_cutout( - source_id=source_id, - stencil=stencil, - input_files=files, - band=band, - output_format=format, - output_path=path, - color=color, - rgb_bands=rgb_bands, - persist=persist, - ) - print(f"[tasks] engine.run_cutout completed result={result}") - except Exception as e: - print(f"[tasks] engine.run_cutout raised: {type(e).__name__}: {e}") - raise - return str(result) - - -def fake_image_cutout( - job_id: str, - source_id: str, - stencil: dict[str, Any], - engine: str, - band: str, - format: str, - path: str, - files: list[str] | None = None, - color: bool = False, - rgb_bands: str | None = None, - persist: bool = False, -) -> dict[str, Any]: - result_path = Path(path) - result_path.parent.mkdir(parents=True, exist_ok=True) - payload = { - "job_id": job_id, - "source_id": source_id, - "stencil": stencil, - "engine": engine, - "band": band, - "format": format, - "path": path, - "files": files or [], - "color": color, - "rgb_bands": rgb_bands, - "persist": persist, - "mode": "fake_async_result", - } - result_path.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8") - - mime_type = "image/png" if str(format).lower() == "png" else "application/fits" - return { - "result_id": result_path.stem, - "file_path": str(result_path), - "mime_type": mime_type, - "size": result_path.stat().st_size, - } - - @celery_app.task( bind=True, autoretry_for=(Job.DoesNotExist, Task.DoesNotExist), @@ -167,81 +15,17 @@ def fake_image_cutout( retry_jitter=True, retry_kwargs={"max_retries": 5}, ) -def run_cutout_for_pos(self, job_id: str, task_id: str) -> dict[str, Any]: - """Execute a single cutout unit. Reads all parameters from the Task row in DB.""" +def perform_cutout_task(self, job_id: str, task_id: str) -> dict[str, Any]: + """Celery wrapper for `perform_cutout`. All parameters are read from the Task row in DB.""" logger = logging.getLogger("cutout") - job_pk = int(str(job_id).strip()) - task_pk = int(str(task_id).strip()) - logger.info( - "[run_cutout_for_pos] celery_task_id=%s retries=%s job_id=%r task_id=%r", + "[perform_cutout_task] celery_task_id=%s retries=%s job_id=%r task_id=%r", self.request.id, self.request.retries, job_id, task_id, ) - - job = Job.objects.get(pk=job_pk) - task = Task.objects.get(pk=task_pk) - - if job.phase == Job.ExecutionPhase.ABORTED: - logger.info("[run_cutout_for_pos] Job %r is ABORTED, skipping", job_id) - return {} - - # First task to run transitions QUEUED → EXECUTING (idempotent under concurrency) - Job.objects.filter(pk=job_pk, phase=Job.ExecutionPhase.QUEUED).update( - phase=Job.ExecutionPhase.EXECUTING, - start_time=timezone.now(), - ) - - # PENDING → EXECUTING (idempotent — only first call wins if multi-band) - Task.objects.filter(pk=task_pk, status=Task.Status.PENDING).update( - status=Task.Status.EXECUTING, - start_time=timezone.now(), - ) - - try: - result = fake_image_cutout( - job_id=job_id, - source_id=task.survey_id, - stencil=task.stencil, - engine=task.engine, - band=task.band, - format=task.output_format, - path=task.output_path, - color=task.color, - rgb_bands=task.rgb_bands, - persist=task.persist, - ) - - job.results.create( - result_id=result["result_id"], - sequence=task.sequence, - size=result.get("size") or 0, - mime_type=result.get("mime_type"), - url=f"/api/async/{job_id}/results/{result['result_id']}", - file_path=result.get("file_path"), - ) - - Task.objects.filter(pk=task_pk).update( - status=Task.Status.COMPLETED, - end_time=timezone.now(), - ) - - logger.info("[run_cutout_for_pos] completed task_id=%r result_id=%s", task_id, result.get("result_id")) - return {"task_id": task_pk, "result_id": result["result_id"]} - - except Exception as exc: - Job.objects.filter(pk=job_pk).update( - phase=Job.ExecutionPhase.ERROR, - end_time=timezone.now(), - ) - Task.objects.filter(pk=task_pk).update( - status=Task.Status.ERROR, - end_time=timezone.now(), - error_message=str(exc), - ) - raise + return perform_cutout(job_id, task_id) @celery_app.task @@ -270,18 +54,11 @@ def finalize_job(_results: list[dict[str, Any]], job_id: str) -> None: logger.info("[finalize_job] job_id=%r marked COMPLETED", job_id) -# @celery_app.task() -# def on_success_cutout(job_id: str, ) -> str: -# return str(result) - - @celery_app.task(bind=True) -# def job_completed(job_id: str, results) -> str: def job_completed(result, **kwargs) -> str: print(result) print(kwargs) return f"TESTE: {result}" - # uws_job_completed(job_id=job_id, results=results) @celery_app.task() diff --git a/cutout/service/teste_classe.py b/cutout/service/teste_classe.py deleted file mode 100755 index c122dfc..0000000 --- a/cutout/service/teste_classe.py +++ /dev/null @@ -1,76 +0,0 @@ -from pathlib import Path - -from celery import group - -# from cutout.lib.cutout import Cutout -# from cutout.lib.des_cutout import DesCutout -# from cutout.service.policy import ImageCutoutPolicy -# from cutout.service.uws.models import JobParameter -# from cutout.service.uws.service import JobService -# from cutout.users.models import User - -if __name__ == "__main__": - from cutout.service.tasks import task_1, task_completed - - header = [task_1.s(1, 2), task_1.s(3, 4)] - - g = group(header) - gresult = g.apply_async() - print(gresult.get()) - - # params = [] - # teste_params = { - # "id": "des_dr2", - # "runid": "MeujobCutout", - # "band": "g", - # "format": "fits", - # "pos": "CIRCLE 36.30911 -10.18749 2", - # } - # for key, value in teste_params.items(): - # if key.lower() == "runid": - # run_id = value - # else: - # params.append(JobParameter(parameter_id=key.lower(), value=value, is_post=False)) - # user = User.objects.get(pk=1) - # policy = ImageCutoutPolicy() - # job_service = JobService(policy=policy) - # job = job_service.create(user=user, params=params, run_id=run_id) - # job_service.start(user, job_id=job.id) - # cutouts = [ - # { - # "id": "des_dr2", - # "stencil": {"type": "circle", "center": {"ra": 36.30911, "dec": -10.18749}, "radius": 2.0}, - # "band": "g", - # "format": "fits", - # } - # ] - # for c in cutouts: - # dc = Cutout(source_id=c["id"], stencil=c["stencil"], band=c["band"], format=c["format"]) - # # filename = "{:.5f}_{:.5f}_{}.fits".format(round(c["stencil"]["center"]["ra"], 5), round(c["stencil"]["center"]["dec"], 5), c["band"]) - # filename = "teste.fits" - # resultfile = Path("/data/results").joinpath(filename) - # dc.create(path=resultfile) - # cutouts = [ - # {"ra": 36.30911, "dec": -10.18749, "size": 2.0, "band": "g", "format": "fits"}, # 1 - Tile - # # {"ra": 36.30911, "dec": -10.18749, "size": 2.0, "band": "gri", "format": "png"}, # 1 - Tile - # # {"ra": 36.15801, "dec": -10.33579, "size": 2.0, "band": "g", "format": "fits"}, # 2 - Tile - # # {"ra": 36.15801, "dec": -10.33579, "size": 2.0, "band": "gri", "format": "png"}, # 2 - Tile - # # {"ra": 35.23676, "dec": -10.33269, "size": 10.0, "band": "g", "format": "fits"}, # 3 - Tile - # # {"ra": 35.23676, "dec": -10.33269, "size": 10.0, "band": "gri", "format": "png"}, # 3 - Tile - # ] - # dc = DesCutout() - # for c in cutouts: - # if c["format"] == "fits": - # filename = "{:.5f}_{:.5f}_{}.fits".format(round(c["ra"], 5), round(c["dec"], 5), c["band"]) - # resultfile = Path("/data/results").joinpath(filename) - # result = dc.single_cutout_fits( - # ra=c["ra"], dec=c["dec"], size_arcmin=c["size"], band=c["band"], path=resultfile - # ) - # print(result) - # if c["format"] == "png": - # filename = "{:.5f}_{:.5f}.png".format(round(c["ra"], 5), round(c["dec"], 5)) - # resultfile = Path("/data/results").joinpath(filename) - # result = dc.single_cutout_png( - # ra=c["ra"], dec=c["dec"], size_arcmin=c["size"], band=c["band"], path=resultfile - # ) - # print(result) diff --git a/cutout/service/teste_cutout_runner.py b/cutout/service/teste_cutout_runner.py new file mode 100644 index 0000000..437dc99 --- /dev/null +++ b/cutout/service/teste_cutout_runner.py @@ -0,0 +1,135 @@ +"""Bateria manual de testes do `perform_cutout` baseada em exemplos.txt. + +Cenários já validados no fluxo sync, usando os FITS reais Y6A1/r4907 em +/data/tiles/des_dr2 (disponíveis no container). + +Uso (dentro do container): + + docker compose exec django python manage.py shell + + >>> from cutout.service.teste_cutout_runner import run_exemplo, run_todos + >>> run_exemplo(1) # cria Job+Task e executa perform_cutout direto + >>> run_todos() # cenários 1 a 8 + >>> job, task = run_exemplo(9, dispatch=False) # só cria; executar via worker: + >>> from cutout.service.tasks import perform_cutout_task + >>> perform_cutout_task.delay(job.id, task.id) +""" + +from __future__ import annotations + +from pathlib import Path + +from cutout.service.cutout_runner import perform_cutout +from cutout.service.uws.models import JobParameter +from cutout.service.uws.service import JobService + +EXEMPLOS = { + 1: { + "params": {"pos": "CIRCLE 0.5 0.017 0.016667", "band": "g", "format": "fits"}, + "esperado": "COMPLETED — FITS ~0.8MB, 1 tile (DES0002+0001, 1')", + }, + 2: { + "params": {"pos": "CIRCLE 1.25 -0.683 0.05", "band": "z", "format": "fits"}, + "esperado": "COMPLETED — FITS ~7.2MB (DES0005-0041, 3')", + }, + 3: { + "params": {"pos": "CIRCLE 0.75 2.867 0.033333", "format": "png", "color": "true", "rgb_bands": "gri"}, + "esperado": "COMPLETED — PNG RGB ~2.0MB (DES0003+0252, 2')", + }, + 4: { + "params": {"pos": "CIRCLE 0.5 0.017 0.116667", "band": "Y", "format": "fits"}, + "esperado": "COMPLETED — FITS ~38.9MB (DES0002+0001, 7')", + }, + 5: { + "params": {"pos": "CIRCLE 1.10 2.50 0.083333", "band": "r", "format": "fits"}, + "esperado": "COMPLETED — mosaico de 2 tiles, header com NINPUTS=2 (5')", + }, + 6: { + "params": {"pos": "CIRCLE 1.07 2.15 0.083333", "band": "r", "format": "fits"}, + "esperado": "COMPLETED — cobertura parcial, borda leste preenchida com zeros (5')", + }, + 7: { + "params": {"pos": "CIRCLE 10.0 10.0 0.016667", "band": "r", "format": "fits"}, + "esperado": "Task ERROR — 'No available files on disk...' (fora do footprint)", + }, + 8: { + "params": {"pos": "CIRCLE 0.5 1.0 0.166667", "band": "r", "format": "fits"}, + "esperado": "Task ERROR — vão entre tiles (Dec entre DES0002+0001 e DES0002+0209)", + }, + 9: { + "params": {"pos": "CIRCLE 0.5 2.15 0.25", "band": "i", "format": "fits"}, + "esperado": "COMPLETED via worker — 15', caso que estoura o timeout do sync", + }, +} + + +def _get_user(username: str | None = None): + from cutout.users.models import User + + if username: + return User.objects.get(username=username) + return User.objects.filter(username="dev").first() or User.objects.first() + + +def create_job_exemplo(n: int, user=None): + """Cria Job + Tasks (sem despachar) para o cenário `n` e retorna (job, task).""" + scenario = EXEMPLOS[n] + params = [JobParameter(parameter_id="id", value="des_dr2", is_post=True)] + params += [JobParameter(parameter_id=key, value=value, is_post=True) for key, value in scenario["params"].items()] + + job = JobService().create(user=user or _get_user(), params=params, run_id=f"teste_exemplo_{n}") + task = job.tasks.order_by("sequence").first() + return job, task + + +def _print_status(job, task) -> None: + job.refresh_from_db() + task.refresh_from_db() + + print(f" Job {job.id}: phase={job.phase} start={job.start_time} end={job.end_time}") + print(f" Task {task.id}: status={task.status} start={task.start_time} end={task.end_time}") + if task.error_message: + print(f" Task error_message: {task.error_message}") + + for result in job.results.order_by("sequence"): + exists = Path(result.file_path).exists() if result.file_path else False + print( + f" Result {result.result_id}: size={result.size} mime={result.mime_type} " + f"path={result.file_path} exists={exists}" + ) + if not job.results.exists(): + print(" Sem JobResult registrado.") + + +def run_exemplo(n: int, user=None, dispatch: bool = True): + """Executa o cenário `n` chamando perform_cutout(job_id, task_id) diretamente. + + Com dispatch=False apenas cria o Job+Task (para despachar manualmente via + perform_cutout_task.delay e observar o worker). + """ + scenario = EXEMPLOS[n] + print(f"=== Exemplo {n}: {scenario['params']}") + print(f" Esperado: {scenario['esperado']}") + + job, task = create_job_exemplo(n, user) + print(f" Criado job_id={job.id} task_id={task.id}") + + if not dispatch: + print(" Despache com: perform_cutout_task.delay(job.id, task.id)") + return job, task + + try: + result = perform_cutout(job.id, task.id) + print(f" perform_cutout retornou: {result}") + except Exception as exc: + print(f" perform_cutout levantou: {type(exc).__name__}: {exc}") + + _print_status(job, task) + return job, task + + +def run_todos(user=None) -> None: + """Roda os cenários 1 a 8 (o 9 é grande — rodar via worker com dispatch=False).""" + for n in range(1, 9): + run_exemplo(n, user=user) + print() diff --git a/cutout/service/teste_funcoes.py b/cutout/service/teste_funcoes.py deleted file mode 100755 index 751013a..0000000 --- a/cutout/service/teste_funcoes.py +++ /dev/null @@ -1,260 +0,0 @@ -from pathlib import Path - -import numpy as np -from astropy import units as u -from astropy.coordinates import SkyCoord -from astropy.io import fits -from astropy.nddata.utils import Cutout2D -from astropy.visualization import make_lupton_rgb -from astropy.wcs import WCS - - -def cutout_verts(RA_center, DEC_center, size_arcmin): - """Defines the position of vertices in each cutout. - See the pos_angle where the vertices are sorted. - - Parameters - ---------- - RA_center : float - Equatorial coordinate of center of cutout. - DEC_center : float - Equatorial coordinate of center of cutout. - size_arcmin : float - Size (length of each side) of cutout, in arcmin. - - Returns - ------- - SkyCoord astropy object - Location of vertices of cutout. - """ - pos_angle = [45, 315, 225, 135] * u.deg - c1 = SkyCoord(RA_center * u.deg, DEC_center * u.deg, frame="icrs") - sep = 0.5 * np.sqrt(2.0) * size_arcmin * u.arcmin - RA = c1.directional_offset_by(pos_angle, sep).ra.deg - DEC = c1.directional_offset_by(pos_angle, sep).dec.deg - return SkyCoord(RA * u.deg, DEC * u.deg, frame="icrs") - - -def tiles_from_cat(cat, file_path): - """Read information about tiles. - TODO: read more information about the vertices of tiles in - order to have a correct overlap in case cutouts are in the - edge of tiles. - - Parameters - ---------- - cat : SkyCoord astropy object - Object with information about coordinates of vertices. - file_path : str - File with tile's information. - - Returns - ------- - list - List of tiles where the vertices of cutout reside. - """ - ra_ll, dec_ll, ra_ul, dec_ul, ra_ur, dec_ur, ra_lr, dec_lr = np.loadtxt( - file_path, usecols=(9, 10, 11, 12, 13, 14, 15, 16), delimiter=",", unpack=True - ) - tile_names = np.loadtxt(file_path, usecols=(2), delimiter=",", dtype=str, unpack=True) - - ra = cat.ra.deg - dec = cat.dec.deg - - idx_ = [] - for j in range(4): - idx_.append(np.argwhere((ra_ll < ra[j]) & (ra_ur > ra[j]) & (dec_ll < dec[j]) & (dec_ur > dec[j]))[0][0]) - tile_match = [tile_names[k] for k in idx_] - # TODO: Remover tiles duplicadas - return tile_match - - -def cutout_fits(RA_center, DEC_center, size_arcmin, tile_name, band, path, mode="partial"): - """Return data (image array and wcs) from tile. - - Parameters - ---------- - RA_center : float - Equatorial coordinate of center of tile. - DEC_center : float - Equatorial coordinate of center of tile. - size_arcmin : float - Size of cutout in arcmin. - tile_name : str - Name of tile where total or part of the cutout image resides. - band : str - Band of image. - path : str - Path to folder where the FITS files are stored. - mode : str, optional - Mode of cutout. See: - https://docs.astropy.org/en/stable/api/astropy.nddata.Cutout2D.html - By default set to 'partial'. - - Returns - ------- - arrays - Two arrays, one with image data and other with WCS astropy object. - """ - # file_name_ = glob.glob(path + "/" + tile_name + "_*_" + band + ".fits") - # file_name = file_name_[0] - # TODO: Ter o tilename completo. - fits_filepath = path.joinpath(f"{tile_name}_r4920p01_{band}.fits") - f = fits.open(fits_filepath) - wcs = WCS(f[1].header) - print(fits_filepath) - cutout1 = Cutout2D( - fits.getdata(fits_filepath, ext=0), - (SkyCoord(ra=RA_center * u.degree, dec=DEC_center * u.degree, frame="icrs")), - size_arcmin * u.arcmin, - wcs=wcs, - mode=mode, - ) - - return cutout1.data, cutout1.wcs.to_header() - - -def get_fits_data(RA, DEC, size, tiles, band, path): - """Access data (image and wcs) from FITS files. - - Parameters - ---------- - RA : float - Equatorial coordinate of the center of cutout (degrees). - DEC : float - Equatorial coordinate of the center of cutout (degrees). - size : float - Size of cutout (arcmin). - tiles : list - List with the tiles where the vertices of cutout reside. - Sorted by: Upper left, Upper right, Lower right, Lower left. - band : str - Band of cutout. - path : str - Path to folder with the FITS files. - """ - - tile_un, ind = np.unique(tiles[:], return_index=True) - tile_un = tile_un[np.argsort(ind)] - - if len(tile_un) == 1: - data_, wcs_ = cutout_fits(RA, DEC, size, tiles[0], band, path) - return (data_, wcs_) - - elif len(tile_un) == 2: - data_1, wcs_1 = cutout_fits(RA, DEC, size, tile_un[0], band, path, "trim") - data_2, wcs_2 = cutout_fits(RA, DEC, size, tile_un[1], band, path, "trim") - - if np.shape(data_1)[1] < np.shape(data_1)[0]: - # side-by-side - data_1 = data_1[:, :-118] - data_2 = data_2[:, 118:] - data_ = np.concatenate((data_1, data_2), axis=1) - return (data_, wcs_1) - - else: - # top-bottom - data_1 = data_1[114:, :] - data_2 = data_2[:-114, :] - data_ = np.concatenate((data_2, data_1), axis=0) - return (data_, wcs_2) - - elif len(tile_un) == 3: - data_1, wcs_1 = cutout_fits(RA, DEC, size, tile_un[0], band, path, "trim") - data_2, wcs_2 = cutout_fits(RA, DEC, size, tile_un[1], band, path, "trim") - data_3, wcs_3 = cutout_fits(RA, DEC, size, tile_un[2], band, path, "trim") - - # Biggest at bottom: - if np.shape(data_1)[1] < np.shape(data_3)[1]: - data_12 = np.concatenate((data_1[118:, :-118], data_2[118:, 118:]), axis=1) - data_ = np.concatenate((data_3[:-118, :], data_12[:, 0 : np.shape(data_3)[1]]), axis=0) - # Biggest at top: - else: - data_23 = np.concatenate((data_2[:-118, 118:], data_3[:-118, :-118]), axis=1) - data_ = np.concatenate((data_23, data_1[118:, 0 : np.shape(data_23)[1]]), axis=0) - return (data_, wcs_3) - - -def write_cutout_file(data, wcs, filename): - """Saves cutout file. - - Parameters - ---------- - data : array - Array with image data. - wcs : astropy object - Information about world coordinate system of cutout. - filename : str - Name of file to be saved. - """ - hdu = fits.PrimaryHDU(data) - hdu.header.update(wcs) - hdu.writeto(filename, overwrite=True) - - -def cutout_lupton(g_data, r_data, i_data, minimum, stretch, Q, filename): - """Make RGB image and saves as png or jpg files using Lupton method. - TODO: Improve quality of image for cutout with saturated data. - - Parameters - ---------- - g_data : array - Cutout data from first band. - r_data : array - Cutout data from second band. - i_data : array - Cutout data from third band. - filename : str - Name of file to be saved. - """ - - rgb_default = make_lupton_rgb(i_data, r_data, g_data, minimum=minimum, stretch=stretch, Q=Q, filename=filename) - - -if __name__ == "__main__": - cutout_1_tile = {"ra": 36.30911, "dec": -10.18749, "size": 2.0, "band": "g"} - cutout_2_tile = {"ra": 36.15801, "dec": -10.33579, "size": 2.0, "band": "g"} - cutout_3_tiles = {"ra": 35.23676, "dec": -10.33269, "size": 10.0, "band": "g"} - - cutout = cutout_3_tiles - - ra = cutout["ra"] - dec = cutout["dec"] - size = cutout["size"] - band = cutout["band"] - # Calculates the cutout's vertices to access tiles - verts = cutout_verts(ra, dec, size) - print(verts) - # Set tiles from vertices - tile_list = Path("/app/cutout/service/coaddtiles-20121015.csv") - tiles = tiles_from_cat(verts, tile_list) - print(tiles) - - # Cutout Fits: - path_to_fits = Path("/data/tiles") - data, wcs_ = get_fits_data(ra, dec, size, tiles, band, path_to_fits) - # print(data) - - # Exemplo Cutout FITS - # result_path = Path("/data/results") - # filename = "{:.5f}_{:.5f}_{}.fits".format(round(ra, 5), round(dec, 5), band) - # filepath = result_path.joinpath(filename) - # if filepath.exists(): - # filepath.unlink() - # write_cutout_file(data, wcs_, filepath) - - # Exemplo Cutout PNG - # Usando 3 bandas, primeiro faz 3 cutouts fits em g, r, i - # Depois gera a png. - result_path = Path("/data/results") - filename = f"{round(ra, 5):.5f}_{round(dec, 5):.5f}.png" - filepath = result_path.joinpath(filename) - if filepath.exists(): - filepath.unlink() - print(filepath) - # Fits g, r, i - data_g, wcs_g = get_fits_data(ra, dec, size, tiles, "g", path_to_fits) - data_r, wcs_r = get_fits_data(ra, dec, size, tiles, "r", path_to_fits) - data_i, wcs_i = get_fits_data(ra, dec, size, tiles, "i", path_to_fits) - - cutout_lupton(data_g, data_r, data_i, 0.05, 10, 0.5, filepath) diff --git a/cutout/service/tests/test_async_api.py b/cutout/service/tests/test_async_api.py index 15168fd..4f7aba2 100644 --- a/cutout/service/tests/test_async_api.py +++ b/cutout/service/tests/test_async_api.py @@ -4,6 +4,8 @@ from django.urls import reverse from rest_framework.test import APIClient +from cutout.service import cutout_runner +from cutout.service.discovery import FileDescriptor from cutout.service.policy import ImageCutoutPolicy from cutout.service.uws.models import JobParameter from cutout.service.uws.service import JobService @@ -13,15 +15,47 @@ def _patch_async_result_path(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - def _fake_path(self, job, task_params, sequence): + def _fake_path(self, job, task_params, sequence, execution_mode): extension = "png" if str(task_params.get("format", "fits")).lower() == "png" else "fits" - return tmp_path / f"job_{job.job_id}_{sequence}.{extension}" + return tmp_path / execution_mode / f"job_{job.job_id}_{sequence}.{extension}" - monkeypatch.setattr(ImageCutoutPolicy, "_build_async_result_path", _fake_path) + monkeypatch.setattr(ImageCutoutPolicy, "_build_task_result_path", _fake_path) + + +class _FakeLocator: + def __init__(self, input_file: Path): + self._input_file = input_file + + def find_files(self, *, survey_id, stencil, band=None): + return [ + FileDescriptor( + tile_id="DES0002+0001", + archive_path="Y6A1/r4907/DES0002+0001/p01/coadd", + file_path=self._input_file, + band=band, + ) + ] + + +class _FakeEngine: + def run_cutout(self, **kwargs): + output_path = Path(kwargs["output_path"]) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_bytes(b"fake fits data") + return output_path + + +def _patch_cutout_execution(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Replace file discovery and the cutout engine so perform_cutout runs without real tiles.""" + input_file = tmp_path / "DES0002+0001_r4907p01_g.fits.fz" + input_file.write_bytes(b"tile data") + monkeypatch.setattr(cutout_runner, "DesCsvFileLocator", lambda: _FakeLocator(input_file)) + monkeypatch.setattr(cutout_runner, "create_cutout_engine", lambda name: _FakeEngine()) def test_async_create_runs_job_and_persists_result(user, settings, monkeypatch, tmp_path): _patch_async_result_path(monkeypatch, tmp_path) + _patch_cutout_execution(monkeypatch, tmp_path) settings.CELERY_TASK_ALWAYS_EAGER = True settings.CELERY_TASK_EAGER_PROPAGATES = True client = APIClient() @@ -58,6 +92,7 @@ def test_async_create_runs_job_and_persists_result(user, settings, monkeypatch, def test_async_phase_run_starts_pending_job(user, settings, monkeypatch, tmp_path): _patch_async_result_path(monkeypatch, tmp_path) + _patch_cutout_execution(monkeypatch, tmp_path) settings.CELERY_TASK_ALWAYS_EAGER = True settings.CELERY_TASK_EAGER_PROPAGATES = True client = APIClient() diff --git a/cutout/service/tests/test_cutout_parameters.py b/cutout/service/tests/test_cutout_parameters.py index ee415fb..c040a4a 100644 --- a/cutout/service/tests/test_cutout_parameters.py +++ b/cutout/service/tests/test_cutout_parameters.py @@ -17,7 +17,7 @@ def test_cutout_parameters_parses_engine() -> None: assert parsed.ids == ["des_dr2"] -def test_cutout_parameters_without_engine_keeps_empty_list() -> None: +def test_cutout_parameters_without_engine_defaults_to_astrocut() -> None: params = [ JobParameter(parameter_id="id", value="des_dr2"), JobParameter(parameter_id="pos", value="CIRCLE 10 0 1"), @@ -27,4 +27,4 @@ def test_cutout_parameters_without_engine_keeps_empty_list() -> None: parsed = CutoutParameters.from_job_parameters(params) - assert parsed.engines == [] + assert parsed.engines == ["astrocut"] diff --git a/cutout/service/tests/test_cutout_runner.py b/cutout/service/tests/test_cutout_runner.py new file mode 100644 index 0000000..b6513dc --- /dev/null +++ b/cutout/service/tests/test_cutout_runner.py @@ -0,0 +1,210 @@ +from pathlib import Path + +import pytest + +from cutout.service import cutout_runner +from cutout.service.cutout_runner import perform_cutout +from cutout.service.discovery import FileDescriptor +from cutout.service.models import Job, Task +from cutout.service.uws.exceptions import ParameterError + +pytestmark = pytest.mark.django_db + +CIRCLE_STENCIL = {"type": "circle", "center": {"ra": 0.5, "dec": 0.017}, "radius": 0.016667} + + +def _create_job_and_task(user, output_path, **task_overrides): + job = Job.objects.create(owner=user, phase=Job.ExecutionPhase.PENDING) + fields = { + "job": job, + "sequence": 1, + "survey_id": "des_dr2", + "stencil": CIRCLE_STENCIL, + "stencil_type": "circle", + "band": "g", + "output_format": "fits", + "engine": "astrocut", + "color": False, + "rgb_bands": "gri", + "persist": False, + "output_path": str(output_path), + } + fields.update(task_overrides) + task = Task.objects.create(**fields) + return job, task + + +class _FakeLocator: + def __init__(self, input_file: Path | None): + self._input_file = input_file + + def find_files(self, *, survey_id, stencil, band=None): + if self._input_file is None: + return [] + return [ + FileDescriptor( + tile_id="DES0002+0001", + archive_path="Y6A1/r4907/DES0002+0001/p01/coadd", + file_path=self._input_file, + band=band, + ) + ] + + +class _FakeEngine: + def __init__(self, payload: bytes = b"fake fits data"): + self.payload = payload + self.calls: list[dict] = [] + + def run_cutout(self, **kwargs): + self.calls.append(kwargs) + output_path = Path(kwargs["output_path"]) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_bytes(self.payload) + return output_path + + +class _FailingEngine: + def run_cutout(self, **kwargs): + raise RuntimeError("engine exploded") + + +def _patch_runner(monkeypatch, tmp_path, engine=None, input_file="present"): + if input_file == "present": + input_file = tmp_path / "DES0002+0001_r4907p01_g.fits.fz" + input_file.write_bytes(b"tile data") + engine = engine or _FakeEngine() + monkeypatch.setattr(cutout_runner, "DesCsvFileLocator", lambda: _FakeLocator(input_file)) + monkeypatch.setattr(cutout_runner, "create_cutout_engine", lambda name: engine) + return engine + + +def test_perform_cutout_success_records_result_and_statuses(user, monkeypatch, tmp_path): + engine = _patch_runner(monkeypatch, tmp_path) + job, task = _create_job_and_task(user, tmp_path / "out" / "job_1_g.fits") + + result = perform_cutout(job.id, task.id) + + task.refresh_from_db() + assert task.status == Task.Status.COMPLETED + assert task.start_time is not None + assert task.end_time is not None + assert task.end_time >= task.start_time + + job.refresh_from_db() + assert job.phase == Job.ExecutionPhase.EXECUTING + assert job.start_time is not None + assert job.end_time is None + + job_result = job.results.get() + assert job_result.result_id == "job_1_g" + assert job_result.sequence == task.sequence + assert job_result.size == len(engine.payload) + assert job_result.mime_type == "application/fits" + assert job_result.file_path == task.output_path + assert job_result.url == f"/api/async/{job.id}/results/job_1_g" + + assert result == { + "task_id": task.id, + "result_id": "job_1_g", + "file_path": task.output_path, + "size": job_result.size, + } + + engine_kwargs = engine.calls[0] + assert engine_kwargs["source_id"] == "des_dr2" + assert engine_kwargs["stencil"] == CIRCLE_STENCIL + assert engine_kwargs["band"] == "g" + assert engine_kwargs["output_format"] == "fits" + assert isinstance(engine_kwargs["input_files"], list) + + +def test_perform_cutout_color_passes_files_per_band(user, monkeypatch, tmp_path): + engine = _patch_runner(monkeypatch, tmp_path) + job, task = _create_job_and_task( + user, + tmp_path / "out" / "job_1_rgb.png", + color=True, + rgb_bands="gri", + output_format="png", + ) + + perform_cutout(job.id, task.id) + + engine_kwargs = engine.calls[0] + assert isinstance(engine_kwargs["input_files"], dict) + assert sorted(engine_kwargs["input_files"].keys()) == ["g", "i", "r"] + assert job.results.get().mime_type == "image/png" + + +def test_perform_cutout_is_idempotent_for_reruns(user, monkeypatch, tmp_path): + _patch_runner(monkeypatch, tmp_path) + job, task = _create_job_and_task(user, tmp_path / "out" / "job_1_g.fits") + + perform_cutout(job.id, task.id) + perform_cutout(job.id, task.id) + + assert job.results.count() == 1 + task.refresh_from_db() + assert task.status == Task.Status.COMPLETED + + +def test_perform_cutout_engine_error_marks_task_and_job(user, monkeypatch, tmp_path): + _patch_runner(monkeypatch, tmp_path, engine=_FailingEngine()) + job, task = _create_job_and_task(user, tmp_path / "out" / "job_1_g.fits") + + with pytest.raises(RuntimeError, match="engine exploded"): + perform_cutout(job.id, task.id) + + task.refresh_from_db() + assert task.status == Task.Status.ERROR + assert task.error_message == "engine exploded" + assert task.end_time is not None + + job.refresh_from_db() + assert job.phase == Job.ExecutionPhase.ERROR + assert job.end_time is not None + assert job.results.count() == 0 + + +def test_perform_cutout_no_files_marks_error(user, monkeypatch, tmp_path): + _patch_runner(monkeypatch, tmp_path, input_file=None) + job, task = _create_job_and_task(user, tmp_path / "out" / "job_1_g.fits") + + with pytest.raises(ParameterError, match="No files found"): + perform_cutout(job.id, task.id) + + task.refresh_from_db() + assert task.status == Task.Status.ERROR + assert "No files found" in task.error_message + + job.refresh_from_db() + assert job.phase == Job.ExecutionPhase.ERROR + + +def test_perform_cutout_skips_aborted_job(user, monkeypatch, tmp_path): + _patch_runner(monkeypatch, tmp_path) + job, task = _create_job_and_task(user, tmp_path / "out" / "job_1_g.fits") + job.phase = Job.ExecutionPhase.ABORTED + job.save() + + result = perform_cutout(job.id, task.id) + + assert result == {} + task.refresh_from_db() + assert task.status == Task.Status.PENDING + assert job.results.count() == 0 + + +def test_perform_cutout_rejects_task_from_another_job(user, monkeypatch, tmp_path): + _patch_runner(monkeypatch, tmp_path) + job_a, _ = _create_job_and_task(user, tmp_path / "out" / "job_a.fits") + job_b, task_b = _create_job_and_task(user, tmp_path / "out" / "job_b.fits") + + with pytest.raises(ValueError, match="does not belong"): + perform_cutout(job_a.id, task_b.id) + + task_b.refresh_from_db() + assert task_b.status == Task.Status.PENDING + job_a.refresh_from_db() + assert job_a.phase == Job.ExecutionPhase.PENDING diff --git a/cutout/service/tests/test_sync_api.py b/cutout/service/tests/test_sync_api.py new file mode 100644 index 0000000..b452c1e --- /dev/null +++ b/cutout/service/tests/test_sync_api.py @@ -0,0 +1,137 @@ +from pathlib import Path + +import pytest +from django.urls import reverse +from rest_framework.test import APIClient + +from cutout.service import cutout_runner +from cutout.service.discovery import FileDescriptor +from cutout.service.models import Job, Task +from cutout.service.policy import ImageCutoutPolicy + +pytestmark = pytest.mark.django_db + +FAKE_PAYLOAD = b"fake fits data" + + +def _patch_async_result_path(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + def _fake_path(self, job, task_params, sequence, execution_mode): + extension = "png" if str(task_params.get("format", "fits")).lower() == "png" else "fits" + return tmp_path / execution_mode / f"job_{job.job_id}_{sequence}.{extension}" + + monkeypatch.setattr(ImageCutoutPolicy, "_build_task_result_path", _fake_path) + + +class _FakeLocator: + def __init__(self, input_file: Path | None): + self._input_file = input_file + + def find_files(self, *, survey_id, stencil, band=None): + if self._input_file is None: + return [] + return [ + FileDescriptor( + tile_id="DES0002+0001", + archive_path="Y6A1/r4907/DES0002+0001/p01/coadd", + file_path=self._input_file, + band=band, + ) + ] + + +class _FakeEngine: + def run_cutout(self, **kwargs): + output_path = Path(kwargs["output_path"]) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_bytes(FAKE_PAYLOAD) + return output_path + + +def _patch_cutout_execution(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, with_files: bool = True) -> None: + input_file = None + if with_files: + input_file = tmp_path / "DES0002+0001_r4907p01_g.fits.fz" + input_file.write_bytes(b"tile data") + monkeypatch.setattr(cutout_runner, "DesCsvFileLocator", lambda: _FakeLocator(input_file)) + monkeypatch.setattr(cutout_runner, "create_cutout_engine", lambda name: _FakeEngine()) + + +def _client(user) -> APIClient: + client = APIClient() + client.force_authenticate(user=user) + return client + + +def test_sync_get_runs_cutout_and_returns_file(user, monkeypatch, tmp_path): + _patch_async_result_path(monkeypatch, tmp_path) + _patch_cutout_execution(monkeypatch, tmp_path) + + response = _client(user).get( + reverse("api:sync_cutout"), + {"id": "des_dr2", "pos": "CIRCLE 10 0 1", "band": "g", "format": "fits"}, + ) + + assert response.status_code == 200 + assert response["Content-Disposition"].startswith("attachment;") + assert response["Content-Type"] == "application/fits" + assert int(response["Content-Length"]) == len(FAKE_PAYLOAD) + + job = Job.objects.get() + assert job.phase == Job.ExecutionPhase.COMPLETED + assert job.start_time is not None + assert job.end_time is not None + + task = job.tasks.get() + assert task.status == Task.Status.COMPLETED + assert task.start_time is not None + assert task.end_time is not None + + job_result = job.results.get() + assert job_result.size == len(FAKE_PAYLOAD) + assert job_result.mime_type == "application/fits" + assert Path(job_result.file_path).exists() + assert "/sync/" in job_result.file_path + + +# transaction=True: on error responses DRF's exception handler calls set_rollback, which would +# poison the test's wrapping atomic block (production uses non_atomic_requests on this route). +@pytest.mark.django_db(transaction=True) +def test_sync_get_no_files_marks_error_and_returns_422(user, monkeypatch, tmp_path): + _patch_async_result_path(monkeypatch, tmp_path) + _patch_cutout_execution(monkeypatch, tmp_path, with_files=False) + + response = _client(user).get( + reverse("api:sync_cutout"), + {"id": "des_dr2", "pos": "CIRCLE 10 0 1", "band": "g", "format": "fits"}, + ) + + assert response.status_code == 422 + assert "No files found" in response.json()["detail"] + + job = Job.objects.get() + assert job.phase == Job.ExecutionPhase.ERROR + assert job.end_time is not None + + task = job.tasks.get() + assert task.status == Task.Status.ERROR + assert "No files found" in task.error_message + assert job.results.count() == 0 + + +@pytest.mark.django_db(transaction=True) +def test_sync_get_rejects_multiple_tasks(user, monkeypatch, tmp_path): + _patch_async_result_path(monkeypatch, tmp_path) + _patch_cutout_execution(monkeypatch, tmp_path) + + response = _client(user).get( + reverse("api:sync_cutout"), + {"id": "des_dr2", "pos": "CIRCLE 10 0 1", "band": ["g", "r"], "format": "fits"}, + ) + + assert response.status_code == 422 + assert "Only one cutout task" in response.json()["detail"] + + job = Job.objects.get() + assert job.phase == Job.ExecutionPhase.ERROR + assert job.tasks.count() == 2 + assert all(task.status == Task.Status.PENDING for task in job.tasks.all()) diff --git a/cutout/service/tests/test_tasks.py b/cutout/service/tests/test_tasks.py index 7721ce9..b724a26 100644 --- a/cutout/service/tests/test_tasks.py +++ b/cutout/service/tests/test_tasks.py @@ -2,7 +2,7 @@ import pytest -from cutout.service.tasks import _validate_input_files +from cutout.service.cutout_runner import _validate_input_files def test_validate_input_files_accepts_none() -> None: diff --git a/cutout/service/uws/policy.py b/cutout/service/uws/policy.py index a16a361..79b4f47 100644 --- a/cutout/service/uws/policy.py +++ b/cutout/service/uws/policy.py @@ -25,22 +25,40 @@ class UWSPolicy(ABC): """ @abstractmethod - def dispatch(self, job: Job): - """Dispatch a job to a backend worker. + def create_tasks_for_job(self, job: Job, params: list[JobParameter], execution_mode: str = "async") -> list: + """Create the Task rows for a job, one per cutout execution unit. - This method is responsible for converting UWS job parameters to the - appropriate arguments for a backend job and invoking it with the - appropriate timeout. + Parameters + ---------- + job + The job the tasks belong to. + params + The job parameters. + execution_mode + "sync" or "async"; selects the results directory for the + generated files. + + Returns + ------- + list + The created Task rows. + """ + + @abstractmethod + def dispatch_async(self, job: Job, message_id: str): + """Dispatch the job's tasks to the backend workers. Parameters ---------- job The job to start. + message_id + Identifier used to track the dispatched work. Returns ------- - dramatiq.Message - The message sent to the backend worker. + celery.result.AsyncResult + The result handle of the dispatched workflow. """ @abstractmethod diff --git a/cutout/service/uws/service.py b/cutout/service/uws/service.py index 77b1adf..f2fcd67 100644 --- a/cutout/service/uws/service.py +++ b/cutout/service/uws/service.py @@ -1,6 +1,5 @@ import logging from pathlib import Path -from typing import List, Optional from celery import uuid from django.db import transaction @@ -19,11 +18,19 @@ def __init__(self) -> None: # TODO: Setup Settings, Logging self._policy = ImageCutoutPolicy() - def create(self, user: User, params: list[JobParameter], run_id: str | None = None) -> Job: - """Create a pending job. - - This does not start execution of the job. That must be done - separately with `start`.""" + def create( + self, + user: User, + params: list[JobParameter], + run_id: str | None = None, + execution_mode: str = "async", + ) -> Job: + """Create a pending job with its Task rows. + + This does not start execution of the job. Async jobs are started + separately with `start_async`; sync jobs execute their task directly + with `perform_cutout`. `execution_mode` ("sync" or "async") selects + the results directory for the generated files.""" self._policy.validate_params(params) job = Job( @@ -35,7 +42,7 @@ def create(self, user: User, params: list[JobParameter], run_id: str | None = No for p in params: job.parameters.create(parameter=p.parameter_id, value=p.value, is_post=p.is_post) - self._policy.create_tasks_for_job(_convert_job(job), params) + self._policy.create_tasks_for_job(_convert_job(job), params, execution_mode=execution_mode) return job @@ -48,25 +55,8 @@ def get_for_user(self, user: User, job_id: int) -> Job: raise PermissionDeniedError(f"Access to job {job_id} denied") return job - def start(self, user: User, job_id: int): - """Start execution of a job.""" - print(f"[JobService.start] called with user={user} job_id={job_id}") - sqljob = self.get_for_user(user, job_id) - if sqljob.phase not in (Job.ExecutionPhase.PENDING, Job.ExecutionPhase.HELD): - raise InvalidPhaseError("Cannot start job in phase {job.phase}") - - print(f"[JobService.start] sqljob.phase={sqljob.phase}") - job = _convert_job(sqljob) - print(f"[JobService.start] calling policy.dispatch with job_id={job.job_id}") - message = self._policy.dispatch(job) - print(f"[JobService.start] policy.dispatch returned: {message}") - - # TODO: Marcar o job como QUEUED - self.mark_queued(job_id, message.id) - return message - def start_async(self, user: User, job_id: int): - """Start async execution using the fake worker pipeline.""" + """Dispatch the job's tasks to the Celery workers.""" logger = logging.getLogger("cutout") logger.info(f"[JobService.start_async] called with user={user} job_id={job_id}") @@ -78,12 +68,16 @@ def start_async(self, user: User, job_id: int): raise InvalidPhaseError(f"Cannot start job in phase {sqljob.phase}") job = _convert_job(sqljob) + message_id = uuid() + logger.info(f"[JobService.start_async] mark_queued with message_id={message_id}") self.mark_queued(job_id, message_id) + logger.info( f"[JobService.start_async] calling policy.dispatch_async with job_id={job.job_id} message_id={message_id}" ) + message = self._policy.dispatch_async(job, message_id=message_id) logger.info(f"[JobService.start_async] policy.dispatch_async returned: {message}") return message @@ -98,12 +92,6 @@ def mark_queued(self, job_id: int, message_id: str) -> None: job.save() - def mark_executing(self, job_id: int) -> None: - job = Job.objects.get(pk=job_id) - job.phase = Job.ExecutionPhase.EXECUTING - job.start_time = timezone.now() - job.save() - def mark_completed(self, job_id: int) -> None: job = Job.objects.get(pk=job_id) job.phase = Job.ExecutionPhase.COMPLETED diff --git a/data/results/sync/.gitkeep b/data/results/sync/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/merge_production_dotenvs_in_dotenv.py b/merge_production_dotenvs_in_dotenv.py deleted file mode 100644 index 35139fb..0000000 --- a/merge_production_dotenvs_in_dotenv.py +++ /dev/null @@ -1,26 +0,0 @@ -import os -from collections.abc import Sequence -from pathlib import Path - -BASE_DIR = Path(__file__).parent.resolve() -PRODUCTION_DOTENVS_DIR = BASE_DIR / ".envs" / ".production" -PRODUCTION_DOTENV_FILES = [ - PRODUCTION_DOTENVS_DIR / ".django", - PRODUCTION_DOTENVS_DIR / ".postgres", -] -DOTENV_FILE = BASE_DIR / ".env" - - -def merge( - output_file: Path, - files_to_merge: Sequence[Path], -) -> None: - merged_content = "" - for merge_file in files_to_merge: - merged_content += merge_file.read_text() - merged_content += os.linesep - output_file.write_text(merged_content) - - -if __name__ == "__main__": - merge(DOTENV_FILE, PRODUCTION_DOTENV_FILES) diff --git a/scripts/debug_band_fits.py b/scripts/debug_band_fits.py deleted file mode 100644 index 1f33e82..0000000 --- a/scripts/debug_band_fits.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python3 -from cutout.lib.des_cutout import DesCutout -from astrocut import fits_cut -from astropy import units as u -from astropy.coordinates import SkyCoord -from pathlib import Path - -OUTDIR = Path('/data/results/debug') -OUTDIR.mkdir(parents=True, exist_ok=True) - -def main(): - ra = 36.30911 - dec = -10.18749 - size = 2.0 - print('Testing per-band fits_cut for coord', ra, dec) - dc = DesCutout() - verts = dc.get_cutout_verts(ra, dec, size) - for b in ['g','r','i']: - print('\n--- band', b, '---') - comp_files = dc.get_fits_files(verts, b) - print('compressed:', comp_files) - files = [] - for c in comp_files: - fits_filename = c.name.split('.fz')[0] - uncompressed = dc.tmp_path.joinpath(fits_filename) - if not uncompressed.exists(): - print(' uncompressing', c, '->', uncompressed) - try: - dc.funpack(c, uncompressed) - except Exception as e: - print(' funpack failed', e) - files.append(str(uncompressed)) - print('uncompressed files:', files) - try: - coord = SkyCoord(ra * u.deg, dec * u.deg, frame='icrs') - res = fits_cut(input_files=files, coordinates=coord, cutout_size=size * u.arcmin, single_outfile=True, cutout_prefix=f'dbg_{b}', output_dir=str(OUTDIR)) - print('fits_cut produced', res) - except Exception as e: - import traceback - print('fits_cut failed for band', b, ':', e) - traceback.print_exc() - -if __name__ == '__main__': - main() diff --git a/scripts/debug_cutout.py b/scripts/debug_cutout.py deleted file mode 100644 index b247b07..0000000 --- a/scripts/debug_cutout.py +++ /dev/null @@ -1,229 +0,0 @@ -#!/usr/bin/env python3 -"""Debug script for cutout generation. - -Creates FITS and PNG artefacts using legacy and DesCutout functions and -inspects the generated files (size, shape, min/max, nan counts). - -Run inside container: - - docker compose exec django python scripts/debug_cutout.py - -""" -from pathlib import Path -import sys -import traceback - -from PIL import Image -import numpy as np -from astropy.io import fits - -from cutout.lib.des_cutout import DesCutout -from cutout.lib.cutout import Cutout -from cutout.service.discovery.des_csv_locator import DesCsvFileLocator - - -OUT_DIR = Path("/data/results/debug") -OUT_DIR.mkdir(parents=True, exist_ok=True) - - -def inspect_fits(path: Path): - print(f"Inspecting FITS: {path}") - if not path.exists(): - print(" MISSING") - return - print(f" size_bytes: {path.stat().st_size}") - try: - with fits.open(path) as hdul: - print(" HDU list:") - hdul.info(output=sys.stdout) - # try to find first HDU with data - found = False - for i, h in enumerate(hdul): - if getattr(h, 'data', None) is not None: - data = h.data - print(f" found data in HDU {i}: dtype={data.dtype}, shape={data.shape}") - arr = np.array(data) - print(f" min: {np.nanmin(arr)}, max: {np.nanmax(arr)}, mean: {np.nanmean(arr)}") - print(f" nans: {np.isnan(arr).sum()} / {arr.size}") - found = True - break - if not found: - print(" No data array found in any HDU") - except Exception as e: - print(" ERROR reading FITS:", e) - traceback.print_exc() - - -def inspect_png(path: Path): - print(f"Inspecting PNG: {path}") - if not path.exists(): - print(" MISSING") - return - print(f" size_bytes: {path.stat().st_size}") - try: - img = Image.open(path) - print(f" mode: {img.mode}, size: {img.size}") - except Exception as e: - print(" ERROR reading PNG:", e) - traceback.print_exc() - - -def run_legacy_fits(ra, dec, size_arcmin, band): - dc = DesCutout() - filename = f"{ra:.5f}_{dec:.5f}_{band}.fits" - out = OUT_DIR.joinpath("legacy_" + filename) - print(f"Running legacy FITS: {out}") - try: - res = dc.single_cutout_fits(ra=ra, dec=dec, size_arcmin=size_arcmin, band=band, path=out) - print(" produced:", res) - inspect_fits(out) - except Exception as e: - print(" legacy FITS failed:", e) - traceback.print_exc() - - -def run_legacy_png(ra, dec, size_arcmin, band): - dc = DesCutout() - filename = f"{ra:.5f}_{dec:.5f}.png" - out = OUT_DIR.joinpath("legacy_" + filename) - print(f"Running legacy PNG: {out}") - try: - res = dc.single_cutout_png(ra=ra, dec=dec, size_arcmin=size_arcmin, band=band, path=out) - print(" produced:", res) - inspect_png(out) - except Exception as e: - print(" legacy PNG failed:", e) - traceback.print_exc() - - -def run_engine(engine_name, stencil, band, fmt, files=None): - print(f"Running engine {engine_name} format={fmt} band={band}") - from cutout.service.cutout_engine import create_cutout_engine - - engine = create_cutout_engine(engine_name) - filename = f"engine_{engine_name}_{stencil['center']['ra']:.5f}_{stencil['center']['dec']:.5f}.{fmt}" - out = OUT_DIR.joinpath(filename) - try: - # If no files provided, try to discover and uncompress via DesCutout - if not files: - try: - dc = DesCutout() - verts = dc.get_cutout_verts(stencil["center"]["ra"], stencil["center"]["dec"], stencil["radius"]) # type: ignore - # If band is multiple letters (e.g. 'gri'), build mapping per band - if isinstance(band, str) and len(band) > 1 and "," not in band and " " not in band: - bands = list(band) - else: - # split by comma or space if present - if isinstance(band, str) and "," in band: - bands = [b.strip() for b in band.split(",") if b.strip()] - elif isinstance(band, str) and " " in band: - bands = [b.strip() for b in band.split() if b.strip()] - else: - bands = [band] - - if len(bands) == 1: - comp_files = dc.get_fits_files(verts, bands[0]) - files = [] - for comp in comp_files: - fits_filename = comp.name.split(".fz")[0] - uncompressed = dc.tmp_path.joinpath(fits_filename) - if not uncompressed.exists(): - print(f" uncompressing {comp} -> {uncompressed}") - try: - dc.funpack(comp, uncompressed) - except Exception as e: - print(" funpack failed:", e) - if uncompressed.exists(): - files.append(str(uncompressed)) - else: - files_map = {} - for b in bands: - comp_files = dc.get_fits_files(verts, b) - files_map[b] = [] - for comp in comp_files: - fits_filename = comp.name.split(".fz")[0] - uncompressed = dc.tmp_path.joinpath(fits_filename) - if not uncompressed.exists(): - print(f" uncompressing {comp} -> {uncompressed}") - try: - dc.funpack(comp, uncompressed) - except Exception as e: - print(" funpack failed:", e) - if uncompressed.exists(): - files_map[b].append(str(uncompressed)) - files = files_map - except Exception: - files = files or [] - - print(' input_files passed to engine:', files) - # Build extra kwargs (e.g., color/rgb_bands) for multi-band PNG tests - extra_kwargs = {} - if engine_name == 'astrocut' and fmt == 'png' and isinstance(band, str) and len(band) > 1 and ',' not in band and ' ' not in band: - extra_kwargs['color'] = True - extra_kwargs['rgb_bands'] = band - - res = engine.run_cutout( - source_id="des_dr2", - stencil=stencil, - input_files=files, - band=band, - output_format=fmt, - output_path=out, - **extra_kwargs, - ) - print(" produced:", res) - if fmt == "fits": - inspect_fits(out) - else: - inspect_png(out) - except Exception as e: - print(f" engine {engine_name} failed:", e) - traceback.print_exc() - - -def main(): - # Coordinates from examples - cutouts = [ - {"ra": 36.30911, "dec": -10.18749, "size": 2.0, "band": "g", "format": "fits"}, - {"ra": 36.30911, "dec": -10.18749, "size": 2.0, "band": "gri", "format": "png"}, - {"ra": 36.15801, "dec": -10.33579, "size": 2.0, "band": "g", "format": "fits"}, - {"ra": 36.15801, "dec": -10.33579, "size": 2.0, "band": "gri", "format": "png"}, - ] - - for c in cutouts: - ra = c["ra"] - dec = c["dec"] - size = c["size"] - band = c["band"] - fmt = c["format"] - - print("\n=== Test case ===") - print(c) - - if fmt == "fits": - # legacy path - run_legacy_fits(ra, dec, size, band) - # engine path (des engine) - stencil = {"type": "circle", "center": {"ra": ra, "dec": dec}, "radius": size} - run_engine("legacy", stencil, band, "fits") - try: - run_engine("astrocut", stencil, band, "fits") - except Exception: - pass - - elif fmt == "png": - # legacy PNG - run_legacy_png(ra, dec, size, band) - # engine mono PNG (if band is single) - stencil = {"type": "circle", "center": {"ra": ra, "dec": dec}, "radius": size} - if len(band) == 1: - run_engine("legacy", stencil, band, "png") - run_engine("astrocut", stencil, band, "png") - else: - # multi-band: let run_engine perform discovery/uncompress for each band - run_engine("legacy", stencil, band, "png", files=None) - run_engine("astrocut", stencil, band, "png", files=None) - - -if __name__ == "__main__": - main() diff --git a/scripts/test_engine_color.py b/scripts/test_engine_color.py deleted file mode 100644 index 7230efa..0000000 --- a/scripts/test_engine_color.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python3 -from cutout.lib.des_cutout import DesCutout -from cutout.service.cutout_engine import create_cutout_engine -from pathlib import Path -from astropy import units as u -from astropy.coordinates import SkyCoord - -OUTDIR = Path('/data/results/debug') -OUTDIR.mkdir(parents=True, exist_ok=True) - -def main(): - ra = 36.30911 - dec = -10.18749 - size = 2.0 - dc = DesCutout() - verts = dc.get_cutout_verts(ra, dec, size) - bands = ['g','r','i'] - files_map = {} - for b in bands: - comp_files = dc.get_fits_files(verts, b) - files_map[b] = [] - for comp in comp_files: - fits_filename = comp.name.split('.fz')[0] - uncompressed = dc.tmp_path.joinpath(fits_filename) - if not uncompressed.exists(): - print('uncompressing', comp, '->', uncompressed) - dc.funpack(comp, uncompressed) - files_map[b].append(str(uncompressed)) - - print('files_map:', files_map) - stencil = {"type": "circle", "center": {"ra": ra, "dec": dec}, "radius": size} - engine = create_cutout_engine('astrocut') - out = OUTDIR.joinpath('engine_astrocut_test_gri.png') - try: - res = engine.run_cutout(source_id='des_dr2', stencil=stencil, input_files=files_map, band='gri', output_format='png', output_path=out, color=True, rgb_bands='gri') - print('engine produced', res) - except Exception as e: - import traceback - print('engine.run_cutout failed:', e) - traceback.print_exc() - -if __name__ == '__main__': - main() diff --git a/test_async_endpoint.md b/test_async_endpoint.md new file mode 100644 index 0000000..b37c17f --- /dev/null +++ b/test_async_endpoint.md @@ -0,0 +1,27 @@ +# 1. Obter o token com usuário/senha +TOKEN=$(curl -s -X POST "http://localhost:80/auth-token/" \ + -d "username=gverde&password=adminadmin" | python3 -c "import sys,json; print(json.load(sys.stdin)['token'])") + +# 2. Submeter o job async (PNG colorido gri, CIRCLE 0.75 2.867 0.033333) +curl -s -X POST "http://localhost:80/api/async" \ + -H "Authorization: Token ${TOKEN}" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "POS=CIRCLE 0.75 2.867 0.033333&format=png&color=true&rgb_bands=gri&id=des_dr2&phase=RUN" +# → retorna JSON com "job_id" e "phase":"QUEUED" + +# 3. Consultar a fase (repita até COMPLETED) — troque 154 pelo job_id retornado +curl -s "http://localhost:80/api/async/154/phase" -H "Authorization: Token ${TOKEN}" + +# 4. Listar os resultados +curl -s "http://localhost:80/api/async/154/results" -H "Authorization: Token ${TOKEN}" + +# 5. Baixar o PNG (result_id vem da listagem acima) +curl -s -o cutout_rgb.png \ + "http://localhost:80/api/async/154/results/job_154_des_dr2_astrocut_rgb_1" \ + -H "Authorization: Token ${TOKEN}" + +Observações: + +- No POST via form-urlencoded os espaços do POS podem ir literais (o curl codifica); só use %20 se passar na query string de um GET. +- O passo 3 devolve texto puro (QUEUED/EXECUTING/COMPLETED/ERROR). No teste que rodei o job completou em menos de 2 segundos. +- Para um exemplo de erro registrado no banco, use uma região fora do footprint (ex.: POS=CIRCLE 10.0 10.0 0.016667&band=r&format=fits) — a fase termina em ERROR e a mensagem fica no error_message da task. diff --git a/test_async_endpoint.py b/test_async_endpoint.py deleted file mode 100644 index af00c7f..0000000 --- a/test_async_endpoint.py +++ /dev/null @@ -1,212 +0,0 @@ -#!/usr/bin/env python3 -"""Smoke test for the sync cutout endpoint using token authentication. - -Examples: - - python test_async_endpoint.py --username gverde --password adminadmin --engine astrocut --pos "CIRCLE 36.30911 -10.18749 0.01" --output /tmp/async_astrocut_result.fits - - python test_async_endpoint.py \ - --username gverde \ - --password adminadmin \ - --engine astrocut \ - --pos "CIRCLE 36.30911 -10.18749 0.01" \ - --output /tmp/async_astrocut_result.fits - - python test_async_endpoint.py \ - --username gverde \ - --password adminadmin \ - --engine legacy \ - --pos "CIRCLE 36.30911 -10.18749 0.01" \ - --output /tmp/async_legacy_result.fits - - python test_async_endpoint.py \ - --username gverde \ - --password adminadmin \ - --id private_survey \ - --engine astrocut -""" - -from __future__ import annotations - -import argparse -import json -import sys -import urllib.error -import urllib.parse -import urllib.request -from pathlib import Path - - -def get_token(base_url: str, username: str, password: str) -> str: - url = f"{base_url.rstrip('/')}/auth-token/" - payload = json.dumps({"username": username, "password": password}).encode("utf-8") - req = urllib.request.Request( - url, - data=payload, - headers={"Content-Type": "application/json"}, - method="POST", - ) - with urllib.request.urlopen(req, timeout=20) as resp: - body = resp.read().decode("utf-8") - data = json.loads(body) - token = data.get("token") - if not token: - raise RuntimeError(f"Token not found in auth response: {body}") - return token - - -import time - - -def call_async_cutout( - *, - base_url: str, - token: str, - survey_id: str, - pos: str, - engine: str, - output_format: str, - band: str, - poll_interval: float = 1.0, - max_polls: int = 30, -) -> tuple[int, str, bytes]: - # 1. Cria job async (urllib segue o 303 automaticamente e entrega o job detail) - url = f"{base_url.rstrip('/')}/api/async" - payload = json.dumps( - { - "id": survey_id, - "pos": pos, - "engine": engine, - "format": output_format, - "band": band, - } - ).encode("utf-8") - - req = urllib.request.Request( - url, - data=payload, - headers={"Authorization": f"Token {token}", "Content-Type": "application/json"}, - method="POST", - ) - with urllib.request.urlopen(req, timeout=30) as resp: - job_url = resp.geturl() - body = resp.read() - - job_data = json.loads(body.decode("utf-8")) - job_id = job_data.get("job_id") - if not job_id: - raise RuntimeError(f"No job_id in response: {body[:1000].decode('utf-8', 'replace')}") - print(f"job created: id={job_id} url={job_url}") - - # 2. Poll job status - for _ in range(max_polls): - job_req = urllib.request.Request( - job_url, - headers={"Authorization": f"Token {token}"}, - method="GET", - ) - with urllib.request.urlopen(job_req, timeout=20) as job_resp: - job_data = json.loads(job_resp.read().decode("utf-8")) - phase = job_data.get("phase") - print(f" phase: {phase}") - if phase == "COMPLETED": - break - elif phase == "ERROR": - raise RuntimeError(f"Job failed: {job_data}") - time.sleep(poll_interval) - else: - raise TimeoutError("Job did not complete in time") - - # 3. Get results - results_url = job_data.get("results_url") - if not results_url: - raise RuntimeError("No results_url in job data") - results_req = urllib.request.Request( - results_url, - headers={"Authorization": f"Token {token}"}, - method="GET", - ) - with urllib.request.urlopen(results_req, timeout=20) as results_resp: - results_data = json.loads(results_resp.read().decode("utf-8")) - results = results_data.get("results", []) - if not results: - raise RuntimeError("No results found for job") - result = results[0] - download_url = result.get("download_url") - if not download_url: - raise RuntimeError("No download_url in result") - print(f"downloading from: {download_url}") - - # 4. Download result - download_req = urllib.request.Request( - download_url, - headers={"Authorization": f"Token {token}"}, - method="GET", - ) - with urllib.request.urlopen(download_req, timeout=60) as dl_resp: - dl_status = dl_resp.getcode() - content_type = dl_resp.headers.get("Content-Type", "") - body = dl_resp.read() - return dl_status, content_type, body - - -def main() -> int: - parser = argparse.ArgumentParser(description="Test /api/async endpoint") - parser.add_argument("--base-url", default="http://localhost:8000", help="API base URL") - parser.add_argument("--username", required=True, help="Django username") - parser.add_argument("--password", required=True, help="Django password") - parser.add_argument("--id", default="des_dr2", dest="survey_id", help="Survey ID") - parser.add_argument("--pos", default="CIRCLE 36.30911 -10.18749 2", help="POS value") - parser.add_argument("--engine", default="astrocut", help="Engine name: astrocut or legacy") - parser.add_argument("--format", default="fits", dest="output_format", help="Output format") - parser.add_argument("--band", default="g", help="Band") - parser.add_argument( - "--output", - default="/tmp/async_result_test.fits", - help="Output file path for successful binary response", - ) - parser.add_argument("--poll-interval", type=float, default=1.0, help="Polling interval in seconds") - parser.add_argument("--max-polls", type=int, default=30, help="Maximum polling attempts") - args = parser.parse_args() - - try: - token = get_token(args.base_url, args.username, args.password) - print("auth: ok") - - status, content_type, body = call_async_cutout( - base_url=args.base_url, - token=token, - survey_id=args.survey_id, - pos=args.pos, - engine=args.engine, - output_format=args.output_format, - band=args.band, - poll_interval=args.poll_interval, - max_polls=args.max_polls, - ) - print(f"status: {status}") - print(f"content-type: {content_type}") - - if status == 200 and "application/json" not in content_type: - output = Path(args.output) - output.parent.mkdir(parents=True, exist_ok=True) - output.write_bytes(body) - print(f"result saved to: {output}") - else: - text = body.decode("utf-8", errors="replace") - print("response body:") - print(text[:2000]) - - return 0 - except urllib.error.HTTPError as e: - body = e.read().decode("utf-8", errors="replace") - print(f"http error: {e.code}") - print(body[:2000]) - return 1 - except Exception as e: # noqa: BLE001 - print(f"error: {e}") - return 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/test_celery_ping.py b/test_celery_ping.py deleted file mode 100644 index e9458c4..0000000 --- a/test_celery_ping.py +++ /dev/null @@ -1,6 +0,0 @@ -from cutout.service.tasks_test_celery import ping - -if __name__ == "__main__": - result = ping.delay(42) - print("Task submitted, waiting result...") - print(result.get(timeout=10)) diff --git a/test_plot_cutout_fits.py b/test_plot_cutout_fits.py deleted file mode 100644 index 30d375b..0000000 --- a/test_plot_cutout_fits.py +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env python3 -"""Inspect and plot a FITS cutout file. - -Examples: - python test_plot_cutout_fits.py --file /tmp/sync_result_test.fits - - python test_plot_cutout_fits.py \ - --file /tmp/sync_result_test.fits \ - --save /tmp/sync_result_test.png \ - --no-show - - python test_plot_cutout_fits.py --file /tmp/sync_result_test.fits --wcs -""" - -from __future__ import annotations - -import argparse -from pathlib import Path - -import matplotlib.pyplot as plt -import numpy as np -from astropy.io import fits -from astropy.wcs import WCS - - -def _find_image_hdu( - hdul: fits.HDUList, preferred_ext: int | None -) -> tuple[int, fits.hdu.base.ExtensionHDU | fits.PrimaryHDU]: - if preferred_ext is not None: - hdu = hdul[preferred_ext] - if hdu.data is None: - raise ValueError(f"HDU {preferred_ext} has no data") - return preferred_ext, hdu - - for idx, hdu in enumerate(hdul): - if hdu.data is not None and getattr(hdu.data, "ndim", 0) >= 2: - return idx, hdu - - raise ValueError("No image HDU found in FITS file") - - -def _print_stats(data: np.ndarray, hdu_index: int) -> None: - finite_mask = np.isfinite(data) - finite_count = int(np.count_nonzero(finite_mask)) - total_count = int(data.size) - - print(f"HDU index: {hdu_index}") - print(f"Shape: {data.shape}") - print(f"dtype: {data.dtype}") - print(f"Finite pixels: {finite_count}/{total_count}") - - if finite_count == 0: - print("No finite pixels available for stats") - return - - finite = data[finite_mask] - print(f"Min: {float(np.min(finite)):.6g}") - print(f"Max: {float(np.max(finite)):.6g}") - print(f"Mean: {float(np.mean(finite)):.6g}") - print(f"Median: {float(np.median(finite)):.6g}") - - -def _plot_image( - data: np.ndarray, header: fits.Header, use_wcs: bool, title: str, save_path: Path | None, show: bool -) -> None: - vmin, vmax = np.nanpercentile(data, [1, 99]) - - if use_wcs: - wcs = WCS(header) - fig = plt.figure(figsize=(8, 7)) - ax = fig.add_subplot(111, projection=wcs) - ax.set_xlabel("RA") - ax.set_ylabel("Dec") - else: - fig, ax = plt.subplots(figsize=(8, 7)) - ax.set_xlabel("X pixel") - ax.set_ylabel("Y pixel") - - image = ax.imshow(data, origin="lower", cmap="gray", vmin=vmin, vmax=vmax) - ax.set_title(title) - fig.colorbar(image, ax=ax, fraction=0.046, pad=0.04, label="Flux") - fig.tight_layout() - - if save_path is not None: - save_path.parent.mkdir(parents=True, exist_ok=True) - fig.savefig(save_path, dpi=150) - print(f"Plot saved to: {save_path}") - - if show: - plt.show() - else: - plt.close(fig) - - -def main() -> int: - parser = argparse.ArgumentParser(description="Inspect and plot a FITS cutout") - parser.add_argument("--file", required=True, help="Path to FITS file") - parser.add_argument("--ext", type=int, default=None, help="HDU extension index (default: first image HDU)") - parser.add_argument("--wcs", action="store_true", help="Use WCS projection when available") - parser.add_argument("--save", default=None, help="Save plot image path (PNG, JPG, etc.)") - parser.add_argument("--no-show", action="store_true", help="Do not open interactive plot window") - args = parser.parse_args() - - fits_path = Path(args.file) - if not fits_path.exists(): - raise FileNotFoundError(f"FITS file does not exist: {fits_path}") - - with fits.open(fits_path) as hdul: - hdu_index, hdu = _find_image_hdu(hdul, args.ext) - data = np.asarray(hdu.data, dtype=float) - if data.ndim > 2: - data = np.squeeze(data) - if data.ndim != 2: - raise ValueError(f"Expected a 2D image after squeeze, got shape={data.shape}") - - _print_stats(data, hdu_index) - _plot_image( - data=data, - header=hdu.header, - use_wcs=args.wcs, - title=f"Cutout: {fits_path.name}", - save_path=Path(args.save) if args.save else None, - show=not args.no_show, - ) - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/test_worker_dispatch.py b/test_worker_dispatch.py deleted file mode 100644 index 27e3d13..0000000 --- a/test_worker_dispatch.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python3 -""" -Dispatch a cutout job directly to Celery workers, bypassing the HTTP API. -Useful for testing worker execution in isolation. - -Usage: - docker compose exec django python test_worker_dispatch.py --job-id 103 - docker compose exec django python test_worker_dispatch.py --job-id 103 --reset -""" -from __future__ import annotations - -import argparse -import os -import sys -import time - -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local") - -import django # noqa: E402 - -django.setup() - -from cutout.service.models import Job as SQLJob # noqa: E402 -from cutout.service.uws.service import JobService # noqa: E402 - - -def main() -> int: - parser = argparse.ArgumentParser(description="Dispatch a cutout job directly to Celery workers") - parser.add_argument("--job-id", required=True, type=int, help="ID of the job to dispatch") - parser.add_argument( - "--reset", - action="store_true", - help="Reset job to PENDING before dispatching (required if job already ran)", - ) - parser.add_argument("--poll-interval", type=float, default=1.0, help="Poll interval in seconds") - parser.add_argument("--max-polls", type=int, default=60, help="Maximum poll attempts before timeout") - args = parser.parse_args() - - try: - sqljob = SQLJob.objects.get(pk=args.job_id) - except SQLJob.DoesNotExist: - print(f"error: job {args.job_id} not found") - return 1 - - print(f"job found: id={sqljob.id} phase={sqljob.phase} owner={sqljob.owner}") - params = list(sqljob.parameters.order_by("id").values_list("parameter", "value")) - print(f" params: {params}") - - terminal_phases = ( - SQLJob.ExecutionPhase.COMPLETED, - SQLJob.ExecutionPhase.ERROR, - SQLJob.ExecutionPhase.ABORTED, - SQLJob.ExecutionPhase.EXECUTING, - SQLJob.ExecutionPhase.QUEUED, - ) - if sqljob.phase in terminal_phases: - if not args.reset: - print(f" job is already in phase={sqljob.phase}. Use --reset to re-dispatch.") - return 1 - sqljob.phase = SQLJob.ExecutionPhase.PENDING - sqljob.message_id = None - sqljob.start_time = None - sqljob.end_time = None - sqljob.results.all().delete() - sqljob.save() - print(" reset to PENDING") - - # Dispatch via the same service layer the API uses - job_service = JobService() - job_service.start_async(sqljob.owner, sqljob.id) - print(f" dispatched — polling every {args.poll_interval}s (max {args.max_polls} attempts)") - - # Poll until terminal state - for i in range(1, args.max_polls + 1): - time.sleep(args.poll_interval) - sqljob.refresh_from_db() - phase = sqljob.phase - print(f" [{i:2d}] phase={phase}") - if phase == SQLJob.ExecutionPhase.COMPLETED: - break - elif phase in (SQLJob.ExecutionPhase.ERROR, SQLJob.ExecutionPhase.ABORTED): - print(" job ended with failure") - return 1 - else: - print(f" timeout: job did not complete after {args.max_polls} polls") - return 1 - - # Print results - results = list(sqljob.results.order_by("sequence")) - print(f"\nresults ({len(results)}):") - for r in results: - print(f" [{r.sequence}] result_id : {r.result_id}") - print(f" file : {r.file_path}") - print(f" mime : {r.mime_type}") - print(f" size : {r.size} bytes") - print(f" url : {r.url}") - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tests/test_e2e_png.py b/tests/test_e2e_png.py deleted file mode 100644 index 4ddf919..0000000 --- a/tests/test_e2e_png.py +++ /dev/null @@ -1,50 +0,0 @@ -import pytest -from pathlib import Path - - -def _has_tiles() -> bool: - return Path("/data/tiles").exists() - - -def test_astrocut_end_to_end_png(): - if not _has_tiles(): - pytest.skip("No tiles available in /data/tiles for E2E test") - - from cutout.lib.des_cutout import DesCutout - from cutout.service.tasks import image_cutout - from PIL import Image - - ra = 36.30911 - dec = -10.18749 - size = 2.0 - dc = DesCutout() - verts = dc.get_cutout_verts(ra, dec, size) - - bands = ["g", "r", "i"] - files_map = {} - for b in bands: - comp_files = dc.get_fits_files(verts, b) - if not comp_files: - pytest.skip(f"No files found for band {b}") - files_map[b] = [str(p.file_path) if hasattr(p, 'file_path') else str(p) for p in comp_files] - - out = "/data/results/test_e2e_astrocut_gri.png" - - res = image_cutout.run( - job_id="e2e-test", - source_id="des_dr2", - stencil={"type": "circle", "center": {"ra": ra, "dec": dec}, "radius": size}, - engine="astrocut", - band="gri", - format="png", - path=out, - files=files_map, - color=True, - rgb_bands="gri", - persist=False, - ) - - assert Path(res).exists(), f"Result file missing: {res}" - im = Image.open(res) - assert im.mode in ("RGB", "RGBA") - assert im.size[0] > 0 and im.size[1] > 0 diff --git a/tests/test_merge_production_dotenvs_in_dotenv.py b/tests/test_merge_production_dotenvs_in_dotenv.py deleted file mode 100644 index c0e68f6..0000000 --- a/tests/test_merge_production_dotenvs_in_dotenv.py +++ /dev/null @@ -1,34 +0,0 @@ -from pathlib import Path - -import pytest - -from merge_production_dotenvs_in_dotenv import merge - - -@pytest.mark.parametrize( - ("input_contents", "expected_output"), - [ - ([], ""), - ([""], "\n"), - (["JANE=doe"], "JANE=doe\n"), - (["SEP=true", "AR=ator"], "SEP=true\nAR=ator\n"), - (["A=0", "B=1", "C=2"], "A=0\nB=1\nC=2\n"), - (["X=x\n", "Y=y", "Z=z\n"], "X=x\n\nY=y\nZ=z\n\n"), - ], -) -def test_merge( - tmp_path: Path, - input_contents: list[str], - expected_output: str, -): - output_file = tmp_path / ".env" - - files_to_merge = [] - for num, input_content in enumerate(input_contents, start=1): - merge_file = tmp_path / f".service{num}" - merge_file.write_text(input_content) - files_to_merge.append(merge_file) - - merge(output_file, files_to_merge) - - assert output_file.read_text() == expected_output diff --git a/tests/test_png_output.py b/tests/test_png_output.py index e5ffa1a..56e4070 100644 --- a/tests/test_png_output.py +++ b/tests/test_png_output.py @@ -1,25 +1,19 @@ -import numpy as np from pathlib import Path +import numpy as np from astropy.io import fits from cutout.service.cutout_engine.astrocut_engine import AstrocutEngine -def make_fits(path: Path, shape=(64, 64), value=1): - data = np.full(shape, value, dtype=float) - fits.writeto(path, data, overwrite=True) - return str(path) +def _hdulist(value: float, shape=(64, 64)) -> fits.HDUList: + data = np.full(shape, value, dtype=np.float32) + return fits.HDUList([fits.PrimaryHDU(data=data)]) def test_mono_png(tmp_path, monkeypatch): - in_fits = tmp_path / "in.fits" - make_fits(in_fits, value=42) - - def mock_fits_cut(**kwargs): - out = tmp_path / f"{kwargs.get('cutout_prefix','cut')}.fits" - make_fits(out, value=42) - return str(out) + def mock_fits_cut(input_files, coordinates, cutout_size, single_outfile=True, memory_only=True): + return [_hdulist(42)] monkeypatch.setattr("cutout.service.cutout_engine.astrocut_engine.fits_cut", mock_fits_cut) @@ -29,7 +23,7 @@ def mock_fits_cut(**kwargs): res = engine.run_cutout( source_id="des_dr2", stencil=stencil, - input_files=[str(in_fits)], + input_files=[str(tmp_path / "in.fits")], band="g", output_format="png", output_path=out_png, @@ -41,27 +35,17 @@ def mock_fits_cut(**kwargs): def test_rgb_png(tmp_path, monkeypatch): - def mock_fits_cut(input_files, coordinates, cutout_size, single_outfile, cutout_prefix, output_dir): - # produce file with value dependent on prefix suffix - val = 10 - if cutout_prefix.endswith("_g"): - val = 50 - elif cutout_prefix.endswith("_r"): - val = 100 - elif cutout_prefix.endswith("_i"): - val = 150 - out = Path(output_dir) / f"{cutout_prefix}.fits" - make_fits(out, value=val) - return str(out) + band_values = {"g1": 50.0, "r1": 100.0, "i1": 150.0} + + def mock_fits_cut(input_files, coordinates, cutout_size, single_outfile=True, memory_only=True): + stem = Path(str(input_files[0])).stem + return [_hdulist(band_values.get(stem, 10.0))] monkeypatch.setattr("cutout.service.cutout_engine.astrocut_engine.fits_cut", mock_fits_cut) engine = AstrocutEngine() stencil = {"type": "circle", "center": {"ra": 36.0, "dec": -10.0}, "radius": 1.0} - # prepare dummy original files (not used by mock but kept for clarity) in_map = {"g": [str(tmp_path / "g1.fits")], "r": [str(tmp_path / "r1.fits")], "i": [str(tmp_path / "i1.fits")]} - for v in in_map.values(): - make_fits(Path(v[0]), value=1) out_png = tmp_path / "out_rgb.png" res = engine.run_cutout(