From b4a38f000179df4512d91ceb3fbc5e10bdaae3ea Mon Sep 17 00:00:00 2001 From: Sebokolodi Date: Thu, 11 Jun 2026 10:09:23 -0400 Subject: [PATCH] fix(ingest): trust CADC over _report.txt when deciding tile status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit do_ingest() was gating the 'Ingested' status on _report.txt showing 24/24 successes AND CADC having all expected products. When a transient minoc HEAD/PUT timeout dropped one file in the current run but the file was already in CADC from a prior partial run, the tile got marked IngestFailed even though every expected product was visible at CADC. Seen on tiles 11720, 11532, 11846 — each reported '_report.txt reports that ingestion failed' and 'CADC contains all products' in the same run. Flip the priority: CADC is the source of truth. If all 17 expected products are in CADC, the tile is Ingested regardless of _report.txt's per-run accounting. Only mark IngestFailed when CADC is actually missing products. This self-heals false-negative status updates without changing real-failure detection (e.g. tile 11519 has a genuinely missing RMSF_real_p3d_v1 product from a separate caom2utils blueprint bug and will still correctly mark IngestFailed). --- possum_pipeline_control/ingest3Dpipeline.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/possum_pipeline_control/ingest3Dpipeline.py b/possum_pipeline_control/ingest3Dpipeline.py index 840ea89..cf065ff 100644 --- a/possum_pipeline_control/ingest3Dpipeline.py +++ b/possum_pipeline_control/ingest3Dpipeline.py @@ -316,14 +316,23 @@ def do_ingest( # if this fails with SSLError, try to run cadc-get-cert -u on CANFAR CADCsuccess, date = check_CADC(tilenumber, band, CADC_cert_file) + # CADC is the source of truth for ingest success: if all expected products are + # in CADC, the tile is ingested regardless of what `_report.txt` says about this + # particular run. _report.txt only records per-file outcomes of the current attempt, + # so transient minoc timeouts on files that were already uploaded by a prior partial + # run produce false-negative IngestFailed status (seen on tiles 11720, 11532, 11846). status = "IngestFailed" - if success: - if CADCsuccess: - status = "Ingested" - else: - print("_report.txt reports succesful ingest, but CADC ingest failed") + if CADCsuccess: + status = "Ingested" + if not success: + print( + "_report.txt reports per-file errors but CADC contains all expected " + "products; treating as Ingested." + ) + elif success: + print("_report.txt reports successful ingest but CADC is missing products.") else: - print("_report.txt reports that ingestion failed") + print("_report.txt reports that ingestion failed and CADC is missing products.") # Record the status in the POSSUM Validation database conn = db.get_database_connection(test=False)