From f7eb0ba9b2412babc9bece3b5c90fb13a0c7f2b1 Mon Sep 17 00:00:00 2001 From: Sebokolodi Date: Fri, 17 Jul 2026 10:07:51 -0400 Subject: [PATCH 1/2] diagnostics: log inputs, computed max_order, and warn when write_new_parameters produces no output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit write_new_parameters currently derives its output count from a max_order heuristic on new_coeffs (any-non-zero-non-nan planes). When new_coeffs is entirely 0.0 or NaN, max_order = -1, the write loop becomes range(0), and NO files are written — with no exception raised and no log line emitted. Downstream consumers that assume newcoeff*.fits exist can be caught out badly by this: POSSUM_Polarimetry_Pipeline's fracpol3d_Irescale step, for example, will destroy its input coeff*.fits files in an ill-fated delete-then-rename block and only crash much later with a FileNotFoundError, far from the actual root cause. That has now been fixed defensively on the pipeline side, but the underlying silent no-op deserves visibility here too. Changes (all logging, no behaviour change on the happy path): * Add a module-level logger and a small _summarize_coeffs helper for compact array descriptors (shape, dtype, nan/zero/finite-nonzero counts). * rescale_I_model_3D logs input coeffs and output new_coeffs summaries at INFO. * write_new_parameters logs input summary, computed max_order, planned file count, and each file it writes at INFO; total file count at end. * When max_order < 0, write_new_parameters now emits both logger.warning and warnings.warn(UserWarning) explaining the condition, and returns explicitly. Preserves the historical 'no files written' behaviour but makes the reason loudly traceable. This surfaces the root cause of the tile 6614 (RMtools 1.4.11) incident observed in the CIRADA POSSUM pipeline, without changing what the function actually writes on any input where it did write something before. --- RMtools_3D/rescale_I_model_3D.py | 64 +++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 2 deletions(-) diff --git a/RMtools_3D/rescale_I_model_3D.py b/RMtools_3D/rescale_I_model_3D.py index a2eb9a7..aa9543c 100644 --- a/RMtools_3D/rescale_I_model_3D.py +++ b/RMtools_3D/rescale_I_model_3D.py @@ -45,8 +45,10 @@ # (make uniform across all pixels) import argparse +import logging import multiprocessing as mp import os +import warnings from functools import partial import astropy.io.fits as pf @@ -54,6 +56,22 @@ from RMutils.util_misc import FitResult, renormalize_StokesI_model +logger = logging.getLogger(__name__) + + +def _summarize_coeffs(coeffs): + """Return a one-line summary string of a coefficient array for logging.""" + if not hasattr(coeffs, 'shape'): + return f"type={type(coeffs).__name__} (no shape)" + total = int(np.prod(coeffs.shape)) + nans = int(np.isnan(coeffs).sum()) + zeros = int((coeffs == 0.0).sum()) + finite_nonzero = int(np.count_nonzero(np.isfinite(coeffs) & (coeffs != 0.0))) + return ( + f"shape={coeffs.shape} dtype={coeffs.dtype} total={total} " + f"nans={nans} zeros={zeros} finite_nonzero={finite_nonzero}" + ) + def command_line(): """Handle invocation from the command line, parsing inputs and running @@ -267,6 +285,11 @@ def rescale_I_model_3D( new_errors (3D array): maps of new parameter uncertainties (highest to lowest) """ + logger.info( + "rescale_I_model_3D: begin. num_cores=%d fit_function=%s input coeffs %s", + num_cores, fit_function, _summarize_coeffs(coeffs), + ) + # Initialize output arrays, keep dtype consistent new_coeffs = np.zeros_like(coeffs, dtype=coeffs.dtype) new_errors = np.zeros_like(coeffs, dtype=coeffs.dtype) @@ -295,6 +318,11 @@ def rescale_I_model_3D( new_coeffs[:, i // rs, i % rs] = p new_errors[:, i // rs, i % rs] = perror + logger.info( + "rescale_I_model_3D: end. output new_coeffs %s", + _summarize_coeffs(new_coeffs), + ) + return new_freq_map, new_coeffs, new_errors @@ -326,19 +354,51 @@ def write_new_parameters( np.sum(np.any((new_coeffs != 0.0) & (~np.isnan(new_coeffs)), axis=(1, 2))) - 1 ) + logger.info( + "write_new_parameters: input new_coeffs %s -> computed max_order=%d, " + "will write %d newcoeff*.fits + %d newcoeff*err.fits files with basename %r", + _summarize_coeffs(new_coeffs), int(max_order), + max(max_order + 1, 0), max(max_order + 1, 0), out_basename, + ) + + if max_order < 0: + # Every plane of new_coeffs is zero or NaN. The write loop below would + # be range(0) — nothing written, no exception raised. Historically this + # was silent; downstream callers that expect newcoeff*.fits to exist + # (e.g. POSSUM_Polarimetry_Pipeline's fracpol3d_Irescale) can then + # destroy the input coeff*.fits and hit a FileNotFoundError much later, + # far from the actual root cause. Make the condition loudly visible. + msg = ( + "RMtools_3D.rescale_I_model_3D.write_new_parameters: no output " + "written because computed max_order=-1 (every plane of new_coeffs " + "is 0.0 or NaN). This usually means rescale_I_model_3D received " + "unusable inputs or the per-pixel rescale failed everywhere. " + f"out_basename={out_basename!r}" + ) + logger.warning(msg) + warnings.warn(msg, UserWarning, stacklevel=2) + return + + written = [] for i in range(max_order + 1): + coeff_path = out_basename + f"newcoeff{i}.fits" + err_path = out_basename + f"newcoeff{i}err.fits" pf.writeto( - out_basename + f"newcoeff{i}.fits", + coeff_path, new_coeffs[5 - i], header=out_header, overwrite=overwrite, ) pf.writeto( - out_basename + f"newcoeff{i}err.fits", + err_path, new_errors[5 - i], header=out_header, overwrite=overwrite, ) + written.extend((coeff_path, err_path)) + logger.info("write_new_parameters: wrote %s and %s", coeff_path, err_path) + + logger.info("write_new_parameters: end. wrote %d files.", len(written)) # -----------------------------------------------------------------------------# From 76e0534f5d66a60e37bd054158ff7154aa108479 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:08:35 +0000 Subject: [PATCH 2/2] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- RMtools_3D/rescale_I_model_3D.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/RMtools_3D/rescale_I_model_3D.py b/RMtools_3D/rescale_I_model_3D.py index aa9543c..2856cf8 100644 --- a/RMtools_3D/rescale_I_model_3D.py +++ b/RMtools_3D/rescale_I_model_3D.py @@ -61,7 +61,7 @@ def _summarize_coeffs(coeffs): """Return a one-line summary string of a coefficient array for logging.""" - if not hasattr(coeffs, 'shape'): + if not hasattr(coeffs, "shape"): return f"type={type(coeffs).__name__} (no shape)" total = int(np.prod(coeffs.shape)) nans = int(np.isnan(coeffs).sum()) @@ -287,7 +287,9 @@ def rescale_I_model_3D( logger.info( "rescale_I_model_3D: begin. num_cores=%d fit_function=%s input coeffs %s", - num_cores, fit_function, _summarize_coeffs(coeffs), + num_cores, + fit_function, + _summarize_coeffs(coeffs), ) # Initialize output arrays, keep dtype consistent @@ -357,8 +359,11 @@ def write_new_parameters( logger.info( "write_new_parameters: input new_coeffs %s -> computed max_order=%d, " "will write %d newcoeff*.fits + %d newcoeff*err.fits files with basename %r", - _summarize_coeffs(new_coeffs), int(max_order), - max(max_order + 1, 0), max(max_order + 1, 0), out_basename, + _summarize_coeffs(new_coeffs), + int(max_order), + max(max_order + 1, 0), + max(max_order + 1, 0), + out_basename, ) if max_order < 0: