From 1b6af65415398be40ff6bd5792a287d2a0bcd5d9 Mon Sep 17 00:00:00 2001 From: Luc Busquin <133058544+Cybis320@users.noreply.github.com> Date: Thu, 2 Jul 2026 14:25:11 -0700 Subject: [PATCH 01/15] Improve star extraction robustness on bright and noisy frames - On candidate overflow, keep a spatially stratified sample of the most prominent candidates (white top-hat ranking, equal quota per image tile) instead of dropping the whole frame. Frames under the limit are unchanged. - Log the max_global_intensity gate at both call sites and raise its default from 150 to 230 (frames at median ~190-225 still hold dozens of real stars). - Skip too-bright chunks in extractStarsImgHandle instead of aborting the whole image handle and discarding stars from all other chunks. - SkyFit2 star detection tab: add Max Global Intensity slider, split the max stars budget into a session-only SkyFit value and a Config value written by Save Config (400 recommended), and add a Reset to Defaults button. - Update the star detection help topic accordingly. --- .config | 5 +- RMS/ConfigReader.py | 2 +- RMS/ExtractStars.py | 85 +++++++++++++++++++++----- RMS/Routines/CustomPyqtgraphClasses.py | 49 ++++++++++++++- RMS/Routines/SkyFitHelp.py | 24 ++++++-- Utils/SkyFit2.py | 39 +++++++++--- 6 files changed, 170 insertions(+), 34 deletions(-) diff --git a/.config b/.config index c8da82d1f..e7ab00b87 100644 --- a/.config +++ b/.config @@ -688,8 +688,9 @@ num_cores: -1 ; Extract stars ; ------------- -; Maximum mean intensity of an image before it is discarded as too bright -max_global_intensity: 140 +; Maximum median intensity of an image (8-bit scale, measured after flat correction) +; before it is discarded as too bright to contain stars +max_global_intensity: 230 ; Apply a mask on the detections by removing all that are too close to the ; given image border (in pixels) diff --git a/RMS/ConfigReader.py b/RMS/ConfigReader.py index 380ce1475..bebe693e4 100644 --- a/RMS/ConfigReader.py +++ b/RMS/ConfigReader.py @@ -551,7 +551,7 @@ def __init__(self): ##### StarExtraction # Extraction parameters - self.max_global_intensity = 150 # maximum mean intensity of an image before it is discarded as too bright + self.max_global_intensity = 230 # maximum median intensity of an image (8-bit scale) before it is discarded as too bright to contain stars self.border = 10 # apply a mask on the detections by removing all that are too close to the given image border (in pixels) self.neighborhood_size = 10 # size of the neighbourhood for the maximum search (in pixels) self.intensity_threshold = 5 # a threshold for cutting the detections which are too faint (0-255) diff --git a/RMS/ExtractStars.py b/RMS/ExtractStars.py index 86f3c0297..140802d03 100644 --- a/RMS/ExtractStars.py +++ b/RMS/ExtractStars.py @@ -66,8 +66,9 @@ def extractStars(img, img_median=None, mask=None, gamma=1.0, max_star_candidates img_median: [float] Median value of the image. If not given, it will be computed. mask: [ndarray] Mask image. None by default. gamma: [float] Gamma correction factor for the image. - max_star_candidates: [int] Maximum number of star candidates to process. If the number of - candidates is larger than this number, the image will be skipped. + max_star_candidates: [int] Maximum number of star candidates to process. If more candidates + are found (e.g. the image is flooded with noise or glare around the Moon), only the + max_star_candidates most prominent ones are kept. border: [int] apply a mask on the detections by removing all that are too close to the given image border (in pixels) neighborhood_size: [int] size of the neighbourhood for the maximum search (in pixels) @@ -129,13 +130,55 @@ def extractStars(img, img_median=None, mask=None, gamma=1.0, max_star_candidates if extra_info is not None: extra_info['num_candidates'] = num_objects - # Skip the image if there are too many maxima to process + label_index = range(1, num_objects + 1) + + # If there are too many candidates (e.g. the image is flooded with sensor noise or glare around + # the Moon), subsample them instead of skipping the image. The PSF fit below rejects the + # non-star candidates. This bounds the PSF fitting cost to max_star_candidates. if num_objects > max_star_candidates: - log.warning('Too many candidate stars to process! {:d}/{:d}'.format(num_objects, max_star_candidates)) - return False + + log.warning('Too many candidate stars ({:d}/{:d}), keeping the {:d} most prominent ones'.format( + num_objects, max_star_candidates, max_star_candidates)) + + # Rank the candidates by their peak height above the local background (white top-hat, + # i.e. the image minus its morphological opening). Unlike the raw pixel value, this is + # insensitive to brightness gradients, so stars rank above noise on glare or clouds. + opening = filters.maximum_filter(img_min, neighborhood_size) + tophat = img_convolved - opening + prominence = np.array(ndimage.maximum(tophat, labeled, label_index)) + + # Stratify the selection on a spatial grid so that the sample stays spatially uniform. + # A purely global prominence ranking would bias the sample toward the image centre + # (vignetting dims the stars near the edges) and let noisy regions (e.g. around the + # Moon) crowd out real stars elsewhere. Each tile gets an equal share of the budget, + # filled by local prominence; any unused share goes to the most prominent leftovers. + # Size the grid so each tile's share stays meaningful (at least ~4 candidates). + n_tiles = int(np.clip(np.sqrt(max_star_candidates/4.0), 2, 8)) + positions = np.array(ndimage.maximum_position(tophat, labeled, label_index)) + tile_y = np.clip(positions[:, 0]*n_tiles//img.shape[0], 0, n_tiles - 1) + tile_x = np.clip(positions[:, 1]*n_tiles//img.shape[1], 0, n_tiles - 1) + tile_id = (tile_y*n_tiles + tile_x).astype(int) + + tile_quota = max(1, max_star_candidates//(n_tiles*n_tiles)) + tile_counts = np.zeros(n_tiles*n_tiles, dtype=int) + + selected = [] + leftover = [] + for i in np.argsort(prominence)[::-1]: + if (tile_counts[tile_id[i]] < tile_quota) and (len(selected) < max_star_candidates): + tile_counts[tile_id[i]] += 1 + selected.append(i) + else: + leftover.append(i) + + # Fill any remaining budget with the most prominent unselected candidates + selected.extend(leftover[:max_star_candidates - len(selected)]) + + # Convert positional indices to labels + label_index = (np.array(selected) + 1).tolist() # Find centres of mass of each labeled objects - xy = np.array(ndimage.center_of_mass(img_convolved, labeled, range(1, num_objects + 1))) + xy = np.array(ndimage.center_of_mass(img_convolved, labeled, label_index)) # Remove all detection on the border #xy = xy[np.where((xy[:, 1] > border) & (xy[:,1] < ff.ncols - border) & (xy[:,0] > border) & (xy[:,0] < ff.nrows - border))] @@ -259,7 +302,7 @@ def extractStarsFF( flat_struct=None, dark=None, mask=None, config=None, border=10, - max_global_intensity=150, + max_global_intensity=230, neighborhood_size=10, intensity_threshold=18, segment_radius=4, roundness_threshold=0.5, max_feature_ratio=0.8, extra_info=None @@ -331,7 +374,10 @@ def extractStarsFF( img_median = np.median(ff.avepixel) # Check if the image is too bright and skip the image (scale the cutoff to the image bit depth) - if img_median > max_global_intensity*(2**(config.bit_depth - 8)): + max_global_intensity_scaled = max_global_intensity*(2**(config.bit_depth - 8)) + if img_median > max_global_intensity_scaled: + log.info('{:s} is too bright, skipping star extraction (median {:.0f} > {:d})'.format( + ff_name, img_median, max_global_intensity_scaled)) return error_return # Get the image data from the average pixel image @@ -365,8 +411,8 @@ def extractStarsImgHandle(img_handle, flat_struct=None, dark=None, mask=None, config=None, border=10, - max_global_intensity=150, - neighborhood_size=10, intensity_threshold=18, + max_global_intensity=230, + neighborhood_size=10, intensity_threshold=18, segment_radius=4, roundness_threshold=0.5, max_feature_ratio=0.8 ): @@ -447,9 +493,15 @@ def extractStarsImgHandle(img_handle, # Calculate image mean and stddev img_median = np.median(avepixel) - # Check if the image is too bright and skip the image (scale the cutoff to the image bit depth) - if img_median > max_global_intensity*(2**(config.bit_depth - 8)): - return error_return + # Check if the chunk is too bright and skip it (scale the cutoff to the image bit depth). + # Only this chunk is skipped - other chunks in the image handle are still processed. + max_global_intensity_scaled = max_global_intensity*(2**(config.bit_depth - 8)) + if img_median > max_global_intensity_scaled: + log.info('Chunk {:d} is too bright, skipping star extraction (median {:.0f} > {:d})'.format( + chunk_no, img_median, max_global_intensity_scaled)) + + img_handle.nextChunk() + continue # Get the image data from the average pixel image img = avepixel.astype(np.float32) @@ -464,10 +516,11 @@ def extractStarsImgHandle(img_handle, max_feature_ratio=max_feature_ratio, bit_depth=config.bit_depth ) - # If the star extraction failed, return an empty list + # If the star extraction failed, skip this chunk if status is False: - return error_return - + img_handle.nextChunk() + continue + # Unpack the star data x_arr, y_arr, amplitude, intensity, fwhm, background, snr, saturated_count = status diff --git a/RMS/Routines/CustomPyqtgraphClasses.py b/RMS/Routines/CustomPyqtgraphClasses.py index 8c3f91ac9..c613d44dc 100644 --- a/RMS/Routines/CustomPyqtgraphClasses.py +++ b/RMS/Routines/CustomPyqtgraphClasses.py @@ -3268,6 +3268,8 @@ class StarDetectionWidget(QtWidgets.QWidget, ScaledSizeHelper): sigIntensityThresholdChanged = QtCore.pyqtSignal(int) sigNeighborhoodSizeChanged = QtCore.pyqtSignal(int) sigMaxStarsChanged = QtCore.pyqtSignal(int) + sigConfigMaxStarsChanged = QtCore.pyqtSignal(int) + sigMaxGlobalIntensityChanged = QtCore.pyqtSignal(int) sigGammaChanged = QtCore.pyqtSignal(float) sigSegmentRadiusChanged = QtCore.pyqtSignal(int) sigMaxFeatureRatioChanged = QtCore.pyqtSignal(float) @@ -3301,7 +3303,9 @@ def __init__(self, gui): slider_data = [ ('Intensity Threshold', 1, 200, 18, '18', self.onIntensityThresholdChanged), ('Neighborhood Size', 5, 40, 10, '10', self.onNeighborhoodSizeChanged), - ('Max Stars', 50, 5000, 200, '200', self.onMaxStarsChanged), + ('SkyFit Max Stars', 50, 5000, 800, '800', self.onMaxStarsChanged), + ('Config Max Stars', 50, 2000, 400, '400', self.onConfigMaxStarsChanged), + ('Max Global Intensity', 30, 255, 230, '230', self.onMaxGlobalIntensityChanged), ('Gamma', 45, 200, 100, '1.00', self.onGammaChanged), ('Segment Radius', 2, 20, 4, '4', self.onSegmentRadiusChanged), ('Max Feature Ratio', 50, 200, 80, '0.80', self.onMaxFeatureRatioChanged), @@ -3310,6 +3314,7 @@ def __init__(self, gui): self.sliders = {} self.slider_labels = {} + self.slider_defaults = {} for name, min_val, max_val, default, default_str, callback in slider_data: key = name.lower().replace(' ', '_') @@ -3332,6 +3337,7 @@ def __init__(self, gui): self.sliders[key] = slider self.slider_labels[key] = val_label + self.slider_defaults[key] = default # Add gamma preset buttons right after gamma slider if key == 'gamma': @@ -3360,8 +3366,12 @@ def __init__(self, gui): self.intensity_threshold_label = self.slider_labels['intensity_threshold'] self.neighborhood_size_slider = self.sliders['neighborhood_size'] self.neighborhood_size_label = self.slider_labels['neighborhood_size'] - self.max_stars_slider = self.sliders['max_stars'] - self.max_stars_label = self.slider_labels['max_stars'] + self.max_stars_slider = self.sliders['skyfit_max_stars'] + self.max_stars_label = self.slider_labels['skyfit_max_stars'] + self.config_max_stars_slider = self.sliders['config_max_stars'] + self.config_max_stars_label = self.slider_labels['config_max_stars'] + self.max_global_intensity_slider = self.sliders['max_global_intensity'] + self.max_global_intensity_label = self.slider_labels['max_global_intensity'] self.gamma_slider = self.sliders['gamma'] self.gamma_label = self.slider_labels['gamma'] self.segment_radius_slider = self.sliders['segment_radius'] @@ -3371,6 +3381,14 @@ def __init__(self, gui): self.roundness_threshold_slider = self.sliders['roundness_threshold'] self.roundness_threshold_label = self.slider_labels['roundness_threshold'] + # Tooltips distinguishing the two star count budgets + self.max_stars_slider.setToolTip( + 'Number of star candidates used by SkyFit re-detection in this session.\n' + 'Initial calibration benefits from a deep sample - this value is never saved.') + self.config_max_stars_slider.setToolTip( + 'max_stars value written to the station config by Save Config.\n' + 'This bounds the star extraction cost of the nightly pipeline - 400 recommended.') + layout.addSpacing(self.scaledSpacing(1)) # Buttons in their own layout with spacing @@ -3390,6 +3408,11 @@ def __init__(self, gui): self.tune_button.clicked.connect(self.sigTuneParameters.emit) btn_layout.addWidget(self.tune_button) + self.defaults_button = QtWidgets.QPushButton('Reset to Defaults') + self.defaults_button.setToolTip('Reset all star detection parameters to the recommended defaults') + self.defaults_button.clicked.connect(self.resetToDefaults) + btn_layout.addWidget(self.defaults_button) + self.save_config_button = QtWidgets.QPushButton('Save Config...') self.save_config_button.setToolTip('Open File Manager to save star detection settings') self.save_config_button.setEnabled(False) @@ -3441,6 +3464,21 @@ def onMaxStarsChanged(self, value): self.max_stars_label.setText(str(value)) self.sigMaxStarsChanged.emit(value) + def onConfigMaxStarsChanged(self, value): + self.config_max_stars_label.setText(str(value)) + self.sigConfigMaxStarsChanged.emit(value) + + def resetToDefaults(self): + """Reset all sliders to the recommended default values.""" + for key, default in self.slider_defaults.items(): + # setValue triggers each slider's callback, so labels and override + # values in SkyFit update through the normal signal path + self.sliders[key].setValue(default) + + def onMaxGlobalIntensityChanged(self, value): + self.max_global_intensity_label.setText(str(value)) + self.sigMaxGlobalIntensityChanged.emit(value) + def onGammaChanged(self, value): gamma = value / 100.0 self.gamma_label.setText(f'{gamma:.2f}') @@ -3512,7 +3550,12 @@ def loadFromConfig(self, config): if hasattr(config, 'neighborhood_size'): self.neighborhood_size_slider.setValue(config.neighborhood_size) if hasattr(config, 'max_stars'): + # The config value seeds both budgets: the session one is free to move, + # the config one is what Save Config writes back self.max_stars_slider.setValue(config.max_stars) + self.config_max_stars_slider.setValue(config.max_stars) + if hasattr(config, 'max_global_intensity'): + self.max_global_intensity_slider.setValue(config.max_global_intensity) if hasattr(config, 'gamma'): self.gamma_slider.setValue(int(config.gamma * 100)) if hasattr(config, 'segment_radius'): diff --git a/RMS/Routines/SkyFitHelp.py b/RMS/Routines/SkyFitHelp.py index fe63a75ff..bdcade9a9 100644 --- a/RMS/Routines/SkyFitHelp.py +++ b/RMS/Routines/SkyFitHelp.py @@ -573,9 +573,20 @@ def _topic_stardetect(gui): "Size of the local window used to pick one peak per star. Larger merges close stars " "(fewer detections); smaller separates them but can split one bright star into " "several. Set it a little larger than your typical star spacing."), - ("Max stars (def. 200)", - "Upper limit on how many detections are kept (brightest first). Raise it for rich, " - "wide-field images; lower it to keep only the brightest."), + ("SkyFit max stars (def. 800)", + "Candidate budget used by Re-Detect in this session only – it is never saved " + "to the config. Initial plate fitting benefits from a deep, frame-wide star sample, so feel " + "free to raise it. When more candidates are found than the budget, they are subsampled " + "evenly across the frame (most prominent first within each region), not simply brightest " + "first."), + ("Config max stars (def. 400)", + "The max_stars value that Save Config writes to the station config. " + "This bounds the star extraction cost of the nightly pipeline on the station, which only " + "needs a modest sample to track calibration drift – 400 is recommended."), + ("Max global intensity (def. 230)", + "Median image level (8-bit scale) above which a frame is considered too bright to contain " + "stars and is skipped entirely. Raise it if twilight or moonlit frames that still show " + "stars are being rejected; frames near saturation are never worth processing."), ("Gamma (def. 1.0)", "Gamma stretch applied to the image for detection only (not the camera gamma and " "not the display gamma). Values below 1 lift faint stars out of the background so they get " @@ -611,8 +622,11 @@ def _topic_stardetect(gui): "so the number of detected stars roughly matches the catalog stars in the field." "
  • Tick Use Override Detections to feed these detections into fitting and Auto " "Fit instead of CALSTARS.
    • " - "
  • Save to Config writes the parameters to your config file so future runs reuse " - "them.
    • " + "
  • Save Config writes the parameters to your config file so the station pipeline " + "reuses them. Note it saves Config max stars, not the SkyFit session budget – " + "deep detection is for calibration here, while the nightly pipeline should stay cheap.
    • " + "
  • Reset to Defaults returns every slider in this tab to the recommended " + "values.
    • " "" "

    Parameters

    " diff --git a/Utils/SkyFit2.py b/Utils/SkyFit2.py index 2250401f0..4b3e11b56 100644 --- a/Utils/SkyFit2.py +++ b/Utils/SkyFit2.py @@ -1876,7 +1876,8 @@ def _saveFile(self, ftype, target_dirs): # Sync in-memory config attrs so modified state is cleared pt.config.intensity_threshold = pt.override_intensity_threshold pt.config.neighborhood_size = pt.override_neighborhood_size - pt.config.max_stars = pt.override_max_stars + pt.config.max_stars = pt.override_config_max_stars + pt.config.max_global_intensity = pt.override_max_global_intensity pt.config.segment_radius = pt.override_segment_radius pt.config.max_feature_ratio = pt.override_max_feature_ratio pt.config.roundness_threshold = pt.override_roundness_threshold @@ -2435,7 +2436,9 @@ def __init__(self, input_path=None, config=None, beginning_time=None, fps=None, self.star_detection_override_data = {} # Store re-detected stars per FF file self.override_intensity_threshold = 18 self.override_neighborhood_size = 10 - self.override_max_stars = 200 + self.override_max_stars = 800 + self.override_config_max_stars = 400 + self.override_max_global_intensity = 230 self.override_gamma = 1.0 self.override_segment_radius = 4 self.override_max_feature_ratio = 0.8 @@ -3352,6 +3355,8 @@ def setupUI(self, loaded_file=False): self.tab.star_detection.sigIntensityThresholdChanged.connect(self.updateIntensityThreshold) self.tab.star_detection.sigNeighborhoodSizeChanged.connect(self.updateNeighborhoodSize) self.tab.star_detection.sigMaxStarsChanged.connect(self.updateMaxStars) + self.tab.star_detection.sigConfigMaxStarsChanged.connect(self.updateConfigMaxStars) + self.tab.star_detection.sigMaxGlobalIntensityChanged.connect(self.updateMaxGlobalIntensity) self.tab.star_detection.sigGammaChanged.connect(self.updateGamma) self.tab.star_detection.sigSegmentRadiusChanged.connect(self.updateSegmentRadius) self.tab.star_detection.sigMaxFeatureRatioChanged.connect(self.updateMaxFeatureRatio) @@ -4646,6 +4651,9 @@ def initStarDetectionOverrides(self): self.override_neighborhood_size = self.config.neighborhood_size if hasattr(self.config, 'max_stars'): self.override_max_stars = self.config.max_stars + self.override_config_max_stars = self.config.max_stars + if hasattr(self.config, 'max_global_intensity'): + self.override_max_global_intensity = self.config.max_global_intensity if hasattr(self.config, 'gamma'): self.override_gamma = self.config.gamma if hasattr(self.config, 'segment_radius'): @@ -4670,8 +4678,17 @@ def updateNeighborhoodSize(self, value): self._updateConfigSaveButtonState() def updateMaxStars(self, value): - """ Update max stars override parameter. """ + """ Update the SkyFit session max stars parameter (used for re-detection, never saved). """ self.override_max_stars = value + + def updateConfigMaxStars(self, value): + """ Update the max_stars value that Save Config writes to the station config. """ + self.override_config_max_stars = value + self._updateConfigSaveButtonState() + + def updateMaxGlobalIntensity(self, value): + """ Update max global intensity (too-bright image cutoff) override parameter. """ + self.override_max_global_intensity = value self._updateConfigSaveButtonState() def updateGamma(self, value): @@ -4707,7 +4724,8 @@ def isConfigModified(self): return ( self.override_intensity_threshold != getattr(cfg, 'intensity_threshold', self.override_intensity_threshold) or self.override_neighborhood_size != getattr(cfg, 'neighborhood_size', self.override_neighborhood_size) - or self.override_max_stars != getattr(cfg, 'max_stars', self.override_max_stars) + or self.override_config_max_stars != getattr(cfg, 'max_stars', self.override_config_max_stars) + or self.override_max_global_intensity != getattr(cfg, 'max_global_intensity', self.override_max_global_intensity) or self.override_segment_radius != getattr(cfg, 'segment_radius', self.override_segment_radius) or abs(self.override_max_feature_ratio - getattr(cfg, 'max_feature_ratio', self.override_max_feature_ratio)) > 1e-6 or abs(self.override_roundness_threshold - getattr(cfg, 'roundness_threshold', self.override_roundness_threshold)) > 1e-6 @@ -4790,6 +4808,7 @@ def redetectStars(self): original_intensity_threshold = getattr(self.config, 'intensity_threshold', 18) original_neighborhood_size = getattr(self.config, 'neighborhood_size', 10) original_max_stars = getattr(self.config, 'max_stars', 200) + original_max_global_intensity = getattr(self.config, 'max_global_intensity', 230) original_gamma = getattr(self.config, 'gamma', 1.0) original_segment_radius = getattr(self.config, 'segment_radius', 4) original_max_feature_ratio = getattr(self.config, 'max_feature_ratio', 0.8) @@ -4799,6 +4818,7 @@ def redetectStars(self): self.config.intensity_threshold = self.override_intensity_threshold self.config.neighborhood_size = self.override_neighborhood_size self.config.max_stars = self.override_max_stars + self.config.max_global_intensity = self.override_max_global_intensity self.config.gamma = self.override_gamma self.config.segment_radius = self.override_segment_radius self.config.max_feature_ratio = self.override_max_feature_ratio @@ -4820,6 +4840,7 @@ def redetectStars(self): self.config.intensity_threshold = original_intensity_threshold self.config.neighborhood_size = original_neighborhood_size self.config.max_stars = original_max_stars + self.config.max_global_intensity = original_max_global_intensity self.config.gamma = original_gamma self.config.segment_radius = original_segment_radius self.config.max_feature_ratio = original_max_feature_ratio @@ -4892,6 +4913,7 @@ def redetectAllImages(self): original_intensity_threshold = getattr(self.config, 'intensity_threshold', 18) original_neighborhood_size = getattr(self.config, 'neighborhood_size', 10) original_max_stars = getattr(self.config, 'max_stars', 200) + original_max_global_intensity = getattr(self.config, 'max_global_intensity', 230) original_gamma = getattr(self.config, 'gamma', 1.0) original_segment_radius = getattr(self.config, 'segment_radius', 4) original_max_feature_ratio = getattr(self.config, 'max_feature_ratio', 0.8) @@ -4901,6 +4923,7 @@ def redetectAllImages(self): self.config.intensity_threshold = self.override_intensity_threshold self.config.neighborhood_size = self.override_neighborhood_size self.config.max_stars = self.override_max_stars + self.config.max_global_intensity = self.override_max_global_intensity self.config.gamma = self.override_gamma self.config.segment_radius = self.override_segment_radius self.config.max_feature_ratio = self.override_max_feature_ratio @@ -4948,6 +4971,7 @@ def redetectAllImages(self): self.config.intensity_threshold = original_intensity_threshold self.config.neighborhood_size = original_neighborhood_size self.config.max_stars = original_max_stars + self.config.max_global_intensity = original_max_global_intensity self.config.gamma = original_gamma self.config.segment_radius = original_segment_radius self.config.max_feature_ratio = original_max_feature_ratio @@ -5512,7 +5536,7 @@ def _extractStarsCurrentImage(self, ff_name, extra_info=None): # Check if image is too bright (scale the cutoff to the image bit depth, matching # extractStarsFF/extractStarsImgHandle so high-bit-depth data is not wrongly skipped) bit_depth = getattr(self.config, 'bit_depth', 8) - max_global_intensity = getattr(self.config, 'max_global_intensity', 150)*(2**(bit_depth - 8)) + max_global_intensity = getattr(self.config, 'max_global_intensity', 230)*(2**(bit_depth - 8)) if img_median > max_global_intensity: print(f" Image too bright (median={img_median:.1f} > {max_global_intensity})") return [] @@ -6050,7 +6074,8 @@ def _writeStarDetectionConfig(self, config_path, catalog_mag_limit): "StarExtraction": { "intensity_threshold": str(self.override_intensity_threshold), "segment_radius": str(self.override_segment_radius), - "max_stars": str(self.override_max_stars), + "max_stars": str(self.override_config_max_stars), + "max_global_intensity": str(self.override_max_global_intensity), "neighborhood_size": str(self.override_neighborhood_size), "max_feature_ratio": str(self.override_max_feature_ratio), "roundness_threshold": str(self.override_roundness_threshold), @@ -6134,7 +6159,7 @@ def _writeStarDetectionConfig(self, config_path, catalog_mag_limit): print(f"Saved star detection settings to: {config_path}") print(f" intensity_threshold: {self.override_intensity_threshold}") print(f" segment_radius: {self.override_segment_radius}") - print(f" max_stars: {self.override_max_stars}") + print(f" max_stars: {self.override_config_max_stars}") print(f" catalog_mag_limit: {catalog_mag_limit:.1f}") qmessagebox(message=f"Star detection settings saved to:\n{config_path}", From 275efc930041e29ce4d7495c09ce1dcb4ccf4766 Mon Sep 17 00:00:00 2001 From: Luc Busquin <133058544+Cybis320@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:08:08 -0700 Subject: [PATCH 02/15] Reorganize SkyFit star detection tab and add auto levels button - Split the star detection sliders into Station Config (written by Save Config) and SkyFit Session Only group boxes - Move gamma to the Station Config group and have Save Config write it to the [Capture] config section; track gamma in the modified-config check - Tighten group box margins and shorten labels so nothing clips at the fixed tab width - Add a checkable Auto Levels button to the Levels tab, kept in sync with the Ctrl+A shortcut via a new sigAutoLevelsToggled signal - Update the star detection and levels help topics --- RMS/Routines/CustomPyqtgraphClasses.py | 85 ++++++++++++++++++++------ RMS/Routines/SkyFitHelp.py | 26 +++++--- Utils/SkyFit2.py | 11 ++++ 3 files changed, 92 insertions(+), 30 deletions(-) diff --git a/RMS/Routines/CustomPyqtgraphClasses.py b/RMS/Routines/CustomPyqtgraphClasses.py index c613d44dc..157960f34 100644 --- a/RMS/Routines/CustomPyqtgraphClasses.py +++ b/RMS/Routines/CustomPyqtgraphClasses.py @@ -1214,6 +1214,10 @@ def mousePressEvent(self, event): class HistogramLUTItem(pg.HistogramLUTItem): + + # Emitted with the new state whenever auto levels is toggled (button or Ctrl+A) + sigAutoLevelsToggled = QtCore.pyqtSignal(bool) + def __init__(self, *args, **kwargs): pg.HistogramLUTItem.__init__(self, *args, **kwargs) self.level_images = [] @@ -1249,6 +1253,7 @@ def toggleAutoLevels(self): self.setLevels(*self.saved_manual_levels) self.auto_levels = not self.auto_levels self.region.setMovable(not self.auto_levels) + self.sigAutoLevelsToggled.emit(self.auto_levels) def paint(self, p, *args): # tbh this is an improvement @@ -1300,7 +1305,22 @@ def __init__(self, gui): self.maximized = True self.setFixedWidth(self.scaledWidth(self.TAB_WIDTH_CHARS)) - self.addTab(self.hist, 'Levels') + # Levels tab: auto-levels toggle button above the histogram. The button state stays in + # sync with the Ctrl+A shortcut through sigAutoLevelsToggled. + self.levels_tab = QtWidgets.QWidget() + levels_layout = QtWidgets.QVBoxLayout() + levels_layout.setContentsMargins(*self.scaledMargins(0.3, 0.2)) + levels_layout.setSpacing(self.scaledSpacing(0.3)) + self.auto_levels_button = QtWidgets.QPushButton('Auto Levels') + self.auto_levels_button.setCheckable(True) + self.auto_levels_button.setToolTip('Toggle automatic image levels (Ctrl+A)') + self.auto_levels_button.clicked.connect(lambda checked: self.hist.item.toggleAutoLevels()) + self.hist.item.sigAutoLevelsToggled.connect(self.auto_levels_button.setChecked) + levels_layout.addWidget(self.auto_levels_button) + levels_layout.addWidget(self.hist) + self.levels_tab.setLayout(levels_layout) + + self.addTab(self.levels_tab, 'Levels') self.addTab(self.param_manager, 'Fit Parameters') self.addTab(self.geolocation, 'Station') self.addTab(self.star_detection, 'Star Detection') @@ -3292,32 +3312,52 @@ def __init__(self, gui): title.setStyleSheet("font-weight: bold; font-size: 11pt;") layout.addWidget(title) - # Use QGridLayout for stable slider layout - grid = QtWidgets.QGridLayout() - grid.setSpacing(self.scaledSpacing(0.3)) - grid.setColumnStretch(0, 1) # Label column stretches - grid.setColumnStretch(1, 0) # Value column fixed - layout.addLayout(grid) - - row = 0 + # Detection parameters, split into what Save Config persists to the station config + # and what only applies to this SkyFit session slider_data = [ - ('Intensity Threshold', 1, 200, 18, '18', self.onIntensityThresholdChanged), - ('Neighborhood Size', 5, 40, 10, '10', self.onNeighborhoodSizeChanged), - ('SkyFit Max Stars', 50, 5000, 800, '800', self.onMaxStarsChanged), - ('Config Max Stars', 50, 2000, 400, '400', self.onConfigMaxStarsChanged), - ('Max Global Intensity', 30, 255, 230, '230', self.onMaxGlobalIntensityChanged), - ('Gamma', 45, 200, 100, '1.00', self.onGammaChanged), - ('Segment Radius', 2, 20, 4, '4', self.onSegmentRadiusChanged), - ('Max Feature Ratio', 50, 200, 80, '0.80', self.onMaxFeatureRatioChanged), - ('Roundness Threshold', 30, 90, 50, '0.50', self.onRoundnessThresholdChanged), + # (key, label, min, max, default, default label, callback, group) + ('intensity_threshold', 'Intensity Threshold', 1, 200, 18, '18', self.onIntensityThresholdChanged, 'config'), + ('neighborhood_size', 'Neighborhood Size', 5, 40, 10, '10', self.onNeighborhoodSizeChanged, 'config'), + ('config_max_stars', 'Max Stars', 50, 2000, 400, '400', self.onConfigMaxStarsChanged, 'config'), + ('max_global_intensity', 'Max Global Intensity', 30, 255, 230, '230', self.onMaxGlobalIntensityChanged, 'config'), + ('gamma', 'Gamma', 45, 200, 100, '1.00', self.onGammaChanged, 'config'), + ('segment_radius', 'Segment Radius', 2, 20, 4, '4', self.onSegmentRadiusChanged, 'config'), + ('max_feature_ratio', 'Max Feature Ratio', 50, 200, 80, '0.80', self.onMaxFeatureRatioChanged, 'config'), + ('roundness_threshold', 'Roundness Threshold', 30, 90, 50, '0.50', self.onRoundnessThresholdChanged, 'config'), + ('skyfit_max_stars', 'Max Stars', 50, 5000, 800, '800', self.onMaxStarsChanged, 'session'), ] self.sliders = {} self.slider_labels = {} self.slider_defaults = {} - for name, min_val, max_val, default, default_str, callback in slider_data: - key = name.lower().replace(' ', '_') + # One group box per parameter scope + group_grids = {} + group_rows = {} + for group_key, group_title, group_tip in [ + ('config', 'Station Config', + 'Written to the station config file by Save Config - these control the nightly pipeline'), + ('session', 'SkyFit Session Only', + 'Only used by SkyFit re-detection in this session - never written to the config')]: + + box = QtWidgets.QGroupBox(group_title) + box.setToolTip(group_tip) + grid = QtWidgets.QGridLayout() + grid.setSpacing(self.scaledSpacing(0.3)) + # Keep the group box padding tight so the label column doesn't get clipped + grid.setContentsMargins(*self.scaledMargins(0.4, 0.3)) + grid.setColumnStretch(0, 1) # Label column stretches + grid.setColumnStretch(1, 0) # Value column fixed + box.setLayout(grid) + layout.addWidget(box) + + group_grids[group_key] = grid + group_rows[group_key] = 0 + + for key, name, min_val, max_val, default, default_str, callback, group in slider_data: + + grid = group_grids[group] + row = group_rows[group] # Row with label and value grid.addWidget(QtWidgets.QLabel(name), row, 0) @@ -3361,6 +3401,8 @@ def __init__(self, gui): grid.setRowMinimumHeight(row, self.scaledHeight(1.75)) row += 1 + group_rows[group] = row + # Create named references for compatibility self.intensity_threshold_slider = self.sliders['intensity_threshold'] self.intensity_threshold_label = self.slider_labels['intensity_threshold'] @@ -3388,6 +3430,9 @@ def __init__(self, gui): self.config_max_stars_slider.setToolTip( 'max_stars value written to the station config by Save Config.\n' 'This bounds the star extraction cost of the nightly pipeline - 400 recommended.') + self.gamma_slider.setToolTip( + 'Camera gamma used for detection and photometry.\n' + 'Written to the config [Capture] section by Save Config, and also stored in the platepar.') layout.addSpacing(self.scaledSpacing(1)) diff --git a/RMS/Routines/SkyFitHelp.py b/RMS/Routines/SkyFitHelp.py index bdcade9a9..48861729e 100644 --- a/RMS/Routines/SkyFitHelp.py +++ b/RMS/Routines/SkyFitHelp.py @@ -239,8 +239,9 @@ def _topic_levels(gui): "distribution – roughly the 0.1st percentile for black and the " "99.95th percentile for white, while ignoring the brightest few percent of pixels so " "hot or saturated pixels don't blow out the stretch. It is a good starting point for almost " - "any image. Press " + _key(c + " + A") + " again to return to your manual levels; while auto " - "is on, the handles are locked.

    " + "any image. Toggle it with the Auto Levels button at the top of the tab or with " + + _key(c + " + A") + "; toggling again returns to your manual levels. While auto is on, " + "the handles are locked.

    " + _callout("Levels are display-only. Set them so you can comfortably see the stars you need " "to pick – they have no effect on the calibration result.") + _nav_links(related=[('tabs', 'Guide to the tabs')]) @@ -573,24 +574,24 @@ def _topic_stardetect(gui): "Size of the local window used to pick one peak per star. Larger merges close stars " "(fewer detections); smaller separates them but can split one bright star into " "several. Set it a little larger than your typical star spacing."), - ("SkyFit max stars (def. 800)", + ("Max stars (Station Config, def. 400)", + "The max_stars value that Save Config writes to the station config. " + "This bounds the star extraction cost of the nightly pipeline on the station, which only " + "needs a modest sample to track calibration drift – 400 is recommended."), + ("Max stars (detection depth) (SkyFit Session, def. 800)", "Candidate budget used by Re-Detect in this session only – it is never saved " "to the config. Initial plate fitting benefits from a deep, frame-wide star sample, so feel " "free to raise it. When more candidates are found than the budget, they are subsampled " "evenly across the frame (most prominent first within each region), not simply brightest " "first."), - ("Config max stars (def. 400)", - "The max_stars value that Save Config writes to the station config. " - "This bounds the star extraction cost of the nightly pipeline on the station, which only " - "needs a modest sample to track calibration drift – 400 is recommended."), ("Max global intensity (def. 230)", "Median image level (8-bit scale) above which a frame is considered too bright to contain " "stars and is skipped entirely. Raise it if twilight or moonlit frames that still show " "stars are being rejected; frames near saturation are never worth processing."), ("Gamma (def. 1.0)", - "Gamma stretch applied to the image for detection only (not the camera gamma and " - "not the display gamma). Values below 1 lift faint stars out of the background so they get " - "detected."), + "Camera gamma used when measuring stars (not the display gamma). Values below 1 lift " + "faint stars out of the background so they get detected. Saved to the config " + "[Capture] section and also stored in the platepar for photometry."), ("Segment radius (def. 4 px)", "Radius of the patch used to centroid and measure each star. Match it to the typical star " "size (FWHM): too small clips the star and worsens the centroid; too large pulls in " @@ -613,6 +614,11 @@ def _topic_stardetect(gui): "use those instead – handy when the default detection misses stars or picks up " "noise.

    " + "

    The parameters are split into two groups: Station Config values are what " + "Save Config writes to the station config file (they control the nightly pipeline), " + "while SkyFit Session Only values are used by re-detection here and are never " + "saved.

    " + "

    How to use it

    " "
      " "
    1. Adjust the parameters below and click Redetect to re-run detection on the " diff --git a/Utils/SkyFit2.py b/Utils/SkyFit2.py index 4b3e11b56..6c0300df1 100644 --- a/Utils/SkyFit2.py +++ b/Utils/SkyFit2.py @@ -1878,6 +1878,8 @@ def _saveFile(self, ftype, target_dirs): pt.config.neighborhood_size = pt.override_neighborhood_size pt.config.max_stars = pt.override_config_max_stars pt.config.max_global_intensity = pt.override_max_global_intensity + pt.config.gamma = pt.override_gamma + pt._original_config_gamma = pt.override_gamma pt.config.segment_radius = pt.override_segment_radius pt.config.max_feature_ratio = pt.override_max_feature_ratio pt.config.roundness_threshold = pt.override_roundness_threshold @@ -4701,6 +4703,7 @@ def updateGamma(self, value): if self.platepar is not None: self.platepar.gamma = value + self._updateConfigSaveButtonState() self.updateLeftLabels() def updateSegmentRadius(self, value): @@ -4729,6 +4732,11 @@ def isConfigModified(self): or self.override_segment_radius != getattr(cfg, 'segment_radius', self.override_segment_radius) or abs(self.override_max_feature_ratio - getattr(cfg, 'max_feature_ratio', self.override_max_feature_ratio)) > 1e-6 or abs(self.override_roundness_threshold - getattr(cfg, 'roundness_threshold', self.override_roundness_threshold)) > 1e-6 + # Compare gamma against the original config value: with override enabled, + # the in-memory config.gamma is synced to the slider and would mask the change + or abs(self.override_gamma - (self._original_config_gamma + if self._original_config_gamma is not None + else getattr(cfg, 'gamma', self.override_gamma))) > 0.005 ) def _updateConfigSaveButtonState(self): @@ -6080,6 +6088,9 @@ def _writeStarDetectionConfig(self, config_path, catalog_mag_limit): "max_feature_ratio": str(self.override_max_feature_ratio), "roundness_threshold": str(self.override_roundness_threshold), }, + "Capture": { + "gamma": f"{self.override_gamma:.2f}", + }, "Calibration": { "catalog_mag_limit": f"{catalog_mag_limit:.1f}", }, From 34c2511f5c28a965f644c761d43cd1b18bc0f2fe Mon Sep 17 00:00:00 2001 From: Luc Busquin <133058544+Cybis320@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:58:24 -0700 Subject: [PATCH 03/15] Snap max stars sliders to 100 and make tab panels scroll when short - Both max stars sliders move in 100-star increments (drag snapping, arrow keys step 100, page step 500, minimum raised to 100) - Wrap the form-style tabs (Fit Parameters, Station, Star Detection, Mask, Settings, Debruijn) in scroll areas so short windows scroll the content instead of compressing it; Levels and Help stay unwrapped - Add tabWidgetFor()/tabIndexOf() resolvers and route identity-based tab lookups through them --- RMS/Routines/CustomPyqtgraphClasses.py | 71 +++++++++++++++++++++----- Utils/SkyFit2.py | 10 ++-- 2 files changed, 62 insertions(+), 19 deletions(-) diff --git a/RMS/Routines/CustomPyqtgraphClasses.py b/RMS/Routines/CustomPyqtgraphClasses.py index 157960f34..91001c331 100644 --- a/RMS/Routines/CustomPyqtgraphClasses.py +++ b/RMS/Routines/CustomPyqtgraphClasses.py @@ -1320,12 +1320,25 @@ def __init__(self, gui): levels_layout.addWidget(self.hist) self.levels_tab.setLayout(levels_layout) + # Wrap the form-style tabs in scroll areas so that short windows scroll the content + # instead of compressing it into unreadability. Levels stretches naturally with the + # window and Help scrolls by itself, so they stay unwrapped. + self._tab_scroll_wrappers = {} + for panel in (self.param_manager, self.geolocation, self.star_detection, + self.mask, self.settings, self.debruijn): + scroll = QtWidgets.QScrollArea() + scroll.setWidgetResizable(True) + scroll.setFrameShape(QtWidgets.QFrame.NoFrame) + scroll.setHorizontalScrollBarPolicy(QtCore.Qt.ScrollBarAlwaysOff) + scroll.setWidget(panel) + self._tab_scroll_wrappers[panel] = scroll + self.addTab(self.levels_tab, 'Levels') - self.addTab(self.param_manager, 'Fit Parameters') - self.addTab(self.geolocation, 'Station') - self.addTab(self.star_detection, 'Star Detection') - self.addTab(self.mask, 'Mask') - self.addTab(self.settings, 'Settings') + self.addTab(self.tabWidgetFor(self.param_manager), 'Fit Parameters') + self.addTab(self.tabWidgetFor(self.geolocation), 'Station') + self.addTab(self.tabWidgetFor(self.star_detection), 'Star Detection') + self.addTab(self.tabWidgetFor(self.mask), 'Mask') + self.addTab(self.tabWidgetFor(self.settings), 'Settings') self.addTab(self.help, 'ⓘ Help') self.setCurrentIndex(self.index) # redundant @@ -1373,14 +1386,24 @@ def onTabBarClicked(self, index): self.gui.view_widget.setFocus() + def tabWidgetFor(self, panel): + """ Return the widget actually inserted in the tab bar for the given panel + (its scroll wrapper if it has one, the panel itself otherwise). + """ + return self._tab_scroll_wrappers.get(panel, panel) + + def tabIndexOf(self, panel): + """ Return the tab index of the given panel, looking through scroll wrappers. """ + return self.indexOf(self.tabWidgetFor(panel)) + def onSkyFit(self): # Remove ManualReduction-specific tabs self.removeTabText('Debruijn') # Add Skyfit-specific tabs - self.insertTab(1, self.param_manager, "Fit Parameters") - self.insertTab(2, self.geolocation, "Station") + self.insertTab(1, self.tabWidgetFor(self.param_manager), "Fit Parameters") + self.insertTab(2, self.tabWidgetFor(self.geolocation), "Station") self.settings.onSkyFit() self.setCurrentIndex(self.index) @@ -1394,7 +1417,7 @@ def onManualReduction(self): # Add ManualReduction-specific tabs if self.gui.img.img_handle.input_type == 'dfn': - self.insertTab(1, self.debruijn, 'Debruijn') + self.insertTab(1, self.tabWidgetFor(self.debruijn), 'Debruijn') self.setCurrentIndex(self.index) @@ -3318,13 +3341,13 @@ def __init__(self, gui): # (key, label, min, max, default, default label, callback, group) ('intensity_threshold', 'Intensity Threshold', 1, 200, 18, '18', self.onIntensityThresholdChanged, 'config'), ('neighborhood_size', 'Neighborhood Size', 5, 40, 10, '10', self.onNeighborhoodSizeChanged, 'config'), - ('config_max_stars', 'Max Stars', 50, 2000, 400, '400', self.onConfigMaxStarsChanged, 'config'), + ('config_max_stars', 'Max Stars', 100, 2000, 400, '400', self.onConfigMaxStarsChanged, 'config'), ('max_global_intensity', 'Max Global Intensity', 30, 255, 230, '230', self.onMaxGlobalIntensityChanged, 'config'), ('gamma', 'Gamma', 45, 200, 100, '1.00', self.onGammaChanged, 'config'), ('segment_radius', 'Segment Radius', 2, 20, 4, '4', self.onSegmentRadiusChanged, 'config'), ('max_feature_ratio', 'Max Feature Ratio', 50, 200, 80, '0.80', self.onMaxFeatureRatioChanged, 'config'), ('roundness_threshold', 'Roundness Threshold', 30, 90, 50, '0.50', self.onRoundnessThresholdChanged, 'config'), - ('skyfit_max_stars', 'Max Stars', 50, 5000, 800, '800', self.onMaxStarsChanged, 'session'), + ('skyfit_max_stars', 'Max Stars', 100, 5000, 800, '800', self.onMaxStarsChanged, 'session'), ] self.sliders = {} @@ -3423,6 +3446,11 @@ def __init__(self, gui): self.roundness_threshold_slider = self.sliders['roundness_threshold'] self.roundness_threshold_label = self.slider_labels['roundness_threshold'] + # Both max stars budgets move in increments of 100 + for s in (self.max_stars_slider, self.config_max_stars_slider): + s.setSingleStep(100) + s.setPageStep(500) + # Tooltips distinguishing the two star count budgets self.max_stars_slider.setToolTip( 'Number of star candidates used by SkyFit re-detection in this session.\n' @@ -3505,13 +3533,28 @@ def onNeighborhoodSizeChanged(self, value): self.neighborhood_size_label.setText(str(value)) self.sigNeighborhoodSizeChanged.emit(value) + @staticmethod + def _snapTo100(value): + """Snap a max stars value to the nearest 100.""" + return max(100, int(round(value/100.0))*100) + def onMaxStarsChanged(self, value): - self.max_stars_label.setText(str(value)) - self.sigMaxStarsChanged.emit(value) + snapped = self._snapTo100(value) + if snapped != value: + # Re-fires this handler with the snapped value + self.max_stars_slider.setValue(snapped) + return + self.max_stars_label.setText(str(snapped)) + self.sigMaxStarsChanged.emit(snapped) def onConfigMaxStarsChanged(self, value): - self.config_max_stars_label.setText(str(value)) - self.sigConfigMaxStarsChanged.emit(value) + snapped = self._snapTo100(value) + if snapped != value: + # Re-fires this handler with the snapped value + self.config_max_stars_slider.setValue(snapped) + return + self.config_max_stars_label.setText(str(snapped)) + self.sigConfigMaxStarsChanged.emit(snapped) def resetToDefaults(self): """Reset all sliders to the recommended default values.""" diff --git a/Utils/SkyFit2.py b/Utils/SkyFit2.py index 6c0300df1..3f31ad6d3 100644 --- a/Utils/SkyFit2.py +++ b/Utils/SkyFit2.py @@ -6763,7 +6763,7 @@ def toggleMaskFlatBackground(self, use_flat): self.mask_use_flat_background = use_flat # Only change the image if we're currently on the mask tab - mask_tab_index = self.tab.indexOf(self.tab.mask) + mask_tab_index = self.tab.tabIndexOf(self.tab.mask) if self.tab.currentIndex() != mask_tab_index: return @@ -6799,11 +6799,11 @@ def onTabChanged(self, old_index, new_index): # Refresh the Help tab content when it is opened, so it reflects the current mode and any # features that were toggled since it was last shown. - if new_index == self.tab.indexOf(self.tab.help): + if new_index == self.tab.tabIndexOf(self.tab.help): self.tab.help.updateHelp() # Mask tab is at index 4 (Levels=0, Fit Parameters=1, Station=2, Star Detection=3, Mask=4) - mask_tab_index = self.tab.indexOf(self.tab.mask) + mask_tab_index = self.tab.tabIndexOf(self.tab.mask) if old_index == mask_tab_index and self.mask_use_flat_background: # Leaving mask tab while flat was shown - restore current image @@ -10150,7 +10150,7 @@ def keyPressEvent(self, event): # Handle brush undo - Ctrl+Z when on mask tab if event.key() == QtCore.Qt.Key_Z and (modifiers == QtCore.Qt.ControlModifier): - mask_tab_index = self.tab.indexOf(self.tab.mask) + mask_tab_index = self.tab.tabIndexOf(self.tab.mask) if self.tab.currentIndex() == mask_tab_index and self.mask_brush_stroke_history: self.undoBrushStroke() return @@ -11835,7 +11835,7 @@ def toggleInfo(self): def _raiseHelpTab(self): """ Select and maximise the Help tab. Returns True if the tab exists. """ - help_index = self.tab.indexOf(self.tab.help) + help_index = self.tab.tabIndexOf(self.tab.help) if help_index == -1: return False self.tab.setCurrentIndex(help_index) From 007166a543dc0a7582cc693671fe127ac271865f Mon Sep 17 00:00:00 2001 From: Luc Busquin <133058544+Cybis320@users.noreply.github.com> Date: Sat, 4 Jul 2026 15:42:42 -0700 Subject: [PATCH 04/15] Hide the top-left info panel by default again The Help tab rework (7b8da601) collapsed the three-state F1 cycle into a show/hide toggle and incidentally flipped the panel's startup state back to visible, reverting 5d7064b1. Restore hidden-by-default, but keep the panel force-shown as the empty-state hint when no data is loaded (and hide it again once data arrives). --- Utils/SkyFit2.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/Utils/SkyFit2.py b/Utils/SkyFit2.py index 3f31ad6d3..ffe9e7500 100644 --- a/Utils/SkyFit2.py +++ b/Utils/SkyFit2.py @@ -2792,7 +2792,10 @@ def setupUI(self, loaded_file=False): self.label1.setTextWidth(label_fm.averageCharWidth() * 35) # ~35 chars wide self.label1.setZValue(1000) self.label1.setParentItem(self.img_frame) - self.label1.show() + + # Hidden by default, toggled with F1. It is force-shown as the empty-state hint when no + # data is loaded, and hidden again once data arrives. + self.label1.hide() self.catalog_stars_visible = True @@ -3447,8 +3450,10 @@ def setupUI(self, loaded_file=False): self._show_calibration_dialog_on_start = False self.showCalibrationFilesDialog() else: - # Show empty-state message + # Show empty-state message (the panel is otherwise hidden by default) self.label1.setText("No data loaded.\nUse File > File Manager to open a folder.") + self.label1.show() + self._label1_empty_hint = True self.image_navigation_slider.hide() self.image_navigation_label.hide() @@ -3932,6 +3937,12 @@ def updateLeftLabels(self): if not self.hasData(): return + # The empty-state hint force-shows the info panel; restore the hidden-by-default state + # once data is loaded (F1 toggles it from then on) + if getattr(self, '_label1_empty_hint', False): + self._label1_empty_hint = False + self.label1.hide() + # Refresh the optical-axis pointing indicator (zenith arrow, elevation, WASD step size). This # is the common path for the pan (WASD), rotation (Q/E), scale and step-size (+/-) keys. self.updatePointingIndicator() From d92d5347a5294ce01a8dbdca1c858efcf9a96334 Mon Sep 17 00:00:00 2001 From: Luc Busquin <133058544+Cybis320@users.noreply.github.com> Date: Mon, 6 Jul 2026 07:51:12 -0700 Subject: [PATCH 05/15] Star Detection tab: protect station-bound values from reset and snap Review fixes for #918 (items 1 and 2): - Reset to Defaults no longer resets gamma to 1.0 - it is a hardware property of the camera, and resetting it silently corrupted photometry the moment Save Config wrote it to the station config. Gamma and the config max stars now return to the loaded station config values; only the tuning sliders return to the recommended defaults. - Programmatic slider seeding bypasses the snap-to-100 handlers, so a config value like max_stars=150 loads exactly instead of snapping to 200, showing as an unsaved modification with zero user input, and being written back snapped. User motion still snaps. Out-of-range config values extend the slider range instead of clamping. --- RMS/Routines/CustomPyqtgraphClasses.py | 88 ++++++++++++++++++++------ 1 file changed, 70 insertions(+), 18 deletions(-) diff --git a/RMS/Routines/CustomPyqtgraphClasses.py b/RMS/Routines/CustomPyqtgraphClasses.py index 91001c331..9a2376bbe 100644 --- a/RMS/Routines/CustomPyqtgraphClasses.py +++ b/RMS/Routines/CustomPyqtgraphClasses.py @@ -3322,6 +3322,15 @@ def __init__(self, gui): QtWidgets.QWidget.__init__(self) self.gui = gui + # While True, programmatic slider seeding bypasses the snap-to-100 handlers, so a + # config value like max_stars=150 survives loading exactly instead of being snapped + # (and then counting as an unsaved modification with zero user input) + self._seeding = False + + # The station config last loaded into the sliders - Reset to Defaults returns the + # station-bound values (gamma, config max stars) to it rather than global defaults + self._loaded_config = None + layout = QtWidgets.QVBoxLayout() layout.setContentsMargins(*self.scaledMargins(1, 0.5)) layout.setSpacing(self.scaledSpacing(0.3)) @@ -3539,30 +3548,67 @@ def _snapTo100(value): return max(100, int(round(value/100.0))*100) def onMaxStarsChanged(self, value): - snapped = self._snapTo100(value) - if snapped != value: - # Re-fires this handler with the snapped value - self.max_stars_slider.setValue(snapped) - return - self.max_stars_label.setText(str(snapped)) - self.sigMaxStarsChanged.emit(snapped) + # Snap only user motion - programmatic seeding keeps the exact config value + if not self._seeding: + snapped = self._snapTo100(value) + if snapped != value: + # Re-fires this handler with the snapped value + self.max_stars_slider.setValue(snapped) + return + self.max_stars_label.setText(str(value)) + self.sigMaxStarsChanged.emit(value) def onConfigMaxStarsChanged(self, value): - snapped = self._snapTo100(value) - if snapped != value: - # Re-fires this handler with the snapped value - self.config_max_stars_slider.setValue(snapped) - return - self.config_max_stars_label.setText(str(snapped)) - self.sigConfigMaxStarsChanged.emit(snapped) + # Snap only user motion - programmatic seeding keeps the exact config value + if not self._seeding: + snapped = self._snapTo100(value) + if snapped != value: + # Re-fires this handler with the snapped value + self.config_max_stars_slider.setValue(snapped) + return + self.config_max_stars_label.setText(str(value)) + self.sigConfigMaxStarsChanged.emit(value) def resetToDefaults(self): - """Reset all sliders to the recommended default values.""" + """Reset the tuning sliders to the recommended defaults. + + Station-bound values return to the loaded config instead: gamma is a hardware + property of the camera (resetting it to 1.0 would corrupt photometry the moment + Save Config is pressed), and the config max stars is the station's pipeline + budget, not a tuning preference. + """ + cfg = self._loaded_config + for key, default in self.slider_defaults.items(): + + if key == 'gamma': + if cfg is not None and hasattr(cfg, 'gamma'): + self.gamma_slider.setValue(int(round(cfg.gamma*100))) + continue + + if key == 'config_max_stars': + if cfg is not None and hasattr(cfg, 'max_stars'): + self._seedSlider(self.config_max_stars_slider, cfg.max_stars) + continue + # setValue triggers each slider's callback, so labels and override # values in SkyFit update through the normal signal path self.sliders[key].setValue(default) + def _seedSlider(self, slider, value): + """ Set a slider value programmatically, bypassing the snap-to-100 handlers so the + exact value survives (the change signal still fires normally). Extends the + slider range if needed so out-of-range config values are not clamped. """ + if value < slider.minimum(): + slider.setMinimum(value) + if value > slider.maximum(): + slider.setMaximum(value) + self._seeding = True + try: + slider.setValue(value) + finally: + self._seeding = False + def onMaxGlobalIntensityChanged(self, value): self.max_global_intensity_label.setText(str(value)) self.sigMaxGlobalIntensityChanged.emit(value) @@ -3620,6 +3666,9 @@ def updateStatus(self, using_override, star_count=None, candidate_count=None): def loadFromConfig(self, config): """Initialize sliders from config values.""" + + self._loaded_config = config + if hasattr(config, 'intensity_threshold'): # Give the threshold slider a bit-depth-appropriate maximum before setting the # value, otherwise a high-bit-depth config threshold is silently clamped to the @@ -3639,9 +3688,12 @@ def loadFromConfig(self, config): self.neighborhood_size_slider.setValue(config.neighborhood_size) if hasattr(config, 'max_stars'): # The config value seeds both budgets: the session one is free to move, - # the config one is what Save Config writes back - self.max_stars_slider.setValue(config.max_stars) - self.config_max_stars_slider.setValue(config.max_stars) + # the config one is what Save Config writes back. Seed without snapping so a + # config value like 150 loads exactly - otherwise the seeded value counts as + # an unsaved config modification with zero user input, and Save Config would + # write the snapped value back to the station config + self._seedSlider(self.max_stars_slider, config.max_stars) + self._seedSlider(self.config_max_stars_slider, config.max_stars) if hasattr(config, 'max_global_intensity'): self.max_global_intensity_slider.setValue(config.max_global_intensity) if hasattr(config, 'gamma'): From 35bf5cb8cc207c4a9c5e4e3bf081f15a73d4ad0d Mon Sep 17 00:00:00 2001 From: Luc Busquin <133058544+Cybis320@users.noreply.github.com> Date: Mon, 6 Jul 2026 07:51:12 -0700 Subject: [PATCH 06/15] Shield raw-CALSTARS consumers from junk detections; fix stale docstrings Review follow-ups for #918 (item 3 cheap fixes and the docstring note): - autoCheckFit seeds its NN alignment from the FF file with the most SNR-filtered stars instead of the most raw entries, so a junk-flooded frame cannot win the seed selection (falls back to the raw count for older CALSTARS files without an SNR column). - The flux sensor FWHM median is computed over SNR-filtered detections for the same reason, with the same fallback. - max_global_intensity docstrings updated: the gate compares the median (not mean) image intensity, default 230. --- RMS/Astrometry/CheckFit.py | 14 ++++++++++++-- RMS/ExtractStars.py | 4 ++-- Utils/Flux.py | 12 ++++++++++-- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/RMS/Astrometry/CheckFit.py b/RMS/Astrometry/CheckFit.py index 3538cbbfe..b4119d9ca 100644 --- a/RMS/Astrometry/CheckFit.py +++ b/RMS/Astrometry/CheckFit.py @@ -504,8 +504,18 @@ def _handleFailure(config, platepar, calstars_data, catalog_stars, _nn_refinemen calstars_dict = {ff_file: star_data for ff_file, star_data in calstars_list} - # Extract star list from CALSTARS file from FF file with most stars - max_len_ff = max(calstars_dict, key=lambda k: len(calstars_dict[k])) + # Extract star list from the FF file with the most high-quality stars. Counting + # raw entries would let a junk-flooded frame (moon glare, noise) win and seed + # the alignment badly - prefer the SNR-filtered count where the data has SNR + # (older CALSTARS files pad the column with -1, in which case fall back to the + # raw count) + def qualityStarCount(ff_name): + star_data = np.array(calstars_dict[ff_name]) + if (star_data.ndim == 2) and (star_data.shape[1] > 6) and np.any(star_data[:, 6] > 0): + return int(np.sum(star_data[:, 6] >= 5.0)) + return len(star_data) + + max_len_ff = max(calstars_dict, key=qualityStarCount) # Pass full CALSTARS data (y, x, intensity, ...) - alignPlatepar will extract what it needs # and use intensities to infer appropriate catalog limiting magnitude diff --git a/RMS/ExtractStars.py b/RMS/ExtractStars.py index 140802d03..a310cc19d 100644 --- a/RMS/ExtractStars.py +++ b/RMS/ExtractStars.py @@ -317,7 +317,7 @@ def extractStarsFF( ff_dir: [str] Path to directory where FF files are. ff_name: [str] Name of the FF file. config: [config object] configuration object (loaded from the .config file) - max_global_intensity: [int] maximum mean intensity of an image before it is discarded as too bright + max_global_intensity: [int] maximum median intensity of an image before it is discarded as too bright. 230 by default. border: [int] apply a mask on the detections by removing all that are too close to the given image border (in pixels) neighborhood_size: [int] size of the neighbourhood for the maximum search (in pixels) @@ -427,7 +427,7 @@ def extractStarsImgHandle(img_handle, dark: [ndarray] Dark frame. None by default. mask: [ndarray] Mask image. None by default. config: [config object] configuration object (loaded from the .config file) - max_global_intensity: [int] maximum mean intensity of an image before it is discarded as too bright + max_global_intensity: [int] maximum median intensity of an image before it is discarded as too bright. 230 by default. border: [int] apply a mask on the detections by removing all that are too close to the given image border (in pixels) neighborhood_size: [int] size of the neighbourhood for the maximum search (in pixels) diff --git a/Utils/Flux.py b/Utils/Flux.py index ece2047b9..dd8bd0b3c 100644 --- a/Utils/Flux.py +++ b/Utils/Flux.py @@ -1817,8 +1817,16 @@ def sensorCharacterization(config, flux_config, dir_path, meteor_data, default_f else: star_data = np.array(star_data) - # Compute the median star FWHM - fwhm_median = np.median(star_data[:, 4]) + # Compute the median star FWHM over high-quality detections: with permissive + # extraction settings a bad night keeps junk detections whose bogus FWHM would + # bias the median. Filter on SNR where the data has it (older CALSTARS files + # pad the column with -1), otherwise use all entries + fwhm_values = star_data[:, 4] + if (star_data.shape[1] > 6) and np.any(star_data[:, 6] > 0): + snr_mask = star_data[:, 6] >= 5.0 + if np.any(snr_mask): + fwhm_values = star_data[snr_mask, 4] + fwhm_median = np.median(fwhm_values) # Store the values to the dictionary sensor_data[ff_name] = [fwhm_median] From 7c375170823b11579deff14350031676a1b1c613 Mon Sep 17 00:00:00 2001 From: Luc Busquin <133058544+Cybis320@users.noreply.github.com> Date: Mon, 6 Jul 2026 10:42:03 -0700 Subject: [PATCH 07/15] Revert the SNR filter on the flux FWHM median - measured, it only harms Empirical check of the previous commit's flux change: on a frame that is 94% junk (866 detections, 54 catalog-matched), the raw FWHM median differs from the matched-only median by just 0.07 px - junk that survives the PSF fit has star-like FWHM by construction, and the median is robust to it exactly as designed. Meanwhile the SNR >= 5 filter shifts the median by +0.43 px on a perfectly normal night (standard-settings detections have median SNR ~3.5, so the cut keeps only the brightest third), which would have changed flux results network-wide. The SNR-filtered seed selection in autoCheckFit stands - for that consumer the same measurement confirms it: junk sits at SNR ~1.3, real stars at ~8, the ranking is unaffected on clean nights and stops junk-flooded frames from winning on bad ones. --- Utils/Flux.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/Utils/Flux.py b/Utils/Flux.py index dd8bd0b3c..9e27b4351 100644 --- a/Utils/Flux.py +++ b/Utils/Flux.py @@ -1817,16 +1817,12 @@ def sensorCharacterization(config, flux_config, dir_path, meteor_data, default_f else: star_data = np.array(star_data) - # Compute the median star FWHM over high-quality detections: with permissive - # extraction settings a bad night keeps junk detections whose bogus FWHM would - # bias the median. Filter on SNR where the data has it (older CALSTARS files - # pad the column with -1), otherwise use all entries - fwhm_values = star_data[:, 4] - if (star_data.shape[1] > 6) and np.any(star_data[:, 6] > 0): - snr_mask = star_data[:, 6] >= 5.0 - if np.any(snr_mask): - fwhm_values = star_data[snr_mask, 4] - fwhm_median = np.median(fwhm_values) + # Compute the median star FWHM. Deliberately NOT SNR-filtered: junk detections + # that survive the PSF fit have star-like FWHM by construction (measured: a 94% + # junk-flooded frame shifts the median by only 0.07 px), while an SNR cut + # biases the median toward bright stars on perfectly normal nights (+0.4 px at + # SNR >= 5, where the median detection SNR of a standard night is ~3.5) + fwhm_median = np.median(star_data[:, 4]) # Store the values to the dictionary sensor_data[ff_name] = [fwhm_median] From e15100c46afd808c81753bed4386bc42749a9bd2 Mon Sep 17 00:00:00 2001 From: Luc Busquin <133058544+Cybis320@users.noreply.github.com> Date: Mon, 6 Jul 2026 11:23:35 -0700 Subject: [PATCH 08/15] Saturate the clear-sky star prediction at the detection cap The cloud-detection ratio compares catalog-matched detections against a predicted star count derived from the photometric zero-point - a model that does not know the extractor never returns more than config.max_stars detections. On star-rich nights with sensitive cameras the prediction can exceed the cap severalfold (at detector LM ~8, a 40x22 deg FOV holds ~900 predicted stars against a 400-star cap), so a perfectly clear sky reads as cloudy (matched ~ cap, ratio < 0.5). Before the subsampler such frames were dropped entirely and always read as cloudy; now they deliver a full cap of matches, so the ratio must compare against what detection is actually allowed to deliver: min(predicted, max_stars). Also guards a division by zero when the prediction is empty. --- Utils/Flux.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Utils/Flux.py b/Utils/Flux.py index 9e27b4351..fc23ba905 100644 --- a/Utils/Flux.py +++ b/Utils/Flux.py @@ -1342,9 +1342,15 @@ def detectClouds(config, dir_path, N=5, mask=None, show_plots=True, save_plots=F recalibrated_platepars, ff_limiting_magnitude, config, mask=mask, show_plot=show_plots ) - # Compute the ratio between matched and predicted stars + # Compute the ratio between matched and predicted stars. The prediction counts every + # catalog star the detector could see, but the extractor never returns more than + # config.max_stars detections - so on star-rich nights with sensitive cameras the + # prediction can exceed what detection is allowed to deliver, and a perfectly clear + # sky would read as cloudy (matched ~ cap, predicted >> cap). Saturate the prediction + # at the detection cap so the ratio compares against what is actually achievable. ratio = { - ff_file: (matched_count[ff_file]/predicted_stars[ff_file] if ff_file in predicted_stars else 0) + ff_file: (matched_count[ff_file]/min(predicted_stars[ff_file], config.max_stars) + if (ff_file in predicted_stars) and (predicted_stars[ff_file] > 0) else 0) for ff_file in recorded_files } From 3081ef41328968ea55aca93c2562dcb2095e008d Mon Sep 17 00:00:00 2001 From: Luc Busquin <133058544+Cybis320@users.noreply.github.com> Date: Mon, 6 Jul 2026 15:03:21 -0700 Subject: [PATCH 09/15] Treat both max-stars values as pure compute budgets Since the subsampler, exceeding max_stars keeps the best-distributed most prominent subset instead of dropping the frame - so neither max-stars value is a detection-quality knob anymore, and tuning or seeding them from sky content no longer makes sense: - The SkyFit session budget defaults to 2000 and is no longer seeded from the station config, whose value is the Pi's pipeline budget - seeding dragged the deep calibration default down on every folder load. - Tune no longer sets max_stars from the candidate count and the seasonal star variation sweep (both fossils from when overflowing the cap lost the frame); it tunes what remains quality-relevant - intensity threshold and segment radius. The now-unused seasonal variation helper is removed. - Reset to Defaults returns the session budget to 2000 and the config budget to the station's configured value, as before. --- RMS/Routines/CustomPyqtgraphClasses.py | 15 ++-- Utils/SkyFit2.py | 106 +++---------------------- 2 files changed, 20 insertions(+), 101 deletions(-) diff --git a/RMS/Routines/CustomPyqtgraphClasses.py b/RMS/Routines/CustomPyqtgraphClasses.py index 9a2376bbe..a7762df8f 100644 --- a/RMS/Routines/CustomPyqtgraphClasses.py +++ b/RMS/Routines/CustomPyqtgraphClasses.py @@ -3356,7 +3356,7 @@ def __init__(self, gui): ('segment_radius', 'Segment Radius', 2, 20, 4, '4', self.onSegmentRadiusChanged, 'config'), ('max_feature_ratio', 'Max Feature Ratio', 50, 200, 80, '0.80', self.onMaxFeatureRatioChanged, 'config'), ('roundness_threshold', 'Roundness Threshold', 30, 90, 50, '0.50', self.onRoundnessThresholdChanged, 'config'), - ('skyfit_max_stars', 'Max Stars', 100, 5000, 800, '800', self.onMaxStarsChanged, 'session'), + ('skyfit_max_stars', 'Max Stars', 100, 5000, 2000, '2000', self.onMaxStarsChanged, 'session'), ] self.sliders = {} @@ -3687,12 +3687,13 @@ def loadFromConfig(self, config): if hasattr(config, 'neighborhood_size'): self.neighborhood_size_slider.setValue(config.neighborhood_size) if hasattr(config, 'max_stars'): - # The config value seeds both budgets: the session one is free to move, - # the config one is what Save Config writes back. Seed without snapping so a - # config value like 150 loads exactly - otherwise the seeded value counts as - # an unsaved config modification with zero user input, and Save Config would - # write the snapped value back to the station config - self._seedSlider(self.max_stars_slider, config.max_stars) + # Only the CONFIG budget tracks the station config (it is what Save Config + # writes back). The session budget is a desktop compute limit for calibration + # work - seeding it from the Pi's pipeline budget would drag the deep session + # default down on every folder load. Seed without snapping so a config value + # like 150 loads exactly - otherwise the seeded value counts as an unsaved + # config modification with zero user input, and Save Config would write the + # snapped value back to the station config self._seedSlider(self.config_max_stars_slider, config.max_stars) if hasattr(config, 'max_global_intensity'): self.max_global_intensity_slider.setValue(config.max_global_intensity) diff --git a/Utils/SkyFit2.py b/Utils/SkyFit2.py index ffe9e7500..9c15a99c7 100644 --- a/Utils/SkyFit2.py +++ b/Utils/SkyFit2.py @@ -4663,7 +4663,9 @@ def initStarDetectionOverrides(self): if hasattr(self.config, 'neighborhood_size'): self.override_neighborhood_size = self.config.neighborhood_size if hasattr(self.config, 'max_stars'): - self.override_max_stars = self.config.max_stars + # The session budget is a desktop compute limit - deep by default for + # calibration work; only the config budget tracks the station config + self.override_max_stars = 2000 self.override_config_max_stars = self.config.max_stars if hasattr(self.config, 'max_global_intensity'): self.override_max_global_intensity = self.config.max_global_intensity @@ -5355,30 +5357,15 @@ def tuneStarDetection(self): # Enable override mode self.tab.star_detection.use_override_checkbox.setChecked(True) - # Compute seasonal headroom: sweep the year to find peak visible star count - # relative to current, so max_stars accommodates the busiest sky. - self.status_bar.showMessage("Tuning... computing seasonal variation") - QtWidgets.QApplication.processEvents() - peak_ratio, peak_count, current_count = self._computeSeasonalStarVariation( - self.catalog_stars, jd) - seasonal_factor = max(peak_ratio, 1.2) # At least 20% headroom - print(f" Seasonal variation: {current_count} stars now, {peak_count} at peak " - f"(ratio={peak_ratio:.2f}, using {seasonal_factor:.2f}x)") - - # Set max_stars from the measured raw candidate count scaled by seasonal - # variation + 15% frame-to-frame buffer, rounded up to nearest 100, - # clamped to [400, 2000]. - num_candidates = detection_info.get('num_candidates', n_detected * 3) - self.override_max_stars = int(np.clip( - np.ceil(num_candidates * seasonal_factor * 1.15 / 100) * 100, 400, 2000)) - print(f" Raw candidates: {num_candidates}, max_stars set to: {self.override_max_stars}") - - # Update the GUI slider to reflect the new max_stars value - # First ensure the slider range can accommodate the value - current_max = self.tab.star_detection.max_stars_slider.maximum() - if self.override_max_stars > current_max: - self.tab.star_detection.max_stars_slider.setMaximum(self.override_max_stars) - self.tab.star_detection.max_stars_slider.setValue(self.override_max_stars) + # max_stars is deliberately NOT tuned: since the subsampler, exceeding the + # cap keeps the best-distributed most prominent subset instead of dropping the + # frame, so both max-stars values are pure compute budgets - there is no + # detection-quality reason to adjust them from candidate counts. (The historical + # tuning of max_stars from candidate counts and seasonal star variation dates + # from when overflowing it lost the frame.) + num_candidates = detection_info.get('num_candidates', n_detected*3) + print(" Raw candidates: {} (max_stars untouched - compute budget, not a " + "tuning target)".format(num_candidates)) # Trigger re-detection with the new parameters self.redetectStars() @@ -5821,75 +5808,6 @@ def _findOptimalThreshold(self, results): return best_threshold - def _computeSeasonalStarVariation(self, catalog_stars, current_jd): - """Compute the ratio of peak-to-current visible star count across all seasons. - - Sweeps 24 evenly-spaced sidereal times (one full year) to find the maximum - number of catalog stars visible in the FOV for this station's pointing. - - Arguments: - catalog_stars: [ndarray] Deep catalog array (ra, dec, mag). - current_jd: [float] Julian date of the current observation. - - Returns: - (peak_ratio, peak_count, current_count): [tuple] - peak_ratio: peak visible stars / current visible stars. - peak_count: maximum visible star count across the year. - current_count: visible star count at the current JD. - """ - pp = self.platepar - fov_radius = getFOVSelectionRadius(pp) - - def _count_visible(jd): - """Count catalog stars visible in the image at a given JD.""" - # Get RA/Dec of FOV center at this JD - img_time = jd2Date(jd) - _, ra_c, dec_c, _ = xyToRaDecPP( - [img_time], [pp.X_res / 2], [pp.Y_res / 2], [1], pp, - extinction_correction=False) - ra_c, dec_c = ra_c[0], dec_c[0] - - # Pre-filter catalog to FOV region - _, subset = subsetCatalog( - catalog_stars, ra_c, dec_c, jd, pp.lat, pp.lon, - fov_radius, 99.0, remove_under_horizon=True) - if len(subset) == 0: - return 0 - - # Project to image coordinates - cat_x, cat_y, _ = getCatalogStarsImagePositions( - np.array(subset), jd, pp) - in_image = ((cat_x >= 0) & (cat_x < pp.X_res) & - (cat_y >= 0) & (cat_y < pp.Y_res)) - - # Apply mask if available - if self.mask is not None and self.mask.img is not None: - if (self.mask.img.shape[0] == pp.Y_res and - self.mask.img.shape[1] == pp.X_res): - x_int = np.clip(cat_x.astype(int), 0, self.mask.img.shape[1] - 1) - y_int = np.clip(cat_y.astype(int), 0, self.mask.img.shape[0] - 1) - in_image = in_image & (self.mask.img[y_int, x_int] != 0) - - return int(np.sum(in_image)) - - # Count at current JD - current_count = _count_visible(current_jd) - - # Sweep 24 points across a full year (~15.2 days apart) - peak_count = current_count - for i in range(24): - sweep_jd = current_jd + i * (365.25 / 24) - count = _count_visible(sweep_jd) - if count > peak_count: - peak_count = count - - if current_count > 0: - peak_ratio = peak_count / current_count - else: - peak_ratio = 2.0 # Fallback if current count is 0 - - return peak_ratio, peak_count, current_count - def _findOptimalCatalogLM(self, jd, detected_x, detected_y, target_matches, match_radius=2.0): """ Find the catalog limiting magnitude where catalog stars match the detected From 2130ae0596134ea5871ab009f0126d9239ae59d6 Mon Sep 17 00:00:00 2001 From: Luc Busquin <133058544+Cybis320@users.noreply.github.com> Date: Tue, 7 Jul 2026 11:59:48 -0700 Subject: [PATCH 10/15] Show the capped star prediction in the observing-periods plot The clear-sky ratio saturates the prediction at max_stars (the extractor never returns more), but the plot still showed the raw prediction with a 'Matched/Predicted stars' ratio above it - on star-rich nights the two panels disagreed (e.g. ratio 0.80 plotted while the panels showed 320/870). Plot the capped prediction the ratio actually uses (only when the cap binds) and note the capping in the ratio axis label. --- Utils/Flux.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Utils/Flux.py b/Utils/Flux.py index fc23ba905..541190fe4 100644 --- a/Utils/Flux.py +++ b/Utils/Flux.py @@ -1378,7 +1378,7 @@ def detectClouds(config, dir_path, N=5, mask=None, show_plots=True, save_plots=F # Plot the computed ratio ax[0].scatter([FFfile.filenameToDatetime(x) for x in ratio.keys()], list(ratio.values()), \ marker='o', s=5, c='k', zorder=6, label='Measurements') - ax[0].set_ylabel("Matched/Predicted stars") + ax[0].set_ylabel("Matched/Predicted stars\n(prediction capped at max_stars)") # Plot the radio threshold times = [FFfile.filenameToDatetime(x) for x in ratio.keys()] @@ -1412,6 +1412,17 @@ def detectClouds(config, dir_path, N=5, mask=None, show_plots=True, save_plots=F [predicted_stars[ff] for ff in predicted_stars], label='Predicted stars', marker='x', color='k', zorder=5, ) + + # Show the prediction the ratio actually uses: detection never returns more than + # max_stars, so the prediction saturates there - without this line the two panels + # disagree on star-rich nights (top ratio uses the cap, bottom raw counts do not) + if any(predicted_stars[ff] > config.max_stars for ff in predicted_stars): + ax[1].scatter( + [FFfile.filenameToDatetime(ff) for ff in predicted_stars], + [min(predicted_stars[ff], config.max_stars) for ff in predicted_stars], + label='Predicted (capped at max_stars={:d})'.format(config.max_stars), + marker='x', color='gray', zorder=5, + ) # Shade the regions with clear skies From 844f88a8b2e3ba31fb035528a0b8d3492366d492 Mon Sep 17 00:00:00 2001 From: Luc Busquin <133058544+Cybis320@users.noreply.github.com> Date: Tue, 7 Jul 2026 12:40:42 -0700 Subject: [PATCH 11/15] Self-calibrate the clear-sky expectation from the night's uncapped frames The clear-sky ratio saturated the prediction at max_stars, but the cap counts candidates while the numerator counts catalog-matched survivors of the PSF fit - a clear sky on a cap-bound night plateaued at ~0.75 rather than 1, eating into the threshold margin on cameras with weaker matching. Since observing periods are computed after the night, the deficit is measurable from the night itself: the median matched/predicted over frames where the cap does not bind and the sky is already clearly clear. The expectation becomes min(predicted, deficit*max_stars) - continuous in the prediction, exact historical semantics below the effective ceiling, and a clear capped sky now reads ~1.0 (measured on a real cap-bound starry night: 0.75-0.80 -> ~1.0, cloudy frames unaffected at ~0.2). Clamped to [0.5, 1] so a contaminated sample cannot lift cloudy frames past the threshold, and inert (previous behavior) on nights with no usable calibration sample. The observing-periods plot shows the expectation series it actually uses. --- Utils/Flux.py | 60 +++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/Utils/Flux.py b/Utils/Flux.py index 541190fe4..e2648ddc6 100644 --- a/Utils/Flux.py +++ b/Utils/Flux.py @@ -1342,15 +1342,43 @@ def detectClouds(config, dir_path, N=5, mask=None, show_plots=True, save_plots=F recalibrated_platepars, ff_limiting_magnitude, config, mask=mask, show_plot=show_plots ) - # Compute the ratio between matched and predicted stars. The prediction counts every - # catalog star the detector could see, but the extractor never returns more than - # config.max_stars detections - so on star-rich nights with sensitive cameras the - # prediction can exceed what detection is allowed to deliver, and a perfectly clear - # sky would read as cloudy (matched ~ cap, predicted >> cap). Saturate the prediction - # at the detection cap so the ratio compares against what is actually achievable. + # Compute the ratio between matched and EXPECTED stars. Two effects separate the raw + # prediction from what detection can deliver on a perfectly clear sky: + # 1. The extractor never returns more than config.max_stars candidates, so the + # prediction saturates at the cap (on star-rich nights predicted >> cap and the raw + # ratio would read a clear sky as cloudy). + # 2. The chain from candidates to matched stars loses a roughly constant fraction + # (PSF-fit acceptance, catalog matching, prediction model error) - a clear-sky + # "deficit" that would otherwise cap the achievable ratio well below 1. + # The deficit is self-calibrated from the night: the median matched/predicted over + # frames where the cap does NOT bind and the sky is already clearly clear. It is + # applied to the CAP ONLY - uncapped frames keep the historical matched/predicted + # semantics the threshold was tuned on; only cap-bound frames compare against what + # the capped detection chain actually delivers. Clamped to [0.5, 1] so a contaminated + # calibration sample cannot inflate cloudy frames past the threshold, and left at 1 + # (conservative, previous behavior) when there is no usable sample (fully capped or + # fully cloudy night). + deficit_sample = [ + matched_count[ff]/predicted_stars[ff] + for ff in recorded_files + if (ff in predicted_stars) and (0 < predicted_stars[ff] <= config.max_stars) + and (matched_count[ff]/predicted_stars[ff] >= ratio_threshold) + ] + if len(deficit_sample) >= 5: + detection_deficit = float(np.clip(np.median(deficit_sample), 0.5, 1.0)) + log.info("Clear-sky detection deficit self-calibrated from {:d} uncapped frames: " + "{:.2f}".format(len(deficit_sample), detection_deficit)) + else: + detection_deficit = 1.0 + + expected_stars = { + ff: min(predicted_stars[ff], detection_deficit*config.max_stars) + for ff in predicted_stars + } + ratio = { - ff_file: (matched_count[ff_file]/min(predicted_stars[ff_file], config.max_stars) - if (ff_file in predicted_stars) and (predicted_stars[ff_file] > 0) else 0) + ff_file: (matched_count[ff_file]/expected_stars[ff_file] + if (ff_file in expected_stars) and (expected_stars[ff_file] > 0) else 0) for ff_file in recorded_files } @@ -1378,7 +1406,7 @@ def detectClouds(config, dir_path, N=5, mask=None, show_plots=True, save_plots=F # Plot the computed ratio ax[0].scatter([FFfile.filenameToDatetime(x) for x in ratio.keys()], list(ratio.values()), \ marker='o', s=5, c='k', zorder=6, label='Measurements') - ax[0].set_ylabel("Matched/Predicted stars\n(prediction capped at max_stars)") + ax[0].set_ylabel("Matched/Expected stars") # Plot the radio threshold times = [FFfile.filenameToDatetime(x) for x in ratio.keys()] @@ -1413,14 +1441,14 @@ def detectClouds(config, dir_path, N=5, mask=None, show_plots=True, save_plots=F label='Predicted stars', marker='x', color='k', zorder=5, ) - # Show the prediction the ratio actually uses: detection never returns more than - # max_stars, so the prediction saturates there - without this line the two panels - # disagree on star-rich nights (top ratio uses the cap, bottom raw counts do not) - if any(predicted_stars[ff] > config.max_stars for ff in predicted_stars): + # Show the expectation the ratio actually uses (cap + self-calibrated clear-sky + # deficit) - without this the two panels disagree on star-rich nights + if any(abs(expected_stars[ff] - predicted_stars[ff]) > 1 for ff in predicted_stars): ax[1].scatter( - [FFfile.filenameToDatetime(ff) for ff in predicted_stars], - [min(predicted_stars[ff], config.max_stars) for ff in predicted_stars], - label='Predicted (capped at max_stars={:d})'.format(config.max_stars), + [FFfile.filenameToDatetime(ff) for ff in expected_stars], + [expected_stars[ff] for ff in expected_stars], + label='Expected (cap {:d}, deficit {:.2f})'.format( + config.max_stars, detection_deficit), marker='x', color='gray', zorder=5, ) From 6d840fa8d23b839e3067e62968cab94d79dd8a31 Mon Sep 17 00:00:00 2001 From: Luc Busquin <133058544+Cybis320@users.noreply.github.com> Date: Tue, 7 Jul 2026 13:00:04 -0700 Subject: [PATCH 12/15] Ground the observing-periods ratio with a clear-sky reference line The ratio is now 'fraction of the clear-sky expectation' rather than a raw matched/predicted fraction, and can legitimately sit slightly above 1 on cap-bound frames (the deficit is calibrated on uncapped frames, whose fainter stars underperform the bright capped subset). A dotted reference at 1.0 makes the reading direct: near the green line = clear, under the red line = cloudy. --- Utils/Flux.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Utils/Flux.py b/Utils/Flux.py index e2648ddc6..fd26595c2 100644 --- a/Utils/Flux.py +++ b/Utils/Flux.py @@ -1414,6 +1414,12 @@ def detectClouds(config, dir_path, N=5, mask=None, show_plots=True, save_plots=F ax[0].plot(time_arr, [ratio_threshold, ratio_threshold], linestyle='dashed', color='r', zorder=5, \ alpha=0.5, label='Threshold') + # Reference for reading the ratio: ~1 = the full clear-sky expectation is delivered + # (values slightly above 1 are normal - the deficit is calibrated on uncapped + # frames, whose fainter star population underperforms the bright capped subset) + ax[0].plot(time_arr, [1.0, 1.0], linestyle='dotted', color='g', zorder=5, + alpha=0.6, label='Clear-sky expectation') + # Shade the regions with clear skies if len(time_intervals): From a99a8ae44595196d8970cd9234a63d10c565467c Mon Sep 17 00:00:00 2001 From: Luc Busquin <133058544+Cybis320@users.noreply.github.com> Date: Tue, 7 Jul 2026 13:11:38 -0700 Subject: [PATCH 13/15] Plain star-count verbiage in the observing-periods plot Legend and labels stick to the three nouns - matched, predicted, expected stars - with the 1.0 reference reading 'Matched = Expected'. The cap and calibrated deficit stay in the log, not the legend. --- Utils/Flux.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Utils/Flux.py b/Utils/Flux.py index fd26595c2..bd164aa1a 100644 --- a/Utils/Flux.py +++ b/Utils/Flux.py @@ -1414,11 +1414,12 @@ def detectClouds(config, dir_path, N=5, mask=None, show_plots=True, save_plots=F ax[0].plot(time_arr, [ratio_threshold, ratio_threshold], linestyle='dashed', color='r', zorder=5, \ alpha=0.5, label='Threshold') - # Reference for reading the ratio: ~1 = the full clear-sky expectation is delivered - # (values slightly above 1 are normal - the deficit is calibrated on uncapped - # frames, whose fainter star population underperforms the bright capped subset) + # Reference for reading the plot: on the line, the frame delivered exactly the + # expected star count (slightly above is normal - the deficit is calibrated on + # uncapped frames, whose fainter star population underperforms the bright capped + # subset) ax[0].plot(time_arr, [1.0, 1.0], linestyle='dotted', color='g', zorder=5, - alpha=0.6, label='Clear-sky expectation') + alpha=0.6, label='Matched = Expected') # Shade the regions with clear skies if len(time_intervals): @@ -1453,8 +1454,7 @@ def detectClouds(config, dir_path, N=5, mask=None, show_plots=True, save_plots=F ax[1].scatter( [FFfile.filenameToDatetime(ff) for ff in expected_stars], [expected_stars[ff] for ff in expected_stars], - label='Expected (cap {:d}, deficit {:.2f})'.format( - config.max_stars, detection_deficit), + label='Expected stars', marker='x', color='gray', zorder=5, ) From 893157432fec3e0e064814e41f2b0d4d798775e1 Mon Sep 17 00:00:00 2001 From: Luc Busquin <133058544+Cybis320@users.noreply.github.com> Date: Tue, 7 Jul 2026 13:24:17 -0700 Subject: [PATCH 14/15] Drop the Matched = Expected reference line from the observing plot A horizontal reference at 1 asserts a calibrated expectation that the prediction model does not deliver: on a second live station the uncapped matched/predicted runs 1.2-1.5 early night and drifts to 0.9 - the limiting-magnitude prediction is biased low for that camera, and time-varyingly so. The deficit calibration correctly refuses to inflate expectations (clamped at 1), so measurements legitimately sit far above the line and it reads as broken. The threshold line is the decision boundary and the bottom panel carries the counts; the reference line only misleads. --- Utils/Flux.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Utils/Flux.py b/Utils/Flux.py index bd164aa1a..cadac07d6 100644 --- a/Utils/Flux.py +++ b/Utils/Flux.py @@ -1414,12 +1414,6 @@ def detectClouds(config, dir_path, N=5, mask=None, show_plots=True, save_plots=F ax[0].plot(time_arr, [ratio_threshold, ratio_threshold], linestyle='dashed', color='r', zorder=5, \ alpha=0.5, label='Threshold') - # Reference for reading the plot: on the line, the frame delivered exactly the - # expected star count (slightly above is normal - the deficit is calibrated on - # uncapped frames, whose fainter star population underperforms the bright capped - # subset) - ax[0].plot(time_arr, [1.0, 1.0], linestyle='dotted', color='g', zorder=5, - alpha=0.6, label='Matched = Expected') # Shade the regions with clear skies if len(time_intervals): From 024f2ae13f2b9dc67bd9bf0f05cf04c1215ad4d3 Mon Sep 17 00:00:00 2001 From: Luc Busquin <133058544+Cybis320@users.noreply.github.com> Date: Tue, 7 Jul 2026 23:54:02 -0700 Subject: [PATCH 15/15] Pass the mask to cloud detection in the Flux CLI and FluxBatch predictStarNumberInFOV filters predicted stars through the mask, but only the nightly pipeline caller passed one - the manual Flux CLI and FluxBatch ran the prediction unmasked, counting stars behind obstructions that matched stars can never contain. On heavily masked cameras this biases the clear-sky ratio low in batch flux runs, rejecting intervals that the same night passes on-station. --- Utils/Flux.py | 6 +++++- Utils/FluxBatch.py | 8 +++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/Utils/Flux.py b/Utils/Flux.py index cadac07d6..e53737a04 100644 --- a/Utils/Flux.py +++ b/Utils/Flux.py @@ -3909,7 +3909,11 @@ def fluxParser(): # Automatically deterine time intervals else: - time_intervals = detectClouds(config, dir_path, show_plots=True, save_plots=False, \ + # Load the mask so the star prediction skips obstructed regions - without it the + # expectation counts stars the detector can never match and the ratio biases low + mask = getMaskFile(dir_path, config, default_as_backup=True) + + time_intervals = detectClouds(config, dir_path, mask=mask, show_plots=True, save_plots=False, \ ratio_threshold=cml_args.ratiothres) for i, interval in enumerate(time_intervals): diff --git a/Utils/FluxBatch.py b/Utils/FluxBatch.py index 6c774b731..efab1a245 100644 --- a/Utils/FluxBatch.py +++ b/Utils/FluxBatch.py @@ -26,6 +26,7 @@ from RMS.Formats.Showers import FluxShowers, loadRadiantShowers from Utils.Flux import calculatePopulationIndex, calculateMassIndex, computeFlux, detectClouds, fluxParser, \ calculateFixedBins, calculateZHR, massVerniani, loadShower +from RMS.Routines.MaskImage import getMaskFile from RMS.Routines.SolarLongitude import unwrapSol from RMS.Misc import formatScientific, roundToSignificantDigits, SegmentedScale, mkdirP from RMS.QueuedPool import QueuedPool @@ -833,8 +834,13 @@ def computeTimeIntervalsPerStation(night_dir_path, time_intervals, binduration, # Find time intervals to compute flux with print('Detecting whether clouds are present...') + # Load the mask so the star prediction skips obstructed regions - without it the + # expectation counts stars the detector can never match and the ratio biases low + mask = getMaskFile(ftp_dir_path, config_station, default_as_backup=True) + time_intervals = detectClouds( - config_station, ftp_dir_path, show_plots=False, ratio_threshold=ratio_threshold + config_station, ftp_dir_path, mask=mask, show_plots=False, + ratio_threshold=ratio_threshold ) print('Cloud detection complete!')