From fe1812fcd74f539f61765c5d88a779a7c6ea5fbb Mon Sep 17 00:00:00 2001 From: David Law Date: Wed, 11 Dec 2024 17:44:29 -0500 Subject: [PATCH 1/5] Fix pixel replace numpy 2.0 issues --- jwst/pixel_replace/pixel_replace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jwst/pixel_replace/pixel_replace.py b/jwst/pixel_replace/pixel_replace.py index 99a540d932..0d83ecc8c9 100644 --- a/jwst/pixel_replace/pixel_replace.py +++ b/jwst/pixel_replace/pixel_replace.py @@ -380,7 +380,7 @@ def fit_profile(self, model): # TODO: check on signs here - absolute max sometimes picks up # large negative outliers norm_scale = minimize(self.profile_mse, x0=np.abs(np.nanmax(norm_current)), - args=(min_median, norm_current)).x + args=(min_median, norm_current), method='Nelder-Mead').x scale = np.max(min_current) From d85959467a29602aae51dea7245e57d1135ae458 Mon Sep 17 00:00:00 2001 From: David Law Date: Wed, 11 Dec 2024 17:50:39 -0500 Subject: [PATCH 2/5] Change log entry --- changes/9004.pixel_replace.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/9004.pixel_replace.rst diff --git a/changes/9004.pixel_replace.rst b/changes/9004.pixel_replace.rst new file mode 100644 index 0000000000..dd6fded54a --- /dev/null +++ b/changes/9004.pixel_replace.rst @@ -0,0 +1 @@ +Change from the default BFGS algorithm to Nelder-Mead when calling scipy.minimize within the fit_profile approach to pixel replacement in order to fix numpy 2.0 compatibility issues. From 23f1a3c2cae338f29b16e5618b3a9b6e4b992d86 Mon Sep 17 00:00:00 2001 From: David Law Date: Fri, 13 Dec 2024 11:57:04 -0500 Subject: [PATCH 3/5] Add safety catches to pixel replacement to not scale on noise --- jwst/pixel_replace/pixel_replace.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/jwst/pixel_replace/pixel_replace.py b/jwst/pixel_replace/pixel_replace.py index 0d83ecc8c9..71305a63e3 100644 --- a/jwst/pixel_replace/pixel_replace.py +++ b/jwst/pixel_replace/pixel_replace.py @@ -319,20 +319,27 @@ def fit_profile(self, model): # Cut out valid neighboring profiles adjacent_condition = self.custom_slice(dispaxis, valid_adjacent_inds) profile_data = model.data[adjacent_condition] + profile_err = model.err[adjacent_condition] # Mask out bad pixels invalid_condition = (model.dq[adjacent_condition] & self.DO_NOT_USE).astype(bool) profile_data[invalid_condition] = np.nan + profile_err[invalid_condition] = np.nan # Add additional cut to pull only from region with valid data # for convenience (may not be necessary) region_condition = self.custom_slice(3 - dispaxis, range(*profile_cut)) profile_data = profile_data[region_condition] + profile_snr = np.abs(profile_data / profile_err[region_condition]) # Normalize profile data # TODO: check on signs here - absolute max sometimes picks up # large negative outliers profile_norm_scale = np.nanmax(np.abs(profile_data), axis=(dispaxis - 1), keepdims=True) + # If profile data has SNR <>> 5 everywhere just use unity scaling + # (so we don't normalize to noise) + if (np.nanmax(profile_snr) < 5): + profile_norm_scale[:] = 1.0 normalized = profile_data / profile_norm_scale # Get corresponding error and variance data and scale and mask to match @@ -362,11 +369,13 @@ def fit_profile(self, model): # Clean current profile of values flagged as bad current_condition = self.custom_slice(dispaxis, ind) current_profile = model.data[current_condition] + current_err = model.err[current_condition] cleaned_current = np.where( model.dq[current_condition] & self.DO_NOT_USE, np.nan, current_profile )[range(*profile_cut)] + cleaned_snr = cleaned_current / current_err[range(*profile_cut)] replace_mask = np.where(~np.isnan(cleaned_current))[0] if len(replace_mask) == 0: @@ -377,12 +386,19 @@ def fit_profile(self, model): norm_current = min_current / np.max(min_current) # Scale median profile to current profile with bad pixel - minimize mse? - # TODO: check on signs here - absolute max sometimes picks up - # large negative outliers - norm_scale = minimize(self.profile_mse, x0=np.abs(np.nanmax(norm_current)), - args=(min_median, norm_current), method='Nelder-Mead').x - - scale = np.max(min_current) + # Only do this scaling if max SNR > 5 so we don't scale on noise. + # Likewise, require input values below 1e20 so that we don't overflow the + # minimization routine with extremely bad noise. + if ((np.nanmax(cleaned_snr > 5)) & (np.nanmax(np.abs(min_median)) < 1e20) + & (np.nanmax(np.abs(norm_current)) < 1e20)): + # TODO: check on signs here - absolute max sometimes picks up + # large negative outliers + norm_scale = minimize(self.profile_mse, x0=np.abs(np.nanmax(norm_current)), + args=(min_median, norm_current), method='Nelder-Mead').x + scale = np.max(min_current) + else: + norm_scale = 1.0 + scale = 1.0 # Replace pixels that are do-not-use but not non-science current_dq = model.dq[current_condition][range(*profile_cut)] From 96ab30588aef5090a17baa1d1e51ef33f3e837cc Mon Sep 17 00:00:00 2001 From: David Law Date: Fri, 13 Dec 2024 11:59:03 -0500 Subject: [PATCH 4/5] Update log message --- changes/9004.pixel_replace.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/changes/9004.pixel_replace.rst b/changes/9004.pixel_replace.rst index dd6fded54a..b6b69d38f3 100644 --- a/changes/9004.pixel_replace.rst +++ b/changes/9004.pixel_replace.rst @@ -1 +1,4 @@ -Change from the default BFGS algorithm to Nelder-Mead when calling scipy.minimize within the fit_profile approach to pixel replacement in order to fix numpy 2.0 compatibility issues. +Change from the default BFGS algorithm to Nelder-Mead when calling scipy.minimize +within the fit_profile approach topixel replacement in order to fix numpy 2.0 +compatibility issues. Additionally, add safety catch to ensure that pixel replacement +profile fitting doesn't attempt to scale based on noise. \ No newline at end of file From e4315aad387fa13906e8ecd30e86d5e9acf35a67 Mon Sep 17 00:00:00 2001 From: David Law Date: Fri, 13 Dec 2024 16:13:00 -0500 Subject: [PATCH 5/5] Handle NIRSpec units failure case --- jwst/pixel_replace/pixel_replace.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/jwst/pixel_replace/pixel_replace.py b/jwst/pixel_replace/pixel_replace.py index 71305a63e3..0a21341ec0 100644 --- a/jwst/pixel_replace/pixel_replace.py +++ b/jwst/pixel_replace/pixel_replace.py @@ -336,7 +336,7 @@ def fit_profile(self, model): # TODO: check on signs here - absolute max sometimes picks up # large negative outliers profile_norm_scale = np.nanmax(np.abs(profile_data), axis=(dispaxis - 1), keepdims=True) - # If profile data has SNR <>> 5 everywhere just use unity scaling + # If profile data has SNR < 5 everywhere just use unity scaling # (so we don't normalize to noise) if (np.nanmax(profile_snr) < 5): profile_norm_scale[:] = 1.0 @@ -369,13 +369,11 @@ def fit_profile(self, model): # Clean current profile of values flagged as bad current_condition = self.custom_slice(dispaxis, ind) current_profile = model.data[current_condition] - current_err = model.err[current_condition] cleaned_current = np.where( model.dq[current_condition] & self.DO_NOT_USE, np.nan, current_profile )[range(*profile_cut)] - cleaned_snr = cleaned_current / current_err[range(*profile_cut)] replace_mask = np.where(~np.isnan(cleaned_current))[0] if len(replace_mask) == 0: @@ -386,15 +384,15 @@ def fit_profile(self, model): norm_current = min_current / np.max(min_current) # Scale median profile to current profile with bad pixel - minimize mse? - # Only do this scaling if max SNR > 5 so we don't scale on noise. - # Likewise, require input values below 1e20 so that we don't overflow the + # Only do this scaling if we didn't default to all-unity scaling above, + # and require input values below 1e20 so that we don't overflow the # minimization routine with extremely bad noise. - if ((np.nanmax(cleaned_snr > 5)) & (np.nanmax(np.abs(min_median)) < 1e20) + if ((np.nanmedian(profile_norm_scale) != 1.0) & (np.nanmax(np.abs(min_median)) < 1e20) & (np.nanmax(np.abs(norm_current)) < 1e20)): # TODO: check on signs here - absolute max sometimes picks up # large negative outliers norm_scale = minimize(self.profile_mse, x0=np.abs(np.nanmax(norm_current)), - args=(min_median, norm_current), method='Nelder-Mead').x + args=(np.abs(min_median), np.abs(norm_current)), method='Nelder-Mead').x scale = np.max(min_current) else: norm_scale = 1.0