Changed the variable names for the general "biggest source" and "bigg…

…est source fraction"

limits which were misleading, and added a new variable name to accommodate the RickerWavelet
biggest source limit.
   OLD                       NEW
   _rw2d_biggest_source  ==> _biggest_source_limit
   _rw2d_source_fraction ==> _biggest_source_fraction_limit
    new                  ==> _rw_biggest_source_limit

These changes required updating the catalog parameter JSON files, the code which reads
the parameter values, and parameter names in segmentation source support functions.

Clarified informational log messages when reporting the configuration limits in use for
the Segmentation catalog.  Moved the computation of the ratio between the "Gaussian
biggest source" and the "Rickerwavlet biggest source" to be computed, as well as applied,
for Round 2. Added more informational messages to keep the user better informed.  Changed
some limit values (though more changes will be needed for other detectors):

ACS/WFC ONLY
   negative_percent: 15.0 ==> 20.0
   rw2d_nsigma: 10.0  ==> 40.0

drizzlepac/haputils/catalog_utils.py

@@ -1516,8 +1516,9 @@ def __init__(self, image, param_dict, param_dict_qc, diagnostic_mode, tp_sources
         self._nsigma = self.param_dict["sourcex"]["segm_nsigma"]
         self._rw2d_size = self.param_dict["sourcex"]["rw2d_size"]
         self._rw2d_nsigma = self.param_dict["sourcex"]["rw2d_nsigma"]
-        self._rw2d_biggest_source = self.param_dict["sourcex"]["rw2d_biggest_source"]
-        self._rw2d_source_fraction = self.param_dict["sourcex"]["rw2d_source_fraction"]
+        self._biggest_source_limit = self.param_dict["sourcex"]["biggest_source_limit"]
+        self._biggest_fraction_limit = self.param_dict["sourcex"]["biggest_fraction_limit"]
+        self._rw_biggest_source_limit = self.param_dict["sourcex"]["rw_biggest_source_limit"]
         self._bs_deblend_limit = self.param_dict["sourcex"]["biggest_source_deblend_limit"]
         self._sf_deblend_limit = self.param_dict["sourcex"]["source_fraction_deblend_limit"]
         self._ratio_bigsource_limit = self.param_dict["sourcex"]["ratio_bigsource_limit"]
@@ -1584,10 +1585,13 @@ def identify_sources(self, **pars):
             log.info("contrast (frac. flux for peak to be separate object, 0=max. deblend, 1=no deblend): {}".format(self._contrast))
             log.info("RickerWavelet nsigma (threshold = nsigma * background_rms): {}".format(self._rw2d_nsigma))
             log.info("RickerWavelet kernel X- and Y-dimension: {}".format(self._rw2d_size))
-            log.info("Percentage limit on biggest source (criterion for  RickerWavelet kernel): {}".format(100.0 * self._rw2d_biggest_source))
-            log.info("Percentage limit on source fraction over the image (criterion for RickerWavelet kernel): {}".format(100.0 * self._rw2d_source_fraction))
-            log.info("Percentage limit on biggest source deblending limit: {}".format(100.0 * self._bs_deblend_limit))
-            log.info("Percentage limit on source fraction deblending limit: {}".format(100.0 * self._sf_deblend_limit))
+            log.info("Maximum % limit on acceptable biggest source: {}".format(100.0 * self._biggest_source_limit))
+            log.info("Maximum % limit on acceptable source fraction over the image: {}".format(100.0 * self._biggest_fraction_limit))
+            log.info("Maximum % limit on acceptable RickerWavelet biggest source: {}".format(100.0 * self._rw_biggest_source_limit))
+            log.info("Minimum acceptable Gaussian vs RickerWavelet kernel big source ratio limit (overlapping PSFs): {}".format(self._ratio_bigsource_limit))
+            log.info("Maximum % limit on acceptable biggest source (RW deblend retry): {}".format(100.0 * self._bs_deblend_limit))
+            log.info("Maximum % limit on acceptable source fraction (RW deblend retry): {}".format(100.0 * self._sf_deblend_limit))
+            log.info("Ignore prohibitively big sources identified by mimimum ratio of the areas (biggest/biggest-1): {}".format(self._ratio_bigsource_deblend_limit))
             log.info("Scaling parameter of the Kron radius: {}".format(self._kron_scaling_radius))
             log.info("Kron minimum circular radius: {}".format(self._kron_minimum_radius))
             log.info("")
@@ -1617,6 +1621,7 @@ def identify_sources(self, **pars):
             # Round 1
             ncount = 0
             log.info("")
+            log.info("ROUND 1")
             log.info("Using Custom kernel or Gaussian to generate a segmentation map.")
             g_segm_img, g_is_big_crowded, g_bs, g_sf = self.detect_and_eval_segments(imgarr,
                                                                                      g2d_kernel,
@@ -1626,8 +1631,8 @@ def identify_sources(self, **pars):
                                                                                      self.image.bkg_background_ra,
                                                                                      self.image.bkg_rms_ra,
                                                                                      check_big_island_only=False,
-                                                                                     rw2d_biggest_source=self._rw2d_biggest_source,
-                                                                                     rw2d_source_fraction=self._rw2d_source_fraction)
+                                                                                     biggest_source=self._biggest_source_limit,
+                                                                                     source_fraction=self._biggest_fraction_limit)
             segm_img_orig = copy.deepcopy(g_segm_img)
 
             # If the science field via the segmentation map is deemed crowded or has big sources/islands, compute the
@@ -1658,14 +1663,8 @@ def identify_sources(self, **pars):
                                                                                              self.image.bkg_background_ra,
                                                                                              self.image.bkg_rms_ra,
                                                                                              check_big_island_only=True,
-                                                                                             rw2d_biggest_source=self._rw2d_biggest_source,
-                                                                                             rw2d_source_fraction=self._rw2d_source_fraction)
-
-                # Compute the ratio of big sources/islands using Custom/Gaussian kernel vs Rickerwavelet kernel
-                # This value can be used as a discriminant between overlapping point sources and nebulousity fields
-                ratio_cg2rw_bigsource = 3.0
-                if rw_bs > 0.0:
-                    ratio_cg2rw_bigsource = g_bs / rw_bs
+                                                                                             biggest_source=self._rw_biggest_source_limit,
+                                                                                             source_fraction=self._biggest_fraction_limit)
 
                 # Check if the RickerWavelet segmentation image still seems to be problematic
                 if rw_is_big_crowded and rw_segm_img:
@@ -1731,6 +1730,7 @@ def identify_sources(self, **pars):
                     # Round 2
                     ncount += 1
                     log.info("")
+                    log.info("ROUND 2")
                     log.info("With alternate background...using Custom/Gaussian kernel to generate a segmentation map.")
                     del g_segm_img
                     g_segm_img, g_is_big_crowded, g_bs, g_sf = self.detect_and_eval_segments(imgarr,
@@ -1741,8 +1741,8 @@ def identify_sources(self, **pars):
                                                                                              self.image.bkg_background_ra,
                                                                                              self.image.bkg_rms_ra,
                                                                                              check_big_island_only=False,
-                                                                                             rw2d_biggest_source=self._rw2d_biggest_source,
-                                                                                             rw2d_source_fraction=self._rw2d_source_fraction)
+                                                                                             biggest_source=self._biggest_source_limit,
+                                                                                             source_fraction=self._biggest_fraction_limit)
 
                     # Check again for big sources/islands or a large source fraction
                     if g_is_big_crowded:
@@ -1763,8 +1763,14 @@ def identify_sources(self, **pars):
                                                                                                      self.image.bkg_background_ra,
                                                                                                      self.image.bkg_rms_ra,
                                                                                                      check_big_island_only=False,
-                                                                                                     rw2d_biggest_source=self._bs_deblend_limit,
-                                                                                                     rw2d_source_fraction=self._sf_deblend_limit)
+                                                                                                     biggest_source=self._bs_deblend_limit,
+                                                                                                     source_fraction=self._sf_deblend_limit)
+
+                        # Compute the ratio of big sources/islands using Custom/Gaussian kernel vs Rickerwavelet kernel
+                        # This value can be used as a discriminant between overlapping point sources and nebulousity fields
+                        ratio_cg2rw_bigsource = 3.0
+                        if rw_bs > 0.0:
+                            ratio_cg2rw_bigsource = g_bs / rw_bs
 
                         # Last chance - The larger "deblend" limits were used in this last detection
                         # attempt based upon the the statistics of processing lots of data - looking
@@ -1923,7 +1929,7 @@ def identify_sources(self, **pars):
 
 # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 
-    def detect_and_eval_segments(self, imgarr, kernel, ncount, size_source_box, nsigma_above_bkg, background_img, background_rms, check_big_island_only=False, rw2d_biggest_source=0.015, rw2d_source_fraction=0.075):
+    def detect_and_eval_segments(self, imgarr, kernel, ncount, size_source_box, nsigma_above_bkg, background_img, background_rms, check_big_island_only=False, biggest_source=0.015, source_fraction=0.075):
 
             # Compute the threshold to use for source detection
             threshold = self.compute_threshold(nsigma_above_bkg, background_img, background_rms)
@@ -1962,8 +1968,8 @@ def detect_and_eval_segments(self, imgarr, kernel, ncount, size_source_box, nsig
                 is_big_crowded, big_island, source_fraction = self._evaluate_segmentation_image(segm_img,
                                                                                                 imgarr,
                                                                                                 big_island_only=check_big_island_only,
-                                                                                                max_biggest_source=rw2d_biggest_source,
-                                                                                                max_source_fraction=rw2d_source_fraction)
+                                                                                                max_biggest_source=biggest_source,
+                                                                                                max_source_fraction=source_fraction)
 
             return segm_img, is_big_crowded, big_island, source_fraction
 
@@ -2053,10 +2059,12 @@ def detect_segments(self, imgarr, threshold, ncount, filter_kernel=None, source_
             img_bkg_sub = np.clip(img_bkg_sub, 0, img_bkg_sub.max())
             # Now set threshold to 0, since it has already been applied to the input data array
             thresh0 = np.zeros_like(img_bkg_sub)
+            log.info("Setting threshold to zero as data has been background subtracted.")
         else:
             # Use inputs provided with the kernel
             img_bkg_sub = imgarr
             thresh0 = threshold
+            log.info("Using computed threshold which includes [bkg + nsigma * rms].")
 
         # Note: SExtractor has "connectivity=8" which is the default for detect_sources().
         segm_img = None
@@ -2689,8 +2697,7 @@ def _evaluate_segmentation_image(self, segm_img, image_data, big_island_only=Fal
         max_biggest_source : float, optional
             Maximum limit on the single largest detected "source".
 
-        max_source_fraction : float, optional
-            Maximum limit on the fraction of pixels identified as part of a "source".
+        max_source_fraction : float, optional Maximum limit on the fraction of pixels identified as part of a "source".
 
         Returns
         -------
@@ -2728,7 +2735,6 @@ def _evaluate_segmentation_image(self, segm_img, image_data, big_island_only=Fal
         n, binedges = np.histogram(segm_img.data, range=(1, nbins))
         real_pixels = (image_data != 0).sum()
         biggest_source = n.max()/float(real_pixels)
-        log.info("Biggest_source: %f", biggest_source)
 
         # Compute which segments are larger than the kernel.
         deb_limit = self.kernel.size
@@ -2745,8 +2751,10 @@ def _evaluate_segmentation_image(self, segm_img, image_data, big_island_only=Fal
         # and is_poor_quality should be set to True.  The is_poor_quality is only an indicator that
         # a different kernel type or background computation could be tried for improved results.
         if biggest_source > max_biggest_source:
-            log.info("Biggest source %.4f percent exceeds %f percent of the image", (100.0*biggest_source), (100.0*max_biggest_source))
+            log.info("Biggest source %.4f percent exceeds %f percent of the image.", (100.0*biggest_source), (100.0*max_biggest_source))
             is_poor_quality = True
+        else:
+            log.info("Biggest source %.4f percent is WITHIN THE ACCEPTABLE limit of %f percent.", (100.0*biggest_source), (100.0*max_biggest_source))
 
         # Filter the big_segments array to remove the prohibitively large segments
         if big_segments.size > 0:
@@ -2793,11 +2801,14 @@ def _evaluate_segmentation_image(self, segm_img, image_data, big_island_only=Fal
         # big_island_only parameter allows control over whether the source_fraction should
         # or should not be ignored.
         source_fraction = n.sum()/float(real_pixels)
-        log.info("Source_fraction: %f", source_fraction)
         if not big_island_only:
             if source_fraction > max_source_fraction:
                 log.info("Total source fraction %.4f percent exceeds %f percent of the image.", (100.0*source_fraction), (100.0*max_source_fraction))
                 is_poor_quality = True
+            else:
+                log.info("Total source fraction %.4f percent is WITHIN THE ACCEPTABLE limit of %f percent.", (100.0*source_fraction), (100.0*max_source_fraction))
+        else:
+                log.info("Total source fraction %.4f percent is computed, but not set to be considered as a rejection criterion.", (100.0*source_fraction))
 
         return is_poor_quality, biggest_source, source_fraction
 

drizzlepac/pars/hap_pars/svm_parameters/acs/hrc/acs_hrc_catalog_generation_all.json

@@ -37,9 +37,10 @@
         "border": 10,
         "rw2d_size": 15,
         "rw2d_nsigma": 10.0,
-        "rw2d_biggest_source": 0.045,
-        "rw2d_source_fraction": 0.15,
+        "biggest_source_limit": 0.045,
+        "biggest_fraction_limit": 0.15,
         "biggest_source_deblend_limit": 0.15,
+        "rw_biggest_source_limit": 0.05,
         "source_fraction_deblend_limit": 0.20,
         "ratio_bigsource_limit": 2,
         "ratio_bigsource_deblend_limit": 10,

drizzlepac/pars/hap_pars/svm_parameters/acs/sbc/acs_sbc_catalog_generation_all.json

@@ -37,8 +37,9 @@
         "border": 10,
         "rw2d_size": 23,
         "rw2d_nsigma": 10.0,
-        "rw2d_biggest_source": 0.045,
-        "rw2d_source_fraction": 0.15,
+        "biggest_source_limit": 0.045,
+        "biggest_fraction_limit": 0.15,
+        "rw_biggest_source_limit": 0.05,
         "biggest_source_deblend_limit": 0.15,
         "source_fraction_deblend_limit": 0.20,
         "ratio_bigsource_limit": 2,

drizzlepac/pars/hap_pars/svm_parameters/acs/wfc/acs_wfc_catalog_generation_all.json

@@ -17,7 +17,7 @@
     "flag_trim_value": 5,
     "simple_bkg": false,
     "zero_percent": 25.0,
-    "negative_percent": 15.0,
+    "negative_percent": 20.0,
     "nsigma_clip": 3.0,
     "maxiters": 3,
     "bkg_skew_threshold": 0.5,
@@ -36,9 +36,10 @@
         "contrast": 0.001,
         "border": 10,
         "rw2d_size": 15,
-        "rw2d_nsigma": 10.0,
-        "rw2d_biggest_source": 0.015,
-        "rw2d_source_fraction": 0.075,
+        "rw2d_nsigma": 40.0,
+        "biggest_source_limit": 0.015,
+        "biggest_fraction_limit": 0.075,
+        "rw_biggest_source_limit": 0.05,
         "biggest_source_deblend_limit": 0.15,
         "source_fraction_deblend_limit": 0.20,
         "ratio_bigsource_limit": 2,

drizzlepac/pars/hap_pars/svm_parameters/wfc3/ir/wfc3_ir_catalog_generation_all.json

@@ -37,8 +37,9 @@
         "border": 10,
         "rw2d_size": 11,
         "rw2d_nsigma": 10.0,
-        "rw2d_biggest_source": 0.045,
-        "rw2d_source_fraction": 0.15,
+        "biggest_source_limit": 0.045,
+        "biggest_fraction_limit": 0.15,
+        "rw_biggest_source_limit": 0.05,
         "biggest_source_deblend_limit": 0.15,
         "source_fraction_deblend_limit": 0.20,
         "ratio_bigsource_limit": 2,

drizzlepac/pars/hap_pars/svm_parameters/wfc3/uvis/wfc3_uvis_catalog_generation_all.json

@@ -37,8 +37,9 @@
         "border": 10,
         "rw2d_size": 15,
         "rw2d_nsigma": 10.0,
-        "rw2d_biggest_source": 0.015,
-        "rw2d_source_fraction": 0.075,
+        "biggest_source_limit": 0.015,
+        "biggest_fraction_limit": 0.075,
+        "rw_biggest_source_limit": 0.05,
         "biggest_source_deblend_limit": 0.15,
         "source_fraction_deblend_limit": 0.20,
         "ratio_bigsource_limit": 2,

drizzlepac/pars/hap_pars/svm_parameters/wfpc2/wfpc2/wfpc2_wfpc2_catalog_generation_all.json

@@ -37,8 +37,9 @@
         "border": 10,
         "rw2d_size": 11,
         "rw2d_nsigma": 10.0,
-        "rw2d_biggest_source": 0.045,
-        "rw2d_source_fraction": 0.15,
+        "biggest_source_limit": 0.045,
+        "biggest_fraction_limit": 0.15,
+        "rw_biggest_source_limit": 0.05,
         "biggest_source_deblend_limit": 0.15,
         "source_fraction_deblend_limit": 0.20,
         "ratio_bigsource_limit": 2,