Optimize mask, flat, and dark loading with caching

RMS/DetectStarsAndMeteors.py

@@ -32,7 +32,7 @@
 from RMS.Formats.FrameInterface import detectInputType
 from RMS.ExtractStars import extractStarsFF
 from RMS.Detection import detectMeteors
-from RMS.DetectionTools import loadImageCalibration
+from RMS.ImageCalibrationLoader import getImageCalibrationLoader
 from RMS.QueuedPool import QueuedPool
 from RMS.Misc import RmsDateTime
 
@@ -78,9 +78,11 @@ def detectStarsAndMeteors(ff_directory, ff_name, config, flat_struct=None, dark=
 
 
     # Load mask, dark, flat
-    mask, dark, flat_struct = loadImageCalibration(ff_directory, config, dtype=img_handle.ff.dtype, \
-        byteswap=img_handle.byteswap)
+    loader = getImageCalibrationLoader()
+    calibration_data = loader.loadImageCalibration(ff_directory, config, dtype=img_handle.ff.dtype,
+                                                          byteswap=img_handle.byteswap)
 
+    (mask, dark, flat_struct), _ = calibration_data
 
     # Run star extraction on FF files
     star_list = extractStarsFF(ff_directory, ff_name, config=config, 
@@ -96,7 +98,7 @@ def detectStarsAndMeteors(ff_directory, ff_name, config, flat_struct=None, dark=
         log.debug('At least ' + str(config.ff_min_stars) + ' stars, detecting meteors...')
 
         # Run the detection
-        meteor_list = detectMeteors(img_handle, config, flat_struct=flat_struct, dark=dark, mask=mask)
+        meteor_list = detectMeteors(img_handle, config, calibration_data=calibration_data)
 
         log.info(ff_name + ' detected meteors: ' + str(len(meteor_list)))
 

RMS/Detection.py

@@ -36,7 +36,8 @@
 # RMS imports
 from RMS.Astrometry.Conversions import jd2Date, raDec2AltAz
 import RMS.ConfigReader as cr
-from RMS.DetectionTools import getThresholdedStripe3DPoints, loadImageCalibration, binImageCalibration
+from RMS.DetectionTools import getThresholdedStripe3DPoints
+from RMS.ImageCalibrationLoader import getImageCalibrationLoader
 from RMS.Formats.AsgardEv import writeEv
 from RMS.Formats.AST import xyToRaDecAST
 from RMS.Formats import FFfile
@@ -1050,7 +1051,7 @@ def thresholdAndCorrectGammaFF(img_handle, config, mask):
 
 
 
-def detectMeteors(img_handle, config, flat_struct=None, dark=None, mask=None, asgard=False, debug=False):
+def detectMeteors(img_handle, config, calibration_data=None, asgard=False, debug=False):
     """ Detect meteors on the given image. Here are the steps in the detection:
             - input image (FF bin format file) is thresholded (converted to black and white)
             - several morphological operations are applied to clean the image
@@ -1067,9 +1068,7 @@ def detectMeteors(img_handle, config, flat_struct=None, dark=None, mask=None, as
         config: [config object] configuration object (loaded from the .config file)
 
     Keyword arguments:
-        flat_struct: [Flat struct] Structure containing the flat field. None by default.
-        dark: [ndarray] Dark frame. None by default.
-        mask: [ndarray] Mask image. None by default.
+        calibration_data: Tuple of (original, binned) calibration data, where each is (mask, dark, flat_struct)
         asgard: [bool] If True, the vid file sequence number will be added in with the frame. False by 
             default, in which case only the frame number will be in the centroids.
         debug: [bool] If True, graphs for testing the detection settings will be shown. False by default.
@@ -1085,12 +1084,17 @@ def detectMeteors(img_handle, config, flat_struct=None, dark=None, mask=None, as
     t1 = time()
     t_all = time()
 
+    mask, dark, flat_struct = None, None, None
+
+    if calibration_data is not None:
+        # Use the binned mask, dark and flat, only when not running on FF files
+        if (img_handle.input_type != 'ff') and (config.detection_binning_factor > 1):
 
-    # Bin the mask, dark and flat, only when not running on FF files
-    if (img_handle.input_type != 'ff') and (config.detection_binning_factor > 1):
-
-        # Bin the calibration images
-        mask, dark, flat_struct = binImageCalibration(config, mask, dark, flat_struct)
+            # Use binned calibration images
+            _, (mask, dark, flat_struct) = calibration_data
+        else:
+            # Use original calibration images
+            (mask, dark, flat_struct), _ = calibration_data
 
 
     # Do all image processing on single FF file, if given
@@ -1747,9 +1751,9 @@ def detectMeteors(img_handle, config, flat_struct=None, dark=None, mask=None, as
 
 
     # Load mask, dark, flat
-    mask, dark, flat_struct = loadImageCalibration(main_dir, config, dtype=img_handle_main.ff.dtype, \
-        byteswap=img_handle_main.byteswap)
-
+    loader = getImageCalibrationLoader()
+    calibration_data = loader.loadImageCalibration(main_dir, config, dtype=img_handle_main.ff.dtype,
+                                                          byteswap=img_handle_main.byteswap)
 
     # Init results list
     results_list = []
@@ -1770,7 +1774,7 @@ def detectMeteors(img_handle, config, flat_struct=None, dark=None, mask=None, as
         logDebug(img_handle.name())
 
         # Run the meteor detection algorithm
-        meteor_detections = detectMeteors(img_handle, config, flat_struct=flat_struct, dark=dark, mask=mask, \
+        meteor_detections = detectMeteors(img_handle, config, calibration_data=calibration_data, \
             debug=cml_args.debugplots, asgard=(cml_args.asgard is not None))
 
         # Suppress numpy scientific notation printing

RMS/DetectionTools.py

@@ -25,129 +25,6 @@
 log = logging.getLogger("logger")
 
 
-def loadImageCalibration(dir_path, config, dtype=None, byteswap=False):
-    """ Load the mask, dark and flat. 
-
-    Arguments:
-        dir_path: [str] Path to the directory with calibration.
-        config: [ConfigStruct]
-
-    Keyword arguments:
-        dtype: [object] Numpy array dtype for the image. None by default, if which case it will be determined
-            from the input image.
-        byteswap: [bool] If the dark and flat should be byteswapped. False by default, and should be True for
-            UWO PNGs.
-
-    Return:
-        mask, dark, flat_struct: [tuple of ndarrays]
-    """
-
-    mask_path = None
-    mask = None
-
-    # Try loading the mask from CaptureFiles directory
-    if os.path.exists(os.path.join(dir_path, config.mask_file)):
-        mask_path = os.path.join(dir_path, config.mask_file)
-
-    # Try loading the default mask
-    elif os.path.exists(os.path.join(config.config_file_path, config.mask_file)):
-        mask_path = os.path.join(config.config_file_path, config.mask_file)
-
-    # Load the mask if given
-    if mask_path:
-
-        mask = MaskImage.loadMask(mask_path)
-
-        # If the mask is all white, set it to None
-        if (mask is not None) and np.all(mask.img == 255):
-            print('Mask is all white, setting it to None.')
-            mask = None
-
-    if mask is not None:
-        print('Loaded mask:', mask_path)
-        log.info('Loaded mask: {:s}'.format(mask_path))
-
-    else:
-        log.info('No mask file has been found.')
-
-
-
-
-    # Try loading the dark frame
-    dark = None
-    if config.use_dark:
-
-        dark_path = None
-
-        # Check if dark is in the data directory
-        if os.path.exists(os.path.join(dir_path, config.dark_file)):
-            dark_path = os.path.join(dir_path, config.dark_file)
-
-        # Try loading the default dark
-        elif os.path.exists(config.dark_file):
-            dark_path = os.path.abspath(config.dark_file)
-
-        if dark_path is not None:
-
-            # Load the dark
-            dark = Image.loadDark(*os.path.split(dark_path), dtype=dtype, byteswap=byteswap)
-
-        if dark is not None:
-            print('Loaded dark:', dark_path)
-            log.info('Loaded dark: {:s}'.format(dark_path))
-
-
-
-    # Try loading a flat field image
-    flat_struct = None
-    if config.use_flat:
-
-        flat_path = None
-
-        # Check if there is flat in the data directory
-        if os.path.exists(os.path.join(dir_path, config.flat_file)):
-            flat_path = os.path.join(dir_path, config.flat_file)
-
-        # Try loading the default flat
-        elif os.path.exists(config.flat_file):
-            flat_path = os.path.abspath(config.flat_file)
-
-        if flat_path is not None:
-
-            # Load the flat
-            flat_struct = Image.loadFlat(*os.path.split(flat_path), dtype=dtype, byteswap=byteswap)
-
-
-        if flat_struct is not None:
-            print('Loaded flat:', flat_path)
-            log.info('Loaded flat: {:s}'.format(flat_path))
-
-
-
-    return mask, dark, flat_struct
-
-
-
-def binImageCalibration(config, mask, dark, flat_struct):
-    """ Bin the calibration images. """
-
-    # Bin the mask
-    if mask is not None:
-        mask.img = Image.binImage(mask.img, config.detection_binning_factor, 'avg')
-
-    # Bin the dark
-    if dark is not None:
-        dark = Image.binImage(dark, config.detection_binning_factor, 'avg')
-
-    # Bin the flat
-    if flat_struct is not None:
-        flat_struct.binFlat(config.detection_binning_factor, 'avg')
-
-
-    return mask, dark, flat_struct
-
-
-
 def htLinePerpendicular(rho, theta, x_inters, y_inters, img_h, img_w):
     """ Compute a parpendicular line to the one given in Hough polar coordinates. The new line will intersect
         the given line in point (x_inters, y_inters).

RMS/ExtractStars.py

@@ -34,7 +34,7 @@
 import RMS.ConfigReader as cr
 from RMS.Formats import FFfile
 from RMS.Formats import CALSTARS
-from RMS.DetectionTools import loadImageCalibration
+from RMS.ImageCalibrationLoader import getImageCalibrationLoader
 from RMS.Routines import MaskImage
 from RMS.Routines import Image
 from RMS.QueuedPool import QueuedPool
@@ -606,7 +606,8 @@ def extractStarsAndSave(config, ff_dir):
     time_start = time.time()
 
     # Load mask, dark, flat
-    mask, dark, flat_struct = loadImageCalibration(ff_dir, config)
+    loader = getImageCalibrationLoader()
+    (mask, dark, flat_struct), _ = loader.loadImageCalibration(ff_dir, config)
 
 
     extraction_list = []