add option for prefix_chars in silcam and holo load classes

pyopia/__init__.py

@@ -1 +1 @@
-__version__ = "2.16.5"
+__version__ = "2.16.6"

pyopia/instrument/holo.py

@@ -20,9 +20,9 @@
 from skimage.filters import sobel
 from skimage.morphology import disk, erosion, dilation
 import pyopia.process
+from pyopia.pipeline import FilesToProcess
 import struct
-from datetime import timedelta, datetime
-from glob import glob
+from datetime import timedelta
 
 import logging
 logger = logging.getLogger()
@@ -64,15 +64,15 @@ def __init__(self, wavelength, n, offset, minZ, maxZ, stepZ):
         self.stepZ = stepZ
 
     def __call__(self, data):
-        logger.info('Using first raw file from list in general settings to determine image dimensions')
-        raw_files = glob(data['settings']['general']['raw_files'])
-        self.filename = raw_files[0]
+        logger.debug('Using first raw file from list in general settings to determine image dimensions')
+        raw_files = FilesToProcess(data['settings']['general']['raw_files'])
+        self.filename = raw_files.files[0]
         imtmp = load_image(self.filename)
         self.pixel_size = data['settings']['general']['pixel_size']
-        logger.info('Build kernel with pixel_size = ', self.pixel_size, 'um')
+        logger.info(f'Build kernel with pixel_size = {self.pixel_size} um')
         kern = create_kernel(imtmp, self.pixel_size, self.wavelength, self.n, self.offset, self.minZ, self.maxZ, self.stepZ)
         im_stack = np.zeros(np.shape(kern)).astype(np.float64)
-        logger.info('HoloInitial done', datetime.now())
+        logger.info('HoloInitial done')
         data['kern'] = kern
         data['im_stack'] = im_stack
         return data
@@ -103,24 +103,33 @@ class Load():
     filename : string
         hologram filename (.pgm)
 
+    prefix_chars : int, optional
+        number of characters to ignore at start of filename when parsing timestamp,
+        by default 1 (e.g. to ignore 'D' in 'D20221101T120000.pgm')
+
+    datetime_format : string, optional
+        Format string for parsing the timestamp from the filename,
+        by default None (automatic parsing with pandas.to_datetime)
+
     Returns
     -------
-    timestamp : timestamp
-        timestamp @todo
-    imraw : np.arraym
+    timestamp : pandas.Timestamp
+        timestamp from filename
+    imraw : np.array
         hologram
     '''
 
-    def __init__(self):
-        pass
+    def __init__(self, prefix_chars=1, datetime_format=None):
+        self.prefix_chars = prefix_chars
+        self.datetime_format = datetime_format
 
     def __call__(self, data):
         logger.info(data['filename'])
         try:
             timestamp = read_lisst_holo_info(data['filename'])
         except ValueError:
-            timestamp = pd.to_datetime(os.path.splitext(os.path.basename(data['filename']))[0][1:])
-        logger.info(timestamp)
+            timestamp = pd.to_datetime(os.path.splitext(os.path.basename(data['filename']))[0][self.prefix_chars:],
+                                       format=self.datetime_format)
         im = load_image(data['filename'])
         data['timestamp'] = timestamp
         data['imraw'] = im
@@ -559,6 +568,16 @@ def __call__(self, data):
         stack_rp = data['stack_rp']
         stack_ifocus = data['stack_ifocus']
 
+        # If there are no particles in the image, then stats has one row of nans
+        # This means that variables created here just need allocating and returning
+        if len(stack_ifocus) == 0:
+            logger.debug(f"stack_ifocus is empty. Allocating nan to ifocus and z. Filename: {data['filename']}")
+            stats['ifocus'] = np.nan
+            stats['z'] = np.nan
+            stats['holo_filename'] = data['filename']
+            data['stats'] = stats
+            return data
+
         bbox = np.empty((0, 4), int)
         for rp in stack_rp:
             bbox = np.append(bbox, [rp.bbox], axis=0)
@@ -603,7 +622,7 @@ def read_lisst_holo_info(filename):
     filenum = int(filenum.rsplit('.', 1)[0])
     timestamp = timestamp + timedelta(microseconds=filenum)
     timestamp = timestamp[0]
-    logger.info(timestamp.strftime('D%Y%m%dT%H%M%S.%f'))
+    logger.debug(timestamp.strftime('D%Y%m%dT%H%M%S.%f'))
     f.close()
 
     return timestamp

pyopia/instrument/silcam.py

@@ -14,21 +14,31 @@
 import skimage.io
 
 
-def timestamp_from_filename(filename):
+def timestamp_from_filename(filename, prefix_chars=1, datetime_format=None):
     '''get a pandas timestamp from a silcam filename
 
     Parameters
     ----------
-    filename (string): silcam filename (.silc)
+    filename : string
+        silcam filename (.silc)
+
+    prefix_chars : int, optional
+        number of characters to ignore at start of filename when parsing timestamp,
+        by default 1 (e.g. to ignore 'D' in 'D20221101T120000.silc')
+
+    datetime_format : string, optional
+        Format string for parsing the timestamp from the filename,
+        by default None (automatic parsing with pandas.to_datetime)
 
     Returns
     -------
-    timestamp: timestamp
+    timestamp : pandas.Timestamp
         timestamp from pandas.to_datetime()
     '''
 
     # get the timestamp of the image (in this case from the filename)
-    timestamp = pd.to_datetime(os.path.splitext(os.path.basename(filename))[0][1:])
+    timestamp = pd.to_datetime(os.path.splitext(os.path.basename(filename))[0][prefix_chars:],
+                               format=datetime_format)
     return timestamp
 
 
@@ -222,6 +232,14 @@ class SilCamLoad():
     image_format : str, optional
         Image file format. Can be either 'infer', 'rgb8', 'bayer_rg8' or 'mono8', by default 'infer'.
 
+    prefix_chars : int, optional
+        number of characters to ignore at start of filename when parsing timestamp,
+        by default 1 (e.g. to ignore 'D' in 'D20221101T120000.silc')
+
+    datetime_format : string, optional
+        Format string for parsing the timestamp from the filename,
+        by default None (automatic parsing with pandas.to_datetime)
+
     Note
     ----
         'infer' uses the file extension to determine the image format using the following convention:
@@ -240,7 +258,9 @@ class SilCamLoad():
         :attr:`pyopia.pipeline.Data.img`
     '''
 
-    def __init__(self, image_format='infer'):
+    def __init__(self, image_format='infer', prefix_chars=1, datetime_format=None):
+        self.prefix_chars = prefix_chars
+        self.datetime_format = datetime_format
         self.image_format = image_format
         self.extension_load = {'.silc': load_rgb8,
                                '.msilc': load_mono8,
@@ -250,7 +270,9 @@ def __init__(self, image_format='infer'):
                             'MONO8': load_mono8, 'BAYER_RG8': load_bayer_rgb8}
 
     def __call__(self, data):
-        data['timestamp'] = timestamp_from_filename(data['filename'])
+        data['timestamp'] = timestamp_from_filename(data['filename'],
+                                                    prefix_chars=self.prefix_chars,
+                                                    datetime_format=self.datetime_format)
         data['imraw'] = self.load_image(data['filename'])
         return data