Merge branch 'exif-capture-reload' into 'master'

Load capture objects from files. Remove YAML database.

See merge request openflexure/openflexure-microscope-server!17

openflexure_microscope/api/app.py

@@ -17,6 +17,8 @@ from openflexure_microscope.camera.pi import StreamingCamera
 from openflexure_microscope.stage.sanga import SangaStage
 from openflexure_microscope.stage.mock import MockStage
 
+from openflexure_microscope.camera.capture import build_captures_from_exif
+
 from openflexure_microscope.config import USER_CONFIG_DIR
 
 import atexit
@@ -58,6 +60,10 @@ else:
 # Create a dummy microscope object, with no hardware attachments
 api_microscope = Microscope(None, None)
 
+# Rebuild the capture list
+# TODO: Offload to a thread?
+stored_image_list = build_captures_from_exif()
+
 # Generate API URI based on version from filename
 def uri(suffix, api_version, base=None):
     if not base:
@@ -81,7 +87,7 @@ handler = JSONExceptionHandler(app)
 @app.before_first_request
 def attach_microscope():
     # Create the microscope object globally (common to all spawned server threads)
-    global api_microscope
+    global api_microscope, capture_list
     logging.debug("First request made. Populating microscope with hardware...")
 
     logging.debug("Creating camera object...")
@@ -101,6 +107,10 @@ def attach_microscope():
             api_stage
         )
 
+        logging.debug("Restoring captures...")
+        if stored_image_list:
+            api_microscope.camera.images = stored_image_list
+
         logging.debug("Microscope successfully attached!")
 
 
@@ -153,10 +163,6 @@ def cleanup():
     if api_microscope.rc:
         api_microscope.rc.save(backup=True)
 
-    # Update capture DB and metadata files before exiting
-    if api_microscope.camera:
-        api_microscope.camera.store_captures()
-
     # Close down the microscope
     api_microscope.close()
     logging.debug("App teardown complete.")

openflexure_microscope/api/v1/blueprints/task.py

@@ -44,8 +44,6 @@ class TaskListAPI(MicroscopeView):
         :>header Content-Type: application/json
         """
 
-        print(self.microscope.task.state)
-
         data = self.microscope.task.state
 
         return jsonify(data)

openflexure_microscope/camera/base.py

@@ -14,7 +14,7 @@ except ImportError:
     except ImportError:
         from _thread import get_ident
 
-from .capture import CaptureObject, capture_from_dict, BASE_CAPTURE_PATH, TEMP_CAPTURE_PATH
+from .capture import CaptureObject, BASE_CAPTURE_PATH, TEMP_CAPTURE_PATH
 from openflexure_microscope.config import USER_CONFIG_DIR
 from openflexure_microscope.utilities import entry_by_id
 from openflexure_microscope.lock import StrictLock
@@ -119,14 +119,6 @@ class BaseCamera(object):
         self.images = []  #: list: List of image capture objects
         self.videos = []  #: list: List of video recording objects
 
-        # Capture data files
-        self.images_db = os.path.join(USER_CONFIG_DIR, "images_db.yaml")
-        self.videos_db = os.path.join(USER_CONFIG_DIR, "videos_db.yaml")
-
-        # Load captures from persistent db
-        self.images = self.load_capture_db(self.images_db)
-        self.videos = self.load_capture_db(self.videos_db)
-
     def apply_config(self, config):
         """Update settings from a config dictionary"""
         self.config.update(config)
@@ -244,50 +236,6 @@ class BaseCamera(object):
         """Return a video StreamObject with a matching ID."""
         return entry_by_id(video_id, self.videos)
 
-    # MANAGE CAPTURE DATABASE
-
-    def save_capture_db(self, capture_list, db_path):
-        capture_state = [capture.state for capture in capture_list]
-
-        with open(db_path, 'w') as outfile:
-            yaml.safe_dump(capture_state, outfile)
-
-    def load_capture_db(self, db_path):
-        if os.path.isfile(db_path):
-            # Load list of capture dictionary representations from db_path
-            with open(db_path, 'r') as infile:
-                try:
-                    capture_dict_list = yaml.load(infile)
-                except yaml.scanner.ScannerError as e:
-                    logging.error(e)
-                    logging.warning('Emptying malformed database. Your captures will not be deleted, but may not appear in the gallery.')
-                    capture_dict_list = []
-
-            # Create capture object list, and validate captures
-            capture_list = []
-
-            if capture_dict_list:
-                for capture_dict in capture_dict_list:
-                    if 'path' in capture_dict and os.path.isfile(capture_dict['path']):
-                        capture_list.append(capture_from_dict(capture_dict))
-
-            return capture_list
-
-        else:
-            return []
-
-    def store_captures(self):
-        # Save metadata files
-        #for image in self.images:
-        #    image.save_metadata()
-        #for video in self.videos:
-        #    video.save_metadata()
-
-        # Update capture database
-        self.save_capture_db(self.images, self.images_db)
-        self.save_capture_db(self.videos, self.videos_db)
-
-
     # CREATING NEW CAPTURES
 
     def shunt_captures(self, target_list: list):
@@ -335,9 +283,6 @@ class BaseCamera(object):
         self.shunt_captures(self.images)
         self.images.append(output)
 
-        # Update capture database
-        self.save_capture_db(self.images, self.images_db)
-
         return output
 
     def new_video(
@@ -381,9 +326,6 @@ class BaseCamera(object):
         self.shunt_captures(self.videos)
         self.videos.append(output)
 
-        # Update capture database
-        self.save_capture_db(self.videos, self.videos_db)
-
         return output
 
     # WORKER THREAD

openflexure_microscope/camera/capture.py

@@ -18,6 +18,7 @@ THUMBNAIL_SIZE = (200, 150)
 BASE_CAPTURE_PATH = os.path.join(os.path.expanduser('~'), 'micrographs')  #: str: Base path to store all captures
 TEMP_CAPTURE_PATH = os.path.join(BASE_CAPTURE_PATH, 'tmp')  #: str: Base path to store all temporary captures (automatically emptied)
 
+
 # TODO: Move these methods to a camera utilities module?
 def clear_tmp():
     """
@@ -38,67 +39,66 @@ def pull_usercomment_dict(filepath):
         filepath: Path to the Exif-containing file
     """
     exif_dict = piexif.load(filepath)
-
     if 'Exif' in exif_dict and 37510 in exif_dict['Exif']:
         return yaml.load(exif_dict['Exif'][37510].decode())
     else:
         return {}
 
 
-def extract_with_priority(key, best_dict, backup_dict):
+def make_file_list(directory, formats):
-    """
+    files = []
-    Extracts a value from one of two dictionaries, prioritising one over the other.
+    for fmt in formats:
-    Second dictionary is used only if the key doesn't exist in the first.
+        files.extend(glob.glob('{}/**/*.{}'.format(directory, fmt.lower()), recursive=True))
 
-    Args:
+    logging.info("{} capture files found on disk".format(len(files)))
-        key: Key to search for
-        best_dict: Ideal dictionary to use
-        backup_dict: Fallback dictionary
 
-    """
+    return files
-    if key in best_dict:
-        return best_dict[key]
-    elif key in backup_dict:
-        logging.warning("Key {} not found in primary dictionary. Falling back to backup.".format(key))
-        return backup_dict[key]
-    else:
-        logging.error("Key {} not found in either dictionary!".format(key))
-        return None
 
 
-def capture_from_dict(capture_dict):
+def build_captures_from_exif():
+    global BASE_CAPTURE_PATH, EXIF_FORMATS
+
+    logging.debug("Reloading captures from {}...".format(BASE_CAPTURE_PATH))
+    files = make_file_list(BASE_CAPTURE_PATH, EXIF_FORMATS)
+    captures = []
+
+    for f in files:
+        logging.debug("Reloading capture {}...".format(f))
+        exif = pull_usercomment_dict(f)
+        capture = capture_from_exif(f, exif)
+        captures.append(capture)
+
+    logging.info("{} capture files successfully reloaded".format(len(captures)))
+
+    return captures
+
+
+def capture_from_exif(path, exif_dict):
     """
     Creates an instance of CaptureObject from a dictionary of capture information.
     This is used when reloading the API server, to restore captures created in the 
     previous session.
 
     Args:
-        capture_dict (dict): Dictionary containing capture information
+        exif_dict (dict): Dictionary containing capture information
     """
-    global EXIF_FORMATS
 
+    # Create a placeholder capture
     capture = CaptureObject(
-        filepath=capture_dict['path']
+        filepath=path
-    )  # Create a placeholder capture
+    ) 
+
+    # Build file path information
     capture.split_file_path(capture.file)
 
-    capture.temporary = capture_dict['temporary']
-
-    if capture.format.upper() in EXIF_FORMATS:
-        md_exif = pull_usercomment_dict(capture.file)
-    else:
-        logging.debug("Unsupported format for EXIF data. Skipping.")
-        md_exif = {}
-
-    md_database = capture_dict['metadata']
-
     # Populate capture parameters
-    capture.id = extract_with_priority('id', md_exif, md_database)
+    capture.id = exif_dict['id']
-    capture.timestring = extract_with_priority('time', md_exif, md_database)
-    capture.format = extract_with_priority('format', md_exif, md_database)
     
-    capture._metadata = extract_with_priority('custom', md_exif, md_database)
+    capture.timestring = exif_dict['time']
-    capture.tags = extract_with_priority('tags', md_exif, md_database)
+    capture.format = exif_dict['format']
+
+    capture._metadata = exif_dict['custom']
+    capture.tags = exif_dict['tags']
 
     return capture
 
@@ -119,7 +119,7 @@ class CaptureObject(object):
 
         # Store a nice ID
         self.id = uuid.uuid4().hex  #: str: Unique capture ID
-        logging.info("Created StreamObject {}".format(self.id))
+        logging.debug("Created StreamObject {}".format(self.id))
         self.timestring = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")  #: str: Timestring of capture creation time
 
         # Keep on disk after close by default

openflexure_microscope/camera/set_picamera_gain.py

@@ -4,7 +4,7 @@ import picamera
 from picamera import mmal, mmalobj, exc
 from picamera.mmalobj import to_rational
 import time
-
+import logging
 
 MMAL_PARAMETER_ANALOG_GAIN = mmal.MMAL_PARAMETER_GROUP_CAMERA + 0x59
 MMAL_PARAMETER_DIGITAL_GAIN = mmal.MMAL_PARAMETER_GROUP_CAMERA + 0x5A
@@ -51,11 +51,11 @@ if __name__ == "__main__":
         # fix the shutter speed
         cam.shutter_speed = cam.exposure_speed
 
-        print("Current a/d gains: {}, {}".format(cam.analog_gain, cam.digital_gain))
+        logging.info("Current a/d gains: {}, {}".format(cam.analog_gain, cam.digital_gain))
 
-        print("Attempting to set analogue gain to 1")
+        logging.info("Attempting to set analogue gain to 1")
         set_analog_gain(cam, 1)
-        print("Attempting to set digital gain to 1")
+        logging.info("Attempting to set digital gain to 1")
         set_digital_gain(cam, 1)
         # The old code is left in here in case it is a useful example...
         #ret = mmal.mmal_port_parameter_set_rational(cam._camera.control._port, 
@@ -65,9 +65,9 @@ if __name__ == "__main__":
 
         try:
             while True:
-                print("Current a/d gains: {}, {}".format(cam.analog_gain, cam.digital_gain))
+                logging.info("Current a/d gains: {}, {}".format(cam.analog_gain, cam.digital_gain))
                 time.sleep(1)
         except KeyboardInterrupt:
-            print("Stopping...")
+            logging.info("Stopping...")
 
 

openflexure_microscope/plugins/default/autofocus/api.py

@@ -1,4 +1,5 @@
 import numpy as np
+import logging
 
 from openflexure_microscope.api.v1.views import MicroscopeViewPlugin
 from openflexure_microscope.api.utilities import JsonPayload
@@ -19,7 +20,7 @@ class AutofocusAPI(MicroscopeViewPlugin):
         # Figure out the range of z values to use
         dz = payload.param("dz", default=np.linspace(-300, 300, 7), convert=np.array)
 
-        print("Running autofocus...")
+        logging.info("Running autofocus...")
         task = self.microscope.task.start(self.plugin.autofocus, dz)
 
         # return a handle on the autofocus task
@@ -35,7 +36,7 @@ class FastAutofocusAPI(MicroscopeViewPlugin):
         if backlash < 0:
             backlash = None
 
-        print("Running autofocus...")
+        logging.info("Running autofocus...")
         task = self.microscope.task.start(self.plugin.fast_autofocus, dz, backlash=backlash)
 
         # return a handle on the autofocus task

openflexure_microscope/plugins/default/camera_calibration/plugin.py

@@ -14,7 +14,7 @@ class RecalibrateAPIView(MicroscopeViewPlugin):
         
         # TODO: Figure out the range of z values to use
 
-        print("Starting microscope recalibration...")
+        logging.info("Starting microscope recalibration...")
         task = self.microscope.task.start(self.plugin.recalibrate)
 
         # Return a handle on the autofocus task

openflexure_microscope/plugins/default/scan/api.py

@@ -2,7 +2,7 @@ from openflexure_microscope.api.v1.views import MicroscopeViewPlugin
 from openflexure_microscope.api.utilities import JsonPayload
 
 from flask import request, jsonify, abort
-
+import logging
 
 class TileScanAPI(MicroscopeViewPlugin):
     def post(self):
@@ -34,7 +34,7 @@ class TileScanAPI(MicroscopeViewPlugin):
         metadata = payload.param('metadata', default={}, convert=dict)
         tags = payload.param('tags', default=[], convert=list)
 
-        print("Running tile scan...")
+        logging.info("Running tile scan...")
         task = self.microscope.task.start(
             self.plugin.tile,
             basename=filename,

openflexure_microscope/plugins/loader.py

@@ -67,8 +67,6 @@ def check_module(module_path):
 def name_from_module(plugin_path):
     path_array = plugin_path.split('.')
 
-    print(path_array)
-
     # Truncate namespace of default plugins
     if path_array[0:2] == ['openflexure_microscope', 'plugins']:
         path_array = path_array[2:]
@@ -129,7 +127,7 @@ class PluginMount(object):
     def __init__(self, parent):
         self.parent = parent
         self.plugins = []
-        print("Creating plugin mount")
+        logging.info("Creating plugin mount")
 
     @property
     def state(self):