Merge remote-tracking branch 'origin/master' into labthings-070

openflexure_microscope/api/app.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-from gevent import monkey
+from labthings.server import monkey
 
 monkey.patch_all()
 

openflexure_microscope/api/default_extensions/scan.py

@@ -83,6 +83,7 @@ def capture(
         annotations=annotations,
         tags=tags,
         metadata=metadata,
+        cache_key=folder
     )
 
 

openflexure_microscope/camera/mock.py

@@ -10,7 +10,6 @@ import time
 import numpy as np
 from PIL import Image, ImageFont, ImageDraw
 from datetime import datetime
-import gevent
 
 import logging
 
@@ -221,7 +220,7 @@ class MissingCamera(BaseCamera):
         # While the iterator is not closed
         try:
             while True:
-                gevent.sleep(1)  # Only serve frames at 1fps
+                time.sleep(1)  # Only serve frames at 1fps
                 # Reset stream
                 self.stream.seek(0)
                 self.stream.truncate()

openflexure_microscope/captures/capture.py

@@ -10,9 +10,6 @@ from PIL import Image
 import dateutil.parser
 import atexit
 
-from gevent.fileobject import FileObjectThread
-import gevent
-
 from collections import OrderedDict
 
 from openflexure_microscope.camera import piexif
@@ -129,10 +126,6 @@ class CaptureObject(object):
         """Create a new StreamObject, to manage capture data."""
         # Stream for buffering capture data
         self.stream = io.BytesIO()
-        # Event to notify when the stream has finished writing to disk
-        self.file_ready = gevent.event.Event()
-        # Access lock for the disk file
-        self.lock = gevent.lock.BoundedSemaphore()
 
         # Store a nice ID
         self.id = uuid.uuid4()  #: str: Unique capture ID
@@ -160,19 +153,12 @@ class CaptureObject(object):
     def write(self, s):
         self.stream.write(s)
 
-    def _stream_to_file(self):
-        with self.lock:
-            logging.info(f"Writing to disk {self.file}")
-            with FileObjectThread(open(self.file, "wb"), 'wb')  as outfile:
-                outfile.write(self.stream.getbuffer())
-            self.stream.close()
-            self.file_ready.set()
-            logging.info(f"Finished writing to disk {self.file}")
-
     def flush(self):
-        logging.debug(f"Flushing {self.file}")
-        gevent.spawn(self._stream_to_file)
-        logging.debug(f"Returning flushing {self.file}")
+        logging.info(f"Writing to disk {self.file}")
+        with open(self.file, "wb")  as outfile:
+            outfile.write(self.stream.getbuffer())
+        self.stream.close()
+        logging.info(f"Finished writing to disk {self.file}")
 
     def open(self, mode):
         return open(self.file, mode)
@@ -275,20 +261,18 @@ class CaptureObject(object):
         global EXIF_FORMATS
 
         if self.format.upper() in EXIF_FORMATS and self.exists:
-            self.file_ready.wait()
-            with self.lock:
-                logging.debug("Writing exif data to capture file")
-                # Extract current Exif data
-                exif_dict = piexif.load(self.file)
-                # Serialize metadata
-                metadata_string = json.dumps(self.metadata, cls=JSONEncoder)
-                logging.debug(f"Saving metadata string to file: {metadata_string}")
-                # Insert metadata into exif_dict
-                exif_dict["Exif"][piexif.ExifIFD.UserComment] = metadata_string.encode()
-                # Convert new exif dict to exif bytes
-                exif_bytes = piexif.dump(exif_dict)
-                # Insert exif into file
-                piexif.insert(exif_bytes, self.file)
+            logging.debug("Writing exif data to capture file")
+            # Extract current Exif data
+            exif_dict = piexif.load(self.file)
+            # Serialize metadata
+            metadata_string = json.dumps(self.metadata, cls=JSONEncoder)
+            logging.debug(f"Saving metadata string to file: {metadata_string}")
+            # Insert metadata into exif_dict
+            exif_dict["Exif"][piexif.ExifIFD.UserComment] = metadata_string.encode()
+            # Convert new exif dict to exif bytes
+            exif_bytes = piexif.dump(exif_dict)
+            # Insert exif into file
+            piexif.insert(exif_bytes, self.file)
             logging.info(f"Finished saving metadata to {self.file}")
 
     @property

openflexure_microscope/microscope.py

@@ -6,8 +6,7 @@ import logging
 import pkg_resources
 import uuid
 from typing import Tuple
-
-import gevent
+from expiringdict import ExpiringDict
 
 from openflexure_microscope.captures import CaptureManager
 
@@ -61,6 +60,10 @@ class Microscope:
         # Apply settings loaded from file
         self.update_settings(self.settings_file.load())
 
+        # Data cache
+        self.configuration_cache = ExpiringDict(max_len=100, max_age_seconds=3600)
+        self.metadata_cache = ExpiringDict(max_len=100, max_age_seconds=3600)
+
     def __enter__(self):
         """Create microscope on context enter."""
         return self
@@ -237,8 +240,7 @@ class Microscope:
         # Save config to file
         self.settings_file.save(current_config, backup=True)
 
-    @property
-    def configuration(self):
+    def force_get_configuration(self):
         with self.lock:
             initial_configuration = self.configuration_file.load()
 
@@ -262,37 +264,62 @@ class Microscope:
             initial_configuration.update(current_configuration)
             return initial_configuration
 
+    def get_configuration(self, cache_key=None):
+        if cache_key:
+            cached_config = self.configuration_cache.get(cache_key, None)
+            if cached_config:
+                return cached_config
+            else:
+                full_config = self.force_get_configuration()
+                self.configuration_cache[cache_key] = full_config
+                return full_config
+        return self.force_get_configuration()
+
     @property
-    def metadata(self):
+    def configuration(self):
+        return self.get_configuration()
+
+    def force_get_metadata(self):
         """
-        Microscope system metadata, to be applied to basically all captures
+        Read cachable bits of microscope metadata.
+        Currently ID, settings, and configuration can be cached
         """
-        with Timer("Reading settings:"):
-            settings = self.read_settings(full=False)
-        with Timer("Reading state:"):
-            state = self.state
-        with Timer("Reading configuration:"):
-            configuration = self.configuration
         system_metadata = {
             "id": self.id,
-            "settings": settings,
-            "state": state,
-            "configuration": configuration,
+            "settings": self.read_settings(full=False),
+            "configuration": self.get_configuration(),
         }
 
         return system_metadata
 
-    def add_metadata_to_capture(self, output, metadata, annotations, tags):
-        logging.debug(f"Waiting for {output.file}")
-        # Wait for the file to be written to disk
-        with Timer("Waiting for file:"):
-            output.file_ready.wait()
+    def get_metadata(self, cache_key=None):
+        """
+        Read microscope metadata, with partial caching
+        """
+        metadata = {}
+
+        # Load cached bits of metadata
+        if cache_key:
+            logging.debug(f"Reading cached microscope metadata: {cache_key}")
+            metadata = self.metadata_cache.get(cache_key, None)
+            if not metadata:
+                logging.debug(f"Building and caching microscope metadata: {cache_key}")
+                metadata = self.force_get_metadata()
+                self.metadata_cache[cache_key] = metadata
+        else:
+            logging.debug(f"Building microscope metadata: {cache_key}")
+            metadata = self.force_get_metadata()
         
-        with Timer("Building metadata"):
-            full_metadata = {"instrument": self.metadata, **metadata}
-        with Timer("Writing metadata to file"):
-            output.put_and_save(tags, annotations, full_metadata)
-        logging.info(f"Finished injecting EXIF data into {output.file}")
+        # Keys that should never be cached
+        metadata.update({
+            "state": self.state,
+        })
+
+        return metadata
+
+    @property
+    def metadata(self):
+        return self.get_metadata()
 
     def capture(
         self,
@@ -306,6 +333,7 @@ class Microscope:
         annotations: dict = None,
         tags: list = None,
         metadata: dict = None,
+        cache_key: str = None
     ):
         logging.debug(f"Microscope capturing to {filename}")
         if not annotations:
@@ -315,6 +343,10 @@ class Microscope:
         if not tags:
             tags = []
 
+        # Read metadata for capture
+        full_metadata = {"instrument": self.get_metadata(cache_key), **metadata}
+
+        # Do capture
         with self.camera.lock:
             # Create output object
             output = self.captures.new_image(
@@ -322,7 +354,7 @@ class Microscope:
             )
 
             # Capture to output object
-            logging.info("Starting microscope capture...")
+            logging.info(f"Starting microscope capture {filename}")
             self.camera.capture(
                 output,
                 use_video_port=use_video_port,
@@ -331,11 +363,7 @@ class Microscope:
                 fmt=fmt,
             )
 
-        # Gether metadata from hardware in a greenlet
-        #gevent.get_hub().threadpool.spawn(self.add_metadata_to_capture, output, metadata, annotations, tags)
-        gevent.spawn(self.add_metadata_to_capture, output, metadata, annotations, tags)
-        #self.add_metadata_to_capture(output, metadata, annotations, tags)
-
-        logging.info(f"Finished capture to {output.file}")
+        output.put_and_save(tags, annotations, full_metadata)
+        logging.debug(f"Finished capture to {output.file}")
 
         return output