Optimised captures by using greenlets for disk IO

openflexure_microscope/captures/capture.py

@@ -10,6 +10,9 @@ from PIL import Image
 import dateutil.parser
 import atexit
 
+from gevent.fileobject import FileObjectThread
+import gevent
+
 from collections import OrderedDict
 
 from openflexure_microscope.camera import piexif
@@ -118,12 +121,19 @@ def capture_from_exif(path, exif_dict):
 
 class CaptureObject(object):
     """
-    StreamObject used to store and process on-disk capture data, and metadata.
+    File-like object used to store and process on-disk capture data, and metadata.
     Serves to simplify modifying properties of on-disk capture data.
     """
 
     def __init__(self, filepath) -> None:
         """Create a new StreamObject, to manage capture data."""
+        # Stream for buffering capture data
+        self.stream = io.BytesIO()
+        # Event to notify when the stream has finished writing to disk
+        #self.file_ready = Event()
+        self.file_ready = gevent.event.Event()
+        # Lock to control disk file access
+        self.file_lock = gevent.lock.BoundedSemaphore()
 
         # Store a nice ID
         self.id = uuid.uuid4()  #: str: Unique capture ID
@@ -148,6 +158,25 @@ class CaptureObject(object):
         # Thumbnail (populated only for PIL captures)
         self.thumb_bytes = None
 
+    def write(self, s):
+        self.stream.write(s)
+        gevent.sleep()
+
+    def _stream_to_file(self):
+        logging.info(f"Writing to disk {self.file}")
+        with FileObjectThread(open(self.file, "wb"), 'wb')  as outfile, self.file_lock:
+            outfile.write(self.stream.getbuffer())
+        self.stream.close()
+        self.file_ready.set()
+        logging.info(f"Finished writing to disk {self.file}")
+        gevent.sleep()
+
+    def flush(self):
+        logging.debug(f"Flushing {self.file}")
+        gevent.spawn(self._stream_to_file)
+        gevent.sleep()
+        logging.debug(f"Returning flushing {self.file}")
+
     def open(self, mode):
         return open(self.file, mode)
 
@@ -221,24 +250,30 @@ class CaptureObject(object):
         self.save_metadata()
 
     def save_metadata(self) -> None:
+        #gevent.get_hub().threadpool.spawn(self.synchronous_save_metadata)
+        gevent.spawn(self.synchronous_save_metadata)
+        gevent.sleep()
+
+    def synchronous_save_metadata(self) -> None:
         """
         Save metadata to exif, if supported
         """
         global EXIF_FORMATS
 
         if self.format.upper() in EXIF_FORMATS and self.exists:
-            logging.debug("Writing exif data to capture file")
-            # Extract current Exif data
-            exif_dict = piexif.load(self.file)
-            # Serialize metadata
-            metadata_string = json.dumps(self.metadata, cls=JSONEncoder)
-            logging.debug(f"Saving metadata string to file: {metadata_string}")
-            # Insert metadata into exif_dict
-            exif_dict["Exif"][piexif.ExifIFD.UserComment] = metadata_string.encode()
-            # Convert new exif dict to exif bytes
-            exif_bytes = piexif.dump(exif_dict)
-            # Insert exif into file
-            piexif.insert(exif_bytes, self.file)
+            with self.file_lock:
+                logging.debug("Writing exif data to capture file")
+                # Extract current Exif data
+                exif_dict = piexif.load(self.file)
+                # Serialize metadata
+                metadata_string = json.dumps(self.metadata, cls=JSONEncoder)
+                logging.debug(f"Saving metadata string to file: {metadata_string}")
+                # Insert metadata into exif_dict
+                exif_dict["Exif"][piexif.ExifIFD.UserComment] = metadata_string.encode()
+                # Convert new exif dict to exif bytes
+                exif_bytes = piexif.dump(exif_dict)
+                # Insert exif into file
+                piexif.insert(exif_bytes, self.file)
 
     @property
     def metadata(self) -> dict:
@@ -286,7 +321,7 @@ class CaptureObject(object):
 
         if self.exists:  # If data file exists
             logging.info("Opening from file {}".format(self.file))
-            with open(self.file, "rb") as f:
+            with open(self.file, "rb") as f, self.file_lock:
                 d = io.BytesIO(f.read())  # Load bytes from file
             d.seek(0)  # Rewind loaded bytestream
             # Create a copy of the bytestream bytes