From 6bf37a0985f048db94655ac0ac51177113f71d77 Mon Sep 17 00:00:00 2001 From: Joel Collins Date: Thu, 11 Jun 2020 12:01:19 +0100 Subject: [PATCH] Threadpool for disk IO, greenlets for metadata --- openflexure_microscope/captures/capture.py | 37 ++++++++++------------ openflexure_microscope/microscope.py | 37 +++++++++++----------- 2 files changed, 35 insertions(+), 39 deletions(-) diff --git a/openflexure_microscope/captures/capture.py b/openflexure_microscope/captures/capture.py index 6ce0f42a..af5fac32 100644 --- a/openflexure_microscope/captures/capture.py +++ b/openflexure_microscope/captures/capture.py @@ -12,6 +12,7 @@ import atexit from gevent.fileobject import FileObjectThread import gevent +import threading from collections import OrderedDict @@ -132,8 +133,6 @@ class CaptureObject(object): # Event to notify when the stream has finished writing to disk #self.file_ready = Event() self.file_ready = gevent.event.Event() - # Lock to control disk file access - self.file_lock = gevent.lock.BoundedSemaphore() # Store a nice ID self.id = uuid.uuid4() #: str: Unique capture ID @@ -164,7 +163,7 @@ class CaptureObject(object): def _stream_to_file(self): logging.info(f"Writing to disk {self.file}") - with FileObjectThread(open(self.file, "wb"), 'wb') as outfile, self.file_lock: + with FileObjectThread(open(self.file, "wb"), 'wb') as outfile: outfile.write(self.stream.getbuffer()) self.stream.close() self.file_ready.set() @@ -173,7 +172,7 @@ class CaptureObject(object): def flush(self): logging.debug(f"Flushing {self.file}") - gevent.spawn(self._stream_to_file) + gevent.get_hub().threadpool.spawn(self._stream_to_file) gevent.sleep() logging.debug(f"Returning flushing {self.file}") @@ -250,8 +249,7 @@ class CaptureObject(object): self.save_metadata() def save_metadata(self) -> None: - #gevent.get_hub().threadpool.spawn(self.synchronous_save_metadata) - gevent.spawn(self.synchronous_save_metadata) + gevent.get_hub().threadpool.spawn(self.synchronous_save_metadata) gevent.sleep() def synchronous_save_metadata(self) -> None: @@ -261,19 +259,18 @@ class CaptureObject(object): global EXIF_FORMATS if self.format.upper() in EXIF_FORMATS and self.exists: - with self.file_lock: - logging.debug("Writing exif data to capture file") - # Extract current Exif data - exif_dict = piexif.load(self.file) - # Serialize metadata - metadata_string = json.dumps(self.metadata, cls=JSONEncoder) - logging.debug(f"Saving metadata string to file: {metadata_string}") - # Insert metadata into exif_dict - exif_dict["Exif"][piexif.ExifIFD.UserComment] = metadata_string.encode() - # Convert new exif dict to exif bytes - exif_bytes = piexif.dump(exif_dict) - # Insert exif into file - piexif.insert(exif_bytes, self.file) + logging.debug("Writing exif data to capture file") + # Extract current Exif data + exif_dict = piexif.load(self.file) + # Serialize metadata + metadata_string = json.dumps(self.metadata, cls=JSONEncoder) + logging.debug(f"Saving metadata string to file: {metadata_string}") + # Insert metadata into exif_dict + exif_dict["Exif"][piexif.ExifIFD.UserComment] = metadata_string.encode() + # Convert new exif dict to exif bytes + exif_bytes = piexif.dump(exif_dict) + # Insert exif into file + piexif.insert(exif_bytes, self.file) @property def metadata(self) -> dict: @@ -321,7 +318,7 @@ class CaptureObject(object): if self.exists: # If data file exists logging.info("Opening from file {}".format(self.file)) - with open(self.file, "rb") as f, self.file_lock: + with open(self.file, "rb") as f: d = io.BytesIO(f.read()) # Load bytes from file d.seek(0) # Rewind loaded bytestream # Create a copy of the bytestream bytes diff --git a/openflexure_microscope/microscope.py b/openflexure_microscope/microscope.py index 7301ec73..4af47379 100644 --- a/openflexure_microscope/microscope.py +++ b/openflexure_microscope/microscope.py @@ -288,6 +288,21 @@ class Microscope: return system_metadata + def add_metadata_to_capture(output, metadata, annotations, tags): + logging.debug(f"Waiting for {output.file}") + # Wait for the file to be written to disk + output.file_ready.wait() + logging.info(f"Asynchronously injecting EXIF data into {output.file}") + # Inject system metadata + output.put_metadata({"instrument": self.metadata}) + # Insert custom metadata + output.put_metadata(metadata) + # Insert custom metadata + output.put_annotations(annotations) + # Insert custom tags + output.put_tags(tags) + logging.info(f"Finished injecting EXIF data into {output.file}") + def capture( self, filename: str = None, @@ -324,26 +339,10 @@ class Microscope: bayer=bayer, fmt=fmt, ) - logging.info("Finished microscope capture...") + # Gether metadata from hardware in a greenlet + gevent.spawn(self.add_metadata_to_capture, output, metadata, annotations, tags) - def inject_metadata(): - logging.debug(f"Waiting for {output.file}") - # Wait for the file to be written to disk - output.file_ready.wait() - logging.info(f"Asynchronously injecting EXIF data into {output.file}") - # Inject system metadata - output.put_metadata({"instrument": self.metadata}) - # Insert custom metadata - output.put_metadata(metadata) - # Insert custom metadata - output.put_annotations(annotations) - # Insert custom tags - output.put_tags(tags) - logging.info(f"Finished injecting EXIF data into {output.file}") - - gevent.spawn(inject_metadata) - - logging.debug(f"Finished capture to {output.file}") + logging.info(f"Finished capture to {output.file}") return output