Handle errors in capture thread

src/openflexure_microscope_server/things/smart_scan.py

@@ -13,7 +13,7 @@ from PIL import Image
 from pydantic import BaseModel
 from datetime import datetime
 from subprocess import CompletedProcess, Popen, PIPE, SubprocessError, STDOUT
-from threading import Event, Thread
+from threading import Event
 import glob
 import json
 import piexif
@@ -30,6 +30,8 @@ from labthings_fastapi.decorators import thing_action, thing_property, fastapi_e
 from labthings_fastapi.outputs.blob import blob_type
 from .camera import CameraDependency as CamDep
 from .stage import StageDependency as StageDep
+
+from openflexure_microscope_server.utilities import ErrorCapturingThread
 from openflexure_microscope_server.things.autofocus import AutofocusThing
 from openflexure_microscope_server.things.camera_stage_mapping import CameraStageMapper
 from openflexure_microscope_server.things.background_detect import BackgroundDetectThing
@@ -290,6 +292,7 @@ class SmartScanThing(Thing):
         """
         # Define these variables so we can use them in the finally: block
         # (after testing they are not None)
+        scan_sucessful = True
         scan_folder = None
         images_folder = None
         starting_position = None
@@ -483,9 +486,15 @@ class SmartScanThing(Thing):
 
                     # Acquire the image in a thread, and continue once it's acquired (i.e. leave saving in the background)
                     if capture_thread:  # wait for the previous capture to be saved, i.e. don't leave more than one image saving in the background
-                        if capture_thread.is_alive():
-                            wait_start = time.time()
-                            capture_thread.join()
+                        wait_start = time.time()
+                        thread_was_alive = capture_thread.is_alive()
+
+                        # If the capture thread has thrown an exception it will be raised when join is called,
+                        # this will cause the scan to end. If we want to retry captures at a later date
+                        # this is where we will need to catch the IOError or CaptureError from the thread.
+                        capture_thread.join()
+                        time.sleep(0.2)
+                        if thread_was_alive:
                             wait_time = time.time() - wait_start
                             logger.info(
                                 f"Waited {wait_time:.1f}s for the previous capture to finish saving."
@@ -494,7 +503,10 @@ class SmartScanThing(Thing):
                     name = f"image_{loc[0]}_{loc[1]}.jpg"
                     jpeg_path = os.path.join(images_folder, name)
                     time.sleep(0.2)
-                    capture_thread = Thread(
+
+                    # Use ErrorCapturingThread intead of Thread. This will raise errors in the calling
+                    # thread only when join() is called, allowing us to handle this appropriately.
+                    capture_thread = ErrorCapturingThread(
                         target=self.capture_and_save,
                         kwargs={
                             "acquired": acquired,
@@ -534,23 +546,39 @@ class SmartScanThing(Thing):
                 )
 
         except InvocationCancelledError:
+            scan_sucessful = False
             logger.error("Stopping scan because it was cancelled.")
         except NotEnoughFreeSpaceError as e:
+            scan_sucessful = False
             logger.error(
                 f"Stopping scan to avoid filling up the disk: {e}",
                 exc_info=e,
             )
             raise e
         except Exception as e:
+            scan_sucessful = False
             logger.error(
-                f"The scan stopped because of an error: {e}",
-                "We will attempt to stitch and archive the images acquired so far.",
+                f"The scan stopped because of an error: {e} "
+                "Attempting to stitch and archive the images acquired so far.",
                 exc_info=e,
             )
             raise e
         finally:
             if capture_thread:
-                capture_thread.join()
+                # If the capture thread had an error we capture it here
+                try:
+                    capture_thread.join()
+                except Exception as e:
+                    # If the thread has already ended due to an exception we will
+                    # ignore any excppetions, but if it appeared to be succesfull
+                    # we will log an error.
+                    if scan_sucessful:
+                        logger.error(
+                            "The appears to have been successful however the final capture raised"
+                            f"the following error: {e}."
+                            "Attempting to stitch and archive images.",
+                            exc_info=e,
+                        )
             try:
                 logger.info("Returning to starting position.")
                 if starting_position is not None:
@@ -606,12 +634,9 @@ class SmartScanThing(Thing):
         logger: InvocationLogger,
     ) -> tuple[np.ndarray, dict]:
         """Capture an image in memory and return it with metadata
-
         This will set the event `acquired` once the image has been acquired, so
         that the stage may be moved while it's saved.
-
         CaptureError raised if the capture fails for any reason
-
         returns tuple with numpy array of image data, and dict of metadata
         """
         try:
@@ -631,11 +656,8 @@ class SmartScanThing(Thing):
         logger: InvocationLogger,
     ) -> None:
         """Saving the captured image and metadata to disk
-
         logger warning (via InvocationLogger) is raised if metadata is failed to be added
-
         IOError is raised if the file cannot be saved
-
         nothing is returned on success"""
         try:
             Image.fromarray(image.astype("uint8"), "RGB").save(

src/openflexure_microscope_server/utilities.py

@@ -0,0 +1,77 @@
+"""
+This module contains some utility functions and classes
+"""
+
+from threading import Thread
+
+
+class ErrorCapturingThread(Thread):
+    """
+    This is a a subclass or Thread. It wraps the target function in a
+    try-except block.
+
+    Execution will stop with an unhandled exception, but the exception will not be raised
+    until the join method is called. When the join method is called, the exception is
+    called in the calling thread.
+
+    This allows for error handling to be performed by the calling thread at the time that
+    join is run.
+    """
+
+    def __init__(self, group=None, target=None, args=None, kwargs=None, daemon=None):
+        """
+        Initialise with the same arguments as Thread
+        """
+        # As all inputs are keywords we need to set the default values for args and kwargs:
+        if args is None:
+            args = ()
+        if kwargs is None:
+            kwargs = {}
+        # Make and empy list for the error.
+        self._error_buffer = []
+
+        # Add the target function end error buffer to the thread to the start of
+        # the argument list so they are passed to _wrap_and_catch_errors()
+        args = (target, self._error_buffer, *args)
+        # Start the thread with _wrap_and_catch_errors() as the target
+        super().__init__(
+            group=group,
+            target=_wrap_and_catch_errors,
+            args=args,
+            kwargs=kwargs,
+            daemon=daemon,
+        )
+
+    def join(self, timeout=None):
+        """
+        Join when the thread is complete. If the thread ended due to an unhandled exception,
+        the exception will be raised when this method is called.
+        """
+        super().join(timeout)
+        # If there is an error in the error buffer clear the buffer and raise it
+        if self._error_buffer:
+            err = self._error_buffer[0]
+            self._error_buffer = []
+            raise err
+
+
+def _wrap_and_catch_errors(target, error_buffer, *args, **kwargs):
+    """
+    This function is designed only to be used by
+    ErrorCapturingThread
+
+    It will run a target function in a try-except block
+    any errors caught are added to an empty list that
+    should be supplied as a keyword argument
+
+    Arguments:
+      * target - The target function to call
+      * error_buffer - An empty list that is used for returning
+          the exception from the thread
+      *args: The arguments for the target function
+      **keargs: The Keyword arguments for the target function
+    """
+    try:
+        target(*args, **kwargs)
+    except BaseException as e:
+        error_buffer.append(e)