diff --git a/src/openflexure_microscope_server/things/smart_scan.py b/src/openflexure_microscope_server/things/smart_scan.py index 211af941..af9a493a 100644 --- a/src/openflexure_microscope_server/things/smart_scan.py +++ b/src/openflexure_microscope_server/things/smart_scan.py @@ -13,7 +13,7 @@ from PIL import Image from pydantic import BaseModel from datetime import datetime from subprocess import CompletedProcess, Popen, PIPE, SubprocessError, STDOUT -from threading import Event, Thread +from threading import Event import glob import json import piexif @@ -30,6 +30,8 @@ from labthings_fastapi.decorators import thing_action, thing_property, fastapi_e from labthings_fastapi.outputs.blob import blob_type from .camera import CameraDependency as CamDep from .stage import StageDependency as StageDep + +from openflexure_microscope_server.utilities import ErrorCapturingThread from openflexure_microscope_server.things.autofocus import AutofocusThing from openflexure_microscope_server.things.camera_stage_mapping import CameraStageMapper from openflexure_microscope_server.things.background_detect import BackgroundDetectThing @@ -290,6 +292,7 @@ class SmartScanThing(Thing): """ # Define these variables so we can use them in the finally: block # (after testing they are not None) + scan_sucessful = True scan_folder = None images_folder = None starting_position = None @@ -483,9 +486,15 @@ class SmartScanThing(Thing): # Acquire the image in a thread, and continue once it's acquired (i.e. leave saving in the background) if capture_thread: # wait for the previous capture to be saved, i.e. don't leave more than one image saving in the background - if capture_thread.is_alive(): - wait_start = time.time() - capture_thread.join() + wait_start = time.time() + thread_was_alive = capture_thread.is_alive() + + # If the capture thread has thrown an exception it will be raised when join is called, + # this will cause the scan to end. If we want to retry captures at a later date + # this is where we will need to catch the IOError or CaptureError from the thread. + capture_thread.join() + time.sleep(0.2) + if thread_was_alive: wait_time = time.time() - wait_start logger.info( f"Waited {wait_time:.1f}s for the previous capture to finish saving." @@ -494,7 +503,10 @@ class SmartScanThing(Thing): name = f"image_{loc[0]}_{loc[1]}.jpg" jpeg_path = os.path.join(images_folder, name) time.sleep(0.2) - capture_thread = Thread( + + # Use ErrorCapturingThread intead of Thread. This will raise errors in the calling + # thread only when join() is called, allowing us to handle this appropriately. + capture_thread = ErrorCapturingThread( target=self.capture_and_save, kwargs={ "acquired": acquired, @@ -534,23 +546,39 @@ class SmartScanThing(Thing): ) except InvocationCancelledError: + scan_sucessful = False logger.error("Stopping scan because it was cancelled.") except NotEnoughFreeSpaceError as e: + scan_sucessful = False logger.error( f"Stopping scan to avoid filling up the disk: {e}", exc_info=e, ) raise e except Exception as e: + scan_sucessful = False logger.error( - f"The scan stopped because of an error: {e}", - "We will attempt to stitch and archive the images acquired so far.", + f"The scan stopped because of an error: {e} " + "Attempting to stitch and archive the images acquired so far.", exc_info=e, ) raise e finally: if capture_thread: - capture_thread.join() + # If the capture thread had an error we capture it here + try: + capture_thread.join() + except Exception as e: + # If the thread has already ended due to an exception we will + # ignore any excppetions, but if it appeared to be succesfull + # we will log an error. + if scan_sucessful: + logger.error( + "The appears to have been successful however the final capture raised" + f"the following error: {e}." + "Attempting to stitch and archive images.", + exc_info=e, + ) try: logger.info("Returning to starting position.") if starting_position is not None: @@ -606,12 +634,9 @@ class SmartScanThing(Thing): logger: InvocationLogger, ) -> tuple[np.ndarray, dict]: """Capture an image in memory and return it with metadata - This will set the event `acquired` once the image has been acquired, so that the stage may be moved while it's saved. - CaptureError raised if the capture fails for any reason - returns tuple with numpy array of image data, and dict of metadata """ try: @@ -631,11 +656,8 @@ class SmartScanThing(Thing): logger: InvocationLogger, ) -> None: """Saving the captured image and metadata to disk - logger warning (via InvocationLogger) is raised if metadata is failed to be added - IOError is raised if the file cannot be saved - nothing is returned on success""" try: Image.fromarray(image.astype("uint8"), "RGB").save( diff --git a/src/openflexure_microscope_server/utilities.py b/src/openflexure_microscope_server/utilities.py new file mode 100644 index 00000000..30bf028f --- /dev/null +++ b/src/openflexure_microscope_server/utilities.py @@ -0,0 +1,77 @@ +""" +This module contains some utility functions and classes +""" + +from threading import Thread + + +class ErrorCapturingThread(Thread): + """ + This is a a subclass or Thread. It wraps the target function in a + try-except block. + + Execution will stop with an unhandled exception, but the exception will not be raised + until the join method is called. When the join method is called, the exception is + called in the calling thread. + + This allows for error handling to be performed by the calling thread at the time that + join is run. + """ + + def __init__(self, group=None, target=None, args=None, kwargs=None, daemon=None): + """ + Initialise with the same arguments as Thread + """ + # As all inputs are keywords we need to set the default values for args and kwargs: + if args is None: + args = () + if kwargs is None: + kwargs = {} + # Make and empy list for the error. + self._error_buffer = [] + + # Add the target function end error buffer to the thread to the start of + # the argument list so they are passed to _wrap_and_catch_errors() + args = (target, self._error_buffer, *args) + # Start the thread with _wrap_and_catch_errors() as the target + super().__init__( + group=group, + target=_wrap_and_catch_errors, + args=args, + kwargs=kwargs, + daemon=daemon, + ) + + def join(self, timeout=None): + """ + Join when the thread is complete. If the thread ended due to an unhandled exception, + the exception will be raised when this method is called. + """ + super().join(timeout) + # If there is an error in the error buffer clear the buffer and raise it + if self._error_buffer: + err = self._error_buffer[0] + self._error_buffer = [] + raise err + + +def _wrap_and_catch_errors(target, error_buffer, *args, **kwargs): + """ + This function is designed only to be used by + ErrorCapturingThread + + It will run a target function in a try-except block + any errors caught are added to an empty list that + should be supplied as a keyword argument + + Arguments: + * target - The target function to call + * error_buffer - An empty list that is used for returning + the exception from the thread + *args: The arguments for the target function + **keargs: The Keyword arguments for the target function + """ + try: + target(*args, **kwargs) + except BaseException as e: + error_buffer.append(e)