Merge branch 'stacking_Thing' into 'v3'

z stacking Thing

See merge request openflexure/openflexure-microscope-server!246

ofm_config_full.json

@@ -13,7 +13,8 @@
                 "path_to_openflexure_stitch": "application/openflexure-stitching/.venv/bin/openflexure-stitch"
             }
         },
-        "/background_detect/": "openflexure_microscope_server.things.background_detect:BackgroundDetectThing"
+        "/background_detect/": "openflexure_microscope_server.things.background_detect:BackgroundDetectThing",
+        "/capture/": "openflexure_microscope_server.things.capture:CaptureThing"
     },
     "settings_folder": "/var/openflexure/settings/"
 }

src/openflexure_microscope_server/things/autofocus.py

@@ -11,21 +11,28 @@ from contextlib import contextmanager
 import logging
 import time
 from typing import Annotated, Mapping, Optional, Sequence
+import os
+import shutil
+import glob
 
 from fastapi import Depends
-
-from labthings_fastapi.thing import Thing
-from labthings_fastapi.dependencies.blocking_portal import BlockingPortal
-from labthings_fastapi.decorators import thing_action
-from labthings_fastapi.types.numpy import NDArray
-from .camera import RawCameraDependency as Camera
-from .camera import CameraDependency as WrappedCamera
-from .stage import StageDependency as Stage
 import numpy as np
 from pydantic import BaseModel
 
+from labthings_fastapi.thing import Thing
+from labthings_fastapi.dependencies.blocking_portal import BlockingPortal
+from labthings_fastapi.decorators import thing_action, thing_property
+from labthings_fastapi.dependencies.metadata import GetThingStates
+from labthings_fastapi.types.numpy import NDArray
+from labthings_fastapi.dependencies.thing import direct_thing_client_dependency
+from labthings_fastapi.dependencies.invocation import InvocationLogger
 
-### Autofocus utilities
+from .camera import RawCameraDependency as Camera
+from .camera import CameraDependency as WrappedCamera
+from .stage import StageDependency as Stage
+from .capture import CaptureThing
+
+CaptureDep = direct_thing_client_dependency(CaptureThing, "/capture/")
 
 
 class JPEGSharpnessMonitor:
@@ -138,10 +145,16 @@ class SharpnessDataArrays(BaseModel):
 
 
 class AutofocusThing(Thing):
+    """The Thing concerned with combinations of z axis movements and the camera.
+
+    Actions here involve moving a stage in z, and using the camera to either
+    capture images (generally, z-stacking) and measuring the sharpness of the
+    field of view to assess focus (autofocus and testing the success of a z-stack)"""
+
     @thing_action
     def fast_autofocus(
         self,
-        m: SharpnessMonitorDep,
+        sharpness_monitor: SharpnessMonitorDep,
         dz: int = 2000,
         start: str = "centre",
     ) -> SharpnessDataArrays:
@@ -151,27 +164,27 @@ class AutofocusThing(Thing):
         the position where the image was sharpest. We'll then move back down, and
         finally up to the sharpest point.
         """
-        with m.run():
+        with sharpness_monitor.run():
             # Move to (-dz / 2)
             if start == "centre":
-                m.focus_rel(-dz / 2)
+                sharpness_monitor.focus_rel(-dz / 2)
             # Move to dz while monitoring sharpness
             # i: Sharpness monitor index for this move
             # z: Final z position after move
-            i, z = m.focus_rel(dz, block_cancellation=True)
+            i, z = sharpness_monitor.focus_rel(dz, block_cancellation=True)
             # Get the z position with highest sharpness from the previous move (index i)
-            fz: int = m.sharpest_z_on_move(i)
+            fz: int = sharpness_monitor.sharpest_z_on_move(i)
             # Move all the way to the start so it's consistent
-            i, z = m.focus_rel(-dz)
+            i, z = sharpness_monitor.focus_rel(-dz)
             # Move to the target position fz (relative move of (fz - z))
-            m.focus_rel(fz - z)
+            sharpness_monitor.focus_rel(fz - z)
             # Return all focus data
-            return m.data_dict()
+            return sharpness_monitor.data_dict()
 
     @thing_action
-    def move_and_measure(
+    def z_move_and_measure_sharpness(
         self,
-        m: SharpnessMonitorDep,
+        sharpness_monitor: SharpnessMonitorDep,
         dz: Sequence[int],
         wait: float = 0,
     ) -> SharpnessDataArrays:
@@ -187,16 +200,20 @@ class AutofocusThing(Thing):
         If `wait` is specified, we will wait for that many seconds
         between moves.
         """
-        with m.run():
+        with sharpness_monitor.run():
             for i, current_dz in enumerate(dz):
                 if i > 0 and wait > 0:
                     time.sleep(wait)
-                m.focus_rel(current_dz)
+                sharpness_monitor.focus_rel(current_dz)
-            return m.data_dict()
+            return sharpness_monitor.data_dict()
 
     @thing_action
     def looping_autofocus(
-        self, stage: Stage, m: SharpnessMonitorDep, dz=2000, start="centre"
+        self,
+        stage: Stage,
+        sharpness_monitor: SharpnessMonitorDep,
+        dz=2000,
+        start="centre",
     ):
         """Repeatedly autofocus the stage until it looks focused.
 
@@ -209,14 +226,14 @@ class AutofocusThing(Thing):
         attempts = 0
         backlash = 200
 
-        with m.run():
+        with sharpness_monitor.run():
             while repeat and attempts < 10:
                 if start == "centre":
                     stage.move_relative(x=0, y=0, z=-(backlash + dz / 2))
                     stage.move_relative(x=0, y=0, z=backlash)
 
-                i, z = m.focus_rel(dz, block_cancellation=True)
+                i, z = sharpness_monitor.focus_rel(dz, block_cancellation=True)
-                _, heights, sizes = m.move_data(i)
+                _, heights, sizes = sharpness_monitor.move_data(i)
 
                 peak_height = heights[np.argmax(sizes)]
                 height_min = np.min(heights)
@@ -251,3 +268,78 @@ class AutofocusThing(Thing):
         cutoff = threshold * (peak - base)
 
         return current_sharpness >= base + cutoff
+
+    @thing_property
+    def stack_images_to_capture(self) -> int:
+        """The number of images to capture and save in a stack
+        Defaults to 1 unless you need to see either side of focus"""
+        return self.thing_settings.get("stack_images_to_capture", 1)
+
+    @stack_images_to_capture.setter
+    def stack_images_to_capture(self, value: int) -> None:
+        self.thing_settings["stack_images_to_capture"] = value
+
+    @thing_property
+    def stack_dz(self) -> int:
+        """Space in steps between images in a z-stack
+        Suggested is 50 for 60-100x
+        100 for 40x
+        200 for 20x"""
+        return self.thing_settings.get("stack_dz", 50)
+
+    @stack_dz.setter
+    def stack_dz(self, value: int) -> None:
+        self.thing_settings["stack_dz"] = value
+
+    @thing_action
+    def run_z_stack(
+        self,
+        cam: WrappedCamera,
+        stage: Stage,
+        logger: InvocationLogger,
+        metadata_getter: GetThingStates,
+        capture: CaptureDep,
+        images_dir: str,
+        stack_dir: str,
+    ) -> None:
+        """Run a z stack, saving all images to stack_dir and copying the
+        central image to stack_dir"""
+        stack_dz = self.stack_dz
+        images_to_capture = self.stack_images_to_capture
+
+        stack_z_range = stack_dz * (images_to_capture - 1)
+        stage.move_relative(z=-stack_z_range / 2)
+
+        for capture_count in range(images_to_capture):
+            jpeg_path = os.path.join(
+                stack_dir,
+                f"{capture_count}.jpeg",
+            )
+            capture._capture_and_save(
+                jpeg_path=jpeg_path,
+                cam=cam,
+                logger=logger,
+                metadata_getter=metadata_getter,
+            )
+
+            # If the stack isn't complete yet, move
+            if capture_count + 1 < images_to_capture:
+                stage.move_relative(z=stack_dz)
+                time.sleep(0.3)
+
+        self.copy_central_image_from_stack(images_dir, stack_dir)
+
+    def copy_central_image_from_stack(
+        self,
+        images_dir: str,
+        stack_dir: str,
+    ):
+        """Gets a list of images in a folder (stack_dir), sorts them, and copies the central image
+        to images dir."""
+        image_list = glob.glob(os.path.join(stack_dir, "*"))
+        image_list.sort()
+        central_index = (len(image_list) - 1) // 2
+        central_image = image_list[central_index]
+        xy_location = os.path.basename(stack_dir)
+
+        shutil.copy(central_image, os.path.join(images_dir, f"{xy_location}.jpeg"))

src/openflexure_microscope_server/things/camera/__init__.py

@@ -44,7 +44,7 @@ class CameraProtocol(Protocol):
 
     def capture_array(
         self,
-        resolution: Literal["lores", "main", "full"] = "main",
+        stream_name: Literal["main", "lores", "raw"] = "main",
     ) -> NDArray: ...
 
     def capture_jpeg(
@@ -166,7 +166,7 @@ class CameraStub(BaseCamera):
     @thing_action
     def capture_array(
         self,
-        resolution: Literal["lores", "main", "full"] = "main",
+        stream_name: Literal["main", "lores", "raw"] = "main",
     ) -> NDArray:
         raise NotImplementedError("Cameras must not inherit from CameraStub")
 

src/openflexure_microscope_server/things/capture.py

@@ -0,0 +1,89 @@
+import numpy as np
+from PIL import Image
+import time
+import piexif
+import json
+
+from labthings_fastapi.dependencies.metadata import GetThingStates
+from labthings_fastapi.thing import Thing
+from labthings_fastapi.decorators import thing_action
+from .camera import CameraDependency as CamDep
+from labthings_fastapi.dependencies.invocation import (
+    InvocationLogger,
+)
+
+
+class CaptureError(RuntimeError):
+    """An error trying to capture from Picamera"""
+
+
+class CaptureThing(Thing):
+    """A temporary Thing to handle capturing to disk with associated metadata
+    Will be moved to the camera Thing or dependency in due course"""
+
+    @thing_action
+    def _capture_and_save(
+        self,
+        jpeg_path: str,
+        cam: CamDep,
+        logger: InvocationLogger,
+        metadata_getter: GetThingStates,
+    ) -> None:
+        """Capture an image and save it to disk
+
+        This will set the event `acquired` once the image has been acquired, so
+        that the stage may be moved while it's saved.
+        """
+        capture_start = time.time()
+        image, metadata = self._capture_image(
+            cam,
+            metadata_getter,
+        )
+        acquisition_time = time.time()
+        self._save_capture(jpeg_path, image, metadata, logger)
+        save_time = time.time()
+        acquisition_duration = round(acquisition_time - capture_start, 1)
+        saving_duration = round(save_time - acquisition_time, 1)
+        logger.info(
+            f"Acquired {jpeg_path} in {acquisition_duration}s then {saving_duration}s saving to disk"
+        )
+
+    def _capture_image(self, cam, metadata_getter) -> tuple[np.ndarray, dict]:
+        """Capture an image in memory and return it with metadata
+        CaptureError raised if the capture fails for any reason
+        returns tuple with numpy array of image data, and dict of metadata
+        """
+        try:
+            metadata = metadata_getter()
+            image = cam.capture_array()[..., :3]
+        except Exception as e:
+            raise CaptureError("An error occurred while capturing") from e
+        return image, metadata
+
+    def _save_capture(
+        self,
+        jpeg_path: str,
+        image: np.ndarray,
+        metadata: dict,
+        logger: InvocationLogger,
+    ) -> None:
+        """Saving the captured image and metadata to disk
+        logger warning (via InvocationLogger) is raised if metadata is failed to be added
+        IOError is raised if the file cannot be saved
+        nothing is returned on success"""
+        try:
+            Image.fromarray(image.astype("uint8"), "RGB").save(
+                jpeg_path, quality=95, subsampling=0
+            )
+            try:
+                exif_dict = piexif.load(jpeg_path)
+                exif_dict["Exif"][piexif.ExifIFD.UserComment] = json.dumps(
+                    metadata
+                ).encode("utf-8")
+                piexif.insert(piexif.dump(exif_dict), jpeg_path)
+            except:  # noqa: E722
+                # We need to capture any exception as there are many reasons metadata
+                # might not be added. We warn rather than log the error.
+                logger.warning(f"Failed to add metadata to {jpeg_path}")
+        except Exception as e:
+            raise IOError(f"An error occurred while saving {jpeg_path}") from e

src/openflexure_microscope_server/things/smart_scan.py

@@ -11,10 +11,8 @@ from PIL import Image
 from pydantic import BaseModel
 from datetime import datetime
 from subprocess import CompletedProcess, Popen, PIPE, SubprocessError, STDOUT
-from threading import Event
 import glob
 import json
-import piexif
 import re
 
 from labthings_fastapi.thing import Thing
@@ -585,23 +583,25 @@ class SmartScanThing(Thing):
                 current_pos_xyz, imaged=True, focused=focused
             )
 
-            # wait for the previous capture to be saved, i.e. don't leave more than one image saving in the background
+            site_folder = os.path.join(
-            if self._capture_thread:
+                self._ongoing_scan_images_dir,
-                self._wait_for_capture_thread()
+                "stacks",
-                # increment capure counter as thread has completed
+                f"{new_pos_xyz[0]}_{new_pos_xyz[1]}",
-                self._scan_images_taken += 1
+            )
-                # Add it to the incremental zip
+            os.makedirs(site_folder, exist_ok=True)
-                self.create_zip_of_scan(
+            self._autofocus.run_z_stack(
-                    logger=self._scan_logger,
+                images_dir=self._ongoing_scan_images_dir,
-                    scan_name=self._ongoing_scan_name,
+                stack_dir=site_folder,
-                    download_zip=False,
+            )
-                )
 
-            name = f"image_{new_pos_xyz[0]}_{new_pos_xyz[1]}.jpg"
+            # increment capure counter as thread has completed
-            jpeg_path = os.path.join(self._ongoing_scan_images_dir, name)
+            self._scan_images_taken += 1
-            self._capture_thread, acquired = self._start_capture_thread(jpeg_path)
+            # Add it to the incremental zip
-            # wait until the image is acquired
+            self.create_zip_of_scan(
-            acquired.wait()
+                logger=self._scan_logger,
+                scan_name=self._ongoing_scan_name,
+                download_zip=False,
+            )
 
     @_scan_running
     def _try_autofocus(
@@ -639,52 +639,6 @@ class SmartScanThing(Thing):
         self._scan_logger.warning("Could not autofocus after 3 attempts.")
         return False, self._stage.position["z"]
 
-    @_scan_running
-    def _wait_for_capture_thread(self) -> None:
-        """
-        Wait for the capture thread to be complete.
-        """
-        wait_start = time.time()
-        thread_was_alive = self._capture_thread.is_alive()
-
-        # If the capture thread has thrown an exception it will be raised
-        # when join is called, this will cause the scan to end. If we want
-        # to retry captures at a later date this is where we will need to
-        # catch the IOError or CaptureError from the thread.
-        self._capture_thread.join()
-        time.sleep(0.2)
-        if thread_was_alive:
-            wait_time = time.time() - wait_start
-            self._scan_logger.info(
-                f"Waited {wait_time:.1f}s for the previous capture to finish saving."
-            )
-
-    @_scan_running
-    def _start_capture_thread(
-        self, jpeg_path: str
-    ) -> tuple[ErrorCapturingThread, Event]:
-        """
-        Start the capture thread.
-
-        Args:
-           jpeg_path, the path to save the image once aquired
-
-        Return the thread and an event that will be set when the image is aquired
-        """
-        acquired = Event()
-        time.sleep(0.2)
-
-        # Acquire the image in a thread, and continue once it's acquired
-        # (i.e. leave saving in the background) Use ErrorCapturingThread
-        # intead of Thread. This will raise errors in the calling thread
-        # only when join() is called, allowing us to handle this appropriately.
-        capture_thread = ErrorCapturingThread(
-            target=self._capture_and_save,
-            kwargs={"acquired": acquired, "jpeg_path": jpeg_path},
-        )
-        capture_thread.start()
-        return capture_thread, acquired
-
     @_scan_running
     def _return_to_starting_position(self):
         self._scan_logger.info("Returning to starting position.")
@@ -743,75 +697,6 @@ class SmartScanThing(Thing):
             raise HTTPException(404, "File not found")
         return FileResponse(path)
 
-    @_scan_running
-    def _capture_and_save(
-        self,
-        acquired: Event,
-        jpeg_path: str,
-    ) -> None:
-        """Capture an image and save it to disk
-
-        This will set the event `acquired` once the image has been acquired, so
-        that the stage may be moved while it's saved.
-        """
-        try:
-            capture_start = time.time()
-            image, metadata = self._capture_image()
-        finally:
-            # Ensure aquired is set even if capture fails or program will hang forever.
-            acquired.set()
-        acquisition_time = time.time()
-        self._save_capture(jpeg_path, image, metadata)
-        save_time = time.time()
-        acquisition_duration = round(acquisition_time - capture_start, 1)
-        saving_duration = round(save_time - acquisition_time, 1)
-        self._scan_logger.debug(
-            f"Acquired {jpeg_path} in {acquisition_duration}s then {saving_duration}s saving to disk"
-        )
-
-    @_scan_running
-    def _capture_image(self) -> tuple[np.ndarray, dict]:
-        """Capture an image in memory and return it with metadata
-        This will set the event `acquired` once the image has been acquired, so
-        that the stage may be moved while it's saved.
-        CaptureError raised if the capture fails for any reason
-        returns tuple with numpy array of image data, and dict of metadata
-        """
-        try:
-            metadata = self._metadata_getter()
-            image = self._cam.capture_array()[..., :3]
-        except Exception as e:
-            raise CaptureError("An error occurred while capturing") from e
-        return image, metadata
-
-    @_scan_running
-    def _save_capture(
-        self,
-        jpeg_path: str,
-        image: np.ndarray,
-        metadata: dict,
-    ) -> None:
-        """Saving the captured image and metadata to disk
-        logger warning (via InvocationLogger) is raised if metadata is failed to be added
-        IOError is raised if the file cannot be saved
-        nothing is returned on success"""
-        try:
-            Image.fromarray(image.astype("uint8"), "RGB").save(
-                jpeg_path, quality=95, subsampling=0
-            )
-            try:
-                exif_dict = piexif.load(jpeg_path)
-                exif_dict["Exif"][piexif.ExifIFD.UserComment] = json.dumps(
-                    metadata
-                ).encode("utf-8")
-                piexif.insert(piexif.dump(exif_dict), jpeg_path)
-            except:  # noqa: E722
-                # We need to capture any exception as there are many reasons metadata
-                # might not be added. We warn rather than log the error.
-                self._scan_logger.warning(f"Failed to add metadata to {jpeg_path}")
-        except Exception as e:
-            raise IOError(f"An error occurred while saving {jpeg_path}") from e
-
     @thing_property
     def max_range(self) -> int:
         """The maximum distance from the centre of the scan before we break"""
@@ -1241,7 +1126,3 @@ class SmartScanThing(Thing):
         """List the relative paths of all files and folders in the zip folder specified"""
         scan_zip = zipfile.ZipFile(zip_path)
         return [os.path.normpath(i) for i in scan_zip.namelist()]
-
-
-class CaptureError(RuntimeError):
-    """An error trying to capture from Picamera"""

webapp/src/components/tabContentComponents/slideScanContent.vue

@@ -24,6 +24,20 @@
                   label="Autofocus range (steps)"
                 />
               </div>
+              <div class="uk-margin">
+                <propertyControl
+                  thing-name="autofocus"
+                  property-name="stack_dz"
+                  label="Stack dz (steps)"
+                />  
+              </div>
+              <div class="uk-margin">
+                <propertyControl
+                  thing-name="autofocus"
+                  property-name="stack_images_to_capture"
+                  label="Images in stack to capture"
+                />
+              </div>
               <div class="uk-margin">
                 <propertyControl
                   thing-name="smart_scan"