Merge branch 'background-detection-for-scans' into 'master'

Background detect and better focus handling in tiled scans

See merge request openflexure/openflexure-microscope-server!153

openflexure_microscope/api/default_extensions/scan.py

@@ -2,10 +2,10 @@ import datetime
 import logging
 import time
 import uuid
-from functools import reduce
+from typing import Callable, Dict, List, Optional, Tuple
-from typing import Dict, List, Optional, Tuple
 
 import marshmallow
+import numpy as np
 from labthings import (
     current_action,
     fields,
@@ -30,12 +30,146 @@ XyzCoordinate = Tuple[int, int, int]
 ### Grid construction
 
 
+class FocusManager:
+    """Manage axial motion during a series of XY moves
+
+    This class keeps track of the focus position as we move around.
+    It currently uses the regular autofocus method, and has support
+    for background detection.
+    """
+
+    initial_position: XyzCoordinate = None  # type: ignore[assignment]
+    focused_positions: List[XyzCoordinate] = None  # type: ignore[assignment]
+    microscope: Microscope = None  # type: ignore[assignment]
+    current_image_is_background_function: Optional[Callable] = None
+    autofocus_function: Optional[Callable] = None
+    axial_jump_threshold: Optional[float] = None
+
+    def __init__(
+        self,
+        microscope: Microscope,
+        initial_position: XyzCoordinate,
+        autofocus_function: Optional[Callable],
+        current_image_is_background_function: Optional[Callable] = None,
+        axial_jump_threshold: Optional[float] = 0.4,
+    ):
+        """Set up management of axial motion.
+
+        The `FocusManager` keeps track of previous positions where the
+        microscope was in focus, and will estimate the best Z value for
+        future XY positions.
+
+        Arguments:
+        * microscope: The microscope object.
+        * initial_position: the XYZ position of the start of the scan
+        * autofocus: A function that performs an autofocus.
+        * current_image_is_background: a function that returns `True`
+        if the microscope is currently looking at an empty field.
+        * axial_jump_threshold: the maximum ratio between axial and
+        lateral moves.  If this is not None, it will not count the 
+        autofocus routine as successful if the focus moves more than
+        this ratio times the lateral move between two points.  This can
+        help avoid focus drift due to accidentally focusing on the coverslip.
+
+        If either of the `autofocus` or `current_image_is_background`
+        functions are None, we will neither perform an autofocus, nor 
+        use background estimation.
+        """
+        self.initial_position = initial_position
+        self.focused_positions = []
+        self.microscope = microscope
+        self.autofocus_function = autofocus_function
+        if not autofocus_function:
+            logging.info("Setting up FocusManager with autofocus disabled.")
+        self.current_image_is_background_function = current_image_is_background_function
+        self.axial_jump_threshold = axial_jump_threshold
+
+    def record_focused_point(self, position: XyzCoordinate):
+        """Add a position to the list of successfully-focused points"""
+        self.focused_positions.append(position)
+
+    def closest_focused_point(self, position: XyCoordinate) -> Optional[XyzCoordinate]:
+        """The closest point in our list of focused points to a given XY position."""
+        return closest_point_in_xy(position, self.focused_positions)
+
+    def estimate_z(self, position: XyCoordinate) -> int:
+        """Estimate the z position most likely to be in focus at an XY point
+        
+        The next z position is estimated based on the closest point that was in focus.
+        For a snake/spiral scan, this should always be the last point, unless
+        it's skipped for some reason.  In a raster scan, this should be the last
+        point, except when we're at the start of a row when it will be the first
+        point of the preceding row.
+        
+        It is possible that for some scan geometries, we won't be using the most
+        recent point (e.g. if X and Y spacing in a raster scan are very different).
+        This does not happen with the default settings used for raster scanning
+        clinical samples.
+        """
+        closest_focused_point = self.closest_focused_point(position)
+        if closest_focused_point:
+            return closest_focused_point[2]
+        else:
+            return self.initial_position[2]
+
+    def check_for_axial_jumps(self, position: XyzCoordinate) -> bool:
+        """Check if a position is inconsistent with previous positions.
+
+        This function returns `True` if the specified position is not consistent
+        with the list of previously-visited positions, i.e. it's made a big axial
+        move but a small lateral move.
+
+        The threshold is set by `self.axial_jump_threshold`, and if that is `None`
+        no check is performed.  Sensible values are probably between 0.1 and 0.5.
+        """
+        if not self.axial_jump_threshold:
+            return False
+        closest_focused_point = self.closest_focused_point(position[:2])
+        if not closest_focused_point:
+            return False
+        move = np.array(position) - np.array(closest_focused_point)
+        lateral_move = np.sqrt(np.sum(move[:2] ** 2))
+        axial_move = np.abs(move[2])
+        return axial_move > lateral_move * self.axial_jump_threshold
+
+    def autofocus(self):
+        """Perform an autofocus routine.
+
+        If autofocus is disabled, nothing happens here.  If it is enabled, we optionally
+        check whether there's anything in the image to focus on, and then run the autofocus
+        routine.
+        """
+        if not self.autofocus_function:
+            logging.debug("Autofocus is disabled, skipping autofocus.")
+            return
+        if self.current_image_is_background_function:
+            # If it's been set, call the background detect function and skip
+            # autofocus if appropriate
+            if self.current_image_is_background_function():
+                here = self.microscope.stage.position
+                logging.info(f"Detected an empty field at {here}, skipping autofocus.")
+                return
+        # Assuming it's not disabled, and we're not skipping it, actually run the autofocus now
+        self.autofocus_function()
+        # Now, check for big jumps and record the new position if we've not made a big jump
+        here = self.microscope.stage.position
+        if self.check_for_axial_jumps(here):
+            logging.warning(
+                f"During a scan, there was a large axial jump from "
+                f"{self.closest_focused_point(here[:2])}"
+                f" to {here}.  This may mean autofocus has failed."
+            )
+        else:
+            # If there has not been a jump in focus, record the point as successful
+            self.record_focused_point(here)
+
+
 def construct_grid(
     initial: XyCoordinate,
     step_sizes: XyCoordinate,
     n_steps: XyCoordinate,
     style: Literal["raster", "snake", "spiral"] = "raster",
-):
+) -> List[List[XyCoordinate]]:
     """
     Given an initial position, step sizes, and number of steps,
     construct a 2-dimensional list of scan x-y positions.
@@ -95,6 +229,50 @@ def construct_grid(
     return arr
 
 
+def construct_grid_1d(
+    initial: XyCoordinate,
+    step_sizes: XyCoordinate,
+    n_steps: XyCoordinate,
+    style: Literal["raster", "snake", "spiral"] = "raster",
+) -> List[XyCoordinate]:
+    """Construct coordinates for a scan, returning a 1D list.
+
+    This is the same set of coordinates returned by `construct_grid`
+    but the list-of-lists is flattened to a simple list.
+    """
+    path = []
+    grid = construct_grid(
+        initial=initial, step_sizes=step_sizes, n_steps=n_steps, style=style
+    )
+    for line in grid:
+        path += line  # concatenate the lists
+    return path
+
+
+def closest_point_in_xy(
+    current_position: XyCoordinate, points: List[XyzCoordinate]
+) -> Optional[XyzCoordinate]:
+    """Find the closest point in a list
+
+    Given a 2D position, find the 3D position that's closest in XY and return it.
+    In the event of a tie, the most recent (i.e. latest in the list) is returned.
+
+    If the list is empty, we return None
+    """
+    if len(points) < 1:
+        return None
+    points_2d = np.asarray(points)[:, :2]
+
+    squared_distances = np.sum((points_2d - current_position) ** 2, axis=1)
+    # We reverse the distances before searching, as argmin will return the first
+    # point in the event of there being multiple points with the same minimum,
+    # and we want to pick the last one.
+    reverse_min_index = np.argmin(squared_distances[::-1])
+    # of course, now we must convert the index to be the right way round
+    min_index = len(points) - 1 - reverse_min_index
+    return points[int(min_index)]  # The explicit cast is necessary for MyPy
+
+
 ### Capturing
 
 
@@ -157,6 +335,62 @@ class ScanExtension(BaseExtension):
         logging.info(progress)
         return progress
 
+    def get_autofocus_function(self, dz: int, use_fast_autofocus: bool) -> Callable:
+        """Return a function that will perform an autofocus routine.
+
+        This should be called at the start of a scan.  It will check that
+        the necessary hardware and software are present, and raise a helpful
+        error if they are not.
+        """
+        microscope = find_component("org.openflexure.microscope")
+        # Locate the autofocus extension
+        autofocus_extension = find_extension("org.openflexure.autofocus")
+        if not autofocus_extension:
+            raise RuntimeError(
+                "The Autofocus extension is missing: select 'None' as your autofocus "
+                "type to scan without autofocusing."
+            )
+        if not (microscope.has_real_stage() and microscope.has_real_camera()):
+            raise RuntimeError(
+                "A real stage and camera are needed in order to autofocus.  You can "
+                "still run a scan without autofocus."
+            )
+
+        if use_fast_autofocus:
+
+            def autofocus():
+                # Run fast autofocus. Client should provide dz ~ 2000
+                autofocus_extension.fast_autofocus(microscope, dz=dz)
+                time.sleep(0.5)
+
+            return autofocus
+        else:
+
+            def autofocus():
+                # Run slow autofocus. Client should provide dz ~ 50
+                autofocus_extension.autofocus(microscope, range(-3 * dz, 4 * dz, dz))
+                time.sleep(0.5)
+
+            return autofocus
+
+    def get_background_detect_function(self):
+        """Return a function that returns true if we are looking at background"""
+        # Check for the background detect extension, raise an error now if it's missing.
+        background_detect_extension = find_extension(
+            "org.openflexure.background-detect"
+        )
+        if not background_detect_extension:
+            raise RuntimeError(
+                "Detecting background fields requires the background detect extension and it was not found."
+            )
+
+        def current_image_is_background():
+            verdict = background_detect_extension.grab_and_classify_image()
+            logging.debug(f"Background detection verdict: {verdict}")
+            return verdict["classification"] == "background"
+
+        return current_image_is_background
+
     ### Scanning
     def tile(
         self,
@@ -175,6 +409,7 @@ class ScanExtension(BaseExtension):
         metadata: Optional[dict] = None,
         annotations: Optional[Dict[str, str]] = None,
         tags: Optional[List[str]] = None,
+        detect_empty_fields_and_skip_autofocus: bool = False,
     ):
         metadata = metadata or {}
         annotations = annotations or {}
@@ -182,17 +417,21 @@ class ScanExtension(BaseExtension):
 
         start = time.time()
 
+        # Store initial position
+        initial_position = microscope.stage.position
+        # Construct an x-y scan path (list of 2D coordinates)
+        path = construct_grid_1d(
+            initial_position[:2], stride_size[:2], grid[:2], style=style
+        )
+
         # Keep task progress
-        self._images_to_be_captured = reduce((lambda x, y: x * y), grid)
+        self._images_to_be_captured = len(path)
         self._images_captured_so_far = 0
 
         # Generate a basename if none given
         if not basename:
             basename = generate_basename()
 
-        # Store initial position
-        initial_position = microscope.stage.position
-
         # Add dataset metadata
         dataset_d = {
             "id": uuid.uuid4(),
@@ -205,93 +444,71 @@ class ScanExtension(BaseExtension):
             "autofocusDz": autofocus_dz,
         }
 
-        # Check if autofocus is enabled
+        # Perform set-up to be able to autofocus, if needed
-        autofocus_extension = find_extension("org.openflexure.autofocus")
+        autofocus: Optional[Callable] = None
-        if (
+        if autofocus_dz:
-            autofocus_dz
+            autofocus = self.get_autofocus_function(
-            and autofocus_extension
+                dz=autofocus_dz, use_fast_autofocus=fast_autofocus
-            and microscope.has_real_stage()
+            )
-            and microscope.has_real_camera()
+        if detect_empty_fields_and_skip_autofocus:
-        ):
+            current_image_is_background = self.get_background_detect_function()
-            autofocus_enabled = True
         else:
-            autofocus_enabled = False
+            current_image_is_background = None
-
+        focus_manager = FocusManager(
-        # Construct an x-y grid (worry about z later)
+            microscope,
-        x_y_grid = construct_grid(
+            initial_position,
-            initial_position[:2], stride_size[:2], grid[:2], style=style
+            autofocus_function=autofocus,
+            current_image_is_background_function=current_image_is_background,
+            axial_jump_threshold=0.4
+            if detect_empty_fields_and_skip_autofocus
+            else None,
         )
 
-        # Keep the initial Z position the same as our current position
-        initial_z = initial_position[2]
-        next_z = initial_z  # Save this value for use in raster scans
-
         # Now step through each point in the x-y coordinate array
-        for line in x_y_grid:
+        for x_y in path:
-            # If rastering, rather than snake (or eventually spiral)
+            next_z = focus_manager.estimate_z(x_y)
-            # Return focus to initial position
+            # Move to new grid position
-            if style == "raster":
+            logging.debug("Moving to step %s", ([x_y[0], x_y[1], next_z]))
-                next_z = initial_z  # Reset z position at start of each new row
+            microscope.stage.move_abs((x_y[0], x_y[1], next_z))
-                logging.debug("Returning to initial z position")
+            # Autofocus (if requested)
-                microscope.stage.move_abs(
+            focus_manager.autofocus()
-                    (line[0][0], line[0][1], next_z)
-                )  # RWB: I think this line is redundant
 
-            for x_y in line:
+            # If we're not doing a z-stack, just capture
-                # Move to new grid position without changing z
+            if grid[2] <= 1:
-                logging.debug("Moving to step %s", ([x_y[0], x_y[1], next_z]))
+                self.capture(
-                microscope.stage.move_abs((x_y[0], x_y[1], next_z))
+                    microscope,
-                # Refocus
+                    basename,
-                if autofocus_enabled:
+                    namemode=namemode,
-                    if fast_autofocus:
+                    temporary=temporary,
-                        # Run fast autofocus. Client should provide dz ~ 2000
+                    use_video_port=use_video_port,
-                        autofocus_extension.fast_autofocus(microscope, dz=autofocus_dz)
+                    resize=resize,
-                    else:
+                    bayer=bayer,
-                        # Run slow autofocus. Client should provide dz ~ 50
+                    dataset=dataset_d,
-                        autofocus_extension.autofocus(
+                    annotations=annotations,
-                            microscope,
+                    tags=tags,
-                            range(-3 * autofocus_dz, 4 * autofocus_dz, autofocus_dz),
+                )
-                        )
+                # Update task progress
-                        logging.debug("Finished autofocus")
+                self._images_captured_so_far += 1
-                        time.sleep(1)
+                update_action_progress(self.progress())
-
+            else:
-                # If we're not doing a z-stack, just capture
+                logging.debug("Entering z-stack")
-                if grid[2] <= 1:
+                self.stack(
-                    self.capture(
+                    microscope=microscope,
-                        microscope,
+                    basename=basename,
-                        basename,
+                    namemode=namemode,
-                        namemode=namemode,
+                    temporary=temporary,
-                        temporary=temporary,
+                    step_size=stride_size[2],
-                        use_video_port=use_video_port,
+                    steps=grid[2],
-                        resize=resize,
+                    use_video_port=use_video_port,
-                        bayer=bayer,
+                    resize=resize,
-                        dataset=dataset_d,
+                    bayer=bayer,
-                        annotations=annotations,
+                    dataset=dataset_d,
-                        tags=tags,
+                    annotations=annotations,
-                    )
+                    tags=tags,
-                    # Update task progress
+                )
-                    self._images_captured_so_far += 1
+            # Gracefully shut down if we have been requested to stop
-                    update_action_progress(self.progress())
+            if current_action() and current_action().stopped:
-                else:
+                return
-                    logging.debug("Entering z-stack")
-                    self.stack(
-                        microscope=microscope,
-                        basename=basename,
-                        namemode=namemode,
-                        temporary=temporary,
-                        step_size=stride_size[2],
-                        steps=grid[2],
-                        use_video_port=use_video_port,
-                        resize=resize,
-                        bayer=bayer,
-                        dataset=dataset_d,
-                        annotations=annotations,
-                        tags=tags,
-                    )
-                if current_action() and current_action().stopped:
-                    return
-                # Make sure we use our current best estimate of focus (i.e. the current position) next point
-                next_z = microscope.stage.position[2]
 
         logging.debug("Returning to %s", (initial_position))
         microscope.stage.move_abs(initial_position)
@@ -373,6 +590,7 @@ class TileScanArgs(FullCaptureArgs):
     stride_size = fields.List(
         fields.Integer, missing=[2000, 1500, 100], example=[2000, 1500, 100]
     )
+    detect_empty_fields_and_skip_autofocus = fields.Boolean(missing=False)
 
 
 class TileScanAPI(ActionView):
@@ -418,4 +636,7 @@ class TileScanAPI(ActionView):
                 fast_autofocus=args.get("fast_autofocus"),
                 annotations=args.get("annotations"),
                 tags=args.get("tags"),
+                detect_empty_fields_and_skip_autofocus=args.get(
+                    "detect_empty_fields_and_skip_autofocus"
+                ),
             )

webapp/src/components/tabContentComponents/captureComponents/paneCapture.vue

@@ -221,6 +221,17 @@
               </select>
             </div>
 
+            <div class="uk-margin-small uk-margin-remove-bottom" v-if="backgroundDetectUri">
+              <label class="uk-form-label" for="form-stacked-text">
+                <input
+                  v-model="detectEmptyFieldsAndSkipAutofocus"
+                  class="uk-checkbox"
+                  type="checkbox"
+                />
+                Skip autofocus for empty fields of view
+              </label>
+            </div>
+
             <div class="uk-margin-small uk-margin-remove-bottom">
               <label class="uk-form-label" for="form-stacked-text"
                 >Scan Style</label
@@ -365,6 +376,7 @@ function defaultCaptureSettings() {
     annotations: {
       Client: "openflexure-microscope-jsclient:builtin"
     },
+    detectEmptyFieldsAndSkipAutofocus: false,
     smartStack: false,
     smartStackThreshold: 0.9,
     smartStackPeakWidth: 200,
@@ -391,7 +403,8 @@ export default {
       ...defaultCaptureSettings(),
       scanCapture: false, // Don't remember the "scan" tickbox in local storage.
       scanUri: null,
-      smartScanUri: null
+      smartScanUri: null,
+      backgroundDetectUri: null
     };
   },
 
@@ -461,7 +474,8 @@ export default {
         style: this.scanStyle.toLowerCase(),
         namemode: this.namingStyle.toLowerCase(),
         autofocus_dz: afDeltas[this.scanDeltaZ],
-        fast_autofocus: this.scanDeltaZ == "Fast"
+        fast_autofocus: this.scanDeltaZ == "Fast",
+        detect_empty_fields_and_skip_autofocus: this.detectEmptyFieldsAndSkipAutofocus
       };
     },
     smartScanPayload: function() {
@@ -508,6 +522,10 @@ export default {
     },
 
     updateScanUri: function() {
+      // Update URIs for the various functions in plugins.
+      // This will only work if the plugins are present, so
+      // checking if the URIs is null will determine which
+      // extensions are available.
       axios
         .get(this.pluginsUri) // Get a list of plugins
         .then(response => {
@@ -540,6 +558,22 @@ export default {
         .catch(error => {
           this.modalError(error); // Let mixin handle error
         });
+
+      axios
+        .get(this.pluginsUri) // Get a list of plugins
+        .then(response => {
+          var plugins = response.data;
+          var foundExtension = plugins.find(
+            e => e.title === "org.openflexure.background-detect"
+          );
+          if (foundExtension) {
+            // Get plugin action link
+            this.backgroundDetectUri = foundExtension.links.grab_and_classify_image.href;
+          }
+        })
+        .catch(error => {
+          this.modalError(error); // Let mixin handle error
+        });
     },
 
     onScanResponse: function() {