Merge branch 'bg_detect_in_camera' into 'v3'

Refactor background detect to allow switching algorithms

See merge request openflexure/openflexure-microscope-server!327

ofm_config_full.json

@@ -11,8 +11,7 @@
             "kwargs": {
                 "scans_folder": "/var/openflexure/scans/"
             }
-        },
+        }
-        "/background_detect/": "openflexure_microscope_server.things.background_detect:BackgroundDetectThing"
     },
     "settings_folder": "/var/openflexure/settings/",
     "log_folder": "/var/openflexure/logs/"

ofm_config_simulation.json

@@ -11,8 +11,7 @@
             "kwargs": {
                 "scans_folder": "./openflexure/scans/"
             }
-        },
+        }
-        "/background_detect/": "openflexure_microscope_server.things.background_detect:BackgroundDetectThing"
     },
     "settings_folder": "./openflexure/settings/",
     "log_folder": "./openflexure/logs/"

src/openflexure_microscope_server/background_detect.py

@@ -0,0 +1,253 @@
+"""Provide functionality to detect if the camera is imaging sample or background.
+
+An example background image is captured by the camera and sent to classes in the module
+for analysis. Information from these images is used to detect whether an image from the
+current camera field of view contains sample.
+"""
+
+from typing import Optional, Any
+import cv2
+import numpy as np
+from pydantic import BaseModel
+from pydantic.errors import PydanticUserError
+from scipy.stats import norm
+from labthings_fastapi.thing_description import type_to_dataschema
+
+
+class MissingBackgroundDataError(RuntimeError):
+    """An error raised if checking for sample without background data set."""
+
+
+class BackgroundDetectorStatus(BaseModel):
+    """The status information about a background detector instance needed for the GUI.
+
+    Each BackgroundDetectAlgorithm must be able to return one of these models when
+    ``status`` is called.
+    """
+
+    ready: bool
+    """True if ready to be used, if False this detector isn't initialised for use.
+
+    This could be called ``has_background_data`` or similar, but the more generic
+    ``ready`` is used in case more complex methods are added in the future, which
+    need different initialisation.
+    """
+    settings: BaseModel
+    """The settings for this this background detect Algorithm"""
+
+    # Setting schema is a dict until LabThings FastAPI issue #154 is fixed and
+    # DataSchema can be used directly. For now `model_dump()` must be used to dump schema
+    # to a dict.
+    settings_schema: dict[str, Any]
+
+
+class BackgroundDetectAlgorithm:
+    """The base class for defining background detect algorithms."""
+
+    background_data_model: BaseModel = BaseModel
+    """The data model of the background data. This must be set by child classes"""
+    settings_data_model: BaseModel = BaseModel
+    """The data model of algorithm settings. This must be set by child classes"""
+
+    def __init__(self):
+        """Initialise the algorithm settings."""
+        try:
+            self._settings: BaseModel = self.settings_data_model()
+        except PydanticUserError as e:
+            raise NotImplementedError(
+                "BackgroundDetectAlgorithms must set their own settings data model."
+            ) from e
+
+    @property
+    def status(self) -> BackgroundDetectorStatus:
+        """The status information needed for the GUI. Read only."""
+        return BackgroundDetectorStatus(
+            ready=self.background_data is not None,
+            settings=self.settings,
+            # Dump model with `model_dump()` for reason explained when defining
+            # BackgroundDetectorStatus
+            settings_schema=type_to_dataschema(self.settings_data_model).model_dump(),
+        )
+
+    # Requires a getter and a setter to support being a BaseModel but being
+    # saved to file as a dict
+    _background_data: Optional[BaseModel] = None
+
+    @property
+    def background_data(self) -> Optional[BaseModel]:
+        """The statistics of the background image."""
+        bd = self._background_data
+        if bd is None:
+            return None
+        return bd
+
+    @background_data.setter
+    def background_data(self, value: Optional[BaseModel | dict]) -> None:
+        """Set the statistics for the background image.
+
+        This should be None, of no data is available. It can be set from either
+        a dictionary or a base model of the type specified in
+        ``self.background_data_model``.
+        """
+        try:
+            if value is None:
+                self._background_data = None
+            elif isinstance(value, self.background_data_model):
+                self._background_data = value
+            elif isinstance(value, dict):
+                self._background_data = self.background_data_model(**value)
+            else:
+                raise TypeError(
+                    f"Cannot set background_data with an object of type {type(value)}"
+                )
+        except PydanticUserError as e:
+            raise NotImplementedError(
+                "BackgroundDetectAlgorithms must set their own background data model."
+            ) from e
+
+    @property
+    def settings(self) -> BaseModel:
+        """The statistics of the background image."""
+        return self._settings
+
+    @settings.setter
+    def settings(self, value: BaseModel | dict) -> None:
+        if isinstance(value, self.settings_data_model):
+            self._settings = value
+        elif isinstance(value, dict):
+            self._settings = self.settings_data_model(**value)
+        else:
+            raise TypeError(f"Cannot set settings with an object of type {type(value)}")
+
+    def image_is_sample(self, image: np.ndarray) -> tuple[bool, str]:
+        """Label the current image as either background or sample.
+
+        :returns: A tuple of the result (boolean), and explanation string. The
+            explanation string is formatted so it can be added into a sentence such as
+            ``An action was taken because the image is {message}.``
+        """
+        raise NotImplementedError(
+            "Each background detect algorithm must implement an image_is_sample method."
+        )
+
+    def set_background(self, image: np.ndarray) -> BaseModel:
+        """Use the input image to update the background data.
+
+        Background data must be a Pydantic BaseModel.
+        """
+        raise NotImplementedError(
+            "Each background detect algorithm must implement an set_background method."
+        )
+
+
+class ChannelDistributions(BaseModel):
+    """A BaseModel for storing the channel distribution of a background image."""
+
+    means: list[float]
+    """The mean of each channel in the colourspace."""
+    standard_deviations: list[float]
+    """The standard deviation of each channel in the colourspace."""
+
+
+class ColourChannelDetectSettings(BaseModel):
+    """A BaseModel for storing the settings for colour channel detectors."""
+
+    channel_tolerance: float = 7.0
+    """Channel Tolerance
+
+    The number of standard deviations a pixel value must be from the background mean
+    to be considered sample.
+    """
+    min_sample_coverage: float = 25.0
+    """Sample Coverage Required (%)
+
+    The minimum percentage of the image that needs to be identified as sample for the
+    image to be labeled as containing sample.
+    """
+
+
+class ColourChannelDetectLUV(BackgroundDetectAlgorithm):
+    """Compare images with a known background in LUV colourspace.
+
+    This uses an LUV colour space checking only the mean and standard deviation of the
+    U and V channels. The LUV colourspace as it collect colours together in a human-
+    intuitive way.
+    """
+
+    background_data_model: BaseModel = ChannelDistributions
+    settings_data_model: BaseModel = ColourChannelDetectSettings
+
+    def background_mask(self, image: np.ndarray) -> np.ndarray:
+        """Calculate a binary image, showing whether each pixel is background.
+
+        True is background.
+
+        The image should be in LUV format, the output will be binary with the
+        same shape in the first two dimensions.
+        """
+        if not self.background_data:
+            raise MissingBackgroundDataError(
+                "Background is not set: you need to calibrate background detection."
+            )
+
+        # The ``[1:]`` selects only the U and V channels of the image.
+        # Only U and V are used as brightness (L) often changes as
+        # the height of the sample changes.
+        # Wrapping in ``[[ ]]`` forces the colour channels to the numpy axis 2
+        # (3rd axis) so they are compared to the colour channel of each pixel.
+        means = np.array([[self.background_data.means[1:]]])
+        stds = np.array([[self.background_data.standard_deviations[1:]]])
+
+        # Compare each image in the pixel with the mean and standard deviation along
+        # axis to (the colour channels).
+        return np.all(
+            np.abs(image[:, :, 1:] - means) < stds * self.settings.channel_tolerance,
+            axis=2,
+        )
+
+    def get_sample_coverage(self, image: np.ndarray) -> float:
+        """Return the percentage of the input image that is background.
+
+        Evaluate whether it is foreground or background by comparing it to the saved
+        statistics for a background image on a per-pixel basis
+
+        :returns: A value (between 0 and 100) is the percentage of the image that is
+            sample.
+        """
+        image_luv = cv2.cvtColor(image, cv2.COLOR_RGB2LUV)
+        mask = self.background_mask(image_luv)
+
+        return (1 - np.count_nonzero(mask) / np.prod(mask.shape)) * 100
+
+    def image_is_sample(self, image: np.ndarray) -> tuple[bool, str]:
+        """Label the current image as either background or sample.
+
+        :returns: A tuple of the result (boolean), and explanation string. The
+            explanation string is formatted so it can be added into a sentence such as
+            ``An action was taken because the image is {message}.``
+        """
+        sample_coverage = self.get_sample_coverage(image)
+
+        # Use bool otherwise get numpy variants of True and False.
+        is_sample = bool(sample_coverage > self.settings.min_sample_coverage)
+        message = f"{sample_coverage:0.1f}% sample"
+        if not is_sample:
+            message = "only " + message
+        return is_sample, message
+
+    def set_background(self, image: np.ndarray) -> None:
+        """Use the input image to update the background distributions."""
+        image_luv = cv2.cvtColor(image, cv2.COLOR_RGB2LUV)
+
+        ch1 = (image_luv.T[0]).flatten()
+        ch2 = (image_luv.T[1]).flatten()
+        ch3 = (image_luv.T[2]).flatten()
+
+        points = np.array([np.asarray(ch1), np.asarray(ch2), np.asarray(ch3)]).T
+
+        # Get the mean and standard deviation of values in each channel
+        mu, std = np.apply_along_axis(norm.fit, 0, points)
+
+        self.background_data = ChannelDistributions(
+            means=mu.tolist(), standard_deviations=std.tolist()
+        )

src/openflexure_microscope_server/things/background_detect.py

@@ -1,163 +0,0 @@
-"""Provide functionality to detect if the camera is imaging sample or background.
-
-An example background image must be captured and analysed by BackgroundDetectThing,
-information from this images is used to detect whether the current camera field of
-view contains sample.
-"""
-
-from typing import Mapping, Optional
-import cv2
-import numpy as np
-from PIL import Image
-from pydantic import BaseModel
-from scipy.stats import norm
-
-import labthings_fastapi as lt
-from .camera import CameraDependency as CamDep
-
-
-class ChannelDistributions(BaseModel):
-    """A BaseModel for storing the channel distribution of a background image."""
-
-    means: list[float]
-    """The mean of each channel in the colourspace."""
-    standard_deviations: list[float]
-    """The standard deviation of each channel in the colourspace."""
-    colorspace: str = "LUV"
-    """The colourspace used."""
-
-
-class BackgroundDetectThing(lt.Thing):
-    """Thing for setting a background image and detecting sample in the field of view.
-
-    This uses an LUV colour space checking only the mean and standard deviation of the
-    UV channels. Over time different, selectable, background detection methods will be
-    added.
-    """
-
-    # Requires a getter and a setter to support being a BaseModel but being
-    # saved to file as a dict
-    _background_distributions: Optional[ChannelDistributions] = None
-
-    @lt.thing_setting
-    def background_distributions(self) -> Optional[ChannelDistributions]:
-        """The statistics of the background image."""
-        bd = self._background_distributions
-        if bd is None:
-            return None
-        return ChannelDistributions(**bd)
-
-    @background_distributions.setter
-    def background_distributions(
-        self, value: Optional[ChannelDistributions | dict]
-    ) -> None:
-        if value is None:
-            self._background_distributions = None
-        elif isinstance(value, ChannelDistributions):
-            self._background_distributions = value.model_dump()
-        elif isinstance(value, dict):
-            self._background_distributions = value
-        else:
-            raise TypeError(
-                f"Cannot set background_distributions with an object of type {type(value)}"
-            )
-
-    tolerance = lt.ThingSetting(
-        initial_value=7.0,
-        model=float,
-    )
-    """How many standard deviations to allow for the background."""
-
-    fraction = lt.ThingSetting(
-        initial_value=25.0,
-        model=float,
-    )
-    """How much of the image needs to be not background to label as sample"""
-
-    def background_mask(self, image: np.ndarray) -> np.ndarray:
-        """Calculate a binary image, showing whether each pixel is background.
-
-        The image should be in LUV format, the output will be binary with the
-        same shape in the first two dimensions.
-        """
-        d = self.background_distributions
-        if not d:
-            raise RuntimeError(
-                "Background is not set: you need to calibrate background detection."
-            )
-        # This image is in LUV space. But the brightness (L) often changes as the
-        # height of the sample changes. Hence in the line below we are only using
-        # the UV (colour) channels.
-        return np.all(
-            np.abs(image[:, :, 1:] - np.array(d.means[1:])[np.newaxis, np.newaxis, :])
-            < np.array(d.standard_deviations[1:])[np.newaxis, np.newaxis, :]
-            * self.tolerance,
-            axis=2,
-        )
-
-    @lt.thing_action
-    def background_fraction(self, cam: CamDep) -> float:
-        """Determine what fraction of the current image is background.
-
-        This action will acquire a new image from the preview stream, then
-        evaluate whether it is foreground or background, by comparing it
-        too the saved statistics. This is done on a per-pixel basis, and
-        the returned value (between 0 and 100) is the fraction of the image
-        that is background.
-        """
-        current_image = cam.grab_jpeg()
-        current_image = np.array(Image.open(current_image.open()))
-
-        # we're working in the LUV colourspace as it collect colours together in a human-intuitive way
-        current_image_luv = cv2.cvtColor(current_image, cv2.COLOR_RGB2LUV)
-        mask = self.background_mask(current_image_luv)
-        return np.count_nonzero(mask) / np.prod(mask.shape) * 100
-
-    @lt.thing_action
-    def image_is_sample(self, cam: CamDep) -> bool:
-        """Label the current image as either background or sample."""
-        b_fraction = self.background_fraction(cam)
-        fraction_threshold = self.fraction
-
-        return (100 - b_fraction) > fraction_threshold
-
-    @lt.thing_action
-    def set_background(self, cam: CamDep):
-        """Grab an image, and use its statistics to set the background.
-
-        This should be run when the microscope is looking at an empty region,
-        and will calculate the mean and standard deviation of the pixel values
-        in the LUV colourspace. These values will then be used to compare
-        future images to the distribution, to determine if each pixel is
-        foreground or background.
-        """
-        background = cam.grab_jpeg()
-        background = np.array(Image.open(background.open()))
-
-        # we're working in the LUV colourspace as it collect colours together in a human-intuitive way
-        background_luv = cv2.cvtColor(background, cv2.COLOR_RGB2LUV)
-
-        ch1 = (background_luv.T[0]).flatten()
-        ch2 = (background_luv.T[1]).flatten()
-        ch3 = (background_luv.T[2]).flatten()
-
-        points = np.array([np.asarray(ch1), np.asarray(ch2), np.asarray(ch3)]).T
-
-        # we get the mean and standard deviation of values in each channel
-        mu, std = np.apply_along_axis(norm.fit, 0, points)
-
-        self.background_distributions = ChannelDistributions(
-            means=mu.tolist(),
-            standard_deviations=std.tolist(),
-            colorspace="LUV",
-        )
-
-    @property
-    def thing_state(self) -> Mapping:
-        """Summary metadata describing the current state of the Thing."""
-        bd = self.background_distributions
-        return {
-            "background_distributions": bd.model_dump() if bd else None,
-            "tolerance": self.tolerance,
-            "fraction": self.fraction,
-        }

src/openflexure_microscope_server/things/camera/__init__.py

@@ -10,6 +10,7 @@ from __future__ import annotations
 from typing import Literal, Optional, Tuple, Any
 import json
 import time
+import logging
 
 import numpy as np
 from pydantic import RootModel
@@ -19,6 +20,14 @@ import piexif
 import labthings_fastapi as lt
 from labthings_fastapi.types.numpy import NDArray
 
+from openflexure_microscope_server.background_detect import (
+    ColourChannelDetectLUV,
+    BackgroundDetectAlgorithm,
+    BackgroundDetectorStatus,
+)
+
+LOGGER = logging.getLogger(__name__)
+
 
 class JPEGBlob(lt.blob.Blob):
     """A class representing a JPEG image as a LabThings FastAPI Blob."""
@@ -155,6 +164,18 @@ class BaseCamera(lt.Thing):
     lores_mjpeg_stream = lt.outputs.MJPEGStreamDescriptor()
     _memory_buffer = CameraMemoryBuffer()
 
+    def __init__(self):
+        """Initialise the base camera, this creates the background detectors.
+
+        This must be run by all child camera classes.
+
+        To add a new background detector to the server it must be added to the
+        dictionary in this function. Configuration will be added at a later date.
+        """
+        super().__init__()
+        self.background_detectors = {"Colour Channels (LUV)": ColourChannelDetectLUV()}
+        self._detector_name = "Colour Channels (LUV)"
+
     def __enter__(self) -> None:
         """Open hardware connection when the Thing context manager is opened."""
         raise NotImplementedError("CameraThings must define their own __enter__ method")
@@ -455,6 +476,86 @@ class BaseCamera(lt.Thing):
         time.sleep(self.settling_time)
         self.discard_frames()
 
+    # Note that the default detector name is set at init. This is over written if
+    # setting is loaded from disk.
+    @lt.thing_setting
+    def detector_name(self) -> str:
+        """The name of the active background selector."""
+        return self._detector_name
+
+    @detector_name.setter
+    def detector_name(self, name: str) -> None:
+        """Validate and set detector_name."""
+        if name not in self.background_detectors:
+            raise ValueError(f"{name} is not a valid background detector name")
+        self._detector_name = name
+
+    @property
+    def active_detector(self) -> BackgroundDetectAlgorithm:
+        """The active background detector instance."""
+        return self.background_detectors[self.detector_name]
+
+    @lt.thing_property
+    def background_detector_status(self) -> BackgroundDetectorStatus:
+        """The status of the active detector for the UI."""
+        return self.active_detector.status
+
+    @lt.thing_setting
+    def background_detector_data(self) -> dict:
+        """The data for each background detector, used to save to disk."""
+        data = {}
+        for name, obj in self.background_detectors.items():
+            bg_data = (
+                None
+                if obj.background_data is None
+                else obj.background_data.model_dump()
+            )
+            data[name] = {
+                "settings": obj.settings.model_dump(),
+                "background_data": bg_data,
+            }
+        return data
+
+    @background_detector_data.setter
+    def background_detector_data(self, data: dict) -> None:
+        """Set the data for each detector. Only to be used as settings are loaded from disk.
+
+        Do not call over HTTP. This needs to be updated once LbaThings Settings can be
+        read-only over HTTP (#484).
+        """
+        for name, instance_data in data.items():
+            if name in self.background_detectors:
+                obj = self.background_detectors[name]
+                obj.settings = instance_data["settings"]
+                obj.background_data = instance_data["background_data"]
+            else:
+                LOGGER.warning(
+                    f"No background detector named {name}, settings will be discarded."
+                )
+
+    @lt.thing_action
+    def image_is_sample(self, portal: lt.deps.BlockingPortal) -> tuple[bool, str]:
+        """Label the current image as either background or sample."""
+        current_image = self.grab_jpeg(portal)
+        current_image = np.array(Image.open(current_image.open()))
+        return self.active_detector.image_is_sample(current_image)
+
+    @lt.thing_action
+    def set_background(self, portal: lt.deps.BlockingPortal) -> None:
+        """Grab an image, and use its statistics to set the background.
+
+        This should be run when the microscope is looking at an empty region,
+        and will calculate the mean and standard deviation of the pixel values
+        in the LUV colourspace. These values will then be used to compare
+        future images to the distribution, to determine if each pixel is
+        foreground or background.
+        """
+        background = self.grab_jpeg(portal)
+        background = np.array(Image.open(background.open()))
+        self.active_detector.set_background(background)
+        # Manually save settings as the setter is not called.
+        self.save_settings()
+
 
 CameraDependency = lt.deps.direct_thing_client_dependency(BaseCamera, "/camera/")
 RawCameraDependency = lt.deps.raw_thing_dependency(BaseCamera)

src/openflexure_microscope_server/things/camera/opencv.py

@@ -30,6 +30,7 @@ class OpenCVCamera(BaseCamera):
 
         :param camera_index: The index of the camera to use for the microscope.
         """
+        super().__init__()
         self.camera_index = camera_index
         self._capture_thread: Optional[Thread] = None
         self._capture_enabled = False

src/openflexure_microscope_server/things/camera/picamera.py

@@ -118,6 +118,7 @@ class StreamingPiCamera2(BaseCamera):
         :param camera_num: The number of the camera. This should generally be left as 0
             as most Raspberry Pi boards only support 1 camera.
         """
+        super().__init__()
         self._setting_save_in_progress = False
         self.camera_num = camera_num
         self.camera_configs: dict[str, dict] = {}
@@ -736,7 +737,7 @@ class StreamingPiCamera2(BaseCamera):
             self._initialise_picamera()
 
     @lt.thing_action
-    def full_auto_calibrate(self) -> None:
+    def full_auto_calibrate(self, portal: lt.deps.BlockingPortal) -> None:
         """Perform a full auto-calibration.
 
         This function will call the other calibration actions in sequence:
@@ -746,12 +747,14 @@ class StreamingPiCamera2(BaseCamera):
         * ``set_static_green_equalisation`` to set geq offset to max
         * ``calibrate_lens_shading``
         * ``calibrate_white_balance``
+        * ``set_background``
         """
         self.flat_lens_shading()
         self.auto_expose_from_minimum()
         self.set_static_green_equalisation()
         self.calibrate_lens_shading()
         self.calibrate_white_balance()
+        self.set_background(portal)
 
     @lt.thing_action
     def flat_lens_shading(self) -> None:

src/openflexure_microscope_server/things/camera/simulation.py

@@ -58,6 +58,7 @@ class SimulatedCamera(BaseCamera):
         :param frame_interval: Nominally the time between frames on the MJPEG stream,
             however the rate may be slower due to calculation time for focus.
         """
+        super().__init__()
         self.shape = shape
         self.glyph_shape = glyph_shape
         self.canvas_shape = canvas_shape

src/openflexure_microscope_server/things/smart_scan.py

@@ -1,7 +1,7 @@
 """The core sample scanning functionality for the OpenFlexure Microscope.
 
 SmartScan provides sample scanning functionality including automatic background
-detection (via the `BackgroundDetectThing`) and automatic path planning via
+detection (via the ``CameraThing``) and automatic path planning via
 `scan_planners`. It manages the directories of past scans via `scan_directories`.
 It also controls external processes for live stitching composite images, and
 the creation of the final stitched images.
@@ -29,7 +29,6 @@ from openflexure_microscope_server import scan_planners
 # Things
 from .autofocus import AutofocusThing
 from .camera_stage_mapping import CameraStageMapper
-from .background_detect import BackgroundDetectThing
 from .camera import CameraDependency as CamDep
 from .stage import StageDependency as StageDep
 
@@ -37,9 +36,6 @@ CSMDep = lt.deps.direct_thing_client_dependency(
     CameraStageMapper, "/camera_stage_mapping/"
 )
 AutofocusDep = lt.deps.direct_thing_client_dependency(AutofocusThing, "/autofocus/")
-BackgroundDep = lt.deps.direct_thing_client_dependency(
-    BackgroundDetectThing, "/background_detect/"
-)
 
 JPEGBlob = lt.blob.blob_type("image/jpeg")
 ZipBlob = lt.blob.blob_type("application/zip")
@@ -110,7 +106,6 @@ class SmartScanThing(lt.Thing):
         self._cam: Optional[CamDep] = None
         self._metadata_getter: Optional[lt.deps.GetThingStates] = None
         self._csm: Optional[CSMDep] = None
-        self._background_detect: Optional[BackgroundDep] = None
         self._ongoing_scan: Optional[scan_directories.ScanDirectory] = None
         self._starting_position: Optional[Mapping[str, int]] = None
         self._capture_thread: Optional[ErrorCapturingThread] = None
@@ -128,14 +123,13 @@ class SmartScanThing(lt.Thing):
         cam: CamDep,
         metadata_getter: lt.deps.GetThingStates,
         csm: CSMDep,
-        background_detect: BackgroundDep,
         scan_name: str = "",
     ):
         """Move the stage to cover an area, taking images that can be tiled together.
 
         The stage will move in a pattern that grows outwards from the starting point,
         stopping once it is surrounded by "background" (as detected by the
-        background_detect Thing) or reaches the "max_range" measured in steps.
+        camera Thing) or reaches the "max_range" measured in steps.
         """
         got_lock = self._scan_lock.acquire(timeout=0.1)
         if not got_lock:
@@ -149,7 +143,6 @@ class SmartScanThing(lt.Thing):
         self._cam = cam
         self._metadata_getter = metadata_getter
         self._csm = csm
-        self._background_detect = background_detect
         self._capture_thread = None
         self._scan_images_taken = 0
 
@@ -183,7 +176,6 @@ class SmartScanThing(lt.Thing):
             self._cam = None
             self._metadata_getter = None
             self._csm = None
-            self._background_detect = None
             self._capture_thread = None
             self._ongoing_scan = None
             self._scan_images_taken = None
@@ -207,7 +199,7 @@ class SmartScanThing(lt.Thing):
             )
 
         if self.skip_background:
-            if not self._background_detect.background_distributions:
+            if not self._cam.background_detector_status.ready:
                 raise RuntimeError(
                     "Background is not set: you need to calibrate background detection."
                 )
@@ -511,15 +503,13 @@ class SmartScanThing(lt.Thing):
             capture_image = True
             # If skipping background, take an image to check if current field of view is background
             if self._scan_data["skip_background"]:
-                capture_image = self._background_detect.image_is_sample()
+                capture_image, bg_message = self._cam.image_is_sample()
 
             if not capture_image:
                 route_planner.mark_location_visited(
                     new_pos_xyz, imaged=False, focused=False
                 )
-                # Background fraction is actually a percentage
+                msg = f"Skipping {new_pos_xyz} as it is {bg_message}."
-                back_perc = round(self._background_detect.background_fraction(), 0)
-                msg = f"Skipping {new_pos_xyz} as it is {back_perc}% background."
                 self._scan_logger.info(msg)
                 continue
 

tests/mock_things/mock_background_detect.py

@@ -1,25 +0,0 @@
-"""Testing submodule with mock Background Detect Things.
-
-These mocks are designed to be inserted as dependencies to provide specific
-functionality and return values.
-
-The mocks do not subclass Things. Instead, they return predefined
-answers to functions.
-"""
-
-from openflexure_microscope_server.things.background_detect import ChannelDistributions
-
-
-class MockBackgroundDetectThing:
-    """A mock background detect Thing that imports no code from BackgroundDetectThing.
-
-    The class needs functionality added to it over time as more complex
-    mocking is needed. It imports no code from BackgroundDetectThing so that coverage
-    is not artificially inflated.
-    """
-
-    background_distributions = ChannelDistributions(
-        means=[128.0, 128.0, 128.0],
-        standard_deviations=[3.0, 3.0, 3.0],
-        colorspace="LUV",
-    )

tests/mock_things/mock_camera.py

@@ -0,0 +1,28 @@
+"""Testing submodule with mock Camera Things.
+
+These mocks are designed to be inserted as dependencies to provide specific
+functionality and return values.
+
+The mocks do not subclass Things. Instead, they return predefined
+answers to functions.
+"""
+
+from openflexure_microscope_server.background_detect import (
+    BackgroundDetectorStatus,
+    ColourChannelDetectSettings,
+)
+
+
+class MockCameraThing:
+    """A mock camera Thing that imports no code from ``BaseCamera``.
+
+    The class needs functionality added to it over time as more complex
+    mocking is needed. It imports no code from ``BaseCamera or any other
+    camera Thing, so that coverage is not artificially inflated.
+    """
+
+    background_detector_status = BackgroundDetectorStatus(
+        ready=True,
+        settings=ColourChannelDetectSettings(),
+        settings_schema={"fake": "schema"},
+    )

tests/test_background_detectors.py

@@ -0,0 +1,182 @@
+"""Test the background detection algorithms.
+
+This tests both the base class an individual algorithms.
+"""
+
+import re
+
+import pytest
+from pydantic import BaseModel
+import numpy as np
+
+from openflexure_microscope_server.background_detect import (
+    MissingBackgroundDataError,
+    BackgroundDetectAlgorithm,
+    ChannelDistributions,
+    ColourChannelDetectSettings,
+    ColourChannelDetectLUV,
+)
+
+RNG = np.random.default_rng()
+IMG_SHAPE = (820, 616, 3)
+BG_COLOR = [220, 215, 217]
+
+PERC_REGEX = re.compile(r"(\d+\.+\d)+%")
+
+
+@pytest.fixture
+def background_image():
+    """Generate a numpy array that simulates a background image."""
+    image = np.ones(IMG_SHAPE, dtype=np.int16)
+    image[:, :, 0] *= BG_COLOR[0]
+    image[:, :, 1] *= BG_COLOR[1]
+    image[:, :, 2] *= BG_COLOR[2]
+    image += RNG.normal(scale=3, size=IMG_SHAPE).astype("int16")
+    image[image < 0] = 0
+    image[image > 255] = 255
+    return image.astype("uint8")
+
+
+@pytest.fixture
+def sample_image(background_image):
+    """Generate a numpy array of a simulated image where 50% is a block of red."""
+    image = background_image.copy()
+    x_cent = IMG_SHAPE[0] // 2
+    image[:x_cent, :, 0] -= 180
+    return image
+
+
+def test_bg_detect_base_class():
+    """Test the base class for background detect.
+
+    If initialised as is it should raise not implemented error.
+    """
+    with pytest.raises(NotImplementedError):
+        BackgroundDetectAlgorithm()
+
+
+def test_partial_base_class(background_image):
+    """Create a partial class to initialise the base class and test other methods.
+
+    This test is to check that if the necessary methods are not set, that an
+    appropriate error is raised.
+    """
+
+    class BadAlgo1(BackgroundDetectAlgorithm):
+        """Only has a settings model so it can initialise."""
+
+        settings_data_model: BaseModel = ColourChannelDetectSettings
+
+    bad_algo1 = BadAlgo1()
+    status = bad_algo1.status
+    assert not status.ready
+    assert isinstance(status.settings, ColourChannelDetectSettings)
+
+    with pytest.raises(NotImplementedError):
+        # Should error on any dictionary input. This simulates loading settings from
+        # disk
+        bad_algo1.background_data = {"key": 1}
+
+    with pytest.raises(NotImplementedError):
+        bad_algo1.set_background(background_image)
+
+    with pytest.raises(NotImplementedError):
+        bad_algo1.image_is_sample(background_image)
+
+
+def test_colour_channel_luv(background_image, sample_image):
+    """Test measuring if a sample is background."""
+    cc_luv = ColourChannelDetectLUV()
+
+    # No background data so it is not ready and will error if image_is_sample is called.
+    assert not cc_luv.status.ready
+    with pytest.raises(MissingBackgroundDataError):
+        cc_luv.image_is_sample(background_image)
+
+    # Set the background
+    cc_luv.set_background(background_image)
+    # Now it is ready
+    assert cc_luv.status.ready
+    sample, message = cc_luv.image_is_sample(background_image)
+    assert not sample
+    assert "0.0%" in message
+    sample, message = cc_luv.image_is_sample(sample_image)
+    assert sample
+    match = PERC_REGEX.search(message)
+    assert match is not None
+    # Should be 50% background. Allowing 49.8-50.2% due to noise.
+    assert 49.8 < float(match.group(1)) < 50.2
+
+    # Require 75% coverage
+    cc_luv.settings = ColourChannelDetectSettings(min_sample_coverage=75.0)
+
+    sample, message = cc_luv.image_is_sample(sample_image)
+    # No longer detected as a sample.
+    assert not sample
+    match = PERC_REGEX.search(message)
+    assert match is not None
+    # Still 50% background.
+    assert 49.8 < float(match.group(1)) < 50.2
+
+
+def test_colour_channel_luv_save_load(background_image, sample_image):
+    """Get settings and data as dicts, and creating new instance using these dicts.
+
+    This is how the camera will load/save settings from/to disk.
+    """
+    cc_luv = ColourChannelDetectLUV()
+    cc_luv.set_background(background_image)
+
+    # Check types for background data
+    assert cc_luv.background_data_model is ChannelDistributions
+    assert isinstance(cc_luv.background_data, cc_luv.background_data_model)
+    # Change Settings
+    cc_luv.settings = ColourChannelDetectSettings(min_sample_coverage=10.0)
+
+    setting_dict = cc_luv.settings.model_dump()
+    data_dict = cc_luv.background_data.model_dump()
+
+    # Remove the old detector so we don't accidentally use it!
+    del cc_luv
+
+    # Create a new instance
+    cc_luv2 = ColourChannelDetectLUV()
+    assert not cc_luv2.status.ready
+    # Load settings and channels from dictionary as the camera will do on init.
+    cc_luv2.settings = setting_dict
+    cc_luv2.background_data = data_dict
+
+    # Now should be ready to use
+    assert cc_luv2.status.ready
+    assert cc_luv2.settings.min_sample_coverage == 10.0
+    sample, _ = cc_luv2.image_is_sample(sample_image)
+    assert sample
+
+    # Remove the 2nd detector so we don't accidentally use it!
+    del cc_luv2
+
+    # One final test that None can be set explicitly to background data as this will
+    # happen if loading with background detect not saved.
+    cc_luv3 = ColourChannelDetectLUV()
+    assert not cc_luv3.status.ready
+    # Load settings and channels from dictionary as the camera will do on init.
+    cc_luv3.settings = setting_dict
+    cc_luv3.background_data = None
+    # Still not ready
+    assert not cc_luv3.status.ready
+
+
+def test_colour_channel_luv_load_bad_data():
+    """Check a type error is thrown on bad data input."""
+
+    class WrongModel(BaseModel):
+        """Using a different BaseModel as this is most likely to cause confusion."""
+
+        prop1: int = 8
+        prop2: str = "foo"
+
+    cc_luv = ColourChannelDetectLUV()
+    with pytest.raises(TypeError):
+        cc_luv.settings = WrongModel()
+    with pytest.raises(TypeError):
+        cc_luv.background_data = WrongModel()

tests/test_smart_scan.py

@@ -30,7 +30,7 @@ from openflexure_microscope_server.things.smart_scan import (
 from .mock_things.mock_csm import MockCSMThing
 from .mock_things.mock_autofocus import MockAutoFocusThing
 from .mock_things.mock_stage import MockStageThing
-from .mock_things.mock_background_detect import MockBackgroundDetectThing
+from .mock_things.mock_camera import MockCameraThing
 
 # A global logger to pass in as an Invocation Logger
 LOGGER = logging.getLogger("mock-invocation_logger")
@@ -169,10 +169,9 @@ def _run_only_outer_scan(adjust_initial_state: Optional[Callable] = None):
     cancel_mock = 1  # not called
     af_mock = MockAutoFocusThing()
     stage_mock = MockStageThing()
-    cam_mock = 4  # not called
+    cam_mock = MockCameraThing()
     meta_mock = 5  # not called
     csm_mock = MockCSMThing()
-    bkgrnd_det_mock = MockBackgroundDetectThing()
 
     class MockedSmartScanThing(SmartScanThing):
         """Mocked version of SmartScanThing with a patched _run_scan method."""
@@ -192,7 +191,6 @@ def _run_only_outer_scan(adjust_initial_state: Optional[Callable] = None):
             assert self._cam is cam_mock
             assert self._metadata_getter is meta_mock
             assert self._csm is csm_mock
-            assert self._background_detect is bkgrnd_det_mock
             assert self._capture_thread is None
             assert self._scan_images_taken == 0
 
@@ -212,7 +210,6 @@ def _run_only_outer_scan(adjust_initial_state: Optional[Callable] = None):
             cam=cam_mock,
             metadata_getter=meta_mock,
             csm=csm_mock,
-            background_detect=bkgrnd_det_mock,
             scan_name="FooBar",
         )
     except Exception as e:
@@ -227,7 +224,6 @@ def _run_only_outer_scan(adjust_initial_state: Optional[Callable] = None):
     assert mock_ss_thing._cam is None
     assert mock_ss_thing._metadata_getter is None
     assert mock_ss_thing._csm is None
-    assert mock_ss_thing._background_detect is None
     assert mock_ss_thing._capture_thread is None
     assert mock_ss_thing._scan_images_taken is None
 

webapp/src/components/tabContentComponents/backgroundDetectComponents/paneBackgroundDetect.vue

@@ -1,44 +1,17 @@
 <template>
   <div class="uk-padding-small">
-    <div v-show="!backendOK" class="uk-alert-danger">
+    <div>
-      The background detect Thing seems to be missing or incompatible.
-    </div>
-    <div v-show="backendOK">
       <ul uk-accordion="multiple: true">
         <li>
           <a class="uk-accordion-title" href="#">Settings</a>
           <div class="uk-accordion-content">
-            <div class="uk-margin">
+            <p> TODO!</p>
-              <propertyControl
-                thing-name="background_detect"
-                property-name="tolerance"
-                label="Tolerance"
-              />
-            </div>
-            <div class="uk-margin">
-              <propertyControl
-                thing-name="background_detect"
-                property-name="fraction"
-                label="Sample Coverage Required (%)"
-              />
-            </div>
-            <div class="uk-margin">
-              <action-button
-                thing="background_detect"
-                action="background_fraction"
-                submit-label="Check Coverage"
-                :can-terminate="false"
-                :poll-interval="0.1"
-                @response="alertBackgroundFraction"
-                @error="backgroundDetectError"
-              />
-            </div>
           </div>
         </li>
       </ul>
       <div class="uk-margin">
         <action-button
-          thing="background_detect"
+          thing="camera"
           action="set_background"
           submit-label="Set Background"
           :can-terminate="false"
@@ -47,8 +20,9 @@
         />
       </div>
       <div class="uk-margin">
+        <!--Once status is read this should be disabled id not ready..-->
         <action-button
-          thing="background_detect"
+          thing="camera"
           action="image_is_sample"
           submit-label="Check Current Image"
           :can-terminate="false"
@@ -63,32 +37,19 @@
 
 <script>
 import ActionButton from "../../labThingsComponents/actionButton.vue";
-import propertyControl from "../../labThingsComponents/propertyControl.vue";
 
 export default {
   components: {
-    ActionButton,
+    ActionButton
-    propertyControl
-  },
-
-  computed: {
-    backendOK() {
-      return this.thingAvailable("background_detect");
-    }
   },
 
   methods: {
-    alertBackgroundFraction(r) {
-      let fraction = r.output;
-      // let percentage = (fraction * 100).toFixed(0);
-      this.modalNotify(`Current image is ${fraction.toFixed(0)}% background.`);
-    },
     alertBackgroundSet() {
       this.modalNotify(`Background image has been updated`);
     },
     alertImageLabel(r) {
-      let label = r.output === true ? "sample" : "background";
+      let label = r.output[0] ? "sample" : "background";
-      this.modalNotify(`Current image is ${label}`);
+      this.modalNotify(`Current image is ${label} (${r.output[1]})`);
     },
     backgroundDetectError() {
       this.modalError(