From b5586a4b320491eb14a7d314f9d542b6504f4e06 Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Wed, 16 Jul 2025 15:38:57 +0100 Subject: [PATCH 01/13] A partial refactor of background detect --- .../background_detect.py | 89 +++++++++++++++++++ .../things/camera/__init__.py | 66 ++++++++++++++ .../things/camera/opencv.py | 1 + .../things/camera/picamera.py | 1 + .../things/camera/simulation.py | 1 + .../paneBackgroundDetect.vue | 8 +- 6 files changed, 162 insertions(+), 4 deletions(-) create mode 100644 src/openflexure_microscope_server/background_detect.py diff --git a/src/openflexure_microscope_server/background_detect.py b/src/openflexure_microscope_server/background_detect.py new file mode 100644 index 00000000..b3818f1f --- /dev/null +++ b/src/openflexure_microscope_server/background_detect.py @@ -0,0 +1,89 @@ + +import cv2 +import numpy as np +from pydantic import BaseModel + +class ChannelDistributions(BaseModel): + """A BaseModel for storing the channel distribution of a background image.""" + + means: list[float] + """The mean of each channel in the colourspace.""" + standard_deviations: list[float] + """The standard deviation of each channel in the colourspace.""" + colorspace: str = "LUV" + """The colourspace used.""" + + +class BackgroundDetectLUV: + """Compare images with a known background in LUV colourspace. + + This uses an LUV colour space checking only the mean and standard deviation of the + U and V channels. The LUV colourspace as it collect colours together in a + """ + background_distributions: Optional[ChannelDistributions] = None + background_tolerance: float = 7.0 + min_sample_coverage: float = 25.0 + + def background_mask(self, image: np.ndarray) -> np.ndarray: + """Calculate a binary image, showing whether each pixel is background. + + The image should be in LUV format, the output will be binary with the + same shape in the first two dimensions. + + # human-intuitive way + """ + d = self.background_distributions + if not d: + raise RuntimeError( + "Background is not set: you need to calibrate background detection." + ) + + # Only use the U and V channels of as brightness (L) often changes as the + # height of the sample changes. + return np.all( + np.abs(image[:, :, 1:] - np.array(d.means[1:])[np.newaxis, np.newaxis, :]) + < np.array(d.standard_deviations[1:])[np.newaxis, np.newaxis, :] + * self.background_tolerance, + axis=2, + ) + + def get_sample_coverage(self, image: np.ndarray) -> float: + """Measure what percentage of the current image is background. + + * Acquire a new image from the preview stream, + * Evaluate whether it is foreground or background, by comparing it to the saved + statistics for a background image on a per-pixel basis + * Returned value (between 0 and 100) is the percentage of the image that is + background. + """ + image_luv = cv2.cvtColor(image, cv2.COLOR_RGB2LUV) + mask = self.background_mask(image_luv) + return (1 - np.count_nonzero(mask) / np.prod(mask.shape)) * 100 + + + def image_is_sample(self, image: np.ndarrayl) -> bool: + """Label the current image as either background or sample.""" + sample_coverage = self.get_sample_coverage(image) + fraction_threshold = self.fraction + + return sample_coverage > self.min_sample_coverage + + + def set_background(self, image: np.ndarray) -> np.ndarray: + + image_luv = cv2.cvtColor(image, cv2.COLOR_RGB2LUV) + + ch1 = (image_luv.T[0]).flatten() + ch2 = (image_luv.T[1]).flatten() + ch3 = (image_luv.T[2]).flatten() + + points = np.array([np.asarray(ch1), np.asarray(ch2), np.asarray(ch3)]).T + + # Get the mean and standard deviation of values in each channel + mu, std = np.apply_along_axis(norm.fit, 0, points) + + self.background_distributions = ChannelDistributions( + means=mu.tolist(), + standard_deviations=std.tolist(), + colorspace="LUV", + ) diff --git a/src/openflexure_microscope_server/things/camera/__init__.py b/src/openflexure_microscope_server/things/camera/__init__.py index dd901ed9..734c01bd 100644 --- a/src/openflexure_microscope_server/things/camera/__init__.py +++ b/src/openflexure_microscope_server/things/camera/__init__.py @@ -19,6 +19,7 @@ import piexif import labthings_fastapi as lt from labthings_fastapi.types.numpy import NDArray +from openflexure_microscope_server.background_detect import ChannelDistributions, BackgroundDetectLUV class JPEGBlob(lt.blob.Blob): """A class representing a JPEG image as a LabThings FastAPI Blob.""" @@ -155,6 +156,10 @@ class BaseCamera(lt.Thing): lores_mjpeg_stream = lt.outputs.MJPEGStreamDescriptor() _memory_buffer = CameraMemoryBuffer() + def __init__(self): + super().__init__() + self.background_detector = BackgroundDetectLUV() + def __enter__(self) -> None: """Open hardware connection when the Thing context manager is opened.""" raise NotImplementedError("CameraThings must define their own __enter__ method") @@ -455,6 +460,67 @@ class BaseCamera(lt.Thing): time.sleep(self.settling_time) self.discard_frames() + background_tolerance = lt.ThingSetting( + initial_value=7.0, + model=float, + ) + """How many standard deviations to allow for the background.""" + + min_sample_coverage = lt.ThingSetting( + initial_value=25.0, + model=float, + ) + """The percentage of the image that needs to be not be background to label as sample.""" + + # Requires a getter and a setter to support being a BaseModel but being + # saved to file as a dict + _background_distributions: Optional[ChannelDistributions] = None + + @lt.thing_setting + def background_distributions(self) -> Optional[ChannelDistributions]: + """The statistics of the background image.""" + bd = self._background_distributions + if bd is None: + return None + return ChannelDistributions(**bd) + + @background_distributions.setter + def background_distributions( + self, value: Optional[ChannelDistributions | dict] + ) -> None: + if value is None: + self._background_distributions = None + elif isinstance(value, ChannelDistributions): + self._background_distributions = value.model_dump() + elif isinstance(value, dict): + self._background_distributions = value + else: + raise TypeError( + f"Cannot set background_distributions with an object of type {type(value)}" + ) + + @lt.thing_action + def image_is_sample(self, portal: lt.deps.BlockingPortal) -> bool: + """Label the current image as either background or sample.""" + current_image = self.grab_jpeg(portal) + current_image = np.array(Image.open(current_image.open())) + return self.background_detector.image_is_sample(current_image) + + @lt.thing_action + def set_background(self, portal: lt.deps.BlockingPortal): + """Grab an image, and use its statistics to set the background. + + This should be run when the microscope is looking at an empty region, + and will calculate the mean and standard deviation of the pixel values + in the LUV colourspace. These values will then be used to compare + future images to the distribution, to determine if each pixel is + foreground or background. + """ + background = self.grab_jpeg(portal) + background = np.array(Image.open(background.open())) + self.background_detector.set_background(current_image) + + CameraDependency = lt.deps.direct_thing_client_dependency(BaseCamera, "/camera/") RawCameraDependency = lt.deps.raw_thing_dependency(BaseCamera) diff --git a/src/openflexure_microscope_server/things/camera/opencv.py b/src/openflexure_microscope_server/things/camera/opencv.py index 7f543cf1..0a3bd047 100644 --- a/src/openflexure_microscope_server/things/camera/opencv.py +++ b/src/openflexure_microscope_server/things/camera/opencv.py @@ -30,6 +30,7 @@ class OpenCVCamera(BaseCamera): :param camera_index: The index of the camera to use for the microscope. """ + super().__init() self.camera_index = camera_index self._capture_thread: Optional[Thread] = None self._capture_enabled = False diff --git a/src/openflexure_microscope_server/things/camera/picamera.py b/src/openflexure_microscope_server/things/camera/picamera.py index c676b82d..fea7b149 100644 --- a/src/openflexure_microscope_server/things/camera/picamera.py +++ b/src/openflexure_microscope_server/things/camera/picamera.py @@ -118,6 +118,7 @@ class StreamingPiCamera2(BaseCamera): :param camera_num: The number of the camera. This should generally be left as 0 as most Raspberry Pi boards only support 1 camera. """ + super().__init() self._setting_save_in_progress = False self.camera_num = camera_num self.camera_configs: dict[str, dict] = {} diff --git a/src/openflexure_microscope_server/things/camera/simulation.py b/src/openflexure_microscope_server/things/camera/simulation.py index 1bb3715c..225e474a 100644 --- a/src/openflexure_microscope_server/things/camera/simulation.py +++ b/src/openflexure_microscope_server/things/camera/simulation.py @@ -58,6 +58,7 @@ class SimulatedCamera(BaseCamera): :param frame_interval: Nominally the time between frames on the MJPEG stream, however the rate may be slower due to calculation time for focus. """ + super().__init() self.shape = shape self.glyph_shape = glyph_shape self.canvas_shape = canvas_shape diff --git a/webapp/src/components/tabContentComponents/backgroundDetectComponents/paneBackgroundDetect.vue b/webapp/src/components/tabContentComponents/backgroundDetectComponents/paneBackgroundDetect.vue index 8d6c8bac..bef06118 100644 --- a/webapp/src/components/tabContentComponents/backgroundDetectComponents/paneBackgroundDetect.vue +++ b/webapp/src/components/tabContentComponents/backgroundDetectComponents/paneBackgroundDetect.vue @@ -10,15 +10,15 @@
From 2245d9357d0d8456e4932ca0463c4e129c25bdb1 Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Thu, 17 Jul 2025 18:00:11 +0100 Subject: [PATCH 02/13] Continue to rebase background detect. --- .../background_detect.py | 166 +++++++++++++++--- .../things/background_detect.py | 163 ----------------- .../things/camera/__init__.py | 89 ++++++---- .../things/smart_scan.py | 6 +- 4 files changed, 193 insertions(+), 231 deletions(-) delete mode 100644 src/openflexure_microscope_server/things/background_detect.py diff --git a/src/openflexure_microscope_server/background_detect.py b/src/openflexure_microscope_server/background_detect.py index b3818f1f..71a1d8a9 100644 --- a/src/openflexure_microscope_server/background_detect.py +++ b/src/openflexure_microscope_server/background_detect.py @@ -1,7 +1,104 @@ +"""Provide functionality to detect if the camera is imaging sample or background. +An example background image is captured by the camera and sent to classes in the module +for analysis. Information from this images is used to detect whether an image from the +current camera field of view contains sample. +""" + +from typing import Optional import cv2 import numpy as np from pydantic import BaseModel +from pydantic.errors import PydanticUserError +from scipy.stats import norm + + +class BackgroundDetectAlgorithm: + """The base class for defining background detect algorithms.""" + + background_data_model: BaseModel + """The data model of the background data. This must be set by child classes""" + settings_data_model: BaseModel + """The data model of algorithm settings. This must be set by child classes""" + + def __init__(self): + """Initialise the algorithm settings.""" + try: + _settings: BaseModel = self.settings_data_model() + except PydanticUserError as e: + raise NotImplementedError( + "BackgroundDetectAlgorithms must set their own settings data model." + ) from e + + # Requires a getter and a setter to support being a BaseModel but being + # saved to file as a dict + _background_data: Optional[BaseModel] = None + + @property + def background_data(self) -> Optional[BaseModel]: + """The statistics of the background image.""" + bd = self._background_data + if bd is None: + return None + try: + return self.background_data_model(**bd) + except PydanticUserError as e: + raise NotImplementedError( + "BackgroundDetectAlgorithms must set their own background data model." + ) from e + + @background_data.setter + def background_data(self, value: Optional[BaseModel | dict]) -> None: + if value is None: + self._background_data = None + elif isinstance(value, self.background_data_model): + self._background_data = value.model_dump() + elif isinstance(value, dict): + self._background_data = value + else: + raise TypeError( + f"Cannot set background_data with an object of type {type(value)}" + ) + + @property + def settings(self) -> BaseModel: + """The statistics of the background image.""" + bd = self._background_data + if bd is None: + return None + return self.settings_data_model(**bd) + + @settings.setter + def settings(self, value: Optional[BaseModel | dict]) -> None: + if value is None: + self._settings = None + elif isinstance(value, self.settings_data_model): + self._settings = value.model_dump() + elif isinstance(value, dict): + self._settings = value + else: + raise TypeError(f"Cannot set settings with an object of type {type(value)}") + + def image_is_sample(self, image: np.ndarrayl) -> tuple[bool, str]: + """Label the current image as either background or sample. + + :returns: A tuple of the result (boolean), and explanation string. The + explanation string is formatted so it can be added into a sentence such as + ``An action was taken because the image is {message}.`` + """ + raise NotImplementedError( + "Each background detect algorithm must implement an image_is_sample method." + ) + + def set_background(self, image: np.ndarray) -> BaseModel: + """Use the input image to update the background data. + + Background data must be a Pydantic BaseModel. + """ + raise NotImplementedError( + "Each background detect algorithm must implement an set_background method." + ) + class ChannelDistributions(BaseModel): """A BaseModel for storing the channel distribution of a background image.""" @@ -10,19 +107,34 @@ class ChannelDistributions(BaseModel): """The mean of each channel in the colourspace.""" standard_deviations: list[float] """The standard deviation of each channel in the colourspace.""" - colorspace: str = "LUV" - """The colourspace used.""" -class BackgroundDetectLUV: +class ColourChannelDetectSettings(BaseModel): + """A BaseModel for storing the settings for colour channel detectors.""" + + channel_tolerance: float = 7.0 + """Channel Tolerance + + The number of standard deviations a pixel value must be from the background mean + to be considered sample. + """ + min_sample_coverage: float = 25.0 + """Sample Coverage Required (%) + + The minimum percentage of the image that needs to be identified as sample for the + image to be labeled as containing sample. + """ + + +class ColourChannelDetectLUV(BackgroundDetectAlgorithm): """Compare images with a known background in LUV colourspace. This uses an LUV colour space checking only the mean and standard deviation of the U and V channels. The LUV colourspace as it collect colours together in a """ - background_distributions: Optional[ChannelDistributions] = None - background_tolerance: float = 7.0 - min_sample_coverage: float = 25.0 + + background_data_model: BaseModel = ChannelDistributions + settings_data_model: BaseModel = ColourChannelDetectSettings def background_mask(self, image: np.ndarray) -> np.ndarray: """Calculate a binary image, showing whether each pixel is background. @@ -32,7 +144,7 @@ class BackgroundDetectLUV: # human-intuitive way """ - d = self.background_distributions + d = self.background_data if not d: raise RuntimeError( "Background is not set: you need to calibrate background detection." @@ -43,34 +155,40 @@ class BackgroundDetectLUV: return np.all( np.abs(image[:, :, 1:] - np.array(d.means[1:])[np.newaxis, np.newaxis, :]) < np.array(d.standard_deviations[1:])[np.newaxis, np.newaxis, :] - * self.background_tolerance, + * self.channel_tolerance, axis=2, ) def get_sample_coverage(self, image: np.ndarray) -> float: - """Measure what percentage of the current image is background. + """Return the percentage of the input image that is background. - * Acquire a new image from the preview stream, - * Evaluate whether it is foreground or background, by comparing it to the saved - statistics for a background image on a per-pixel basis - * Returned value (between 0 and 100) is the percentage of the image that is - background. + Evaluate whether it is foreground or background by comparing it to the saved + statistics for a background image on a per-pixel basis + + :returns: A value (between 0 and 100) is the percentage of the image that is + sample. """ image_luv = cv2.cvtColor(image, cv2.COLOR_RGB2LUV) mask = self.background_mask(image_luv) return (1 - np.count_nonzero(mask) / np.prod(mask.shape)) * 100 + def image_is_sample(self, image: np.ndarrayl) -> tuple[bool, str]: + """Label the current image as either background or sample. - def image_is_sample(self, image: np.ndarrayl) -> bool: - """Label the current image as either background or sample.""" + :returns: A tuple of the result (boolean), and explanation string. The + explanation string is formatted so it can be added into a sentence such as + ``An action was taken because the image is {message}.`` + """ sample_coverage = self.get_sample_coverage(image) - fraction_threshold = self.fraction - return sample_coverage > self.min_sample_coverage - - - def set_background(self, image: np.ndarray) -> np.ndarray: + is_sample = sample_coverage > self.min_sample_coverage + message = f"{sample_coverage}% sample" + if not is_sample: + message = "only " + message + return is_sample, message + def set_background(self, image: np.ndarray) -> ChannelDistributions: + """Use the input image to update the background distributions.""" image_luv = cv2.cvtColor(image, cv2.COLOR_RGB2LUV) ch1 = (image_luv.T[0]).flatten() @@ -82,8 +200,6 @@ class BackgroundDetectLUV: # Get the mean and standard deviation of values in each channel mu, std = np.apply_along_axis(norm.fit, 0, points) - self.background_distributions = ChannelDistributions( - means=mu.tolist(), - standard_deviations=std.tolist(), - colorspace="LUV", + self.background_data = ChannelDistributions( + means=mu.tolist(), standard_deviations=std.tolist() ) diff --git a/src/openflexure_microscope_server/things/background_detect.py b/src/openflexure_microscope_server/things/background_detect.py deleted file mode 100644 index 1c147093..00000000 --- a/src/openflexure_microscope_server/things/background_detect.py +++ /dev/null @@ -1,163 +0,0 @@ -"""Provide functionality to detect if the camera is imaging sample or background. - -An example background image must be captured and analysed by BackgroundDetectThing, -information from this images is used to detect whether the current camera field of -view contains sample. -""" - -from typing import Mapping, Optional -import cv2 -import numpy as np -from PIL import Image -from pydantic import BaseModel -from scipy.stats import norm - -import labthings_fastapi as lt -from .camera import CameraDependency as CamDep - - -class ChannelDistributions(BaseModel): - """A BaseModel for storing the channel distribution of a background image.""" - - means: list[float] - """The mean of each channel in the colourspace.""" - standard_deviations: list[float] - """The standard deviation of each channel in the colourspace.""" - colorspace: str = "LUV" - """The colourspace used.""" - - -class BackgroundDetectThing(lt.Thing): - """Thing for setting a background image and detecting sample in the field of view. - - This uses an LUV colour space checking only the mean and standard deviation of the - UV channels. Over time different, selectable, background detection methods will be - added. - """ - - # Requires a getter and a setter to support being a BaseModel but being - # saved to file as a dict - _background_distributions: Optional[ChannelDistributions] = None - - @lt.thing_setting - def background_distributions(self) -> Optional[ChannelDistributions]: - """The statistics of the background image.""" - bd = self._background_distributions - if bd is None: - return None - return ChannelDistributions(**bd) - - @background_distributions.setter - def background_distributions( - self, value: Optional[ChannelDistributions | dict] - ) -> None: - if value is None: - self._background_distributions = None - elif isinstance(value, ChannelDistributions): - self._background_distributions = value.model_dump() - elif isinstance(value, dict): - self._background_distributions = value - else: - raise TypeError( - f"Cannot set background_distributions with an object of type {type(value)}" - ) - - tolerance = lt.ThingSetting( - initial_value=7.0, - model=float, - ) - """How many standard deviations to allow for the background.""" - - fraction = lt.ThingSetting( - initial_value=25.0, - model=float, - ) - """How much of the image needs to be not background to label as sample""" - - def background_mask(self, image: np.ndarray) -> np.ndarray: - """Calculate a binary image, showing whether each pixel is background. - - The image should be in LUV format, the output will be binary with the - same shape in the first two dimensions. - """ - d = self.background_distributions - if not d: - raise RuntimeError( - "Background is not set: you need to calibrate background detection." - ) - # This image is in LUV space. But the brightness (L) often changes as the - # height of the sample changes. Hence in the line below we are only using - # the UV (colour) channels. - return np.all( - np.abs(image[:, :, 1:] - np.array(d.means[1:])[np.newaxis, np.newaxis, :]) - < np.array(d.standard_deviations[1:])[np.newaxis, np.newaxis, :] - * self.tolerance, - axis=2, - ) - - @lt.thing_action - def background_fraction(self, cam: CamDep) -> float: - """Determine what fraction of the current image is background. - - This action will acquire a new image from the preview stream, then - evaluate whether it is foreground or background, by comparing it - too the saved statistics. This is done on a per-pixel basis, and - the returned value (between 0 and 100) is the fraction of the image - that is background. - """ - current_image = cam.grab_jpeg() - current_image = np.array(Image.open(current_image.open())) - - # we're working in the LUV colourspace as it collect colours together in a human-intuitive way - current_image_luv = cv2.cvtColor(current_image, cv2.COLOR_RGB2LUV) - mask = self.background_mask(current_image_luv) - return np.count_nonzero(mask) / np.prod(mask.shape) * 100 - - @lt.thing_action - def image_is_sample(self, cam: CamDep) -> bool: - """Label the current image as either background or sample.""" - b_fraction = self.background_fraction(cam) - fraction_threshold = self.fraction - - return (100 - b_fraction) > fraction_threshold - - @lt.thing_action - def set_background(self, cam: CamDep): - """Grab an image, and use its statistics to set the background. - - This should be run when the microscope is looking at an empty region, - and will calculate the mean and standard deviation of the pixel values - in the LUV colourspace. These values will then be used to compare - future images to the distribution, to determine if each pixel is - foreground or background. - """ - background = cam.grab_jpeg() - background = np.array(Image.open(background.open())) - - # we're working in the LUV colourspace as it collect colours together in a human-intuitive way - background_luv = cv2.cvtColor(background, cv2.COLOR_RGB2LUV) - - ch1 = (background_luv.T[0]).flatten() - ch2 = (background_luv.T[1]).flatten() - ch3 = (background_luv.T[2]).flatten() - - points = np.array([np.asarray(ch1), np.asarray(ch2), np.asarray(ch3)]).T - - # we get the mean and standard deviation of values in each channel - mu, std = np.apply_along_axis(norm.fit, 0, points) - - self.background_distributions = ChannelDistributions( - means=mu.tolist(), - standard_deviations=std.tolist(), - colorspace="LUV", - ) - - @property - def thing_state(self) -> Mapping: - """Summary metadata describing the current state of the Thing.""" - bd = self.background_distributions - return { - "background_distributions": bd.model_dump() if bd else None, - "tolerance": self.tolerance, - "fraction": self.fraction, - } diff --git a/src/openflexure_microscope_server/things/camera/__init__.py b/src/openflexure_microscope_server/things/camera/__init__.py index 734c01bd..7a26335c 100644 --- a/src/openflexure_microscope_server/things/camera/__init__.py +++ b/src/openflexure_microscope_server/things/camera/__init__.py @@ -10,6 +10,7 @@ from __future__ import annotations from typing import Literal, Optional, Tuple, Any import json import time +import logging import numpy as np from pydantic import RootModel @@ -19,7 +20,10 @@ import piexif import labthings_fastapi as lt from labthings_fastapi.types.numpy import NDArray -from openflexure_microscope_server.background_detect import ChannelDistributions, BackgroundDetectLUV +from openflexure_microscope_server.background_detect import BackgroundDetectLUV + +LOGGER = logging.getLogger(__name__) + class JPEGBlob(lt.blob.Blob): """A class representing a JPEG image as a LabThings FastAPI Blob.""" @@ -157,8 +161,13 @@ class BaseCamera(lt.Thing): _memory_buffer = CameraMemoryBuffer() def __init__(self): + """Initialise the base camera, this creates the background detectors. + + This must be run by all child camera classes. + """ super().__init__() - self.background_detector = BackgroundDetectLUV() + self.background_detectors = {"Colour Channels (LUV)": BackgroundDetectLUV()} + self._detector_name = "Colour Channels (LUV)" def __enter__(self) -> None: """Open hardware connection when the Thing context manager is opened.""" @@ -460,51 +469,54 @@ class BaseCamera(lt.Thing): time.sleep(self.settling_time) self.discard_frames() - background_tolerance = lt.ThingSetting( - initial_value=7.0, - model=float, - ) - """How many standard deviations to allow for the background.""" + # Note that the default detector name is set at init. This is over written if + # setting is loaded from disk. + @lt.thing_setting + def detector_name(self) -> str: + """The name of the active background selector.""" + return self._detector_name - min_sample_coverage = lt.ThingSetting( - initial_value=25.0, - model=float, - ) - """The percentage of the image that needs to be not be background to label as sample.""" + @detector_name.setter + def detector_name(self, name: str) -> None: + """Validate and set detector_name.""" + if name not in self.background_detectors: + raise ValueError(f"{name} is not a valid background detector name") + self._detector_name = name - # Requires a getter and a setter to support being a BaseModel but being - # saved to file as a dict - _background_distributions: Optional[ChannelDistributions] = None + @property + def active_detector(self): + """The active background detector instance.""" + return self.background_detectors[self.detector_name] @lt.thing_setting - def background_distributions(self) -> Optional[ChannelDistributions]: - """The statistics of the background image.""" - bd = self._background_distributions - if bd is None: - return None - return ChannelDistributions(**bd) + def background_detector_data(self) -> str: + """The name of the active background selector.""" + data = {} + for name, obj in self.background_detectors.items(): + data[name] = { + "settings": obj.settings.model_dump(), + "background_data": obj.background_data.model_dump(), + } - @background_distributions.setter - def background_distributions( - self, value: Optional[ChannelDistributions | dict] - ) -> None: - if value is None: - self._background_distributions = None - elif isinstance(value, ChannelDistributions): - self._background_distributions = value.model_dump() - elif isinstance(value, dict): - self._background_distributions = value - else: - raise TypeError( - f"Cannot set background_distributions with an object of type {type(value)}" - ) + @detector_name.setter + def detector_name(self, data: str) -> None: + """Validate and set detector_name.""" + for name, instance_data in data.items(): + if name in self.background_detectors: + obj = self.background_detectors[name] + obj.settings = instance_data["settings"] + obj.background_data = instance_data["background_data"] + else: + LOGGER.warning( + f"No background detector named {name}, settings will be discarded." + ) @lt.thing_action - def image_is_sample(self, portal: lt.deps.BlockingPortal) -> bool: + def image_is_sample(self, portal: lt.deps.BlockingPortal) -> tuple[bool, str]: """Label the current image as either background or sample.""" current_image = self.grab_jpeg(portal) current_image = np.array(Image.open(current_image.open())) - return self.background_detector.image_is_sample(current_image) + return self.active_detector.image_is_sample(current_image) @lt.thing_action def set_background(self, portal: lt.deps.BlockingPortal): @@ -518,8 +530,7 @@ class BaseCamera(lt.Thing): """ background = self.grab_jpeg(portal) background = np.array(Image.open(background.open())) - self.background_detector.set_background(current_image) - + self.active_detector.set_background(background) CameraDependency = lt.deps.direct_thing_client_dependency(BaseCamera, "/camera/") diff --git a/src/openflexure_microscope_server/things/smart_scan.py b/src/openflexure_microscope_server/things/smart_scan.py index f7e5d675..3db02813 100644 --- a/src/openflexure_microscope_server/things/smart_scan.py +++ b/src/openflexure_microscope_server/things/smart_scan.py @@ -511,15 +511,13 @@ class SmartScanThing(lt.Thing): capture_image = True # If skipping background, take an image to check if current field of view is background if self._scan_data["skip_background"]: - capture_image = self._background_detect.image_is_sample() + capture_image, bg_message = self._background_detect.image_is_sample() if not capture_image: route_planner.mark_location_visited( new_pos_xyz, imaged=False, focused=False ) - # Background fraction is actually a percentage - back_perc = round(self._background_detect.background_fraction(), 0) - msg = f"Skipping {new_pos_xyz} as it is {back_perc}% background." + msg = f"Skipping {new_pos_xyz} as it is {bg_message}." self._scan_logger.info(msg) continue From fe1b84a92266bca48aaeff47c3f11d3d99404747 Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Fri, 18 Jul 2025 19:49:36 +0100 Subject: [PATCH 03/13] Complete first pass (untested on hardware) of bg detect refactor. --- ofm_config_full.json | 3 +- ofm_config_simulation.json | 3 +- .../background_detect.py | 63 ++++++++++++------- .../things/camera/__init__.py | 29 +++++++-- .../things/camera/picamera.py | 2 +- .../things/camera/simulation.py | 2 +- .../things/smart_scan.py | 16 ++--- ...ck_background_detect.py => mock_camera.py} | 17 ++--- tests/test_smart_scan.py | 8 +-- .../paneBackgroundDetect.vue | 35 ++--------- 10 files changed, 90 insertions(+), 88 deletions(-) rename tests/mock_things/{mock_background_detect.py => mock_camera.py} (57%) diff --git a/ofm_config_full.json b/ofm_config_full.json index 1cfedb26..98a1cbc7 100644 --- a/ofm_config_full.json +++ b/ofm_config_full.json @@ -11,8 +11,7 @@ "kwargs": { "scans_folder": "/var/openflexure/scans/" } - }, - "/background_detect/": "openflexure_microscope_server.things.background_detect:BackgroundDetectThing" + } }, "settings_folder": "/var/openflexure/settings/", "log_folder": "/var/openflexure/logs/" diff --git a/ofm_config_simulation.json b/ofm_config_simulation.json index a254fa85..fd9dcae2 100644 --- a/ofm_config_simulation.json +++ b/ofm_config_simulation.json @@ -11,8 +11,7 @@ "kwargs": { "scans_folder": "./openflexure/scans/" } - }, - "/background_detect/": "openflexure_microscope_server.things.background_detect:BackgroundDetectThing" + } }, "settings_folder": "./openflexure/settings/", "log_folder": "./openflexure/logs/" diff --git a/src/openflexure_microscope_server/background_detect.py b/src/openflexure_microscope_server/background_detect.py index 71a1d8a9..e2286b9f 100644 --- a/src/openflexure_microscope_server/background_detect.py +++ b/src/openflexure_microscope_server/background_detect.py @@ -5,12 +5,32 @@ for analysis. Information from this images is used to detect whether an image fr current camera field of view contains sample. """ -from typing import Optional +from typing import Optional, Any import cv2 import numpy as np from pydantic import BaseModel from pydantic.errors import PydanticUserError from scipy.stats import norm +from labthings_fastapi.thing_description import type_to_dataschema + + +class BackgroundDetectorStatus(BaseModel): + """The status information about a background detector instance needed for the GUI. + + Each BackgroundDetectAlgorithm must be able to return one of these models when + ``status`` is called. + """ + + ready: bool + """True if ready to be used, if False this detector isn't initialised for use. + + This could be called ``has_background_data`` or similar, but the more generic + ``ready`` is used in case more complex methods are added in the future, which + need different initialisation. + """ + settings: BaseModel + """The settings for this this background detect Algorithm""" + settings_schema: dict[str, Any] class BackgroundDetectAlgorithm: @@ -24,12 +44,21 @@ class BackgroundDetectAlgorithm: def __init__(self): """Initialise the algorithm settings.""" try: - _settings: BaseModel = self.settings_data_model() + self._settings: BaseModel = self.settings_data_model() except PydanticUserError as e: raise NotImplementedError( "BackgroundDetectAlgorithms must set their own settings data model." ) from e + @property + def status(self) -> BackgroundDetectorStatus: + """The status information needed for the GUI. Read only.""" + return BackgroundDetectorStatus( + ready=self.background_data is not None, + settings=self.settings, + settings_schema=type_to_dataschema(self.settings_data_model), + ) + # Requires a getter and a setter to support being a BaseModel but being # saved to file as a dict _background_data: Optional[BaseModel] = None @@ -40,21 +69,16 @@ class BackgroundDetectAlgorithm: bd = self._background_data if bd is None: return None - try: - return self.background_data_model(**bd) - except PydanticUserError as e: - raise NotImplementedError( - "BackgroundDetectAlgorithms must set their own background data model." - ) from e + return bd @background_data.setter def background_data(self, value: Optional[BaseModel | dict]) -> None: if value is None: self._background_data = None elif isinstance(value, self.background_data_model): - self._background_data = value.model_dump() - elif isinstance(value, dict): self._background_data = value + elif isinstance(value, dict): + self._background_data = self.background_data_model(**value) else: raise TypeError( f"Cannot set background_data with an object of type {type(value)}" @@ -63,23 +87,18 @@ class BackgroundDetectAlgorithm: @property def settings(self) -> BaseModel: """The statistics of the background image.""" - bd = self._background_data - if bd is None: - return None - return self.settings_data_model(**bd) + return self._settings @settings.setter - def settings(self, value: Optional[BaseModel | dict]) -> None: - if value is None: - self._settings = None - elif isinstance(value, self.settings_data_model): - self._settings = value.model_dump() - elif isinstance(value, dict): + def settings(self, value: BaseModel | dict) -> None: + if isinstance(value, self.settings_data_model): self._settings = value + elif isinstance(value, dict): + self._settings = self.settings_data_model(**value) else: raise TypeError(f"Cannot set settings with an object of type {type(value)}") - def image_is_sample(self, image: np.ndarrayl) -> tuple[bool, str]: + def image_is_sample(self, image: np.ndarray) -> tuple[bool, str]: """Label the current image as either background or sample. :returns: A tuple of the result (boolean), and explanation string. The @@ -172,7 +191,7 @@ class ColourChannelDetectLUV(BackgroundDetectAlgorithm): mask = self.background_mask(image_luv) return (1 - np.count_nonzero(mask) / np.prod(mask.shape)) * 100 - def image_is_sample(self, image: np.ndarrayl) -> tuple[bool, str]: + def image_is_sample(self, image: np.ndarray) -> tuple[bool, str]: """Label the current image as either background or sample. :returns: A tuple of the result (boolean), and explanation string. The diff --git a/src/openflexure_microscope_server/things/camera/__init__.py b/src/openflexure_microscope_server/things/camera/__init__.py index 7a26335c..3beeb7b5 100644 --- a/src/openflexure_microscope_server/things/camera/__init__.py +++ b/src/openflexure_microscope_server/things/camera/__init__.py @@ -20,7 +20,11 @@ import piexif import labthings_fastapi as lt from labthings_fastapi.types.numpy import NDArray -from openflexure_microscope_server.background_detect import BackgroundDetectLUV +from openflexure_microscope_server.background_detect import ( + ColourChannelDetectLUV, + BackgroundDetectAlgorithm, + BackgroundDetectorStatus, +) LOGGER = logging.getLogger(__name__) @@ -166,7 +170,7 @@ class BaseCamera(lt.Thing): This must be run by all child camera classes. """ super().__init__() - self.background_detectors = {"Colour Channels (LUV)": BackgroundDetectLUV()} + self.background_detectors = {"Colour Channels (LUV)": ColourChannelDetectLUV()} self._detector_name = "Colour Channels (LUV)" def __enter__(self) -> None: @@ -484,23 +488,36 @@ class BaseCamera(lt.Thing): self._detector_name = name @property - def active_detector(self): + def active_detector(self) -> BackgroundDetectAlgorithm: """The active background detector instance.""" return self.background_detectors[self.detector_name] + @lt.thing_property + def background_detector_status(self) -> BackgroundDetectorStatus: + """The status of the active detector for the UI.""" + return self.active_detector.status + @lt.thing_setting def background_detector_data(self) -> str: - """The name of the active background selector.""" + """The data for each background detector, used to save to disk.""" data = {} for name, obj in self.background_detectors.items(): + bg_data = ( + None + if obj.background_data is None + else obj.background_data.model_dump() + ) data[name] = { "settings": obj.settings.model_dump(), - "background_data": obj.background_data.model_dump(), + "background_data": bg_data, } @detector_name.setter def detector_name(self, data: str) -> None: - """Validate and set detector_name.""" + """Set the data for each detector. This should only be called on init. + + Do not call over HTTP. Would be good to have a way to make this private. + """ for name, instance_data in data.items(): if name in self.background_detectors: obj = self.background_detectors[name] diff --git a/src/openflexure_microscope_server/things/camera/picamera.py b/src/openflexure_microscope_server/things/camera/picamera.py index fea7b149..fba605e0 100644 --- a/src/openflexure_microscope_server/things/camera/picamera.py +++ b/src/openflexure_microscope_server/things/camera/picamera.py @@ -118,7 +118,7 @@ class StreamingPiCamera2(BaseCamera): :param camera_num: The number of the camera. This should generally be left as 0 as most Raspberry Pi boards only support 1 camera. """ - super().__init() + super().__init__() self._setting_save_in_progress = False self.camera_num = camera_num self.camera_configs: dict[str, dict] = {} diff --git a/src/openflexure_microscope_server/things/camera/simulation.py b/src/openflexure_microscope_server/things/camera/simulation.py index 225e474a..76d11eb9 100644 --- a/src/openflexure_microscope_server/things/camera/simulation.py +++ b/src/openflexure_microscope_server/things/camera/simulation.py @@ -58,7 +58,7 @@ class SimulatedCamera(BaseCamera): :param frame_interval: Nominally the time between frames on the MJPEG stream, however the rate may be slower due to calculation time for focus. """ - super().__init() + super().__init__() self.shape = shape self.glyph_shape = glyph_shape self.canvas_shape = canvas_shape diff --git a/src/openflexure_microscope_server/things/smart_scan.py b/src/openflexure_microscope_server/things/smart_scan.py index 3db02813..c830c93c 100644 --- a/src/openflexure_microscope_server/things/smart_scan.py +++ b/src/openflexure_microscope_server/things/smart_scan.py @@ -1,7 +1,7 @@ """The core sample scanning functionality for the OpenFlexure Microscope. SmartScan provides sample scanning functionality including automatic background -detection (via the `BackgroundDetectThing`) and automatic path planning via +detection (via the `CameraThing`) and automatic path planning via `scan_planners`. It manages the directories of past scans via `scan_directories`. It also controls external processes for live stitching composite images, and the creation of the final stitched images. @@ -29,7 +29,6 @@ from openflexure_microscope_server import scan_planners # Things from .autofocus import AutofocusThing from .camera_stage_mapping import CameraStageMapper -from .background_detect import BackgroundDetectThing from .camera import CameraDependency as CamDep from .stage import StageDependency as StageDep @@ -37,9 +36,6 @@ CSMDep = lt.deps.direct_thing_client_dependency( CameraStageMapper, "/camera_stage_mapping/" ) AutofocusDep = lt.deps.direct_thing_client_dependency(AutofocusThing, "/autofocus/") -BackgroundDep = lt.deps.direct_thing_client_dependency( - BackgroundDetectThing, "/background_detect/" -) JPEGBlob = lt.blob.blob_type("image/jpeg") ZipBlob = lt.blob.blob_type("application/zip") @@ -110,7 +106,6 @@ class SmartScanThing(lt.Thing): self._cam: Optional[CamDep] = None self._metadata_getter: Optional[lt.deps.GetThingStates] = None self._csm: Optional[CSMDep] = None - self._background_detect: Optional[BackgroundDep] = None self._ongoing_scan: Optional[scan_directories.ScanDirectory] = None self._starting_position: Optional[Mapping[str, int]] = None self._capture_thread: Optional[ErrorCapturingThread] = None @@ -128,14 +123,13 @@ class SmartScanThing(lt.Thing): cam: CamDep, metadata_getter: lt.deps.GetThingStates, csm: CSMDep, - background_detect: BackgroundDep, scan_name: str = "", ): """Move the stage to cover an area, taking images that can be tiled together. The stage will move in a pattern that grows outwards from the starting point, stopping once it is surrounded by "background" (as detected by the - background_detect Thing) or reaches the "max_range" measured in steps. + camera Thing) or reaches the "max_range" measured in steps. """ got_lock = self._scan_lock.acquire(timeout=0.1) if not got_lock: @@ -149,7 +143,6 @@ class SmartScanThing(lt.Thing): self._cam = cam self._metadata_getter = metadata_getter self._csm = csm - self._background_detect = background_detect self._capture_thread = None self._scan_images_taken = 0 @@ -183,7 +176,6 @@ class SmartScanThing(lt.Thing): self._cam = None self._metadata_getter = None self._csm = None - self._background_detect = None self._capture_thread = None self._ongoing_scan = None self._scan_images_taken = None @@ -207,7 +199,7 @@ class SmartScanThing(lt.Thing): ) if self.skip_background: - if not self._background_detect.background_distributions: + if not self._cam.background_detector_status.ready: raise RuntimeError( "Background is not set: you need to calibrate background detection." ) @@ -511,7 +503,7 @@ class SmartScanThing(lt.Thing): capture_image = True # If skipping background, take an image to check if current field of view is background if self._scan_data["skip_background"]: - capture_image, bg_message = self._background_detect.image_is_sample() + capture_image, bg_message = self._cam.image_is_sample() if not capture_image: route_planner.mark_location_visited( diff --git a/tests/mock_things/mock_background_detect.py b/tests/mock_things/mock_camera.py similarity index 57% rename from tests/mock_things/mock_background_detect.py rename to tests/mock_things/mock_camera.py index 80419030..1c2ecdbf 100644 --- a/tests/mock_things/mock_background_detect.py +++ b/tests/mock_things/mock_camera.py @@ -1,4 +1,4 @@ -"""Testing submodule with mock Background Detect Things. +"""Testing submodule with mock Camera Things. These mocks are designed to be inserted as dependencies to provide specific functionality and return values. @@ -7,10 +7,13 @@ The mocks do not subclass Things. Instead, they return predefined answers to functions. """ -from openflexure_microscope_server.things.background_detect import ChannelDistributions +from openflexure_microscope_server.background_detect import ( + BackgroundDetectorStatus, + ColourChannelDetectSettings, +) -class MockBackgroundDetectThing: +class MockCameraThing: """A mock background detect Thing that imports no code from BackgroundDetectThing. The class needs functionality added to it over time as more complex @@ -18,8 +21,8 @@ class MockBackgroundDetectThing: is not artificially inflated. """ - background_distributions = ChannelDistributions( - means=[128.0, 128.0, 128.0], - standard_deviations=[3.0, 3.0, 3.0], - colorspace="LUV", + background_detector_status = BackgroundDetectorStatus( + ready=True, + settings=ColourChannelDetectSettings(), + settings_schema={"fake": "schema"}, ) diff --git a/tests/test_smart_scan.py b/tests/test_smart_scan.py index 0ec1ae3f..1a0be750 100644 --- a/tests/test_smart_scan.py +++ b/tests/test_smart_scan.py @@ -30,7 +30,7 @@ from openflexure_microscope_server.things.smart_scan import ( from .mock_things.mock_csm import MockCSMThing from .mock_things.mock_autofocus import MockAutoFocusThing from .mock_things.mock_stage import MockStageThing -from .mock_things.mock_background_detect import MockBackgroundDetectThing +from .mock_things.mock_camera import MockCameraThing # A global logger to pass in as an Invocation Logger LOGGER = logging.getLogger("mock-invocation_logger") @@ -169,10 +169,9 @@ def _run_only_outer_scan(adjust_initial_state: Optional[Callable] = None): cancel_mock = 1 # not called af_mock = MockAutoFocusThing() stage_mock = MockStageThing() - cam_mock = 4 # not called + cam_mock = MockCameraThing() meta_mock = 5 # not called csm_mock = MockCSMThing() - bkgrnd_det_mock = MockBackgroundDetectThing() class MockedSmartScanThing(SmartScanThing): """Mocked version of SmartScanThing with a patched _run_scan method.""" @@ -192,7 +191,6 @@ def _run_only_outer_scan(adjust_initial_state: Optional[Callable] = None): assert self._cam is cam_mock assert self._metadata_getter is meta_mock assert self._csm is csm_mock - assert self._background_detect is bkgrnd_det_mock assert self._capture_thread is None assert self._scan_images_taken == 0 @@ -212,7 +210,6 @@ def _run_only_outer_scan(adjust_initial_state: Optional[Callable] = None): cam=cam_mock, metadata_getter=meta_mock, csm=csm_mock, - background_detect=bkgrnd_det_mock, scan_name="FooBar", ) except Exception as e: @@ -227,7 +224,6 @@ def _run_only_outer_scan(adjust_initial_state: Optional[Callable] = None): assert mock_ss_thing._cam is None assert mock_ss_thing._metadata_getter is None assert mock_ss_thing._csm is None - assert mock_ss_thing._background_detect is None assert mock_ss_thing._capture_thread is None assert mock_ss_thing._scan_images_taken is None diff --git a/webapp/src/components/tabContentComponents/backgroundDetectComponents/paneBackgroundDetect.vue b/webapp/src/components/tabContentComponents/backgroundDetectComponents/paneBackgroundDetect.vue index bef06118..be57f9ac 100644 --- a/webapp/src/components/tabContentComponents/backgroundDetectComponents/paneBackgroundDetect.vue +++ b/webapp/src/components/tabContentComponents/backgroundDetectComponents/paneBackgroundDetect.vue @@ -8,37 +8,13 @@
  • Settings
    -
    - -
    -
    - -
    -
    - -
    +

    TODO!

    • + Date: Fri, 18 Jul 2025 21:21:25 +0100 Subject: [PATCH 04/13] Get new background detect running in simulation mode. --- .../background_detect.py | 13 ++++++++----- .../things/camera/__init__.py | 9 ++++++--- .../paneBackgroundDetect.vue | 15 ++------------- 3 files changed, 16 insertions(+), 21 deletions(-) diff --git a/src/openflexure_microscope_server/background_detect.py b/src/openflexure_microscope_server/background_detect.py index e2286b9f..a3a6fa82 100644 --- a/src/openflexure_microscope_server/background_detect.py +++ b/src/openflexure_microscope_server/background_detect.py @@ -158,23 +158,23 @@ class ColourChannelDetectLUV(BackgroundDetectAlgorithm): def background_mask(self, image: np.ndarray) -> np.ndarray: """Calculate a binary image, showing whether each pixel is background. + True is background. + The image should be in LUV format, the output will be binary with the same shape in the first two dimensions. - - # human-intuitive way """ d = self.background_data if not d: raise RuntimeError( "Background is not set: you need to calibrate background detection." ) - + print(d) # Only use the U and V channels of as brightness (L) often changes as the # height of the sample changes. return np.all( np.abs(image[:, :, 1:] - np.array(d.means[1:])[np.newaxis, np.newaxis, :]) < np.array(d.standard_deviations[1:])[np.newaxis, np.newaxis, :] - * self.channel_tolerance, + * self.settings.channel_tolerance, axis=2, ) @@ -189,6 +189,9 @@ class ColourChannelDetectLUV(BackgroundDetectAlgorithm): """ image_luv = cv2.cvtColor(image, cv2.COLOR_RGB2LUV) mask = self.background_mask(image_luv) + print(np.count_nonzero(mask)) + print(np.prod(mask.shape)) + print((1 - np.count_nonzero(mask) / np.prod(mask.shape)) * 100) return (1 - np.count_nonzero(mask) / np.prod(mask.shape)) * 100 def image_is_sample(self, image: np.ndarray) -> tuple[bool, str]: @@ -200,7 +203,7 @@ class ColourChannelDetectLUV(BackgroundDetectAlgorithm): """ sample_coverage = self.get_sample_coverage(image) - is_sample = sample_coverage > self.min_sample_coverage + is_sample = sample_coverage > self.settings.min_sample_coverage message = f"{sample_coverage}% sample" if not is_sample: message = "only " + message diff --git a/src/openflexure_microscope_server/things/camera/__init__.py b/src/openflexure_microscope_server/things/camera/__init__.py index 3beeb7b5..fbe5fc04 100644 --- a/src/openflexure_microscope_server/things/camera/__init__.py +++ b/src/openflexure_microscope_server/things/camera/__init__.py @@ -498,7 +498,7 @@ class BaseCamera(lt.Thing): return self.active_detector.status @lt.thing_setting - def background_detector_data(self) -> str: + def background_detector_data(self) -> dict: """The data for each background detector, used to save to disk.""" data = {} for name, obj in self.background_detectors.items(): @@ -511,9 +511,10 @@ class BaseCamera(lt.Thing): "settings": obj.settings.model_dump(), "background_data": bg_data, } + return data - @detector_name.setter - def detector_name(self, data: str) -> None: + @background_detector_data.setter + def background_detector_data(self, data: dict) -> None: """Set the data for each detector. This should only be called on init. Do not call over HTTP. Would be good to have a way to make this private. @@ -548,6 +549,8 @@ class BaseCamera(lt.Thing): background = self.grab_jpeg(portal) background = np.array(Image.open(background.open())) self.active_detector.set_background(background) + # Manually save settings as the setter is not called. + self.save_settings() CameraDependency = lt.deps.direct_thing_client_dependency(BaseCamera, "/camera/") diff --git a/webapp/src/components/tabContentComponents/backgroundDetectComponents/paneBackgroundDetect.vue b/webapp/src/components/tabContentComponents/backgroundDetectComponents/paneBackgroundDetect.vue index be57f9ac..28eb4c21 100644 --- a/webapp/src/components/tabContentComponents/backgroundDetectComponents/paneBackgroundDetect.vue +++ b/webapp/src/components/tabContentComponents/backgroundDetectComponents/paneBackgroundDetect.vue @@ -1,9 +1,6 @@