diff --git a/src/openflexure_microscope_server/background_detect.py b/src/openflexure_microscope_server/background_detect.py new file mode 100644 index 00000000..b3818f1f --- /dev/null +++ b/src/openflexure_microscope_server/background_detect.py @@ -0,0 +1,89 @@ + +import cv2 +import numpy as np +from pydantic import BaseModel + +class ChannelDistributions(BaseModel): + """A BaseModel for storing the channel distribution of a background image.""" + + means: list[float] + """The mean of each channel in the colourspace.""" + standard_deviations: list[float] + """The standard deviation of each channel in the colourspace.""" + colorspace: str = "LUV" + """The colourspace used.""" + + +class BackgroundDetectLUV: + """Compare images with a known background in LUV colourspace. + + This uses an LUV colour space checking only the mean and standard deviation of the + U and V channels. The LUV colourspace as it collect colours together in a + """ + background_distributions: Optional[ChannelDistributions] = None + background_tolerance: float = 7.0 + min_sample_coverage: float = 25.0 + + def background_mask(self, image: np.ndarray) -> np.ndarray: + """Calculate a binary image, showing whether each pixel is background. + + The image should be in LUV format, the output will be binary with the + same shape in the first two dimensions. + + # human-intuitive way + """ + d = self.background_distributions + if not d: + raise RuntimeError( + "Background is not set: you need to calibrate background detection." + ) + + # Only use the U and V channels of as brightness (L) often changes as the + # height of the sample changes. + return np.all( + np.abs(image[:, :, 1:] - np.array(d.means[1:])[np.newaxis, np.newaxis, :]) + < np.array(d.standard_deviations[1:])[np.newaxis, np.newaxis, :] + * self.background_tolerance, + axis=2, + ) + + def get_sample_coverage(self, image: np.ndarray) -> float: + """Measure what percentage of the current image is background. + + * Acquire a new image from the preview stream, + * Evaluate whether it is foreground or background, by comparing it to the saved + statistics for a background image on a per-pixel basis + * Returned value (between 0 and 100) is the percentage of the image that is + background. + """ + image_luv = cv2.cvtColor(image, cv2.COLOR_RGB2LUV) + mask = self.background_mask(image_luv) + return (1 - np.count_nonzero(mask) / np.prod(mask.shape)) * 100 + + + def image_is_sample(self, image: np.ndarrayl) -> bool: + """Label the current image as either background or sample.""" + sample_coverage = self.get_sample_coverage(image) + fraction_threshold = self.fraction + + return sample_coverage > self.min_sample_coverage + + + def set_background(self, image: np.ndarray) -> np.ndarray: + + image_luv = cv2.cvtColor(image, cv2.COLOR_RGB2LUV) + + ch1 = (image_luv.T[0]).flatten() + ch2 = (image_luv.T[1]).flatten() + ch3 = (image_luv.T[2]).flatten() + + points = np.array([np.asarray(ch1), np.asarray(ch2), np.asarray(ch3)]).T + + # Get the mean and standard deviation of values in each channel + mu, std = np.apply_along_axis(norm.fit, 0, points) + + self.background_distributions = ChannelDistributions( + means=mu.tolist(), + standard_deviations=std.tolist(), + colorspace="LUV", + ) diff --git a/src/openflexure_microscope_server/things/camera/__init__.py b/src/openflexure_microscope_server/things/camera/__init__.py index dd901ed9..734c01bd 100644 --- a/src/openflexure_microscope_server/things/camera/__init__.py +++ b/src/openflexure_microscope_server/things/camera/__init__.py @@ -19,6 +19,7 @@ import piexif import labthings_fastapi as lt from labthings_fastapi.types.numpy import NDArray +from openflexure_microscope_server.background_detect import ChannelDistributions, BackgroundDetectLUV class JPEGBlob(lt.blob.Blob): """A class representing a JPEG image as a LabThings FastAPI Blob.""" @@ -155,6 +156,10 @@ class BaseCamera(lt.Thing): lores_mjpeg_stream = lt.outputs.MJPEGStreamDescriptor() _memory_buffer = CameraMemoryBuffer() + def __init__(self): + super().__init__() + self.background_detector = BackgroundDetectLUV() + def __enter__(self) -> None: """Open hardware connection when the Thing context manager is opened.""" raise NotImplementedError("CameraThings must define their own __enter__ method") @@ -455,6 +460,67 @@ class BaseCamera(lt.Thing): time.sleep(self.settling_time) self.discard_frames() + background_tolerance = lt.ThingSetting( + initial_value=7.0, + model=float, + ) + """How many standard deviations to allow for the background.""" + + min_sample_coverage = lt.ThingSetting( + initial_value=25.0, + model=float, + ) + """The percentage of the image that needs to be not be background to label as sample.""" + + # Requires a getter and a setter to support being a BaseModel but being + # saved to file as a dict + _background_distributions: Optional[ChannelDistributions] = None + + @lt.thing_setting + def background_distributions(self) -> Optional[ChannelDistributions]: + """The statistics of the background image.""" + bd = self._background_distributions + if bd is None: + return None + return ChannelDistributions(**bd) + + @background_distributions.setter + def background_distributions( + self, value: Optional[ChannelDistributions | dict] + ) -> None: + if value is None: + self._background_distributions = None + elif isinstance(value, ChannelDistributions): + self._background_distributions = value.model_dump() + elif isinstance(value, dict): + self._background_distributions = value + else: + raise TypeError( + f"Cannot set background_distributions with an object of type {type(value)}" + ) + + @lt.thing_action + def image_is_sample(self, portal: lt.deps.BlockingPortal) -> bool: + """Label the current image as either background or sample.""" + current_image = self.grab_jpeg(portal) + current_image = np.array(Image.open(current_image.open())) + return self.background_detector.image_is_sample(current_image) + + @lt.thing_action + def set_background(self, portal: lt.deps.BlockingPortal): + """Grab an image, and use its statistics to set the background. + + This should be run when the microscope is looking at an empty region, + and will calculate the mean and standard deviation of the pixel values + in the LUV colourspace. These values will then be used to compare + future images to the distribution, to determine if each pixel is + foreground or background. + """ + background = self.grab_jpeg(portal) + background = np.array(Image.open(background.open())) + self.background_detector.set_background(current_image) + + CameraDependency = lt.deps.direct_thing_client_dependency(BaseCamera, "/camera/") RawCameraDependency = lt.deps.raw_thing_dependency(BaseCamera) diff --git a/src/openflexure_microscope_server/things/camera/opencv.py b/src/openflexure_microscope_server/things/camera/opencv.py index 7f543cf1..0a3bd047 100644 --- a/src/openflexure_microscope_server/things/camera/opencv.py +++ b/src/openflexure_microscope_server/things/camera/opencv.py @@ -30,6 +30,7 @@ class OpenCVCamera(BaseCamera): :param camera_index: The index of the camera to use for the microscope. """ + super().__init() self.camera_index = camera_index self._capture_thread: Optional[Thread] = None self._capture_enabled = False diff --git a/src/openflexure_microscope_server/things/camera/picamera.py b/src/openflexure_microscope_server/things/camera/picamera.py index c676b82d..fea7b149 100644 --- a/src/openflexure_microscope_server/things/camera/picamera.py +++ b/src/openflexure_microscope_server/things/camera/picamera.py @@ -118,6 +118,7 @@ class StreamingPiCamera2(BaseCamera): :param camera_num: The number of the camera. This should generally be left as 0 as most Raspberry Pi boards only support 1 camera. """ + super().__init() self._setting_save_in_progress = False self.camera_num = camera_num self.camera_configs: dict[str, dict] = {} diff --git a/src/openflexure_microscope_server/things/camera/simulation.py b/src/openflexure_microscope_server/things/camera/simulation.py index 1bb3715c..225e474a 100644 --- a/src/openflexure_microscope_server/things/camera/simulation.py +++ b/src/openflexure_microscope_server/things/camera/simulation.py @@ -58,6 +58,7 @@ class SimulatedCamera(BaseCamera): :param frame_interval: Nominally the time between frames on the MJPEG stream, however the rate may be slower due to calculation time for focus. """ + super().__init() self.shape = shape self.glyph_shape = glyph_shape self.canvas_shape = canvas_shape diff --git a/webapp/src/components/tabContentComponents/backgroundDetectComponents/paneBackgroundDetect.vue b/webapp/src/components/tabContentComponents/backgroundDetectComponents/paneBackgroundDetect.vue index 8d6c8bac..bef06118 100644 --- a/webapp/src/components/tabContentComponents/backgroundDetectComponents/paneBackgroundDetect.vue +++ b/webapp/src/components/tabContentComponents/backgroundDetectComponents/paneBackgroundDetect.vue @@ -10,15 +10,15 @@