openflexure-microscope-server/src/openflexure_microscope_server/things/background_detect.py

"""Provide functionality to detect if the camera is imaging sample or background.

An example background image is captured by the camera and sent to classes in the module
for analysis. Information from these images is used to detect whether an image from the
current camera field of view contains sample.
"""

from typing import Optional, TypeVar

import cv2
import numpy as np
from pydantic import BaseModel

import labthings_fastapi as lt

from openflexure_microscope_server.ui import (
    UI_ELEMENT_RESPONSE,
    UIElementList,
    property_control_for,
)

SettingsType = TypeVar("SettingsType", bound=BaseModel)
BackgroundType = TypeVar("BackgroundType", bound=BaseModel)


class MissingBackgroundDataError(lt.exceptions.InvocationError):
    """An error raised if checking for sample without background data set."""


class ChannelBlankError(lt.exceptions.InvocationError):
    """An error raised if a channel has no measured standard deviation.

    This is not physical and usually means the camera has not yet booted or changed
    mode fully.
    """


class BackgroundDetectAlgorithm(lt.Thing):
    """The base class for defining background detect algorithms."""

    _class_settings = {"validate_properties_on_set": True}

    display_name: str = lt.property(default="Base Detector", readonly=True)

    def __init__(self, thing_server_interface: lt.ThingServerInterface) -> None:
        """Initialise and create the lock."""
        if self.display_name == "Base Detector":
            raise NotImplementedError(
                "Do not try to use the BackgroundDetectAlgorithm directly. "
                " Use a subclass"
            )
        super().__init__(thing_server_interface)

    @lt.property
    def ready(self) -> bool:
        """Whether the background detector is ready."""
        raise NotImplementedError(
            "Each background detect algorithm must implement a ready property."
        )

    def image_is_sample(self, image: np.ndarray) -> tuple[bool, str]:
        """Label the current image as either background or sample.

        :returns: A tuple of the result (boolean), and explanation string. The
            explanation string is formatted so it can be added into a sentence such as
            ``An action was taken because the image is {message}.``
        """
        raise NotImplementedError(
            "Each background detect algorithm must implement an image_is_sample method."
        )

    def set_background(self, image: np.ndarray) -> None:
        """Use the input image to update the background data.

        Background data must be a Pydantic BaseModel.
        """
        raise NotImplementedError(
            "Each background detect algorithm must implement an set_background method."
        )

    @lt.endpoint("get", "settings_ui", responses=UI_ELEMENT_RESPONSE)
    def settings_ui(self) -> UIElementList:
        """Return the UI for the background detect settings."""
        raise NotImplementedError(
            "Each background detect algorithm must implement an settings_ui method."
        )


class ColourChannelDetectLUV(BackgroundDetectAlgorithm):
    """Compare images with a known background in LUV colourspace.

    This uses an LUV colour space checking only the mean and standard deviation of the
    U and V channels. The LUV colourspace as it collect colours together in a human-
    intuitive way.
    """

    display_name: str = lt.property(default="Colour Channel (LUV)", readonly=True)

    background_means: Optional[list[float]] = lt.setting(default=None, readonly=True)
    """The mean of each channel in the colourspace."""
    background_stds: Optional[list[float]] = lt.setting(default=None, readonly=True)
    """The standard deviation of each channel in the colourspace."""

    # These are the same as those used for ChannelDeviationLUV. More detail is
    # provided there.
    min_stds = [0.5, 0.3, 0.5]

    channel_tolerance: float = lt.setting(default=7.0, ge=0)
    """Channel Tolerance

    The number of standard deviations a pixel value must be from the background mean
    to be considered sample.
    """

    min_sample_coverage: float = lt.setting(default=25, ge=0, le=100)
    """Sample Coverage Required (%)

    The minimum percentage of the image that needs to be identified as sample for the
    image to be labeled as containing sample.
    """

    @lt.endpoint("get", "settings_ui", responses=UI_ELEMENT_RESPONSE)
    def settings_ui(self) -> UIElementList:
        """Return the UI for the background detect settings."""
        return UIElementList(
            [
                property_control_for(
                    self, "channel_tolerance", label="Channel Tolerance"
                ),
                property_control_for(
                    self, "min_sample_coverage", label="Sample Coverage Required (%)"
                ),
            ]
        )

    @lt.property
    def ready(self) -> bool:
        """Whether the background detector is ready."""
        return self.background_means is not None and self.background_stds is not None

    def background_mask(self, image: np.ndarray) -> np.ndarray:
        """Calculate a binary image, showing whether each pixel is background.

        True is background.

        The image should be in LUV format, the output will be binary with the
        same shape in the first two dimensions.
        """
        if self.background_means is None or self.background_stds is None:
            raise RuntimeError(
                "Cannot calculated background mask if no background is set."
            )
        # The ``[1:]`` selects only the U and V channels of the image.
        # Only U and V are used as brightness (L) often changes as
        # the height of the sample changes.
        # Wrapping in ``[[ ]]`` forces the colour channels to the numpy axis 2
        # (3rd axis) so they are compared to the colour channel of each pixel.
        means = np.array([[self.background_means[1:]]])
        stds = np.array([[self.background_stds[1:]]])

        # Compare each image in the pixel with the mean and standard deviation along
        # axis to (the colour channels).
        return np.all(
            np.abs(image[:, :, 1:] - means) < stds * self.channel_tolerance,
            axis=2,
        )

    def get_sample_coverage(self, image: np.ndarray) -> float:
        """Return the percentage of the input image that is background.

        Evaluate whether it is foreground or background by comparing it to the saved
        statistics for a background image on a per-pixel basis

        :returns: A value (between 0 and 100) is the percentage of the image that is
            sample.
        """
        if self.background_means is None or self.background_stds is None:
            raise MissingBackgroundDataError(
                "Background is not set: you need to calibrate background detection."
            )
        image_luv = cv2.cvtColor(image, cv2.COLOR_RGB2LUV)
        mask = self.background_mask(image_luv)

        return float(1 - np.count_nonzero(mask) / np.prod(mask.shape)) * 100

    def image_is_sample(self, image: np.ndarray) -> tuple[bool, str]:
        """Label the current image as either background or sample.

        :returns: A tuple of the result (boolean), and explanation string. The
            explanation string is formatted so it can be added into a sentence such as
            ``An action was taken because the image is {message}.``
        """
        sample_coverage = self.get_sample_coverage(image)

        # Use bool otherwise get numpy variants of True and False.
        is_sample = bool(sample_coverage > self.min_sample_coverage)
        message = f"{sample_coverage:0.1f}% sample"
        if not is_sample:
            message = "only " + message
        return is_sample, message

    def set_background(self, image: np.ndarray) -> None:
        """Use the input image to update the background distributions."""
        image_luv = cv2.cvtColor(image, cv2.COLOR_RGB2LUV)

        mu = np.mean(image_luv, axis=(0, 1))
        std = np.std(image_luv, axis=(0, 1))

        if np.any(std == 0):
            raise ChannelBlankError("Some LUV channels have no standard deviation.")
        std = np.maximum(std, self.min_stds)

        self.background_means = mu.tolist()
        self.background_stds = std.tolist()


class ChannelDeviationLUV(BackgroundDetectAlgorithm):
    """Compare the standard deviations of the LUV channels in a grid to background data.

    Using an LUV colour space, each image is divided into an 8x8 grid of images.
    The standard deviation of each channel of each image is calculated and compared
    to the median standard deviation for a grid of background images.
    """

    display_name: str = lt.property(default="Channel Deviation (LUV)", readonly=True)

    background_stds: Optional[list[float]] = lt.setting(default=None, readonly=True)
    """The standard deviation of each channel in the colourspace."""

    # Empirically, 0.5 seems to be approximate the standard deviation for a good image
    # in L and V. U appears to be about 60% of this value. U is about 65% of V when
    # converting white-noise in RGB into LUV (note that we are using cv2's internal
    # LUV colour space not converting to the CIELUV numbers)
    min_stds = [0.5, 0.3, 0.5]

    channel_tolerance: float = lt.setting(default=7.0, ge=0)
    """Channel Tolerance

    The number of standard deviations a pixel value must be from the background mean
    to be considered sample.
    """

    min_sample_coverage: float = lt.setting(default=25, ge=0, le=100)
    """Sample Coverage Required (%)

    The minimum percentage of the image that needs to be identified as sample for the
    image to be labeled as containing sample.
    """

    @lt.endpoint("get", "settings_ui", responses=UI_ELEMENT_RESPONSE)
    def settings_ui(self) -> UIElementList:
        """Return the UI for the background detect settings."""
        return UIElementList(
            [
                property_control_for(
                    self, "channel_tolerance", label="Channel Tolerance"
                ),
                property_control_for(
                    self, "min_sample_coverage", label="Sample Coverage Required (%)"
                ),
            ]
        )

    @lt.property
    def ready(self) -> bool:
        """Whether the background detector is ready."""
        return self.background_stds is not None

    def get_sample_coverage(self, image: np.ndarray) -> float:
        """Return the percentage of the input image that is background.

        Evaluate whether it is foreground or background by comparing the standard
        deviations of an 8x8 grid of sub-images to the median standard deviation
        from a background image.

        :returns: A value (between 0 and 100) that is the percentage of the image that is
            sample.
        """
        if self.background_stds is None:
            raise MissingBackgroundDataError(
                "Background is not set: you need to calibrate background detection."
            )
        image_luv = cv2.cvtColor(image, cv2.COLOR_RGB2LUV)
        stds = _chunked_stds(image_luv, 8, 8)

        bg_stds = self.background_stds
        l_cut = bg_stds[0] * self.channel_tolerance
        u_cut = bg_stds[1] * self.channel_tolerance
        v_cut = bg_stds[2] * self.channel_tolerance

        populated_regions = (
            (stds[:, :, 0] > l_cut) | (stds[:, :, 1] > u_cut) | (stds[:, :, 2] > v_cut)
        )

        return float(100 * np.sum(populated_regions) / 64)

    def image_is_sample(self, image: np.ndarray) -> tuple[bool, str]:
        """Label the current image as either background or sample.

        :returns: A tuple of the result (boolean), and explanation string. The
            explanation string is formatted so it can be added into a sentence such as
            ``An action was taken because the image is {message}.``
        """
        sample_coverage = self.get_sample_coverage(image)

        # Use bool otherwise get numpy variants of True and False.
        is_sample = bool(sample_coverage > self.min_sample_coverage)
        message = f"{sample_coverage:0.1f}% sample"
        if not is_sample:
            message = "only " + message
        return is_sample, message

    def set_background(self, image: np.ndarray) -> None:
        """Use the input image to update the background distributions."""
        image_luv = cv2.cvtColor(image, cv2.COLOR_RGB2LUV)
        c_stds = _chunked_stds(image_luv, 8, 8)
        channel_blank = np.all(c_stds == 0, axis=(0, 1))
        if np.any(channel_blank):
            raise ChannelBlankError("Some LUV channels have no standard devaition.")

        std = np.median(c_stds, axis=(0, 1))
        std = np.maximum(std, self.min_stds)
        self.background_stds = std.tolist()


def _chunked_stds(img: np.ndarray, n_rows: int = 8, n_cols: int = 8) -> np.ndarray:
    """Split image into a grid and calculate std of each channel in each chunk.

    :param img: The image to analyse
    :param n_rows: The number of rows in the grid
    :param n_cols: The number of cols in the grid
    :return: A numpy array of the grid of standard deviations.
    """
    h, w = img.shape[:2]
    row_height = h // n_rows
    col_width = w // n_cols
    out = np.zeros((n_rows, n_cols, 3))
    for i in range(n_rows):
        for j in range(n_cols):
            chunk = img[
                i * row_height : (i + 1) * row_height,
                j * col_width : (j + 1) * col_width,
            ]
            out[i, j, :] = np.std(chunk, axis=(0, 1))
    return out