Merge branch 'bg_detect_in_camera' into 'v3'
Refactor background detect to allow switching algorithms See merge request openflexure/openflexure-microscope-server!327
This commit is contained in:
Julian Stirling
commit
547704fdc0
14 changed files with 587 additions and 261 deletions
|
|
@ -1,163 +0,0 @@
|
|||
"""Provide functionality to detect if the camera is imaging sample or background.
|
||||
|
||||
An example background image must be captured and analysed by BackgroundDetectThing,
|
||||
information from this images is used to detect whether the current camera field of
|
||||
view contains sample.
|
||||
"""
|
||||
|
||||
from typing import Mapping, Optional
|
||||
import cv2
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
from pydantic import BaseModel
|
||||
from scipy.stats import norm
|
||||
|
||||
import labthings_fastapi as lt
|
||||
from .camera import CameraDependency as CamDep
|
||||
|
||||
|
||||
class ChannelDistributions(BaseModel):
|
||||
"""A BaseModel for storing the channel distribution of a background image."""
|
||||
|
||||
means: list[float]
|
||||
"""The mean of each channel in the colourspace."""
|
||||
standard_deviations: list[float]
|
||||
"""The standard deviation of each channel in the colourspace."""
|
||||
colorspace: str = "LUV"
|
||||
"""The colourspace used."""
|
||||
|
||||
|
||||
class BackgroundDetectThing(lt.Thing):
|
||||
"""Thing for setting a background image and detecting sample in the field of view.
|
||||
|
||||
This uses an LUV colour space checking only the mean and standard deviation of the
|
||||
UV channels. Over time different, selectable, background detection methods will be
|
||||
added.
|
||||
"""
|
||||
|
||||
# Requires a getter and a setter to support being a BaseModel but being
|
||||
# saved to file as a dict
|
||||
_background_distributions: Optional[ChannelDistributions] = None
|
||||
|
||||
@lt.thing_setting
|
||||
def background_distributions(self) -> Optional[ChannelDistributions]:
|
||||
"""The statistics of the background image."""
|
||||
bd = self._background_distributions
|
||||
if bd is None:
|
||||
return None
|
||||
return ChannelDistributions(**bd)
|
||||
|
||||
@background_distributions.setter
|
||||
def background_distributions(
|
||||
self, value: Optional[ChannelDistributions | dict]
|
||||
) -> None:
|
||||
if value is None:
|
||||
self._background_distributions = None
|
||||
elif isinstance(value, ChannelDistributions):
|
||||
self._background_distributions = value.model_dump()
|
||||
elif isinstance(value, dict):
|
||||
self._background_distributions = value
|
||||
else:
|
||||
raise TypeError(
|
||||
f"Cannot set background_distributions with an object of type {type(value)}"
|
||||
)
|
||||
|
||||
tolerance = lt.ThingSetting(
|
||||
initial_value=7.0,
|
||||
model=float,
|
||||
)
|
||||
"""How many standard deviations to allow for the background."""
|
||||
|
||||
fraction = lt.ThingSetting(
|
||||
initial_value=25.0,
|
||||
model=float,
|
||||
)
|
||||
"""How much of the image needs to be not background to label as sample"""
|
||||
|
||||
def background_mask(self, image: np.ndarray) -> np.ndarray:
|
||||
"""Calculate a binary image, showing whether each pixel is background.
|
||||
|
||||
The image should be in LUV format, the output will be binary with the
|
||||
same shape in the first two dimensions.
|
||||
"""
|
||||
d = self.background_distributions
|
||||
if not d:
|
||||
raise RuntimeError(
|
||||
"Background is not set: you need to calibrate background detection."
|
||||
)
|
||||
# This image is in LUV space. But the brightness (L) often changes as the
|
||||
# height of the sample changes. Hence in the line below we are only using
|
||||
# the UV (colour) channels.
|
||||
return np.all(
|
||||
np.abs(image[:, :, 1:] - np.array(d.means[1:])[np.newaxis, np.newaxis, :])
|
||||
< np.array(d.standard_deviations[1:])[np.newaxis, np.newaxis, :]
|
||||
* self.tolerance,
|
||||
axis=2,
|
||||
)
|
||||
|
||||
@lt.thing_action
|
||||
def background_fraction(self, cam: CamDep) -> float:
|
||||
"""Determine what fraction of the current image is background.
|
||||
|
||||
This action will acquire a new image from the preview stream, then
|
||||
evaluate whether it is foreground or background, by comparing it
|
||||
too the saved statistics. This is done on a per-pixel basis, and
|
||||
the returned value (between 0 and 100) is the fraction of the image
|
||||
that is background.
|
||||
"""
|
||||
current_image = cam.grab_jpeg()
|
||||
current_image = np.array(Image.open(current_image.open()))
|
||||
|
||||
# we're working in the LUV colourspace as it collect colours together in a human-intuitive way
|
||||
current_image_luv = cv2.cvtColor(current_image, cv2.COLOR_RGB2LUV)
|
||||
mask = self.background_mask(current_image_luv)
|
||||
return np.count_nonzero(mask) / np.prod(mask.shape) * 100
|
||||
|
||||
@lt.thing_action
|
||||
def image_is_sample(self, cam: CamDep) -> bool:
|
||||
"""Label the current image as either background or sample."""
|
||||
b_fraction = self.background_fraction(cam)
|
||||
fraction_threshold = self.fraction
|
||||
|
||||
return (100 - b_fraction) > fraction_threshold
|
||||
|
||||
@lt.thing_action
|
||||
def set_background(self, cam: CamDep):
|
||||
"""Grab an image, and use its statistics to set the background.
|
||||
|
||||
This should be run when the microscope is looking at an empty region,
|
||||
and will calculate the mean and standard deviation of the pixel values
|
||||
in the LUV colourspace. These values will then be used to compare
|
||||
future images to the distribution, to determine if each pixel is
|
||||
foreground or background.
|
||||
"""
|
||||
background = cam.grab_jpeg()
|
||||
background = np.array(Image.open(background.open()))
|
||||
|
||||
# we're working in the LUV colourspace as it collect colours together in a human-intuitive way
|
||||
background_luv = cv2.cvtColor(background, cv2.COLOR_RGB2LUV)
|
||||
|
||||
ch1 = (background_luv.T[0]).flatten()
|
||||
ch2 = (background_luv.T[1]).flatten()
|
||||
ch3 = (background_luv.T[2]).flatten()
|
||||
|
||||
points = np.array([np.asarray(ch1), np.asarray(ch2), np.asarray(ch3)]).T
|
||||
|
||||
# we get the mean and standard deviation of values in each channel
|
||||
mu, std = np.apply_along_axis(norm.fit, 0, points)
|
||||
|
||||
self.background_distributions = ChannelDistributions(
|
||||
means=mu.tolist(),
|
||||
standard_deviations=std.tolist(),
|
||||
colorspace="LUV",
|
||||
)
|
||||
|
||||
@property
|
||||
def thing_state(self) -> Mapping:
|
||||
"""Summary metadata describing the current state of the Thing."""
|
||||
bd = self.background_distributions
|
||||
return {
|
||||
"background_distributions": bd.model_dump() if bd else None,
|
||||
"tolerance": self.tolerance,
|
||||
"fraction": self.fraction,
|
||||
}
|
||||
|
|
@ -10,6 +10,7 @@ from __future__ import annotations
|
|||
from typing import Literal, Optional, Tuple, Any
|
||||
import json
|
||||
import time
|
||||
import logging
|
||||
|
||||
import numpy as np
|
||||
from pydantic import RootModel
|
||||
|
|
@ -19,6 +20,14 @@ import piexif
|
|||
import labthings_fastapi as lt
|
||||
from labthings_fastapi.types.numpy import NDArray
|
||||
|
||||
from openflexure_microscope_server.background_detect import (
|
||||
ColourChannelDetectLUV,
|
||||
BackgroundDetectAlgorithm,
|
||||
BackgroundDetectorStatus,
|
||||
)
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class JPEGBlob(lt.blob.Blob):
|
||||
"""A class representing a JPEG image as a LabThings FastAPI Blob."""
|
||||
|
|
@ -155,6 +164,18 @@ class BaseCamera(lt.Thing):
|
|||
lores_mjpeg_stream = lt.outputs.MJPEGStreamDescriptor()
|
||||
_memory_buffer = CameraMemoryBuffer()
|
||||
|
||||
def __init__(self):
|
||||
"""Initialise the base camera, this creates the background detectors.
|
||||
|
||||
This must be run by all child camera classes.
|
||||
|
||||
To add a new background detector to the server it must be added to the
|
||||
dictionary in this function. Configuration will be added at a later date.
|
||||
"""
|
||||
super().__init__()
|
||||
self.background_detectors = {"Colour Channels (LUV)": ColourChannelDetectLUV()}
|
||||
self._detector_name = "Colour Channels (LUV)"
|
||||
|
||||
def __enter__(self) -> None:
|
||||
"""Open hardware connection when the Thing context manager is opened."""
|
||||
raise NotImplementedError("CameraThings must define their own __enter__ method")
|
||||
|
|
@ -455,6 +476,86 @@ class BaseCamera(lt.Thing):
|
|||
time.sleep(self.settling_time)
|
||||
self.discard_frames()
|
||||
|
||||
# Note that the default detector name is set at init. This is over written if
|
||||
# setting is loaded from disk.
|
||||
@lt.thing_setting
|
||||
def detector_name(self) -> str:
|
||||
"""The name of the active background selector."""
|
||||
return self._detector_name
|
||||
|
||||
@detector_name.setter
|
||||
def detector_name(self, name: str) -> None:
|
||||
"""Validate and set detector_name."""
|
||||
if name not in self.background_detectors:
|
||||
raise ValueError(f"{name} is not a valid background detector name")
|
||||
self._detector_name = name
|
||||
|
||||
@property
|
||||
def active_detector(self) -> BackgroundDetectAlgorithm:
|
||||
"""The active background detector instance."""
|
||||
return self.background_detectors[self.detector_name]
|
||||
|
||||
@lt.thing_property
|
||||
def background_detector_status(self) -> BackgroundDetectorStatus:
|
||||
"""The status of the active detector for the UI."""
|
||||
return self.active_detector.status
|
||||
|
||||
@lt.thing_setting
|
||||
def background_detector_data(self) -> dict:
|
||||
"""The data for each background detector, used to save to disk."""
|
||||
data = {}
|
||||
for name, obj in self.background_detectors.items():
|
||||
bg_data = (
|
||||
None
|
||||
if obj.background_data is None
|
||||
else obj.background_data.model_dump()
|
||||
)
|
||||
data[name] = {
|
||||
"settings": obj.settings.model_dump(),
|
||||
"background_data": bg_data,
|
||||
}
|
||||
return data
|
||||
|
||||
@background_detector_data.setter
|
||||
def background_detector_data(self, data: dict) -> None:
|
||||
"""Set the data for each detector. Only to be used as settings are loaded from disk.
|
||||
|
||||
Do not call over HTTP. This needs to be updated once LbaThings Settings can be
|
||||
read-only over HTTP (#484).
|
||||
"""
|
||||
for name, instance_data in data.items():
|
||||
if name in self.background_detectors:
|
||||
obj = self.background_detectors[name]
|
||||
obj.settings = instance_data["settings"]
|
||||
obj.background_data = instance_data["background_data"]
|
||||
else:
|
||||
LOGGER.warning(
|
||||
f"No background detector named {name}, settings will be discarded."
|
||||
)
|
||||
|
||||
@lt.thing_action
|
||||
def image_is_sample(self, portal: lt.deps.BlockingPortal) -> tuple[bool, str]:
|
||||
"""Label the current image as either background or sample."""
|
||||
current_image = self.grab_jpeg(portal)
|
||||
current_image = np.array(Image.open(current_image.open()))
|
||||
return self.active_detector.image_is_sample(current_image)
|
||||
|
||||
@lt.thing_action
|
||||
def set_background(self, portal: lt.deps.BlockingPortal) -> None:
|
||||
"""Grab an image, and use its statistics to set the background.
|
||||
|
||||
This should be run when the microscope is looking at an empty region,
|
||||
and will calculate the mean and standard deviation of the pixel values
|
||||
in the LUV colourspace. These values will then be used to compare
|
||||
future images to the distribution, to determine if each pixel is
|
||||
foreground or background.
|
||||
"""
|
||||
background = self.grab_jpeg(portal)
|
||||
background = np.array(Image.open(background.open()))
|
||||
self.active_detector.set_background(background)
|
||||
# Manually save settings as the setter is not called.
|
||||
self.save_settings()
|
||||
|
||||
|
||||
CameraDependency = lt.deps.direct_thing_client_dependency(BaseCamera, "/camera/")
|
||||
RawCameraDependency = lt.deps.raw_thing_dependency(BaseCamera)
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ class OpenCVCamera(BaseCamera):
|
|||
|
||||
:param camera_index: The index of the camera to use for the microscope.
|
||||
"""
|
||||
super().__init__()
|
||||
self.camera_index = camera_index
|
||||
self._capture_thread: Optional[Thread] = None
|
||||
self._capture_enabled = False
|
||||
|
|
|
|||
|
|
@ -118,6 +118,7 @@ class StreamingPiCamera2(BaseCamera):
|
|||
:param camera_num: The number of the camera. This should generally be left as 0
|
||||
as most Raspberry Pi boards only support 1 camera.
|
||||
"""
|
||||
super().__init__()
|
||||
self._setting_save_in_progress = False
|
||||
self.camera_num = camera_num
|
||||
self.camera_configs: dict[str, dict] = {}
|
||||
|
|
@ -736,7 +737,7 @@ class StreamingPiCamera2(BaseCamera):
|
|||
self._initialise_picamera()
|
||||
|
||||
@lt.thing_action
|
||||
def full_auto_calibrate(self) -> None:
|
||||
def full_auto_calibrate(self, portal: lt.deps.BlockingPortal) -> None:
|
||||
"""Perform a full auto-calibration.
|
||||
|
||||
This function will call the other calibration actions in sequence:
|
||||
|
|
@ -746,12 +747,14 @@ class StreamingPiCamera2(BaseCamera):
|
|||
* ``set_static_green_equalisation`` to set geq offset to max
|
||||
* ``calibrate_lens_shading``
|
||||
* ``calibrate_white_balance``
|
||||
* ``set_background``
|
||||
"""
|
||||
self.flat_lens_shading()
|
||||
self.auto_expose_from_minimum()
|
||||
self.set_static_green_equalisation()
|
||||
self.calibrate_lens_shading()
|
||||
self.calibrate_white_balance()
|
||||
self.set_background(portal)
|
||||
|
||||
@lt.thing_action
|
||||
def flat_lens_shading(self) -> None:
|
||||
|
|
|
|||
|
|
@ -58,6 +58,7 @@ class SimulatedCamera(BaseCamera):
|
|||
:param frame_interval: Nominally the time between frames on the MJPEG stream,
|
||||
however the rate may be slower due to calculation time for focus.
|
||||
"""
|
||||
super().__init__()
|
||||
self.shape = shape
|
||||
self.glyph_shape = glyph_shape
|
||||
self.canvas_shape = canvas_shape
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
"""The core sample scanning functionality for the OpenFlexure Microscope.
|
||||
|
||||
SmartScan provides sample scanning functionality including automatic background
|
||||
detection (via the `BackgroundDetectThing`) and automatic path planning via
|
||||
detection (via the ``CameraThing``) and automatic path planning via
|
||||
`scan_planners`. It manages the directories of past scans via `scan_directories`.
|
||||
It also controls external processes for live stitching composite images, and
|
||||
the creation of the final stitched images.
|
||||
|
|
@ -29,7 +29,6 @@ from openflexure_microscope_server import scan_planners
|
|||
# Things
|
||||
from .autofocus import AutofocusThing
|
||||
from .camera_stage_mapping import CameraStageMapper
|
||||
from .background_detect import BackgroundDetectThing
|
||||
from .camera import CameraDependency as CamDep
|
||||
from .stage import StageDependency as StageDep
|
||||
|
||||
|
|
@ -37,9 +36,6 @@ CSMDep = lt.deps.direct_thing_client_dependency(
|
|||
CameraStageMapper, "/camera_stage_mapping/"
|
||||
)
|
||||
AutofocusDep = lt.deps.direct_thing_client_dependency(AutofocusThing, "/autofocus/")
|
||||
BackgroundDep = lt.deps.direct_thing_client_dependency(
|
||||
BackgroundDetectThing, "/background_detect/"
|
||||
)
|
||||
|
||||
JPEGBlob = lt.blob.blob_type("image/jpeg")
|
||||
ZipBlob = lt.blob.blob_type("application/zip")
|
||||
|
|
@ -110,7 +106,6 @@ class SmartScanThing(lt.Thing):
|
|||
self._cam: Optional[CamDep] = None
|
||||
self._metadata_getter: Optional[lt.deps.GetThingStates] = None
|
||||
self._csm: Optional[CSMDep] = None
|
||||
self._background_detect: Optional[BackgroundDep] = None
|
||||
self._ongoing_scan: Optional[scan_directories.ScanDirectory] = None
|
||||
self._starting_position: Optional[Mapping[str, int]] = None
|
||||
self._capture_thread: Optional[ErrorCapturingThread] = None
|
||||
|
|
@ -128,14 +123,13 @@ class SmartScanThing(lt.Thing):
|
|||
cam: CamDep,
|
||||
metadata_getter: lt.deps.GetThingStates,
|
||||
csm: CSMDep,
|
||||
background_detect: BackgroundDep,
|
||||
scan_name: str = "",
|
||||
):
|
||||
"""Move the stage to cover an area, taking images that can be tiled together.
|
||||
|
||||
The stage will move in a pattern that grows outwards from the starting point,
|
||||
stopping once it is surrounded by "background" (as detected by the
|
||||
background_detect Thing) or reaches the "max_range" measured in steps.
|
||||
camera Thing) or reaches the "max_range" measured in steps.
|
||||
"""
|
||||
got_lock = self._scan_lock.acquire(timeout=0.1)
|
||||
if not got_lock:
|
||||
|
|
@ -149,7 +143,6 @@ class SmartScanThing(lt.Thing):
|
|||
self._cam = cam
|
||||
self._metadata_getter = metadata_getter
|
||||
self._csm = csm
|
||||
self._background_detect = background_detect
|
||||
self._capture_thread = None
|
||||
self._scan_images_taken = 0
|
||||
|
||||
|
|
@ -183,7 +176,6 @@ class SmartScanThing(lt.Thing):
|
|||
self._cam = None
|
||||
self._metadata_getter = None
|
||||
self._csm = None
|
||||
self._background_detect = None
|
||||
self._capture_thread = None
|
||||
self._ongoing_scan = None
|
||||
self._scan_images_taken = None
|
||||
|
|
@ -207,7 +199,7 @@ class SmartScanThing(lt.Thing):
|
|||
)
|
||||
|
||||
if self.skip_background:
|
||||
if not self._background_detect.background_distributions:
|
||||
if not self._cam.background_detector_status.ready:
|
||||
raise RuntimeError(
|
||||
"Background is not set: you need to calibrate background detection."
|
||||
)
|
||||
|
|
@ -511,15 +503,13 @@ class SmartScanThing(lt.Thing):
|
|||
capture_image = True
|
||||
# If skipping background, take an image to check if current field of view is background
|
||||
if self._scan_data["skip_background"]:
|
||||
capture_image = self._background_detect.image_is_sample()
|
||||
capture_image, bg_message = self._cam.image_is_sample()
|
||||
|
||||
if not capture_image:
|
||||
route_planner.mark_location_visited(
|
||||
new_pos_xyz, imaged=False, focused=False
|
||||
)
|
||||
# Background fraction is actually a percentage
|
||||
back_perc = round(self._background_detect.background_fraction(), 0)
|
||||
msg = f"Skipping {new_pos_xyz} as it is {back_perc}% background."
|
||||
msg = f"Skipping {new_pos_xyz} as it is {bg_message}."
|
||||
self._scan_logger.info(msg)
|
||||
continue
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue