From 25552215f92c6a3b2d47c95956e537c377c7492d Mon Sep 17 00:00:00 2001 From: Richard Bowman Date: Wed, 3 Jan 2024 16:56:20 +0000 Subject: [PATCH] Refactored background detection into a Thing The new background detect Thing (currently in smart_scan.py) manages its own settings and exposes a few more actions. --- src/openflexure_microscope_server/server.py | 3 +- .../things/smart_scan.py | 146 ++++++++++++------ 2 files changed, 99 insertions(+), 50 deletions(-) diff --git a/src/openflexure_microscope_server/server.py b/src/openflexure_microscope_server/server.py index 9f64d4ae..30236581 100644 --- a/src/openflexure_microscope_server/server.py +++ b/src/openflexure_microscope_server/server.py @@ -10,7 +10,7 @@ from .things.camera_stage_mapping import CameraStageMapper from .things.system_control import SystemControlThing from .things.settings_manager import SettingsManager from .things.auto_recentre_stage import RecentringThing -from .things.smart_scan import SmartScanThing +from .things.smart_scan import SmartScanThing, BackgroundDetectThing from .serve_static_files import add_static_files from .logging import configure_logging, retrieve_log @@ -25,6 +25,7 @@ thing_server.add_thing(CameraStageMapper(), "/camera_stage_mapping/") thing_server.add_thing(SystemControlThing(), "/system_control/") thing_server.add_thing(SettingsManager(), "/settings/") thing_server.add_thing(SmartScanThing(), "/smart_scan/") +thing_server.add_thing(BackgroundDetectThing(), "/background_detect/") try: add_static_files(thing_server.app) except RuntimeError: diff --git a/src/openflexure_microscope_server/things/smart_scan.py b/src/openflexure_microscope_server/things/smart_scan.py index 547beb3b..1978af3a 100644 --- a/src/openflexure_microscope_server/things/smart_scan.py +++ b/src/openflexure_microscope_server/things/smart_scan.py @@ -1,39 +1,28 @@ +from typing import Mapping, Optional import cv2 import numpy as np import os import time from PIL import Image +from pydantic import BaseModel from scipy.stats import norm import logging from copy import deepcopy from labthings_fastapi.thing import Thing from labthings_fastapi.dependencies.thing import direct_thing_client_dependency -from labthings_fastapi.decorators import thing_action +from labthings_fastapi.decorators import thing_action, thing_property from labthings_sangaboard import SangaboardThing from labthings_picamera2.thing import StreamingPiCamera2 from openflexure_microscope_server.things.autofocus import AutofocusThing from openflexure_microscope_server.things.camera_stage_mapping import CameraStageMapper from openflexure_microscope_server.things.auto_recentre_stage import RecentringThing -from openflexure_microscope_server.things.settings_manager import SettingsManager StageDep = direct_thing_client_dependency(SangaboardThing, "/stage/") CamDep = direct_thing_client_dependency(StreamingPiCamera2, "/camera/") CSMDep = direct_thing_client_dependency(CameraStageMapper, "/camera_stage_mapping/") AutofocusDep = direct_thing_client_dependency(AutofocusThing, "/autofocus/") RecentreStage = direct_thing_client_dependency(RecentringThing, "/auto_recentre_stage/") -SettingsDep = direct_thing_client_dependency(SettingsManager, "/settings/") - - -def check_dist(image, stats_list, multiple): - """tests whether each pixel in image is within multiple stdevs (stats_list[1]) of the mean (stats_list[0]) for all three channels - returns a mask with True for pixels within that range (similar to stats list) and False otherwise - """ - return np.all( - np.abs(image - stats_list[np.newaxis, np.newaxis, 0, :]) - < stats_list[np.newaxis, np.newaxis, 1, :] * multiple, - axis=2, - ) def closest(current, focused_path): @@ -150,9 +139,82 @@ def generate_config(folder_path: str, positions: list, names: list): loc = positions[i] - mean_loc fp.write(f'{names[i]}; ; {loc[1], loc[0]} \n') -class SmartScanThing(Thing): + +class ChannelDistributions(BaseModel): + means: list[float] + standard_deviations: list[float] + colorspace: str = "LUV" + +class BackgroundDetectThing(Thing): + @thing_property + def background_distributions(self) -> Optional[ChannelDistributions]: + """The statistics of the background image""" + bd = self.thing_settings.get("background_distributions", None) + if bd: + return ChannelDistributions(**bd) + else: + return None + + @background_distributions.setter + def background_distributions(self, value: Optional[ChannelDistributions]) -> None: + try: + self.thing_settings["background_distributions"] = value.model_dump() + except AttributeError: + self.thing_settings["background_distributions"] = None + + @thing_property + def tolerance(self) -> float: + """How many standard deviations to allow for the background""" + return self.thing_settings.get("tolerance", 5) + + @tolerance.setter + def tolerance(self, value: float) -> None: + self.thing_settings["tolerance"] = value + + def background_mask(self, image: np.ndarray) -> np.ndarray: + """Calculate a binary image, showing whether each pixel is background + + The image should be in LUV format, the ouput will be binary with the + same shape in the first two dimensions. + """ + d = self.background_distributions + if not d: + raise RuntimeError("Background is not set: you need to calibrate background detection.") + return np.all( + np.abs(image - np.array(d.means)[np.newaxis, np.newaxis, :]) + < np.array(d.standard_deviations)[np.newaxis, np.newaxis, :] * self.tolerance, + axis=2, + ) + @thing_action - def set_template(self, settings: SettingsDep, cam: CamDep): + def background_fraction(self, cam: CamDep) -> float: + """Determine what fraction of the current image is background + + This action will acquire a new image from the preview stream, then + evaluate whether it is foreground or background, by comparing it + too the saved statistics. This is done on a per-pixel basis, and + the returned value (between 0 and 1) is the fraction of the image + that is background. + """ + background = cam.grab_jpeg() + background = np.array(Image.open(background.open())) + + # we're working in the LUV colourspace as it collect colours together in a human-intuitive way + background_LUV = cv2.cvtColor(background, cv2.COLOR_RGB2LUV) + mask = self.background_mask(background_LUV) + return np.count_nonzero(mask) / np.prod(mask.shape) + + + @thing_action + def set_background(self, cam: CamDep): + """Grab an image, and use its statistics to set the background + + This should be run when the microscope is looking at an empty region, + and will calculate the mean and standard deviation of the pixel values + in the LUV colourspace. These values will then be used to compare + future images to the distribution, to determine if each pixel is + foreground or background. + """ background = cam.grab_jpeg() background = np.array(Image.open(background.open())) @@ -166,24 +228,27 @@ class SmartScanThing(Thing): points = np.array([np.asarray(ch1), np.asarray(ch2), np.asarray(ch3)]).T # we get the mean and standard deviation of values in each channel - mu, std = np.apply_along_axis(norm.fit, 0, points) - stats_list = np.vstack([mu, std]) - settings.update_external_metadata( - data={"background_stats": stats_list.tolist()} + self.background_distributions = ChannelDistributions( + means = mu.tolist(), + standard_deviations = std.tolist(), + colorspace = "LUV", ) - current_keys = settings.external_metadata_in_state - logging.info(type(current_keys)) + @property + def thing_state(self) -> Mapping: + bd = self.background_distributions + return { + "background_distributions": bd.model_dump() if bd else None, + "tolerance": self.tolerance, + } - if "background_stats" not in current_keys: - current_keys.append("background_stats") - settings.external_metadata_in_state = current_keys - - logging.info(settings.external_metadata_in_state) - logging.info(settings.external_metadata) + +BackgroundDep = direct_thing_client_dependency(BackgroundDetectThing, "/background_detect/") +class SmartScanThing(Thing): + @thing_action def sample_scan( self, @@ -191,16 +256,9 @@ class SmartScanThing(Thing): stage: StageDep, cam: CamDep, csm: CSMDep, - settings: SettingsDep, + background_detect: BackgroundDep, recentre: RecentreStage, ): - # try: - stats_list = np.array(settings.external_metadata["background_stats"]) - # print(stats_list) - # except: - # logging.warning("No template set") - # raise Exception - names = [] positions = [] @@ -277,20 +335,10 @@ class SmartScanThing(Thing): x=int(loc[0]), y=int(loc[1]), z=int(focused_path[z_index][2]) ) - # capture an image, convert it to LUV, then make lists of the 3 channels' values - img = cam.grab_jpeg() - img = np.array(Image.open(img.open())) - - img2 = cv2.cvtColor(img, cv2.COLOR_RGB2LUV) - ch4 = (img2.T[0]).flatten() - ch5 = (img2.T[1]).flatten() - ch6 = (img2.T[2]).flatten() - - # mask the image to only include pixels outside 5 stds of the mean of all three channels. - # get the percent of pixels that are in the mask (assumed sample) - img_mask = check_dist(img2, stats_list, 5) + # Check if the image is background + background_fraction = background_detect.background_fraction() background_coverage = round( - 100 * np.count_nonzero(img_mask) / img_mask.size, 1 + 100 * background_fraction, 1 ) # if more than 92% of the image is background, treat it as background and continue