From 603b9c7d1627e011418e5bbbbcdebe83aae9d5f5 Mon Sep 17 00:00:00 2001 From: jaknapper Date: Wed, 16 Apr 2025 09:40:14 +0100 Subject: [PATCH 01/56] Functional z stack Thing --- .../things/smart_scan.py | 5 + .../things/z_stack.py | 137 ++++++++++++++++++ 2 files changed, 142 insertions(+) create mode 100644 src/openflexure_microscope_server/things/z_stack.py diff --git a/src/openflexure_microscope_server/things/smart_scan.py b/src/openflexure_microscope_server/things/smart_scan.py index 50a8c442..905f5968 100644 --- a/src/openflexure_microscope_server/things/smart_scan.py +++ b/src/openflexure_microscope_server/things/smart_scan.py @@ -32,6 +32,7 @@ from openflexure_microscope_server.utilities import ErrorCapturingThread from openflexure_microscope_server.things.autofocus import AutofocusThing from openflexure_microscope_server.things.camera_stage_mapping import CameraStageMapper from openflexure_microscope_server.things.background_detect import BackgroundDetectThing +from openflexure_microscope_server.things.z_stack import ZStackThing from openflexure_microscope_server import scan_planners CSMDep = direct_thing_client_dependency(CameraStageMapper, "/camera_stage_mapping/") @@ -139,6 +140,7 @@ class SmartScanThing(Thing): self._background_detect: Optional[BackgroundDep] = None self._ongoing_scan_name: Optional[str] = None self._starting_position: Optional[Mapping[str, int]] = None + self._z_stack: Optional[ZStackDep] = None self._capture_thread: Optional[ErrorCapturingThread] = None self._scan_images_taken: Optional[int] = None # TODO Scan data is a dict during refactoring, should become a dataclass @@ -155,6 +157,7 @@ class SmartScanThing(Thing): metadata_getter: GetThingStates, csm: CSMDep, background_detect: BackgroundDep, + z_stack: ZStackDep, scan_name: str = "", ): """Move the stage to cover an area, taking images that can be tiled together. @@ -178,6 +181,7 @@ class SmartScanThing(Thing): self._csm = csm self._background_detect = background_detect self._capture_thread = None + self._z_stack = z_stack self._scan_images_taken = 0 # Don't set self._scan_data dictionary. This is done at the start of _run_scan @@ -213,6 +217,7 @@ class SmartScanThing(Thing): self._ongoing_scan_name = None self._scan_images_taken = None self._scan_data = None + self._z_stack = None self._scan_lock.release() def promote_stitch_files( diff --git a/src/openflexure_microscope_server/things/z_stack.py b/src/openflexure_microscope_server/things/z_stack.py new file mode 100644 index 00000000..97cd83a7 --- /dev/null +++ b/src/openflexure_microscope_server/things/z_stack.py @@ -0,0 +1,137 @@ +import os +from typing import Mapping, Optional +import cv2 +import numpy as np +from PIL import Image +from pydantic import BaseModel +from scipy.stats import norm +import time +import piexif +import json + +from labthings_fastapi.dependencies.metadata import GetThingStates +from labthings_fastapi.thing import Thing +from labthings_fastapi.decorators import thing_action, thing_property +from .camera import CameraDependency as CamDep +from .stage import StageDependency as StageDep +from labthings_fastapi.dependencies.invocation import ( + CancelHook, + InvocationLogger, + InvocationCancelledError, +) + + +class CaptureError(RuntimeError): + """An error trying to capture from Picamera""" + +class ZStackThing(Thing): + @thing_property + def images_to_capture(self) -> int: + """The number of images to capture and save in a stack + Defaults to 1 unless you need to see either side of focus""" + return self.thing_settings.get("stack_height", 9) + + @images_to_capture.setter + def images_to_capture(self, value: int) -> None: + self.thing_settings["images_to_capture"] = value + + @thing_property + def stack_dz(self) -> int: + """Space in steps between images in a z-stack + Suggested is 50 for 60-100x + 100 for 40x + 200 for 20x""" + return self.thing_settings.get("stack_dz", 50) + + @stack_dz.setter + def stack_dz(self, value: int) -> None: + self.thing_settings["stack_dz"] = value + + @thing_action + def smart_stack( + self, + cam: CamDep, + stage: StageDep, + images_dir: str, + logger: InvocationLogger, + metadata_getter: GetThingStates, + ) -> None: + stack_dz = self.stack_dz + images_to_capture = self.images_to_capture + + stack_z_range = stack_dz * (images_to_capture - 1) + stage.move_relative(z=-stack_z_range / 2) + + for capture_count in range(images_to_capture): + jpeg_path = os.path.join( + images_dir, + f"{capture_count}.jpeg", + ) + self._capture_and_save( + jpeg_path, cam, logger, metadata_getter, + ) + stage.move_relative(z=stack_dz) + + def _capture_and_save( + self, + jpeg_path: str, + cam: CamDep, + logger: InvocationLogger, + metadata_getter: GetThingStates, + ) -> None: + """Capture an image and save it to disk + + This will set the event `acquired` once the image has been acquired, so + that the stage may be moved while it's saved. + """ + capture_start = time.time() + image, metadata = self._capture_image(cam, metadata_getter,) + acquisition_time = time.time() + self._save_capture(jpeg_path, image, metadata) + save_time = time.time() + acquisition_duration = round(acquisition_time - capture_start, 1) + saving_duration = round(save_time - acquisition_time, 1) + logger.info( + f"Acquired {jpeg_path} in {acquisition_duration}s then {saving_duration}s saving to disk" + ) + + def _capture_image(self, cam, metadata_getter) -> tuple[np.ndarray, dict]: + """Capture an image in memory and return it with metadata + This will set the event `acquired` once the image has been acquired, so + that the stage may be moved while it's saved. + CaptureError raised if the capture fails for any reason + returns tuple with numpy array of image data, and dict of metadata + """ + try: + metadata = metadata_getter() + image = cam.capture_array()[..., :3] + except Exception as e: + raise CaptureError("An error occurred while capturing") from e + return image, metadata + + def _save_capture( + self, + jpeg_path: str, + image: np.ndarray, + metadata: dict, + ) -> None: + """Saving the captured image and metadata to disk + logger warning (via InvocationLogger) is raised if metadata is failed to be added + IOError is raised if the file cannot be saved + nothing is returned on success""" + try: + Image.fromarray(image.astype("uint8"), "RGB").save( + jpeg_path, quality=95, subsampling=0 + ) + try: + exif_dict = piexif.load(jpeg_path) + exif_dict["Exif"][piexif.ExifIFD.UserComment] = json.dumps( + metadata + ).encode("utf-8") + piexif.insert(piexif.dump(exif_dict), jpeg_path) + except: # noqa: E722 + # We need to capture any exception as there are many reasons metadata + # might not be added. We warn rather than log the error. + logger.warning(f"Failed to add metadata to {jpeg_path}") + except Exception as e: + raise IOError(f"An error occurred while saving {jpeg_path}") from e From a1784941feadde26a6b2f8feee1b52fae8d407cc Mon Sep 17 00:00:00 2001 From: jaknapper Date: Wed, 16 Apr 2025 11:17:55 +0100 Subject: [PATCH 02/56] Ruff formatting and removing all capturing from smart_scan --- src/openflexure_microscope_server/things/z_stack.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/openflexure_microscope_server/things/z_stack.py b/src/openflexure_microscope_server/things/z_stack.py index 97cd83a7..9b1edfe6 100644 --- a/src/openflexure_microscope_server/things/z_stack.py +++ b/src/openflexure_microscope_server/things/z_stack.py @@ -1,10 +1,6 @@ import os -from typing import Mapping, Optional -import cv2 import numpy as np from PIL import Image -from pydantic import BaseModel -from scipy.stats import norm import time import piexif import json @@ -15,9 +11,7 @@ from labthings_fastapi.decorators import thing_action, thing_property from .camera import CameraDependency as CamDep from .stage import StageDependency as StageDep from labthings_fastapi.dependencies.invocation import ( - CancelHook, InvocationLogger, - InvocationCancelledError, ) @@ -87,7 +81,7 @@ class ZStackThing(Thing): capture_start = time.time() image, metadata = self._capture_image(cam, metadata_getter,) acquisition_time = time.time() - self._save_capture(jpeg_path, image, metadata) + self._save_capture(jpeg_path, image, metadata, logger) save_time = time.time() acquisition_duration = round(acquisition_time - capture_start, 1) saving_duration = round(save_time - acquisition_time, 1) @@ -114,6 +108,7 @@ class ZStackThing(Thing): jpeg_path: str, image: np.ndarray, metadata: dict, + logger: InvocationLogger, ) -> None: """Saving the captured image and metadata to disk logger warning (via InvocationLogger) is raised if metadata is failed to be added From 46458ca8a1c0235cecd75890e0bdc3b6265cde5d Mon Sep 17 00:00:00 2001 From: jaknapper Date: Wed, 16 Apr 2025 18:14:24 +0100 Subject: [PATCH 03/56] Create a stack folder and copy the middle image to image_dir --- .../things/z_stack.py | 42 +++++++++++++++---- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/src/openflexure_microscope_server/things/z_stack.py b/src/openflexure_microscope_server/things/z_stack.py index 9b1edfe6..1e6922e6 100644 --- a/src/openflexure_microscope_server/things/z_stack.py +++ b/src/openflexure_microscope_server/things/z_stack.py @@ -4,6 +4,8 @@ from PIL import Image import time import piexif import json +import shutil +import glob from labthings_fastapi.dependencies.metadata import GetThingStates from labthings_fastapi.thing import Thing @@ -18,12 +20,13 @@ from labthings_fastapi.dependencies.invocation import ( class CaptureError(RuntimeError): """An error trying to capture from Picamera""" + class ZStackThing(Thing): @thing_property def images_to_capture(self) -> int: """The number of images to capture and save in a stack Defaults to 1 unless you need to see either side of focus""" - return self.thing_settings.get("stack_height", 9) + return self.thing_settings.get("images_to_capture", 1) @images_to_capture.setter def images_to_capture(self, value: int) -> None: @@ -46,9 +49,10 @@ class ZStackThing(Thing): self, cam: CamDep, stage: StageDep, - images_dir: str, logger: InvocationLogger, metadata_getter: GetThingStates, + images_dir: str, + stack_dir: str, ) -> None: stack_dz = self.stack_dz images_to_capture = self.images_to_capture @@ -58,13 +62,19 @@ class ZStackThing(Thing): for capture_count in range(images_to_capture): jpeg_path = os.path.join( - images_dir, + stack_dir, f"{capture_count}.jpeg", ) self._capture_and_save( - jpeg_path, cam, logger, metadata_getter, + jpeg_path=jpeg_path, + cam=cam, + logger=logger, + metadata_getter=metadata_getter, ) stage.move_relative(z=stack_dz) + time.sleep(0.3) + + self.copy_central_image(images_dir, stack_dir, logger) def _capture_and_save( self, @@ -79,7 +89,10 @@ class ZStackThing(Thing): that the stage may be moved while it's saved. """ capture_start = time.time() - image, metadata = self._capture_image(cam, metadata_getter,) + image, metadata = self._capture_image( + cam, + metadata_getter, + ) acquisition_time = time.time() self._save_capture(jpeg_path, image, metadata, logger) save_time = time.time() @@ -91,8 +104,6 @@ class ZStackThing(Thing): def _capture_image(self, cam, metadata_getter) -> tuple[np.ndarray, dict]: """Capture an image in memory and return it with metadata - This will set the event `acquired` once the image has been acquired, so - that the stage may be moved while it's saved. CaptureError raised if the capture fails for any reason returns tuple with numpy array of image data, and dict of metadata """ @@ -130,3 +141,20 @@ class ZStackThing(Thing): logger.warning(f"Failed to add metadata to {jpeg_path}") except Exception as e: raise IOError(f"An error occurred while saving {jpeg_path}") from e + + def copy_central_image( + self, + images_dir: str, + stack_dir: str, + logger: InvocationLogger, + ): + """Gets a list of images in a folder (stack_dir), sorts them, and copies the central image + to images dir.""" + image_list = glob.glob(os.path.join(stack_dir, "*")) + image_list.sort() + central_index = (len(image_list) - 1) // 2 + central_image = image_list[central_index] + logger.warning(central_image) + xy_location = os.path.basename(stack_dir) + + shutil.copy(central_image, os.path.join(images_dir, f"{xy_location}.jpeg")) From d8878e40da6784cbd983f02e8c476cdfc1ba26b6 Mon Sep 17 00:00:00 2001 From: jaknapper Date: Wed, 23 Apr 2025 19:16:23 +0100 Subject: [PATCH 04/56] Stacking in autofocus with temp capture Thing --- .../things/autofocus.py | 6 + .../things/smart_scan.py | 5 - .../things/z_stack.py | 160 ------------------ 3 files changed, 6 insertions(+), 165 deletions(-) delete mode 100644 src/openflexure_microscope_server/things/z_stack.py diff --git a/src/openflexure_microscope_server/things/autofocus.py b/src/openflexure_microscope_server/things/autofocus.py index dd4a388a..efcc7aca 100644 --- a/src/openflexure_microscope_server/things/autofocus.py +++ b/src/openflexure_microscope_server/things/autofocus.py @@ -14,6 +14,9 @@ from typing import Annotated, Mapping, Optional, Sequence import os import shutil import glob +import os +import shutil +import glob from fastapi import Depends import numpy as np @@ -31,6 +34,9 @@ from .camera import RawCameraDependency as Camera from .camera import CameraDependency as WrappedCamera from .stage import StageDependency as Stage from .capture import CaptureThing +import numpy as np +from pydantic import BaseModel + CaptureDep = direct_thing_client_dependency(CaptureThing, "/capture/") diff --git a/src/openflexure_microscope_server/things/smart_scan.py b/src/openflexure_microscope_server/things/smart_scan.py index 905f5968..50a8c442 100644 --- a/src/openflexure_microscope_server/things/smart_scan.py +++ b/src/openflexure_microscope_server/things/smart_scan.py @@ -32,7 +32,6 @@ from openflexure_microscope_server.utilities import ErrorCapturingThread from openflexure_microscope_server.things.autofocus import AutofocusThing from openflexure_microscope_server.things.camera_stage_mapping import CameraStageMapper from openflexure_microscope_server.things.background_detect import BackgroundDetectThing -from openflexure_microscope_server.things.z_stack import ZStackThing from openflexure_microscope_server import scan_planners CSMDep = direct_thing_client_dependency(CameraStageMapper, "/camera_stage_mapping/") @@ -140,7 +139,6 @@ class SmartScanThing(Thing): self._background_detect: Optional[BackgroundDep] = None self._ongoing_scan_name: Optional[str] = None self._starting_position: Optional[Mapping[str, int]] = None - self._z_stack: Optional[ZStackDep] = None self._capture_thread: Optional[ErrorCapturingThread] = None self._scan_images_taken: Optional[int] = None # TODO Scan data is a dict during refactoring, should become a dataclass @@ -157,7 +155,6 @@ class SmartScanThing(Thing): metadata_getter: GetThingStates, csm: CSMDep, background_detect: BackgroundDep, - z_stack: ZStackDep, scan_name: str = "", ): """Move the stage to cover an area, taking images that can be tiled together. @@ -181,7 +178,6 @@ class SmartScanThing(Thing): self._csm = csm self._background_detect = background_detect self._capture_thread = None - self._z_stack = z_stack self._scan_images_taken = 0 # Don't set self._scan_data dictionary. This is done at the start of _run_scan @@ -217,7 +213,6 @@ class SmartScanThing(Thing): self._ongoing_scan_name = None self._scan_images_taken = None self._scan_data = None - self._z_stack = None self._scan_lock.release() def promote_stitch_files( diff --git a/src/openflexure_microscope_server/things/z_stack.py b/src/openflexure_microscope_server/things/z_stack.py deleted file mode 100644 index 1e6922e6..00000000 --- a/src/openflexure_microscope_server/things/z_stack.py +++ /dev/null @@ -1,160 +0,0 @@ -import os -import numpy as np -from PIL import Image -import time -import piexif -import json -import shutil -import glob - -from labthings_fastapi.dependencies.metadata import GetThingStates -from labthings_fastapi.thing import Thing -from labthings_fastapi.decorators import thing_action, thing_property -from .camera import CameraDependency as CamDep -from .stage import StageDependency as StageDep -from labthings_fastapi.dependencies.invocation import ( - InvocationLogger, -) - - -class CaptureError(RuntimeError): - """An error trying to capture from Picamera""" - - -class ZStackThing(Thing): - @thing_property - def images_to_capture(self) -> int: - """The number of images to capture and save in a stack - Defaults to 1 unless you need to see either side of focus""" - return self.thing_settings.get("images_to_capture", 1) - - @images_to_capture.setter - def images_to_capture(self, value: int) -> None: - self.thing_settings["images_to_capture"] = value - - @thing_property - def stack_dz(self) -> int: - """Space in steps between images in a z-stack - Suggested is 50 for 60-100x - 100 for 40x - 200 for 20x""" - return self.thing_settings.get("stack_dz", 50) - - @stack_dz.setter - def stack_dz(self, value: int) -> None: - self.thing_settings["stack_dz"] = value - - @thing_action - def smart_stack( - self, - cam: CamDep, - stage: StageDep, - logger: InvocationLogger, - metadata_getter: GetThingStates, - images_dir: str, - stack_dir: str, - ) -> None: - stack_dz = self.stack_dz - images_to_capture = self.images_to_capture - - stack_z_range = stack_dz * (images_to_capture - 1) - stage.move_relative(z=-stack_z_range / 2) - - for capture_count in range(images_to_capture): - jpeg_path = os.path.join( - stack_dir, - f"{capture_count}.jpeg", - ) - self._capture_and_save( - jpeg_path=jpeg_path, - cam=cam, - logger=logger, - metadata_getter=metadata_getter, - ) - stage.move_relative(z=stack_dz) - time.sleep(0.3) - - self.copy_central_image(images_dir, stack_dir, logger) - - def _capture_and_save( - self, - jpeg_path: str, - cam: CamDep, - logger: InvocationLogger, - metadata_getter: GetThingStates, - ) -> None: - """Capture an image and save it to disk - - This will set the event `acquired` once the image has been acquired, so - that the stage may be moved while it's saved. - """ - capture_start = time.time() - image, metadata = self._capture_image( - cam, - metadata_getter, - ) - acquisition_time = time.time() - self._save_capture(jpeg_path, image, metadata, logger) - save_time = time.time() - acquisition_duration = round(acquisition_time - capture_start, 1) - saving_duration = round(save_time - acquisition_time, 1) - logger.info( - f"Acquired {jpeg_path} in {acquisition_duration}s then {saving_duration}s saving to disk" - ) - - def _capture_image(self, cam, metadata_getter) -> tuple[np.ndarray, dict]: - """Capture an image in memory and return it with metadata - CaptureError raised if the capture fails for any reason - returns tuple with numpy array of image data, and dict of metadata - """ - try: - metadata = metadata_getter() - image = cam.capture_array()[..., :3] - except Exception as e: - raise CaptureError("An error occurred while capturing") from e - return image, metadata - - def _save_capture( - self, - jpeg_path: str, - image: np.ndarray, - metadata: dict, - logger: InvocationLogger, - ) -> None: - """Saving the captured image and metadata to disk - logger warning (via InvocationLogger) is raised if metadata is failed to be added - IOError is raised if the file cannot be saved - nothing is returned on success""" - try: - Image.fromarray(image.astype("uint8"), "RGB").save( - jpeg_path, quality=95, subsampling=0 - ) - try: - exif_dict = piexif.load(jpeg_path) - exif_dict["Exif"][piexif.ExifIFD.UserComment] = json.dumps( - metadata - ).encode("utf-8") - piexif.insert(piexif.dump(exif_dict), jpeg_path) - except: # noqa: E722 - # We need to capture any exception as there are many reasons metadata - # might not be added. We warn rather than log the error. - logger.warning(f"Failed to add metadata to {jpeg_path}") - except Exception as e: - raise IOError(f"An error occurred while saving {jpeg_path}") from e - - def copy_central_image( - self, - images_dir: str, - stack_dir: str, - logger: InvocationLogger, - ): - """Gets a list of images in a folder (stack_dir), sorts them, and copies the central image - to images dir.""" - image_list = glob.glob(os.path.join(stack_dir, "*")) - image_list.sort() - central_index = (len(image_list) - 1) // 2 - central_image = image_list[central_index] - logger.warning(central_image) - xy_location = os.path.basename(stack_dir) - - shutil.copy(central_image, os.path.join(images_dir, f"{xy_location}.jpeg")) From ab33de7ccd569b4b4af5787ee9773c3177721f84 Mon Sep 17 00:00:00 2001 From: jaknapper Date: Mon, 28 Apr 2025 14:01:14 +0100 Subject: [PATCH 05/56] Capture a jpeg to disk --- .../things/autofocus.py | 7 +------ src/openflexure_microscope_server/things/capture.py | 13 ++++++++++--- .../things/smart_scan.py | 4 ++++ 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/openflexure_microscope_server/things/autofocus.py b/src/openflexure_microscope_server/things/autofocus.py index efcc7aca..45f43dcd 100644 --- a/src/openflexure_microscope_server/things/autofocus.py +++ b/src/openflexure_microscope_server/things/autofocus.py @@ -310,12 +310,7 @@ class AutofocusThing(Thing): stack_dir, f"{capture_count}.jpeg", ) - capture._capture_and_save( - jpeg_path=jpeg_path, - cam=cam, - logger=logger, - metadata_getter=metadata_getter, - ) + capture.capture_jpeg(filename=jpeg_path, cam=cam) # If the stack isn't complete yet, move if capture_count + 1 < images_to_capture: diff --git a/src/openflexure_microscope_server/things/capture.py b/src/openflexure_microscope_server/things/capture.py index 214c05b9..5c82da95 100644 --- a/src/openflexure_microscope_server/things/capture.py +++ b/src/openflexure_microscope_server/things/capture.py @@ -35,7 +35,7 @@ class CaptureThing(Thing): that the stage may be moved while it's saved. """ capture_start = time.time() - image, metadata = self._capture_image( + image, metadata = self._capture_array( cam, metadata_getter, ) @@ -48,14 +48,21 @@ class CaptureThing(Thing): f"Acquired {jpeg_path} in {acquisition_duration}s then {saving_duration}s saving to disk" ) - def _capture_image(self, cam, metadata_getter) -> tuple[np.ndarray, dict]: + @thing_action + def capture_jpeg(self, filename: str, cam: CamDep): + """Capture a JPEG (from a JPEGBlob) to disk""" + jpeg = cam.capture_jpeg(resolution="full") + jpeg.save(filename) + + @thing_action + def _capture_array(self, cam: CamDep, metadata_getter: GetThingStates): """Capture an image in memory and return it with metadata CaptureError raised if the capture fails for any reason returns tuple with numpy array of image data, and dict of metadata """ try: metadata = metadata_getter() - image = cam.capture_array()[..., :3] + image = cam.capture_array(stream_name="main")[..., :3] except Exception as e: raise CaptureError("An error occurred while capturing") from e return image, metadata diff --git a/src/openflexure_microscope_server/things/smart_scan.py b/src/openflexure_microscope_server/things/smart_scan.py index 50a8c442..f0d45d1d 100644 --- a/src/openflexure_microscope_server/things/smart_scan.py +++ b/src/openflexure_microscope_server/things/smart_scan.py @@ -988,6 +988,8 @@ class SmartScanThing(Thing): "preview_stitch", "--minimum_overlap", f"{min_overlap}", + "--resize", + "0.25", self._ongoing_scan_images_dir, ] ) @@ -1104,6 +1106,8 @@ class SmartScanThing(Thing): "--stitch_dzi", "--minimum_overlap", f"{round(overlap * 0.9, 2)}", + "--resize", + "0.25", images_folder, ], ) From 26163dc97e1b786198a0394f8ad16cf087f62539 Mon Sep 17 00:00:00 2001 From: jaknapper Date: Tue, 29 Apr 2025 18:52:59 +0100 Subject: [PATCH 06/56] Added a new way to capture full res JPEGs, need to do a comparison of results --- .../things/autofocus.py | 24 ++++++++++++-- .../things/camera/__init__.py | 11 +++++++ .../things/capture.py | 32 +++++++++++++++++++ 3 files changed, 65 insertions(+), 2 deletions(-) diff --git a/src/openflexure_microscope_server/things/autofocus.py b/src/openflexure_microscope_server/things/autofocus.py index 45f43dcd..21223f6f 100644 --- a/src/openflexure_microscope_server/things/autofocus.py +++ b/src/openflexure_microscope_server/things/autofocus.py @@ -33,7 +33,7 @@ from labthings_fastapi.dependencies.invocation import InvocationLogger from .camera import RawCameraDependency as Camera from .camera import CameraDependency as WrappedCamera from .stage import StageDependency as Stage -from .capture import CaptureThing +from .capture import CaptureThing, _save_capture import numpy as np from pydantic import BaseModel @@ -310,7 +310,27 @@ class AutofocusThing(Thing): stack_dir, f"{capture_count}.jpeg", ) - capture.capture_jpeg(filename=jpeg_path, cam=cam) + # The pre-existing capture_jpeg + # capture.capture_jpeg(filename=jpeg_path, cam=cam) + + # A new way to get the full array + img = capture.capture_jpeg_array() + + metadata = metadata_getter() + _save_capture( + jpeg_path, + img, + metadata, + logger + ) + + # The old capture and save a "main" res image + # capture._capture_and_save( + # jpeg_path=jpeg_path, + # cam=cam, + # logger=logger, + # metadata_getter=metadata_getter, + # ) # If the stack isn't complete yet, move if capture_count + 1 < images_to_capture: diff --git a/src/openflexure_microscope_server/things/camera/__init__.py b/src/openflexure_microscope_server/things/camera/__init__.py index 8d3fab23..5b4f8c4a 100644 --- a/src/openflexure_microscope_server/things/camera/__init__.py +++ b/src/openflexure_microscope_server/things/camera/__init__.py @@ -77,6 +77,11 @@ class CameraProtocol(Protocol): """Acquire one image from the preview stream and return its size""" ... + def capture_jpeg_array( + self, + ): + ... + class BaseCamera(Thing): """A Thing representing a camera @@ -170,6 +175,12 @@ class CameraStub(BaseCamera): ) -> NDArray: raise NotImplementedError("Cameras must not inherit from CameraStub") + @thing_action + def capture_jpeg_array( + self, + ): + raise NotImplementedError("Cameras must not inherit from CameraStub") + @thing_action def capture_jpeg( self, diff --git a/src/openflexure_microscope_server/things/capture.py b/src/openflexure_microscope_server/things/capture.py index 5c82da95..f99e2791 100644 --- a/src/openflexure_microscope_server/things/capture.py +++ b/src/openflexure_microscope_server/things/capture.py @@ -54,6 +54,11 @@ class CaptureThing(Thing): jpeg = cam.capture_jpeg(resolution="full") jpeg.save(filename) + @thing_action + def capture_jpeg_array(self, cam: CamDep): + """Return a full resolution stream array""" + return cam.capture_jpeg_array() + @thing_action def _capture_array(self, cam: CamDep, metadata_getter: GetThingStates): """Capture an image in memory and return it with metadata @@ -94,3 +99,30 @@ class CaptureThing(Thing): logger.warning(f"Failed to add metadata to {jpeg_path}") except Exception as e: raise IOError(f"An error occurred while saving {jpeg_path}") from e + +def _save_capture( + jpeg_path: str, + image: np.ndarray, + metadata: dict, + logger: InvocationLogger, +) -> None: + """Saving the captured image and metadata to disk + logger warning (via InvocationLogger) is raised if metadata is failed to be added + IOError is raised if the file cannot be saved + nothing is returned on success""" + try: + Image.fromarray(image.astype("uint8"), "RGB").save( + jpeg_path, quality=95, subsampling=0 + ) + try: + exif_dict = piexif.load(jpeg_path) + exif_dict["Exif"][piexif.ExifIFD.UserComment] = json.dumps( + metadata + ).encode("utf-8") + piexif.insert(piexif.dump(exif_dict), jpeg_path) + except: # noqa: E722 + # We need to capture any exception as there are many reasons metadata + # might not be added. We warn rather than log the error. + logger.warning(f"Failed to add metadata to {jpeg_path}") + except Exception as e: + raise IOError(f"An error occurred while saving {jpeg_path}") from e From 132efa8033bf909d9302cca7d775a352df49b5f0 Mon Sep 17 00:00:00 2001 From: jaknapper Date: Thu, 1 May 2025 14:47:12 +0100 Subject: [PATCH 07/56] Options for capture method, default downsampling of correlations --- .../things/autofocus.py | 46 ++++++++++--------- .../things/smart_scan.py | 34 ++++++++++++-- 2 files changed, 54 insertions(+), 26 deletions(-) diff --git a/src/openflexure_microscope_server/things/autofocus.py b/src/openflexure_microscope_server/things/autofocus.py index 21223f6f..33b42f98 100644 --- a/src/openflexure_microscope_server/things/autofocus.py +++ b/src/openflexure_microscope_server/things/autofocus.py @@ -294,6 +294,7 @@ class AutofocusThing(Thing): capture: CaptureDep, images_dir: str, stack_dir: str, + capture_method: str = 'array' ) -> None: """Run a z stack, saving all images to stack_dir and copying the central image to stack_dir""" @@ -307,30 +308,31 @@ class AutofocusThing(Thing): time.sleep(SETTLING_TIME) jpeg_path = os.path.join( - stack_dir, - f"{capture_count}.jpeg", - ) - # The pre-existing capture_jpeg - # capture.capture_jpeg(filename=jpeg_path, cam=cam) - - # A new way to get the full array - img = capture.capture_jpeg_array() - + stack_dir, + f"{capture_count}.jpeg", + ) metadata = metadata_getter() - _save_capture( - jpeg_path, - img, - metadata, - logger - ) - + if capture_method == "blob": + capture.capture_jpeg(filename=jpeg_path, cam=cam) + elif capture_method == "hires_array": + # A new way to get the full array + img = capture.capture_jpeg_array() + _save_capture( + jpeg_path, + img, + metadata, + logger + ) + elif capture_method == "array": # The old capture and save a "main" res image - # capture._capture_and_save( - # jpeg_path=jpeg_path, - # cam=cam, - # logger=logger, - # metadata_getter=metadata_getter, - # ) + capture._capture_and_save( + jpeg_path=jpeg_path, + cam=cam, + logger=logger, + metadata_getter=metadata_getter, + ) + else: + raise ValueError('Capture method must be one of "array", "blob" or "hires_array"') # If the stack isn't complete yet, move if capture_count + 1 < images_to_capture: diff --git a/src/openflexure_microscope_server/things/smart_scan.py b/src/openflexure_microscope_server/things/smart_scan.py index f0d45d1d..8094f632 100644 --- a/src/openflexure_microscope_server/things/smart_scan.py +++ b/src/openflexure_microscope_server/things/smart_scan.py @@ -143,6 +143,7 @@ class SmartScanThing(Thing): self._scan_images_taken: Optional[int] = None # TODO Scan data is a dict during refactoring, should become a dataclass self._scan_data: Optional[dict] = None + self._stitch_resize: Optional[float] = None @thing_action def sample_scan( @@ -179,6 +180,7 @@ class SmartScanThing(Thing): self._background_detect = background_detect self._capture_thread = None self._scan_images_taken = 0 + self._stitch_resize = 1 # Don't set self._scan_data dictionary. This is done at the start of _run_scan @@ -214,6 +216,7 @@ class SmartScanThing(Thing): self._scan_images_taken = None self._scan_data = None self._scan_lock.release() + self._stitch_resize = None def promote_stitch_files( self, @@ -375,8 +378,24 @@ class SmartScanThing(Thing): return (next_point[0], next_point[1], z_estimate) + # @_scan_running + # def _calc_resize_from_test_image(self): + # """ + # Take a test image to set the amount to downsample images for stitching + + # Return the decimal value to scale x and y by when stitching + # """ + + # #TODO: This needs to match how the capture in the z stack is done + # test_jpg = self._cam.capture_jpeg(resolution="full") + # test_jpg = test_jpg.content + + # test_image = Image.open(io.BytesIO(test_jpg)) + # test_image_res = list(test_image.size) + # return STITCH_IMAGE_WIDTH / test_image_res[0] + @_scan_running - def _take_test_image_to_calc_displacement(self, overlap): + def _calc_displacement_from_test_image(self, overlap): """ Take a test image and use camera stage mapping to calculate x and y displacement @@ -421,7 +440,14 @@ class SmartScanThing(Thing): dataclass. """ overlap = self.overlap - dx, dy = self._take_test_image_to_calc_displacement(overlap) + dx, dy = self._calc_displacement_from_test_image(overlap) + # self._stitch_resize = self._calc_resize_from_test_image() + self._stitch_resize = 0.25 + + self._scan_logger.info( + f'Resizing images by {self._stitch_resize}' + ) + self._scan_logger.info( f"Based on an overlap of {overlap}, we will make steps of {dx}, {dy}" ) @@ -989,7 +1015,7 @@ class SmartScanThing(Thing): "--minimum_overlap", f"{min_overlap}", "--resize", - "0.25", + f"{self._stitch_resize}", self._ongoing_scan_images_dir, ] ) @@ -1107,7 +1133,7 @@ class SmartScanThing(Thing): "--minimum_overlap", f"{round(overlap * 0.9, 2)}", "--resize", - "0.25", + f"{self._stitch_resize}", images_folder, ], ) From 9d1385231ddb0b2fbaaa6a1350663386004793c3 Mon Sep 17 00:00:00 2001 From: jaknapper Date: Thu, 1 May 2025 15:00:16 +0100 Subject: [PATCH 08/56] Rename jpeg_array to highres_array --- src/openflexure_microscope_server/things/autofocus.py | 2 +- src/openflexure_microscope_server/things/camera/__init__.py | 4 ++-- src/openflexure_microscope_server/things/capture.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/openflexure_microscope_server/things/autofocus.py b/src/openflexure_microscope_server/things/autofocus.py index 33b42f98..528195a4 100644 --- a/src/openflexure_microscope_server/things/autofocus.py +++ b/src/openflexure_microscope_server/things/autofocus.py @@ -316,7 +316,7 @@ class AutofocusThing(Thing): capture.capture_jpeg(filename=jpeg_path, cam=cam) elif capture_method == "hires_array": # A new way to get the full array - img = capture.capture_jpeg_array() + img = capture.capture_highres_array() _save_capture( jpeg_path, img, diff --git a/src/openflexure_microscope_server/things/camera/__init__.py b/src/openflexure_microscope_server/things/camera/__init__.py index 5b4f8c4a..6f84fe66 100644 --- a/src/openflexure_microscope_server/things/camera/__init__.py +++ b/src/openflexure_microscope_server/things/camera/__init__.py @@ -77,7 +77,7 @@ class CameraProtocol(Protocol): """Acquire one image from the preview stream and return its size""" ... - def capture_jpeg_array( + def capture_highres_array( self, ): ... @@ -176,7 +176,7 @@ class CameraStub(BaseCamera): raise NotImplementedError("Cameras must not inherit from CameraStub") @thing_action - def capture_jpeg_array( + def capture_highres_array( self, ): raise NotImplementedError("Cameras must not inherit from CameraStub") diff --git a/src/openflexure_microscope_server/things/capture.py b/src/openflexure_microscope_server/things/capture.py index f99e2791..c4ca7cae 100644 --- a/src/openflexure_microscope_server/things/capture.py +++ b/src/openflexure_microscope_server/things/capture.py @@ -55,9 +55,9 @@ class CaptureThing(Thing): jpeg.save(filename) @thing_action - def capture_jpeg_array(self, cam: CamDep): + def capture_highres_array(self, cam: CamDep): """Return a full resolution stream array""" - return cam.capture_jpeg_array() + return cam.capture_highres_array() @thing_action def _capture_array(self, cam: CamDep, metadata_getter: GetThingStates): From 8bc0573ab1342e4d2cfec3a9063277812332e63c Mon Sep 17 00:00:00 2001 From: jaknapper Date: Thu, 1 May 2025 16:07:05 +0100 Subject: [PATCH 09/56] Capture_array can now use "full" stream --- .../things/autofocus.py | 15 ++------ .../things/camera/__init__.py | 16 +------- .../things/capture.py | 38 ++----------------- 3 files changed, 9 insertions(+), 60 deletions(-) diff --git a/src/openflexure_microscope_server/things/autofocus.py b/src/openflexure_microscope_server/things/autofocus.py index 528195a4..7131cd0d 100644 --- a/src/openflexure_microscope_server/things/autofocus.py +++ b/src/openflexure_microscope_server/things/autofocus.py @@ -311,25 +311,16 @@ class AutofocusThing(Thing): stack_dir, f"{capture_count}.jpeg", ) - metadata = metadata_getter() if capture_method == "blob": capture.capture_jpeg(filename=jpeg_path, cam=cam) - elif capture_method == "hires_array": - # A new way to get the full array - img = capture.capture_highres_array() - _save_capture( - jpeg_path, - img, - metadata, - logger - ) - elif capture_method == "array": - # The old capture and save a "main" res image + elif capture_method == "hires_array" or capture_method == "array": + stream = 'main' if capture_method == "array" else "full" capture._capture_and_save( jpeg_path=jpeg_path, cam=cam, logger=logger, metadata_getter=metadata_getter, + stream_name=stream, ) else: raise ValueError('Capture method must be one of "array", "blob" or "hires_array"') diff --git a/src/openflexure_microscope_server/things/camera/__init__.py b/src/openflexure_microscope_server/things/camera/__init__.py index 6f84fe66..076d2efd 100644 --- a/src/openflexure_microscope_server/things/camera/__init__.py +++ b/src/openflexure_microscope_server/things/camera/__init__.py @@ -44,7 +44,7 @@ class CameraProtocol(Protocol): def capture_array( self, - stream_name: Literal["main", "lores", "raw"] = "main", + stream_name: Literal["main", "lores", "raw", "full"] = "main", ) -> NDArray: ... def capture_jpeg( @@ -77,12 +77,6 @@ class CameraProtocol(Protocol): """Acquire one image from the preview stream and return its size""" ... - def capture_highres_array( - self, - ): - ... - - class BaseCamera(Thing): """A Thing representing a camera @@ -171,16 +165,10 @@ class CameraStub(BaseCamera): @thing_action def capture_array( self, - stream_name: Literal["main", "lores", "raw"] = "main", + stream_name: Literal["main", "lores", "raw", "full"] = "main", ) -> NDArray: raise NotImplementedError("Cameras must not inherit from CameraStub") - @thing_action - def capture_highres_array( - self, - ): - raise NotImplementedError("Cameras must not inherit from CameraStub") - @thing_action def capture_jpeg( self, diff --git a/src/openflexure_microscope_server/things/capture.py b/src/openflexure_microscope_server/things/capture.py index c4ca7cae..dc5a1572 100644 --- a/src/openflexure_microscope_server/things/capture.py +++ b/src/openflexure_microscope_server/things/capture.py @@ -28,6 +28,7 @@ class CaptureThing(Thing): cam: CamDep, logger: InvocationLogger, metadata_getter: GetThingStates, + stream: str = "main", ) -> None: """Capture an image and save it to disk @@ -38,6 +39,7 @@ class CaptureThing(Thing): image, metadata = self._capture_array( cam, metadata_getter, + stream=stream, ) acquisition_time = time.time() self._save_capture(jpeg_path, image, metadata, logger) @@ -55,19 +57,14 @@ class CaptureThing(Thing): jpeg.save(filename) @thing_action - def capture_highres_array(self, cam: CamDep): - """Return a full resolution stream array""" - return cam.capture_highres_array() - - @thing_action - def _capture_array(self, cam: CamDep, metadata_getter: GetThingStates): + def _capture_array(self, cam: CamDep, metadata_getter: GetThingStates, stream: str = 'main'): """Capture an image in memory and return it with metadata CaptureError raised if the capture fails for any reason returns tuple with numpy array of image data, and dict of metadata """ try: metadata = metadata_getter() - image = cam.capture_array(stream_name="main")[..., :3] + image = cam.capture_array(stream_name=stream)[..., :3] except Exception as e: raise CaptureError("An error occurred while capturing") from e return image, metadata @@ -99,30 +96,3 @@ class CaptureThing(Thing): logger.warning(f"Failed to add metadata to {jpeg_path}") except Exception as e: raise IOError(f"An error occurred while saving {jpeg_path}") from e - -def _save_capture( - jpeg_path: str, - image: np.ndarray, - metadata: dict, - logger: InvocationLogger, -) -> None: - """Saving the captured image and metadata to disk - logger warning (via InvocationLogger) is raised if metadata is failed to be added - IOError is raised if the file cannot be saved - nothing is returned on success""" - try: - Image.fromarray(image.astype("uint8"), "RGB").save( - jpeg_path, quality=95, subsampling=0 - ) - try: - exif_dict = piexif.load(jpeg_path) - exif_dict["Exif"][piexif.ExifIFD.UserComment] = json.dumps( - metadata - ).encode("utf-8") - piexif.insert(piexif.dump(exif_dict), jpeg_path) - except: # noqa: E722 - # We need to capture any exception as there are many reasons metadata - # might not be added. We warn rather than log the error. - logger.warning(f"Failed to add metadata to {jpeg_path}") - except Exception as e: - raise IOError(f"An error occurred while saving {jpeg_path}") from e From de7ffb9d1ecdcc36249d52aac2d10bb7d8b00869 Mon Sep 17 00:00:00 2001 From: jaknapper Date: Fri, 2 May 2025 13:40:41 +0100 Subject: [PATCH 10/56] Stream_name and capture_method args updated --- src/openflexure_microscope_server/things/autofocus.py | 2 +- src/openflexure_microscope_server/things/capture.py | 4 ++-- src/openflexure_microscope_server/things/smart_scan.py | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/openflexure_microscope_server/things/autofocus.py b/src/openflexure_microscope_server/things/autofocus.py index 7131cd0d..79a517bd 100644 --- a/src/openflexure_microscope_server/things/autofocus.py +++ b/src/openflexure_microscope_server/things/autofocus.py @@ -33,7 +33,7 @@ from labthings_fastapi.dependencies.invocation import InvocationLogger from .camera import RawCameraDependency as Camera from .camera import CameraDependency as WrappedCamera from .stage import StageDependency as Stage -from .capture import CaptureThing, _save_capture +from .capture import CaptureThing import numpy as np from pydantic import BaseModel diff --git a/src/openflexure_microscope_server/things/capture.py b/src/openflexure_microscope_server/things/capture.py index dc5a1572..904a5bf9 100644 --- a/src/openflexure_microscope_server/things/capture.py +++ b/src/openflexure_microscope_server/things/capture.py @@ -28,7 +28,7 @@ class CaptureThing(Thing): cam: CamDep, logger: InvocationLogger, metadata_getter: GetThingStates, - stream: str = "main", + stream_name: str = "main", ) -> None: """Capture an image and save it to disk @@ -39,7 +39,7 @@ class CaptureThing(Thing): image, metadata = self._capture_array( cam, metadata_getter, - stream=stream, + stream=stream_name, ) acquisition_time = time.time() self._save_capture(jpeg_path, image, metadata, logger) diff --git a/src/openflexure_microscope_server/things/smart_scan.py b/src/openflexure_microscope_server/things/smart_scan.py index 8094f632..ea024bca 100644 --- a/src/openflexure_microscope_server/things/smart_scan.py +++ b/src/openflexure_microscope_server/things/smart_scan.py @@ -694,6 +694,7 @@ class SmartScanThing(Thing): self._autofocus.run_z_stack( images_dir=self._ongoing_scan_images_dir, stack_dir=site_folder, + capture_method="hires_array", ) # increment capure counter as thread has completed From a9fdd0ea3e294cce41d23803dbce064aaa499ed2 Mon Sep 17 00:00:00 2001 From: jaknapper Date: Fri, 2 May 2025 17:25:43 +0100 Subject: [PATCH 11/56] Wait params in captures, multiple attempts to capture --- .../things/camera/__init__.py | 6 +++- .../things/capture.py | 31 +++++++++++++------ .../things/smart_scan.py | 2 +- 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/openflexure_microscope_server/things/camera/__init__.py b/src/openflexure_microscope_server/things/camera/__init__.py index 076d2efd..8e095e75 100644 --- a/src/openflexure_microscope_server/things/camera/__init__.py +++ b/src/openflexure_microscope_server/things/camera/__init__.py @@ -8,7 +8,7 @@ See repository root for licensing information. from __future__ import annotations import logging -from typing import Literal, Protocol, runtime_checkable +from typing import Literal, Protocol, runtime_checkable, Optional from labthings_fastapi.thing import Thing from labthings_fastapi.decorators import thing_action, thing_property @@ -45,12 +45,14 @@ class CameraProtocol(Protocol): def capture_array( self, stream_name: Literal["main", "lores", "raw", "full"] = "main", + wait: Optional[float] = 5, ) -> NDArray: ... def capture_jpeg( self, metadata_getter: GetThingStates, resolution: Literal["lores", "main", "full"] = "main", + wait: Optional[float] = 5, ) -> JPEGBlob: """Acquire one image from the camera and return as a JPEG blob""" ... @@ -166,6 +168,7 @@ class CameraStub(BaseCamera): def capture_array( self, stream_name: Literal["main", "lores", "raw", "full"] = "main", + wait: Optional[float] = 5, ) -> NDArray: raise NotImplementedError("Cameras must not inherit from CameraStub") @@ -174,6 +177,7 @@ class CameraStub(BaseCamera): self, metadata_getter: GetThingStates, resolution: Literal["lores", "main", "full"] = "main", + wait: Optional[float] = 5, ) -> JPEGBlob: """Acquire one image from the camera and return as a JPEG blob""" raise NotImplementedError("Cameras must not inherit from CameraStub") diff --git a/src/openflexure_microscope_server/things/capture.py b/src/openflexure_microscope_server/things/capture.py index 904a5bf9..8d3dc413 100644 --- a/src/openflexure_microscope_server/things/capture.py +++ b/src/openflexure_microscope_server/things/capture.py @@ -40,6 +40,7 @@ class CaptureThing(Thing): cam, metadata_getter, stream=stream_name, + logger=logger, ) acquisition_time = time.time() self._save_capture(jpeg_path, image, metadata, logger) @@ -51,23 +52,33 @@ class CaptureThing(Thing): ) @thing_action - def capture_jpeg(self, filename: str, cam: CamDep): + def capture_jpeg(self, filename: str, cam: CamDep, logger: InvocationLogger): """Capture a JPEG (from a JPEGBlob) to disk""" - jpeg = cam.capture_jpeg(resolution="full") - jpeg.save(filename) + for capture_attempts in range(5): + try: + jpeg = cam.capture_jpeg(resolution="full", wait=5) + jpeg.save(filename) + return + except TimeoutError: + logger.warning(f'Attempt {capture_attempts+1} to capture image timed out. Do you have enough RAM?') + except Exception as e: + logger.warning(e) + raise CaptureError("An error occurred while capturing after 5 attempts") @thing_action - def _capture_array(self, cam: CamDep, metadata_getter: GetThingStates, stream: str = 'main'): + def _capture_array(self, cam: CamDep, metadata_getter: GetThingStates, logger: InvocationLogger, stream: str = 'main'): """Capture an image in memory and return it with metadata CaptureError raised if the capture fails for any reason returns tuple with numpy array of image data, and dict of metadata """ - try: - metadata = metadata_getter() - image = cam.capture_array(stream_name=stream)[..., :3] - except Exception as e: - raise CaptureError("An error occurred while capturing") from e - return image, metadata + for capture_attempts in range(5): + try: + metadata = metadata_getter() + image = cam.capture_array(stream_name=stream, wait=5)[..., :3] + return image, metadata + except TimeoutError: + logger.warning(f'Attempt {capture_attempts+1} to capture image timed out. Do you have enough RAM?') + raise CaptureError("An error occurred while capturing after 5 attempts") def _save_capture( self, diff --git a/src/openflexure_microscope_server/things/smart_scan.py b/src/openflexure_microscope_server/things/smart_scan.py index ea024bca..46799104 100644 --- a/src/openflexure_microscope_server/things/smart_scan.py +++ b/src/openflexure_microscope_server/things/smart_scan.py @@ -694,7 +694,7 @@ class SmartScanThing(Thing): self._autofocus.run_z_stack( images_dir=self._ongoing_scan_images_dir, stack_dir=site_folder, - capture_method="hires_array", + capture_method="blob", ) # increment capure counter as thread has completed From 17f3bbe646f4bd26919a0dbd270bb0b5c94e78e6 Mon Sep 17 00:00:00 2001 From: jaknapper Date: Wed, 7 May 2025 10:39:05 +0100 Subject: [PATCH 12/56] Low res stream during scanning for instant high res captures --- .../things/camera/__init__.py | 21 +++++++++++++++++++ .../things/capture.py | 8 +++++++ .../things/smart_scan.py | 19 +++++++++-------- 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/src/openflexure_microscope_server/things/camera/__init__.py b/src/openflexure_microscope_server/things/camera/__init__.py index 8e095e75..cf765e99 100644 --- a/src/openflexure_microscope_server/things/camera/__init__.py +++ b/src/openflexure_microscope_server/things/camera/__init__.py @@ -79,6 +79,16 @@ class CameraProtocol(Protocol): """Acquire one image from the preview stream and return its size""" ... + def start_highres_streaming(self) -> None: + """Start (or stop and restart) the camera in a configuration for low + res streaming and instant high res captures""" + ... + + def start_streaming(self) -> None: + """Start (or stop and restart) the camera in the default streaming + configuration which balances stream and capture quality""" + ... + class BaseCamera(Thing): """A Thing representing a camera @@ -182,6 +192,17 @@ class CameraStub(BaseCamera): """Acquire one image from the camera and return as a JPEG blob""" raise NotImplementedError("Cameras must not inherit from CameraStub") + @thing_action + def start_highres_streaming(self): + """Start (or stop and restart) the camera in a configuration for low + res streaming and instant high res captures""" + raise NotImplementedError("Cameras must not inherit from CameraStub") + + @thing_action + def start_streaming(self) -> None: + """Start (or stop and restart) the camera in the default streaming + configuration which balances stream and capture quality""" + raise NotImplementedError("Cameras must not inherit from CameraStub") CameraDependency = direct_thing_client_dependency(CameraStub, "/camera/") RawCameraDependency = raw_thing_dependency(CameraProtocol) diff --git a/src/openflexure_microscope_server/things/capture.py b/src/openflexure_microscope_server/things/capture.py index 8d3dc413..7b8d260e 100644 --- a/src/openflexure_microscope_server/things/capture.py +++ b/src/openflexure_microscope_server/things/capture.py @@ -65,6 +65,14 @@ class CaptureThing(Thing): logger.warning(e) raise CaptureError("An error occurred while capturing after 5 attempts") + @thing_action + def stop_streaming(self, cam: CamDep): + cam.stop_streaming() + + @thing_action + def restart_stream(self, cam:CamDep): + cam.restart_stream() + @thing_action def _capture_array(self, cam: CamDep, metadata_getter: GetThingStates, logger: InvocationLogger, stream: str = 'main'): """Capture an image in memory and return it with metadata diff --git a/src/openflexure_microscope_server/things/smart_scan.py b/src/openflexure_microscope_server/things/smart_scan.py index 46799104..61cc00c2 100644 --- a/src/openflexure_microscope_server/things/smart_scan.py +++ b/src/openflexure_microscope_server/things/smart_scan.py @@ -182,6 +182,8 @@ class SmartScanThing(Thing): self._scan_images_taken = 0 self._stitch_resize = 1 + self._cam.start_highres_streaming() + # Don't set self._scan_data dictionary. This is done at the start of _run_scan try: @@ -202,6 +204,7 @@ class SmartScanThing(Thing): # Error must be raised so UI gives correct output raise e finally: + self._cam.start_streaming() # However the scan finishes, unset all variables and release lock self._cancel = None self._scan_logger = None @@ -404,19 +407,17 @@ class SmartScanThing(Thing): test_jpg = self._cam.grab_jpeg() test_image = np.array(Image.open(test_jpg.open())) - test_image_res = list(test_image.shape[:2]) + test_image_res = list(test_image.shape) csm_image_res = [int(i) for i in self._csm.image_resolution] - if test_image_res != csm_image_res: - raise RuntimeError( - "Cannot start scan as it is set up to capture with a resolution that " - "has not been mapped.\n" - f"Scan resolution: {test_image_res}\n" - f"camera-stage-mapping resolution {csm_image_res}." - ) + + # If current stream width is different to csm calibration width, + # perform the conversion here + res_ratio = csm_image_res[0] / test_image_res[0] # get displacement matrix. note it is for (y, x) not (x, y) coordinates - csm_disp_matrix = self._csm.image_to_stage_displacement_matrix + csm_disp_matrix = np.array(self._csm.image_to_stage_displacement_matrix) + csm_disp_matrix *= res_ratio # Calculate displacements in image coordinates dx_img = test_image.shape[1] * (1 - overlap) From f5c1ff262192096b0cfb1ab80f035579b68ebcf9 Mon Sep 17 00:00:00 2001 From: jaknapper Date: Wed, 7 May 2025 10:44:54 +0100 Subject: [PATCH 13/56] Ruff formating --- .../things/autofocus.py | 16 +++++++++------- .../things/camera/__init__.py | 6 ++++-- .../things/capture.py | 18 ++++++++++++++---- .../things/smart_scan.py | 5 +---- 4 files changed, 28 insertions(+), 17 deletions(-) diff --git a/src/openflexure_microscope_server/things/autofocus.py b/src/openflexure_microscope_server/things/autofocus.py index 79a517bd..b4825e08 100644 --- a/src/openflexure_microscope_server/things/autofocus.py +++ b/src/openflexure_microscope_server/things/autofocus.py @@ -294,7 +294,7 @@ class AutofocusThing(Thing): capture: CaptureDep, images_dir: str, stack_dir: str, - capture_method: str = 'array' + capture_method: str = "array", ) -> None: """Run a z stack, saving all images to stack_dir and copying the central image to stack_dir""" @@ -308,13 +308,13 @@ class AutofocusThing(Thing): time.sleep(SETTLING_TIME) jpeg_path = os.path.join( - stack_dir, - f"{capture_count}.jpeg", - ) - if capture_method == "blob": + stack_dir, + f"{capture_count}.jpeg", + ) + if capture_method == "blob": capture.capture_jpeg(filename=jpeg_path, cam=cam) elif capture_method == "hires_array" or capture_method == "array": - stream = 'main' if capture_method == "array" else "full" + stream = "main" if capture_method == "array" else "full" capture._capture_and_save( jpeg_path=jpeg_path, cam=cam, @@ -323,7 +323,9 @@ class AutofocusThing(Thing): stream_name=stream, ) else: - raise ValueError('Capture method must be one of "array", "blob" or "hires_array"') + raise ValueError( + 'Capture method must be one of "array", "blob" or "hires_array"' + ) # If the stack isn't complete yet, move if capture_count + 1 < images_to_capture: diff --git a/src/openflexure_microscope_server/things/camera/__init__.py b/src/openflexure_microscope_server/things/camera/__init__.py index cf765e99..ac471031 100644 --- a/src/openflexure_microscope_server/things/camera/__init__.py +++ b/src/openflexure_microscope_server/things/camera/__init__.py @@ -83,12 +83,13 @@ class CameraProtocol(Protocol): """Start (or stop and restart) the camera in a configuration for low res streaming and instant high res captures""" ... - + def start_streaming(self) -> None: """Start (or stop and restart) the camera in the default streaming configuration which balances stream and capture quality""" ... + class BaseCamera(Thing): """A Thing representing a camera @@ -197,12 +198,13 @@ class CameraStub(BaseCamera): """Start (or stop and restart) the camera in a configuration for low res streaming and instant high res captures""" raise NotImplementedError("Cameras must not inherit from CameraStub") - + @thing_action def start_streaming(self) -> None: """Start (or stop and restart) the camera in the default streaming configuration which balances stream and capture quality""" raise NotImplementedError("Cameras must not inherit from CameraStub") + CameraDependency = direct_thing_client_dependency(CameraStub, "/camera/") RawCameraDependency = raw_thing_dependency(CameraProtocol) diff --git a/src/openflexure_microscope_server/things/capture.py b/src/openflexure_microscope_server/things/capture.py index 7b8d260e..ebcd7312 100644 --- a/src/openflexure_microscope_server/things/capture.py +++ b/src/openflexure_microscope_server/things/capture.py @@ -60,7 +60,9 @@ class CaptureThing(Thing): jpeg.save(filename) return except TimeoutError: - logger.warning(f'Attempt {capture_attempts+1} to capture image timed out. Do you have enough RAM?') + logger.warning( + f"Attempt {capture_attempts + 1} to capture image timed out. Do you have enough RAM?" + ) except Exception as e: logger.warning(e) raise CaptureError("An error occurred while capturing after 5 attempts") @@ -70,11 +72,17 @@ class CaptureThing(Thing): cam.stop_streaming() @thing_action - def restart_stream(self, cam:CamDep): + def restart_stream(self, cam: CamDep): cam.restart_stream() @thing_action - def _capture_array(self, cam: CamDep, metadata_getter: GetThingStates, logger: InvocationLogger, stream: str = 'main'): + def _capture_array( + self, + cam: CamDep, + metadata_getter: GetThingStates, + logger: InvocationLogger, + stream: str = "main", + ): """Capture an image in memory and return it with metadata CaptureError raised if the capture fails for any reason returns tuple with numpy array of image data, and dict of metadata @@ -85,7 +93,9 @@ class CaptureThing(Thing): image = cam.capture_array(stream_name=stream, wait=5)[..., :3] return image, metadata except TimeoutError: - logger.warning(f'Attempt {capture_attempts+1} to capture image timed out. Do you have enough RAM?') + logger.warning( + f"Attempt {capture_attempts + 1} to capture image timed out. Do you have enough RAM?" + ) raise CaptureError("An error occurred while capturing after 5 attempts") def _save_capture( diff --git a/src/openflexure_microscope_server/things/smart_scan.py b/src/openflexure_microscope_server/things/smart_scan.py index 61cc00c2..f82fbf43 100644 --- a/src/openflexure_microscope_server/things/smart_scan.py +++ b/src/openflexure_microscope_server/things/smart_scan.py @@ -410,7 +410,6 @@ class SmartScanThing(Thing): test_image_res = list(test_image.shape) csm_image_res = [int(i) for i in self._csm.image_resolution] - # If current stream width is different to csm calibration width, # perform the conversion here res_ratio = csm_image_res[0] / test_image_res[0] @@ -445,9 +444,7 @@ class SmartScanThing(Thing): # self._stitch_resize = self._calc_resize_from_test_image() self._stitch_resize = 0.25 - self._scan_logger.info( - f'Resizing images by {self._stitch_resize}' - ) + self._scan_logger.info(f"Resizing images by {self._stitch_resize}") self._scan_logger.info( f"Based on an overlap of {overlap}, we will make steps of {dx}, {dy}" From 9f2a131f1ed0d534451165b5d2e7f3c25caa67e9 Mon Sep 17 00:00:00 2001 From: jaknapper Date: Wed, 7 May 2025 19:34:13 +0100 Subject: [PATCH 14/56] Swap stream for scan, target_resolution as arg --- .../things/autofocus.py | 24 +++++++------------ .../things/camera/__init__.py | 23 +++++------------- .../things/capture.py | 18 ++++++++------ .../things/smart_scan.py | 4 ++-- 4 files changed, 27 insertions(+), 42 deletions(-) diff --git a/src/openflexure_microscope_server/things/autofocus.py b/src/openflexure_microscope_server/things/autofocus.py index b4825e08..f332278f 100644 --- a/src/openflexure_microscope_server/things/autofocus.py +++ b/src/openflexure_microscope_server/things/autofocus.py @@ -294,7 +294,7 @@ class AutofocusThing(Thing): capture: CaptureDep, images_dir: str, stack_dir: str, - capture_method: str = "array", + target_resolution: tuple[int, int], ) -> None: """Run a z stack, saving all images to stack_dir and copying the central image to stack_dir""" @@ -311,21 +311,13 @@ class AutofocusThing(Thing): stack_dir, f"{capture_count}.jpeg", ) - if capture_method == "blob": - capture.capture_jpeg(filename=jpeg_path, cam=cam) - elif capture_method == "hires_array" or capture_method == "array": - stream = "main" if capture_method == "array" else "full" - capture._capture_and_save( - jpeg_path=jpeg_path, - cam=cam, - logger=logger, - metadata_getter=metadata_getter, - stream_name=stream, - ) - else: - raise ValueError( - 'Capture method must be one of "array", "blob" or "hires_array"' - ) + capture._capture_and_save( + jpeg_path=jpeg_path, + cam=cam, + logger=logger, + metadata_getter=metadata_getter, + target_resolution=target_resolution, + ) # If the stack isn't complete yet, move if capture_count + 1 < images_to_capture: diff --git a/src/openflexure_microscope_server/things/camera/__init__.py b/src/openflexure_microscope_server/things/camera/__init__.py index ac471031..6c18765a 100644 --- a/src/openflexure_microscope_server/things/camera/__init__.py +++ b/src/openflexure_microscope_server/things/camera/__init__.py @@ -79,14 +79,9 @@ class CameraProtocol(Protocol): """Acquire one image from the preview stream and return its size""" ... - def start_highres_streaming(self) -> None: - """Start (or stop and restart) the camera in a configuration for low - res streaming and instant high res captures""" - ... - - def start_streaming(self) -> None: - """Start (or stop and restart) the camera in the default streaming - configuration which balances stream and capture quality""" + def start_streaming(self, main_resolution) -> None: + """Start (or stop and restart) the camera with the given resolution + for the main stream""" ... @@ -194,15 +189,9 @@ class CameraStub(BaseCamera): raise NotImplementedError("Cameras must not inherit from CameraStub") @thing_action - def start_highres_streaming(self): - """Start (or stop and restart) the camera in a configuration for low - res streaming and instant high res captures""" - raise NotImplementedError("Cameras must not inherit from CameraStub") - - @thing_action - def start_streaming(self) -> None: - """Start (or stop and restart) the camera in the default streaming - configuration which balances stream and capture quality""" + def start_streaming(self, main_resolution) -> None: + """Start (or stop and restart) the camera with the given resolution + for the main stream""" raise NotImplementedError("Cameras must not inherit from CameraStub") diff --git a/src/openflexure_microscope_server/things/capture.py b/src/openflexure_microscope_server/things/capture.py index ebcd7312..e353ecba 100644 --- a/src/openflexure_microscope_server/things/capture.py +++ b/src/openflexure_microscope_server/things/capture.py @@ -28,7 +28,7 @@ class CaptureThing(Thing): cam: CamDep, logger: InvocationLogger, metadata_getter: GetThingStates, - stream_name: str = "main", + target_resolution: tuple[int, int], ) -> None: """Capture an image and save it to disk @@ -39,7 +39,7 @@ class CaptureThing(Thing): image, metadata = self._capture_array( cam, metadata_getter, - stream=stream_name, + target_resolution=target_resolution, logger=logger, ) acquisition_time = time.time() @@ -81,7 +81,7 @@ class CaptureThing(Thing): cam: CamDep, metadata_getter: GetThingStates, logger: InvocationLogger, - stream: str = "main", + target_resolution: tuple[int, int] = (1640, 1232), ): """Capture an image in memory and return it with metadata CaptureError raised if the capture fails for any reason @@ -90,7 +90,13 @@ class CaptureThing(Thing): for capture_attempts in range(5): try: metadata = metadata_getter() - image = cam.capture_array(stream_name=stream, wait=5)[..., :3] + image = Image.fromarray( + cam.capture_array(stream_name="main", wait=5)[..., :3].astype( + "uint8" + ), + "RGB", + ) + image = image.resize(target_resolution, Image.LANCZOS) return image, metadata except TimeoutError: logger.warning( @@ -110,9 +116,7 @@ class CaptureThing(Thing): IOError is raised if the file cannot be saved nothing is returned on success""" try: - Image.fromarray(image.astype("uint8"), "RGB").save( - jpeg_path, quality=95, subsampling=0 - ) + image.save(jpeg_path, quality=95, subsampling=0) try: exif_dict = piexif.load(jpeg_path) exif_dict["Exif"][piexif.ExifIFD.UserComment] = json.dumps( diff --git a/src/openflexure_microscope_server/things/smart_scan.py b/src/openflexure_microscope_server/things/smart_scan.py index f82fbf43..af7f6569 100644 --- a/src/openflexure_microscope_server/things/smart_scan.py +++ b/src/openflexure_microscope_server/things/smart_scan.py @@ -182,7 +182,7 @@ class SmartScanThing(Thing): self._scan_images_taken = 0 self._stitch_resize = 1 - self._cam.start_highres_streaming() + self._cam.start_streaming(main_resolution=(3280, 2464)) # Don't set self._scan_data dictionary. This is done at the start of _run_scan @@ -692,7 +692,7 @@ class SmartScanThing(Thing): self._autofocus.run_z_stack( images_dir=self._ongoing_scan_images_dir, stack_dir=site_folder, - capture_method="blob", + target_resolution=(1640, 1232), ) # increment capure counter as thread has completed From dbed1c69716e506b9d76fede99b29be001e892db Mon Sep 17 00:00:00 2001 From: jaknapper Date: Wed, 7 May 2025 22:15:11 +0100 Subject: [PATCH 15/56] Default resize in stitching for scan tab stitching --- src/openflexure_microscope_server/things/smart_scan.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/openflexure_microscope_server/things/smart_scan.py b/src/openflexure_microscope_server/things/smart_scan.py index af7f6569..56f0d305 100644 --- a/src/openflexure_microscope_server/things/smart_scan.py +++ b/src/openflexure_microscope_server/things/smart_scan.py @@ -733,6 +733,7 @@ class SmartScanThing(Thing): logger=self._scan_logger, scan_name=self._ongoing_scan_name, overlap=self._scan_data["overlap"], + stitch_resize=self._stitch_resize, ) self.promote_stitch_files(self._scan_logger) except SubprocessError as e: @@ -1086,6 +1087,7 @@ class SmartScanThing(Thing): logger: InvocationLogger, scan_name: str, overlap: float = 0.0, + stitch_resize: float = 0.25, ) -> None: """Generate a stitched image based on stage position metadata @@ -1132,7 +1134,7 @@ class SmartScanThing(Thing): "--minimum_overlap", f"{round(overlap * 0.9, 2)}", "--resize", - f"{self._stitch_resize}", + f"{stitch_resize}", images_folder, ], ) From 744ec28f93429541c98f12453fb2ec27a294064d Mon Sep 17 00:00:00 2001 From: jaknapper Date: Wed, 7 May 2025 22:16:38 +0100 Subject: [PATCH 16/56] Get images as a PIL image from camera, use resizing and saving as settling time in stack --- .../things/autofocus.py | 38 ++++++++++++------- .../things/camera/__init__.py | 10 +++++ .../things/capture.py | 29 ++++++-------- 3 files changed, 46 insertions(+), 31 deletions(-) diff --git a/src/openflexure_microscope_server/things/autofocus.py b/src/openflexure_microscope_server/things/autofocus.py index f332278f..0d90e70d 100644 --- a/src/openflexure_microscope_server/things/autofocus.py +++ b/src/openflexure_microscope_server/things/autofocus.py @@ -14,13 +14,11 @@ from typing import Annotated, Mapping, Optional, Sequence import os import shutil import glob -import os -import shutil -import glob from fastapi import Depends import numpy as np from pydantic import BaseModel +from PIL import Image from labthings_fastapi.thing import Thing from labthings_fastapi.dependencies.blocking_portal import BlockingPortal @@ -34,13 +32,12 @@ from .camera import RawCameraDependency as Camera from .camera import CameraDependency as WrappedCamera from .stage import StageDependency as Stage from .capture import CaptureThing -import numpy as np -from pydantic import BaseModel CaptureDep = direct_thing_client_dependency(CaptureThing, "/capture/") SETTLING_TIME = 0.3 +STACK_OVERSHOOT = 200 class JPEGSharpnessMonitor: @@ -302,7 +299,8 @@ class AutofocusThing(Thing): images_to_capture = self.stack_images_to_capture stack_z_range = stack_dz * (images_to_capture - 1) - stage.move_relative(z=-stack_z_range / 2) + stage.move_relative(z=-(STACK_OVERSHOOT + stack_z_range / 2)) + stage.move_relative(z=STACK_OVERSHOOT) for capture_count in range(images_to_capture): time.sleep(SETTLING_TIME) @@ -311,17 +309,29 @@ class AutofocusThing(Thing): stack_dir, f"{capture_count}.jpeg", ) - capture._capture_and_save( - jpeg_path=jpeg_path, + start = time.time() + image, metadata = capture._capture_image( cam=cam, - logger=logger, metadata_getter=metadata_getter, - target_resolution=target_resolution, + logger=logger, ) - - # If the stack isn't complete yet, move - if capture_count + 1 < images_to_capture: - stage.move_relative(z=stack_dz) + captured = time.time() + # There's an unnecessary move up at the end of the stack + stage.move_relative(z=stack_dz) + moved = time.time() + image = image.resize(target_resolution, Image.LANCZOS) + downsampled = time.time() + capture._save_capture( + jpeg_path=jpeg_path, + image=image, + metadata=metadata, + logger=logger, + ) + saved = time.time() + logger.info(f"Capturing took {round(captured - start, 2)} s") + logger.info(f"Resizing took {round(downsampled - moved, 2)} s") + logger.info(f"Saving took {round(saved - downsampled, 2)} s") + logger.info(f"Effective settling time was {round(saved - moved, 2)} s") self.copy_central_image_from_stack(images_dir, stack_dir) diff --git a/src/openflexure_microscope_server/things/camera/__init__.py b/src/openflexure_microscope_server/things/camera/__init__.py index 6c18765a..12424116 100644 --- a/src/openflexure_microscope_server/things/camera/__init__.py +++ b/src/openflexure_microscope_server/things/camera/__init__.py @@ -84,6 +84,11 @@ class CameraProtocol(Protocol): for the main stream""" ... + @thing_action + def capture_image(self, stream_name, wait): + """Capture a PIL image from stream stream_name with timeout wait""" + ... + class BaseCamera(Thing): """A Thing representing a camera @@ -194,6 +199,11 @@ class CameraStub(BaseCamera): for the main stream""" raise NotImplementedError("Cameras must not inherit from CameraStub") + @thing_action + def capture_image(self, stream_name, wait): + """Capture a PIL image from stream stream_name with timeout wait""" + raise Exception + CameraDependency = direct_thing_client_dependency(CameraStub, "/camera/") RawCameraDependency = raw_thing_dependency(CameraProtocol) diff --git a/src/openflexure_microscope_server/things/capture.py b/src/openflexure_microscope_server/things/capture.py index e353ecba..71b7c00c 100644 --- a/src/openflexure_microscope_server/things/capture.py +++ b/src/openflexure_microscope_server/things/capture.py @@ -1,5 +1,3 @@ -import numpy as np -from PIL import Image import time import piexif import json @@ -28,7 +26,6 @@ class CaptureThing(Thing): cam: CamDep, logger: InvocationLogger, metadata_getter: GetThingStates, - target_resolution: tuple[int, int], ) -> None: """Capture an image and save it to disk @@ -36,14 +33,18 @@ class CaptureThing(Thing): that the stage may be moved while it's saved. """ capture_start = time.time() - image, metadata = self._capture_array( + image, metadata = self._capture_image( cam, metadata_getter, - target_resolution=target_resolution, logger=logger, ) acquisition_time = time.time() - self._save_capture(jpeg_path, image, metadata, logger) + self._save_capture( + jpeg_path, + image, + metadata, + logger, + ) save_time = time.time() acquisition_duration = round(acquisition_time - capture_start, 1) saving_duration = round(save_time - acquisition_time, 1) @@ -56,7 +57,7 @@ class CaptureThing(Thing): """Capture a JPEG (from a JPEGBlob) to disk""" for capture_attempts in range(5): try: - jpeg = cam.capture_jpeg(resolution="full", wait=5) + jpeg = cam.capture_jpeg(resolution="main", wait=5) jpeg.save(filename) return except TimeoutError: @@ -76,12 +77,11 @@ class CaptureThing(Thing): cam.restart_stream() @thing_action - def _capture_array( + def _capture_image( self, cam: CamDep, metadata_getter: GetThingStates, logger: InvocationLogger, - target_resolution: tuple[int, int] = (1640, 1232), ): """Capture an image in memory and return it with metadata CaptureError raised if the capture fails for any reason @@ -90,13 +90,7 @@ class CaptureThing(Thing): for capture_attempts in range(5): try: metadata = metadata_getter() - image = Image.fromarray( - cam.capture_array(stream_name="main", wait=5)[..., :3].astype( - "uint8" - ), - "RGB", - ) - image = image.resize(target_resolution, Image.LANCZOS) + image = cam.capture_image(stream_name="main", wait=5) return image, metadata except TimeoutError: logger.warning( @@ -104,10 +98,11 @@ class CaptureThing(Thing): ) raise CaptureError("An error occurred while capturing after 5 attempts") + @thing_action def _save_capture( self, jpeg_path: str, - image: np.ndarray, + image, metadata: dict, logger: InvocationLogger, ) -> None: From 0d32983b62b5dabcaa2f318cffabb64d1d37db29 Mon Sep 17 00:00:00 2001 From: jaknapper Date: Thu, 8 May 2025 20:05:16 +0100 Subject: [PATCH 17/56] Only undershoot z stack if image count > 1, settle before first capture --- src/openflexure_microscope_server/things/autofocus.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/openflexure_microscope_server/things/autofocus.py b/src/openflexure_microscope_server/things/autofocus.py index 0d90e70d..1edde53f 100644 --- a/src/openflexure_microscope_server/things/autofocus.py +++ b/src/openflexure_microscope_server/things/autofocus.py @@ -299,8 +299,10 @@ class AutofocusThing(Thing): images_to_capture = self.stack_images_to_capture stack_z_range = stack_dz * (images_to_capture - 1) - stage.move_relative(z=-(STACK_OVERSHOOT + stack_z_range / 2)) - stage.move_relative(z=STACK_OVERSHOOT) + if stack_z_range > 0: + stage.move_relative(z=-(STACK_OVERSHOOT + stack_z_range / 2)) + stage.move_relative(z=STACK_OVERSHOOT) + time.sleep(0.3) for capture_count in range(images_to_capture): time.sleep(SETTLING_TIME) From d39bcbaf5cf0a8f1c51921819c2786014cb8a2f1 Mon Sep 17 00:00:00 2001 From: jaknapper Date: Mon, 12 May 2025 12:31:43 +0100 Subject: [PATCH 18/56] Moved streaming changing to inner loop --- src/openflexure_microscope_server/things/smart_scan.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/openflexure_microscope_server/things/smart_scan.py b/src/openflexure_microscope_server/things/smart_scan.py index 56f0d305..53e324cd 100644 --- a/src/openflexure_microscope_server/things/smart_scan.py +++ b/src/openflexure_microscope_server/things/smart_scan.py @@ -63,7 +63,7 @@ def ensure_free_disk_space(path: str, min_space: int = 500000000) -> None: if du.free < min_space: raise NotEnoughFreeSpaceError( "There is not enough free disk space to continue. " - f"(Required: {min_space}, {du})." + f"(Required: {min_space >> 20}MB, free: {du.free >> 20}MB)." ) @@ -182,8 +182,6 @@ class SmartScanThing(Thing): self._scan_images_taken = 0 self._stitch_resize = 1 - self._cam.start_streaming(main_resolution=(3280, 2464)) - # Don't set self._scan_data dictionary. This is done at the start of _run_scan try: @@ -204,7 +202,6 @@ class SmartScanThing(Thing): # Error must be raised so UI gives correct output raise e finally: - self._cam.start_streaming() # However the scan finishes, unset all variables and release lock self._cancel = None self._scan_logger = None @@ -559,6 +556,7 @@ class SmartScanThing(Thing): scan_successful = True try: + self._cam.start_streaming(main_resolution = (3280,2464)) self._set_scan_data() self._save_scan_inputs_json() if self._scan_images_taken != 0: @@ -590,6 +588,8 @@ class SmartScanThing(Thing): ) raise e finally: + # Start streaming in the default resolution again as soon as possible + self._cam.start_streaming() if self._capture_thread: # If the capture thread had an error, we capture it here try: From 52b58241cc088f90b89b821a21133c7bc919b244 Mon Sep 17 00:00:00 2001 From: jaknapper Date: Mon, 12 May 2025 12:32:51 +0100 Subject: [PATCH 19/56] Added copy_sharpest_image to autofocus --- .../things/autofocus.py | 32 +++++++++++++++---- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/src/openflexure_microscope_server/things/autofocus.py b/src/openflexure_microscope_server/things/autofocus.py index 1edde53f..edb208e3 100644 --- a/src/openflexure_microscope_server/things/autofocus.py +++ b/src/openflexure_microscope_server/things/autofocus.py @@ -319,9 +319,10 @@ class AutofocusThing(Thing): ) captured = time.time() # There's an unnecessary move up at the end of the stack - stage.move_relative(z=stack_dz) + if capture_count + 1 < images_to_capture: + stage.move_relative(z=stack_dz) moved = time.time() - image = image.resize(target_resolution, Image.LANCZOS) + image = image.resize(target_resolution, Image.BOX) downsampled = time.time() capture._save_capture( jpeg_path=jpeg_path, @@ -330,12 +331,12 @@ class AutofocusThing(Thing): logger=logger, ) saved = time.time() - logger.info(f"Capturing took {round(captured - start, 2)} s") - logger.info(f"Resizing took {round(downsampled - moved, 2)} s") - logger.info(f"Saving took {round(saved - downsampled, 2)} s") - logger.info(f"Effective settling time was {round(saved - moved, 2)} s") + logger.debug(f"Capturing took {round(captured - start, 2)} s") + logger.debug(f"Resizing took {round(downsampled - moved, 2)} s") + logger.debug(f"Saving took {round(saved - downsampled, 2)} s") + logger.debug(f"Effective settling time was {round(saved - moved, 2)} s") - self.copy_central_image_from_stack(images_dir, stack_dir) + self.copy_sharpest_image_from_stack(images_dir, stack_dir, logger) def copy_central_image_from_stack( self, @@ -351,3 +352,20 @@ class AutofocusThing(Thing): xy_location = os.path.basename(stack_dir) shutil.copy(central_image, os.path.join(images_dir, f"{xy_location}.jpeg")) + + def copy_sharpest_image_from_stack( + self, + images_dir: str, + stack_dir: str, + logger: InvocationLogger, + ): + """Gets a list of images in a folder (stack_dir), sorts them by filesize, and copies the sharpest + image to images dir.""" + image_list = glob.glob(os.path.join(stack_dir, "*")) + image_list = sorted(image_list, key=os.path.getsize) + central_index = (len(image_list) - 1) // 2 + central_image = image_list[central_index] + xy_location = os.path.basename(stack_dir) + + logger.info(central_image) + shutil.copy(central_image, os.path.join(images_dir, f"{xy_location}.jpeg")) From 478c3629c9f88c09fae0abbe5559e27ec13c6cbf Mon Sep 17 00:00:00 2001 From: jaknapper Date: Mon, 12 May 2025 18:35:20 +0100 Subject: [PATCH 20/56] Scan_resize from global vars, added to scan_data --- .../things/smart_scan.py | 34 +++++-------------- 1 file changed, 8 insertions(+), 26 deletions(-) diff --git a/src/openflexure_microscope_server/things/smart_scan.py b/src/openflexure_microscope_server/things/smart_scan.py index 53e324cd..efdb5b73 100644 --- a/src/openflexure_microscope_server/things/smart_scan.py +++ b/src/openflexure_microscope_server/things/smart_scan.py @@ -92,6 +92,8 @@ SCAN_DATA_FILENAME = "scan_data.json" STITCHING_CMD = "openflexure-stitch" SCAN_ZERO_PAD_DIGITS = 4 +TARGET_RESOLUTION = (1640, 1232) +STITCHING_RESOLUTION = (820, 616) def _scan_running(method): @@ -143,7 +145,6 @@ class SmartScanThing(Thing): self._scan_images_taken: Optional[int] = None # TODO Scan data is a dict during refactoring, should become a dataclass self._scan_data: Optional[dict] = None - self._stitch_resize: Optional[float] = None @thing_action def sample_scan( @@ -180,7 +181,6 @@ class SmartScanThing(Thing): self._background_detect = background_detect self._capture_thread = None self._scan_images_taken = 0 - self._stitch_resize = 1 # Don't set self._scan_data dictionary. This is done at the start of _run_scan @@ -216,7 +216,6 @@ class SmartScanThing(Thing): self._scan_images_taken = None self._scan_data = None self._scan_lock.release() - self._stitch_resize = None def promote_stitch_files( self, @@ -378,22 +377,6 @@ class SmartScanThing(Thing): return (next_point[0], next_point[1], z_estimate) - # @_scan_running - # def _calc_resize_from_test_image(self): - # """ - # Take a test image to set the amount to downsample images for stitching - - # Return the decimal value to scale x and y by when stitching - # """ - - # #TODO: This needs to match how the capture in the z stack is done - # test_jpg = self._cam.capture_jpeg(resolution="full") - # test_jpg = test_jpg.content - - # test_image = Image.open(io.BytesIO(test_jpg)) - # test_image_res = list(test_image.size) - # return STITCH_IMAGE_WIDTH / test_image_res[0] - @_scan_running def _calc_displacement_from_test_image(self, overlap): """ @@ -438,10 +421,9 @@ class SmartScanThing(Thing): """ overlap = self.overlap dx, dy = self._calc_displacement_from_test_image(overlap) - # self._stitch_resize = self._calc_resize_from_test_image() - self._stitch_resize = 0.25 + stitch_resize = STITCHING_RESOLUTION[0] / TARGET_RESOLUTION[0] - self._scan_logger.info(f"Resizing images by {self._stitch_resize}") + self._scan_logger.debug(f"Resizing images when stitching by {stitch_resize}") self._scan_logger.info( f"Based on an overlap of {overlap}, we will make steps of {dx}, {dy}" @@ -469,6 +451,7 @@ class SmartScanThing(Thing): "start_time": time.strftime("%H_%M_%S-%d_%m_%Y"), "skip_background": self.skip_background, "stitch_automatically": self.stitch_automatically, + "stitch_resize": stitch_resize, } @_scan_running @@ -556,7 +539,7 @@ class SmartScanThing(Thing): scan_successful = True try: - self._cam.start_streaming(main_resolution = (3280,2464)) + self._cam.start_streaming(main_resolution=(3280, 2464)) self._set_scan_data() self._save_scan_inputs_json() if self._scan_images_taken != 0: @@ -692,7 +675,7 @@ class SmartScanThing(Thing): self._autofocus.run_z_stack( images_dir=self._ongoing_scan_images_dir, stack_dir=site_folder, - target_resolution=(1640, 1232), + target_resolution=TARGET_RESOLUTION, ) # increment capure counter as thread has completed @@ -733,7 +716,6 @@ class SmartScanThing(Thing): logger=self._scan_logger, scan_name=self._ongoing_scan_name, overlap=self._scan_data["overlap"], - stitch_resize=self._stitch_resize, ) self.promote_stitch_files(self._scan_logger) except SubprocessError as e: @@ -1015,7 +997,7 @@ class SmartScanThing(Thing): "--minimum_overlap", f"{min_overlap}", "--resize", - f"{self._stitch_resize}", + f"{self._scan_data['stitch_resize']}", self._ongoing_scan_images_dir, ] ) From 968bd339c209a7075133c1d5c5f229e787264a6b Mon Sep 17 00:00:00 2001 From: jaknapper Date: Wed, 14 May 2025 19:49:42 +0100 Subject: [PATCH 21/56] SETTLING_TIME global in stacking --- src/openflexure_microscope_server/things/autofocus.py | 10 ++++++++-- .../things/camera/__init__.py | 1 - 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/openflexure_microscope_server/things/autofocus.py b/src/openflexure_microscope_server/things/autofocus.py index edb208e3..89bbc8a8 100644 --- a/src/openflexure_microscope_server/things/autofocus.py +++ b/src/openflexure_microscope_server/things/autofocus.py @@ -36,8 +36,8 @@ from .capture import CaptureThing CaptureDep = direct_thing_client_dependency(CaptureThing, "/capture/") -SETTLING_TIME = 0.3 STACK_OVERSHOOT = 200 +SETTLING_TIME = 0.3 class JPEGSharpnessMonitor: @@ -322,7 +322,8 @@ class AutofocusThing(Thing): if capture_count + 1 < images_to_capture: stage.move_relative(z=stack_dz) moved = time.time() - image = image.resize(target_resolution, Image.BOX) + if image.size != target_resolution: + image = image.resize(target_resolution, Image.BOX) downsampled = time.time() capture._save_capture( jpeg_path=jpeg_path, @@ -331,6 +332,11 @@ class AutofocusThing(Thing): logger=logger, ) saved = time.time() + if saved - start < SETTLING_TIME: + time.sleep(SETTLING_TIME - (saved - start)) + logger.info( + f"Settled for an extra {round(SETTLING_TIME - (saved - start), 3)} seconds" + ) logger.debug(f"Capturing took {round(captured - start, 2)} s") logger.debug(f"Resizing took {round(downsampled - moved, 2)} s") logger.debug(f"Saving took {round(saved - downsampled, 2)} s") diff --git a/src/openflexure_microscope_server/things/camera/__init__.py b/src/openflexure_microscope_server/things/camera/__init__.py index 12424116..24219f09 100644 --- a/src/openflexure_microscope_server/things/camera/__init__.py +++ b/src/openflexure_microscope_server/things/camera/__init__.py @@ -84,7 +84,6 @@ class CameraProtocol(Protocol): for the main stream""" ... - @thing_action def capture_image(self, stream_name, wait): """Capture a PIL image from stream stream_name with timeout wait""" ... From 2d946637eb13bf55f7e3da3672ad692c1599dcaf Mon Sep 17 00:00:00 2001 From: jaknapper Date: Fri, 16 May 2025 15:26:49 +0100 Subject: [PATCH 22/56] Added buffer_count as an argument of streaming --- src/openflexure_microscope_server/things/camera/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/openflexure_microscope_server/things/camera/__init__.py b/src/openflexure_microscope_server/things/camera/__init__.py index 24219f09..dcdf3e8f 100644 --- a/src/openflexure_microscope_server/things/camera/__init__.py +++ b/src/openflexure_microscope_server/things/camera/__init__.py @@ -79,7 +79,7 @@ class CameraProtocol(Protocol): """Acquire one image from the preview stream and return its size""" ... - def start_streaming(self, main_resolution) -> None: + def start_streaming(self, main_resolution, buffer_count) -> None: """Start (or stop and restart) the camera with the given resolution for the main stream""" ... @@ -193,7 +193,7 @@ class CameraStub(BaseCamera): raise NotImplementedError("Cameras must not inherit from CameraStub") @thing_action - def start_streaming(self, main_resolution) -> None: + def start_streaming(self, main_resolution, buffer_count) -> None: """Start (or stop and restart) the camera with the given resolution for the main stream""" raise NotImplementedError("Cameras must not inherit from CameraStub") From 8c8c6568c3f3ad0ab98a567242132b6a783d502f Mon Sep 17 00:00:00 2001 From: jaknapper Date: Fri, 16 May 2025 15:29:31 +0100 Subject: [PATCH 23/56] Updated docstring for start_streaming --- src/openflexure_microscope_server/things/camera/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/openflexure_microscope_server/things/camera/__init__.py b/src/openflexure_microscope_server/things/camera/__init__.py index dcdf3e8f..4916e40f 100644 --- a/src/openflexure_microscope_server/things/camera/__init__.py +++ b/src/openflexure_microscope_server/things/camera/__init__.py @@ -81,7 +81,7 @@ class CameraProtocol(Protocol): def start_streaming(self, main_resolution, buffer_count) -> None: """Start (or stop and restart) the camera with the given resolution - for the main stream""" + for the main stream, and buffer_count number of images in the buffer""" ... def capture_image(self, stream_name, wait): @@ -195,7 +195,7 @@ class CameraStub(BaseCamera): @thing_action def start_streaming(self, main_resolution, buffer_count) -> None: """Start (or stop and restart) the camera with the given resolution - for the main stream""" + for the main stream, and buffer_count number of images in the buffer""" raise NotImplementedError("Cameras must not inherit from CameraStub") @thing_action From 985d20e03259168687ac91fee4ebe6a66b294169 Mon Sep 17 00:00:00 2001 From: jaknapper Date: Mon, 19 May 2025 16:49:54 +0100 Subject: [PATCH 24/56] Capture resolution as a Thing property --- .../things/autofocus.py | 6 +++--- .../things/smart_scan.py | 17 ++++++++++++++--- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/openflexure_microscope_server/things/autofocus.py b/src/openflexure_microscope_server/things/autofocus.py index 89bbc8a8..58ca704e 100644 --- a/src/openflexure_microscope_server/things/autofocus.py +++ b/src/openflexure_microscope_server/things/autofocus.py @@ -291,7 +291,7 @@ class AutofocusThing(Thing): capture: CaptureDep, images_dir: str, stack_dir: str, - target_resolution: tuple[int, int], + capture_resolution: tuple[int, int], ) -> None: """Run a z stack, saving all images to stack_dir and copying the central image to stack_dir""" @@ -322,8 +322,8 @@ class AutofocusThing(Thing): if capture_count + 1 < images_to_capture: stage.move_relative(z=stack_dz) moved = time.time() - if image.size != target_resolution: - image = image.resize(target_resolution, Image.BOX) + if image.size != capture_resolution: + image = image.resize(capture_resolution, Image.BOX) downsampled = time.time() capture._save_capture( jpeg_path=jpeg_path, diff --git a/src/openflexure_microscope_server/things/smart_scan.py b/src/openflexure_microscope_server/things/smart_scan.py index efdb5b73..6f2b69cd 100644 --- a/src/openflexure_microscope_server/things/smart_scan.py +++ b/src/openflexure_microscope_server/things/smart_scan.py @@ -92,7 +92,6 @@ SCAN_DATA_FILENAME = "scan_data.json" STITCHING_CMD = "openflexure-stitch" SCAN_ZERO_PAD_DIGITS = 4 -TARGET_RESOLUTION = (1640, 1232) STITCHING_RESOLUTION = (820, 616) @@ -421,7 +420,7 @@ class SmartScanThing(Thing): """ overlap = self.overlap dx, dy = self._calc_displacement_from_test_image(overlap) - stitch_resize = STITCHING_RESOLUTION[0] / TARGET_RESOLUTION[0] + stitch_resize = STITCHING_RESOLUTION[0] / self.capture_resolution[0] self._scan_logger.debug(f"Resizing images when stitching by {stitch_resize}") @@ -452,6 +451,7 @@ class SmartScanThing(Thing): "skip_background": self.skip_background, "stitch_automatically": self.stitch_automatically, "stitch_resize": stitch_resize, + "capture_resolution": self.capture_resolution, } @_scan_running @@ -470,6 +470,7 @@ class SmartScanThing(Thing): "dy": self._scan_data["dy"], "start time": self._scan_data["start_time"], "skipping background": self._scan_data["skip_background"], + "capture resolution": self._scan_data["capture_resolution"], } scan_inputs_fname = os.path.join( @@ -675,7 +676,7 @@ class SmartScanThing(Thing): self._autofocus.run_z_stack( images_dir=self._ongoing_scan_images_dir, stack_dir=site_folder, - target_resolution=TARGET_RESOLUTION, + capture_resolution=self._scan_data["capture_resolution"], ) # increment capure counter as thread has completed @@ -744,6 +745,16 @@ class SmartScanThing(Thing): raise HTTPException(404, "File not found") return FileResponse(path) + @thing_property + def capture_resolution(self) -> tuple[int, int]: + """A tuple of the image resolution to capture. Should be in a + 4:3 aspect ratio""" + return self.thing_settings.get("capture_resolution", ((1640, 1232))) + + @capture_resolution.setter + def capture_resolution(self, value: tuple[int, int]) -> None: + self.thing_settings["capture_resolution"] = value + @thing_property def max_range(self) -> int: """The maximum distance from the centre of the scan before we break in steps""" From 2f9e37a02fbc8c54d6bf2821af5a912480909a0f Mon Sep 17 00:00:00 2001 From: jaknapper Date: Tue, 20 May 2025 14:27:53 +0100 Subject: [PATCH 25/56] More descriptive timing variable names --- .../things/autofocus.py | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/openflexure_microscope_server/things/autofocus.py b/src/openflexure_microscope_server/things/autofocus.py index 58ca704e..c5848e23 100644 --- a/src/openflexure_microscope_server/things/autofocus.py +++ b/src/openflexure_microscope_server/things/autofocus.py @@ -311,36 +311,38 @@ class AutofocusThing(Thing): stack_dir, f"{capture_count}.jpeg", ) - start = time.time() + start_time = time.time() image, metadata = capture._capture_image( cam=cam, metadata_getter=metadata_getter, logger=logger, ) - captured = time.time() + captured_time = time.time() # There's an unnecessary move up at the end of the stack if capture_count + 1 < images_to_capture: stage.move_relative(z=stack_dz) - moved = time.time() + moved_time = time.time() if image.size != capture_resolution: image = image.resize(capture_resolution, Image.BOX) - downsampled = time.time() + downsampled_time = time.time() capture._save_capture( jpeg_path=jpeg_path, image=image, metadata=metadata, logger=logger, ) - saved = time.time() - if saved - start < SETTLING_TIME: - time.sleep(SETTLING_TIME - (saved - start)) + saved_time = time.time() + if saved_time - start_time < SETTLING_TIME: + time.sleep(SETTLING_TIME - (saved_time - start_time)) logger.info( - f"Settled for an extra {round(SETTLING_TIME - (saved - start), 3)} seconds" + f"Settled for an extra {round(SETTLING_TIME - (saved_time - start_time), 3)} seconds" ) - logger.debug(f"Capturing took {round(captured - start, 2)} s") - logger.debug(f"Resizing took {round(downsampled - moved, 2)} s") - logger.debug(f"Saving took {round(saved - downsampled, 2)} s") - logger.debug(f"Effective settling time was {round(saved - moved, 2)} s") + logger.debug(f"Capturing took {round(captured_time - start_time, 2)} s") + logger.debug(f"Resizing took {round(downsampled_time - moved_time, 2)} s") + logger.debug(f"Saving took {round(saved_time - downsampled_time, 2)} s") + logger.debug( + f"Effective settling time was {round(saved_time - moved_time, 2)} s" + ) self.copy_sharpest_image_from_stack(images_dir, stack_dir, logger) From 7052917ff9d37d7a709f6df15becf568ad3152fe Mon Sep 17 00:00:00 2001 From: jaknapper Date: Tue, 20 May 2025 14:40:13 +0100 Subject: [PATCH 26/56] capture_image in stub raises NotImplemented --- src/openflexure_microscope_server/things/camera/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/openflexure_microscope_server/things/camera/__init__.py b/src/openflexure_microscope_server/things/camera/__init__.py index 4916e40f..c1fb9a17 100644 --- a/src/openflexure_microscope_server/things/camera/__init__.py +++ b/src/openflexure_microscope_server/things/camera/__init__.py @@ -201,7 +201,7 @@ class CameraStub(BaseCamera): @thing_action def capture_image(self, stream_name, wait): """Capture a PIL image from stream stream_name with timeout wait""" - raise Exception + raise NotImplementedError("Cameras must not inherit from CameraStub") CameraDependency = direct_thing_client_dependency(CameraStub, "/camera/") From 4271c28045ccb4abadf61f3ebd7abe1986e6e330 Mon Sep 17 00:00:00 2001 From: jaknapper Date: Tue, 20 May 2025 14:41:08 +0100 Subject: [PATCH 27/56] More descriptive variable names in smart_scan, use resize in stitching with no default --- src/openflexure_microscope_server/things/smart_scan.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/openflexure_microscope_server/things/smart_scan.py b/src/openflexure_microscope_server/things/smart_scan.py index 6f2b69cd..5ec78a4d 100644 --- a/src/openflexure_microscope_server/things/smart_scan.py +++ b/src/openflexure_microscope_server/things/smart_scan.py @@ -59,11 +59,11 @@ def ensure_free_disk_space(path: str, min_space: int = 500000000) -> None: Raises: NotEnoughFreeSpaceError if the remaining storage is below min_space """ - du = shutil.disk_usage(path) - if du.free < min_space: + disk_usage = shutil.disk_usage(path) + if disk_usage.free < min_space: raise NotEnoughFreeSpaceError( "There is not enough free disk space to continue. " - f"(Required: {min_space >> 20}MB, free: {du.free >> 20}MB)." + f"(Required: {min_space >> 20}MB, free: {disk_usage.free >> 20}MB)." ) @@ -716,6 +716,7 @@ class SmartScanThing(Thing): self.stitch_scan( logger=self._scan_logger, scan_name=self._ongoing_scan_name, + stitch_resize=self._scan_data["stitch_resize"], overlap=self._scan_data["overlap"], ) self.promote_stitch_files(self._scan_logger) @@ -1079,8 +1080,8 @@ class SmartScanThing(Thing): self, logger: InvocationLogger, scan_name: str, + stitch_resize: float, overlap: float = 0.0, - stitch_resize: float = 0.25, ) -> None: """Generate a stitched image based on stage position metadata From 35e1d6cdee6117e1ed4f3abf7354c01954584de0 Mon Sep 17 00:00:00 2001 From: jaknapper Date: Tue, 20 May 2025 14:41:35 +0100 Subject: [PATCH 28/56] Remove unused actions in capture --- .../things/capture.py | 26 +------------------ 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/src/openflexure_microscope_server/things/capture.py b/src/openflexure_microscope_server/things/capture.py index 71b7c00c..7f2e5469 100644 --- a/src/openflexure_microscope_server/things/capture.py +++ b/src/openflexure_microscope_server/things/capture.py @@ -52,30 +52,6 @@ class CaptureThing(Thing): f"Acquired {jpeg_path} in {acquisition_duration}s then {saving_duration}s saving to disk" ) - @thing_action - def capture_jpeg(self, filename: str, cam: CamDep, logger: InvocationLogger): - """Capture a JPEG (from a JPEGBlob) to disk""" - for capture_attempts in range(5): - try: - jpeg = cam.capture_jpeg(resolution="main", wait=5) - jpeg.save(filename) - return - except TimeoutError: - logger.warning( - f"Attempt {capture_attempts + 1} to capture image timed out. Do you have enough RAM?" - ) - except Exception as e: - logger.warning(e) - raise CaptureError("An error occurred while capturing after 5 attempts") - - @thing_action - def stop_streaming(self, cam: CamDep): - cam.stop_streaming() - - @thing_action - def restart_stream(self, cam: CamDep): - cam.restart_stream() - @thing_action def _capture_image( self, @@ -85,7 +61,7 @@ class CaptureThing(Thing): ): """Capture an image in memory and return it with metadata CaptureError raised if the capture fails for any reason - returns tuple with numpy array of image data, and dict of metadata + returns tuple with PIL Image, and dict of metadata """ for capture_attempts in range(5): try: From 628fd145f335823b68b2d95240b4249253a904b6 Mon Sep 17 00:00:00 2001 From: jaknapper Date: Tue, 20 May 2025 14:42:02 +0100 Subject: [PATCH 29/56] Cleaned up settling time maths to only calculate once --- src/openflexure_microscope_server/things/autofocus.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/openflexure_microscope_server/things/autofocus.py b/src/openflexure_microscope_server/things/autofocus.py index c5848e23..b651085a 100644 --- a/src/openflexure_microscope_server/things/autofocus.py +++ b/src/openflexure_microscope_server/things/autofocus.py @@ -318,7 +318,7 @@ class AutofocusThing(Thing): logger=logger, ) captured_time = time.time() - # There's an unnecessary move up at the end of the stack + if capture_count + 1 < images_to_capture: stage.move_relative(z=stack_dz) moved_time = time.time() @@ -332,11 +332,10 @@ class AutofocusThing(Thing): logger=logger, ) saved_time = time.time() - if saved_time - start_time < SETTLING_TIME: - time.sleep(SETTLING_TIME - (saved_time - start_time)) - logger.info( - f"Settled for an extra {round(SETTLING_TIME - (saved_time - start_time), 3)} seconds" - ) + time_remaining = SETTLING_TIME - (saved_time - start_time) + if time_remaining > 0: + time.sleep(time_remaining) + logger.info(f"Settled for an extra {round(time_remaining, 3)} seconds") logger.debug(f"Capturing took {round(captured_time - start_time, 2)} s") logger.debug(f"Resizing took {round(downsampled_time - moved_time, 2)} s") logger.debug(f"Saving took {round(saved_time - downsampled_time, 2)} s") From b5606984ae87a23b325f252c5558b31dd20f6cb5 Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Wed, 21 May 2025 20:31:51 +0100 Subject: [PATCH 30/56] Consolidate the BaseCamera, CameraStub, and CameraProtocol, move StreamingPiCamera2 to this repo --- .../picamera2/test_acquisition.py | 32 + .../picamera2/test_exposure_time_drift.py | 32 + .../picamera2/test_sensor_mode.py | 27 + .../picamera2/test_tuning.py | 81 ++ ofm_config_full.json | 2 +- pyproject.toml | 2 +- .../things/camera/__init__.py | 141 +-- .../things/camera/opencv.py | 3 - .../things/camera/picamera.py | 868 ++++++++++++++++++ .../camera/picamera_recalibrate_utils.py | 561 +++++++++++ .../things/camera/simulation.py | 11 +- 11 files changed, 1617 insertions(+), 143 deletions(-) create mode 100644 hardware-specific-tests/picamera2/test_acquisition.py create mode 100644 hardware-specific-tests/picamera2/test_exposure_time_drift.py create mode 100644 hardware-specific-tests/picamera2/test_sensor_mode.py create mode 100644 hardware-specific-tests/picamera2/test_tuning.py create mode 100644 src/openflexure_microscope_server/things/camera/picamera.py create mode 100644 src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py diff --git a/hardware-specific-tests/picamera2/test_acquisition.py b/hardware-specific-tests/picamera2/test_acquisition.py new file mode 100644 index 00000000..9c8ce20f --- /dev/null +++ b/hardware-specific-tests/picamera2/test_acquisition.py @@ -0,0 +1,32 @@ +from labthings_picamera2 import StreamingPiCamera2 +from labthings_fastapi.server import ThingServer +from labthings_fastapi.client import ThingClient +from fastapi.testclient import TestClient +from PIL import Image +import numpy as np +from pytest import fixture + +@fixture(scope="module") +def client(): + server = ThingServer() + server.add_thing(StreamingPiCamera2(), "/camera/") + with TestClient(server.app) as test_client: + client = ThingClient.from_url("/camera/", client=test_client) + yield client + +def test_calibration(client): + client.full_auto_calibrate() + +def test_jpeg_and_array(client): + blob = client.grab_jpeg() + mjpeg_frame = Image.open(blob.open()) + assert mjpeg_frame + blob = client.capture_jpeg(resolution="main") + jpeg_capture = Image.open(blob.open()) + assert jpeg_capture + arrlist = client.capture_array(stream_name="main") + array_main = np.array(arrlist) + assert mjpeg_frame.size == jpeg_capture.size + assert array_main.shape[1::-1] == jpeg_capture.size + + diff --git a/hardware-specific-tests/picamera2/test_exposure_time_drift.py b/hardware-specific-tests/picamera2/test_exposure_time_drift.py new file mode 100644 index 00000000..ee75d179 --- /dev/null +++ b/hardware-specific-tests/picamera2/test_exposure_time_drift.py @@ -0,0 +1,32 @@ +import logging +import time + +from fastapi.testclient import TestClient + +from labthings_fastapi.server import ThingServer +from labthings_fastapi.client import ThingClient +from labthings_picamera2.thing import StreamingPiCamera2 + +logging.basicConfig(level=logging.DEBUG) + + +def test_exposure_time_drift(): + cam = StreamingPiCamera2() + server = ThingServer() + server.add_thing(cam, "/camera/") + + with TestClient(server.app) as test_client: + client = ThingClient.from_url("/camera/", client=test_client) + client.exposure_time = 50000 + time.sleep(0.1) + initial_et = client.exposure_time + print(f"Before capture, et is {client.exposure_time}") + for i in range(10): + client.capture_jpeg(resolution="full") + print(f"After capture, et is {client.exposure_time}") + final_et = client.exposure_time + assert initial_et == final_et + + +if __name__ == "__main__": + test_exposure_time_drift() diff --git a/hardware-specific-tests/picamera2/test_sensor_mode.py b/hardware-specific-tests/picamera2/test_sensor_mode.py new file mode 100644 index 00000000..91694a89 --- /dev/null +++ b/hardware-specific-tests/picamera2/test_sensor_mode.py @@ -0,0 +1,27 @@ +import logging + +from fastapi.testclient import TestClient +import numpy as np + +from labthings_fastapi.server import ThingServer +from labthings_fastapi.client import ThingClient +from labthings_picamera2.thing import StreamingPiCamera2 + +logging.basicConfig(level=logging.DEBUG) + + +def test_sensor_mode(): + cam = StreamingPiCamera2() + server = ThingServer() + server.add_thing(cam, "/camera/") + + with TestClient(server.app) as test_client: + client = ThingClient.from_url("/camera/", client=test_client) + for size in [(3280, 2464), (1640, 1232)]: + client.sensor_mode = {"output_size": size, "bit_depth": 10} + arr = np.array(client.capture_array(stream_name="raw")) + assert arr.shape[0] == size[1] + + +if __name__ == "__main__": + test_sensor_mode() diff --git a/hardware-specific-tests/picamera2/test_tuning.py b/hardware-specific-tests/picamera2/test_tuning.py new file mode 100644 index 00000000..f284badc --- /dev/null +++ b/hardware-specific-tests/picamera2/test_tuning.py @@ -0,0 +1,81 @@ +import os +from picamera2 import Picamera2 +from labthings_picamera2 import recalibrate_utils + +import pytest + + +MODEL = Picamera2.global_camera_info()[0]['Model'] + + +def check_camera_available(): + assert len(Picamera2.global_camera_info()) >= 1 + + +def load_default_tuning(): + fname = f"{MODEL}.json" + return Picamera2.load_tuning_file(fname) + + +def generate_bad_tuning(): + default_tuning = load_default_tuning() + bad_tuning = default_tuning.copy() + bad_tuning["version"] = 999 + return bad_tuning + + +def print_tuning(read_file=False): + key = "LIBCAMERA_RPI_TUNING_FILE" + if key in os.environ: + print(f"Tuning file environment variable: {os.environ[key]}") + if read_file: + with open(os.environ[key], "r") as f: + print(f.read()) + else: + print("Tuning file environment variable not set") + + +def _test_bad_tuning_after_good_tuning(configure): + bad_tuning = generate_bad_tuning() + default_tuning = load_default_tuning() + print_tuning() + print("opening camera with explicitly specified tuning") + with Picamera2(tuning=default_tuning) as cam: + print_tuning() + if configure: + cam.configure(cam.create_preview_configuration()) + del cam + recalibrate_utils.recreate_camera_manager() + print(f"Opening camera with tuning['version'] = {bad_tuning['version']}") + with pytest.raises(IndexError): + # The bad version should cause a problem + cam = Picamera2(tuning=bad_tuning) + print_tuning() + print("Success (not expected)!") + del cam + recalibrate_utils.recreate_camera_manager() + with Picamera2(tuning=default_tuning) as cam: + # Reload the camera with working tuning, or it will stop responding + # and fail future tests + pass + del cam + + +@pytest.mark.filterwarnings("ignore: Exception ignored") +def test_bad_tuning_after_good_tuning_noconfigure(): + _test_bad_tuning_after_good_tuning(False) + + +@pytest.mark.filterwarnings("ignore: Exception ignored") +def test_bad_tuning_after_good_tuning_configure(): + _test_bad_tuning_after_good_tuning(True) + + +@pytest.mark.filterwarnings("ignore: Exception ignored") +def test_bad_tuning_after_good_tuning_noconfigure2(): + _test_bad_tuning_after_good_tuning(False) + + +@pytest.mark.filterwarnings("ignore: Exception ignored") +def test_bad_tuning_after_good_tuning_configure2(): + _test_bad_tuning_after_good_tuning(True) diff --git a/ofm_config_full.json b/ofm_config_full.json index a73e4e40..bf9e29cc 100644 --- a/ofm_config_full.json +++ b/ofm_config_full.json @@ -1,6 +1,6 @@ { "things": { - "/camera/": "labthings_picamera2.thing:StreamingPiCamera2", + "/camera/": "openflexure_microscope_server.things.camera.picamera:StreamingPiCamera2", "/stage/": "labthings_sangaboard:SangaboardThing", "/auto_recentre_stage/": "openflexure_microscope_server.things.auto_recentre_stage:RecentringThing", "/autofocus/": "openflexure_microscope_server.things.autofocus:AutofocusThing", diff --git a/pyproject.toml b/pyproject.toml index bfa7b2c5..5b00f07b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ dev = [ "matplotlib~=3.10" ] pi = [ - "labthings-picamera2 == 0.0.2", + "picamera2~=0.3.12", ] [project.scripts] diff --git a/src/openflexure_microscope_server/things/camera/__init__.py b/src/openflexure_microscope_server/things/camera/__init__.py index c1fb9a17..71c7406d 100644 --- a/src/openflexure_microscope_server/things/camera/__init__.py +++ b/src/openflexure_microscope_server/things/camera/__init__.py @@ -10,6 +10,8 @@ from __future__ import annotations import logging from typing import Literal, Protocol, runtime_checkable, Optional +from pydantic import RootModel + from labthings_fastapi.thing import Thing from labthings_fastapi.decorators import thing_action, thing_property from labthings_fastapi.dependencies.metadata import GetThingStates @@ -24,129 +26,14 @@ from labthings_fastapi.types.numpy import NDArray class JPEGBlob(Blob): media_type: str = "image/jpeg" +class PNGBlob(Blob): + media_type: str = "image/png" -@runtime_checkable -class CameraProtocol(Protocol): - """A Thing representing a camera""" - - def __enter__(self) -> None: ... - - def __exit__(self, _exc_type, _exc_value, _traceback) -> None: ... - - @property - def stream_active(self) -> bool: - "Whether the MJPEG stream is active" - ... - - def snap_image(self) -> NDArray: - """Acquire one image from the camera.""" - ... - - def capture_array( - self, - stream_name: Literal["main", "lores", "raw", "full"] = "main", - wait: Optional[float] = 5, - ) -> NDArray: ... - - def capture_jpeg( - self, - metadata_getter: GetThingStates, - resolution: Literal["lores", "main", "full"] = "main", - wait: Optional[float] = 5, - ) -> JPEGBlob: - """Acquire one image from the camera and return as a JPEG blob""" - ... - - def grab_jpeg( - self, - portal: BlockingPortal, - stream_name: Literal["main", "lores"] = "main", - ) -> JPEGBlob: - """Acquire one image from the preview stream and return as an array - - This differs from `capture_jpeg` in that it does not pause the MJPEG - preview stream. Instead, we simply return the next frame from that - stream (either "main" for the preview stream, or "lores" for the low - resolution preview). No metadata is returned. - """ - ... - - def grab_jpeg_size( - self, - portal: BlockingPortal, - stream_name: Literal["main", "lores"] = "main", - ) -> int: - """Acquire one image from the preview stream and return its size""" - ... - - def start_streaming(self, main_resolution, buffer_count) -> None: - """Start (or stop and restart) the camera with the given resolution - for the main stream, and buffer_count number of images in the buffer""" - ... - - def capture_image(self, stream_name, wait): - """Capture a PIL image from stream stream_name with timeout wait""" - ... - +class ArrayModel(RootModel): + """A model for an array""" + root: NDArray class BaseCamera(Thing): - """A Thing representing a camera - - This is a concrete base class for `Thing`s implementing the `CameraProtocol`. - It provides the stream descriptors and actions to grab from the stream. - """ - - mjpeg_stream = MJPEGStreamDescriptor() - lores_mjpeg_stream = MJPEGStreamDescriptor() - - @thing_action - def snap_image(self) -> NDArray: - """Acquire one image from the camera. - - This action cannot run if the camera is in use by a background thread, for - example if a preview stream is running. - """ - return self.capture_array() - - @thing_action - def grab_jpeg( - self, - portal: BlockingPortal, - stream_name: Literal["main", "lores"] = "main", - ) -> JPEGBlob: - """Acquire one image from the preview stream and return as an array - - This differs from `capture_jpeg` in that it does not pause the MJPEG - preview stream. Instead, we simply return the next frame from that - stream (either "main" for the preview stream, or "lores" for the low - resolution preview). No metadata is returned. - """ - logging.info( - f"StreamingPiCamera2.grab_jpeg(stream_name={stream_name}) starting" - ) - stream = ( - self.lores_mjpeg_stream if stream_name == "lores" else self.mjpeg_stream - ) - frame = portal.call(stream.grab_frame) - logging.info( - f"StreamingPiCamera2.grab_jpeg(stream_name={stream_name}) got frame" - ) - return JPEGBlob.from_bytes(frame) - - @thing_action - def grab_jpeg_size( - self, - portal: BlockingPortal, - stream_name: Literal["main", "lores"] = "main", - ) -> int: - """Acquire one image from the preview stream and return its size""" - stream = ( - self.lores_mjpeg_stream if stream_name == "lores" else self.mjpeg_stream - ) - return portal.call(stream.next_frame_size) - - -class CameraStub(BaseCamera): """A stub for a camera, to allow dependencies @@ -158,8 +45,11 @@ class CameraStub(BaseCamera): This stub class should be used for dependencies on the CameraProtocol. """ + mjpeg_stream = MJPEGStreamDescriptor() + lores_mjpeg_stream = MJPEGStreamDescriptor() + def __enter__(self) -> None: - raise NotImplementedError("Cameras must not inherit from CameraStub") + raise NotImplementedError("CameraThings must define their own __enter__ method") def __exit__(self, _exc_type, _exc_value, _traceback) -> None: raise NotImplementedError("Cameras must not inherit from CameraStub") @@ -169,11 +59,6 @@ class CameraStub(BaseCamera): "Whether the MJPEG stream is active" raise NotImplementedError("Cameras must not inherit from CameraStub") - @thing_action - def snap_image(self) -> NDArray: - """Acquire one image from the camera.""" - raise NotImplementedError("Cameras must not inherit from CameraStub") - @thing_action def capture_array( self, @@ -204,5 +89,5 @@ class CameraStub(BaseCamera): raise NotImplementedError("Cameras must not inherit from CameraStub") -CameraDependency = direct_thing_client_dependency(CameraStub, "/camera/") -RawCameraDependency = raw_thing_dependency(CameraProtocol) +CameraDependency = direct_thing_client_dependency(BaseCamera, "/camera/") +RawCameraDependency = raw_thing_dependency(BaseCamera) diff --git a/src/openflexure_microscope_server/things/camera/opencv.py b/src/openflexure_microscope_server/things/camera/opencv.py index abb50252..2278e2bb 100644 --- a/src/openflexure_microscope_server/things/camera/opencv.py +++ b/src/openflexure_microscope_server/things/camera/opencv.py @@ -53,9 +53,6 @@ class OpenCVCamera(BaseCamera): return self._capture_thread.is_alive() return False - mjpeg_stream = MJPEGStreamDescriptor() - lores_mjpeg_stream = MJPEGStreamDescriptor() - def _capture_frames(self): portal = get_blocking_portal(self) while self._capture_enabled: diff --git a/src/openflexure_microscope_server/things/camera/picamera.py b/src/openflexure_microscope_server/things/camera/picamera.py new file mode 100644 index 00000000..81434918 --- /dev/null +++ b/src/openflexure_microscope_server/things/camera/picamera.py @@ -0,0 +1,868 @@ +from __future__ import annotations +from dataclasses import dataclass +from datetime import datetime +import io +import json +import logging +import os +import tempfile +import time +from tempfile import TemporaryDirectory + +from pydantic import BaseModel, BeforeValidator + +from labthings_fastapi.descriptors.property import PropertyDescriptor +from labthings_fastapi.thing import Thing +from labthings_fastapi.decorators import thing_action, thing_property +from labthings_fastapi.outputs.mjpeg_stream import MJPEGStream +from labthings_fastapi.utilities import get_blocking_portal +from labthings_fastapi.types.numpy import NDArray +from labthings_fastapi.dependencies.metadata import GetThingStates +from labthings_fastapi.dependencies.blocking_portal import BlockingPortal +from typing import Annotated, Any, Iterator, Literal, Mapping, Optional, Self +from contextlib import contextmanager +import piexif +from scipy.ndimage import zoom +from scipy.interpolate import interp1d +from PIL import Image +from threading import RLock +import picamera2 +from picamera2 import Picamera2 +from picamera2.encoders import MJPEGEncoder +from picamera2.outputs import Output +import numpy as np +from . import picamera_recalibrate_utils as recalibrate_utils + +from . import BaseCamera, JPEGBlob, PNGBlob, ArrayModel + + +class PicameraControl(PropertyDescriptor): + def __init__( + self, control_name: str, model: type = float, description: Optional[str] = None + ): + """A property descriptor controlling a picamera control""" + PropertyDescriptor.__init__( + self, model, observable=False, description=description + ) + self.control_name = control_name + + def _getter(self, obj: StreamingPiCamera2): + with obj.picamera() as cam: + ret = cam.capture_metadata()[self.control_name] + return ret + + def _setter(self, obj: StreamingPiCamera2, value: Any): + with obj.picamera() as cam: + cam.set_controls({self.control_name: value}) + + +class PicameraStreamOutput(Output): + """An Output class that sends frames to a stream""" + + def __init__(self, stream: MJPEGStream, portal: BlockingPortal): + """Create an output that puts frames in an MJPEGStream + + We need to pass the stream object, and also the blocking portal, because + new frame notifications happen in the anyio event loop and frames are + sent from a thread. The blocking portal enables thread-to-async + communication. + """ + Output.__init__(self) + self.stream = stream + self.portal = portal + + def outputframe( + self, frame, _keyframe=True, _timestamp=None, _packet=None, _audio=False + ): + """Add a frame to the stream's ringbuffer""" + self.stream.add_frame(frame, self.portal) + + +class SensorMode(BaseModel): + unpacked: str + bit_depth: int + size: tuple[int, int] + fps: float + crop_limits: tuple[int, int, int, int] + exposure_limits: tuple[Optional[int], Optional[int], Optional[int]] + format: Annotated[str, BeforeValidator(repr)] + + +class SensorModeSelector(BaseModel): + output_size: tuple[int, int] + bit_depth: int + + +class LensShading(BaseModel): + luminance: list[list[float]] + Cr: list[list[float]] + Cb: list[list[float]] + + +class StreamingPiCamera2(BaseCamera): + """A Thing that represents an OpenCV camera""" + + def __init__(self, camera_num: int = 0): + self.camera_num = camera_num + self.camera_configs: dict[str, dict] = {} + # NB persistent controls will be updated with settings, in __enter__. + self.persistent_controls = { + "AeEnable": False, + "AnalogueGain": 1.0, + "AwbEnable": False, + "Brightness": 0, + "ColourGains": (1, 1), + "Contrast": 1, + "ExposureTime": 0, + "Saturation": 1, + "Sharpness": 1, + } + self.persistent_control_tolerances = { + "ExposureTime": 30, + } + + def update_persistent_controls(self, discard_frames: int = 1): + """Update the persistent controls dict from the camera + + Query the camera and update the value of `persistent_controls` to + match the current state of the camera. + + There is a work-around here, that will suppress small updates. There + appears to be a bug in the camera code that causes a slight drift in + `ExposureTime` each time the camera is reinitialised: this can + add up over time, particularly if the camera is reconfigured many + times. To get around this, we look in `self.persistent_control_tolerances` + and only update `self.persistent_controls` if the change is greater than + this tolerance. + """ + with self.picamera() as cam: + for i in range(discard_frames): + # Discard frames, so we know our data is fresh + cam.capture_metadata() + for k, v in cam.capture_metadata().items(): + if k in self.persistent_controls: + if k in self.persistent_control_tolerances: + if ( + np.abs(self.persistent_controls[k] - v) + < self.persistent_control_tolerances[k] + ): + logging.debug( + f"Ignoring a small change in persistent control {k}" + f"from {self.persistent_controls[k]} to {v}" + "while updating persistent controls." + ) + continue # Ignore small changes, to avoid drift + self.persistent_controls[k] = v + self.thing_settings.update( + self.persistent_controls + ) # TODO: Is this saving to the wrong place? + + def settings_to_persistent_controls(self): + """Update the persistent controls dict from the settings dict + + NB this must be called **after** self.thing_settings is initialised, + i.e. during or after `__enter__`. + """ + try: + pc = self.thing_settings["persistent_controls"] + except KeyError: + return # If there are no saved settings, use defaults + for k in self.persistent_controls: + try: + self.persistent_controls[k] = pc[k] + except KeyError: + pass # If controls are missing, leave at default + + stream_resolution = PropertyDescriptor( + tuple[int, int], + initial_value=(820, 616), + description="Resolution to use for the MJPEG stream", + ) + mjpeg_bitrate = PropertyDescriptor( + Optional[int], + initial_value=100000000, + description="Bitrate for MJPEG stream (None for default)", + ) + + @mjpeg_bitrate.setter + def mjpeg_bitrate(self, value: Optional[int]): + """Restart the stream when we set the bitrate""" + with self.picamera(pause_stream=True): + pass # just pausing and restarting the stream is enough. + + stream_active = PropertyDescriptor( + bool, + initial_value=False, + description="Whether the MJPEG stream is active", + observable=True, + readonly=True, + ) + + analogue_gain = PicameraControl("AnalogueGain", float) + colour_gains = PicameraControl("ColourGains", tuple[float, float]) + + exposure_time = PicameraControl( + "ExposureTime", int, description="The exposure time in microseconds" + ) + + _sensor_modes = None + + @thing_property + def sensor_modes(self) -> list[SensorMode]: + """All the available modes the current sensor supports""" + if not self._sensor_modes: + with self.picamera() as cam: + self._sensor_modes = cam.sensor_modes + return self._sensor_modes + + @thing_property + def sensor_mode(self) -> Optional[SensorModeSelector]: + """The intended sensor mode of the camera""" + return self.thing_settings["sensor_mode"] + + @sensor_mode.setter + def sensor_mode(self, new_mode: Optional[SensorModeSelector]): + """Change the sensor mode used""" + if isinstance(new_mode, SensorModeSelector): + new_mode = new_mode.model_dump() + with self.picamera(pause_stream=True): + self.thing_settings["sensor_mode"] = new_mode + + @thing_property + def sensor_resolution(self) -> tuple[int, int]: + """The native resolution of the camera's sensor""" + with self.picamera() as cam: + return cam.sensor_resolution + + tuning = PropertyDescriptor(Optional[dict], None, readonly=True) + + def settings_to_properties(self): + """Set the values of properties based on the settings dict""" + try: + props = self.thing_settings["properties"] + except KeyError: + return + for k, v in props.items(): + setattr(self, k, v) + + def properties_to_settings(self): + """Save certain properties to the settings dictionary""" + props = {} + for k in ["mjpeg_bitrate", "stream_resolution"]: + props[k] = getattr(self, k) + self.thing_settings["properties"] = props + + def initialise_tuning(self): + """Read the tuning from the settings, or load default tuning + + NB this relies on `self.thing_settings` and `self.default_tuning` + so will fail if it's run before those are populated in `__enter__`. + """ + if "tuning" in self.thing_settings: + # TODO: should this be a separate file? + self.tuning = self.thing_settings["tuning"].dict + else: + logging.info("Did not find tuning in settings, reading from camera...") + self.tuning = self.default_tuning + + def initialise_picamera(self): + """Acquire the picamera device and store it as `self._picamera`""" + if hasattr(self, "_picamera_lock"): + # Don't close the camera if it's in use + self._picamera_lock.acquire() + with tempfile.NamedTemporaryFile("w") as tuning_file: + # This duplicates logic in `Picamera2.__init__` to provide a tuning file + # that will be read when the camera system initialises. + # This is a necessary work-around until `picamera2` better supports + # reinitialisation of the camera with new tuning. + json.dump(self.tuning, tuning_file) + tuning_file.flush() # but leave it open as closing it will delete it + os.environ["LIBCAMERA_RPI_TUNING_FILE"] = tuning_file.name + # NB even though we've put the tuning file in the environment, we will + # need to specify the filename in the `Picamera2` initialiser as otherwise + # it will be overwritten with None. + if hasattr(self, "_picamera") and self._picamera: + print("Closing picamera object for reinitialisation") + logging.info( + "Camera object already exists, closing for reinitialisation" + ) + self._picamera.close() + print("closed, deleting picamera") + del self._picamera + recalibrate_utils.recreate_camera_manager() + print("[re]creating Picamera2 object") + self._picamera = picamera2.Picamera2( + camera_num=self.camera_num, + tuning=self.tuning, + ) + self._picamera_lock = RLock() + + def __enter__(self): + self.populate_default_tuning() + self.initialise_tuning() + self.initialise_picamera() + self.sensor_modes + self.settings_to_persistent_controls() + self.settings_to_properties() + self.start_streaming() + return self + + @contextmanager + def picamera(self, pause_stream=False) -> Iterator[Picamera2]: + """Return the underlying `Picamera2` instance, optionally pausing the stream. + + If pause_stream is True (default is False), we will stop the MJPEG stream + before yielding control of the camera, and restart afterwards. If you make + changes to the camera settings, these may be ignored when the stream is + restarted: you may nened to call `update_persistent_controls()` to ensure + your changes persist after the stream restarts. + """ + already_streaming = self.stream_active + with self._picamera_lock: + if pause_stream and already_streaming: + self.update_persistent_controls() + self.stop_streaming(stop_web_stream=False) + try: + yield self._picamera + finally: + if pause_stream and already_streaming: + self.start_streaming() + + def populate_default_tuning(self): + """Sensor modes are enumerated and stored, once, on start-up (`__enter__`). + + This opens and closes the camera - must be run before the camera is + initialised. + """ + logging.info("Starting & reconfiguring camera to populate sensor_modes.") + with Picamera2(camera_num=self.camera_num) as cam: + self.default_tuning = recalibrate_utils.load_default_tuning(cam) + logging.info("Done reading sensor modes & default tuning.") + + def __exit__(self, exc_type, exc_value, traceback): + # Allow key controls to persist across restarts + self.update_persistent_controls() + self.thing_settings["persistent_controls"] = self.persistent_controls + self.thing_settings["tuning"] = self.tuning + self.properties_to_settings() + self.thing_settings.write_to_file() + # Shut down the camera + self.stop_streaming() + with self.picamera() as cam: + cam.close() + del self._picamera + + @thing_action + def start_streaming( + self, main_resolution: tuple[int, int] = (820, 616), buffer_count: int = 6 + ) -> None: + """ + Start the MJPEG stream + + Sets the camera resolutions based on input parameters, and sets the low-res + resolution to (320, 240). Note: (320, 240) is a standard from the Pi Camera + manual. + + Create two streams: + - `lores_mjpeg_stream` for autofocus at low-res resolution + - `mjpeg_stream` for preview. This is the `main_resolution` if this is less + than (1280, 960), or the low-res resolution if above. This allows for + high resolution capture without streaming high resolution video. + + main_resolution: the resolution for the main configuration. Defaults to + (820, 616), 1/4 sensor size. + buffer_count: the number of frames to hold in the buffer. Higher uses more memory, + lower may cause dropped frames. Defaults to 6. + """ + with self.picamera() as picam: + # TODO: Filip: can we use the lores output to keep preview stream going + # while recording? According to picamera2 docs 4.2.1.6 this should work + try: + if picam.started: + picam.stop() + picam.stop_encoder() # make sure there are no other encoders going + stream_config = picam.create_video_configuration( + main={"size": main_resolution}, + lores={"size": (320, 240), "format": "YUV420"}, + sensor=self.thing_settings.get("sensor_mode", None), + controls=self.persistent_controls, + ) + # Set buffer count - can't be negative + stream_config["buffer_count"] = buffer_count + picam.configure(stream_config) + logging.info("Starting picamera MJPEG stream...") + stream_name = "lores" if main_resolution[0] > 1280 else "main" + picam.start_recording( + MJPEGEncoder(self.mjpeg_bitrate), + PicameraStreamOutput( + self.mjpeg_stream, + get_blocking_portal(self), + ), + name=stream_name, + ) + picam.start_encoder( + MJPEGEncoder(100000000), + PicameraStreamOutput( + self.lores_mjpeg_stream, + get_blocking_portal(self), + ), + name="lores", + ) + except Exception as e: + logging.exception("Error while starting preview: {e}") + logging.exception(e) + else: + self.stream_active = True + logging.debug( + "Started MJPEG stream at %s on port %s", self.stream_resolution, 1 + ) + + @thing_action + def stop_streaming(self, stop_web_stream=True) -> None: + """ + Stop the MJPEG stream + """ + with self.picamera() as picam: + try: + picam.stop_recording() # This should also stop the extra lores encoder + except Exception as e: + logging.info("Stopping recording failed") + logging.exception(e) + else: + self.stream_active = False + if stop_web_stream: + self.mjpeg_stream.stop() + self.lores_mjpeg_stream.stop() + logging.info("Stopped MJPEG stream.") + + # Increase the resolution for taking an image + time.sleep( + 0.2 + ) # Sprinkled a sleep to prevent camera getting confused by rapid commands + + @thing_action + def capture_image( + self, + stream_name: Literal["main", "lores", "raw", "full"] = "main", + wait: Optional[float] = 0.9, + ): + """Acquire one image from the camera. + + Return it as a PIL Image + + stream_name: (Optional) The PiCamera2 stream to use, should be one of ["main", "lores", "raw", "full"]. Default = "main" + wait: (Optional, float) Set a timeout in seconds. + A TimeoutError is raised if this time is exceeded during capture. + Default = 0.9s, lower than the 1s timeout default in picamera yaml settings + """ + with self.picamera() as cam: + return cam.capture_image(stream_name, wait=wait) + + @thing_action + def capture_array( + self, + stream_name: Literal["main", "lores", "raw", "full"] = "main", + wait: Optional[float] = 0.9, + ) -> ArrayModel: + """Acquire one image from the camera and return as an array + + This function will produce a nested list containing an uncompressed RGB image. + It's likely to be highly inefficient - raw and/or uncompressed captures using + binary image formats will be added in due course. + + stream_name: (Optional) The PiCamera2 stream to use, should be one of ["main", "lores", "raw", "full"]. Default = "main" + wait: (Optional, float) Set a timeout in seconds. + A TimeoutError is raised if this time is exceeded during capture. + Default = 0.9s, lower than the 1s timeout default in picamera yaml settings + """ + + # This was slower than capture_image for our use case, but directly returning + # an image as an array is still a useful feature + if stream_name == "full": + with self.picamera(pause_stream=True) as picam2: + capture_config = picam2.create_still_configuration() + return picam2.switch_mode_and_capture_array(capture_config, wait=wait) + with self.picamera() as cam: + return cam.capture_array(stream_name, wait=wait) + + @thing_property + def camera_configuration(self) -> Mapping: + """The "configuration" dictionary of the picamera2 object + + The "configuration" sets the resolution and format of the camera's streams. + Together with the "tuning" it determines how the sensor is configured and + how the data is processed. + + Note that the configuration may be modified when taking still images, and + this property refers to whatever configuration is currently in force - + usually the one used for the preview stream. + """ + with self.picamera() as cam: + return cam.camera_configuration() + + @thing_action + def capture_jpeg( + self, + metadata_getter: GetThingStates, + resolution: Literal["lores", "main", "full"] = "main", + wait: Optional[float] = 0.9, + ) -> JPEGBlob: + """Acquire one image from the camera as a JPEG + + The JPEG will be acquired using `Picamera2.capture_file`. If the + `resolution` parameter is `main` or `lores`, it will be captured + from the main preview stream, or the low-res preview stream, + respectively. This means the camera won't be reconfigured, and + the stream will not pause (though it may miss one frame). + + If `full` resolution is requested, we will briefly pause the + MJPEG stream and reconfigure the camera to capture a full + resolution image. + + wait: (Optional, float) Set a timeout in seconds. + A TimeoutError is raised if this time is exceeded during capture. + Default = 0.9s, lower than the 1s timeout default in picamera yaml settings + + Note that this always uses the image processing pipeline - to + bypass this, you must use a raw capture. + """ + fname = datetime.now().strftime("%Y-%m-%d-%H%M%S.jpeg") + folder = TemporaryDirectory() + path = os.path.join(folder.name, fname) + config = self.camera_configuration + # Low-res and main streams are running already - so we don't need + # to reconfigure for these + if resolution in ("lores", "main") and config[resolution]: + with self.picamera() as cam: + cam.capture_file(path, name=resolution, format="jpeg", wait=wait) + else: + if resolution != "full": + logging.warning( + f"There was no {resolution} stream, capturing full resolution" + ) + with self.picamera(pause_stream=True) as cam: + logging.info("Reconfiguring camera for full resolution capture") + cam.configure(cam.create_still_configuration()) + cam.start() + cam.options["quality"] = 95 + logging.info("capturing") + cam.capture_file(path, name="main", format="jpeg", wait=wait) + logging.info("done") + # After the file is written, add metadata about the current Things + exif_dict = piexif.load(path) + exif_dict["Exif"][piexif.ExifIFD.UserComment] = json.dumps( + metadata_getter() + ).encode("utf-8") + piexif.insert(piexif.dump(exif_dict), path) + return JPEGBlob.from_temporary_directory(folder, fname) + + @thing_action + def grab_jpeg( + self, + portal: BlockingPortal, + stream_name: Literal["main", "lores"] = "main", + ) -> JPEGBlob: + """Acquire one image from the preview stream and return as an array + + This differs from `capture_jpeg` in that it does not pause the MJPEG + preview stream. Instead, we simply return the next frame from that + stream (either "main" for the preview stream, or "lores" for the low + resolution preview). No metadata is returned. + """ + logging.debug( + f"StreamingPiCamera2.grab_jpeg(stream_name={stream_name}) starting" + ) + stream = ( + self.lores_mjpeg_stream if stream_name == "lores" else self.mjpeg_stream + ) + frame = portal.call(stream.grab_frame) + logging.debug( + f"StreamingPiCamera2.grab_jpeg(stream_name={stream_name}) got frame" + ) + return JPEGBlob.from_bytes(frame) + + @thing_action + def grab_jpeg_size( + self, + portal: BlockingPortal, + stream_name: Literal["main", "lores"] = "main", + ) -> int: + """Acquire one image from the preview stream and return its size""" + stream = ( + self.lores_mjpeg_stream if stream_name == "lores" else self.mjpeg_stream + ) + return portal.call(stream.next_frame_size) + + @thing_property + def exposure(self) -> float: + """An alias for `exposure_time` to fit the micromanager API""" + return self.exposure_time + + @exposure.setter # type: ignore + def exposure(self, value): + self.exposure_time = value + + @thing_property + def capture_metadata(self) -> dict: + """Return the metadata from the camera""" + with self.picamera() as cam: + return cam.capture_metadata() + + @thing_action + def auto_expose_from_minimum( + self, + target_white_level: int = 700, + percentile: float = 99.9, + ): + """Adjust exposure to hit the target white level + + Starting from the minimum exposure, we gradually increase exposure until + we hit the specified white level. We use a percentile rather than the + maximum, in order to be robust to a small number of noisy/bright pixels. + """ + with self.picamera(pause_stream=True) as cam: + recalibrate_utils.adjust_shutter_and_gain_from_raw( + cam, + target_white_level=target_white_level, + percentile=percentile, + ) + self.update_persistent_controls() + + @thing_action + def calibrate_white_balance( + self, + method: Literal["percentile", "centre"] = "centre", + luminance_power: float = 1.0, + ): + """Correct the white balance of the image + + This calibration requires a neutral image, such that the 99th centile + of each colour channel should correspond to white. We calculate the + centiles and use this to set the colour gains. This is done on the raw + image with the lens shading correction applied, which should mean + that the image is uniform, rather than weighted towards the centre. + + If `method` is `"centre"`, we will correct the mean of the central 10% + of the image. + """ + with self.picamera(pause_stream=True) as cam: + if self.lens_shading_is_static: + lst: LensShading = self.lens_shading_tables + recalibrate_utils.adjust_white_balance_from_raw( + cam, + percentile=99, + luminance=lst.luminance, + Cr=lst.Cr, + Cb=lst.Cb, + luminance_power=luminance_power, + method=method, + ) + else: + recalibrate_utils.adjust_white_balance_from_raw( + cam, percentile=99, method=method + ) + self.update_persistent_controls() + + @thing_action + def calibrate_lens_shading(self): + """Take an image and use it for flat-field correction. + + This method requires an empty (i.e. bright) field of view. It will take + a raw image and effectively divide every subsequent image by the current + one. This uses the camera's "tuning" file to correct the preview and + the processed images. It should not affect raw images. + """ + with self.picamera(pause_stream=True) as cam: + L, Cr, Cb = recalibrate_utils.lst_from_camera(cam) + recalibrate_utils.set_static_lst(self.tuning, L, Cr, Cb) + self.initialise_picamera() + + @thing_property + def colour_correction_matrix( + self, + ) -> tuple[float, float, float, float, float, float, float, float, float]: + """An alias for `colour_correction_matrix` to fit the micromanager API""" + return self.thing_settings.get( + "colour_correction_matrix", + tuple(recalibrate_utils.get_static_ccm(self.tuning)[0]["ccm"]), + ) + + @colour_correction_matrix.setter # type: ignore + def colour_correction_matrix(self, value): + self.thing_settings["colour_correction_matrix"] = value + self.calibrate_colour_correction(value) + + @thing_action + def reset_ccm(self): + """Overwrite the colour correction matrix in camera tuning with default values from the documentation""" + c = [ + 1.80439, + -0.73699, + -0.06739, + -0.36073, + 1.83327, + -0.47255, + -0.08378, + -0.56403, + 1.64781, + ] + self.colour_correction_matrix = c + + @thing_action + def calibrate_colour_correction(self, c: tuple): + """Overwrite the colour correction matrix in camera tuning""" + with self.picamera(pause_stream=True): + recalibrate_utils.set_static_ccm(self.tuning, c) + self.initialise_picamera() + + @thing_action + def set_static_green_equalisation(self, offset: int = 65535): + """Set the green equalisation to a static value. + + Green equalisation avoids the debayering algorithm becoming confused + by the two green channels having different values, which is a problem + when the chief ray angle isn't what the sensor was designed for, and + that's the case in e.g. a microscope using camera module v2. + + A value of 0 here does nothing, a value of 65535 is maximum correction. + """ + with self.picamera(pause_stream=True): + recalibrate_utils.set_static_geq(self.tuning, offset) + self.initialise_picamera() + + @thing_action + def full_auto_calibrate(self): + """Perform a full auto-calibration + + This function will call the other calibration actions in sequence: + + * `flat_lens_shading` to disable flat-field + * `auto_expose_from_minimum` + * `set_static_green_equalisation` to set geq offset to max + * `calibrate_lens_shading` + * `calibrate_white_balance` + """ + self.flat_lens_shading() + self.auto_expose_from_minimum() + self.set_static_green_equalisation() + self.calibrate_lens_shading() + self.calibrate_white_balance() + + @thing_action + def flat_lens_shading(self): + """Disable flat-field correction + + This method will set a completely flat lens shading table. It is not the + same as the default behaviour, which is to use an adaptive lens shading + table. + """ + with self.picamera(pause_stream=True): + f = np.ones((12, 16)) + recalibrate_utils.set_static_lst(self.tuning, f, f, f) + self.initialise_picamera() + + @thing_property + def lens_shading_tables(self) -> Optional[LensShading]: + """The current lens shading (i.e. flat-field correction) + + This returns the current lens shading correction, as three 2D lists + each with dimensions 16x12. This assumes that we are using a static + lens shading table - if adaptive control is enabled, or if there + are multiple LSTs in use for different colour temperatures, + we return a null value to avoid confusion. + """ + if not self.lens_shading_is_static: + return None + alsc = Picamera2.find_tuning_algo(self.tuning, "rpi.alsc") + if any(len(alsc[f"calibrations_C{c}"]) != 1 for c in ("r", "b")): + return None + + def reshape_lst(lin: list[float]) -> list[list[float]]: + w, h = 16, 12 + return [lin[w * i : w * (i + 1)] for i in range(h)] + + return LensShading( + luminance=reshape_lst(alsc["luminance_lut"]), + Cr=reshape_lst(alsc["calibrations_Cr"][0]["table"]), + Cb=reshape_lst(alsc["calibrations_Cb"][0]["table"]), + ) + + @lens_shading_tables.setter + def lens_shading_tables(self, lst: LensShading) -> None: + """Set the lens shading tables""" + with self.picamera(pause_stream=True): + recalibrate_utils.set_static_lst( + self.tuning, + luminance=lst.luminance, + cr=lst.Cr, + cb=lst.Cb, + ) + self.initialise_picamera() + + def correct_colour_gains_for_lens_shading( + self, colour_gains: tuple[float, float] + ) -> tuple[float, float]: + """Correct white balance gains for the effect of lens shading + + The white balance algorithm we use assumes the brightest pixels + should be white, and that the only thing affecting the colour of + said pixels is the `colour_gains`. + + The lens shading correction is normalised such that the *minimum* + gain in the `Cr` and `Cb` channels is 1. The white balance + assumption above requires that the gain for the brightest pixels + is 1. The solution might be that, when calibrating, we note which + pixels are brightest (usually the centre) and explicitly use + the LST values for there. However, for now I will assume that we + need to normalise by the **maximum** of the `Cr` and `Cb` + channels, which is correct the majority of the time. + """ + if not self.lens_shading_is_static: + return colour_gains + lst = self.lens_shading_tables + # The Cr and Cb corrections are normalised to have a minimum of 1, + # but the white balance algorithm normalises the brightest pixels + # to be white, assuming the brightest pixels have equal gain from + # the LST. + gain_r, gain_b = colour_gains + return ( + float(gain_r / np.max(lst.Cr)), + float(gain_b / np.max(lst.Cb)), + ) + + @thing_action + def flat_lens_shading_chrominance(self): + """Disable flat-field correction + + This method will set the chrominance of the lens shading table to be + flat, i.e. we'll correct vignetting of intensity, but not any change in + colour across the image. + """ + with self.picamera(pause_stream=True): + alsc = Picamera2.find_tuning_algo(self.tuning, "rpi.alsc") + luminance = alsc["luminance_lut"] + flat = np.ones((12, 16)) + recalibrate_utils.set_static_lst(self.tuning, luminance, flat, flat) + self.initialise_picamera() + + @thing_action + def reset_lens_shading(self): + """Revert to default lens shading settings + + This method will restore the default "adaptive" lens shading method used + by the Raspberry Pi camera. + """ + with self.picamera(pause_stream=True): + recalibrate_utils.copy_alsc_section(self.default_tuning, self.tuning) + self.initialise_picamera() + + @thing_property + def lens_shading_is_static(self) -> bool: + """Whether the lens shading is static + + This property is true if the lens shading correction has been set to use + a static table (i.e. the number of automatic correction iterations is zero). + The default LST is not static, but all the calibration controls will set it + to be static (except "reset") + """ + return recalibrate_utils.lst_is_static(self.tuning) diff --git a/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py b/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py new file mode 100644 index 00000000..0370ba33 --- /dev/null +++ b/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py @@ -0,0 +1,561 @@ +""" +Functions to set up a Raspberry Pi Camera v2 for scientific use + +This module provides slower, simpler functions to set the +gain, exposure, and white balance of a Raspberry Pi camera, using +the `picamera2` Python library. It's mostly used by the OpenFlexure +Microscope, though it deliberately has no hard dependencies on +said software, so that it's useful on its own. + +There are three main calibration steps: + +* Setting exposure time and gain to get a reasonably bright + image. +* Fixing the white balance to get a neutral image +* Taking a uniform white image and using it to calibrate + the Lens Shading Table + +The most reliable way to do this, avoiding any issues relating +to "memory" or nonlinearities in the camera's image processing +pipeline, is to use raw images. This is quite slow, but very +reliable. The three steps above can be accomplished by: + +``` +picamera = picamera2.Picamera2() + +adjust_shutter_and_gain_from_raw(picamera) +adjust_white_balance_from_raw(picamera) +lst = lst_from_camera(picamera) +picamera.lens_shading_table = lst +``` +""" + +from __future__ import annotations +import gc +import logging +import time +from typing import List, Literal, Optional, Tuple +from pydantic import BaseModel +import numpy as np +from scipy.ndimage import zoom + +from picamera2 import Picamera2 +import picamera2 + + +def load_default_tuning(cam: Picamera2) -> dict: + """Load the default tuning file for the camera + + This will open and close the camera to determine its model. If you are + using a model that's supported by `picamera2` it should have a tuning + file built in. If not, this will probably crash with an error. + + Error handling for unsupported cameras is not something we are likely + to test in the short term. + """ + cp = cam.camera_properties + fname = f"{cp['Model']}.json" + try: + return cam.load_tuning_file(fname) + except RuntimeError: + dir = "/usr/share/libcamera/ipa/raspberrypi" # from picamera2 v0.3.9 + # The directory above has been removed from the search path, which I + # find odd - as that's where the files currently are on a default + # Raspbian image. This may need updating if the files have moved + # in future updates to the system libcamera package + return cam.load_tuning_file(fname, dir=dir) + + +def set_minimum_exposure(camera: Picamera2): + """Enable manual exposure, with low gain and shutter speed + + We set exposure mode to manual, analog and digital gain + to 1, and shutter speed to the minimum (8us for Pi Camera v2) + NB ISO is left at auto, because this is needed for the gains + to be set correctly. + """ + camera.set_controls({"AeEnable": False, "AnalogueGain": 1, "ExposureTime": 1}) + # camera.iso = 0 # We must set ISO=0 (auto) or we can't set gain + # camera.analog_gain = 1 + # camera.digital_gain = 1 (not configurable) + # Setting the shutter speed to 1us will result in it being set + # to the minimum possible, which is probably 8us for PiCamera v2 + # camera.shutter_speed = 1 + time.sleep(0.5) + + +class ExposureTest(BaseModel): + """Record the results of testing the camera's current exposure settings""" + + level: int + exposure_time: int + analog_gain: float + + +def test_exposure_settings(camera: Picamera2, percentile: float) -> ExposureTest: + """Evaluate current exposure settings using a raw image + + CAMERA SHOULD BE STARTED! + + We will acquire a raw image and calculate the given percentile + of the pixel values. We return a dictionary containing the + percentile (which will be compared to the target), as well as + the camera's shutter and gain values. + """ + camera.capture_array("raw") # controls might not be updated for the first frame? + max_brightness = np.percentile( + channels_from_bayer_array(camera.capture_array("raw")), + percentile, + ) + # The reported brightness can, theoretically, be negative or zero + # because of black level compensation. The line below forces a + # minimum value of 1 which will keep things well-behaved! + if max_brightness < 1: + logging.warning( + f"Measured brightness of {max_brightness}. " + "This should normally be >= 1, and may indicate the " + "camera's black level compensation has gone wrong." + ) + max_brightness = 1 + metadata = camera.capture_metadata() + result = ExposureTest( + level=max_brightness, + exposure_time=int(metadata["ExposureTime"]), + analog_gain=float(metadata["AnalogueGain"]), + ) + logging.info(f"{result.model_dump()}") + return result + + +def check_convergence(test: ExposureTest, target: int, tolerance: float): + """Check whether the brightness is within the specified target range""" + converged = abs(test.level - target) < target * tolerance + return converged + + +def adjust_shutter_and_gain_from_raw( + camera: Picamera2, + target_white_level: int = 700, + max_iterations: int = 20, + tolerance: float = 0.05, + percentile: float = 99.9, +) -> float: + """Adjust exposure and analog gain based on raw images. + + This routine is slow but effective. It uses raw images, so we + are not affected by white balance or digital gain. + + + Arguments: + target_white_level: + The raw, 10-bit value we aim for. The brightest pixels + should be approximately this bright. Maximum possible + is about 900, 700 is reasonable. + max_iterations: + We will terminate once we perform this many iterations, + whether or not we converge. More than 10 shouldn't happen. + tolerance: + How close to the target value we consider "done". Expressed + as a fraction of the ``target_white_level`` so 0.05 means + +/- 5% + percentile: + Rather then use the maximum value for each channel, we + calculate a percentile. This makes us robust to single + pixels that are bright/noisy. 99.9% still picks the top + of the brightness range, but seems much more reliable + than just ``np.max()``. + """ + # TODO: read black level and bit depth from camera? + if target_white_level * (tolerance + 1) >= 959: + raise ValueError( + "The target level is too high - a saturated image would be " + "considered successful. target_white_level * (tolerance + 1) " + "must be less than 959." + ) + + config = camera.create_still_configuration(raw={"format": "SBGGR10"}) + camera.configure(config) + camera.start() + set_minimum_exposure(camera) + + # We start with very low exposure settings and work up + # until either the brightness is high enough, or we can't increase the + # shutter speed any more. + iterations = 0 + while iterations < max_iterations: + test = test_exposure_settings(camera, percentile) + if check_convergence(test, target_white_level, tolerance): + break + iterations += 1 + + # Adjust shutter speed so that the brightness approximates the target + # NB we put a maximum of 8 on this, to stop it increasing too quickly. + new_time = int(test.exposure_time * min(target_white_level / test.level, 8)) + camera.controls.ExposureTime = new_time + camera.controls.AeEnable = False + time.sleep(0.5) + + # Check whether the shutter speed is still going up - if not, we've hit a maximum + if camera.capture_metadata()["ExposureTime"] == test.exposure_time: + logging.info(f"Shutter speed has maxed out at {test.exposure_time}") + break + + # Now, if we've not converged, increase gain until we converge or run out of options. + while iterations < max_iterations: + test = test_exposure_settings(camera, percentile) + if check_convergence(test, target_white_level, tolerance): + break + iterations += 1 + + # Adjust gain to make the white level hit the target, again with a maximum + camera.controls.AnalogueGain = test.analog_gain * min( + target_white_level / test.level, 2 + ) + time.sleep(0.5) + + # Check the gain is still changing - if not, we have probably hit the maximum + if camera.capture_metadata()["AnalogueGain"] == test.analog_gain: + logging.info(f"Gain has maxed out. at {test.analog_gain}") + break + + if check_convergence(test, target_white_level, tolerance): + logging.info(f"Brightness has converged to within {tolerance * 100 :.0f}%.") + else: + logging.warning( + f"Failed to reach target brightness of {target_white_level}." + f"Brightness reached {test.level} after {iterations} iterations." + ) + + return test.level + + +def adjust_white_balance_from_raw( + camera: Picamera2, + percentile: float = 99, + luminance: Optional[np.ndarray] = None, + Cr: Optional[np.ndarray] = None, + Cb: Optional[np.ndarray] = None, + luminance_power: float = 1.0, + method: Literal["percentile", "centre"] = "centre", +) -> Tuple[float, float]: + """Adjust the white balance in a single shot, based on the raw image. + + NB if ``channels_from_raw_image`` is broken, this will go haywire. + We should probably have better logic to verify the channels really + are BGGR... + """ + config = camera.create_still_configuration(raw={"format": "SBGGR10"}) + camera.configure(config) + camera.start() + channels = channels_from_bayer_array(camera.capture_array("raw")) + # logging.info(f"White balance: channels were retrieved with shape {channels.shape}.") + if luminance is not None and Cr is not None and Cb is not None: + # Reconstruct a low-resolution image from the lens shading tables + # and use it to normalise the raw image, to compensate for + # the brightest pixels in each channel not coinciding. + grids = grids_from_lst(np.array(luminance) ** luminance_power, Cr, Cb) + channel_gains = 1 / grids + if channel_gains.shape[1:] != channels.shape[1:]: + channel_gains = upsample_channels(channel_gains, channels.shape[1:]) + logging.info(f"Before gains, channel maxima are {np.max(channels, axis=(1,2))}") + channels = channels * channel_gains + logging.info(f"After gains, channel maxima are {np.max(channels, axis=(1,2))}") + if method == "centre": + _, h, w = channels.shape + blue, g1, g2, red = ( + np.mean( + channels[:, 9 * h // 20 : 11 * h // 20, 9 * w // 20 : 11 * w // 20], + axis=(1, 2), + ) + - 64 + ) + else: + # TODO: read black level from camera rather than hard-coding 64 + blue, g1, g2, red = np.percentile(channels, percentile, axis=(1, 2)) - 64 + green = (g1 + g2) / 2.0 + new_awb_gains = (green / red, green / blue) + if Cr is not None and Cb is not None: + # The LST algorithm normalises Cr and Cb by their minimum. + # The lens shading correction only ever boosts the red and blue values. + # Here, we decrease the gains by the minimum value of Cr and Cb. + new_awb_gains = (green / red * np.min(Cr), green / blue * np.min(Cb)) + + logging.info( + f"Raw white point is R: {red} G: {green} B: {blue}, " + f"setting AWB gains to ({new_awb_gains[0]:.2f}, " + f"{new_awb_gains[1]:.2f})." + ) + camera.controls.AwbEnable = False + camera.controls.ColourGains = new_awb_gains + time.sleep(0.2) + m = camera.capture_metadata() + print(f"Camera confirms gains are now {m['ColourGains']}") + return new_awb_gains + + +def channels_from_bayer_array(bayer_array: np.ndarray) -> np.ndarray: + """Given the 'array' from a PiBayerArray, return the 4 channels.""" + bayer_pattern: List[Tuple[int, int]] = [(0, 0), (0, 1), (1, 0), (1, 1)] + bayer_array = bayer_array.view(np.uint16) + channels_shape: Tuple[int, int, int] = ( + 4, + bayer_array.shape[0] // 2, + bayer_array.shape[1] // 2, + ) + channels: np.ndarray = np.zeros(channels_shape, dtype=bayer_array.dtype) + for i, offset in enumerate(bayer_pattern): + # We simplify life by dealing with only one channel at a time. + channels[i, :, :] = bayer_array[offset[0] :: 2, offset[1] :: 2] + + return channels + + +LensShadingTables = tuple[np.ndarray, np.ndarray, np.ndarray] + + +def get_16x12_grid(chan: np.ndarray, dx: int, dy: int): + """Compresses channel down to a 16x12 grid - from libcamera + + This is taken from https://git.linuxtv.org/libcamera.git/tree/utils/raspberrypi/ctt/ctt_alsc.py + for consistency. + """ + grid = [] + """ + since left and bottom border will not necessarily have rectangles of + dimension dx x dy, the 32nd iteration has to be handled separately. + """ + for i in range(11): + for j in range(15): + grid.append(np.mean(chan[dy * i : dy * (1 + i), dx * j : dx * (1 + j)])) + grid.append(np.mean(chan[dy * i : dy * (1 + i), 15 * dx :])) + for j in range(15): + grid.append(np.mean(chan[11 * dy :, dx * j : dx * (1 + j)])) + grid.append(np.mean(chan[11 * dy :, 15 * dx :])) + """ + return as np.array, ready for further manipulation + """ + return np.reshape(np.array(grid), (12, 16)) + + +def upsample_channels(grids: np.ndarray, shape: tuple[int]): + """Zoom an image in the last two dimensions + + This is effectively the inverse operation of `get_16x12_grid` + """ + zoom_factors = [ + 1, + ] + list(np.ceil(np.array(shape) / np.array(grids.shape[1:]))) + return zoom(grids, zoom_factors, order=1)[:, : shape[0], : shape[1]] + + +def downsampled_channels(channels: np.ndarray, blacklevel=64) -> list[np.ndarray]: + """Generate a downsampled, un-normalised image from which to calculate the LST + + TODO: blacklevel probably ought to be determined from the camera... + """ + channel_shape = np.array(channels.shape[1:]) + lst_shape = np.array([12, 16]) + step = np.ceil(channel_shape / lst_shape).astype(int) + return np.stack( + [ + get_16x12_grid( + channels[i, ...].astype(float) - blacklevel, step[1], step[0] + ) + for i in range(channels.shape[0]) + ], + axis=0, + ) + + +def lst_from_channels(channels: np.ndarray) -> LensShadingTables: + """Given the 4 Bayer colour channels from a white image, generate a LST. + + Internally, is just calls `downsampled_channels` and `lst_from_grids`. + """ + grids = downsampled_channels(channels) + return lst_from_grids(grids) + + +def lst_from_grids(grids: np.ndarray) -> LensShadingTables: + """Given 4 downsampled grids, generate the luminance and chrominance tables + + The LST format has changed with `picamera2` and now uses a fixed resolution, + and is in luminance, Cr, Cb format. This function returns three ndarrays of + luminance, Cr, Cb, each with shape (12, 16). + + # TODO: make consistent with + https://git.linuxtv.org/libcamera.git/tree/utils/raspberrypi/ctt/ctt_alsc.py + """ + r: np.ndarray = grids[3, ...] + g: np.ndarray = np.mean(grids[1:3, ...], axis=0) + b: np.ndarray = grids[0, ...] + + # What we actually want to calculate is the gains needed to compensate for the + # lens shading - that's 1/lens_shading_table_float as we currently have it. + luminance_gains: np.ndarray = np.max(g) / g # Minimum luminance gain is 1 + cr_gains: np.ndarray = g / r + # cr_gains /= cr_gains[5, 7] # Normalise so the central colour doesn't change + cb_gains: np.ndarray = g / b + # cb_gains /= cb_gains[5, 7] + return luminance_gains, cr_gains, cb_gains + + +def grids_from_lst(lum: np.ndarray, Cr: np.ndarray, Cb: np.ndarray) -> np.ndarray: + """Convert form luminance/chrominance dict to four RGGB channels + + Note that these will be normalised - the maximum green value is always 1. + Also, note that the channels are BGGR, to be consistent with the + `channels_from_raw_image` function. This should probably change in the + future. + """ + G = 1 / np.array(lum) + R = G / np.array(Cr) + B = G / np.array(Cb) + return np.stack([B, G, G, R], axis=0) + + +def set_static_lst( + tuning: dict, + luminance: np.ndarray, + cr: np.ndarray, + cb: np.ndarray, +) -> None: + """Update the `rpi.alsc` section of a camera tuning dict to use a static correcton. + + `tuning` will be updated in-place to set its shading to static, and disable any + adaptive tweaking by the algorithm. + """ + for table in luminance, cr, cb: + assert np.array(table).shape == (12, 16), "Lens shading tables must be 12x16!" + alsc = Picamera2.find_tuning_algo(tuning, "rpi.alsc") + alsc["n_iter"] = 0 # disable the adaptive part + alsc["luminance_strength"] = 1.0 + alsc["calibrations_Cr"] = [ + {"ct": 4500, "table": np.reshape(cr, (-1)).round(3).tolist()} + ] + alsc["calibrations_Cb"] = [ + {"ct": 4500, "table": np.reshape(cb, (-1)).round(3).tolist()} + ] + alsc["luminance_lut"] = np.reshape(luminance, (-1)).round(3).tolist() + + +def set_static_ccm(tuning: dict, c: list) -> None: + """Update the `rpi.alsc` section of a camera tuning dict to use a static correcton. + + `tuning` will be updated in-place to set its shading to static, and disable any + adaptive tweaking by the algorithm. + """ + ccm = Picamera2.find_tuning_algo(tuning, "rpi.ccm") + ccm["ccms"] = [{"ct": 2860, "ccm": c}] + + +def get_static_ccm(tuning: dict) -> None: + """Get the `rpi.ccm` section of a camera tuning dict""" + ccm = Picamera2.find_tuning_algo(tuning, "rpi.ccm") + return ccm["ccms"] + + +def lst_is_static(tuning: dict) -> bool: + """Whether the lens shading table is set to static""" + alsc = Picamera2.find_tuning_algo(tuning, "rpi.alsc") + return alsc["n_iter"] == 0 + + +def set_static_geq( + tuning: dict, + offset: int = 65535, +) -> None: + """Update the `rpi.geq` section of a camera tuning dict to always use green + equalisation that averages the green pixels in the red and blue rows. + + `tuning` will be updated in-place to set the geq offest to the given value. + The default 65535 is the maximum allowed value. This means + the brightness will always be below the threshold where averaging is used. + """ + + geq = Picamera2.find_tuning_algo(tuning, "rpi.geq") + geq["offset"] = offset # max out offset to disable the adaptive green equalisation + + +def _geq_is_static(tuning: dict) -> bool: + """Whether the green equalisation is set to static""" + geq = Picamera2.find_tuning_algo(tuning, "rpi.geq") + return geq["offset"] == 65535 + + +def index_of_algorithm(algorithms: list[dict], algorithm: str): + """Find the index of an algorithm's section in the tuning file""" + for i, a in enumerate(algorithms): + if algorithm in a: + return i + + +def copy_alsc_section(from_tuning: dict, to_tuning: dict): + """Copy the `rpi.alsc` algorithm from one tuning to another. + + This is done in-place, i.e. modifying to_tuning. + """ + from_i = index_of_algorithm(from_tuning["algorithms"], "rpi.alsc") + to_i = index_of_algorithm(to_tuning["algorithms"], "rpi.alsc") + # Please excuse the clumsy update-and-delete - this lets us use + # the nice Picamera2 function to find the relevant sub-dict. + to_tuning["algorithms"][to_i] = from_tuning["algorithms"][from_i] + + +def lst_from_camera(camera: Picamera2) -> LensShadingTables: + """Acquire a raw image and use it to calculate a lens shading table.""" + channels = raw_channels_from_camera(camera) + return lst_from_channels(channels) + + +def raw_channels_from_camera(camera: Picamera2) -> LensShadingTables: + """Acquire a raw image and return a 4xNxM array of the colour channels.""" + if camera.started: + camera.stop_recording() + # We will acquire a raw image with unpacked pixels, which is what the + # format below requests. Bit depth and Bayer order may be overwritten. + # TODO: don't assume 10-bit - the high quality camera uses 12. + # TODO: what's the best mode to use here? + config = camera.create_still_configuration(raw={"format": "SBGGR10"}) + camera.configure(config) + camera.start() + raw_image = camera.capture_array("raw") + camera.stop() + # Now we need to calculate a lens shading table that would make this flat. + # raw_image is a 3D array, with full resolution and 3 colour channels. No + # de-mosaicing has been done, so 2/3 of the values are zero (3/4 for R and B + # channels, 1/2 for green because there's twice as many green pixels). + format = camera.camera_configuration()["raw"]["format"] + print(f"Acquired a raw image in format {format}") + return channels_from_bayer_array(raw_image) + + +def recreate_camera_manager(): + """Delete and recreate the camera manager. + + This is necessary to ensure the tuning file is re-read. + """ + del Picamera2._cm + gc.collect() + Picamera2._cm = picamera2.picamera2.CameraManager() + + +if __name__ == "__main__": + """This block is untested but has been updated.""" + with Picamera2() as cam: + tuning = load_default_tuning(cam) + f = np.ones((12, 16)) + set_static_lst(tuning, f, f, f) + set_static_geq(tuning) + with Picamera2(tuning=tuning) as cam: + cam.start_preview() + time.sleep(3) + logging.info("Recalibrating...") + adjust_shutter_and_gain_from_raw(cam) + adjust_white_balance_from_raw(cam) + lst = lst_from_camera(cam) + set_static_lst(tuning, *lst) + logging.info("Done.") + with Picamera2(tuning=tuning) as cam: + cam.start_preview() + time.sleep(2) diff --git a/src/openflexure_microscope_server/things/camera/simulation.py b/src/openflexure_microscope_server/things/camera/simulation.py index ad0ad1c2..723b52cc 100644 --- a/src/openflexure_microscope_server/things/camera/simulation.py +++ b/src/openflexure_microscope_server/things/camera/simulation.py @@ -22,12 +22,11 @@ from scipy.ndimage import gaussian_filter from labthings_fastapi.utilities import get_blocking_portal from labthings_fastapi.decorators import thing_action, thing_property from labthings_fastapi.dependencies.metadata import GetThingStates -from labthings_fastapi.outputs.mjpeg_stream import MJPEGStreamDescriptor from labthings_fastapi.types.numpy import NDArray from labthings_fastapi.server import ThingServer from pydantic import RootModel -from . import BaseCamera, JPEGBlob +from . import BaseCamera, JPEGBlob, ArrayModel from ..stage import StageProtocol as Stage # The ratio between "motor" steps and pixels @@ -35,11 +34,6 @@ from ..stage import StageProtocol as Stage RATIO = 0.2 -class ArrayModel(RootModel): - """A model for an array""" - - root: NDArray - class SimulatedCamera(BaseCamera): """A Thing representing an OpenCV camera""" @@ -160,9 +154,6 @@ class SimulatedCamera(BaseCamera): return self._capture_thread.is_alive() return False - mjpeg_stream = MJPEGStreamDescriptor() - lores_mjpeg_stream = MJPEGStreamDescriptor() - def _capture_frames(self): portal = get_blocking_portal(self) while self._capture_enabled: From 4368dc3d907a21e2ea67eb2a400913a69619cf64 Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Wed, 21 May 2025 22:09:01 +0100 Subject: [PATCH 31/56] Consolidate functionality into BaseCamera from Capture Thing and StreamingPicamera2 --- .../picamera2/test_acquisition.py | 5 +- .../picamera2/test_tuning.py | 2 +- ofm_config_full.json | 3 +- .../things/autofocus.py | 30 +-- .../things/camera/__init__.py | 204 ++++++++++++++++-- .../things/camera/picamera.py | 37 ---- .../camera/picamera_recalibrate_utils.py | 10 +- .../things/camera/simulation.py | 1 - .../things/capture.py | 102 --------- 9 files changed, 202 insertions(+), 192 deletions(-) delete mode 100644 src/openflexure_microscope_server/things/capture.py diff --git a/hardware-specific-tests/picamera2/test_acquisition.py b/hardware-specific-tests/picamera2/test_acquisition.py index 9c8ce20f..2929d83d 100644 --- a/hardware-specific-tests/picamera2/test_acquisition.py +++ b/hardware-specific-tests/picamera2/test_acquisition.py @@ -6,6 +6,7 @@ from PIL import Image import numpy as np from pytest import fixture + @fixture(scope="module") def client(): server = ThingServer() @@ -14,9 +15,11 @@ def client(): client = ThingClient.from_url("/camera/", client=test_client) yield client + def test_calibration(client): client.full_auto_calibrate() + def test_jpeg_and_array(client): blob = client.grab_jpeg() mjpeg_frame = Image.open(blob.open()) @@ -28,5 +31,3 @@ def test_jpeg_and_array(client): array_main = np.array(arrlist) assert mjpeg_frame.size == jpeg_capture.size assert array_main.shape[1::-1] == jpeg_capture.size - - diff --git a/hardware-specific-tests/picamera2/test_tuning.py b/hardware-specific-tests/picamera2/test_tuning.py index f284badc..65ab87f8 100644 --- a/hardware-specific-tests/picamera2/test_tuning.py +++ b/hardware-specific-tests/picamera2/test_tuning.py @@ -5,7 +5,7 @@ from labthings_picamera2 import recalibrate_utils import pytest -MODEL = Picamera2.global_camera_info()[0]['Model'] +MODEL = Picamera2.global_camera_info()[0]["Model"] def check_camera_available(): diff --git a/ofm_config_full.json b/ofm_config_full.json index bf9e29cc..4e5beeee 100644 --- a/ofm_config_full.json +++ b/ofm_config_full.json @@ -13,8 +13,7 @@ "scans_folder": "/var/openflexure/scans/" } }, - "/background_detect/": "openflexure_microscope_server.things.background_detect:BackgroundDetectThing", - "/capture/": "openflexure_microscope_server.things.capture:CaptureThing" + "/background_detect/": "openflexure_microscope_server.things.background_detect:BackgroundDetectThing" }, "settings_folder": "/var/openflexure/settings/", "log_folder": "/var/openflexure/logs/" diff --git a/src/openflexure_microscope_server/things/autofocus.py b/src/openflexure_microscope_server/things/autofocus.py index b651085a..2e87aac1 100644 --- a/src/openflexure_microscope_server/things/autofocus.py +++ b/src/openflexure_microscope_server/things/autofocus.py @@ -18,7 +18,6 @@ import glob from fastapi import Depends import numpy as np from pydantic import BaseModel -from PIL import Image from labthings_fastapi.thing import Thing from labthings_fastapi.dependencies.blocking_portal import BlockingPortal @@ -31,11 +30,8 @@ from labthings_fastapi.dependencies.invocation import InvocationLogger from .camera import RawCameraDependency as Camera from .camera import CameraDependency as WrappedCamera from .stage import StageDependency as Stage -from .capture import CaptureThing -CaptureDep = direct_thing_client_dependency(CaptureThing, "/capture/") - STACK_OVERSHOOT = 200 SETTLING_TIME = 0.3 @@ -288,7 +284,6 @@ class AutofocusThing(Thing): stage: Stage, logger: InvocationLogger, metadata_getter: GetThingStates, - capture: CaptureDep, images_dir: str, stack_dir: str, capture_resolution: tuple[int, int], @@ -307,29 +302,19 @@ class AutofocusThing(Thing): for capture_count in range(images_to_capture): time.sleep(SETTLING_TIME) - jpeg_path = os.path.join( - stack_dir, - f"{capture_count}.jpeg", - ) + jpeg_path = os.path.join(stack_dir, f"{capture_count}.jpeg") start_time = time.time() - image, metadata = capture._capture_image( - cam=cam, - metadata_getter=metadata_getter, - logger=logger, - ) + cam.capture_to_memory(logger=logger, metadata_getter=logger) captured_time = time.time() if capture_count + 1 < images_to_capture: stage.move_relative(z=stack_dz) moved_time = time.time() - if image.size != capture_resolution: - image = image.resize(capture_resolution, Image.BOX) - downsampled_time = time.time() - capture._save_capture( + + cam.save_from_memory( jpeg_path=jpeg_path, - image=image, - metadata=metadata, logger=logger, + save_resolution=capture_resolution, ) saved_time = time.time() time_remaining = SETTLING_TIME - (saved_time - start_time) @@ -337,10 +322,9 @@ class AutofocusThing(Thing): time.sleep(time_remaining) logger.info(f"Settled for an extra {round(time_remaining, 3)} seconds") logger.debug(f"Capturing took {round(captured_time - start_time, 2)} s") - logger.debug(f"Resizing took {round(downsampled_time - moved_time, 2)} s") - logger.debug(f"Saving took {round(saved_time - downsampled_time, 2)} s") + logger.debug(f"Saving took {round(saved_time - moved_time, 2)} s") logger.debug( - f"Effective settling time was {round(saved_time - moved_time, 2)} s" + f"Effective settling time was {round(time.time() - moved_time, 2)} s" ) self.copy_sharpest_image_from_stack(images_dir, stack_dir, logger) diff --git a/src/openflexure_microscope_server/things/camera/__init__.py b/src/openflexure_microscope_server/things/camera/__init__.py index 71c7406d..91a7efb1 100644 --- a/src/openflexure_microscope_server/things/camera/__init__.py +++ b/src/openflexure_microscope_server/things/camera/__init__.py @@ -8,9 +8,10 @@ See repository root for licensing information. from __future__ import annotations import logging -from typing import Literal, Protocol, runtime_checkable, Optional +from typing import Literal, Optional, Tuple from pydantic import RootModel +from PIL import Image from labthings_fastapi.thing import Thing from labthings_fastapi.decorators import thing_action, thing_property @@ -18,6 +19,7 @@ from labthings_fastapi.dependencies.metadata import GetThingStates from labthings_fastapi.dependencies.blocking_portal import BlockingPortal from labthings_fastapi.dependencies.thing import direct_thing_client_dependency from labthings_fastapi.dependencies.raw_thing import raw_thing_dependency +from labthings_fastapi.dependencies.invocation import InvocationLogger from labthings_fastapi.outputs.mjpeg_stream import MJPEGStreamDescriptor from labthings_fastapi.outputs.blob import Blob from labthings_fastapi.types.numpy import NDArray @@ -26,25 +28,30 @@ from labthings_fastapi.types.numpy import NDArray class JPEGBlob(Blob): media_type: str = "image/jpeg" + class PNGBlob(Blob): media_type: str = "image/png" + class ArrayModel(RootModel): """A model for an array""" + root: NDArray + +class CaptureError(RuntimeError): + """An error trying to capture from a CameraThing""" + + +class NoImageInMemoryError(RuntimeError): + """An error called if no image in in memory when an method is called to use that image""" + + class BaseCamera(Thing): - """A stub for a camera, to allow dependencies - - - As of LabThings-FastAPI 0.0.7, we can't make a thing client dependency - based on a protocol, because the protocol is not a Thing, and its - methods/properties are not decorated as Affordances. This stub class - is a workaround for that limitation, and should not be used directly. - - This stub class should be used for dependencies on the CameraProtocol. - """ + """The base class for all cameras. All cameras must directly inherit from this class""" + _memory_image: Optional[Image] = None + _memory_metadata: Optional[dict] = None mjpeg_stream = MJPEGStreamDescriptor() lores_mjpeg_stream = MJPEGStreamDescriptor() @@ -52,12 +59,33 @@ class BaseCamera(Thing): raise NotImplementedError("CameraThings must define their own __enter__ method") def __exit__(self, _exc_type, _exc_value, _traceback) -> None: - raise NotImplementedError("Cameras must not inherit from CameraStub") + raise NotImplementedError("CameraThings must define their own __exit__ method") + + @thing_property + def image_in_memory(self) -> bool: + """True if an image is in memory ready to be saved""" + return self._memory_image is not None and self._memory_metadata is not None + + @thing_action + def clear_image_memory(self) -> None: + """Clear any image in memory""" + self._memory_image = None + self._memory_metadata = None + + @thing_action + def start_streaming(self, main_resolution, buffer_count) -> None: + """Start (or stop and restart) the camera with the given resolution + for the main stream, and buffer_count number of images in the buffer""" + raise NotImplementedError( + "CameraThings must define their own start_streaming method" + ) @thing_property def stream_active(self) -> bool: "Whether the MJPEG stream is active" - raise NotImplementedError("Cameras must not inherit from CameraStub") + raise NotImplementedError( + "CameraThings must define their own stream_active method" + ) @thing_action def capture_array( @@ -65,7 +93,9 @@ class BaseCamera(Thing): stream_name: Literal["main", "lores", "raw", "full"] = "main", wait: Optional[float] = 5, ) -> NDArray: - raise NotImplementedError("Cameras must not inherit from CameraStub") + raise NotImplementedError( + "CameraThings must define their own capture_array method" + ) @thing_action def capture_jpeg( @@ -75,18 +105,152 @@ class BaseCamera(Thing): wait: Optional[float] = 5, ) -> JPEGBlob: """Acquire one image from the camera and return as a JPEG blob""" - raise NotImplementedError("Cameras must not inherit from CameraStub") + raise NotImplementedError( + "CameraThings must define their own capture_jpeg method" + ) @thing_action - def start_streaming(self, main_resolution, buffer_count) -> None: - """Start (or stop and restart) the camera with the given resolution - for the main stream, and buffer_count number of images in the buffer""" - raise NotImplementedError("Cameras must not inherit from CameraStub") + def grab_jpeg( + self, + portal: BlockingPortal, + stream_name: Literal["main", "lores"] = "main", + ) -> JPEGBlob: + """Acquire one image from the preview stream and return as an array + + This differs from `capture_jpeg` in that it does not pause the MJPEG + preview stream. Instead, we simply return the next frame from that + stream (either "main" for the preview stream, or "lores" for the low + resolution preview). No metadata is returned. + """ + stream = ( + self.lores_mjpeg_stream if stream_name == "lores" else self.mjpeg_stream + ) + frame = portal.call(stream.grab_frame) + return JPEGBlob.from_bytes(frame) @thing_action def capture_image(self, stream_name, wait): """Capture a PIL image from stream stream_name with timeout wait""" - raise NotImplementedError("Cameras must not inherit from CameraStub") + raise NotImplementedError( + "CameraThings must define their own capture_image method" + ) + + @thing_action + def capture_and_save( + self, + jpeg_path: str, + logger: InvocationLogger, + metadata_getter: GetThingStates, + save_resolution: Optional[Tuple[int, int]] = None, + ) -> None: + """Capture an image and save it to disk + + This will set the event `acquired` once the image has been acquired, so + that the stage may be moved while it's saved. + + save_resolution can be set to resize the image before saving. By default this is None + meaning that the image is saved at original resoltion. + """ + image, metadata = self._robust_image_capture( + metadata_getter, + logger=logger, + ) + + self._save_capture( + jpeg_path, + image, + metadata, + logger, + save_resolution, + ) + + @thing_action + def capture_to_memory( + self, + logger: InvocationLogger, + metadata_getter: GetThingStates, + ) -> None: + """ + Capture an image to memory. This can be saved later with `save_from_memory` + + Note that only one image is held in memory so this will overwrite any image + in memory. + """ + self._memory_image, self._memory_metadata = self._robust_image_capture( + metadata_getter, + logger=logger, + ) + + @thing_action + def save_from_memory( + self, + jpeg_path: str, + logger: InvocationLogger, + save_resolution: Optional[Tuple[int, int]] = None, + ) -> None: + """ + Save an image that has been captured to memory. + """ + if not self.image_in_memory: + raise NoImageInMemoryError("No image in memory to save.") + + self._save_capture( + jpeg_path=jpeg_path, + image=self._memory_image, + metadata=self._memory_metadata, + logger=logger, + save_resolution=save_resolution, + ) + self.clear_image_memory() + + def _robust_image_capture( + self, + metadata_getter: GetThingStates, + logger: InvocationLogger, + ) -> Image: + """Capture an image in memory and return it with metadata + CaptureError raised if the capture fails for any reason + returns tuple with PIL Image, and dict of metadata + """ + for capture_attempts in range(5): + try: + metadata = metadata_getter() + image = self.capture_image(stream_name="main", wait=5) + return image, metadata + except TimeoutError: + logger.warning( + f"Attempt {capture_attempts + 1} to capture image timed out. Do you have enough RAM?" + ) + raise CaptureError("An error occurred while capturing after 5 attempts") + + def _save_capture( + self, + jpeg_path: str, + image: Image, + metadata: dict, + logger: InvocationLogger, + save_resolution: Optional[Tuple[int, int]] = None, + ) -> None: + """Saving the captured image and metadata to disk + logger warning (via InvocationLogger) is raised if metadata is failed to be added + IOError is raised if the file cannot be saved + nothing is returned on success""" + if save_resolution is not None and image.size != save_resolution: + image = image.resize(save_resolution, Image.BOX) + try: + image.save(jpeg_path, quality=95, subsampling=0) + try: + exif_dict = piexif.load(jpeg_path) + exif_dict["Exif"][piexif.ExifIFD.UserComment] = json.dumps( + metadata + ).encode("utf-8") + piexif.insert(piexif.dump(exif_dict), jpeg_path) + except: # noqa: E722 + # We need to capture any exception as there are many reasons metadata + # might not be added. We warn rather than log the error. + logger.warning(f"Failed to add metadata to {jpeg_path}") + except Exception as e: + raise IOError(f"An error occurred while saving {jpeg_path}") from e CameraDependency = direct_thing_client_dependency(BaseCamera, "/camera/") diff --git a/src/openflexure_microscope_server/things/camera/picamera.py b/src/openflexure_microscope_server/things/camera/picamera.py index 81434918..4c72c6ff 100644 --- a/src/openflexure_microscope_server/things/camera/picamera.py +++ b/src/openflexure_microscope_server/things/camera/picamera.py @@ -556,43 +556,6 @@ class StreamingPiCamera2(BaseCamera): piexif.insert(piexif.dump(exif_dict), path) return JPEGBlob.from_temporary_directory(folder, fname) - @thing_action - def grab_jpeg( - self, - portal: BlockingPortal, - stream_name: Literal["main", "lores"] = "main", - ) -> JPEGBlob: - """Acquire one image from the preview stream and return as an array - - This differs from `capture_jpeg` in that it does not pause the MJPEG - preview stream. Instead, we simply return the next frame from that - stream (either "main" for the preview stream, or "lores" for the low - resolution preview). No metadata is returned. - """ - logging.debug( - f"StreamingPiCamera2.grab_jpeg(stream_name={stream_name}) starting" - ) - stream = ( - self.lores_mjpeg_stream if stream_name == "lores" else self.mjpeg_stream - ) - frame = portal.call(stream.grab_frame) - logging.debug( - f"StreamingPiCamera2.grab_jpeg(stream_name={stream_name}) got frame" - ) - return JPEGBlob.from_bytes(frame) - - @thing_action - def grab_jpeg_size( - self, - portal: BlockingPortal, - stream_name: Literal["main", "lores"] = "main", - ) -> int: - """Acquire one image from the preview stream and return its size""" - stream = ( - self.lores_mjpeg_stream if stream_name == "lores" else self.mjpeg_stream - ) - return portal.call(stream.next_frame_size) - @thing_property def exposure(self) -> float: """An alias for `exposure_time` to fit the micromanager API""" diff --git a/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py b/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py index 0370ba33..ca2fa629 100644 --- a/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py +++ b/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py @@ -219,7 +219,7 @@ def adjust_shutter_and_gain_from_raw( break if check_convergence(test, target_white_level, tolerance): - logging.info(f"Brightness has converged to within {tolerance * 100 :.0f}%.") + logging.info(f"Brightness has converged to within {tolerance * 100:.0f}%.") else: logging.warning( f"Failed to reach target brightness of {target_white_level}." @@ -257,9 +257,11 @@ def adjust_white_balance_from_raw( channel_gains = 1 / grids if channel_gains.shape[1:] != channels.shape[1:]: channel_gains = upsample_channels(channel_gains, channels.shape[1:]) - logging.info(f"Before gains, channel maxima are {np.max(channels, axis=(1,2))}") + logging.info( + f"Before gains, channel maxima are {np.max(channels, axis=(1, 2))}" + ) channels = channels * channel_gains - logging.info(f"After gains, channel maxima are {np.max(channels, axis=(1,2))}") + logging.info(f"After gains, channel maxima are {np.max(channels, axis=(1, 2))}") if method == "centre": _, h, w = channels.shape blue, g1, g2, red = ( @@ -532,7 +534,7 @@ def raw_channels_from_camera(camera: Picamera2) -> LensShadingTables: def recreate_camera_manager(): """Delete and recreate the camera manager. - + This is necessary to ensure the tuning file is re-read. """ del Picamera2._cm diff --git a/src/openflexure_microscope_server/things/camera/simulation.py b/src/openflexure_microscope_server/things/camera/simulation.py index 723b52cc..def8f443 100644 --- a/src/openflexure_microscope_server/things/camera/simulation.py +++ b/src/openflexure_microscope_server/things/camera/simulation.py @@ -34,7 +34,6 @@ from ..stage import StageProtocol as Stage RATIO = 0.2 - class SimulatedCamera(BaseCamera): """A Thing representing an OpenCV camera""" diff --git a/src/openflexure_microscope_server/things/capture.py b/src/openflexure_microscope_server/things/capture.py deleted file mode 100644 index 7f2e5469..00000000 --- a/src/openflexure_microscope_server/things/capture.py +++ /dev/null @@ -1,102 +0,0 @@ -import time -import piexif -import json - -from labthings_fastapi.dependencies.metadata import GetThingStates -from labthings_fastapi.thing import Thing -from labthings_fastapi.decorators import thing_action -from .camera import CameraDependency as CamDep -from labthings_fastapi.dependencies.invocation import ( - InvocationLogger, -) - - -class CaptureError(RuntimeError): - """An error trying to capture from Picamera""" - - -class CaptureThing(Thing): - """A temporary Thing to handle capturing to disk with associated metadata - Will be moved to the camera Thing or dependency in due course""" - - @thing_action - def _capture_and_save( - self, - jpeg_path: str, - cam: CamDep, - logger: InvocationLogger, - metadata_getter: GetThingStates, - ) -> None: - """Capture an image and save it to disk - - This will set the event `acquired` once the image has been acquired, so - that the stage may be moved while it's saved. - """ - capture_start = time.time() - image, metadata = self._capture_image( - cam, - metadata_getter, - logger=logger, - ) - acquisition_time = time.time() - self._save_capture( - jpeg_path, - image, - metadata, - logger, - ) - save_time = time.time() - acquisition_duration = round(acquisition_time - capture_start, 1) - saving_duration = round(save_time - acquisition_time, 1) - logger.info( - f"Acquired {jpeg_path} in {acquisition_duration}s then {saving_duration}s saving to disk" - ) - - @thing_action - def _capture_image( - self, - cam: CamDep, - metadata_getter: GetThingStates, - logger: InvocationLogger, - ): - """Capture an image in memory and return it with metadata - CaptureError raised if the capture fails for any reason - returns tuple with PIL Image, and dict of metadata - """ - for capture_attempts in range(5): - try: - metadata = metadata_getter() - image = cam.capture_image(stream_name="main", wait=5) - return image, metadata - except TimeoutError: - logger.warning( - f"Attempt {capture_attempts + 1} to capture image timed out. Do you have enough RAM?" - ) - raise CaptureError("An error occurred while capturing after 5 attempts") - - @thing_action - def _save_capture( - self, - jpeg_path: str, - image, - metadata: dict, - logger: InvocationLogger, - ) -> None: - """Saving the captured image and metadata to disk - logger warning (via InvocationLogger) is raised if metadata is failed to be added - IOError is raised if the file cannot be saved - nothing is returned on success""" - try: - image.save(jpeg_path, quality=95, subsampling=0) - try: - exif_dict = piexif.load(jpeg_path) - exif_dict["Exif"][piexif.ExifIFD.UserComment] = json.dumps( - metadata - ).encode("utf-8") - piexif.insert(piexif.dump(exif_dict), jpeg_path) - except: # noqa: E722 - # We need to capture any exception as there are many reasons metadata - # might not be added. We warn rather than log the error. - logger.warning(f"Failed to add metadata to {jpeg_path}") - except Exception as e: - raise IOError(f"An error occurred while saving {jpeg_path}") from e From 0e3f4305b5de9e5d8d97e8608657098da759b1d2 Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Thu, 22 May 2025 09:13:42 +0100 Subject: [PATCH 32/56] Tidy post refactor of cameras --- pyproject.toml | 2 +- .../things/autofocus.py | 1 - .../things/camera/__init__.py | 3 +- .../things/camera/opencv.py | 1 - .../things/camera/picamera.py | 15 +++---- .../camera/picamera_recalibrate_utils.py | 43 ++++++------------- .../things/camera/simulation.py | 3 -- 7 files changed, 22 insertions(+), 46 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5b00f07b..53f992e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,8 +83,8 @@ addopts = [ ] norecursedirs = [".git", "build", "node_modules"] - pythonpath = ["."] +testpaths = ["tests"] [tool.ruff.format] # Use native line endings for all files diff --git a/src/openflexure_microscope_server/things/autofocus.py b/src/openflexure_microscope_server/things/autofocus.py index 2e87aac1..4040ae7f 100644 --- a/src/openflexure_microscope_server/things/autofocus.py +++ b/src/openflexure_microscope_server/things/autofocus.py @@ -24,7 +24,6 @@ from labthings_fastapi.dependencies.blocking_portal import BlockingPortal from labthings_fastapi.decorators import thing_action, thing_property from labthings_fastapi.dependencies.metadata import GetThingStates from labthings_fastapi.types.numpy import NDArray -from labthings_fastapi.dependencies.thing import direct_thing_client_dependency from labthings_fastapi.dependencies.invocation import InvocationLogger from .camera import RawCameraDependency as Camera diff --git a/src/openflexure_microscope_server/things/camera/__init__.py b/src/openflexure_microscope_server/things/camera/__init__.py index 91a7efb1..7d771af6 100644 --- a/src/openflexure_microscope_server/things/camera/__init__.py +++ b/src/openflexure_microscope_server/things/camera/__init__.py @@ -7,11 +7,12 @@ See repository root for licensing information. """ from __future__ import annotations -import logging from typing import Literal, Optional, Tuple +import json from pydantic import RootModel from PIL import Image +import piexif from labthings_fastapi.thing import Thing from labthings_fastapi.decorators import thing_action, thing_property diff --git a/src/openflexure_microscope_server/things/camera/opencv.py b/src/openflexure_microscope_server/things/camera/opencv.py index 2278e2bb..f793f209 100644 --- a/src/openflexure_microscope_server/things/camera/opencv.py +++ b/src/openflexure_microscope_server/things/camera/opencv.py @@ -19,7 +19,6 @@ import piexif from labthings_fastapi.utilities import get_blocking_portal from labthings_fastapi.decorators import thing_action, thing_property from labthings_fastapi.dependencies.metadata import GetThingStates -from labthings_fastapi.outputs.mjpeg_stream import MJPEGStreamDescriptor from labthings_fastapi.types.numpy import NDArray from . import BaseCamera, JPEGBlob diff --git a/src/openflexure_microscope_server/things/camera/picamera.py b/src/openflexure_microscope_server/things/camera/picamera.py index 4c72c6ff..9920050d 100644 --- a/src/openflexure_microscope_server/things/camera/picamera.py +++ b/src/openflexure_microscope_server/things/camera/picamera.py @@ -1,7 +1,5 @@ from __future__ import annotations -from dataclasses import dataclass from datetime import datetime -import io import json import logging import os @@ -12,19 +10,14 @@ from tempfile import TemporaryDirectory from pydantic import BaseModel, BeforeValidator from labthings_fastapi.descriptors.property import PropertyDescriptor -from labthings_fastapi.thing import Thing from labthings_fastapi.decorators import thing_action, thing_property from labthings_fastapi.outputs.mjpeg_stream import MJPEGStream from labthings_fastapi.utilities import get_blocking_portal -from labthings_fastapi.types.numpy import NDArray from labthings_fastapi.dependencies.metadata import GetThingStates from labthings_fastapi.dependencies.blocking_portal import BlockingPortal -from typing import Annotated, Any, Iterator, Literal, Mapping, Optional, Self +from typing import Annotated, Any, Iterator, Literal, Mapping, Optional from contextlib import contextmanager import piexif -from scipy.ndimage import zoom -from scipy.interpolate import interp1d -from PIL import Image from threading import RLock import picamera2 from picamera2 import Picamera2 @@ -33,7 +26,7 @@ from picamera2.outputs import Output import numpy as np from . import picamera_recalibrate_utils as recalibrate_utils -from . import BaseCamera, JPEGBlob, PNGBlob, ArrayModel +from . import BaseCamera, JPEGBlob, ArrayModel class PicameraControl(PropertyDescriptor): @@ -636,7 +629,9 @@ class StreamingPiCamera2(BaseCamera): the processed images. It should not affect raw images. """ with self.picamera(pause_stream=True) as cam: - L, Cr, Cb = recalibrate_utils.lst_from_camera(cam) + # Suppress lint warning that L, Cr, and Cb are not lowercase, as this is the + # Standard format for these mathematical vars. + L, Cr, Cb = recalibrate_utils.lst_from_camera(cam) # noqa: N806 recalibrate_utils.set_static_lst(self.tuning, L, Cr, Cb) self.initialise_picamera() diff --git a/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py b/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py index ca2fa629..dee31127 100644 --- a/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py +++ b/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py @@ -30,6 +30,11 @@ picamera.lens_shading_table = lst ``` """ +# Disable N806 & 803, which checks that all variables and args are lowercase. +# This is due to the number of matrix calculations and colour channel +# calculations that are clearer using the standard R, G, B, or L, Cr, Cb terms. +# ruff: noqa: N806 N803 + from __future__ import annotations import gc import logging @@ -43,6 +48,9 @@ from picamera2 import Picamera2 import picamera2 +LensShadingTables = tuple[np.ndarray, np.ndarray, np.ndarray] + + def load_default_tuning(cam: Picamera2) -> dict: """Load the default tuning file for the camera @@ -58,12 +66,13 @@ def load_default_tuning(cam: Picamera2) -> dict: try: return cam.load_tuning_file(fname) except RuntimeError: - dir = "/usr/share/libcamera/ipa/raspberrypi" # from picamera2 v0.3.9 - # The directory above has been removed from the search path, which I + tuning_dir = "/usr/share/libcamera/ipa/raspberrypi" + # from picamera2 v0.3.9 + # The directory above has been removed from the search path seems # find odd - as that's where the files currently are on a default # Raspbian image. This may need updating if the files have moved # in future updates to the system libcamera package - return cam.load_tuning_file(fname, dir=dir) + return cam.load_tuning_file(fname, dir=tuning_dir) def set_minimum_exposure(camera: Picamera2): @@ -312,9 +321,6 @@ def channels_from_bayer_array(bayer_array: np.ndarray) -> np.ndarray: return channels -LensShadingTables = tuple[np.ndarray, np.ndarray, np.ndarray] - - def get_16x12_grid(chan: np.ndarray, dx: int, dy: int): """Compresses channel down to a 16x12 grid - from libcamera @@ -527,8 +533,8 @@ def raw_channels_from_camera(camera: Picamera2) -> LensShadingTables: # raw_image is a 3D array, with full resolution and 3 colour channels. No # de-mosaicing has been done, so 2/3 of the values are zero (3/4 for R and B # channels, 1/2 for green because there's twice as many green pixels). - format = camera.camera_configuration()["raw"]["format"] - print(f"Acquired a raw image in format {format}") + raw_format = camera.camera_configuration()["raw"]["format"] + print(f"Acquired a raw image in format {raw_format}") return channels_from_bayer_array(raw_image) @@ -540,24 +546,3 @@ def recreate_camera_manager(): del Picamera2._cm gc.collect() Picamera2._cm = picamera2.picamera2.CameraManager() - - -if __name__ == "__main__": - """This block is untested but has been updated.""" - with Picamera2() as cam: - tuning = load_default_tuning(cam) - f = np.ones((12, 16)) - set_static_lst(tuning, f, f, f) - set_static_geq(tuning) - with Picamera2(tuning=tuning) as cam: - cam.start_preview() - time.sleep(3) - logging.info("Recalibrating...") - adjust_shutter_and_gain_from_raw(cam) - adjust_white_balance_from_raw(cam) - lst = lst_from_camera(cam) - set_static_lst(tuning, *lst) - logging.info("Done.") - with Picamera2(tuning=tuning) as cam: - cam.start_preview() - time.sleep(2) diff --git a/src/openflexure_microscope_server/things/camera/simulation.py b/src/openflexure_microscope_server/things/camera/simulation.py index def8f443..2494c6c6 100644 --- a/src/openflexure_microscope_server/things/camera/simulation.py +++ b/src/openflexure_microscope_server/things/camera/simulation.py @@ -22,9 +22,7 @@ from scipy.ndimage import gaussian_filter from labthings_fastapi.utilities import get_blocking_portal from labthings_fastapi.decorators import thing_action, thing_property from labthings_fastapi.dependencies.metadata import GetThingStates -from labthings_fastapi.types.numpy import NDArray from labthings_fastapi.server import ThingServer -from pydantic import RootModel from . import BaseCamera, JPEGBlob, ArrayModel from ..stage import StageProtocol as Stage @@ -33,7 +31,6 @@ from ..stage import StageProtocol as Stage # higher related to a faster movement RATIO = 0.2 - class SimulatedCamera(BaseCamera): """A Thing representing an OpenCV camera""" From 72faba0272ce6908fb13dac7b787dd95f3935468 Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Mon, 23 Jun 2025 12:33:42 +0100 Subject: [PATCH 33/56] Fix issues found by linter in PiCamera Code Issue #423 created based on lens shading TODOs --- .../things/autofocus.py | 2 +- .../things/camera/picamera.py | 6 +-- .../camera/picamera_recalibrate_utils.py | 46 +++++++++---------- .../things/camera/simulation.py | 1 + 4 files changed, 26 insertions(+), 29 deletions(-) diff --git a/src/openflexure_microscope_server/things/autofocus.py b/src/openflexure_microscope_server/things/autofocus.py index 4040ae7f..ab6a4c0a 100644 --- a/src/openflexure_microscope_server/things/autofocus.py +++ b/src/openflexure_microscope_server/things/autofocus.py @@ -303,7 +303,7 @@ class AutofocusThing(Thing): jpeg_path = os.path.join(stack_dir, f"{capture_count}.jpeg") start_time = time.time() - cam.capture_to_memory(logger=logger, metadata_getter=logger) + cam.capture_to_memory(logger=logger, metadata_getter=metadata_getter) captured_time = time.time() if capture_count + 1 < images_to_capture: diff --git a/src/openflexure_microscope_server/things/camera/picamera.py b/src/openflexure_microscope_server/things/camera/picamera.py index 9920050d..da86022c 100644 --- a/src/openflexure_microscope_server/things/camera/picamera.py +++ b/src/openflexure_microscope_server/things/camera/picamera.py @@ -41,8 +41,7 @@ class PicameraControl(PropertyDescriptor): def _getter(self, obj: StreamingPiCamera2): with obj.picamera() as cam: - ret = cam.capture_metadata()[self.control_name] - return ret + return cam.capture_metadata()[self.control_name] def _setter(self, obj: StreamingPiCamera2, value: Any): with obj.picamera() as cam: @@ -171,6 +170,7 @@ class StreamingPiCamera2(BaseCamera): initial_value=(820, 616), description="Resolution to use for the MJPEG stream", ) + mjpeg_bitrate = PropertyDescriptor( Optional[int], initial_value=100000000, @@ -368,8 +368,6 @@ class StreamingPiCamera2(BaseCamera): lower may cause dropped frames. Defaults to 6. """ with self.picamera() as picam: - # TODO: Filip: can we use the lores output to keep preview stream going - # while recording? According to picamera2 docs 4.2.1.6 this should work try: if picam.started: picam.stop() diff --git a/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py b/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py index dee31127..790cf04f 100644 --- a/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py +++ b/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py @@ -80,16 +80,15 @@ def set_minimum_exposure(camera: Picamera2): We set exposure mode to manual, analog and digital gain to 1, and shutter speed to the minimum (8us for Pi Camera v2) - NB ISO is left at auto, because this is needed for the gains + + Note ISO is left at auto, because this is needed for the gains to be set correctly. """ - camera.set_controls({"AeEnable": False, "AnalogueGain": 1, "ExposureTime": 1}) - # camera.iso = 0 # We must set ISO=0 (auto) or we can't set gain - # camera.analog_gain = 1 - # camera.digital_gain = 1 (not configurable) + # Disable Automatic exposure and gain algoritm (AeEnable), and set analogue + # gain and exposure time. # Setting the shutter speed to 1us will result in it being set - # to the minimum possible, which is probably 8us for PiCamera v2 - # camera.shutter_speed = 1 + # to the minimum possible, which is ~8us for PiCamera v2 + camera.set_controls({"AeEnable": False, "AnalogueGain": 1, "ExposureTime": 1}) time.sleep(0.5) @@ -138,8 +137,7 @@ def test_exposure_settings(camera: Picamera2, percentile: float) -> ExposureTest def check_convergence(test: ExposureTest, target: int, tolerance: float): """Check whether the brightness is within the specified target range""" - converged = abs(test.level - target) < target * tolerance - return converged + return abs(test.level - target) < target * tolerance def adjust_shutter_and_gain_from_raw( @@ -257,7 +255,6 @@ def adjust_white_balance_from_raw( camera.configure(config) camera.start() channels = channels_from_bayer_array(camera.capture_array("raw")) - # logging.info(f"White balance: channels were retrieved with shape {channels.shape}.") if luminance is not None and Cr is not None and Cb is not None: # Reconstruct a low-resolution image from the lens shading tables # and use it to normalise the raw image, to compensate for @@ -328,10 +325,9 @@ def get_16x12_grid(chan: np.ndarray, dx: int, dy: int): for consistency. """ grid = [] - """ - since left and bottom border will not necessarily have rectangles of - dimension dx x dy, the 32nd iteration has to be handled separately. - """ + + # since left and bottom border will not necessarily have rectangles of + # dimension dx x dy, the final iteration has to be handled separately. for i in range(11): for j in range(15): grid.append(np.mean(chan[dy * i : dy * (1 + i), dx * j : dx * (1 + j)])) @@ -339,9 +335,7 @@ def get_16x12_grid(chan: np.ndarray, dx: int, dy: int): for j in range(15): grid.append(np.mean(chan[11 * dy :, dx * j : dx * (1 + j)])) grid.append(np.mean(chan[11 * dy :, 15 * dx :])) - """ - return as np.array, ready for further manipulation - """ + # return as np.array, ready for further manipulation return np.reshape(np.array(grid), (12, 16)) @@ -387,24 +381,27 @@ def lst_from_channels(channels: np.ndarray) -> LensShadingTables: def lst_from_grids(grids: np.ndarray) -> LensShadingTables: """Given 4 downsampled grids, generate the luminance and chrominance tables + The grids are the 4 BAYER channels RGGB + The LST format has changed with `picamera2` and now uses a fixed resolution, and is in luminance, Cr, Cb format. This function returns three ndarrays of luminance, Cr, Cb, each with shape (12, 16). - - # TODO: make consistent with - https://git.linuxtv.org/libcamera.git/tree/utils/raspberrypi/ctt/ctt_alsc.py """ + + # Calculated red, green, and blue channels from Bayer data r: np.ndarray = grids[3, ...] g: np.ndarray = np.mean(grids[1:3, ...], axis=0) b: np.ndarray = grids[0, ...] # What we actually want to calculate is the gains needed to compensate for the # lens shading - that's 1/lens_shading_table_float as we currently have it. - luminance_gains: np.ndarray = np.max(g) / g # Minimum luminance gain is 1 + + # Minimum luminance gain is 1 + luminance_gains: np.ndarray = np.max(g) / g + cr_gains: np.ndarray = g / r - # cr_gains /= cr_gains[5, 7] # Normalise so the central colour doesn't change cb_gains: np.ndarray = g / b - # cb_gains /= cb_gains[5, 7] + return luminance_gains, cr_gains, cb_gains @@ -491,11 +488,12 @@ def _geq_is_static(tuning: dict) -> bool: return geq["offset"] == 65535 -def index_of_algorithm(algorithms: list[dict], algorithm: str): +def index_of_algorithm(algorithms: list[dict], algorithm: str) -> int: """Find the index of an algorithm's section in the tuning file""" for i, a in enumerate(algorithms): if algorithm in a: return i + raise ValueError(f"Algorithm {algorithm} is not available.") def copy_alsc_section(from_tuning: dict, to_tuning: dict): diff --git a/src/openflexure_microscope_server/things/camera/simulation.py b/src/openflexure_microscope_server/things/camera/simulation.py index 2494c6c6..c27ebc20 100644 --- a/src/openflexure_microscope_server/things/camera/simulation.py +++ b/src/openflexure_microscope_server/things/camera/simulation.py @@ -31,6 +31,7 @@ from ..stage import StageProtocol as Stage # higher related to a faster movement RATIO = 0.2 + class SimulatedCamera(BaseCamera): """A Thing representing an OpenCV camera""" From dd63d74651060fd406264a99cb029d7ecb4bf7c8 Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Mon, 23 Jun 2025 12:39:41 +0100 Subject: [PATCH 34/56] Removing mjpeg bitrate setter from UI and Python as it wasn't implemented fully and isn't a setting that should need changing --- .../things/camera/picamera.py | 6 --- .../settingsComponents/cameraSettings.vue | 50 ------------------- 2 files changed, 56 deletions(-) diff --git a/src/openflexure_microscope_server/things/camera/picamera.py b/src/openflexure_microscope_server/things/camera/picamera.py index da86022c..222ab108 100644 --- a/src/openflexure_microscope_server/things/camera/picamera.py +++ b/src/openflexure_microscope_server/things/camera/picamera.py @@ -177,12 +177,6 @@ class StreamingPiCamera2(BaseCamera): description="Bitrate for MJPEG stream (None for default)", ) - @mjpeg_bitrate.setter - def mjpeg_bitrate(self, value: Optional[int]): - """Restart the stream when we set the bitrate""" - with self.picamera(pause_stream=True): - pass # just pausing and restarting the stream is enough. - stream_active = PropertyDescriptor( bool, initial_value=False, diff --git a/webapp/src/components/tabContentComponents/settingsComponents/cameraSettings.vue b/webapp/src/components/tabContentComponents/settingsComponents/cameraSettings.vue index 69f3f837..ddcef921 100644 --- a/webapp/src/components/tabContentComponents/settingsComponents/cameraSettings.vue +++ b/webapp/src/components/tabContentComponents/settingsComponents/cameraSettings.vue @@ -32,56 +32,6 @@ /> -
  • - Image Quality -
    - - -
    -
    • - From c659ba8f5e83152a611696821103e13daced34bf Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Mon, 23 Jun 2025 13:07:26 +0100 Subject: [PATCH 35/56] Minor tidying to Picamera Thing --- .../things/camera/picamera.py | 55 +++++++++++-------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/src/openflexure_microscope_server/things/camera/picamera.py b/src/openflexure_microscope_server/things/camera/picamera.py index 222ab108..4372dabc 100644 --- a/src/openflexure_microscope_server/things/camera/picamera.py +++ b/src/openflexure_microscope_server/things/camera/picamera.py @@ -97,7 +97,7 @@ class StreamingPiCamera2(BaseCamera): def __init__(self, camera_num: int = 0): self.camera_num = camera_num self.camera_configs: dict[str, dict] = {} - # NB persistent controls will be updated with settings, in __enter__. + # Note persistent controls will be updated with settings, in __enter__. self.persistent_controls = { "AeEnable": False, "AnalogueGain": 1.0, @@ -129,25 +129,34 @@ class StreamingPiCamera2(BaseCamera): """ with self.picamera() as cam: for i in range(discard_frames): - # Discard frames, so we know our data is fresh + # Discard frames, so data is fresh cam.capture_metadata() - for k, v in cam.capture_metadata().items(): - if k in self.persistent_controls: - if k in self.persistent_control_tolerances: - if ( - np.abs(self.persistent_controls[k] - v) - < self.persistent_control_tolerances[k] - ): - logging.debug( - f"Ignoring a small change in persistent control {k}" - f"from {self.persistent_controls[k]} to {v}" - "while updating persistent controls." - ) - continue # Ignore small changes, to avoid drift - self.persistent_controls[k] = v - self.thing_settings.update( - self.persistent_controls - ) # TODO: Is this saving to the wrong place? + for key, value in cam.capture_metadata().items(): + if key not in self.persistent_controls: + # Skip keys that are not persistent controls + continue + if self._control_value_within_tolerance(key, value): + logging.debug( + "Ignoring a small change in persistent control %s from %s to " + "%s while updating persistent controls.", + key, + self.persistent_controls[key], + self.persistent_controls[key], + ) + else: + self.persistent_controls[key] = value + self.thing_settings.update(self.persistent_controls) + + def _control_value_within_tolerance(self, key: str, value: float) -> bool: + """Return True if the updated value for a persistent control is within + tolerance of current value. This allows ignoring small changes + """ + if key not in self.persistent_control_tolerances: + # Not tolerance range set, so it is not within tollerance + return False + + difference = np.abs(self.persistent_controls[key] - value) + return difference < self.persistent_control_tolerances[key] def settings_to_persistent_controls(self): """Update the persistent controls dict from the settings dict @@ -246,7 +255,6 @@ class StreamingPiCamera2(BaseCamera): so will fail if it's run before those are populated in `__enter__`. """ if "tuning" in self.thing_settings: - # TODO: should this be a separate file? self.tuning = self.thing_settings["tuning"].dict else: logging.info("Did not find tuning in settings, reading from camera...") @@ -288,6 +296,7 @@ class StreamingPiCamera2(BaseCamera): self.populate_default_tuning() self.initialise_tuning() self.initialise_picamera() + # populate sensor modes by reading the property self.sensor_modes self.settings_to_persistent_controls() self.settings_to_properties() @@ -420,10 +429,8 @@ class StreamingPiCamera2(BaseCamera): self.lores_mjpeg_stream.stop() logging.info("Stopped MJPEG stream.") - # Increase the resolution for taking an image - time.sleep( - 0.2 - ) # Sprinkled a sleep to prevent camera getting confused by rapid commands + # Adding a sleep to prevent camera getting confused by rapid commands + time.sleep(0.2) @thing_action def capture_image( From 62a07622fad6390e1e6cd1ff11c121f4df26ae87 Mon Sep 17 00:00:00 2001 From: jaknapper Date: Mon, 23 Jun 2025 14:58:57 +0100 Subject: [PATCH 36/56] If not provided, find resize from image size when stitching --- .../things/smart_scan.py | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/openflexure_microscope_server/things/smart_scan.py b/src/openflexure_microscope_server/things/smart_scan.py index 5ec78a4d..618fec2f 100644 --- a/src/openflexure_microscope_server/things/smart_scan.py +++ b/src/openflexure_microscope_server/things/smart_scan.py @@ -1080,7 +1080,7 @@ class SmartScanThing(Thing): self, logger: InvocationLogger, scan_name: str, - stitch_resize: float, + stitch_resize: Optional[float] = None, overlap: float = 0.0, ) -> None: """Generate a stitched image based on stage position metadata @@ -1100,7 +1100,6 @@ class SmartScanThing(Thing): try: with open(json_fpath, "r", encoding="utf-8") as data_file: data_loaded = json.load(data_file) - logger.info(data_loaded) overlap = data_loaded["overlap"] except (json.decoder.JSONDecodeError, FileNotFoundError, TypeError): # As there is no schema or pydantic model this should handle @@ -1117,6 +1116,29 @@ class SmartScanThing(Thing): "Attempting stitch with overlap value of 0.1" ) overlap = 0.1 + + if stitch_resize is None: + try: + with open(json_fpath, "r", encoding="utf-8") as data_file: + data_loaded = json.load(data_file) + capture_resolution = data_loaded["capture resolution"] + stitch_resize = STITCHING_RESOLUTION[0] / capture_resolution[0] + except (json.decoder.JSONDecodeError, FileNotFoundError, TypeError): + # As there is no schema or pydantic model this should handle + # the file not being there, it not being json in the file, + # or the imported data not being indexable + logger.warning( + f"Couldn't read scan data, is {SCAN_DATA_FILENAME} missing or corrupt? " + "Attempting stitch with resize value of 0.5" + ) + stitch_resize = 0.5 + except KeyError: + logger.warning( + "Value for capture resolution not found in scan data. " + "Attempting stitch with resize value of 0.5" + ) + stitch_resize = 0.5 + self.run_subprocess( logger, [ From afea1e36581ec2d5fcdd70f6f36409fbcf8dbd83 Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Mon, 23 Jun 2025 17:22:28 +0100 Subject: [PATCH 37/56] Create function for flattening and rounding array and converting to a list --- .../things/camera/picamera_recalibrate_utils.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py b/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py index 790cf04f..6b4e0fc0 100644 --- a/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py +++ b/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py @@ -436,12 +436,12 @@ def set_static_lst( alsc["n_iter"] = 0 # disable the adaptive part alsc["luminance_strength"] = 1.0 alsc["calibrations_Cr"] = [ - {"ct": 4500, "table": np.reshape(cr, (-1)).round(3).tolist()} + {"ct": 4500, "table": as_flat_rounded_list(cr, round_to=3)} ] alsc["calibrations_Cb"] = [ - {"ct": 4500, "table": np.reshape(cb, (-1)).round(3).tolist()} + {"ct": 4500, "table": as_flat_rounded_list(cb, round_to=3)} ] - alsc["luminance_lut"] = np.reshape(luminance, (-1)).round(3).tolist() + alsc["luminance_lut"] = as_flat_rounded_list(luminance, round_to=3) def set_static_ccm(tuning: dict, c: list) -> None: @@ -544,3 +544,8 @@ def recreate_camera_manager(): del Picamera2._cm gc.collect() Picamera2._cm = picamera2.picamera2.CameraManager() + + +def as_flat_rounded_list(array: np.ndarray, round_to: int = 3) -> list[float]: + """Flatten array, round, and then convert to list""" + return np.reshape(array, -1).round(round_to).tolist() From cdae90013a17c7c4d44e6de2c96f99f91c9a1684 Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Mon, 23 Jun 2025 16:24:40 +0000 Subject: [PATCH 38/56] Modify comment for clarity in the function that copies the lens shading algorithm --- .../things/camera/picamera_recalibrate_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py b/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py index 6b4e0fc0..812f8379 100644 --- a/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py +++ b/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py @@ -501,10 +501,10 @@ def copy_alsc_section(from_tuning: dict, to_tuning: dict): This is done in-place, i.e. modifying to_tuning. """ + # Using Picamera2 function to find the relevant sub-dict for each tuning file from_i = index_of_algorithm(from_tuning["algorithms"], "rpi.alsc") to_i = index_of_algorithm(to_tuning["algorithms"], "rpi.alsc") - # Please excuse the clumsy update-and-delete - this lets us use - # the nice Picamera2 function to find the relevant sub-dict. + # Updating the dictionary in place. to_tuning["algorithms"][to_i] = from_tuning["algorithms"][from_i] From febb0d2690e5b6298263787087320c53d0354e08 Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Mon, 23 Jun 2025 17:47:18 +0100 Subject: [PATCH 39/56] Clarify white balance code. --- .../camera/picamera_recalibrate_utils.py | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py b/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py index 812f8379..4f4d5224 100644 --- a/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py +++ b/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py @@ -255,6 +255,8 @@ def adjust_white_balance_from_raw( camera.configure(config) camera.start() channels = channels_from_bayer_array(camera.capture_array("raw")) + # TODO: read black level from camera rather than hard-coding 64 + blacklevel = 64 if luminance is not None and Cr is not None and Cb is not None: # Reconstruct a low-resolution image from the lens shading tables # and use it to normalise the raw image, to compensate for @@ -269,17 +271,23 @@ def adjust_white_balance_from_raw( channels = channels * channel_gains logging.info(f"After gains, channel maxima are {np.max(channels, axis=(1, 2))}") if method == "centre": - _, h, w = channels.shape - blue, g1, g2, red = ( - np.mean( - channels[:, 9 * h // 20 : 11 * h // 20, 9 * w // 20 : 11 * w // 20], - axis=(1, 2), - ) - - 64 + _, height, width = channels.shape + # Cut out the central 10% from 9/20 to 11/20... + low_y_range = 9 * height // 20 + hi_y_range = 11 * height // 20 + low_x_range = 9 * width // 20 + hi_x_range = 11 * width // 20 + # ... and then take the mean of each bayer channel. + centre_means = np.mean( + channels[:, low_y_range:hi_y_range, low_x_range:hi_x_range], + axis=(1, 2), ) + # Subtract blacklevel before splitting into channels + blue, g1, g2, red = centre_means - blacklevel else: - # TODO: read black level from camera rather than hard-coding 64 - blue, g1, g2, red = np.percentile(channels, percentile, axis=(1, 2)) - 64 + blue, g1, g2, red = ( + np.percentile(channels, percentile, axis=(1, 2)) - blacklevel + ) green = (g1 + g2) / 2.0 new_awb_gains = (green / red, green / blue) if Cr is not None and Cb is not None: From 7c3a9ad01fdd615b9be0d67e4e2ca088c3cb7edb Mon Sep 17 00:00:00 2001 From: jaknapper Date: Mon, 23 Jun 2025 18:08:27 +0100 Subject: [PATCH 40/56] Copy sharpest image from stack, not the most average sharpness! --- src/openflexure_microscope_server/things/autofocus.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/openflexure_microscope_server/things/autofocus.py b/src/openflexure_microscope_server/things/autofocus.py index ab6a4c0a..64013551 100644 --- a/src/openflexure_microscope_server/things/autofocus.py +++ b/src/openflexure_microscope_server/things/autofocus.py @@ -353,8 +353,7 @@ class AutofocusThing(Thing): image to images dir.""" image_list = glob.glob(os.path.join(stack_dir, "*")) image_list = sorted(image_list, key=os.path.getsize) - central_index = (len(image_list) - 1) // 2 - central_image = image_list[central_index] + central_image = image_list[-1] xy_location = os.path.basename(stack_dir) logger.info(central_image) From 11c8f47e20d0b14a25ff52643f584969c16f064a Mon Sep 17 00:00:00 2001 From: jaknapper Date: Mon, 23 Jun 2025 18:13:46 +0100 Subject: [PATCH 41/56] Rename central image to sharpest --- src/openflexure_microscope_server/things/autofocus.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/openflexure_microscope_server/things/autofocus.py b/src/openflexure_microscope_server/things/autofocus.py index 64013551..7e0d652c 100644 --- a/src/openflexure_microscope_server/things/autofocus.py +++ b/src/openflexure_microscope_server/things/autofocus.py @@ -353,8 +353,8 @@ class AutofocusThing(Thing): image to images dir.""" image_list = glob.glob(os.path.join(stack_dir, "*")) image_list = sorted(image_list, key=os.path.getsize) - central_image = image_list[-1] + sharpest_image = image_list[-1] xy_location = os.path.basename(stack_dir) - logger.info(central_image) - shutil.copy(central_image, os.path.join(images_dir, f"{xy_location}.jpeg")) + logger.info(sharpest_image) + shutil.copy(sharpest_image, os.path.join(images_dir, f"{xy_location}.jpeg")) From a43a82213d7f2431e5768feb7b5bbe71bc6a2cc8 Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Tue, 24 Jun 2025 14:07:58 +0100 Subject: [PATCH 42/56] Improve docs for hardware specific tests --- .../picamera2/test_acquisition.py | 15 +++++-- .../picamera2/test_exposure_time_drift.py | 8 +++- .../picamera2/test_sensor_mode.py | 6 ++- .../picamera2/test_tuning.py | 43 ++++++++++++++++--- 4 files changed, 61 insertions(+), 11 deletions(-) diff --git a/hardware-specific-tests/picamera2/test_acquisition.py b/hardware-specific-tests/picamera2/test_acquisition.py index 2929d83d..6d66cc5e 100644 --- a/hardware-specific-tests/picamera2/test_acquisition.py +++ b/hardware-specific-tests/picamera2/test_acquisition.py @@ -1,11 +1,13 @@ -from labthings_picamera2 import StreamingPiCamera2 -from labthings_fastapi.server import ThingServer -from labthings_fastapi.client import ThingClient from fastapi.testclient import TestClient from PIL import Image import numpy as np from pytest import fixture +from labthings_fastapi.server import ThingServer +from labthings_fastapi.client import ThingClient + +from openflexure_microscope_server.things.camera.picamera import StreamingPiCamera2 + @fixture(scope="module") def client(): @@ -17,10 +19,17 @@ def client(): def test_calibration(client): + """ + Check that full auto calibrate completes without an exception + """ client.full_auto_calibrate() def test_jpeg_and_array(client): + """ + Check that grabbing a jpeg from the stream results in the same size + image as a array capture or a jpeg capture. + """ blob = client.grab_jpeg() mjpeg_frame = Image.open(blob.open()) assert mjpeg_frame diff --git a/hardware-specific-tests/picamera2/test_exposure_time_drift.py b/hardware-specific-tests/picamera2/test_exposure_time_drift.py index ee75d179..d7a24f00 100644 --- a/hardware-specific-tests/picamera2/test_exposure_time_drift.py +++ b/hardware-specific-tests/picamera2/test_exposure_time_drift.py @@ -5,12 +5,18 @@ from fastapi.testclient import TestClient from labthings_fastapi.server import ThingServer from labthings_fastapi.client import ThingClient -from labthings_picamera2.thing import StreamingPiCamera2 + +from openflexure_microscope_server.things.camera.picamera import StreamingPiCamera2 logging.basicConfig(level=logging.DEBUG) def test_exposure_time_drift(): + """ + Capture 10 full resolution images and check that the exposure time remains constant + + This confirms that automatic exposure time adjustment is fully turned off + """ cam = StreamingPiCamera2() server = ThingServer() server.add_thing(cam, "/camera/") diff --git a/hardware-specific-tests/picamera2/test_sensor_mode.py b/hardware-specific-tests/picamera2/test_sensor_mode.py index 91694a89..84f6ce7c 100644 --- a/hardware-specific-tests/picamera2/test_sensor_mode.py +++ b/hardware-specific-tests/picamera2/test_sensor_mode.py @@ -5,12 +5,16 @@ import numpy as np from labthings_fastapi.server import ThingServer from labthings_fastapi.client import ThingClient -from labthings_picamera2.thing import StreamingPiCamera2 + +from openflexure_microscope_server.things.camera.picamera import StreamingPiCamera2 logging.basicConfig(level=logging.DEBUG) def test_sensor_mode(): + """ + Test capturing raw arrays in two different sensor modes + """ cam = StreamingPiCamera2() server = ThingServer() server.add_thing(cam, "/camera/") diff --git a/hardware-specific-tests/picamera2/test_tuning.py b/hardware-specific-tests/picamera2/test_tuning.py index 65ab87f8..96691cc8 100644 --- a/hardware-specific-tests/picamera2/test_tuning.py +++ b/hardware-specific-tests/picamera2/test_tuning.py @@ -1,23 +1,30 @@ +""" +Tests that check that a tuning file can be reloaded +""" + import os -from picamera2 import Picamera2 -from labthings_picamera2 import recalibrate_utils import pytest +from picamera2 import Picamera2 + +from openflexure_microscope_server.things.camera import recalibrate_utils MODEL = Picamera2.global_camera_info()[0]["Model"] -def check_camera_available(): - assert len(Picamera2.global_camera_info()) >= 1 - - def load_default_tuning(): + """ + Return the default tuning file for the connected camera. + """ fname = f"{MODEL}.json" return Picamera2.load_tuning_file(fname) def generate_bad_tuning(): + """ + Return a tuning file with an invalid version number to force an error when loaded. + """ default_tuning = load_default_tuning() bad_tuning = default_tuning.copy() bad_tuning["version"] = 999 @@ -25,6 +32,14 @@ def generate_bad_tuning(): def print_tuning(read_file=False): + """ + Print the path of the default tuning file from the the environment variable. + + :param read_file: Boolean, set true to also print the file contents. + + This is useful for debuging. As PyTest suppresses the printing by default the + -s option is needed when running pylint to see this. + """ key = "LIBCAMERA_RPI_TUNING_FILE" if key in os.environ: print(f"Tuning file environment variable: {os.environ[key]}") @@ -36,6 +51,16 @@ def print_tuning(read_file=False): def _test_bad_tuning_after_good_tuning(configure): + """ + Load the default tuning file into the camera, re-load with a broken tuning file, + check it errors. Finally check the default tuning file will load again afterwards. + + :param configure: Boolean, set true to configure the camera on initial loading + + This test checks that: + recalibrate_utils.recreate_camera_manager() is working as expected. As the default + PiCamera2 behaviour does not expect the tuning file to be reloaded. + """ bad_tuning = generate_bad_tuning() default_tuning = load_default_tuning() print_tuning() @@ -61,6 +86,12 @@ def _test_bad_tuning_after_good_tuning(configure): del cam +# Note: The tests below are marked with +# @pytest.mark.filterwarnings("ignore: Exception ignored") +# as the picamera will throw an exception ignored warning when deleting +# the picamera object. This is expected + + @pytest.mark.filterwarnings("ignore: Exception ignored") def test_bad_tuning_after_good_tuning_noconfigure(): _test_bad_tuning_after_good_tuning(False) From 83832bf375a1cf582b37f035a5fb8f9ffe92e598 Mon Sep 17 00:00:00 2001 From: jaknapper Date: Tue, 24 Jun 2025 15:53:57 +0100 Subject: [PATCH 43/56] Update import name of recalibrate_utils --- hardware-specific-tests/picamera2/test_tuning.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hardware-specific-tests/picamera2/test_tuning.py b/hardware-specific-tests/picamera2/test_tuning.py index 96691cc8..572562b1 100644 --- a/hardware-specific-tests/picamera2/test_tuning.py +++ b/hardware-specific-tests/picamera2/test_tuning.py @@ -7,7 +7,9 @@ import os import pytest from picamera2 import Picamera2 -from openflexure_microscope_server.things.camera import recalibrate_utils +from openflexure_microscope_server.things.camera import ( + picamera_recalibrate_utils as recalibrate_utils, +) MODEL = Picamera2.global_camera_info()[0]["Model"] From 9966cb0fbb86d4a4cbc2ef734fecbe3a82f517ad Mon Sep 17 00:00:00 2001 From: jaknapper Date: Tue, 24 Jun 2025 15:55:19 +0100 Subject: [PATCH 44/56] Longer wait in exposure time test with fewer loops and lower exposure time --- .../picamera2/test_exposure_time_drift.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/hardware-specific-tests/picamera2/test_exposure_time_drift.py b/hardware-specific-tests/picamera2/test_exposure_time_drift.py index d7a24f00..cf8e1976 100644 --- a/hardware-specific-tests/picamera2/test_exposure_time_drift.py +++ b/hardware-specific-tests/picamera2/test_exposure_time_drift.py @@ -23,11 +23,12 @@ def test_exposure_time_drift(): with TestClient(server.app) as test_client: client = ThingClient.from_url("/camera/", client=test_client) - client.exposure_time = 50000 - time.sleep(0.1) + # 5444 is an exposure time supported by PiCamera2 + client.exposure_time = 5444 + time.sleep(0.5) initial_et = client.exposure_time print(f"Before capture, et is {client.exposure_time}") - for i in range(10): + for i in range(5): client.capture_jpeg(resolution="full") print(f"After capture, et is {client.exposure_time}") final_et = client.exposure_time From 4e78f3d104b111cef2fb86d04c50a4b79162a978 Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Tue, 24 Jun 2025 17:01:16 +0100 Subject: [PATCH 45/56] Fix exposure time adjusting when setting the same value and add test Closes #433 --- .../picamera2/test_exposure_time_drift.py | 49 +++++++++++++++---- .../things/camera/picamera.py | 7 +++ 2 files changed, 47 insertions(+), 9 deletions(-) diff --git a/hardware-specific-tests/picamera2/test_exposure_time_drift.py b/hardware-specific-tests/picamera2/test_exposure_time_drift.py index cf8e1976..1fe65e53 100644 --- a/hardware-specific-tests/picamera2/test_exposure_time_drift.py +++ b/hardware-specific-tests/picamera2/test_exposure_time_drift.py @@ -1,3 +1,10 @@ +""" +Check exposure times do not drift. + +This can get very tedious. Recommende running pytest with -s option +to monitor progress. +""" + import logging import time @@ -11,7 +18,7 @@ from openflexure_microscope_server.things.camera.picamera import StreamingPiCame logging.basicConfig(level=logging.DEBUG) -def test_exposure_time_drift(): +def _test_exposure_time_drift(desired_time): """ Capture 10 full resolution images and check that the exposure time remains constant @@ -23,16 +30,40 @@ def test_exposure_time_drift(): with TestClient(server.app) as test_client: client = ThingClient.from_url("/camera/", client=test_client) - # 5444 is an exposure time supported by PiCamera2 - client.exposure_time = 5444 + exposure_tol = cam.persistent_control_tolerances["ExposureTime"] + client.exposure_time = desired_time + print(f"Setting desired time of {desired_time}") time.sleep(0.5) - initial_et = client.exposure_time - print(f"Before capture, et is {client.exposure_time}") - for i in range(5): + pre_capture_et = client.exposure_time + print(f"Pre-capture the time is set to {pre_capture_et}") + # Check exp is set correctly within known tolerance + assert abs(pre_capture_et - desired_time) < exposure_tol + for i in range(10): client.capture_jpeg(resolution="full") - print(f"After capture, et is {client.exposure_time}") - final_et = client.exposure_time - assert initial_et == final_et + if i == 0: + # Exposure can update on first capture, due to frame rate restrictions + first_et = client.exposure_time + assert abs(first_et - pre_capture_et) < exposure_tol + frame_et = client.exposure_time + print(f"Frame {i} captured with exposure time {frame_et}") + # Check no further drift in value + assert first_et == frame_et + + # Set the exposure time to the value it already is. To check it doesn't shift + print(f"Setting exposure time to {frame_et} to check it doesn't change") + client.exposure_time = frame_et + time.sleep(0.5) + # Check before and after capture + assert client.exposure_time == frame_et + client.capture_jpeg(resolution="full") + assert client.exposure_time == frame_et + print("Exposure time didn't change!!") + print(f"End of test for exposure target {desired_time}") + + +def test_exposure_time_drift(): + for desired_time in [100, 1000, 10000]: + _test_exposure_time_drift(desired_time) if __name__ == "__main__": diff --git a/src/openflexure_microscope_server/things/camera/picamera.py b/src/openflexure_microscope_server/things/camera/picamera.py index 4372dabc..ecba1694 100644 --- a/src/openflexure_microscope_server/things/camera/picamera.py +++ b/src/openflexure_microscope_server/things/camera/picamera.py @@ -201,6 +201,13 @@ class StreamingPiCamera2(BaseCamera): "ExposureTime", int, description="The exposure time in microseconds" ) + @exposure_time.setter + def exposure_time(self, value: int): + with self.picamera() as cam: + # Note: This set a value 1 higher than requested as picamera2 always sets + # a lower value than requested, even if the requested is allowed + cam.set_controls({"ExposureTime": value + 1}) + _sensor_modes = None @thing_property From 8a4ad49c03f2406046770c23b73397c60aae5c72 Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Wed, 25 Jun 2025 09:17:32 +0000 Subject: [PATCH 46/56] Apply suggestions from code review of branch jpeg-capture-in-stacking Co-authored-by: Richard Bowman --- src/openflexure_microscope_server/things/camera/__init__.py | 2 -- src/openflexure_microscope_server/things/camera/picamera.py | 6 +++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/openflexure_microscope_server/things/camera/__init__.py b/src/openflexure_microscope_server/things/camera/__init__.py index 7d771af6..543db45d 100644 --- a/src/openflexure_microscope_server/things/camera/__init__.py +++ b/src/openflexure_microscope_server/things/camera/__init__.py @@ -146,8 +146,6 @@ class BaseCamera(Thing): ) -> None: """Capture an image and save it to disk - This will set the event `acquired` once the image has been acquired, so - that the stage may be moved while it's saved. save_resolution can be set to resize the image before saving. By default this is None meaning that the image is saved at original resoltion. diff --git a/src/openflexure_microscope_server/things/camera/picamera.py b/src/openflexure_microscope_server/things/camera/picamera.py index ecba1694..bef59148 100644 --- a/src/openflexure_microscope_server/things/camera/picamera.py +++ b/src/openflexure_microscope_server/things/camera/picamera.py @@ -141,7 +141,7 @@ class StreamingPiCamera2(BaseCamera): "%s while updating persistent controls.", key, self.persistent_controls[key], - self.persistent_controls[key], + value, ) else: self.persistent_controls[key] = value @@ -442,14 +442,14 @@ class StreamingPiCamera2(BaseCamera): @thing_action def capture_image( self, - stream_name: Literal["main", "lores", "raw", "full"] = "main", + stream_name: Literal["main", "lores", "raw"] = "main", wait: Optional[float] = 0.9, ): """Acquire one image from the camera. Return it as a PIL Image - stream_name: (Optional) The PiCamera2 stream to use, should be one of ["main", "lores", "raw", "full"]. Default = "main" + stream_name: (Optional) The PiCamera2 stream to use, should be one of ["main", "lores", "raw"]. Default = "main" wait: (Optional, float) Set a timeout in seconds. A TimeoutError is raised if this time is exceeded during capture. Default = 0.9s, lower than the 1s timeout default in picamera yaml settings From 59d15fb186ee2a5a655e5133558bfa0af6b7c66d Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Tue, 24 Jun 2025 17:41:05 +0100 Subject: [PATCH 47/56] Add further comment to picamera tests --- hardware-specific-tests/picamera2/test_tuning.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/hardware-specific-tests/picamera2/test_tuning.py b/hardware-specific-tests/picamera2/test_tuning.py index 572562b1..6ce4562b 100644 --- a/hardware-specific-tests/picamera2/test_tuning.py +++ b/hardware-specific-tests/picamera2/test_tuning.py @@ -93,6 +93,11 @@ def _test_bad_tuning_after_good_tuning(configure): # as the picamera will throw an exception ignored warning when deleting # the picamera object. This is expected +# The 2 pairs of repeated identical tests exist because pytest only +# starts once, and so by running the test more than one time checks +# that the camera can be initialised multiple times without restarting +# python. + @pytest.mark.filterwarnings("ignore: Exception ignored") def test_bad_tuning_after_good_tuning_noconfigure(): From 2d49bf20613b5b99f277caf117724d49824e0323 Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Wed, 25 Jun 2025 10:21:54 +0100 Subject: [PATCH 48/56] Remove duplicate exposure property from picamera thing --- .../things/camera/picamera.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/openflexure_microscope_server/things/camera/picamera.py b/src/openflexure_microscope_server/things/camera/picamera.py index bef59148..fcbffc4f 100644 --- a/src/openflexure_microscope_server/things/camera/picamera.py +++ b/src/openflexure_microscope_server/things/camera/picamera.py @@ -555,15 +555,6 @@ class StreamingPiCamera2(BaseCamera): piexif.insert(piexif.dump(exif_dict), path) return JPEGBlob.from_temporary_directory(folder, fname) - @thing_property - def exposure(self) -> float: - """An alias for `exposure_time` to fit the micromanager API""" - return self.exposure_time - - @exposure.setter # type: ignore - def exposure(self, value): - self.exposure_time = value - @thing_property def capture_metadata(self) -> dict: """Return the metadata from the camera""" From 0933ece63aeae990d92984f8cec1aec4de16340b Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Wed, 25 Jun 2025 14:22:01 +0000 Subject: [PATCH 49/56] Apply suggestions from code review of branch jpeg-capture-in-stacking Co-authored-by: Beth Probert --- .../picamera2/test_acquisition.py | 12 ++++++++++++ .../picamera2/test_exposure_time_drift.py | 15 ++++++++++----- .../picamera2/test_sensor_mode.py | 3 +++ hardware-specific-tests/picamera2/test_tuning.py | 12 ++++++------ pyproject.toml | 2 +- .../things/autofocus.py | 3 ++- .../things/camera/__init__.py | 2 ++ .../things/smart_scan.py | 9 +++++++-- 8 files changed, 43 insertions(+), 15 deletions(-) diff --git a/hardware-specific-tests/picamera2/test_acquisition.py b/hardware-specific-tests/picamera2/test_acquisition.py index 6d66cc5e..3d8ef5c0 100644 --- a/hardware-specific-tests/picamera2/test_acquisition.py +++ b/hardware-specific-tests/picamera2/test_acquisition.py @@ -11,6 +11,11 @@ from openflexure_microscope_server.things.camera.picamera import StreamingPiCame @fixture(scope="module") def client(): + """ + A pytest fixture that initialises a test client for the StreamingPiCamera2 Thing. + This fixture sets up a ThingServer, registers a StreamingPiCamera2 instance at the + "/camera/" endpoint, and provides a ThingClient for interacting with it during tests. + """ server = ThingServer() server.add_thing(StreamingPiCamera2(), "/camera/") with TestClient(server.app) as test_client: @@ -30,13 +35,20 @@ def test_jpeg_and_array(client): Check that grabbing a jpeg from the stream results in the same size image as a array capture or a jpeg capture. """ + # Grab a jpeg from the stream blob = client.grab_jpeg() mjpeg_frame = Image.open(blob.open()) assert mjpeg_frame + + # Capture a jpeg blob = client.capture_jpeg(resolution="main") jpeg_capture = Image.open(blob.open()) assert jpeg_capture + + # Capture an array arrlist = client.capture_array(stream_name="main") array_main = np.array(arrlist) + + # Verify image sizes are the same assert mjpeg_frame.size == jpeg_capture.size assert array_main.shape[1::-1] == jpeg_capture.size diff --git a/hardware-specific-tests/picamera2/test_exposure_time_drift.py b/hardware-specific-tests/picamera2/test_exposure_time_drift.py index 1fe65e53..d7ea142b 100644 --- a/hardware-specific-tests/picamera2/test_exposure_time_drift.py +++ b/hardware-specific-tests/picamera2/test_exposure_time_drift.py @@ -1,7 +1,7 @@ """ Check exposure times do not drift. -This can get very tedious. Recommende running pytest with -s option +This can get very tedious. Recommend running pytest with -s option to monitor progress. """ @@ -38,16 +38,18 @@ def _test_exposure_time_drift(desired_time): print(f"Pre-capture the time is set to {pre_capture_et}") # Check exp is set correctly within known tolerance assert abs(pre_capture_et - desired_time) < exposure_tol + for i in range(10): client.capture_jpeg(resolution="full") if i == 0: # Exposure can update on first capture, due to frame rate restrictions first_et = client.exposure_time assert abs(first_et - pre_capture_et) < exposure_tol - frame_et = client.exposure_time - print(f"Frame {i} captured with exposure time {frame_et}") - # Check no further drift in value - assert first_et == frame_et + else: + frame_et = client.exposure_time + print(f"Frame {i} captured with exposure time {frame_et}") + # Check no further drift in value + assert first_et == frame_et # Set the exposure time to the value it already is. To check it doesn't shift print(f"Setting exposure time to {frame_et} to check it doesn't change") @@ -62,6 +64,9 @@ def _test_exposure_time_drift(desired_time): def test_exposure_time_drift(): + """ + Performs the exposure time test for a range of exposure time values. + """ for desired_time in [100, 1000, 10000]: _test_exposure_time_drift(desired_time) diff --git a/hardware-specific-tests/picamera2/test_sensor_mode.py b/hardware-specific-tests/picamera2/test_sensor_mode.py index 84f6ce7c..57da8463 100644 --- a/hardware-specific-tests/picamera2/test_sensor_mode.py +++ b/hardware-specific-tests/picamera2/test_sensor_mode.py @@ -24,6 +24,9 @@ def test_sensor_mode(): for size in [(3280, 2464), (1640, 1232)]: client.sensor_mode = {"output_size": size, "bit_depth": 10} arr = np.array(client.capture_array(stream_name="raw")) + # Check that the array dimensions match the requested image size. + # Note: Numpy array shape is (y,x), but the sensor is set with (x,y) + # hence the need to compare index 0 with index 1. assert arr.shape[0] == size[1] diff --git a/hardware-specific-tests/picamera2/test_tuning.py b/hardware-specific-tests/picamera2/test_tuning.py index 6ce4562b..a6fd9b76 100644 --- a/hardware-specific-tests/picamera2/test_tuning.py +++ b/hardware-specific-tests/picamera2/test_tuning.py @@ -33,14 +33,14 @@ def generate_bad_tuning(): return bad_tuning -def print_tuning(read_file=False): +def print_tuning(read_file: bool = False): """ Print the path of the default tuning file from the the environment variable. :param read_file: Boolean, set true to also print the file contents. - This is useful for debuging. As PyTest suppresses the printing by default the - -s option is needed when running pylint to see this. + This is useful for debugging. As pytest suppresses the printing by default the + -s option is needed when running pytest to see this. """ key = "LIBCAMERA_RPI_TUNING_FILE" if key in os.environ: @@ -52,7 +52,7 @@ def print_tuning(read_file=False): print("Tuning file environment variable not set") -def _test_bad_tuning_after_good_tuning(configure): +def _test_bad_tuning_after_good_tuning(configure: bool = False): """ Load the default tuning file into the camera, re-load with a broken tuning file, check it errors. Finally check the default tuning file will load again afterwards. @@ -66,14 +66,14 @@ def _test_bad_tuning_after_good_tuning(configure): bad_tuning = generate_bad_tuning() default_tuning = load_default_tuning() print_tuning() - print("opening camera with explicitly specified tuning") + print("opening camera with default tuning") with Picamera2(tuning=default_tuning) as cam: print_tuning() if configure: cam.configure(cam.create_preview_configuration()) del cam recalibrate_utils.recreate_camera_manager() - print(f"Opening camera with tuning['version'] = {bad_tuning['version']}") + print(f"Opening camera with bad tuning - ['version'] = {bad_tuning['version']}") with pytest.raises(IndexError): # The bad version should cause a problem cam = Picamera2(tuning=bad_tuning) diff --git a/pyproject.toml b/pyproject.toml index 53f992e5..6d038360 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ dev = [ "matplotlib~=3.10" ] pi = [ - "picamera2~=0.3.12", + "picamera2~=0.3.27", ] [project.scripts] diff --git a/src/openflexure_microscope_server/things/autofocus.py b/src/openflexure_microscope_server/things/autofocus.py index 7e0d652c..e17d792d 100644 --- a/src/openflexure_microscope_server/things/autofocus.py +++ b/src/openflexure_microscope_server/things/autofocus.py @@ -294,6 +294,7 @@ class AutofocusThing(Thing): stack_z_range = stack_dz * (images_to_capture - 1) if stack_z_range > 0: + # Perform backlash corrected move. See issue #420 stage.move_relative(z=-(STACK_OVERSHOOT + stack_z_range / 2)) stage.move_relative(z=STACK_OVERSHOOT) time.sleep(0.3) @@ -348,7 +349,7 @@ class AutofocusThing(Thing): images_dir: str, stack_dir: str, logger: InvocationLogger, - ): + ) -> None: """Gets a list of images in a folder (stack_dir), sorts them by filesize, and copies the sharpest image to images dir.""" image_list = glob.glob(os.path.join(stack_dir, "*")) diff --git a/src/openflexure_microscope_server/things/camera/__init__.py b/src/openflexure_microscope_server/things/camera/__init__.py index 543db45d..b16ff8cc 100644 --- a/src/openflexure_microscope_server/things/camera/__init__.py +++ b/src/openflexure_microscope_server/things/camera/__init__.py @@ -31,6 +31,8 @@ class JPEGBlob(Blob): class PNGBlob(Blob): + """A class representing a PNG image as a LabThings FastAPI Blob""" + media_type: str = "image/png" diff --git a/src/openflexure_microscope_server/things/smart_scan.py b/src/openflexure_microscope_server/things/smart_scan.py index 618fec2f..0f7ee409 100644 --- a/src/openflexure_microscope_server/things/smart_scan.py +++ b/src/openflexure_microscope_server/things/smart_scan.py @@ -377,10 +377,13 @@ class SmartScanThing(Thing): return (next_point[0], next_point[1], z_estimate) @_scan_running - def _calc_displacement_from_test_image(self, overlap): + def _calc_displacement_from_test_image(self, overlap: int) -> tuple[int, int]: """ Take a test image and use camera stage mapping to calculate x and y displacement + :param overlap: The desired overlap as a fraction of the image. i.e. 0.5 means + that each image should overlap its nearest neighbour by 50%. + Return (dx, dy) - the x and y displacments in steps """ test_jpg = self._cam.grab_jpeg() @@ -422,7 +425,9 @@ class SmartScanThing(Thing): dx, dy = self._calc_displacement_from_test_image(overlap) stitch_resize = STITCHING_RESOLUTION[0] / self.capture_resolution[0] - self._scan_logger.debug(f"Resizing images when stitching by {stitch_resize}") + self._scan_logger.debug( + f"Resizing images when stitching by a factor of {stitch_resize}" + ) self._scan_logger.info( f"Based on an overlap of {overlap}, we will make steps of {dx}, {dy}" From ce5b5b781cff46cfde82900e8c49c0a7c963d9d2 Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Wed, 25 Jun 2025 15:55:48 +0100 Subject: [PATCH 50/56] Add typehints and docstrings as suggested in review --- .../things/camera/__init__.py | 15 ++++++++++++--- .../things/camera/picamera.py | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/openflexure_microscope_server/things/camera/__init__.py b/src/openflexure_microscope_server/things/camera/__init__.py index b16ff8cc..6c5652bd 100644 --- a/src/openflexure_microscope_server/things/camera/__init__.py +++ b/src/openflexure_microscope_server/things/camera/__init__.py @@ -76,7 +76,9 @@ class BaseCamera(Thing): self._memory_metadata = None @thing_action - def start_streaming(self, main_resolution, buffer_count) -> None: + def start_streaming( + self, main_resolution: tuple[int, int], buffer_count: int + ) -> None: """Start (or stop and restart) the camera with the given resolution for the main stream, and buffer_count number of images in the buffer""" raise NotImplementedError( @@ -132,7 +134,11 @@ class BaseCamera(Thing): return JPEGBlob.from_bytes(frame) @thing_action - def capture_image(self, stream_name, wait): + def capture_image( + self, + stream_name: Literal["main", "lores", "raw"], + wait: Optional[float], + ) -> None: """Capture a PIL image from stream stream_name with timeout wait""" raise NotImplementedError( "CameraThings must define their own capture_image method" @@ -211,7 +217,10 @@ class BaseCamera(Thing): ) -> Image: """Capture an image in memory and return it with metadata CaptureError raised if the capture fails for any reason - returns tuple with PIL Image, and dict of metadata + returns tuple with PIL Image, and dict of metadata. + + This robust capturing method attempts to capture the image five times + each time with a 5 second timeout set. """ for capture_attempts in range(5): try: diff --git a/src/openflexure_microscope_server/things/camera/picamera.py b/src/openflexure_microscope_server/things/camera/picamera.py index fcbffc4f..991f9381 100644 --- a/src/openflexure_microscope_server/things/camera/picamera.py +++ b/src/openflexure_microscope_server/things/camera/picamera.py @@ -444,7 +444,7 @@ class StreamingPiCamera2(BaseCamera): self, stream_name: Literal["main", "lores", "raw"] = "main", wait: Optional[float] = 0.9, - ): + ) -> None: """Acquire one image from the camera. Return it as a PIL Image From 0751457ed3a5cf04c21ab1d524d9f77174d68695 Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Wed, 25 Jun 2025 16:01:59 +0100 Subject: [PATCH 51/56] Remove rednundant if __main__ from hardware tests --- hardware-specific-tests/picamera2/test_exposure_time_drift.py | 4 ---- hardware-specific-tests/picamera2/test_sensor_mode.py | 4 ---- 2 files changed, 8 deletions(-) diff --git a/hardware-specific-tests/picamera2/test_exposure_time_drift.py b/hardware-specific-tests/picamera2/test_exposure_time_drift.py index d7ea142b..4e712e5e 100644 --- a/hardware-specific-tests/picamera2/test_exposure_time_drift.py +++ b/hardware-specific-tests/picamera2/test_exposure_time_drift.py @@ -69,7 +69,3 @@ def test_exposure_time_drift(): """ for desired_time in [100, 1000, 10000]: _test_exposure_time_drift(desired_time) - - -if __name__ == "__main__": - test_exposure_time_drift() diff --git a/hardware-specific-tests/picamera2/test_sensor_mode.py b/hardware-specific-tests/picamera2/test_sensor_mode.py index 57da8463..fbc575d6 100644 --- a/hardware-specific-tests/picamera2/test_sensor_mode.py +++ b/hardware-specific-tests/picamera2/test_sensor_mode.py @@ -28,7 +28,3 @@ def test_sensor_mode(): # Note: Numpy array shape is (y,x), but the sensor is set with (x,y) # hence the need to compare index 0 with index 1. assert arr.shape[0] == size[1] - - -if __name__ == "__main__": - test_sensor_mode() From 83e0e08ccce35c6188fae644e72131b0addf8980 Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Wed, 25 Jun 2025 15:12:28 +0000 Subject: [PATCH 52/56] Apply suggestions from code review of branch jpeg-capture-in-stacking Co-authored-by: Beth Probert --- .../things/camera/__init__.py | 1 + .../things/camera/picamera.py | 20 +++++++++---------- .../camera/picamera_recalibrate_utils.py | 12 +++++------ 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/openflexure_microscope_server/things/camera/__init__.py b/src/openflexure_microscope_server/things/camera/__init__.py index 6c5652bd..31cfe0e3 100644 --- a/src/openflexure_microscope_server/things/camera/__init__.py +++ b/src/openflexure_microscope_server/things/camera/__init__.py @@ -250,6 +250,7 @@ class BaseCamera(Thing): try: image.save(jpeg_path, quality=95, subsampling=0) try: + # Load EXIF metadata from image so it can be added to. exif_dict = piexif.load(jpeg_path) exif_dict["Exif"][piexif.ExifIFD.UserComment] = json.dumps( metadata diff --git a/src/openflexure_microscope_server/things/camera/picamera.py b/src/openflexure_microscope_server/things/camera/picamera.py index 991f9381..24fa2f94 100644 --- a/src/openflexure_microscope_server/things/camera/picamera.py +++ b/src/openflexure_microscope_server/things/camera/picamera.py @@ -152,7 +152,7 @@ class StreamingPiCamera2(BaseCamera): tolerance of current value. This allows ignoring small changes """ if key not in self.persistent_control_tolerances: - # Not tolerance range set, so it is not within tollerance + # Not tolerance range set, so it is not within tolerance return False difference = np.abs(self.persistent_controls[key] - value) @@ -419,7 +419,7 @@ class StreamingPiCamera2(BaseCamera): ) @thing_action - def stop_streaming(self, stop_web_stream=True) -> None: + def stop_streaming(self, stop_web_stream: bool = True) -> None: """ Stop the MJPEG stream """ @@ -617,7 +617,7 @@ class StreamingPiCamera2(BaseCamera): self.update_persistent_controls() @thing_action - def calibrate_lens_shading(self): + def calibrate_lens_shading(self) -> None: """Take an image and use it for flat-field correction. This method requires an empty (i.e. bright) field of view. It will take @@ -643,7 +643,7 @@ class StreamingPiCamera2(BaseCamera): ) @colour_correction_matrix.setter # type: ignore - def colour_correction_matrix(self, value): + def colour_correction_matrix(self, value) -> None: self.thing_settings["colour_correction_matrix"] = value self.calibrate_colour_correction(value) @@ -664,14 +664,14 @@ class StreamingPiCamera2(BaseCamera): self.colour_correction_matrix = c @thing_action - def calibrate_colour_correction(self, c: tuple): + def calibrate_colour_correction(self, c: tuple) -> None: """Overwrite the colour correction matrix in camera tuning""" with self.picamera(pause_stream=True): recalibrate_utils.set_static_ccm(self.tuning, c) self.initialise_picamera() @thing_action - def set_static_green_equalisation(self, offset: int = 65535): + def set_static_green_equalisation(self, offset: int = 65535) -> None: """Set the green equalisation to a static value. Green equalisation avoids the debayering algorithm becoming confused @@ -686,7 +686,7 @@ class StreamingPiCamera2(BaseCamera): self.initialise_picamera() @thing_action - def full_auto_calibrate(self): + def full_auto_calibrate(self) -> None: """Perform a full auto-calibration This function will call the other calibration actions in sequence: @@ -704,7 +704,7 @@ class StreamingPiCamera2(BaseCamera): self.calibrate_white_balance() @thing_action - def flat_lens_shading(self): + def flat_lens_shading(self) -> None: """Disable flat-field correction This method will set a completely flat lens shading table. It is not the @@ -786,7 +786,7 @@ class StreamingPiCamera2(BaseCamera): ) @thing_action - def flat_lens_shading_chrominance(self): + def flat_lens_shading_chrominance(self) -> None: """Disable flat-field correction This method will set the chrominance of the lens shading table to be @@ -801,7 +801,7 @@ class StreamingPiCamera2(BaseCamera): self.initialise_picamera() @thing_action - def reset_lens_shading(self): + def reset_lens_shading(self) -> None: """Revert to default lens shading settings This method will restore the default "adaptive" lens shading method used diff --git a/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py b/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py index 4f4d5224..d3073f75 100644 --- a/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py +++ b/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py @@ -69,13 +69,13 @@ def load_default_tuning(cam: Picamera2) -> dict: tuning_dir = "/usr/share/libcamera/ipa/raspberrypi" # from picamera2 v0.3.9 # The directory above has been removed from the search path seems - # find odd - as that's where the files currently are on a default + # odd - as that's where the files currently are on a default # Raspbian image. This may need updating if the files have moved # in future updates to the system libcamera package return cam.load_tuning_file(fname, dir=tuning_dir) -def set_minimum_exposure(camera: Picamera2): +def set_minimum_exposure(camera: Picamera2) -> None: """Enable manual exposure, with low gain and shutter speed We set exposure mode to manual, analog and digital gain @@ -135,7 +135,7 @@ def test_exposure_settings(camera: Picamera2, percentile: float) -> ExposureTest return result -def check_convergence(test: ExposureTest, target: int, tolerance: float): +def check_convergence(test: ExposureTest, target: int, tolerance: float) -> bool: """Check whether the brightness is within the specified target range""" return abs(test.level - target) < target * tolerance @@ -222,7 +222,7 @@ def adjust_shutter_and_gain_from_raw( # Check the gain is still changing - if not, we have probably hit the maximum if camera.capture_metadata()["AnalogueGain"] == test.analog_gain: - logging.info(f"Gain has maxed out. at {test.analog_gain}") + logging.info(f"Gain has maxed out at {test.analog_gain}") break if check_convergence(test, target_white_level, tolerance): @@ -504,7 +504,7 @@ def index_of_algorithm(algorithms: list[dict], algorithm: str) -> int: raise ValueError(f"Algorithm {algorithm} is not available.") -def copy_alsc_section(from_tuning: dict, to_tuning: dict): +def copy_alsc_section(from_tuning: dict, to_tuning: dict) -> None: """Copy the `rpi.alsc` algorithm from one tuning to another. This is done in-place, i.e. modifying to_tuning. @@ -544,7 +544,7 @@ def raw_channels_from_camera(camera: Picamera2) -> LensShadingTables: return channels_from_bayer_array(raw_image) -def recreate_camera_manager(): +def recreate_camera_manager() -> None: """Delete and recreate the camera manager. This is necessary to ensure the tuning file is re-read. From ee8c58c99fa60b10fa12d72929798d38e72839ff Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Wed, 25 Jun 2025 17:18:55 +0100 Subject: [PATCH 53/56] Docstrings, typehints, and clarifications in PiCamera and its utils --- .../things/camera/picamera.py | 126 +++++++++++++++--- .../camera/picamera_recalibrate_utils.py | 16 ++- 2 files changed, 119 insertions(+), 23 deletions(-) diff --git a/src/openflexure_microscope_server/things/camera/picamera.py b/src/openflexure_microscope_server/things/camera/picamera.py index 24fa2f94..fe60aa14 100644 --- a/src/openflexure_microscope_server/things/camera/picamera.py +++ b/src/openflexure_microscope_server/things/camera/picamera.py @@ -1,3 +1,20 @@ +""" +This SubModule interacts with a Raspberry Pi camera using the Picamera2 library. + +The Picamera2 library uses LibCamera as the underlying camera stack. This gives us +some control of the GPU pipeline for the image. + +The API documentation for PiCamera2 is unfortunatly not in a standard auto-generated +website. For documentation of the PiCamera2 API there is a PDF called +"The Picamera2 Library" available at: +https://datasheets.raspberrypi.com/camera/picamera2-manual.pdf + +For information on the algorithms used to tune/calibrate the Raspberry Pi Camera see +the guide called "Raspberry Pi Camera Algorithm and Tuning Guide" +Available at: +https://datasheets.raspberrypi.com/camera/raspberry-pi-camera-guide.pdf +""" + from __future__ import annotations from datetime import datetime import json @@ -71,6 +88,11 @@ class PicameraStreamOutput(Output): class SensorMode(BaseModel): + """ + A Pydantic model holding all the information about a specific + sensor mode as reported by the PiCamera. + """ + unpacked: str bit_depth: int size: tuple[int, int] @@ -81,11 +103,28 @@ class SensorMode(BaseModel): class SensorModeSelector(BaseModel): + """ + A Pydantic model holding the two values needed to select a PiCamera + Sensor mode. The output size and the bit depth. + + This is a Pydantic modell so that it can be saved to the disk. + """ + output_size: tuple[int, int] bit_depth: int class LensShading(BaseModel): + """ + A Pydantic model holding the lens shading tables. + + PiCamera needs three numpy arrays for lens shading correction. Each array is + (12, 16) in size. The arrays are luminance, red-difference chroma (Cr), and + blue-difference chroma (Cb). + + This is a Pydantic modell so that it can be saved to the disk. + """ + luminance: list[list[float]] Cr: list[list[float]] Cb: list[list[float]] @@ -203,6 +242,14 @@ class StreamingPiCamera2(BaseCamera): @exposure_time.setter def exposure_time(self, value: int): + """ + Custom setter for the above exposure_time PicameraControl. + + This is overriding the standard setter for a PicameraControl, as + the value needs to be adjusted before setting to behave as expected. + + See comment within the function for more detail. + """ with self.picamera() as cam: # Note: This set a value 1 higher than requested as picamera2 always sets # a lower value than requested, even if the requested is allowed @@ -375,8 +422,18 @@ class StreamingPiCamera2(BaseCamera): main_resolution: the resolution for the main configuration. Defaults to (820, 616), 1/4 sensor size. buffer_count: the number of frames to hold in the buffer. Higher uses more memory, - lower may cause dropped frames. Defaults to 6. + lower may cause dropped frames. Value must be between 1 and 8, Defaults to 6. """ + + # Buffer count can't be negative, zero, or too high. + if buffer_count < 1 or buffer_count < 8: + # 8 is slightly arbitrary. 6 is the PiCamera default for video + # and the documentation only says that setting values higher gives + # diminishing returns, and that the true maximum is hardware dependent + raise ValueError( + f"Can't set a buffer count of {buffer_count}. " + "Buffer count must be an integer from 1-8" + ) with self.picamera() as picam: try: if picam.started: @@ -388,7 +445,6 @@ class StreamingPiCamera2(BaseCamera): sensor=self.thing_settings.get("sensor_mode", None), controls=self.persistent_controls, ) - # Set buffer count - can't be negative stream_config["buffer_count"] = buffer_count picam.configure(stream_config) logging.info("Starting picamera MJPEG stream...") @@ -626,8 +682,10 @@ class StreamingPiCamera2(BaseCamera): the processed images. It should not affect raw images. """ with self.picamera(pause_stream=True) as cam: - # Suppress lint warning that L, Cr, and Cb are not lowercase, as this is the - # Standard format for these mathematical vars. + # Suppress lint warning that L, Cr, and Cb are not lowercase, as these are + # the standard mathematical terms for: + # luminance (L), red-difference chroma (Cr), and blue-difference chroma + # (Cb). L, Cr, Cb = recalibrate_utils.lst_from_camera(cam) # noqa: N806 recalibrate_utils.set_static_lst(self.tuning, L, Cr, Cb) self.initialise_picamera() @@ -649,8 +707,14 @@ class StreamingPiCamera2(BaseCamera): @thing_action def reset_ccm(self): - """Overwrite the colour correction matrix in camera tuning with default values from the documentation""" - c = [ + """ + Overwrite the colour correction matrix in camera tuning with default values. + + These values are from the Raspberry Pi Camera Algorithm and Tuning Guide, page + 45. + """ + # This is flattened 3x3 matrix. See `calibrate_colour_correction` + col_corr_matrix = [ 1.80439, -0.73699, -0.06739, @@ -661,13 +725,25 @@ class StreamingPiCamera2(BaseCamera): -0.56403, 1.64781, ] - self.colour_correction_matrix = c + self.colour_correction_matrix = col_corr_matrix @thing_action - def calibrate_colour_correction(self, c: tuple) -> None: - """Overwrite the colour correction matrix in camera tuning""" + def calibrate_colour_correction( + self, + col_corr_matrix: tuple[ + float, float, float, float, float, float, float, float, float + ], + ) -> None: + """Overwrite the colour correction matrix in camera tuning + + col_corr_matrix: This is a 9 value tuple used to specify the 3x3 + matrix that the GPU pipeline uses to convert from the camera R,G,B vector + to the standard R,G,B. + + See page Raspberry Pi Camera Algorithm and Tuning Guide, page 45. + """ with self.picamera(pause_stream=True): - recalibrate_utils.set_static_ccm(self.tuning, c) + recalibrate_utils.set_static_ccm(self.tuning, col_corr_matrix) self.initialise_picamera() @thing_action @@ -710,29 +786,43 @@ class StreamingPiCamera2(BaseCamera): This method will set a completely flat lens shading table. It is not the same as the default behaviour, which is to use an adaptive lens shading table. + + This flat table is used to take an image wth no lens shading so that the + correct lens shading table can be calibrated. """ with self.picamera(pause_stream=True): - f = np.ones((12, 16)) - recalibrate_utils.set_static_lst(self.tuning, f, f, f) + # Generate and array of ones of the correct size for each channel + flat_array = np.ones((12, 16)) + recalibrate_utils.set_static_lst( + self.tuning, flat_array, flat_array, flat_array + ) self.initialise_picamera() @thing_property def lens_shading_tables(self) -> Optional[LensShading]: """The current lens shading (i.e. flat-field correction) - This returns the current lens shading correction, as three 2D lists - each with dimensions 16x12. This assumes that we are using a static - lens shading table - if adaptive control is enabled, or if there - are multiple LSTs in use for different colour temperatures, - we return a null value to avoid confusion. + Return the current lens shading correction, as three 2D lists each with + dimensions 16x12, if a static lens shading table is in use. + + Return None if: + - adaptive control is enabled + - multiple LSTs in use (for different colour temperatures), """ if not self.lens_shading_is_static: return None + + # Note "alsc" is the Picamera2 term for "Automatic Lens Shading Correction" alsc = Picamera2.find_tuning_algo(self.tuning, "rpi.alsc") - if any(len(alsc[f"calibrations_C{c}"]) != 1 for c in ("r", "b")): + + # Check there is exactly 1 correction table for red-difference chroma (Cr) + # and blue-difference chroma (Cb) + if len(alsc["calibrations_Cr"]) != 1 or len(alsc["calibrations_Cb"]) != 1: + # If there is not exactly one table, then lens shading isn't static. return None def reshape_lst(lin: list[float]) -> list[list[float]]: + """Reshape the 192 element list into a 2D 16x12 list""" w, h = 16, 12 return [lin[w * i : w * (i + 1)] for i in range(h)] diff --git a/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py b/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py index d3073f75..cf51efe8 100644 --- a/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py +++ b/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py @@ -326,10 +326,11 @@ def channels_from_bayer_array(bayer_array: np.ndarray) -> np.ndarray: return channels -def get_16x12_grid(chan: np.ndarray, dx: int, dy: int): +def get_16x12_grid(chan: np.ndarray, dx: int, dy: int) -> np.ndarray: """Compresses channel down to a 16x12 grid - from libcamera - This is taken from https://git.linuxtv.org/libcamera.git/tree/utils/raspberrypi/ctt/ctt_alsc.py + This is taken from + https://git.linuxtv.org/libcamera.git/tree/utils/raspberrypi/ctt/ctt_alsc.py for consistency. """ grid = [] @@ -347,7 +348,7 @@ def get_16x12_grid(chan: np.ndarray, dx: int, dy: int): return np.reshape(np.array(grid), (12, 16)) -def upsample_channels(grids: np.ndarray, shape: tuple[int]): +def upsample_channels(grids: np.ndarray, shape: tuple[int]) -> np.ndarray: """Zoom an image in the last two dimensions This is effectively the inverse operation of `get_16x12_grid` @@ -452,14 +453,19 @@ def set_static_lst( alsc["luminance_lut"] = as_flat_rounded_list(luminance, round_to=3) -def set_static_ccm(tuning: dict, c: list) -> None: +def set_static_ccm( + tuning: dict, + col_corr_matrix: tuple[ + float, float, float, float, float, float, float, float, float + ], +) -> None: """Update the `rpi.alsc` section of a camera tuning dict to use a static correcton. `tuning` will be updated in-place to set its shading to static, and disable any adaptive tweaking by the algorithm. """ ccm = Picamera2.find_tuning_algo(tuning, "rpi.ccm") - ccm["ccms"] = [{"ct": 2860, "ccm": c}] + ccm["ccms"] = [{"ct": 2860, "ccm": col_corr_matrix}] def get_static_ccm(tuning: dict) -> None: From 0389ab760ae7bd7f36bc2302ae2e0d111bb8ea63 Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Wed, 25 Jun 2025 16:52:23 +0000 Subject: [PATCH 54/56] Apply suggestions from code review of branch jpeg-capture-in-stacking Co-authored-by: Joe Knapper --- src/openflexure_microscope_server/things/camera/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/openflexure_microscope_server/things/camera/__init__.py b/src/openflexure_microscope_server/things/camera/__init__.py index 31cfe0e3..14376e0f 100644 --- a/src/openflexure_microscope_server/things/camera/__init__.py +++ b/src/openflexure_microscope_server/things/camera/__init__.py @@ -248,6 +248,10 @@ class BaseCamera(Thing): if save_resolution is not None and image.size != save_resolution: image = image.resize(save_resolution, Image.BOX) try: + # Per PIL documentation, (https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html#jpeg) + # there are two factors when saving a JPEG. subsampling affects the colour, quality the pixels + # subsampling = 0 disables subsampling of colour + # quality = 95 is the maximum recommended - above this, JPEG compression is disabled, file size increases and quality is barely or not affected image.save(jpeg_path, quality=95, subsampling=0) try: # Load EXIF metadata from image so it can be added to. From 761957d7fb3e32ac11f4e143bc69711a38466939 Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Wed, 25 Jun 2025 17:53:43 +0100 Subject: [PATCH 55/56] Clarify exposure variables in docstring --- .../things/camera/picamera.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/openflexure_microscope_server/things/camera/picamera.py b/src/openflexure_microscope_server/things/camera/picamera.py index fe60aa14..fce23500 100644 --- a/src/openflexure_microscope_server/things/camera/picamera.py +++ b/src/openflexure_microscope_server/things/camera/picamera.py @@ -623,11 +623,17 @@ class StreamingPiCamera2(BaseCamera): target_white_level: int = 700, percentile: float = 99.9, ): - """Adjust exposure to hit the target white level + """Adjust exposure until a the target white level is reached - Starting from the minimum exposure, we gradually increase exposure until - we hit the specified white level. We use a percentile rather than the - maximum, in order to be robust to a small number of noisy/bright pixels. + Starting from the minimum exposure, gradually increase exposure until + the image reaches the specified white level. + + :param target_white_level: The target 10bit white level. 10-bit data has a + theoretical maximum of 1023, but with black level correction the true maxiumum + is about 950. Default is 700 as this is approximately 70% saturated. + :param percentile: The percentile to use instead of maximum. Default 99.9. When + calculating the brightest pixel, a percentile is used rather than the + maximum in order to be robust to a small number of noisy/bright pixels. """ with self.picamera(pause_stream=True) as cam: recalibrate_utils.adjust_shutter_and_gain_from_raw( From b49aecb8027d9fffd1e95a315863ff0d1cd5cd12 Mon Sep 17 00:00:00 2001 From: Joe Knapper Date: Wed, 25 Jun 2025 17:32:05 +0000 Subject: [PATCH 56/56] Apply suggestions from code review of branch jpeg-capture-in-stacking --- src/openflexure_microscope_server/things/camera/picamera.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/openflexure_microscope_server/things/camera/picamera.py b/src/openflexure_microscope_server/things/camera/picamera.py index fce23500..56f8ff84 100644 --- a/src/openflexure_microscope_server/things/camera/picamera.py +++ b/src/openflexure_microscope_server/things/camera/picamera.py @@ -4,7 +4,7 @@ This SubModule interacts with a Raspberry Pi camera using the Picamera2 library. The Picamera2 library uses LibCamera as the underlying camera stack. This gives us some control of the GPU pipeline for the image. -The API documentation for PiCamera2 is unfortunatly not in a standard auto-generated +The API documentation for PiCamera2 is unfortunately not in a standard auto-generated website. For documentation of the PiCamera2 API there is a PDF called "The Picamera2 Library" available at: https://datasheets.raspberrypi.com/camera/picamera2-manual.pdf @@ -426,7 +426,7 @@ class StreamingPiCamera2(BaseCamera): """ # Buffer count can't be negative, zero, or too high. - if buffer_count < 1 or buffer_count < 8: + if buffer_count < 1 or buffer_count > 8: # 8 is slightly arbitrary. 6 is the PiCamera default for video # and the documentation only says that setting values higher gives # diminishing returns, and that the true maximum is hardware dependent