From f24272bc7f12457b85bc810776a3679a148b37a6 Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Wed, 17 Sep 2025 23:54:17 +0100 Subject: [PATCH] Set sensor_model as kwarg to SteamingPicameraThing. From this load sensor information. --- .../picamera2/test_tuning.py | 4 +- .../things/camera/picamera.py | 72 +++++--- .../camera/picamera_recalibrate_utils.py | 157 ++++++++++++------ .../camera/picamera_tuning_file_utils.py | 16 +- 4 files changed, 164 insertions(+), 85 deletions(-) diff --git a/hardware-specific-tests/picamera2/test_tuning.py b/hardware-specific-tests/picamera2/test_tuning.py index 5932af6f..b193131d 100644 --- a/hardware-specific-tests/picamera2/test_tuning.py +++ b/hardware-specific-tests/picamera2/test_tuning.py @@ -18,7 +18,7 @@ MODEL = Picamera2.global_camera_info()[0]["Model"] def generate_bad_tuning(): """Return a tuning file with an invalid version number to force an error when loaded.""" - default_tuning = tf_utils.load_default_tuning() + default_tuning = tf_utils.load_default_tuning("imx219") bad_tuning = default_tuning.copy() bad_tuning["version"] = 999 return bad_tuning @@ -55,7 +55,7 @@ def _test_bad_tuning_after_good_tuning(configure: bool = False): PiCamera2 behaviour does not expect the tuning file to be reloaded. """ bad_tuning = generate_bad_tuning() - default_tuning = tf_utils.load_default_tuning() + default_tuning = tf_utils.load_default_tuning("imx219") print_tuning() print("opening camera with default tuning") with Picamera2(tuning=default_tuning) as cam: diff --git a/src/openflexure_microscope_server/things/camera/picamera.py b/src/openflexure_microscope_server/things/camera/picamera.py index a6096372..8d00b007 100644 --- a/src/openflexure_microscope_server/things/camera/picamera.py +++ b/src/openflexure_microscope_server/things/camera/picamera.py @@ -47,6 +47,15 @@ from . import picamera_tuning_file_utils as tf_utils from . import BaseCamera, ArrayModel +SUPPORTED_SENSOR_INFO = { + "imx219": recalibrate_utils.IMX219_SENSOR_INFO, + "imx477": recalibrate_utils.IMX477_SENSOR_INFO, +} + + +class PicameraModelError(RuntimeError): + """There is a problem Picamera sensor model set by the configuration.""" + class MissingCalibrationError(RuntimeError): """Picamera tuning file is missing or doesn't contain the requested algorithm.""" @@ -130,24 +139,30 @@ class StreamingPiCamera2(BaseCamera): generalisation. """ - def __init__(self, camera_num: int = 0) -> None: + def __init__(self, camera_num: int = 0, sensor_model: str = "imx219") -> None: """Initialise the camera with the given camera number. This makes no connection to the camera (except to get the default tuning file). :param camera_num: The number of the camera. This should generally be left as 0 as most Raspberry Pi boards only support 1 camera. + :param sensor_model: The sensor model of the image sensor on this picamera. """ super().__init__() self._setting_save_in_progress = False - self.camera_num = camera_num - self.camera_configs: dict[str, dict] = {} + self._camera_num = camera_num + self._sensor_model = sensor_model + if sensor_model not in SUPPORTED_SENSOR_INFO: + raise PicameraModelError( + f"The sensor model {sensor_model} is not supported." + ) + self._sensor_info = SUPPORTED_SENSOR_INFO[sensor_model] self._picamera_lock = None self._picamera = None - logging.info("Starting & reconfiguring camera to populate sensor_modes.") - with Picamera2(camera_num=self.camera_num) as cam: - self.default_tuning = tf_utils.load_default_tuning(cam) - logging.info("Done reading sensor modes & default tuning.") + + # Load the tuning file for the specified sensor mode. + self.default_tuning = tf_utils.load_default_tuning(sensor_model) + # Set tuning to default tuning. This will be overwritten when the Thing is # connects to the server if tuning is saved to disk. try: @@ -356,11 +371,18 @@ class StreamingPiCamera2(BaseCamera): ) from e return None - def _initialise_picamera(self) -> None: + def _initialise_picamera(self, check_sensor_model: bool = False) -> None: """Acquire the picamera device and store it as ``self._picamera``. This duplicates logic in ``Picamera2.__init__`` to provide a tuning file that will be read when the camera system initialises. + + :param check_sensor_model: Set to true to check the sensor model is the + expected sensor model. This is used on ``__enter__`` to confirm that the + real camera matches the expected camera. + + :raises PicameraModelError: If check_sensor_model is True and the real + camera sensor model doesn't match the expected sensor model. """ if self._picamera_lock is not None: # Don't close the camera if it's in use @@ -383,9 +405,16 @@ class StreamingPiCamera2(BaseCamera): logging.info("Creating new Picamera2 object") # Specify tuning file otherwise it will be overwritten with None. self._picamera = Picamera2( - camera_num=self.camera_num, + camera_num=self._camera_num, tuning=self.tuning, ) + if check_sensor_model: + hw_sensor_model = self._picamera.camera_properties["Model"] + if hw_sensor_model != self._sensor_model: + raise PicameraModelError( + f"Wrong Picamera model. Expecting {self._sensor_model}, but " + f"found {hw_sensor_model}." + ) self._picamera_lock = RLock() def __enter__(self) -> None: @@ -394,7 +423,7 @@ class StreamingPiCamera2(BaseCamera): This opens the picamera connection, initialises the camera, sets the sensor_modes property, and then starts the streams. """ - self._initialise_picamera() + self._initialise_picamera(check_sensor_model=True) # Sensor modes is a cached property read it once after initialising the camera _modes = self.sensor_modes self.start_streaming() @@ -531,7 +560,7 @@ class StreamingPiCamera2(BaseCamera): logging.info("Stopped MJPEG stream.") # Adding a sleep to prevent camera getting confused by rapid commands - time.sleep(0.2) + time.sleep(self._sensor_info.short_pause) @lt.thing_action def discard_frames(self) -> None: @@ -549,7 +578,7 @@ class StreamingPiCamera2(BaseCamera): logging.debug("Reconfiguring camera for full resolution capture") cam.configure(cam.create_still_configuration(sensor=self._sensor_mode)) cam.start() - time.sleep(0.2) + time.sleep(self._sensor_info.short_pause) yield cam def capture_image( @@ -648,7 +677,7 @@ class StreamingPiCamera2(BaseCamera): @lt.thing_action def auto_expose_from_minimum( self, - target_white_level: int = 3000, + target_white_level: Optional[int] = None, percentile: float = 99.9, ) -> None: """Adjust exposure until a the target white level is reached. @@ -656,17 +685,21 @@ class StreamingPiCamera2(BaseCamera): Starting from the minimum exposure, gradually increase exposure until the image reaches the specified white level. - :param target_white_level: The target 10bit white level. 10-bit data has a - theoretical maximum of 1023, but with black level correction the true - maximum is about 950. Default is 700 as this is approximately 70% - saturated. + :param target_white_level: Raw target white level, this should be an integer + within the range set by the bit-depth of the camera sensor (10-bit for + PiCamera v2, 12 Bit for Picamera HQ. If None the default will be used for + the current sensor. This is approximately 70% saturated. :param percentile: The percentile to use instead of maximum. Default 99.9. When calculating the brightest pixel, a percentile is used rather than the maximum in order to be robust to a small number of noisy/bright pixels. """ + if target_white_level is None: + target_white_level = self._sensor_info.default_target_white_level + with self._streaming_picamera(pause_stream=True) as cam: recalibrate_utils.adjust_shutter_and_gain_from_raw( cam, + self._sensor_info, target_white_level=target_white_level, percentile=percentile, ) @@ -693,6 +726,7 @@ class StreamingPiCamera2(BaseCamera): lst: LensShading = self.lens_shading_tables recalibrate_utils.adjust_white_balance_from_raw( cam, + self._sensor_info, percentile=99, luminance=lst.luminance, Cr=lst.Cr, @@ -702,7 +736,7 @@ class StreamingPiCamera2(BaseCamera): ) else: recalibrate_utils.adjust_white_balance_from_raw( - cam, percentile=99, method=method + cam, self._sensor_info, percentile=99, method=method ) @lt.thing_action @@ -719,7 +753,7 @@ class StreamingPiCamera2(BaseCamera): # the standard mathematical terms for: # luminance (L), red-difference chroma (Cr), and blue-difference chroma # (Cb). - L, Cr, Cb = recalibrate_utils.lst_from_camera(cam) # noqa: N806 + L, Cr, Cb = recalibrate_utils.lst_from_camera(cam, self._sensor_info) # noqa: N806 tf_utils.set_static_lst(self.tuning, L, Cr, Cb) self._initialise_picamera() diff --git a/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py b/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py index 837ba4a7..dc425443 100644 --- a/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py +++ b/src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py @@ -22,10 +22,15 @@ reliable. The three steps above can be accomplished by: .. code-block:: python picamera = picamera2.Picamera2() + sensor_info = IMX219_SENSOR_INFO - adjust_shutter_and_gain_from_raw(picamera) - adjust_white_balance_from_raw(picamera) - lst = lst_from_camera(picamera) + adjust_shutter_and_gain_from_raw( + picamera, + sensor_info, + target_white_level=sensor_info.default_target_white_level, + ) + adjust_white_balance_from_raw(picamera, sensor_info) + lst = lst_from_camera(picamera, sensor_info) picamera.lens_shading_table = lst """ @@ -48,29 +53,54 @@ from picamera2 import Picamera2 import picamera2 +class SensorInfo(BaseModel): + """Information about the sensor used for calibration and property setting.""" + + unpacked_pixel_format: str + """The format of the unpacked pixels.""" + + bit_depth: int + """The bit depth of each pixel.""" + + blacklevel: int + """The sensor black level.""" + + default_target_white_level: int + """The default target white level during exposure setting.""" + + short_pause: float + """The time to pause for actions that update quickly.""" + + long_pause: float + """Time to pause for actions that are known to update slowly.""" + + +IMX219_SENSOR_INFO = SensorInfo( + unpacked_pixel_format="SBGGR10", + bit_depth=10, + blacklevel=64, + default_target_white_level=700, + short_pause=0.2, + long_pause=0.5, +) + +IMX477_SENSOR_INFO = SensorInfo( + unpacked_pixel_format="SBGGR12", + bit_depth=12, + blacklevel=256, + default_target_white_level=2800, + short_pause=0.2, + long_pause=0.5, +) + + LensShadingTables = tuple[np.ndarray, np.ndarray, np.ndarray] -def set_minimum_exposure(camera: Picamera2) -> None: - """Enable manual exposure, with low gain and shutter speed. - - We set exposure mode to manual, analog and digital gain - to 1, and shutter speed to the minimum (8us for Pi Camera v2) - - Note ISO is left at auto, because this is needed for the gains - to be set correctly. - """ - # Disable Automatic exposure and gain algorithm (AeEnable), and set analogue - # gain and exposure time. - # Setting the shutter speed to 1us will result in it being set - # to the minimum possible, which is ~8us for PiCamera v2 - camera.set_controls({"AeEnable": False, "AnalogueGain": 1, "ExposureTime": 1}) - time.sleep(1) - - def adjust_shutter_and_gain_from_raw( camera: Picamera2, - target_white_level: int = 3000, + sensor_info: SensorInfo, + target_white_level: int, max_iterations: int = 20, tolerance: float = 0.05, percentile: float = 99.9, @@ -81,9 +111,12 @@ def adjust_shutter_and_gain_from_raw( are not affected by white balance or digital gain. :param camera: A Picamera2 object. - :param target_white_level: The raw, 10-bit value we aim for. The brightest pixels - should be approximately this bright. Maximum possible is about 900, 700 is - reasonable. + :param target_white_level: The raw value we aim for, the raw value of the brightest + pixels should be approximately this bright. The value to set depends on the + sensor bit depth. We recommend values of 700 for 10-bit sensors and 2800 for + 12-bit sensors. This is about 70% of saturated once the blacklevel is + subtracted. The maximum possible value depends on the sensor bit depth, the + sensor blackleve, the tolerance argument. :param max_iterations: We will terminate once we perform this many iterations, whether or not we converge. More than 10 shouldn't happen. :param tolerance: How close to the target value we consider "done". Expressed as a @@ -94,18 +127,20 @@ def adjust_shutter_and_gain_from_raw( than just ``np.max()``. """ - # TODO: read black level and bit depth from camera? - if target_white_level * (tolerance + 1) >= 3850: + max_level = 2**sensor_info.bit_depth - 1 - sensor_info.blacklevel + if target_white_level * (tolerance + 1) >= max_level: raise ValueError( "The target level is too high - a saturated image would be " "considered successful. target_white_level * (tolerance + 1) " - "must be less than 3850." + f"must be less than {max_level}." ) - config = camera.create_still_configuration(raw={"format": "SBGGR12"}) + config = camera.create_still_configuration( + raw={"format": sensor_info.unpacked_pixel_format} + ) camera.configure(config) camera.start() - set_minimum_exposure(camera) + _set_minimum_exposure(camera, sensor_info) # We start with very low exposure settings and work up # until either the brightness is high enough, or we can't increase the @@ -122,7 +157,7 @@ def adjust_shutter_and_gain_from_raw( new_time = int(test.exposure_time * min(target_white_level / test.level, 8)) camera.controls.ExposureTime = new_time camera.controls.AeEnable = False - time.sleep(1) + time.sleep(sensor_info.long_pause) # Check whether the shutter speed is still going up - if not, we've hit a maximum if camera.capture_metadata()["ExposureTime"] == test.exposure_time: @@ -140,7 +175,7 @@ def adjust_shutter_and_gain_from_raw( camera.controls.AnalogueGain = test.analog_gain * min( target_white_level / test.level, 2 ) - time.sleep(1) + time.sleep(sensor_info.long_pause) # Check the gain is still changing - if not, we have probably hit the maximum if camera.capture_metadata()["AnalogueGain"] == test.analog_gain: @@ -160,6 +195,7 @@ def adjust_shutter_and_gain_from_raw( def adjust_white_balance_from_raw( camera: Picamera2, + sensor_info: SensorInfo, percentile: float = 99, luminance: Optional[np.ndarray] = None, Cr: Optional[np.ndarray] = None, @@ -173,12 +209,13 @@ def adjust_white_balance_from_raw( We should probably have better logic to verify the channels really are BGGR... """ - config = camera.create_still_configuration(raw={"format": "SBGGR12"}) + config = camera.create_still_configuration( + raw={"format": sensor_info.unpacked_pixel_format} + ) camera.configure(config) camera.start() channels = _channels_from_bayer_array(camera.capture_array("raw")) - # TODO: read black level from camera rather than hard-coding 64 - blacklevel = 256 + if luminance is not None and Cr is not None and Cb is not None: # Reconstruct a low-resolution image from the lens shading tables # and use it to normalise the raw image, to compensate for @@ -205,10 +242,10 @@ def adjust_white_balance_from_raw( axis=(1, 2), ) # Subtract blacklevel before splitting into channels - blue, g1, g2, red = centre_means - blacklevel + blue, g1, g2, red = centre_means - sensor_info.blacklevel else: blue, g1, g2, red = ( - np.percentile(channels, percentile, axis=(1, 2)) - blacklevel + np.percentile(channels, percentile, axis=(1, 2)) - sensor_info.blacklevel ) green = (g1 + g2) / 2.0 new_awb_gains = (green / red, green / blue) @@ -225,16 +262,16 @@ def adjust_white_balance_from_raw( ) camera.controls.AwbEnable = False camera.controls.ColourGains = new_awb_gains - time.sleep(1) + time.sleep(sensor_info.long_pause) m = camera.capture_metadata() print(f"Camera confirms gains are now {m['ColourGains']}") return new_awb_gains -def lst_from_camera(camera: Picamera2) -> LensShadingTables: +def lst_from_camera(camera: Picamera2, sensor_info: SensorInfo) -> LensShadingTables: """Acquire a raw image and use it to calculate a lens shading table.""" - channels = _raw_channels_from_camera(camera) - return _lst_from_channels(channels) + channels = _raw_channels_from_camera(camera, sensor_info) + return _lst_from_channels(channels, sensor_info.blacklevel) def recreate_camera_manager() -> None: @@ -255,6 +292,23 @@ class _ExposureTest(BaseModel): analog_gain: float +def _set_minimum_exposure(camera: Picamera2, sensor_info: SensorInfo) -> None: + """Enable manual exposure, with low gain and shutter speed. + + Set exposure mode to manual, analog and digital gain to 1, and + shutter speed to the minimum (8us for Pi Camera v2) + + Note ISO is left at auto, because this is needed for the gains + to be set correctly. + """ + # Disable Automatic exposure and gain algorithm (AeEnable), and set analogue + # gain and exposure time. + # Setting the shutter speed to 1us will result in it being set + # to the minimum possible, which is ~8us for PiCamera v2 + camera.set_controls({"AeEnable": False, "AnalogueGain": 1, "ExposureTime": 1}) + time.sleep(sensor_info.long_pause) + + def _test_exposure_settings(camera: Picamera2, percentile: float) -> _ExposureTest: """Evaluate current exposure settings using a raw image. @@ -345,13 +399,8 @@ def _upsample_channels(grids: np.ndarray, shape: tuple[int]) -> np.ndarray: return zoom(grids, zoom_factors, order=1)[:, : shape[0], : shape[1]] -def _downsampled_channels( - channels: np.ndarray, blacklevel: int = 256 -) -> list[np.ndarray]: - """Generate a downsampled, un-normalised image from which to calculate the LST. - - TODO: blacklevel probably ought to be determined from the camera... - """ +def _downsampled_channels(channels: np.ndarray, blacklevel: int) -> list[np.ndarray]: + """Generate a downsampled, un-normalised image from which to calculate the LST.""" channel_shape = np.array(channels.shape[1:]) lst_shape = np.array([12, 16]) step = np.ceil(channel_shape / lst_shape).astype(int) @@ -366,12 +415,12 @@ def _downsampled_channels( ) -def _lst_from_channels(channels: np.ndarray) -> LensShadingTables: +def _lst_from_channels(channels: np.ndarray, blacklevel: int) -> LensShadingTables: """Given the 4 Bayer colour channels from a white image, generate a LST. Internally, is just calls ``_downsampled_channels`` and ``_lst_from_grids``. """ - grids = _downsampled_channels(channels) + grids = _downsampled_channels(channels, blacklevel) return _lst_from_grids(grids) @@ -415,15 +464,17 @@ def _grids_from_lst(lum: np.ndarray, Cr: np.ndarray, Cb: np.ndarray) -> np.ndarr return np.stack([B, G, G, R], axis=0) -def _raw_channels_from_camera(camera: Picamera2) -> LensShadingTables: +def _raw_channels_from_camera( + camera: Picamera2, sensor_info: SensorInfo +) -> LensShadingTables: """Acquire a raw image and return a 4xNxM array of the colour channels.""" if camera.started: camera.stop_recording() # We will acquire a raw image with unpacked pixels, which is what the # format below requests. Bit depth and Bayer order may be overwritten. - # TODO: don't assume 10-bit - the high quality camera uses 12. - # TODO: what's the best mode to use here? - config = camera.create_still_configuration(raw={"format": "SBGGR12"}) + config = camera.create_still_configuration( + raw={"format": sensor_info.unpacked_pixel_format} + ) camera.configure(config) camera.start() raw_image = camera.capture_array("raw") diff --git a/src/openflexure_microscope_server/things/camera/picamera_tuning_file_utils.py b/src/openflexure_microscope_server/things/camera/picamera_tuning_file_utils.py index 95df25b1..00e5a0d3 100644 --- a/src/openflexure_microscope_server/things/camera/picamera_tuning_file_utils.py +++ b/src/openflexure_microscope_server/things/camera/picamera_tuning_file_utils.py @@ -8,20 +8,14 @@ from picamera2 import Picamera2 import numpy as np -def load_default_tuning(cam: Picamera2) -> dict: +def load_default_tuning(sensor_model: str) -> dict: """Load the default tuning file for the camera. - This will open and close the camera to determine its model. If you are - using a model that's supported by ``picamera2`` it should have a tuning - file built in. If not, this will probably crash with an error. - - Error handling for unsupported cameras is not something we are likely - to test in the short term. + This will loat the tuning file based on the specified sensor model. """ - cp = cam.camera_properties - fname = f"{cp['Model']}.json" + fname = f"{sensor_model}.json" try: - return cam.load_tuning_file(fname) + return Picamera2.load_tuning_file(fname) except RuntimeError: tuning_dir = "/usr/share/libcamera/ipa/raspberrypi" # from picamera2 v0.3.9 @@ -29,7 +23,7 @@ def load_default_tuning(cam: Picamera2) -> dict: # odd - as that's where the files currently are on a default # Raspbian image. This may need updating if the files have moved # in future updates to the system libcamera package - return cam.load_tuning_file(fname, dir=tuning_dir) + return Picamera2.load_tuning_file(fname, dir=tuning_dir) def set_static_lst(