rahix/openflexure-microscope-server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Set sensor_model as kwarg to SteamingPicameraThing. From this load sensor information.

Browse source
This commit is contained in:
Julian Stirling 2025-09-17 23:54:17 +01:00
parent c16a0391df
commit f24272bc7f
4 changed files with 164 additions and 85 deletions
Download patch file Download diff file Expand all files Collapse all files

72
src/openflexure_microscope_server/things/camera/picamera.py
View file

@ -47,6 +47,15 @@ from . import picamera_tuning_file_utils as tf_utils
from . import BaseCamera, ArrayModel
SUPPORTED_SENSOR_INFO = {
"imx219": recalibrate_utils.IMX219_SENSOR_INFO,
"imx477": recalibrate_utils.IMX477_SENSOR_INFO,
}
class PicameraModelError(RuntimeError):
"""There is a problem Picamera sensor model set by the configuration."""
class MissingCalibrationError(RuntimeError):
"""Picamera tuning file is missing or doesn't contain the requested algorithm."""
@ -130,24 +139,30 @@ class StreamingPiCamera2(BaseCamera):
generalisation.
"""
def __init__(self, camera_num: int = 0) -> None:
def __init__(self, camera_num: int = 0, sensor_model: str = "imx219") -> None:
"""Initialise the camera with the given camera number.
This makes no connection to the camera (except to get the default tuning file).
:param camera_num: The number of the camera. This should generally be left as 0
as most Raspberry Pi boards only support 1 camera.
:param sensor_model: The sensor model of the image sensor on this picamera.
"""
super().__init__()
self._setting_save_in_progress = False
self.camera_num = camera_num
self.camera_configs: dict[str, dict] = {}
self._camera_num = camera_num
self._sensor_model = sensor_model
if sensor_model not in SUPPORTED_SENSOR_INFO:
raise PicameraModelError(
f"The sensor model {sensor_model} is not supported."
)
self._sensor_info = SUPPORTED_SENSOR_INFO[sensor_model]
self._picamera_lock = None
self._picamera = None
logging.info("Starting & reconfiguring camera to populate sensor_modes.")
with Picamera2(camera_num=self.camera_num) as cam:
self.default_tuning = tf_utils.load_default_tuning(cam)
logging.info("Done reading sensor modes & default tuning.")
# Load the tuning file for the specified sensor mode.
self.default_tuning = tf_utils.load_default_tuning(sensor_model)
# Set tuning to default tuning. This will be overwritten when the Thing is
# connects to the server if tuning is saved to disk.
try:
@ -356,11 +371,18 @@ class StreamingPiCamera2(BaseCamera):
) from e
return None
def _initialise_picamera(self) -> None:
def _initialise_picamera(self, check_sensor_model: bool = False) -> None:
"""Acquire the picamera device and store it as ``self._picamera``.
This duplicates logic in ``Picamera2.__init__`` to provide a tuning file that
will be read when the camera system initialises.
:param check_sensor_model: Set to true to check the sensor model is the
expected sensor model. This is used on ``__enter__`` to confirm that the
real camera matches the expected camera.
:raises PicameraModelError: If check_sensor_model is True and the real
camera sensor model doesn't match the expected sensor model.
"""
if self._picamera_lock is not None:
# Don't close the camera if it's in use
@ -383,9 +405,16 @@ class StreamingPiCamera2(BaseCamera):
logging.info("Creating new Picamera2 object")
# Specify tuning file otherwise it will be overwritten with None.
self._picamera = Picamera2(
camera_num=self.camera_num,
camera_num=self._camera_num,
tuning=self.tuning,
)
if check_sensor_model:
hw_sensor_model = self._picamera.camera_properties["Model"]
if hw_sensor_model != self._sensor_model:
raise PicameraModelError(
f"Wrong Picamera model. Expecting {self._sensor_model}, but "
f"found {hw_sensor_model}."
)
self._picamera_lock = RLock()
def __enter__(self) -> None:
@ -394,7 +423,7 @@ class StreamingPiCamera2(BaseCamera):
This opens the picamera connection, initialises the camera, sets the
sensor_modes property, and then starts the streams.
"""
self._initialise_picamera()
self._initialise_picamera(check_sensor_model=True)
# Sensor modes is a cached property read it once after initialising the camera
_modes = self.sensor_modes
self.start_streaming()
@ -531,7 +560,7 @@ class StreamingPiCamera2(BaseCamera):
logging.info("Stopped MJPEG stream.")
# Adding a sleep to prevent camera getting confused by rapid commands
time.sleep(0.2)
time.sleep(self._sensor_info.short_pause)
@lt.thing_action
def discard_frames(self) -> None:
@ -549,7 +578,7 @@ class StreamingPiCamera2(BaseCamera):
logging.debug("Reconfiguring camera for full resolution capture")
cam.configure(cam.create_still_configuration(sensor=self._sensor_mode))
cam.start()
time.sleep(0.2)
time.sleep(self._sensor_info.short_pause)
yield cam
def capture_image(
@ -648,7 +677,7 @@ class StreamingPiCamera2(BaseCamera):
@lt.thing_action
def auto_expose_from_minimum(
self,
target_white_level: int = 3000,
target_white_level: Optional[int] = None,
percentile: float = 99.9,
) -> None:
"""Adjust exposure until a the target white level is reached.
@ -656,17 +685,21 @@ class StreamingPiCamera2(BaseCamera):
Starting from the minimum exposure, gradually increase exposure until
the image reaches the specified white level.
:param target_white_level: The target 10bit white level. 10-bit data has a
theoretical maximum of 1023, but with black level correction the true
maximum is about 950. Default is 700 as this is approximately 70%
saturated.
:param target_white_level: Raw target white level, this should be an integer
within the range set by the bit-depth of the camera sensor (10-bit for
PiCamera v2, 12 Bit for Picamera HQ. If None the default will be used for
the current sensor. This is approximately 70% saturated.
:param percentile: The percentile to use instead of maximum. Default 99.9. When
calculating the brightest pixel, a percentile is used rather than the
maximum in order to be robust to a small number of noisy/bright pixels.
"""
if target_white_level is None:
target_white_level = self._sensor_info.default_target_white_level
with self._streaming_picamera(pause_stream=True) as cam:
recalibrate_utils.adjust_shutter_and_gain_from_raw(
cam,
self._sensor_info,
target_white_level=target_white_level,
percentile=percentile,
)
@ -693,6 +726,7 @@ class StreamingPiCamera2(BaseCamera):
lst: LensShading = self.lens_shading_tables
recalibrate_utils.adjust_white_balance_from_raw(
cam,
self._sensor_info,
percentile=99,
luminance=lst.luminance,
Cr=lst.Cr,
@ -702,7 +736,7 @@ class StreamingPiCamera2(BaseCamera):
)
else:
recalibrate_utils.adjust_white_balance_from_raw(
cam, percentile=99, method=method
cam, self._sensor_info, percentile=99, method=method
)
@lt.thing_action
@ -719,7 +753,7 @@ class StreamingPiCamera2(BaseCamera):
# the standard mathematical terms for:
# luminance (L), red-difference chroma (Cr), and blue-difference chroma
# (Cb).
L, Cr, Cb = recalibrate_utils.lst_from_camera(cam) # noqa: N806
L, Cr, Cb = recalibrate_utils.lst_from_camera(cam, self._sensor_info) # noqa: N806
tf_utils.set_static_lst(self.tuning, L, Cr, Cb)
self._initialise_picamera()

157
src/openflexure_microscope_server/things/camera/picamera_recalibrate_utils.py
View file

@ -22,10 +22,15 @@ reliable. The three steps above can be accomplished by:
.. code-block:: python
picamera = picamera2.Picamera2()
sensor_info = IMX219_SENSOR_INFO
adjust_shutter_and_gain_from_raw(picamera)
adjust_white_balance_from_raw(picamera)
lst = lst_from_camera(picamera)
adjust_shutter_and_gain_from_raw(
picamera,
sensor_info,
target_white_level=sensor_info.default_target_white_level,
)
adjust_white_balance_from_raw(picamera, sensor_info)
lst = lst_from_camera(picamera, sensor_info)
picamera.lens_shading_table = lst
"""
@ -48,29 +53,54 @@ from picamera2 import Picamera2
import picamera2
class SensorInfo(BaseModel):
"""Information about the sensor used for calibration and property setting."""
unpacked_pixel_format: str
"""The format of the unpacked pixels."""
bit_depth: int
"""The bit depth of each pixel."""
blacklevel: int
"""The sensor black level."""
default_target_white_level: int
"""The default target white level during exposure setting."""
short_pause: float
"""The time to pause for actions that update quickly."""
long_pause: float
"""Time to pause for actions that are known to update slowly."""
IMX219_SENSOR_INFO = SensorInfo(
unpacked_pixel_format="SBGGR10",
bit_depth=10,
blacklevel=64,
default_target_white_level=700,
short_pause=0.2,
long_pause=0.5,
)
IMX477_SENSOR_INFO = SensorInfo(
unpacked_pixel_format="SBGGR12",
bit_depth=12,
blacklevel=256,
default_target_white_level=2800,
short_pause=0.2,
long_pause=0.5,
)
LensShadingTables = tuple[np.ndarray, np.ndarray, np.ndarray]
def set_minimum_exposure(camera: Picamera2) -> None:
"""Enable manual exposure, with low gain and shutter speed.
We set exposure mode to manual, analog and digital gain
to 1, and shutter speed to the minimum (8us for Pi Camera v2)
Note ISO is left at auto, because this is needed for the gains
to be set correctly.
"""
# Disable Automatic exposure and gain algorithm (AeEnable), and set analogue
# gain and exposure time.
# Setting the shutter speed to 1us will result in it being set
# to the minimum possible, which is ~8us for PiCamera v2
camera.set_controls({"AeEnable": False, "AnalogueGain": 1, "ExposureTime": 1})
time.sleep(1)
def adjust_shutter_and_gain_from_raw(
camera: Picamera2,
target_white_level: int = 3000,
sensor_info: SensorInfo,
target_white_level: int,
max_iterations: int = 20,
tolerance: float = 0.05,
percentile: float = 99.9,
@ -81,9 +111,12 @@ def adjust_shutter_and_gain_from_raw(
are not affected by white balance or digital gain.
:param camera: A Picamera2 object.
:param target_white_level: The raw, 10-bit value we aim for. The brightest pixels
should be approximately this bright. Maximum possible is about 900, 700 is
reasonable.
:param target_white_level: The raw value we aim for, the raw value of the brightest
pixels should be approximately this bright. The value to set depends on the
sensor bit depth. We recommend values of 700 for 10-bit sensors and 2800 for
12-bit sensors. This is about 70% of saturated once the blacklevel is
subtracted. The maximum possible value depends on the sensor bit depth, the
sensor blackleve, the tolerance argument.
:param max_iterations: We will terminate once we perform this many iterations,
whether or not we converge. More than 10 shouldn't happen.
:param tolerance: How close to the target value we consider "done". Expressed as a
@ -94,18 +127,20 @@ def adjust_shutter_and_gain_from_raw(
than just ``np.max()``.
"""
# TODO: read black level and bit depth from camera?
if target_white_level * (tolerance + 1) >= 3850:
max_level = 2**sensor_info.bit_depth - 1 - sensor_info.blacklevel
if target_white_level * (tolerance + 1) >= max_level:
raise ValueError(
"The target level is too high - a saturated image would be "
"considered successful. target_white_level * (tolerance + 1) "
"must be less than 3850."
f"must be less than {max_level}."
)
config = camera.create_still_configuration(raw={"format": "SBGGR12"})
config = camera.create_still_configuration(
raw={"format": sensor_info.unpacked_pixel_format}
)
camera.configure(config)
camera.start()
set_minimum_exposure(camera)
_set_minimum_exposure(camera, sensor_info)
# We start with very low exposure settings and work up
# until either the brightness is high enough, or we can't increase the
@ -122,7 +157,7 @@ def adjust_shutter_and_gain_from_raw(
new_time = int(test.exposure_time * min(target_white_level / test.level, 8))
camera.controls.ExposureTime = new_time
camera.controls.AeEnable = False
time.sleep(1)
time.sleep(sensor_info.long_pause)
# Check whether the shutter speed is still going up - if not, we've hit a maximum
if camera.capture_metadata()["ExposureTime"] == test.exposure_time:
@ -140,7 +175,7 @@ def adjust_shutter_and_gain_from_raw(
camera.controls.AnalogueGain = test.analog_gain * min(
target_white_level / test.level, 2
)
time.sleep(1)
time.sleep(sensor_info.long_pause)
# Check the gain is still changing - if not, we have probably hit the maximum
if camera.capture_metadata()["AnalogueGain"] == test.analog_gain:
@ -160,6 +195,7 @@ def adjust_shutter_and_gain_from_raw(
def adjust_white_balance_from_raw(
camera: Picamera2,
sensor_info: SensorInfo,
percentile: float = 99,
luminance: Optional[np.ndarray] = None,
Cr: Optional[np.ndarray] = None,
@ -173,12 +209,13 @@ def adjust_white_balance_from_raw(
We should probably have better logic to verify the channels really
are BGGR...
"""
config = camera.create_still_configuration(raw={"format": "SBGGR12"})
config = camera.create_still_configuration(
raw={"format": sensor_info.unpacked_pixel_format}
)
camera.configure(config)
camera.start()
channels = _channels_from_bayer_array(camera.capture_array("raw"))
# TODO: read black level from camera rather than hard-coding 64
blacklevel = 256
if luminance is not None and Cr is not None and Cb is not None:
# Reconstruct a low-resolution image from the lens shading tables
# and use it to normalise the raw image, to compensate for
@ -205,10 +242,10 @@ def adjust_white_balance_from_raw(
axis=(1, 2),
)
# Subtract blacklevel before splitting into channels
blue, g1, g2, red = centre_means - blacklevel
blue, g1, g2, red = centre_means - sensor_info.blacklevel
else:
blue, g1, g2, red = (
np.percentile(channels, percentile, axis=(1, 2)) - blacklevel
np.percentile(channels, percentile, axis=(1, 2)) - sensor_info.blacklevel
)
green = (g1 + g2) / 2.0
new_awb_gains = (green / red, green / blue)
@ -225,16 +262,16 @@ def adjust_white_balance_from_raw(
)
camera.controls.AwbEnable = False
camera.controls.ColourGains = new_awb_gains
time.sleep(1)
time.sleep(sensor_info.long_pause)
m = camera.capture_metadata()
print(f"Camera confirms gains are now {m['ColourGains']}")
return new_awb_gains
def lst_from_camera(camera: Picamera2) -> LensShadingTables:
def lst_from_camera(camera: Picamera2, sensor_info: SensorInfo) -> LensShadingTables:
"""Acquire a raw image and use it to calculate a lens shading table."""
channels = _raw_channels_from_camera(camera)
return _lst_from_channels(channels)
channels = _raw_channels_from_camera(camera, sensor_info)
return _lst_from_channels(channels, sensor_info.blacklevel)
def recreate_camera_manager() -> None:
@ -255,6 +292,23 @@ class _ExposureTest(BaseModel):
analog_gain: float
def _set_minimum_exposure(camera: Picamera2, sensor_info: SensorInfo) -> None:
"""Enable manual exposure, with low gain and shutter speed.
Set exposure mode to manual, analog and digital gain to 1, and
shutter speed to the minimum (8us for Pi Camera v2)
Note ISO is left at auto, because this is needed for the gains
to be set correctly.
"""
# Disable Automatic exposure and gain algorithm (AeEnable), and set analogue
# gain and exposure time.
# Setting the shutter speed to 1us will result in it being set
# to the minimum possible, which is ~8us for PiCamera v2
camera.set_controls({"AeEnable": False, "AnalogueGain": 1, "ExposureTime": 1})
time.sleep(sensor_info.long_pause)
def _test_exposure_settings(camera: Picamera2, percentile: float) -> _ExposureTest:
"""Evaluate current exposure settings using a raw image.
@ -345,13 +399,8 @@ def _upsample_channels(grids: np.ndarray, shape: tuple[int]) -> np.ndarray:
return zoom(grids, zoom_factors, order=1)[:, : shape[0], : shape[1]]
def _downsampled_channels(
channels: np.ndarray, blacklevel: int = 256
) -> list[np.ndarray]:
"""Generate a downsampled, un-normalised image from which to calculate the LST.
TODO: blacklevel probably ought to be determined from the camera...
"""
def _downsampled_channels(channels: np.ndarray, blacklevel: int) -> list[np.ndarray]:
"""Generate a downsampled, un-normalised image from which to calculate the LST."""
channel_shape = np.array(channels.shape[1:])
lst_shape = np.array([12, 16])
step = np.ceil(channel_shape / lst_shape).astype(int)
@ -366,12 +415,12 @@ def _downsampled_channels(
)
def _lst_from_channels(channels: np.ndarray) -> LensShadingTables:
def _lst_from_channels(channels: np.ndarray, blacklevel: int) -> LensShadingTables:
"""Given the 4 Bayer colour channels from a white image, generate a LST.
Internally, is just calls ``_downsampled_channels`` and ``_lst_from_grids``.
"""
grids = _downsampled_channels(channels)
grids = _downsampled_channels(channels, blacklevel)
return _lst_from_grids(grids)
@ -415,15 +464,17 @@ def _grids_from_lst(lum: np.ndarray, Cr: np.ndarray, Cb: np.ndarray) -> np.ndarr
return np.stack([B, G, G, R], axis=0)
def _raw_channels_from_camera(camera: Picamera2) -> LensShadingTables:
def _raw_channels_from_camera(
camera: Picamera2, sensor_info: SensorInfo
) -> LensShadingTables:
"""Acquire a raw image and return a 4xNxM array of the colour channels."""
if camera.started:
camera.stop_recording()
# We will acquire a raw image with unpacked pixels, which is what the
# format below requests. Bit depth and Bayer order may be overwritten.
# TODO: don't assume 10-bit - the high quality camera uses 12.
# TODO: what's the best mode to use here?
config = camera.create_still_configuration(raw={"format": "SBGGR12"})
config = camera.create_still_configuration(
raw={"format": sensor_info.unpacked_pixel_format}
)
camera.configure(config)
camera.start()
raw_image = camera.capture_array("raw")

16
src/openflexure_microscope_server/things/camera/picamera_tuning_file_utils.py
View file

@ -8,20 +8,14 @@ from picamera2 import Picamera2
import numpy as np
def load_default_tuning(cam: Picamera2) -> dict:
def load_default_tuning(sensor_model: str) -> dict:
"""Load the default tuning file for the camera.
This will open and close the camera to determine its model. If you are
using a model that's supported by ``picamera2`` it should have a tuning
file built in. If not, this will probably crash with an error.
Error handling for unsupported cameras is not something we are likely
to test in the short term.
This will loat the tuning file based on the specified sensor model.
"""
cp = cam.camera_properties
fname = f"{cp['Model']}.json"
fname = f"{sensor_model}.json"
try:
return cam.load_tuning_file(fname)
return Picamera2.load_tuning_file(fname)
except RuntimeError:
tuning_dir = "/usr/share/libcamera/ipa/raspberrypi"
# from picamera2 v0.3.9
@ -29,7 +23,7 @@ def load_default_tuning(cam: Picamera2) -> dict:
# odd - as that's where the files currently are on a default
# Raspbian image. This may need updating if the files have moved
# in future updates to the system libcamera package
return cam.load_tuning_file(fname, dir=tuning_dir)
return Picamera2.load_tuning_file(fname, dir=tuning_dir)
def set_static_lst(