diff --git a/hardware-specific-tests/picamera2/test_tuning.py b/hardware-specific-tests/picamera2/test_tuning.py index d2733b8f..6d58d8e6 100644 --- a/hardware-specific-tests/picamera2/test_tuning.py +++ b/hardware-specific-tests/picamera2/test_tuning.py @@ -1,7 +1,5 @@ """Tests that check that a tuning file can be reloaded.""" -import os - import pytest from picamera2 import Picamera2 @@ -24,24 +22,6 @@ def generate_bad_tuning(): return bad_tuning -def print_tuning(read_file: bool = False): - """Print the path of the default tuning file from the the environment variable. - - :param read_file: Boolean, set true to also print the file contents. - - This is useful for debugging. As pytest suppresses the printing by default the - -s option is needed when running pytest to see this. - """ - key = "LIBCAMERA_RPI_TUNING_FILE" - if key in os.environ: - print(f"Tuning file environment variable: {os.environ[key]}") - if read_file: - with open(os.environ[key], "r") as f: - print(f.read()) - else: - print("Tuning file environment variable not set") - - def _test_bad_tuning_after_good_tuning(configure: bool = False): """Test loading good, bad, then another good tuning files in sequence. @@ -56,10 +36,8 @@ def _test_bad_tuning_after_good_tuning(configure: bool = False): """ bad_tuning = generate_bad_tuning() default_tuning = tf_utils.load_default_tuning("imx219") - print_tuning() print("opening camera with default tuning") with Picamera2(tuning=default_tuning) as cam: - print_tuning() if configure: cam.configure(cam.create_preview_configuration()) del cam diff --git a/src/openflexure_microscope_server/things/camera/picamera.py b/src/openflexure_microscope_server/things/camera/picamera.py index 9cee3856..95eb06e5 100644 --- a/src/openflexure_microscope_server/things/camera/picamera.py +++ b/src/openflexure_microscope_server/things/camera/picamera.py @@ -706,7 +706,7 @@ class StreamingPiCamera2(BaseCamera): It is a 9 value tuple used to specify the 3x3 matrix that the GPU pipeline uses to convert from the camera R,G,B vector to the standard R,G,B. """ - return tuple(tf_utils.get_ccm(self.tuning)[0]["ccm"]) + return tuple(tf_utils.get_ccm(self.tuning)) @colour_correction_matrix.setter # type: ignore def colour_correction_matrix( diff --git a/src/openflexure_microscope_server/things/camera/picamera_tuning_file_utils.py b/src/openflexure_microscope_server/things/camera/picamera_tuning_file_utils.py index 20f2d7a1..991296dc 100644 --- a/src/openflexure_microscope_server/things/camera/picamera_tuning_file_utils.py +++ b/src/openflexure_microscope_server/things/camera/picamera_tuning_file_utils.py @@ -73,11 +73,20 @@ def find_tuning_algo(tuning: dict[str, dict], name: str) -> dict[str, Any]: # Version 1 of the tuning files was simply a dictionary of algorithms. Later # versions have an "algorithms" key, the value of which is a list of algorithms. if version == 1: - return tuning[name] + try: + return tuning[name] + except KeyError as e: + raise KeyError(f"No algorithm {name} in tuning.") from e + # The tuning file "algorithms" is a list of dictionaries + if "algorithms" not in tuning: + raise KeyError("A v2 tuning file must specify an algorithms key") algorithms = tuning["algorithms"] # The list is a list of dictionaries that have 1 key: the algorithm name - algo_dict = next(algo for algo in algorithms if name in algo) + try: + algo_dict = next(algo for algo in algorithms if name in algo) + except StopIteration as e: + raise KeyError(f"No algorithm {name} in tuning.") from e # We want the value for that key, which is a dictionary of algorithm parameters return algo_dict[name] @@ -183,9 +192,7 @@ def lst_calibrated(tuning: dict) -> bool: def set_ccm( tuning: dict, - col_corr_matrix: tuple[ - float, float, float, float, float, float, float, float, float - ], + col_corr_matrix: list, ) -> dict: """Update the ``rpi.alsc`` section of a camera tuning dict set the colour correction matrix. @@ -194,15 +201,17 @@ def set_ccm( :return: an updated tuning dict with the new colour correction matrix. """ output_tuning = deepcopy(tuning) + if len(col_corr_matrix) != 9: + raise ValueError("col_corr_matrix should be a list of 9 floats") ccm = find_tuning_algo(output_tuning, "rpi.ccm") - ccm["ccms"] = [{"ct": CALIBRATED_COLOUR_TEMP, "ccm": col_corr_matrix}] + ccm["ccms"] = [{"ct": CALIBRATED_COLOUR_TEMP, "ccm": list(col_corr_matrix)}] return output_tuning def get_ccm(tuning: dict) -> None: """Get a copy of the the ``rpi.ccm`` section of a camera tuning dict.""" ccm = find_tuning_algo(tuning, "rpi.ccm") - return deepcopy(ccm["ccms"]) + return deepcopy(ccm["ccms"][0]["ccm"]) def set_static_geq( diff --git a/tests/test_cameras.py b/tests/test_cameras.py index 307df610..fc10370b 100644 --- a/tests/test_cameras.py +++ b/tests/test_cameras.py @@ -112,7 +112,6 @@ def test_thing_description_equivalence(mock_picam_thing): # defined as PiCamera specific, or by replicating the functionality for other # cameras. picamera_extra_actions = { - "flat_lens_shading_chrominance", "set_static_green_equalisation", "set_ce_enable_to_off", "stop_streaming", @@ -130,6 +129,7 @@ def test_thing_description_equivalence(mock_picam_thing): "sensor_modes", "sensor_mode", } + # Note these are only the action not exposed as calibration actions. for action in picamera_extra_actions: assert action in picamera_actions for props in picamera_extra_props: diff --git a/tests/test_picamera_tuning_files.py b/tests/test_picamera_tuning_files.py new file mode 100644 index 00000000..cb2ea8a6 --- /dev/null +++ b/tests/test_picamera_tuning_files.py @@ -0,0 +1,258 @@ +"""Tests for interactions with tuning files and dictionaries.""" + +from copy import deepcopy + +import pytest +import numpy as np + + +from openflexure_microscope_server.things.camera import ( + picamera_tuning_file_utils as tf_utils, +) + +RNG = np.random.default_rng() + + +@pytest.fixture +def imx219_tuning(): + """Load the default tuning file for the PiCamera v2.""" + return tf_utils.load_default_tuning("imx219") + + +@pytest.fixture +def imx477_tuning(): + """Load the default tuning file for the PiCamera HQ.""" + return tf_utils.load_default_tuning("imx477") + + +def test_load_tuning(imx219_tuning, imx477_tuning): + """Check the sctandard tuning files load and contain the correct information.""" + assert imx219_tuning != imx477_tuning + + for tuning in [imx219_tuning, imx477_tuning]: + assert tuning["version"] == 2.0 + assert tuning["target"] == "bcm2835" + assert isinstance(tuning["algorithms"], list) + + # The algorithms in order, and whether we expect them to be the same for the cameras. + algos = [ + ("rpi.black_level", True), + ("rpi.dpc", True), + ("rpi.lux", True), + ("rpi.noise", True), + ("rpi.geq", True), + ("rpi.sdn", True), + ("rpi.awb", True), + ("rpi.agc", True), + ("rpi.alsc", True), + ("rpi.contrast", True), + ("rpi.ccm", False), + ("rpi.sharpen", True), + ("rpi.hdr", True), + ("rpi.sync", True), + ] + + # Strict= true so this will error if the tuning files are not the same length as + # the algorithms + zipped_algos = zip( + algos, imx219_tuning["algorithms"], imx477_tuning["algorithms"], strict=True + ) + + # Split up the algorithm name, either we expect them to be the same in both files + # and the algorithm for the two tuning files. + for (algo_name, algo_same), algo219, algo477 in zipped_algos: + # Check this is the correct algorithm + assert algo_name in algo219 + assert algo_name in algo477 + # Check they are they are the same if expected to be. + if algo_same: + assert algo219 == algo477 + else: + assert algo219 != algo477 + + +def test_find_tuning_algo_v1(): + """Check tuning algorithms are read directly from dict if no version is set.""" + # Version 1 onf the tuning file was just a dictionary of algorithms + mock_tuning = {"mock": {"param": 1}, "foo": {"param": "bar"}} + assert tf_utils.find_tuning_algo(mock_tuning, "mock") == {"param": 1} + assert tf_utils.find_tuning_algo(mock_tuning, "foo") == {"param": "bar"} + with pytest.raises(KeyError, match="No algorithm who in tuning."): + assert tf_utils.find_tuning_algo(mock_tuning, "who") + + # Also check that if version is not 1 it is read differently. + mock_tuning["version"] = 2 + with pytest.raises(KeyError, match="A v2 tuning file must specify"): + assert tf_utils.find_tuning_algo(mock_tuning, "mock") + + +def test_find_tuning_algo_v2(): + """Check reading version 2 dictionary structure.""" + mock_tuning = { + "version": 2.0, + "algorithms": [{"mock": {"param": 1}}, {"foo": {"param": "bar"}}], + } + assert tf_utils.find_tuning_algo(mock_tuning, "mock") == {"param": 1} + assert tf_utils.find_tuning_algo(mock_tuning, "foo") == {"param": "bar"} + with pytest.raises(KeyError, match="No algorithm who in tuning."): + assert tf_utils.find_tuning_algo(mock_tuning, "who") + + +def _assert_lst_table_equivalence(array, table): + """Check equivalence between input numpy array and output table.""" + for i in range(12): + for j in range(16): + assert table[i][j] == round(float(array[i, j]), 3) + assert table[i][j] == round(float(array[i, j]), 3) + + +def test_tuning_lst_tools(imx219_tuning, imx477_tuning): + """Check reading the default lens shading tables.""" + for tuning in (imx219_tuning, imx477_tuning): + assert not tf_utils.lst_calibrated(tuning) + + lst_model = tf_utils.get_lst(tuning) + assert isinstance(lst_model, tf_utils.LensShading) + # Check the tables are lists of lists + assert isinstance(lst_model.luminance, list) + assert isinstance(lst_model.luminance[0], list) + assert isinstance(lst_model.Cr, list) + assert isinstance(lst_model.Cr[0], list) + assert isinstance(lst_model.Cb, list) + assert isinstance(lst_model.Cb[0], list) + # Loaded table has default lens tuning, + assert lst_model.colour_temp == tf_utils.DEFAULT_COLOUR_TEMP + + +def test_setting_lst(imx219_tuning): + """Check that the LST can be set.""" + lum = RNG.normal(size=(12, 16)) + cr = RNG.normal(size=(12, 16)) + cb = RNG.normal(size=(12, 16)) + + updated_tuning = tf_utils.set_lst( + imx219_tuning, + luminance=lum, + cr=cr, + cb=cb, + colour_temp=tf_utils.CALIBRATED_COLOUR_TEMP, + ) + + # Check it wasn't modified in place. + assert updated_tuning != imx219_tuning + # Check it now reports as calibrated + assert tf_utils.lst_calibrated(updated_tuning) + + lst_model = tf_utils.get_lst(updated_tuning) + assert lst_model.colour_temp == tf_utils.CALIBRATED_COLOUR_TEMP + # Check the numbers are equivalent from the input array and the output table + # except for formatting and rounding. + _assert_lst_table_equivalence(lum, lst_model.luminance) + _assert_lst_table_equivalence(cr, lst_model.Cr) + _assert_lst_table_equivalence(cb, lst_model.Cb) + + # Flatten the lens shading tables + flat_tuning = tf_utils.flatten_lst(updated_tuning) + # In this case only the colour channels + flat_chroma_tuning = tf_utils.flatten_lst(updated_tuning, keep_luminance=True) + + # Check each was changed + assert updated_tuning != flat_tuning + assert updated_tuning != flat_chroma_tuning + assert flat_tuning != flat_chroma_tuning + + # Check the flattened tunings report as not calibrated + assert not tf_utils.lst_calibrated(flat_tuning) + assert not tf_utils.lst_calibrated(flat_chroma_tuning) + + flat_lst_model = tf_utils.get_lst(flat_tuning) + flat_chroma_lst_model = tf_utils.get_lst(flat_chroma_tuning) + + flat_array = np.ones((12, 16)) + + # Check the that it really was flattened for the flat tuning + _assert_lst_table_equivalence(flat_array, flat_lst_model.luminance) + _assert_lst_table_equivalence(flat_array, flat_lst_model.Cr) + _assert_lst_table_equivalence(flat_array, flat_lst_model.Cb) + + # Check only colour channels flattened when keep_luminance=True + _assert_lst_table_equivalence(lum, flat_chroma_lst_model.luminance) + _assert_lst_table_equivalence(flat_array, flat_chroma_lst_model.Cr) + _assert_lst_table_equivalence(flat_array, flat_chroma_lst_model.Cb) + + +def test_setting_ccm(imx219_tuning): + """Check the colour correction matrix can be set.""" + # CCM is set as a flat list. Create a mock ccm + new_ccm = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0] + + updated_tuning = tf_utils.set_ccm(imx219_tuning, new_ccm) + + # Check it wasn't modified in place. + assert updated_tuning != imx219_tuning + + # Check the value was set + assert tf_utils.get_ccm(updated_tuning) == new_ccm + + # Check that our dictionary doesn't update if the input list is updated. + new_ccm[0] = 0.0 + assert tf_utils.get_ccm(updated_tuning) != new_ccm + + new_ccm.append(1.0) + with pytest.raises( + ValueError, match="col_corr_matrix should be a list of 9 floats" + ): + tf_utils.set_ccm(imx219_tuning, new_ccm) + + +def test_green_equalisation(imx219_tuning): + """Test setting the offset parameter of green equalisation.""" + assert tf_utils.geq_is_static(imx219_tuning) + bad_geq_tuning = tf_utils.set_static_geq(imx219_tuning, offset=1234) + # Check it wasn't modified in place. + assert bad_geq_tuning != imx219_tuning + + # No longer static + assert not tf_utils.geq_is_static(bad_geq_tuning) + + fixed_geq_tuning = tf_utils.set_static_geq(bad_geq_tuning) + + assert tf_utils.geq_is_static(fixed_geq_tuning) + + +def test_ce_enable(imx219_tuning): + """Check the setting of ce enable in the contrast algorithm.""" + assert tf_utils.ce_enable_is_static(imx219_tuning) + + # No way to enable it so do it manually + bad_tuning = deepcopy(imx219_tuning) + contrast = tf_utils.find_tuning_algo(bad_tuning, "rpi.contrast") + contrast["ce_enable"] = 1 + + assert not tf_utils.ce_enable_is_static(bad_tuning) + + fixed_tuning = tf_utils.set_ce_to_disabled(bad_tuning) + assert tf_utils.ce_enable_is_static(fixed_tuning) + + +def test_copying_algorithms(imx219_tuning): + """Check an algorithm can be copied from one file to another.""" + # Update the CCM + new_ccm = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0] + updated_ccm_tuning = tf_utils.set_ccm(imx219_tuning, new_ccm) + # It has changed + assert updated_ccm_tuning != imx219_tuning + + # Create another tuning file with the GEQ updated. + updated_geq_tuning = tf_utils.set_static_geq(imx219_tuning, offset=1234) + # It has changed + assert updated_geq_tuning != imx219_tuning + + # Copy the CCM from the dict with the bad GEQ value into the one with the updated + # CCM + original_tuning = tf_utils.copy_algo_from_other_tuning( + "rpi.ccm", base_tuning_file=updated_ccm_tuning, copy_from=updated_geq_tuning + ) + + # This should now be equivalent to the original. + assert original_tuning == imx219_tuning