From 92e55412671f8741b92f29b70626ec8a490db3f5 Mon Sep 17 00:00:00 2001 From: Joe Knapper Date: Thu, 26 Mar 2026 16:48:50 +0000 Subject: [PATCH 1/6] Add test that stack checking agrees with at least 80% of manual labels --- .../unit_tests/data/sharpness_test_cases.json | 43 ++++++++++ tests/unit_tests/test_stack_examples.py | 80 +++++++++++++++++++ 2 files changed, 123 insertions(+) create mode 100644 tests/unit_tests/data/sharpness_test_cases.json create mode 100644 tests/unit_tests/test_stack_examples.py diff --git a/tests/unit_tests/data/sharpness_test_cases.json b/tests/unit_tests/data/sharpness_test_cases.json new file mode 100644 index 00000000..0a50f44f --- /dev/null +++ b/tests/unit_tests/data/sharpness_test_cases.json @@ -0,0 +1,43 @@ +[ + {"sharpnesses": [100000, 300000, 500000, 300000, 100000], "label": "success"}, + {"sharpnesses": [100000, 200000, 400000, 600000, 400000, 200000, 100000], "label": "success"}, + {"sharpnesses": [100000, 200000, 300000, 400000, 500000], "label": "continue"}, + {"sharpnesses": [500000, 400000, 300000, 200000, 100000], "label": "restart"}, + {"sharpnesses": [100000, 200000, 300000, 200000, 100000, 50000], "label": ["success"]}, + {"sharpnesses": [0, 100000, 200000, 300000, 400000, 500000], "label": "continue"}, + {"sharpnesses": [500000, 500000, 500000, 500000, 500000], "label": "continue"}, + {"sharpnesses": [100000, 200000, 300000, 300000, 200000, 100000], "label": ["success"]}, + {"sharpnesses": [100000, 200000, 300000, 400000, 400000, 300000, 200000], "label": ["success", "continue"]}, + {"sharpnesses": [100000, 300000, 500000, 500000, 500000, 300000, 100000], "label": ["success", "continue"]}, + {"sharpnesses": [100000, 300000, 500000, 400000, 500000, 300000, 100000], "label": ["continue", "success"]}, + {"sharpnesses": [100000, 400000, 600000, 500000, 300000, 200000, 100000], "label": "success"}, + {"sharpnesses": [100000, 200000, 500000, 700000, 600000, 300000, 100000], "label": "success"}, + {"sharpnesses": [100000, 200000, 300000, 500000, 700000, 600000, 400000], "label": ["success", "continue"]}, + {"sharpnesses": [700000, 600000, 500000, 400000, 300000, 200000, 100000, 0], "label": "restart"}, + {"sharpnesses": [100000, 300000, 600000, 900000, 600000, 300000, 200000, 100000], "label": "success"}, + {"sharpnesses": [100000, 200000, 400000, 800000, 900000, 800000, 400000, 200000], "label": "success"}, + {"sharpnesses": [100000, 200000, 300000, 400000, 500000, 600000, 700000, 800000], "label": "continue"}, + {"sharpnesses": [800000, 700000, 600000, 500000, 400000, 300000, 200000, 100000], "label": "restart"}, + {"sharpnesses": [100000, 200000, 300000, 400000, 400000, 400000, 300000, 200000], "label": ["success", "continue"]}, + {"sharpnesses": [100000, 200000, 300000, 500000, 500000, 500000, 500000, 500000], "label": "continue"}, + {"sharpnesses": [500000, 500000, 500000, 500000, 500000, 400000, 300000, 200000], "label": "restart"}, + {"sharpnesses": [100000, 200000, 300000, 600000, 900000, 600000, 300000, 200000, 100000], "label": "success"}, + {"sharpnesses": [100000, 200000, 400000, 700000, 1000000, 900000, 700000, 400000, 200000], "label": "success"}, + {"sharpnesses": [100000, 200000, 300000, 400000, 500000, 600000, 700000, 800000, 900000], "label": "continue"}, + {"sharpnesses": [900000, 800000, 700000, 600000, 500000, 400000, 300000, 200000, 100000], "label": "restart"}, + {"sharpnesses": [100000, 200000, 1000000, 200000, 1000000, 200000, 100000], "label": "continue"}, + {"sharpnesses": [100000, 500000, 100000, 500000, 100000, 500000, 100000], "label": ["restart", "continue"]}, + {"sharpnesses": [300000, 300000, 300000, 300000, 300000, 300000, 300000], "label": "continue"}, + {"sharpnesses": [500000, 600000, 800000, 1100000, 1500000, 1100000, 800000, 600000, 500000], "label": "success"}, + {"sharpnesses": [1000000, 1100000, 1300000, 1600000, 2000000, 1600000, 1300000, 1100000, 1000000], "label": "success"}, + {"sharpnesses": [2000000, 1800000, 1500000, 1200000, 1000000, 900000, 800000, 700000, 600000], "label": "restart"}, + {"sharpnesses": [600000, 700000, 800000, 900000, 1100000, 1300000, 1500000, 1700000, 1800000], "label": "continue"}, + {"sharpnesses": [500000, 700000, 1000000, 1400000, 1800000, 1700000, 1400000, 1000000, 700000], "label": "success"}, + {"sharpnesses": [500000, 600000, 900000, 1300000, 1800000, 1900000, 1700000, 1300000, 900000], "label": "success"}, + {"sharpnesses": [500000, 500000, 500000, 500000, 600000, 900000, 600000, 500000, 500000], "label": ["success"]}, + {"sharpnesses": [500000, 500000, 500000, 500000, 500000, 600000, 900000, 600000, 500000], "label": ["success", "continue"]}, + {"sharpnesses": [500000, 500000, 500000, 500000, 500000, 500000, 600000, 900000, 600000], "label": "continue"}, + {"sharpnesses": [100000, 100000, 100000, 100000, 200000, 500000, 200000, 100000, 100000], "label": ["success"]}, + {"sharpnesses": [100000, 100000, 100000, 100000, 100000, 200000, 500000, 200000, 100000], "label": ["success", "continue"]}, + {"sharpnesses": [100000, 100000, 100000, 100000, 100000, 100000, 200000, 500000, 200000], "label": "continue"} +] \ No newline at end of file diff --git a/tests/unit_tests/test_stack_examples.py b/tests/unit_tests/test_stack_examples.py new file mode 100644 index 00000000..217eedf7 --- /dev/null +++ b/tests/unit_tests/test_stack_examples.py @@ -0,0 +1,80 @@ +"""Unit tests for validating stack classification logic in AutofocusThing. + +This module tests whether the `check_stack_result` method correctly classifies +z-stack sharpness profiles into "success", "continue", or "restart" categories +based on predefined test cases. + +Sharpness profiles are loaded from a JSON file and converted into mock capture +objects to simulate real camera captures. +""" + +import json + +from labthings_fastapi.testing import create_thing_without_server + +from openflexure_microscope_server.things.autofocus import AutofocusThing + + +class MockCapture: + """Simple mock object representing a captured image. + + This class mimics the minimal interface required by + `check_stack_result` by adding the `sharpness` and `buffer_id` + attributes. + + :param sharpness: The sharpness value associated with the image. + :param buffer_id: A unique identifier for the image buffer. + """ + + def __init__(self, sharpness, buffer_id): + """Give each capture a sharpness and buffer_id.""" + self.sharpness = sharpness + self.buffer_id = buffer_id + + +def make_captures(sharpness_list): + """Convert a list of sharpness values into mock capture objects. + + :param sharpness_list: A list of numeric sharpness values. + :returns: A list of MockCapture instances with sequential buffer IDs. + """ + return [MockCapture(s, i) for i, s in enumerate(sharpness_list)] + + +def test_stack_labelling(): + """Test stack classification accuracy against labelled sharpness cases. + + This test loads predefined sharpness profiles and their expected labels + from a JSON file, converts them into mock capture objects, and evaluates + the classification returned by `check_stack_result`. + + The test asserts that at least 90% of cases are correctly classified. + + Expected labels may be a single value or a list of acceptable values. + """ + autofocus_thing = create_thing_without_server(AutofocusThing, mock_all_slots=True) + with open(r"tests\unit_tests\data\sharpness_test_cases.json") as f: + data = json.load(f) + + success = 0 + total = len(data) + + for _i, case in enumerate(data): + sharpnesses = case["sharpnesses"] + expected = case["label"] + + # Allow multiple acceptable labels + if not isinstance(expected, list): + expected = [expected] + + # Convert to capture objects + captures = make_captures(sharpnesses) + + # Call the method under test + result, _ = autofocus_thing.check_stack_result( + captures, check_turning_points=False + ) + if result in expected: + success += 1 + + assert success > 0.8 * total From 5a2787a03937e1c26e957fd568a8af4e79be2649 Mon Sep 17 00:00:00 2001 From: Joe Knapper Date: Thu, 26 Mar 2026 17:25:33 +0000 Subject: [PATCH 2/6] Data path based on containing DIR location --- tests/unit_tests/test_stack_examples.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/unit_tests/test_stack_examples.py b/tests/unit_tests/test_stack_examples.py index 217eedf7..81fda2d2 100644 --- a/tests/unit_tests/test_stack_examples.py +++ b/tests/unit_tests/test_stack_examples.py @@ -9,11 +9,14 @@ objects to simulate real camera captures. """ import json +import os from labthings_fastapi.testing import create_thing_without_server from openflexure_microscope_server.things.autofocus import AutofocusThing +THIS_DIR = os.path.dirname(__file__) +DATA_PATH = os.path.join(THIS_DIR, "data", "sharpness_test_cases.json") class MockCapture: """Simple mock object representing a captured image. @@ -53,7 +56,8 @@ def test_stack_labelling(): Expected labels may be a single value or a list of acceptable values. """ autofocus_thing = create_thing_without_server(AutofocusThing, mock_all_slots=True) - with open(r"tests\unit_tests\data\sharpness_test_cases.json") as f: + + with open(DATA_PATH) as f: data = json.load(f) success = 0 From 70c36333a8485e0e70f3a1b14bb2b7d9bdb3d678 Mon Sep 17 00:00:00 2001 From: Joe Knapper Date: Thu, 26 Mar 2026 17:25:47 +0000 Subject: [PATCH 3/6] Data path based on containing DIR location --- tests/unit_tests/test_stack_examples.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit_tests/test_stack_examples.py b/tests/unit_tests/test_stack_examples.py index 81fda2d2..27b5123c 100644 --- a/tests/unit_tests/test_stack_examples.py +++ b/tests/unit_tests/test_stack_examples.py @@ -18,6 +18,7 @@ from openflexure_microscope_server.things.autofocus import AutofocusThing THIS_DIR = os.path.dirname(__file__) DATA_PATH = os.path.join(THIS_DIR, "data", "sharpness_test_cases.json") + class MockCapture: """Simple mock object representing a captured image. From 3b53473475c9eaad03da6d3521fc5daa314a5d1c Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Fri, 27 Mar 2026 16:01:26 +0000 Subject: [PATCH 4/6] Bump stitching version --- CHANGELOG.md | 2 ++ pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4740dc9..ad9c32b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -105,6 +105,8 @@ The following merge requests have been merged into v3: * !554 Move smart stacking into a mixin so it can be re-used in other workflows * !551 Sort scans by created time * !538 Update dependency to labthings 0.1.0 +* !557 Prep for v3.0.0 alpha5 +* !560 Bump stitching version # [v3.0.0-alpha4](https://gitlab.com/openflexure/openflexure-microscope-server/compare/v3.0.0-alpha3...v3.0.0-alpha4) (2025-12-15) diff --git a/pyproject.toml b/pyproject.toml index f65f8279..1242790a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ dependencies = [ "sangaboard~=0.4.0", "camera-stage-mapping ~= 0.1.10", "opencv-python-headless ~= 4.13.0", - "openflexure-stitching[libvips]==0.2.4", + "openflexure-stitching[libvips]==0.2.5", "pillow ~= 10.4", "anyio ~= 4.0", "numpy ~= 2.2", From bc6ea9a1cabc75443229cd2f32335cd3013a2094 Mon Sep 17 00:00:00 2001 From: jaknapper Date: Thu, 2 Apr 2026 17:22:59 +0100 Subject: [PATCH 5/6] Manually test for increasing or decreasing stacks --- .../things/autofocus.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/openflexure_microscope_server/things/autofocus.py b/src/openflexure_microscope_server/things/autofocus.py index 5cbc050c..ca9992a0 100644 --- a/src/openflexure_microscope_server/things/autofocus.py +++ b/src/openflexure_microscope_server/things/autofocus.py @@ -676,10 +676,11 @@ class AutofocusThing(lt.Thing): sharpness=self._cam.grab_jpeg_size(stream_name="lores"), ) - # Silence too many returns in this situation as refactoring to reduce returns is - # unlikely to improve readability. This function is basically a complex switch - # statement, having an explicit return after each option is clear. - def check_stack_result( # noqa: PLR0911 + # Silence too many returns and complexity in this situation as refactoring to + # reduce returns is unlikely to improve readability. + # This function is basically a complex switch statement, having an explicit + # return after each option is clear. + def check_stack_result( # noqa: PLR0911 C901 self, captures: list[CaptureInfo], check_turning_points: bool ) -> tuple[Literal["success", "continue", "restart"], int]: """Check if the sharpest image in a list of captures is central enough. @@ -688,7 +689,7 @@ class AutofocusThing(lt.Thing): sharpness has converged in the centre :param check_turning_points: Whether to check the number of turning points in the sharpnesses of the images in the stack is exactly 1. (May fail with - thick samples) + thick samples and stacks with many images) :returns: A tuple with two values: @@ -719,6 +720,13 @@ class AutofocusThing(lt.Thing): return "restart", capture_id return "continue", capture_id + # Manually test for monotomically increasing or decreasing sharpnesses, as + # fitting can struggle with these and their behaviour is simpler to hardcode + if np.array_equal(sharpnesses, np.sort(sharpnesses)): + return "continue", capture_id + if np.array_equal(sharpnesses, np.sort(sharpnesses)[::-1]): + return "restart", capture_id + try: turning = _get_peak_turning_point(sharpnesses) except NotAPeakError: From 4da8172a165bd0208fccf08c54b72ec7f384f98a Mon Sep 17 00:00:00 2001 From: Joe Knapper Date: Thu, 16 Apr 2026 11:49:04 +0100 Subject: [PATCH 6/6] Updates from review, mark tests as allowed to fail --- .../things/autofocus.py | 4 +- .../unit_tests/data/sharpness_test_cases.json | 84 +++++++++---------- tests/unit_tests/test_stack_examples.py | 65 ++++++++------ 3 files changed, 85 insertions(+), 68 deletions(-) diff --git a/src/openflexure_microscope_server/things/autofocus.py b/src/openflexure_microscope_server/things/autofocus.py index ca9992a0..61d1ff0a 100644 --- a/src/openflexure_microscope_server/things/autofocus.py +++ b/src/openflexure_microscope_server/things/autofocus.py @@ -722,9 +722,9 @@ class AutofocusThing(lt.Thing): # Manually test for monotomically increasing or decreasing sharpnesses, as # fitting can struggle with these and their behaviour is simpler to hardcode - if np.array_equal(sharpnesses, np.sort(sharpnesses)): + if np.all(sharpnesses[:-1] <= sharpnesses[1:]): return "continue", capture_id - if np.array_equal(sharpnesses, np.sort(sharpnesses)[::-1]): + if np.all(sharpnesses[:-1] >= sharpnesses[1:]): return "restart", capture_id try: diff --git a/tests/unit_tests/data/sharpness_test_cases.json b/tests/unit_tests/data/sharpness_test_cases.json index 0a50f44f..27967eea 100644 --- a/tests/unit_tests/data/sharpness_test_cases.json +++ b/tests/unit_tests/data/sharpness_test_cases.json @@ -1,43 +1,43 @@ [ - {"sharpnesses": [100000, 300000, 500000, 300000, 100000], "label": "success"}, - {"sharpnesses": [100000, 200000, 400000, 600000, 400000, 200000, 100000], "label": "success"}, - {"sharpnesses": [100000, 200000, 300000, 400000, 500000], "label": "continue"}, - {"sharpnesses": [500000, 400000, 300000, 200000, 100000], "label": "restart"}, - {"sharpnesses": [100000, 200000, 300000, 200000, 100000, 50000], "label": ["success"]}, - {"sharpnesses": [0, 100000, 200000, 300000, 400000, 500000], "label": "continue"}, - {"sharpnesses": [500000, 500000, 500000, 500000, 500000], "label": "continue"}, - {"sharpnesses": [100000, 200000, 300000, 300000, 200000, 100000], "label": ["success"]}, - {"sharpnesses": [100000, 200000, 300000, 400000, 400000, 300000, 200000], "label": ["success", "continue"]}, - {"sharpnesses": [100000, 300000, 500000, 500000, 500000, 300000, 100000], "label": ["success", "continue"]}, - {"sharpnesses": [100000, 300000, 500000, 400000, 500000, 300000, 100000], "label": ["continue", "success"]}, - {"sharpnesses": [100000, 400000, 600000, 500000, 300000, 200000, 100000], "label": "success"}, - {"sharpnesses": [100000, 200000, 500000, 700000, 600000, 300000, 100000], "label": "success"}, - {"sharpnesses": [100000, 200000, 300000, 500000, 700000, 600000, 400000], "label": ["success", "continue"]}, - {"sharpnesses": [700000, 600000, 500000, 400000, 300000, 200000, 100000, 0], "label": "restart"}, - {"sharpnesses": [100000, 300000, 600000, 900000, 600000, 300000, 200000, 100000], "label": "success"}, - {"sharpnesses": [100000, 200000, 400000, 800000, 900000, 800000, 400000, 200000], "label": "success"}, - {"sharpnesses": [100000, 200000, 300000, 400000, 500000, 600000, 700000, 800000], "label": "continue"}, - {"sharpnesses": [800000, 700000, 600000, 500000, 400000, 300000, 200000, 100000], "label": "restart"}, - {"sharpnesses": [100000, 200000, 300000, 400000, 400000, 400000, 300000, 200000], "label": ["success", "continue"]}, - {"sharpnesses": [100000, 200000, 300000, 500000, 500000, 500000, 500000, 500000], "label": "continue"}, - {"sharpnesses": [500000, 500000, 500000, 500000, 500000, 400000, 300000, 200000], "label": "restart"}, - {"sharpnesses": [100000, 200000, 300000, 600000, 900000, 600000, 300000, 200000, 100000], "label": "success"}, - {"sharpnesses": [100000, 200000, 400000, 700000, 1000000, 900000, 700000, 400000, 200000], "label": "success"}, - {"sharpnesses": [100000, 200000, 300000, 400000, 500000, 600000, 700000, 800000, 900000], "label": "continue"}, - {"sharpnesses": [900000, 800000, 700000, 600000, 500000, 400000, 300000, 200000, 100000], "label": "restart"}, - {"sharpnesses": [100000, 200000, 1000000, 200000, 1000000, 200000, 100000], "label": "continue"}, - {"sharpnesses": [100000, 500000, 100000, 500000, 100000, 500000, 100000], "label": ["restart", "continue"]}, - {"sharpnesses": [300000, 300000, 300000, 300000, 300000, 300000, 300000], "label": "continue"}, - {"sharpnesses": [500000, 600000, 800000, 1100000, 1500000, 1100000, 800000, 600000, 500000], "label": "success"}, - {"sharpnesses": [1000000, 1100000, 1300000, 1600000, 2000000, 1600000, 1300000, 1100000, 1000000], "label": "success"}, - {"sharpnesses": [2000000, 1800000, 1500000, 1200000, 1000000, 900000, 800000, 700000, 600000], "label": "restart"}, - {"sharpnesses": [600000, 700000, 800000, 900000, 1100000, 1300000, 1500000, 1700000, 1800000], "label": "continue"}, - {"sharpnesses": [500000, 700000, 1000000, 1400000, 1800000, 1700000, 1400000, 1000000, 700000], "label": "success"}, - {"sharpnesses": [500000, 600000, 900000, 1300000, 1800000, 1900000, 1700000, 1300000, 900000], "label": "success"}, - {"sharpnesses": [500000, 500000, 500000, 500000, 600000, 900000, 600000, 500000, 500000], "label": ["success"]}, - {"sharpnesses": [500000, 500000, 500000, 500000, 500000, 600000, 900000, 600000, 500000], "label": ["success", "continue"]}, - {"sharpnesses": [500000, 500000, 500000, 500000, 500000, 500000, 600000, 900000, 600000], "label": "continue"}, - {"sharpnesses": [100000, 100000, 100000, 100000, 200000, 500000, 200000, 100000, 100000], "label": ["success"]}, - {"sharpnesses": [100000, 100000, 100000, 100000, 100000, 200000, 500000, 200000, 100000], "label": ["success", "continue"]}, - {"sharpnesses": [100000, 100000, 100000, 100000, 100000, 100000, 200000, 500000, 200000], "label": "continue"} -] \ No newline at end of file + {"sharpnesses":[100000,300000,500000,300000,100000],"label":"success"}, + {"sharpnesses":[100000,200000,400000,600000,400000,200000,100000],"label":"success"}, + {"sharpnesses":[100000,200000,300000,400000,500000],"label":"continue"}, + {"sharpnesses":[500000,400000,300000,200000,100000],"label":"restart"}, + {"sharpnesses":[100000,200000,300000,200000,100000,50000],"label":["success"]}, + {"sharpnesses":[0,100000,200000,300000,400000,500000],"label":"continue"}, + {"sharpnesses":[500000,500000,500000,500000,500000],"label":"continue"}, + {"sharpnesses":[100000,200000,300000,300000,200000,100000],"label":["success"]}, + {"sharpnesses":[100000,200000,300000,400000,400000,300000,200000],"label":["success","continue"]}, + {"sharpnesses":[100000,300000,500000,500000,500000,300000,100000],"label":["success","continue"]}, + {"sharpnesses":[100000,300000,500000,400000,500000,300000,100000],"label":["continue","success"]}, + {"sharpnesses":[100000,400000,600000,500000,300000,200000,100000],"label":"success"}, + {"sharpnesses":[100000,200000,500000,700000,600000,300000,100000],"label":"success"}, + {"sharpnesses":[100000,200000,300000,500000,700000,600000,400000],"label":["success","continue"]}, + {"sharpnesses":[700000,600000,500000,400000,300000,200000,100000,0],"label":"restart"}, + {"sharpnesses":[100000,300000,600000,900000,600000,300000,200000,100000],"label":"success"}, + {"sharpnesses":[100000,200000,400000,800000,900000,800000,400000,200000],"label":"success"}, + {"sharpnesses":[100000,200000,300000,400000,500000,600000,700000,800000],"label":"continue"}, + {"sharpnesses":[800000,700000,600000,500000,400000,300000,200000,100000],"label":"restart"}, + {"sharpnesses":[100000,200000,300000,400000,400000,400000,300000,200000],"label":["success","continue"]}, + {"sharpnesses":[100000,200000,300000,500000,500000,500000,500000,500000],"label":"continue"}, + {"sharpnesses":[500000,500000,500000,500000,500000,400000,300000,200000],"label":"restart"}, + {"sharpnesses":[100000,200000,300000,600000,900000,600000,300000,200000,100000],"label":"success"}, + {"sharpnesses":[100000,200000,400000,700000,1000000,900000,700000,400000,200000],"label":"success"}, + {"sharpnesses":[100000,200000,300000,400000,500000,600000,700000,800000,900000],"label":"continue"}, + {"sharpnesses":[900000,800000,700000,600000,500000,400000,300000,200000,100000],"label":"restart"}, + {"sharpnesses":[100000,200000,1000000,200000,1000000,200000,100000],"label":"continue"}, + {"sharpnesses":[100000,500000,100000,500000,100000,500000,100000],"label":["restart","continue"]}, + {"sharpnesses":[300000,300000,300000,300000,300000,300000,300000],"label":"continue"}, + {"sharpnesses":[500000,600000,800000,1100000,1500000,1100000,800000,600000,500000],"label":"success"}, + {"sharpnesses":[1000000,1100000,1300000,1600000,2000000,1600000,1300000,1100000,1000000],"label":"success"}, + {"sharpnesses":[2000000,1800000,1500000,1200000,1000000,900000,800000,700000,600000],"label":"restart"}, + {"sharpnesses":[600000,700000,800000,900000,1100000,1300000,1500000,1700000,1800000],"label":"continue"}, + {"sharpnesses":[500000,700000,1000000,1400000,1800000,1700000,1400000,1000000,700000],"label":"success"}, + {"sharpnesses":[500000,600000,900000,1300000,1800000,1900000,1700000,1300000,900000],"label":"success"}, + {"sharpnesses":[500000,500000,500000,500000,600000,900000,600000,500000,500000],"label":["success"],"allow_failure":true}, + {"sharpnesses":[500000,500000,500000,500000,500000,600000,900000,600000,500000],"label":["success","continue"]}, + {"sharpnesses":[500000,500000,500000,500000,500000,500000,600000,900000,600000],"label":"continue"}, + {"sharpnesses":[100000,100000,100000,100000,200000,500000,200000,100000,100000],"label":["success"],"allow_failure":true}, + {"sharpnesses":[100000,100000,100000,100000,100000,200000,500000,200000,100000],"label":["success","continue"]}, + {"sharpnesses":[100000,100000,100000,100000,100000,100000,200000,500000,200000],"label":"continue"} +] diff --git a/tests/unit_tests/test_stack_examples.py b/tests/unit_tests/test_stack_examples.py index 27b5123c..6c3ecbda 100644 --- a/tests/unit_tests/test_stack_examples.py +++ b/tests/unit_tests/test_stack_examples.py @@ -11,6 +11,8 @@ objects to simulate real camera captures. import json import os +import pytest + from labthings_fastapi.testing import create_thing_without_server from openflexure_microscope_server.things.autofocus import AutofocusThing @@ -45,41 +47,56 @@ def make_captures(sharpness_list): return [MockCapture(s, i) for i, s in enumerate(sharpness_list)] -def test_stack_labelling(): - """Test stack classification accuracy against labelled sharpness cases. +def load_cases(): + """Load the sharpness data from the json. - This test loads predefined sharpness profiles and their expected labels - from a JSON file, converts them into mock capture objects, and evaluates - the classification returned by `check_stack_result`. - - The test asserts that at least 90% of cases are correctly classified. - - Expected labels may be a single value or a list of acceptable values. + Includes the sharpnesses to test, manually written labels + on the required result, and whether the test is allowed to + fail. This ensures future tests will flag any regression on + tests while allowing improvements. """ - autofocus_thing = create_thing_without_server(AutofocusThing, mock_all_slots=True) - with open(DATA_PATH) as f: data = json.load(f) - success = 0 - total = len(data) - - for _i, case in enumerate(data): - sharpnesses = case["sharpnesses"] + params = [] + for i, case in enumerate(data): expected = case["label"] # Allow multiple acceptable labels if not isinstance(expected, list): expected = [expected] - # Convert to capture objects - captures = make_captures(sharpnesses) + marks = [] - # Call the method under test - result, _ = autofocus_thing.check_stack_result( - captures, check_turning_points=False + if case.get("allow_failure", False): + marks.append(pytest.mark.xfail(reason="Known failing case")) + + params.append( + pytest.param( + case["sharpnesses"], + expected, + marks=marks, + id=f"case_{i}", + ) ) - if result in expected: - success += 1 + return params - assert success > 0.8 * total + +@pytest.mark.parametrize(("sharpnesses", "expected"), load_cases()) +def test_stack_labelling(sharpnesses, expected): + """Test stack classification accuracy against labelled sharpness cases. + + This test loads the test from load_cases and evaluates + the classification returned by `check_stack_result`. + + The test ensures that only cases marked with "allow_failure: true" can fail. + + Expected labels may be a single value or a list of acceptable values. + """ + autofocus_thing = create_thing_without_server(AutofocusThing, mock_all_slots=True) + + captures = make_captures(sharpnesses) + + result, _ = autofocus_thing.check_stack_result(captures, check_turning_points=False) + + assert result in expected