diff --git a/src/openflexure_microscope_server/things/autofocus.py b/src/openflexure_microscope_server/things/autofocus.py index ca9992a0..61d1ff0a 100644 --- a/src/openflexure_microscope_server/things/autofocus.py +++ b/src/openflexure_microscope_server/things/autofocus.py @@ -722,9 +722,9 @@ class AutofocusThing(lt.Thing): # Manually test for monotomically increasing or decreasing sharpnesses, as # fitting can struggle with these and their behaviour is simpler to hardcode - if np.array_equal(sharpnesses, np.sort(sharpnesses)): + if np.all(sharpnesses[:-1] <= sharpnesses[1:]): return "continue", capture_id - if np.array_equal(sharpnesses, np.sort(sharpnesses)[::-1]): + if np.all(sharpnesses[:-1] >= sharpnesses[1:]): return "restart", capture_id try: diff --git a/tests/unit_tests/data/sharpness_test_cases.json b/tests/unit_tests/data/sharpness_test_cases.json index 0a50f44f..27967eea 100644 --- a/tests/unit_tests/data/sharpness_test_cases.json +++ b/tests/unit_tests/data/sharpness_test_cases.json @@ -1,43 +1,43 @@ [ - {"sharpnesses": [100000, 300000, 500000, 300000, 100000], "label": "success"}, - {"sharpnesses": [100000, 200000, 400000, 600000, 400000, 200000, 100000], "label": "success"}, - {"sharpnesses": [100000, 200000, 300000, 400000, 500000], "label": "continue"}, - {"sharpnesses": [500000, 400000, 300000, 200000, 100000], "label": "restart"}, - {"sharpnesses": [100000, 200000, 300000, 200000, 100000, 50000], "label": ["success"]}, - {"sharpnesses": [0, 100000, 200000, 300000, 400000, 500000], "label": "continue"}, - {"sharpnesses": [500000, 500000, 500000, 500000, 500000], "label": "continue"}, - {"sharpnesses": [100000, 200000, 300000, 300000, 200000, 100000], "label": ["success"]}, - {"sharpnesses": [100000, 200000, 300000, 400000, 400000, 300000, 200000], "label": ["success", "continue"]}, - {"sharpnesses": [100000, 300000, 500000, 500000, 500000, 300000, 100000], "label": ["success", "continue"]}, - {"sharpnesses": [100000, 300000, 500000, 400000, 500000, 300000, 100000], "label": ["continue", "success"]}, - {"sharpnesses": [100000, 400000, 600000, 500000, 300000, 200000, 100000], "label": "success"}, - {"sharpnesses": [100000, 200000, 500000, 700000, 600000, 300000, 100000], "label": "success"}, - {"sharpnesses": [100000, 200000, 300000, 500000, 700000, 600000, 400000], "label": ["success", "continue"]}, - {"sharpnesses": [700000, 600000, 500000, 400000, 300000, 200000, 100000, 0], "label": "restart"}, - {"sharpnesses": [100000, 300000, 600000, 900000, 600000, 300000, 200000, 100000], "label": "success"}, - {"sharpnesses": [100000, 200000, 400000, 800000, 900000, 800000, 400000, 200000], "label": "success"}, - {"sharpnesses": [100000, 200000, 300000, 400000, 500000, 600000, 700000, 800000], "label": "continue"}, - {"sharpnesses": [800000, 700000, 600000, 500000, 400000, 300000, 200000, 100000], "label": "restart"}, - {"sharpnesses": [100000, 200000, 300000, 400000, 400000, 400000, 300000, 200000], "label": ["success", "continue"]}, - {"sharpnesses": [100000, 200000, 300000, 500000, 500000, 500000, 500000, 500000], "label": "continue"}, - {"sharpnesses": [500000, 500000, 500000, 500000, 500000, 400000, 300000, 200000], "label": "restart"}, - {"sharpnesses": [100000, 200000, 300000, 600000, 900000, 600000, 300000, 200000, 100000], "label": "success"}, - {"sharpnesses": [100000, 200000, 400000, 700000, 1000000, 900000, 700000, 400000, 200000], "label": "success"}, - {"sharpnesses": [100000, 200000, 300000, 400000, 500000, 600000, 700000, 800000, 900000], "label": "continue"}, - {"sharpnesses": [900000, 800000, 700000, 600000, 500000, 400000, 300000, 200000, 100000], "label": "restart"}, - {"sharpnesses": [100000, 200000, 1000000, 200000, 1000000, 200000, 100000], "label": "continue"}, - {"sharpnesses": [100000, 500000, 100000, 500000, 100000, 500000, 100000], "label": ["restart", "continue"]}, - {"sharpnesses": [300000, 300000, 300000, 300000, 300000, 300000, 300000], "label": "continue"}, - {"sharpnesses": [500000, 600000, 800000, 1100000, 1500000, 1100000, 800000, 600000, 500000], "label": "success"}, - {"sharpnesses": [1000000, 1100000, 1300000, 1600000, 2000000, 1600000, 1300000, 1100000, 1000000], "label": "success"}, - {"sharpnesses": [2000000, 1800000, 1500000, 1200000, 1000000, 900000, 800000, 700000, 600000], "label": "restart"}, - {"sharpnesses": [600000, 700000, 800000, 900000, 1100000, 1300000, 1500000, 1700000, 1800000], "label": "continue"}, - {"sharpnesses": [500000, 700000, 1000000, 1400000, 1800000, 1700000, 1400000, 1000000, 700000], "label": "success"}, - {"sharpnesses": [500000, 600000, 900000, 1300000, 1800000, 1900000, 1700000, 1300000, 900000], "label": "success"}, - {"sharpnesses": [500000, 500000, 500000, 500000, 600000, 900000, 600000, 500000, 500000], "label": ["success"]}, - {"sharpnesses": [500000, 500000, 500000, 500000, 500000, 600000, 900000, 600000, 500000], "label": ["success", "continue"]}, - {"sharpnesses": [500000, 500000, 500000, 500000, 500000, 500000, 600000, 900000, 600000], "label": "continue"}, - {"sharpnesses": [100000, 100000, 100000, 100000, 200000, 500000, 200000, 100000, 100000], "label": ["success"]}, - {"sharpnesses": [100000, 100000, 100000, 100000, 100000, 200000, 500000, 200000, 100000], "label": ["success", "continue"]}, - {"sharpnesses": [100000, 100000, 100000, 100000, 100000, 100000, 200000, 500000, 200000], "label": "continue"} -] \ No newline at end of file + {"sharpnesses":[100000,300000,500000,300000,100000],"label":"success"}, + {"sharpnesses":[100000,200000,400000,600000,400000,200000,100000],"label":"success"}, + {"sharpnesses":[100000,200000,300000,400000,500000],"label":"continue"}, + {"sharpnesses":[500000,400000,300000,200000,100000],"label":"restart"}, + {"sharpnesses":[100000,200000,300000,200000,100000,50000],"label":["success"]}, + {"sharpnesses":[0,100000,200000,300000,400000,500000],"label":"continue"}, + {"sharpnesses":[500000,500000,500000,500000,500000],"label":"continue"}, + {"sharpnesses":[100000,200000,300000,300000,200000,100000],"label":["success"]}, + {"sharpnesses":[100000,200000,300000,400000,400000,300000,200000],"label":["success","continue"]}, + {"sharpnesses":[100000,300000,500000,500000,500000,300000,100000],"label":["success","continue"]}, + {"sharpnesses":[100000,300000,500000,400000,500000,300000,100000],"label":["continue","success"]}, + {"sharpnesses":[100000,400000,600000,500000,300000,200000,100000],"label":"success"}, + {"sharpnesses":[100000,200000,500000,700000,600000,300000,100000],"label":"success"}, + {"sharpnesses":[100000,200000,300000,500000,700000,600000,400000],"label":["success","continue"]}, + {"sharpnesses":[700000,600000,500000,400000,300000,200000,100000,0],"label":"restart"}, + {"sharpnesses":[100000,300000,600000,900000,600000,300000,200000,100000],"label":"success"}, + {"sharpnesses":[100000,200000,400000,800000,900000,800000,400000,200000],"label":"success"}, + {"sharpnesses":[100000,200000,300000,400000,500000,600000,700000,800000],"label":"continue"}, + {"sharpnesses":[800000,700000,600000,500000,400000,300000,200000,100000],"label":"restart"}, + {"sharpnesses":[100000,200000,300000,400000,400000,400000,300000,200000],"label":["success","continue"]}, + {"sharpnesses":[100000,200000,300000,500000,500000,500000,500000,500000],"label":"continue"}, + {"sharpnesses":[500000,500000,500000,500000,500000,400000,300000,200000],"label":"restart"}, + {"sharpnesses":[100000,200000,300000,600000,900000,600000,300000,200000,100000],"label":"success"}, + {"sharpnesses":[100000,200000,400000,700000,1000000,900000,700000,400000,200000],"label":"success"}, + {"sharpnesses":[100000,200000,300000,400000,500000,600000,700000,800000,900000],"label":"continue"}, + {"sharpnesses":[900000,800000,700000,600000,500000,400000,300000,200000,100000],"label":"restart"}, + {"sharpnesses":[100000,200000,1000000,200000,1000000,200000,100000],"label":"continue"}, + {"sharpnesses":[100000,500000,100000,500000,100000,500000,100000],"label":["restart","continue"]}, + {"sharpnesses":[300000,300000,300000,300000,300000,300000,300000],"label":"continue"}, + {"sharpnesses":[500000,600000,800000,1100000,1500000,1100000,800000,600000,500000],"label":"success"}, + {"sharpnesses":[1000000,1100000,1300000,1600000,2000000,1600000,1300000,1100000,1000000],"label":"success"}, + {"sharpnesses":[2000000,1800000,1500000,1200000,1000000,900000,800000,700000,600000],"label":"restart"}, + {"sharpnesses":[600000,700000,800000,900000,1100000,1300000,1500000,1700000,1800000],"label":"continue"}, + {"sharpnesses":[500000,700000,1000000,1400000,1800000,1700000,1400000,1000000,700000],"label":"success"}, + {"sharpnesses":[500000,600000,900000,1300000,1800000,1900000,1700000,1300000,900000],"label":"success"}, + {"sharpnesses":[500000,500000,500000,500000,600000,900000,600000,500000,500000],"label":["success"],"allow_failure":true}, + {"sharpnesses":[500000,500000,500000,500000,500000,600000,900000,600000,500000],"label":["success","continue"]}, + {"sharpnesses":[500000,500000,500000,500000,500000,500000,600000,900000,600000],"label":"continue"}, + {"sharpnesses":[100000,100000,100000,100000,200000,500000,200000,100000,100000],"label":["success"],"allow_failure":true}, + {"sharpnesses":[100000,100000,100000,100000,100000,200000,500000,200000,100000],"label":["success","continue"]}, + {"sharpnesses":[100000,100000,100000,100000,100000,100000,200000,500000,200000],"label":"continue"} +] diff --git a/tests/unit_tests/test_stack_examples.py b/tests/unit_tests/test_stack_examples.py index 27b5123c..6c3ecbda 100644 --- a/tests/unit_tests/test_stack_examples.py +++ b/tests/unit_tests/test_stack_examples.py @@ -11,6 +11,8 @@ objects to simulate real camera captures. import json import os +import pytest + from labthings_fastapi.testing import create_thing_without_server from openflexure_microscope_server.things.autofocus import AutofocusThing @@ -45,41 +47,56 @@ def make_captures(sharpness_list): return [MockCapture(s, i) for i, s in enumerate(sharpness_list)] -def test_stack_labelling(): - """Test stack classification accuracy against labelled sharpness cases. +def load_cases(): + """Load the sharpness data from the json. - This test loads predefined sharpness profiles and their expected labels - from a JSON file, converts them into mock capture objects, and evaluates - the classification returned by `check_stack_result`. - - The test asserts that at least 90% of cases are correctly classified. - - Expected labels may be a single value or a list of acceptable values. + Includes the sharpnesses to test, manually written labels + on the required result, and whether the test is allowed to + fail. This ensures future tests will flag any regression on + tests while allowing improvements. """ - autofocus_thing = create_thing_without_server(AutofocusThing, mock_all_slots=True) - with open(DATA_PATH) as f: data = json.load(f) - success = 0 - total = len(data) - - for _i, case in enumerate(data): - sharpnesses = case["sharpnesses"] + params = [] + for i, case in enumerate(data): expected = case["label"] # Allow multiple acceptable labels if not isinstance(expected, list): expected = [expected] - # Convert to capture objects - captures = make_captures(sharpnesses) + marks = [] - # Call the method under test - result, _ = autofocus_thing.check_stack_result( - captures, check_turning_points=False + if case.get("allow_failure", False): + marks.append(pytest.mark.xfail(reason="Known failing case")) + + params.append( + pytest.param( + case["sharpnesses"], + expected, + marks=marks, + id=f"case_{i}", + ) ) - if result in expected: - success += 1 + return params - assert success > 0.8 * total + +@pytest.mark.parametrize(("sharpnesses", "expected"), load_cases()) +def test_stack_labelling(sharpnesses, expected): + """Test stack classification accuracy against labelled sharpness cases. + + This test loads the test from load_cases and evaluates + the classification returned by `check_stack_result`. + + The test ensures that only cases marked with "allow_failure: true" can fail. + + Expected labels may be a single value or a list of acceptable values. + """ + autofocus_thing = create_thing_without_server(AutofocusThing, mock_all_slots=True) + + captures = make_captures(sharpnesses) + + result, _ = autofocus_thing.check_stack_result(captures, check_turning_points=False) + + assert result in expected