Updates from review, mark tests as allowed to fail

src/openflexure_microscope_server/things/autofocus.py

@@ -722,9 +722,9 @@ class AutofocusThing(lt.Thing):
 
         # Manually test for monotomically increasing or decreasing sharpnesses, as
         # fitting can struggle with these and their behaviour is simpler to hardcode
-        if np.array_equal(sharpnesses, np.sort(sharpnesses)):
+        if np.all(sharpnesses[:-1] <= sharpnesses[1:]):
             return "continue", capture_id
-        if np.array_equal(sharpnesses, np.sort(sharpnesses)[::-1]):
+        if np.all(sharpnesses[:-1] >= sharpnesses[1:]):
             return "restart", capture_id
 
         try:

tests/unit_tests/data/sharpness_test_cases.json

@@ -34,10 +34,10 @@
   {"sharpnesses":[600000,700000,800000,900000,1100000,1300000,1500000,1700000,1800000],"label":"continue"},
   {"sharpnesses":[500000,700000,1000000,1400000,1800000,1700000,1400000,1000000,700000],"label":"success"},
   {"sharpnesses":[500000,600000,900000,1300000,1800000,1900000,1700000,1300000,900000],"label":"success"},
-  {"sharpnesses": [500000, 500000, 500000, 500000, 600000, 900000, 600000, 500000, 500000], "label": ["success"]},
+  {"sharpnesses":[500000,500000,500000,500000,600000,900000,600000,500000,500000],"label":["success"],"allow_failure":true},
   {"sharpnesses":[500000,500000,500000,500000,500000,600000,900000,600000,500000],"label":["success","continue"]},
   {"sharpnesses":[500000,500000,500000,500000,500000,500000,600000,900000,600000],"label":"continue"},
-  {"sharpnesses": [100000, 100000, 100000, 100000, 200000, 500000, 200000, 100000, 100000], "label": ["success"]},
+  {"sharpnesses":[100000,100000,100000,100000,200000,500000,200000,100000,100000],"label":["success"],"allow_failure":true},
   {"sharpnesses":[100000,100000,100000,100000,100000,200000,500000,200000,100000],"label":["success","continue"]},
   {"sharpnesses":[100000,100000,100000,100000,100000,100000,200000,500000,200000],"label":"continue"}
 ]

tests/unit_tests/test_stack_examples.py

@@ -11,6 +11,8 @@ objects to simulate real camera captures.
 import json
 import os
 
+import pytest
+
 from labthings_fastapi.testing import create_thing_without_server
 
 from openflexure_microscope_server.things.autofocus import AutofocusThing
@@ -45,41 +47,56 @@ def make_captures(sharpness_list):
     return [MockCapture(s, i) for i, s in enumerate(sharpness_list)]
 
 
-def test_stack_labelling():
+def load_cases():
-    """Test stack classification accuracy against labelled sharpness cases.
+    """Load the sharpness data from the json.
 
-    This test loads predefined sharpness profiles and their expected labels
+    Includes the sharpnesses to test, manually written labels
-    from a JSON file, converts them into mock capture objects, and evaluates
+    on the required result, and whether the test is allowed to
-    the classification returned by `check_stack_result`.
+    fail. This ensures future tests will flag any regression on
-
+    tests while allowing improvements.
-    The test asserts that at least 90% of cases are correctly classified.
-
-    Expected labels may be a single value or a list of acceptable values.
     """
-    autofocus_thing = create_thing_without_server(AutofocusThing, mock_all_slots=True)
-
     with open(DATA_PATH) as f:
         data = json.load(f)
 
-    success = 0
+    params = []
-    total = len(data)
+    for i, case in enumerate(data):
-
-    for _i, case in enumerate(data):
-        sharpnesses = case["sharpnesses"]
         expected = case["label"]
 
         # Allow multiple acceptable labels
         if not isinstance(expected, list):
             expected = [expected]
 
-        # Convert to capture objects
+        marks = []
+
+        if case.get("allow_failure", False):
+            marks.append(pytest.mark.xfail(reason="Known failing case"))
+
+        params.append(
+            pytest.param(
+                case["sharpnesses"],
+                expected,
+                marks=marks,
+                id=f"case_{i}",
+            )
+        )
+    return params
+
+
+@pytest.mark.parametrize(("sharpnesses", "expected"), load_cases())
+def test_stack_labelling(sharpnesses, expected):
+    """Test stack classification accuracy against labelled sharpness cases.
+
+    This test loads the test from load_cases and evaluates
+    the classification returned by `check_stack_result`.
+
+    The test ensures that only cases marked with "allow_failure: true" can fail.
+
+    Expected labels may be a single value or a list of acceptable values.
+    """
+    autofocus_thing = create_thing_without_server(AutofocusThing, mock_all_slots=True)
+
     captures = make_captures(sharpnesses)
 
-        # Call the method under test
+    result, _ = autofocus_thing.check_stack_result(captures, check_turning_points=False)
-        result, _ = autofocus_thing.check_stack_result(
-            captures, check_turning_points=False
-        )
-        if result in expected:
-            success += 1
 
-    assert success > 0.8 * total
+    assert result in expected