Updates from review, mark tests as allowed to fail

tests/unit_tests/test_stack_examples.py

@@ -11,6 +11,8 @@ objects to simulate real camera captures.
 import json
 import os
 
+import pytest
+
 from labthings_fastapi.testing import create_thing_without_server
 
 from openflexure_microscope_server.things.autofocus import AutofocusThing
@@ -45,41 +47,56 @@ def make_captures(sharpness_list):
     return [MockCapture(s, i) for i, s in enumerate(sharpness_list)]
 
 
-def test_stack_labelling():
-    """Test stack classification accuracy against labelled sharpness cases.
+def load_cases():
+    """Load the sharpness data from the json.
 
-    This test loads predefined sharpness profiles and their expected labels
-    from a JSON file, converts them into mock capture objects, and evaluates
-    the classification returned by `check_stack_result`.
-
-    The test asserts that at least 90% of cases are correctly classified.
-
-    Expected labels may be a single value or a list of acceptable values.
+    Includes the sharpnesses to test, manually written labels
+    on the required result, and whether the test is allowed to
+    fail. This ensures future tests will flag any regression on
+    tests while allowing improvements.
     """
-    autofocus_thing = create_thing_without_server(AutofocusThing, mock_all_slots=True)
-
     with open(DATA_PATH) as f:
         data = json.load(f)
 
-    success = 0
-    total = len(data)
-
-    for _i, case in enumerate(data):
-        sharpnesses = case["sharpnesses"]
+    params = []
+    for i, case in enumerate(data):
         expected = case["label"]
 
         # Allow multiple acceptable labels
         if not isinstance(expected, list):
             expected = [expected]
 
-        # Convert to capture objects
-        captures = make_captures(sharpnesses)
+        marks = []
 
-        # Call the method under test
-        result, _ = autofocus_thing.check_stack_result(
-            captures, check_turning_points=False
+        if case.get("allow_failure", False):
+            marks.append(pytest.mark.xfail(reason="Known failing case"))
+
+        params.append(
+            pytest.param(
+                case["sharpnesses"],
+                expected,
+                marks=marks,
+                id=f"case_{i}",
+            )
         )
-        if result in expected:
-            success += 1
+    return params
 
-    assert success > 0.8 * total
+
+@pytest.mark.parametrize(("sharpnesses", "expected"), load_cases())
+def test_stack_labelling(sharpnesses, expected):
+    """Test stack classification accuracy against labelled sharpness cases.
+
+    This test loads the test from load_cases and evaluates
+    the classification returned by `check_stack_result`.
+
+    The test ensures that only cases marked with "allow_failure: true" can fail.
+
+    Expected labels may be a single value or a list of acceptable values.
+    """
+    autofocus_thing = create_thing_without_server(AutofocusThing, mock_all_slots=True)
+
+    captures = make_captures(sharpnesses)
+
+    result, _ = autofocus_thing.check_stack_result(captures, check_turning_points=False)
+
+    assert result in expected