Updates from review, mark tests as allowed to fail
This commit is contained in:
Joe Knapper
parent
bc6ea9a1ca
commit
4da8172a16
3 changed files with 85 additions and 68 deletions
|
|
@ -722,9 +722,9 @@ class AutofocusThing(lt.Thing):
|
|||
|
||||
# Manually test for monotomically increasing or decreasing sharpnesses, as
|
||||
# fitting can struggle with these and their behaviour is simpler to hardcode
|
||||
if np.array_equal(sharpnesses, np.sort(sharpnesses)):
|
||||
if np.all(sharpnesses[:-1] <= sharpnesses[1:]):
|
||||
return "continue", capture_id
|
||||
if np.array_equal(sharpnesses, np.sort(sharpnesses)[::-1]):
|
||||
if np.all(sharpnesses[:-1] >= sharpnesses[1:]):
|
||||
return "restart", capture_id
|
||||
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -1,43 +1,43 @@
|
|||
[
|
||||
{"sharpnesses": [100000, 300000, 500000, 300000, 100000], "label": "success"},
|
||||
{"sharpnesses": [100000, 200000, 400000, 600000, 400000, 200000, 100000], "label": "success"},
|
||||
{"sharpnesses": [100000, 200000, 300000, 400000, 500000], "label": "continue"},
|
||||
{"sharpnesses": [500000, 400000, 300000, 200000, 100000], "label": "restart"},
|
||||
{"sharpnesses": [100000, 200000, 300000, 200000, 100000, 50000], "label": ["success"]},
|
||||
{"sharpnesses": [0, 100000, 200000, 300000, 400000, 500000], "label": "continue"},
|
||||
{"sharpnesses": [500000, 500000, 500000, 500000, 500000], "label": "continue"},
|
||||
{"sharpnesses": [100000, 200000, 300000, 300000, 200000, 100000], "label": ["success"]},
|
||||
{"sharpnesses": [100000, 200000, 300000, 400000, 400000, 300000, 200000], "label": ["success", "continue"]},
|
||||
{"sharpnesses": [100000, 300000, 500000, 500000, 500000, 300000, 100000], "label": ["success", "continue"]},
|
||||
{"sharpnesses": [100000, 300000, 500000, 400000, 500000, 300000, 100000], "label": ["continue", "success"]},
|
||||
{"sharpnesses": [100000, 400000, 600000, 500000, 300000, 200000, 100000], "label": "success"},
|
||||
{"sharpnesses": [100000, 200000, 500000, 700000, 600000, 300000, 100000], "label": "success"},
|
||||
{"sharpnesses": [100000, 200000, 300000, 500000, 700000, 600000, 400000], "label": ["success", "continue"]},
|
||||
{"sharpnesses": [700000, 600000, 500000, 400000, 300000, 200000, 100000, 0], "label": "restart"},
|
||||
{"sharpnesses": [100000, 300000, 600000, 900000, 600000, 300000, 200000, 100000], "label": "success"},
|
||||
{"sharpnesses": [100000, 200000, 400000, 800000, 900000, 800000, 400000, 200000], "label": "success"},
|
||||
{"sharpnesses": [100000, 200000, 300000, 400000, 500000, 600000, 700000, 800000], "label": "continue"},
|
||||
{"sharpnesses": [800000, 700000, 600000, 500000, 400000, 300000, 200000, 100000], "label": "restart"},
|
||||
{"sharpnesses": [100000, 200000, 300000, 400000, 400000, 400000, 300000, 200000], "label": ["success", "continue"]},
|
||||
{"sharpnesses": [100000, 200000, 300000, 500000, 500000, 500000, 500000, 500000], "label": "continue"},
|
||||
{"sharpnesses": [500000, 500000, 500000, 500000, 500000, 400000, 300000, 200000], "label": "restart"},
|
||||
{"sharpnesses": [100000, 200000, 300000, 600000, 900000, 600000, 300000, 200000, 100000], "label": "success"},
|
||||
{"sharpnesses": [100000, 200000, 400000, 700000, 1000000, 900000, 700000, 400000, 200000], "label": "success"},
|
||||
{"sharpnesses": [100000, 200000, 300000, 400000, 500000, 600000, 700000, 800000, 900000], "label": "continue"},
|
||||
{"sharpnesses": [900000, 800000, 700000, 600000, 500000, 400000, 300000, 200000, 100000], "label": "restart"},
|
||||
{"sharpnesses": [100000, 200000, 1000000, 200000, 1000000, 200000, 100000], "label": "continue"},
|
||||
{"sharpnesses": [100000, 500000, 100000, 500000, 100000, 500000, 100000], "label": ["restart", "continue"]},
|
||||
{"sharpnesses": [300000, 300000, 300000, 300000, 300000, 300000, 300000], "label": "continue"},
|
||||
{"sharpnesses": [500000, 600000, 800000, 1100000, 1500000, 1100000, 800000, 600000, 500000], "label": "success"},
|
||||
{"sharpnesses": [1000000, 1100000, 1300000, 1600000, 2000000, 1600000, 1300000, 1100000, 1000000], "label": "success"},
|
||||
{"sharpnesses": [2000000, 1800000, 1500000, 1200000, 1000000, 900000, 800000, 700000, 600000], "label": "restart"},
|
||||
{"sharpnesses": [600000, 700000, 800000, 900000, 1100000, 1300000, 1500000, 1700000, 1800000], "label": "continue"},
|
||||
{"sharpnesses": [500000, 700000, 1000000, 1400000, 1800000, 1700000, 1400000, 1000000, 700000], "label": "success"},
|
||||
{"sharpnesses": [500000, 600000, 900000, 1300000, 1800000, 1900000, 1700000, 1300000, 900000], "label": "success"},
|
||||
{"sharpnesses": [500000, 500000, 500000, 500000, 600000, 900000, 600000, 500000, 500000], "label": ["success"]},
|
||||
{"sharpnesses": [500000, 500000, 500000, 500000, 500000, 600000, 900000, 600000, 500000], "label": ["success", "continue"]},
|
||||
{"sharpnesses": [500000, 500000, 500000, 500000, 500000, 500000, 600000, 900000, 600000], "label": "continue"},
|
||||
{"sharpnesses": [100000, 100000, 100000, 100000, 200000, 500000, 200000, 100000, 100000], "label": ["success"]},
|
||||
{"sharpnesses": [100000, 100000, 100000, 100000, 100000, 200000, 500000, 200000, 100000], "label": ["success", "continue"]},
|
||||
{"sharpnesses": [100000, 100000, 100000, 100000, 100000, 100000, 200000, 500000, 200000], "label": "continue"}
|
||||
]
|
||||
{"sharpnesses":[100000,300000,500000,300000,100000],"label":"success"},
|
||||
{"sharpnesses":[100000,200000,400000,600000,400000,200000,100000],"label":"success"},
|
||||
{"sharpnesses":[100000,200000,300000,400000,500000],"label":"continue"},
|
||||
{"sharpnesses":[500000,400000,300000,200000,100000],"label":"restart"},
|
||||
{"sharpnesses":[100000,200000,300000,200000,100000,50000],"label":["success"]},
|
||||
{"sharpnesses":[0,100000,200000,300000,400000,500000],"label":"continue"},
|
||||
{"sharpnesses":[500000,500000,500000,500000,500000],"label":"continue"},
|
||||
{"sharpnesses":[100000,200000,300000,300000,200000,100000],"label":["success"]},
|
||||
{"sharpnesses":[100000,200000,300000,400000,400000,300000,200000],"label":["success","continue"]},
|
||||
{"sharpnesses":[100000,300000,500000,500000,500000,300000,100000],"label":["success","continue"]},
|
||||
{"sharpnesses":[100000,300000,500000,400000,500000,300000,100000],"label":["continue","success"]},
|
||||
{"sharpnesses":[100000,400000,600000,500000,300000,200000,100000],"label":"success"},
|
||||
{"sharpnesses":[100000,200000,500000,700000,600000,300000,100000],"label":"success"},
|
||||
{"sharpnesses":[100000,200000,300000,500000,700000,600000,400000],"label":["success","continue"]},
|
||||
{"sharpnesses":[700000,600000,500000,400000,300000,200000,100000,0],"label":"restart"},
|
||||
{"sharpnesses":[100000,300000,600000,900000,600000,300000,200000,100000],"label":"success"},
|
||||
{"sharpnesses":[100000,200000,400000,800000,900000,800000,400000,200000],"label":"success"},
|
||||
{"sharpnesses":[100000,200000,300000,400000,500000,600000,700000,800000],"label":"continue"},
|
||||
{"sharpnesses":[800000,700000,600000,500000,400000,300000,200000,100000],"label":"restart"},
|
||||
{"sharpnesses":[100000,200000,300000,400000,400000,400000,300000,200000],"label":["success","continue"]},
|
||||
{"sharpnesses":[100000,200000,300000,500000,500000,500000,500000,500000],"label":"continue"},
|
||||
{"sharpnesses":[500000,500000,500000,500000,500000,400000,300000,200000],"label":"restart"},
|
||||
{"sharpnesses":[100000,200000,300000,600000,900000,600000,300000,200000,100000],"label":"success"},
|
||||
{"sharpnesses":[100000,200000,400000,700000,1000000,900000,700000,400000,200000],"label":"success"},
|
||||
{"sharpnesses":[100000,200000,300000,400000,500000,600000,700000,800000,900000],"label":"continue"},
|
||||
{"sharpnesses":[900000,800000,700000,600000,500000,400000,300000,200000,100000],"label":"restart"},
|
||||
{"sharpnesses":[100000,200000,1000000,200000,1000000,200000,100000],"label":"continue"},
|
||||
{"sharpnesses":[100000,500000,100000,500000,100000,500000,100000],"label":["restart","continue"]},
|
||||
{"sharpnesses":[300000,300000,300000,300000,300000,300000,300000],"label":"continue"},
|
||||
{"sharpnesses":[500000,600000,800000,1100000,1500000,1100000,800000,600000,500000],"label":"success"},
|
||||
{"sharpnesses":[1000000,1100000,1300000,1600000,2000000,1600000,1300000,1100000,1000000],"label":"success"},
|
||||
{"sharpnesses":[2000000,1800000,1500000,1200000,1000000,900000,800000,700000,600000],"label":"restart"},
|
||||
{"sharpnesses":[600000,700000,800000,900000,1100000,1300000,1500000,1700000,1800000],"label":"continue"},
|
||||
{"sharpnesses":[500000,700000,1000000,1400000,1800000,1700000,1400000,1000000,700000],"label":"success"},
|
||||
{"sharpnesses":[500000,600000,900000,1300000,1800000,1900000,1700000,1300000,900000],"label":"success"},
|
||||
{"sharpnesses":[500000,500000,500000,500000,600000,900000,600000,500000,500000],"label":["success"],"allow_failure":true},
|
||||
{"sharpnesses":[500000,500000,500000,500000,500000,600000,900000,600000,500000],"label":["success","continue"]},
|
||||
{"sharpnesses":[500000,500000,500000,500000,500000,500000,600000,900000,600000],"label":"continue"},
|
||||
{"sharpnesses":[100000,100000,100000,100000,200000,500000,200000,100000,100000],"label":["success"],"allow_failure":true},
|
||||
{"sharpnesses":[100000,100000,100000,100000,100000,200000,500000,200000,100000],"label":["success","continue"]},
|
||||
{"sharpnesses":[100000,100000,100000,100000,100000,100000,200000,500000,200000],"label":"continue"}
|
||||
]
|
||||
|
|
|
|||
|
|
@ -11,6 +11,8 @@ objects to simulate real camera captures.
|
|||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from labthings_fastapi.testing import create_thing_without_server
|
||||
|
||||
from openflexure_microscope_server.things.autofocus import AutofocusThing
|
||||
|
|
@ -45,41 +47,56 @@ def make_captures(sharpness_list):
|
|||
return [MockCapture(s, i) for i, s in enumerate(sharpness_list)]
|
||||
|
||||
|
||||
def test_stack_labelling():
|
||||
"""Test stack classification accuracy against labelled sharpness cases.
|
||||
def load_cases():
|
||||
"""Load the sharpness data from the json.
|
||||
|
||||
This test loads predefined sharpness profiles and their expected labels
|
||||
from a JSON file, converts them into mock capture objects, and evaluates
|
||||
the classification returned by `check_stack_result`.
|
||||
|
||||
The test asserts that at least 90% of cases are correctly classified.
|
||||
|
||||
Expected labels may be a single value or a list of acceptable values.
|
||||
Includes the sharpnesses to test, manually written labels
|
||||
on the required result, and whether the test is allowed to
|
||||
fail. This ensures future tests will flag any regression on
|
||||
tests while allowing improvements.
|
||||
"""
|
||||
autofocus_thing = create_thing_without_server(AutofocusThing, mock_all_slots=True)
|
||||
|
||||
with open(DATA_PATH) as f:
|
||||
data = json.load(f)
|
||||
|
||||
success = 0
|
||||
total = len(data)
|
||||
|
||||
for _i, case in enumerate(data):
|
||||
sharpnesses = case["sharpnesses"]
|
||||
params = []
|
||||
for i, case in enumerate(data):
|
||||
expected = case["label"]
|
||||
|
||||
# Allow multiple acceptable labels
|
||||
if not isinstance(expected, list):
|
||||
expected = [expected]
|
||||
|
||||
# Convert to capture objects
|
||||
captures = make_captures(sharpnesses)
|
||||
marks = []
|
||||
|
||||
# Call the method under test
|
||||
result, _ = autofocus_thing.check_stack_result(
|
||||
captures, check_turning_points=False
|
||||
if case.get("allow_failure", False):
|
||||
marks.append(pytest.mark.xfail(reason="Known failing case"))
|
||||
|
||||
params.append(
|
||||
pytest.param(
|
||||
case["sharpnesses"],
|
||||
expected,
|
||||
marks=marks,
|
||||
id=f"case_{i}",
|
||||
)
|
||||
)
|
||||
if result in expected:
|
||||
success += 1
|
||||
return params
|
||||
|
||||
assert success > 0.8 * total
|
||||
|
||||
@pytest.mark.parametrize(("sharpnesses", "expected"), load_cases())
|
||||
def test_stack_labelling(sharpnesses, expected):
|
||||
"""Test stack classification accuracy against labelled sharpness cases.
|
||||
|
||||
This test loads the test from load_cases and evaluates
|
||||
the classification returned by `check_stack_result`.
|
||||
|
||||
The test ensures that only cases marked with "allow_failure: true" can fail.
|
||||
|
||||
Expected labels may be a single value or a list of acceptable values.
|
||||
"""
|
||||
autofocus_thing = create_thing_without_server(AutofocusThing, mock_all_slots=True)
|
||||
|
||||
captures = make_captures(sharpnesses)
|
||||
|
||||
result, _ = autofocus_thing.check_stack_result(captures, check_turning_points=False)
|
||||
|
||||
assert result in expected
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue