Updates from review, mark tests as allowed to fail
This commit is contained in:
Joe Knapper
parent
bc6ea9a1ca
commit
4da8172a16
3 changed files with 85 additions and 68 deletions
|
|
@ -722,9 +722,9 @@ class AutofocusThing(lt.Thing):
|
||||||
|
|
||||||
# Manually test for monotomically increasing or decreasing sharpnesses, as
|
# Manually test for monotomically increasing or decreasing sharpnesses, as
|
||||||
# fitting can struggle with these and their behaviour is simpler to hardcode
|
# fitting can struggle with these and their behaviour is simpler to hardcode
|
||||||
if np.array_equal(sharpnesses, np.sort(sharpnesses)):
|
if np.all(sharpnesses[:-1] <= sharpnesses[1:]):
|
||||||
return "continue", capture_id
|
return "continue", capture_id
|
||||||
if np.array_equal(sharpnesses, np.sort(sharpnesses)[::-1]):
|
if np.all(sharpnesses[:-1] >= sharpnesses[1:]):
|
||||||
return "restart", capture_id
|
return "restart", capture_id
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
|
|
@ -1,43 +1,43 @@
|
||||||
[
|
[
|
||||||
{"sharpnesses": [100000, 300000, 500000, 300000, 100000], "label": "success"},
|
{"sharpnesses":[100000,300000,500000,300000,100000],"label":"success"},
|
||||||
{"sharpnesses": [100000, 200000, 400000, 600000, 400000, 200000, 100000], "label": "success"},
|
{"sharpnesses":[100000,200000,400000,600000,400000,200000,100000],"label":"success"},
|
||||||
{"sharpnesses": [100000, 200000, 300000, 400000, 500000], "label": "continue"},
|
{"sharpnesses":[100000,200000,300000,400000,500000],"label":"continue"},
|
||||||
{"sharpnesses": [500000, 400000, 300000, 200000, 100000], "label": "restart"},
|
{"sharpnesses":[500000,400000,300000,200000,100000],"label":"restart"},
|
||||||
{"sharpnesses": [100000, 200000, 300000, 200000, 100000, 50000], "label": ["success"]},
|
{"sharpnesses":[100000,200000,300000,200000,100000,50000],"label":["success"]},
|
||||||
{"sharpnesses": [0, 100000, 200000, 300000, 400000, 500000], "label": "continue"},
|
{"sharpnesses":[0,100000,200000,300000,400000,500000],"label":"continue"},
|
||||||
{"sharpnesses": [500000, 500000, 500000, 500000, 500000], "label": "continue"},
|
{"sharpnesses":[500000,500000,500000,500000,500000],"label":"continue"},
|
||||||
{"sharpnesses": [100000, 200000, 300000, 300000, 200000, 100000], "label": ["success"]},
|
{"sharpnesses":[100000,200000,300000,300000,200000,100000],"label":["success"]},
|
||||||
{"sharpnesses": [100000, 200000, 300000, 400000, 400000, 300000, 200000], "label": ["success", "continue"]},
|
{"sharpnesses":[100000,200000,300000,400000,400000,300000,200000],"label":["success","continue"]},
|
||||||
{"sharpnesses": [100000, 300000, 500000, 500000, 500000, 300000, 100000], "label": ["success", "continue"]},
|
{"sharpnesses":[100000,300000,500000,500000,500000,300000,100000],"label":["success","continue"]},
|
||||||
{"sharpnesses": [100000, 300000, 500000, 400000, 500000, 300000, 100000], "label": ["continue", "success"]},
|
{"sharpnesses":[100000,300000,500000,400000,500000,300000,100000],"label":["continue","success"]},
|
||||||
{"sharpnesses": [100000, 400000, 600000, 500000, 300000, 200000, 100000], "label": "success"},
|
{"sharpnesses":[100000,400000,600000,500000,300000,200000,100000],"label":"success"},
|
||||||
{"sharpnesses": [100000, 200000, 500000, 700000, 600000, 300000, 100000], "label": "success"},
|
{"sharpnesses":[100000,200000,500000,700000,600000,300000,100000],"label":"success"},
|
||||||
{"sharpnesses": [100000, 200000, 300000, 500000, 700000, 600000, 400000], "label": ["success", "continue"]},
|
{"sharpnesses":[100000,200000,300000,500000,700000,600000,400000],"label":["success","continue"]},
|
||||||
{"sharpnesses": [700000, 600000, 500000, 400000, 300000, 200000, 100000, 0], "label": "restart"},
|
{"sharpnesses":[700000,600000,500000,400000,300000,200000,100000,0],"label":"restart"},
|
||||||
{"sharpnesses": [100000, 300000, 600000, 900000, 600000, 300000, 200000, 100000], "label": "success"},
|
{"sharpnesses":[100000,300000,600000,900000,600000,300000,200000,100000],"label":"success"},
|
||||||
{"sharpnesses": [100000, 200000, 400000, 800000, 900000, 800000, 400000, 200000], "label": "success"},
|
{"sharpnesses":[100000,200000,400000,800000,900000,800000,400000,200000],"label":"success"},
|
||||||
{"sharpnesses": [100000, 200000, 300000, 400000, 500000, 600000, 700000, 800000], "label": "continue"},
|
{"sharpnesses":[100000,200000,300000,400000,500000,600000,700000,800000],"label":"continue"},
|
||||||
{"sharpnesses": [800000, 700000, 600000, 500000, 400000, 300000, 200000, 100000], "label": "restart"},
|
{"sharpnesses":[800000,700000,600000,500000,400000,300000,200000,100000],"label":"restart"},
|
||||||
{"sharpnesses": [100000, 200000, 300000, 400000, 400000, 400000, 300000, 200000], "label": ["success", "continue"]},
|
{"sharpnesses":[100000,200000,300000,400000,400000,400000,300000,200000],"label":["success","continue"]},
|
||||||
{"sharpnesses": [100000, 200000, 300000, 500000, 500000, 500000, 500000, 500000], "label": "continue"},
|
{"sharpnesses":[100000,200000,300000,500000,500000,500000,500000,500000],"label":"continue"},
|
||||||
{"sharpnesses": [500000, 500000, 500000, 500000, 500000, 400000, 300000, 200000], "label": "restart"},
|
{"sharpnesses":[500000,500000,500000,500000,500000,400000,300000,200000],"label":"restart"},
|
||||||
{"sharpnesses": [100000, 200000, 300000, 600000, 900000, 600000, 300000, 200000, 100000], "label": "success"},
|
{"sharpnesses":[100000,200000,300000,600000,900000,600000,300000,200000,100000],"label":"success"},
|
||||||
{"sharpnesses": [100000, 200000, 400000, 700000, 1000000, 900000, 700000, 400000, 200000], "label": "success"},
|
{"sharpnesses":[100000,200000,400000,700000,1000000,900000,700000,400000,200000],"label":"success"},
|
||||||
{"sharpnesses": [100000, 200000, 300000, 400000, 500000, 600000, 700000, 800000, 900000], "label": "continue"},
|
{"sharpnesses":[100000,200000,300000,400000,500000,600000,700000,800000,900000],"label":"continue"},
|
||||||
{"sharpnesses": [900000, 800000, 700000, 600000, 500000, 400000, 300000, 200000, 100000], "label": "restart"},
|
{"sharpnesses":[900000,800000,700000,600000,500000,400000,300000,200000,100000],"label":"restart"},
|
||||||
{"sharpnesses": [100000, 200000, 1000000, 200000, 1000000, 200000, 100000], "label": "continue"},
|
{"sharpnesses":[100000,200000,1000000,200000,1000000,200000,100000],"label":"continue"},
|
||||||
{"sharpnesses": [100000, 500000, 100000, 500000, 100000, 500000, 100000], "label": ["restart", "continue"]},
|
{"sharpnesses":[100000,500000,100000,500000,100000,500000,100000],"label":["restart","continue"]},
|
||||||
{"sharpnesses": [300000, 300000, 300000, 300000, 300000, 300000, 300000], "label": "continue"},
|
{"sharpnesses":[300000,300000,300000,300000,300000,300000,300000],"label":"continue"},
|
||||||
{"sharpnesses": [500000, 600000, 800000, 1100000, 1500000, 1100000, 800000, 600000, 500000], "label": "success"},
|
{"sharpnesses":[500000,600000,800000,1100000,1500000,1100000,800000,600000,500000],"label":"success"},
|
||||||
{"sharpnesses": [1000000, 1100000, 1300000, 1600000, 2000000, 1600000, 1300000, 1100000, 1000000], "label": "success"},
|
{"sharpnesses":[1000000,1100000,1300000,1600000,2000000,1600000,1300000,1100000,1000000],"label":"success"},
|
||||||
{"sharpnesses": [2000000, 1800000, 1500000, 1200000, 1000000, 900000, 800000, 700000, 600000], "label": "restart"},
|
{"sharpnesses":[2000000,1800000,1500000,1200000,1000000,900000,800000,700000,600000],"label":"restart"},
|
||||||
{"sharpnesses": [600000, 700000, 800000, 900000, 1100000, 1300000, 1500000, 1700000, 1800000], "label": "continue"},
|
{"sharpnesses":[600000,700000,800000,900000,1100000,1300000,1500000,1700000,1800000],"label":"continue"},
|
||||||
{"sharpnesses": [500000, 700000, 1000000, 1400000, 1800000, 1700000, 1400000, 1000000, 700000], "label": "success"},
|
{"sharpnesses":[500000,700000,1000000,1400000,1800000,1700000,1400000,1000000,700000],"label":"success"},
|
||||||
{"sharpnesses": [500000, 600000, 900000, 1300000, 1800000, 1900000, 1700000, 1300000, 900000], "label": "success"},
|
{"sharpnesses":[500000,600000,900000,1300000,1800000,1900000,1700000,1300000,900000],"label":"success"},
|
||||||
{"sharpnesses": [500000, 500000, 500000, 500000, 600000, 900000, 600000, 500000, 500000], "label": ["success"]},
|
{"sharpnesses":[500000,500000,500000,500000,600000,900000,600000,500000,500000],"label":["success"],"allow_failure":true},
|
||||||
{"sharpnesses": [500000, 500000, 500000, 500000, 500000, 600000, 900000, 600000, 500000], "label": ["success", "continue"]},
|
{"sharpnesses":[500000,500000,500000,500000,500000,600000,900000,600000,500000],"label":["success","continue"]},
|
||||||
{"sharpnesses": [500000, 500000, 500000, 500000, 500000, 500000, 600000, 900000, 600000], "label": "continue"},
|
{"sharpnesses":[500000,500000,500000,500000,500000,500000,600000,900000,600000],"label":"continue"},
|
||||||
{"sharpnesses": [100000, 100000, 100000, 100000, 200000, 500000, 200000, 100000, 100000], "label": ["success"]},
|
{"sharpnesses":[100000,100000,100000,100000,200000,500000,200000,100000,100000],"label":["success"],"allow_failure":true},
|
||||||
{"sharpnesses": [100000, 100000, 100000, 100000, 100000, 200000, 500000, 200000, 100000], "label": ["success", "continue"]},
|
{"sharpnesses":[100000,100000,100000,100000,100000,200000,500000,200000,100000],"label":["success","continue"]},
|
||||||
{"sharpnesses": [100000, 100000, 100000, 100000, 100000, 100000, 200000, 500000, 200000], "label": "continue"}
|
{"sharpnesses":[100000,100000,100000,100000,100000,100000,200000,500000,200000],"label":"continue"}
|
||||||
]
|
]
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,8 @@ objects to simulate real camera captures.
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
from labthings_fastapi.testing import create_thing_without_server
|
from labthings_fastapi.testing import create_thing_without_server
|
||||||
|
|
||||||
from openflexure_microscope_server.things.autofocus import AutofocusThing
|
from openflexure_microscope_server.things.autofocus import AutofocusThing
|
||||||
|
|
@ -45,41 +47,56 @@ def make_captures(sharpness_list):
|
||||||
return [MockCapture(s, i) for i, s in enumerate(sharpness_list)]
|
return [MockCapture(s, i) for i, s in enumerate(sharpness_list)]
|
||||||
|
|
||||||
|
|
||||||
def test_stack_labelling():
|
def load_cases():
|
||||||
"""Test stack classification accuracy against labelled sharpness cases.
|
"""Load the sharpness data from the json.
|
||||||
|
|
||||||
This test loads predefined sharpness profiles and their expected labels
|
Includes the sharpnesses to test, manually written labels
|
||||||
from a JSON file, converts them into mock capture objects, and evaluates
|
on the required result, and whether the test is allowed to
|
||||||
the classification returned by `check_stack_result`.
|
fail. This ensures future tests will flag any regression on
|
||||||
|
tests while allowing improvements.
|
||||||
The test asserts that at least 90% of cases are correctly classified.
|
|
||||||
|
|
||||||
Expected labels may be a single value or a list of acceptable values.
|
|
||||||
"""
|
"""
|
||||||
autofocus_thing = create_thing_without_server(AutofocusThing, mock_all_slots=True)
|
|
||||||
|
|
||||||
with open(DATA_PATH) as f:
|
with open(DATA_PATH) as f:
|
||||||
data = json.load(f)
|
data = json.load(f)
|
||||||
|
|
||||||
success = 0
|
params = []
|
||||||
total = len(data)
|
for i, case in enumerate(data):
|
||||||
|
|
||||||
for _i, case in enumerate(data):
|
|
||||||
sharpnesses = case["sharpnesses"]
|
|
||||||
expected = case["label"]
|
expected = case["label"]
|
||||||
|
|
||||||
# Allow multiple acceptable labels
|
# Allow multiple acceptable labels
|
||||||
if not isinstance(expected, list):
|
if not isinstance(expected, list):
|
||||||
expected = [expected]
|
expected = [expected]
|
||||||
|
|
||||||
# Convert to capture objects
|
marks = []
|
||||||
captures = make_captures(sharpnesses)
|
|
||||||
|
|
||||||
# Call the method under test
|
if case.get("allow_failure", False):
|
||||||
result, _ = autofocus_thing.check_stack_result(
|
marks.append(pytest.mark.xfail(reason="Known failing case"))
|
||||||
captures, check_turning_points=False
|
|
||||||
|
params.append(
|
||||||
|
pytest.param(
|
||||||
|
case["sharpnesses"],
|
||||||
|
expected,
|
||||||
|
marks=marks,
|
||||||
|
id=f"case_{i}",
|
||||||
|
)
|
||||||
)
|
)
|
||||||
if result in expected:
|
return params
|
||||||
success += 1
|
|
||||||
|
|
||||||
assert success > 0.8 * total
|
|
||||||
|
@pytest.mark.parametrize(("sharpnesses", "expected"), load_cases())
|
||||||
|
def test_stack_labelling(sharpnesses, expected):
|
||||||
|
"""Test stack classification accuracy against labelled sharpness cases.
|
||||||
|
|
||||||
|
This test loads the test from load_cases and evaluates
|
||||||
|
the classification returned by `check_stack_result`.
|
||||||
|
|
||||||
|
The test ensures that only cases marked with "allow_failure: true" can fail.
|
||||||
|
|
||||||
|
Expected labels may be a single value or a list of acceptable values.
|
||||||
|
"""
|
||||||
|
autofocus_thing = create_thing_without_server(AutofocusThing, mock_all_slots=True)
|
||||||
|
|
||||||
|
captures = make_captures(sharpnesses)
|
||||||
|
|
||||||
|
result, _ = autofocus_thing.check_stack_result(captures, check_turning_points=False)
|
||||||
|
|
||||||
|
assert result in expected
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue