Turn scan_data into a BaseModel with validation for custom formatting

src/openflexure_microscope_server/scan_directories.py

@@ -6,8 +6,9 @@ import re
 import shutil
 import zipfile
 import threading
+from datetime import datetime, timedelta
 
-from pydantic import BaseModel
+from pydantic import BaseModel, field_validator, field_serializer
 
 from openflexure_microscope_server.utilities import requires_lock
 
@@ -17,6 +18,8 @@ SCAN_ZERO_PAD_DIGITS = 4
 STITCH_REGEX = re.compile(r"stitched\.jpe?g$")
 IMAGE_REGEX = re.compile(r"-?[0-9]+_-?[0-9]+\.jpe?g$")
 
+SCAN_DATA_FILENAME = "scan_data.json"
+
 
 class NotEnoughFreeSpaceError(IOError):
     """An exception raised if there is not enough free space on disk to scan."""
@@ -33,6 +36,96 @@ class ScanInfo(BaseModel):
     dzi: Optional[str]
 
 
+class ScanData(BaseModel):
+    """Data about a scan to be saved to a JSON file in the directory.
+
+    This serialises into a human readable format where possible with
+
+    timestamps in %H_%M_%S-%d_%m_%Y format
+    timedeltas in %H:%M:%S format
+
+    Properties that are not known until the end have ``None`` serialised as "Unknown"
+    """
+
+    # TODO: Docs for each variable
+    # TODO: Make note about timestamp format. Is it too ambiguous?
+    scan_name: str
+    overlap: float
+    max_dist: int
+    dx: int
+    dy: int
+    autofocus_range: int
+    autofocus_on: bool
+    start_time: datetime
+    skip_background: str
+    stitch_automatically: str
+    stitch_resize: int
+    save_resolution: tuple[int, int]
+    final_image_count: Optional[int] = None
+    duration: Optional[timedelta] = None
+    scan_result: Optional[str] = None
+
+    def set_final_data(self, result: str, final_image_count: int):
+        """Set the final data for the scan, scan duration is automatically calculated.
+
+        :param result: A string describing the result.
+        :param final_image_count: The total number of images captures.
+        """
+        self.duration = datetime.now() - self.start_time
+        self.final_image_count = final_image_count
+        self.scan_result = result
+
+    @field_validator("start_time", mode="before")
+    @classmethod
+    def parse_timestamp(cls, value: str | datetime) -> datetime:
+        """Validate a timestamp that may be a string in Hrs_Min_Sec-Day_Month_Year format."""
+        if isinstance(value, str):
+            return datetime.strptime(value, "%H_%M_%S-%d_%m_%Y")
+        return value
+
+    @field_serializer("start_time")
+    def serialize_timestamp(self, value: datetime) -> str:
+        """Serialise timestamp to Hrs_Min_Sec-Day_Month_Year format."""
+        return value.strftime("%H_%M_%S-%d_%m_%Y")
+
+    @field_validator("duration", mode="before")
+    @classmethod
+    def parse_timedelta(cls, value: Optional[str | timedelta]) -> Optional[timedelta]:
+        """Validate a timedelta that may be a string in Hrs:Min:Sec format or "Unknown"."""
+        if isinstance(value, str):
+            if value == "Unknown":
+                return None
+            hrs, mins, secs = map(int, value.split(":"))
+            return timedelta(hours=hrs, minutes=mins, seconds=secs)
+        return value
+
+    @field_serializer("duration")
+    def serialize_timedelta(self, value: Optional[timedelta]) -> str:
+        """Serialise timedelta to Hrs:Min:Sec (or "Unknown" if None)."""
+        if value is None:
+            return "Unknown"
+        # Calculate the string manually rather than use str(), as this may convert to
+        # days for a very very long scan, and then it is much harder to parse.
+        total_secs = value.total_seconds()
+        hrs = int(total_secs // 3600)
+        mins = int((total_secs % 3600) // 60)
+        secs = int((total_secs % 60))
+        return f"{hrs}:{mins}:{secs}"
+
+    @field_validator("final_image_count", "scan_result", mode="before")
+    @classmethod
+    def parse_unknown_as_none(cls, value: Optional[str | int]) -> Optional[str | int]:
+        """Validate the string "Unknown" as None."""
+        if value == "Unknown":
+            return None
+        return value
+
+    @field_serializer("final_image_count", "scan_result")
+    def serialize_none_as_unknown(self, value: Optional[str | int]) -> str:
+        """Serialise None as "Unknown" for a more human readable result."""
+        return "Unknown" if value is None else str(value)
+
+
 class ScanDirectoryManager:
     """A class for managing interactions with scan directories."""
 
@@ -111,6 +204,17 @@ class ScanDirectoryManager:
             return None
         return self.get_file_path_from_img_dir(scan_name, stitch_fname)
 
+    @requires_lock
+    def get_scan_data_path(self, scan_name: str) -> Optional[str]:
+        """Return the file full path scan data JSON file.
+
+        If no scan data JSON file is found, return None
+        """
+        scan_data_path = ScanDirectory(scan_name, self.base_dir).scan_data_path
+        if not os.path.isfile(scan_data_path):
+            return None
+        return scan_data_path
+
     @property
     @requires_lock
     def all_scans(self) -> list[str]:
@@ -250,6 +354,16 @@ class ScanDirectory:
             return im_path
         return None
 
+    @property
+    def scan_data_path(self) -> Optional[str]:
+        """The path to the scan data json file for this directory.
+
+        Returns None if there is no images dir to write to.
+        """
+        if self.images_dir is not None:
+            return os.path.join(self.images_dir, SCAN_DATA_FILENAME)
+        return None
+
     @property
     def created_time(self) -> float:
         """The time the directory was created on disk."""
@@ -341,6 +455,16 @@ class ScanDirectory:
                 files.append(os.path.relpath(full_path, self.dir_path))
         return files
 
+    def save_scan_data(self, scan_data: ScanData):
+        """Save the scan data for this scan to disk."""
+        if self.scan_data_path is None:
+            raise FileNotFoundError(
+                "There is no images directory to save scan data into."
+            )
+
+        with open(self.scan_data_path, "w", encoding="utf-8") as f:
+            f.write(scan_data.model_dump_json(indent=4))
+
     def zip_files(self, final_version: bool = False) -> str:
         """Zips any images from the scan not yet zipped, return full path to zip.
 

src/openflexure_microscope_server/things/smart_scan.py

@@ -12,7 +12,6 @@ import threading
 import os
 import time
 import json
-from datetime import datetime
 from subprocess import CompletedProcess, Popen, PIPE, SubprocessError, STDOUT
 
 from fastapi import HTTPException
@@ -40,7 +39,7 @@ AutofocusDep = lt.deps.direct_thing_client_dependency(AutofocusThing, "/autofocu
 JPEGBlob = lt.blob.blob_type("image/jpeg")
 ZipBlob = lt.blob.blob_type("application/zip")
 
-SCAN_DATA_FILENAME = "scan_data.json"
+
 STITCHING_CMD = "openflexure-stitch"
 
 STITCHING_RESOLUTION = (820, 616)
@@ -111,7 +110,7 @@ class SmartScanThing(lt.Thing):
         self._capture_thread: Optional[ErrorCapturingThread] = None
         self._scan_images_taken: Optional[int] = None
         # TODO Scan data is a dict during refactoring, should become a dataclass
-        self._scan_data: Optional[dict] = None
+        self._scan_data: Optional[scan_directories.ScanData] = None
 
     @lt.thing_action
     def sample_scan(
@@ -141,6 +140,7 @@ class SmartScanThing(lt.Thing):
         self._autofocus = autofocus
         self._stage = stage
         self._cam = cam
+        # TODO check if metadata_getter this can be removed without error?
         self._metadata_getter = metadata_getter
         self._csm = csm
         self._capture_thread = None
@@ -280,14 +280,11 @@ class SmartScanThing(lt.Thing):
         return dx, dy
 
     @_scan_running
-    def _set_scan_data(self):
+    def _collect_scan_data(self) -> scan_directories.ScanData:
-        """Set date for this scan to the ``self._scan_data`` variable.
+        """Collect and return the data for this scan so it cannot be changed mid-scan."""
-
-        This needs to become a dataclass at some point.
-        """
         overlap = self.overlap
         dx, dy = self._calc_displacement_from_test_image(overlap)
-        stitch_resize = STITCHING_RESOLUTION[0] / self.save_resolution[0]
+        stitch_resize = STITCHING_RESOLUTION[0] // self.save_resolution[0]
 
         self._scan_logger.debug(
             f"Resizing images when stitching by a factor of {stitch_resize}"
@@ -308,78 +305,33 @@ class SmartScanThing(lt.Thing):
             autofocus_dz = 0
 
         # Fix scan parameters in case UI is updated during scan.
-        self._scan_data = {
+        return scan_directories.ScanData(
-            "scan_name": self._ongoing_scan.name,
+            scan_name=self._ongoing_scan.name,
-            "overlap": overlap,
+            overlap=overlap,
-            "max_dist": self.max_range,
+            max_dist=self.max_range,
-            "dx": dx,
+            dx=dx,
-            "dy": dy,
+            dy=dy,
-            "autofocus_dz": autofocus_dz,
+            autofocus_dz=autofocus_dz,
-            "autofocus_on": bool(autofocus_dz),
+            autofocus_on=bool(autofocus_dz),
-            "start_time": time.strftime("%H_%M_%S-%d_%m_%Y"),
+            start_time=time.strftime("%H_%M_%S-%d_%m_%Y"),
-            "skip_background": self.skip_background,
+            skip_background=self.skip_background,
-            "stitch_automatically": self.stitch_automatically,
+            stitch_automatically=self.stitch_automatically,
-            "stitch_resize": stitch_resize,
+            stitch_resize=stitch_resize,
-            "save_resolution": self.save_resolution,
+            save_resolution=self.save_resolution,
-        }
-
-    @_scan_running
-    def _save_scan_inputs_json(self):
-        """Save scan inputs as a JSON file in the scan folder.
-
-        This file allows the user to review the settings used in the scan.
-        """
-        # Should this be a method of the scan_data dataclass?
-
-        data = {
-            "scan_name": self._ongoing_scan.name,
-            "overlap": self._scan_data["overlap"],
-            "autofocus range": self._scan_data["autofocus_dz"],
-            "dx": self._scan_data["dx"],
-            "dy": self._scan_data["dy"],
-            "start time": self._scan_data["start_time"],
-            "skipping background": self._scan_data["skip_background"],
-            "capture resolution": self._scan_data["save_resolution"],
-        }
-
-        scan_inputs_fname = os.path.join(
-            self._ongoing_scan.images_dir, SCAN_DATA_FILENAME
         )
-        with open(scan_inputs_fname, "w", encoding="utf-8") as f:
-            json.dump(data, f, ensure_ascii=False, indent=4)
 
     @_scan_running
-    def _update_scan_data_json(self, scan_result: str):
+    def _save_final_scan_data(self, scan_result: str):
         """Update scan data JSON file with data only known at the end of the scan.
 
         Takes scan_result, a string that is either "success", "cancelled by user",
         or the error that ended the scan.
         """
-        # Should this be a method of the scan_data dataclass?
+        self.scan_data.set_final_data(
-        current_time = datetime.now().replace(microsecond=0)
+            result=scan_result,
-        start_time = datetime.strptime(
+            final_image_count=self._scan_images_taken,
-            self._scan_data["start_time"], "%H_%M_%S-%d_%m_%Y"
-        ).replace(microsecond=0)
-
-        duration = current_time - start_time
-
-        outputs = {
-            "image_count": self._scan_images_taken,
-            "duration": str(duration),
-            "scan_result": scan_result,
-        }
-
-        scan_data_fname = os.path.join(
-            self._ongoing_scan.images_dir, SCAN_DATA_FILENAME
         )
-
+        self._ongoing_scan.save_scan_data(self._scan_data)
-        with open(scan_data_fname, encoding="utf-8") as f:
-            data = json.load(f)
-
-        data.update(outputs)
-
-        with open(scan_data_fname, "w", encoding="utf-8") as f:
-            json.dump(data, f, ensure_ascii=False, indent=4)
 
     @_scan_running
     def _manage_stitching_threads(self):
@@ -388,7 +340,7 @@ class SmartScanThing(lt.Thing):
         # well constrained.
         if self._scan_images_taken > 3:
             if not self._preview_stitch_running():
-                self._preview_stitch_start(overlap=self._scan_data["overlap"])
+                self._preview_stitch_start(overlap=self._scan_data.overlap)
 
     @_scan_running
     def _run_scan(self):
@@ -404,25 +356,25 @@ class SmartScanThing(lt.Thing):
 
         try:
             self._cam.start_streaming(main_resolution=(3280, 2464))
-            self._set_scan_data()
+            self._scan_data = self._collect_scan_data()
-            self._save_scan_inputs_json()
+            self._ongoing_scan.save_scan_data(self._scan_data)
             if self._scan_images_taken != 0:
                 msg = "_scan_images_taken should be zero before starting scanning"
                 raise RuntimeError(msg)
 
             # This is the main loop of the scan!
             self._main_scan_loop()
-            self._update_scan_data_json(scan_result="success")
+            self._save_final_scan_data(scan_result="success")
 
         except lt.exceptions.InvocationCancelledError:
             scan_successful = False
             # Reset the cancel event so it can be thrown again
             self._cancel.clear()
             self._scan_logger.info("Stopping scan because it was cancelled.")
-            self._update_scan_data_json(scan_result="cancelled by user")
+            self._save_final_scan_data(scan_result="cancelled by user")
         except scan_directories.NotEnoughFreeSpaceError as e:
             scan_successful = False
-            self._update_scan_data_json(scan_result=str(e))
+            self._save_final_scan_data(scan_result=str(e))
             self._scan_logger.error(
                 f"Stopping scan to avoid filling up the disk: {e}",
                 exc_info=e,
@@ -430,6 +382,7 @@ class SmartScanThing(lt.Thing):
             raise e
         except Exception as e:
             scan_successful = False
+            self._save_final_scan_data(scan_result=str(e))
             self._scan_logger.error(
                 f"The scan stopped because of an error: {e} "
                 "Attempting to stitch and archive the images acquired so far.",
@@ -475,9 +428,9 @@ class SmartScanThing(lt.Thing):
         # have multiple starting positions, each of which will be visited before the
         # scan can end.
         planner_settings = {
-            "dx": self._scan_data["dx"],
+            "dx": self._scan_data.dx,
-            "dy": self._scan_data["dy"],
+            "dy": self._scan_data.dy,
-            "max_dist": self._scan_data["max_dist"],
+            "max_dist": self._scan_data.max_dist,
         }
         route_planner = scan_planners.SmartSpiral(
             initial_position=(self._stage.position["x"], self._stage.position["y"]),
@@ -502,7 +455,7 @@ class SmartScanThing(lt.Thing):
 
             capture_image = True
             # If skipping background, take an image to check if current field of view is background
-            if self._scan_data["skip_background"]:
+            if self._scan_data.skip_background:
                 capture_image, bg_message = self._cam.image_is_sample()
 
             if not capture_image:
@@ -515,8 +468,8 @@ class SmartScanThing(lt.Thing):
 
             focused, focused_height = self._autofocus.run_smart_stack(
                 images_dir=self._ongoing_scan.images_dir,
-                autofocus_dz=self._scan_data["autofocus_dz"],
+                autofocus_dz=self._scan_data.autofocus_dz,
-                save_resolution=self._scan_data["save_resolution"],
+                save_resolution=self._scan_data.save_resolution,
             )
 
             current_pos_xyz = (new_pos_xyz[0], new_pos_xyz[1], focused_height)
@@ -552,14 +505,14 @@ class SmartScanThing(lt.Thing):
 
         self._preview_stitch_wait()
         try:
-            if self._scan_data["stitch_automatically"]:
+            if self._scan_data.stitch_automatically:
                 self._scan_logger.info("Stitching final image (may take some time)...")
                 self.stitch_scan(
                     logger=self._scan_logger,
                     cancel=self._cancel,
                     scan_name=self._ongoing_scan.name,
-                    stitch_resize=self._scan_data["stitch_resize"],
+                    stitch_resize=self._scan_data.stitch_resize,
-                    overlap=self._scan_data["overlap"],
+                    overlap=self._scan_data.overlap,
                 )
         except SubprocessError as e:
             self._scan_logger.error(f"Stitching failed: {e}", exc_info=e)
@@ -798,7 +751,7 @@ class SmartScanThing(lt.Thing):
                     "--minimum_overlap",
                     f"{min_overlap}",
                     "--resize",
-                    f"{self._scan_data['stitch_resize']}",
+                    f"{self._scan_data.stitch_resize}",
                     self._ongoing_scan.images_dir,
                 ]
             )
@@ -885,9 +838,9 @@ class SmartScanThing(lt.Thing):
         Note that as this is a lt.thing_action it needs the logger passed as
         a variable if called from another thing action
         """
-        json_fpath = self._scan_dir_manager.get_file_path_from_img_dir(
+        # TODO tidy this function
-            scan_name=scan_name, filename=SCAN_DATA_FILENAME
+        # TODO handle if json_path is None (easier once tidied)
-        )
+        json_fpath = self._scan_dir_manager.get_scan_data_path()
 
         if self.stitch_tiff:
             tiff_arg = "--stitch_tiff"
@@ -904,7 +857,7 @@ class SmartScanThing(lt.Thing):
                 # the file not being there, it not being json in the file,
                 # or the imported data not being indexable
                 logger.warning(
-                    f"Couldn't read scan data, is {SCAN_DATA_FILENAME} missing or corrupt? "
+                    f"Couldn't read scan data, is {json_fpath} missing or corrupt? "
                     "Attempting stitch with overlap value of 0.1"
                 )
                 overlap = 0.1
@@ -919,20 +872,28 @@ class SmartScanThing(lt.Thing):
             try:
                 with open(json_fpath, "r", encoding="utf-8") as data_file:
                     data_loaded = json.load(data_file)
-                save_resolution = data_loaded["capture resolution"]
+
+                # Handle "capture resolution" being used to store the save resolution
+                # in old scans.
+                key = (
+                    "capture resolution"
+                    if "capture resolution" in data_loaded
+                    else "save_resolution"
+                )
+                save_resolution = data_loaded[key]
                 stitch_resize = STITCHING_RESOLUTION[0] / save_resolution[0]
             except (json.decoder.JSONDecodeError, FileNotFoundError, TypeError):
                 # As there is no schema or pydantic model this should handle
                 # the file not being there, it not being json in the file,
                 # or the imported data not being indexable
                 logger.warning(
-                    f"Couldn't read scan data, is {SCAN_DATA_FILENAME} missing or corrupt? "
+                    f"Couldn't read scan data, is {json_fpath} missing or corrupt? "
                     "Attempting stitch with resize value of 0.5"
                 )
                 stitch_resize = 0.5
             except KeyError:
                 logger.warning(
-                    "Value for capture resolution not found in scan data. "
+                    "Value for save resolution not found in scan data. "
                     "Attempting stitch with resize value of 0.5"
                 )
                 stitch_resize = 0.5