Record timestamp when adding frames to stream

src/openflexure_microscope_server/things/autofocus.py

@@ -345,12 +345,14 @@ class JPEGSharpnessMonitor:
     async def monitor_sharpness(self) -> None:
         """Start monitoring sharpness metrics."""
         self.running = True
-        async for frame in self.camera.lores_mjpeg_stream.frame_async_generator():
-            self._jpeg_times.append(time.time())
+        while self.running:
+            i = await self.camera.lores_mjpeg_stream.next_frame()
+            entry = await self.camera.lores_mjpeg_stream.ringbuffer_entry(i)
+            self._jpeg_times.append(entry.timestamp.timestamp())
 
             # JPEG sharpness metric
             if self.record & SharpnessMethod.JPEG:
-                self._jpeg_sizes.append(len(frame))
+                self._jpeg_sizes.append(len(entry.frame))
 
             # FocusFoM metric
             if self.record & SharpnessMethod.FOCUS_FOM:

src/openflexure_microscope_server/things/camera/__init__.py

@@ -216,6 +216,41 @@ class CaptureGalleryInfo(BaseModel):
     card_type: Literal["Capture"] = "Capture"
 
 
+class MJPEGStreamWithTimestamp(lt.outputs.MJPEGStream):
+    """An MJPEG stream where the frame time can be recorded with the frame.
+
+    This can be used to capture when the image was taken by the sensor if known
+    rather than take the time it is added into the ring buffer.
+    """
+
+    def add_frame(self, frame: bytes, timestamp: Optional[datetime]) -> None:
+        """Add a JPEG to the MJPEG stream.
+
+        Modify the standard function to have an option to send in the capture time.
+
+        :param frame: The frame to add
+        :param timestamp: The time the frame was captured.
+
+        :raise ValueError: if the supplied frame does not start with the JPEG
+            start bytes and end with the end bytes.
+        """
+        if not (
+            frame[0] == 0xFF
+            and frame[1] == 0xD8
+            and frame[-2] == 0xFF
+            and frame[-1] == 0xD9
+        ):
+            raise ValueError("Invalid JPEG")
+        with self._lock:
+            entry = self._ringbuffer[(self.last_frame_i + 1) % len(self._ringbuffer)]
+            entry.timestamp = timestamp if timestamp is not None else datetime.now()
+            entry.frame = frame
+            entry.index = self.last_frame_i + 1
+            self._thing_server_interface.start_async_task_soon(
+                self.notify_new_frame, entry.index
+            )
+
+
 class BaseCamera(OFMThing, ABC):
     """The base class for all cameras. All cameras must directly inherit from this class.
 

src/openflexure_microscope_server/things/camera/picamera.py

@@ -24,6 +24,7 @@ import tempfile
 import time
 from abc import ABC, abstractmethod
 from contextlib import contextmanager
+from datetime import datetime
 from threading import RLock
 from types import TracebackType
 from typing import (
@@ -54,7 +55,7 @@ from openflexure_microscope_server.ui import (
     property_control_for,
 )
 
-from . import BaseCamera, CaptureMode, StreamingMode
+from . import BaseCamera, CaptureMode, MJPEGStreamWithTimestamp, StreamingMode
 from . import picamera_recalibrate_utils as recalibrate_utils
 from . import picamera_tuning_file_utils as tf_utils
 
@@ -75,24 +76,36 @@ class MissingCalibrationError(RuntimeError):
 class PicameraStreamOutput(Output):
     """An Output class that sends frames to a stream."""
 
-    def __init__(self, stream: lt.outputs.MJPEGStream) -> None:
+    def __init__(self, stream: MJPEGStreamWithTimestamp, encoder: MJPEGEncoder) -> None:
         """Create an output that puts frames in an MJPEGStream.
 
         :param stream: The labthings MJPEGStream to send frames to.
+        :param encoder: The encoder that is encoding the frames. This is needed to get
+            the full timestamp.
         """
         Output.__init__(self)
         self.stream = stream
+        self.encoder = encoder
+        # The difference between unix time and the CPU time
+        self.cpu_dt = time.time() - time.monotonic()
 
     def outputframe(
         self,
         frame: bytes,
         _keyframe: Optional[bool] = True,
-        _timestamp: Optional[int] = None,
+        timestamp: Optional[int] = None,
         _packet: Any = None,
         _audio: bool = False,
     ) -> None:
         """Add a frame to the stream's ringbuffer."""
-        self.stream.add_frame(frame)
+        if timestamp is None or self.encoder.firsttimestamp is None:
+            frame_timestamp = None
+        else:
+            # Reconstruct full CPU time of the frame in ns
+            timestamp += self.encoder.firsttimestamp
+            # Convert to datetime
+            frame_timestamp = datetime.fromtimestamp(self.cpu_dt + timestamp / 1e9)
+        self.stream.add_frame(frame, frame_timestamp)
 
 
 class PiCamera2StreamingMode(StreamingMode):
@@ -487,14 +500,16 @@ class StreamingPiCamera2(BaseCamera, ABC):
                 picam.configure(stream_config)
                 LOGGER.info("Starting picamera MJPEG stream...")
                 stream_name = "lores" if mode_info.use_lores_as_preview else "main"
+                encoder = MJPEGEncoder(self.mjpeg_bitrate)
                 picam.start_recording(
-                    MJPEGEncoder(self.mjpeg_bitrate),
-                    PicameraStreamOutput(self.mjpeg_stream),
+                    encoder,
+                    PicameraStreamOutput(self.mjpeg_stream, encoder),
                     name=stream_name,
                 )
+                encoder = MJPEGEncoder(100000000)
                 picam.start_encoder(
-                    MJPEGEncoder(100000000),
-                    PicameraStreamOutput(self.lores_mjpeg_stream),
+                    encoder,
+                    PicameraStreamOutput(self.lores_mjpeg_stream, encoder),
                     name="lores",
                 )
             except Exception as e: