From ad7b0d8da8615e2842ebead1e99e3250d24cb96a Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Fri, 10 Jul 2026 09:39:09 +0100 Subject: [PATCH 1/5] Record timestamp when adding frames to stream --- .../things/autofocus.py | 8 +++-- .../things/camera/__init__.py | 35 +++++++++++++++++++ .../things/camera/picamera.py | 31 +++++++++++----- 3 files changed, 63 insertions(+), 11 deletions(-) diff --git a/src/openflexure_microscope_server/things/autofocus.py b/src/openflexure_microscope_server/things/autofocus.py index bff91e81..3eedcb2b 100644 --- a/src/openflexure_microscope_server/things/autofocus.py +++ b/src/openflexure_microscope_server/things/autofocus.py @@ -345,12 +345,14 @@ class JPEGSharpnessMonitor: async def monitor_sharpness(self) -> None: """Start monitoring sharpness metrics.""" self.running = True - async for frame in self.camera.lores_mjpeg_stream.frame_async_generator(): - self._jpeg_times.append(time.time()) + while self.running: + i = await self.camera.lores_mjpeg_stream.next_frame() + entry = await self.camera.lores_mjpeg_stream.ringbuffer_entry(i) + self._jpeg_times.append(entry.timestamp.timestamp()) # JPEG sharpness metric if self.record & SharpnessMethod.JPEG: - self._jpeg_sizes.append(len(frame)) + self._jpeg_sizes.append(len(entry.frame)) # FocusFoM metric if self.record & SharpnessMethod.FOCUS_FOM: diff --git a/src/openflexure_microscope_server/things/camera/__init__.py b/src/openflexure_microscope_server/things/camera/__init__.py index a4fe8fce..8891b2bf 100644 --- a/src/openflexure_microscope_server/things/camera/__init__.py +++ b/src/openflexure_microscope_server/things/camera/__init__.py @@ -216,6 +216,41 @@ class CaptureGalleryInfo(BaseModel): card_type: Literal["Capture"] = "Capture" +class MJPEGStreamWithTimestamp(lt.outputs.MJPEGStream): + """An MJPEG stream where the frame time can be recorded with the frame. + + This can be used to capture when the image was taken by the sensor if known + rather than take the time it is added into the ring buffer. + """ + + def add_frame(self, frame: bytes, timestamp: Optional[datetime]) -> None: + """Add a JPEG to the MJPEG stream. + + Modify the standard function to have an option to send in the capture time. + + :param frame: The frame to add + :param timestamp: The time the frame was captured. + + :raise ValueError: if the supplied frame does not start with the JPEG + start bytes and end with the end bytes. + """ + if not ( + frame[0] == 0xFF + and frame[1] == 0xD8 + and frame[-2] == 0xFF + and frame[-1] == 0xD9 + ): + raise ValueError("Invalid JPEG") + with self._lock: + entry = self._ringbuffer[(self.last_frame_i + 1) % len(self._ringbuffer)] + entry.timestamp = timestamp if timestamp is not None else datetime.now() + entry.frame = frame + entry.index = self.last_frame_i + 1 + self._thing_server_interface.start_async_task_soon( + self.notify_new_frame, entry.index + ) + + class BaseCamera(OFMThing, ABC): """The base class for all cameras. All cameras must directly inherit from this class. diff --git a/src/openflexure_microscope_server/things/camera/picamera.py b/src/openflexure_microscope_server/things/camera/picamera.py index c769947d..fd00fc88 100644 --- a/src/openflexure_microscope_server/things/camera/picamera.py +++ b/src/openflexure_microscope_server/things/camera/picamera.py @@ -24,6 +24,7 @@ import tempfile import time from abc import ABC, abstractmethod from contextlib import contextmanager +from datetime import datetime from threading import RLock from types import TracebackType from typing import ( @@ -54,7 +55,7 @@ from openflexure_microscope_server.ui import ( property_control_for, ) -from . import BaseCamera, CaptureMode, StreamingMode +from . import BaseCamera, CaptureMode, MJPEGStreamWithTimestamp, StreamingMode from . import picamera_recalibrate_utils as recalibrate_utils from . import picamera_tuning_file_utils as tf_utils @@ -75,24 +76,36 @@ class MissingCalibrationError(RuntimeError): class PicameraStreamOutput(Output): """An Output class that sends frames to a stream.""" - def __init__(self, stream: lt.outputs.MJPEGStream) -> None: + def __init__(self, stream: MJPEGStreamWithTimestamp, encoder: MJPEGEncoder) -> None: """Create an output that puts frames in an MJPEGStream. :param stream: The labthings MJPEGStream to send frames to. + :param encoder: The encoder that is encoding the frames. This is needed to get + the full timestamp. """ Output.__init__(self) self.stream = stream + self.encoder = encoder + # The difference between unix time and the CPU time + self.cpu_dt = time.time() - time.monotonic() def outputframe( self, frame: bytes, _keyframe: Optional[bool] = True, - _timestamp: Optional[int] = None, + timestamp: Optional[int] = None, _packet: Any = None, _audio: bool = False, ) -> None: """Add a frame to the stream's ringbuffer.""" - self.stream.add_frame(frame) + if timestamp is None or self.encoder.firsttimestamp is None: + frame_timestamp = None + else: + # Reconstruct full CPU time of the frame in ns + timestamp += self.encoder.firsttimestamp + # Convert to datetime + frame_timestamp = datetime.fromtimestamp(self.cpu_dt + timestamp / 1e9) + self.stream.add_frame(frame, frame_timestamp) class PiCamera2StreamingMode(StreamingMode): @@ -487,14 +500,16 @@ class StreamingPiCamera2(BaseCamera, ABC): picam.configure(stream_config) LOGGER.info("Starting picamera MJPEG stream...") stream_name = "lores" if mode_info.use_lores_as_preview else "main" + encoder = MJPEGEncoder(self.mjpeg_bitrate) picam.start_recording( - MJPEGEncoder(self.mjpeg_bitrate), - PicameraStreamOutput(self.mjpeg_stream), + encoder, + PicameraStreamOutput(self.mjpeg_stream, encoder), name=stream_name, ) + encoder = MJPEGEncoder(100000000) picam.start_encoder( - MJPEGEncoder(100000000), - PicameraStreamOutput(self.lores_mjpeg_stream), + encoder, + PicameraStreamOutput(self.lores_mjpeg_stream, encoder), name="lores", ) except Exception as e: From f85361ae67e7b77314b3d2f29701e63fad9dfe7e Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Fri, 10 Jul 2026 09:48:13 +0100 Subject: [PATCH 2/5] Subclass the mjpegstream descriptor --- .../things/camera/__init__.py | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/openflexure_microscope_server/things/camera/__init__.py b/src/openflexure_microscope_server/things/camera/__init__.py index 8891b2bf..648264fd 100644 --- a/src/openflexure_microscope_server/things/camera/__init__.py +++ b/src/openflexure_microscope_server/things/camera/__init__.py @@ -250,6 +250,24 @@ class MJPEGStreamWithTimestamp(lt.outputs.MJPEGStream): self.notify_new_frame, entry.index ) +class MJPEGStreamDescriptorWithTimestamp(lt.outputs.MJPEGStreamDescriptor): + """Subclass the Labthings descriptor to return our own class.""" + + def __get__( + self, obj: Optional[lt.Thing], type: type[lt.Thing] | None = None # noqa: A002 + ) -> MJPEGStreamWithTimestamp | Self: + """Return our own MJPEG Stream with timestamp.""" + if obj is None: + return self + try: + return obj.__dict__[self.name] + except KeyError: + obj.__dict__[self.name] = MJPEGStreamWithTimestamp( + **self._kwargs, + thing_server_interface=obj._thing_server_interface, + ) + return obj.__dict__[self.name] + class BaseCamera(OFMThing, ABC): """The base class for all cameras. All cameras must directly inherit from this class. @@ -262,8 +280,8 @@ class BaseCamera(OFMThing, ABC): _all_background_detectors: Mapping[str, BackgroundDetectAlgorithm] = lt.thing_slot() - mjpeg_stream = lt.outputs.MJPEGStreamDescriptor() - lores_mjpeg_stream = lt.outputs.MJPEGStreamDescriptor() + mjpeg_stream = MJPEGStreamDescriptorWithTimestamp() + lores_mjpeg_stream = MJPEGStreamDescriptorWithTimestamp() _memory_buffer = CameraMemoryBuffer() supports_focus_fom: bool = False From 7ba2c03f61495710a26689e2fe1ba256e9c0961e Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Fri, 10 Jul 2026 10:09:17 +0100 Subject: [PATCH 3/5] Use us not ns for picamera timestamp and fix descriptor typing --- .../things/camera/__init__.py | 19 +++++++++++++++---- .../things/camera/picamera.py | 4 ++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/openflexure_microscope_server/things/camera/__init__.py b/src/openflexure_microscope_server/things/camera/__init__.py index 648264fd..e93fb938 100644 --- a/src/openflexure_microscope_server/things/camera/__init__.py +++ b/src/openflexure_microscope_server/things/camera/__init__.py @@ -16,7 +16,7 @@ from abc import ABC, abstractmethod from copy import deepcopy from datetime import datetime from types import TracebackType -from typing import Any, Literal, Mapping, Optional, Self +from typing import Any, Literal, Mapping, Optional, Self, Union, overload import numpy as np import piexif @@ -223,7 +223,7 @@ class MJPEGStreamWithTimestamp(lt.outputs.MJPEGStream): rather than take the time it is added into the ring buffer. """ - def add_frame(self, frame: bytes, timestamp: Optional[datetime]) -> None: + def add_frame(self, frame: bytes, timestamp: Optional[datetime] = None) -> None: """Add a JPEG to the MJPEG stream. Modify the standard function to have an option to send in the capture time. @@ -250,12 +250,23 @@ class MJPEGStreamWithTimestamp(lt.outputs.MJPEGStream): self.notify_new_frame, entry.index ) + class MJPEGStreamDescriptorWithTimestamp(lt.outputs.MJPEGStreamDescriptor): """Subclass the Labthings descriptor to return our own class.""" + @overload + def __get__(self, obj: Literal[None], type: type | None = None) -> Self: ... + + @overload def __get__( - self, obj: Optional[lt.Thing], type: type[lt.Thing] | None = None # noqa: A002 - ) -> MJPEGStreamWithTimestamp | Self: + self, obj: lt.Thing, type: type | None = None + ) -> MJPEGStreamWithTimestamp: ... + + def __get__( + self, + obj: Optional[lt.Thing], + type: type[lt.Thing] | None = None, # noqa: A002 + ) -> Union[MJPEGStreamWithTimestamp | Self]: """Return our own MJPEG Stream with timestamp.""" if obj is None: return self diff --git a/src/openflexure_microscope_server/things/camera/picamera.py b/src/openflexure_microscope_server/things/camera/picamera.py index fd00fc88..f3f0ff1d 100644 --- a/src/openflexure_microscope_server/things/camera/picamera.py +++ b/src/openflexure_microscope_server/things/camera/picamera.py @@ -101,10 +101,10 @@ class PicameraStreamOutput(Output): if timestamp is None or self.encoder.firsttimestamp is None: frame_timestamp = None else: - # Reconstruct full CPU time of the frame in ns + # Reconstruct full CPU time of the frame in us timestamp += self.encoder.firsttimestamp # Convert to datetime - frame_timestamp = datetime.fromtimestamp(self.cpu_dt + timestamp / 1e9) + frame_timestamp = datetime.fromtimestamp(self.cpu_dt + timestamp / 1e6) self.stream.add_frame(frame, frame_timestamp) From 764306544cea01ac4144c85023240a9640484fb2 Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Fri, 10 Jul 2026 10:37:35 +0100 Subject: [PATCH 4/5] Send timestamp with simulation frames --- .../things/camera/simulation.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/openflexure_microscope_server/things/camera/simulation.py b/src/openflexure_microscope_server/things/camera/simulation.py index e5fe90c1..4c71ed39 100644 --- a/src/openflexure_microscope_server/things/camera/simulation.py +++ b/src/openflexure_microscope_server/things/camera/simulation.py @@ -12,6 +12,7 @@ import io import logging import re import time +from datetime import datetime from threading import Thread from types import TracebackType from typing import Optional, Self, overload @@ -467,9 +468,10 @@ class SimulatedCamera(BaseCamera): last_frame_t = time.time() frame = self.generate_frame() - self.mjpeg_stream.add_frame(_frame2bytes(frame)) + timestamp = datetime.fromtimestamp(last_frame_t) + self.mjpeg_stream.add_frame(_frame2bytes(frame), timestamp) ds_frame = frame.resize((320, 240), resample=Image.Resampling.NEAREST) - self.lores_mjpeg_stream.add_frame(_frame2bytes(ds_frame)) + self.lores_mjpeg_stream.add_frame(_frame2bytes(ds_frame), timestamp) @lt.action def discard_frames(self) -> None: From 07cd60db2987764e5ce368b6899aca26852ca673 Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Fri, 10 Jul 2026 10:47:59 +0100 Subject: [PATCH 5/5] update picamera coverage zip --- picamera_coverage.zip | Bin 54038 -> 54038 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/picamera_coverage.zip b/picamera_coverage.zip index da2039b1b98ef1a3675c9eb19c3f2224ba39e418..95a69c4b118181ac436d21f7a9bf463370b436a2 100644 GIT binary patch delta 723 zcmbQXjCtBJW}yIYW)=|!5XcFA74wg&OJt*vN`uNW2L7M?Gx;s}nfadc#qnwL&g13h zxy-YWr-1tfcNMoD*DtPRn;ivexF)~o>ZtePXJzDU6yasMFrCqWVIC_-gvLFkox2WhiK{Ww^lj3uGS%HwmyXGcd3)FfcGOG%zr*B``4O@PW*V zU}pNmP++@VrkYJGA@Ugmd&+{>3@i*U8r>LJ96)BdG51_#FlFEYnPLqh4lpw`FfuSC zG%y-~xFF07A{dwrBp4X}s4{HZ&)G1aAwhwGf!ToJ!}J3T3=B6K7&j~Ra4<62c~92q zm8<9BWM$-RlHg}z_*KtvpEt&6^MR}Sv;OKzpMMa^ZgG&uJ%hWzcgueF-@i?N{Qk1v zn5E%+oVaX{enmFJkLdT-%h^BaGI;&{{{FC4%`LeFx_{#ioa19)TEKW<12e;KNr8VW zSQD6;uQB8mFn`#ez`y`@AIQ->49q(iC2GXq>ffHW{>P1?_leRB3=9&DTpS?xh)llM zE6dJu;PdfsyXq(Z>OB=Ku)v*xfkB3WfkB~xf#J}Bw_uYt@G~$p)PnuYz`#%qW;F0M z*fB7?0J9kw6c`u|FhbcIz$4k$R2o!vF!2B6pUH2*&&>CnFOE-}cNwoF z&t0C?JXPGExVyO>xOupCY<3jrBw*jgNsrg86_PgT(Pe6OE}Zj z8FH9D$T8)6?VsxKXX$<=9|m?^Mu&~<%#B=}tc;vZBK%AYzw8^{v+2!@jM)Ds|7Hc- zd8-4QX%Hzcw%d^1+~6Bx*SfQ@Yp_8<%5)rHOgK^SZSt?) zQ^71R1Q{3@*cliYI20Hf3<`dO&3(hnz>pvhW->4wkOMOoFfZU|V6XzS89+f>0A(_~ zfiMm*AMjydU=Y|W-mmP)0ZJkv#b2IJcDj(Jk!X^ZYLS+lYGG`aWRPZ-VqlV(W{{d> zW}IYcoSJB1Vq|P&m~5DwJo&-}XN_cYGfPVo^AzJ$gCvVY^Tgz21IyGzQ!{fTgR~@z zq|}t;mxe3=i*^I6EkCDlN96RL}SZTOH%{OG~*=WG$XTQQzJ9e llr+l}OG9&`#3XYQrP=^*MkWzv)O0+#_mTy|#b+;h0sxvU<}v^P