Merge branch 'pil-simulation' into 'v3'

Improve simulation frame rate

See merge request openflexure/openflexure-microscope-server!419

src/openflexure_microscope_server/things/camera/simulation.py

@@ -12,11 +12,10 @@ from typing import Literal, Optional, Mapping
 from types import TracebackType
 from threading import Thread
 import time
+import io
 
-import cv2
 import numpy as np
-from PIL import Image
+from PIL import Image, ImageFilter
-from scipy.ndimage import gaussian_filter
 
 import labthings_fastapi as lt
 
@@ -194,7 +193,7 @@ class SimulatedCamera(BaseCamera):
 
         self.canvas[top:bottom, left:right] -= sprite
 
-    def generate_image(self, pos: tuple[int, int, int]) -> np.ndarray:
+    def generate_image(self, pos: tuple[int, int, int]) -> Image:
         """Generate an image with blobs based on supplied coordinates.
 
         :param pos: a 3-item tuple containing the x,y,z coordinates of the 'stage'
@@ -214,11 +213,9 @@ class SimulatedCamera(BaseCamera):
         canvas = self.canvas if self._show_sample else self.blank_canvas
         # Use npx to make each 1d index list 3D
         focused_image = canvas[np.ix_(x_indices, y_indices, z_indices)]
-        image = gaussian_filter(
+
-            focused_image,
+        image = fast_pil_blur(focused_image, sigma=np.abs(pos[2]) / 5)
-            sigma=np.abs(pos[2]) / 5,
+
-            axes=(0, 1),
-        )
         if image.shape != self.shape:
             raise ValueError(
                 f"Image shape {image.shape} does not match intended shape {self.shape}"
@@ -228,7 +225,7 @@ class SimulatedCamera(BaseCamera):
         image += RNG.normal(scale=self.noise_level, size=self.shape).astype("int16")
         image[image < 0] = 0
         image[image > 255] = 255
-        return image.astype("uint8")
+        return Image.fromarray(image.astype("uint8"))
 
     def attach_to_server(
         self, server: lt.ThingServer, path: str, setting_storage_path: str
@@ -251,7 +248,7 @@ class SimulatedCamera(BaseCamera):
             self._stage = self._server.things["/stage/"]
         return self._stage.instantaneous_position
 
-    def generate_frame(self) -> np.ndarray:
+    def generate_frame(self) -> Image:
         """Generate a frame with blobs based on the stage coordinates."""
         try:
             pos = self.get_stage_position()
@@ -313,18 +310,18 @@ class SimulatedCamera(BaseCamera):
 
     def _capture_frames(self) -> None:
         portal = lt.get_blocking_portal(self)
+        last_frame_t = time.time()
         while self._capture_enabled:
-            time.sleep(self.frame_interval)
+            wait_time = last_frame_t - time.time() - self.frame_interval
+            if wait_time > 0:
+                time.sleep(wait_time)
+            last_frame_t = time.time()
             try:
                 frame = self.generate_frame()
-                jpeg = cv2.imencode(".jpg", frame)[1].tobytes()
+                self.mjpeg_stream.add_frame(_frame2bytes(frame), portal)
-                self.mjpeg_stream.add_frame(jpeg, portal)
+                ds_frame = frame.resize((320, 240), resample=Image.NEAREST)
-                # Downsample for lores
+                self.lores_mjpeg_stream.add_frame(_frame2bytes(ds_frame), portal)
-                ds_frame = cv2.resize(
+
-                    frame, (320, 240), interpolation=cv2.INTER_NEAREST
-                )
-                jpeg_lores = cv2.imencode(".jpg", ds_frame)[1].tobytes()
-                self.lores_mjpeg_stream.add_frame(jpeg_lores, portal)
             except Exception as e:
                 LOGGER.exception(f"Failed to capture frame: {e}, retrying...")
 
@@ -353,7 +350,7 @@ class SimulatedCamera(BaseCamera):
         if wait is not None:
             LOGGER.warning("Simulation camera has no wait option. Use None.")
         LOGGER.warning(f"Simulation camera camera doesn't respect {stream_name=}")
-        return self.generate_frame()
+        return np.array(self.generate_frame())
 
     def capture_image(
         self,
@@ -370,7 +367,7 @@ class SimulatedCamera(BaseCamera):
         if wait is not None:
             LOGGER.warning("Simulation camera has no wait option. Use None.")
         LOGGER.warning(f"Simulation camera camera doesn't respect {stream_name=}")
-        return Image.fromarray(self.generate_frame())
+        return self.generate_frame()
 
     @lt.thing_action
     def full_auto_calibrate(self, portal: lt.deps.BlockingPortal) -> None:
@@ -423,3 +420,23 @@ class SimulatedCamera(BaseCamera):
     def manual_camera_settings(self) -> list[PropertyControl]:
         """The camera settings to expose as property controls in the settings panel."""
         return [property_control_for(self, "noise_level", label="Noise Level")]
+
+
+def _frame2bytes(frame: Image) -> bytes:
+    """Convert frame to bytes."""
+    with io.BytesIO() as buf:
+        # Save in low quality for speed.
+        frame.save(buf, format="JPEG", quality=85)
+        return buf.getvalue()
+
+
+def fast_pil_blur(array: np.ndarray, sigma: float) -> np.ndarray:
+    """Apply Gaussian blur using PIL (faster than scipy)."""
+    if sigma < 0.5:
+        return array  # no visible blur needed
+
+    img_pil = Image.fromarray(array.astype(np.uint8))
+    img_pil = img_pil.filter(ImageFilter.GaussianBlur(radius=sigma))
+
+    # Convert back to NumPy array
+    return np.array(img_pil, dtype=array.dtype)