From 574f4a615d424bab49b1bb5c102a623142799195 Mon Sep 17 00:00:00 2001 From: Joel Collins Date: Mon, 26 Oct 2020 13:16:58 +0000 Subject: [PATCH 1/3] Fast AF use normal stream --- .../api/default_extensions/autofocus.py | 65 +--------- openflexure_microscope/camera/base.py | 47 +++++++ openflexure_microscope/camera/mock.py | 4 - openflexure_microscope/camera/pi.py | 115 +++--------------- 4 files changed, 72 insertions(+), 159 deletions(-) diff --git a/openflexure_microscope/api/default_extensions/autofocus.py b/openflexure_microscope/api/default_extensions/autofocus.py index 51b2e575..67306399 100644 --- a/openflexure_microscope/api/default_extensions/autofocus.py +++ b/openflexure_microscope/api/default_extensions/autofocus.py @@ -1,6 +1,5 @@ import logging import time -from collections import namedtuple from contextlib import contextmanager import numpy as np @@ -14,40 +13,6 @@ from openflexure_microscope.utilities import set_properties ### Autofocus utilities -# Class to store a frames metadata -TrackerFrame = namedtuple("TrackerFrame", ["size", "time"]) - - -class FrameTracker(object): - """ - A file-like object used to analyse MJPEG frames. - - Instead of analysing a load of real MJPEG frames - after they've been stored in a BytesIO stream, - we tell the camera to write frames to this class instead. - - We then do analysis as the frames are written, and discard the heavier - image data. - """ - - def __init__(self): - # Array of TrackerFrame objects - self.frames = [] - self.last = None - - def write(self, s): - # Store the incoming frame metadata - # Discards the actual frame image data - # NOTE: This could be used for smarter processing - # E.g. different sharpness metrics - frame = TrackerFrame(size=len(s), time=time.time()) - self.frames.append(frame) - self.last = frame - - def flush(self): - # Called at end of recording - pass - class JPEGSharpnessMonitor: def __init__(self, microscope): @@ -62,36 +27,17 @@ class JPEGSharpnessMonitor: self.jpeg_times = [] self.jpeg_sizes = [] - # Splitter port 3 is reserved for autofocus - self.splitter_port = 3 - self.stream = FrameTracker() - - def stop_recording(self): - logging.info("Stopping recording for autofocus...") - self.camera.camera.stop_recording(splitter_port=self.splitter_port) - logging.info("Stopped recording for autofocus!") - def start(self): # Log the recording start time self.recording_start_time = time.time() - # Start the stream recording - self.camera.camera.start_recording( - self.stream, - format="mjpeg", - quality=100, # Use best quality (we don't store the frames anyway!) - bitrate=-1, # RWB: disable bitrate control - # (bitrate control makes JPEG size less good as a focus - # metric) - splitter_port=self.splitter_port, - ) - logging.info("Started recording for autofocus") def stop(self): - self.stop_recording() - logging.info("Stopped recording for autofocus") + self.camera.stream.stop_tracking() + self.camera.stream.reset_tracking() def focus_rel(self, dz, backlash=False, **kwargs): # Store the start time and position + self.camera.stream.start_tracking() self.stage_times.append(time.time()) self.stage_positions.append(self.stage.position) @@ -99,11 +45,12 @@ class JPEGSharpnessMonitor: self.stage.move_rel([0, 0, dz], backlash=backlash, **kwargs) # Store the end time and position + self.camera.stream.stop_tracking() self.stage_times.append(time.time()) self.stage_positions.append(self.stage.position) # Retrieve frame data - for frame in self.stream.frames: + for frame in self.camera.stream.frames: # Make timestamp absolute Unix time self.jpeg_times.append(frame.time) self.jpeg_sizes.append(frame.size) @@ -329,7 +276,7 @@ def fast_up_down_up_autofocus( # We've deliberately undershot - figure out how much further we should move based on the curve logging.debug("Calculate remining movement") - current_js = m.stream.last.size + current_js = m.camera.stream.last.size imax = np.argmax(js) # we want to crop out just the bit below the peak js = js[imax:] # NB z is in DECREASING order jz = jz[imax:] diff --git a/openflexure_microscope/camera/base.py b/openflexure_microscope/camera/base.py index c2451dbe..2c499430 100644 --- a/openflexure_microscope/camera/base.py +++ b/openflexure_microscope/camera/base.py @@ -2,10 +2,56 @@ import logging import threading import time +import io from abc import ABCMeta, abstractmethod +from collections import namedtuple from labthings import ClientEvent, StrictLock +# Class to store a frames metadata +TrackerFrame = namedtuple("TrackerFrame", ["size", "time"]) + + +class FrameStream(io.BytesIO): + """ + A file-like object used to analyse MJPEG frames. + + Instead of analysing a load of real MJPEG frames + after they've been stored in a BytesIO stream, + we tell the camera to write frames to this class instead. + + We then do analysis as the frames are written, and discard the heavier + image data. + """ + + def __init__(self, *args, **kwargs): + # Array of TrackerFrame objects + io.BytesIO.__init__(self, *args, **kwargs) + self.frames = [] + self.last = None + + self.tracking = False + + def start_tracking(self): + if not self.tracking: + logging.info("Started tracking frame data") + self.tracking = True + + def stop_tracking(self): + if self.tracking: + logging.info("Stopped tracking frame data") + self.tracking = False + + def reset_tracking(self): + self.frames = [] + + def write(self, s): + if self.tracking: + frame = TrackerFrame(size=len(s), time=time.time()) + self.frames.append(frame) + self.last = frame + io.BytesIO.write(self, s) + class BaseCamera(metaclass=ABCMeta): """ @@ -18,6 +64,7 @@ class BaseCamera(metaclass=ABCMeta): self.lock = StrictLock(name="Camera", timeout=None) + self.stream = FrameStream() self.frames_iterator = None self.frame = None self.last_access = 0 diff --git a/openflexure_microscope/camera/mock.py b/openflexure_microscope/camera/mock.py index 2d20ef39..f2c84762 100644 --- a/openflexure_microscope/camera/mock.py +++ b/openflexure_microscope/camera/mock.py @@ -2,7 +2,6 @@ from __future__ import division -import io import logging import time from datetime import datetime @@ -32,9 +31,6 @@ class MissingCamera(BaseCamera): self.jpeg_quality = 75 self.framerate = 10 - # Create an empty stream - self.stream = io.BytesIO() - # Generate an initial dummy image self.generate_new_dummy_image() diff --git a/openflexure_microscope/camera/pi.py b/openflexure_microscope/camera/pi.py index 1ef0948e..4ea5e691 100644 --- a/openflexure_microscope/camera/pi.py +++ b/openflexure_microscope/camera/pi.py @@ -26,9 +26,6 @@ Still capture (if use_video_port == False) uses pause_stream to temporarily increase the capture resolution. """ -from __future__ import division - -import io import logging import time @@ -111,7 +108,6 @@ class PiCameraStreamer(BaseCamera): ) #: str: Path of .npy lens shading table file # Start the stream worker on init - self.stream = io.BytesIO() # Create a stream object self.start_worker() @property @@ -403,21 +399,18 @@ class PiCameraStreamer(BaseCamera): # Update state self.record_active = False - def stop_stream_recording( - self, splitter_port: int = 1, resolution: Tuple[int, int] = None - ) -> None: + def stop_stream_recording(self, splitter_port: int = 1, **kwargs) -> None: """ Sets the camera resolution to the still-image resolution, and stops recording if the stream is active. Args: splitter_port (int): Splitter port to stop recording on - resolution ((int, int)): Resolution to set the camera to, after stopping recording. """ + for k in kwargs.keys(): + logging.warning( + "Warning, kwarg %s is invalid for stop_stream_recording.", k + ) with self.lock: - # If no resolution is specified, default to image_resolution - if not resolution: - resolution = self.image_resolution - # Stop the camera video recording on port 1 try: self.camera.stop_recording(splitter_port=splitter_port) @@ -426,7 +419,7 @@ class PiCameraStreamer(BaseCamera): else: logging.info( "Stopped MJPEG stream on port {1}. Switching to {0}.".format( - resolution, splitter_port + self.image_resolution, splitter_port ) ) @@ -434,30 +427,20 @@ class PiCameraStreamer(BaseCamera): time.sleep( 0.2 ) # Sprinkled a sleep to prevent camera getting confused by rapid commands - self.camera.resolution = resolution + self.camera.resolution = self.image_resolution - def start_stream_recording( - self, splitter_port: int = 1, resolution: Tuple[int, int] = None - ) -> None: + def start_stream_recording(self, splitter_port: int = 1, **kwargs) -> None: """ Sets the camera resolution to the video/stream resolution, and starts recording if the stream should be active. Args: splitter_port (int): Splitter port to start recording on - resolution ((int, int)): Resolution to set the camera to, before starting recording. - Defaults to `self.stream_resolution`. """ + for k in kwargs.keys(): + logging.warning( + "Warning, kwarg %s is invalid for stop_stream_recording.", k + ) with self.lock(timeout=None): - # If stream object was destroyed - if not hasattr(self, "stream"): - self.stream = io.BytesIO() # Create a stream object - - # If no explicit resolution is passed - if not resolution: - resolution = ( - self.stream_resolution - ) # Default to video recording resolution - # Reduce the resolution for video streaming try: self.camera._check_recording_stopped() # pylint: disable=W0212 @@ -466,7 +449,9 @@ class PiCameraStreamer(BaseCamera): "Error while changing resolution: Recording already running." ) else: - self.camera.resolution = resolution + self.camera.resolution = self.stream_resolution + # Sprinkled a sleep to prevent camera getting confused by rapid commands + time.sleep(0.2) # If the stream should be active if self.stream_active: @@ -488,7 +473,7 @@ class PiCameraStreamer(BaseCamera): else: logging.debug( "Started MJPEG stream at {} on port {}".format( - resolution, splitter_port + self.stream_resolution, splitter_port ) ) @@ -522,7 +507,6 @@ class PiCameraStreamer(BaseCamera): # Set resolution and stop stream recording if necessary if not use_video_port: self.stop_stream_recording() - time.sleep(0.1) self.camera.capture( output, @@ -532,7 +516,6 @@ class PiCameraStreamer(BaseCamera): bayer=(not use_video_port) and bayer, use_video_port=use_video_port, ) - # time.sleep(0.1) # Set resolution and start stream recording if necessary if not use_video_port: @@ -540,49 +523,7 @@ class PiCameraStreamer(BaseCamera): return output - def yuv( - self, use_video_port: bool = True, resize: Tuple[int, int] = None - ) -> np.ndarray: - """Capture an uncompressed still YUV image to a Numpy array. - - Args: - use_video_port (bool): Capture from the video port used for streaming. Lower resolution, faster. - resize ((int, int)): Resize the captured image. - - Returns: - output_array (np.ndarray): Output array of capture - """ - with self.lock: - if use_video_port: - resolution = self.stream_resolution - else: - resolution = self.numpy_resolution - - if resize: - size = resize - else: - size = resolution - - if not use_video_port: - self.stop_stream_recording(resolution=resolution) - - logging.debug("Creating PiYUVArray") - with picamerax.array.PiYUVArray(self.camera, size=size) as output: - - logging.info("Capturing to {}".format(output)) - - self.camera.capture( - output, resize=size, format="yuv", use_video_port=use_video_port - ) - - if not use_video_port: - self.start_stream_recording() - - return output.array - - def array( - self, use_video_port: bool = True, resize: Tuple[int, int] = None - ) -> np.ndarray: + def array(self) -> np.ndarray: """Capture an uncompressed still RGB image to a Numpy array. Args: @@ -593,30 +534,12 @@ class PiCameraStreamer(BaseCamera): output_array (np.ndarray): Output array of capture """ with self.lock: - if use_video_port: - resolution = self.stream_resolution - else: - resolution = self.numpy_resolution - - if resize: - size = resize - else: - size = resolution - - # Always pause stream, to prevent resizer memory issues - self.stop_stream_recording(resolution=resolution) - logging.debug("Creating PiRGBArray") - with picamerax.array.PiRGBArray(self.camera, size=size) as output: + with picamerax.array.PiRGBArray(self.camera) as output: logging.info("Capturing to {}".format(output)) - self.camera.capture( - output, resize=size, format="rgb", use_video_port=use_video_port - ) - - # Resume stream - self.start_stream_recording() + self.camera.capture(output, format="rgb", use_video_port=True) return output.array From 459a8dc2a3ad31efa7fc1374734fd621a1883619 Mon Sep 17 00:00:00 2001 From: Joel Collins Date: Mon, 26 Oct 2020 13:36:36 +0000 Subject: [PATCH 2/3] Reset tracker frames after each mode --- openflexure_microscope/api/default_extensions/autofocus.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/openflexure_microscope/api/default_extensions/autofocus.py b/openflexure_microscope/api/default_extensions/autofocus.py index 67306399..089eae4c 100644 --- a/openflexure_microscope/api/default_extensions/autofocus.py +++ b/openflexure_microscope/api/default_extensions/autofocus.py @@ -54,6 +54,8 @@ class JPEGSharpnessMonitor: # Make timestamp absolute Unix time self.jpeg_times.append(frame.time) self.jpeg_sizes.append(frame.size) + # Clear frame data for this move from the stream + self.camera.stream.reset_tracking() # Index of the data for this movement data_index = len(self.stage_positions) - 2 @@ -62,7 +64,7 @@ class JPEGSharpnessMonitor: return data_index, final_z_position def move_data(self, istart, istop=None): - "Extract sharpness as a function of (interpolated) z" + """Extract sharpness as a function of (interpolated) z""" if istop is None: istop = istart + 2 jpeg_times = np.array(self.jpeg_times) From a258b6338d93f416d26f787d90db6327161eedef Mon Sep 17 00:00:00 2001 From: Joel Collins Date: Mon, 26 Oct 2020 13:48:33 +0000 Subject: [PATCH 3/3] Close #148 --- .../static/src/components/controlComponents/paneNavigate.vue | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openflexure_microscope/api/static/src/components/controlComponents/paneNavigate.vue b/openflexure_microscope/api/static/src/components/controlComponents/paneNavigate.vue index fbaa567c..0810cfe1 100644 --- a/openflexure_microscope/api/static/src/components/controlComponents/paneNavigate.vue +++ b/openflexure_microscope/api/static/src/components/controlComponents/paneNavigate.vue @@ -122,7 +122,7 @@