rahix/openflexure-microscope-server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Optimised captures by using greenlets for disk IO

Browse source
This commit is contained in:
Joel Collins 2020-06-11 11:42:31 +01:00
parent 1a0ede609d
commit 7c9f42f7f3
8 changed files with 86 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

9
openflexure_microscope/api/app.py
View file

@ -3,13 +3,20 @@ from gevent import monkey
monkey.patch_all()
import sys
import time
import atexit
import logging, logging.handlers
# Look for debug flag
if "-d" in sys.argv or "--debug" in sys.argv:
log_level = logging.DEBUG
else:
log_level = logging.INFO
# Set root logger level
logger = logging.getLogger()
logger.setLevel(logging.INFO)
logger.setLevel(log_level)
import os
import pkg_resources

15
openflexure_microscope/camera/mock.py
View file

@ -197,19 +197,14 @@ class MissingCamera(BaseCamera):
bayer (bool): Store raw bayer data in capture
"""
if isinstance(output, CaptureObject):
target = output.file
else:
target = output
with self.lock:
if isinstance(target, str):
target = open(target, "wb")
if isinstance(output, str):
output = open(output, "wb")
target.write(self.stream.getvalue())
output.write(self.stream.getvalue())
if isinstance(target, str):
target.close()
if isinstance(output, str):
output.close()
# HANDLE STREAM FRAMES

17
openflexure_microscope/camera/pi.py
View file

@ -115,11 +115,6 @@ class PiCameraStreamer(BaseCamera):
"picamera_lst.npy"
) #: str: Path of .npy lens shading table file
# Create an empty stream
self.stream = io.BytesIO()
# Start streaming
self.start_worker()
@property
def configuration(self):
@ -520,12 +515,6 @@ class PiCameraStreamer(BaseCamera):
Returns:
output_object (str/BytesIO): Target object.
"""
if isinstance(output, CaptureObject):
target = output.file
else:
target = output
with self.lock:
logging.info("Capturing to {}".format(output))
@ -535,20 +524,20 @@ class PiCameraStreamer(BaseCamera):
time.sleep(0.1)
self.camera.capture(
target,
output,
format=fmt,
quality=100,
resize=resize,
bayer=(not use_video_port) and bayer,
use_video_port=use_video_port,
)
time.sleep(0.1)
#time.sleep(0.1)
# Set resolution and start stream recording if necessary
if not use_video_port:
self.start_stream_recording()
return target
return output
def yuv(
self, use_video_port: bool = True, resize: Tuple[int, int] = None

63
openflexure_microscope/captures/capture.py
View file

@ -10,6 +10,9 @@ from PIL import Image
import dateutil.parser
import atexit
from gevent.fileobject import FileObjectThread
import gevent
from collections import OrderedDict
from openflexure_microscope.camera import piexif
@ -118,12 +121,19 @@ def capture_from_exif(path, exif_dict):
class CaptureObject(object):
"""
StreamObject used to store and process on-disk capture data, and metadata.
File-like object used to store and process on-disk capture data, and metadata.
Serves to simplify modifying properties of on-disk capture data.
"""
def __init__(self, filepath) -> None:
"""Create a new StreamObject, to manage capture data."""
# Stream for buffering capture data
self.stream = io.BytesIO()
# Event to notify when the stream has finished writing to disk
#self.file_ready = Event()
self.file_ready = gevent.event.Event()
# Lock to control disk file access
self.file_lock = gevent.lock.BoundedSemaphore()
# Store a nice ID
self.id = uuid.uuid4() #: str: Unique capture ID
@ -148,6 +158,25 @@ class CaptureObject(object):
# Thumbnail (populated only for PIL captures)
self.thumb_bytes = None
def write(self, s):
self.stream.write(s)
gevent.sleep()
def _stream_to_file(self):
logging.info(f"Writing to disk {self.file}")
with FileObjectThread(open(self.file, "wb"), 'wb') as outfile, self.file_lock:
outfile.write(self.stream.getbuffer())
self.stream.close()
self.file_ready.set()
logging.info(f"Finished writing to disk {self.file}")
gevent.sleep()
def flush(self):
logging.debug(f"Flushing {self.file}")
gevent.spawn(self._stream_to_file)
gevent.sleep()
logging.debug(f"Returning flushing {self.file}")
def open(self, mode):
return open(self.file, mode)
@ -221,24 +250,30 @@ class CaptureObject(object):
self.save_metadata()
def save_metadata(self) -> None:
#gevent.get_hub().threadpool.spawn(self.synchronous_save_metadata)
gevent.spawn(self.synchronous_save_metadata)
gevent.sleep()
def synchronous_save_metadata(self) -> None:
"""
Save metadata to exif, if supported
"""
global EXIF_FORMATS
if self.format.upper() in EXIF_FORMATS and self.exists:
logging.debug("Writing exif data to capture file")
# Extract current Exif data
exif_dict = piexif.load(self.file)
# Serialize metadata
metadata_string = json.dumps(self.metadata, cls=JSONEncoder)
logging.debug(f"Saving metadata string to file: {metadata_string}")
# Insert metadata into exif_dict
exif_dict["Exif"][piexif.ExifIFD.UserComment] = metadata_string.encode()
# Convert new exif dict to exif bytes
exif_bytes = piexif.dump(exif_dict)
# Insert exif into file
piexif.insert(exif_bytes, self.file)
with self.file_lock:
logging.debug("Writing exif data to capture file")
# Extract current Exif data
exif_dict = piexif.load(self.file)
# Serialize metadata
metadata_string = json.dumps(self.metadata, cls=JSONEncoder)
logging.debug(f"Saving metadata string to file: {metadata_string}")
# Insert metadata into exif_dict
exif_dict["Exif"][piexif.ExifIFD.UserComment] = metadata_string.encode()
# Convert new exif dict to exif bytes
exif_bytes = piexif.dump(exif_dict)
# Insert exif into file
piexif.insert(exif_bytes, self.file)
@property
def metadata(self) -> dict:
@ -286,7 +321,7 @@ class CaptureObject(object):
if self.exists: # If data file exists
logging.info("Opening from file {}".format(self.file))
with open(self.file, "rb") as f:
with open(self.file, "rb") as f, self.file_lock:
d = io.BytesIO(f.read()) # Load bytes from file
d.seek(0) # Rewind loaded bytestream
# Create a copy of the bytestream bytes

18
openflexure_microscope/microscope.py
View file

@ -7,6 +7,8 @@ import pkg_resources
import uuid
from typing import Tuple
import gevent
from openflexure_microscope.captures import CaptureManager
from openflexure_microscope.stage.mock import MissingStage
@ -299,6 +301,7 @@ class Microscope:
tags: list = None,
metadata: dict = None,
):
logging.debug(f"Microscope capturing to {filename}")
if not annotations:
annotations = {}
if not metadata:
@ -313,14 +316,22 @@ class Microscope:
)
# Capture to output object
logging.info("Starting microscope capture...")
self.camera.capture(
output.file,
output,
use_video_port=use_video_port,
resize=resize,
bayer=bayer,
fmt=fmt,
)
logging.info("Finished microscope capture...")
def inject_metadata():
logging.debug(f"Waiting for {output.file}")
# Wait for the file to be written to disk
output.file_ready.wait()
logging.info(f"Asynchronously injecting EXIF data into {output.file}")
# Inject system metadata
output.put_metadata({"instrument": self.metadata})
# Insert custom metadata
@ -329,5 +340,10 @@ class Microscope:
output.put_annotations(annotations)
# Insert custom tags
output.put_tags(tags)
logging.info(f"Finished injecting EXIF data into {output.file}")
gevent.spawn(inject_metadata)
logging.debug(f"Finished capture to {output.file}")
return output

2
openflexure_microscope/stage/sanga.py
View file

@ -108,6 +108,7 @@ class SangaStage(BaseStage):
backlash: (default: True) whether to correct for backlash.
"""
with self.lock:
logging.info(f"Moving sangaboard by {displacement}")
if not backlash or self.backlash is None:
return self.board.move_rel(displacement, axis=axis)
@ -145,6 +146,7 @@ class SangaStage(BaseStage):
"""Make an absolute move to a position
"""
with self.lock:
logging.info(f"Moving sangaboard to {final}")
self.board.move_abs(final, **kwargs)
def zero_position(self):

1
openflexure_microscope/utilities.py
View file

@ -8,6 +8,7 @@ import logging
from collections import abc
from functools import reduce
from contextlib import contextmanager
import gevent
def deserialise_array_b64(b64_string, dtype, shape):