From 4e985fe80d2395134e851e9e48cf00f91349c1f9 Mon Sep 17 00:00:00 2001 From: Joel Collins Date: Tue, 22 Jan 2019 11:23:15 +0000 Subject: [PATCH] Capture list now persists through server restart --- openflexure_microscope/api/static/main_v1.js | 8 +- .../api/templates/index_v1.html | 2 +- .../api/v1/blueprints/camera/capture.py | 10 +- openflexure_microscope/camera/base.py | 93 +++++++++++++++---- openflexure_microscope/camera/capture.py | 61 +++++++----- 5 files changed, 124 insertions(+), 50 deletions(-) diff --git a/openflexure_microscope/api/static/main_v1.js b/openflexure_microscope/api/static/main_v1.js index 54cf7f97..2962f087 100644 --- a/openflexure_microscope/api/static/main_v1.js +++ b/openflexure_microscope/api/static/main_v1.js @@ -143,7 +143,7 @@ function getStagePositions() { } // Capture methods -function newCapture(filename, keep_on_disk, use_video_port, bayer, resizeWidth=null, resizeHeight=null) { +function newCapture(filename, temporary, use_video_port, bayer, resizeWidth=null, resizeHeight=null) { // Make a position request function newCaptureCallback(response, status) { if (status != 200) { @@ -155,7 +155,7 @@ function newCapture(filename, keep_on_disk, use_video_port, bayer, resizeWidth=n payload = { "filename": filename, - "keep_on_disk": keep_on_disk, + "temporary": temporary, "use_video_port": use_video_port, "bayer": bayer } @@ -173,7 +173,7 @@ function newCapture(filename, keep_on_disk, use_video_port, bayer, resizeWidth=n function newCaptureFromInput() { captureFilenameInput = document.getElementById('captureFilenameInput'); - captureKeepOnDiskCheck = document.getElementById('captureKeepOnDiskCheck'); + captureTemporaryCheck = document.getElementById('captureTemporaryCheck'); captureFullResolutionCheck = document.getElementById('captureFullResolutionCheck'); captureBayerCheck = document.getElementById('captureBayerCheck') @@ -200,7 +200,7 @@ function newCaptureFromInput() { newCapture( filename, - captureKeepOnDiskCheck.checked, + captureTemporaryCheck.checked, !(captureFullResolutionCheck.checked), captureBayerCheck.checked, resizeWidth, diff --git a/openflexure_microscope/api/templates/index_v1.html b/openflexure_microscope/api/templates/index_v1.html index 249b3862..d93702d6 100644 --- a/openflexure_microscope/api/templates/index_v1.html +++ b/openflexure_microscope/api/templates/index_v1.html @@ -91,7 +91,7 @@

Capture

Filename:

- Store on Pi:
+ Temporary:
Full resolution:
Keep raw data:
Resize capture:
diff --git a/openflexure_microscope/api/v1/blueprints/camera/capture.py b/openflexure_microscope/api/v1/blueprints/camera/capture.py index cf1e110f..7f03800a 100644 --- a/openflexure_microscope/api/v1/blueprints/camera/capture.py +++ b/openflexure_microscope/api/v1/blueprints/camera/capture.py @@ -68,7 +68,7 @@ class ListAPI(MicroscopeView): { "filename": "myfirstcapture", - "keep_on_disk": true, + "temporary": false, "use_video_port": true, "bayer": true, "size": { @@ -80,7 +80,7 @@ class ListAPI(MicroscopeView): :>header Accept: application/json :json boolean available: availability of capture data :>json string filename: filename of capture :>json string id: unique id of the capture object - :>json boolean keep_on_disk: keep the capture file on microscope after closing + :>json boolean temporary: delete the capture file on microscope after closing :>json boolean locked: file locked for modifications (mostly used for video recording) :>json string path: path on pi storage to the capture file, if available :>json boolean stream: capture stored in-memory as a BytesIO stream @@ -102,7 +102,7 @@ class ListAPI(MicroscopeView): payload = JsonPayload(request) filename = payload.param('filename') - keep_on_disk = payload.param('keep_on_disk', default=True, convert=bool) + temporary = payload.param('temporary', default=True, convert=bool) use_video_port = payload.param('use_video_port', default=False, convert=bool) bayer = payload.param('bayer', default=True, convert=bool) @@ -115,7 +115,7 @@ class ListAPI(MicroscopeView): output = self.microscope.camera.new_image( write_to_file=True, - keep_on_disk=keep_on_disk, + temporary=temporary, filename=filename) self.microscope.camera.capture( diff --git a/openflexure_microscope/camera/base.py b/openflexure_microscope/camera/base.py index 0a5b31b9..35a0afd8 100644 --- a/openflexure_microscope/camera/base.py +++ b/openflexure_microscope/camera/base.py @@ -5,6 +5,7 @@ import os import threading from PIL import Image import datetime +import yaml import logging try: @@ -15,7 +16,8 @@ except ImportError: except ImportError: from _thread import get_ident -from .capture import CaptureObject, BASE_CAPTURE_PATH +from .capture import CaptureObject, capture_from_dict, BASE_CAPTURE_PATH +from openflexure_microscope.config import USER_CONFIG_DIR def last_entry(object_list: list): @@ -110,6 +112,14 @@ class BaseCamera(object): self.images = [] #: list: List of image capture objects self.videos = [] #: list: List of video recording objects + # Capture data files + self.images_db = os.path.join(USER_CONFIG_DIR, "images_db.yaml") + self.videos_db = os.path.join(USER_CONFIG_DIR, "videos_db.yaml") + + # Load captures from persistent db + self.images = self.load_capture_db(self.images_db) + self.videos = self.load_capture_db(self.videos_db) + def __enter__(self): """Create camera on context enter.""" return self @@ -208,19 +218,51 @@ class BaseCamera(object): """Return a video StreamObject with a matching ID.""" return entry_by_id(id, self.videos) - # CREATING NEW CAPTURES + # MANAGE CAPTURE DATABASE - def new_stream_object( - self, - stream_object, - target_list: list, - shunt_others: bool=True): - """Add a new capture to a list of captures, and shunt all others.""" - if shunt_others: # If shunting all older captures - for obj in target_list: # For each older capture - obj.shunt() # Shunt capture from memory to storage - target_list.append(stream_object) - return stream_object + def save_capture_db(self, capture_list, db_path): + capture_state = [capture.state for capture in capture_list] + + with open(db_path, 'w') as outfile: + yaml.safe_dump(capture_state, outfile) + + def validate_captures(self, capture_list): + # Purge captures with missing data files + valid_captures = [] + for capture in capture_list: + if capture.file_exists: + valid_captures.append(capture) + else: + if os.path.isfile(capture.metadata_file): + logging.warning("Capture data missing. Deleting metadata file {}.".format(capture.metadata_file)) + os.remove(capture.metadata_file) + + # Validate and repair captures + for capture in valid_captures: + # If the captures metadata file is missing + if not os.path.isfile(capture.metadata_file): + # Recreate the metadata file + capture.save_metadata() + + return valid_captures + + def load_capture_db(self, db_path): + if os.path.isfile(db_path): + # Load list of capture dictionary representations from db_path + with open(db_path, 'r') as infile: + capture_dict_list = yaml.load(infile) + + # Create capture object list, and validate captures + capture_list = self.validate_captures( + [capture_from_dict(capture_dict) for capture_dict in capture_dict_list] + ) + + return capture_list + + else: + return [] + + # CREATING NEW CAPTURES def shunt_captures(self, target_list: list): for obj in target_list: # For each older capture @@ -229,7 +271,7 @@ class BaseCamera(object): def new_image( self, write_to_file: bool=False, - keep_on_disk: bool=True, + temporary: bool=True, filename: str=None, fmt: str='jpeg', shunt_others: bool=True): @@ -239,31 +281,37 @@ class BaseCamera(object): Args: write_to_file (bool): Should the StreamObject write to a file, or an in-memory byte stream. - keep_on_disk (bool): Should the data be kept on disk after session ends. Creating the capture with a content manager sets this to false. + keep_on_disk (bool): Should the data be deleted after session ends. Creating the capture with a content manager sets this to true. filename (str): Name of the stored file. Defaults to timestamp. fmt (str): Format of the capture. """ + # Generate file name if not filename: filename = self.generate_basename(self.images) logging.debug(filename) + # Create capture object output = CaptureObject( write_to_file=write_to_file, - keep_on_disk=keep_on_disk, + temporary=temporary, filename=filename, folder=self.paths['image'], fmt=fmt) + # Update capture list self.shunt_captures(self.images) self.images.append(output) + # Update capture database + self.save_capture_db(self.images, self.images_db) + return output def new_video( self, write_to_file: bool=True, - keep_on_disk: bool=True, + temporary: bool=False, filename: str=None, fmt: str='h264', shunt_others: bool=True): @@ -273,28 +321,35 @@ class BaseCamera(object): Args: write_to_file (bool): Should the StreamObject write to a file, or an in-memory byte stream. - keep_on_disk (bool): Should the data be kept on disk after session ends. Creating the capture with a content manager sets this to false. + temporary (bool): Should the data be deleted after session ends. Creating the capture with a content manager sets this to true. filename (str): Name of the stored file. Defaults to timestamp. fmt (str): Format of the capture. """ + # Generate file name if not filename: filename = self.generate_basename(self.videos) logging.debug(filename) + # Create capture object output = CaptureObject( write_to_file=write_to_file, - keep_on_disk=keep_on_disk, + temporary=temporary, filename=filename, folder=self.paths['video'], fmt=fmt) + # Update capture list self.shunt_captures(self.videos) self.videos.append(output) + # Update capture database + self.save_capture_db(self.videos, self.videos_db) + return output # INTELLIGENTLY GENERATE FILENAMES + def generate_basename(self, obj_list: list) -> str: initial_basename = generate_basename() basename = initial_basename diff --git a/openflexure_microscope/camera/capture.py b/openflexure_microscope/camera/capture.py index a27d84f7..e9185718 100644 --- a/openflexure_microscope/camera/capture.py +++ b/openflexure_microscope/camera/capture.py @@ -24,6 +24,24 @@ def clear_tmp(): os.remove(f) logging.debug("Removed {}".format(f)) +def capture_from_dict(capture_dict): + capture = CaptureObject(create_metadata_file=False) # Create a placeholder capture + + # Populate capture parameters + capture.temporary = capture_dict['temporary'] + capture.id = capture_dict['metadata']['id'] + capture.timestring = capture_dict['metadata']['time'] + capture.format = capture_dict['metadata']['format'] + + if 'custom' in capture_dict['metadata']: + capture._metadata = capture_dict['metadata']['custom'] + + capture.file = capture_dict['metadata']['path'] + capture.split_file_path(capture.file) + + return capture + + class CaptureObject(object): """ StreamObject used to store and process capture data, and metadata. @@ -33,10 +51,11 @@ class CaptureObject(object): def __init__( self, write_to_file: bool=False, - keep_on_disk: bool=True, + temporary: bool=False, filename: str='', folder: str='', - fmt: str='') -> None: + fmt: str='', + create_metadata_file: bool=True) -> None: """Create a new StreamObject, to manage capture data.""" # Store a nice ID @@ -48,7 +67,7 @@ class CaptureObject(object): self.format = fmt # Keep on disk after close by default - self.keep_on_disk = keep_on_disk + self.temporary = temporary # Explicitally state if capture should be written to a file, not a bytestream self.write_to_file = write_to_file @@ -56,15 +75,18 @@ class CaptureObject(object): # Create file name. Default to UUID if not filename: filename = self.id - self.filename = "{}.{}".format(filename, fmt) + + # Create folder path. Default to BASE_CAPTURE_PATH + if not folder: + folder = BASE_CAPTURE_PATH self.folder = folder # Dictionary for storing custom metadata self._metadata = {} # Initialise the capture stream - self.initialise_capture() + self.initialise_capture(create_metadata_file=create_metadata_file) # Log if created by context manager self.context_manager = False @@ -78,7 +100,7 @@ class CaptureObject(object): def __enter__(self): """Create StreamObject in context, to auto-clean disk data.""" logging.debug("Entering context for {}. Stored files will be cleaned up automatically regardless of location.".format(self.id)) - self.keep_on_disk = False # Flag file to be removed on close. + self.temporary = True # Flag file to be removed on close. self.context_manager = True # Used in metadata logging.info("Rebuilding as a temporary capture...") @@ -91,7 +113,7 @@ class CaptureObject(object): logging.info("Cleaning up {}".format(self.id)) self.close() - def initialise_capture(self): + def initialise_capture(self, create_metadata_file=True): self.build_file_path(self.filename, self.folder) # Byte bytestream properties @@ -106,7 +128,8 @@ class CaptureObject(object): self.stream = self.file # Save initial metadata file - self.save_metadata() + if create_metadata_file: + self.save_metadata() def build_file_path( self, @@ -128,7 +151,7 @@ class CaptureObject(object): raise Exception("Captures cannot be stored in a lower-level directory than {}.".format(BASE_CAPTURE_PATH)) # Move to tmp directory if not being kept - if not self.keep_on_disk: + if self.temporary: self.file_notmp = self.file # Store originally defined folder self.filefolder = TEMP_CAPTURE_PATH self.file = os.path.join(self.filefolder, self.filename) @@ -201,11 +224,12 @@ class CaptureObject(object): 'id': self.id, 'filename': self.filename, 'path': self.file, - 'time': self.timestring + 'time': self.timestring, + 'format': self.format } # Add custom metadata to dictionary - d.update(self._metadata) + d['custom'] = self._metadata return d @@ -224,7 +248,7 @@ class CaptureObject(object): # Create basic state dictionary d = { 'locked': self.locked, - 'keep_on_disk': self.keep_on_disk, + 'temporary': self.temporary, } # Add metadata to state @@ -243,10 +267,6 @@ class CaptureObject(object): else: d['available'] = False - # Check if file was manually deleted - if self.keep_on_disk and not self.file_exists: - d['metadata']['path'] = "{} (Deleted)".format(d['metadata']['path']) - return d @property @@ -281,11 +301,10 @@ class CaptureObject(object): # If no thumbnail exists, try and make one if not self.thumb_bytes: logging.info("Building thumbnail") + self.thumb_bytes = io.BytesIO() if self.format.upper() in pil_formats: im = Image.open(self.data) im.thumbnail(thumbnail_size) - - self.thumb_bytes = io.BytesIO() im.save(self.thumb_bytes, self.format) self.thumb_bytes.seek(0) else: @@ -307,9 +326,9 @@ class CaptureObject(object): def save_file(self) -> bool: """Write the StreamObjects bytestream to a file.""" - if not self.keep_on_disk: # If capture is currently temporary + if self.temporary: # If capture is currently temporary self.load_file() # Load data from tmp file into bytestream, if tmp file exists - self.keep_on_disk = True # Flag as kept on disk + self.temporary = False # Flag as kept on disk self.file = self.file_notmp # Reset file path to non-temporary path self.split_file_path(self.file) # Set split properties based on new path logging.info("Moved temporary file out to {}".format(self.file)) @@ -367,7 +386,7 @@ class CaptureObject(object): """Both clear the bytestream, and delete any associated on-disk data.""" logging.info("Closing {}".format(self.id)) self.delete_stream() - if not self.keep_on_disk: + if self.temporary: self.delete() atexit.register(clear_tmp) \ No newline at end of file