From 61f15d610f985d8507f1537347c1ad4a62c51aca Mon Sep 17 00:00:00 2001 From: Joel Collins Date: Tue, 9 Apr 2019 14:52:08 +0100 Subject: [PATCH 1/7] Restructured how temporary paths are calculated --- openflexure_microscope/camera/capture.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/openflexure_microscope/camera/capture.py b/openflexure_microscope/camera/capture.py index 6f33fc34..fa3ae215 100644 --- a/openflexure_microscope/camera/capture.py +++ b/openflexure_microscope/camera/capture.py @@ -215,6 +215,12 @@ class CaptureObject(object): """ global TEMP_CAPTURE_PATH + if self.temporary: + # Store original file path + self.file_notmp = os.path.join(folder, filename) + # Move user-specified folder to TEMP + self.folder = os.path.join(TEMP_CAPTURE_PATH, self.folder) + self.file = os.path.join(folder, filename) # Full file name by joining given folder to given name self.split_file_path(self.file) # Split file path into folder, filename, and basename @@ -223,12 +229,6 @@ class CaptureObject(object): if not os.path.commonprefix([self.file, BASE_CAPTURE_PATH]) == BASE_CAPTURE_PATH: raise Exception("Captures cannot be stored in a lower-level directory than {}.".format(BASE_CAPTURE_PATH)) - # Move to tmp directory if not being kept - if self.temporary: - self.file_notmp = self.file # Store originally defined folder - self.filefolder = TEMP_CAPTURE_PATH - self.file = os.path.join(self.filefolder, self.filename) - # Create folder and file if not os.path.exists(self.filefolder): os.makedirs(self.filefolder) From f9fb6dc7212817be5e11345fd7aeec5ae80f54ab Mon Sep 17 00:00:00 2001 From: Joel Collins Date: Tue, 9 Apr 2019 14:57:19 +0100 Subject: [PATCH 2/7] Default to not temporary --- openflexure_microscope/api/v1/blueprints/camera/capture.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openflexure_microscope/api/v1/blueprints/camera/capture.py b/openflexure_microscope/api/v1/blueprints/camera/capture.py index a2a5dc6f..ffa41bdd 100644 --- a/openflexure_microscope/api/v1/blueprints/camera/capture.py +++ b/openflexure_microscope/api/v1/blueprints/camera/capture.py @@ -101,7 +101,7 @@ class ListAPI(MicroscopeView): payload = JsonPayload(request) filename = payload.param('filename') - temporary = payload.param('temporary', default=True, convert=bool) + temporary = payload.param('temporary', default=False, convert=bool) use_video_port = payload.param('use_video_port', default=False, convert=bool) bayer = payload.param('bayer', default=True, convert=bool) metadata = payload.param('metadata', default={}, convert=dict) From 915958885be6b51a9035c6079faaf6d0f6538ace Mon Sep 17 00:00:00 2001 From: Joel Collins Date: Tue, 9 Apr 2019 14:57:28 +0100 Subject: [PATCH 3/7] Added temporary support to scans --- .../plugins/default/scan/api.py | 42 +------------------ .../plugins/default/scan/plugin.py | 9 +++- 2 files changed, 9 insertions(+), 42 deletions(-) diff --git a/openflexure_microscope/plugins/default/scan/api.py b/openflexure_microscope/plugins/default/scan/api.py index 45e514cc..d025f039 100644 --- a/openflexure_microscope/plugins/default/scan/api.py +++ b/openflexure_microscope/plugins/default/scan/api.py @@ -10,6 +10,7 @@ class TileScanAPI(MicroscopeViewPlugin): # Get params filename = payload.param('filename') + temporary = payload.param('temporary', default=False, convert=bool) step_size = payload.param('step_size', default=[2000, 1500, 100], convert=list) step_size = [int(i) for i in step_size] @@ -36,6 +37,7 @@ class TileScanAPI(MicroscopeViewPlugin): task = self.microscope.task.start( self.plugin.tile, basename=filename, + temporary=temporary, step_size=step_size, grid=grid, style=style, @@ -49,43 +51,3 @@ class TileScanAPI(MicroscopeViewPlugin): # return a handle on the autofocus task return jsonify(task.state), 202 - - -class ZStackAPI(MicroscopeViewPlugin): - def post(self): - payload = JsonPayload(request) - - # Get params - name = payload.param('name') - step_size = payload.param('step_size', default=100, convert=int) - steps = payload.param('steps', default=5, convert=int) - center = payload.param('center', default=True, convert=bool) - - use_video_port = payload.param('use_video_port', default=False, convert=bool) - resize = payload.param('size', default=None) - if resize: - if ('width' in resize) and ('height' in resize): - resize = (int(resize['width']), int(resize['height'])) # Convert dict to tuple - else: - abort(404) - - bayer = payload.param('bayer', default=True, convert=bool) - metadata = payload.param('metadata', default={}, convert=dict) - tags = payload.param('tags', default=[], convert=list) - - print("Running tile scan...") - task = self.microscope.task.start( - self.plugin.stack, - basename=name, - step_size=step_size, - steps=steps, - center=center, - use_video_port=use_video_port, - resize=resize, - bayer=bayer, - metadata=metadata, - tags=tags - ) - - # return a handle on the autofocus task - return jsonify(task.state), 202 diff --git a/openflexure_microscope/plugins/default/scan/plugin.py b/openflexure_microscope/plugins/default/scan/plugin.py index e8c957a6..d8f26dba 100644 --- a/openflexure_microscope/plugins/default/scan/plugin.py +++ b/openflexure_microscope/plugins/default/scan/plugin.py @@ -48,12 +48,12 @@ class ScanPlugin(MicroscopePlugin): api_views = { '/tile': TileScanAPI, - '/stack': ZStackAPI, } def capture(self, basename, scan_id, + temporary: bool = False, use_video_port: bool = False, resize: Tuple[int, int] = None, bayer: bool = False, @@ -67,7 +67,7 @@ class ScanPlugin(MicroscopePlugin): # Create output object output = self.microscope.camera.new_image( write_to_file=True, - temporary=False, + temporary=temporary, filename=filename, folder=foldername) @@ -94,6 +94,7 @@ class ScanPlugin(MicroscopePlugin): def tile( self, basename: str = None, + temporary: bool = False, step_size: int = [2000, 1500, 100], grid: list = [3, 3, 5], style='raster', @@ -159,6 +160,7 @@ class ScanPlugin(MicroscopePlugin): self.capture( basename, scan_id, + temporary=temporary, use_video_port=use_video_port, resize=resize, bayer=bayer, @@ -169,6 +171,7 @@ class ScanPlugin(MicroscopePlugin): logging.debug("Entering z-stack") self.stack( basename=basename, + temporary=temporary, scan_id=scan_id, step_size=step_size[2], steps=grid[2], @@ -186,6 +189,7 @@ class ScanPlugin(MicroscopePlugin): def stack( self, basename: str = None, + temporary: bool = False, scan_id: str = None, step_size: int = 100, steps: int = 5, @@ -224,6 +228,7 @@ class ScanPlugin(MicroscopePlugin): self.capture( basename, scan_id, + temporary=temporary, use_video_port=use_video_port, resize=resize, bayer=bayer, From e281b1ecc32a5a862ddf0542e45533ddc1ed7dfc Mon Sep 17 00:00:00 2001 From: Joel Collins Date: Tue, 9 Apr 2019 15:05:56 +0100 Subject: [PATCH 4/7] Removed a bit of logging vomit --- openflexure_microscope/camera/capture.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/openflexure_microscope/camera/capture.py b/openflexure_microscope/camera/capture.py index fa3ae215..d73dfd6b 100644 --- a/openflexure_microscope/camera/capture.py +++ b/openflexure_microscope/camera/capture.py @@ -233,11 +233,6 @@ class CaptureObject(object): if not os.path.exists(self.filefolder): os.makedirs(self.filefolder) - logging.debug(self.file) - logging.debug(self.filename) - logging.debug(self.basename) - logging.debug(self.filefolder) - def split_file_path(self, filepath): """ Take a full file path, and split it into separated class properties. From a1965f606cfc19bbe79e7e0f72d017965c22584b Mon Sep 17 00:00:00 2001 From: Joel Collins Date: Tue, 9 Apr 2019 15:12:31 +0100 Subject: [PATCH 5/7] Removed redundant z-stack import --- openflexure_microscope/plugins/default/scan/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openflexure_microscope/plugins/default/scan/plugin.py b/openflexure_microscope/plugins/default/scan/plugin.py index d8f26dba..cdbdb37a 100644 --- a/openflexure_microscope/plugins/default/scan/plugin.py +++ b/openflexure_microscope/plugins/default/scan/plugin.py @@ -7,7 +7,7 @@ import logging from openflexure_microscope.camera.base import generate_basename from openflexure_microscope.plugins import MicroscopePlugin -from .api import TileScanAPI, ZStackAPI +from .api import TileScanAPI def construct_grid(initial, step_sizes, n_steps, style='raster'): """ From 7cf54240be1f36e76fb22814a681e248e245b60e Mon Sep 17 00:00:00 2001 From: Joel Collins Date: Thu, 11 Apr 2019 07:13:32 +0100 Subject: [PATCH 6/7] Temp capture folder now handled by camera --- openflexure_microscope/camera/base.py | 18 ++++----- openflexure_microscope/camera/capture.py | 38 ++++++------------- .../plugins/default/scan/plugin.py | 4 +- 3 files changed, 21 insertions(+), 39 deletions(-) diff --git a/openflexure_microscope/camera/base.py b/openflexure_microscope/camera/base.py index c3f237d8..6a643898 100644 --- a/openflexure_microscope/camera/base.py +++ b/openflexure_microscope/camera/base.py @@ -14,7 +14,7 @@ except ImportError: except ImportError: from _thread import get_ident -from .capture import CaptureObject, capture_from_dict, BASE_CAPTURE_PATH +from .capture import CaptureObject, capture_from_dict, BASE_CAPTURE_PATH, TEMP_CAPTURE_PATH from openflexure_microscope.config import USER_CONFIG_DIR from openflexure_microscope.utilities import entry_by_id from openflexure_microscope.lock import StrictLock @@ -111,6 +111,8 @@ class BaseCamera(object): self.paths = { 'image': BASE_CAPTURE_PATH, 'video': BASE_CAPTURE_PATH, + 'image_tmp': TEMP_CAPTURE_PATH, + 'video_tpm': TEMP_CAPTURE_PATH } #: dict: Dictionary of capture paths # Capture data @@ -284,7 +286,7 @@ class BaseCamera(object): write_to_file: bool = False, temporary: bool = True, filename: str = None, - folder: str = None, + folder: str = "", fmt: str = 'jpeg'): """ @@ -303,10 +305,8 @@ class BaseCamera(object): logging.debug(filename) # Generate folder - if folder: - folder = os.path.join(self.paths['image'], folder) - else: - folder = self.paths['image'] + base_folder = self.paths['image_tmp'] if temporary else self.paths['image'] + folder = os.path.join(base_folder, folder) # Create capture object output = CaptureObject( @@ -349,10 +349,8 @@ class BaseCamera(object): logging.debug(filename) # Generate folder - if folder: - folder = os.path.join(self.paths['video'], folder) - else: - folder = self.paths['video'] + base_folder = self.paths['video_tmp'] if temporary else self.paths['video'] + folder = os.path.join(base_folder, folder) # Create capture object output = CaptureObject( diff --git a/openflexure_microscope/camera/capture.py b/openflexure_microscope/camera/capture.py index d73dfd6b..37588cf8 100644 --- a/openflexure_microscope/camera/capture.py +++ b/openflexure_microscope/camera/capture.py @@ -81,8 +81,8 @@ def capture_from_dict(capture_dict): # Get inherent capture information from database capture.file = capture_dict['path'] - capture.split_file_path(capture.file) capture.temporary = capture_dict['temporary'] + capture.split_file_path(capture.file) if capture.format.upper() in EXIF_FORMATS: md_exif = pull_usercomment_dict(capture.file) @@ -137,9 +137,7 @@ class CaptureObject(object): filename = self.id self.filename = "{}.{}".format(filename, fmt) - # Create folder path. Default to BASE_CAPTURE_PATH - if not folder: - folder = BASE_CAPTURE_PATH + # Create folder path self.folder = folder # Dictionary for storing custom metadata @@ -185,7 +183,7 @@ class CaptureObject(object): file on disk. """ - self.build_file_path(self.filename, self.folder) + self.build_file_path() # Byte bytestream properties self.bytestream = io.BytesIO() # Byte bytestream that data will be written to @@ -201,31 +199,23 @@ class CaptureObject(object): # Save initial metadata file self.save_metadata() - def build_file_path( - self, - filename: str, - folder: str): + def build_file_path(self): """ Construct a full file path, based on filename, folder, and file format. Defaults to UUID. - - Args: - filename (str): Filename of capture - folder (str): Directory on disk to store capture file """ global TEMP_CAPTURE_PATH + # TODO: Combine this and split_file_path, and tidy. Let the device (camera) handle folders. This should be more basic. + # TODO: Even let the base camera manage moving captures to temp folder + # This module will clear out the temp folder, but won't MOVE anything there. + # In Base cameras new_image method, the full folder is constructed. Temp folder should be inserted there. - if self.temporary: - # Store original file path - self.file_notmp = os.path.join(folder, filename) - # Move user-specified folder to TEMP - self.folder = os.path.join(TEMP_CAPTURE_PATH, self.folder) - - self.file = os.path.join(folder, filename) # Full file name by joining given folder to given name + self.file = os.path.join(self.folder, self.filename) # Full file name by joining given folder to given name self.split_file_path(self.file) # Split file path into folder, filename, and basename # Check directory is a subdirectory of BASE_CAPTURE_PATH + # TODO: Do we need this? if not os.path.commonprefix([self.file, BASE_CAPTURE_PATH]) == BASE_CAPTURE_PATH: raise Exception("Captures cannot be stored in a lower-level directory than {}.".format(BASE_CAPTURE_PATH)) @@ -448,13 +438,6 @@ class CaptureObject(object): def save_file(self) -> bool: """Write the StreamObjects bytestream to a file.""" - if self.temporary: # If capture is currently temporary - self.load_file() # Load data from tmp file into bytestream, if tmp file exists - self.temporary = False # Flag as kept on disk - self.file = self.file_notmp # Reset file path to non-temporary path - self.split_file_path(self.file) # Set split properties based on new path - logging.info("Moved temporary file out to {}".format(self.file)) - if self.stream_exists: # If there's a bytestream to save with open(self.file, 'ab') as f: # Load file as bytes logging.debug("Writing bytestream to file {}".format(self.file)) @@ -504,6 +487,7 @@ class CaptureObject(object): """Both clear the bytestream, and delete any associated on-disk data.""" logging.info("Closing {}".format(self.id)) self.delete_stream() + # Delete the file from disk if temporary if self.temporary: self.delete() diff --git a/openflexure_microscope/plugins/default/scan/plugin.py b/openflexure_microscope/plugins/default/scan/plugin.py index cdbdb37a..82a2d060 100644 --- a/openflexure_microscope/plugins/default/scan/plugin.py +++ b/openflexure_microscope/plugins/default/scan/plugin.py @@ -62,14 +62,14 @@ class ScanPlugin(MicroscopePlugin): # Construct a tile filename filename = "{}_{}_{}_{}".format(basename, *self.microscope.stage.position) - foldername = "SCAN_{}".format(basename) + folder = "SCAN_{}".format(basename) # Create output object output = self.microscope.camera.new_image( write_to_file=True, temporary=temporary, filename=filename, - folder=foldername) + folder=folder) # Capture self.microscope.camera.capture( From 62b70e5bae3ab1d0c761da3171e5e25f849a46d2 Mon Sep 17 00:00:00 2001 From: Joel Collins Date: Sun, 14 Apr 2019 09:55:51 +0100 Subject: [PATCH 7/7] clear_tmp now properly removes the tmp capture tree --- openflexure_microscope/camera/capture.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/openflexure_microscope/camera/capture.py b/openflexure_microscope/camera/capture.py index 37588cf8..eed2d6e6 100644 --- a/openflexure_microscope/camera/capture.py +++ b/openflexure_microscope/camera/capture.py @@ -1,6 +1,7 @@ import uuid import io import os +import shutil import glob import datetime import yaml @@ -25,10 +26,7 @@ def clear_tmp(): global TEMP_CAPTURE_PATH logging.info("Clearing {}...".format(TEMP_CAPTURE_PATH)) - files = glob.glob('{}/*'.format(TEMP_CAPTURE_PATH)) - for f in files: - os.remove(f) - logging.debug("Removed {}".format(f)) + shutil.rmtree(TEMP_CAPTURE_PATH) logging.debug("Cleared {}.".format(TEMP_CAPTURE_PATH))