diff --git a/openflexure_microscope/api/app.py b/openflexure_microscope/api/app.py index 3ca9f857..a97be602 100644 --- a/openflexure_microscope/api/app.py +++ b/openflexure_microscope/api/app.py @@ -24,7 +24,7 @@ from flask_cors import CORS from openflexure_microscope.api.utilities import list_routes from openflexure_microscope.api.exceptions import JSONExceptionHandler -from openflexure_microscope import Microscope, config +from openflexure_microscope import Microscope from openflexure_microscope.exceptions import LockError from openflexure_microscope.camera.pi import StreamingCamera from openflexure_microscope.stage.openflexure import Stage @@ -35,10 +35,7 @@ import logging, sys logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) # Create a dummy microscope object, with no hardware attachments -openflexurerc = config.load_config() # Load default user config -api_microscope = Microscope(None, None, config=openflexurerc) -logging.debug("Created an empty microscope in global.") - +api_microscope = Microscope(None, None) # Generate API URI based on version from filename def uri(suffix, api_version, base=None): @@ -61,11 +58,11 @@ handler = JSONExceptionHandler(app) @app.before_first_request def attach_microscope(): # Create the microscope object globally (common to all spawned server threads) - global api_microscope, openflexurerc + global api_microscope logging.debug("First request made. Populating microscope with hardware...") logging.debug("Creating camera object...") - api_camera = StreamingCamera(config=openflexurerc) + api_camera = StreamingCamera(config=api_microscope.rc.config) logging.debug("Creating stage object...") api_stage = Stage("/dev/ttyUSB0") @@ -115,7 +112,7 @@ task_blueprint = blueprints.task.construct_blueprint(api_microscope) app.register_blueprint(task_blueprint, url_prefix=uri('/task', 'v1')) # List all routes -list_routes(app) +#list_routes(app) # Automatically clean up microscope at exit diff --git a/openflexure_microscope/camera/base.py b/openflexure_microscope/camera/base.py index 98e738a8..059b4c56 100644 --- a/openflexure_microscope/camera/base.py +++ b/openflexure_microscope/camera/base.py @@ -97,7 +97,7 @@ class BaseCamera(object): self.stream_timeout_enabled = True #: bool: Enable or disable timing out the stream self.state = {} #: dict: Dictionary for capture state - self._config = {} #: dict: Dictionary of base camera settings + self._config = {} #: dict: Dictionary of base camera config self.paths = { 'image': BASE_CAPTURE_PATH, 'video': BASE_CAPTURE_PATH, diff --git a/openflexure_microscope/camera/pi.py b/openflexure_microscope/camera/pi.py index 5458856e..74571538 100644 --- a/openflexure_microscope/camera/pi.py +++ b/openflexure_microscope/camera/pi.py @@ -51,24 +51,28 @@ from .base import BaseCamera, CaptureObject # Richard's fix gain from .set_picamera_gain import set_analog_gain, set_digital_gain -PICAMERA_KEYS = [ - 'exposure_mode', - 'awb_mode', - 'awb_gains', - 'framerate', - 'shutter_speed', - 'saturation', ] - -CONFIG_KEYS = [ - 'video_resolution', - 'image_resolution', - 'numpy_resolution', - 'jpeg_quality', - 'analog_gain', - 'digital_gain', - 'shading_table_path'] +# Handle config and picamera settings +CONFIG_KEYS = { + 'video_resolution': (832, 624), + 'image_resolution': (2592, 1944), + 'numpy_resolution': (1312, 976), + 'jpeg_quality': 75, + 'picamera_settings': { + 'exposure_mode': None, + 'awb_mode': None, + 'awb_gains': None, + 'framerate': None, + 'shutter_speed': None, + 'saturation': None, + 'analog_gain': None, + 'digital_gain': None, + 'lens_shading_table': None, + }, +} +# Useful methods +# TODO: Remove this def fractions_to_floats(value): """Deal with horrible, horrible PiCamera fractions""" result = value @@ -81,6 +85,7 @@ def fractions_to_floats(value): result = float(value) return result +# MAIN CLASS class StreamingCamera(BaseCamera): """Raspberry Pi camera implementation of StreamingCamera. @@ -88,7 +93,9 @@ class StreamingCamera(BaseCamera): Args: config (dict): Dictionary of config parameters to apply on init. If None, will default to basic config. """ - def __init__(self, config: dict=None): + def __init__(self, config: dict=None, picamera_settings: dict=None): + global CONFIG_KEYS + # Run BaseCamera init BaseCamera.__init__(self) # Attach to Pi camera @@ -103,13 +110,8 @@ class StreamingCamera(BaseCamera): # Reset variable states self.set_zoom(1.0) - # Default camera config - self._config.update({ - 'video_resolution': (832, 624), - 'image_resolution': (2592, 1944), - 'numpy_resolution': (1312, 976), - 'jpeg_quality': 75, - }) + # Populate config and settings with all available keys + self._config.update(CONFIG_KEYS) # Load config dictionary if passed if config: @@ -144,44 +146,27 @@ class StreamingCamera(BaseCamera): """ return hasattr(self.camera, "lens_shading_table") - def apply_shading_table(self): - if not self.supports_lens_shading: - logging.error("the currently-installed picamera library does not support all " - "the features of the openflexure microscope software. These features " - "include lens shading control and setting the analog/digital gain.\n" - "\n" - "See the installation instructions for how to fix this:\n" - "https://github.com/rwb27/openflexure_microscope_software") - elif 'shading_table_path' not in self.config: - logging.warning("No shading table path given in config.") - else: - logging.debug("Applying shading table from {}".format(self.config['shading_table_path'])) - npy = np.load(self.config['shading_table_path']) - self.camera.lens_shading_table = npy - @property def config(self): """ Return config dictionary of the StreamingCamera. """ - global PICAMERA_KEYS, CONFIG_KEYS - conf_dict = { - 'picamera_params': {}, + 'picamera_settings': {}, } # PiCamera parameters (obtained directly from PiCamera object) - for key in PICAMERA_KEYS: + for key, _ in self._config['picamera_settings'].items(): try: value = getattr(self.camera, key) value = fractions_to_floats(value) - conf_dict['picamera_params'][key] = value + conf_dict['picamera_settings'][key] = value except AttributeError: logging.warning("Unable to read PiCamera attribute {}".format(key)) # StreamingCamera parameters (obtained from StreamingCamera _config) for key in CONFIG_KEYS: - if key in self._config: + if (key != 'picamera_settings') and (key in self._config): conf_dict[key] = self._config[key] return conf_dict @@ -197,8 +182,7 @@ class StreamingCamera(BaseCamera): Args: config (dict): Dictionary of config parameters. """ - global PICAMERA_KEYS, CONFIG_KEYS - + paused_stream = False with self.lock: @@ -216,27 +200,17 @@ class StreamingCamera(BaseCamera): paused_stream = True # Remember to unpause stream when done # PiCamera parameters (applied directly to PiCamera object) - if 'picamera_params' in config: - for key in PICAMERA_KEYS: - if key in config['picamera_params']: - logging.debug("Setting parameter {}: {}".format(key, config['picamera_params'][key])) - setattr(self.camera, key, config['picamera_params'][key]) + if 'picamera_settings' in config: # If new settings are given + for key, value in config['picamera_settings'].items(): # For each given setting + self._config['picamera_settings'][key] = None # Add the key to the list of returned settings + logging.debug("Setting parameter {}: {}".format(key, config['picamera_settings'][key])) + setattr(self.camera, key, value) # Write setting to camera # StreamingCamera parameters (applied via StreamingCamera config) - for key in CONFIG_KEYS: - if key in config: - logging.debug("Setting parameter {}: {}".format(key, config[key])) - self._config[key] = config[key] - - # Richard's library to set analog and digital gains - if 'analog_gain' in config: - set_analog_gain(self.camera, config['analog_gain']) - if 'digital_gain' in config: - set_digital_gain(self.camera, config['digital_gain']) - - # Handle lens shading, if a new one is given - if 'shading_table_path' in config: - self.apply_shading_table() + for key, value in config.items(): # For each provided setting + if key != 'picamera_settings': # We already handled this + logging.debug("Setting parameter {}: {}".format(key, value)) + self._config[key] = value # If stream was paused to update config, unpause if paused_stream: diff --git a/openflexure_microscope/config.py b/openflexure_microscope/config.py index 00b6de1c..2ed7be05 100644 --- a/openflexure_microscope/config.py +++ b/openflexure_microscope/config.py @@ -1,7 +1,9 @@ import yaml import os +import errno import logging import shutil +import copy HERE = os.path.abspath(os.path.dirname(__file__)) DEFAULT_CONFIG_PATH = os.path.join(HERE, 'microscoperc.default.yaml') @@ -9,100 +11,158 @@ DEFAULT_CONFIG_PATH = os.path.join(HERE, 'microscoperc.default.yaml') USER_CONFIG_DIR = os.path.join(os.path.expanduser("~"), ".openflexure") #: str: Default path of the user-config directory, containing runtime-config and calibration files. Obtained from ``os.path.join(os.path.expanduser("~"), ".openflexure")``. USER_CONFIG_FILE = os.path.join(USER_CONFIG_DIR, "microscoperc.yaml") #: str: Default path of the user microscoperc.yaml runtime-config file. Obtained from ``os.path.join(USER_CONFIG_DIR, "microscoperc.yaml")`` -TYPES = { - 'stream_resolution': tuple, - 'video_resolution': tuple, - 'image_resolution': tuple, - 'numpy_resolution': tuple, - 'jpeg_quality': int, - 'analog_gain': float, - 'digital_gain': float, -} #: dict: Dictionary of expected types for common config parameters. Used by ``convert_config()``. +with open(DEFAULT_CONFIG_PATH, 'r') as defaultrc: + DEFAULT_CONFIG = defaultrc.read() -def convert_config(config: dict) -> dict: - """ - Convert datatype of top-level config parameters based on the global TYPES dictionary. +class OpenflexureConfig(): + expandable_keys = [ + 'picamera_settings', + 'openflexure_stage_settings', + ] #: List of keys that can be passed as a file path string and expanded automatically + + def __init__(self, config_path: str=None, expand: bool=True): + global DEFAULT_CONFIG, USER_CONFIG_FILE + + # Set arguments + self.config_path = config_path or USER_CONFIG_FILE + self.expand = expand + + # Create empty config dictionaries + self.raw_config = {} + # Expanded config dictionary (used by server) + self.config = copy.copy(self.raw_config) + + # Initialise basic config file with defaults if it doesn't exist + self.initialise_file(self.config_path, populate=DEFAULT_CONFIG) + + # Load the config in, setting self._config and self.config + self.load() + + + def update(self, update_dict: dict): + self.config.update(update_dict) + + + def overwrite(self, new_dict: dict): + self.config = new_dict + + + def load(self): + # Unexpanded config dictionary (used at load/save time) + self.raw_config = self.load_yaml_file(self.config_path) + + # If the loaded config is in contracted format + if self.expand: + # Expand self.raw_config into self._config + self.expand_config() + + + def save(self, backup: bool=True): + # If the loaded config was in contracted format + if self.expand: + # Contract self._config into self.raw_config + self.contract_config() - Args: - config (dict): Dictionary representation of a loaded runtime-config. - """ - global TYPES + if backup: + if os.path.isfile(self.config_path): + shutil.copyfile(self.config_path, self.config_path+".bk") - for key in config: - if key in TYPES: - config[key] = TYPES[key](config[key]) + self.save_yaml_file(self.config_path, self.raw_config) + - return config + def expand_config(self): + # For each value in the raw loaded config + for key, value in self.raw_config.items(): + # If it's a valid expandable parameter + if (key in OpenflexureConfig.expandable_keys and + type(value) is str): + + logging.debug("Expanding {}".format(value)) + # Initialise, load and expand + self.initialise_file(value) + self.config[key] = self.load_yaml_file(value) or {} + else: + self.config[key] = value -def load_config(config_path: str=None) -> dict: - """ - Open openflexurerc.yaml runtime-config file and write to the microscope devices. + def contract_config(self): + for key, value in self.config.items(): + # If it's a valid expandable parameter + if (key in OpenflexureConfig.expandable_keys and + type(value) is dict): - Args: - config_path (str): Path to the config YAML file. If `None`, defaults to `DEFAULT_CONFIG_PATH` - """ - global DEFAULT_CONFIG_PATH, USER_CONFIG_FILE - - if not config_path: - if os.path.exists(USER_CONFIG_FILE): # If user config file already exists - config_path = USER_CONFIG_FILE # Load it - else: # If user config file doesn't yet exist - logging.warning("No user config found. Loading system defaults...") - if not os.path.exists(USER_CONFIG_DIR): - logging.info("Making user config directory...") - os.makedirs(USER_CONFIG_DIR) - logging.info("Copying default config to user config...") - shutil.copyfile(DEFAULT_CONFIG_PATH, USER_CONFIG_FILE) - logging.info("Loading user config...") - config_path = USER_CONFIG_FILE # Load defaults in - - with open(config_path) as config_file: - config_data = yaml.load(config_file) - - # Store config dictionary to self - return convert_config(config_data) + # Create the file if it doesn't exist + self.initialise_file(value) + self.save_yaml_file(self.raw_config[key], value) + else: + self.raw_config = value -def save_config(config_dict: dict, config_path: str=None, safe: bool=False): - """ - Save current config dictionary to a YAML file. + def load_yaml_file(self, config_path) -> dict: + """ + Open a .yaml config file - Args: - config_dict (dict): Dictionary of config data to save. - config_path (str): Path to the config YAML file. If `None`, defaults to `DEFAULT_CONFIG_PATH` - """ - global USER_CONFIG_FILE - if not config_path: - config_path = USER_CONFIG_FILE + Args: + config_path (str): Path to the config YAML file. If `None`, defaults to `DEFAULT_CONFIG_PATH` + """ + config_path = os.path.expanduser(config_path) - with open(config_path, 'w') as outfile: - if not safe: - yaml.dump(config_dict, outfile) - else: - yaml.safe_dump(config_dict, outfile) + logging.info("Loading {}...".format(config_path)) + + with open(config_path) as config_file: + config_data = yaml.load(config_file) + + # Return loaded config dictionary + return config_data -def merge_config(config_dict: dict, config_path: str=None, safe: bool=False, backup: bool=True): - """ - merge current config dictionary with an existing YAML file. + def save_yaml_file(self, config_path: str, config_dict: dict, safe: bool=False): + """ + Save a .yaml config file - Args: - config_dict (dict): Dictionary of config data to save. - config_path (str): Path to the config YAML file. If `None`, defaults to `DEFAULT_CONFIG_PATH` - """ - global USER_CONFIG_FILE - if not config_path: - config_path = USER_CONFIG_FILE + Args: + config_dict (dict): Dictionary of config data to save. + config_path (str): Path to the config YAML file. + """ + config_path = os.path.expanduser(config_path) - config_data = load_config(config_path=config_path) + logging.info("Saving {}...".format(config_path)) - for key, value in config_dict.items(): - config_data[key] = value + with open(config_path, 'w') as outfile: + if not safe: + yaml.dump(config_dict, outfile) + else: + yaml.safe_dump(config_dict, outfile) - if backup: - if os.path.isfile(config_path): - shutil.copyfile(config_path, config_path+".bk") - save_config(config_data, config_path=config_path, safe=safe) + def initialise_file(self, config_path, populate: str=""): + """ + Check if a file exists, and if not, create it + and optionally populate it with content + + Args: + config_path (str): Path to the file. + populate (str): String to dump to the file, if it is being newly created + """ + config_path = os.path.expanduser(config_path) + + logging.debug("Initialising {}".format(config_path)) + logging.debug("Exists: {}".format(os.path.exists(config_path))) + + if not os.path.exists(config_path): # If user config file doesn't exist + logging.warning("No config file found at {}. Creating...".format(config_path)) + self.create_file(config_path) + + logging.info("Populating {}...".format(config_path)) + with open(config_path, 'w') as outfile: + outfile.write(populate) + + + def create_file(self, config_path): + if not os.path.exists(os.path.dirname(config_path)): + try: + os.makedirs(os.path.dirname(config_path)) + except OSError as exc: # Guard against race condition + if exc.errno != errno.EEXIST: + raise diff --git a/openflexure_microscope/microscope.py b/openflexure_microscope/microscope.py index ea940336..1a75b241 100644 --- a/openflexure_microscope/microscope.py +++ b/openflexure_microscope/microscope.py @@ -11,10 +11,10 @@ from openflexure_stage import OpenFlexureStage from .camera.pi import StreamingCamera from .plugins import PluginMount -from .config import load_config, save_config, convert_config from .utilities import axes_to_array from .task import TaskOrchestrator from .lock import CompositeLock +from .config import OpenflexureConfig class Microscope(object): @@ -29,14 +29,14 @@ class Microscope(object): config (dict): Dictionary of the runtime config to apply to the microscope. Does not automatically apply to camera and stage. attach_plugins (bool): Should the microscope attach plugins listen in 'config' immediately. """ - def __init__(self, camera: StreamingCamera, stage: OpenFlexureStage, config: dict={}, attach_plugins: bool=True): + def __init__(self, camera: StreamingCamera, stage: OpenFlexureStage, config_path: str=None, attach_plugins: bool=True): + + # Create RC object + self.rc = OpenflexureConfig(config_path=config_path, expand=True) # Attach initial hardware (may be NoneTypes) self.attach(camera, stage) - # Store entire runtime-config - self._config = config - # Create a task orchestrator self.task = TaskOrchestrator() #: :py:class:`openflexure_microscope.task.TaskOrchestrator`: Threaded task orchestrator @@ -48,11 +48,11 @@ class Microscope(object): self.attach_plugins() # Set default parameters - if not 'name' in self._config: - self._config['name'] = uuid.uuid4().hex + if not 'name' in self.rc.config: + self.rc.config['name'] = uuid.uuid4().hex - if not 'fov' in self._config: - self._config['fov'] = [0, 0] # Assumes pi camera 2, and 40x objective + if not 'fov' in self.rc.config: + self.rc.config['fov'] = [0, 0] # Initialise with an empty composite lock self.lock = CompositeLock([]) #: :py:class:`openflexure_microscope.lock.CompositeLock`: Composite lock controlling thread access to multiple pieces of hardware @@ -73,7 +73,7 @@ class Microscope(object): if self.stage: self.stage.close() - def attach(self, camera: StreamingCamera, stage: OpenFlexureStage, apply_config: bool=True): + def attach(self, camera: StreamingCamera, stage: OpenFlexureStage): """ Retroactively attaches a camera and stage to the microscope object. @@ -128,8 +128,8 @@ class Microscope(object): """ Automatically search for plugin maps in self.config, and attach. """ - if 'plugins' in self._config: - self.attach_plugin_maps(self._config['plugins']) + if 'plugins' in self.rc.config: + self.attach_plugin_maps(self.rc.config['plugins']) else: logging.warning("No plugins specified in microscoperc.yaml. Skipping.") @@ -178,7 +178,7 @@ class Microscope(object): # If attached to a camera if self.camera: # Update camera config params from StreamingCamera object - self._config.update(self.camera.config) + self.rc.update(self.camera.config) # If attached to a stage if self.stage: @@ -186,13 +186,17 @@ class Microscope(object): else: backlash = [0, 0, 0] - self._config['backlash'] = { - 'x': backlash[0], - 'y': backlash[1], - 'z': backlash[2], + backlash = { + 'backlash': { + 'x': backlash[0], + 'y': backlash[1], + 'z': backlash[2], + } } - return self._config + self.rc.update(backlash) + + return self.rc.config @config.setter def config(self, config: dict) -> None: @@ -209,4 +213,6 @@ class Microscope(object): self.stage.backlash = backlash # Cache config - self._config.update(config) \ No newline at end of file + self.rc.update(config) + # Save to disk + self.rc.save() \ No newline at end of file diff --git a/openflexure_microscope/microscoperc.default.yaml b/openflexure_microscope/microscoperc.default.yaml index 4a553c29..44fd3a59 100644 --- a/openflexure_microscope/microscoperc.default.yaml +++ b/openflexure_microscope/microscoperc.default.yaml @@ -1,7 +1,3 @@ -# The analog gain offers higher sensitivity and less noise than using digital gain only -analog_gain: 1. -digital_gain: 1. - # Resolutions for streaming and capture video_resolution: [832, 624] image_resolution: [2592, 1944] @@ -14,14 +10,7 @@ fov: [4100, 3146] jpeg_quality: 75 # Parameters specific to PiCamera objects -picamera_params: - exposure_mode: 'off' - awb_mode: 'off' - awb_gains: [0.9, 2.8] - framerate: 24 - shutter_speed: 30000 - saturation: 0 - led: false +picamera_settings: ~/.openflexure/picamera_settings.yaml # Default plugins plugins: