Overhauled config loader to allow split configs

openflexure_microscope/api/app.py

@@ -24,7 +24,7 @@ from flask_cors import CORS
 from openflexure_microscope.api.utilities import list_routes
 from openflexure_microscope.api.exceptions import JSONExceptionHandler
 
-from openflexure_microscope import Microscope, config
+from openflexure_microscope import Microscope
 from openflexure_microscope.exceptions import LockError
 from openflexure_microscope.camera.pi import StreamingCamera
 from openflexure_microscope.stage.openflexure import Stage
@@ -35,10 +35,7 @@ import logging, sys
 logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
 
 # Create a dummy microscope object, with no hardware attachments
-openflexurerc = config.load_config()  # Load default user config
+api_microscope = Microscope(None, None)
-api_microscope = Microscope(None, None, config=openflexurerc)
-logging.debug("Created an empty microscope in global.")
-
 
 # Generate API URI based on version from filename
 def uri(suffix, api_version, base=None):
@@ -61,11 +58,11 @@ handler = JSONExceptionHandler(app)
 @app.before_first_request
 def attach_microscope():
     # Create the microscope object globally (common to all spawned server threads)
-    global api_microscope, openflexurerc
+    global api_microscope
     logging.debug("First request made. Populating microscope with hardware...")
 
     logging.debug("Creating camera object...")
-    api_camera = StreamingCamera(config=openflexurerc)
+    api_camera = StreamingCamera(config=api_microscope.rc.config)
     
     logging.debug("Creating stage object...")
     api_stage = Stage("/dev/ttyUSB0")
@@ -115,7 +112,7 @@ task_blueprint = blueprints.task.construct_blueprint(api_microscope)
 app.register_blueprint(task_blueprint, url_prefix=uri('/task', 'v1'))
 
 # List all routes
-list_routes(app)
+#list_routes(app)
 
 
 # Automatically clean up microscope at exit

openflexure_microscope/camera/base.py

@@ -97,7 +97,7 @@ class BaseCamera(object):
         self.stream_timeout_enabled = True  #: bool: Enable or disable timing out the stream
 
         self.state = {}  #: dict: Dictionary for capture state
-        self._config = {}  #: dict: Dictionary of base camera settings
+        self._config = {}  #: dict: Dictionary of base camera config
         self.paths = {
             'image': BASE_CAPTURE_PATH,
             'video': BASE_CAPTURE_PATH,

openflexure_microscope/camera/pi.py

@@ -51,24 +51,28 @@ from .base import BaseCamera, CaptureObject
 # Richard's fix gain
 from .set_picamera_gain import set_analog_gain, set_digital_gain
 
-PICAMERA_KEYS = [
+# Handle config and picamera settings
-    'exposure_mode',
+CONFIG_KEYS = {
-    'awb_mode',
+    'video_resolution': (832, 624),
-    'awb_gains',
+    'image_resolution': (2592, 1944),
-    'framerate',
+    'numpy_resolution': (1312, 976),
-    'shutter_speed',
+    'jpeg_quality': 75,
-    'saturation', ]
+    'picamera_settings': {
-
+        'exposure_mode': None,
-CONFIG_KEYS = [
+        'awb_mode': None,
-    'video_resolution',
+        'awb_gains': None,
-    'image_resolution',
+        'framerate': None,
-    'numpy_resolution',
+        'shutter_speed': None,
-    'jpeg_quality',
+        'saturation': None,
-    'analog_gain',
+        'analog_gain': None,
-    'digital_gain',
+        'digital_gain': None,
-    'shading_table_path']
+        'lens_shading_table': None,
+    },
+}
 
+# Useful methods
 
+# TODO: Remove this
 def fractions_to_floats(value):
     """Deal with horrible, horrible PiCamera fractions"""
     result = value
@@ -81,6 +85,7 @@ def fractions_to_floats(value):
             result = float(value)
     return result
 
+# MAIN CLASS
 
 class StreamingCamera(BaseCamera):
     """Raspberry Pi camera implementation of StreamingCamera.
@@ -88,7 +93,9 @@ class StreamingCamera(BaseCamera):
     Args:
         config (dict): Dictionary of config parameters to apply on init. If None, will default to basic config.
     """
-    def __init__(self, config: dict=None):
+    def __init__(self, config: dict=None, picamera_settings: dict=None):
+        global CONFIG_KEYS
+
         # Run BaseCamera init
         BaseCamera.__init__(self)
         # Attach to Pi camera
@@ -103,13 +110,8 @@ class StreamingCamera(BaseCamera):
         # Reset variable states
         self.set_zoom(1.0)
 
-        # Default camera config
+        # Populate config and settings with all available keys
-        self._config.update({
+        self._config.update(CONFIG_KEYS)
-            'video_resolution': (832, 624),
-            'image_resolution': (2592, 1944),
-            'numpy_resolution': (1312, 976),
-            'jpeg_quality': 75,
-        })
 
         # Load config dictionary if passed
         if config:
@@ -144,44 +146,27 @@ class StreamingCamera(BaseCamera):
         """
         return hasattr(self.camera, "lens_shading_table")
 
-    def apply_shading_table(self):
-        if not self.supports_lens_shading:
-            logging.error("the currently-installed picamera library does not support all "
-            "the features of the openflexure microscope software.  These features "
-            "include lens shading control and setting the analog/digital gain.\n"
-            "\n"
-            "See the installation instructions for how to fix this:\n"
-            "https://github.com/rwb27/openflexure_microscope_software")
-        elif 'shading_table_path' not in self.config:
-            logging.warning("No shading table path given in config.")
-        else:
-            logging.debug("Applying shading table from {}".format(self.config['shading_table_path']))
-            npy = np.load(self.config['shading_table_path'])
-            self.camera.lens_shading_table = npy
-
     @property
     def config(self):
         """
         Return config dictionary of the StreamingCamera.
         """
-        global PICAMERA_KEYS, CONFIG_KEYS
-
         conf_dict = {
-            'picamera_params': {},
+            'picamera_settings': {},
         }
 
         # PiCamera parameters (obtained directly from PiCamera object)
-        for key in PICAMERA_KEYS:
+        for key, _ in self._config['picamera_settings'].items():
             try:
                 value = getattr(self.camera, key)
                 value = fractions_to_floats(value)
-                conf_dict['picamera_params'][key] = value
+                conf_dict['picamera_settings'][key] = value
             except AttributeError:
                 logging.warning("Unable to read PiCamera attribute {}".format(key))
 
         # StreamingCamera parameters (obtained from StreamingCamera _config)
         for key in CONFIG_KEYS:
-            if key in self._config:
+            if (key != 'picamera_settings') and (key in self._config):
                 conf_dict[key] = self._config[key]
 
         return conf_dict
@@ -197,8 +182,7 @@ class StreamingCamera(BaseCamera):
         Args:
             config (dict): Dictionary of config parameters.
         """
-        global PICAMERA_KEYS, CONFIG_KEYS
+        
-
         paused_stream = False
 
         with self.lock:
@@ -216,27 +200,17 @@ class StreamingCamera(BaseCamera):
                     paused_stream = True  # Remember to unpause stream when done
 
                 # PiCamera parameters (applied directly to PiCamera object)
-                if 'picamera_params' in config:
+                if 'picamera_settings' in config:  # If new settings are given
-                    for key in PICAMERA_KEYS:
+                    for key, value in config['picamera_settings'].items():  # For each given setting
-                        if key in config['picamera_params']:
+                        self._config['picamera_settings'][key] = None  # Add the key to the list of returned settings
-                            logging.debug("Setting parameter {}: {}".format(key, config['picamera_params'][key]))
+                        logging.debug("Setting parameter {}: {}".format(key, config['picamera_settings'][key]))
-                            setattr(self.camera, key, config['picamera_params'][key])
+                        setattr(self.camera, key, value)  # Write setting to camera
 
                 # StreamingCamera parameters (applied via StreamingCamera config)
-                for key in CONFIG_KEYS:
+                for key, value in config.items():  # For each provided setting
-                    if key in config:
+                    if key != 'picamera_settings':  # We already handled this
-                        logging.debug("Setting parameter {}: {}".format(key, config[key]))
+                        logging.debug("Setting parameter {}: {}".format(key, value))
-                        self._config[key] = config[key]
+                        self._config[key] = value
-
-                # Richard's library to set analog and digital gains
-                if 'analog_gain' in config:
-                    set_analog_gain(self.camera, config['analog_gain'])
-                if 'digital_gain' in config:
-                    set_digital_gain(self.camera, config['digital_gain'])
-
-                # Handle lens shading, if a new one is given
-                if 'shading_table_path' in config:
-                    self.apply_shading_table()
 
                 # If stream was paused to update config, unpause
                 if paused_stream:

openflexure_microscope/config.py

@@ -1,7 +1,9 @@
 import yaml
 import os
+import errno
 import logging
 import shutil
+import copy
 
 HERE = os.path.abspath(os.path.dirname(__file__))
 DEFAULT_CONFIG_PATH = os.path.join(HERE, 'microscoperc.default.yaml')
@@ -9,100 +11,158 @@ DEFAULT_CONFIG_PATH = os.path.join(HERE, 'microscoperc.default.yaml')
 USER_CONFIG_DIR = os.path.join(os.path.expanduser("~"), ".openflexure")  #: str: Default path of the user-config directory, containing runtime-config and calibration files. Obtained from ``os.path.join(os.path.expanduser("~"), ".openflexure")``.
 USER_CONFIG_FILE = os.path.join(USER_CONFIG_DIR, "microscoperc.yaml")  #: str: Default path of the user microscoperc.yaml runtime-config file. Obtained from ``os.path.join(USER_CONFIG_DIR, "microscoperc.yaml")``
 
-TYPES = {
+with open(DEFAULT_CONFIG_PATH, 'r') as defaultrc:
-    'stream_resolution': tuple,
+    DEFAULT_CONFIG = defaultrc.read()
-    'video_resolution': tuple,
-    'image_resolution': tuple,
-    'numpy_resolution': tuple,
-    'jpeg_quality': int,
-    'analog_gain': float,
-    'digital_gain': float,
-}  #: dict: Dictionary of expected types for common config parameters. Used by ``convert_config()``.
 
 
-def convert_config(config: dict) -> dict:
+class OpenflexureConfig():
-    """
+    expandable_keys = [
-    Convert datatype of top-level config parameters based on the global TYPES dictionary.
+        'picamera_settings',
+        'openflexure_stage_settings',
+    ]  #: List of keys that can be passed as a file path string and expanded automatically
+
+    def __init__(self, config_path: str=None, expand: bool=True):
+        global DEFAULT_CONFIG, USER_CONFIG_FILE
+
+        # Set arguments
+        self.config_path = config_path or USER_CONFIG_FILE
+        self.expand = expand
+
+        # Create empty config dictionaries
+        self.raw_config = {}
+        # Expanded config dictionary (used by server)
+        self.config = copy.copy(self.raw_config)
+
+        # Initialise basic config file with defaults if it doesn't exist
+        self.initialise_file(self.config_path, populate=DEFAULT_CONFIG)
+
+        # Load the config in, setting self._config and self.config
+        self.load()
+
+
+    def update(self, update_dict: dict):
+        self.config.update(update_dict)
+
+
+    def overwrite(self, new_dict: dict):
+        self.config = new_dict
+
+
+    def load(self):
+        # Unexpanded config dictionary (used at load/save time)
+        self.raw_config = self.load_yaml_file(self.config_path)
+
+        # If the loaded config is in contracted format
+        if self.expand:
+            # Expand self.raw_config into self._config
+            self.expand_config()
+
+
+    def save(self, backup: bool=True):
+        # If the loaded config was in contracted format
+        if self.expand:
+            # Contract self._config into self.raw_config
+            self.contract_config()
     
-    Args:
+        if backup:
-        config (dict): Dictionary representation of a loaded runtime-config.
+            if os.path.isfile(self.config_path):
-    """
+                shutil.copyfile(self.config_path, self.config_path+".bk")
-    global TYPES
 
-    for key in config:
+        self.save_yaml_file(self.config_path, self.raw_config)
-        if key in TYPES:
+        
-            config[key] = TYPES[key](config[key])
 
-    return config
+    def expand_config(self):
+        # For each value in the raw loaded config
+        for key, value in self.raw_config.items():
+            # If it's a valid expandable parameter
+            if (key in OpenflexureConfig.expandable_keys and 
+                type(value) is str):
+
+                logging.debug("Expanding {}".format(value))
+                # Initialise, load and expand
+                self.initialise_file(value)
+                self.config[key] = self.load_yaml_file(value) or {}
+            else:
+                self.config[key] = value
 
 
-def load_config(config_path: str=None) -> dict:
+    def contract_config(self):
-    """
+        for key, value in self.config.items():
-    Open openflexurerc.yaml runtime-config file and write to the microscope devices.
+            # If it's a valid expandable parameter
+            if (key in OpenflexureConfig.expandable_keys and 
+                type(value) is dict):
 
-    Args:
+                # Create the file if it doesn't exist
-        config_path (str): Path to the config YAML file. If `None`, defaults to `DEFAULT_CONFIG_PATH`
+                self.initialise_file(value)
-    """
+                self.save_yaml_file(self.raw_config[key], value)
-    global DEFAULT_CONFIG_PATH, USER_CONFIG_FILE
+            else:
-
+                self.raw_config = value
-    if not config_path:
-        if os.path.exists(USER_CONFIG_FILE):  # If user config file already exists
-            config_path = USER_CONFIG_FILE  # Load it
-        else:  # If user config file doesn't yet exist
-            logging.warning("No user config found. Loading system defaults...")
-            if not os.path.exists(USER_CONFIG_DIR):
-                logging.info("Making user config directory...")
-                os.makedirs(USER_CONFIG_DIR)
-            logging.info("Copying default config to user config...")
-            shutil.copyfile(DEFAULT_CONFIG_PATH, USER_CONFIG_FILE)
-            logging.info("Loading user config...")
-            config_path = USER_CONFIG_FILE  # Load defaults in
-
-    with open(config_path) as config_file:
-        config_data = yaml.load(config_file)
-
-    # Store config dictionary to self
-    return convert_config(config_data)
 
 
-def save_config(config_dict: dict, config_path: str=None, safe: bool=False):
+    def load_yaml_file(self, config_path) -> dict:
-    """
+        """
-    Save current config dictionary to a YAML file.
+        Open a .yaml config file
 
-    Args:
+        Args:
-        config_dict (dict): Dictionary of config data to save.
+            config_path (str): Path to the config YAML file. If `None`, defaults to `DEFAULT_CONFIG_PATH`
-        config_path (str): Path to the config YAML file. If `None`, defaults to `DEFAULT_CONFIG_PATH`
+        """
-    """
+        config_path = os.path.expanduser(config_path)
-    global USER_CONFIG_FILE
-    if not config_path:
-        config_path = USER_CONFIG_FILE
 
-    with open(config_path, 'w') as outfile:
+        logging.info("Loading {}...".format(config_path))
-        if not safe:
+
-            yaml.dump(config_dict, outfile)
+        with open(config_path) as config_file:
-        else:
+            config_data = yaml.load(config_file)
-            yaml.safe_dump(config_dict, outfile)
+
+        # Return loaded config dictionary
+        return config_data
 
 
-def merge_config(config_dict: dict, config_path: str=None, safe: bool=False, backup: bool=True):
+    def save_yaml_file(self, config_path: str, config_dict: dict, safe: bool=False):
-    """
+        """
-    merge current config dictionary with an existing YAML file.
+        Save a .yaml config file
 
-    Args:
+        Args:
-        config_dict (dict): Dictionary of config data to save.
+            config_dict (dict): Dictionary of config data to save.
-        config_path (str): Path to the config YAML file. If `None`, defaults to `DEFAULT_CONFIG_PATH`
+            config_path (str): Path to the config YAML file. 
-    """
+        """
-    global USER_CONFIG_FILE
+        config_path = os.path.expanduser(config_path)
-    if not config_path:
-        config_path = USER_CONFIG_FILE
 
-    config_data = load_config(config_path=config_path)
+        logging.info("Saving {}...".format(config_path))
 
-    for key, value in config_dict.items():
+        with open(config_path, 'w') as outfile:
-        config_data[key] = value
+            if not safe:
+                yaml.dump(config_dict, outfile)
+            else:
+                yaml.safe_dump(config_dict, outfile)
 
-    if backup:
-        if os.path.isfile(config_path):
-            shutil.copyfile(config_path, config_path+".bk")
 
-    save_config(config_data, config_path=config_path, safe=safe)
+    def initialise_file(self, config_path, populate: str=""):
+        """
+        Check if a file exists, and if not, create it
+        and optionally populate it with content
+
+        Args:
+            config_path (str): Path to the file. 
+            populate (str): String to dump to the file, if it is being newly created
+        """
+        config_path = os.path.expanduser(config_path)
+
+        logging.debug("Initialising {}".format(config_path))
+        logging.debug("Exists: {}".format(os.path.exists(config_path)))
+
+        if not os.path.exists(config_path):  # If user config file doesn't exist
+            logging.warning("No config file found at {}. Creating...".format(config_path))
+            self.create_file(config_path)
+        
+            logging.info("Populating {}...".format(config_path))
+            with open(config_path, 'w') as outfile:
+                outfile.write(populate)
+
+
+    def create_file(self, config_path):
+        if not os.path.exists(os.path.dirname(config_path)):
+            try:
+                os.makedirs(os.path.dirname(config_path))
+            except OSError as exc: # Guard against race condition
+                if exc.errno != errno.EEXIST:
+                    raise

openflexure_microscope/microscope.py

@@ -11,10 +11,10 @@ from openflexure_stage import OpenFlexureStage
 from .camera.pi import StreamingCamera
 
 from .plugins import PluginMount
-from .config import load_config, save_config, convert_config
 from .utilities import axes_to_array
 from .task import TaskOrchestrator
 from .lock import CompositeLock
+from .config import OpenflexureConfig
 
 
 class Microscope(object):
@@ -29,14 +29,14 @@ class Microscope(object):
         config (dict): Dictionary of the runtime config to apply to the microscope. Does not automatically apply to camera and stage.
         attach_plugins (bool): Should the microscope attach plugins listen in 'config' immediately.
     """
-    def __init__(self, camera: StreamingCamera, stage: OpenFlexureStage, config: dict={}, attach_plugins: bool=True):
+    def __init__(self, camera: StreamingCamera, stage: OpenFlexureStage, config_path: str=None, attach_plugins: bool=True):
+
+        # Create RC object
+        self.rc = OpenflexureConfig(config_path=config_path, expand=True)
 
         # Attach initial hardware (may be NoneTypes)
         self.attach(camera, stage)
 
-        # Store entire runtime-config
-        self._config = config
-
         # Create a task orchestrator
         self.task = TaskOrchestrator()  #: :py:class:`openflexure_microscope.task.TaskOrchestrator`: Threaded task orchestrator
 
@@ -48,11 +48,11 @@ class Microscope(object):
             self.attach_plugins()
         
         # Set default parameters
-        if not 'name' in self._config:
+        if not 'name' in self.rc.config:
-            self._config['name'] = uuid.uuid4().hex
+            self.rc.config['name'] = uuid.uuid4().hex
         
-        if not 'fov' in self._config:
+        if not 'fov' in self.rc.config:
-            self._config['fov'] = [0, 0]  # Assumes pi camera 2, and 40x objective
+            self.rc.config['fov'] = [0, 0]
         
         # Initialise with an empty composite lock
         self.lock = CompositeLock([])  #: :py:class:`openflexure_microscope.lock.CompositeLock`: Composite lock controlling thread access to multiple pieces of hardware
@@ -73,7 +73,7 @@ class Microscope(object):
         if self.stage:
             self.stage.close()
 
-    def attach(self, camera: StreamingCamera, stage: OpenFlexureStage, apply_config: bool=True):
+    def attach(self, camera: StreamingCamera, stage: OpenFlexureStage):
         """
         Retroactively attaches a camera and stage to the microscope object.
 
@@ -128,8 +128,8 @@ class Microscope(object):
         """
         Automatically search for plugin maps in self.config, and attach.
         """
-        if 'plugins' in self._config:
+        if 'plugins' in self.rc.config:
-            self.attach_plugin_maps(self._config['plugins'])
+            self.attach_plugin_maps(self.rc.config['plugins'])
         else:
             logging.warning("No plugins specified in microscoperc.yaml. Skipping.")
 
@@ -178,7 +178,7 @@ class Microscope(object):
         # If attached to a camera
         if self.camera:
             # Update camera config params from StreamingCamera object
-            self._config.update(self.camera.config)
+            self.rc.update(self.camera.config)
         
         # If attached to a stage
         if self.stage:
@@ -186,13 +186,17 @@ class Microscope(object):
         else:
             backlash = [0, 0, 0]
 
-        self._config['backlash'] = {
+        backlash = {
-            'x': backlash[0],
+            'backlash': {
-            'y': backlash[1],
+                'x': backlash[0],
-            'z': backlash[2],
+                'y': backlash[1],
+                'z': backlash[2],
+            }
         }
 
-        return self._config
+        self.rc.update(backlash)
+
+        return self.rc.config
 
     @config.setter
     def config(self, config: dict) -> None:
@@ -209,4 +213,6 @@ class Microscope(object):
                 self.stage.backlash = backlash
 
         # Cache config
-        self._config.update(config)
+        self.rc.update(config)
+        # Save to disk
+        self.rc.save()

openflexure_microscope/microscoperc.default.yaml

@@ -1,7 +1,3 @@
-# The analog gain offers higher sensitivity and less noise than using digital gain only
-analog_gain: 1.
-digital_gain: 1.
-
 # Resolutions for streaming and capture
 video_resolution: [832, 624]
 image_resolution: [2592, 1944]
@@ -14,14 +10,7 @@ fov: [4100, 3146]
 jpeg_quality: 75
 
 # Parameters specific to PiCamera objects
-picamera_params:
+picamera_settings: ~/.openflexure/picamera_settings.yaml
-  exposure_mode: 'off'
-  awb_mode: 'off'
-  awb_gains: [0.9, 2.8]
-  framerate: 24
-  shutter_speed: 30000
-  saturation: 0
-  led: false
 
 # Default plugins
 plugins: