rahix/openflexure-microscope-server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Overhauled config loader to allow split configs

Browse source
This commit is contained in:
Joel Collins 2019-01-30 17:45:11 +00:00
parent f720e67700
commit f2a3261b03
6 changed files with 210 additions and 184 deletions
Download patch file Download diff file Expand all files Collapse all files

13
openflexure_microscope/api/app.py
View file

@ -24,7 +24,7 @@ from flask_cors import CORS
from openflexure_microscope.api.utilities import list_routes
from openflexure_microscope.api.exceptions import JSONExceptionHandler
from openflexure_microscope import Microscope, config
from openflexure_microscope import Microscope
from openflexure_microscope.exceptions import LockError
from openflexure_microscope.camera.pi import StreamingCamera
from openflexure_microscope.stage.openflexure import Stage
@ -35,10 +35,7 @@ import logging, sys
logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
# Create a dummy microscope object, with no hardware attachments
openflexurerc = config.load_config() # Load default user config
api_microscope = Microscope(None, None, config=openflexurerc)
logging.debug("Created an empty microscope in global.")
api_microscope = Microscope(None, None)
# Generate API URI based on version from filename
def uri(suffix, api_version, base=None):
@ -61,11 +58,11 @@ handler = JSONExceptionHandler(app)
@app.before_first_request
def attach_microscope():
# Create the microscope object globally (common to all spawned server threads)
global api_microscope, openflexurerc
global api_microscope
logging.debug("First request made. Populating microscope with hardware...")
logging.debug("Creating camera object...")
api_camera = StreamingCamera(config=openflexurerc)
api_camera = StreamingCamera(config=api_microscope.rc.config)
logging.debug("Creating stage object...")
api_stage = Stage("/dev/ttyUSB0")
@ -115,7 +112,7 @@ task_blueprint = blueprints.task.construct_blueprint(api_microscope)
app.register_blueprint(task_blueprint, url_prefix=uri('/task', 'v1'))
# List all routes
list_routes(app)
#list_routes(app)
# Automatically clean up microscope at exit

2
openflexure_microscope/camera/base.py
View file

@ -97,7 +97,7 @@ class BaseCamera(object):
self.stream_timeout_enabled = True #: bool: Enable or disable timing out the stream
self.state = {} #: dict: Dictionary for capture state
self._config = {} #: dict: Dictionary of base camera settings
self._config = {} #: dict: Dictionary of base camera config
self.paths = {
'image': BASE_CAPTURE_PATH,
'video': BASE_CAPTURE_PATH,

106
openflexure_microscope/camera/pi.py
View file

@ -51,24 +51,28 @@ from .base import BaseCamera, CaptureObject
# Richard's fix gain
from .set_picamera_gain import set_analog_gain, set_digital_gain
PICAMERA_KEYS = [
'exposure_mode',
'awb_mode',
'awb_gains',
'framerate',
'shutter_speed',
'saturation', ]
CONFIG_KEYS = [
'video_resolution',
'image_resolution',
'numpy_resolution',
'jpeg_quality',
'analog_gain',
'digital_gain',
'shading_table_path']
# Handle config and picamera settings
CONFIG_KEYS = {
'video_resolution': (832, 624),
'image_resolution': (2592, 1944),
'numpy_resolution': (1312, 976),
'jpeg_quality': 75,
'picamera_settings': {
'exposure_mode': None,
'awb_mode': None,
'awb_gains': None,
'framerate': None,
'shutter_speed': None,
'saturation': None,
'analog_gain': None,
'digital_gain': None,
'lens_shading_table': None,
},
}
# Useful methods
# TODO: Remove this
def fractions_to_floats(value):
"""Deal with horrible, horrible PiCamera fractions"""
result = value
@ -81,6 +85,7 @@ def fractions_to_floats(value):
result = float(value)
return result
# MAIN CLASS
class StreamingCamera(BaseCamera):
"""Raspberry Pi camera implementation of StreamingCamera.
@ -88,7 +93,9 @@ class StreamingCamera(BaseCamera):
Args:
config (dict): Dictionary of config parameters to apply on init. If None, will default to basic config.
"""
def __init__(self, config: dict=None):
def __init__(self, config: dict=None, picamera_settings: dict=None):
global CONFIG_KEYS
# Run BaseCamera init
BaseCamera.__init__(self)
# Attach to Pi camera
@ -103,13 +110,8 @@ class StreamingCamera(BaseCamera):
# Reset variable states
self.set_zoom(1.0)
# Default camera config
self._config.update({
'video_resolution': (832, 624),
'image_resolution': (2592, 1944),
'numpy_resolution': (1312, 976),
'jpeg_quality': 75,
})
# Populate config and settings with all available keys
self._config.update(CONFIG_KEYS)
# Load config dictionary if passed
if config:
@ -144,44 +146,27 @@ class StreamingCamera(BaseCamera):
"""
return hasattr(self.camera, "lens_shading_table")
def apply_shading_table(self):
if not self.supports_lens_shading:
logging.error("the currently-installed picamera library does not support all "
"the features of the openflexure microscope software. These features "
"include lens shading control and setting the analog/digital gain.\n"
"\n"
"See the installation instructions for how to fix this:\n"
"https://github.com/rwb27/openflexure_microscope_software")
elif 'shading_table_path' not in self.config:
logging.warning("No shading table path given in config.")
else:
logging.debug("Applying shading table from {}".format(self.config['shading_table_path']))
npy = np.load(self.config['shading_table_path'])
self.camera.lens_shading_table = npy
@property
def config(self):
"""
Return config dictionary of the StreamingCamera.
"""
global PICAMERA_KEYS, CONFIG_KEYS
conf_dict = {
'picamera_params': {},
'picamera_settings': {},
}
# PiCamera parameters (obtained directly from PiCamera object)
for key in PICAMERA_KEYS:
for key, _ in self._config['picamera_settings'].items():
try:
value = getattr(self.camera, key)
value = fractions_to_floats(value)
conf_dict['picamera_params'][key] = value
conf_dict['picamera_settings'][key] = value
except AttributeError:
logging.warning("Unable to read PiCamera attribute {}".format(key))
# StreamingCamera parameters (obtained from StreamingCamera _config)
for key in CONFIG_KEYS:
if key in self._config:
if (key != 'picamera_settings') and (key in self._config):
conf_dict[key] = self._config[key]
return conf_dict
@ -197,8 +182,7 @@ class StreamingCamera(BaseCamera):
Args:
config (dict): Dictionary of config parameters.
"""
global PICAMERA_KEYS, CONFIG_KEYS
paused_stream = False
with self.lock:
@ -216,27 +200,17 @@ class StreamingCamera(BaseCamera):
paused_stream = True # Remember to unpause stream when done
# PiCamera parameters (applied directly to PiCamera object)
if 'picamera_params' in config:
for key in PICAMERA_KEYS:
if key in config['picamera_params']:
logging.debug("Setting parameter {}: {}".format(key, config['picamera_params'][key]))
setattr(self.camera, key, config['picamera_params'][key])
if 'picamera_settings' in config: # If new settings are given
for key, value in config['picamera_settings'].items(): # For each given setting
self._config['picamera_settings'][key] = None # Add the key to the list of returned settings
logging.debug("Setting parameter {}: {}".format(key, config['picamera_settings'][key]))
setattr(self.camera, key, value) # Write setting to camera
# StreamingCamera parameters (applied via StreamingCamera config)
for key in CONFIG_KEYS:
if key in config:
logging.debug("Setting parameter {}: {}".format(key, config[key]))
self._config[key] = config[key]
# Richard's library to set analog and digital gains
if 'analog_gain' in config:
set_analog_gain(self.camera, config['analog_gain'])
if 'digital_gain' in config:
set_digital_gain(self.camera, config['digital_gain'])
# Handle lens shading, if a new one is given
if 'shading_table_path' in config:
self.apply_shading_table()
for key, value in config.items(): # For each provided setting
if key != 'picamera_settings': # We already handled this
logging.debug("Setting parameter {}: {}".format(key, value))
self._config[key] = value
# If stream was paused to update config, unpause
if paused_stream:

216
openflexure_microscope/config.py
View file

@ -1,7 +1,9 @@
import yaml
import os
import errno
import logging
import shutil
import copy
HERE = os.path.abspath(os.path.dirname(__file__))
DEFAULT_CONFIG_PATH = os.path.join(HERE, 'microscoperc.default.yaml')
@ -9,100 +11,158 @@ DEFAULT_CONFIG_PATH = os.path.join(HERE, 'microscoperc.default.yaml')
USER_CONFIG_DIR = os.path.join(os.path.expanduser("~"), ".openflexure") #: str: Default path of the user-config directory, containing runtime-config and calibration files. Obtained from ``os.path.join(os.path.expanduser("~"), ".openflexure")``.
USER_CONFIG_FILE = os.path.join(USER_CONFIG_DIR, "microscoperc.yaml") #: str: Default path of the user microscoperc.yaml runtime-config file. Obtained from ``os.path.join(USER_CONFIG_DIR, "microscoperc.yaml")``
TYPES = {
'stream_resolution': tuple,
'video_resolution': tuple,
'image_resolution': tuple,
'numpy_resolution': tuple,
'jpeg_quality': int,
'analog_gain': float,
'digital_gain': float,
} #: dict: Dictionary of expected types for common config parameters. Used by ``convert_config()``.
with open(DEFAULT_CONFIG_PATH, 'r') as defaultrc:
DEFAULT_CONFIG = defaultrc.read()
def convert_config(config: dict) -> dict:
"""
Convert datatype of top-level config parameters based on the global TYPES dictionary.
class OpenflexureConfig():
expandable_keys = [
'picamera_settings',
'openflexure_stage_settings',
] #: List of keys that can be passed as a file path string and expanded automatically
def __init__(self, config_path: str=None, expand: bool=True):
global DEFAULT_CONFIG, USER_CONFIG_FILE
# Set arguments
self.config_path = config_path or USER_CONFIG_FILE
self.expand = expand
# Create empty config dictionaries
self.raw_config = {}
# Expanded config dictionary (used by server)
self.config = copy.copy(self.raw_config)
# Initialise basic config file with defaults if it doesn't exist
self.initialise_file(self.config_path, populate=DEFAULT_CONFIG)
# Load the config in, setting self._config and self.config
self.load()
def update(self, update_dict: dict):
self.config.update(update_dict)
def overwrite(self, new_dict: dict):
self.config = new_dict
def load(self):
# Unexpanded config dictionary (used at load/save time)
self.raw_config = self.load_yaml_file(self.config_path)
# If the loaded config is in contracted format
if self.expand:
# Expand self.raw_config into self._config
self.expand_config()
def save(self, backup: bool=True):
# If the loaded config was in contracted format
if self.expand:
# Contract self._config into self.raw_config
self.contract_config()
Args:
config (dict): Dictionary representation of a loaded runtime-config.
"""
global TYPES
if backup:
if os.path.isfile(self.config_path):
shutil.copyfile(self.config_path, self.config_path+".bk")
for key in config:
if key in TYPES:
config[key] = TYPES[key](config[key])
self.save_yaml_file(self.config_path, self.raw_config)
return config
def expand_config(self):
# For each value in the raw loaded config
for key, value in self.raw_config.items():
# If it's a valid expandable parameter
if (key in OpenflexureConfig.expandable_keys and
type(value) is str):
logging.debug("Expanding {}".format(value))
# Initialise, load and expand
self.initialise_file(value)
self.config[key] = self.load_yaml_file(value) or {}
else:
self.config[key] = value
def load_config(config_path: str=None) -> dict:
"""
Open openflexurerc.yaml runtime-config file and write to the microscope devices.
def contract_config(self):
for key, value in self.config.items():
# If it's a valid expandable parameter
if (key in OpenflexureConfig.expandable_keys and
type(value) is dict):
Args:
config_path (str): Path to the config YAML file. If `None`, defaults to `DEFAULT_CONFIG_PATH`
"""
global DEFAULT_CONFIG_PATH, USER_CONFIG_FILE
if not config_path:
if os.path.exists(USER_CONFIG_FILE): # If user config file already exists
config_path = USER_CONFIG_FILE # Load it
else: # If user config file doesn't yet exist
logging.warning("No user config found. Loading system defaults...")
if not os.path.exists(USER_CONFIG_DIR):
logging.info("Making user config directory...")
os.makedirs(USER_CONFIG_DIR)
logging.info("Copying default config to user config...")
shutil.copyfile(DEFAULT_CONFIG_PATH, USER_CONFIG_FILE)
logging.info("Loading user config...")
config_path = USER_CONFIG_FILE # Load defaults in
with open(config_path) as config_file:
config_data = yaml.load(config_file)
# Store config dictionary to self
return convert_config(config_data)
# Create the file if it doesn't exist
self.initialise_file(value)
self.save_yaml_file(self.raw_config[key], value)
else:
self.raw_config = value
def save_config(config_dict: dict, config_path: str=None, safe: bool=False):
"""
Save current config dictionary to a YAML file.
def load_yaml_file(self, config_path) -> dict:
"""
Open a .yaml config file
Args:
config_dict (dict): Dictionary of config data to save.
config_path (str): Path to the config YAML file. If `None`, defaults to `DEFAULT_CONFIG_PATH`
"""
global USER_CONFIG_FILE
if not config_path:
config_path = USER_CONFIG_FILE
Args:
config_path (str): Path to the config YAML file. If `None`, defaults to `DEFAULT_CONFIG_PATH`
"""
config_path = os.path.expanduser(config_path)
with open(config_path, 'w') as outfile:
if not safe:
yaml.dump(config_dict, outfile)
else:
yaml.safe_dump(config_dict, outfile)
logging.info("Loading {}...".format(config_path))
with open(config_path) as config_file:
config_data = yaml.load(config_file)
# Return loaded config dictionary
return config_data
def merge_config(config_dict: dict, config_path: str=None, safe: bool=False, backup: bool=True):
"""
merge current config dictionary with an existing YAML file.
def save_yaml_file(self, config_path: str, config_dict: dict, safe: bool=False):
"""
Save a .yaml config file
Args:
config_dict (dict): Dictionary of config data to save.
config_path (str): Path to the config YAML file. If `None`, defaults to `DEFAULT_CONFIG_PATH`
"""
global USER_CONFIG_FILE
if not config_path:
config_path = USER_CONFIG_FILE
Args:
config_dict (dict): Dictionary of config data to save.
config_path (str): Path to the config YAML file.
"""
config_path = os.path.expanduser(config_path)
config_data = load_config(config_path=config_path)
logging.info("Saving {}...".format(config_path))
for key, value in config_dict.items():
config_data[key] = value
with open(config_path, 'w') as outfile:
if not safe:
yaml.dump(config_dict, outfile)
else:
yaml.safe_dump(config_dict, outfile)
if backup:
if os.path.isfile(config_path):
shutil.copyfile(config_path, config_path+".bk")
save_config(config_data, config_path=config_path, safe=safe)
def initialise_file(self, config_path, populate: str=""):
"""
Check if a file exists, and if not, create it
and optionally populate it with content
Args:
config_path (str): Path to the file.
populate (str): String to dump to the file, if it is being newly created
"""
config_path = os.path.expanduser(config_path)
logging.debug("Initialising {}".format(config_path))
logging.debug("Exists: {}".format(os.path.exists(config_path)))
if not os.path.exists(config_path): # If user config file doesn't exist
logging.warning("No config file found at {}. Creating...".format(config_path))
self.create_file(config_path)
logging.info("Populating {}...".format(config_path))
with open(config_path, 'w') as outfile:
outfile.write(populate)
def create_file(self, config_path):
if not os.path.exists(os.path.dirname(config_path)):
try:
os.makedirs(os.path.dirname(config_path))
except OSError as exc: # Guard against race condition
if exc.errno != errno.EEXIST:
raise

44
openflexure_microscope/microscope.py
View file

@ -11,10 +11,10 @@ from openflexure_stage import OpenFlexureStage
from .camera.pi import StreamingCamera
from .plugins import PluginMount
from .config import load_config, save_config, convert_config
from .utilities import axes_to_array
from .task import TaskOrchestrator
from .lock import CompositeLock
from .config import OpenflexureConfig
class Microscope(object):
@ -29,14 +29,14 @@ class Microscope(object):
config (dict): Dictionary of the runtime config to apply to the microscope. Does not automatically apply to camera and stage.
attach_plugins (bool): Should the microscope attach plugins listen in 'config' immediately.
"""
def __init__(self, camera: StreamingCamera, stage: OpenFlexureStage, config: dict={}, attach_plugins: bool=True):
def __init__(self, camera: StreamingCamera, stage: OpenFlexureStage, config_path: str=None, attach_plugins: bool=True):
# Create RC object
self.rc = OpenflexureConfig(config_path=config_path, expand=True)
# Attach initial hardware (may be NoneTypes)
self.attach(camera, stage)
# Store entire runtime-config
self._config = config
# Create a task orchestrator
self.task = TaskOrchestrator() #: :py:class:`openflexure_microscope.task.TaskOrchestrator`: Threaded task orchestrator
@ -48,11 +48,11 @@ class Microscope(object):
self.attach_plugins()
# Set default parameters
if not 'name' in self._config:
self._config['name'] = uuid.uuid4().hex
if not 'name' in self.rc.config:
self.rc.config['name'] = uuid.uuid4().hex
if not 'fov' in self._config:
self._config['fov'] = [0, 0] # Assumes pi camera 2, and 40x objective
if not 'fov' in self.rc.config:
self.rc.config['fov'] = [0, 0]
# Initialise with an empty composite lock
self.lock = CompositeLock([]) #: :py:class:`openflexure_microscope.lock.CompositeLock`: Composite lock controlling thread access to multiple pieces of hardware
@ -73,7 +73,7 @@ class Microscope(object):
if self.stage:
self.stage.close()
def attach(self, camera: StreamingCamera, stage: OpenFlexureStage, apply_config: bool=True):
def attach(self, camera: StreamingCamera, stage: OpenFlexureStage):
"""
Retroactively attaches a camera and stage to the microscope object.
@ -128,8 +128,8 @@ class Microscope(object):
"""
Automatically search for plugin maps in self.config, and attach.
"""
if 'plugins' in self._config:
self.attach_plugin_maps(self._config['plugins'])
if 'plugins' in self.rc.config:
self.attach_plugin_maps(self.rc.config['plugins'])
else:
logging.warning("No plugins specified in microscoperc.yaml. Skipping.")
@ -178,7 +178,7 @@ class Microscope(object):
# If attached to a camera
if self.camera:
# Update camera config params from StreamingCamera object
self._config.update(self.camera.config)
self.rc.update(self.camera.config)
# If attached to a stage
if self.stage:
@ -186,13 +186,17 @@ class Microscope(object):
else:
backlash = [0, 0, 0]
self._config['backlash'] = {
'x': backlash[0],
'y': backlash[1],
'z': backlash[2],
backlash = {
'backlash': {
'x': backlash[0],
'y': backlash[1],
'z': backlash[2],
}
}
return self._config
self.rc.update(backlash)
return self.rc.config
@config.setter
def config(self, config: dict) -> None:
@ -209,4 +213,6 @@ class Microscope(object):
self.stage.backlash = backlash
# Cache config
self._config.update(config)
self.rc.update(config)
# Save to disk
self.rc.save()

13
openflexure_microscope/microscoperc.default.yaml
View file

@ -1,7 +1,3 @@
# The analog gain offers higher sensitivity and less noise than using digital gain only
analog_gain: 1.
digital_gain: 1.
# Resolutions for streaming and capture
video_resolution: [832, 624]
image_resolution: [2592, 1944]
@ -14,14 +10,7 @@ fov: [4100, 3146]
jpeg_quality: 75
# Parameters specific to PiCamera objects
picamera_params:
exposure_mode: 'off'
awb_mode: 'off'
awb_gains: [0.9, 2.8]
framerate: 24
shutter_speed: 30000
saturation: 0
led: false
picamera_settings: ~/.openflexure/picamera_settings.yaml
# Default plugins
plugins: