rahix/openflexure-microscope-server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Converted all configs to JSON

Browse source
This commit is contained in:
Joel Collins 2019-11-12 14:36:39 +00:00
parent 94a8774ceb
commit b81cfaf9e7
13 changed files with 175 additions and 218 deletions
Download patch file Download diff file Expand all files Collapse all files

2
docs/source/config.rst
View file

@ -11,7 +11,7 @@ Microscope RC file
------------------
Microscope configuration is made persistent via a microscope runtime-config (RC) file. By default, this
file exists at ``~/.openflexure/microscope_settings.yaml``, and contains basic parameters to set up the microscope.
file exists at ``~/.openflexure/microscope_settings.json``, and contains basic parameters to set up the microscope.
Additionally, by default, configurations for specific pieces of hardware (i.e. the stage and camera) are located
in separate "auxillary" config files, and are linked together by adding the config file's path to your main

15
docs/source/plugins/introduction.rst
View file

@ -25,17 +25,18 @@ Generally, for adding anything other than very simple functionality, plugins sho
The main restriction is that the plugin package must be importable using an absolute import from within the Python environment being used to load your microscope.
Loading plugins with microscope_settings.yaml
Loading plugins with microscope_settings.json
---------------------------------------------
Both types of plugin are loaded by specifying the plugin class in your :ref:`MicroscopeRC`. In the case of a single-file plugin, specify the path to the plugin file, followed by the name of your :py:class:`openflexure_microscope.plugins.MicroscopePlugin` child class, separated by a colon. For packaged plugins, specify the absolute module name in place of the path.
For example, the plugins section of your microscope_settings.yaml file may look like:
For example, the plugins section of your microscope_settings.json file may look like:
.. code-block:: yaml
.. code-block:: json
...
plugins:
- openflexure_microscope.plugins.default:Plugin
- ~/my_plugins/my_plugin_file.py:MyPluginClass
- my_openflexure_plugins.microscope.mypluginpackage:MyPluginClass
"plugins": [
"openflexure_microscope.plugins.default:Plugin",
"~/my_plugins/my_plugin_file.py:MyPluginClass",
"my_openflexure_plugins.microscope.mypluginpackage:MyPluginClass"
]
...

8
openflexure_microscope/api/app.py
View file

@ -19,7 +19,7 @@ from openflexure_microscope.api.utilities import list_routes
from openflexure_microscope import Microscope
from openflexure_microscope.camera.capture import build_captures_from_exif
from openflexure_microscope.config import USER_CONFIG_DIR
from openflexure_microscope.config import settings_file_path, JSONEncoder
from openflexure_microscope.api.v1 import blueprints
# Import device modules
@ -38,7 +38,7 @@ from openflexure_microscope.stage.mock import MockStage
# Handle logging
is_gunicorn = "gunicorn" in os.environ.get("SERVER_SOFTWARE", "")
DEFAULT_LOGFILE = os.path.join(USER_CONFIG_DIR, "openflexure_microscope.log")
DEFAULT_LOGFILE = settings_file_path("openflexure_microscope.log")
if (__name__ == "__main__") or (not is_gunicorn):
# If imported, but not by gunicorn
@ -109,6 +109,10 @@ def uri(suffix, api_version, base=None):
app = Flask(__name__)
app.url_map.strict_slashes = False
# Use custom JSON encoder
app.json_encoder = JSONEncoder
# Enable CORS everywhere
CORS(app, resources=r"*")
# Make errors more API friendly

15
openflexure_microscope/api/v1/blueprints/camera/__init__.py
View file

@ -9,21 +9,6 @@ def construct_blueprint(microscope_obj):
blueprint = Blueprint("camera_blueprint", __name__)
# Metadata routes
blueprint.add_url_rule(
"/capture/<capture_id>/metadata/<filename>",
view_func=capture.MetadataAPI.as_view(
"metadata_download", microscope=microscope_obj
),
)
blueprint.add_url_rule(
"/capture/<capture_id>/metadata",
view_func=capture.MetadataRedirectAPI.as_view(
"metadata_download_redirect", microscope=microscope_obj
),
)
# Tag routes
blueprint.add_url_rule(
"/capture/<capture_id>/tags",

78
openflexure_microscope/api/v1/blueprints/camera/capture.py
View file

@ -236,7 +236,7 @@ class CaptureAPI(MicroscopeView):
def put(self, capture_id):
"""
Add arbitrary metadata to the capture (stored in an accompanying capture `.yaml` file)
Add arbitrary metadata to the capture
.. :quickref: Capture; Update capture metadata
@ -351,76 +351,6 @@ class DownloadAPI(MicroscopeView):
return send_file(img, mimetype="image/jpeg")
class MetadataRedirectAPI(MicroscopeView):
def get(self, capture_id):
"""
Return metadatadata for a capture.
Return capture metadata as a YAML file with the requested filename.
I.e., `/(capture_id)/download/bar.yaml` will download the file as
`bar.yaml`, regardless of the capture's initially set filename.
Route automatically redirects to download the capture metadata under it's currently set filename.
I.e., `/(capture_id)/download` will
redirect to `/(capture_id)/download/(filename)`.
.. :quickref: Capture; Download capture metadata
**Example request**:
.. sourcecode:: http
GET /camera/capture/d0b2067abbb946f19351e075c5e7cd5b/metadata/2018-11-20_16-04-17.yaml HTTP/1.1
Accept: text/yaml
:>header Accept: text/yaml
:query as_attachment: return the image as an attachment download e.g. ?as_attachment=true
:>header Content-Type: text/yaml
:status 200: capture data found
:status 404: no capture found with that id
"""
capture_obj = self.microscope.camera.image_from_id(capture_id)
if not capture_obj or not capture_obj.state["available"]:
return abort(404) # 404 Not Found
return redirect(
url_for(
".metadata_download",
capture_id=capture_id,
filename=capture_obj.metadataname,
),
code=307,
)
class MetadataAPI(MicroscopeView):
def get(self, capture_id, filename):
capture_obj = self.microscope.camera.image_from_id(capture_id)
if not capture_obj or not capture_obj.state["available"]:
return abort(404) # 404 Not Found
# If no filename is specified, redirect to the capture's currently set filename
if not filename:
return redirect(
url_for(
"capture_download",
capture_id=capture_id,
filename=capture_obj.metadataname,
),
code=307,
)
# Download the metadata using the requested filename
data = capture_obj.yaml
return Response(data, mimetype="text/yaml")
class TagsAPI(MicroscopeView):
def get(self, capture_id):
"""
@ -433,11 +363,11 @@ class TagsAPI(MicroscopeView):
.. sourcecode:: http
GET /camera/capture/d0b2067abbb946f19351e075c5e7cd5b/tags HTTP/1.1
Accept: text/yaml
Accept: text/json
:>header Accept: text/yaml
:>header Accept: text/json
:>header Content-Type: text/yaml
:>header Content-Type: text/json
:status 200: capture data found
:status 404: no capture found with that id
"""

4
openflexure_microscope/camera/base.py
View file

@ -143,6 +143,10 @@ class BaseCamera(metaclass=ABCMeta):
"""Return the current settings as a dictionary"""
return {"paths": self.paths}
def save_config(self):
"""(Optional) Save any settings to disk that need to be stored"""
return
def __enter__(self):
"""Create camera on context enter."""
return self

12
openflexure_microscope/camera/capture.py
View file

@ -5,13 +5,14 @@ import shutil
import glob
import datetime
import yaml
import json
import logging
from PIL import Image
import atexit
from openflexure_microscope.camera import piexif
from openflexure_microscope.camera.piexif._exceptions import InvalidImageDataError
from openflexure_microscope.config import JSONEncoder
PIL_FORMATS = ["JPG", "JPEG", "PNG", "TIF", "TIFF"]
EXIF_FORMATS = ["JPG", "JPEG", "TIF", "TIFF"]
@ -30,7 +31,12 @@ def pull_usercomment_dict(filepath):
logging.warning("Invalid data at {}. Skipping.".format(filepath))
return None
if "Exif" in exif_dict and 37510 in exif_dict["Exif"]:
return yaml.load(exif_dict["Exif"][37510].decode())
try:
return json.loads(exif_dict["Exif"][37510].decode())
except json.decoder.JSONDecodeError:
# TODO: Remove YAML support in a later version
logging.warning(f"Capture {filepath} has metadata stored in YAML format. This is now deprecated in favour of JSON.")
return yaml.load(exif_dict["Exif"][37510].decode())
else:
return None
@ -210,7 +216,7 @@ class CaptureObject(object):
# Extract current Exif data
exif_dict = piexif.load(self.file)
# Serialize metadata
metadata_string = yaml.safe_dump(self.metadata)
metadata_string = json.dumps(self.metadata, cls=JSONEncoder)
# Insert metadata into exif_dict
exif_dict["Exif"][piexif.ExifIFD.UserComment] = metadata_string.encode()
# Convert new exif dict to exif bytes

87
openflexure_microscope/camera/pi.py
View file

@ -26,6 +26,7 @@ from __future__ import division
import io
import time
import numpy as np
import os
import logging
# Pi camera
@ -40,6 +41,7 @@ from .base import BaseCamera, CaptureObject
# Richard's fix gain
from .set_picamera_gain import set_analog_gain, set_digital_gain
from openflexure_microscope.config import settings_file_path
# MAIN CLASS
class PiCameraStreamer(BaseCamera):
@ -54,10 +56,20 @@ class PiCameraStreamer(BaseCamera):
"awb_mode",
"framerate",
"saturation",
"lens_shading_table",
"iso",
"brightness",
"contrast",
"crop",
"drc_strength",
"exposure_compensation",
"image_effect",
"meter_mode",
"sharpness"
]
def __init__(self):
global USER_CONFIG_DIR
# Run BaseCamera init
BaseCamera.__init__(self)
# Attach to Pi camera
@ -78,11 +90,13 @@ class PiCameraStreamer(BaseCamera):
# Reset variable states
self.set_zoom(1.0)
# Update config properties
self.image_resolution = tuple(self.camera.MAX_RESOLUTION)
self.stream_resolution = (832, 624)
self.numpy_resolution = (1312, 976)
self.jpeg_quality = 75
# Set default settings
self.image_resolution = tuple(self.camera.MAX_RESOLUTION) #: tuple: Resolution for image captures
self.stream_resolution = (832, 624) #: tuple: Resolution for stream and video captures
self.numpy_resolution = (1312, 976) #: tuple: Resolution for numpy array captures
self.jpeg_quality = 75 #: int: JPEG quality
# Set default lens shading table path
self.picamera_lst_path = settings_file_path("picamera_lst.npy") #: str: Path of .npy lens shading table file
# Update board identifier
self.state.update({})
@ -121,12 +135,12 @@ class PiCameraStreamer(BaseCamera):
"image_resolution": self.image_resolution,
"numpy_resolution": self.numpy_resolution,
"jpeg_quality": self.jpeg_quality,
"picamera_lst_path": self.picamera_lst_path if os.path.isfile(self.picamera_lst_path) else None,
"picamera_settings": {},
}
)
# Include a subset of picamera properties
# TODO: Expand this subset so we have more metadata?
# Include a subset of picamera properties. Excludes lens shading table
for key in PiCameraStreamer.picamera_settings_keys:
try:
value = getattr(self.camera, key)
@ -137,6 +151,11 @@ class PiCameraStreamer(BaseCamera):
return conf_dict
def save_config(self):
"""Save lens-shading table to disk"""
logging.info("Saving picamera_lst to {}".format(self.picamera_lst_path))
self.save_lens_shading_table()
def apply_config(self, config: dict):
"""
Write a config dictionary to the PiCameraStreamer config.
@ -174,6 +193,17 @@ class PiCameraStreamer(BaseCamera):
if (key != "picamera_settings") and hasattr(self, key):
setattr(self, key, value)
# Handle lens shading if camera supports it
if ("picamera_lst_path" in config) and hasattr(
self.camera, "lens_shading_table"
):
logging.debug(
"Applying lens_shading_table from file: {}".format(
config["picamera_lst_path"]
)
)
self.apply_lens_shading_table(config["picamera_lst_path"])
# If stream was paused to update config, unpause
if paused_stream:
logging.info("Resuming stream.")
@ -232,21 +262,40 @@ class PiCameraStreamer(BaseCamera):
logging.debug("Applying {}: {}".format(key, settings_dict[key]))
setattr(self.camera, key, settings_dict[key])
# Handle lens shading if camera supports it
if ("lens_shading_table" in settings_dict) and hasattr(
self.camera, "lens_shading_table"
):
logging.debug(
"Applying lens_shading_table: {}".format(
settings_dict["lens_shading_table"]
)
)
self.camera.lens_shading_table = settings_dict["lens_shading_table"]
# Final optional pause to settle
if pause_for_effect:
time.sleep(0.2)
def read_lens_shading_table(self):
"""
Read the current lens shading table as a numpy array, if it exists. Return None otherwise.
"""
if hasattr(self.camera, 'lens_shading_table'):
return self.camera.lens_shading_table
else:
return None
def save_lens_shading_table(self):
"""
Save the current lens shading table to an .npy file, if it exists.
"""
logging.debug(self.read_lens_shading_table())
if self.read_lens_shading_table() is not None:
np.save(self.picamera_lst_path, self.read_lens_shading_table())
else:
logging.warning("Unable to save a nonexistant lens shading table")
def apply_lens_shading_table(self, lst_array_or_path):
"""
Apply a lens shading table from an .npy file, or numpy array.
"""
if isinstance(lst_array_or_path, np.ndarray):
self.camera.lens_shading_table = lst_array_or_path
elif (type(lst_array_or_path) == str) and os.path.isfile(lst_array_or_path):
self.camera.lens_shading_table = np.load(lst_array_or_path)
else:
logging.error("Unsupported or missing data for camera lens_shading_table. Must be numpy ndarray, or .npy file path string. Skipping.")
def set_zoom(self, zoom_value: float = 1.0) -> None:
"""
Change the camera zoom, handling re-centering and scaling.

127
openflexure_microscope/config.py
View file

@ -1,4 +1,4 @@
import yaml
import json
import os
import errno
import logging
@ -13,103 +13,57 @@ from .utilities import recursively_apply
Attributes:
USER_CONFIG_DIR (str): Default path of the user-config directory, containing runtime-config and
calibration files. Obtained from ``os.path.join(os.path.expanduser("~"), ".openflexure")``.
USER_CONFIG_FILE (str): Default path of the user microscope_settings.yaml runtime-config file.
Obtained from ``os.path.join(USER_CONFIG_DIR, "microscope_settings.yaml")``
USER_CONFIG_FILE (str): Default path of the user microscope_settings.json runtime-config file.
Obtained from ``os.path.join(USER_CONFIG_DIR, "microscope_settings.json")``
"""
# HANDLE THE DEFAULT CONFIGURATION FILE
HERE = os.path.abspath(os.path.dirname(__file__))
DEFAULT_CONFIG_PATH = os.path.join(HERE, "microscope_settings.default.yaml")
DEFAULT_CONFIG_PATH = os.path.join(HERE, "microscope_settings.default.json")
USER_CONFIG_DIR = os.path.join(os.path.expanduser("~"), ".openflexure")
USER_CONFIG_FILE = os.path.join(USER_CONFIG_DIR, "microscope_settings.yaml")
USER_CONFIG_FILE = os.path.join(USER_CONFIG_DIR, "microscope_settings.json")
USER_CONFIG_FILE_OLD = os.path.join(USER_CONFIG_DIR, "microscoperc.yaml")
USER_CONFIG_FILE_OLD = os.path.join(USER_CONFIG_DIR, "microscoperc.json")
# Load the default config
with open(DEFAULT_CONFIG_PATH, "r") as default_rc:
DEFAULT_CONFIG = default_rc.read()
# HANDLE CUSTOM YAML PARSING
def construct_yaml_bool(self, node):
# HANDLE BASIC LOADING AND SAVING OF SETTINGS FILES
def load_json_file(config_path) -> dict:
"""
Override PyYAML constructors handling of bools.
YAML otherwise caused grief for picamera properties where 'off' is a valid string value.
"""
override_bool_values = {
"yes": True,
"no": False,
"true": True,
"false": False,
"on": "on",
"off": "off",
}
value = self.construct_scalar(node)
return override_bool_values[value.lower()]
yaml.Loader.add_constructor(u"tag:yaml.org,2002:bool", construct_yaml_bool)
# HANDLE CONVERTING ARBITRARY CONFIG TO JSON-SAFE JSON
def convert_type_to_json_safe(v):
"""Make an individual attribute JSON-safe"""
if isinstance(v, Fraction):
return float(v)
elif isinstance(v, np.integer):
return int(v)
elif isinstance(v, np.ndarray):
return v.tolist()
else:
return v
def settings_to_json(data: dict):
"""
Make a copy of an input dictionary that's safe for JSON return
Open a .json config file
Args:
data: Input dictionary
"""
# Do not overwrite original data dictionary
d = copy.deepcopy(data)
return recursively_apply(d, convert_type_to_json_safe)
# HANDLE BASIC LOADING AND SAVING OF YAML FILES
def load_yaml_file(config_path) -> dict:
"""
Open a .yaml config file
Args:
config_path (str): Path to the config YAML file. If `None`, defaults to `DEFAULT_CONFIG_PATH`
config_path (str): Path to the config JSON file. If `None`, defaults to `DEFAULT_CONFIG_PATH`
"""
config_path = os.path.expanduser(config_path)
logging.info("Loading {}...".format(config_path))
with open(config_path) as config_file:
config_data = yaml.load(config_file)
try:
config_data = json.load(config_file)
except json.decoder.JSONDecodeError as e:
logging.error(e)
config_data = {}
# Return loaded config dictionary
return config_data
def save_yaml_file(config_path: str, config_dict: dict, safe: bool = False):
def save_json_file(config_path: str, config_dict: dict):
"""
Save a .yaml config file
Save a .json config file
Args:
config_dict (dict): Dictionary of config data to save.
config_path (str): Path to the config YAML file.
safe (bool): Whether to use PyYAML safe_dump instead of dump
config_path (str): Path to the config JSON file.
"""
config_path = os.path.expanduser(config_path)
@ -117,10 +71,7 @@ def save_yaml_file(config_path: str, config_dict: dict, safe: bool = False):
logging.debug(config_dict)
with open(config_path, "w") as outfile:
if not safe:
yaml.dump(config_dict, outfile)
else:
yaml.safe_dump(config_dict, outfile)
json.dump(config_dict, outfile, cls=JSONEncoder, indent=2, sort_keys=True)
def create_file(config_path):
@ -132,7 +83,7 @@ def create_file(config_path):
raise
def initialise_file(config_path, populate: str = ""):
def initialise_file(config_path, populate: str = "{}\n"):
"""
Check if a file exists, and if not, create it
and optionally populate it with content
@ -155,13 +106,35 @@ def initialise_file(config_path, populate: str = ""):
outfile.write(populate)
def settings_file_path(filename:str):
"""Generate a full file path for a filename to be stored in user settings"""
global USER_CONFIG_DIR
return os.path.join(USER_CONFIG_DIR, filename)
class JSONEncoder(json.JSONEncoder):
def default(self, o, markers=None):
# PiCamera fractions
if isinstance(o, Fraction):
return float(o)
# Numpy integers
elif isinstance(o, np.integer):
return int(o)
# Numpy arrays
elif isinstance(o, np.ndarray):
return o.tolist()
else:
# call base class implementation which takes care of
# raising exceptions for unsupported types
return json.JSONEncoder.default(self, o)
# MAIN CONFIG CLASS
class OpenflexureSettingsFile:
"""
An object to handle expansion, conversion, and saving of the microscope configuration.
Args:
config_path (str): Path to the config YAML file (None falls back to default location)
config_path (str): Path to the config JSON file (None falls back to default location)
expand (bool): Expand paths to valid auxillary config files.
"""
@ -185,7 +158,7 @@ class OpenflexureSettingsFile:
Loads config from a file on-disk, and expands auxillary config files if available.
"""
# Unexpanded config dictionary (used at load/save time)
loaded_config = load_yaml_file(self.config_path)
loaded_config = load_json_file(self.config_path)
# If the loaded config is in contracted format
if self.expand:
@ -211,7 +184,7 @@ class OpenflexureSettingsFile:
shutil.copyfile(self.config_path, self.config_path + ".bk")
logging.debug("Saving settings dictionary to disk")
save_yaml_file(self.config_path, save_config)
save_json_file(self.config_path, save_config)
def merge(self, config: dict, backup: bool = True):
logging.debug("Merging settings with file on disk")
@ -243,7 +216,7 @@ class OpenflexureSettingsFile:
# Create the expansion file if it doesn't yet exist
initialise_file(value)
# Load the expansion file into _config
return_config[key] = load_yaml_file(value) or {}
return_config[key] = load_json_file(value) or {}
else:
return_config[key] = value
@ -270,7 +243,7 @@ class OpenflexureSettingsFile:
# Create the file if it doesn't exist
initialise_file(self.expandable_keys[key])
# Save the expanded config dictionary to the file
save_yaml_file(self.expandable_keys[key], value)
save_json_file(self.expandable_keys[key], value)
# Replace the expanded dictionary with a file path
return_config[key] = self.expandable_keys[key]
else:

10
openflexure_microscope/microscope.py
View file

@ -14,7 +14,7 @@ from openflexure_microscope.camera.mock import MockStreamer
from openflexure_microscope.plugins import PluginMount
from openflexure_microscope.task import TaskOrchestrator
from openflexure_microscope.common.lock import CompositeLock
from openflexure_microscope.config import OpenflexureSettingsFile, settings_to_json
from openflexure_microscope.config import OpenflexureSettingsFile
class Microscope:
@ -225,8 +225,8 @@ class Microscope:
settings_full = self.settings_file.merge(settings_current)
if json_safe:
settings_full = settings_to_json(settings_full)
#if json_safe:
# settings_full = settings_to_json(settings_full)
return settings_full
@ -241,6 +241,10 @@ class Microscope:
"openflexure_microscope"
).version
# Save config to file
if self.camera:
self.camera.save_config()
if self.stage:
self.stage.save_config()
self.settings_file.save(current_config, backup=True)
@property

14
openflexure_microscope/microscope_settings.default.json Normal file
View file

@ -0,0 +1,14 @@
{
"fov": [
4100,
3146
],
"jpeg_quality": 75,
"camera_settings": "~/.openflexure/camera_settings.json",
"stage_settings": "~/.openflexure/stage_settings.json",
"plugins": [
"openflexure_microscope.plugins.default.autofocus:AutofocusPlugin",
"openflexure_microscope.plugins.default.scan:ScanPlugin",
"openflexure_microscope.plugins.default.camera_calibration:Plugin"
]
}

17
openflexure_microscope/microscope_settings.default.yaml
View file

@ -1,17 +0,0 @@
# Field of view, in stage steps
fov: [4100, 3146]
# Capture quality
jpeg_quality: 75
# Parameters specific to camera objects
camera_settings: ~/.openflexure/camera_settings.yaml
# Parameters specific to stage objects
stage_settings: ~/.openflexure/stage_settings.yaml
# Default plugins
plugins:
- openflexure_microscope.plugins.default.autofocus:AutofocusPlugin
- openflexure_microscope.plugins.default.scan:ScanPlugin
- openflexure_microscope.plugins.default.camera_calibration:Plugin

4
openflexure_microscope/stage/base.py
View file

@ -23,6 +23,10 @@ class BaseStage(metaclass=ABCMeta):
"""Return the current settings as a dictionary"""
pass
def save_config(self):
"""(Optional) Save any settings to disk that need to be stored"""
return
@property
@abstractmethod
def state(self):