rahix/openflexure-microscope-server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Converted all configs to JSON

Browse source
This commit is contained in:
Joel Collins 2019-11-12 14:36:39 +00:00
parent 94a8774ceb
commit b81cfaf9e7
13 changed files with 175 additions and 218 deletions
Download patch file Download diff file Expand all files Collapse all files

8
openflexure_microscope/api/app.py
View file

@ -19,7 +19,7 @@ from openflexure_microscope.api.utilities import list_routes
from openflexure_microscope import Microscope
from openflexure_microscope.camera.capture import build_captures_from_exif
from openflexure_microscope.config import USER_CONFIG_DIR
from openflexure_microscope.config import settings_file_path, JSONEncoder
from openflexure_microscope.api.v1 import blueprints
# Import device modules
@ -38,7 +38,7 @@ from openflexure_microscope.stage.mock import MockStage
# Handle logging
is_gunicorn = "gunicorn" in os.environ.get("SERVER_SOFTWARE", "")
DEFAULT_LOGFILE = os.path.join(USER_CONFIG_DIR, "openflexure_microscope.log")
DEFAULT_LOGFILE = settings_file_path("openflexure_microscope.log")
if (__name__ == "__main__") or (not is_gunicorn):
# If imported, but not by gunicorn
@ -109,6 +109,10 @@ def uri(suffix, api_version, base=None):
app = Flask(__name__)
app.url_map.strict_slashes = False
# Use custom JSON encoder
app.json_encoder = JSONEncoder
# Enable CORS everywhere
CORS(app, resources=r"*")
# Make errors more API friendly

15
openflexure_microscope/api/v1/blueprints/camera/__init__.py
View file

@ -9,21 +9,6 @@ def construct_blueprint(microscope_obj):
blueprint = Blueprint("camera_blueprint", __name__)
# Metadata routes
blueprint.add_url_rule(
"/capture/<capture_id>/metadata/<filename>",
view_func=capture.MetadataAPI.as_view(
"metadata_download", microscope=microscope_obj
),
)
blueprint.add_url_rule(
"/capture/<capture_id>/metadata",
view_func=capture.MetadataRedirectAPI.as_view(
"metadata_download_redirect", microscope=microscope_obj
),
)
# Tag routes
blueprint.add_url_rule(
"/capture/<capture_id>/tags",

78
openflexure_microscope/api/v1/blueprints/camera/capture.py
View file

@ -236,7 +236,7 @@ class CaptureAPI(MicroscopeView):
def put(self, capture_id):
"""
Add arbitrary metadata to the capture (stored in an accompanying capture `.yaml` file)
Add arbitrary metadata to the capture
.. :quickref: Capture; Update capture metadata
@ -351,76 +351,6 @@ class DownloadAPI(MicroscopeView):
return send_file(img, mimetype="image/jpeg")
class MetadataRedirectAPI(MicroscopeView):
def get(self, capture_id):
"""
Return metadatadata for a capture.
Return capture metadata as a YAML file with the requested filename.
I.e., `/(capture_id)/download/bar.yaml` will download the file as
`bar.yaml`, regardless of the capture's initially set filename.
Route automatically redirects to download the capture metadata under it's currently set filename.
I.e., `/(capture_id)/download` will
redirect to `/(capture_id)/download/(filename)`.
.. :quickref: Capture; Download capture metadata
**Example request**:
.. sourcecode:: http
GET /camera/capture/d0b2067abbb946f19351e075c5e7cd5b/metadata/2018-11-20_16-04-17.yaml HTTP/1.1
Accept: text/yaml
:>header Accept: text/yaml
:query as_attachment: return the image as an attachment download e.g. ?as_attachment=true
:>header Content-Type: text/yaml
:status 200: capture data found
:status 404: no capture found with that id
"""
capture_obj = self.microscope.camera.image_from_id(capture_id)
if not capture_obj or not capture_obj.state["available"]:
return abort(404) # 404 Not Found
return redirect(
url_for(
".metadata_download",
capture_id=capture_id,
filename=capture_obj.metadataname,
),
code=307,
)
class MetadataAPI(MicroscopeView):
def get(self, capture_id, filename):
capture_obj = self.microscope.camera.image_from_id(capture_id)
if not capture_obj or not capture_obj.state["available"]:
return abort(404) # 404 Not Found
# If no filename is specified, redirect to the capture's currently set filename
if not filename:
return redirect(
url_for(
"capture_download",
capture_id=capture_id,
filename=capture_obj.metadataname,
),
code=307,
)
# Download the metadata using the requested filename
data = capture_obj.yaml
return Response(data, mimetype="text/yaml")
class TagsAPI(MicroscopeView):
def get(self, capture_id):
"""
@ -433,11 +363,11 @@ class TagsAPI(MicroscopeView):
.. sourcecode:: http
GET /camera/capture/d0b2067abbb946f19351e075c5e7cd5b/tags HTTP/1.1
Accept: text/yaml
Accept: text/json
:>header Accept: text/yaml
:>header Accept: text/json
:>header Content-Type: text/yaml
:>header Content-Type: text/json
:status 200: capture data found
:status 404: no capture found with that id
"""

4
openflexure_microscope/camera/base.py
View file

@ -143,6 +143,10 @@ class BaseCamera(metaclass=ABCMeta):
"""Return the current settings as a dictionary"""
return {"paths": self.paths}
def save_config(self):
"""(Optional) Save any settings to disk that need to be stored"""
return
def __enter__(self):
"""Create camera on context enter."""
return self

12
openflexure_microscope/camera/capture.py
View file

@ -5,13 +5,14 @@ import shutil
import glob
import datetime
import yaml
import json
import logging
from PIL import Image
import atexit
from openflexure_microscope.camera import piexif
from openflexure_microscope.camera.piexif._exceptions import InvalidImageDataError
from openflexure_microscope.config import JSONEncoder
PIL_FORMATS = ["JPG", "JPEG", "PNG", "TIF", "TIFF"]
EXIF_FORMATS = ["JPG", "JPEG", "TIF", "TIFF"]
@ -30,7 +31,12 @@ def pull_usercomment_dict(filepath):
logging.warning("Invalid data at {}. Skipping.".format(filepath))
return None
if "Exif" in exif_dict and 37510 in exif_dict["Exif"]:
return yaml.load(exif_dict["Exif"][37510].decode())
try:
return json.loads(exif_dict["Exif"][37510].decode())
except json.decoder.JSONDecodeError:
# TODO: Remove YAML support in a later version
logging.warning(f"Capture {filepath} has metadata stored in YAML format. This is now deprecated in favour of JSON.")
return yaml.load(exif_dict["Exif"][37510].decode())
else:
return None
@ -210,7 +216,7 @@ class CaptureObject(object):
# Extract current Exif data
exif_dict = piexif.load(self.file)
# Serialize metadata
metadata_string = yaml.safe_dump(self.metadata)
metadata_string = json.dumps(self.metadata, cls=JSONEncoder)
# Insert metadata into exif_dict
exif_dict["Exif"][piexif.ExifIFD.UserComment] = metadata_string.encode()
# Convert new exif dict to exif bytes

87
openflexure_microscope/camera/pi.py
View file

@ -26,6 +26,7 @@ from __future__ import division
import io
import time
import numpy as np
import os
import logging
# Pi camera
@ -40,6 +41,7 @@ from .base import BaseCamera, CaptureObject
# Richard's fix gain
from .set_picamera_gain import set_analog_gain, set_digital_gain
from openflexure_microscope.config import settings_file_path
# MAIN CLASS
class PiCameraStreamer(BaseCamera):
@ -54,10 +56,20 @@ class PiCameraStreamer(BaseCamera):
"awb_mode",
"framerate",
"saturation",
"lens_shading_table",
"iso",
"brightness",
"contrast",
"crop",
"drc_strength",
"exposure_compensation",
"image_effect",
"meter_mode",
"sharpness"
]
def __init__(self):
global USER_CONFIG_DIR
# Run BaseCamera init
BaseCamera.__init__(self)
# Attach to Pi camera
@ -78,11 +90,13 @@ class PiCameraStreamer(BaseCamera):
# Reset variable states
self.set_zoom(1.0)
# Update config properties
self.image_resolution = tuple(self.camera.MAX_RESOLUTION)
self.stream_resolution = (832, 624)
self.numpy_resolution = (1312, 976)
self.jpeg_quality = 75
# Set default settings
self.image_resolution = tuple(self.camera.MAX_RESOLUTION) #: tuple: Resolution for image captures
self.stream_resolution = (832, 624) #: tuple: Resolution for stream and video captures
self.numpy_resolution = (1312, 976) #: tuple: Resolution for numpy array captures
self.jpeg_quality = 75 #: int: JPEG quality
# Set default lens shading table path
self.picamera_lst_path = settings_file_path("picamera_lst.npy") #: str: Path of .npy lens shading table file
# Update board identifier
self.state.update({})
@ -121,12 +135,12 @@ class PiCameraStreamer(BaseCamera):
"image_resolution": self.image_resolution,
"numpy_resolution": self.numpy_resolution,
"jpeg_quality": self.jpeg_quality,
"picamera_lst_path": self.picamera_lst_path if os.path.isfile(self.picamera_lst_path) else None,
"picamera_settings": {},
}
)
# Include a subset of picamera properties
# TODO: Expand this subset so we have more metadata?
# Include a subset of picamera properties. Excludes lens shading table
for key in PiCameraStreamer.picamera_settings_keys:
try:
value = getattr(self.camera, key)
@ -137,6 +151,11 @@ class PiCameraStreamer(BaseCamera):
return conf_dict
def save_config(self):
"""Save lens-shading table to disk"""
logging.info("Saving picamera_lst to {}".format(self.picamera_lst_path))
self.save_lens_shading_table()
def apply_config(self, config: dict):
"""
Write a config dictionary to the PiCameraStreamer config.
@ -174,6 +193,17 @@ class PiCameraStreamer(BaseCamera):
if (key != "picamera_settings") and hasattr(self, key):
setattr(self, key, value)
# Handle lens shading if camera supports it
if ("picamera_lst_path" in config) and hasattr(
self.camera, "lens_shading_table"
):
logging.debug(
"Applying lens_shading_table from file: {}".format(
config["picamera_lst_path"]
)
)
self.apply_lens_shading_table(config["picamera_lst_path"])
# If stream was paused to update config, unpause
if paused_stream:
logging.info("Resuming stream.")
@ -232,21 +262,40 @@ class PiCameraStreamer(BaseCamera):
logging.debug("Applying {}: {}".format(key, settings_dict[key]))
setattr(self.camera, key, settings_dict[key])
# Handle lens shading if camera supports it
if ("lens_shading_table" in settings_dict) and hasattr(
self.camera, "lens_shading_table"
):
logging.debug(
"Applying lens_shading_table: {}".format(
settings_dict["lens_shading_table"]
)
)
self.camera.lens_shading_table = settings_dict["lens_shading_table"]
# Final optional pause to settle
if pause_for_effect:
time.sleep(0.2)
def read_lens_shading_table(self):
"""
Read the current lens shading table as a numpy array, if it exists. Return None otherwise.
"""
if hasattr(self.camera, 'lens_shading_table'):
return self.camera.lens_shading_table
else:
return None
def save_lens_shading_table(self):
"""
Save the current lens shading table to an .npy file, if it exists.
"""
logging.debug(self.read_lens_shading_table())
if self.read_lens_shading_table() is not None:
np.save(self.picamera_lst_path, self.read_lens_shading_table())
else:
logging.warning("Unable to save a nonexistant lens shading table")
def apply_lens_shading_table(self, lst_array_or_path):
"""
Apply a lens shading table from an .npy file, or numpy array.
"""
if isinstance(lst_array_or_path, np.ndarray):
self.camera.lens_shading_table = lst_array_or_path
elif (type(lst_array_or_path) == str) and os.path.isfile(lst_array_or_path):
self.camera.lens_shading_table = np.load(lst_array_or_path)
else:
logging.error("Unsupported or missing data for camera lens_shading_table. Must be numpy ndarray, or .npy file path string. Skipping.")
def set_zoom(self, zoom_value: float = 1.0) -> None:
"""
Change the camera zoom, handling re-centering and scaling.

127
openflexure_microscope/config.py
View file

@ -1,4 +1,4 @@
import yaml
import json
import os
import errno
import logging
@ -13,103 +13,57 @@ from .utilities import recursively_apply
Attributes:
USER_CONFIG_DIR (str): Default path of the user-config directory, containing runtime-config and
calibration files. Obtained from ``os.path.join(os.path.expanduser("~"), ".openflexure")``.
USER_CONFIG_FILE (str): Default path of the user microscope_settings.yaml runtime-config file.
Obtained from ``os.path.join(USER_CONFIG_DIR, "microscope_settings.yaml")``
USER_CONFIG_FILE (str): Default path of the user microscope_settings.json runtime-config file.
Obtained from ``os.path.join(USER_CONFIG_DIR, "microscope_settings.json")``
"""
# HANDLE THE DEFAULT CONFIGURATION FILE
HERE = os.path.abspath(os.path.dirname(__file__))
DEFAULT_CONFIG_PATH = os.path.join(HERE, "microscope_settings.default.yaml")
DEFAULT_CONFIG_PATH = os.path.join(HERE, "microscope_settings.default.json")
USER_CONFIG_DIR = os.path.join(os.path.expanduser("~"), ".openflexure")
USER_CONFIG_FILE = os.path.join(USER_CONFIG_DIR, "microscope_settings.yaml")
USER_CONFIG_FILE = os.path.join(USER_CONFIG_DIR, "microscope_settings.json")
USER_CONFIG_FILE_OLD = os.path.join(USER_CONFIG_DIR, "microscoperc.yaml")
USER_CONFIG_FILE_OLD = os.path.join(USER_CONFIG_DIR, "microscoperc.json")
# Load the default config
with open(DEFAULT_CONFIG_PATH, "r") as default_rc:
DEFAULT_CONFIG = default_rc.read()
# HANDLE CUSTOM YAML PARSING
def construct_yaml_bool(self, node):
# HANDLE BASIC LOADING AND SAVING OF SETTINGS FILES
def load_json_file(config_path) -> dict:
"""
Override PyYAML constructors handling of bools.
YAML otherwise caused grief for picamera properties where 'off' is a valid string value.
"""
override_bool_values = {
"yes": True,
"no": False,
"true": True,
"false": False,
"on": "on",
"off": "off",
}
value = self.construct_scalar(node)
return override_bool_values[value.lower()]
yaml.Loader.add_constructor(u"tag:yaml.org,2002:bool", construct_yaml_bool)
# HANDLE CONVERTING ARBITRARY CONFIG TO JSON-SAFE JSON
def convert_type_to_json_safe(v):
"""Make an individual attribute JSON-safe"""
if isinstance(v, Fraction):
return float(v)
elif isinstance(v, np.integer):
return int(v)
elif isinstance(v, np.ndarray):
return v.tolist()
else:
return v
def settings_to_json(data: dict):
"""
Make a copy of an input dictionary that's safe for JSON return
Open a .json config file
Args:
data: Input dictionary
"""
# Do not overwrite original data dictionary
d = copy.deepcopy(data)
return recursively_apply(d, convert_type_to_json_safe)
# HANDLE BASIC LOADING AND SAVING OF YAML FILES
def load_yaml_file(config_path) -> dict:
"""
Open a .yaml config file
Args:
config_path (str): Path to the config YAML file. If `None`, defaults to `DEFAULT_CONFIG_PATH`
config_path (str): Path to the config JSON file. If `None`, defaults to `DEFAULT_CONFIG_PATH`
"""
config_path = os.path.expanduser(config_path)
logging.info("Loading {}...".format(config_path))
with open(config_path) as config_file:
config_data = yaml.load(config_file)
try:
config_data = json.load(config_file)
except json.decoder.JSONDecodeError as e:
logging.error(e)
config_data = {}
# Return loaded config dictionary
return config_data
def save_yaml_file(config_path: str, config_dict: dict, safe: bool = False):
def save_json_file(config_path: str, config_dict: dict):
"""
Save a .yaml config file
Save a .json config file
Args:
config_dict (dict): Dictionary of config data to save.
config_path (str): Path to the config YAML file.
safe (bool): Whether to use PyYAML safe_dump instead of dump
config_path (str): Path to the config JSON file.
"""
config_path = os.path.expanduser(config_path)
@ -117,10 +71,7 @@ def save_yaml_file(config_path: str, config_dict: dict, safe: bool = False):
logging.debug(config_dict)
with open(config_path, "w") as outfile:
if not safe:
yaml.dump(config_dict, outfile)
else:
yaml.safe_dump(config_dict, outfile)
json.dump(config_dict, outfile, cls=JSONEncoder, indent=2, sort_keys=True)
def create_file(config_path):
@ -132,7 +83,7 @@ def create_file(config_path):
raise
def initialise_file(config_path, populate: str = ""):
def initialise_file(config_path, populate: str = "{}\n"):
"""
Check if a file exists, and if not, create it
and optionally populate it with content
@ -155,13 +106,35 @@ def initialise_file(config_path, populate: str = ""):
outfile.write(populate)
def settings_file_path(filename:str):
"""Generate a full file path for a filename to be stored in user settings"""
global USER_CONFIG_DIR
return os.path.join(USER_CONFIG_DIR, filename)
class JSONEncoder(json.JSONEncoder):
def default(self, o, markers=None):
# PiCamera fractions
if isinstance(o, Fraction):
return float(o)
# Numpy integers
elif isinstance(o, np.integer):
return int(o)
# Numpy arrays
elif isinstance(o, np.ndarray):
return o.tolist()
else:
# call base class implementation which takes care of
# raising exceptions for unsupported types
return json.JSONEncoder.default(self, o)
# MAIN CONFIG CLASS
class OpenflexureSettingsFile:
"""
An object to handle expansion, conversion, and saving of the microscope configuration.
Args:
config_path (str): Path to the config YAML file (None falls back to default location)
config_path (str): Path to the config JSON file (None falls back to default location)
expand (bool): Expand paths to valid auxillary config files.
"""
@ -185,7 +158,7 @@ class OpenflexureSettingsFile:
Loads config from a file on-disk, and expands auxillary config files if available.
"""
# Unexpanded config dictionary (used at load/save time)
loaded_config = load_yaml_file(self.config_path)
loaded_config = load_json_file(self.config_path)
# If the loaded config is in contracted format
if self.expand:
@ -211,7 +184,7 @@ class OpenflexureSettingsFile:
shutil.copyfile(self.config_path, self.config_path + ".bk")
logging.debug("Saving settings dictionary to disk")
save_yaml_file(self.config_path, save_config)
save_json_file(self.config_path, save_config)
def merge(self, config: dict, backup: bool = True):
logging.debug("Merging settings with file on disk")
@ -243,7 +216,7 @@ class OpenflexureSettingsFile:
# Create the expansion file if it doesn't yet exist
initialise_file(value)
# Load the expansion file into _config
return_config[key] = load_yaml_file(value) or {}
return_config[key] = load_json_file(value) or {}
else:
return_config[key] = value
@ -270,7 +243,7 @@ class OpenflexureSettingsFile:
# Create the file if it doesn't exist
initialise_file(self.expandable_keys[key])
# Save the expanded config dictionary to the file
save_yaml_file(self.expandable_keys[key], value)
save_json_file(self.expandable_keys[key], value)
# Replace the expanded dictionary with a file path
return_config[key] = self.expandable_keys[key]
else:

10
openflexure_microscope/microscope.py
View file

@ -14,7 +14,7 @@ from openflexure_microscope.camera.mock import MockStreamer
from openflexure_microscope.plugins import PluginMount
from openflexure_microscope.task import TaskOrchestrator
from openflexure_microscope.common.lock import CompositeLock
from openflexure_microscope.config import OpenflexureSettingsFile, settings_to_json
from openflexure_microscope.config import OpenflexureSettingsFile
class Microscope:
@ -225,8 +225,8 @@ class Microscope:
settings_full = self.settings_file.merge(settings_current)
if json_safe:
settings_full = settings_to_json(settings_full)
#if json_safe:
# settings_full = settings_to_json(settings_full)
return settings_full
@ -241,6 +241,10 @@ class Microscope:
"openflexure_microscope"
).version
# Save config to file
if self.camera:
self.camera.save_config()
if self.stage:
self.stage.save_config()
self.settings_file.save(current_config, backup=True)
@property

14
openflexure_microscope/microscope_settings.default.json Normal file
View file

@ -0,0 +1,14 @@
{
"fov": [
4100,
3146
],
"jpeg_quality": 75,
"camera_settings": "~/.openflexure/camera_settings.json",
"stage_settings": "~/.openflexure/stage_settings.json",
"plugins": [
"openflexure_microscope.plugins.default.autofocus:AutofocusPlugin",
"openflexure_microscope.plugins.default.scan:ScanPlugin",
"openflexure_microscope.plugins.default.camera_calibration:Plugin"
]
}

17
openflexure_microscope/microscope_settings.default.yaml
View file

@ -1,17 +0,0 @@
# Field of view, in stage steps
fov: [4100, 3146]
# Capture quality
jpeg_quality: 75
# Parameters specific to camera objects
camera_settings: ~/.openflexure/camera_settings.yaml
# Parameters specific to stage objects
stage_settings: ~/.openflexure/stage_settings.yaml
# Default plugins
plugins:
- openflexure_microscope.plugins.default.autofocus:AutofocusPlugin
- openflexure_microscope.plugins.default.scan:ScanPlugin
- openflexure_microscope.plugins.default.camera_calibration:Plugin

4
openflexure_microscope/stage/base.py
View file

@ -23,6 +23,10 @@ class BaseStage(metaclass=ABCMeta):
"""Return the current settings as a dictionary"""
pass
def save_config(self):
"""(Optional) Save any settings to disk that need to be stored"""
return
@property
@abstractmethod
def state(self):