Merge branch 'autostorage-test' into 'master'

Autostorage extension

See merge request openflexure/openflexure-microscope-server!41

openflexure_microscope/api/default_extensions/__init__.py

@@ -4,6 +4,7 @@ import traceback
 from .autofocus import autofocus_extension_v2
 from .scan import scan_extension_v2
 from .zip_builder import zip_extension_v2
+from .autostorage import autostorage_extension_v2
 
 # "Gracefully" handle cases where picamera cannot be imported (eg test server)
 try:

openflexure_microscope/api/default_extensions/autostorage.py

@@ -0,0 +1,270 @@
+from labthings.server.extensions import BaseExtension
+from labthings.server.view import View
+from labthings.server.decorators import ThingProperty, PropertySchema, use_args
+from labthings.server import fields
+from labthings.server.find import find_component
+
+from openflexure_microscope.paths import settings_file_path, check_rw
+from openflexure_microscope.config import OpenflexureSettingsFile
+from openflexure_microscope.camera.base import BASE_CAPTURE_PATH
+from openflexure_microscope.camera.capture import build_captures_from_exif
+
+from openflexure_microscope.api.utilities.gui import build_gui
+
+from flask import abort
+import logging
+import os
+import psutil
+
+from sys import platform
+
+
+AS_SETTINGS_PATH = settings_file_path("autostorage_settings.json")
+
+
+def get_partitions():
+    return [disk.mountpoint for disk in psutil.disk_partitions() if "rw" in disk.opts]
+
+
+def get_permissive_partitions():
+    return [partition for partition in get_partitions() if check_rw(partition)]
+
+
+def get_permissive_locations():
+    return [
+        (partition, os.path.join(partition, "openflexure", "data", "micrographs"))
+        for partition in get_permissive_partitions()
+    ]
+
+
+def get_current_location(camera):
+    return camera.paths.get("default")
+
+
+def set_current_location(camera, location: str):
+    if not os.path.isdir(location):
+        os.makedirs(location)
+    logging.debug("Updating location...")
+    camera.paths.update({"default": location})
+    logging.debug("Rebuilding captures...")
+    camera.rebuild_captures()
+    logging.debug("Capture location changed successfully.")
+
+
+def get_default_location():
+    return BASE_CAPTURE_PATH
+
+
+def get_all_locations():
+    locations = {}
+    # If default is not already listed (e.g. if it's currently set)
+    if get_default_location() not in locations.values():
+        locations["Default"] = get_default_location()
+
+    for ppartition, plocation in get_permissive_locations():
+        pdrive = os.path.splitdrive(plocation)[0]
+        if not (
+            pdrive  # If path actually has a drive (basically just Windows?)
+            and any(  # And shares a common drive with an existing location
+                [
+                    pdrive == os.path.splitdrive(location)[0]
+                    for location in locations.values()
+                ]
+            )
+        ):
+            locations[ppartition] = plocation
+
+    # Strip out Nones
+    return {k: v for k, v in locations.items() if v}
+
+
+class CaptureStorageLocation:
+    def __init__(self, mountpoint):
+        pass
+
+
+class AutostorageExtension(BaseExtension):
+    def __init__(self):
+        BaseExtension.__init__(
+            self,
+            "org.openflexure.autostorage",
+            version="2.0.0-beta.1",
+            description="Handle switching capture storage devices",
+        )
+
+        # We'll store a reference to a camera object, who's capture paths will be modified
+        self.camera = None
+
+        self.initial_location = get_default_location()
+
+        # Register the on_microscope function to run when the microscope is attached
+        self.on_component("org.openflexure.microscope", self.on_microscope)
+
+    def on_microscope(self, microscope_obj):
+        """Function to automatically call when the parent LabThing has a microscope attached."""
+        logging.debug(f"Autostorage extension found microscope {microscope_obj}")
+        if hasattr(microscope_obj, "camera"):
+            logging.debug(f"Autostorage extension bound to camera {self.camera}")
+
+            # Store a reference to the camera
+            self.camera = microscope_obj.camera
+            # Store the initial storage location
+            self.initial_location = get_current_location(self.camera)
+
+            # If preferred path does not exist, or cannot be written to
+            self.check_location(self.initial_location)
+
+            logging.debug(self.get_locations())
+
+    def check_location(self, location=None):
+        if not location:
+            location = get_current_location(self.camera)
+        # If preferred path does not exist, or cannot be written to
+        if not (os.path.isdir(location) and check_rw(location)):
+            logging.error(
+                f"Preferred capture path {location} is missing or cannot be written to. Restoring defaults."
+            )
+            # Reset the storage location to default
+            set_current_location(self.camera, get_default_location())
+
+    def get_locations(self):
+        if self.camera:
+            locations = get_all_locations()
+
+            current_location = get_current_location(self.camera)
+            if current_location not in locations.values():
+                locations.update({"Custom": current_location})
+            # Add location from the cameras settings file
+            return locations
+        else:
+            return {}
+
+    def get_preferred_key(self):
+        current = get_current_location(self.camera)
+        locations = self.get_locations()
+
+        matches = [k for k, v in locations.items() if v == current]
+
+        if len(matches) > 1:
+            logging.warning(
+                "Multiple path matches found. Weird, but carrying on using zeroth."
+            )
+
+        return matches[0]
+
+    def set_preferred_key(self, new_path_key: str):
+        if not new_path_key in self.get_locations().keys():
+            raise KeyError(f"No location named {new_path_key}")
+
+        location = self.get_locations().get(new_path_key)
+        set_current_location(self.camera, location)
+
+    def key_to_title(self, path_key: str):
+        if not path_key in self.get_locations().keys():
+            raise KeyError(f"No location named {path_key}")
+
+        return f"{path_key} ({self.get_locations().get(path_key)})"
+
+    def title_to_key(self, path_title: str):
+        matches = []
+        for loc_key in self.get_locations().keys():
+            if path_title.startswith(loc_key):
+                matches.append(loc_key)
+
+        if len(matches) > 1:
+            logging.warning(
+                "Multiple path matches found. Weird, but carrying on using zeroth."
+            )
+
+        return matches[0]
+
+    def get_titles(self):
+        return [self.key_to_title(key) for key in self.get_locations().keys()]
+
+    def get_preferred_title(self):
+        return self.key_to_title(self.get_preferred_key())
+
+
+autostorage_extension_v2 = AutostorageExtension()
+
+
+@ThingProperty
+class GetLocationsView(View):
+    def get(self):
+        global autostorage_extension_v2
+
+        autostorage_extension_v2.check_location()
+        return autostorage_extension_v2.get_locations()
+
+
+@ThingProperty
+@PropertySchema(fields.String(required=True, example="Default"))
+class PreferredLocationView(View):
+    def get(self):
+        global autostorage_extension_v2
+
+        autostorage_extension_v2.check_location()
+        return autostorage_extension_v2.get_preferred_key()
+
+    def post(self, new_path_key):
+        global autostorage_extension_v2
+        microscope = find_component("org.openflexure.microscope")
+
+        if not microscope:
+            abort(503, "No microscope connected. Unable to autofocus.")
+
+        autostorage_extension_v2.check_location()
+        autostorage_extension_v2.set_preferred_key(new_path_key)
+        microscope.save_settings()
+
+
+class PreferredLocationGUIView(View):
+    @use_args({"new_path_title": fields.String(required=True)})
+    def post(self, args):
+        global autostorage_extension_v2
+
+        new_path_title = args.get("new_path_title")
+        logging.debug(new_path_title)
+
+        new_path_key = autostorage_extension_v2.title_to_key(new_path_title)
+        logging.debug(f"{new_path_key}")
+
+        autostorage_extension_v2.check_location()
+        autostorage_extension_v2.set_preferred_key(new_path_key)
+
+        return new_path_title
+
+
+def dynamic_form():
+    global autostorage_extension_v2
+    autostorage_extension_v2.check_location()
+    return {
+        "icon": "sd_storage",
+        "forms": [
+            {
+                "name": "Autostorage",
+                "isCollapsible": False,
+                "isTask": False,
+                "route": "/location-from-title",
+                "emitOnResponse": "globalUpdateCaptures",
+                "submitLabel": "Set path",
+                "schema": [
+                    {
+                        "fieldType": "selectList",
+                        "name": "new_path_title",
+                        "label": "Capture storage path",
+                        "options": autostorage_extension_v2.get_titles(),
+                        "value": autostorage_extension_v2.get_preferred_title(),
+                    }
+                ],
+            }
+        ],
+    }
+
+
+autostorage_extension_v2.add_view(GetLocationsView, "list-locations")
+autostorage_extension_v2.add_view(PreferredLocationView, "location")
+autostorage_extension_v2.add_view(PreferredLocationGUIView, "location-from-title")
+autostorage_extension_v2.add_meta(
+    "gui", build_gui(dynamic_form, autostorage_extension_v2)
+)

openflexure_microscope/api/microscope.py

@@ -1,5 +1,4 @@
 from openflexure_microscope import Microscope
-from openflexure_microscope.camera.capture import build_captures_from_exif
 
 import logging
 
@@ -8,8 +7,6 @@ default_microscope = Microscope()
 
 # Restore loaded capture array to camera object
 logging.debug("Restoring captures...")
-default_microscope.camera.images = build_captures_from_exif(
+default_microscope.camera.rebuild_captures()
-    default_microscope.camera.paths["default"]
-)
 
 logging.debug("Microscope successfully attached!")

openflexure_microscope/api/utilities/gui.py

@@ -10,6 +10,11 @@ def build_gui_from_dict(gui_description, extension_object):
     # Expand shorthand routes into full relative URLs
     if "forms" in gui_description and isinstance(api_gui["forms"], list):
         for form in api_gui["forms"]:
+            # Clean leading slashes from rule
+            if "route" in form:
+                while form["route"][0] == "/":
+                    form["route"] = form["route"][1:]
+            # Match rule in extension object
             if "route" in form and form["route"] in extension_object._rules.keys():
                 form["route"] = extension_object._rules[form["route"]]["rule"]
             else:

openflexure_microscope/camera/base.py

@@ -8,7 +8,7 @@ import logging
 
 from abc import ABCMeta, abstractmethod
 
-from .capture import CaptureObject
+from .capture import CaptureObject, build_captures_from_exif
 from openflexure_microscope.utilities import entry_by_uuid
 from labthings.core.lock import StrictLock
 
@@ -177,6 +177,9 @@ class BaseCamera(metaclass=ABCMeta):
             shutil.rmtree(self.paths["temp"])
             logging.debug("Cleared {}.".format(self.paths["temp"]))
 
+    def rebuild_captures(self):
+        self.images = build_captures_from_exif(self.paths["default"])
+
     # START AND STOP WORKER THREAD
 
     def start_worker(self, timeout: int = 5) -> bool:

openflexure_microscope/config.py

@@ -32,7 +32,8 @@ class OpenflexureSettingsFile:
         self.path = path
 
         # Initialise basic config file with defaults if it doesn't exist
-        initialise_file(self.path, populate=defaults)
+        defaults_str = json.dumps(defaults, cls=JSONEncoder, indent=2, sort_keys=True)
+        initialise_file(self.path, populate=defaults_str)
 
     def load(self) -> dict:
         """

poetry.lock

@@ -229,9 +229,10 @@ marshmallow = "^3.4.0"
 webargs = "^5.5.3"
 
 [package.source]
-reference = "2a8213651083ad9387bf3f7f27e9b8ec64c6e2bc"
+reference = "12b6dda521f00e449c67c5ea8bf34005918fd649"
 type = "git"
 url = "https://github.com/labthings/python-labthings.git"
+
 [[package]]
 category = "dev"
 description = "A fast and thorough lazy object proxy."
@@ -307,6 +308,7 @@ test = ["coverage", "pytest", "mock", "pillow", "numpy"]
 reference = "b39c2b6e42f5f7f57bb46eafcb5c9e2bbdb5d0cb"
 type = "git"
 url = "https://github.com/rwb27/picamera.git"
+
 [[package]]
 category = "main"
 description = "Python Imaging Library (Fork)"
@@ -315,6 +317,17 @@ optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 version = "5.4.1"
 
+[[package]]
+category = "main"
+description = "Cross-platform lib for process and system monitoring in Python."
+name = "psutil"
+optional = false
+python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+version = "5.6.7"
+
+[package.extras]
+enum = ["enum34"]
+
 [[package]]
 category = "dev"
 description = "Pygments is a syntax highlighting package written in Python."
@@ -623,7 +636,7 @@ version = "1.11.2"
 rpi = ["picamera", "RPi.GPIO"]
 
 [metadata]
-content-hash = "03ea9560d58dababb9f7f86d533aa9effa087872bcbed58073923325e190ee32"
+content-hash = "3727711968c7246075a9b1e250158bcf546aab4654a0cf4fde8a77869180c29f"
 python-versions = "^3.6"
 
 [metadata.files]
@@ -755,11 +768,6 @@ markupsafe = [
     {file = "MarkupSafe-1.1.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6"},
     {file = "MarkupSafe-1.1.1-cp37-cp37m-win32.whl", hash = "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2"},
     {file = "MarkupSafe-1.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c"},
-    {file = "MarkupSafe-1.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15"},
-    {file = "MarkupSafe-1.1.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2"},
-    {file = "MarkupSafe-1.1.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42"},
-    {file = "MarkupSafe-1.1.1-cp38-cp38-win32.whl", hash = "sha256:596510de112c685489095da617b5bcbbac7dd6384aeebeda4df6025d0256a81b"},
-    {file = "MarkupSafe-1.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be"},
     {file = "MarkupSafe-1.1.1.tar.gz", hash = "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b"},
 ]
 marshmallow = [
@@ -851,6 +859,19 @@ pillow = [
     {file = "Pillow-5.4.1.win32-py3.6.exe", hash = "sha256:4132c78200372045bb348fcad8d52518c8f5cfc077b1089949381ee4a61f1c6d"},
     {file = "Pillow-5.4.1.win32-py3.7.exe", hash = "sha256:75d1f20bd8072eff92c5f457c266a61619a02d03ece56544195c56d41a1a0522"},
 ]
+psutil = [
+    {file = "psutil-5.6.7-cp27-none-win32.whl", hash = "sha256:1b1575240ca9a90b437e5a40db662acd87bbf181f6aa02f0204978737b913c6b"},
+    {file = "psutil-5.6.7-cp27-none-win_amd64.whl", hash = "sha256:28f771129bfee9fc6b63d83a15d857663bbdcae3828e1cb926e91320a9b5b5cd"},
+    {file = "psutil-5.6.7-cp35-cp35m-win32.whl", hash = "sha256:21231ef1c1a89728e29b98a885b8e0a8e00d09018f6da5cdc1f43f988471a995"},
+    {file = "psutil-5.6.7-cp35-cp35m-win_amd64.whl", hash = "sha256:b74b43fecce384a57094a83d2778cdfc2e2d9a6afaadd1ebecb2e75e0d34e10d"},
+    {file = "psutil-5.6.7-cp36-cp36m-win32.whl", hash = "sha256:e85f727ffb21539849e6012f47b12f6dd4c44965e56591d8dec6e8bc9ab96f4a"},
+    {file = "psutil-5.6.7-cp36-cp36m-win_amd64.whl", hash = "sha256:b560f5cd86cf8df7bcd258a851ca1ad98f0d5b8b98748e877a0aec4e9032b465"},
+    {file = "psutil-5.6.7-cp37-cp37m-win32.whl", hash = "sha256:094f899ac3ef72422b7e00411b4ed174e3c5a2e04c267db6643937ddba67a05b"},
+    {file = "psutil-5.6.7-cp37-cp37m-win_amd64.whl", hash = "sha256:fd2e09bb593ad9bdd7429e779699d2d47c1268cbde4dda95fcd1bd17544a0217"},
+    {file = "psutil-5.6.7-cp38-cp38-win32.whl", hash = "sha256:70387772f84fa5c3bb6a106915a2445e20ac8f9821c5914d7cbde148f4d7ff73"},
+    {file = "psutil-5.6.7-cp38-cp38-win_amd64.whl", hash = "sha256:10b7f75cc8bd676cfc6fa40cd7d5c25b3f45a0e06d43becd7c2d2871cbb5e806"},
+    {file = "psutil-5.6.7.tar.gz", hash = "sha256:ffad8eb2ac614518bbe3c0b8eb9dffdb3a8d2e3a7d5da51c5b974fb723a5c5aa"},
+]
 pygments = [
     {file = "Pygments-2.5.2-py2.py3-none-any.whl", hash = "sha256:2a3fe295e54a20164a9df49c75fa58526d3be48e14aceba6d6b1e8ac0bfd6f1b"},
     {file = "Pygments-2.5.2.tar.gz", hash = "sha256:98c8aa5a9f778fcd1026a17361ddaf7330d1b7c62ae97c3bb0ae73e0b9b6b0fe"},

pyproject.toml

@@ -35,6 +35,7 @@ pyserial = "^3.4" # Used for sangaboard (basic_serial_instrument) until we move
 
 labthings = { git = "https://github.com/labthings/python-labthings.git", branch = "master"} # TODO: Pin to a fixed version before 2.0.0 stable
 python-dateutil = "^2.8"
+psutil = "^5.6.7" # Autostorage extension
 
 [tool.poetry.extras]
 rpi = ["picamera", "RPi.GPIO"]