rahix/openflexure-microscope-server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'better-rescue' into 'master'

Browse source 
Better rescue

See merge request openflexure/openflexure-microscope-server!81
This commit is contained in:
Joel Collins 2020-11-06 17:36:28 +00:00
commit 065825dfdd
7 changed files with 352 additions and 190 deletions
Download patch file Download diff file Expand all files Collapse all files

246
openflexure_microscope/captures/capture.py
View file

@ -18,29 +18,6 @@ EXIF_FORMATS = ["JPG", "JPEG", "TIF", "TIFF"]
THUMBNAIL_SIZE = (200, 150)
def pull_usercomment_dict(filepath):
"""
Reads UserComment Exif data from a file, and returns the contained bytes as a dictionary.
Args:
filepath: Path to the Exif-containing file
"""
try:
exif_dict = piexif.load(filepath)
except InvalidImageDataError:
logging.warning("Invalid data at {}. Skipping.".format(filepath))
return None
if "Exif" in exif_dict and piexif.ExifIFD.UserComment in exif_dict["Exif"]:
try:
return json.loads(exif_dict["Exif"][piexif.ExifIFD.UserComment].decode())
except json.decoder.JSONDecodeError:
logging.error(
"Capture %s has old, corrupt, or missing OpenFlexure metadata. Unable to reload to server.",
filepath,
)
else:
return None
def make_file_list(directory, formats):
files = []
for fmt in formats:
@ -63,8 +40,6 @@ def build_captures_from_exif(capture_path):
capture = capture_from_path(f)
if capture:
captures[capture.id] = capture
else:
logging.error("Invalid data at {}. Skipping.".format(f))
logging.info("{} capture files successfully reloaded".format(len(captures)))
@ -72,37 +47,18 @@ def build_captures_from_exif(capture_path):
def capture_from_path(path):
exif_dict = pull_usercomment_dict(path)
if exif_dict:
# Create a placeholder capture
capture = CaptureObject(filepath=path)
# Create a placeholder capture
capture = CaptureObject(filepath=path)
# Build file path information
capture.split_file_path(capture.file)
# Image metadata
try:
image_metadata = exif_dict.pop("image")
except KeyError:
logging.error(
"Unable to obtain valid 2.0 OpenFlexure metadata from file %s", path
)
return None
# Populate capture parameters
capture.id = image_metadata.get("id")
capture.time = dateutil.parser.isoparse(
image_metadata.get("time") or image_metadata.get("acquisitionDate")
)
capture.format = image_metadata.get("format")
capture.tags = image_metadata.get("tags")
capture.annotations = image_metadata.get("annotations")
# Since we popped the "image" key, we dump whatever is left in _metadata
capture.set_metadata(exif_dict)
# Build file path information
capture.split_file_path(capture.file)
# Check and sync basic metadata
try:
capture.sync_basic_metadata()
return capture
else:
except (InvalidImageDataError, json.decoder.JSONDecodeError):
logging.error("Invalid metadata at {}.".format(path))
return None
@ -131,10 +87,8 @@ class CaptureObject(object):
if not os.path.exists(self.filefolder):
os.makedirs(self.filefolder)
# Dictionary for adding top-level metadata
# This can ONLY be modified by the server application
# Top level metadata cannot be modified via the web API
self._metadata = {}
# Dataset information
self._dataset = None
# Dictionary for storing custom annotations
# Can be modified via the web API
self.annotations = {}
@ -147,10 +101,12 @@ class CaptureObject(object):
self.stream.write(s)
def flush(self):
logging.info("Writing to disk %s", self.file)
logging.info("Writing image data to disk %s", self.file)
with open(self.file, "wb") as outfile:
outfile.write(self.stream.getbuffer())
self.stream.close()
logging.info("Writing metadata to disk %s", self.file)
self._init_metadata()
logging.info("Finished writing to disk %s", self.file)
def open(self, mode):
@ -169,6 +125,65 @@ class CaptureObject(object):
self.basename = os.path.splitext(self.name)[0]
self.format = self.name.split(".")[-1]
def _read_exif(self):
return piexif.load(self.file)
def _decode_usercomment(self, exif_dict: dict):
if "Exif" not in exif_dict:
raise InvalidImageDataError
if piexif.ExifIFD.UserComment not in exif_dict["Exif"]:
return {}
return json.loads(exif_dict["Exif"][piexif.ExifIFD.UserComment].decode())
def _init_metadata(self):
self.put_and_save(
metadata={
"image": {
"id": self.id,
"name": self.name,
"time": self.time.isoformat(),
"format": self.format,
"tags": self.tags,
"annotations": self.annotations,
}
}
)
def sync_basic_metadata(self):
exif_dict = self._read_exif()
metadata_dict = self._decode_usercomment(exif_dict) or {}
image_metadata = metadata_dict.get("image")
if not image_metadata:
raise InvalidImageDataError("No capture metadata found")
self.id = image_metadata.get("id")
self.format = image_metadata.get("format")
self.time = dateutil.parser.isoparse(
image_metadata.get("time") or image_metadata.get("acquisitionDate")
)
self.tags = image_metadata.get("tags")
self.annotations = image_metadata.get("annotations")
dataset = metadata_dict.get("dataset")
if dataset:
self._dataset = {
"id": dataset.get("id"),
"name": dataset.get("name"),
"type": dataset.get("type"),
}
def read_full_metadata(self):
logging.info("Reading full capture metadata from %s...", self.file)
exif_dict = self._read_exif()
return self._decode_usercomment(exif_dict)
@property
def dataset(self):
"""Read-only dataset property"""
return self._dataset
@property
def exists(self) -> bool:
"""Check if capture data file exists on disk."""
@ -177,21 +192,6 @@ class CaptureObject(object):
else:
return False
@property
def dataset(self) -> str:
"""
If capture is part of a dataset, return basic dataset info.
Otherwise return None
"""
dataset = self.metadata.get("dataset")
if not dataset:
return None
return {
"id": dataset.get("id"),
"name": dataset.get("name"),
"type": dataset.get("type"),
}
# HANDLE TAGS
def put_tags(self, tags: list):
"""
@ -200,11 +200,7 @@ class CaptureObject(object):
Args:
tags (list): List of tags to be added
"""
for tag in tags:
if tag not in self.tags:
self.tags.append(tag)
self.save_metadata()
self.put_and_save(tags=tags)
def delete_tag(self, tag: str):
"""
@ -213,10 +209,12 @@ class CaptureObject(object):
Args:
tag (str): Tag to be removed
"""
# Update in-memory tag list
if tag in self.tags:
self.tags = [new_tag for new_tag in self.tags if new_tag != tag]
self.save_metadata()
# Write in-memory metadata to file
self.put_and_save()
# HANDLE ANNOTATIONS
@ -227,13 +225,15 @@ class CaptureObject(object):
Args:
data (dict): Dictionary of metadata to be added
"""
self.annotations.update(data)
self.save_metadata()
self.put_and_save(annotations=data)
def delete_annotation(self, key: str) -> None:
# Update in-memory annotations list
if key in self.annotations:
del self.annotations[key]
self.save_metadata()
# Write in-memory metadata to file
self.put_and_save()
# HANDLE METADATA
@ -244,17 +244,7 @@ class CaptureObject(object):
Args:
data (dict): Dictionary of metadata to be added
"""
self._metadata.update(data)
self.save_metadata()
def set_metadata(self, data: dict) -> None:
"""
Write root metadata from a passed dictionary into the capture metadata.
Args:
data (dict): Dictionary of metadata to be added
"""
self._metadata = data
self.put_and_save(metadata=data)
# BULK OPERATIONS
@ -271,35 +261,42 @@ class CaptureObject(object):
if not metadata:
metadata = {}
# Tags
# Update in-memory tags array
for tag in tags:
if tag not in self.tags:
self.tags.append(tag)
# Annotations
# Update in-memory annotations dictionary
self.annotations.update(annotations)
# Metadata
self._metadata.update(metadata)
self.save_metadata()
def save_metadata(self) -> None:
"""
Save metadata to exif, if supported
"""
# Write new data to file EXIF, if supported
if self.format.upper() in EXIF_FORMATS and self.exists:
logging.debug("Writing exif data to capture file")
logging.info("Writing Exif data to %s", self.file)
# Extract current Exif data
exif_dict = piexif.load(self.file)
exif_dict = self._read_exif()
metadata_dict = self._decode_usercomment(exif_dict) or {}
# Add new tags to exif dictionary
metadata_dict.get("image", {})["tags"] = self.tags
# Add new annotations to exif dictionary
metadata_dict.get("image", {})["annotations"] = self.annotations
# Add new custom metadata to exif dictionary
metadata_dict.update(metadata)
# Serialize metadata
metadata_string = json.dumps(self.metadata, cls=JSONEncoder)
metadata_string = json.dumps(metadata_dict, cls=JSONEncoder)
logging.debug("Saving metadata string to file: %s", metadata_string)
# Insert metadata into exif_dict
exif_dict["Exif"][piexif.ExifIFD.UserComment] = metadata_string.encode()
# Convert new exif dict to exif bytes
exif_bytes = piexif.dump(exif_dict)
# Insert exif into file
piexif.insert(exif_bytes, self.file)
logging.info("Finished saving metadata to %s", self.file)
logging.info("Finished writing Exif data to %s", self.file)
# PROPERTIES
@ -309,42 +306,13 @@ class CaptureObject(object):
Create basic metadata dictionary from basic capture data,
and any added custom metadata and tags.
"""
d = {
"image": {
"id": self.id,
"name": self.name,
"time": self.time.isoformat(),
"format": self.format,
"tags": self.tags,
"annotations": self.annotations,
},
**self._metadata,
}
# Add custom metadata to dictionary
return d
@property
def state(self) -> dict:
"""
Return a dictionary of objects full state, including metadata.
"""
# Create basic state dictionary
d = {"path": self.file, "name": self.name, "metadata": self.metadata}
# Combined availability of data
if self.exists:
d["available"] = True
else:
d["available"] = False
return d
return self.read_full_metadata()
@property
def data(self) -> io.BytesIO:
"""
Return a byte string of the capture data.
Return a BytesIO object of the capture data.
"""
if self.exists: # If data file exists
@ -390,7 +358,7 @@ class CaptureObject(object):
"""Write stream to file, and save/update metadata file"""
# If a stream OR file exists, save the metadata file
if self.exists:
self.save_metadata()
self.put_and_save()
def delete(self) -> bool:
"""If the StreamObject has been saved, delete the file."""

78
openflexure_microscope/rescue/auto.py
View file

@ -1,12 +1,23 @@
import logging
import os
import sys
import platform
import pkg_resources
from .error_sources import bcolors
from openflexure_microscope.paths import (
FALLBACK_OPENFLEXURE_VAR_PATH,
PREFERRED_OPENFLEXURE_VAR_PATH,
)
from . import check_capture_reload, check_settings, check_picamera
from . import (
check_capture_reload,
check_settings,
check_picamera,
check_sangaboard,
check_system,
)
# Paths for suggestions
LOGS_PATHS = [
@ -26,55 +37,54 @@ DATA_PATHS = [
os.path.join(FALLBACK_OPENFLEXURE_VAR_PATH, "data"),
]
# Look for debug flag
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
logging.debug("Testing debug logger. One two one two.")
if "-d" in sys.argv or "--debug" in sys.argv:
logger.setLevel(logging.DEBUG)
logging.debug("Testing debug logger. One two one two.")
else:
logger.setLevel(logging.WARNING)
class bcolors:
HEADER = "\033[95m"
OKBLUE = "\033[94m"
OKGREEN = "\033[92m"
WARNING = "\033[93m"
FAIL = "\033[91m"
ENDC = "\033[0m"
BOLD = "\033[1m"
UNDERLINE = "\033[4m"
error_keys = {
"config_settings_import_error": "The configuration submodule could not be imported properly. This is usually because the default config or settings files are badly broken somehow. See further errors for details",
"default_config_empty": "The default configuration file is empty, and could not be automatically populated.",
"default_settings_empty": "The default settings file is empty, and could not be automatically populated.",
"default_config_error": f"The default configuration file could not be parsed. To fix, consider backing up and deleting your configuration files from {CONFIG_PATHS} to reset the configuration.",
"default_settings_error": f"The default settings file could not be parsed. To fix, consider backing up and deleting your configuration files from {SETTINGS_PATHS} to reset the configuration.",
"capture_rebuild_timeout": f"Capture database rebuilding took a long time. This may not cause catastrophic errors, but rather will cause the server to hang for a while. To fix, consider moving your captures from {DATA_PATHS} to another location.",
"picamera_import_error": "Picamera module could not be imported. Check physical connections to the camera as it may be damaged."
}
if __name__ == "__main__":
spoof = False
print()
print(bcolors.HEADER + "OpenFlexure Rescue" + bcolors.ENDC)
print()
print(
"This script attempts to identify common issues for a microscope not working properly."
)
print(
"It is not designed to identify bugs in the code, but rather configuration or setup issues."
)
print()
print("Any identified warnings [?] or errors [!] will be reported.")
print()
error_sources = []
if spoof:
error_sources = list(error_keys.keys())
error_sources.extend(check_system.main())
error_sources.extend(check_settings.main())
error_sources.extend(check_picamera.main())
#error_sources.extend(check_capture_reload.main())
error_sources.extend(check_capture_reload.main())
error_sources.extend(check_sangaboard.main())
dist = pkg_resources.get_distribution("openflexure-microscope-server")
print()
print(f"Server Version: {dist.version}")
print(f"Platform: {platform.platform()}")
if not error_sources:
print()
print(bcolors.OKGREEN + "No errors found!" + bcolors.ENDC)
print(bcolors.OKGREEN + "No issues found!" + bcolors.ENDC)
print(
"That's not to say everything is fine, only that our automatic diagnostics couldn't find much."
)
print(f"You can check through the server logs at {LOGS_PATHS}")
else:
for err_code in error_sources:
logging.error(err_code)
if error_keys.get(err_code):
print(bcolors.FAIL + error_keys.get(err_code) + bcolors.ENDC)
print()
for err in error_sources:
print(err.message)

56
openflexure_microscope/rescue/check_capture_reload.py
View file

@ -1,23 +1,18 @@
import logging
from openflexure_microscope.captures.capture import build_captures_from_exif
from openflexure_microscope.captures.capture import (
build_captures_from_exif,
make_file_list,
EXIF_FORMATS,
)
from openflexure_microscope.captures.capture_manager import BASE_CAPTURE_PATH
from openflexure_microscope.config import user_settings
from openflexure_microscope.rescue.monitor_timeout import launch_timeout_test_process
from .error_sources import ErrorSource, WarningSource
def check_capture_reload():
logging.info("Loading user settings...")
settings = user_settings.load()
cap_path = str(settings.get("captures", {}).get("paths", {}).get("default"))
logging.info("Capture path found: %s", cap_path)
if not cap_path:
logging.error(
"No capture path defined in settings. This is unusual for anything other than a first-run. \nFalling back to default path."
)
cap_path = BASE_CAPTURE_PATH
def check_capture_reload(cap_path):
logging.info("Starting capture reload with a 60 second timeout...")
passed_timeout_test = launch_timeout_test_process(
build_captures_from_exif, args=(cap_path,), timeout=60
@ -28,7 +23,40 @@ def check_capture_reload():
def main():
error_sources = []
passed_timeout = check_capture_reload()
logging.info("Loading user settings...")
settings = user_settings.load()
cap_path = str(settings.get("captures", {}).get("paths", {}).get("default"))
logging.info("Capture path found: %s", cap_path)
if not cap_path:
logging.error(
"No capture path defined in settings. This is unusual for anything other than a first-run. \nFalling back to default path."
)
cap_path = BASE_CAPTURE_PATH
# Check number of captures being restored
files = make_file_list(cap_path, EXIF_FORMATS)
if len(files) >= 10000:
error_sources.append(
WarningSource(
(
"Over 10000 captures are being restored. This may slow down server startup.",
f"Consider moving your captures from {cap_path} to another location.",
)
)
)
# Check restore time of captures
passed_timeout = check_capture_reload(cap_path)
if not passed_timeout:
error_sources.append("capture_rebuild_timeout")
error_sources.append(
ErrorSource(
(
"Capture database rebuilding took a long time. This may not cause catastrophic errors, but rather will cause the server to hang for a while.",
f"To fix, consider moving your captures from {cap_path} to another location.",
)
)
)
return error_sources

16
openflexure_microscope/rescue/check_picamera.py
View file

@ -1,15 +1,25 @@
import logging
from .error_sources import ErrorSource
picamera_import_error = ErrorSource(
(
"Picamera module could not be imported.",
"Check physical connections to the camera as it may be damaged or disconnected.",
)
)
def main():
error_sources = []
logging.info("Attempting to import picamera...")
try:
from picamerax import PiCamera
from picamerax import PiCamera as _
except Exception as e: # pylint: disable=W0703
# TODO: Parse exception object for a few different issues, and append different error sources
# E.g. pi with camera already in use, disconnected, unsupported OS, etc
logging.error(e)
error_sources.append("picamera_import_error")
error_sources.append(picamera_import_error)
return error_sources
return error_sources

45
openflexure_microscope/rescue/check_sangaboard.py Normal file
View file

@ -0,0 +1,45 @@
from .error_sources import ErrorSource
from openflexure_microscope.config import user_configuration
def main():
error_sources = []
try:
from sangaboard import Sangaboard
except ImportError as e:
error_sources.append(ErrorSource(str(e)))
else:
configuration = user_configuration.load()
stage_type = configuration["stage"].get("type")
stage_port = configuration["stage"].get("port")
# If any Sangaboard-based stage is configured for use
if stage_type in ("SangaBoard", "SangaStage", "SangaDeltaStage"):
# Try connecting on the specified port
try:
stage = Sangaboard(stage_port)
except FileNotFoundError as e:
if stage_port:
error_sources.append(
ErrorSource(
f"No {stage_type} device was found on the configured port {stage_port}."
)
)
else:
error_sources.append(
ErrorSource(
f"No {stage_type} device was found during port scanning."
)
)
else:
stage.close()
else:
error_sources.append(
ErrorSource(
f"Invalid stage type {stage_type} specified in configuration file."
)
)
return error_sources

68
openflexure_microscope/rescue/check_system.py Normal file
View file

@ -0,0 +1,68 @@
from urllib.request import urlopen
from urllib.error import URLError
import psutil
from .error_sources import WarningSource
def internet_on():
test_urls = (
"https://build.openflexure.org", # Bath server
"https://openflexure.org", # GitHub pages
)
for url in test_urls:
try:
urlopen(url, timeout=10)
# If any test passes, return True
return True
except URLError:
pass
# If no test passed, return False
return False
def main():
error_sources = []
# Check internet
if not internet_on():
error_sources.append(
WarningSource(
"No internet connection detected. Updates may not be available."
)
)
# Check memory
mem = psutil.virtual_memory()
total_gb = mem.total / 1e9
if total_gb <= 1:
error_sources.append(
WarningSource(
(
"Less than 1GB total memory available.",
"For small scans, or control from another device, this is usually fine.",
"\n More complex usage may require additional resources.",
)
)
)
# Check disks
data_partitions = [
disk
for disk in psutil.disk_partitions()
if "rw" in disk.opts and disk.mountpoint != "/boot"
]
for part in data_partitions:
usage = psutil.disk_usage(part.mountpoint)
if int(usage.percent) >= 90:
error_sources.append(
WarningSource(
(
f"Disk {part.device}, at {part.mountpoint} is {int(usage.percent)}% full.",
"Captures may fail to save soon.",
)
)
)
return error_sources

33
openflexure_microscope/rescue/error_sources.py Normal file
View file

@ -0,0 +1,33 @@
class bcolors:
HEADER = "\033[95m"
OKBLUE = "\033[94m"
OKGREEN = "\033[92m"
WARNING = "\033[93m"
FAIL = "\033[91m"
ENDC = "\033[0m"
BOLD = "\033[1m"
UNDERLINE = "\033[4m"
class Source:
def __init__(self, message):
if isinstance(message, tuple):
self._message = " ".join(message)
else:
self._message = message
@property
def message(self):
return self._message
class WarningSource(Source):
@property
def message(self):
return "[?] " + bcolors.WARNING + self._message + bcolors.ENDC
class ErrorSource(Source):
@property
def message(self):
return "[!] " + bcolors.FAIL + self._message + bcolors.ENDC