diff --git a/openflexure_microscope/captures/capture.py b/openflexure_microscope/captures/capture.py index e45f80cc..fcddc98f 100644 --- a/openflexure_microscope/captures/capture.py +++ b/openflexure_microscope/captures/capture.py @@ -18,29 +18,6 @@ EXIF_FORMATS = ["JPG", "JPEG", "TIF", "TIFF"] THUMBNAIL_SIZE = (200, 150) -def pull_usercomment_dict(filepath): - """ - Reads UserComment Exif data from a file, and returns the contained bytes as a dictionary. - Args: - filepath: Path to the Exif-containing file - """ - try: - exif_dict = piexif.load(filepath) - except InvalidImageDataError: - logging.warning("Invalid data at {}. Skipping.".format(filepath)) - return None - if "Exif" in exif_dict and piexif.ExifIFD.UserComment in exif_dict["Exif"]: - try: - return json.loads(exif_dict["Exif"][piexif.ExifIFD.UserComment].decode()) - except json.decoder.JSONDecodeError: - logging.error( - "Capture %s has old, corrupt, or missing OpenFlexure metadata. Unable to reload to server.", - filepath, - ) - else: - return None - - def make_file_list(directory, formats): files = [] for fmt in formats: @@ -63,8 +40,6 @@ def build_captures_from_exif(capture_path): capture = capture_from_path(f) if capture: captures[capture.id] = capture - else: - logging.error("Invalid data at {}. Skipping.".format(f)) logging.info("{} capture files successfully reloaded".format(len(captures))) @@ -72,37 +47,18 @@ def build_captures_from_exif(capture_path): def capture_from_path(path): - exif_dict = pull_usercomment_dict(path) - if exif_dict: - # Create a placeholder capture - capture = CaptureObject(filepath=path) + # Create a placeholder capture + capture = CaptureObject(filepath=path) - # Build file path information - capture.split_file_path(capture.file) - - # Image metadata - try: - image_metadata = exif_dict.pop("image") - except KeyError: - logging.error( - "Unable to obtain valid 2.0 OpenFlexure metadata from file %s", path - ) - return None - - # Populate capture parameters - capture.id = image_metadata.get("id") - capture.time = dateutil.parser.isoparse( - image_metadata.get("time") or image_metadata.get("acquisitionDate") - ) - capture.format = image_metadata.get("format") - capture.tags = image_metadata.get("tags") - capture.annotations = image_metadata.get("annotations") - - # Since we popped the "image" key, we dump whatever is left in _metadata - capture.set_metadata(exif_dict) + # Build file path information + capture.split_file_path(capture.file) + # Check and sync basic metadata + try: + capture.sync_basic_metadata() return capture - else: + except (InvalidImageDataError, json.decoder.JSONDecodeError): + logging.error("Invalid metadata at {}.".format(path)) return None @@ -131,10 +87,8 @@ class CaptureObject(object): if not os.path.exists(self.filefolder): os.makedirs(self.filefolder) - # Dictionary for adding top-level metadata - # This can ONLY be modified by the server application - # Top level metadata cannot be modified via the web API - self._metadata = {} + # Dataset information + self._dataset = None # Dictionary for storing custom annotations # Can be modified via the web API self.annotations = {} @@ -147,10 +101,12 @@ class CaptureObject(object): self.stream.write(s) def flush(self): - logging.info("Writing to disk %s", self.file) + logging.info("Writing image data to disk %s", self.file) with open(self.file, "wb") as outfile: outfile.write(self.stream.getbuffer()) self.stream.close() + logging.info("Writing metadata to disk %s", self.file) + self._init_metadata() logging.info("Finished writing to disk %s", self.file) def open(self, mode): @@ -169,6 +125,65 @@ class CaptureObject(object): self.basename = os.path.splitext(self.name)[0] self.format = self.name.split(".")[-1] + def _read_exif(self): + return piexif.load(self.file) + + def _decode_usercomment(self, exif_dict: dict): + if "Exif" not in exif_dict: + raise InvalidImageDataError + if piexif.ExifIFD.UserComment not in exif_dict["Exif"]: + return {} + return json.loads(exif_dict["Exif"][piexif.ExifIFD.UserComment].decode()) + + def _init_metadata(self): + self.put_and_save( + metadata={ + "image": { + "id": self.id, + "name": self.name, + "time": self.time.isoformat(), + "format": self.format, + "tags": self.tags, + "annotations": self.annotations, + } + } + ) + + def sync_basic_metadata(self): + exif_dict = self._read_exif() + + metadata_dict = self._decode_usercomment(exif_dict) or {} + image_metadata = metadata_dict.get("image") + + if not image_metadata: + raise InvalidImageDataError("No capture metadata found") + + self.id = image_metadata.get("id") + self.format = image_metadata.get("format") + self.time = dateutil.parser.isoparse( + image_metadata.get("time") or image_metadata.get("acquisitionDate") + ) + self.tags = image_metadata.get("tags") + self.annotations = image_metadata.get("annotations") + + dataset = metadata_dict.get("dataset") + if dataset: + self._dataset = { + "id": dataset.get("id"), + "name": dataset.get("name"), + "type": dataset.get("type"), + } + + def read_full_metadata(self): + logging.info("Reading full capture metadata from %s...", self.file) + exif_dict = self._read_exif() + return self._decode_usercomment(exif_dict) + + @property + def dataset(self): + """Read-only dataset property""" + return self._dataset + @property def exists(self) -> bool: """Check if capture data file exists on disk.""" @@ -177,21 +192,6 @@ class CaptureObject(object): else: return False - @property - def dataset(self) -> str: - """ - If capture is part of a dataset, return basic dataset info. - Otherwise return None - """ - dataset = self.metadata.get("dataset") - if not dataset: - return None - return { - "id": dataset.get("id"), - "name": dataset.get("name"), - "type": dataset.get("type"), - } - # HANDLE TAGS def put_tags(self, tags: list): """ @@ -200,11 +200,7 @@ class CaptureObject(object): Args: tags (list): List of tags to be added """ - for tag in tags: - if tag not in self.tags: - self.tags.append(tag) - - self.save_metadata() + self.put_and_save(tags=tags) def delete_tag(self, tag: str): """ @@ -213,10 +209,12 @@ class CaptureObject(object): Args: tag (str): Tag to be removed """ + # Update in-memory tag list if tag in self.tags: self.tags = [new_tag for new_tag in self.tags if new_tag != tag] - self.save_metadata() + # Write in-memory metadata to file + self.put_and_save() # HANDLE ANNOTATIONS @@ -227,13 +225,15 @@ class CaptureObject(object): Args: data (dict): Dictionary of metadata to be added """ - self.annotations.update(data) - self.save_metadata() + self.put_and_save(annotations=data) def delete_annotation(self, key: str) -> None: + # Update in-memory annotations list if key in self.annotations: del self.annotations[key] - self.save_metadata() + + # Write in-memory metadata to file + self.put_and_save() # HANDLE METADATA @@ -244,17 +244,7 @@ class CaptureObject(object): Args: data (dict): Dictionary of metadata to be added """ - self._metadata.update(data) - self.save_metadata() - - def set_metadata(self, data: dict) -> None: - """ - Write root metadata from a passed dictionary into the capture metadata. - - Args: - data (dict): Dictionary of metadata to be added - """ - self._metadata = data + self.put_and_save(metadata=data) # BULK OPERATIONS @@ -271,35 +261,42 @@ class CaptureObject(object): if not metadata: metadata = {} - # Tags + # Update in-memory tags array for tag in tags: if tag not in self.tags: self.tags.append(tag) - # Annotations + + # Update in-memory annotations dictionary self.annotations.update(annotations) - # Metadata - self._metadata.update(metadata) - self.save_metadata() - - def save_metadata(self) -> None: - """ - Save metadata to exif, if supported - """ + # Write new data to file EXIF, if supported if self.format.upper() in EXIF_FORMATS and self.exists: - logging.debug("Writing exif data to capture file") + logging.info("Writing Exif data to %s", self.file) + # Extract current Exif data - exif_dict = piexif.load(self.file) + exif_dict = self._read_exif() + metadata_dict = self._decode_usercomment(exif_dict) or {} + + # Add new tags to exif dictionary + metadata_dict.get("image", {})["tags"] = self.tags + # Add new annotations to exif dictionary + metadata_dict.get("image", {})["annotations"] = self.annotations + # Add new custom metadata to exif dictionary + metadata_dict.update(metadata) + # Serialize metadata - metadata_string = json.dumps(self.metadata, cls=JSONEncoder) + metadata_string = json.dumps(metadata_dict, cls=JSONEncoder) logging.debug("Saving metadata string to file: %s", metadata_string) + # Insert metadata into exif_dict exif_dict["Exif"][piexif.ExifIFD.UserComment] = metadata_string.encode() + # Convert new exif dict to exif bytes exif_bytes = piexif.dump(exif_dict) + # Insert exif into file piexif.insert(exif_bytes, self.file) - logging.info("Finished saving metadata to %s", self.file) + logging.info("Finished writing Exif data to %s", self.file) # PROPERTIES @@ -309,42 +306,13 @@ class CaptureObject(object): Create basic metadata dictionary from basic capture data, and any added custom metadata and tags. """ - d = { - "image": { - "id": self.id, - "name": self.name, - "time": self.time.isoformat(), - "format": self.format, - "tags": self.tags, - "annotations": self.annotations, - }, - **self._metadata, - } - # Add custom metadata to dictionary - return d - - @property - def state(self) -> dict: - """ - Return a dictionary of objects full state, including metadata. - """ - - # Create basic state dictionary - d = {"path": self.file, "name": self.name, "metadata": self.metadata} - - # Combined availability of data - if self.exists: - d["available"] = True - else: - d["available"] = False - - return d + return self.read_full_metadata() @property def data(self) -> io.BytesIO: """ - Return a byte string of the capture data. + Return a BytesIO object of the capture data. """ if self.exists: # If data file exists @@ -390,7 +358,7 @@ class CaptureObject(object): """Write stream to file, and save/update metadata file""" # If a stream OR file exists, save the metadata file if self.exists: - self.save_metadata() + self.put_and_save() def delete(self) -> bool: """If the StreamObject has been saved, delete the file.""" diff --git a/openflexure_microscope/rescue/auto.py b/openflexure_microscope/rescue/auto.py index 08625cd2..5bb95f22 100644 --- a/openflexure_microscope/rescue/auto.py +++ b/openflexure_microscope/rescue/auto.py @@ -1,12 +1,23 @@ import logging import os +import sys +import platform +import pkg_resources + +from .error_sources import bcolors from openflexure_microscope.paths import ( FALLBACK_OPENFLEXURE_VAR_PATH, PREFERRED_OPENFLEXURE_VAR_PATH, ) -from . import check_capture_reload, check_settings, check_picamera +from . import ( + check_capture_reload, + check_settings, + check_picamera, + check_sangaboard, + check_system, +) # Paths for suggestions LOGS_PATHS = [ @@ -26,55 +37,54 @@ DATA_PATHS = [ os.path.join(FALLBACK_OPENFLEXURE_VAR_PATH, "data"), ] +# Look for debug flag logger = logging.getLogger() -logger.setLevel(logging.DEBUG) -logging.debug("Testing debug logger. One two one two.") +if "-d" in sys.argv or "--debug" in sys.argv: + logger.setLevel(logging.DEBUG) + logging.debug("Testing debug logger. One two one two.") +else: + logger.setLevel(logging.WARNING) -class bcolors: - HEADER = "\033[95m" - OKBLUE = "\033[94m" - OKGREEN = "\033[92m" - WARNING = "\033[93m" - FAIL = "\033[91m" - ENDC = "\033[0m" - BOLD = "\033[1m" - UNDERLINE = "\033[4m" - - -error_keys = { - "config_settings_import_error": "The configuration submodule could not be imported properly. This is usually because the default config or settings files are badly broken somehow. See further errors for details", - "default_config_empty": "The default configuration file is empty, and could not be automatically populated.", - "default_settings_empty": "The default settings file is empty, and could not be automatically populated.", - "default_config_error": f"The default configuration file could not be parsed. To fix, consider backing up and deleting your configuration files from {CONFIG_PATHS} to reset the configuration.", - "default_settings_error": f"The default settings file could not be parsed. To fix, consider backing up and deleting your configuration files from {SETTINGS_PATHS} to reset the configuration.", - "capture_rebuild_timeout": f"Capture database rebuilding took a long time. This may not cause catastrophic errors, but rather will cause the server to hang for a while. To fix, consider moving your captures from {DATA_PATHS} to another location.", - "picamera_import_error": "Picamera module could not be imported. Check physical connections to the camera as it may be damaged." -} - if __name__ == "__main__": spoof = False + print() + print(bcolors.HEADER + "OpenFlexure Rescue" + bcolors.ENDC) + print() + print( + "This script attempts to identify common issues for a microscope not working properly." + ) + print( + "It is not designed to identify bugs in the code, but rather configuration or setup issues." + ) + print() + print("Any identified warnings [?] or errors [!] will be reported.") + print() + error_sources = [] - if spoof: - error_sources = list(error_keys.keys()) - + error_sources.extend(check_system.main()) error_sources.extend(check_settings.main()) error_sources.extend(check_picamera.main()) - #error_sources.extend(check_capture_reload.main()) + error_sources.extend(check_capture_reload.main()) + error_sources.extend(check_sangaboard.main()) + + dist = pkg_resources.get_distribution("openflexure-microscope-server") + + print() + print(f"Server Version: {dist.version}") + print(f"Platform: {platform.platform()}") if not error_sources: - print() - print(bcolors.OKGREEN + "No errors found!" + bcolors.ENDC) + print(bcolors.OKGREEN + "No issues found!" + bcolors.ENDC) print( "That's not to say everything is fine, only that our automatic diagnostics couldn't find much." ) print(f"You can check through the server logs at {LOGS_PATHS}") else: - for err_code in error_sources: - logging.error(err_code) - if error_keys.get(err_code): - print(bcolors.FAIL + error_keys.get(err_code) + bcolors.ENDC) + print() + for err in error_sources: + print(err.message) diff --git a/openflexure_microscope/rescue/check_capture_reload.py b/openflexure_microscope/rescue/check_capture_reload.py index bddfe28c..54459665 100644 --- a/openflexure_microscope/rescue/check_capture_reload.py +++ b/openflexure_microscope/rescue/check_capture_reload.py @@ -1,23 +1,18 @@ import logging -from openflexure_microscope.captures.capture import build_captures_from_exif +from openflexure_microscope.captures.capture import ( + build_captures_from_exif, + make_file_list, + EXIF_FORMATS, +) from openflexure_microscope.captures.capture_manager import BASE_CAPTURE_PATH from openflexure_microscope.config import user_settings from openflexure_microscope.rescue.monitor_timeout import launch_timeout_test_process +from .error_sources import ErrorSource, WarningSource -def check_capture_reload(): - logging.info("Loading user settings...") - settings = user_settings.load() - - cap_path = str(settings.get("captures", {}).get("paths", {}).get("default")) - logging.info("Capture path found: %s", cap_path) - if not cap_path: - logging.error( - "No capture path defined in settings. This is unusual for anything other than a first-run. \nFalling back to default path." - ) - cap_path = BASE_CAPTURE_PATH +def check_capture_reload(cap_path): logging.info("Starting capture reload with a 60 second timeout...") passed_timeout_test = launch_timeout_test_process( build_captures_from_exif, args=(cap_path,), timeout=60 @@ -28,7 +23,40 @@ def check_capture_reload(): def main(): error_sources = [] - passed_timeout = check_capture_reload() + + logging.info("Loading user settings...") + settings = user_settings.load() + cap_path = str(settings.get("captures", {}).get("paths", {}).get("default")) + logging.info("Capture path found: %s", cap_path) + + if not cap_path: + logging.error( + "No capture path defined in settings. This is unusual for anything other than a first-run. \nFalling back to default path." + ) + cap_path = BASE_CAPTURE_PATH + + # Check number of captures being restored + files = make_file_list(cap_path, EXIF_FORMATS) + if len(files) >= 10000: + error_sources.append( + WarningSource( + ( + "Over 10000 captures are being restored. This may slow down server startup.", + f"Consider moving your captures from {cap_path} to another location.", + ) + ) + ) + + # Check restore time of captures + passed_timeout = check_capture_reload(cap_path) if not passed_timeout: - error_sources.append("capture_rebuild_timeout") + error_sources.append( + ErrorSource( + ( + "Capture database rebuilding took a long time. This may not cause catastrophic errors, but rather will cause the server to hang for a while.", + f"To fix, consider moving your captures from {cap_path} to another location.", + ) + ) + ) + return error_sources diff --git a/openflexure_microscope/rescue/check_picamera.py b/openflexure_microscope/rescue/check_picamera.py index e5f3210a..9b21305d 100644 --- a/openflexure_microscope/rescue/check_picamera.py +++ b/openflexure_microscope/rescue/check_picamera.py @@ -1,15 +1,25 @@ import logging +from .error_sources import ErrorSource + +picamera_import_error = ErrorSource( + ( + "Picamera module could not be imported.", + "Check physical connections to the camera as it may be damaged or disconnected.", + ) +) + + def main(): error_sources = [] logging.info("Attempting to import picamera...") try: - from picamerax import PiCamera + from picamerax import PiCamera as _ except Exception as e: # pylint: disable=W0703 # TODO: Parse exception object for a few different issues, and append different error sources # E.g. pi with camera already in use, disconnected, unsupported OS, etc logging.error(e) - error_sources.append("picamera_import_error") + error_sources.append(picamera_import_error) - return error_sources \ No newline at end of file + return error_sources diff --git a/openflexure_microscope/rescue/check_sangaboard.py b/openflexure_microscope/rescue/check_sangaboard.py new file mode 100644 index 00000000..8c7f6aa7 --- /dev/null +++ b/openflexure_microscope/rescue/check_sangaboard.py @@ -0,0 +1,45 @@ +from .error_sources import ErrorSource + +from openflexure_microscope.config import user_configuration + + +def main(): + error_sources = [] + + try: + from sangaboard import Sangaboard + except ImportError as e: + error_sources.append(ErrorSource(str(e))) + else: + configuration = user_configuration.load() + stage_type = configuration["stage"].get("type") + stage_port = configuration["stage"].get("port") + + # If any Sangaboard-based stage is configured for use + if stage_type in ("SangaBoard", "SangaStage", "SangaDeltaStage"): + # Try connecting on the specified port + try: + stage = Sangaboard(stage_port) + except FileNotFoundError as e: + if stage_port: + error_sources.append( + ErrorSource( + f"No {stage_type} device was found on the configured port {stage_port}." + ) + ) + else: + error_sources.append( + ErrorSource( + f"No {stage_type} device was found during port scanning." + ) + ) + else: + stage.close() + else: + error_sources.append( + ErrorSource( + f"Invalid stage type {stage_type} specified in configuration file." + ) + ) + + return error_sources diff --git a/openflexure_microscope/rescue/check_system.py b/openflexure_microscope/rescue/check_system.py new file mode 100644 index 00000000..1c3f31ee --- /dev/null +++ b/openflexure_microscope/rescue/check_system.py @@ -0,0 +1,68 @@ +from urllib.request import urlopen +from urllib.error import URLError +import psutil + +from .error_sources import WarningSource + + +def internet_on(): + test_urls = ( + "https://build.openflexure.org", # Bath server + "https://openflexure.org", # GitHub pages + ) + for url in test_urls: + try: + urlopen(url, timeout=10) + # If any test passes, return True + return True + except URLError: + pass + # If no test passed, return False + return False + + +def main(): + error_sources = [] + + # Check internet + if not internet_on(): + error_sources.append( + WarningSource( + "No internet connection detected. Updates may not be available." + ) + ) + + # Check memory + mem = psutil.virtual_memory() + total_gb = mem.total / 1e9 + + if total_gb <= 1: + error_sources.append( + WarningSource( + ( + "Less than 1GB total memory available.", + "For small scans, or control from another device, this is usually fine.", + "\n More complex usage may require additional resources.", + ) + ) + ) + + # Check disks + data_partitions = [ + disk + for disk in psutil.disk_partitions() + if "rw" in disk.opts and disk.mountpoint != "/boot" + ] + for part in data_partitions: + usage = psutil.disk_usage(part.mountpoint) + if int(usage.percent) >= 90: + error_sources.append( + WarningSource( + ( + f"Disk {part.device}, at {part.mountpoint} is {int(usage.percent)}% full.", + "Captures may fail to save soon.", + ) + ) + ) + + return error_sources diff --git a/openflexure_microscope/rescue/error_sources.py b/openflexure_microscope/rescue/error_sources.py new file mode 100644 index 00000000..67910701 --- /dev/null +++ b/openflexure_microscope/rescue/error_sources.py @@ -0,0 +1,33 @@ +class bcolors: + HEADER = "\033[95m" + OKBLUE = "\033[94m" + OKGREEN = "\033[92m" + WARNING = "\033[93m" + FAIL = "\033[91m" + ENDC = "\033[0m" + BOLD = "\033[1m" + UNDERLINE = "\033[4m" + + +class Source: + def __init__(self, message): + if isinstance(message, tuple): + self._message = " ".join(message) + else: + self._message = message + + @property + def message(self): + return self._message + + +class WarningSource(Source): + @property + def message(self): + return "[?] " + bcolors.WARNING + self._message + bcolors.ENDC + + +class ErrorSource(Source): + @property + def message(self): + return "[!] " + bcolors.FAIL + self._message + bcolors.ENDC