From 9dab2421a430dabe6d057a900f6f99b22f0a85a0 Mon Sep 17 00:00:00 2001 From: Joel Collins Date: Fri, 6 Nov 2020 12:09:05 +0000 Subject: [PATCH 01/17] Removed in-memory full metadata from CaptureObject --- openflexure_microscope/captures/capture.py | 244 +++++++++------------ 1 file changed, 107 insertions(+), 137 deletions(-) diff --git a/openflexure_microscope/captures/capture.py b/openflexure_microscope/captures/capture.py index e45f80cc..2ad8da37 100644 --- a/openflexure_microscope/captures/capture.py +++ b/openflexure_microscope/captures/capture.py @@ -18,29 +18,6 @@ EXIF_FORMATS = ["JPG", "JPEG", "TIF", "TIFF"] THUMBNAIL_SIZE = (200, 150) -def pull_usercomment_dict(filepath): - """ - Reads UserComment Exif data from a file, and returns the contained bytes as a dictionary. - Args: - filepath: Path to the Exif-containing file - """ - try: - exif_dict = piexif.load(filepath) - except InvalidImageDataError: - logging.warning("Invalid data at {}. Skipping.".format(filepath)) - return None - if "Exif" in exif_dict and piexif.ExifIFD.UserComment in exif_dict["Exif"]: - try: - return json.loads(exif_dict["Exif"][piexif.ExifIFD.UserComment].decode()) - except json.decoder.JSONDecodeError: - logging.error( - "Capture %s has old, corrupt, or missing OpenFlexure metadata. Unable to reload to server.", - filepath, - ) - else: - return None - - def make_file_list(directory, formats): files = [] for fmt in formats: @@ -72,37 +49,18 @@ def build_captures_from_exif(capture_path): def capture_from_path(path): - exif_dict = pull_usercomment_dict(path) - if exif_dict: - # Create a placeholder capture - capture = CaptureObject(filepath=path) + # Create a placeholder capture + capture = CaptureObject(filepath=path) - # Build file path information - capture.split_file_path(capture.file) - - # Image metadata - try: - image_metadata = exif_dict.pop("image") - except KeyError: - logging.error( - "Unable to obtain valid 2.0 OpenFlexure metadata from file %s", path - ) - return None - - # Populate capture parameters - capture.id = image_metadata.get("id") - capture.time = dateutil.parser.isoparse( - image_metadata.get("time") or image_metadata.get("acquisitionDate") - ) - capture.format = image_metadata.get("format") - capture.tags = image_metadata.get("tags") - capture.annotations = image_metadata.get("annotations") - - # Since we popped the "image" key, we dump whatever is left in _metadata - capture.set_metadata(exif_dict) + # Build file path information + capture.split_file_path(capture.file) + # Check and sync basic metadata + try: + capture.sync_basic_metadata() return capture - else: + except (InvalidImageDataError, json.decoder.JSONDecodeError): + logging.error("Invalid metadata at {}. Skipping.".format(path)) return None @@ -131,10 +89,8 @@ class CaptureObject(object): if not os.path.exists(self.filefolder): os.makedirs(self.filefolder) - # Dictionary for adding top-level metadata - # This can ONLY be modified by the server application - # Top level metadata cannot be modified via the web API - self._metadata = {} + # Dataset information + self._dataset = None # Dictionary for storing custom annotations # Can be modified via the web API self.annotations = {} @@ -147,10 +103,12 @@ class CaptureObject(object): self.stream.write(s) def flush(self): - logging.info("Writing to disk %s", self.file) + logging.info("Writing image data to disk %s", self.file) with open(self.file, "wb") as outfile: outfile.write(self.stream.getbuffer()) self.stream.close() + logging.info("Writing metadata to disk %s", self.file) + self._init_metadata() logging.info("Finished writing to disk %s", self.file) def open(self, mode): @@ -169,6 +127,65 @@ class CaptureObject(object): self.basename = os.path.splitext(self.name)[0] self.format = self.name.split(".")[-1] + def _read_exif(self): + return piexif.load(self.file) + + def _decode_usercomment(self, exif_dict: dict): + if "Exif" not in exif_dict: + raise InvalidImageDataError + if piexif.ExifIFD.UserComment not in exif_dict["Exif"]: + return {} + return json.loads(exif_dict["Exif"][piexif.ExifIFD.UserComment].decode()) + + def _init_metadata(self): + self.put_and_save( + metadata={ + "image": { + "id": self.id, + "name": self.name, + "time": self.time.isoformat(), + "format": self.format, + "tags": self.tags, + "annotations": self.annotations, + } + } + ) + + def sync_basic_metadata(self): + exif_dict = self._read_exif() + + metadata_dict = self._decode_usercomment(exif_dict) or {} + image_metadata = metadata_dict.get("image") + + if not image_metadata: + raise InvalidImageDataError("No capture metadata found") + + self.id = image_metadata.get("id") + self.format = image_metadata.get("format") + self.time = dateutil.parser.isoparse( + image_metadata.get("time") or image_metadata.get("acquisitionDate") + ) + self.tags = image_metadata.get("tags") + self.annotations = image_metadata.get("annotations") + + dataset = metadata_dict.get("dataset") + if dataset: + self._dataset = { + "id": dataset.get("id"), + "name": dataset.get("name"), + "type": dataset.get("type"), + } + + def read_full_metadata(self): + logging.info("Reading full capture metadata from %s...", self.file) + exif_dict = self._read_exif() + return self._decode_usercomment(exif_dict) + + @property + def dataset(self): + """Read-only dataset property""" + return self._dataset + @property def exists(self) -> bool: """Check if capture data file exists on disk.""" @@ -177,21 +194,6 @@ class CaptureObject(object): else: return False - @property - def dataset(self) -> str: - """ - If capture is part of a dataset, return basic dataset info. - Otherwise return None - """ - dataset = self.metadata.get("dataset") - if not dataset: - return None - return { - "id": dataset.get("id"), - "name": dataset.get("name"), - "type": dataset.get("type"), - } - # HANDLE TAGS def put_tags(self, tags: list): """ @@ -200,11 +202,7 @@ class CaptureObject(object): Args: tags (list): List of tags to be added """ - for tag in tags: - if tag not in self.tags: - self.tags.append(tag) - - self.save_metadata() + self.put_and_save(tags=tags) def delete_tag(self, tag: str): """ @@ -213,10 +211,12 @@ class CaptureObject(object): Args: tag (str): Tag to be removed """ + # Update in-memory tag list if tag in self.tags: self.tags = [new_tag for new_tag in self.tags if new_tag != tag] - self.save_metadata() + # Write in-memory metadata to file + self.put_and_save() # HANDLE ANNOTATIONS @@ -227,13 +227,15 @@ class CaptureObject(object): Args: data (dict): Dictionary of metadata to be added """ - self.annotations.update(data) - self.save_metadata() + self.put_and_save(annotations=data) def delete_annotation(self, key: str) -> None: + # Update in-memory annotations list if key in self.annotations: del self.annotations[key] - self.save_metadata() + + # Write in-memory metadata to file + self.put_and_save() # HANDLE METADATA @@ -244,17 +246,7 @@ class CaptureObject(object): Args: data (dict): Dictionary of metadata to be added """ - self._metadata.update(data) - self.save_metadata() - - def set_metadata(self, data: dict) -> None: - """ - Write root metadata from a passed dictionary into the capture metadata. - - Args: - data (dict): Dictionary of metadata to be added - """ - self._metadata = data + self.put_and_save(metadata=data) # BULK OPERATIONS @@ -271,35 +263,42 @@ class CaptureObject(object): if not metadata: metadata = {} - # Tags + # Update in-memory tags array for tag in tags: if tag not in self.tags: self.tags.append(tag) - # Annotations + + # Update in-memory annotations dictionary self.annotations.update(annotations) - # Metadata - self._metadata.update(metadata) - self.save_metadata() - - def save_metadata(self) -> None: - """ - Save metadata to exif, if supported - """ + # Write new data to file EXIF, if supported if self.format.upper() in EXIF_FORMATS and self.exists: - logging.debug("Writing exif data to capture file") + logging.info("Writing Exif data to %s", self.file) + # Extract current Exif data - exif_dict = piexif.load(self.file) + exif_dict = self._read_exif() + metadata_dict = self._decode_usercomment(exif_dict) or {} + + # Add new tags to exif dictionary + metadata_dict.get("image", {})["tags"] = self.tags + # Add new annotations to exif dictionary + metadata_dict.get("image", {})["annotations"] = self.annotations + # Add new custom metadata to exif dictionary + metadata_dict.update(metadata) + # Serialize metadata - metadata_string = json.dumps(self.metadata, cls=JSONEncoder) + metadata_string = json.dumps(metadata_dict, cls=JSONEncoder) logging.debug("Saving metadata string to file: %s", metadata_string) + # Insert metadata into exif_dict exif_dict["Exif"][piexif.ExifIFD.UserComment] = metadata_string.encode() + # Convert new exif dict to exif bytes exif_bytes = piexif.dump(exif_dict) + # Insert exif into file piexif.insert(exif_bytes, self.file) - logging.info("Finished saving metadata to %s", self.file) + logging.info("Finished writing Exif data to %s", self.file) # PROPERTIES @@ -309,42 +308,13 @@ class CaptureObject(object): Create basic metadata dictionary from basic capture data, and any added custom metadata and tags. """ - d = { - "image": { - "id": self.id, - "name": self.name, - "time": self.time.isoformat(), - "format": self.format, - "tags": self.tags, - "annotations": self.annotations, - }, - **self._metadata, - } - # Add custom metadata to dictionary - return d - - @property - def state(self) -> dict: - """ - Return a dictionary of objects full state, including metadata. - """ - - # Create basic state dictionary - d = {"path": self.file, "name": self.name, "metadata": self.metadata} - - # Combined availability of data - if self.exists: - d["available"] = True - else: - d["available"] = False - - return d + return self.read_full_metadata() @property def data(self) -> io.BytesIO: """ - Return a byte string of the capture data. + Return a BytesIO object of the capture data. """ if self.exists: # If data file exists @@ -390,7 +360,7 @@ class CaptureObject(object): """Write stream to file, and save/update metadata file""" # If a stream OR file exists, save the metadata file if self.exists: - self.save_metadata() + self.put_and_save() def delete(self) -> bool: """If the StreamObject has been saved, delete the file.""" From 70437212b120c379d8485b592b5589034316f0de Mon Sep 17 00:00:00 2001 From: Joel Collins Date: Fri, 6 Nov 2020 12:19:41 +0000 Subject: [PATCH 02/17] Added classes for error sources --- openflexure_microscope/rescue/auto.py | 14 ++------ .../rescue/check_picamera.py | 3 +- .../rescue/error_sources.py | 36 +++++++++++++++++++ 3 files changed, 41 insertions(+), 12 deletions(-) create mode 100644 openflexure_microscope/rescue/error_sources.py diff --git a/openflexure_microscope/rescue/auto.py b/openflexure_microscope/rescue/auto.py index 08625cd2..b092a366 100644 --- a/openflexure_microscope/rescue/auto.py +++ b/openflexure_microscope/rescue/auto.py @@ -31,15 +31,7 @@ logger.setLevel(logging.DEBUG) logging.debug("Testing debug logger. One two one two.") -class bcolors: - HEADER = "\033[95m" - OKBLUE = "\033[94m" - OKGREEN = "\033[92m" - WARNING = "\033[93m" - FAIL = "\033[91m" - ENDC = "\033[0m" - BOLD = "\033[1m" - UNDERLINE = "\033[4m" + error_keys = { @@ -49,7 +41,7 @@ error_keys = { "default_config_error": f"The default configuration file could not be parsed. To fix, consider backing up and deleting your configuration files from {CONFIG_PATHS} to reset the configuration.", "default_settings_error": f"The default settings file could not be parsed. To fix, consider backing up and deleting your configuration files from {SETTINGS_PATHS} to reset the configuration.", "capture_rebuild_timeout": f"Capture database rebuilding took a long time. This may not cause catastrophic errors, but rather will cause the server to hang for a while. To fix, consider moving your captures from {DATA_PATHS} to another location.", - "picamera_import_error": "Picamera module could not be imported. Check physical connections to the camera as it may be damaged." + "picamera_import_error": "Picamera module could not be imported. Check physical connections to the camera as it may be damaged.", } if __name__ == "__main__": @@ -62,7 +54,7 @@ if __name__ == "__main__": error_sources.extend(check_settings.main()) error_sources.extend(check_picamera.main()) - #error_sources.extend(check_capture_reload.main()) + # error_sources.extend(check_capture_reload.main()) if not error_sources: diff --git a/openflexure_microscope/rescue/check_picamera.py b/openflexure_microscope/rescue/check_picamera.py index e5f3210a..e31ed26d 100644 --- a/openflexure_microscope/rescue/check_picamera.py +++ b/openflexure_microscope/rescue/check_picamera.py @@ -1,5 +1,6 @@ import logging + def main(): error_sources = [] logging.info("Attempting to import picamera...") @@ -12,4 +13,4 @@ def main(): logging.error(e) error_sources.append("picamera_import_error") - return error_sources \ No newline at end of file + return error_sources diff --git a/openflexure_microscope/rescue/error_sources.py b/openflexure_microscope/rescue/error_sources.py new file mode 100644 index 00000000..9ed6f476 --- /dev/null +++ b/openflexure_microscope/rescue/error_sources.py @@ -0,0 +1,36 @@ +class bcolors: + HEADER = "\033[95m" + OKBLUE = "\033[94m" + OKGREEN = "\033[92m" + WARNING = "\033[93m" + FAIL = "\033[91m" + ENDC = "\033[0m" + BOLD = "\033[1m" + UNDERLINE = "\033[4m" + + +class Source: + def __init__(self, message): + self._message = message + + @property + def message(self): + return self._message + + +class WarningSource: + def __init__(self, message): + self.message = message + + @property + def message(self): + print(bcolors.WARNING + self._message + bcolors.ENDC) + + +class ErrorSource: + def __init__(self, message): + self.message = message + + @property + def message(self): + print(bcolors.FAIL + self._message + bcolors.ENDC) \ No newline at end of file From 0989a505587a5fc8ae76f13d0874916f08270d99 Mon Sep 17 00:00:00 2001 From: Joel Collins Date: Fri, 6 Nov 2020 15:34:34 +0000 Subject: [PATCH 03/17] Added stage tests and tidied up rescue --- openflexure_microscope/rescue/auto.py | 40 +++++++---------- .../rescue/check_picamera.py | 7 ++- .../rescue/check_sangaboard.py | 44 +++++++++++++++++++ .../rescue/error_sources.py | 14 ++---- 4 files changed, 69 insertions(+), 36 deletions(-) create mode 100644 openflexure_microscope/rescue/check_sangaboard.py diff --git a/openflexure_microscope/rescue/auto.py b/openflexure_microscope/rescue/auto.py index b092a366..8fcce894 100644 --- a/openflexure_microscope/rescue/auto.py +++ b/openflexure_microscope/rescue/auto.py @@ -1,12 +1,15 @@ import logging import os +import sys + +from .error_sources import bcolors from openflexure_microscope.paths import ( FALLBACK_OPENFLEXURE_VAR_PATH, PREFERRED_OPENFLEXURE_VAR_PATH, ) -from . import check_capture_reload, check_settings, check_picamera +from . import check_capture_reload, check_settings, check_picamera, check_sangaboard # Paths for suggestions LOGS_PATHS = [ @@ -26,38 +29,26 @@ DATA_PATHS = [ os.path.join(FALLBACK_OPENFLEXURE_VAR_PATH, "data"), ] +# Look for debug flag logger = logging.getLogger() -logger.setLevel(logging.DEBUG) -logging.debug("Testing debug logger. One two one two.") +if "-d" in sys.argv or "--debug" in sys.argv: + logger.setLevel(logging.DEBUG) + logging.debug("Testing debug logger. One two one two.") +else: + logger.setLevel(logging.INFO) - - - -error_keys = { - "config_settings_import_error": "The configuration submodule could not be imported properly. This is usually because the default config or settings files are badly broken somehow. See further errors for details", - "default_config_empty": "The default configuration file is empty, and could not be automatically populated.", - "default_settings_empty": "The default settings file is empty, and could not be automatically populated.", - "default_config_error": f"The default configuration file could not be parsed. To fix, consider backing up and deleting your configuration files from {CONFIG_PATHS} to reset the configuration.", - "default_settings_error": f"The default settings file could not be parsed. To fix, consider backing up and deleting your configuration files from {SETTINGS_PATHS} to reset the configuration.", - "capture_rebuild_timeout": f"Capture database rebuilding took a long time. This may not cause catastrophic errors, but rather will cause the server to hang for a while. To fix, consider moving your captures from {DATA_PATHS} to another location.", - "picamera_import_error": "Picamera module could not be imported. Check physical connections to the camera as it may be damaged.", -} - if __name__ == "__main__": spoof = False error_sources = [] - if spoof: - error_sources = list(error_keys.keys()) - error_sources.extend(check_settings.main()) error_sources.extend(check_picamera.main()) - # error_sources.extend(check_capture_reload.main()) + error_sources.extend(check_capture_reload.main()) + error_sources.extend(check_sangaboard.main()) if not error_sources: - print() print(bcolors.OKGREEN + "No errors found!" + bcolors.ENDC) print( @@ -66,7 +57,6 @@ if __name__ == "__main__": print(f"You can check through the server logs at {LOGS_PATHS}") else: - for err_code in error_sources: - logging.error(err_code) - if error_keys.get(err_code): - print(bcolors.FAIL + error_keys.get(err_code) + bcolors.ENDC) + print() + for err in error_sources: + print(err.message) diff --git a/openflexure_microscope/rescue/check_picamera.py b/openflexure_microscope/rescue/check_picamera.py index e31ed26d..bd9582f2 100644 --- a/openflexure_microscope/rescue/check_picamera.py +++ b/openflexure_microscope/rescue/check_picamera.py @@ -1,5 +1,10 @@ import logging +from .error_sources import ErrorSource, WarningSource + +picamera_import_error = ErrorSource( + "Picamera module could not be imported. Check physical connections to the camera as it may be damaged." +) def main(): error_sources = [] @@ -11,6 +16,6 @@ def main(): # TODO: Parse exception object for a few different issues, and append different error sources # E.g. pi with camera already in use, disconnected, unsupported OS, etc logging.error(e) - error_sources.append("picamera_import_error") + error_sources.append(picamera_import_error) return error_sources diff --git a/openflexure_microscope/rescue/check_sangaboard.py b/openflexure_microscope/rescue/check_sangaboard.py new file mode 100644 index 00000000..76821686 --- /dev/null +++ b/openflexure_microscope/rescue/check_sangaboard.py @@ -0,0 +1,44 @@ +from .error_sources import ErrorSource, WarningSource + +from openflexure_microscope.config import user_configuration + + +def main(): + error_sources = [] + + try: + from sangaboard import Sangaboard + except Exception as e: # pylint: disable=W0703 + error_sources.append(ErrorSource( + "Sangaboard module could not be imported." + )) + else: + configuration = user_configuration.load() + stage_type = configuration["stage"].get("type") + stage_port = configuration["stage"].get("port") + + error_sources.append(WarningSource( + "Stage serial port is not specified in configuration. Application will scan ports for a Sangaboard." + )) + + + # If any Sangaboard-based stage is configured for use + if stage_type in ("SangaBoard", "SangaStage", "SangaDeltaStage"): + # Try connecting on the specified port + try: + _ = Sangaboard(stage_port) + except FileNotFoundError as e: # pylint: disable=W0703 + if stage_port: + error_sources.append(ErrorSource( + f"No {stage_type} device was found on the configured port {stage_port}." + )) + else: + error_sources.append(ErrorSource( + f"No {stage_type} device was found during port scanning." + )) + else: + error_sources.append(ErrorSource( + f"Invalid stage type {stage_type} specified in configuration file." + )) + + return error_sources diff --git a/openflexure_microscope/rescue/error_sources.py b/openflexure_microscope/rescue/error_sources.py index 9ed6f476..13cda56f 100644 --- a/openflexure_microscope/rescue/error_sources.py +++ b/openflexure_microscope/rescue/error_sources.py @@ -18,19 +18,13 @@ class Source: return self._message -class WarningSource: - def __init__(self, message): - self.message = message - +class WarningSource(Source): @property def message(self): - print(bcolors.WARNING + self._message + bcolors.ENDC) + return bcolors.WARNING + self._message + bcolors.ENDC -class ErrorSource: - def __init__(self, message): - self.message = message - +class ErrorSource(Source): @property def message(self): - print(bcolors.FAIL + self._message + bcolors.ENDC) \ No newline at end of file + return bcolors.FAIL + self._message + bcolors.ENDC \ No newline at end of file From f12671fa3f29718b1955e8c5c7a7390eb3dc884e Mon Sep 17 00:00:00 2001 From: Joel Collins Date: Fri, 6 Nov 2020 15:53:20 +0000 Subject: [PATCH 04/17] Support tuple-split strings as error messages --- openflexure_microscope/rescue/error_sources.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/openflexure_microscope/rescue/error_sources.py b/openflexure_microscope/rescue/error_sources.py index 13cda56f..49f36826 100644 --- a/openflexure_microscope/rescue/error_sources.py +++ b/openflexure_microscope/rescue/error_sources.py @@ -11,7 +11,10 @@ class bcolors: class Source: def __init__(self, message): - self._message = message + if isinstance(message, tuple): + self._message = " ".join(message) + else: + self._message = message @property def message(self): @@ -27,4 +30,4 @@ class WarningSource(Source): class ErrorSource(Source): @property def message(self): - return bcolors.FAIL + self._message + bcolors.ENDC \ No newline at end of file + return bcolors.FAIL + self._message + bcolors.ENDC From de135abc957a71f765c9e439eb1aca614a2d9284 Mon Sep 17 00:00:00 2001 From: Joel Collins Date: Fri, 6 Nov 2020 15:53:25 +0000 Subject: [PATCH 05/17] Style and linting --- .../rescue/check_picamera.py | 8 ++-- .../rescue/check_sangaboard.py | 41 +++++++++++-------- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/openflexure_microscope/rescue/check_picamera.py b/openflexure_microscope/rescue/check_picamera.py index bd9582f2..f7ddbbd7 100644 --- a/openflexure_microscope/rescue/check_picamera.py +++ b/openflexure_microscope/rescue/check_picamera.py @@ -1,17 +1,19 @@ import logging -from .error_sources import ErrorSource, WarningSource +from .error_sources import ErrorSource picamera_import_error = ErrorSource( - "Picamera module could not be imported. Check physical connections to the camera as it may be damaged." + ("Picamera module could not be imported.", + "Check physical connections to the camera as it may be damaged or disconnected.") ) + def main(): error_sources = [] logging.info("Attempting to import picamera...") try: - from picamerax import PiCamera + from picamerax import PiCamera as _ except Exception as e: # pylint: disable=W0703 # TODO: Parse exception object for a few different issues, and append different error sources # E.g. pi with camera already in use, disconnected, unsupported OS, etc diff --git a/openflexure_microscope/rescue/check_sangaboard.py b/openflexure_microscope/rescue/check_sangaboard.py index 76821686..de8c480b 100644 --- a/openflexure_microscope/rescue/check_sangaboard.py +++ b/openflexure_microscope/rescue/check_sangaboard.py @@ -8,37 +8,42 @@ def main(): try: from sangaboard import Sangaboard - except Exception as e: # pylint: disable=W0703 - error_sources.append(ErrorSource( - "Sangaboard module could not be imported." - )) + except ImportError as e: + error_sources.append(ErrorSource(str(e))) else: configuration = user_configuration.load() stage_type = configuration["stage"].get("type") stage_port = configuration["stage"].get("port") - error_sources.append(WarningSource( - "Stage serial port is not specified in configuration. Application will scan ports for a Sangaboard." - )) - + error_sources.append( + WarningSource( + "Stage serial port is not specified in configuration. Application will scan ports for a Sangaboard." + ) + ) # If any Sangaboard-based stage is configured for use if stage_type in ("SangaBoard", "SangaStage", "SangaDeltaStage"): # Try connecting on the specified port try: _ = Sangaboard(stage_port) - except FileNotFoundError as e: # pylint: disable=W0703 + except FileNotFoundError as e: if stage_port: - error_sources.append(ErrorSource( - f"No {stage_type} device was found on the configured port {stage_port}." - )) + error_sources.append( + ErrorSource( + f"No {stage_type} device was found on the configured port {stage_port}." + ) + ) else: - error_sources.append(ErrorSource( - f"No {stage_type} device was found during port scanning." - )) + error_sources.append( + ErrorSource( + f"No {stage_type} device was found during port scanning." + ) + ) else: - error_sources.append(ErrorSource( - f"Invalid stage type {stage_type} specified in configuration file." - )) + error_sources.append( + ErrorSource( + f"Invalid stage type {stage_type} specified in configuration file." + ) + ) return error_sources From 9e887c0a4886420f884a8d871361533943587aaa Mon Sep 17 00:00:00 2001 From: Joel Collins Date: Fri, 6 Nov 2020 16:11:44 +0000 Subject: [PATCH 06/17] Added capture count warning --- .../rescue/check_capture_reload.py | 45 +++++++++++++------ 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/openflexure_microscope/rescue/check_capture_reload.py b/openflexure_microscope/rescue/check_capture_reload.py index bddfe28c..ad26e0f1 100644 --- a/openflexure_microscope/rescue/check_capture_reload.py +++ b/openflexure_microscope/rescue/check_capture_reload.py @@ -1,23 +1,15 @@ import logging +from logging import error -from openflexure_microscope.captures.capture import build_captures_from_exif +from openflexure_microscope.captures.capture import build_captures_from_exif, make_file_list, EXIF_FORMATS from openflexure_microscope.captures.capture_manager import BASE_CAPTURE_PATH from openflexure_microscope.config import user_settings from openflexure_microscope.rescue.monitor_timeout import launch_timeout_test_process +from .error_sources import ErrorSource, WarningSource -def check_capture_reload(): - logging.info("Loading user settings...") - settings = user_settings.load() - - cap_path = str(settings.get("captures", {}).get("paths", {}).get("default")) - logging.info("Capture path found: %s", cap_path) - if not cap_path: - logging.error( - "No capture path defined in settings. This is unusual for anything other than a first-run. \nFalling back to default path." - ) - cap_path = BASE_CAPTURE_PATH +def check_capture_reload(cap_path): logging.info("Starting capture reload with a 60 second timeout...") passed_timeout_test = launch_timeout_test_process( build_captures_from_exif, args=(cap_path,), timeout=60 @@ -28,7 +20,32 @@ def check_capture_reload(): def main(): error_sources = [] - passed_timeout = check_capture_reload() + + logging.info("Loading user settings...") + settings = user_settings.load() + cap_path = str(settings.get("captures", {}).get("paths", {}).get("default")) + logging.info("Capture path found: %s", cap_path) + + if not cap_path: + logging.error( + "No capture path defined in settings. This is unusual for anything other than a first-run. \nFalling back to default path." + ) + cap_path = BASE_CAPTURE_PATH + + # Check number of captures being restored + files = make_file_list(cap_path, EXIF_FORMATS) + if len(files) >= 2000: + error_sources.append(WarningSource( + ("Over 2000 captures are being restored. This may slow down server startup.", + f"Consider moving your captures from {cap_path} to another location.") + )) + + # Check restore time of captures + passed_timeout = check_capture_reload(cap_path) if not passed_timeout: - error_sources.append("capture_rebuild_timeout") + error_sources.append(ErrorSource( + ("Capture database rebuilding took a long time. This may not cause catastrophic errors, but rather will cause the server to hang for a while.", + f"To fix, consider moving your captures from {cap_path} to another location.") + )) + return error_sources From 121afb44b88040e14f4c0978cf615d61ce83a4db Mon Sep 17 00:00:00 2001 From: Joel Collins Date: Fri, 6 Nov 2020 16:11:53 +0000 Subject: [PATCH 07/17] Added newline symbols --- openflexure_microscope/rescue/error_sources.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openflexure_microscope/rescue/error_sources.py b/openflexure_microscope/rescue/error_sources.py index 49f36826..8f7f7697 100644 --- a/openflexure_microscope/rescue/error_sources.py +++ b/openflexure_microscope/rescue/error_sources.py @@ -24,10 +24,10 @@ class Source: class WarningSource(Source): @property def message(self): - return bcolors.WARNING + self._message + bcolors.ENDC + return "[!] " + bcolors.WARNING + self._message + bcolors.ENDC class ErrorSource(Source): @property def message(self): - return bcolors.FAIL + self._message + bcolors.ENDC + return "[X] " + bcolors.FAIL + self._message + bcolors.ENDC From 061618d9e6a9813f7d2af31102bbf93d7af6c140 Mon Sep 17 00:00:00 2001 From: Joel Collins Date: Fri, 6 Nov 2020 16:15:26 +0000 Subject: [PATCH 08/17] Changed newline symbols --- openflexure_microscope/rescue/error_sources.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openflexure_microscope/rescue/error_sources.py b/openflexure_microscope/rescue/error_sources.py index 8f7f7697..67910701 100644 --- a/openflexure_microscope/rescue/error_sources.py +++ b/openflexure_microscope/rescue/error_sources.py @@ -24,10 +24,10 @@ class Source: class WarningSource(Source): @property def message(self): - return "[!] " + bcolors.WARNING + self._message + bcolors.ENDC + return "[?] " + bcolors.WARNING + self._message + bcolors.ENDC class ErrorSource(Source): @property def message(self): - return "[X] " + bcolors.FAIL + self._message + bcolors.ENDC + return "[!] " + bcolors.FAIL + self._message + bcolors.ENDC From 0647acd47dbd8ad9d8b6006663215aa8d3bcde5b Mon Sep 17 00:00:00 2001 From: Joel Collins Date: Fri, 6 Nov 2020 16:15:34 +0000 Subject: [PATCH 09/17] Upped capture count warning to 10000 --- openflexure_microscope/rescue/check_capture_reload.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openflexure_microscope/rescue/check_capture_reload.py b/openflexure_microscope/rescue/check_capture_reload.py index ad26e0f1..03f17748 100644 --- a/openflexure_microscope/rescue/check_capture_reload.py +++ b/openflexure_microscope/rescue/check_capture_reload.py @@ -34,9 +34,9 @@ def main(): # Check number of captures being restored files = make_file_list(cap_path, EXIF_FORMATS) - if len(files) >= 2000: + if len(files) >= 10000: error_sources.append(WarningSource( - ("Over 2000 captures are being restored. This may slow down server startup.", + ("Over 10000 captures are being restored. This may slow down server startup.", f"Consider moving your captures from {cap_path} to another location.") )) From 06fb81ec4a847dd5760e9d7f783fe68277d026b2 Mon Sep 17 00:00:00 2001 From: Joel Collins Date: Fri, 6 Nov 2020 16:24:55 +0000 Subject: [PATCH 10/17] Tidied up logging --- openflexure_microscope/captures/capture.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/openflexure_microscope/captures/capture.py b/openflexure_microscope/captures/capture.py index 2ad8da37..fcddc98f 100644 --- a/openflexure_microscope/captures/capture.py +++ b/openflexure_microscope/captures/capture.py @@ -40,8 +40,6 @@ def build_captures_from_exif(capture_path): capture = capture_from_path(f) if capture: captures[capture.id] = capture - else: - logging.error("Invalid data at {}. Skipping.".format(f)) logging.info("{} capture files successfully reloaded".format(len(captures))) @@ -60,7 +58,7 @@ def capture_from_path(path): capture.sync_basic_metadata() return capture except (InvalidImageDataError, json.decoder.JSONDecodeError): - logging.error("Invalid metadata at {}. Skipping.".format(path)) + logging.error("Invalid metadata at {}.".format(path)) return None From cd61ad3b21f0f6c03f0818a43194e84ebbf87ddf Mon Sep 17 00:00:00 2001 From: Joel Collins Date: Fri, 6 Nov 2020 16:25:56 +0000 Subject: [PATCH 11/17] Formatting and linting --- .../rescue/check_capture_reload.py | 31 +++++++++++++------ .../rescue/check_picamera.py | 6 ++-- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/openflexure_microscope/rescue/check_capture_reload.py b/openflexure_microscope/rescue/check_capture_reload.py index 03f17748..54459665 100644 --- a/openflexure_microscope/rescue/check_capture_reload.py +++ b/openflexure_microscope/rescue/check_capture_reload.py @@ -1,7 +1,10 @@ import logging -from logging import error -from openflexure_microscope.captures.capture import build_captures_from_exif, make_file_list, EXIF_FORMATS +from openflexure_microscope.captures.capture import ( + build_captures_from_exif, + make_file_list, + EXIF_FORMATS, +) from openflexure_microscope.captures.capture_manager import BASE_CAPTURE_PATH from openflexure_microscope.config import user_settings from openflexure_microscope.rescue.monitor_timeout import launch_timeout_test_process @@ -35,17 +38,25 @@ def main(): # Check number of captures being restored files = make_file_list(cap_path, EXIF_FORMATS) if len(files) >= 10000: - error_sources.append(WarningSource( - ("Over 10000 captures are being restored. This may slow down server startup.", - f"Consider moving your captures from {cap_path} to another location.") - )) + error_sources.append( + WarningSource( + ( + "Over 10000 captures are being restored. This may slow down server startup.", + f"Consider moving your captures from {cap_path} to another location.", + ) + ) + ) # Check restore time of captures passed_timeout = check_capture_reload(cap_path) if not passed_timeout: - error_sources.append(ErrorSource( - ("Capture database rebuilding took a long time. This may not cause catastrophic errors, but rather will cause the server to hang for a while.", - f"To fix, consider moving your captures from {cap_path} to another location.") - )) + error_sources.append( + ErrorSource( + ( + "Capture database rebuilding took a long time. This may not cause catastrophic errors, but rather will cause the server to hang for a while.", + f"To fix, consider moving your captures from {cap_path} to another location.", + ) + ) + ) return error_sources diff --git a/openflexure_microscope/rescue/check_picamera.py b/openflexure_microscope/rescue/check_picamera.py index f7ddbbd7..9b21305d 100644 --- a/openflexure_microscope/rescue/check_picamera.py +++ b/openflexure_microscope/rescue/check_picamera.py @@ -3,8 +3,10 @@ import logging from .error_sources import ErrorSource picamera_import_error = ErrorSource( - ("Picamera module could not be imported.", - "Check physical connections to the camera as it may be damaged or disconnected.") + ( + "Picamera module could not be imported.", + "Check physical connections to the camera as it may be damaged or disconnected.", + ) ) From 5afff59785e9b534bf4880d8075638f8c14a2b9e Mon Sep 17 00:00:00 2001 From: Joel Collins Date: Fri, 6 Nov 2020 16:59:43 +0000 Subject: [PATCH 12/17] Added internet, RAM, and storage checkers --- openflexure_microscope/rescue/auto.py | 3 +- openflexure_microscope/rescue/check_system.py | 53 +++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 openflexure_microscope/rescue/check_system.py diff --git a/openflexure_microscope/rescue/auto.py b/openflexure_microscope/rescue/auto.py index 8fcce894..842c9c17 100644 --- a/openflexure_microscope/rescue/auto.py +++ b/openflexure_microscope/rescue/auto.py @@ -9,7 +9,7 @@ from openflexure_microscope.paths import ( PREFERRED_OPENFLEXURE_VAR_PATH, ) -from . import check_capture_reload, check_settings, check_picamera, check_sangaboard +from . import check_capture_reload, check_settings, check_picamera, check_sangaboard, check_system # Paths for suggestions LOGS_PATHS = [ @@ -43,6 +43,7 @@ if __name__ == "__main__": error_sources = [] + error_sources.extend(check_system.main()) error_sources.extend(check_settings.main()) error_sources.extend(check_picamera.main()) error_sources.extend(check_capture_reload.main()) diff --git a/openflexure_microscope/rescue/check_system.py b/openflexure_microscope/rescue/check_system.py new file mode 100644 index 00000000..b5736195 --- /dev/null +++ b/openflexure_microscope/rescue/check_system.py @@ -0,0 +1,53 @@ +from urllib.request import urlopen +from urllib.error import URLError +import psutil + +from .error_sources import WarningSource + +def internet_on(): + test_urls = ( + 'https://build.openflexure.org', # Bath server + 'https://openflexure.org' # GitHub pages + ) + for url in test_urls: + try: + urlopen(url, timeout=10) + # If any test passes, return True + return True + except URLError: + pass + # If no test passed, return False + return False + + +def main(): + error_sources = [] + + # Check internet + if not internet_on(): + error_sources.append(WarningSource( + "No internet connection detected. Updates may not be available." + )) + + # Check memory + mem = psutil.virtual_memory() + total_gb = mem.total / 1E9 + + if total_gb <= 1: + error_sources.append(WarningSource( + ("Less than 1GB total memory available.", + "For small scans, or control from another device, this is usually fine.", + "\n More complex usage may require additional resources.") + )) + + # Check disks + data_partitions = [disk for disk in psutil.disk_partitions() if "rw" in disk.opts and disk.mountpoint != "/boot"] + for part in data_partitions: + usage = psutil.disk_usage(part.mountpoint) + if int(usage.percent) >= 90: + error_sources.append(WarningSource( + (f"Disk {part.device}, at {part.mountpoint} is {int(usage.percent)}% full.", + "Captures may fail to save soon.") + )) + + return error_sources From 5319beb735b4d2509ea8c30ae7ee5194e1163ec9 Mon Sep 17 00:00:00 2001 From: Joel Collins Date: Fri, 6 Nov 2020 17:11:07 +0000 Subject: [PATCH 13/17] Removed port scanning warning --- openflexure_microscope/rescue/check_sangaboard.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/openflexure_microscope/rescue/check_sangaboard.py b/openflexure_microscope/rescue/check_sangaboard.py index de8c480b..13470ed2 100644 --- a/openflexure_microscope/rescue/check_sangaboard.py +++ b/openflexure_microscope/rescue/check_sangaboard.py @@ -15,17 +15,11 @@ def main(): stage_type = configuration["stage"].get("type") stage_port = configuration["stage"].get("port") - error_sources.append( - WarningSource( - "Stage serial port is not specified in configuration. Application will scan ports for a Sangaboard." - ) - ) - # If any Sangaboard-based stage is configured for use if stage_type in ("SangaBoard", "SangaStage", "SangaDeltaStage"): # Try connecting on the specified port try: - _ = Sangaboard(stage_port) + stage = Sangaboard(stage_port) except FileNotFoundError as e: if stage_port: error_sources.append( @@ -39,6 +33,8 @@ def main(): f"No {stage_type} device was found during port scanning." ) ) + else: + stage.close() else: error_sources.append( ErrorSource( From e41656355fa51143c6de02633407dc6cdf6c8844 Mon Sep 17 00:00:00 2001 From: Joel Collins Date: Fri, 6 Nov 2020 17:11:17 +0000 Subject: [PATCH 14/17] Formatting and linting --- openflexure_microscope/rescue/auto.py | 25 +++++++-- openflexure_microscope/rescue/check_system.py | 53 ++++++++++++------- 2 files changed, 56 insertions(+), 22 deletions(-) diff --git a/openflexure_microscope/rescue/auto.py b/openflexure_microscope/rescue/auto.py index 842c9c17..cca0134e 100644 --- a/openflexure_microscope/rescue/auto.py +++ b/openflexure_microscope/rescue/auto.py @@ -9,7 +9,13 @@ from openflexure_microscope.paths import ( PREFERRED_OPENFLEXURE_VAR_PATH, ) -from . import check_capture_reload, check_settings, check_picamera, check_sangaboard, check_system +from . import ( + check_capture_reload, + check_settings, + check_picamera, + check_sangaboard, + check_system, +) # Paths for suggestions LOGS_PATHS = [ @@ -35,12 +41,25 @@ if "-d" in sys.argv or "--debug" in sys.argv: logger.setLevel(logging.DEBUG) logging.debug("Testing debug logger. One two one two.") else: - logger.setLevel(logging.INFO) + logger.setLevel(logging.WARNING) if __name__ == "__main__": spoof = False + print() + print(bcolors.HEADER + "OpenFlexure Rescue" + bcolors.ENDC) + print() + print( + "This script attempts to identify common issues for a microscope not working properly." + ) + print( + "It is not designed to identify bugs in the code, but rather configuration or setup issues." + ) + print() + print("Any identified warnings [?] or errors [!] will be reported.") + print() + error_sources = [] error_sources.extend(check_system.main()) @@ -51,7 +70,7 @@ if __name__ == "__main__": if not error_sources: print() - print(bcolors.OKGREEN + "No errors found!" + bcolors.ENDC) + print(bcolors.OKGREEN + "No issues found!" + bcolors.ENDC) print( "That's not to say everything is fine, only that our automatic diagnostics couldn't find much." ) diff --git a/openflexure_microscope/rescue/check_system.py b/openflexure_microscope/rescue/check_system.py index b5736195..1c3f31ee 100644 --- a/openflexure_microscope/rescue/check_system.py +++ b/openflexure_microscope/rescue/check_system.py @@ -4,17 +4,18 @@ import psutil from .error_sources import WarningSource + def internet_on(): test_urls = ( - 'https://build.openflexure.org', # Bath server - 'https://openflexure.org' # GitHub pages + "https://build.openflexure.org", # Bath server + "https://openflexure.org", # GitHub pages ) for url in test_urls: try: urlopen(url, timeout=10) # If any test passes, return True return True - except URLError: + except URLError: pass # If no test passed, return False return False @@ -22,32 +23,46 @@ def internet_on(): def main(): error_sources = [] - + # Check internet if not internet_on(): - error_sources.append(WarningSource( - "No internet connection detected. Updates may not be available." - )) - + error_sources.append( + WarningSource( + "No internet connection detected. Updates may not be available." + ) + ) + # Check memory mem = psutil.virtual_memory() - total_gb = mem.total / 1E9 + total_gb = mem.total / 1e9 if total_gb <= 1: - error_sources.append(WarningSource( - ("Less than 1GB total memory available.", - "For small scans, or control from another device, this is usually fine.", - "\n More complex usage may require additional resources.") - )) + error_sources.append( + WarningSource( + ( + "Less than 1GB total memory available.", + "For small scans, or control from another device, this is usually fine.", + "\n More complex usage may require additional resources.", + ) + ) + ) # Check disks - data_partitions = [disk for disk in psutil.disk_partitions() if "rw" in disk.opts and disk.mountpoint != "/boot"] + data_partitions = [ + disk + for disk in psutil.disk_partitions() + if "rw" in disk.opts and disk.mountpoint != "/boot" + ] for part in data_partitions: usage = psutil.disk_usage(part.mountpoint) if int(usage.percent) >= 90: - error_sources.append(WarningSource( - (f"Disk {part.device}, at {part.mountpoint} is {int(usage.percent)}% full.", - "Captures may fail to save soon.") - )) + error_sources.append( + WarningSource( + ( + f"Disk {part.device}, at {part.mountpoint} is {int(usage.percent)}% full.", + "Captures may fail to save soon.", + ) + ) + ) return error_sources From 8349b3719caa85c6e9817ee4526e3828d2fb53cf Mon Sep 17 00:00:00 2001 From: Joel Collins Date: Fri, 6 Nov 2020 17:26:00 +0000 Subject: [PATCH 15/17] Added platform info to output --- openflexure_microscope/rescue/auto.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/openflexure_microscope/rescue/auto.py b/openflexure_microscope/rescue/auto.py index cca0134e..12107d43 100644 --- a/openflexure_microscope/rescue/auto.py +++ b/openflexure_microscope/rescue/auto.py @@ -1,6 +1,8 @@ import logging import os import sys +import platform +import pkg_resources from .error_sources import bcolors @@ -68,6 +70,14 @@ if __name__ == "__main__": error_sources.extend(check_capture_reload.main()) error_sources.extend(check_sangaboard.main()) + dist = pkg_resources.get_distribution("openflexure-microscope-server") + + print() + print(f"Server Version: {dist.version}") + print(f"Server Location: {dist.location}") + print(f"Platform: {platform.platform()}") + print(f"Python: {sys.version}") + if not error_sources: print() print(bcolors.OKGREEN + "No issues found!" + bcolors.ENDC) From cbea2f51be20749440e63cac064a7e42bfa3b828 Mon Sep 17 00:00:00 2001 From: Joel Collins Date: Fri, 6 Nov 2020 17:26:07 +0000 Subject: [PATCH 16/17] Removed unused import --- openflexure_microscope/rescue/check_sangaboard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openflexure_microscope/rescue/check_sangaboard.py b/openflexure_microscope/rescue/check_sangaboard.py index 13470ed2..8c7f6aa7 100644 --- a/openflexure_microscope/rescue/check_sangaboard.py +++ b/openflexure_microscope/rescue/check_sangaboard.py @@ -1,4 +1,4 @@ -from .error_sources import ErrorSource, WarningSource +from .error_sources import ErrorSource from openflexure_microscope.config import user_configuration From 471ee5e33d2c348f1532647e1a6a19737e060ada Mon Sep 17 00:00:00 2001 From: Joel Collins Date: Fri, 6 Nov 2020 17:28:08 +0000 Subject: [PATCH 17/17] Removed unused output --- openflexure_microscope/rescue/auto.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/openflexure_microscope/rescue/auto.py b/openflexure_microscope/rescue/auto.py index 12107d43..5bb95f22 100644 --- a/openflexure_microscope/rescue/auto.py +++ b/openflexure_microscope/rescue/auto.py @@ -74,9 +74,7 @@ if __name__ == "__main__": print() print(f"Server Version: {dist.version}") - print(f"Server Location: {dist.location}") print(f"Platform: {platform.platform()}") - print(f"Python: {sys.version}") if not error_sources: print()