diff --git a/src/openflexure_microscope_server/scan_directories.py b/src/openflexure_microscope_server/scan_directories.py index e2a33fbc..361abf40 100644 --- a/src/openflexure_microscope_server/scan_directories.py +++ b/src/openflexure_microscope_server/scan_directories.py @@ -14,9 +14,11 @@ import zipfile from pydantic import BaseModel IMG_DIR_NAME = "images" -IMAGE_REGEX = re.compile(r"-?[0-9]+_-?[0-9]+\.jpe?g$") SCAN_ZERO_PAD_DIGITS = 4 +STITCH_REGEX = re.compile(r"stitched\.jpe?g$") +IMAGE_REGEX = re.compile(r"-?[0-9]+_-?[0-9]+\.jpe?g$") + class NotEnoughFreeSpaceError(IOError): pass @@ -68,10 +70,10 @@ class ScanDirectoryManager: """ return os.path.join(self._base_scan_dir, scan_name, IMG_DIR_NAME) - def get_file_from( + def get_file_path_from( self, scan_name: str, filename: str, check_exists: bool = False ) -> Optional[str]: - """Return the file path for the file within a scan directory + """Return the full file path for the file within a scan directory If check_exists is True then None will be returned if the file does not exist. @@ -82,13 +84,13 @@ class ScanDirectoryManager: return None return file_path - def get_file_from_img_dir( + def get_file_path_from_img_dir( self, scan_name: str, filename: str, check_exists: bool = False ) -> Optional[str]: - """Return the file path for the file within a scan directory + """Return the full file path for the file within a scan directory If check_exists is True, None is returned if the file does not exist. If False - then the path is returned anway + then the path is returned anyway """ file_path = os.path.join(self.img_dir_for(scan_name), filename) if check_exists: @@ -96,6 +98,16 @@ class ScanDirectoryManager: return None return file_path + def get_final_stitch_path(self, scan_name: str) -> Optional[str]: + """Return the file full path for the final stitch. + + If no final stitch is found, return None + """ + stitch_fname = ScanDirectory(scan_name, self.base_dir).get_final_stitch_name() + if stitch_fname is None: + return None + return self.get_file_path_from_img_dir(scan_name, stitch_fname) + @property def all_scans(self) -> list[str]: """Return a list of the scan names in the base directory""" @@ -242,25 +254,58 @@ class ScanDirectory: return [] return os.listdir(self.images_dir) + def _extract_scan_images(self, file_list: list[str]): + """Extract files which match the naming convention for scan images + + :param file_list: The list of files to search. Normally this would be + `self.get_scan_files()` + + :returns: The list of files that match the naming convention for scan images + """ + return [i for i in file_list if IMAGE_REGEX.search(i)] + + def _extract_final_stitches(self, file_list: list[str]): + """Extract files which match the naming convention for final stitches + + :param file_list: The list of files to search. + + :returns: The list of files that match the naming convention for final stitches + """ + return [i for i in file_list if STITCH_REGEX.search(i)] + + def _extract_dzi_files(self, file_list: list[str]): + """Extract files which match the naming convention for dzi_files + + :param file_list: The list of files to search. + + :returns: The list of files that match the naming convention for dzi_files + """ + return [i for i in file_list if i.endswith("dzi")] + + def get_final_stitch_name(self) -> Optional[str]: + """Return the filename for the final stitch (in the images dir) + + If no final stitch is found, return None + """ + stitches = self._extract_final_stitches(self.get_scan_files()) + if not stitches: + return None + return stitches[0] + def get_modified_time(self) -> float: """Return the modified time of the directory""" return max(os.stat(root).st_mtime for root, _, _ in os.walk(self.dir_path)) - def scan_info(self): + def scan_info(self) -> ScanInfo: """Return the information for the scan directory as a ScanInfo object""" - folder_contents = self.get_scan_files() - if folder_contents: - scan_images = [i for i in folder_contents if IMAGE_REGEX.search(i)] - stitches = [i for i in folder_contents if i.endswith("_stitched.jpg")] - dzi_files = [i for i in folder_contents if i.endswith("dzi")] - number_of_images = len(scan_images) - stitch_available = len(stitches) > 0 - dzi = None if not dzi_files else str(dzi_files[0]) - else: - number_of_images = 0 - stitch_available = False - dzi = None + scan_files = self.get_scan_files() + scan_images = self._extract_scan_images(scan_files) + stitches = self._extract_final_stitches(scan_files) + dzi_files = self._extract_dzi_files(scan_files) + number_of_images = len(scan_images) + stitch_available = len(stitches) > 0 + dzi = None if not dzi_files else str(dzi_files[0]) return ScanInfo( name=self.name, @@ -271,10 +316,20 @@ class ScanDirectory: dzi=dzi, ) - def all_files(self): - """Return a list of all files in the scan dir relative to the dir""" + def all_files(self, skip_dirs: Optional[list[str]] = None) -> list[str]: + """Return a list of all files in the scan dir relative to the dir. + + :param skip_dirs: Skip any file in a directory that is on this list. The list + should be the basename of the directory. e.g. "scan_0001_files" not + "images/scan_0001_files" + """ + if skip_dirs is None: + skip_dirs = [] files = [] - for file_root, _, filenames in os.walk(self.dir_path): + for file_root, dirs, filenames in os.walk(self.dir_path, topdown=True): + # Skip any skipped directories. + # Note: we must use slice assignment to edit in place. + dirs[:] = [d for d in dirs if d not in skip_dirs] for filename in filenames: full_path = os.path.join(file_root, filename) files.append(os.path.relpath(full_path, self.dir_path)) @@ -296,10 +351,14 @@ class ScanDirectory: else: zip_files = [] + # For each `filename.dzi` we need to skip the `filename_files` directory + dzi_files = self._extract_dzi_files(self.get_scan_files()) + dzi_dirs = [dzi[:-4] + "_files" for dzi in dzi_files] + with zipfile.ZipFile(zip_fname, mode="a") as scan_zip: - for file in self.all_files(): - # Don't zip zipfiles, or files in the zip - if file.endswith(".zip") or file in zip_files: + for file in self.all_files(skip_dirs=dzi_dirs): + # Don't zip zipfiles, dzi files, or files already in the zip + if file.endswith((".zip", ".dzi")) or file in zip_files: continue # If this is not the final version, then only zip image files. if not final_version and not IMAGE_REGEX.search(file): diff --git a/src/openflexure_microscope_server/things/smart_scan.py b/src/openflexure_microscope_server/things/smart_scan.py index 832f6170..22937721 100644 --- a/src/openflexure_microscope_server/things/smart_scan.py +++ b/src/openflexure_microscope_server/things/smart_scan.py @@ -580,7 +580,7 @@ class SmartScanThing(Thing): This endpoint allows files to be downloaded from a scan. """ - preview_path = self._scan_dir_manager.get_file_from_img_dir( + preview_path = self._scan_dir_manager.get_file_path_from_img_dir( scan_name=scan_name, filename="stitched_thumbnail.jpg", check_exists=True ) if preview_path is None: @@ -666,6 +666,30 @@ class SmartScanThing(Thing): """ return self._scan_dir_manager.all_scans_info() + @fastapi_endpoint( + "get", + "get_stitch/{scan_name}", + responses={ + 200: { + "description": "Successfully downloading file", + "content": {"*/*": {}}, + }, + 403: {"description": "Filename not permitted"}, + 404: {"description": "File not found"}, + }, + ) + def get_stitch_file(self, scan_name: str) -> FileResponse: + """Return the stitched image corresponding to a given scan name, if it exists. + + Will only return a file ending in suffix STITCH_SUFFIX + Note: when downloading this, the default filename will be `scan_name`.jpeg""" + + stitch_path = self._scan_dir_manager.get_final_stitch_path(scan_name) + + if stitch_path is None: + raise HTTPException(404, "File not found") + return FileResponse(stitch_path) + @fastapi_endpoint( "delete", "scans/{scan_name}", @@ -734,7 +758,7 @@ class SmartScanThing(Thing): if not self.latest_scan_name: return None - return self._scan_dir_manager.get_file_from_img_dir( + return self._scan_dir_manager.get_file_path_from_img_dir( scan_name=self.latest_scan_name, filename="preview.jpg", check_exists=True ) @@ -875,7 +899,7 @@ class SmartScanThing(Thing): Note that as this is a thing_action it needs the logger passed as a variable if called from another thing action """ - json_fpath = self._scan_dir_manager.get_file_from_img_dir( + json_fpath = self._scan_dir_manager.get_file_path_from_img_dir( scan_name=scan_name, filename=SCAN_DATA_FILENAME ) diff --git a/tests/test_scan_directories.py b/tests/test_scan_directories.py index ee76c9ab..6df56f1b 100644 --- a/tests/test_scan_directories.py +++ b/tests/test_scan_directories.py @@ -1,5 +1,6 @@ import tempfile import os +import math import shutil import logging import random @@ -32,10 +33,13 @@ def _clear_scan_dir() -> None: def _add_fake_image(scan_dir: ScanDirectory) -> None: """Make a fake image on disk in the scan directory""" - x_pos = random.randint(-100, 100) - y_pos = random.randint(-100, 100) - filename = f"image_{x_pos}_{y_pos}.jpg" - filepath = os.path.join(scan_dir.images_dir, filename) + unique = False + while not unique: + x_pos = random.randint(-10000, 10000) + y_pos = random.randint(-10000, 10000) + filename = f"image_{x_pos}_{y_pos}.jpg" + filepath = os.path.join(scan_dir.images_dir, filename) + unique = not os.path.exists(filepath) with open(filepath, "w") as f_obj: f_obj.write("fake") @@ -56,6 +60,40 @@ def _add_fake_file( f_obj.write("fake") +def _make_fake_dzi(scan_dir: ScanDirectory, n_layers: int = 8) -> None: + """Create a fake DZI in a scan + + :param n_layers: The number of layers of tiles. I.e. tile directories numbered + 0...(n_layers-1) will be created. Default 8 + """ + + # Add an a dzi image + dzi_fname = scan_dir.name + ".dzi" + dzi_path = os.path.join(scan_dir.images_dir, dzi_fname) + with open(dzi_path, "w") as f_obj: + f_obj.write("This should be xml") + + # and the directory for the dzi tiles + dzi_tile_dir = scan_dir.name + "_files" + dzi_tile_dir_path = os.path.join(scan_dir.images_dir, dzi_tile_dir) + os.makedirs(dzi_tile_dir_path) + + # this directory then contains a directories numbered from 0-n + for i in range(n_layers): + layer_dir_path = os.path.join(dzi_tile_dir_path, str(i)) + os.makedirs(layer_dir_path) + + # Number of images (in each axis) in this layer. Number of tiles doubles each + # layer, but the first few layers always have 1 image. + n_ims = math.ceil(2 ** (i - 4)) + for x_index in range(n_ims): + for y_index in range(n_ims): + tile_name = f"{x_index}_{y_index}.jpg" + tile_path = os.path.join(layer_dir_path, tile_name) + with open(tile_path, "w") as f_obj: + f_obj.write("This would normally be jpeg data") + + def test_basic_directory_operations(): """Test some basic operations @@ -84,17 +122,19 @@ def test_basic_directory_operations(): assert scan_dir_manager.img_dir_for(scan_name) == scan_im_dir # Get the path of a fake file - fake_file = scan_dir_manager.get_file_from(scan_name, "foo.zip") + fake_file = scan_dir_manager.get_file_path_from(scan_name, "foo.zip") assert fake_file == os.path.join(scan_path, "foo.zip") # But this is none if we check it exists - fake_file = scan_dir_manager.get_file_from(scan_name, "foo.zip", check_exists=True) + fake_file = scan_dir_manager.get_file_path_from( + scan_name, "foo.zip", check_exists=True + ) assert fake_file is None # Get the path of another fake file - fake_file = scan_dir_manager.get_file_from_img_dir(scan_name, "bar.img") + fake_file = scan_dir_manager.get_file_path_from_img_dir(scan_name, "bar.img") assert fake_file == os.path.join(scan_im_dir, "bar.img") # But this is none if we check it exists - fake_file = scan_dir_manager.get_file_from_img_dir( + fake_file = scan_dir_manager.get_file_path_from_img_dir( scan_name, "bar.img", check_exists=True ) assert fake_file is None @@ -211,13 +251,42 @@ def test_scan_info(): assert not info.stitch_available assert info.dzi is None - # Add a fake scan and check this is recognised as a stitch not a scan image + # Add a fake stitched images and check this is recognised as a stitch not + # a scan image _add_fake_file(scan_dir, "fake_scan_0001_stitched.jpg", in_im_dir=True) info = scan_dir.scan_info() assert info.number_of_images == 17 assert info.stitch_available +def test_get_final_stitch(): + """Check that the final stitch can be retrieved""" + _clear_scan_dir() + scan_dir_manager = ScanDirectoryManager(BASE_SCAN_DIR) + scan_dir = scan_dir_manager.new_scan_dir("fake_scan") + + # Create some scan images files + for i in range(17): + _add_fake_image(scan_dir) + + # No scans, so None should be returned, from both manager and scan dir + assert scan_dir_manager.get_final_stitch_path(scan_dir.name) is None + assert scan_dir.get_final_stitch_name() is None + + fake_scan_name = "fake_scan_0001_stitched.jpg" + fake_scan_path = scan_dir_manager.get_file_path_from_img_dir( + scan_name=scan_dir.name, + filename="fake_scan_0001_stitched.jpg", + check_exists=False, + ) + # Add a fake scan + _add_fake_file(scan_dir, fake_scan_name, in_im_dir=True) + # Manager returns full path + assert scan_dir_manager.get_final_stitch_path(scan_dir.name) == fake_scan_path + # ScanDirectory object returns just the filename + assert scan_dir.get_final_stitch_name() == fake_scan_name + + def test_empty_scan_info(): """Test the scan info is correct even if the scan is empty""" _clear_scan_dir() @@ -252,6 +321,9 @@ def test_zipping_scan_data(): _add_fake_file(scan_dir, "fake_scan_0001_stitched.jpg", in_im_dir=True) _add_fake_file(scan_dir, "zipfile.zip") + # This fake dzi should have loads of images. The DZI should not be zipped! + _make_fake_dzi(scan_dir) + # zip the directory without setting as the final version. It should only # zip the 21 scan images if caller == "scan_dir": @@ -271,13 +343,47 @@ def test_zipping_scan_data(): # Check the zips are not in the zip for file in zip_files: assert not file.endswith(".zip") + assert not file.endswith(".dzi") + + +def test_all_files(): + """Test all_files returns the path, and respects skipped directories""" + _clear_scan_dir() + scan_dir_manager = ScanDirectoryManager(BASE_SCAN_DIR) + scan_dir = scan_dir_manager.new_scan_dir("fake_scan") + # This fake dzi should have loads of images. The DZI should not be zipped! + _make_fake_dzi(scan_dir) + all_files = scan_dir.all_files() + + # As standard for 8 layers there are 89 jpegs and 1 dzi file. + assert len(set(all_files)) == 90 + assert len(all_files) == 90 + dzi_file = None + # Check all files exist + for file in all_files: + assert os.path.exists(os.path.join(scan_dir.dir_path, file)) + if file.endswith(".dzi"): + # There is only 1 dzi file, so this should be None + assert dzi_file is None + dzi_file = file + # Once loop is complete there should be a dzi file + assert dzi_file is not None + # Get the dzi tile directory name + dzi_dir = os.path.basename(dzi_file[:-4] + "_files") + + # Get all files skipping the dzi tile directory + all_files = scan_dir.all_files(skip_dirs=dzi_dir) + # There is now only one file + assert len(all_files) == 1 + # It is the DZI file + assert all_files[0] == dzi_file def test_creating_scan_dir_for_missing_scan(): """Check creating ScanDirectory object for a dir that doesn't exist fails.""" _clear_scan_dir() with pytest.raises(FileNotFoundError): - ScanDirectory(BASE_SCAN_DIR, "not_real_0001") + ScanDirectory("not_real_0001", BASE_SCAN_DIR) def test_none_returned_for_missing_images_dir(): @@ -290,12 +396,60 @@ def test_none_returned_for_missing_images_dir(): """ _clear_scan_dir() os.makedirs(os.path.join(BASE_SCAN_DIR, "fake_scan_0001")) - scan_dir = ScanDirectory(BASE_SCAN_DIR, "fake_scan_0001") + scan_dir = ScanDirectory("fake_scan_0001", BASE_SCAN_DIR) assert scan_dir.images_dir is None # Also check that get scan files returns and empty list assert scan_dir.get_scan_files() == [] +def test_extracting_files(): + """Test the private _find_files method of ScanDirectories + + Add files to directory and check expected returns. + """ + _clear_scan_dir() + os.makedirs(os.path.join(BASE_SCAN_DIR, "fake_scan_0001", "images")) + scan_dir = ScanDirectory("fake_scan_0001", BASE_SCAN_DIR) + + # Starting all lists should be empty + scan_files = scan_dir.get_scan_files() + assert scan_dir._extract_scan_images(scan_files) == [] + assert scan_dir._extract_final_stitches(scan_files) == [] + assert scan_dir._extract_dzi_files(scan_files) == [] + + # Add a number of images + for i in range(2321): + _add_fake_image(scan_dir) + + scan_files = scan_dir.get_scan_files() + assert len(set(scan_dir._extract_scan_images(scan_files))) == 2321 + assert len(set(scan_dir._extract_final_stitches(scan_files))) == 0 + assert len(set(scan_dir._extract_dzi_files(scan_files))) == 0 + + # Add and a stitched image + _add_fake_file(scan_dir, "fake_scan_0001_stitched.jpg", in_im_dir=True) + + scan_files = scan_dir.get_scan_files() + assert len(set(scan_dir._extract_scan_images(scan_files))) == 2321 + assert len(set(scan_dir._extract_final_stitches(scan_files))) == 1 + assert len(set(scan_dir._extract_dzi_files(scan_files))) == 0 + + _make_fake_dzi(scan_dir) + + # check totals are still correct after adding a dzi with lots of tiles. + scan_files = scan_dir.get_scan_files() + scan_images = scan_dir._extract_scan_images(scan_files) + stitches = scan_dir._extract_final_stitches(scan_files) + dzi_files = scan_dir._extract_dzi_files(scan_files) + assert len(set(scan_images)) == 2321 + assert len(set(stitches)) == 1 + assert len(set(dzi_files)) == 1 + + # And check the names are as expected + assert stitches[0] == "fake_scan_0001_stitched.jpg" + assert dzi_files[0] == "fake_scan_0001.dzi" + + DiskUsage = namedtuple("DiskUsage", ["total", "used", "free"]) diff --git a/webapp/src/assets/less/theme.less b/webapp/src/assets/less/theme.less index a8c6e80f..c3c89bf6 100644 --- a/webapp/src/assets/less/theme.less +++ b/webapp/src/assets/less/theme.less @@ -17,6 +17,7 @@ @global-primary-background: #C32280; @inverse-primary-muted-color: lighten(@global-primary-background, 15%); +@darkened-primary-color: darken(@global-primary-background, 15%); @inverse-global-color: fade(@global-inverse-color, 80%); @global-border: #d5d5d5; @@ -201,7 +202,6 @@ h4, .uk-h4 { .uk-card { border-radius: @paper-border-radius; - //border: 1px solid rgba(180, 180, 180, 0.25); box-shadow: @small-shadow; } @@ -220,31 +220,13 @@ h4, .uk-h4 { margin: 0 auto; } -.uk-card-default .uk-card-footer { - border-top: 1px solid rgba(180, 180, 180, 0.25); -} - -.uk-card-primary .uk-card-footer { - border-top: 1px solid rgba(0, 0, 0, 0.075); -} - -.uk-card-default { - color: @global-color; -} - .hook-inverse() { // Override background colour in dark mode - .uk-card-default { + .uk-card { background-color: #2a2a2a !important; color: @inverse-global-color; } - - // Lighten on hover to show depth in dark mode - .uk-card-default:hover { - transition: background-color @animation-fast-duration ease; - background-color: #333 !important; - } } /* @@ -383,9 +365,15 @@ a:hover { border: 1px solid #f0506e; } +.uk-button-disabled { + border: 1px solid #999999 !important; + background-color: #cccccc !important; + color: #919191 !important; +} + .hook-inverse() { .uk-button { - background-color: rgba(180, 180, 180, 0.15); + background-color: rgba(27, 13, 13, 0.15); color: @inverse-primary-muted-color; border-color: @inverse-primary-muted-color; } @@ -396,7 +384,7 @@ a:hover { } .uk-button-primary { - background-color: rgba(180, 180, 180, 0.15); + background-color: rgb(0, 0, 0); color: @inverse-primary-muted-color; border-color: @inverse-primary-muted-color; } @@ -406,6 +394,12 @@ a:hover { color: #fff; border-color: #fff; } + + .uk-button-disabled { + border: 1px solid @darkened-primary-color !important; + background-color: #333333 !important; + color: #777777 !important; + } } .uk-icon-button, diff --git a/webapp/src/components/labThingsComponents/endpointButton.vue b/webapp/src/components/labThingsComponents/endpointButton.vue new file mode 100644 index 00000000..d7882ef6 --- /dev/null +++ b/webapp/src/components/labThingsComponents/endpointButton.vue @@ -0,0 +1,50 @@ + + + + + diff --git a/webapp/src/components/tabContentComponents/loggingContent.vue b/webapp/src/components/tabContentComponents/loggingContent.vue index 38f3520c..f1628462 100644 --- a/webapp/src/components/tabContentComponents/loggingContent.vue +++ b/webapp/src/components/tabContentComponents/loggingContent.vue @@ -30,12 +30,12 @@
- Download Log File +
@@ -94,12 +94,14 @@