From 666dced86d787956f29c8ed16b068b10b46354d0 Mon Sep 17 00:00:00 2001 From: jaknapper Date: Wed, 16 Apr 2025 20:28:33 +0100 Subject: [PATCH 1/7] Update the GUI download links to point to new download_zip function --- webapp/src/components/tabContentComponents/scanListContent.vue | 2 +- webapp/src/components/tabContentComponents/slideScanContent.vue | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/webapp/src/components/tabContentComponents/scanListContent.vue b/webapp/src/components/tabContentComponents/scanListContent.vue index 7bbb4a8a..3d018826 100644 --- a/webapp/src/components/tabContentComponents/scanListContent.vue +++ b/webapp/src/components/tabContentComponents/scanListContent.vue @@ -68,7 +68,7 @@

{{ item.name }}

Date: Wed, 16 Apr 2025 20:29:35 +0100 Subject: [PATCH 2/7] Split out zip functions and copying stitched image --- .../things/smart_scan.py | 117 +++++++++++++----- 1 file changed, 84 insertions(+), 33 deletions(-) diff --git a/src/openflexure_microscope_server/things/smart_scan.py b/src/openflexure_microscope_server/things/smart_scan.py index 21a541ac..2d8578f6 100644 --- a/src/openflexure_microscope_server/things/smart_scan.py +++ b/src/openflexure_microscope_server/things/smart_scan.py @@ -203,6 +203,9 @@ class SmartScanThing(Thing): try: self._check_background_and_csm_set() self._ongoing_scan_name = self._get_unique_scan_name_and_dir(scan_name) + self.create_zip_of_scan( + logger=self._scan_logger, scan_name=self._ongoing_scan_name + ) self._autofocus.looping_autofocus(dz=self.autofocus_dz, start="centre") # record starting position so we can return there self._starting_position = self._stage.position @@ -232,6 +235,34 @@ class SmartScanThing(Thing): self._scan_data = None self._scan_lock.release() + def promote_stitch_files( + self, + logger, + ): + """Copy the stitched image from the scan images folder to the top level""" + + # Search the scan images dir for a file ending in '_stitched.jpg + stitched_image_path = glob.glob( + os.path.join(self._ongoing_scan_images_dir, "*_stitched.jpg") + ) + + if len(stitched_image_path) == 0: + logger.warning("Could't find a stitched image to copy") + else: + if len(stitched_image_path) > 1: + logger.warning( + "Found more than one stitched image to copy, you should check them carefully" + ) + for stitched_image in stitched_image_path: + stitch_name = os.path.basename(stitched_image) + + shutil.copy( + stitched_image_path, + os.path.join( + self.dir_for_scan(self._ongoing_scan_name), stitch_name + ), + ) + @_scan_running def _check_background_and_csm_set(self): """Before starting a scan check that background and camera-stage-mapping are set @@ -597,11 +628,10 @@ class SmartScanThing(Thing): # increment capure counter as thread has completed self._scan_images_taken += 1 # Add it to the incremental zip - self.create_zip_of_scan( - logger=self._scan_logger, - scan_name=self._ongoing_scan_name, - download_zip=False, - ) + self.update_zip( + logger=self._scan_logger, + scan_name=self._ongoing_scan_name, + ) @_scan_running def _try_autofocus( @@ -655,10 +685,10 @@ class SmartScanThing(Thing): self._scan_logger.info("Not performing a stitch as 3 or fewer images taken") return - self.create_zip_of_scan( + self.update_zip( logger=self._scan_logger, scan_name=self._ongoing_scan_name, - download_zip=False, + final_version=False, ) self._scan_logger.info("Waiting for background processes to finish...") @@ -671,6 +701,7 @@ class SmartScanThing(Thing): scan_name=self._ongoing_scan_name, overlap=self._scan_data["overlap"], ) + self.promote_stitch_files(self._scan_logger) except SubprocessError as e: self._scan_logger.error(f"Stitching failed: {e}", exc_info=e) @@ -1044,13 +1075,8 @@ class SmartScanThing(Thing): self, logger: InvocationLogger, scan_name: str, - download_zip=True, - ) -> ZipBlob: - """Generate a zip file that can be downloaded, with all the scan files in it. - - Note that as this is a thing_action it needs the logger passed as - a variable if called from another thing action - """ + ) -> None: + """Generate an empty zip file for the current scan""" images_folder = self.images_dir_for_scan(scan_name=scan_name) scan_folder = self.dir_for_scan(scan_name=scan_name) @@ -1062,12 +1088,29 @@ class SmartScanThing(Thing): zip_fname = os.path.join(scan_folder, "images.zip") # Create an empty zip file - we don't want to autofill it with files, - # as some of them should only be added at the end (as we can't overwrite) + # as some of them should only be added at the end, as we can't overwrite # them once they change if not os.path.isfile(zip_fname): - with zipfile.ZipFile(zip_fname, mode="w") as scan_zip: + with zipfile.ZipFile(zip_fname, mode="w"): pass + def update_zip( + self, + logger: InvocationLogger, + scan_name: str, + final_version: bool = False, + ) -> None: + """Update the zip file with any images added to the folder since the last run, + except for files containing 'files_to_delay', which are files to only include + once the scan is finished or the user wants to download the zip + """ + scan_folder = self.dir_for_scan(scan_name=scan_name) + + zip_fname = os.path.join(scan_folder, "images.zip") + + # if not os.path.isfile(zip_fname): + # raise RuntimeError("zip file hasn't been created yet") + # get a list of files in the existing zip current_zip = self.get_files_in_zip(zip_fname) @@ -1085,43 +1128,51 @@ class SmartScanThing(Thing): "stitched.jp", "stitched_from", "stitched.om", + "stitching_correlations", ] - tiff_name = "" with zipfile.ZipFile(zip_fname, mode="a") as scan_zip: for file in files: - if "stitched.jp" in file: - stitch_name = os.path.split(file)[1] - if ".ome.tiff" in file: - tiff_name = os.path.split(file)[1] - if any(banned_name in file for banned_name in files_to_delay): + if any(skipped_name in file for skipped_name in files_to_delay): pass elif file in current_zip: pass - elif ".zip" in file or "raw" in file: + elif ".zip" in file: pass else: logger.info(f"appending {file} to zip") scan_zip.write(os.path.join(scan_folder, file), arcname=file) - # Promote key files to the top level of the zip only at the end of the scan (when downloading) - # and finally zip some of the final files + # Finally zip the updating files we skipped previously # TODO: if you download multiple times, you get duplicate files - is this a problem? - if download_zip: + # TODO: check if they're already in + if final_version: with zipfile.ZipFile(zip_fname, mode="a") as scan_zip: - for fname in ["stitched_from_stage.jpg", stitch_name, tiff_name]: - fpath = os.path.join(images_folder, fname) - if os.path.exists(fpath): - logger.info(f"copying {fpath} to upper level") - scan_zip.write(fpath, arcname=fname) for file in files: if any(banned_name in file for banned_name in files_to_delay): logger.info(f"we are finally adding {file} into zip") scan_zip.write(os.path.join(scan_folder, file), arcname=file) - logger.info("about to download zip") - return ZipBlob.from_file(zip_fname) @thing_action + def download_zip( + self, + scan_name: str, + logger: InvocationLogger, + ): + """Update the zip to include the files we leave to the end, then return the + zip file as a Blob""" + self.update_zip( + logger=logger, + scan_name=scan_name, + final_version=True, + ) + + scan_folder = self.dir_for_scan(scan_name=scan_name) + + zip_fname = os.path.join(scan_folder, "images.zip") + logger.info("about to download zip") + return ZipBlob.from_file(zip_fname) + def get_files_in_zip(self, zip_path): """List the relative paths of all files and folders in the zip folder specified""" scan_zip = zipfile.ZipFile(zip_path) From a88349dbfe4b0c24368d0ad7d446723d586c8a10 Mon Sep 17 00:00:00 2001 From: jaknapper Date: Wed, 7 May 2025 17:56:29 +0100 Subject: [PATCH 3/7] Loop through all stitched images --- src/openflexure_microscope_server/things/smart_scan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/openflexure_microscope_server/things/smart_scan.py b/src/openflexure_microscope_server/things/smart_scan.py index 2d8578f6..43da4892 100644 --- a/src/openflexure_microscope_server/things/smart_scan.py +++ b/src/openflexure_microscope_server/things/smart_scan.py @@ -257,7 +257,7 @@ class SmartScanThing(Thing): stitch_name = os.path.basename(stitched_image) shutil.copy( - stitched_image_path, + stitched_image, os.path.join( self.dir_for_scan(self._ongoing_scan_name), stitch_name ), From a9d06f7d7e3a3d3fa84f5ce08f11db9522885919 Mon Sep 17 00:00:00 2001 From: jaknapper Date: Wed, 7 May 2025 18:26:26 +0100 Subject: [PATCH 4/7] Remove commented out code, unused arguments and extra logging --- src/openflexure_microscope_server/things/smart_scan.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/openflexure_microscope_server/things/smart_scan.py b/src/openflexure_microscope_server/things/smart_scan.py index 43da4892..2fce0186 100644 --- a/src/openflexure_microscope_server/things/smart_scan.py +++ b/src/openflexure_microscope_server/things/smart_scan.py @@ -1096,7 +1096,6 @@ class SmartScanThing(Thing): def update_zip( self, - logger: InvocationLogger, scan_name: str, final_version: bool = False, ) -> None: @@ -1108,9 +1107,6 @@ class SmartScanThing(Thing): zip_fname = os.path.join(scan_folder, "images.zip") - # if not os.path.isfile(zip_fname): - # raise RuntimeError("zip file hasn't been created yet") - # get a list of files in the existing zip current_zip = self.get_files_in_zip(zip_fname) @@ -1140,7 +1136,6 @@ class SmartScanThing(Thing): elif ".zip" in file: pass else: - logger.info(f"appending {file} to zip") scan_zip.write(os.path.join(scan_folder, file), arcname=file) # Finally zip the updating files we skipped previously @@ -1150,19 +1145,16 @@ class SmartScanThing(Thing): with zipfile.ZipFile(zip_fname, mode="a") as scan_zip: for file in files: if any(banned_name in file for banned_name in files_to_delay): - logger.info(f"we are finally adding {file} into zip") scan_zip.write(os.path.join(scan_folder, file), arcname=file) @thing_action def download_zip( self, scan_name: str, - logger: InvocationLogger, ): """Update the zip to include the files we leave to the end, then return the zip file as a Blob""" self.update_zip( - logger=logger, scan_name=scan_name, final_version=True, ) @@ -1170,7 +1162,6 @@ class SmartScanThing(Thing): scan_folder = self.dir_for_scan(scan_name=scan_name) zip_fname = os.path.join(scan_folder, "images.zip") - logger.info("about to download zip") return ZipBlob.from_file(zip_fname) def get_files_in_zip(self, zip_path): From b31a5e44fb3823c5a923cf216006f9af7e50182f Mon Sep 17 00:00:00 2001 From: jaknapper Date: Wed, 7 May 2025 18:40:57 +0100 Subject: [PATCH 5/7] Rename banned names to delayed --- src/openflexure_microscope_server/things/smart_scan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/openflexure_microscope_server/things/smart_scan.py b/src/openflexure_microscope_server/things/smart_scan.py index 2fce0186..471bd93c 100644 --- a/src/openflexure_microscope_server/things/smart_scan.py +++ b/src/openflexure_microscope_server/things/smart_scan.py @@ -1144,7 +1144,7 @@ class SmartScanThing(Thing): if final_version: with zipfile.ZipFile(zip_fname, mode="a") as scan_zip: for file in files: - if any(banned_name in file for banned_name in files_to_delay): + if any(delayed_name in file for delayed_name in files_to_delay): scan_zip.write(os.path.join(scan_folder, file), arcname=file) @thing_action From 6f8e91afe7439b8856d9c6a65b00f1a4bd9101b6 Mon Sep 17 00:00:00 2001 From: Joe Knapper Date: Wed, 7 May 2025 18:42:41 +0000 Subject: [PATCH 6/7] Apply 1 suggestion(s) to 1 file(s) --- src/openflexure_microscope_server/things/smart_scan.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/openflexure_microscope_server/things/smart_scan.py b/src/openflexure_microscope_server/things/smart_scan.py index 471bd93c..a312b173 100644 --- a/src/openflexure_microscope_server/things/smart_scan.py +++ b/src/openflexure_microscope_server/things/smart_scan.py @@ -249,10 +249,6 @@ class SmartScanThing(Thing): if len(stitched_image_path) == 0: logger.warning("Could't find a stitched image to copy") else: - if len(stitched_image_path) > 1: - logger.warning( - "Found more than one stitched image to copy, you should check them carefully" - ) for stitched_image in stitched_image_path: stitch_name = os.path.basename(stitched_image) From 736b82d07144dd66539d586eef1d1d8a3598fbb4 Mon Sep 17 00:00:00 2001 From: jaknapper Date: Wed, 7 May 2025 20:31:06 +0100 Subject: [PATCH 7/7] Remove logger from zipping --- src/openflexure_microscope_server/things/smart_scan.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/openflexure_microscope_server/things/smart_scan.py b/src/openflexure_microscope_server/things/smart_scan.py index a312b173..2c9f1913 100644 --- a/src/openflexure_microscope_server/things/smart_scan.py +++ b/src/openflexure_microscope_server/things/smart_scan.py @@ -625,9 +625,8 @@ class SmartScanThing(Thing): self._scan_images_taken += 1 # Add it to the incremental zip self.update_zip( - logger=self._scan_logger, - scan_name=self._ongoing_scan_name, - ) + scan_name=self._ongoing_scan_name, + ) @_scan_running def _try_autofocus( @@ -682,7 +681,6 @@ class SmartScanThing(Thing): return self.update_zip( - logger=self._scan_logger, scan_name=self._ongoing_scan_name, final_version=False, )