From 528406137f6746eafd1ad4c7f87c876c145f0277 Mon Sep 17 00:00:00 2001 From: Joe Knapper Date: Mon, 29 Jan 2024 18:10:11 +0000 Subject: [PATCH 1/4] Start autofocus from base of range for speed --- .../things/auto_recentre_stage.py | 12 ++++++------ .../things/autofocus.py | 6 ++++-- .../things/smart_scan.py | 12 ++++++------ 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/openflexure_microscope_server/things/auto_recentre_stage.py b/src/openflexure_microscope_server/things/auto_recentre_stage.py index 0e46e97b..ba11a900 100644 --- a/src/openflexure_microscope_server/things/auto_recentre_stage.py +++ b/src/openflexure_microscope_server/things/auto_recentre_stage.py @@ -33,14 +33,14 @@ def unpack_autofocus(scan_data): jpeg_heights = np.interp(jpeg_times, stage_times, stage_height) - turning = np.where(turningpoints(jpeg_heights))[0] + 1 + turning = np.where(turningpoints(jpeg_heights))[0] return jpeg_heights[turning[0] : turning[1]], jpeg_sizes_MB[turning[0] : turning[1]] class RecentringThing(Thing): @thing_action - def looping_autofocus(self, autofocus: AutofocusDep, stage: StageDep, dz=2000): + def looping_autofocus(self, autofocus: AutofocusDep, stage: StageDep, dz=2000, start='centre'): """Repeatedly autofocus the stage until it looks focused. This action will run the `fast_autofocus` action until it settles on a point @@ -51,17 +51,17 @@ class RecentringThing(Thing): repeat = True attempts = 0 while repeat and attempts < 10: - height_min = stage.position["z"] - dz / 2 - height_max = stage.position["z"] + dz / 2 - data = autofocus.fast_autofocus(dz=dz) + data = autofocus.fast_autofocus(dz=dz, start=start) heights, _ = unpack_autofocus(data) - time.sleep(0.3) + height_min = np.min(heights) + height_max = np.max(heights) # TODO: max heights seems badly wrong! Something about turning? if ( stage.position["z"] - height_min < dz / 5 or height_max - stage.position["z"] < dz / 5 ): attempts += 1 + start = 'centre' else: repeat = False diff --git a/src/openflexure_microscope_server/things/autofocus.py b/src/openflexure_microscope_server/things/autofocus.py index 426fcdac..b70f8a44 100644 --- a/src/openflexure_microscope_server/things/autofocus.py +++ b/src/openflexure_microscope_server/things/autofocus.py @@ -143,7 +143,8 @@ class AutofocusThing(Thing): def fast_autofocus( self, m: SharpnessMonitorDep, - dz: int=2000 + dz: int=2000, + start: str='centre', ) -> SharpnessDataArrays: """Sweep the stage up and down, then move to the sharpest point @@ -153,7 +154,8 @@ class AutofocusThing(Thing): """ with m.run(): # Move to (-dz / 2) - m.focus_rel(-dz / 2) + if start == 'centre': + m.focus_rel(-dz / 2) # Move to dz while monitoring sharpness # i: Sharpness monitor index for this move # z: Final z position after move diff --git a/src/openflexure_microscope_server/things/smart_scan.py b/src/openflexure_microscope_server/things/smart_scan.py index e3a75083..0230d16b 100644 --- a/src/openflexure_microscope_server/things/smart_scan.py +++ b/src/openflexure_microscope_server/things/smart_scan.py @@ -480,15 +480,15 @@ class SmartScanThing(Thing): # TODO: combine this with the move below for speed (I think this could just be "else") logger.info(f"Moving to {loc}") - stage.move_absolute(x=int(loc[0]), y=int(loc[1]), z=int(loc[2])) if len(focused_path) > 1: z_index = closest(loc, focused_path) + z=int(focused_path[z_index][2]) # print('Moving to {0}'.format([coords[0], coords[1], focused_path[z_index][2]])) # print(focused_path) - stage.move_absolute( - x=int(loc[0]), y=int(loc[1]), z=int(focused_path[z_index][2]) - ) + stage.move_absolute( + x=int(loc[0]), y=int(loc[1]), z = z - self.autofocus_dz / 2 + ) # Check if the image is background if self.skip_background: @@ -517,7 +517,7 @@ class SmartScanThing(Thing): attempts = 0 if self.autofocus_dz > 200: while True: - recentre.looping_autofocus(dz=self.autofocus_dz) + recentre.looping_autofocus(dz=self.autofocus_dz, start = 'base') current_height = stage.position["z"] # if there have been successful autofocuses in this scan, find the closest one in x-y @@ -542,7 +542,7 @@ class SmartScanThing(Thing): focused_path.append(loc) break if attempts >= 3: - logger.warning("Could not autofocus after 5 attempts.") + logger.warning("Could not autofocus after 3 attempts.") break # if the autofocus was rejected, we return to the height of the closest successful autofocus. not perfect, but better than wandering out of focus logger.info( From a0b98a1adce89c01f6281059823da2e85c291916 Mon Sep 17 00:00:00 2001 From: jaknapper Date: Mon, 29 Jan 2024 19:56:24 +0000 Subject: [PATCH 2/4] Fixed autofocus to not combine logs --- .../things/auto_recentre_stage.py | 26 +------------------ .../things/autofocus.py | 6 ++--- .../things/smart_scan.py | 2 ++ 3 files changed, 6 insertions(+), 28 deletions(-) diff --git a/src/openflexure_microscope_server/things/auto_recentre_stage.py b/src/openflexure_microscope_server/things/auto_recentre_stage.py index ba11a900..2eb77055 100644 --- a/src/openflexure_microscope_server/things/auto_recentre_stage.py +++ b/src/openflexure_microscope_server/things/auto_recentre_stage.py @@ -15,29 +15,6 @@ CamDep = direct_thing_client_dependency(StreamingPiCamera2, "/camera/") CSMDep = direct_thing_client_dependency(CameraStageMapper, "/camera_stage_mapping/") AutofocusDep = direct_thing_client_dependency(AutofocusThing, "/autofocus/") - -def turningpoints(lst): - dx = np.diff(lst) - return dx[1:] * dx[:-1] < 0 - - -def unpack_autofocus(scan_data): - """Extract z, sharpness data from a move_and_measure call""" - scan_data = dict(scan_data) - jpeg_times = scan_data["jpeg_times"] - jpeg_sizes = scan_data["jpeg_sizes"] - jpeg_sizes_MB = [x / 10**3 for x in jpeg_sizes] - stage_times = scan_data["stage_times"] - stage_positions = scan_data["stage_positions"] - stage_height = [pos["z"] for pos in stage_positions] - - jpeg_heights = np.interp(jpeg_times, stage_times, stage_height) - - turning = np.where(turningpoints(jpeg_heights))[0] - - return jpeg_heights[turning[0] : turning[1]], jpeg_sizes_MB[turning[0] : turning[1]] - - class RecentringThing(Thing): @thing_action def looping_autofocus(self, autofocus: AutofocusDep, stage: StageDep, dz=2000, start='centre'): @@ -51,8 +28,7 @@ class RecentringThing(Thing): repeat = True attempts = 0 while repeat and attempts < 10: - data = autofocus.fast_autofocus(dz=dz, start=start) - heights, _ = unpack_autofocus(data) + heights, sizes = autofocus.fast_autofocus(dz=dz, start=start) height_min = np.min(heights) height_max = np.max(heights) # TODO: max heights seems badly wrong! Something about turning? diff --git a/src/openflexure_microscope_server/things/autofocus.py b/src/openflexure_microscope_server/things/autofocus.py index b70f8a44..01f41e58 100644 --- a/src/openflexure_microscope_server/things/autofocus.py +++ b/src/openflexure_microscope_server/things/autofocus.py @@ -119,7 +119,7 @@ class JPEGSharpnessMonitor: raise ValueError( "No images were captured during the move of the stage. Perhaps the camera is not streaming images?" ) - return jz[np.argmax(js)] + return jz[np.argmax(js)], jz, js def data_dict(self) -> SharpnessDataArrays: """Return the gathered data as a single convenient dictionary""" @@ -161,13 +161,13 @@ class AutofocusThing(Thing): # z: Final z position after move i, z = m.focus_rel(dz, block_cancellation=True) # Get the z position with highest sharpness from the previous move (index i) - fz: int = m.sharpest_z_on_move(i) + fz, heights, sizes = m.sharpest_z_on_move(i) # Move all the way to the start so it's consistent i, z = m.focus_rel(-dz) # Move to the target position fz (relative move of (fz - z)) m.focus_rel(fz - z) # Return all focus data - return m.data_dict() + return heights, sizes @thing_action def move_and_measure( diff --git a/src/openflexure_microscope_server/things/smart_scan.py b/src/openflexure_microscope_server/things/smart_scan.py index 0230d16b..6f9791d7 100644 --- a/src/openflexure_microscope_server/things/smart_scan.py +++ b/src/openflexure_microscope_server/things/smart_scan.py @@ -484,6 +484,8 @@ class SmartScanThing(Thing): if len(focused_path) > 1: z_index = closest(loc, focused_path) z=int(focused_path[z_index][2]) + else: + z = loc[2] # print('Moving to {0}'.format([coords[0], coords[1], focused_path[z_index][2]])) # print(focused_path) stage.move_absolute( From e6e809cea45e2618f0be7481cb6d014521eb734f Mon Sep 17 00:00:00 2001 From: jaknapper Date: Tue, 30 Jan 2024 18:46:34 +0000 Subject: [PATCH 3/4] Zip files as we scan, saving the regenerated ones to the end Removed maximum scan size of 750 images Save json of scan inputs and use it for future stitching (ensures consistent parameters) Moved looping autofocus to autofocus thing from recentering thing Rewrote looping autofocus to skip unneeded moves, as if peak is out of range, no need to nicely focus --- .../things/auto_recentre_stage.py | 31 +---- .../things/autofocus.py | 43 ++++++- .../things/smart_scan.py | 118 ++++++++++++++---- 3 files changed, 139 insertions(+), 53 deletions(-) diff --git a/src/openflexure_microscope_server/things/auto_recentre_stage.py b/src/openflexure_microscope_server/things/auto_recentre_stage.py index 2eb77055..f06a148b 100644 --- a/src/openflexure_microscope_server/things/auto_recentre_stage.py +++ b/src/openflexure_microscope_server/things/auto_recentre_stage.py @@ -16,31 +16,6 @@ CSMDep = direct_thing_client_dependency(CameraStageMapper, "/camera_stage_mappin AutofocusDep = direct_thing_client_dependency(AutofocusThing, "/autofocus/") class RecentringThing(Thing): - @thing_action - def looping_autofocus(self, autofocus: AutofocusDep, stage: StageDep, dz=2000, start='centre'): - """Repeatedly autofocus the stage until it looks focused. - - This action will run the `fast_autofocus` action until it settles on a point - in the middle 3/5 of its range. Such logic can be helpful if the microscope - is close to focus, but not quite within `dz/2`. It will attempt to autofocus - up to 10 times. - """ - repeat = True - attempts = 0 - while repeat and attempts < 10: - heights, sizes = autofocus.fast_autofocus(dz=dz, start=start) - height_min = np.min(heights) - height_max = np.max(heights) - # TODO: max heights seems badly wrong! Something about turning? - if ( - stage.position["z"] - height_min < dz / 5 - or height_max - stage.position["z"] < dz / 5 - ): - attempts += 1 - start = 'centre' - else: - repeat = False - @thing_action def recentre( self, @@ -79,7 +54,7 @@ class RecentringThing(Thing): # A list of all the positions we've focused focused_pos = [[], []] - self.looping_autofocus(autofocus, stage) + autofocus.looping_autofocus() for direction in [0, 1]: # Start off with the current position, and moving in the positive direction @@ -109,7 +84,7 @@ class RecentringThing(Thing): stage.move_absolute( x=int(destination[0]), y=int(destination[1]), z=destination[2] ) - self.looping_autofocus(autofocus, stage) + autofocus.looping_autofocus(autofocus, stage) position = list(stage.position.values()) focused_pos[direction].append(position) @@ -166,7 +141,7 @@ class RecentringThing(Thing): direction ] stage.move_absolute(x=centre[0], y=centre[1], z=centre[2]) - self.looping_autofocus(autofocus, stage) + autofocus.looping_autofocus() logging.info(f"Centre of ROM is at {centre, stage.position['z']} \n") diff --git a/src/openflexure_microscope_server/things/autofocus.py b/src/openflexure_microscope_server/things/autofocus.py index 01f41e58..4e966d27 100644 --- a/src/openflexure_microscope_server/things/autofocus.py +++ b/src/openflexure_microscope_server/things/autofocus.py @@ -119,7 +119,7 @@ class JPEGSharpnessMonitor: raise ValueError( "No images were captured during the move of the stage. Perhaps the camera is not streaming images?" ) - return jz[np.argmax(js)], jz, js + return jz[np.argmax(js)] def data_dict(self) -> SharpnessDataArrays: """Return the gathered data as a single convenient dictionary""" @@ -161,13 +161,13 @@ class AutofocusThing(Thing): # z: Final z position after move i, z = m.focus_rel(dz, block_cancellation=True) # Get the z position with highest sharpness from the previous move (index i) - fz, heights, sizes = m.sharpest_z_on_move(i) + fz: int = m.sharpest_z_on_move(i) # Move all the way to the start so it's consistent i, z = m.focus_rel(-dz) # Move to the target position fz (relative move of (fz - z)) m.focus_rel(fz - z) # Return all focus data - return heights, sizes + return m.data_dict() @thing_action def move_and_measure( @@ -194,3 +194,40 @@ class AutofocusThing(Thing): time.sleep(wait) m.focus_rel(current_dz) return m.data_dict() + + @thing_action + def looping_autofocus(self, stage: Stage, m: SharpnessMonitorDep, dz=2000, start='centre'): + """Repeatedly autofocus the stage until it looks focused. + + This action will run the `fast_autofocus` action until it settles on a point + in the middle 3/5 of its range. Such logic can be helpful if the microscope + is close to focus, but not quite within `dz/2`. It will attempt to autofocus + up to 10 times. + """ + repeat = True + attempts = 0 + + with m.run(): + while repeat and attempts < 10: + + if start == 'centre': + stage.move_relative(x = 0, y = 0, z = -dz / 2) + + i, z = m.focus_rel(dz, block_cancellation=True) + _, heights, sizes = m.move_data(i) + + peak_height = heights[np.argmax(sizes)] + height_min = np.min(heights) + height_max = np.max(heights) + + if ( + peak_height - height_min < dz / 5 + or height_max - peak_height < dz / 5 + ): + attempts += 1 + start = 'centre' + stage.move_absolute(z = peak_height) + else: + repeat = False + stage.move_relative(x = 0, y = 0, z = -dz) + stage.move_absolute(z = peak_height) \ No newline at end of file diff --git a/src/openflexure_microscope_server/things/smart_scan.py b/src/openflexure_microscope_server/things/smart_scan.py index 6f9791d7..0f8ee137 100644 --- a/src/openflexure_microscope_server/things/smart_scan.py +++ b/src/openflexure_microscope_server/things/smart_scan.py @@ -14,6 +14,9 @@ from scipy.stats import norm from copy import deepcopy from datetime import datetime from subprocess import CompletedProcess, Popen, PIPE, SubprocessError, run +import glob +import zipfile +import json from labthings_fastapi.thing import Thing from labthings_fastapi.dependencies.thing import direct_thing_client_dependency @@ -470,6 +473,19 @@ class SmartScanThing(Thing): os.mkdir(raw_images_folder) logger.info(f"Saving images to {images_folder}") + data = { + 'scan_name' : scan_name, + 'overlap' : overlap, + 'autofocus range' : self.autofocus_dz, + 'dx' : dx, + 'dy' : dy, + 'start time' : start_time, + 'skipping background' : self.skip_background + } + + with open(os.path.join(images_folder, 'scan_inputs.json'), 'w', encoding='utf-8') as f: + json.dump(data, f, ensure_ascii=False, indent=4) + # move to each x-y position. in z, move to the height of the closest x-y position that successfully focused while len(path) > 0: ensure_free_disk_space(scan_folder) @@ -519,7 +535,7 @@ class SmartScanThing(Thing): attempts = 0 if self.autofocus_dz > 200: while True: - recentre.looping_autofocus(dz=self.autofocus_dz, start = 'base') + autofocus.looping_autofocus(dz=self.autofocus_dz, start = 'base') current_height = stage.position["z"] # if there have been successful autofocuses in this scan, find the closest one in x-y @@ -581,6 +597,8 @@ class SmartScanThing(Thing): if not self.correlate_running(): self.correlate_start(images_folder, overlap=overlap) + self.create_zip_of_scan(logger = logger, scan_name = scan_folder.split('scans/')[1], download_zip = False) + # add the current position to the list of all positions visited true_path.append(loc) @@ -599,8 +617,6 @@ class SmartScanThing(Thing): path = sorted(path, key=lambda x: (steps_from_centre(x, true_path[0][:2], dx, dy), distance_to_site(loc[:2], x))) - if len(true_path) > 750: - break except InvocationCancelledError: logger.error("Stopping scan because it was cancelled.") except NotEnoughFreeSpaceError as e: @@ -847,7 +863,7 @@ class SmartScanThing(Thing): raise RuntimeError("Only one subprocess is allowed at a time") with self._correlate_popen_lock: self._correlate_popen = Popen( - [self._script, "--stitching_mode", "only_correlate", "--minimum_overlap", f"{overlap*0.9}", images_folder] + [self._script, "--stitching_mode", "only_correlate", "--minimum_overlap", f"{round(overlap*0.9, 2)}", images_folder] ) def correlate_running(self) -> bool: @@ -877,13 +893,23 @@ class SmartScanThing(Thing): return output @thing_action - def stitch_scan(self, logger: InvocationLogger, scan_name: Optional[str]=None, overlap: float = 0.1) -> None: + def stitch_scan(self, logger: InvocationLogger, scan_name: Optional[str]=None, overlap: float = 0.0) -> None: """Generate a stitched image based on stage position metadata""" images_folder = self.images_folder(scan_name=scan_name) - self.run_subprocess(logger, [self._script, "--stitching_mode", "all", "--minimum_overlap", f"{overlap*0.9}", images_folder]) + + if overlap == 0.0: + try: + with open(os.path.join(images_folder, 'scan_inputs.json')) as data_file: + data_loaded = json.load(data_file) + logger.info(data_loaded) + overlap = data_loaded['overlap'] + except: + overlap = 0.1 + logger.info(overlap) + self.run_subprocess(logger, [self._script, "--stitching_mode", "all", "--minimum_overlap", f"{round(overlap*0.9,3)}", images_folder]) @thing_action - def create_zip_of_scan(self, logger: InvocationLogger, scan_name: Optional[str]=None) -> ZipBlob: + def create_zip_of_scan(self, logger: InvocationLogger, scan_name: Optional[str]=None, download_zip = True) -> ZipBlob: """Generate a zip file that can be downloaded, with all the scan files in it.""" images_folder = self.images_folder(scan_name=scan_name) scan_folder = self.scan_folder_path(scan_name=scan_name) @@ -895,19 +921,67 @@ class SmartScanThing(Thing): if not os.path.isdir(images_folder): raise FileNotFoundError(f"Tried to make a zip archive of {images_folder} but it does not exist.") logger.info("Creating zip archive of images (may take some time)...") - shutil.make_archive( - os.path.join(scan_folder, "images"), - "zip", - scan_folder, - "images/", - logger=logger - ) - zip_fname = os.path.join(scan_folder, "images.zip") - # Promote key files to the top level of the zip - with zipfile.ZipFile(zip_fname, mode="a") as zip: - for fname in ["stitched_from_stage.jpg", "stitched.jpg"]: - fpath = os.path.join(images_folder, fname) - if os.path.exists(fpath): - zip.write(fpath, arcname=fname) - return ZipBlob.from_file(zip_fname) + + zip_fname = f'{os.path.join(scan_folder, "images")}.zip' + + # Create an empty zip file - we don't want to autofill it with files, + # as some of them should only be added at the end (as we can't overwrite) + # them once they change + if not os.path.isfile(zip_fname): + with zipfile.ZipFile(zip_fname, mode="w") as zip: + pass + + # get a list of files in the existing zip + current_zip = self.get_files_in_zip(zip_fname) + + logger.info(current_zip) + + # get a list of files in the folder we're zipping + folder_path = self.scan_folder_path(scan_name) + files = glob.glob(folder_path + '/**/*', recursive=True) + files = [i.split(f'{folder_path}/')[1] for i in files] + + # This is a list of file names that are updated as the scan goes, + # and should only be zipped at the end of the scan - otherwise they'll + # be appended on every loop as we can't overwrite files in the zip + files_to_delay = ['TileConfiguration', 'tiling_cache', 'stitched.jp'] + + with zipfile.ZipFile(zip_fname, mode="a") as zip: + for file in files: + if any(banned_name in file for banned_name in files_to_delay): + logger.info(f'we only add {file} into zip at the end of the scan') + elif file in current_zip: + logger.info(f'{file} is already in zip') + elif ".zip" in file: + logger.info('Not adding the .zip to itself') + else: + logger.info(f'appending {file} to zip') + zip.write(os.path.join(folder_path, file), arcname=file) + + + images_folder = os.path.join(folder_path, 'images') + # Promote key files to the top level of the zip only at the end of the scan (when downloading) + # and finally zip some of the final files + # TODO: if you download multiple times, you get duplicate files - is this a problem? + if download_zip: + with zipfile.ZipFile(zip_fname, mode="a") as zip: + for fname in ["stitched_from_stage.jpg", "stitched.jpg"]: + fpath = os.path.join(images_folder, fname) + if os.path.exists(fpath): + logger.info(f'copying {fpath} to upper level') + zip.write(fpath, arcname=fname) + for file in files: + if any(banned_name in file for banned_name in files_to_delay): + logger.info(f'we are finally adding {file} into zip') + zip.write(os.path.join(folder_path, file), arcname=file) + + return ZipBlob.from_file(zip_fname) + + @thing_action + def get_files_in_zip(self, zip_path): + """List the relative paths of all files and folders in the zip folder specified""" + zip = zipfile.ZipFile(zip_path) + zip = [os.path.normpath(i) for i in zip.namelist()] + + return zip From db51cd8067dcf260fb8c702939852c797e454dc1 Mon Sep 17 00:00:00 2001 From: jaknapper Date: Thu, 1 Feb 2024 15:31:14 +0000 Subject: [PATCH 4/4] Make overlap value consistent --- src/openflexure_microscope_server/things/smart_scan.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/openflexure_microscope_server/things/smart_scan.py b/src/openflexure_microscope_server/things/smart_scan.py index 0f8ee137..ae9eb7a9 100644 --- a/src/openflexure_microscope_server/things/smart_scan.py +++ b/src/openflexure_microscope_server/things/smart_scan.py @@ -597,8 +597,6 @@ class SmartScanThing(Thing): if not self.correlate_running(): self.correlate_start(images_folder, overlap=overlap) - self.create_zip_of_scan(logger = logger, scan_name = scan_folder.split('scans/')[1], download_zip = False) - # add the current position to the list of all positions visited true_path.append(loc) @@ -640,6 +638,7 @@ class SmartScanThing(Thing): stage.move_absolute(**starting_position, block_cancellation=True) finally: self._scan_lock.release() + self.create_zip_of_scan(logger = logger, scan_name = scan_folder.split('scans/')[1], download_zip = False) logger.info("Waiting for background processes to finish...") self.preview_stitch_wait() self.correlate_wait() @@ -905,8 +904,7 @@ class SmartScanThing(Thing): overlap = data_loaded['overlap'] except: overlap = 0.1 - logger.info(overlap) - self.run_subprocess(logger, [self._script, "--stitching_mode", "all", "--minimum_overlap", f"{round(overlap*0.9,3)}", images_folder]) + self.run_subprocess(logger, [self._script, "--stitching_mode", "all", "--minimum_overlap", f"{round(overlap*0.9,2)}", images_folder]) @thing_action def create_zip_of_scan(self, logger: InvocationLogger, scan_name: Optional[str]=None, download_zip = True) -> ZipBlob: @@ -944,7 +942,7 @@ class SmartScanThing(Thing): # This is a list of file names that are updated as the scan goes, # and should only be zipped at the end of the scan - otherwise they'll # be appended on every loop as we can't overwrite files in the zip - files_to_delay = ['TileConfiguration', 'tiling_cache', 'stitched.jp'] + files_to_delay = ['TileConfiguration', 'tiling_cache', 'stitched.jp', 'stitched_from'] with zipfile.ZipFile(zip_fname, mode="a") as zip: for file in files: