rahix/openflexure-microscope-server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Zip files as we scan, saving the regenerated ones to the end

Browse source 
Removed maximum scan size of 750 images

Save json of scan inputs and use it for future stitching (ensures consistent parameters)

Moved looping autofocus to autofocus thing from recentering thing

Rewrote looping autofocus to skip unneeded moves, as if peak is out of range, no need to nicely focus
This commit is contained in:
jaknapper 2024-01-30 18:46:34 +00:00
parent a0b98a1adc
commit e6e809cea4
3 changed files with 139 additions and 53 deletions
Download patch file Download diff file Expand all files Collapse all files

118
src/openflexure_microscope_server/things/smart_scan.py
View file

@ -14,6 +14,9 @@ from scipy.stats import norm
from copy import deepcopy
from datetime import datetime
from subprocess import CompletedProcess, Popen, PIPE, SubprocessError, run
import glob
import zipfile
import json
from labthings_fastapi.thing import Thing
from labthings_fastapi.dependencies.thing import direct_thing_client_dependency
@ -470,6 +473,19 @@ class SmartScanThing(Thing):
os.mkdir(raw_images_folder)
logger.info(f"Saving images to {images_folder}")
data = {
'scan_name' : scan_name,
'overlap' : overlap,
'autofocus range' : self.autofocus_dz,
'dx' : dx,
'dy' : dy,
'start time' : start_time,
'skipping background' : self.skip_background
}
with open(os.path.join(images_folder, 'scan_inputs.json'), 'w', encoding='utf-8') as f:
json.dump(data, f, ensure_ascii=False, indent=4)
# move to each x-y position. in z, move to the height of the closest x-y position that successfully focused
while len(path) > 0:
ensure_free_disk_space(scan_folder)
@ -519,7 +535,7 @@ class SmartScanThing(Thing):
attempts = 0
if self.autofocus_dz > 200:
while True:
recentre.looping_autofocus(dz=self.autofocus_dz, start = 'base')
autofocus.looping_autofocus(dz=self.autofocus_dz, start = 'base')
current_height = stage.position["z"]
# if there have been successful autofocuses in this scan, find the closest one in x-y
@ -581,6 +597,8 @@ class SmartScanThing(Thing):
if not self.correlate_running():
self.correlate_start(images_folder, overlap=overlap)
self.create_zip_of_scan(logger = logger, scan_name = scan_folder.split('scans/')[1], download_zip = False)
# add the current position to the list of all positions visited
true_path.append(loc)
@ -599,8 +617,6 @@ class SmartScanThing(Thing):
path = sorted(path, key=lambda x: (steps_from_centre(x, true_path[0][:2], dx, dy), distance_to_site(loc[:2], x)))
if len(true_path) > 750:
break
except InvocationCancelledError:
logger.error("Stopping scan because it was cancelled.")
except NotEnoughFreeSpaceError as e:
@ -847,7 +863,7 @@ class SmartScanThing(Thing):
raise RuntimeError("Only one subprocess is allowed at a time")
with self._correlate_popen_lock:
self._correlate_popen = Popen(
[self._script, "--stitching_mode", "only_correlate", "--minimum_overlap", f"{overlap*0.9}", images_folder]
[self._script, "--stitching_mode", "only_correlate", "--minimum_overlap", f"{round(overlap*0.9, 2)}", images_folder]
)
def correlate_running(self) -> bool:
@ -877,13 +893,23 @@ class SmartScanThing(Thing):
return output
@thing_action
def stitch_scan(self, logger: InvocationLogger, scan_name: Optional[str]=None, overlap: float = 0.1) -> None:
def stitch_scan(self, logger: InvocationLogger, scan_name: Optional[str]=None, overlap: float = 0.0) -> None:
"""Generate a stitched image based on stage position metadata"""
images_folder = self.images_folder(scan_name=scan_name)
self.run_subprocess(logger, [self._script, "--stitching_mode", "all", "--minimum_overlap", f"{overlap*0.9}", images_folder])
if overlap == 0.0:
try:
with open(os.path.join(images_folder, 'scan_inputs.json')) as data_file:
data_loaded = json.load(data_file)
logger.info(data_loaded)
overlap = data_loaded['overlap']
except:
overlap = 0.1
logger.info(overlap)
self.run_subprocess(logger, [self._script, "--stitching_mode", "all", "--minimum_overlap", f"{round(overlap*0.9,3)}", images_folder])
@thing_action
def create_zip_of_scan(self, logger: InvocationLogger, scan_name: Optional[str]=None) -> ZipBlob:
def create_zip_of_scan(self, logger: InvocationLogger, scan_name: Optional[str]=None, download_zip = True) -> ZipBlob:
"""Generate a zip file that can be downloaded, with all the scan files in it."""
images_folder = self.images_folder(scan_name=scan_name)
scan_folder = self.scan_folder_path(scan_name=scan_name)
@ -895,19 +921,67 @@ class SmartScanThing(Thing):
if not os.path.isdir(images_folder):
raise FileNotFoundError(f"Tried to make a zip archive of {images_folder} but it does not exist.")
logger.info("Creating zip archive of images (may take some time)...")
shutil.make_archive(
os.path.join(scan_folder, "images"),
"zip",
scan_folder,
"images/",
logger=logger
)
zip_fname = os.path.join(scan_folder, "images.zip")
# Promote key files to the top level of the zip
with zipfile.ZipFile(zip_fname, mode="a") as zip:
for fname in ["stitched_from_stage.jpg", "stitched.jpg"]:
fpath = os.path.join(images_folder, fname)
if os.path.exists(fpath):
zip.write(fpath, arcname=fname)
return ZipBlob.from_file(zip_fname)
zip_fname = f'{os.path.join(scan_folder, "images")}.zip'
# Create an empty zip file - we don't want to autofill it with files,
# as some of them should only be added at the end (as we can't overwrite)
# them once they change
if not os.path.isfile(zip_fname):
with zipfile.ZipFile(zip_fname, mode="w") as zip:
pass
# get a list of files in the existing zip
current_zip = self.get_files_in_zip(zip_fname)
logger.info(current_zip)
# get a list of files in the folder we're zipping
folder_path = self.scan_folder_path(scan_name)
files = glob.glob(folder_path + '/**/*', recursive=True)
files = [i.split(f'{folder_path}/')[1] for i in files]
# This is a list of file names that are updated as the scan goes,
# and should only be zipped at the end of the scan - otherwise they'll
# be appended on every loop as we can't overwrite files in the zip
files_to_delay = ['TileConfiguration', 'tiling_cache', 'stitched.jp']
with zipfile.ZipFile(zip_fname, mode="a") as zip:
for file in files:
if any(banned_name in file for banned_name in files_to_delay):
logger.info(f'we only add {file} into zip at the end of the scan')
elif file in current_zip:
logger.info(f'{file} is already in zip')
elif ".zip" in file:
logger.info('Not adding the .zip to itself')
else:
logger.info(f'appending {file} to zip')
zip.write(os.path.join(folder_path, file), arcname=file)
images_folder = os.path.join(folder_path, 'images')
# Promote key files to the top level of the zip only at the end of the scan (when downloading)
# and finally zip some of the final files
# TODO: if you download multiple times, you get duplicate files - is this a problem?
if download_zip:
with zipfile.ZipFile(zip_fname, mode="a") as zip:
for fname in ["stitched_from_stage.jpg", "stitched.jpg"]:
fpath = os.path.join(images_folder, fname)
if os.path.exists(fpath):
logger.info(f'copying {fpath} to upper level')
zip.write(fpath, arcname=fname)
for file in files:
if any(banned_name in file for banned_name in files_to_delay):
logger.info(f'we are finally adding {file} into zip')
zip.write(os.path.join(folder_path, file), arcname=file)
return ZipBlob.from_file(zip_fname)
@thing_action
def get_files_in_zip(self, zip_path):
"""List the relative paths of all files and folders in the zip folder specified"""
zip = zipfile.ZipFile(zip_path)
zip = [os.path.normpath(i) for i in zip.namelist()]
return zip