Updated tasks nomenclature

This commit is contained in:
Joel Collins 2020-07-21 12:54:42 +01:00
parent 56051f8863
commit 24398ba7dc
27 changed files with 100 additions and 136 deletions

View file

@ -1,5 +1,5 @@
from labthings.server.extensions import BaseExtension
from labthings.server.find import find_component
from labthings.extensions import BaseExtension
from labthings import find_component
def identify():

View file

@ -1,5 +1,5 @@
from labthings.server.extensions import BaseExtension
from labthings.server.find import find_component
from labthings.extensions import BaseExtension
from labthings import find_component
# Create the extension class

View file

@ -1,8 +1,6 @@
from labthings.server.extensions import BaseExtension
from labthings.server.find import find_component
from labthings.server.view import View
from labthings.server import fields
from labthings.extensions import BaseExtension
from labthings import find_component, fields
from labthings.views import View
## Extension methods

View file

@ -1,9 +1,6 @@
from labthings.server.extensions import BaseExtension
from labthings.server.find import find_component
from labthings.server.view import View
from labthings.server.schema import Schema
from labthings.server import fields
from labthings.extensions import BaseExtension
from labthings import find_component, Schema, fields
from labthings.views import View
## Extension methods

View file

@ -1,9 +1,6 @@
from labthings.server.extensions import BaseExtension
from labthings.server.find import find_component
from labthings.server.view import View, PropertyView
from labthings.server.schema import Schema
from labthings.server import fields
from labthings.extensions import BaseExtension
from labthings import find_component, Schema, fields
from labthings.views import View, PropertyView
## Extension methods

View file

@ -1,9 +1,6 @@
from labthings.server.extensions import BaseExtension
from labthings.server.find import find_component
from labthings.server.view import View, ActionView, PropertyView
from labthings.server.schema import Schema
from labthings.server import fields
from labthings.extensions import BaseExtension
from labthings import find_component, Schema, fields
from labthings.views import View, ActionView, PropertyView
from flask import send_file # Used to send images from our server
import io # Used in our capture action

View file

@ -1,9 +1,6 @@
from labthings.server.extensions import BaseExtension
from labthings.server.find import find_component
from labthings.server.view import View, ActionView
from labthings.server.schema import Schema
from labthings.server import fields
from labthings.extensions import BaseExtension
from labthings import find_component, Schema, fields, update_action_progress
from labthings.views import View, ActionView, PropertyView
from flask import send_file # Used to send images from our server
import io # Used in our capture action
@ -12,9 +9,6 @@ import time # Used in our timelapse function
# Used in our timelapse function
from openflexure_microscope.captures.capture_manager import generate_basename
# Used to run our timelapse in a background thread
from labthings.core.tasks import update_task_progress
## Extension methods
@ -49,7 +43,7 @@ def timelapse(microscope, n_images, t_between):
# Update task progress (only does anyting if the function is running in a LabThings task)
progress_pct = ((n + 1) / n_images) * 100 # Progress, in percent
update_task_progress(progress_pct)
update_action_progress(progress_pct)
# Wait for the specified time
time.sleep(t_between)

View file

@ -1,9 +1,6 @@
from labthings.server.extensions import BaseExtension
from labthings.server.find import find_component
from labthings.server.view import View, ActionView
from labthings.server.schema import Schema
from labthings.server import fields
from labthings.extensions import BaseExtension
from labthings import find_component, Schema, fields, update_action_progress
from labthings.views import View, ActionView, PropertyView
from flask import send_file # Used to send images from our server
import io # Used in our capture action
@ -12,9 +9,6 @@ import time # Used in our timelapse function
# Used in our timelapse function
from openflexure_microscope.captures.capture_manager import generate_basename
# Used to run our timelapse in a background thread
from labthings.core.tasks import update_task_progress
# Used to convert our GUI dictionary into a complete eV extension GUI
from openflexure_microscope.api.utilities.gui import build_gui
@ -51,7 +45,7 @@ def timelapse(microscope, n_images, t_between):
# Update task progress (only does anyting if the function is running in a LabThings task)
progress_pct = ((n + 1) / n_images) * 100 # Progress, in percent
update_task_progress(progress_pct)
update_action_progress(progress_pct)
# Wait for the specified time
time.sleep(t_between)

View file

@ -1,5 +1,5 @@
Tasks and Locks
===============
Threads and Locks
=================
Introduction
------------
@ -10,20 +10,22 @@ This introduces a couple of problems. Firstly, a request that triggers a long fu
Similarly, if your functionality takes a long time, it may be possible for other requests to interfere with your function. For example, in our hypothetical timelapse extension, while the timelapse is running, another user could open a connection and start moving the stage around, ruining the timelapse.
We get around these issues by making use of background tasks, and component locks.
We get around these issues by making use of action threads, and component locks.
Background tasks
----------------
Action threads
--------------
Tasks are introduced to manage long-running functions in a way that does not block HTTP requests. Any API Action will automatically run as a background task (that is, any View that subclasses `ActionView`).
Action threads are introduced to manage long-running functions in a way that does not block HTTP requests. Any API Action will automatically run as a background thread.
Internally, the ``tasks`` submodule stores a list of all requested tasks, and their states. This state stores the running status of the task (if itis idle, running, error, or success), information about the start and end times, a unique task ID, and, upon completion, the return value of the long-running function.
Internally, the :class:`labthings.LabThing` object stores a list of all requested actions, and their states. This state stores the running status of the action (if itis idle, running, error, or success), information about the start and end times, a unique ID, and, upon completion, the return value of the long-running function.
By using tasks, a function can be started in the background, and it's return value fetched at a later time once it has reported success. If a long-running task is started by some client, it should note the ID returned in the task state JSON, and use this to periodically check on the status of that particular task.
By using threads, a function can be started in the background, and it's return value fetched at a later time once it has reported success. If a long-running action is started by some client, it should note the ID returned in the action state JSON, and use this to periodically check on the status of that particular action.
API routes have been created to allow checking the state of all tasks (GET ``/tasks``), a particular task by ID (GET ``/tasks/<task_id>``), and terminating or removing individual tasks (DELETE ``/tasks/<task_id>``).
API routes have been created to allow checking the state of all actions (GET ``/actions``), a particular action by ID (GET ``/actions/<action_id>``), and terminating or removing individual actions (DELETE ``/actions/<action_id>``).
All Actions will return a serialized representation of the task, when your POST request returns. If the task completes within a default timeout period (usually 1 second) then the completed Task representation will be returned. If the task is still running after this timeout period, the "in-progress" Task representation will be returned. The final output value can then be retrieved at a later time.
All actions will return a serialized representation of the action state when your POST request returns. If the action completes within a default timeout period (usually 1 second) then the completed action representation will be returned. If the action is still running after this timeout period, the "in-progress" action representation will be returned. The final output value can then be retrieved at a later time.
Most users will not need to create instances of this class. Instead, they will be created automatically when a function is started by an API Action view.
An example of a long running task may look like:
@ -42,21 +44,21 @@ After some time, once the task has completed, it could be retreived using:
.. code-block:: python
...
from labthings import tasks
from labthings import current_labthing
def get_result(task_id):
matching_task = tasks.dict.get(task_id)
return matching_task.state
def get_result(action_id):
matching_action = current_labthing().actions.get(task_id)
return matching_action.state
or by making GET requests to the ``http://microscope.local/api/v2/tasks/<task_id>`` view.
Updating task progress
++++++++++++++++++++++
Updating action progress
++++++++++++++++++++++++
Some applications such as OpenFlexure eV are able to display progress bars showing the progress of a background task. Implementing progress updates in your extension is made easy with the :py:meth:`labthings.tasks.update_task_progress` function. This function takes a single argument, which is the task progress as an integer percent (0 - 100).
Some applications such as OpenFlexure eV are able to display progress bars showing the progress of an action thread. Implementing progress updates in your extension is made easy with the :py:meth:`labthings.update_action_progress` function. This function takes a single argument, which is the action progress as an integer percent (0 - 100).
If your long running function was started within a background task, this function will update the state of the corresponding task object. If your function is called outside of a long-running task (e.g. by another extension, directly), then this function will silently do nothing.
If your long running function was started within a background thread, this function will update the state of the corresponding action thread object. If your function is called outside of a long-running task (e.g. by another extension, directly), then this function will silently do nothing.
An example of task progress is included in the example later on this page.
@ -64,11 +66,11 @@ An example of task progress is included in the example later on this page.
Component Locks
---------------
Locks have been implemented to solve a distinct issue, most obvious when considering long-running tasks. During a long task such as a tile-scan or autofocus, it is absolutely necesarry to block any completing interaction with the microscope hardware. For example, even if the stage is not actively moving (for example during a capture phase within a tile scan), another user should not be able to move the microscope, interrupting the task. Thread locks act to prevent this.
Locks have been implemented to solve a distinct issue, most obvious when considering long-running actions. During a long action such as a tile-scan or autofocus, it is absolutely necesarry to block any completing interaction with the microscope hardware. For example, even if the stage is not actively moving (for example during a capture phase within a tile scan), another user should not be able to move the microscope, interrupting the action. Thread locks act to prevent this.
The camera and stage both contain an instance of :py:class:`labthings.lock.StrictLock`, named ``lock``. Built-in functions such as capture and move will always acquire this lock for the duration of the function. This ensures that, for example, simultaneous attemps to move do not occur.
More importantly, however, tasks can hold on to these locks for longer periods of time, blocking any other calls to the hardware.
More importantly, however, threads can hold on to these locks for longer periods of time, blocking any other calls to the hardware.
Locks are acquired using context managers, i.e. ``with component.lock: ...``
@ -76,7 +78,7 @@ Locks are acquired using context managers, i.e. ``with component.lock: ...``
Complete example
----------------
Implementing both tasks and locks in a new timelapse extension may look like:
Implementing both action threads and locks in a new timelapse extension may look like:
.. literalinclude:: ./example_extension/06_tasks_locks.py

View file

@ -1,8 +1,4 @@
#!/usr/bin/env python
from labthings.server import monkey
monkey.patch_all()
import sys
import time
import atexit
@ -37,8 +33,8 @@ from openflexure_microscope.paths import (
logs_file_path,
)
from labthings.server.quick import create_app
from labthings.server.extensions import find_extensions
from labthings import create_app
from labthings.extensions import find_extensions
from openflexure_microscope.api.microscope import default_microscope as api_microscope
@ -205,7 +201,7 @@ atexit.register(cleanup)
# Start the app
if __name__ == "__main__":
from labthings.server.wsgi import Server
from labthings import Server
logging.info("Starting OpenFlexure Microscope Server...")
server = Server(app)
server.run(host="::", port=5000, debug=False, zeroconf=True)

View file

@ -1,7 +1,6 @@
from labthings.server.find import find_component
from labthings.server.extensions import BaseExtension
from labthings.server.view import View, ActionView, PropertyView
from labthings.server import fields
from labthings import fields, find_component
from labthings.extensions import BaseExtension
from labthings.views import View, ActionView, PropertyView
from openflexure_microscope.devel import JsonResponse, request, abort
from openflexure_microscope.utilities import set_properties

View file

@ -1,7 +1,6 @@
from labthings.server.extensions import BaseExtension
from labthings.server.view import View, PropertyView
from labthings.server import fields
from labthings.server.find import find_component
from labthings.extensions import BaseExtension
from labthings.views import View, PropertyView
from labthings import fields, find_component
from openflexure_microscope.paths import settings_file_path, check_rw
from openflexure_microscope.config import OpenflexureSettingsFile

View file

@ -1,6 +1,6 @@
from labthings.server.view import View, ActionView
from labthings.server.find import find_component
from labthings.server.extensions import BaseExtension
from labthings import find_component
from labthings.views import View, ActionView
from labthings.extensions import BaseExtension
from flask import abort

View file

@ -6,14 +6,13 @@ from typing import Tuple
from functools import reduce
from openflexure_microscope.captures.capture_manager import generate_basename
from labthings.server.find import find_component, find_extension
from labthings.server.extensions import BaseExtension
from labthings.server import fields
from labthings.actions import current_action
from openflexure_microscope.devel import abort, update_task_progress
from labthings import fields, find_component, find_extension, current_action, update_action_progress
from labthings.views import View, ActionView
from labthings.extensions import BaseExtension
from openflexure_microscope.devel import abort
from labthings.server.view import View, ActionView
import time
@ -209,7 +208,7 @@ class ScanExtension(BaseExtension):
)
# Update task progress
self._images_captured_so_far += 1
update_task_progress(self.progress())
update_action_progress(self.progress())
else:
logging.debug("Entering z-stack")
self.stack(
@ -279,7 +278,7 @@ class ScanExtension(BaseExtension):
)
# Update task progress
self._images_captured_so_far += 1
update_task_progress(self.progress())
update_action_progress(self.progress())
if current_action() and current_action().stopped:
return

View file

@ -1,7 +1,6 @@
from openflexure_microscope.devel import (
JsonResponse,
request,
update_task_progress,
)
from flask import send_file, abort, url_for
@ -12,12 +11,11 @@ import zipfile
import tempfile
import logging
from labthings.server.find import find_component
from labthings.server.view import View, ActionView, PropertyView
from labthings.server.schema import Schema, pre_dump
from labthings.server import fields
from labthings.server.extensions import BaseExtension
from labthings.server.utilities import description_from_view
from labthings import fields, find_component, update_action_progress
from labthings.views import View, ActionView, PropertyView
from labthings.schema import Schema, pre_dump
from labthings.extensions import BaseExtension
from labthings.utilities import description_from_view
class ZipObjectSchema(Schema):
@ -107,7 +105,7 @@ class ZipManager:
)
zipObj.write(file_path, arcname=rel_path)
# Update task progress
update_task_progress(int((index / n_files) * 100))
update_action_progress(int((index / n_files) * 100))
session_id = str(uuid.uuid4())
session_description = ZipObjectDescription(

View file

@ -1,7 +1,6 @@
from labthings.server.extensions import BaseExtension
from labthings.server.view import ActionView
from labthings.server import fields
from labthings import fields
from labthings.extensions import BaseExtension
from labthings.views import ActionView
import logging
import time

View file

@ -4,9 +4,9 @@ Top-level representation of enabled actions
from flask import url_for
from . import camera, stage, system
from labthings.server.view import View
from labthings.server.find import current_labthing
from labthings.server.utilities import description_from_view
from labthings import current_labthing
from labthings.views import View
from labthings.utilities import description_from_view
_actions = {
"capture": {

View file

@ -1,8 +1,7 @@
from openflexure_microscope.api.utilities import get_bool, JsonResponse
from labthings.server.view import View, ActionView
from labthings.server.find import find_component
from labthings.views import View, ActionView
from labthings import find_component, fields
from labthings.server import fields
from openflexure_microscope.utilities import filter_dict
from openflexure_microscope.api.v2.views.captures import CaptureSchema

View file

@ -1,7 +1,6 @@
from openflexure_microscope.api.utilities import JsonResponse
from labthings.server.view import View, ActionView
from labthings.server.find import find_component
from labthings.server import fields
from labthings.views import View, ActionView
from labthings import find_component, fields
from openflexure_microscope.utilities import axes_to_array, filter_dict

View file

@ -1,4 +1,4 @@
from labthings.server.view import View, ActionView
from labthings.views import View, ActionView
import subprocess
import os
from sys import platform

View file

@ -1,9 +1,8 @@
from openflexure_microscope.api.utilities import JsonResponse
from labthings.core.utilities import get_by_path, set_by_path, create_from_path
from labthings.server.find import find_component
from labthings.server.view import View, PropertyView
from labthings import find_component
from labthings.views import View, PropertyView
from labthings.utilities import get_by_path, set_by_path, create_from_path
from flask import request, abort
import logging

View file

@ -1,10 +1,10 @@
from openflexure_microscope.api.utilities import gen, JsonResponse
from labthings.core.utilities import get_by_path, set_by_path, create_from_path
from labthings import find_component
from labthings.utilities import get_by_path, set_by_path, create_from_path
from labthings.views import View, PropertyView
from labthings.server.find import find_component
from labthings.server.view import View, PropertyView
from labthings.server.responses import Response
from flask import Response
class MjpegStream(PropertyView):

View file

@ -9,9 +9,7 @@ import threading
from abc import ABCMeta, abstractmethod
from labthings.core.lock import StrictLock
from labthings.core.event import ClientEvent
from labthings import StrictLock, ClientEvent
class BaseCamera(metaclass=ABCMeta):

View file

@ -4,7 +4,7 @@ import shutil
import logging
from collections import OrderedDict
from labthings.core.lock import StrictLock
from labthings import StrictLock
from openflexure_microscope.utilities import entry_by_uuid
from openflexure_microscope.paths import data_file_path

View file

@ -8,10 +8,10 @@ as well as some Flask imports to simplify API route development
from openflexure_microscope.api.utilities import JsonResponse
# Task management
from labthings.core.tasks import (
current_task,
update_task_progress,
update_task_data,
from labthings import (
current_action as current_task,
update_action_progress as update_task_progress,
update_action_data as update_task_data,
)

View file

@ -24,7 +24,7 @@ from openflexure_microscope.camera.mock import MissingCamera
from openflexure_microscope.utilities import serialise_array_b64, Timer
from openflexure_microscope.config import user_settings, user_configuration
from labthings.core.lock import CompositeLock
from labthings import CompositeLock
class Microscope:
@ -49,7 +49,7 @@ class Microscope:
self.extension_settings = {}
# Initialise with an empty composite lock
#: :py:class:`labthings.lock.CompositeLock`: Composite lock for locking both camera and stage
#: :py:class:`labthings.CompositeLock`: Composite lock for locking both camera and stage
self.lock = CompositeLock([])
self.camera = None #: Currently connected camera object

View file

@ -1,12 +1,12 @@
import numpy as np
from abc import ABCMeta, abstractmethod
from labthings.core.lock import StrictLock
from labthings import StrictLock
class BaseStage(metaclass=ABCMeta):
"""
Attributes:
lock (:py:class:`labthings.lock.StrictLock`): Strict lock controlling thread
lock (:py:class:`labthings.StrictLock`): Strict lock controlling thread
access to camera hardware
"""