Merge branch 'sangaboard-v4' into 'master'

Support latest sangaboard firmware

See merge request openflexure/openflexure-microscope-server!154

.gitlab-ci.yml

@@ -6,8 +6,12 @@ stages:
   - deploy
 
 # Re-usable block to install (and cache) Poetry and openflexure-microscope-server
-.python-install-template: &python-install
+.python:
+  image: balenalib/raspberry-pi:buster
+  retry: 1
   before_script:
+    - apt-get update
+    - apt-get install apt-utils curl wget python3 python3-pip python3-venv raspi-gpio libgfortran3 libatlas3-base libffi-dev vim git-lfs libtiff5 libpango-1.0-0 libavcodec58 libgdk-pixbuf2.0-0 libjasper1 libswscale5 libpangocairo-1.0-0 libilmbase23 libatk1.0-0 libgtk-3-0 libwebp6 libcairo2 libavutil56 libcairo-gobject2 libopenexr23 libavformat58 libxslt1.1
     - python3 -m venv .venv        # NB this does not overwrite an existing cached venv
     - source .venv/bin/activate
     - pip install pipenv           # Should be cached after first run
@@ -17,9 +21,14 @@ stages:
     paths:
       - .cache/pip
       - .venv
+  only:
+    - branches
+    - merge_requests
+    - tags
+    - web
 
-# Re-usable block to install (and cache) openflexure-microscope-server static app
+.javascript:
-.node-install-template: &node-install
+  image: node:15
   before_script:
     - cd webapp
     - npm install
@@ -27,93 +36,54 @@ stages:
     key: "${CI_COMMIT_REF_SLUG}"
     paths:
       - webapp/node_modules
+  only:
+    - branches
+    - merge_requests
+    - tags
+    - web
 
 # Python static analysis with PyLint
 pylint:
   stage: analysis
-  image: python:3.7
+  extends: .python
-  retry: 1
-
-  <<: *python-install
-
   script:
     - poe pylint
 
-  only:
-    - branches
-    - merge_requests
-    - tags
-    - web
-
 # Python type checking with Mypy
 mypy:
   stage: analysis
-  image: python:3.7
+  extends: .python
-  retry: 1
-
-  <<: *python-install
-
   script:
     - poe mypy
-
   artifacts:
     reports:
-      cobertura: openflexure_microscope/cobertura.xml
+      coverage_report: 
-
+        coverage_format: cobertura
-  only:
+        path: openflexure_microscope/cobertura.xml
-    - branches
-    - merge_requests
-    - tags
-    - web
 
 # Python style analysis with Black
 black:
   stage: analysis
-  image: python:3.7
-  retry: 1
   allow_failure: true
-
+  extends: .python
-  <<: *python-install
-
   script:
-    # Run static build script
     - poe black_check
 
-  only:
-    - branches
-    - merge_requests
-    - tags
-    - web
-
 # Python unit tests with PyTest
 pytest:
   stage: testing
-  image: python:3.7
+  extends: .python
-  retry: 1
-
-  <<: *python-install
-
   script:
     - poe test
-
-  only:
-    - branches
-    - merge_requests
-    - tags
-    - web
-
   artifacts:
     when: always
     reports:
       junit: pytest_report.xml
 
+# Generate and validate OpenAPI description
 openapi:
   stage: testing
-  image: python:3.7
+  extends: .python
-  retry: 1
-
-  <<: *python-install
-
   script:
     - mkdir -p docs/build/
     - ofm-generate-openapi -o docs/build/swagger.yaml --validate
@@ -123,53 +93,28 @@ openapi:
     expire_in: 1 week
     paths:
       - "./docs/build/swagger.yaml"
-  
-  only:
-    - branches
-    - merge_requests
-    - tags
-    - web
 
 # JavaScript linting with ESLint (via Vue CLI)
 eslint:
   stage: analysis
-  image: node:15
+  extends: .javascript
-
-  <<: *node-install
-
   script:
     # Build JS application for production
     - npm run lint
 
-  only:
-    - branches
-    - merge_requests
-    - tags
-    - web
-
 # Build JS app
 build:
   stage: build
-  image: node:15
+  extends: .javascript
-
-  <<: *node-install
-
   script:
     # Build JS application for production
     - npm run build
-
   artifacts:
     name: "dist"
     expire_in: 1 week
     paths:
       - "openflexure_microscope/api/static/dist/"
 
-  only:
-    - master
-    - merge_requests
-    - tags
-    - web
-
 # Package application into distribution tarball
 package:
   stage: package

CHANGELOG.md

@@ -1,3 +1,9 @@
+# [v2.11.0](https://gitlab.com/openflexure/openflexure-microscope-server/compare/v2.10.1...v2.11.0) (2022-08-08)
+## New features
+* Background detection can now be used in scans, if you have the background-detect extension. ([!153](https://gitlab.com/openflexure/openflexure-microscope-server/-/merge_requests/153))
+* Support for Sangaboard firmware v1, via updated `sangaboard` dependency ([!154](https://gitlab.com/openflexure/openflexure-microscope-server/-/merge_requests/154))
+* Fixed links in the OpenAPI documentation describing actions etc. ([!155](https://gitlab.com/openflexure/openflexure-microscope-server/-/merge_requests/154))
+
 # [v2.10.1](https://gitlab.com/openflexure/openflexure-microscope-server/compare/v2.10.0...v2.10.1) (2022-01-17)
 ## Bug fixes
 * Fixed a typo in the serialisation of numpy arrays (#243) ([!148](https://gitlab.com/openflexure/openflexure-microscope-server/-/merge_requests/148))

Pipfile

@@ -1,8 +1,3 @@
-[[source]]
-url = "https://pypi.python.org/simple"
-verify_ssl = true
-name = "pypi"
-
 [[source]]
 url = "https://www.piwheels.org/simple"
 verify_ssl = true

README.md

@@ -69,7 +69,7 @@ To set up a development version of the software (most likely using emulated came
 * Create a virtual environment and activate it:
   * `python -m venv .venv`
   * `source .venv/bin/activate` (on Linux) or `.venv/Scripts/activate` (on Windows)
-  * `pip install pipenv`
+  * `pip install --upgrade pip wheel pipenv`
   * `pipenv install --dev` (This will install development dependencies.  If you don't need these, `pipenv install` will get you just the dependencies needed to run the server, which takes about half the time.)
 * Finally, run the server:
   * You can use `ofm serve` or `ofm restart` on the Raspberry Pi to manage the server.
@@ -146,6 +146,10 @@ As of `2.10.0b0` we have switched to using `pipenv` for managing dependencies, a
 * `pyproject.toml` is retained, but *no longer includes package metadata or dependencies* and does not have a `[tool.poetry]` table.  It does define the build system as per PEP517, which is `setuptools`, and it also contains settings for `black`, `poe` and `isort`.
 * `Pipfile` is very minimal, and only declares dependencies on the current module, i.e. the dependencies declared in `setup.py`.  It also specifies the Python version.  This allows us to single-source dependency information from `setup.py` but use the dependency resolution/locking functionality of `pipenv`
 * `Pipfile.lock` locks the dependency versions in the same way as `poetry.lock` used to, i.e. it specifies exact versions of everything, derived from the looser specifications in `setup.py`.
+* Dependency resolution is a pain, in part because compiling packages like scipy and numpy from source on a Pi is slow and unreliable. To avoid this, we usually lock the dependencies on a Raspberry Pi, using:
+  `PIP_ONLY_BINARY="numpy, scipy, matplotlib" pipenv lock --dev`
+  This forces the use of wheels only, and thus restricts us to things that built OK on Piwheels.  The resulting Pipfile.lock should then work nicely on other Raspberry Pis.
+* **The Pipfile we use is now Raspberry Pi-specific** so if you attempt to use it on other platforms it will probably fail. This is because we've removed `pypi` to avoid hash conflicts. The CI now runs on a Raspberry Pi image, so that the tests use a similar environment to our deployment.  Currently, changing the `[source]` entry back to PyPi will allow you to re-lock the dependencies on other platforms. In the future we may need to maintain two Pipfiles and two Pipfile.lock files.
 
 ## Creating releases
 
@@ -166,3 +170,6 @@ As of `2.10.0b0` we have switched to using `pipenv` for managing dependencies, a
 
 The Microscope module, and Flask app, both support plugins for extending lower-level functionality not well suited to web API calls. The current documentation can be found [here](https://openflexure-microscope-software.readthedocs.io/en/latest/plugins.html).
 If you want to add functions to the microscope software, this is probably the best mechanism to use if it works for you.
+
+
+```

docs/source/extensions/actions.rst

@@ -18,7 +18,7 @@ Like properties, we use a special view class to identify a view as an action: ``
         Take an image capture and return it without saving
         """
         # Expect a "use_video_port" boolean, which defaults to True if none is given
-        args = {"use_video_port": fields.Boolean(missing=True)}
+        args = {"use_video_port": fields.Boolean(load_default=True)}
 
         # Our success response (200) returns an image (image/jpeg mimetype)
         responses = {

docs/source/extensions/example_extension/02_adding_views.py

@@ -46,7 +46,7 @@ class ExampleIdentifyView(View):
 class ExampleRenameView(View):
     # Expect a request parameter called "name", which is a string.
     # Passed to the argument "args".
-    args = fields.String(required=True, example="My Example Microscope")
+    args = fields.String(required=True, metadata={"example": "My Example Microscope"})
 
     def post(self, args):
         # Look for our new name in the request body

docs/source/extensions/example_extension/03_marshaling_data.py

@@ -49,7 +49,11 @@ class ExampleRenameView(View):
     # Format our returned object using MicroscopeIdentifySchema
     schema = MicroscopeIdentifySchema()
     # Expect a request parameter called "name", which is a string. Pass to argument "args".
-    args = {"name": fields.String(required=True, example="My Example Microscope")}
+    args = {
+        "name": fields.String(
+            required=True, metadata={"example": "My Example Microscope"}
+        )
+    }
 
     def post(self, args):
         # Look for our "name" parameter in the request arguments

docs/source/extensions/example_extension/04_properties.py

@@ -53,7 +53,11 @@ class ExampleIdentifyView(PropertyView):
 # We can use a single schema as the input and output will be formatted identically
 # Eg. We always expect a "name" string argument, and always return a "name" string attribute
 class ExampleRenameView(PropertyView):
-    schema = {"name": fields.String(required=True, example="My Example Microscope")}
+    schema = {
+        "name": fields.String(
+            required=True, metadata={"example": "My Example Microscope"}
+        )
+    }
 
     def get(self):
         """

docs/source/extensions/example_extension/05_actions.py

@@ -56,7 +56,11 @@ class ExampleIdentifyView(PropertyView):
 # We can use a single schema as the input and output will be formatted identically
 # Eg. We always expect a "name" string argument, and always return a "name" string attribute
 class ExampleRenameView(PropertyView):
-    schema = {"name": fields.String(required=True, example="My Example Microscope")}
+    schema = {
+        "name": fields.String(
+            required=True, metadata={"example": "My Example Microscope"}
+        )
+    }
 
     def get(self):
         """
@@ -91,7 +95,7 @@ class QuickCaptureAPI(ActionView):
     """
 
     # Expect a "use_video_port" boolean, which defaults to True if none is given
-    args = {"use_video_port": fields.Boolean(missing=True)}
+    args = {"use_video_port": fields.Boolean(load_default=True)}
 
     # Our success response (200) returns an image (image/jpeg mimetype)
     responses = {200: {"content": {"image/jpeg": {}}}}

docs/source/extensions/example_extension/06_tasks_locks.py

@@ -66,10 +66,11 @@ class TimelapseAPIView(ActionView):
 
     args = {
         "n_images": fields.Integer(
-            required=True, example=5, description="Number of images"
+            required=True, metadata={"example": 5, "description": "Number of images"}
         ),
         "t_between": fields.Number(
-            missing=1, example=1, description="Time (seconds) between images"
+            load_default=1,
+            metadata={"example": 1, "description": "Time (seconds) between images"},
         ),
     }
 

docs/source/extensions/example_extension/07_ev_gui.py

@@ -99,10 +99,11 @@ class TimelapseAPIView(ActionView):
 
     args = {
         "n_images": fields.Integer(
-            required=True, example=5, description="Number of images"
+            required=True, metadata={"example": 5, "description": "Number of images"}
         ),
         "t_between": fields.Number(
-            missing=1, example=1, description="Time (seconds) between images"
+            load_default=1,
+            metadata={"example": 1, "description": "Time (seconds) between images"},
         ),
     }
 

docs/source/extensions/marshaling.rst

@@ -23,7 +23,7 @@ A **field** describes the data type of a single parameter, as well as any other
 
 .. code-block:: python
 
-    fields.String(required=False, missing="Default value", example="Example value")
+    fields.String(required=False, load_default="Default value", metadata={"example: "Example value"})
 
 A **schema** is a collection of keys and fields describing how an object should be serialized/deserialized. Schemas can be created in several ways, either by creating a ``Schema`` class, or by passing a dictionary of key-field pairs. Both methods will be discussed in the following examples.
 
@@ -43,7 +43,7 @@ For example, if you are creating an API route, in which you expect parameters ``
     class UserSchema(Schema):
         name = fields.String(required=True)
         age = fields.Integer(required=True)
-        job = fields.String(required=False, missing="Unknown")
+        job = fields.String(required=False, load_default="Unknown")
 
 To inform your POST method to expect these arguments, use the ``args`` class attribute:
 
@@ -63,7 +63,7 @@ Alternatively, if your schema is only used in a single location, it may be simpl
         args = {
             "name": fields.String(required=True),
             "age": fields.Integer(required=True),
-            "job": fields.String(required=False, missing="Unknown")
+            "job": fields.String(required=False, load_default="Unknown")
         }
 
         def post(self, args):
@@ -88,7 +88,7 @@ This JSON data is the parsed, converted into a Python dictionary, and passed as
         args = {
             "name": fields.String(required=True),
             "age": fields.Integer(required=True),
-            "job": fields.String(required=False, missing="Unknown")
+            "job": fields.String(required=False, load_default="Unknown")
         }
 
         def post(self, args):
@@ -146,7 +146,7 @@ Our ``rename`` view class may now look like:
         # Format our returned object using MicroscopeIdentifySchema
         schema = MicroscopeIdentifySchema()
         # Expect a request parameter called "name", which is a string. Pass to argument "args".
-        args = {"name": fields.String(required=True, example="My Example Microscope")}
+        args = {"name": fields.String(required=True, metadata={"example": "My Example Microscope"})}
 
         def post(self, args):
             # Look for our "name" parameter in the request arguments

docs/source/extensions/properties.rst

@@ -78,7 +78,7 @@ We will implement the ``schema`` attribute in our ``ExampleRenameView`` view fro
     # We can use a single schema as the input and output will be formatted identically
     # Eg. We always expect a "name" string argument, and always return a "name" string attribute
     class ExampleRenameView(PropertyView):
-        schema = {"name": fields.String(required=True, example="My Example Microscope")}
+        schema = {"name": fields.String(required=True, metadata={"example": "My Example Microscope"})}
     
         def get(self):
             """

openflexure_microscope/api/app.py

@@ -261,10 +261,10 @@ def generate_openapi():
     if fname.endswith(".json"):
         import json
 
-        with open(fname, "w") as fd:
+        with open(fname, "w", encoding="utf-8") as fd:
             json.dump(labthing.spec.to_dict(), fd)
     else:
-        with open(fname, "w") as fd:
+        with open(fname, "w", encoding="utf-8") as fd:
             fd.write(labthing.spec.to_yaml())
 
 

openflexure_microscope/api/default_extensions/autofocus.py

@@ -245,7 +245,9 @@ def extension_action(args=None):
                 # Run the action
                 return func(self.extension, **arguments)
 
-            def get(self, *args, **kwargs):  # pylint: disable=useless-super-delegation
+            def get(
+                self, *args, **kwargs
+            ):  # pylint: disable=useless-super-delegation,arguments-differ
                 # Explicitly wrap the `get` method to allow us to add a docstring
                 return super().get(*args, **kwargs)
 
@@ -347,8 +349,10 @@ class AutofocusExtension(BaseExtension):
         args={
             "dz": fields.List(
                 fields.Int(),
-                description="An ascending list of relative z positions",
+                metadata={
-                example=[int(x) for x in np.linspace(-300, 300, 7)],
+                    "description": "An ascending list of relative z positions",
+                    "example": [int(x) for x in np.linspace(-300, 300, 7)],
+                },
             )
         }
     )
@@ -408,7 +412,9 @@ class AutofocusExtension(BaseExtension):
 
     @extension_action(
         args={
-            "dz": fields.Int(required=True, description="The relative Z move to make")
+            "dz": fields.Int(
+                required=True, metadata={"description": "The relative Z move to make"}
+            )
         }
     )
     def move_and_measure(
@@ -430,9 +436,11 @@ class AutofocusExtension(BaseExtension):
     @extension_action(
         args={
             "dz": fields.Int(
-                missing=2000,
+                load_default=2000,
-                example=2000,
+                metadata={
-                description="Total Z range to search over (in stage steps)",
+                    "description": "Total Z range to search over (in stage steps)",
+                    "example": 2000,
+                },
             )
         }
     )
@@ -526,14 +534,18 @@ class AutofocusExtension(BaseExtension):
     @extension_action(
         args={
             "dz": fields.Int(
-                missing=500,
+                load_default=500,
-                example=500,
+                metadata={
-                description="Total Z range to move down, then up (in stage steps)",
+                    "description": "Total Z range to move down, then up (in stage steps)",
+                    "example": 500,
+                },
             ),
             "delay": fields.Int(
-                missing=5,
+                load_default=5,
-                example=5,
+                metadata={
-                description="How long to measure sharpness for after the move",
+                    "description": "How long to measure sharpness for after the move, in seconds",
+                    "example": 5,
+                },
             ),
         }
     )
@@ -554,23 +566,31 @@ class AutofocusExtension(BaseExtension):
     @extension_action(
         args={
             "dz": fields.Int(
-                missing=2000,
+                load_default=2000,
-                example=2000,
+                metadata={
-                description="Total Z range to search over (in stage steps)",
+                    "description": "Total Z range to search over (in stage steps)",
+                    "example": 2000,
+                },
             ),
             "target_z": fields.Int(
-                missing=0,
+                load_default=0,
-                example=-100,
+                metadata={
-                description="Target finishing position, relative to the focus.",
+                    "description": "Target finishing position, relative to the focus.",
+                    "example": -100,
+                },
             ),
             "initial_move_up": fields.Bool(
-                missing=True,
+                load_default=True,
-                description="Set to Flase to disable the initial move upwards",
+                metadata={
+                    "description": "Set to False to disable the initial move upwards"
+                },
             ),
             "backlash": fields.Int(
-                missing=25,
+                load_default=25,
-                minimum=0,
+                metadata={
-                description="Distance to undershoot, before correction move.",
+                    "description": "Distance to undershoot, before correction move.",
+                    "minimum": 0,
+                },
             ),
         }
     )

openflexure_microscope/api/default_extensions/autostorage.py

@@ -242,7 +242,7 @@ class GetLocationsView(PropertyView):
 
 
 class PreferredLocationView(PropertyView):
-    schema = fields.String(required=True, example="Default")
+    schema = fields.String(required=True, metadata={"example": "Default"})
 
     def get(self):
         self.extension.check_location()

openflexure_microscope/api/default_extensions/camera_stage_mapping.py

@@ -200,7 +200,7 @@ class CSMExtension(BaseExtension):
             "linear_calibration_y": cal_y,
         }
 
-        with open(CSM_DATAFILE_PATH, "w") as f:
+        with open(CSM_DATAFILE_PATH, "w", encoding="utf-8") as f:
             json.dump(data, f, cls=JSONEncoder)
 
         return data
@@ -279,7 +279,11 @@ class CSMExtension(BaseExtension):
 
 
 class Calibrate1DView(ActionView):
-    args = {"direction": fields.List(fields.Float(), required=True, example=[1, 0, 0])}
+    args = {
+        "direction": fields.List(
+            fields.Float(), required=True, metadata={"example": [1, 0, 0]}
+        )
+    }
 
     def post(self, args):
         """Calibrate one axis of the microscope stage against the camera."""
@@ -298,10 +302,18 @@ class CalibrateXYView(ActionView):
 class MoveInImageCoordinatesView(ActionView):
     args = {
         "x": fields.Float(
-            description="The number of pixels to move in X", required=True, example=100
+            metadata={
+                "description": "The number of pixels to move in X",
+                "example": 100,
+            },
+            required=True,
         ),
         "y": fields.Float(
-            description="The number of pixels to move in Y", required=True, example=100
+            metadata={
+                "description": "The number of pixels to move in Y",
+                "example": 100,
+            },
+            required=True,
         ),
     }
 
@@ -316,10 +328,18 @@ class MoveInImageCoordinatesView(ActionView):
 class ClosedLoopMoveInImageCoordinatesView(ActionView):
     args = {
         "x": fields.Float(
-            description="The number of pixels to move in X", required=True, example=100
+            metadata={
+                "description": "The number of pixels to move in X",
+                "example": 100,
+            },
+            required=True,
         ),
         "y": fields.Float(
-            description="The number of pixels to move in Y", required=True, example=100
+            metadata={
+                "description": "The number of pixels to move in Y",
+                "example": 100,
+            },
+            required=True,
         ),
     }
 
@@ -336,15 +356,25 @@ class ClosedLoopMoveInImageCoordinatesView(ActionView):
 class TestClosedLoopSpiralScanView(ActionView):
     args = {
         "x_step": fields.Float(
-            description="The number of pixels to move in X", required=True, example=100
+            metadata={
+                "description": "The number of pixels to move in X",
+                "example": 100,
+            },
+            required=True,
         ),
         "y_step": fields.Float(
-            description="The number of pixels to move in Y", required=True, example=100
+            metadata={
+                "description": "The number of pixels to move in Y",
+                "example": 100,
+            },
+            required=True,
         ),
         "N": fields.Int(
-            description="The number of rings in the spiral scan",
+            metadata={
+                "description": "The number of rings in the spiral scan",
+                "example": 100,
+            },
             required=True,
-            example=3,
         ),
     }
 

openflexure_microscope/api/default_extensions/picamera_autocalibrate/extension.py

@@ -169,39 +169,50 @@ class DeleteLSTView(ActionView):
             microscope.save_settings()
 
 
+percentile_field = fields.Float(
+    load_default=99.9,
+    metadata={
+        "example": 99.9,
+        "description": (
+            "A float between 0 and 100 setting the centile to use "
+            "to measure the white point of the image.  A value "
+            "of 99.9 allows 0.1% of the pixels to be erroneously "
+            "bright - this helps stability in low light."
+        ),
+    },
+)
+
+
 class AutoExposureFromRawView(ActionView):
     args = {
         "target_white_level": fields.Int(
-            missing=700,
+            load_default=700,
-            example=700,
+            metadata={
-            description=(
+                "example": 700,
-                "The pixel value (10-bit format) that we aim for when adjusting shutter/gain."
+                "description": (
-            ),
+                    "The pixel value (10-bit format) that we aim for when adjusting shutter/gain."
+                ),
+            },
         ),
         "max_iterations": fields.Int(
-            missing=20,
+            load_default=20,
-            description=(
+            metadata={
-                "The number of adjustments to the camera's settings to make before giving up."
+                "description": (
-            ),
+                    "The number of adjustments to the camera's settings to make before giving up."
+                )
+            },
         ),
         "tolerance": fields.Float(
-            missing=0.05,
+            load_default=0.05,
-            example=0.05,
+            metadata={
-            description=(
+                "example": 0.05,
-                "We stop adjusting when we get within this fraction of the target "
+                "description": (
-                "value.  It is a number between 0 and 1, usually 0.01--0.1."
+                    "We stop adjusting when we get within this fraction of the target "
-            ),
+                    "value.  It is a number between 0 and 1, usually 0.01--0.1."
-        ),
+                ),
-        "percentile": fields.Float(
+            },
-            missing=99.9,
-            example=99.9,
-            description=(
-                "A float between 0 and 100 setting the centile to use "
-                "to measure the white point of the image.  A value "
-                "of 99.9 allows 0.1% of the pixels to be erroneously "
-                "bright - this helps stability in low light."
-            ),
         ),
+        "percentile": percentile_field,
     }
 
     def post(self, args):
@@ -210,18 +221,7 @@ class AutoExposureFromRawView(ActionView):
 
 
 class AutoWhiteBalanceFromRawView(ActionView):
-    args = {
+    args = {"percentile": percentile_field}
-        "percentile": fields.Float(
-            missing=99.9,
-            example=99.9,
-            description=(
-                "A float between 0 and 100 setting the centile to use "
-                "to measure the white point of the image.  A value "
-                "of 99.9 allows 0.1% of the pixels to be erroneously "
-                "bright - this helps stability in low light."
-            ),
-        )
-    }
 
     def post(self, args):
         with find_picamera() as (picamera, _, _):
@@ -231,8 +231,10 @@ class AutoWhiteBalanceFromRawView(ActionView):
 class GetRawChannelPercentilesView(ActionView):
     args = {
         "percentile": fields.Float(
-            example=99.9,
+            metadata={
-            description="A float between 0 and 100 setting the centile to calculate",
+                "description": "A float between 0 and 100 setting the centile to calculate",
+                "example": 99.9,
+            }
         )
     }
     schema = fields.List(fields.Integer)

openflexure_microscope/api/default_extensions/scan.py

@@ -578,19 +578,23 @@ class ScanExtension(BaseExtension):
 
 
 class TileScanArgs(FullCaptureArgs):
-    namemode = fields.String(missing="coordinates", example="coordinates")
+    namemode = fields.String(
+        load_default="coordinates", metadata={"example": "coordinates"}
+    )
     grid = fields.List(
         fields.Integer(validate=marshmallow.validate.Range(min=1)),
-        missing=[3, 3, 3],
+        load_default=[3, 3, 3],
-        example=[3, 3, 3],
+        metadata={"example": [3, 3, 3]},
     )
-    style = fields.String(missing="raster")
+    style = fields.String(load_default="raster")
-    autofocus_dz = fields.Integer(missing=50)
+    autofocus_dz = fields.Integer(load_default=50)
-    fast_autofocus = fields.Boolean(missing=False)
+    fast_autofocus = fields.Boolean(load_default=False)
     stride_size = fields.List(
-        fields.Integer, missing=[2000, 1500, 100], example=[2000, 1500, 100]
+        fields.Integer,
+        load_default=[2000, 1500, 100],
+        metadata={"example": [2000, 1500, 100]},
     )
-    detect_empty_fields_and_skip_autofocus = fields.Boolean(missing=False)
+    detect_empty_fields_and_skip_autofocus = fields.Boolean(load_default=False)
 
 
 class TileScanAPI(ActionView):

openflexure_microscope/api/dev_extensions/tools.py

@@ -24,7 +24,11 @@ class RaiseException(ActionView):
 
 class SleepFor(ActionView):
     schema = {"TimeAsleep": fields.Float()}
-    args = {"time": fields.Float(description="Time to sleep, in seconds", example=0.5)}
+    args = {
+        "time": fields.Float(
+            metadata={"description": "Time to sleep, in seconds", "example": 0.5}
+        )
+    }
 
     def post(self, args):
         sleep_time: int = args.get("time", 0)

openflexure_microscope/api/openapi.py

@@ -8,8 +8,8 @@ API_TAGS = [
             "embedded in the JSON action description."
         ),
         "externalDocs": {
-            "url": "https://iot.mozilla.org/wot/#action-resource",
+            "url": "https://www.w3.org/TR/wot-thing-description/#actionaffordance",
-            "description": "Mozilla's description of Web of Things 'Action' resources.",
+            "description": "W3C's description of Web of Things 'Action' resources.",
         },
     },
     {
@@ -19,8 +19,8 @@ API_TAGS = [
             "state of the microscope."
         ),
         "externalDocs": {
-            "url": "https://iot.mozilla.org/wot/#property-resource",
+            "url": "https://www.w3.org/TR/wot-thing-description/#propertyaffordance",
-            "description": "Mozilla's description of Web of Things 'Property' resources.",
+            "description": "W3C's description of Web of Things 'Property' resources.",
         },
     },
     {"name": "captures", "description": ""},

openflexure_microscope/api/utilities/__init__.py

@@ -81,7 +81,7 @@ def init_default_extensions(extension_dir: str):
         create_file(default_ext_path)
 
         logging.info("Populating %s...", (default_ext_path))
-        with open(default_ext_path, "w") as outfile:
+        with open(default_ext_path, "w", encoding="utf-8") as outfile:
             outfile.write(_DEFAULT_EXTENSION_INIT)
 
 

openflexure_microscope/api/v2/views/actions/camera.py

@@ -10,23 +10,28 @@ from openflexure_microscope.api.v2.views.captures import CaptureSchema
 
 
 class CaptureResizeSchema(Schema):
-    width = fields.Integer(example=640, required=True)
+    width = fields.Integer(metadata={"example": 640}, required=True)
-    height = fields.Integer(example=480, required=True)
+    height = fields.Integer(metadata={"example": 480}, required=True)
 
 
 class BasicCaptureArgs(Schema):
-    use_video_port = fields.Boolean(missing=False)
+    use_video_port = fields.Boolean(load_default=False)
     bayer = fields.Boolean(
-        missing=False, description="Include raw bayer data in capture"
+        load_default=False,
+        metadata={"description": "Include raw bayer data in capture"},
     )
     resize = fields.Nested(CaptureResizeSchema(), required=False)
 
 
 class FullCaptureArgs(BasicCaptureArgs):
-    filename = fields.String(example="MyFileName")
+    filename = fields.String(metadata={"example": "MyFileName"})
-    temporary = fields.Boolean(missing=False, description="Delete capture on shutdown")
+    temporary = fields.Boolean(
-    annotations = fields.Dict(missing={}, example={"Client": "SwaggerUI"})
+        load_default=False, metadata={"description": "Delete capture on shutdown"}
-    tags = fields.List(fields.String, missing=[], example=["docs"])
+    )
+    annotations = fields.Dict(
+        load_default={}, metadata={"example": {"Client": "SwaggerUI"}}
+    )
+    tags = fields.List(fields.String, load_default=[], metadata={"example": ["docs"]})
 
 
 class CaptureAPI(ActionView):
@@ -113,7 +118,11 @@ class GPUPreviewStartAPI(ActionView):
     in the format ``[x, y, width, height]``.
     """
 
-    args = {"window": fields.List(fields.Integer, missing=[], example=[0, 0, 640, 480])}
+    args = {
+        "window": fields.List(
+            fields.Integer, load_default=[], metadata={"example": [0, 0, 640, 480]}
+        )
+    }
 
     def post(self, args):
         """

openflexure_microscope/api/v2/views/actions/stage.py

@@ -7,11 +7,12 @@ from labthings.views import ActionView
 class MoveStageAPI(ActionView):
     args = {
         "absolute": fields.Boolean(
-            missing=False, example=False, description="Move to an absolute position"
+            load_default=False,
+            metadata={"description": "Move to an absolute position", "example": False},
         ),
-        "x": fields.Int(missing=None, example=100, allow_none=False),
+        "x": fields.Int(load_default=None, metadata={"example": 100}, allow_none=True),
-        "y": fields.Int(missing=None, example=100, allow_none=False),
+        "y": fields.Int(load_default=None, metadata={"example": 100}, allow_none=True),
-        "z": fields.Int(missing=None, example=20, allow_none=False),
+        "z": fields.Int(load_default=None, metadata={"example": 20}, allow_none=True),
     }
 
     def post(self, args):

openflexure_microscope/api/v2/views/captures.py

@@ -24,7 +24,7 @@ class InstrumentSchema(Schema):
 
 class ImageSchema(Schema):
     id = fields.UUID()
-    time = fields.String(format="date")
+    time = fields.String(metadata={"format": "date"})
     format = fields.String()
     name = fields.String()
     tags = fields.List(fields.String())
@@ -59,7 +59,7 @@ class CaptureSchema(ImageSchema):
     # without the server having to do a tonne of file IO
     dataset = fields.Nested(BasicDatasetSchema())
     file = fields.String(
-        data_key="path", description="Path of file on microscope device"
+        data_key="path", metadata={"description": "Path of file on microscope device"}
     )
     # No need to make a schema for links as we only ever
     # create the dictionary right here in `generate_links`

openflexure_microscope/api/v2/views/stage.py

@@ -7,10 +7,12 @@ class StageTypeProperty(PropertyView):
     """The type of the stage"""
 
     schema = fields.String(
-        missing=None,
+        load_default="SangaStage",
-        example="SangaStage",
         validate=validate.OneOf(["SangaStage", "SangaDeltaStage"]),
-        description="The translation stage geometry",
+        metadata={
+            "description": "The translation stage geometry",
+            "example": "SangaStage",
+        },
         allow_none=False,
     )
 

openflexure_microscope/camera/base.py

@@ -54,7 +54,7 @@ class FrameStream(io.BytesIO):
         t: Optional[Type[BaseException]],
         value: Optional[BaseException],
         traceback: Optional[TracebackType],
-    ) -> Optional[bool]:
+    ) -> None:
         self.stop_tracking()
         return super().__exit__(t, value, traceback)
 

openflexure_microscope/camera/pi.py

@@ -30,7 +30,7 @@ import logging
 import time
 
 # Type hinting
-from typing import BinaryIO, Tuple, Union
+from typing import BinaryIO, Optional, Tuple, Union
 
 import numpy as np
 
@@ -307,7 +307,9 @@ class PiCameraStreamer(BaseCamera):
             self.picamera.zoom = new_fov
 
     def start_preview(
-        self, fullscreen: bool = True, window: Tuple[int, int, int, int] = None
+        self,
+        fullscreen: bool = True,
+        window: Optional[Tuple[int, int, int, int]] = None,
     ):
         """Start the on board GPU camera preview."""
         with self.lock(timeout=1):
@@ -462,9 +464,9 @@ class PiCameraStreamer(BaseCamera):
         output: Union[str, BinaryIO],
         fmt: str = "jpeg",
         use_video_port: bool = False,
-        resize: Tuple[int, int] = None,
+        resize: Optional[Tuple[int, int]] = None,
         bayer: bool = True,
-        thumbnail: Tuple[int, int, int] = None,
+        thumbnail: Optional[Tuple[int, int, int]] = None,
     ):
         """
         Capture a still image to a StreamObject.

openflexure_microscope/captures/capture.py

@@ -120,7 +120,8 @@ class CaptureObject(object):
         logging.debug("Finished writing to disk %s", self.file)
 
     def open(self, mode):
-        return open(self.file, mode)
+        # We don't specify an encoding because this will be a binary file.
+        return open(self.file, mode)  # pylint: disable=unspecified-encoding
 
     def split_file_path(self, filepath: str):
         """

openflexure_microscope/config.py

@@ -3,6 +3,7 @@ import json
 import logging
 import os
 import shutil
+from typing import Optional
 
 from .json import JSONEncoder
 from .paths import CONFIGURATION_FILE_PATH, SETTINGS_FILE_PATH
@@ -17,7 +18,7 @@ class OpenflexureSettingsFile:
         expand (bool): Expand paths to valid auxillary config files.
     """
 
-    def __init__(self, path: str, defaults: dict = None):
+    def __init__(self, path: str, defaults: Optional[dict] = None):
         defaults = defaults or {}
 
         # Set arguments
@@ -88,7 +89,7 @@ def load_json_file(config_path) -> dict:
 
     logging.info("Loading %s...", config_path)
 
-    with open(config_path) as config_file:
+    with open(config_path, encoding="utf-8") as config_file:
         try:
             config_data = json.load(config_file)
         except json.decoder.JSONDecodeError as e:
@@ -112,7 +113,7 @@ def save_json_file(config_path: str, config_dict: dict):
     logging.info("Saving %s...", config_path)
     logging.debug(config_dict)
 
-    with open(config_path, "w") as outfile:
+    with open(config_path, "w", encoding="utf-8") as outfile:
         json.dump(config_dict, outfile, cls=JSONEncoder, indent=2, sort_keys=True)
 
 
@@ -150,7 +151,7 @@ def initialise_file(config_path, populate: str = "{}\n"):
         create_file(config_path)
 
         logging.info("Populating %s...", (config_path))
-        with open(config_path, "w") as outfile:
+        with open(config_path, "w", encoding="utf-8") as outfile:
             outfile.write(populate)
 
 

openflexure_microscope/microscope.py

@@ -370,7 +370,7 @@ class Microscope:
         folder: str = "",
         temporary: bool = False,
         use_video_port: bool = False,
-        resize: Tuple[int, int] = None,
+        resize: Optional[Tuple[int, int]] = None,
         bayer: bool = True,
         fmt: str = "jpeg",
         annotations: Optional[Dict[str, str]] = None,

openflexure_microscope/rescue/check_picamera.py

@@ -26,11 +26,11 @@ def main():
 
     try:
         import picamerax
-    except Exception as e:  # pylint: disable=W0703
+    except Exception as _e:  # pylint: disable=W0703
         error_sources.append(PICAMERA_IMPORT_ERROR)
     else:
         try:
-            _ = picamerax.PiCamera()
+            _cam = picamerax.PiCamera()
         except picamerax.PiCameraError as e:
             msg = e.args[0]
             if msg in PICAMERA_ERROR_MAP:

openflexure_microscope/rescue/check_sangaboard.py

@@ -20,7 +20,7 @@ def main():
             # Try connecting on the specified port
             try:
                 stage = Sangaboard(stage_port)
-            except FileNotFoundError as e:
+            except FileNotFoundError as _:
                 if stage_port:
                     error_sources.append(
                         ErrorSource(

scripts/zenodo/upload_to_zenodo.py

@@ -31,7 +31,7 @@ def script_directory(path):
 
 
 def get_meta():
-    with open(script_directory("metadata.yaml")) as f:
+    with open(script_directory("metadata.yaml"), encoding="utf-8") as f:
         metadata = f.read()
 
     return yaml.safe_load(metadata)
@@ -51,7 +51,7 @@ def main():
         zenodo.upload_file(deposit["id"], path)
 
     link = deposit["links"]["latest_draft_html"]
-    with open("zenodo-link.html", "w") as f:
+    with open("zenodo-link.html", "w", encoding="utf-8") as f:
         f.write(f'<a href="{link}">{link}</a>')
 
 

setup.py

@@ -22,7 +22,7 @@ with open(path.join(here, "README.md"), encoding="utf-8") as f:
 
 setup(
     name="openflexure-microscope-server",
-    version="2.10.1",
+    version="2.11.0",
     description="Python module, and Flask-based web API, to run the OpenFlexure Microscope.",
     long_description=long_description,
     long_description_content_type="text/markdown",
@@ -51,8 +51,9 @@ setup(
         "scipy ~= 1.6.1",
         "python-dateutil ~= 2.8",
         "psutil ~= 5.6.7",  # Autostorage extension
+        "markupsafe ~= 2.0.0",  # 2.1.x breaks jinja2
         "opencv-python-headless ~= 4.5.1",
-        "sangaboard ~= 0.2",
+        "sangaboard ~= 0.3.3",
         "expiringdict ~= 1.2.1",
         "camera-stage-mapping == 0.1.4",
         "picamerax ~= 20.9.1",
@@ -60,7 +61,7 @@ setup(
         "pytest-cov ~= 2.10.1",
         "piexif ~= 1.1.3",
         "labthings ~= 1.3.0",
-        "typing-extensions ~= 3.7.4",  # Needed for some type-hints in Python < 3.8 (e.g. Literal)
+        "typing-extensions ~= 4.3.0",  # Needed for some type-hints in Python < 3.8 (e.g. Literal)
         "RPi.GPIO ~= 0.7.0; platform_machine == 'armv7l'",
     ],
     # "dev" specifies extra packages used for development (linting, testing, etc.)
@@ -71,19 +72,15 @@ setup(
             "sphinx < 4.0",  # Currently httpdomain isn't ready for 4.0
             "sphinxcontrib-openapi ~= 0.7",
             "sphinx_rtd_theme ~=0.5.2",
-            "rope ~= 0.14.0",
+            "pylint ~= 2.14.0",  # 2.9.2 crashes and I've not yet figured out why.
-            "pylint ~= 2.8.0",  # 2.9.2 crashes and I've not yet figured out why.
+            "pytest ~= 7.1.2",
-            "pytest ~= 6.1.2",
+            "mypy ~= 0.971",
-            "mypy ~= 0.790",
             "types-python-dateutil",
             "types-setuptools",
-            "poethepoet ~= 0.10.0",
+            "poethepoet ~= 0.16.0",
-            "freezegun ~= 1.0.0",
+            "freezegun ~= 1.2.1",
-            "lxml ~= 4.6",
+            "lxml ~= 4.9",
-            "black == 18.9b0",  # Because black is in prerelease, giving a flexible version
+            "black == 18.9b0",
-            # specifier causes all manner of problems.  A fixed version
-            # avoids issues with prereleases (which are normally not
-            # allowed by pipenv) but can be a bit of a pain sometimes!
         ]
     },
     dependency_links=[],

tests/test_app.py

@@ -28,7 +28,7 @@ def test_thing_description_valid():
     # TODO: it would be nice to put this into LabThings
     # First load the schema from file and check it validates
     schema_fname = os.path.join(os.path.dirname(__file__), "w3c_td_schema.json")
-    schema = json.load(open(schema_fname, "r"))
+    schema = json.load(open(schema_fname, "r"), encoding="utf-8")
     jsonschema.Draft7Validator.check_schema(schema)
 
     # Build a TD dictionary

tests/w3c_td_schema.json

@@ -66,10 +66,10 @@
         "thing-context": {
             "oneOf": [{
                     "type": "array",
-                    "items": [{
+                    "prefixItems": [{
                         "$ref": "#/definitions/thing-context-w3c-uri"
                     }],
-                    "additionalItems": {
+                    "items": {
                         "anyOf": [{
                                 "$ref": "#/definitions/anyUri"
                             },