Build docker image and run ruff/pytest

.gitlab-ci.yml

@@ -1,21 +1,37 @@
 stages:
+  - prepare
   - analysis
   - testing
   - build
   - package
   - deploy
 
+meta-build-image:
+  image: docker:stable
+  services:
+    - docker:dind
+  stage: prepare
+  script:
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - cd .meta-testimage-python
+    - docker build -t $CI_REGISTRY/openflexure/openflexure-microscope-server/testimage-python:latest .
+    - docker push $CI_REGISTRY/openflexure/openflexure-microscope-server/testimage-python:latest
+  #only:
+    #refs:    # commented out for initial run
+    #  - main
+    # changes:
+    #   - .meta/Dockerfile
+
+
 # Re-usable block to install (and cache) Poetry and openflexure-microscope-server
 .python:
-  image: balenalib/raspberry-pi:buster
+  image: registry.gitlab.com/openflexure/openflexure-microscope-server/testimage-python
   retry: 1
   before_script:
-    - apt-get update
+    - python --version
-    - apt-get install apt-utils curl wget python3 python3-pip python3-venv raspi-gpio libgfortran3 libatlas3-base libffi-dev vim git-lfs libtiff5 libpango-1.0-0 libavcodec58 libgdk-pixbuf2.0-0 libjasper1 libswscale5 libpangocairo-1.0-0 libilmbase23 libatk1.0-0 libgtk-3-0 libwebp6 libcairo2 libavutil56 libcairo-gobject2 libopenexr23 libavformat58 libxslt1.1
+    - python -m venv .venv        # NB this does not overwrite an existing cached venv
-    - python3 -m venv .venv        # NB this does not overwrite an existing cached venv
     - source .venv/bin/activate
-    - pip install pipenv           # Should be cached after first run
+    - pip install -e .[dev]
-    - pipenv install --dev --deploy        # Should be cached after first run - will just check packages are present
   cache:
     key: "${CI_COMMIT_REF_SLUG}"
     paths:
@@ -42,57 +58,60 @@ stages:
     - tags
     - web
 
-# Python static analysis with PyLint
+# Python formatting and static analysis with ruff
-pylint:
+ruff:
   stage: analysis
   extends: .python
   script:
-    - poe pylint
+    - ruff check
+    - ruff format --check
 
 # Python type checking with Mypy
 mypy:
   stage: analysis
   extends: .python
   script:
-    - poe mypy
+    - mypy --cobertura-xml-report mypy --no-error-summary src | tee mypy/stdout.txt
+    - cat mypy/stdout.txt | mypy-gitlab-code-quality > codequality.json
+  allow_failure: true
+
   artifacts:
     reports:
+      codequality: codequality.json
       coverage_report: 
         coverage_format: cobertura
-        path: openflexure_microscope/cobertura.xml
+        path: mypy/cobertura.xml
-
-# Python style analysis with Black
-black:
-  stage: analysis
-  allow_failure: true
-  extends: .python
-  script:
-    - poe black_check
 
 # Python unit tests with PyTest
 pytest:
   stage: testing
   extends: .python
   script:
-    - poe test
+    - pytest --cov --cov-report term --cov-report xml:coverage.xml --junitxml=report.xml
+  coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
   artifacts:
     when: always
-    reports:
-      junit: pytest_report.xml
-
-# Generate and validate OpenAPI description
-openapi:
-  stage: testing
-  extends: .python
-  script:
-    - mkdir -p docs/build/
-    - ofm-generate-openapi -o docs/build/swagger.yaml --validate
-  artifacts:
-    name: "openapi"
-    expose_as: "OpenAPI Description"
-    expire_in: 1 week
     paths:
-      - "./docs/build/swagger.yaml"
+      - report.xml
+    reports:
+      junit: report.xml
+      coverage_report:
+        coverage_format: cobertura
+        path: coverage.xml
+
+# Generate and validate OpenAPI description - needs updating to work with v3
+# openapi:
+#   stage: testing
+#   extends: .python
+#   script:
+#     - mkdir -p docs/build/
+#     - ofm-generate-openapi -o docs/build/swagger.yaml --validate
+#   artifacts:
+#     name: "openapi"
+#     expose_as: "OpenAPI Description"
+#     expire_in: 1 week
+#     paths:
+#       - "./docs/build/swagger.yaml"
 
 # JavaScript linting with ESLint (via Vue CLI)
 eslint:
@@ -115,86 +134,86 @@ build:
     paths:
       - "src/openflexure_microscope/static/"
 
-# Package application into distribution tarball
+# # Package application into distribution tarball
-package:
+# package:
-  stage: package
+#   stage: package
-  dependencies:
+#   dependencies:
-    - build
+#     - build
-    - openapi
+#     - openapi
 
-  image: ubuntu:latest
+#   image: ubuntu:latest
 
-  script:
+#   script:
-    # Build distribution archive
+#     # Build distribution archive
-    - mkdir -p dist
+#     - mkdir -p dist
-    - tar -c --exclude-vcs --exclude-from .tarignore -vzf dist/openflexure-microscope-server-${CI_COMMIT_REF_NAME}.tar.gz .
+#     - tar -c --exclude-vcs --exclude-from .tarignore -vzf dist/openflexure-microscope-server-${CI_COMMIT_REF_NAME}.tar.gz .
-    - tar -c --exclude-vcs --exclude-from .tarignore -vzf dist/openflexure-microscope-webapp-${CI_COMMIT_REF_NAME}.tar.gz ./openflexure_microscope_server/static/
+#     - tar -c --exclude-vcs --exclude-from .tarignore -vzf dist/openflexure-microscope-webapp-${CI_COMMIT_REF_NAME}.tar.gz ./openflexure_microscope_server/static/
-    - cp docs/build/swagger.yaml dist/openflexure-microscope-api-${CI_COMMIT_REF_NAME}.yaml
+#     - cp docs/build/swagger.yaml dist/openflexure-microscope-api-${CI_COMMIT_REF_NAME}.yaml
-    - cd dist/
+#     - cd dist/
-    - sha256sum openflexure-microscope-server-${CI_COMMIT_REF_NAME}.tar.gz > openflexure-microscope-server-${CI_COMMIT_REF_NAME}.tar.gz.sha256
+#     - sha256sum openflexure-microscope-server-${CI_COMMIT_REF_NAME}.tar.gz > openflexure-microscope-server-${CI_COMMIT_REF_NAME}.tar.gz.sha256
-    - sha256sum openflexure-microscope-webapp-${CI_COMMIT_REF_NAME}.tar.gz > openflexure-microscope-webapp-${CI_COMMIT_REF_NAME}.tar.gz.sha256
+#     - sha256sum openflexure-microscope-webapp-${CI_COMMIT_REF_NAME}.tar.gz > openflexure-microscope-webapp-${CI_COMMIT_REF_NAME}.tar.gz.sha256
-    - sha256sum openflexure-microscope-api-${CI_COMMIT_REF_NAME}.yaml > openflexure-microscope-api-${CI_COMMIT_REF_NAME}.yaml.sha256
+#     - sha256sum openflexure-microscope-api-${CI_COMMIT_REF_NAME}.yaml > openflexure-microscope-api-${CI_COMMIT_REF_NAME}.yaml.sha256
 
-  artifacts:
+#   artifacts:
-    name: "dist"
+#     name: "dist"
-    expire_in: 1 week
+#     expire_in: 1 week
-    paths:
+#     paths:
-      - "./dist/"
+#       - "./dist/"
 
-  only:
+#   only:
-    - master
+#     - master
-    - merge_requests
+#     - merge_requests
-    - tags
+#     - tags
-    - web
+#     - web
 
-# Deploy to builds.openflexure.org
+# # Deploy to builds.openflexure.org
-deploy:
+# deploy:
-  stage: deploy
+#   stage: deploy
-  dependencies:
+#   dependencies:
-    - package
+#     - package
 
-  image: ubuntu:latest
+#   image: ubuntu:latest
 
-  before_script:
+#   before_script:
-    - "which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )"
+#     - "which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )"
-    - eval $(ssh-agent -s)
+#     - eval $(ssh-agent -s)
-    - ssh-add <(echo "$SSH_PRIVATE_KEY_BATH_OPENFLEXURE_BASE64" | base64 --decode)
+#     - ssh-add <(echo "$SSH_PRIVATE_KEY_BATH_OPENFLEXURE_BASE64" | base64 --decode)
-    - mkdir -p ~/.ssh
+#     - mkdir -p ~/.ssh
-    - echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config
+#     - echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config
 
-  script:
+#   script:
-    # Install rsync if not already installed
+#     # Install rsync if not already installed
-    - "which rsync || ( apt-get update -y && apt-get install rsync -y )"
+#     - "which rsync || ( apt-get update -y && apt-get install rsync -y )"
 
-    # Upload the builds folder to openflexure-microscope builds
+#     # Upload the builds folder to openflexure-microscope builds
-    - rsync -hrvz -e ssh dist/ ci-user@openflexure.bath.ac.uk:/var/www/build/openflexure-microscope-server
+#     - rsync -hrvz -e ssh dist/ ci-user@openflexure.bath.ac.uk:/var/www/build/openflexure-microscope-server
 
-    # Run update-latest.py on the build server
+#     # Run update-latest.py on the build server
-    - ssh -t ci-user@openflexure.bath.ac.uk "/var/www/build/update-latest.py"
+#     - ssh -t ci-user@openflexure.bath.ac.uk "/var/www/build/update-latest.py"
 
-  only:
+#   only:
-    - tags
+#     - tags
-    - web
+#     - web
 
-zenodo:
+# zenodo:
-  stage: deploy
+#   stage: deploy
-  image: ubuntu:20.04
+#   image: ubuntu:20.04
-  before_script:
+#   before_script:
-    - apt-get update -qq
+#     - apt-get update -qq
-    - apt-get -y -qq install git python3-pip
+#     - apt-get -y -qq install git python3-pip
-    - pip3 install -r scripts/zenodo/requirements.txt
+#     - pip3 install -r scripts/zenodo/requirements.txt
-  variables:
+#   variables:
-    ZENODO_USE_SANDBOX: "false"
+#     ZENODO_USE_SANDBOX: "false"
-  script:
+#   script:
-    - git archive "${CI_COMMIT_REF_NAME}" --format=zip --output="${CI_PROJECT_NAME}-${CI_COMMIT_REF_NAME}-source.zip"
+#     - git archive "${CI_COMMIT_REF_NAME}" --format=zip --output="${CI_PROJECT_NAME}-${CI_COMMIT_REF_NAME}-source.zip"
-    - python3 scripts/zenodo/upload_to_zenodo.py "${CI_PROJECT_NAME}-${CI_COMMIT_REF_NAME}-source.zip" dist/*
+#     - python3 scripts/zenodo/upload_to_zenodo.py "${CI_PROJECT_NAME}-${CI_COMMIT_REF_NAME}-source.zip" dist/*
-  artifacts:
+#   artifacts:
-    # this is only a small html link, let's just keep it forever
+#     # this is only a small html link, let's just keep it forever
-    # gitlab doesn't understand "expire_in: never" yet though
+#     # gitlab doesn't understand "expire_in: never" yet though
-    expire_in: 100000 years
+#     expire_in: 100000 years
-    name: zenodo-${CI_PROJECT_NAME}-${CI_COMMIT_REF_NAME}-link.html
+#     name: zenodo-${CI_PROJECT_NAME}-${CI_COMMIT_REF_NAME}-link.html
-    paths:
+#     paths:
-      - zenodo-link.html
+#       - zenodo-link.html
-  only:
+#   only:
-    - tags
+#     - tags
-    - web
+#     - web

.meta-testimage-python/Dockerfile

@@ -0,0 +1,4 @@
+# A docker image with system-level dependencies installed already
+FROM python:3.11-bookworm
+RUN apt-get -yq update
+RUN apt-get -yq install python3-opencv python3-numpy python3-scipy

pyproject.toml

@@ -30,7 +30,8 @@ dependencies = [
 dev = [
   "labthings-fastapi[dev]",
   "ruff",
-  "mypy",
+  "mypy[reports]",
+  "mypy-gitlab-code-quality",
   "pytest",
 ]
 pi = [