Build docker image and run ruff/pytest
This commit is contained in:
Richard Bowman
parent
77794c94eb
commit
71e6d84aa4
3 changed files with 131 additions and 107 deletions
231
.gitlab-ci.yml
231
.gitlab-ci.yml
|
|
@ -1,21 +1,37 @@
|
|||
stages:
|
||||
- prepare
|
||||
- analysis
|
||||
- testing
|
||||
- build
|
||||
- package
|
||||
- deploy
|
||||
|
||||
meta-build-image:
|
||||
image: docker:stable
|
||||
services:
|
||||
- docker:dind
|
||||
stage: prepare
|
||||
script:
|
||||
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
|
||||
- cd .meta-testimage-python
|
||||
- docker build -t $CI_REGISTRY/openflexure/openflexure-microscope-server/testimage-python:latest .
|
||||
- docker push $CI_REGISTRY/openflexure/openflexure-microscope-server/testimage-python:latest
|
||||
#only:
|
||||
#refs: # commented out for initial run
|
||||
# - main
|
||||
# changes:
|
||||
# - .meta/Dockerfile
|
||||
|
||||
|
||||
# Re-usable block to install (and cache) Poetry and openflexure-microscope-server
|
||||
.python:
|
||||
image: balenalib/raspberry-pi:buster
|
||||
image: registry.gitlab.com/openflexure/openflexure-microscope-server/testimage-python
|
||||
retry: 1
|
||||
before_script:
|
||||
- apt-get update
|
||||
- apt-get install apt-utils curl wget python3 python3-pip python3-venv raspi-gpio libgfortran3 libatlas3-base libffi-dev vim git-lfs libtiff5 libpango-1.0-0 libavcodec58 libgdk-pixbuf2.0-0 libjasper1 libswscale5 libpangocairo-1.0-0 libilmbase23 libatk1.0-0 libgtk-3-0 libwebp6 libcairo2 libavutil56 libcairo-gobject2 libopenexr23 libavformat58 libxslt1.1
|
||||
- python3 -m venv .venv # NB this does not overwrite an existing cached venv
|
||||
- python --version
|
||||
- python -m venv .venv # NB this does not overwrite an existing cached venv
|
||||
- source .venv/bin/activate
|
||||
- pip install pipenv # Should be cached after first run
|
||||
- pipenv install --dev --deploy # Should be cached after first run - will just check packages are present
|
||||
- pip install -e .[dev]
|
||||
cache:
|
||||
key: "${CI_COMMIT_REF_SLUG}"
|
||||
paths:
|
||||
|
|
@ -42,57 +58,60 @@ stages:
|
|||
- tags
|
||||
- web
|
||||
|
||||
# Python static analysis with PyLint
|
||||
pylint:
|
||||
# Python formatting and static analysis with ruff
|
||||
ruff:
|
||||
stage: analysis
|
||||
extends: .python
|
||||
script:
|
||||
- poe pylint
|
||||
- ruff check
|
||||
- ruff format --check
|
||||
|
||||
# Python type checking with Mypy
|
||||
mypy:
|
||||
stage: analysis
|
||||
extends: .python
|
||||
script:
|
||||
- poe mypy
|
||||
- mypy --cobertura-xml-report mypy --no-error-summary src | tee mypy/stdout.txt
|
||||
- cat mypy/stdout.txt | mypy-gitlab-code-quality > codequality.json
|
||||
allow_failure: true
|
||||
|
||||
artifacts:
|
||||
reports:
|
||||
codequality: codequality.json
|
||||
coverage_report:
|
||||
coverage_format: cobertura
|
||||
path: openflexure_microscope/cobertura.xml
|
||||
|
||||
# Python style analysis with Black
|
||||
black:
|
||||
stage: analysis
|
||||
allow_failure: true
|
||||
extends: .python
|
||||
script:
|
||||
- poe black_check
|
||||
path: mypy/cobertura.xml
|
||||
|
||||
# Python unit tests with PyTest
|
||||
pytest:
|
||||
stage: testing
|
||||
extends: .python
|
||||
script:
|
||||
- poe test
|
||||
- pytest --cov --cov-report term --cov-report xml:coverage.xml --junitxml=report.xml
|
||||
coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
|
||||
artifacts:
|
||||
when: always
|
||||
reports:
|
||||
junit: pytest_report.xml
|
||||
|
||||
# Generate and validate OpenAPI description
|
||||
openapi:
|
||||
stage: testing
|
||||
extends: .python
|
||||
script:
|
||||
- mkdir -p docs/build/
|
||||
- ofm-generate-openapi -o docs/build/swagger.yaml --validate
|
||||
artifacts:
|
||||
name: "openapi"
|
||||
expose_as: "OpenAPI Description"
|
||||
expire_in: 1 week
|
||||
paths:
|
||||
- "./docs/build/swagger.yaml"
|
||||
- report.xml
|
||||
reports:
|
||||
junit: report.xml
|
||||
coverage_report:
|
||||
coverage_format: cobertura
|
||||
path: coverage.xml
|
||||
|
||||
# Generate and validate OpenAPI description - needs updating to work with v3
|
||||
# openapi:
|
||||
# stage: testing
|
||||
# extends: .python
|
||||
# script:
|
||||
# - mkdir -p docs/build/
|
||||
# - ofm-generate-openapi -o docs/build/swagger.yaml --validate
|
||||
# artifacts:
|
||||
# name: "openapi"
|
||||
# expose_as: "OpenAPI Description"
|
||||
# expire_in: 1 week
|
||||
# paths:
|
||||
# - "./docs/build/swagger.yaml"
|
||||
|
||||
# JavaScript linting with ESLint (via Vue CLI)
|
||||
eslint:
|
||||
|
|
@ -115,86 +134,86 @@ build:
|
|||
paths:
|
||||
- "src/openflexure_microscope/static/"
|
||||
|
||||
# Package application into distribution tarball
|
||||
package:
|
||||
stage: package
|
||||
dependencies:
|
||||
- build
|
||||
- openapi
|
||||
# # Package application into distribution tarball
|
||||
# package:
|
||||
# stage: package
|
||||
# dependencies:
|
||||
# - build
|
||||
# - openapi
|
||||
|
||||
image: ubuntu:latest
|
||||
# image: ubuntu:latest
|
||||
|
||||
script:
|
||||
# Build distribution archive
|
||||
- mkdir -p dist
|
||||
- tar -c --exclude-vcs --exclude-from .tarignore -vzf dist/openflexure-microscope-server-${CI_COMMIT_REF_NAME}.tar.gz .
|
||||
- tar -c --exclude-vcs --exclude-from .tarignore -vzf dist/openflexure-microscope-webapp-${CI_COMMIT_REF_NAME}.tar.gz ./openflexure_microscope_server/static/
|
||||
- cp docs/build/swagger.yaml dist/openflexure-microscope-api-${CI_COMMIT_REF_NAME}.yaml
|
||||
- cd dist/
|
||||
- sha256sum openflexure-microscope-server-${CI_COMMIT_REF_NAME}.tar.gz > openflexure-microscope-server-${CI_COMMIT_REF_NAME}.tar.gz.sha256
|
||||
- sha256sum openflexure-microscope-webapp-${CI_COMMIT_REF_NAME}.tar.gz > openflexure-microscope-webapp-${CI_COMMIT_REF_NAME}.tar.gz.sha256
|
||||
- sha256sum openflexure-microscope-api-${CI_COMMIT_REF_NAME}.yaml > openflexure-microscope-api-${CI_COMMIT_REF_NAME}.yaml.sha256
|
||||
# script:
|
||||
# # Build distribution archive
|
||||
# - mkdir -p dist
|
||||
# - tar -c --exclude-vcs --exclude-from .tarignore -vzf dist/openflexure-microscope-server-${CI_COMMIT_REF_NAME}.tar.gz .
|
||||
# - tar -c --exclude-vcs --exclude-from .tarignore -vzf dist/openflexure-microscope-webapp-${CI_COMMIT_REF_NAME}.tar.gz ./openflexure_microscope_server/static/
|
||||
# - cp docs/build/swagger.yaml dist/openflexure-microscope-api-${CI_COMMIT_REF_NAME}.yaml
|
||||
# - cd dist/
|
||||
# - sha256sum openflexure-microscope-server-${CI_COMMIT_REF_NAME}.tar.gz > openflexure-microscope-server-${CI_COMMIT_REF_NAME}.tar.gz.sha256
|
||||
# - sha256sum openflexure-microscope-webapp-${CI_COMMIT_REF_NAME}.tar.gz > openflexure-microscope-webapp-${CI_COMMIT_REF_NAME}.tar.gz.sha256
|
||||
# - sha256sum openflexure-microscope-api-${CI_COMMIT_REF_NAME}.yaml > openflexure-microscope-api-${CI_COMMIT_REF_NAME}.yaml.sha256
|
||||
|
||||
artifacts:
|
||||
name: "dist"
|
||||
expire_in: 1 week
|
||||
paths:
|
||||
- "./dist/"
|
||||
# artifacts:
|
||||
# name: "dist"
|
||||
# expire_in: 1 week
|
||||
# paths:
|
||||
# - "./dist/"
|
||||
|
||||
only:
|
||||
- master
|
||||
- merge_requests
|
||||
- tags
|
||||
- web
|
||||
# only:
|
||||
# - master
|
||||
# - merge_requests
|
||||
# - tags
|
||||
# - web
|
||||
|
||||
# Deploy to builds.openflexure.org
|
||||
deploy:
|
||||
stage: deploy
|
||||
dependencies:
|
||||
- package
|
||||
# # Deploy to builds.openflexure.org
|
||||
# deploy:
|
||||
# stage: deploy
|
||||
# dependencies:
|
||||
# - package
|
||||
|
||||
image: ubuntu:latest
|
||||
# image: ubuntu:latest
|
||||
|
||||
before_script:
|
||||
- "which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )"
|
||||
- eval $(ssh-agent -s)
|
||||
- ssh-add <(echo "$SSH_PRIVATE_KEY_BATH_OPENFLEXURE_BASE64" | base64 --decode)
|
||||
- mkdir -p ~/.ssh
|
||||
- echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config
|
||||
# before_script:
|
||||
# - "which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )"
|
||||
# - eval $(ssh-agent -s)
|
||||
# - ssh-add <(echo "$SSH_PRIVATE_KEY_BATH_OPENFLEXURE_BASE64" | base64 --decode)
|
||||
# - mkdir -p ~/.ssh
|
||||
# - echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config
|
||||
|
||||
script:
|
||||
# Install rsync if not already installed
|
||||
- "which rsync || ( apt-get update -y && apt-get install rsync -y )"
|
||||
# script:
|
||||
# # Install rsync if not already installed
|
||||
# - "which rsync || ( apt-get update -y && apt-get install rsync -y )"
|
||||
|
||||
# Upload the builds folder to openflexure-microscope builds
|
||||
- rsync -hrvz -e ssh dist/ ci-user@openflexure.bath.ac.uk:/var/www/build/openflexure-microscope-server
|
||||
# # Upload the builds folder to openflexure-microscope builds
|
||||
# - rsync -hrvz -e ssh dist/ ci-user@openflexure.bath.ac.uk:/var/www/build/openflexure-microscope-server
|
||||
|
||||
# Run update-latest.py on the build server
|
||||
- ssh -t ci-user@openflexure.bath.ac.uk "/var/www/build/update-latest.py"
|
||||
# # Run update-latest.py on the build server
|
||||
# - ssh -t ci-user@openflexure.bath.ac.uk "/var/www/build/update-latest.py"
|
||||
|
||||
only:
|
||||
- tags
|
||||
- web
|
||||
# only:
|
||||
# - tags
|
||||
# - web
|
||||
|
||||
zenodo:
|
||||
stage: deploy
|
||||
image: ubuntu:20.04
|
||||
before_script:
|
||||
- apt-get update -qq
|
||||
- apt-get -y -qq install git python3-pip
|
||||
- pip3 install -r scripts/zenodo/requirements.txt
|
||||
variables:
|
||||
ZENODO_USE_SANDBOX: "false"
|
||||
script:
|
||||
- git archive "${CI_COMMIT_REF_NAME}" --format=zip --output="${CI_PROJECT_NAME}-${CI_COMMIT_REF_NAME}-source.zip"
|
||||
- python3 scripts/zenodo/upload_to_zenodo.py "${CI_PROJECT_NAME}-${CI_COMMIT_REF_NAME}-source.zip" dist/*
|
||||
artifacts:
|
||||
# this is only a small html link, let's just keep it forever
|
||||
# gitlab doesn't understand "expire_in: never" yet though
|
||||
expire_in: 100000 years
|
||||
name: zenodo-${CI_PROJECT_NAME}-${CI_COMMIT_REF_NAME}-link.html
|
||||
paths:
|
||||
- zenodo-link.html
|
||||
only:
|
||||
- tags
|
||||
- web
|
||||
# zenodo:
|
||||
# stage: deploy
|
||||
# image: ubuntu:20.04
|
||||
# before_script:
|
||||
# - apt-get update -qq
|
||||
# - apt-get -y -qq install git python3-pip
|
||||
# - pip3 install -r scripts/zenodo/requirements.txt
|
||||
# variables:
|
||||
# ZENODO_USE_SANDBOX: "false"
|
||||
# script:
|
||||
# - git archive "${CI_COMMIT_REF_NAME}" --format=zip --output="${CI_PROJECT_NAME}-${CI_COMMIT_REF_NAME}-source.zip"
|
||||
# - python3 scripts/zenodo/upload_to_zenodo.py "${CI_PROJECT_NAME}-${CI_COMMIT_REF_NAME}-source.zip" dist/*
|
||||
# artifacts:
|
||||
# # this is only a small html link, let's just keep it forever
|
||||
# # gitlab doesn't understand "expire_in: never" yet though
|
||||
# expire_in: 100000 years
|
||||
# name: zenodo-${CI_PROJECT_NAME}-${CI_COMMIT_REF_NAME}-link.html
|
||||
# paths:
|
||||
# - zenodo-link.html
|
||||
# only:
|
||||
# - tags
|
||||
# - web
|
||||
|
|
|
|||
4
.meta-testimage-python/Dockerfile
Normal file
4
.meta-testimage-python/Dockerfile
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
# A docker image with system-level dependencies installed already
|
||||
FROM python:3.11-bookworm
|
||||
RUN apt-get -yq update
|
||||
RUN apt-get -yq install python3-opencv python3-numpy python3-scipy
|
||||
|
|
@ -30,7 +30,8 @@ dependencies = [
|
|||
dev = [
|
||||
"labthings-fastapi[dev]",
|
||||
"ruff",
|
||||
"mypy",
|
||||
"mypy[reports]",
|
||||
"mypy-gitlab-code-quality",
|
||||
"pytest",
|
||||
]
|
||||
pi = [
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue