Add some extra rules to ruff, also add a more complete ruff config that fails

Over time we can enable more rules for `ruff check`

.gitlab-ci.yml

@@ -58,14 +58,24 @@ meta-build-image:
     - tags
     - web
 
-# Python formatting and static analysis with ruff
+# Python static analysis with ruff
 ruff-lint:
   stage: analysis
   extends: .python
   script:
     - ruff check
 
-# Python formatting and static analysis with ruff
+# Increased checking of the code with ruff this is allowed to fail
+# while we improve the code quality. Rules from this are
+# slowly moved into the main check
+ruff-lint-increased:
+  stage: analysis
+  extends: .python
+  allow_failure: true
+  script:
+    - ruff --config ruff-increased-checking.toml check
+
+# Python formatting with ruff
 ruff-format:
   stage: analysis
   extends: .python

pyproject.toml

@@ -78,3 +78,37 @@ addopts = [
 [tool.ruff.format]
 # Use native line endings for all files
 line-ending = "native"
+
+[tool.ruff.lint]
+
+# Slowly add linting rules from ruff-increased-checking.toml commented
+# out rules can be checked by running
+# ruff --config ruff-increased-checking.toml check
+select = [
+    "E", # pycodestyle errors
+    "W", #pycodestyle warnings
+    "F", # PyFlakes
+#    "PL", # Pylint (a subset of, catches far less than pylint does)
+#    "B", # Flake8 bugbear
+    "A", # Flake8 builtins checker
+#    "C", # Flake8 comprehensions
+#    "FIX", #TODO/FIXME's in code. These should be added during develoment for ongoing tasks.
+           # If they are not fixed before merge they should be converted to GitLab issues
+#    "LOG", # Flake8 logging issues (pedantic logger formatting issues can be added with "G")
+#    "T20", # Warns for print statments, production code should log
+#    "PT", # pytest linting
+#    "RET", # Consistent clear return statments
+    "RSE", # Raise parentheses
+#    "SIM", # Simplifications detected
+#    "ARG", # unused arguments
+#    "C90", # McCabe complexity!
+    "NPY", # Numpy linting
+    "N", # PEP8 naming
+#    "DOC", # Enforce argument, return, and raise to be mentioned in docstrings there is
+           # also "D" but this even warns about docstring punctuation. Perhaps a step too
+           # far?
+]
+
+ignore = [
+    "E501" # Ignore long lines for now
+]

ruff-increased-checking.toml

@@ -0,0 +1,37 @@
+# This can be passed into ruff with nano ruff-increased-checking.toml
+#
+# Once the codebase is cleaner this can be added to pyproject.toml
+# under the [tool.ruff.lint] heading 
+#
+# Select more than the bare minimum checking in ruff
+# This is only a subset of possible rules
+# See: https://docs.astral.sh/ruff/rules
+# 
+# It will take a long time to get this all passing. We should
+# slowly move rules into the pyproject.toml as we improve the
+# codebase
+
+select = [
+    "E", # pycodestyle errors
+    "W", #pycodestyle warnings
+    "F", # PyFlakes
+    "PL", # Pylint (a subset of, catches far less than pylint does)
+    "B", # Flake8 bugbear
+    "A", # Flake8 builtins checker
+    "C", # Flake8 comprehensions
+    "FIX", #TODO/FIXME's in code. These should be added during develoment for ongoing tasks.
+           # If they are not fixed before merge they should be converted to GitLab issues
+    "LOG", # Flake8 logging issues (pedantic logger formatting issues can be added with "G")
+    "T20", # Warns for print statments, production code should log
+    "PT", # pytest linting
+    "RET", # Consistent clear return statments
+    "RSE", # Raise parentheses
+    "SIM", # Simplifications detected
+    "ARG", # unused arguments
+    "C90", # McCabe complexity!
+    "NPY", # Numpy linting
+    "N", # PEP8 naming
+    "DOC", # Enforce argument, return, and raise to be mentioned in docstrings there is
+           # also "D" but this even warns about docstring punctuation. Perhaps a step too
+           # far?
+]