From 9f38b041a67727aca75313cc3f9271f68da9fb50 Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Tue, 29 Jul 2025 15:48:10 +0100 Subject: [PATCH 1/7] Create script to zip up picamera tests and hashes of releavant source files --- hardware-specific-tests/README.md | 23 ++++++ picamera_tests.py | 120 ++++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+) create mode 100644 hardware-specific-tests/README.md create mode 100755 picamera_tests.py diff --git a/hardware-specific-tests/README.md b/hardware-specific-tests/README.md new file mode 100644 index 00000000..62172610 --- /dev/null +++ b/hardware-specific-tests/README.md @@ -0,0 +1,23 @@ +## Hardware specific tests for the PiCamera. + +These are very slow as they run on hardware. They can be run with: + + pytest hardware-specific-tests + +It is essential to stop the server first: + + ofm stop + +However, this will not archive the tests in the Git repository for reporting the coverage. For this see the section below on reporting the coverage. + +When writing and debugging these unit tests it is often best to run a specific test and to use the `-s` flag to see the print statments. It is also often useful to use `--pdb` to drop you into a python debug session on any failure. For example, you might run: + + /picamera2/test_exposure_time_drift.py::test_exposure_time_saves_and_loads -s --pdb + +### Reporting the coverage + +To create a coverage report that will be included into the repository run: + + ./picamera_tests.py run + +This will create a zip of test results along with hashes of the picamera files and the base camera file. The server will include this overage report if the source files remain the same. \ No newline at end of file diff --git a/picamera_tests.py b/picamera_tests.py new file mode 100755 index 00000000..da648d4e --- /dev/null +++ b/picamera_tests.py @@ -0,0 +1,120 @@ +#! /usr/bin/env python3 +"""Run the Piamera tests and archive the results in the git repository.""" + +import subprocess +import shutil +import os +import posixpath +import argparse +import json +import sys +import zipfile + +CAM_DIR = os.path.join("src", "openflexure_microscope_server", "things", "camera") +ZIP_FILE = "picamera_coverage.zip" +COVERAGE_FILE = ".coverage.picamera" +HASH_FILE = "picamera_test_hashes.json" + +# If any of the HASHED_FILES change, then the picamera tests need to be re-run. +# System independent can be checked on windows even if running tests must be on a Pi +HASHED_FILES = [ + os.path.join(CAM_DIR, "__init__.py"), + os.path.join(CAM_DIR, "picamera.py"), + os.path.join(CAM_DIR, "picamera_recalibrate_utils.py"), +] + + +def _get_hashes(include_coverage=False): + """Return a dictonary of Git hashes for the HASHED FILES. + + The key is the posixpath to the file so matching works even if the dictionary is + compared to a dictionary created on another system. + + The coverage file is hashed when running for debug purposes. + + Git hashes are so the files are line ending indpendent. + """ + hashes = {} + filepaths = HASHED_FILES + if include_coverage: + filepaths.append(COVERAGE_FILE) + for filepath in HASHED_FILES: + if not os.path.isfile(filepath): + # Sys exit rather than raise for better command line experience + print(f"ERROR: {filepath} does not exist!") + sys.exit(-1) + # Create git hashes for file system independence + file_hash = subprocess.check_output(["git", "hash-object", filepath]) + hashes[posixpath.normpath(filepath)] = file_hash.decode("utf-8").strip() + return hashes + + +def run_tests(): + """Run the picamera tests, zip the coverage database, hash relevant source files.""" + subprocess.run(["pytest", "hardware-specific-tests"], check=True) + shutil.copyfile(".coverage", COVERAGE_FILE) + hash_dict = _get_hashes(include_coverage=True) + with open(HASH_FILE, "w", encoding="utf-8") as json_file: + json.dump(hash_dict, json_file) + with zipfile.ZipFile(ZIP_FILE, "w") as zipf: + zipf.write(COVERAGE_FILE) + zipf.write(HASH_FILE) + + +def unzip_coverage(): + """Unzip coverage database if hashes match relevant source files.""" + if zipfile.is_zipfile(ZIP_FILE): + with zipfile.ZipFile(ZIP_FILE, "r") as zip_obj: + zip_obj.extractall() + else: + print("ERROR: Zipfile with coverage results not found.") + sys.exit(-1) + + hash_dict = _get_hashes() + + if os.path.isfile(HASH_FILE): + with open(HASH_FILE, "r", encoding="utf-8") as json_file: + zipped_hash_dict = json.load(json_file) + else: + print("ERROR: JSON hashes not available. Zipfile is incomplete.") + sys.exit(-1) + del zipped_hash_dict[COVERAGE_FILE] + + if hash_dict != zipped_hash_dict: + # Sys exit rather than raise for better command line experience + print( + "ERROR: hashes don't match. This probably means the source has changed " + "since the tests were last run on the Pi." + ) + sys.exit(-1) + + +def main(): + """Either run tests or unzip previous results.""" + parser = argparse.ArgumentParser( + description=( + "picamera_tests: run or validate PiCamera hardware test coverage.\n\n" + "Use 'run' on the Pi to execute tests and package coverage data.\n" + "Use 'unzip' on another machine to extract and verify that the test\n" + "data matches the source code before unzipping it." + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + subparsers.add_parser("run", help="Run hardware-specific tests on the PiCamera") + + subparsers.add_parser( + "unzip", help="Unzip and verify coverage data on another machine" + ) + + args = parser.parse_args() + + if args.command == "run": + run_tests() + elif args.command == "unzip": + unzip_coverage() + + +if __name__ == "__main__": + main() From 727d683c48497f05999c992c877948d2c2b52a7d Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Tue, 29 Jul 2025 16:03:30 +0100 Subject: [PATCH 2/7] Add zip of picamera coverage and update CI to use it --- .gitignore | 3 +++ .gitlab-ci.yml | 20 ++++++++++++++++++-- hardware-specific-tests/README.md | 2 +- picamera_coverage.zip | Bin 0 -> 53913 bytes picamera_tests.py | 4 ++-- 5 files changed, 24 insertions(+), 5 deletions(-) create mode 100644 picamera_coverage.zip diff --git a/.gitignore b/.gitignore index b94a21d8..acc08e3c 100644 --- a/.gitignore +++ b/.gitignore @@ -50,6 +50,9 @@ nosetests.xml coverage.xml report.xml pytest_report.xml +.coverage.picamera +.coverage.main +picamera_test_hashes.json # Translations *.mo diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4bbb1e73..244f5025 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -130,15 +130,31 @@ pytest: - git config --global user.name "Sir Unit of Test" - git config --global user.email "fake@email.no" - pytest + - mv .coverage .coverage.main # --gitlab-code-quality-report=pytest-warnings.json # can restore when it installs ok - coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/' artifacts: when: always paths: - report.xml + - .coverage.main + +# Combine CI coverage with coverage from Picamera +combined-tests: + stage: testing + extends: .python + needs: + - job: pytest + artifacts: true + script: + - ./picamera_tests.py unzip + - coverage combine + - coverage report + - coverage xml + coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/' + artifacts: + when: always reports: junit: report.xml -# codequality: pytest-warnings.json coverage_report: coverage_format: cobertura path: coverage.xml diff --git a/hardware-specific-tests/README.md b/hardware-specific-tests/README.md index 62172610..d5bc7191 100644 --- a/hardware-specific-tests/README.md +++ b/hardware-specific-tests/README.md @@ -10,7 +10,7 @@ It is essential to stop the server first: However, this will not archive the tests in the Git repository for reporting the coverage. For this see the section below on reporting the coverage. -When writing and debugging these unit tests it is often best to run a specific test and to use the `-s` flag to see the print statments. It is also often useful to use `--pdb` to drop you into a python debug session on any failure. For example, you might run: +When writing and debugging these unit tests it is often best to run a specific test and to use the `-s` flag to see the print statements. It is also often useful to use `--pdb` to drop you into a python debug session on any failure. For example, you might run: /picamera2/test_exposure_time_drift.py::test_exposure_time_saves_and_loads -s --pdb diff --git a/picamera_coverage.zip b/picamera_coverage.zip new file mode 100644 index 0000000000000000000000000000000000000000..0eab078c6e4e15823d138a5c44391a34947971e7 GIT binary patch literal 53913 zcmWIWW@Zs#0D%Q{f1|9kqAM9LFff3y5Ca2)UUGg}YEfc(s$M~6a$;_3QDShQPi9G~ zLRx-NZeodoF@pdDBO`-@f&v2r11AFm124!VC}wA1U|@l=K@=m1Ab>d;^pY8Q`NJ4^ z*m@cG<9Pk~b$RCSs&QZB$>r?gp2wcZnatKpz>HDp(GVC7fzc2c4T0ev0*!$@?BbG= zj7_yAiAg!BB}IwJsYMWy*Ez`5F~n6N#L>yeRRJQVpuwe}prD|utKg_mlwYn;kXodW zmYI{PkeQc~nVeXXnU}6ml938iU0R%)0;=c~N-`2lzzX%i`axRaGgA~Y^GZ_FQ;QS| ziZXK(iz*eeQ!91A3ZS-?q*jzb1k&=0QZv)@Kq?e8pxQMRic-^3i&FEFQ;WfdD`;e< zXmV*b*K@Ip3kx&0r-J>ISd^Stl+2Cdo5Z4IgjYP%6pB)l^NUhIu1_jT%uCKlE!I&; zg!?c#KM&**h;NJXbM;H|6>>83z~0O&&Me8y&r?XPNKGy+Nln4(K^;(_!LY6_C`jPG zDaltz0C^|@q7!NtQs^KwW6=p!4%M7il%JcI4+~a=Mv&FIx)3+#mF6a;7Aa(v<`#e= zrzk%cVsS}6E}h``2KyIPZ%IBxC-HHEB`%>Jh|f$xiB706I7T75O7k)cOH&nKk*}iw zRT7_>qN4!uxQ+tI6PjF_s@!bimcon-nRzLx6~%=)nI)<5iKQj^V19gJQF6RtVo@?r ziy#NPxV$`Ldm%WXfV>!=lvz>?XKa#2o>LVg-J%Ybw% zKmxcJlt2)9Mjz@^g~TFIMpMYoO9d;;EGdRYKxR%V)&v8|s>ty}&C~{RY-UNZLQ+nC z5^7olrm{XdnP>`5eR17LMpf!L3NMTB1NuokxQhsR(IQ@dS;N+Q| zT8ZRdhyj@H&rYq>1S{9oRS3v0F3wEKNd;9~#a1v;1@(-?Vo)hmtgeuhpP!SOms-Ni0b%$;?ev04V`G2BcM?Jh50IxhOReT$gI9GviAJpxBBx1d*U#G6$%! z)+o=&F0QT3*jNfq4lsAd7bj<=<|d+XnNSi5nj|8<_~)g9+5*Vw1k}hu3Jwq#Vgsmf zgT!DGBott(pluy4O;87)k^eUX|8J0cNAYL~jE2By2#kinXb6mkz-S1JhQMeDjE2By z2#kinXb6mkz%UB|W=1APc2NJHiT@!3|9k$2!_0r9-Wd&n(GVC7fzc2c4S~@R7!85Z z5Eu=C(GVC7fzc2c4S|sp0z6F2viz_m9tQl3%+lh}MF8e{mU@O<3=9lR{4W^zU+{k& zIe{|j)6ozZ4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Ez~zz{4!d2pa`p;$oKO zgbo6L=KsZ6)fo6E@yqfphn&)S|M~BK_i`WMmcb2o>=#6;OpBD!#ZRu_QA&9<&9mSg)Y6 zSd4|CF`FLN$H!;pWtPOp>lIX%in1^?=F`K{oYeHhJ?O0 z2(mCVmeRvs$bm!PY>-^0S5R3az{1d2OGlfFGjmIG5<$DZLCKq+=E)nD&T!!qAvb4+j)i7MG;v>J?Ph@vty7 zG19|I(6I{fMXAZDc_l@u5U=YMRF-hFFf``U%l@?d;h;`ZQEDVj&G_@`#KRq3i zTN-)TSQwgQX=)p^n1B|Ku!Gq^$v815GpQ)CBsCu7e^5|Yv9d5URt|t`Km{fX%?r$= z#N_PsqWsdll=zg?lGNl9y@JXrW;%NrsZK6tqItUu8u>_;<}k7_G{(|2j*)Bw&Hsb@ z|C9N&c{X#OHdq>uryZF z)>>$li>3Q7QpM6(PDjg;f?K$prLmBXR>QjgLX|9yrF5|u+P(#ipMksog0(D-we++Z zrTZ^H`{WHvXUN@u{ze9tCT2SN8dR2`^#A#aSQ^vm;sAL6pSPZ+iIFZ=V(HYOHEPCh0kRz?<1PCh2k{68E24F>+-{9pLr z@;~Fh&wpd+rHD}%kA}c#2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz{m^%HWo%s zMm9Du#mWk%SXjUmGc%ZCVq#(8iNzVI z#d=xA`FYh!#YM^bpu2q2a#AZwi&EorGn0$*i<9#UQsbd_SwpWjgx+h3bi<*Nm4cFq zk&$VViE)a7p-D<=Qi^G^QBsmgqM<>mftjU=nTbKFS&~7TnPH-mjsnS6!Cb9ZPzkci zEH%|K)yTjgIVH)`(84Iu)YLNBBq=4u(#XUpH7PYU+0ekq*d!&D6072iQj-&NGLwoD zOH$)Ox0Mz{El*2IvoJDGNlUXxGD%4@w@fxROtmmbPD(UNu{27yOiMJkv@lIfG5}ew zmz-agT9lZcst5A{$oZ+}Mky&NsVT+=rk06`DaM9|$z~==sitXZNy#ax28L Date: Tue, 29 Jul 2025 16:46:37 +0100 Subject: [PATCH 3/7] Use relative paths in coverage report for combining Pi and CI reports --- .coveragerc | 1 + picamera_coverage.zip | Bin 53913 -> 53913 bytes 2 files changed, 1 insertion(+) diff --git a/.coveragerc b/.coveragerc index 4f0eb412..02daf61c 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,3 +2,4 @@ concurrency = multiprocessing, thread parallel = true sigterm = true +relative_files = True diff --git a/picamera_coverage.zip b/picamera_coverage.zip index 0eab078c6e4e15823d138a5c44391a34947971e7..c72822888cdf36bb69a9cd5f8134e3c6f83353d8 100644 GIT binary patch delta 956 zcmbQalzHY-W}yIYW)=|!5IElYH|qbwSb>c~$_|EHO$_|!_^0xR@JsUD;A`az;Zx*& z&bxuPjMtFo7ta=+N**)rAKZJmYq+hr{&OAUYTE25;KRk{B+kOn$T|6;r_$uLZtRl} zxQeoxi!n2l!bR-exLGYlnHegfA|h-KA}kDztdkkTZP@IDSr{4_Co_iYPhRKF&1xmY z%uo&D@urmK=2q$zRN4qKGt^HmjId;L6kuU!WS>0I*I=@YCoiidKgdYNa4n(S#G;b; z;^f3Uy@E;?J{E>X-pLO=9av3ynHdTvGlpwT?(yPabLL@TXyl&!&{K2rKQA#>3vOnH zaxkyXn}^ku3#1g}g#;f~fr6aGyu8$+V!eV&3r>hKrO63C!mNfI%nX?zUkH|#Waea+ zWTt}D=(963q=G~Ra`MyDGxO5*3MvEHSQr}BCr|YCWwU2xVQ6HYJki&JL$9FHk_F^y zkWG92MA+<^p>hV3XI|iDHD`i^o2Ec~d}dx|NqoFsL8SpBGebJa_{pjP0)|}84E*Q$ zrT9X5U+`|^I?m0&)yzMQ*NC^AKa~3?-%Y+Y9&?_pJXPGa+ zxt?)yVT2W{6H6mIBRqx#zy_kn5Wg!+BQN9RhmnpfX3T{kle8xHL~*jZurzXmtk<0U zFG`%nlDQnhsf*@?r~`!=TI_H^RVhtQh!J5iV$K9qQ{W@Hj! SM$M6v1uj{@4V)Zs$rAts!5!BC literal 53913 zcmWIWW@Zs#0D%Q{f1|9kqAM9LFff3y5Ca2)UUGg}YEfc(s$M~6a$;_3QDShQPi9G~ zLRx-NZeodoF@pdDBO`-@f&v2r11AFm124!VC}wA1U|@l=K@=m1Ab>d;^pY8Q`NJ4^ z*m@cG<9Pk~b$RCSs&QZB$>r?gp2wcZnatKpz>HDp(GVC7fzc2c4T0ev0*!$@?BbG= zj7_yAiAg!BB}IwJsYMWy*Ez`5F~n6N#L>yeRRJQVpuwe}prD|utKg_mlwYn;kXodW zmYI{PkeQc~nVeXXnU}6ml938iU0R%)0;=c~N-`2lzzX%i`axRaGgA~Y^GZ_FQ;QS| ziZXK(iz*eeQ!91A3ZS-?q*jzb1k&=0QZv)@Kq?e8pxQMRic-^3i&FEFQ;WfdD`;e< zXmV*b*K@Ip3kx&0r-J>ISd^Stl+2Cdo5Z4IgjYP%6pB)l^NUhIu1_jT%uCKlE!I&; zg!?c#KM&**h;NJXbM;H|6>>83z~0O&&Me8y&r?XPNKGy+Nln4(K^;(_!LY6_C`jPG zDaltz0C^|@q7!NtQs^KwW6=p!4%M7il%JcI4+~a=Mv&FIx)3+#mF6a;7Aa(v<`#e= zrzk%cVsS}6E}h``2KyIPZ%IBxC-HHEB`%>Jh|f$xiB706I7T75O7k)cOH&nKk*}iw zRT7_>qN4!uxQ+tI6PjF_s@!bimcon-nRzLx6~%=)nI)<5iKQj^V19gJQF6RtVo@?r ziy#NPxV$`Ldm%WXfV>!=lvz>?XKa#2o>LVg-J%Ybw% zKmxcJlt2)9Mjz@^g~TFIMpMYoO9d;;EGdRYKxR%V)&v8|s>ty}&C~{RY-UNZLQ+nC z5^7olrm{XdnP>`5eR17LMpf!L3NMTB1NuokxQhsR(IQ@dS;N+Q| zT8ZRdhyj@H&rYq>1S{9oRS3v0F3wEKNd;9~#a1v;1@(-?Vo)hmtgeuhpP!SOms-Ni0b%$;?ev04V`G2BcM?Jh50IxhOReT$gI9GviAJpxBBx1d*U#G6$%! z)+o=&F0QT3*jNfq4lsAd7bj<=<|d+XnNSi5nj|8<_~)g9+5*Vw1k}hu3Jwq#Vgsmf zgT!DGBott(pluy4O;87)k^eUX|8J0cNAYL~jE2By2#kinXb6mkz-S1JhQMeDjE2By z2#kinXb6mkz%UB|W=1APc2NJHiT@!3|9k$2!_0r9-Wd&n(GVC7fzc2c4S~@R7!85Z z5Eu=C(GVC7fzc2c4S|sp0z6F2viz_m9tQl3%+lh}MF8e{mU@O<3=9lR{4W^zU+{k& zIe{|j)6ozZ4S~@R7!85Z5Eu=C(GVC7fzc2c4S~@R7!85Z5Ez~zz{4!d2pa`p;$oKO zgbo6L=KsZ6)fo6E@yqfphn&)S|M~BK_i`WMmcb2o>=#6;OpBD!#ZRu_QA&9<&9mSg)Y6 zSd4|CF`FLN$H!;pWtPOp>lIX%in1^?=F`K{oYeHhJ?O0 z2(mCVmeRvs$bm!PY>-^0S5R3az{1d2OGlfFGjmIG5<$DZLCKq+=E)nD&T!!qAvb4+j)i7MG;v>J?Ph@vty7 zG19|I(6I{fMXAZDc_l@u5U=YMRF-hFFf``U%l@?d;h;`ZQEDVj&G_@`#KRq3i zTN-)TSQwgQX=)p^n1B|Ku!Gq^$v815GpQ)CBsCu7e^5|Yv9d5URt|t`Km{fX%?r$= z#N_PsqWsdll=zg?lGNl9y@JXrW;%NrsZK6tqItUu8u>_;<}k7_G{(|2j*)Bw&Hsb@ z|C9N&c{X#OHdq>uryZF z)>>$li>3Q7QpM6(PDjg;f?K$prLmBXR>QjgLX|9yrF5|u+P(#ipMksog0(D-we++Z zrTZ^H`{WHvXUN@u{ze9tCT2SN8dR2`^#A#aSQ^vm;sAL6pSPZ+iIFZ=V(HYOHEPCh0kRz?<1PCh2k{68E24F>+-{9pLr z@;~Fh&wpd+rHD}%kA}c#2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz{m^%HWo%s zMm9Du#mWk%SXjUmGc%ZCVq#(8iNzVI z#d=xA`FYh!#YM^bpu2q2a#AZwi&EorGn0$*i<9#UQsbd_SwpWjgx+h3bi<*Nm4cFq zk&$VViE)a7p-D<=Qi^G^QBsmgqM<>mftjU=nTbKFS&~7TnPH-mjsnS6!Cb9ZPzkci zEH%|K)yTjgIVH)`(84Iu)YLNBBq=4u(#XUpH7PYU+0ekq*d!&D6072iQj-&NGLwoD zOH$)Ox0Mz{El*2IvoJDGNlUXxGD%4@w@fxROtmmbPD(UNu{27yOiMJkv@lIfG5}ew zmz-agT9lZcst5A{$oZ+}Mky&NsVT+=rk06`DaM9|$z~==sitXZNy#ax28L Date: Tue, 29 Jul 2025 18:24:48 +0100 Subject: [PATCH 4/7] Add typehints to picamera_tests script --- picamera_tests.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/picamera_tests.py b/picamera_tests.py index 17dd5662..e8cadfe5 100755 --- a/picamera_tests.py +++ b/picamera_tests.py @@ -24,7 +24,7 @@ HASHED_FILES = [ ] -def _get_hashes(include_coverage=False): +def _get_hashes(include_coverage: bool = False) -> dict[str, str]: """Return a dictionary of Git hashes for the HASHED FILES. The key is the posixpath to the file so matching works even if the dictionary is @@ -49,7 +49,7 @@ def _get_hashes(include_coverage=False): return hashes -def run_tests(): +def run_tests() -> None: """Run the picamera tests, zip the coverage database, hash relevant source files.""" subprocess.run(["pytest", "hardware-specific-tests"], check=True) shutil.copyfile(".coverage", COVERAGE_FILE) @@ -61,7 +61,7 @@ def run_tests(): zipf.write(HASH_FILE) -def unzip_coverage(): +def unzip_coverage() -> None: """Unzip coverage database if hashes match relevant source files.""" if zipfile.is_zipfile(ZIP_FILE): with zipfile.ZipFile(ZIP_FILE, "r") as zip_obj: @@ -89,7 +89,7 @@ def unzip_coverage(): sys.exit(-1) -def main(): +def main() -> None: """Either run tests or unzip previous results.""" parser = argparse.ArgumentParser( description=( From e7d5b8bd1f99c35f67e1ee6cc0143504c7a08a42 Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Tue, 29 Jul 2025 18:42:15 +0100 Subject: [PATCH 5/7] Improve README a for hardware specific tests, and make fixes to picamera_tests script --- hardware-specific-tests/README.md | 28 ++++++++++++++++++++++++++-- picamera_coverage.zip | Bin 53913 -> 53913 bytes picamera_tests.py | 2 +- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/hardware-specific-tests/README.md b/hardware-specific-tests/README.md index d5bc7191..7e6e83e9 100644 --- a/hardware-specific-tests/README.md +++ b/hardware-specific-tests/README.md @@ -12,7 +12,7 @@ However, this will not archive the tests in the Git repository for reporting the When writing and debugging these unit tests it is often best to run a specific test and to use the `-s` flag to see the print statements. It is also often useful to use `--pdb` to drop you into a python debug session on any failure. For example, you might run: - /picamera2/test_exposure_time_drift.py::test_exposure_time_saves_and_loads -s --pdb + pytest hardware-specific-tests/picamera2/test_exposure_time_drift.py::test_exposure_time_saves_and_loads -s --pdb ### Reporting the coverage @@ -20,4 +20,28 @@ To create a coverage report that will be included into the repository run: ./picamera_tests.py run -This will create a zip of test results along with hashes of the picamera files and the base camera file. The server will include this overage report if the source files remain the same. \ No newline at end of file +This will create a zip of test results along with hashes of the picamera files and the base camera file. The server will include this coverage report if the source files remain the same. + +### Creating a combined report locally + +To create a combined coverage report locally (similar to the one created on CI) first run all normal tests with `pytest`. + +Copy the `.coverage` file to `.coverage.main` + + cp .coverage .coverage.main + +Then run: + + ./picamera_tests.py unzip + +to get the `.coverage.picamera` file. + +To combine reports run: + + coverage combine + +It is then possible to run: + +* `coverage report` to get the in-terminal report of the coverage. +* `coverage html` to get the HTML overview of covered lines. +* `coverage xml` to get an XML report any software that may need it. diff --git a/picamera_coverage.zip b/picamera_coverage.zip index c72822888cdf36bb69a9cd5f8134e3c6f83353d8..38b3714374152b266402b9829af91e734b7d1558 100644 GIT binary patch delta 45 xcmbQalzHY-W}X0VW)=|!5a64-kw@ dict[str, str]: filepaths = HASHED_FILES if include_coverage: filepaths.append(COVERAGE_FILE) - for filepath in HASHED_FILES: + for filepath in filepaths: if not os.path.isfile(filepath): # Sys exit rather than raise for better command line experience print(f"ERROR: {filepath} does not exist!") From 3b563a9987418e6c6e779265ff6c2d390094405d Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Wed, 30 Jul 2025 14:48:09 +0100 Subject: [PATCH 6/7] Improve hardware specific tests README --- hardware-specific-tests/README.md | 40 +++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/hardware-specific-tests/README.md b/hardware-specific-tests/README.md index 7e6e83e9..57b02f8f 100644 --- a/hardware-specific-tests/README.md +++ b/hardware-specific-tests/README.md @@ -1,4 +1,16 @@ -## Hardware specific tests for the PiCamera. +# Hardware specific tests + +This directory contains unit tests that require specific hardware to run. Current they implement tests for code that interacts with the Raspberry Pi Camera. + +## Hardware specific tests for the Pi Camera. + +The OpenFlexure Microscope Server runs automated test in the cloud (via the GitLab CI) to test each version of the code as changes are proposed and merged. As these tests run in the cloud they do not have access to actual microscope hardware. As such, these tests either test specific blocks of code that don't need hardware access or make use of simulated hardware via the simulation camera and dummy stage. + +To test the code that interacts with the hardware itself, hardware specific tests are needed and they must be run with the hardware available. This directory provides these tests. For now the only hardware specific tests are for the Pi Camera. They must be run on a Raspberry Pi with a Pi Camera attached. + +In GitLab we track our "code coverage", this tells us which lines of code have been executed during testing. The `picamera_tests.py` script in the root of the repository can be used to report the coverage from these tests. See the section below on "Reporting the coverage to GitLab". + +### Running these tests during development These are very slow as they run on hardware. They can be run with: @@ -14,13 +26,33 @@ When writing and debugging these unit tests it is often best to run a specific t pytest hardware-specific-tests/picamera2/test_exposure_time_drift.py::test_exposure_time_saves_and_loads -s --pdb -### Reporting the coverage +### Reporting the coverage to GitLab -To create a coverage report that will be included into the repository run: +To create a coverage report that will be included into the repository (and reported to GitLab) run: ./picamera_tests.py run -This will create a zip of test results along with hashes of the picamera files and the base camera file. The server will include this coverage report if the source files remain the same. +This will first run `pytest` on the hardware specific tests. This creates a `.coverage` database that can be used to analyse the code coverage of these tests. This will be copied to `.coverage.picamera`. The script will then create a zip of these test results along with hashes of source code for the Pi Camera: + +* `picamera.py` +* `picamera_recalibrate_utils.py` +* `__init__.py` in the `camera` directory that defines `BaseCamera` + +This zip should then be committed to the repository. + +When the CI runs on GitLab to calculate code coverage. It will first run the tests in the `tests` directory, in one job and archive this as `.coverage.main`, and then it will run a second job that: + +* Imports the archive of `.coverage.main` for the tests just run on the server +* Unzip the zip of the results of running tests on the Pi Camera (`.coverage.picamera`) +* Check that the hashes for the Pi Camera source code have not changed. If they have changed it will error and ask for the picamera tests to be re-run on a Raspberry Pi. +* It will then run `coverage combine` to create a single `.coverage` report that combines the coverage from both `.coverage.main` and `.coverage.picamera`. +* It then generates the information needed to display the coverage in GitLab + +This ensures that: + +* Hardware specific tests are re-run if the relevant source code changes. +* That the coverage for hardware specific tests is reported correctly +* That the hardware specific tests do not need re-running when other code that does not affect PiCamera interaction is updated. ### Creating a combined report locally From d9db73a8d9ff9350bc30c3ff9a62f4be9118db6e Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Wed, 30 Jul 2025 15:12:21 +0100 Subject: [PATCH 7/7] Fix coverage combination when other sources are removed --- .gitlab-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 244f5025..b9e9b873 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -148,8 +148,9 @@ combined-tests: script: - ./picamera_tests.py unzip - coverage combine - - coverage report - - coverage xml + # Use -i to ignore file changing between the two runs. We check the relevant ones. + - coverage report -i + - coverage xml -i coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/' artifacts: when: always