From bd96585825e3793879a091d3ebc16bcf7d32b19f Mon Sep 17 00:00:00 2001 From: Julian Stirling Date: Wed, 9 Jul 2025 12:59:14 +0100 Subject: [PATCH] Create failing test for graceful shutdown. --- integration-tests/testfile.py | 110 ++++++++++++++++++++++++++++++---- 1 file changed, 99 insertions(+), 11 deletions(-) diff --git a/integration-tests/testfile.py b/integration-tests/testfile.py index d767945d..d07ce7ef 100755 --- a/integration-tests/testfile.py +++ b/integration-tests/testfile.py @@ -2,7 +2,10 @@ """This module fires up a server in a subprocess to allow for integration tests."" These are seperated from the unit tests to stop them artificially inflating the test -coverage. For now this file should be run directly not with a test framework +coverage. For now this file should be run directly not with a test framework. + +These tests are designed to run on CI. They should work on Linux or WSL for local +debugging. """ import subprocess @@ -10,12 +13,15 @@ import os import shutil from time import sleep +from PIL import Image + +import labthings_fastapi as lt THIS_DIR = os.path.dirname(os.path.realpath(__file__)) WORKING_DIR = os.path.join(THIS_DIR, "working_dir") REPO_DIR = os.path.dirname(THIS_DIR) CONFIG_FILE = os.path.join(REPO_DIR, "ofm_config_simulation.json") -CMD = ["openflexure-microscope-server", "--fallback", "-c", CONFIG_FILE] +SERVER_CMD = ["openflexure-microscope-server", "--fallback", "-c", CONFIG_FILE] def main(): @@ -30,15 +36,71 @@ def main(): sleep(15) error_if_server_not_started(server_process) + test_client_connection() + + # This also sleeps for 2s and checks it is still connected + subscriber_process = subscribe_to_mjpeg_stream() + + server_process.terminate() + sleep(3) + + check_for_graceful_shudown(server_process) + finally: - # Ensure the server process is really dead + # Ensure the server and subscriber processes are really dead + if subscriber_process.poll() is None: + subscriber_process.kill() if server_process.poll() is None: - print("Server still running, shutting down.") - server_process.terminate() - sleep(2) - if server_process.poll() is None: - server_process.kill() - raise RuntimeError("Server needed killing at end of test.") + server_process.kill() + raise RuntimeError("Server needed killing at end of test.") + + +def test_client_connection() -> None: + """Check a ThingClient can interact with the simulation microscope camera""" + print("Connecting Python client to microscope, and capturing image") + cam_client = lt.ThingClient.from_url("http://localhost:5000/camera/") + img = Image.open(cam_client.grab_jpeg().open()) + print(f"Successfully grabbed image of size {img.size}") + assert img.size == (800, 600) + print("Successfully grabbed image from camera mjpeg stream") + + +def subscribe_to_mjpeg_stream() -> subprocess.Popen: + """Start a background process subscribed to the mjpeg stream. + + :return: The Popen object for the ongoing process. + + :raises: RuntimeError if the stream is not still connected after 2s + """ + # Use nohup to stop process hanging up unexpectedly. Explicitly forward stream + # to /dev/null to keep curl connected. + + curl_command = [ + "nohup", + "curl", + "-s", + "http://localhost:5000/camera/mjpeg_stream", + ">", + "/dev/null", + "2>&1", + ] + + process = subprocess.Popen( + curl_command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + bufsize=1, + universal_newlines=True, + ) + os.set_blocking(process.stdout.fileno(), False) + + sleep(2) + if process.poll() is not None: + raise RuntimeError( + "MJPEG Subscriber process is not running. This likley means it could not" + " stay connected to the stream.\n" + ) + return process def set_up_working_dir() -> None: @@ -58,7 +120,7 @@ def start_server() -> subprocess.Popen: :return: Popen object for the ongoing process """ process = subprocess.Popen( - CMD, + SERVER_CMD, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=1, @@ -75,6 +137,10 @@ def error_if_server_not_started(server_process: subprocess.Popen) -> None: Check the subprocess is running Check the logs have no errors Read the Logs to check uvicorn is running on http://127.0.0.1:5000 + + :param server_process: The Popen object for the the server process. + + :rasies RuntimeError: If the server is not running as expected. """ stdout, stderr = read_process_buffers(server_process) @@ -83,7 +149,7 @@ def error_if_server_not_started(server_process: subprocess.Popen) -> None: raise RuntimeError(f"Server process is not running!\n{output}") confirmed_uvicorn_is_running = False - # Note that logs go to stderr once OFM logging process stars, but if there is an + # Note that logs go to stderr once OFM logging process starts, but if there is an # error very early (such as on import) on logs may go to stdout for line in stdout + stderr: if line.startswith("ERROR:"): @@ -101,6 +167,28 @@ def error_if_server_not_started(server_process: subprocess.Popen) -> None: print(output) +def check_for_graceful_shudown(server_process: subprocess.Popen) -> None: + """Check the server shutdown gracefully + + Check the subprocess is not running + Check the logs have no errors + Read the Logs to check the shutdown was graceful, rather than killed on Uvicorn + timeout + + :param server_process: The Popen object for the the server process. + + :rasies RuntimeError: If the server didn't shutdown gracefully. + """ + stdout, stderr = read_process_buffers(server_process) + output = format_stdout_stderr(stdout, stderr) + for line in stdout + stderr: + if line.startswith("ERROR:"): + raise RuntimeError(f"Server encountered error\n{line}\n\n{output}") + if "graceful shutdown exceeded" in line: + # This should be logged as an ERROR. This check is belts and braces. + raise RuntimeError(f"Server failed to shutdown gracefully.\n\n{output}") + + def read_process_buffers(process: subprocess.Popen) -> tuple[list[str], list[str]]: """Return STDOUT and STDERR from a process""" stdout = []