From 590a9a2796636f96facd0209542bc23f904dc445 Mon Sep 17 00:00:00 2001 From: Matthew Kosarek Date: Tue, 26 Sep 2023 13:20:38 -0400 Subject: [PATCH] Benchmarker implementation based on a cgroups backend --- .gitignore | 1 + codecov.yml | 3 + mir-ci/mir_ci/apps.py | 22 ++- mir-ci/mir_ci/benchmarker.py | 88 +++++++++++ mir-ci/mir_ci/cgroups.py | 74 ++++++++++ mir-ci/mir_ci/clients/drag_and_drop_demo.py | 7 +- mir-ci/mir_ci/conftest.py | 14 +- mir-ci/mir_ci/display_server.py | 21 ++- mir-ci/mir_ci/interfaces/benchmarkable.py | 15 ++ .../mir_ci/interfaces/benchmarker_backend.py | 22 +++ mir-ci/mir_ci/program.py | 37 ++++- mir-ci/mir_ci/pytest.ini | 3 + mir-ci/mir_ci/test_apps_can_run.py | 15 +- mir-ci/mir_ci/test_drag_and_drop.py | 4 +- mir-ci/mir_ci/test_screencopy_bandwidth.py | 8 +- mir-ci/mir_ci/test_tests.py | 139 +++++++++++++++++- 16 files changed, 428 insertions(+), 45 deletions(-) create mode 100644 mir-ci/mir_ci/benchmarker.py create mode 100644 mir-ci/mir_ci/cgroups.py create mode 100644 mir-ci/mir_ci/interfaces/benchmarkable.py create mode 100644 mir-ci/mir_ci/interfaces/benchmarker_backend.py diff --git a/.gitignore b/.gitignore index 7e888290..6bb1eca6 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ __pycache__ *.pyc mir-ci/mir_ci/junit.xml .python-version +venv diff --git a/codecov.yml b/codecov.yml index 53b2086e..8bd59aaa 100644 --- a/codecov.yml +++ b/codecov.yml @@ -6,3 +6,6 @@ coverage: patch: default: target: 0% + +ignore: + - "mir-ci/mir_ci/interfaces" \ No newline at end of file diff --git a/mir-ci/mir_ci/apps.py b/mir-ci/mir_ci/apps.py index 0b408701..3613b7dd 100644 --- a/mir-ci/mir_ci/apps.py +++ b/mir-ci/mir_ci/apps.py @@ -1,9 +1,20 @@ import pytest -from typing import Any, Optional, Union, Collection +from typing import Any, Optional, Union, Collection, Literal + +AppType = Literal["snap", "deb", "pip"] + +class App: + command: Collection[str] + app_type: Optional[AppType] + + def __init__(self, command: Collection[str], app_type: Optional[AppType] = None) -> None: + self.command = command + self.app_type = app_type def _dependency( cmd: Collection[str], + app_type: AppType, snap: Optional[str] = None, channel: str = 'latest/stable', debs: Collection[str] = (), @@ -20,14 +31,15 @@ def _dependency( ret = (ret, extra) return pytest.param( - ret, + App(ret, app_type), marks=( # type: ignore pytest.mark.deps( cmd=cmd, snap=snap, debs=debs, pip_pkgs=pip_pkgs, - channel=channel), + channel=channel, + app_type=app_type), *marks), id=id) @@ -40,6 +52,7 @@ def snap( return _dependency( cmd=cmd or (snap, *args), + app_type="snap", snap=snap, id=id or snap, **kwargs @@ -52,9 +65,9 @@ def deb( debs: Collection[str] = (), id: Optional[str] = None, **kwargs): - return _dependency( cmd=cmd or (deb, *args), + app_type="deb", debs=debs or (deb,), id=id or deb, **kwargs) @@ -67,6 +80,7 @@ def pip( **kwargs): return _dependency( pip_pkgs=(pkg,), + app_type="pip", cmd=cmd or ('python3', '-m', pkg, *args), id=id or pkg, **kwargs) diff --git a/mir-ci/mir_ci/benchmarker.py b/mir-ci/mir_ci/benchmarker.py new file mode 100644 index 00000000..0d6c29f0 --- /dev/null +++ b/mir-ci/mir_ci/benchmarker.py @@ -0,0 +1,88 @@ +import asyncio +import logging +from typing import Dict, Callable, Optional +from contextlib import suppress + +from mir_ci.interfaces.benchmarkable import Benchmarkable +from mir_ci.interfaces.benchmarker_backend import BenchmarkBackend + + +logger = logging.getLogger(__name__) + + +class Benchmarker: + def __init__(self, programs: Dict[str, Benchmarkable], poll_time_seconds: float = 1.0): + self.programs = programs + self.backend: BenchmarkBackend = CgroupsBackend() + self.poll_time_seconds = poll_time_seconds + self.task: Optional[asyncio.Task[None]] = None + self.running: bool = False + + async def _run(self) -> None: + while self.running: + await self.backend.poll() + await asyncio.sleep(self.poll_time_seconds) + + async def __aenter__(self): + if self.running is True: + return self + + self.running = True + for program_id, program in self.programs.items(): + await program.__aenter__() + self.backend.add(program_id, program) + + self.task = asyncio.ensure_future(self._run()) + return self + + async def __aexit__(self, *args): + if self.running is False: + return + + self.running = False + if self.task: + self.task.cancel() + with suppress(asyncio.CancelledError): + await self.task + + for program_id, program in self.programs.items(): + await program.__aexit__() + + def generate_report(self, record_property: Callable[[str, object], None]) -> None: + report = self.backend.generate_report() + for key, value in report.items(): + record_property(key, value) + + +class CgroupsBackend(BenchmarkBackend): + class ProcessInfo: + program: Benchmarkable + cpu_time_microseconds: int = 0 + mem_bytes_accumulator: int = 0 + mem_bytes_max: int = 0 + num_data_points: int = 0 + + def __init__(self, program: Benchmarkable) -> None: + self.program = program + + def __init__(self) -> None: + self.data_records: Dict[str, CgroupsBackend.ProcessInfo] = {} + + def add(self, name: str, program: Benchmarkable) -> None: + self.data_records[name] = CgroupsBackend.ProcessInfo(program) + + async def poll(self) -> None: + for name, info in self.data_records.items(): + cgroup = await info.program.get_cgroup() + self.data_records[name].cpu_time_microseconds = cgroup.get_cpu_time_microseconds() + self.data_records[name].mem_bytes_accumulator += cgroup.get_current_memory() + self.data_records[name].mem_bytes_max = cgroup.get_peak_memory() + self.data_records[name].num_data_points += 1 + + def generate_report(self) -> Dict[str, object]: + result: Dict[str, object] = {} + for name, info in self.data_records.items(): + result[f"{name}_cpu_time_microseconds"] = info.cpu_time_microseconds + result[f"{name}_max_mem_bytes"] = info.mem_bytes_max + result[f"{name}_avg_mem_bytes"] = 0 if info.num_data_points == 0 else info.mem_bytes_accumulator / info.num_data_points + return result diff --git a/mir-ci/mir_ci/cgroups.py b/mir-ci/mir_ci/cgroups.py new file mode 100644 index 00000000..034c8029 --- /dev/null +++ b/mir-ci/mir_ci/cgroups.py @@ -0,0 +1,74 @@ +import os +from typing import Iterator, TYPE_CHECKING +import pathlib +import asyncio + +if TYPE_CHECKING: + CreateReturnType = asyncio.Task["Cgroup"] +else: + CreateReturnType = asyncio.Task + +class Cgroup: + def __init__(self, pid: int, path: pathlib.Path) -> None: + self.pid = pid + self.path = path + + @staticmethod + def create(pid: int) -> CreateReturnType: + async def inner(): + path = await Cgroup.get_cgroup_dir(pid) + return Cgroup(pid, path) + task = asyncio.create_task(inner()) + return task + + @staticmethod + async def get_cgroup_dir(pid: int) -> pathlib.Path: + MAX_ATTEMPTS = 10 + parent_path = Cgroup._get_cgroup_dir_internal(os.getpid()) + path = Cgroup._get_cgroup_dir_internal(pid) + + for attempt in range(MAX_ATTEMPTS): + await asyncio.sleep(0.1) + path = Cgroup._get_cgroup_dir_internal(pid) + if path != parent_path: + return path + else: + raise RuntimeError(f"Unable to read cgroup directory for pid: {pid}") + + @staticmethod + def _get_cgroup_dir_internal(pid: int) -> pathlib.Path: + cgroup_file = f"/proc/{pid}/cgroup" + + with open(cgroup_file, "r") as group_file: + for line in group_file.readlines(): + assert line.startswith("0::"), f"Line in cgroup file does not start with 0:: for pid: {pid}" + return pathlib.Path(f"/sys/fs/cgroup/{line[3:]}".strip()) + raise RuntimeError(f"Unable to find path for process with pid: {pid}") + + def _read_file(self, file_name: str) -> Iterator[str]: + file_path = f"{self.path}/{file_name}" + with open(file_path, "r") as file: + yield file.readline() + + def get_cpu_time_microseconds(self) -> int: + try: + for line in self._read_file("cpu.stat"): + split_line = line.split(' ') + if split_line[0] == "usage_usec": + return int(split_line[1]) + + raise RuntimeError("usage_usec line not found") + except Exception as ex: + raise RuntimeError(f"Unable to get the cpu time for cgroup with pid: {self.pid}") from ex + + def get_current_memory(self) -> int: + try: + return int(next(self._read_file("memory.current"))) + except Exception as ex: + raise RuntimeError(f"Unable to get the current memory for cgroup with pid: {self.pid}") from ex + + def get_peak_memory(self) -> int: + try: + return int(next(self._read_file("memory.peak"))) + except Exception as ex: + raise RuntimeError(f"Unable to get the peak memory for cgroup with pid: {self.pid}") from ex diff --git a/mir-ci/mir_ci/clients/drag_and_drop_demo.py b/mir-ci/mir_ci/clients/drag_and_drop_demo.py index 0f10e890..9af8c9e8 100644 --- a/mir-ci/mir_ci/clients/drag_and_drop_demo.py +++ b/mir-ci/mir_ci/clients/drag_and_drop_demo.py @@ -1,5 +1,8 @@ import gi import sys +import logging + +logger = logging.getLogger(__name__) gi.require_version("Gtk", "3.0") from gi.repository import Gtk, Gdk, GdkPixbuf @@ -43,7 +46,7 @@ def __init__(self, source_mode, target_mode, expect): def result_callback(self, result): self.result = result if self.expect != EXCHANGE_TYPE_NONE: - print("expect=", self.expect, ", actual=", self.result) + logger.info("expect=", self.expect, ", actual=", self.result) exit(self.result != self.expect) class DragSourceIconView(Gtk.IconView): @@ -197,7 +200,7 @@ def on_drag_data_received(self, widget, drag_context, x, y, data, info, time): else: assert False, f'invalid argument: {arg}' except Exception as e: - print('Argument error:', str(e)) + logger.error('Argument error:', str(e)) exit(1) win = DragDropWindow(source_mode=source_mode, target_mode=target_mode, expect=expect) win.connect("destroy", Gtk.main_quit) diff --git a/mir-ci/mir_ci/conftest.py b/mir-ci/mir_ci/conftest.py index 8ad9938d..18334960 100644 --- a/mir-ci/mir_ci/conftest.py +++ b/mir-ci/mir_ci/conftest.py @@ -44,7 +44,7 @@ def _deps_skip(request: pytest.FixtureRequest) -> None: if request.keywords['depfixtures'] == DEP_FIXTURES.intersection(request.fixturenames): pytest.skip('dependency-only run') -def _deps_install(request: pytest.FixtureRequest, spec: Union[str, Mapping[str, Any]]) -> List[str]: +def _deps_install(request: pytest.FixtureRequest, spec: Union[str, Mapping[str, Any]]) -> apps.App: ''' Install dependencies for the command spec provided. If `spec` is a string, it's assumed to be a snap and command name. @@ -62,12 +62,14 @@ def _deps_install(request: pytest.FixtureRequest, spec: Union[str, Mapping[str, snap: Optional[str] = spec.get('snap') channel: str = spec.get('channel', 'latest/stable') pip_pkgs: tuple[str, ...] = spec.get('pip_pkgs', ()) + app_type: Optional[apps.AppType] = spec.get('app_type') elif isinstance(spec, str): cmd = [spec] debs = None snap = spec channel = 'latest/stable' pip_pkgs = () + app_type = "snap" else: raise TypeError('Bad value for argument `spec`: ' + repr(spec)) @@ -111,7 +113,7 @@ def _deps_install(request: pytest.FixtureRequest, spec: Union[str, Mapping[str, _deps_skip(request) - return cmd + return apps.App(cmd, app_type) @pytest.fixture(scope='session') def ppa() -> None: @@ -133,7 +135,7 @@ def ppa() -> None: apps.mir_test_tools, apps.mir_demo_server, )) -def server(request: pytest.FixtureRequest) -> List[str]: +def server(request: pytest.FixtureRequest) -> apps.App: ''' Parameterizes the servers (ubuntu-frame, mir-kiosk, confined-shell, mir_demo_server), or installs them if `--deps` is given on the command line. @@ -143,9 +145,9 @@ def server(request: pytest.FixtureRequest) -> List[str]: return _deps_install(request, request.param().marks[0].kwargs) @pytest.fixture(scope='function') -def deps(request: pytest.FixtureRequest) -> List[str]: +def deps(request: pytest.FixtureRequest) -> Optional[apps.App]: ''' - Ensures the dependenciesa are available, or installs them if `--deps` is given on the command line. + Ensures the dependencies are available, or installs them if `--deps` is given on the command line. You need to provide data through the `deps` mark: ``` @@ -162,7 +164,7 @@ def deps(request: pytest.FixtureRequest) -> List[str]: closest: pytest.Mark = next(marks) except StopIteration: _deps_skip(request) - return [] + return None for mark in request.node.iter_markers('deps'): _deps_install(request, mark.kwargs and dict({'cmd': mark.args}, **mark.kwargs) or mark.args[0]) diff --git a/mir-ci/mir_ci/display_server.py b/mir-ci/mir_ci/display_server.py index 93d0d528..d208f9e9 100644 --- a/mir-ci/mir_ci/display_server.py +++ b/mir-ci/mir_ci/display_server.py @@ -3,9 +3,13 @@ import time import asyncio -from typing import Dict, Tuple +from typing import Dict, Tuple, Optional from mir_ci.program import Program, Command +from mir_ci.apps import AppType +from mir_ci.interfaces.benchmarkable import Benchmarkable +from mir_ci.cgroups import Cgroup +from mir_ci.apps import App display_appear_timeout = 10 min_mir_run_time = 0.1 @@ -33,17 +37,20 @@ def wait_for_wayland_display(runtime_dir: str, name: str) -> None: return raise RuntimeError('Wayland display ' + name + ' did not appear') -class DisplayServer: - def __init__(self, command: Command, add_extensions: Tuple[str, ...] = ()) -> None: - self.command = command +class DisplayServer(Benchmarkable): + def __init__(self, app: App, add_extensions: Tuple[str, ...] = ()) -> None: + self.app: App = app self.add_extensions = add_extensions # Snaps require the display to be in the form "waland-". The 00 prefix lets us # easily identify displays created by this test suit and remove them in bulk if a bunch # don't get cleaned up properly. self.display_name = 'wayland-00' + str(os.getpid()) - def program(self, command: Command, env: Dict[str, str] = {}) -> Program: - return Program(command, env=dict({ + async def get_cgroup(self) -> Cgroup: + return await self.server.get_cgroup() + + def program(self, app: App, env: Dict[str, str] = {}) -> Program: + return Program(app, env=dict({ 'DISPLAY': 'no', 'QT_QPA_PLATFORM': 'wayland', 'WAYLAND_DISPLAY': self.display_name @@ -54,7 +61,7 @@ def program(self, command: Command, env: Dict[str, str] = {}) -> Program: async def __aenter__(self) -> 'DisplayServer': runtime_dir = os.environ['XDG_RUNTIME_DIR'] clear_wayland_display(runtime_dir, self.display_name) - self.server = await Program(self.command, env={ + self.server = await Program(self.app, env={ 'WAYLAND_DISPLAY': self.display_name, 'MIR_SERVER_ADD_WAYLAND_EXTENSIONS': ':'.join(self.add_extensions), }).__aenter__() diff --git a/mir-ci/mir_ci/interfaces/benchmarkable.py b/mir-ci/mir_ci/interfaces/benchmarkable.py new file mode 100644 index 00000000..50a7212b --- /dev/null +++ b/mir-ci/mir_ci/interfaces/benchmarkable.py @@ -0,0 +1,15 @@ +from abc import ABC, abstractmethod +from mir_ci.cgroups import Cgroup + +class Benchmarkable(ABC): + @abstractmethod + async def get_cgroup(self) -> Cgroup: + raise NotImplementedError + + @abstractmethod + async def __aenter__(self): + raise NotImplementedError + + @abstractmethod + async def __aexit__(self, *args): + raise NotImplementedError diff --git a/mir-ci/mir_ci/interfaces/benchmarker_backend.py b/mir-ci/mir_ci/interfaces/benchmarker_backend.py new file mode 100644 index 00000000..34ec4a17 --- /dev/null +++ b/mir-ci/mir_ci/interfaces/benchmarker_backend.py @@ -0,0 +1,22 @@ +from mir_ci.interfaces.benchmarkable import Benchmarkable +from typing import Dict +from abc import ABC, abstractmethod + +class BenchmarkBackend(ABC): + """ + Abstract class that aggregates programs together and emits process stats as it is requested + """ + @abstractmethod + def add(self, name: str, program: Benchmarkable) -> None: + """ + Add a process to be benchmarked. + """ + raise NotImplementedError + + @abstractmethod + async def poll(self) -> None: + raise NotImplementedError + + @abstractmethod + def generate_report(self) -> Dict[str, object]: + raise NotImplementedError diff --git a/mir-ci/mir_ci/program.py b/mir-ci/mir_ci/program.py index 978942be..fc6d70b7 100644 --- a/mir-ci/mir_ci/program.py +++ b/mir-ci/mir_ci/program.py @@ -2,6 +2,14 @@ import signal from typing import Dict, List, Tuple, Union, Optional, Awaitable import asyncio +import logging +import uuid + +logger = logging.getLogger(__name__) + +from mir_ci.apps import App +from mir_ci.interfaces.benchmarkable import Benchmarkable +from mir_ci.cgroups import Cgroup default_wait_timeout = default_term_timeout = 10 @@ -27,12 +35,13 @@ def format_output(name: str, output: str) -> str: class ProgramError(RuntimeError): pass -class Program: - def __init__(self, command: Command, env: Dict[str, str] = {}): - if isinstance(command, str): - self.command: tuple[str, ...] = (command,) +class Program(Benchmarkable): + def __init__(self, app: App, env: Dict[str, str] = {}): + if isinstance(app.command, str): + self.command: tuple[str, ...] = (app.command,) else: - self.command = tuple(command) + self.command = tuple(app.command) + self.name = self.command[0] self.env = env self.process: Optional[asyncio.subprocess.Process] = None @@ -40,10 +49,15 @@ def __init__(self, command: Command, env: Dict[str, str] = {}): self.send_signals_task: Optional[asyncio.Task[None]] = None self.output = '' self.sigkill_sent = False + self.with_systemd_run = app.app_type == "deb" or app.app_type == "pip" def is_running(self) -> bool: return self.process is not None and self.process.returncode is None + async def get_cgroup(self) -> Cgroup: + await self.cgroups_task + return self.cgroups_task.result() + async def send_kill_signals(self, timeout: int, term_timeout: int) -> None: '''Assigned to self.send_signals_task, cancelled when process ends''' assert self.is_running(), self.name + ' is dead' @@ -82,8 +96,13 @@ async def kill(self, timeout=default_term_timeout) -> None: pass async def __aenter__(self) -> 'Program': + command = self.command + if self.with_systemd_run is True: + slice = f"mirci-{uuid.uuid4()}" + prefix = ("systemd-run", "--user", "--quiet", "--scope", f"--slice={slice}") + command = (*prefix, *command) process = await asyncio.create_subprocess_exec( - *self.command, + *command, env=dict(os.environ, **self.env), stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT, @@ -91,8 +110,6 @@ async def __aenter__(self) -> 'Program': preexec_fn=os.setsid) # Without setsid killing the subprocess doesn't kill the whole process tree, # see https://pymotw.com/2/subprocess/#process-groups-sessions - # Without setsid killing the subprocess doesn't kill the whole process tree, - # see https://pymotw.com/2/subprocess/#process-groups-sessions async def communicate() -> None: raw_output, _ = await process.communicate() if self.send_signals_task is not None and not self.send_signals_task.done(): @@ -100,9 +117,13 @@ async def communicate() -> None: self.output = raw_output.decode('utf-8').strip() self.process = process self.process_end = communicate() + self.cgroups_task = Cgroup.create(process.pid) return self async def __aexit__(self, *args) -> None: + if self.cgroups_task: + self.cgroups_task.cancel() + if self.process_end is not None: assert self.is_running(), self.name + ' died without being waited for or killed' await self.kill() diff --git a/mir-ci/mir_ci/pytest.ini b/mir-ci/mir_ci/pytest.ini index ed71a931..21b7d9d0 100644 --- a/mir-ci/mir_ci/pytest.ini +++ b/mir-ci/mir_ci/pytest.ini @@ -8,3 +8,6 @@ markers = xdg: mark tests usefixtures = deps xdg asyncio_mode=auto +log_cli = True +log_format = %(asctime)s %(levelname)s %(message)s +log_date_format = %Y-%m-%d %H:%M:%S \ No newline at end of file diff --git a/mir-ci/mir_ci/test_apps_can_run.py b/mir-ci/mir_ci/test_apps_can_run.py index 24bec36b..71f14c2f 100644 --- a/mir-ci/mir_ci/test_apps_can_run.py +++ b/mir-ci/mir_ci/test_apps_can_run.py @@ -1,9 +1,10 @@ from mir_ci import SLOWDOWN from mir_ci.display_server import DisplayServer import pytest -import time +import asyncio from mir_ci import apps +from mir_ci.benchmarker import Benchmarker short_wait_time = 3 * SLOWDOWN @@ -17,7 +18,11 @@ class TestAppsCanRun: apps.pluma(), apps.qterminal(), ]) - async def test_app_can_run(self, server, app) -> None: - async with DisplayServer(server) as server: - async with server.program(app): - time.sleep(short_wait_time) + async def test_app_can_run(self, server, app, record_property) -> None: + server_instance = DisplayServer(server) + program = server_instance.program(app) + benchmarker = Benchmarker({"compositor": server_instance, "client": program}, poll_time_seconds=0.1) + async with benchmarker: + await asyncio.sleep(short_wait_time) + + benchmarker.generate_report(record_property) diff --git a/mir-ci/mir_ci/test_drag_and_drop.py b/mir-ci/mir_ci/test_drag_and_drop.py index 3507e4b0..c2abf85d 100644 --- a/mir-ci/mir_ci/test_drag_and_drop.py +++ b/mir-ci/mir_ci/test_drag_and_drop.py @@ -29,7 +29,7 @@ class TestDragAndDrop: async def test_source_and_dest_match(self, modern_server, app) -> None: modern_server = DisplayServer(modern_server, add_extensions=VirtualPointer.required_extensions) pointer = VirtualPointer(modern_server.display_name) - program = modern_server.program(app) + program = modern_server.program(apps.App(app)) async with modern_server, program, pointer: await asyncio.sleep(STARTUP_TIME) @@ -54,7 +54,7 @@ async def test_source_and_dest_match(self, modern_server, app) -> None: async def test_source_and_dest_mismatch(self, modern_server, app) -> None: modern_server = DisplayServer(modern_server, add_extensions=VirtualPointer.required_extensions) pointer = VirtualPointer(modern_server.display_name) - program = modern_server.program(app) + program = modern_server.program(apps.App(app)) async with modern_server, program, pointer: await asyncio.sleep(STARTUP_TIME) diff --git a/mir-ci/mir_ci/test_screencopy_bandwidth.py b/mir-ci/mir_ci/test_screencopy_bandwidth.py index 2fe8369a..6a27b855 100644 --- a/mir-ci/mir_ci/test_screencopy_bandwidth.py +++ b/mir-ci/mir_ci/test_screencopy_bandwidth.py @@ -35,9 +35,9 @@ class TestScreencopyBandwidth: async def test_active_app(self, record_property, server, app) -> None: server = DisplayServer(server, add_extensions=ScreencopyTracker.required_extensions) tracker = ScreencopyTracker(server.display_name) - async with server as s, tracker, s.program(app[0]) as p: - if app[1]: - await asyncio.wait_for(p.wait(timeout=app[1]), timeout=app[1] + 1) + async with server as s, tracker, s.program(apps.App(app.command[0], app.app_type)) as p: + if app.command[1]: + await asyncio.wait_for(p.wait(timeout=app.command[1]), timeout=app.command[1] + 1) else: await asyncio.sleep(long_wait_time) _record_properties(record_property, server, tracker, 10) @@ -73,7 +73,7 @@ async def pause(): extensions = ScreencopyTracker.required_extensions + VirtualPointer.required_extensions app_path = Path(__file__).parent / 'clients' / 'maximizing_gtk_app.py' server = DisplayServer(local_server, add_extensions=extensions) - app = server.program(('python3', str(app_path))) + app = server.program(apps.App(('python3', str(app_path)))) tracker = ScreencopyTracker(server.display_name) pointer = VirtualPointer(server.display_name) async with server, tracker, app, pointer: diff --git a/mir-ci/mir_ci/test_tests.py b/mir-ci/mir_ci/test_tests.py index 809704f6..41808c69 100644 --- a/mir-ci/mir_ci/test_tests.py +++ b/mir-ci/mir_ci/test_tests.py @@ -1,10 +1,15 @@ import os -import subprocess -import time import pytest import asyncio +import time +import subprocess +from unittest.mock import Mock, MagicMock, patch, mock_open from mir_ci.program import Program +from mir_ci.benchmarker import Benchmarker +from mir_ci.cgroups import Cgroup +from mir_ci.apps import App, ubuntu_frame +from mir_ci.display_server import DisplayServer class TestTest: @pytest.mark.self @@ -13,8 +18,9 @@ def test_project_typechecks(self, deps) -> None: from mir_ci.protocols import WlOutput, WlShm, ZwlrScreencopyManagerV1 # noqa:F401 project_path = os.path.dirname(__file__) assert os.path.isfile(os.path.join(project_path, 'pytest.ini')), 'project path not detected correctly' + command = deps.command result = subprocess.run( - [*deps, project_path], + [*command, project_path], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) @@ -24,14 +30,14 @@ def test_project_typechecks(self, deps) -> None: @pytest.mark.self class TestProgram: async def test_program_gives_output(self) -> None: - p = Program(['printf', '%s - %s', 'abc', 'xyz']) + p = Program(App(['printf', '%s - %s', 'abc', 'xyz'])) async with p: await p.wait() assert p.output == 'abc - xyz' async def test_program_can_be_waited_for(self) -> None: start = time.time() - p = Program(['sh', '-c', 'sleep 1; echo abc']) + p = Program(App(['sh', '-c', 'sleep 1; echo abc'])) async with p: await p.wait() elapsed = time.time() - start @@ -40,7 +46,7 @@ async def test_program_can_be_waited_for(self) -> None: async def test_program_can_be_terminated(self) -> None: start = time.time() - p = Program(['sh', '-c', 'echo abc; sleep 1; echo ijk']) + p = Program(App(['sh', '-c', 'echo abc; sleep 1; echo ijk'])) async with p: await asyncio.sleep(0.5) await p.kill(2) @@ -50,10 +56,129 @@ async def test_program_can_be_terminated(self) -> None: async def test_program_is_killed_when_terminate_fails(self) -> None: start = time.time() - p = Program(['sh', '-c', 'trap "" TERM; echo abc; sleep 1; echo ijk; sleep 5; echo xyz']) + p = Program(App(['sh', '-c', 'trap "" TERM; echo abc; sleep 1; echo ijk; sleep 5; echo xyz'])) async with p: await asyncio.sleep(0.5) await p.kill(2) elapsed = time.time() - start assert p.output.strip() == 'abc\nijk' assert abs(elapsed - 2.5) < 0.1 + + @patch("uuid.uuid4") + async def test_program_runs_with_systemd_when_flag_is_set(self, mock_uuid) -> None: + mock_uuid.return_value = "12345" + start = time.time() + p = Program(App(['sh', '-c', 'sleep 1'], "deb")) + async with p: + await asyncio.sleep(0.5) + await p.kill(2) + mock_uuid.assert_called_once() + + async def test_program_can_get_cgroup(self) -> None: + p = Program(App(['sh', '-c', 'sleep 100'], "deb")) + async with p: + cgroup = await p.get_cgroup() + assert cgroup is not None + await p.kill(2) + + +@pytest.mark.self +class TestBenchmarker: + @staticmethod + def create_program_mock(): + def async_return(): + f = asyncio.Future() + f.set_result(MagicMock()) + return f + p = MagicMock() + p.get_cgroup = Mock(return_value=async_return()) + return p + + async def test_benchmarker_with_program(self) -> None: + p = TestBenchmarker.create_program_mock() + benchmarker = Benchmarker({"program": p}, poll_time_seconds=0.1) + async with benchmarker: + await asyncio.sleep(1) + + p.get_cgroup.assert_called() + p.__aenter__.assert_called_once() + p.__aexit__.assert_called_once() + + async def test_benchmarker_can_generate_report(self) -> None: + p = TestBenchmarker.create_program_mock() + benchmarker = Benchmarker({"program": p}, poll_time_seconds=0.1) + async with benchmarker: + await asyncio.sleep(1) + + callback = Mock() + benchmarker.generate_report(callback) + callback.assert_called() + + async def test_benchmarker_cant_enter_twice(self) -> None: + p = TestBenchmarker.create_program_mock() + benchmarker = Benchmarker({"program": p}, poll_time_seconds=0.1) + async with benchmarker: + async with benchmarker: + await asyncio.sleep(1) + + p.__aenter__.assert_called_once() + +@pytest.mark.self +class TestCgroup: + @patch("builtins.open", new_callable=mock_open, read_data="usage_usec 100\nline_two 30\nline_three 40\nline_four 50") + def test_cgroup_can_get_cpu_time_microseconds(self, mock_open): + cgroup = Cgroup(12345, "/fake/path") + assert(cgroup.get_cpu_time_microseconds() == 100) + + @patch("builtins.open", new_callable=mock_open, read_data="usage_usec string") + def test_cgroup_get_cpu_time_microseconds_raises_when_not_integer(self, mock_open): + cgroup = Cgroup(12345, "/fake/path") + with pytest.raises(RuntimeError, match="Unable to get the cpu time for cgroup with pid: 12345"): + cgroup.get_cpu_time_microseconds() + + @patch("builtins.open", new_callable=mock_open, read_data="100") + def test_cgroup_get_cpu_time_microseconds_raises_when_usage_usec_not_found(self, mock_open): + cgroup = Cgroup(12345, "/fake/path") + with pytest.raises(RuntimeError, match="Unable to get the cpu time for cgroup with pid: 12345"): + cgroup.get_cpu_time_microseconds() + + @patch("builtins.open", new_callable=mock_open, read_data="100") + def test_cgroup_can_get_current_memory(self, mock_open): + cgroup = Cgroup(12345, "/fake/path") + assert(cgroup.get_current_memory() == 100) + + @patch("builtins.open", new_callable=mock_open, read_data="string") + def test_cgroup_get_current_memory_raises_when_not_integer(self, mock_open): + cgroup = Cgroup(12345, "/fake/path") + with pytest.raises(RuntimeError, match="Unable to get the current memory for cgroup with pid: 12345"): + cgroup.get_current_memory() + + @patch("builtins.open", new_callable=mock_open, read_data="100") + def test_cgroup_can_get_peak_memory(self, mock_open): + cgroup = Cgroup(12345, "/fake/path") + assert(cgroup.get_peak_memory() == 100) + + @patch("builtins.open", new_callable=mock_open, read_data="string") + def test_cgroup_get_peak_memory_raises_when_not_integer(self, mock_open): + cgroup = Cgroup(12345, "/fake/path") + with pytest.raises(RuntimeError, match="Unable to get the peak memory for cgroup with pid: 12345"): + cgroup.get_peak_memory() + + @patch("builtins.open", new_callable=mock_open, read_data="string") + async def test_cgroup_path_raises_assertion_error_when_contents_are_incorrect(self, mock_open): + with pytest.raises(AssertionError, match=f"Line in cgroup file does not start with 0:: for pid: {os.getpid()}"): + await Cgroup.get_cgroup_dir(12345) + + @patch("builtins.open", new_callable=mock_open) + async def test_cgroup_path_raises_runtime_error_when_contents_are_none(self, mock_open): + with pytest.raises(RuntimeError, match=f"Unable to find path for process with pid: {os.getpid()}"): + await Cgroup.get_cgroup_dir(12345) + + +@pytest.mark.self +class TestDisplayServer: + async def test_can_get_cgroup(self, server): + server_instance = DisplayServer(server) + async with server_instance: + cgroup = await server_instance.get_cgroup() + assert cgroup is not None