Benchmarking for the DisplayServer and its spawned application

.gitignore

@@ -2,3 +2,4 @@ __pycache__
 *.pyc
 mir-ci/mir_ci/junit.xml
 .python-version
+venv

codecov.yml

@@ -6,3 +6,6 @@ coverage:
     patch:
       default:
         target: 0%
+
+ignore:
+  - "mir-ci/mir_ci/interfaces"

mir-ci/mir_ci/apps.py

@@ -1,9 +1,20 @@
 import pytest
 
-from typing import Any, Optional, Union, Collection
+from typing import Any, Optional, Union, Collection, Literal
+
+AppType = Literal["snap", "deb", "pip"]
+
+class App:
+    command: Collection[str]
+    app_type: Optional[AppType]
+
+    def __init__(self, command: Collection[str], app_type: Optional[AppType] = None) -> None:
+        self.command = command
+        self.app_type = app_type
 
 def _dependency(
         cmd: Collection[str],
+        app_type: AppType,
         snap: Optional[str] = None,
         channel: str = 'latest/stable',
         debs: Collection[str] = (),
@@ -20,14 +31,15 @@ def _dependency(
         ret = (ret, extra)
 
     return pytest.param(
-        ret,
+        App(ret, app_type),
         marks=(                             # type: ignore
             pytest.mark.deps(
                 cmd=cmd,
                 snap=snap,
                 debs=debs,
                 pip_pkgs=pip_pkgs,
-                channel=channel),
+                channel=channel,
+                app_type=app_type),
             *marks),
         id=id)
 
@@ -40,6 +52,7 @@ def snap(
 
     return _dependency(
         cmd=cmd or (snap, *args),
+        app_type="snap",
         snap=snap,
         id=id or snap,
         **kwargs
@@ -52,9 +65,9 @@ def deb(
         debs: Collection[str] = (),
         id: Optional[str] = None,
         **kwargs):
-
     return _dependency(
         cmd=cmd or (deb, *args),
+        app_type="deb",
         debs=debs or (deb,),
         id=id or deb,
         **kwargs)
@@ -67,6 +80,7 @@ def pip(
         **kwargs):
     return _dependency(
         pip_pkgs=(pkg,),
+        app_type="pip",
         cmd=cmd or ('python3', '-m', pkg, *args),
         id=id or pkg,
         **kwargs)

mir-ci/mir_ci/benchmarker.py

@@ -0,0 +1,88 @@
+import asyncio
+import logging
+from typing import Dict, Callable, Optional
+from contextlib import suppress
+
+from mir_ci.interfaces.benchmarkable import Benchmarkable
+from mir_ci.interfaces.benchmarker_backend import BenchmarkBackend
+
+
+logger = logging.getLogger(__name__)
+
+
+class Benchmarker:
+    def __init__(self, programs: Dict[str, Benchmarkable], poll_time_seconds: float = 1.0):
+        self.programs = programs
+        self.backend: BenchmarkBackend = CgroupsBackend()
+        self.poll_time_seconds = poll_time_seconds
+        self.task: Optional[asyncio.Task[None]] = None
+        self.running: bool = False
+
+    async def _run(self) -> None:
+        while self.running:
+            await self.backend.poll()
+            await asyncio.sleep(self.poll_time_seconds)
+
+    async def __aenter__(self):
+        if self.running is True:
+            return self
+
+        self.running = True
+        for program_id, program in self.programs.items():
+            await program.__aenter__()
+            self.backend.add(program_id, program)
+
+        self.task = asyncio.ensure_future(self._run())
+        return self
+
+    async def __aexit__(self, *args):
+        if self.running is False:
+            return
+
+        self.running = False
+        if self.task:
+            self.task.cancel()
+            with suppress(asyncio.CancelledError):
+                await self.task
+
+        for program_id, program in self.programs.items():
+            await program.__aexit__()
+
+    def generate_report(self, record_property: Callable[[str, object], None]) -> None:
+        report = self.backend.generate_report()
+        for key, value in report.items():
+            record_property(key, value)
+
+
+class CgroupsBackend(BenchmarkBackend):
+    class ProcessInfo:
+        program: Benchmarkable
+        cpu_time_microseconds: int = 0
+        mem_bytes_accumulator: int = 0
+        mem_bytes_max: int = 0
+        num_data_points: int = 0
+
+        def __init__(self, program: Benchmarkable) -> None:
+            self.program = program
+
+    def __init__(self) -> None:
+        self.data_records: Dict[str, CgroupsBackend.ProcessInfo] = {}
+
+    def add(self, name: str, program: Benchmarkable) -> None:
+        self.data_records[name] = CgroupsBackend.ProcessInfo(program)
+
+    async def poll(self) -> None:
+        for name, info in self.data_records.items():
+            cgroup = await info.program.get_cgroup()
+            self.data_records[name].cpu_time_microseconds = cgroup.get_cpu_time_microseconds()
+            self.data_records[name].mem_bytes_accumulator += cgroup.get_current_memory()
+            self.data_records[name].mem_bytes_max = cgroup.get_peak_memory()
+            self.data_records[name].num_data_points += 1
+
+    def generate_report(self) -> Dict[str, object]:
+        result: Dict[str, object] = {}
+        for name, info in self.data_records.items():
+            result[f"{name}_cpu_time_microseconds"] = info.cpu_time_microseconds
+            result[f"{name}_max_mem_bytes"] = info.mem_bytes_max
+            result[f"{name}_avg_mem_bytes"] = 0 if info.num_data_points == 0 else info.mem_bytes_accumulator / info.num_data_points
+        return result

mir-ci/mir_ci/cgroups.py

@@ -0,0 +1,74 @@
+import os
+from typing import Iterator, TYPE_CHECKING
+import pathlib
+import asyncio
+
+if TYPE_CHECKING:
+    CreateReturnType = asyncio.Task["Cgroup"]
+else:
+    CreateReturnType = asyncio.Task
+
+class Cgroup:
+    def __init__(self, pid: int, path: pathlib.Path) -> None:
+        self.pid = pid
+        self.path = path
+
+    @staticmethod
+    def create(pid: int) -> CreateReturnType:
+        async def inner():
+            path = await Cgroup.get_cgroup_dir(pid)
+            return Cgroup(pid, path)
+        task = asyncio.create_task(inner())
+        return task
+
+    @staticmethod
+    async def get_cgroup_dir(pid: int) -> pathlib.Path:
+        MAX_ATTEMPTS = 10
+        parent_path = Cgroup._get_cgroup_dir_internal(os.getpid())
+        path = Cgroup._get_cgroup_dir_internal(pid)
+
+        for attempt in range(MAX_ATTEMPTS):
+            await asyncio.sleep(0.1)
+            path = Cgroup._get_cgroup_dir_internal(pid)
+            if path != parent_path:
+                return path
+        else:
+            raise RuntimeError(f"Unable to read cgroup directory for pid: {pid}")
+
+    @staticmethod
+    def _get_cgroup_dir_internal(pid: int) -> pathlib.Path:
+        cgroup_file = f"/proc/{pid}/cgroup"
+
+        with open(cgroup_file, "r") as group_file:
+            for line in group_file.readlines():
+                assert line.startswith("0::"), f"Line in cgroup file does not start with 0:: for pid: {pid}"
+                return pathlib.Path(f"/sys/fs/cgroup/{line[3:]}".strip())
+        raise RuntimeError(f"Unable to find path for process with pid: {pid}")
+
+    def _read_file(self, file_name: str) -> Iterator[str]:
+        file_path = f"{self.path}/{file_name}"
+        with open(file_path, "r") as file:
+            yield file.readline()
+
+    def get_cpu_time_microseconds(self) -> int:
+        try:
+            for line in self._read_file("cpu.stat"):
+                split_line = line.split(' ')
+                if split_line[0] == "usage_usec":
+                    return int(split_line[1])
+
+            raise RuntimeError("usage_usec line not found")
+        except Exception as ex:
+            raise RuntimeError(f"Unable to get the cpu time for cgroup with pid: {self.pid}") from ex
+
+    def get_current_memory(self) -> int:
+        try:
+            return int(next(self._read_file("memory.current")))
+        except Exception as ex:
+            raise RuntimeError(f"Unable to get the current memory for cgroup with pid: {self.pid}") from ex
+
+    def get_peak_memory(self) -> int:
+        try:
+            return int(next(self._read_file("memory.peak")))
+        except Exception as ex:
+            raise RuntimeError(f"Unable to get the peak memory for cgroup with pid: {self.pid}") from ex

mir-ci/mir_ci/clients/drag_and_drop_demo.py

@@ -1,5 +1,8 @@
 import gi
 import sys
+import logging
+
+logger = logging.getLogger(__name__)
 
 gi.require_version("Gtk", "3.0")
 from gi.repository import Gtk, Gdk, GdkPixbuf
@@ -43,7 +46,7 @@ def __init__(self, source_mode, target_mode, expect):
     def result_callback(self, result):
         self.result = result
         if self.expect != EXCHANGE_TYPE_NONE:
-            print("expect=", self.expect, ", actual=", self.result)
+            logger.info("expect=", self.expect, ", actual=", self.result)
             exit(self.result != self.expect)
 
 class DragSourceIconView(Gtk.IconView):
@@ -197,7 +200,7 @@ def on_drag_data_received(self, widget, drag_context, x, y, data, info, time):
             else:
                 assert False, f'invalid argument: {arg}'
     except Exception as e:
-        print('Argument error:', str(e))
+        logger.error('Argument error:', str(e))
         exit(1)
     win = DragDropWindow(source_mode=source_mode, target_mode=target_mode, expect=expect)
     win.connect("destroy", Gtk.main_quit)

mir-ci/mir_ci/conftest.py

@@ -44,7 +44,7 @@
         if request.keywords['depfixtures'] == DEP_FIXTURES.intersection(request.fixturenames):
             pytest.skip('dependency-only run')
 
-def _deps_install(request: pytest.FixtureRequest, spec: Union[str, Mapping[str, Any]]) -> List[str]:
+def _deps_install(request: pytest.FixtureRequest, spec: Union[str, Mapping[str, Any]]) -> apps.App:
     '''
     Install dependencies for the command spec provided. If `spec` is a string, it's assumed
     to be a snap and command name.
@@ -62,12 +62,14 @@
         snap: Optional[str] = spec.get('snap')
         channel: str = spec.get('channel', 'latest/stable')
         pip_pkgs: tuple[str, ...] = spec.get('pip_pkgs', ())
+        app_type: Optional[apps.AppType] = spec.get('app_type')
     elif isinstance(spec, str):
         cmd = [spec]
         debs = None
         snap = spec
         channel = 'latest/stable'
         pip_pkgs = ()
+        app_type = "snap"
     else:
         raise TypeError('Bad value for argument `spec`: ' + repr(spec))
 
@@ -111,7 +113,7 @@
 
         _deps_skip(request)
 
-    return cmd
+    return apps.App(cmd, app_type)
 
 @pytest.fixture(scope='session')
 def ppa() -> None:
@@ -133,7 +135,7 @@
     apps.mir_test_tools,
     apps.mir_demo_server,
 ))
-def server(request: pytest.FixtureRequest) -> List[str]:
+def server(request: pytest.FixtureRequest) -> apps.App:
     '''
     Parameterizes the servers (ubuntu-frame, mir-kiosk, confined-shell, mir_demo_server),
     or installs them if `--deps` is given on the command line.
@@ -143,9 +145,9 @@
     return _deps_install(request, request.param().marks[0].kwargs)
 
 @pytest.fixture(scope='function')
-def deps(request: pytest.FixtureRequest) -> List[str]:
+def deps(request: pytest.FixtureRequest) -> Optional[apps.App]:
     '''
-    Ensures the dependenciesa are available, or installs them if `--deps` is given on the command line.
+    Ensures the dependencies are available, or installs them if `--deps` is given on the command line.
 
     You need to provide data through the `deps` mark:
     ```
@@ -162,7 +164,7 @@
         closest: pytest.Mark = next(marks)
     except StopIteration:
         _deps_skip(request)
-        return []
+        return None
 
     for mark in request.node.iter_markers('deps'):
         _deps_install(request, mark.kwargs and dict({'cmd': mark.args}, **mark.kwargs) or mark.args[0])

mir-ci/mir_ci/display_server.py

@@ -3,9 +3,13 @@
 import time
 import asyncio
 
-from typing import Dict, Tuple
+from typing import Dict, Tuple, Optional
 
 from mir_ci.program import Program, Command
+from mir_ci.apps import AppType
+from mir_ci.interfaces.benchmarkable import Benchmarkable
+from mir_ci.cgroups import Cgroup
+from mir_ci.apps import App
 
 display_appear_timeout = 10
 min_mir_run_time = 0.1
@@ -33,17 +37,20 @@
             return
     raise RuntimeError('Wayland display ' + name + ' did not appear')
 
-class DisplayServer:
-    def __init__(self, command: Command, add_extensions: Tuple[str, ...] = ()) -> None:
-        self.command = command
+class DisplayServer(Benchmarkable):
+    def __init__(self, app: App, add_extensions: Tuple[str, ...] = ()) -> None:
+        self.app: App = app
         self.add_extensions = add_extensions
         # Snaps require the display to be in the form "waland-<number>". The 00 prefix lets us
         # easily identify displays created by this test suit and remove them in bulk if a bunch
         # don't get cleaned up properly.
         self.display_name = 'wayland-00' + str(os.getpid())
 
-    def program(self, command: Command, env: Dict[str, str] = {}) -> Program:
-        return Program(command, env=dict({
+    async def get_cgroup(self) -> Cgroup:
+        return await self.server.get_cgroup()
+
+    def program(self, app: App, env: Dict[str, str] = {}) -> Program:
+        return Program(app, env=dict({
                 'DISPLAY': 'no',
                 'QT_QPA_PLATFORM': 'wayland',
                 'WAYLAND_DISPLAY': self.display_name
@@ -54,7 +61,7 @@
     async def __aenter__(self) -> 'DisplayServer':
         runtime_dir = os.environ['XDG_RUNTIME_DIR']
         clear_wayland_display(runtime_dir, self.display_name)
-        self.server = await Program(self.command, env={
+        self.server = await Program(self.app, env={
             'WAYLAND_DISPLAY': self.display_name,
             'MIR_SERVER_ADD_WAYLAND_EXTENSIONS': ':'.join(self.add_extensions),
         }).__aenter__()

mir-ci/mir_ci/interfaces/benchmarkable.py

@@ -0,0 +1,15 @@
+from abc import ABC, abstractmethod
+from mir_ci.cgroups import Cgroup
+
+class Benchmarkable(ABC):
+    @abstractmethod
+    async def get_cgroup(self) -> Cgroup:
+        raise NotImplementedError
+
+    @abstractmethod
+    async def __aenter__(self):
+        raise NotImplementedError
+
+    @abstractmethod
+    async def __aexit__(self, *args):
+        raise NotImplementedError