feat(benchmark): initial setup

benchmark/benchmark.py

@@ -1,33 +1,151 @@
-import glob
+import logging
 import os
+import platform
+import re
+import subprocess
+import sys
 from importlib.metadata import version
 
+import PyOpenColorIO
+import pyopencl as cl
 
-def latest_project() -> str | None:
-    project_path = os.path.dirname(__file__)
-    files = glob.glob(os.path.join(project_path, 'benchmark*.json'))
-    if files:
-        project = sorted(files)[-1]
-        return project
+from realflare.api.tasks import opencl
+from markdownTable import markdownTable
 
+import realflare
 
-def hardware_specs():
-    # should be queried from opencl
-    pass
 
+def markdown_table(data: dict, headers: tuple[str, str]) -> str:
+    if not data:
+        return ''
+    data_list = [
+        {f' {headers[0]} ': f' {name} ', f' {headers[1]} ': f' {value} '}
+        for name, value in data.items()
+    ]
+    table = (
+        markdownTable(data_list)
+        .setParams(
+            row_sep='markdown',
+            quote=False,
+            padding_weight='right',
+        )
+        .getMarkdown()
+    )
+    return table
 
-def run():
-    if 'REALFLARE_REBUILD' in os.environ:
-        del os.environ['REALFLARE_REBUILD']
 
-    versions = {
-        'realflare': version('realflare'),
+def build_report(args: list, output: str):
+
+    # hardware
+    queue = opencl.queue()
+    device = queue.device
+
+    hardware = {'processor': platform.processor(), 'OpenCL Device': device.name}
+
+    for name in (
+        'MAX_COMPUTE_UNITS',
+        'MAX_WORK_GROUP_SIZE',
+        'LOCAL_MEM_SIZE',
+        'GLOBAL_MEM_SIZE',
+        'MAX_CONSTANT_BUFFER_SIZE',
+    ):
+        parameter = getattr(cl.device_info, name)
+        hardware[name] = device.get_info(parameter)
+
+    hardware_table = markdown_table(hardware, ('Name', 'Value'))
+
+    # software
+
+    # using __version__ ensures the correct version for editable installs
+    realflare_version = realflare.__version__
+    software = {
+        'platform': platform.platform(),
+        'opencl': device.get_info(cl.device_info.OPENCL_C_VERSION),
+        'python': platform.python_version(),
+        'realflare': realflare_version,
         'qt_extensions': version('qt_extensions'),
-        'PySide2': version('PySide2'),
-        'numpy': version('numpy'),
         'pyopencl': version('pyopencl'),
-        'PyOpenColorIO': version('PyOpenColorIO'),
+        'numpy': version('numpy'),
+        'PyOpenColorIO': PyOpenColorIO.__version__,
+        'PySide2': version('PySide2'),
+    }
+    software_table = markdown_table(software, ('Name', 'Version'))
+
+    # realflare
+    project_path = args[-1]
+    package_path = os.path.dirname(os.path.dirname(__file__))
+    args[0] = os.path.relpath(args[0], package_path)
+    args[-1] = os.path.relpath(args[-1], package_path)
+    command = '`' + ' '.join(args) + '`'
+
+    # score
+    score = {}
+    pattern = re.compile(r'([\w.]+):\s*(\d+(?:\.\d+)?ms)$')
+    for line in output.split('\n'):
+        match = pattern.search(line.strip())
+        if match:
+            func = match.group(1)
+            time = match.group(2)
+            if func == 'Engine.render':
+                func = f'**{func}**'
+                time = f'**{time}**'
+            score[func] = time
+
+    score_table = markdown_table(score, ('Function', 'Time'))
+
+    fields = {
+        'version': f'v{realflare_version}',
+        'hardware_table': hardware_table,
+        'software_table': software_table,
+        'command': command,
+        'score_table': score_table,
     }
 
-    with open('/output/output.md', 'w') as f:
-        f.write('hello')
+    # read template
+    template_path = os.path.join(os.path.dirname(__file__), 'report_template.md')
+    with open(template_path, 'r') as f:
+        report = f.read()
+
+    # create text
+    for key, value in fields.items():
+        placeholder = f'<!--{key}-->'
+        report = report.replace(placeholder, value)
+
+    report_name = f'report_v{realflare_version}.md'
+    project_dir = os.path.dirname(project_path)
+    report_path = os.path.join(project_dir, report_name)
+    with open(report_path, 'w') as f:
+        f.write(report)
+
+
+def run(name: str = 'nikon_ai_50_135mm'):
+
+    # set environment variables
+    env = os.environ.copy()
+    if 'REALFLARE_DEV' in env:
+        del env['REALFLARE_DEV']
+    if 'REALFLARE_REBUILD' in env:
+        del env['REALFLARE_REBUILD']
+
+    project_path = os.path.join(os.path.dirname(__file__), name, 'project.json')
+
+    command = [sys.executable, '-m', 'realflare', '--project', project_path]
+
+    output = subprocess.check_output(command, env=env, stderr=subprocess.STDOUT)
+    # process = processing.popen(command, env=env)
+    # output = []
+    # while process.poll() is None:
+    #     output.append(process.stdout.readline())
+    # process.stdout.close()
+    #
+    # return_code = process.returncode
+    # if return_code:
+    #     logging.error(subprocess.CalledProcessError(return_code, command))
+
+    build_report(command, output.decode('utf-8'))
+    # return return_code
+
+
+if __name__ == '__main__':
+    logging.getLogger().setLevel(logging.DEBUG)
+    run()

benchmark/projects/benchmark_v0.0.1.json → benchmark/nikon_ai_50_135mm/project.json

@@ -175,7 +175,7 @@
     }
   },
   "render": {
-    "output_path": "D:/files/dev/027_flare/render/wavelengths_v002.exr",
+    "output_path": "",
     "colorspace": "ACES - ACEScg",
     "quality": {
       "resolution": [
@@ -222,4 +222,4 @@
   "elements": [
     "FLARE"
   ]
-}
+}

benchmark/nikon_ai_50_135mm/report_v0.1.0.md

@@ -0,0 +1,45 @@
+# Benchmark Report v0.1.0
+
+## Score
+
+| Function                      | Time            |
+|-------------------------------|-----------------|
+| ApertureTask.run              | 2.151ms         |
+| StarburstTask.run             | 11.171ms        |
+| GhostTask.run                 | 491.196ms       |
+| RaytracingTask.run            | 3282.587ms      |
+| RasterizingTask.prim_shader   | 41.878ms        |
+| RasterizingTask.vertex_shader | 38.800ms        |
+| RasterizingTask.binner        | 66.161ms        |
+| RasterizingTask.rasterizer    | 21586.057ms     |
+| RasterizingTask.rasterize     | 21801.390ms     |
+| **Engine.render**             | **28699.693ms** |
+
+## Software
+
+| Name          | Version                   |
+|---------------|---------------------------|
+| platform      | Windows-10-10.0.19044-SP0 |
+| opencl        | OpenCL C 1.2              |
+| python        | 3.10.10                   |
+| realflare     | 0.1.0                     |
+| qt_extensions | 0.1.3                     |
+| pyopencl      | 2022.3.1                  |
+| numpy         | 1.24.2                    |
+| PyOpenColorIO | 2.2.1                     |
+| PySide2       | 5.15.2.1                  |
+
+## Hardware
+| Name                     | Value                                              |
+|--------------------------|----------------------------------------------------|
+| processor                | Intel64 Family 6 Model 79 Stepping 1, GenuineIntel |
+| OpenCL Device            | NVIDIA GeForce GTX 1080 Ti                         |
+| MAX_COMPUTE_UNITS        | 28                                                 |
+| MAX_WORK_GROUP_SIZE      | 1024                                               |
+| LOCAL_MEM_SIZE           | 49152                                              |
+| GLOBAL_MEM_SIZE          | 11810897920                                        |
+| MAX_CONSTANT_BUFFER_SIZE | 65536                                              |
+
+## Realflare
+
+`venv\Scripts\python.exe -m realflare --project benchmark\nikon_ai_50_135mm\project.json`

benchmark/report_template.md

@@ -0,0 +1,16 @@
+# Benchmark Report <!--version-->
+
+## Score
+
+<!--score_table-->
+
+## Software
+
+<!--software_table-->
+
+## Hardware
+<!--hardware_table-->
+
+## Realflare
+
+<!--command-->

pyproject.toml

@@ -27,7 +27,8 @@ classifiers = [
 [project.optional-dependencies]
 dev = [
     "mkdocs-material",
-    "python-semantic-release"
+    "python-semantic-release",
+    "py-markdown-table"
 ]
 
 [build-system]

realflare/__main__.py

@@ -29,7 +29,6 @@ def parse_args(args):
     parser.add_argument(
         '--output',
         type=str,
-        default='render.$F4.exr',
         help='Output image',
     )
     parser.add_argument(

realflare/api/engine.py

@@ -178,6 +178,10 @@ def write_image(
         array: np.ndarray | None = None,
         colorspace: str = 'ACES - ACEScg',
     ) -> None:
+        if not output_path:
+            logging.warning('no output path specified')
+            return
+
         # TODO: clean up, image/array etc
         if array is None:
             if image is None:
@@ -189,9 +193,6 @@ def write_image(
                     )
             array = image.array
 
-        if not output_path:
-            raise ValueError('output_path cannot be empty')
-
         if not os.path.isdir(os.path.dirname(output_path)):
             os.makedirs(os.path.dirname(output_path))
 
@@ -204,6 +205,9 @@ def write_image(
 
     @staticmethod
     def parse_output_path(path: str, frame: int) -> str:
+        if not path:
+            # abspath assumes '' as relative path
+            return path
         path = re.sub(r'\$F(\d)?', r'{:0\g<1>d}', path)
         path = path.format(frame)
         path = os.path.abspath(path)

realflare/utils/timing.py

@@ -10,7 +10,7 @@ def wrapper(*args, **kwargs):
         result = func(*args, **kwargs)
         label = f'{func.__qualname__}:'
         ms = (time.perf_counter() - start_time) * 1000
-        logging.info(f'{label: <40}{ms:9.3f}')
+        logging.info(f'{label: <40}{ms:9.3f}ms')
         return result
 
     return wrapper
@@ -25,4 +25,4 @@ def __enter__(self):
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         ms = (time.perf_counter() - self.start_time) * 1000
-        logging.info(f'{self.label: <40}{ms:9.3f}')
+        logging.info(f'{self.label: <40}{ms:9.3f}ms')