diff --git a/docs/src/benchcomp-conf.md b/docs/src/benchcomp-conf.md index 77236d0917bf..de57ac831221 100644 --- a/docs/src/benchcomp-conf.md +++ b/docs/src/benchcomp-conf.md @@ -4,6 +4,26 @@ This page lists the different visualizations that are available. +## Variants + +A *variant* is a single invocation of a benchmark suite. Benchcomp runs several +variants, so that their performance can be compared later. A variant consists of +a command-line argument, working directory, and environment. Benchcomp invokes +the command using the operating system environment, updated with the keys and +values in `env`. If any values in `env` contain strings of the form `${var}`, +Benchcomp expands them to the value of the environment variable `$var`. + +```yaml +variants: + variant_1: + config: + command_line: echo "Hello, world" + directory: /tmp + env: + PATH: /my/local/directory:${PATH} +``` + + ## Built-in visualizations The following visualizations are available; these can be added to the `visualize` list of `benchcomp.yaml`. diff --git a/tools/benchcomp/benchcomp/entry/run.py b/tools/benchcomp/benchcomp/entry/run.py index a870e7e9a1b0..28457381da2b 100644 --- a/tools/benchcomp/benchcomp/entry/run.py +++ b/tools/benchcomp/benchcomp/entry/run.py @@ -13,6 +13,7 @@ import logging import os import pathlib +import re import shutil import subprocess import typing @@ -53,9 +54,10 @@ def __post_init__(self): else: self.working_copy = pathlib.Path(self.directory) + def __call__(self): - env = dict(os.environ) - env.update(self.env) + update_environment_with = _EnvironmentUpdater() + env = update_environment_with(self.env) if self.copy_benchmarks_dir: shutil.copytree( @@ -128,6 +130,44 @@ def __call__(self): tmp_symlink.rename(self.out_symlink) + +@dataclasses.dataclass +class _EnvironmentUpdater: + """Update the OS environment with keys and values containing variables + + When called, this class returns the operating environment updated with new + keys and values. The values can contain variables of the form '${var_name}'. + The class evaluates those variables using values already in the environment. + """ + + os_environment: dict = dataclasses.field( + default_factory=lambda : dict(os.environ)) + pattern: re.Pattern = re.compile(r"\$\{(\w+?)\}") + + + def _evaluate(self, key, value): + """Evaluate all ${var} in value using self.os_environment""" + old_value = value + + for variable in re.findall(self.pattern, value): + if variable not in self.os_environment: + logging.error( + "Couldn't evaluate ${%s} in the value '%s' for environment " + "variable '%s'. Ensure the environment variable $%s is set", + variable, old_value, key, variable) + sys.exit(1) + value = re.sub( + r"\$\{" + variable + "\}", self.os_environment[variable], value) + return value + + + def __call__(self, new_environment): + ret = dict(self.os_environment) + for key, value in new_environment.items(): + ret[key] = self._evaluate(key, value) + return ret + + def get_default_out_symlink(): return "latest" diff --git a/tools/benchcomp/test/test_regression.py b/tools/benchcomp/test/test_regression.py index 87df67a071cc..51f9dbb597db 100644 --- a/tools/benchcomp/test/test_regression.py +++ b/tools/benchcomp/test/test_regression.py @@ -646,6 +646,43 @@ def test_return_0_on_fail(self): result = yaml.safe_load(handle) + def test_env_expansion(self): + """Ensure that config parser expands '${}' in env key""" + + with tempfile.TemporaryDirectory() as tmp: + run_bc = Benchcomp({ + "variants": { + "env_set": { + "config": { + "command_line": 'echo "$__BENCHCOMP_ENV_VAR" > out', + "directory": tmp, + "env": {"__BENCHCOMP_ENV_VAR": "foo:${PATH}"} + } + }, + }, + "run": { + "suites": { + "suite_1": { + "parser": { + # The word 'bin' typically appears in $PATH, so + # check that what was echoed contains 'bin'. + "command": textwrap.dedent("""\ + grep bin out && grep '^foo:' out && echo '{ + "benchmarks": {}, + "metrics": {} + }' + """) + }, + "variants": ["env_set"] + } + } + }, + "visualize": [], + }) + run_bc() + self.assertEqual(run_bc.proc.returncode, 0, msg=run_bc.stderr) + + def test_env(self): """Ensure that benchcomp reads the 'env' key of variant config""" diff --git a/tools/benchcomp/test/test_unit.py b/tools/benchcomp/test/test_unit.py new file mode 100644 index 000000000000..12320116f217 --- /dev/null +++ b/tools/benchcomp/test/test_unit.py @@ -0,0 +1,66 @@ +# Copyright Kani Contributors +# SPDX-License-Identifier: Apache-2.0 OR MIT +# +# Benchcomp regression testing suite. This suite uses Python's stdlib unittest +# module, but nevertheless actually runs the binary rather than running unit +# tests. + +import unittest +import uuid + +import benchcomp.entry.run + + + +class TestEnvironmentUpdater(unittest.TestCase): + def test_environment_construction(self): + """Test that the default constructor reads the OS environment""" + + update_environment = benchcomp.entry.run._EnvironmentUpdater() + environment = update_environment({}) + self.assertIn("PATH", environment) + + + def test_placeholder_construction(self): + """Test that the placeholder constructor reads the placeholder""" + + key, value = [str(uuid.uuid4()) for _ in range(2)] + update_environment = benchcomp.entry.run._EnvironmentUpdater({ + key: value, + }) + environment = update_environment({}) + self.assertIn(key, environment) + self.assertEqual(environment[key], value) + + + def test_environment_update(self): + """Test that the environment is updated""" + + key, value, update = [str(uuid.uuid4()) for _ in range(3)] + update_environment = benchcomp.entry.run._EnvironmentUpdater({ + key: value, + }) + environment = update_environment({ + key: update + }) + self.assertIn(key, environment) + self.assertEqual(environment[key], update) + + + def test_environment_update_variable(self): + """Test that the environment is updated""" + + old_env = { + "key1": str(uuid.uuid4()), + "key2": str(uuid.uuid4()), + } + + actual_update = "${key2}xxx${key1}" + expected_update = f"{old_env['key2']}xxx{old_env['key1']}" + + update_environment = benchcomp.entry.run._EnvironmentUpdater(old_env) + environment = update_environment({ + "key1": actual_update, + }) + self.assertIn("key1", environment) + self.assertEqual(environment["key1"], expected_update)