diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py index 9ec932e5d..0a17218db 100644 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -59,7 +59,7 @@ def __exit__( self.foutput.close() @contextlib.contextmanager - def set_env(self, *env_varss: dict[str, str] | None) -> Iterator[None]: + def set_env(self, *env_varss: Env_VarsType) -> Iterator[None]: """Set environment variables. All the arguments are dicts of name:value, or None. All are applied @@ -116,12 +116,13 @@ def run_command(self, cmd: str) -> str: return output.strip() -def rmrf(path: Path) -> None: +def remake(path: Path) -> None: """ - Remove a directory tree. It's OK if it doesn't exist. + Remove a directory tree and recreate it. It's OK if it doesn't exist. """ if path.exists(): shutil.rmtree(path) + path.mkdir(parents=True) @contextlib.contextmanager @@ -204,8 +205,9 @@ def shell(self) -> ShellSession: def make_dir(self) -> None: self.dir = Path(f"work_{self.slug}") - if self.dir.exists(): - rmrf(self.dir) + remake(self.dir) + self.tmpdir = Path(f"tmp_{self.slug}").resolve() + remake(self.tmpdir) def get_source(self, shell: ShellSession, retries: int = 5) -> None: """Get the source of the project.""" @@ -277,11 +279,18 @@ def run_with_coverage(self, env: Env, cov_ver: Coverage) -> float: class ToxProject(ProjectToTest): """A project using tox to run the test suite.""" + ALLOWABLE_ENV_VARS = [ + "COVERAGE_DEBUG", + "COVERAGE_CORE", + "COVERAGE_FORCE_CONFIG", + "PATH", + "TMPDIR", + ] + env_vars: Env_VarsType = { **(ProjectToTest.env_vars or {}), # Allow some environment variables into the tox execution. - "TOX_OVERRIDE": "testenv.pass_env+=COVERAGE_DEBUG,COVERAGE_CORE,COVERAGE_FORCE_CONFIG", - "COVERAGE_DEBUG": "config,sys", + "TOX_OVERRIDE": "testenv.pass_env+=" + ",".join(ALLOWABLE_ENV_VARS), } def prep_environment(self, env: Env) -> None: @@ -423,22 +432,19 @@ def __init__(self, more_pytest_args: str = ""): def prep_environment(self, env: Env) -> None: env.shell.run_command(f"{env.python} -m pip install tox") - Path("/tmp/operator_tmp").mkdir(exist_ok=True) env.shell.run_command(f"{env.python} -m tox -e unit --notest") env.shell.run_command(f"{env.python} -m tox -e unitnocov --notest") def run_no_coverage(self, env: Env) -> float: env.shell.run_command( - f"TMPDIR=/tmp/operator_tmp {env.python} -m tox -e unitnocov --skip-pkg-install" - + f" -- {self.more_pytest_args}" + f"{env.python} -m tox -e unitnocov --skip-pkg-install -- {self.more_pytest_args}" ) return env.shell.last_duration def run_with_coverage(self, env: Env, cov_ver: Coverage) -> float: env.shell.run_command(f"{env.python} -m pip install {cov_ver.pip_args}") env.shell.run_command( - f"TMPDIR=/tmp/operator_tmp {env.python} -m tox -e unit --skip-pkg-install" - + f" -- {self.more_pytest_args}" + f"{env.python} -m tox -e unit --skip-pkg-install -- {self.more_pytest_args}" ) duration = env.shell.last_duration report = env.shell.run_command(f"{env.python} -m coverage report --precision=6") @@ -848,18 +854,17 @@ def __init__( py_versions: list[PyVersion], cov_versions: list[Coverage], projects: list[ProjectToTest], - results_file: str = "results.json", + results_file: Path = Path("results.json"), load: bool = False, - cwd: str = "", ): self.py_versions = py_versions self.cov_versions = cov_versions self.projects = projects - self.results_file = Path(cwd) / Path(results_file) - self.result_data: dict[ResultKey, list[float]] = ( - self.load_results() if load else {} - ) + self.results_file = results_file + self.result_data: dict[ResultKey, list[float]] = {} self.summary_data: dict[ResultKey, float] = {} + if load: + self.result_data = self.load_results() def save_results(self) -> None: """Save current results to the JSON file.""" @@ -933,8 +938,13 @@ def run(self, num_runs: int = 3) -> None: ) print(banner) env.shell.print_banner(banner) + env_vars = [ + proj.env_vars, + cov_ver.env_vars, + {"TMPDIR": str(proj.tmpdir)}, + ] with change_dir(proj.dir): - with env.shell.set_env(proj.env_vars, cov_ver.env_vars): + with env.shell.set_env(*env_vars): try: if cov_ver.pip_args is None: dur = proj.run_no_coverage(env) @@ -1056,16 +1066,16 @@ def run_experiment( ) print(f"Removing and re-making {PERF_DIR}") - rmrf(PERF_DIR) + remake(PERF_DIR) - cwd = str(Path.cwd()) + results_file = Path("results.json").resolve() with change_dir(PERF_DIR): exp = Experiment( py_versions=py_versions, cov_versions=cov_versions, projects=projects, + results_file=results_file, load=load, - cwd=cwd, ) exp.run(num_runs=int(num_runs)) exp.show_results(rows=rows, column=column, ratios=ratios)