From 8b9846d9a9d29b427860891c9b699eb4305e348b Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Wed, 20 Nov 2024 08:58:10 +0100 Subject: [PATCH] mtest: move determine_worker_count to utils, generalize It is useful to apply a limit to the number of processes even outside "meson test", and specifically for clang tools. In preparation for this, generalize determine_worker_count() to accept a variable MESON_NUM_PROCESSES instead of MESON_TESTTHREADS, and use it throughout instead of multiprocessing.cpu_count(). Signed-off-by: Paolo Bonzini --- docs/markdown/Unit-tests.md | 8 +++++--- docs/markdown/snippets/num-processes.md | 7 +++++++ mesonbuild/compilers/mixins/gnu.py | 6 +++--- mesonbuild/mtest.py | 27 +++---------------------- mesonbuild/scripts/externalproject.py | 5 ++--- mesonbuild/utils/universal.py | 27 +++++++++++++++++++++++++ 6 files changed, 47 insertions(+), 33 deletions(-) create mode 100644 docs/markdown/snippets/num-processes.md diff --git a/docs/markdown/Unit-tests.md b/docs/markdown/Unit-tests.md index 898366095b05..13f6093f2e7b 100644 --- a/docs/markdown/Unit-tests.md +++ b/docs/markdown/Unit-tests.md @@ -83,16 +83,18 @@ possible. By default Meson uses as many concurrent processes as there are cores on the test machine. You can override this with the environment -variable `MESON_TESTTHREADS` like this. +variable `MESON_TESTTHREADS` or, *since 1.7.0*, `MESON_NUM_PROCESSES`: ```console -$ MESON_TESTTHREADS=5 meson test +$ MESON_NUM_PROCESSES=5 meson test ``` -Setting `MESON_TESTTHREADS` to 0 enables the default behavior (core +Setting `MESON_NUM_PROCESSES` to 0 enables the default behavior (core count), whereas setting an invalid value results in setting the job count to 1. +If both environment variables are present, `MESON_NUM_PROCESSES` prevails. + ## Priorities *(added in version 0.52.0)* diff --git a/docs/markdown/snippets/num-processes.md b/docs/markdown/snippets/num-processes.md new file mode 100644 index 000000000000..e4ffdd1dc5c1 --- /dev/null +++ b/docs/markdown/snippets/num-processes.md @@ -0,0 +1,7 @@ +## Control the number of child processes with an environment variable + +Previously, `meson test` checked the `MESON_TESTTHREADS` variable to control +the amount of parallel jobs to run; this was useful when `meson test` is +invoked through `ninja test` for example. With this version, a new variable +`MESON_NUM_PROCESSES` is supported with a broader scope: in addition to +`meson test`, it is also used by the `external_project` module. diff --git a/mesonbuild/compilers/mixins/gnu.py b/mesonbuild/compilers/mixins/gnu.py index 21a57b44fef1..976fa78714c6 100644 --- a/mesonbuild/compilers/mixins/gnu.py +++ b/mesonbuild/compilers/mixins/gnu.py @@ -8,7 +8,6 @@ import abc import functools import os -import multiprocessing import pathlib import re import subprocess @@ -617,8 +616,9 @@ def get_lto_compile_args(self, *, threads: int = 0, mode: str = 'default') -> T. if threads == 0: if self._has_lto_auto_support: return ['-flto=auto'] - # This matches clang's behavior of using the number of cpus - return [f'-flto={multiprocessing.cpu_count()}'] + # This matches clang's behavior of using the number of cpus, but + # obeying meson's MESON_NUM_PROCESSES convention. + return [f'-flto={mesonlib.determine_worker_count()}'] elif threads > 0: return [f'-flto={threads}'] return super().get_lto_compile_args(threads=threads) diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py index d41c676ef110..556451c5e6ab 100644 --- a/mesonbuild/mtest.py +++ b/mesonbuild/mtest.py @@ -14,7 +14,6 @@ import datetime import enum import json -import multiprocessing import os import pickle import platform @@ -36,7 +35,8 @@ from .coredata import MesonVersionMismatchException, major_versions_differ from .coredata import version as coredata_version from .mesonlib import (MesonException, OrderedSet, RealPathAction, - get_wine_shortpath, join_args, split_args, setup_vsenv) + get_wine_shortpath, join_args, split_args, setup_vsenv, + determine_worker_count) from .options import OptionKey from .programs import ExternalProgram from .backend.backends import TestProtocol, TestSerialisation @@ -99,27 +99,6 @@ def uniwidth(s: str) -> int: result += UNIWIDTH_MAPPING[w] return result -def determine_worker_count() -> int: - varname = 'MESON_TESTTHREADS' - num_workers = 0 - if varname in os.environ: - try: - num_workers = int(os.environ[varname]) - if num_workers < 0: - raise ValueError - except ValueError: - print(f'Invalid value in {varname}, using 1 thread.') - num_workers = 1 - - if num_workers == 0: - try: - # Fails in some weird environments such as Debian - # reproducible build. - num_workers = multiprocessing.cpu_count() - except Exception: - num_workers = 1 - return num_workers - # Note: when adding arguments, please also add them to the completion # scripts in $MESONSRC/data/shell-completions/ def add_arguments(parser: argparse.ArgumentParser) -> None: @@ -154,7 +133,7 @@ def add_arguments(parser: argparse.ArgumentParser) -> None: help="Run benchmarks instead of tests.") parser.add_argument('--logbase', default='testlog', help="Base name for log file.") - parser.add_argument('-j', '--num-processes', default=determine_worker_count(), type=int, + parser.add_argument('-j', '--num-processes', default=determine_worker_count(['MESON_TESTTHREADS']), type=int, help='How many parallel processes to use.') parser.add_argument('-v', '--verbose', default=False, action='store_true', help='Do not redirect stdout and stderr') diff --git a/mesonbuild/scripts/externalproject.py b/mesonbuild/scripts/externalproject.py index ce49fbcbf26e..4013b0acf233 100644 --- a/mesonbuild/scripts/externalproject.py +++ b/mesonbuild/scripts/externalproject.py @@ -5,12 +5,11 @@ import os import argparse -import multiprocessing import subprocess from pathlib import Path import typing as T -from ..mesonlib import Popen_safe, split_args +from ..mesonlib import Popen_safe, split_args, determine_worker_count class ExternalProject: def __init__(self, options: argparse.Namespace): @@ -48,7 +47,7 @@ def supports_jobs_flag(self) -> bool: def build(self) -> int: make_cmd = self.make.copy() if self.supports_jobs_flag(): - make_cmd.append(f'-j{multiprocessing.cpu_count()}') + make_cmd.append(f'-j{determine_worker_count()}') rc = self._run('build', make_cmd) if rc != 0: return rc diff --git a/mesonbuild/utils/universal.py b/mesonbuild/utils/universal.py index 8012dcee3d12..ea49a065a446 100644 --- a/mesonbuild/utils/universal.py +++ b/mesonbuild/utils/universal.py @@ -13,6 +13,7 @@ import stat import time import abc +import multiprocessing import platform, subprocess, operator, os, shlex, shutil, re import collections from functools import lru_cache, wraps @@ -94,6 +95,7 @@ class _VerPickleLoadable(Protocol): 'default_sysconfdir', 'detect_subprojects', 'detect_vcs', + 'determine_worker_count', 'do_conf_file', 'do_conf_str', 'do_replacement', @@ -1086,6 +1088,31 @@ def default_sysconfdir() -> str: return 'etc' +def determine_worker_count(varnames: T.Optional[T.List[str]] = None) -> int: + num_workers = 0 + varnames = varnames or [] + # Add MESON_NUM_PROCESSES last, so it will prevail if more than one + # variable is present. + varnames.append('MESON_NUM_PROCESSES') + for varname in varnames: + if varname in os.environ: + try: + num_workers = int(os.environ[varname]) + if num_workers < 0: + raise ValueError + except ValueError: + print(f'Invalid value in {varname}, using 1 thread.') + num_workers = 1 + + if num_workers == 0: + try: + # Fails in some weird environments such as Debian + # reproducible build. + num_workers = multiprocessing.cpu_count() + except Exception: + num_workers = 1 + return num_workers + def has_path_sep(name: str, sep: str = '/\\') -> bool: 'Checks if any of the specified @sep path separators are in @name' for each in sep: