mtest: move determine_worker_count to utils, generalize

It is useful to apply a limit to the number of processes even outside "meson test",
and specifically for clang tools.  In preparation for this, generalize
determine_worker_count() to accept a variable MESON_NUM_PROCESSES instead of
MESON_TESTTHREADS, and use it throughout instead of multiprocessing.cpu_count().

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

docs/markdown/Unit-tests.md

@@ -83,16 +83,18 @@ possible.
 
 By default Meson uses as many concurrent processes as there are cores
 on the test machine. You can override this with the environment
-variable `MESON_TESTTHREADS` like this.
+variable `MESON_TESTTHREADS` or, *since 1.7.0*, `MESON_NUM_PROCESSES`:
 
 ```console
-$ MESON_TESTTHREADS=5 meson test
+$ MESON_NUM_PROCESSES=5 meson test
 ```
 
-Setting `MESON_TESTTHREADS` to 0 enables the default behavior (core
+Setting `MESON_NUM_PROCESSES` to 0 enables the default behavior (core
 count), whereas setting an invalid value results in setting the job
 count to 1.
 
+If both environment variables are present, `MESON_NUM_PROCESSES` prevails.
+
 ## Priorities
 
 *(added in version 0.52.0)*

docs/markdown/snippets/num-processes.md

@@ -0,0 +1,7 @@
+## Control the number of child processes with an environment variable
+
+Previously, `meson test` checked the `MESON_TESTTHREADS` variable to control
+the amount of parallel jobs to run; this was useful when `meson test` is
+invoked through `ninja test` for example.  With this version, a new variable
+`MESON_NUM_PROCESSES` is supported with a broader scope: in addition to
+`meson test`, it is also used by the `external_project` module.

mesonbuild/compilers/mixins/gnu.py

@@ -8,7 +8,6 @@
 import abc
 import functools
 import os
-import multiprocessing
 import pathlib
 import re
 import subprocess
@@ -617,8 +616,9 @@ def get_lto_compile_args(self, *, threads: int = 0, mode: str = 'default') -> T.
         if threads == 0:
             if self._has_lto_auto_support:
                 return ['-flto=auto']
-            # This matches clang's behavior of using the number of cpus
-            return [f'-flto={multiprocessing.cpu_count()}']
+            # This matches clang's behavior of using the number of cpus, but
+            # obeying meson's MESON_NUM_PROCESSES convention.
+            return [f'-flto={mesonlib.determine_worker_count()}']
         elif threads > 0:
             return [f'-flto={threads}']
         return super().get_lto_compile_args(threads=threads)

mesonbuild/mtest.py

@@ -14,7 +14,6 @@
 import datetime
 import enum
 import json
-import multiprocessing
 import os
 import pickle
 import platform
@@ -36,7 +35,8 @@
 from .coredata import MesonVersionMismatchException, major_versions_differ
 from .coredata import version as coredata_version
 from .mesonlib import (MesonException, OrderedSet, RealPathAction,
-                       get_wine_shortpath, join_args, split_args, setup_vsenv)
+                       get_wine_shortpath, join_args, split_args, setup_vsenv,
+                       determine_worker_count)
 from .options import OptionKey
 from .programs import ExternalProgram
 from .backend.backends import TestProtocol, TestSerialisation
@@ -99,27 +99,6 @@ def uniwidth(s: str) -> int:
         result += UNIWIDTH_MAPPING[w]
     return result
 
-def determine_worker_count() -> int:
-    varname = 'MESON_TESTTHREADS'
-    num_workers = 0
-    if varname in os.environ:
-        try:
-            num_workers = int(os.environ[varname])
-            if num_workers < 0:
-                raise ValueError
-        except ValueError:
-            print(f'Invalid value in {varname}, using 1 thread.')
-            num_workers = 1
-
-    if num_workers == 0:
-        try:
-            # Fails in some weird environments such as Debian
-            # reproducible build.
-            num_workers = multiprocessing.cpu_count()
-        except Exception:
-            num_workers = 1
-    return num_workers
-
 # Note: when adding arguments, please also add them to the completion
 # scripts in $MESONSRC/data/shell-completions/
 def add_arguments(parser: argparse.ArgumentParser) -> None:
@@ -154,7 +133,7 @@ def add_arguments(parser: argparse.ArgumentParser) -> None:
                         help="Run benchmarks instead of tests.")
     parser.add_argument('--logbase', default='testlog',
                         help="Base name for log file.")
-    parser.add_argument('-j', '--num-processes', default=determine_worker_count(), type=int,
+    parser.add_argument('-j', '--num-processes', default=determine_worker_count(['MESON_TESTTHREADS']), type=int,
                         help='How many parallel processes to use.')
     parser.add_argument('-v', '--verbose', default=False, action='store_true',
                         help='Do not redirect stdout and stderr')

mesonbuild/scripts/externalproject.py

@@ -5,12 +5,11 @@
 
 import os
 import argparse
-import multiprocessing
 import subprocess
 from pathlib import Path
 import typing as T
 
-from ..mesonlib import Popen_safe, split_args
+from ..mesonlib import Popen_safe, split_args, determine_worker_count
 
 class ExternalProject:
     def __init__(self, options: argparse.Namespace):
@@ -48,7 +47,7 @@ def supports_jobs_flag(self) -> bool:
     def build(self) -> int:
         make_cmd = self.make.copy()
         if self.supports_jobs_flag():
-            make_cmd.append(f'-j{multiprocessing.cpu_count()}')
+            make_cmd.append(f'-j{determine_worker_count()}')
         rc = self._run('build', make_cmd)
         if rc != 0:
             return rc

mesonbuild/utils/universal.py

@@ -13,6 +13,7 @@
 import stat
 import time
 import abc
+import multiprocessing
 import platform, subprocess, operator, os, shlex, shutil, re
 import collections
 from functools import lru_cache, wraps
@@ -94,6 +95,7 @@ class _VerPickleLoadable(Protocol):
     'default_sysconfdir',
     'detect_subprojects',
     'detect_vcs',
+    'determine_worker_count',
     'do_conf_file',
     'do_conf_str',
     'do_replacement',
@@ -1086,6 +1088,31 @@ def default_sysconfdir() -> str:
     return 'etc'
 
 
+def determine_worker_count(varnames: T.Optional[T.List[str]] = None) -> int:
+    num_workers = 0
+    varnames = varnames or []
+    # Add MESON_NUM_PROCESSES last, so it will prevail if more than one
+    # variable is present.
+    varnames.append('MESON_NUM_PROCESSES')
+    for varname in varnames:
+        if varname in os.environ:
+            try:
+                num_workers = int(os.environ[varname])
+                if num_workers < 0:
+                    raise ValueError
+            except ValueError:
+                print(f'Invalid value in {varname}, using 1 thread.')
+                num_workers = 1
+
+    if num_workers == 0:
+        try:
+            # Fails in some weird environments such as Debian
+            # reproducible build.
+            num_workers = multiprocessing.cpu_count()
+        except Exception:
+            num_workers = 1
+    return num_workers
+
 def has_path_sep(name: str, sep: str = '/\\') -> bool:
     'Checks if any of the specified @sep path separators are in @name'
     for each in sep: