From d1c7df8231df76d0b4c2c8b3033bc767f486865f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 23 Sep 2022 10:20:33 +0200 Subject: [PATCH 01/16] refactor git_log.py --- docs/index.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 199611037..aa6b3f4bc 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3036,4 +3036,3 @@ Timeline .. _`threading.Thread`: https://docs.python.org/3/library/threading.html#threading.Thread .. _Tidelift security contact: https://tidelift.com/security .. _Tidelift Subscription: https://tidelift.com/subscription/pkg/pypi-psutil?utm_source=pypi-psutil&utm_medium=referral&utm_campaign=readme -.. _Tidelift Subscription: https://tidelift.com/subscription/pkg/pypi-psutil?utm_source=pypi-psutil&utm_medium=referral&utm_campaign=readme From 3f81c62ecb37098318fd1f80eebb0bdffc8f097c Mon Sep 17 00:00:00 2001 From: Matthieu Darbois Date: Tue, 18 Oct 2022 20:05:17 +0200 Subject: [PATCH 02/16] fix #2021, fix #1954, provide OSX arm64 bins + add pyproject.toml (#2040) This commit updates the build workflow to use the latest cibuildwheel as a GitHub Action. cibuildwheel configuration is now in its own file (as there's no `pyproject.toml` yet) Signed-off-by: mayeut --- .github/workflows/build.yml | 29 ++--------------------------- .gitignore | 1 + pyproject.toml | 15 +++++++++++++++ 3 files changed, 18 insertions(+), 27 deletions(-) create mode 100644 pyproject.toml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7d3c4224..4c0ef79bd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -28,17 +28,7 @@ jobs: strategy: fail-fast: false matrix: - # os: [ubuntu-latest, macos-latest, windows-latest] os: [ubuntu-latest, macos-12] - include: - - {name: Linux, python: '3.9', os: ubuntu-latest} - env: - CIBW_TEST_COMMAND: - PYTHONWARNINGS=always PYTHONUNBUFFERED=1 PSUTIL_DEBUG=1 python {project}/psutil/tests/runner.py && - PYTHONWARNINGS=always PYTHONUNBUFFERED=1 PSUTIL_DEBUG=1 python {project}/psutil/tests/test_memleaks.py - CIBW_TEST_EXTRAS: test - CIBW_BUILD: 'cp36-* cp37-* cp38-* cp39-* cp310-*' - CIBW_SKIP: '*-musllinux_*' steps: - name: Cancel previous runs @@ -53,16 +43,8 @@ jobs: cache: pip cache-dependency-path: .github/workflows/build.yml - - name: Install cibuildwheel - run: pip install cibuildwheel - - # - name: (Windows) install Visual C++ for Python 2.7 - # if: matrix.os == 'windows-latest' - # run: | - # choco install vcpython27 -f -y - - name: Run tests - run: cibuildwheel . + uses: pypa/cibuildwheel@v2.11.1 - name: Create wheels uses: actions/upload-artifact@v3 @@ -87,16 +69,12 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-12] - include: - - {name: Linux, python: '3.9', os: ubuntu-latest} env: - CIBW_ARCHS_LINUX: 'x86_64 i686' CIBW_TEST_COMMAND: PYTHONWARNINGS=always PYTHONUNBUFFERED=1 PSUTIL_DEBUG=1 python {project}/psutil/tests/runner.py && PYTHONWARNINGS=always PYTHONUNBUFFERED=1 PSUTIL_DEBUG=1 python {project}/psutil/tests/test_memleaks.py CIBW_TEST_EXTRAS: test CIBW_BUILD: 'cp27-*' - CIBW_SKIP: '*-musllinux_*' steps: - name: Cancel previous runs @@ -111,11 +89,8 @@ jobs: cache: pip cache-dependency-path: .github/workflows/build.yml - - name: Install cibuildwheel - run: pip install cibuildwheel==1.12.0 - - name: Run tests - run: cibuildwheel . + uses: pypa/cibuildwheel@v1.12.0 - name: Create wheels uses: actions/upload-artifact@v3 diff --git a/.gitignore b/.gitignore index 3d22b0b35..ddafc64c6 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ syntax: glob .tox/ build/ dist/ +wheelhouse/ diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..47031d268 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,15 @@ +[build-system] +requires = ["setuptools>=43"] +build-backend = "setuptools.build_meta" + +[tool.cibuildwheel] +build = ["cp36-*", "cp37-*", "cp38-*", "cp39-*", "cp310-*"] +skip = ["*-musllinux*"] +test-extras = "test" +test-command = [ + "PYTHONWARNINGS=always PYTHONUNBUFFERED=1 PSUTIL_DEBUG=1 python {project}/psutil/tests/runner.py", + "PYTHONWARNINGS=always PYTHONUNBUFFERED=1 PSUTIL_DEBUG=1 python {project}/psutil/tests/test_memleaks.py" +] + +[tool.cibuildwheel.macos] +archs = ["x86_64", "arm64"] From 8ad2d5ba0cd2665e28e6ab84af11f74a83163471 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 18 Oct 2022 21:12:05 +0200 Subject: [PATCH 03/16] move isort and coverage config into pyproject.toml ...since pyproject.toml was introduced in #2040. CC @mayeut Signed-off-by: Giampaolo Rodola --- .coveragerc | 32 ------------------------ .github/workflows/build.yml | 2 +- .isort.cfg | 7 ------ MANIFEST.in | 3 +-- Makefile | 6 ++--- psutil/_psposix.py | 3 ++- pyproject.toml | 39 ++++++++++++++++++++++++++++++ scripts/internal/git_pre_commit.py | 3 +-- 8 files changed, 47 insertions(+), 48 deletions(-) delete mode 100644 .coveragerc delete mode 100644 .isort.cfg diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index c6772e953..000000000 --- a/.coveragerc +++ /dev/null @@ -1,32 +0,0 @@ -[report] - -include = - *psutil* -omit = - psutil/_compat.py - psutil/tests/* - setup.py -exclude_lines = - enum.IntEnum - except ImportError: - globals().update - if __name__ == .__main__.: - if _WINDOWS: - if BSD - if enum is None: - if enum is not None: - if FREEBSD - if has_enums: - if LINUX - if LITTLE_ENDIAN: - if NETBSD - if OPENBSD - if MACOS - if ppid_map is None: - if PY3: - if SUNOS - if sys.platform.startswith - if WINDOWS - import enum - pragma: no cover - raise NotImplementedError diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4c0ef79bd..e572b0297 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -151,6 +151,6 @@ jobs: # py3 python3 -m pip install flake8 isort python3 -m flake8 . - python3 -m isort --settings=.isort.cfg . + python3 -m isort . # clinter find . -type f \( -iname "*.c" -o -iname "*.h" \) | xargs python3 scripts/internal/clinter.py diff --git a/.isort.cfg b/.isort.cfg deleted file mode 100644 index 58f66946c..000000000 --- a/.isort.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# See: https://pycqa.github.io/isort/docs/configuration/options - -[settings] -# one import per line -force_single_line = true -# blank spaces after import section -lines_after_imports = 2 diff --git a/MANIFEST.in b/MANIFEST.in index c9cd85e99..e19c7e2de 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,5 @@ -include .coveragerc include .flake8 include .gitignore -include .isort.cfg include CONTRIBUTING.md include CREDITS include HISTORY.rst @@ -123,6 +121,7 @@ include psutil/tests/test_system.py include psutil/tests/test_testutils.py include psutil/tests/test_unicode.py include psutil/tests/test_windows.py +include pyproject.toml include scripts/battery.py include scripts/cpu_distribution.py include scripts/disk_usage.py diff --git a/Makefile b/Makefile index caf4c9e9d..fb7f89ae0 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ TSCRIPT = psutil/tests/runner.py # Internal. DEPS = \ - git+https://github.com/PyCQA/autoflake.git \ + autoflake \ autopep8 \ check-manifest \ concurrencytest \ @@ -195,7 +195,7 @@ flake8: ## Run flake8 linter. @git ls-files '*.py' | xargs $(PYTHON) -m flake8 --config=.flake8 isort: ## Run isort linter. - @git ls-files '*.py' | xargs $(PYTHON) -m isort --settings=.isort.cfg --check-only + @git ls-files '*.py' | xargs $(PYTHON) -m isort --check-only c-linter: ## Run C linter. @git ls-files '*.c' '*.h' | xargs $(PYTHON) scripts/internal/clinter.py @@ -214,7 +214,7 @@ fix-flake8: ## Run autopep8, fix some Python flake8 / pep8 issues. @git ls-files '*.py' | xargs $(PYTHON) -m autoflake --in-place --jobs 0 --remove-all-unused-imports --remove-unused-variables --remove-duplicate-keys fix-imports: ## Fix imports with isort. - @git ls-files '*.py' | xargs $(PYTHON) -m isort --settings=.isort.cfg + @git ls-files '*.py' | xargs $(PYTHON) -m isort fix-all: ## Run all code fixers. ${MAKE} fix-flake8 diff --git a/psutil/_psposix.py b/psutil/_psposix.py index 39912d97e..1d250bf75 100644 --- a/psutil/_psposix.py +++ b/psutil/_psposix.py @@ -10,11 +10,11 @@ import sys import time +from ._common import MACOS from ._common import TimeoutExpired from ._common import memoize from ._common import sdiskusage from ._common import usage_percent -from ._common import MACOS from ._compat import PY3 from ._compat import ChildProcessError from ._compat import FileNotFoundError @@ -23,6 +23,7 @@ from ._compat import ProcessLookupError from ._compat import unicode + if MACOS: from . import _psutil_osx diff --git a/pyproject.toml b/pyproject.toml index 47031d268..52d4c7286 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,42 @@ +[tool.isort] +force_single_line = true # one import per line +lines_after_imports = 2 # blank spaces after import section + +[tool.coverage.report] +include = [ + "*psutil*" +] +omit = [ + "psutil/_compat.py", + "psutil/tests/*", + "setup.py", +] +exclude_lines = [ + "enum.IntEnum", + "except ImportError:", + "globals().update", + "if __name__ == .__main__.:", + "if _WINDOWS:", + "if BSD", + "if enum is None:", + "if enum is not None:", + "if FREEBSD", + "if has_enums:", + "if LINUX", + "if LITTLE_ENDIAN:", + "if MACOS", + "if NETBSD", + "if OPENBSD", + "if ppid_map is None:", + "if PY3:", + "if SUNOS", + "if sys.platform.startswith", + "if WINDOWS", + "import enum", + "pragma: no cover", + "raise NotImplementedError", +] + [build-system] requires = ["setuptools>=43"] build-backend = "setuptools.build_meta" diff --git a/scripts/internal/git_pre_commit.py b/scripts/internal/git_pre_commit.py index 46b3bc533..877836272 100755 --- a/scripts/internal/git_pre_commit.py +++ b/scripts/internal/git_pre_commit.py @@ -128,8 +128,7 @@ def main(): return exit("python code didn't pass 'flake8' style check; " "try running 'make fix-flake8'") # isort - assert os.path.exists('.isort.cfg') - cmd = "%s -m isort --settings=.isort.cfg --check-only %s" % ( + cmd = "%s -m isort --check-only %s" % ( PYTHON, " ".join(py_files)) ret = subprocess.call(shlex.split(cmd)) if ret != 0: From e1e48a8822f734a9d5ce1729c08984e975fabb1f Mon Sep 17 00:00:00 2001 From: iam-py-test <84232764+iam-py-test@users.noreply.github.com> Date: Tue, 18 Oct 2022 15:27:09 -0400 Subject: [PATCH 04/16] Fix a typo (#2047) --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 5f0afcdfa..4961c695f 100644 --- a/README.rst +++ b/README.rst @@ -220,7 +220,7 @@ Disks >>> psutil.disk_partitions() [sdiskpart(device='/dev/sda1', mountpoint='/', fstype='ext4', opts='rw,nosuid', maxfile=255, maxpath=4096), - sdiskpart(device='/dev/sda2', mountpoint='/home', fstype='ext, opts='rw', maxfile=255, maxpath=4096)] + sdiskpart(device='/dev/sda2', mountpoint='/home', fstype='ext', opts='rw', maxfile=255, maxpath=4096)] >>> >>> psutil.disk_usage('/') sdiskusage(total=21378641920, used=4809781248, free=15482871808, percent=22.5) From aae4346a560601e0417188f55cdd2d2a0e606fa7 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 18 Oct 2022 22:11:08 +0200 Subject: [PATCH 05/16] pre-release + give CREDITS to @mayeut (PR #2040) and @eallrich (new supporter) Signed-off-by: Giampaolo Rodola --- CREDITS | 2 +- HISTORY.rst | 11 ++++++++--- README.rst | 1 + 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CREDITS b/CREDITS index e53c1577d..6f1b6e5b8 100644 --- a/CREDITS +++ b/CREDITS @@ -781,7 +781,7 @@ I: 1956 N: Matthieu Darbois W: https://github.com/mayeut -I: 2039, 2142, 2147, 2153 +I: 2039, 2142, 2147, 2153, 2040 N: Hugo van Kemenade W: https://github.com/hugovk diff --git a/HISTORY.rst b/HISTORY.rst index f77b20607..e98339901 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,9 +1,14 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* -5.9.3 (IN DEVELOPMENT) -====================== +5.9.3 +===== + +2022-10-18 + +**Enhancements** -XXXX-XX-XX +- 2040_, [macOS]: provide wheels for arm64 architecture. (patch by Matthieu + Darbois) **Bug fixes** diff --git a/README.rst b/README.rst index 5f0afcdfa..6eb3cfe96 100644 --- a/README.rst +++ b/README.rst @@ -142,6 +142,7 @@ Supporters + add your avatar From 1da9f7928e79ecc127768e7a5bc4eb7f785f4513 Mon Sep 17 00:00:00 2001 From: Matthieu Darbois Date: Tue, 18 Oct 2022 22:18:05 +0200 Subject: [PATCH 06/16] chore: skip test_cpu_freq on macOS arm64 (#2146) --- psutil/tests/test_contracts.py | 3 +++ psutil/tests/test_memleaks.py | 3 +++ psutil/tests/test_osx.py | 3 +++ psutil/tests/test_system.py | 3 +++ 4 files changed, 12 insertions(+) diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py index d376a3385..fde857b67 100755 --- a/psutil/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -12,6 +12,7 @@ import errno import multiprocessing import os +import platform import signal import stat import sys @@ -235,6 +236,8 @@ def test_cpu_times_percent(self): def test_cpu_count(self): self.assertIsInstance(psutil.cpu_count(), int) + # TODO: remove this once 1892 is fixed + @unittest.skipIf(MACOS and platform.machine() == 'arm64', "skipped due to #1892") @unittest.skipIf(not HAS_CPU_FREQ, "not supported") def test_cpu_freq(self): if psutil.cpu_freq() is None: diff --git a/psutil/tests/test_memleaks.py b/psutil/tests/test_memleaks.py index e507e837c..8d31193b0 100755 --- a/psutil/tests/test_memleaks.py +++ b/psutil/tests/test_memleaks.py @@ -19,6 +19,7 @@ import functools import os +import platform import unittest import psutil @@ -364,6 +365,8 @@ def test_cpu_stats(self): self.execute(psutil.cpu_stats) @fewtimes_if_linux() + # TODO: remove this once 1892 is fixed + @unittest.skipIf(MACOS and platform.machine() == 'arm64', "skipped due to #1892") @unittest.skipIf(not HAS_CPU_FREQ, "not supported") def test_cpu_freq(self): self.execute(psutil.cpu_freq) diff --git a/psutil/tests/test_osx.py b/psutil/tests/test_osx.py index d0f588ad6..8abddb528 100755 --- a/psutil/tests/test_osx.py +++ b/psutil/tests/test_osx.py @@ -6,6 +6,7 @@ """macOS specific tests.""" +import platform import re import time import unittest @@ -144,6 +145,8 @@ def test_cpu_count_cores(self): num = sysctl("sysctl hw.physicalcpu") self.assertEqual(num, psutil.cpu_count(logical=False)) + # TODO: remove this once 1892 is fixed + @unittest.skipIf(platform.machine() == 'arm64', "skipped due to #1892") def test_cpu_freq(self): freq = psutil.cpu_freq() self.assertEqual( diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 753249bc3..42b29e8fa 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -10,6 +10,7 @@ import datetime import errno import os +import platform import pprint import shutil import signal @@ -511,6 +512,8 @@ def test_cpu_stats(self): if not AIX and name in ('ctx_switches', 'interrupts'): self.assertGreater(value, 0) + # TODO: remove this once 1892 is fixed + @unittest.skipIf(MACOS and platform.machine() == 'arm64', "skipped due to #1892") @unittest.skipIf(not HAS_CPU_FREQ, "not supported") def test_cpu_freq(self): def check_ls(ls): From 669b672dc7acc696376e5ca8104cf01086fe6f8c Mon Sep 17 00:00:00 2001 From: Matthieu Darbois Date: Tue, 18 Oct 2022 22:30:29 +0200 Subject: [PATCH 07/16] fix: linter issues from #2146 (#2155) Signed-off-by: mayeut --- psutil/tests/test_contracts.py | 3 ++- psutil/tests/test_memleaks.py | 3 ++- psutil/tests/test_system.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py index fde857b67..3b806ee37 100755 --- a/psutil/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -237,7 +237,8 @@ def test_cpu_count(self): self.assertIsInstance(psutil.cpu_count(), int) # TODO: remove this once 1892 is fixed - @unittest.skipIf(MACOS and platform.machine() == 'arm64', "skipped due to #1892") + @unittest.skipIf(MACOS and platform.machine() == 'arm64', + "skipped due to #1892") @unittest.skipIf(not HAS_CPU_FREQ, "not supported") def test_cpu_freq(self): if psutil.cpu_freq() is None: diff --git a/psutil/tests/test_memleaks.py b/psutil/tests/test_memleaks.py index 8d31193b0..dbd1588df 100755 --- a/psutil/tests/test_memleaks.py +++ b/psutil/tests/test_memleaks.py @@ -366,7 +366,8 @@ def test_cpu_stats(self): @fewtimes_if_linux() # TODO: remove this once 1892 is fixed - @unittest.skipIf(MACOS and platform.machine() == 'arm64', "skipped due to #1892") + @unittest.skipIf(MACOS and platform.machine() == 'arm64', + "skipped due to #1892") @unittest.skipIf(not HAS_CPU_FREQ, "not supported") def test_cpu_freq(self): self.execute(psutil.cpu_freq) diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 42b29e8fa..1722b515d 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -513,7 +513,8 @@ def test_cpu_stats(self): self.assertGreater(value, 0) # TODO: remove this once 1892 is fixed - @unittest.skipIf(MACOS and platform.machine() == 'arm64', "skipped due to #1892") + @unittest.skipIf(MACOS and platform.machine() == 'arm64', + "skipped due to #1892") @unittest.skipIf(not HAS_CPU_FREQ, "not supported") def test_cpu_freq(self): def check_ls(ls): From 085691ac46b060aeb6d17a202fcc33062f61d9b5 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 20 Oct 2022 22:38:18 +0200 Subject: [PATCH 08/16] improve API speed benchmark script #2102 --- scripts/internal/print_api_speed.py | 147 ++++++++++++++++++++++------ 1 file changed, 118 insertions(+), 29 deletions(-) diff --git a/scripts/internal/print_api_speed.py b/scripts/internal/print_api_speed.py index ee2e3254e..f41d0466f 100755 --- a/scripts/internal/print_api_speed.py +++ b/scripts/internal/print_api_speed.py @@ -4,30 +4,75 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -"""Benchmark all API calls. +""" +Benchmark all API calls and print them from fastest to slowest. $ make print_api_speed -SYSTEM APIS SECONDS ----------------------------------- -cpu_count 0.000014 -disk_usage 0.000027 -cpu_times 0.000037 -cpu_percent 0.000045 -... - -PROCESS APIS SECONDS ----------------------------------- -create_time 0.000001 -nice 0.000005 -cwd 0.000011 -cpu_affinity 0.000011 -ionice 0.000013 -... +SYSTEM APIS NUM CALLS SECONDS +------------------------------------------------- +disk_usage 300 0.00157 +cpu_count 300 0.00255 +pid_exists 300 0.00792 +cpu_times 300 0.01044 +boot_time 300 0.01136 +cpu_percent 300 0.01290 +cpu_times_percent 300 0.01515 +virtual_memory 300 0.01594 +users 300 0.01964 +net_io_counters 300 0.02027 +cpu_stats 300 0.02034 +net_if_addrs 300 0.02962 +swap_memory 300 0.03209 +sensors_battery 300 0.05186 +pids 300 0.07954 +net_if_stats 300 0.09321 +disk_io_counters 300 0.09406 +cpu_count (cores) 300 0.10293 +disk_partitions 300 0.10345 +cpu_freq 300 0.20817 +sensors_fans 300 0.63476 +sensors_temperatures 231 2.00039 +process_iter (all) 171 2.01300 +net_connections 97 2.00206 + +PROCESS APIS NUM CALLS SECONDS +------------------------------------------------- +create_time 300 0.00009 +exe 300 0.00015 +nice 300 0.00057 +ionice 300 0.00091 +cpu_affinity 300 0.00091 +cwd 300 0.00151 +num_fds 300 0.00391 +memory_info 300 0.00597 +memory_percent 300 0.00648 +io_counters 300 0.00707 +name 300 0.00894 +status 300 0.00900 +ppid 300 0.00906 +num_threads 300 0.00932 +cpu_num 300 0.00933 +num_ctx_switches 300 0.00943 +uids 300 0.00979 +gids 300 0.01002 +cpu_times 300 0.01008 +cmdline 300 0.01009 +terminal 300 0.01059 +is_running 300 0.01063 +threads 300 0.01209 +connections 300 0.01276 +cpu_percent 300 0.01463 +open_files 300 0.01630 +username 300 0.01655 +environ 300 0.02250 +memory_full_info 300 0.07066 +memory_maps 300 0.74281 """ from __future__ import division from __future__ import print_function +import argparse import inspect import os from timeit import default_timer as timer @@ -36,16 +81,23 @@ from psutil._common import print_color +TIMES = 300 timings = [] -templ = "%-25s %s" +templ = "%-25s %10s %10s" + + +def print_header(what): + s = templ % (what, "NUM CALLS", "SECONDS") + print_color(s, color=None, bold=True) + print("-" * len(s)) def print_timings(): - timings.sort(key=lambda x: x[1]) + timings.sort(key=lambda x: (x[1], -x[2]), reverse=True) i = 0 while timings[:]: - title, elapsed = timings.pop(0) - s = templ % (title, "%f" % elapsed) + title, times, elapsed = timings.pop(0) + s = templ % (title, str(times), "%.5f" % elapsed) if i > len(timings) - 5: print_color(s, color="red") else: @@ -53,13 +105,48 @@ def print_timings(): def timecall(title, fun, *args, **kw): + print("%-50s" % title, end="", flush=True) t = timer() - fun(*args, **kw) - elapsed = timer() - t - timings.append((title, elapsed)) + for n in range(TIMES): + fun(*args, **kw) + elapsed = timer() - t + if elapsed > 2: + break + print("\r", end="", flush=True) + timings.append((title, n + 1, elapsed)) + + +def set_highest_priority(): + """Set highest CPU and I/O priority (requires root).""" + p = psutil.Process() + if psutil.WINDOWS: + p.nice(psutil.HIGH_PRIORITY_CLASS) + else: + p.nice(-20) + + if psutil.LINUX: + p.ionice(psutil.IOPRIO_CLASS_RT, value=7) + elif psutil.WINDOWS: + p.ionice(psutil.IOPRIO_HIGH) def main(): + global TIMES + + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument('-t', '--times', type=int, default=TIMES) + args = parser.parse_args() + TIMES = args.times + assert TIMES > 1, TIMES + + try: + set_highest_priority() + except psutil.AccessDenied: + prio_set = False + else: + prio_set = True + # --- system public_apis = [] @@ -73,8 +160,7 @@ def main(): if name not in ignore: public_apis.append(name) - print_color(templ % ("SYSTEM APIS", "SECONDS"), color=None, bold=True) - print("-" * 34) + print_header("SYSTEM APIS") for name in public_apis: fun = getattr(psutil, name) args = () @@ -89,11 +175,10 @@ def main(): # --- process print("") - print_color(templ % ("PROCESS APIS", "SECONDS"), color=None, bold=True) - print("-" * 34) + print_header("PROCESS APIS") ignore = ['send_signal', 'suspend', 'resume', 'terminate', 'kill', 'wait', 'as_dict', 'parent', 'parents', 'memory_info_ex', 'oneshot', - 'pid', 'rlimit'] + 'pid', 'rlimit', 'children'] if psutil.MACOS: ignore.append('memory_maps') # XXX p = psutil.Process() @@ -103,6 +188,10 @@ def main(): timecall(name, fun) print_timings() + if not prio_set: + print_color("\nWARN: couldn't set highest process priority " + + "(requires root)", "red") + if __name__ == '__main__': main() From 1b09c1b54cb2ae46145de7f710e9b990cd264016 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 20 Oct 2022 23:56:43 +0200 Subject: [PATCH 09/16] fix py2 compatibility Signed-off-by: Giampaolo Rodola --- scripts/internal/print_api_speed.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/internal/print_api_speed.py b/scripts/internal/print_api_speed.py index f41d0466f..e85b70382 100755 --- a/scripts/internal/print_api_speed.py +++ b/scripts/internal/print_api_speed.py @@ -75,6 +75,7 @@ import argparse import inspect import os +import sys from timeit import default_timer as timer import psutil @@ -105,14 +106,16 @@ def print_timings(): def timecall(title, fun, *args, **kw): - print("%-50s" % title, end="", flush=True) + print("%-50s" % title, end="") + sys.stdout.flush() t = timer() for n in range(TIMES): fun(*args, **kw) elapsed = timer() - t if elapsed > 2: break - print("\r", end="", flush=True) + print("\r", end="") + sys.stdout.flush() timings.append((title, n + 1, elapsed)) From 614e91158ced7a65aabbe7244d52de1cebfc494b Mon Sep 17 00:00:00 2001 From: Matthieu Darbois Date: Fri, 21 Oct 2022 01:02:23 +0200 Subject: [PATCH 10/16] feature: use ABI3 for cp36+ (#2102) --- .ci/appveyor/install.ps1 | 85 ------------------------------ .github/workflows/build.yml | 49 +++++++++++++++-- appveyor.yml | 45 ---------------- psutil/_psutil_linux.c | 32 +++++------ psutil/arch/windows/process_info.c | 2 +- pyproject.toml | 20 +++++-- setup.py | 52 ++++++++++++++---- 7 files changed, 120 insertions(+), 165 deletions(-) delete mode 100644 .ci/appveyor/install.ps1 diff --git a/.ci/appveyor/install.ps1 b/.ci/appveyor/install.ps1 deleted file mode 100644 index 3f0562825..000000000 --- a/.ci/appveyor/install.ps1 +++ /dev/null @@ -1,85 +0,0 @@ -# Sample script to install Python and pip under Windows -# Authors: Olivier Grisel and Kyle Kastner -# License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ - -$BASE_URL = "https://www.python.org/ftp/python/" -$GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py" -$GET_PIP_PATH = "C:\get-pip.py" - - -function DownloadPython ($python_version, $platform_suffix) { - $webclient = New-Object System.Net.WebClient - $filename = "python-" + $python_version + $platform_suffix + ".msi" - $url = $BASE_URL + $python_version + "/" + $filename - - $basedir = $pwd.Path + "\" - $filepath = $basedir + $filename - if (Test-Path $filename) { - Write-Host "Reusing" $filepath - return $filepath - } - - # Download and retry up to 5 times in case of network transient errors. - Write-Host "Downloading" $filename "from" $url - $retry_attempts = 3 - for($i=0; $i -lt $retry_attempts; $i++){ - try { - $webclient.DownloadFile($url, $filepath) - break - } - Catch [Exception]{ - Start-Sleep 1 - } - } - Write-Host "File saved at" $filepath - return $filepath -} - - -function InstallPython ($python_version, $architecture, $python_home) { - Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home - if (Test-Path $python_home) { - Write-Host $python_home "already exists, skipping." - return $false - } - if ($architecture -eq "32") { - $platform_suffix = "" - } else { - $platform_suffix = ".amd64" - } - $filepath = DownloadPython $python_version $platform_suffix - Write-Host "Installing" $filepath "to" $python_home - $args = "/qn /i $filepath TARGETDIR=$python_home" - Write-Host "msiexec.exe" $args - Start-Process -FilePath "msiexec.exe" -ArgumentList $args -Wait -Passthru - Write-Host "Python $python_version ($architecture) installation complete" - return $true -} - - -function InstallPip ($python_home) { - $pip_path = $python_home + "/Scripts/pip.exe" - $python_path = $python_home + "/python.exe" - if (-not(Test-Path $pip_path)) { - Write-Host "Installing pip..." - $webclient = New-Object System.Net.WebClient - $webclient.DownloadFile($GET_PIP_URL, $GET_PIP_PATH) - Write-Host "Executing:" $python_path $GET_PIP_PATH - Start-Process -FilePath "$python_path" -ArgumentList "$GET_PIP_PATH" -Wait -Passthru - } else { - Write-Host "pip already installed." - } -} - -function InstallPackage ($python_home, $pkg) { - $pip_path = $python_home + "/Scripts/pip.exe" - & $pip_path install $pkg -} - -function main () { - InstallPython $env:PYTHON_VERSION $env:PYTHON_ARCH $env:PYTHON - InstallPip $env:PYTHON - InstallPackage $env:PYTHON wheel -} - -main diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e572b0297..3a52c8fe7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,15 +20,15 @@ on: [push, pull_request] name: build jobs: - # Linux + macOS + Python 3 - linux-macos-py3: + # Linux + macOS + Windows Python 3 + py3: name: ${{ matrix.os }}-py3 runs-on: ${{ matrix.os }} timeout-minutes: 20 strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-12] + os: [ubuntu-latest, macos-12, windows-2019] steps: - name: Cancel previous runs @@ -60,6 +60,49 @@ jobs: mv dist/psutil*.tar.gz wheelhouse/ python scripts/internal/print_hashes.py wheelhouse/ + # Windows cp37+ tests + # psutil tests do not like running from a virtualenv with python>=3.7 so + # not using cibuildwheel for those. run them "manually" with this job. + windows-py3-test: + name: windows-py3-test-${{ matrix.python }}-${{ matrix.architecture }} + needs: py3 + runs-on: windows-2019 + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + python: ["3.7", "3.8", "3.9", "3.10", "~3.11.0-0"] + architecture: ["x86", "x64"] + + steps: + - name: Cancel previous runs + uses: styfle/cancel-workflow-action@0.9.1 + with: + access_token: ${{ github.token }} + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "${{ matrix.python }}" + architecture: "${{ matrix.architecture }}" + cache: pip + cache-dependency-path: .github/workflows/build.yml + - name: Download wheels + uses: actions/download-artifact@v3 + with: + name: wheels + path: wheelhouse + - name: Run tests + run: | + mkdir .tests + cd .tests + pip install $(find ../wheelhouse -name '*-cp36-abi3-${{ matrix.architecture == 'x86' && 'win32' || 'win_amd64'}}.whl')[test] + export PYTHONWARNINGS=always + export PYTHONUNBUFFERED=1 + export PSUTIL_DEBUG=1 + python ../psutil/tests/runner.py + python ../psutil/tests/test_memleaks.py + shell: bash + # Linux + macOS + Python 2 linux-macos-py2: name: ${{ matrix.os }}-py2 diff --git a/appveyor.yml b/appveyor.yml index 4bbd51aeb..0752e610c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -24,62 +24,17 @@ environment: PYTHON_VERSION: "2.7.x" PYTHON_ARCH: "32" - - PYTHON: "C:\\Python36" - PYTHON_VERSION: "3.6.x" - PYTHON_ARCH: "32" - - - PYTHON: "C:\\Python37" - PYTHON_VERSION: "3.7.x" - PYTHON_ARCH: "32" - - - PYTHON: "C:\\Python38" - PYTHON_VERSION: "3.8.x" - PYTHON_ARCH: "32" - - - PYTHON: "C:\\Python39" - PYTHON_VERSION: "3.9.x" - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 - PYTHON_ARCH: "32" - - - PYTHON: "C:\\Python310" - PYTHON_VERSION: "3.10.x" - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 - PYTHON_ARCH: "32" - # 64 bits - PYTHON: "C:\\Python27-x64" PYTHON_VERSION: "2.7.x" PYTHON_ARCH: "64" - - PYTHON: "C:\\Python36-x64" - PYTHON_VERSION: "3.6.x" - PYTHON_ARCH: "64" - - - PYTHON: "C:\\Python37-x64" - PYTHON_VERSION: "3.7.x" - PYTHON_ARCH: "64" - - - PYTHON: "C:\\Python38-x64" - PYTHON_VERSION: "3.8.x" - PYTHON_ARCH: "64" - - - PYTHON: "C:\\Python39-x64" - PYTHON_VERSION: "3.9.x" - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 - PYTHON_ARCH: "64" - - - PYTHON: "C:\\Python310-x64" - PYTHON_VERSION: "3.10.x" - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 - PYTHON_ARCH: "64" init: - "ECHO %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH%" install: - - "powershell .ci\\appveyor\\install.ps1" - # - ps: (new-object net.webclient).DownloadFile('https://raw.github.com/pypa/pip/master/contrib/get-pip.py', 'C:/get-pip.py') - "%WITH_COMPILER% %PYTHON%/python.exe -m pip --version" - "%WITH_COMPILER% %PYTHON%/python.exe -m pip install --upgrade --user setuptools pip" - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py setup-dev-env" diff --git a/psutil/_psutil_linux.c b/psutil/_psutil_linux.c index 64cdf0b55..1ed1f91a0 100644 --- a/psutil/_psutil_linux.c +++ b/psutil/_psutil_linux.c @@ -296,52 +296,46 @@ psutil_proc_cpu_affinity_set(PyObject *self, PyObject *args) { cpu_set_t cpu_set; size_t len; pid_t pid; - int i, seq_len; + Py_ssize_t i, seq_len; PyObject *py_cpu_set; - PyObject *py_cpu_seq = NULL; if (!PyArg_ParseTuple(args, _Py_PARSE_PID "O", &pid, &py_cpu_set)) return NULL; if (!PySequence_Check(py_cpu_set)) { - PyErr_Format(PyExc_TypeError, "sequence argument expected, got %s", - Py_TYPE(py_cpu_set)->tp_name); - goto error; + return PyErr_Format(PyExc_TypeError, "sequence argument expected, got %R", Py_TYPE(py_cpu_set)); } - py_cpu_seq = PySequence_Fast(py_cpu_set, "expected a sequence or integer"); - if (!py_cpu_seq) - goto error; - seq_len = PySequence_Fast_GET_SIZE(py_cpu_seq); + seq_len = PySequence_Size(py_cpu_set); + if (seq_len < 0) { + return NULL; + } CPU_ZERO(&cpu_set); for (i = 0; i < seq_len; i++) { - PyObject *item = PySequence_Fast_GET_ITEM(py_cpu_seq, i); + PyObject *item = PySequence_GetItem(py_cpu_set, i); + if (!item) { + return NULL; + } #if PY_MAJOR_VERSION >= 3 long value = PyLong_AsLong(item); #else long value = PyInt_AsLong(item); #endif + Py_XDECREF(item); if ((value == -1) || PyErr_Occurred()) { if (!PyErr_Occurred()) PyErr_SetString(PyExc_ValueError, "invalid CPU value"); - goto error; + return NULL; } CPU_SET(value, &cpu_set); } len = sizeof(cpu_set); if (sched_setaffinity(pid, len, &cpu_set)) { - PyErr_SetFromErrno(PyExc_OSError); - goto error; + return PyErr_SetFromErrno(PyExc_OSError); } - Py_DECREF(py_cpu_seq); Py_RETURN_NONE; - -error: - if (py_cpu_seq != NULL) - Py_DECREF(py_cpu_seq); - return NULL; } #endif /* PSUTIL_HAVE_CPU_AFFINITY */ diff --git a/psutil/arch/windows/process_info.c b/psutil/arch/windows/process_info.c index d44c4eb75..1981d306a 100644 --- a/psutil/arch/windows/process_info.c +++ b/psutil/arch/windows/process_info.c @@ -578,7 +578,7 @@ psutil_get_cmdline(DWORD pid, int use_peb) { wcslen(szArglist[i])); if (py_unicode == NULL) goto out; - PyList_SET_ITEM(py_retlist, i, py_unicode); + PyList_SetItem(py_retlist, i, py_unicode); py_unicode = NULL; } ret = py_retlist; diff --git a/pyproject.toml b/pyproject.toml index 52d4c7286..87b84fc67 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,12 +38,11 @@ exclude_lines = [ ] [build-system] -requires = ["setuptools>=43"] +requires = ["setuptools>=43", "wheel"] build-backend = "setuptools.build_meta" [tool.cibuildwheel] -build = ["cp36-*", "cp37-*", "cp38-*", "cp39-*", "cp310-*"] -skip = ["*-musllinux*"] +skip = ["pp*", "*-musllinux*"] test-extras = "test" test-command = [ "PYTHONWARNINGS=always PYTHONUNBUFFERED=1 PSUTIL_DEBUG=1 python {project}/psutil/tests/runner.py", @@ -52,3 +51,18 @@ test-command = [ [tool.cibuildwheel.macos] archs = ["x86_64", "arm64"] + +[tool.cibuildwheel.windows] +# psutil tests do not like running from a virtualenv with python>=3.7 +# restrict build & tests to cp36 +# cp36-abi3 wheels will need to be tested outside cibuildwheel for python>=3.7 +build = "cp36-*" +test-command = [ + "python {project}/psutil/tests/runner.py", + "python {project}/psutil/tests/test_memleaks.py" +] + +[tool.cibuildwheel.windows.environment] +PYTHONWARNINGS = "always" +PYTHONUNBUFFERED = "1" +PSUTIL_DEBUG = "1" diff --git a/setup.py b/setup.py index 0f6716b3b..b3f20666d 100755 --- a/setup.py +++ b/setup.py @@ -31,6 +31,12 @@ setuptools = None from distutils.core import Extension from distutils.core import setup + try: + from wheel.bdist_wheel import bdist_wheel + except ImportError: + if "CIBUILDWHEEL" in os.environ: + raise + bdist_wheel = None HERE = os.path.abspath(os.path.dirname(__file__)) @@ -53,6 +59,9 @@ PYPY = '__pypy__' in sys.builtin_module_names +PY36_PLUS = sys.version_info[:2] >= (3, 6) +CP36_PLUS = PY36_PLUS and sys.implementation.name == "cpython" + macros = [] if POSIX: macros.append(("PSUTIL_POSIX", 1)) @@ -100,6 +109,12 @@ def get_version(): VERSION = get_version() macros.append(('PSUTIL_VERSION', int(VERSION.replace('.', '')))) +if bdist_wheel and CP36_PLUS and (MACOS or LINUX or WINDOWS): + py_limited_api = {"py_limited_api": True} + macros.append(('Py_LIMITED_API', '0x03060000')) +else: + py_limited_api = {} + def get_description(): script = os.path.join(HERE, "scripts", "internal", "convert_readme.py") @@ -182,7 +197,8 @@ def get_winver(): "ws2_32", "PowrProf", "pdh", ], # extra_compile_args=["/W 4"], - # extra_link_args=["/DEBUG"] + # extra_link_args=["/DEBUG"], + **py_limited_api ) elif MACOS: @@ -197,7 +213,8 @@ def get_winver(): define_macros=macros, extra_link_args=[ '-framework', 'CoreFoundation', '-framework', 'IOKit' - ]) + ], + **py_limited_api) elif FREEBSD: macros.append(("PSUTIL_FREEBSD", 1)) @@ -214,7 +231,8 @@ def get_winver(): 'psutil/arch/freebsd/proc_socks.c', ], define_macros=macros, - libraries=["devstat"]) + libraries=["devstat"], + **py_limited_api) elif OPENBSD: macros.append(("PSUTIL_OPENBSD", 1)) @@ -228,7 +246,8 @@ def get_winver(): 'psutil/arch/openbsd/proc.c', ], define_macros=macros, - libraries=["kvm"]) + libraries=["kvm"], + **py_limited_api) elif NETBSD: macros.append(("PSUTIL_NETBSD", 1)) @@ -240,7 +259,8 @@ def get_winver(): 'psutil/arch/netbsd/socks.c', ], define_macros=macros, - libraries=["kvm"]) + libraries=["kvm"], + **py_limited_api) elif LINUX: def get_ethtool_macro(): @@ -276,7 +296,8 @@ def get_ethtool_macro(): ext = Extension( 'psutil._psutil_linux', sources=sources + ['psutil/_psutil_linux.c'], - define_macros=macros) + define_macros=macros, + **py_limited_api) elif SUNOS: macros.append(("PSUTIL_SUNOS", 1)) @@ -288,7 +309,8 @@ def get_ethtool_macro(): 'psutil/arch/solaris/environ.c' ], define_macros=macros, - libraries=['kstat', 'nsl', 'socket']) + libraries=['kstat', 'nsl', 'socket'], + **py_limited_api) elif AIX: macros.append(("PSUTIL_AIX", 1)) @@ -300,7 +322,8 @@ def get_ethtool_macro(): 'psutil/arch/aix/common.c', 'psutil/arch/aix/ifaddrs.c'], libraries=['perfstat'], - define_macros=macros) + define_macros=macros, + **py_limited_api) else: sys.exit('platform %s is not supported' % sys.platform) @@ -310,7 +333,8 @@ def get_ethtool_macro(): posix_extension = Extension( 'psutil._psutil_posix', define_macros=macros, - sources=sources) + sources=sources, + **py_limited_api) if SUNOS: def get_sunos_update(): # See https://serverfault.com/q/524883 @@ -341,11 +365,21 @@ def get_sunos_update(): else: extensions = [ext] +cmdclass = {} +if py_limited_api: + class bdist_wheel_abi3(bdist_wheel): + def get_tag(self): + python, abi, plat = bdist_wheel.get_tag(self) + return python, "abi3", plat + + cmdclass["bdist_wheel"] = bdist_wheel_abi3 + def main(): kwargs = dict( name='psutil', version=VERSION, + cmdclass=cmdclass, description=__doc__ .replace('\n', ' ').strip() if __doc__ else '', long_description=get_description(), long_description_content_type='text/x-rst', From cd8827d065d63114060aae69f36b447006c5a70f Mon Sep 17 00:00:00 2001 From: Daniel Widdis Date: Thu, 20 Oct 2022 16:11:21 -0700 Subject: [PATCH 11/16] Use system-level values for Windows virtual memory (#2077) * Use system-level values for Windows virtual memory Submitting this as a draft PR as I believe it will address #2074. I do not have either a C or Python development environment to test these changes, so they are provided in the hopes someone else can test and confirm they fix the issue. There's probably some room to simplify the code with temporary variables to reduce redundancy. For background, I have implemented precisely this logic in my own (Java-based) project [here](https://github.com/oshi/oshi/blob/3bb9eafbe2062edad4108b7b12501b1f9be27361/oshi-core/src/main/java/oshi/hardware/platform/windows/WindowsVirtualMemory.java#L110-L118) and [here](https://github.com/oshi/oshi/blob/3bb9eafbe2062edad4108b7b12501b1f9be27361/oshi-core/src/main/java/oshi/hardware/platform/windows/WindowsGlobalMemory.java#L253-L263) following a detailed investigation in https://github.com/oshi/oshi/issues/1182 Signed-off-by: Daniel Widdis * Formatting, credits, change log Signed-off-by: Daniel Widdis Signed-off-by: Daniel Widdis --- CREDITS | 4 ++++ HISTORY.rst | 10 ++++++++++ psutil/_psutil_windows.c | 25 +++++++++++++++---------- psutil/_pswindows.py | 6 +++--- 4 files changed, 32 insertions(+), 13 deletions(-) diff --git a/CREDITS b/CREDITS index 6f1b6e5b8..29298ca3e 100644 --- a/CREDITS +++ b/CREDITS @@ -801,3 +801,7 @@ I: 2135 N: Daniel Li I: 2150 + +N: Daniel Widdis +W: https://github.com/dbwiddis +I: 2077 diff --git a/HISTORY.rst b/HISTORY.rst index e98339901..f0d0905d4 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,5 +1,15 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* +5.9.4 (IN DEVELOPMENT) +====================== + +XXXX-XX-XX + +**Bug fixes** + +- 2077_, [Windows]: Use system-level values for `virtual_memory()`. (patch by + Daniel Widdis) + 5.9.3 ===== diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 83da3a26c..6e51c449d 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -610,20 +610,25 @@ psutil_proc_memory_uss(PyObject *self, PyObject *args) { */ static PyObject * psutil_virtual_mem(PyObject *self, PyObject *args) { - MEMORYSTATUSEX memInfo; - memInfo.dwLength = sizeof(MEMORYSTATUSEX); + unsigned long long totalPhys, availPhys, totalSys, availSys, pageSize; + PERFORMANCE_INFORMATION perfInfo; - if (! GlobalMemoryStatusEx(&memInfo)) { + if (! GetPerformanceInfo(&perfInfo, sizeof(PERFORMANCE_INFORMATION))) { PyErr_SetFromWindowsErr(0); return NULL; } - return Py_BuildValue("(LLLLLL)", - memInfo.ullTotalPhys, // total - memInfo.ullAvailPhys, // avail - memInfo.ullTotalPageFile, // total page file - memInfo.ullAvailPageFile, // avail page file - memInfo.ullTotalVirtual, // total virtual - memInfo.ullAvailVirtual); // avail virtual + // values are size_t, widen (if needed) to long long + pageSize = perfInfo.PageSize; + totalPhys = perfInfo.PhysicalTotal * pageSize; + availPhys = perfInfo.PhysicalAvailable * pageSize; + totalSys = perfInfo.CommitLimit * pageSize; + availSys = totalSys - perfInfo.CommitTotal * pageSize; + return Py_BuildValue( + "(LLLL)", + totalPhys, + availPhys, + totalSys, + availSys); } diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index 7d882b774..b546f15d8 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -229,7 +229,7 @@ def getpagesize(): def virtual_memory(): """System virtual memory as a namedtuple.""" mem = cext.virtual_mem() - totphys, availphys, totpagef, availpagef, totvirt, freevirt = mem + totphys, availphys, totsys, availsys = mem # total = totphys avail = availphys @@ -248,8 +248,8 @@ def swap_memory(): total_system = mem[2] free_system = mem[3] - # Despite the name PageFile refers to total system memory here - # thus physical memory values need to be subtracted to get swap values + # system memory (commit total/limit) is the sum of physical and swap + # thus physical memory values need to be substracted to get swap values total = total_system - total_phys free = min(total, free_system - free_phys) used = total - free From 8e58bce0409db7415b963fc394e93a2a4c68d61c Mon Sep 17 00:00:00 2001 From: Amir Rossert Date: Thu, 20 Oct 2022 19:13:28 -0400 Subject: [PATCH 12/16] Linux: fix missing SPEED_UNKNOWN definition (#2156) Signed-off-by: Amir Rossert --- psutil/_psutil_linux.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/psutil/_psutil_linux.c b/psutil/_psutil_linux.c index 1ed1f91a0..0bacb2048 100644 --- a/psutil/_psutil_linux.c +++ b/psutil/_psutil_linux.c @@ -58,6 +58,11 @@ static const int NCPUS_START = sizeof(unsigned long) * CHAR_BIT; #endif +#ifndef SPEED_UNKNOWN + #define SPEED_UNKNOWN -1 +#endif + + #if PSUTIL_HAVE_IOPRIO enum { IOPRIO_WHO_PROCESS = 1, From b0f780a1f46775e7d18a5176c839dac1a0977191 Mon Sep 17 00:00:00 2001 From: Lawrence D'Anna Date: Thu, 20 Oct 2022 19:29:01 -0400 Subject: [PATCH 13/16] build fix for Mac OS, Apple Silicon (#2010) On MacOS, arm64 IFM_1000_TX and IFM_1000_T are the same value, causing a build failure. Signed-off-by: Lawrence D'Anna --- psutil/_psutil_posix.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/psutil/_psutil_posix.c b/psutil/_psutil_posix.c index 07c10398b..24628afc7 100644 --- a/psutil/_psutil_posix.c +++ b/psutil/_psutil_posix.c @@ -708,13 +708,14 @@ int psutil_get_nic_speed(int ifm_active) { case(IFM_1000_LX): // 1000baseLX - single-mode fiber case(IFM_1000_CX): // 1000baseCX - 150ohm STP #if defined(IFM_1000_TX) && !defined(PSUTIL_OPENBSD) + #define HAS_CASE_IFM_1000_TX 1 // FreeBSD 4 and others (but NOT OpenBSD) -> #define IFM_1000_T in net/if_media.h case(IFM_1000_TX): #endif #ifdef IFM_1000_FX case(IFM_1000_FX): #endif -#ifdef IFM_1000_T +#if defined(IFM_1000_T) && (!HAS_CASE_IFM_1000_TX || IFM_1000_T != IFM_1000_TX) case(IFM_1000_T): #endif return 1000; From 57a7a7047e22f79cabb984e883a70888eb370558 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 21 Oct 2022 01:40:23 +0200 Subject: [PATCH 14/16] update HISTORY + give CREDITS for @arossert, @smoofra, @mayeut for #2102, #2156, #2010 --- CREDITS | 10 +++++++++- HISTORY.rst | 12 +++++++++++- psutil/__init__.py | 2 +- psutil/tests/__init__.py | 2 +- psutil/tests/test_osx.py | 4 ++-- 5 files changed, 24 insertions(+), 6 deletions(-) diff --git a/CREDITS b/CREDITS index 29298ca3e..199023159 100644 --- a/CREDITS +++ b/CREDITS @@ -781,7 +781,7 @@ I: 1956 N: Matthieu Darbois W: https://github.com/mayeut -I: 2039, 2142, 2147, 2153, 2040 +I: 2039, 2142, 2147, 2153, 2040, 2102 N: Hugo van Kemenade W: https://github.com/hugovk @@ -805,3 +805,11 @@ I: 2150 N: Daniel Widdis W: https://github.com/dbwiddis I: 2077 + +N: Amir Rossert +W: https://github.com/arossert +I: 2156 + +N: Lawrence D'Anna +W: https://github.com/smoofra +I: 2010 diff --git a/HISTORY.rst b/HISTORY.rst index f0d0905d4..2b1257bd4 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,10 +5,20 @@ XXXX-XX-XX +**Enhancements** + +- 2102_: use Limited API when building wheels with CPython 3.6+ on Linux, + macOS and Windows. This allows to use pre-built wheels in all future versions + of cPython 3. (patch by Matthieu Darbois) + **Bug fixes** -- 2077_, [Windows]: Use system-level values for `virtual_memory()`. (patch by +- 2077_, [Windows]: Use system-level values for `virtual_memory()`_. (patch by Daniel Widdis) +- 2156_, [Linux]: compilation may fail on very old gcc compilers due to missing + ``SPEED_UNKNOWN`` definition. (patch by Amir Rossert) +- 2010_, [macOS]: on MacOS, arm64 ``IFM_1000_TX`` and ``IFM_1000_T`` are the + same value, causing a build failure. (patch by Lawrence D'Anna) 5.9.3 ===== diff --git a/psutil/__init__.py b/psutil/__init__.py index ca2d9273c..567427918 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -211,7 +211,7 @@ AF_LINK = _psplatform.AF_LINK __author__ = "Giampaolo Rodola'" -__version__ = "5.9.3" +__version__ = "5.9.4" version_info = tuple([int(num) for num in __version__.split('.')]) _timer = getattr(time, 'monotonic', time.time) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index a7da8d237..ec9c7480a 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -174,7 +174,7 @@ def macos_version(): if CI_TESTING: NO_RETRIES *= 3 GLOBAL_TIMEOUT *= 3 - TOLERANCE_SYS_MEM *= 3 + TOLERANCE_SYS_MEM *= 4 TOLERANCE_DISK_USAGE *= 3 # --- file names diff --git a/psutil/tests/test_osx.py b/psutil/tests/test_osx.py index 8abddb528..3141cef90 100755 --- a/psutil/tests/test_osx.py +++ b/psutil/tests/test_osx.py @@ -192,13 +192,13 @@ def test_vmem_wired(self): def test_swapmem_sin(self): vmstat_val = vm_stat("Pageins") psutil_val = psutil.swap_memory().sin - self.assertEqual(psutil_val, vmstat_val) + self.assertEqual(psutil_val, vmstat_val, delta=TOLERANCE_SYS_MEM) @retry_on_failure() def test_swapmem_sout(self): vmstat_val = vm_stat("Pageout") psutil_val = psutil.swap_memory().sout - self.assertEqual(psutil_val, vmstat_val) + self.assertEqual(psutil_val, vmstat_val, delta=TOLERANCE_SYS_MEM) # Not very reliable. # def test_swapmem_total(self): From 55b588ac48d9b8e37f01c8428090066c01771627 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 21 Oct 2022 01:43:31 +0200 Subject: [PATCH 15/16] fix OSX tests broken by accident --- psutil/tests/test_osx.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/psutil/tests/test_osx.py b/psutil/tests/test_osx.py index 3141cef90..af1264841 100755 --- a/psutil/tests/test_osx.py +++ b/psutil/tests/test_osx.py @@ -192,13 +192,13 @@ def test_vmem_wired(self): def test_swapmem_sin(self): vmstat_val = vm_stat("Pageins") psutil_val = psutil.swap_memory().sin - self.assertEqual(psutil_val, vmstat_val, delta=TOLERANCE_SYS_MEM) + self.assertAlmostEqual(psutil_val, vmstat_val, delta=TOLERANCE_SYS_MEM) @retry_on_failure() def test_swapmem_sout(self): vmstat_val = vm_stat("Pageout") psutil_val = psutil.swap_memory().sout - self.assertEqual(psutil_val, vmstat_val, delta=TOLERANCE_SYS_MEM) + self.assertAlmostEqual(psutil_val, vmstat_val, delta=TOLERANCE_SYS_MEM) # Not very reliable. # def test_swapmem_total(self): From 8e4099d9f063ceb4ee3da5845562c5b934f83544 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 21 Oct 2022 11:34:55 -0700 Subject: [PATCH 16/16] add windows test for free physical mem #2074 --- psutil/tests/test_windows.py | 39 ++++++++++++++++++++++++++++++++++++ scripts/internal/winmake.py | 4 ++-- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 3bf458079..cb2642d19 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -23,6 +23,7 @@ import psutil from psutil import WINDOWS from psutil._compat import FileNotFoundError +from psutil._compat import which from psutil._compat import super from psutil.tests import APPVEYOR from psutil.tests import GITHUB_ACTIONS @@ -31,6 +32,7 @@ from psutil.tests import PY3 from psutil.tests import PYPY from psutil.tests import TOLERANCE_DISK_USAGE +from psutil.tests import TOLERANCE_SYS_MEM from psutil.tests import PsutilTestCase from psutil.tests import mock from psutil.tests import retry_on_failure @@ -62,6 +64,37 @@ class WindowsTestCase(PsutilTestCase): pass +def powershell(cmd): + """Currently not used, but avalable just in case. Usage: + + >>> powershell( + "Get-CIMInstance Win32_PageFileUsage | Select AllocatedBaseSize") + """ + if not which("powershell.exe"): + raise unittest.SkipTest("powershell.exe not available") + cmdline = \ + 'powershell.exe -ExecutionPolicy Bypass -NoLogo -NonInteractive ' + \ + '-NoProfile -WindowStyle Hidden -Command "%s"' % cmd + return sh(cmdline) + + +def wmic(path, what, converter=int): + """Currently not used, but avalable just in case. Usage: + + >>> wmic("Win32_OperatingSystem", "FreePhysicalMemory") + 2134124534 + """ + out = sh("wmic path %s get %s" % (path, what)).strip() + data = "".join(out.splitlines()[1:]).strip() # get rid of the header + if converter is not None: + if "," in what: + return tuple([converter(x) for x in data.split()]) + else: + return converter(data) + else: + return data + + # =================================================================== # System APIs # =================================================================== @@ -123,6 +156,12 @@ def test_total_phymem(self): self.assertEqual(int(w.TotalPhysicalMemory), psutil.virtual_memory().total) + def test_free_phymem(self): + w = wmi.WMI().Win32_PerfRawData_PerfOS_Memory()[0] + self.assertAlmostEqual( + int(w.AvailableBytes), psutil.virtual_memory().free, + delta=TOLERANCE_SYS_MEM) + # @unittest.skipIf(wmi is None, "wmi module is not installed") # def test__UPTIME(self): # # _UPTIME constant is not public but it is used internally diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index bb8800519..466887c2c 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -383,7 +383,7 @@ def setup_dev_env(): sh("%s -m pip install -U %s" % (PYTHON, " ".join(DEPS))) -def check_flake8(): +def flake8(): """Run flake8 against all py files""" py_files = subprocess.check_output("git ls-files") if PY3: @@ -565,7 +565,7 @@ def main(): sp.add_parser('install', help="build + install in develop/edit mode") sp.add_parser('install-git-hooks', help="install GIT pre-commit hook") sp.add_parser('install-pip', help="install pip") - sp.add_parser('check_flake8', help="run flake8 against all py files") + sp.add_parser('flake8', help="run flake8 against all py files") sp.add_parser('print-access-denied', help="print AD exceptions") sp.add_parser('print-api-speed', help="benchmark all API calls") sp.add_parser('setup-dev-env', help="install deps")