From 135baf16aba1348e9dda0c3dd889ddf94aa2cf94 Mon Sep 17 00:00:00 2001 From: joncrall Date: Sat, 12 Aug 2023 15:52:30 -0400 Subject: [PATCH 1/7] Fixing docs --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index efac8533..40141f6f 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -27,7 +27,7 @@ https://docs.readthedocs.io/en/stable/guides/autobuild-docs-for-pull-requests.html - https://readthedocs.org/dashboard/line-profiler/advanced/ + https://readthedocs.org/dashboard/kernprof/advanced/ ensure your github account is connected to readthedocs https://readthedocs.org/accounts/social/connections/ From 2ea46111f46eaabe57711b2db29470786f7f612c Mon Sep 17 00:00:00 2001 From: joncrall Date: Sat, 12 Aug 2023 16:02:18 -0400 Subject: [PATCH 2/7] Better version test --- tests/test_import.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_import.py b/tests/test_import.py index 936989ba..ab9a8f29 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -7,4 +7,9 @@ def test_import(): def test_version(): import line_profiler from packaging.version import Version - assert Version(line_profiler.__version__) + import kernprof + line_profiler_version1 = Version(line_profiler.__version__) + line_profiler_version2 = Version(line_profiler.line_profiler.__version__) + kernprof_version = Version(kernprof.__version__) + assert line_profiler_version1 == line_profiler_version2 == kernprof_version, ( + 'All 3 places should have the same version') From b7f43906bc128453dcfd271e3ca6e1c88ce58e74 Mon Sep 17 00:00:00 2001 From: joncrall Date: Sat, 12 Aug 2023 18:07:53 -0400 Subject: [PATCH 3/7] Add tests and fix explicit profiler issue with cProfile --- README.rst | 14 ++++ kernprof.py | 13 ++- line_profiler/__init__.py | 64 +++++++++++++++ line_profiler/explicit_profiler.py | 4 + requirements/tests.txt | 2 +- tests/complex_example.py | 81 +++++++++++++++++-- tests/test_complex_case.py | 126 +++++++++++++++++++++++++---- tests/test_explicit_profile.py | 58 ++----------- 8 files changed, 284 insertions(+), 78 deletions(-) diff --git a/README.rst b/README.rst index 0da18216..ff014366 100644 --- a/README.rst +++ b/README.rst @@ -288,6 +288,20 @@ built on ``cProfile`` or ``line_profiler`` are as follows: .. _spyder_line_profiler_plugin: https://github.com/spyder-ide/spyder-line-profiler .. _web_profiler_ui: https://github.com/mirecl/pprof + +Related Work +============ + +Check out these other Python profilers: + +* `Scalene `_: A CPU+GPU+memory sampling based profiler. + +* `PyInstrument `_: A call stack profiler. + +* `Yappi `_: A tracing profiler that is multithreading, asyncio and gevent aware. + +* `profile / cProfile `_: The builtin profile module. + Frequently Asked Questions ========================== diff --git a/kernprof.py b/kernprof.py index f25bed83..563c1622 100755 --- a/kernprof.py +++ b/kernprof.py @@ -239,10 +239,17 @@ def positive_float(value): import line_profiler prof = line_profiler.LineProfiler() options.builtin = True - # Overwrite the explicit profile decorator - line_profiler.profile._kernprof_overwrite(prof) else: prof = ContextualProfile() + + # If line_profiler is installed, then overwrite the explicit decorator + try: + import line_profiler + except ImportError: + ... + else: + line_profiler.profile._kernprof_overwrite(prof) + if options.builtin: builtins.__dict__['profile'] = prof @@ -275,7 +282,7 @@ def positive_float(value): print('Wrote profile results to %s' % options.outfile) if options.view: if isinstance(prof, ContextualProfile): - prof.print_stats(stream=original_stdout) + prof.print_stats() else: prof.print_stats(output_unit=options.unit, stripzeros=options.skip_zero, diff --git a/line_profiler/__init__.py b/line_profiler/__init__.py index 1e1ca838..18db6f5d 100644 --- a/line_profiler/__init__.py +++ b/line_profiler/__init__.py @@ -20,6 +20,70 @@ pip install line_profiler + + + +Demo +---- + +The following code gives a demonstration: + +First we generate a demo python script: + +.. code:: bash + + python -c "if 1: + import textwrap + + # Define the script + text = textwrap.dedent( + ''' + from line_profiler import profile + + @profile + def plus(a, b): + return a + b + + @profile + def fib(n): + a, b = 0, 1 + while a < n: + a, b = b, plus(a, b) + + @profile + def main(): + import math + import time + start = time.time() + + print('start calculating') + while time.time() - start < 1: + fib(10) + math.factorial(1000) + print('done calculating') + + main() + ''' + ).strip() + + # Write the script to disk + with open('demo_script.py', 'w') as file: + file.write(text) + " + + +Run the script with kernprof + +.. code:: bash + + python -m kernprof demo_script.py + +.. code:: bash + + python -m pstats demo_script.py.prof + + + """ __submodules__ = [ 'line_profiler', diff --git a/line_profiler/explicit_profiler.py b/line_profiler/explicit_profiler.py index 4b12378e..2d2bf972 100644 --- a/line_profiler/explicit_profiler.py +++ b/line_profiler/explicit_profiler.py @@ -302,6 +302,10 @@ def __call__(self, func): Returns: Callable: a potentially wrapped function """ + # from multiprocessing import current_process + # if current_process().name != 'MainProcess': + # return func + if self.enabled is None: # Force a setup if we haven't done it before. self._implicit_setup() diff --git a/requirements/tests.txt b/requirements/tests.txt index cb3030b0..a9cf4214 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -11,7 +11,7 @@ pytest-cov>=2.8.1 ; python_version < '3.5.0' and python_version >= '3 pytest-cov>=2.8.1 ; python_version < '2.8.0' and python_version >= '2.7.0' # Python 2.7 coverage[toml] >= 5.3 -ubelt >= 1.0.1 +ubelt >= 1.3.3 -r ipython.txt diff --git a/tests/complex_example.py b/tests/complex_example.py index 08db1616..3a5a4707 100644 --- a/tests/complex_example.py +++ b/tests/complex_example.py @@ -1,16 +1,81 @@ """ A script used in test_complex_case.py -""" -import line_profiler -import atexit +python ~/code/line_profiler/tests/complex_example.py +LINE_PROFILE=1 python ~/code/line_profiler/tests/complex_example.py +python -m kernprof -v ~/code/line_profiler/tests/complex_example.py +python -m kernprof -lv ~/code/line_profiler/tests/complex_example.py + + +cd ~/code/line_profiler/tests/ + + +# Run the code by itself without any profiling +PROFILE_TYPE=none python complex_example.py + + +# NOTE: this fails because we are not running with kernprof +PROFILE_TYPE=implicit python complex_example.py + + +# Kernprof with the implicit profile decorator +# NOTE: multiprocessing breaks this invocation, so set process size to zero +PROFILE_TYPE=implicit python -m kernprof -b complex_example.py --process_size=0 +python -m pstats ./complex_example.py.prof -profile = line_profiler.LineProfiler() +# Explicit decorator with line kernprof +# NOTE: again, multiprocessing breaks when using kernprof (does this happen in older verions?) +PROFILE_TYPE=explicit python -m kernprof -l complex_example.py --process_size=0 +python -m line_profiler complex_example.py.lprof -@atexit.register -def _show_profile_on_end(): - profile.print_stats(summarize=1, sort=1, stripzeros=1) + +# Explicit decorator with cProfile kernprof +# NOTE: again, multiprocessing breaks when using kernprof (does this happen in older verions?) +PROFILE_TYPE=explicit python -m kernprof -b complex_example.py --process_size=0 +python -m pstats ./complex_example.py.prof + +# Explicit decorator with environment enabling +PROFILE_TYPE=explicit LINE_PROFILE=1 python complex_example.py + + +# Explicit decorator without enabling it +PROFILE_TYPE=explicit LINE_PROFILE=0 python complex_example.py + + +# Use a custom defined line profiler object +PROFILE_TYPE=custom python complex_example.py + +""" +import os + +# The test will define how we expect the profile decorator to exist +PROFILE_TYPE = os.environ.get('PROFILE_TYPE', '') + + +if PROFILE_TYPE == 'implicit': + # Do nothing, assume kernprof will inject profile in for us + ... +elif PROFILE_TYPE == 'none': + # Define a no-op profiler + def profile(func): + return func +elif PROFILE_TYPE == 'explicit': + # Use the explicit profile decorator + import line_profiler + profile = line_profiler.profile +elif PROFILE_TYPE == 'custom': + # Create a custom profile decorator + import line_profiler + import atexit + profile = line_profiler.LineProfiler() + + @atexit.register + def _show_profile_on_end(): + ... + profile.print_stats(summarize=1, sort=1, stripzeros=1, rich=1) +else: + raise KeyError('') @profile @@ -92,7 +157,7 @@ def main(): @profile def funcy_fib(n): """ - Alternatite fib function where code splits out over multiple lines + Alternative fib function where code splits out over multiple lines """ a, b = ( 0, 1 diff --git a/tests/test_complex_case.py b/tests/test_complex_case.py index 6ce65cec..53a9f833 100644 --- a/tests/test_complex_case.py +++ b/tests/test_complex_case.py @@ -1,21 +1,117 @@ -def test_complex_example(): - import sys - import pathlib - import subprocess - import os +# TODO: use tempdir to run each command in isolation +import ubelt as ub + +def get_complex_example_fpath(): try: - test_dpath = pathlib.Path(__file__).parent + test_dpath = ub.Path(__file__).parent except NameError: # for development - test_dpath = pathlib.Path('~/code/line_profiler/tests').expanduser() - + test_dpath = ub.Path('~/code/line_profiler/tests').expanduser() complex_fpath = test_dpath / 'complex_example.py' + return complex_fpath + + +def test_complex_example_python_none(): + complex_fpath = get_complex_example_fpath() + info = ub.cmd(f'PROFILE_TYPE=none python {complex_fpath}', shell=True, verbose=3) + assert info.stdout() == '' + info.check_returncode() + + +def test_varied_complex_invocations(): + import sys + import os + # import tempfile + + # Enumerate valid cases to test + cases = [] + for runner in ['python', 'kernprof']: + for env_line_profile in ['0', '1']: + if runner == 'kernprof': + for profile_type in ['explicit', 'implicit']: + for kern_flags in ['-l', '-b']: + if 'l' in kern_flags: + outpath = 'complex_example.py.lprof' + else: + outpath = 'complex_example.py.prof' + + cases.append({ + 'runner': runner, + 'kern_flags': kern_flags, + 'env_line_profile': env_line_profile, + 'profile_type': profile_type, + 'outpath': outpath, + }) + else: + if env_line_profile == '1': + outpath = 'profile_output.txt' + else: + outpath = None + cases.append({ + 'runner': runner, + 'env_line_profile': env_line_profile, + 'profile_type': 'explicit', + 'outpath': outpath, + }) + + complex_fpath = get_complex_example_fpath() + # os.environ.copy() + + results = [] + + for item in cases: + # temp_dpath = tempfile.mkdtemp() + + env = {} + + outpath = item['outpath'] + if outpath: + outpath = ub.Path(outpath) + + # Construct the invocation for each case + if item['runner'] == 'kernprof': + kern_flags = item['kern_flags'] + # Note: kernprof doesn't seem to play well with multiprocessing + prog_flags = ' --process_size=0' + runner = f'{sys.executable} -m kernprof {kern_flags}' + else: + env['LINE_PROFILE'] = item["env_line_profile"] + runner = f'{sys.executable}' + prog_flags = '' + env['PROFILE_TYPE'] = item["profile_type"] + command = f'{runner} {complex_fpath}' + prog_flags + + HAS_SHELL = 1 + if HAS_SHELL: + # Use shell because it gives a indication of what is happening + environ_prefix = ' '.join([f'{k}={v}' for k, v in env.items()]) + info = ub.cmd(environ_prefix + ' ' + command, shell=True, verbose=3) + else: + env = ub.udict(os.environ) | env + info = ub.cmd(command, env=env, verbose=3) + + info.check_returncode() + + result = item.copy() + if outpath: + result['outsize'] = outpath.stat().st_size + else: + result['outsize'] = None + results.append(result) + + if outpath: + assert outpath.exists() + assert outpath.is_file() + outpath.delete() + + if 0: + import pandas as pd + import rich + table = pd.DataFrame(results) + rich.print(table) - from subprocess import PIPE - proc = subprocess.run([sys.executable, os.fspath(complex_fpath)], stdout=PIPE, - stderr=PIPE, universal_newlines=True) - print(proc.stdout) - print(proc.stderr) - print(proc.returncode) - proc.check_returncode() + # Ensure the scripts that produced output produced non-trivial output + for result in results: + if result['outpath'] is not None: + assert result['outsize'] > 100 diff --git a/tests/test_explicit_profile.py b/tests/test_explicit_profile.py index f260d0c4..d56265ff 100644 --- a/tests/test_explicit_profile.py +++ b/tests/test_explicit_profile.py @@ -6,6 +6,7 @@ import subprocess import textwrap from subprocess import PIPE +import ubelt as ub def _demo_explicit_profile_script(): @@ -28,7 +29,7 @@ def test_explicit_profile_with_nothing(): Test that no profiling happens when we dont request it. """ temp_dpath = pathlib.Path(tempfile.mkdtemp()) - with ChDir(temp_dpath): + with ub.ChDir(temp_dpath): script_fpath = pathlib.Path('script.py') script_fpath.write_text(_demo_explicit_profile_script()) @@ -54,7 +55,7 @@ def test_explicit_profile_with_environ_on(): env = os.environ.copy() env['LINE_PROFILE'] = '1' - with ChDir(temp_dpath): + with ub.ChDir(temp_dpath): script_fpath = pathlib.Path('script.py') script_fpath.write_text(_demo_explicit_profile_script()) @@ -80,7 +81,7 @@ def test_explicit_profile_with_environ_off(): env = os.environ.copy() env['LINE_PROFILE'] = '0' - with ChDir(temp_dpath): + with ub.ChDir(temp_dpath): script_fpath = pathlib.Path('script.py') script_fpath.write_text(_demo_explicit_profile_script()) @@ -107,7 +108,7 @@ def test_explicit_profile_with_cmdline(): """ temp_dpath = pathlib.Path(tempfile.mkdtemp()) - with ChDir(temp_dpath): + with ub.ChDir(temp_dpath): script_fpath = pathlib.Path('script.py') script_fpath.write_text(_demo_explicit_profile_script()) @@ -132,7 +133,7 @@ def test_explicit_profile_with_kernprof(): """ temp_dpath = pathlib.Path(tempfile.mkdtemp()) - with ChDir(temp_dpath): + with ub.ChDir(temp_dpath): script_fpath = pathlib.Path('script.py') script_fpath.write_text(_demo_explicit_profile_script()) @@ -188,7 +189,7 @@ def func4(a): func3(1) func4(1) ''').strip() - with ChDir(temp_dpath): + with ub.ChDir(temp_dpath): script_fpath = pathlib.Path('script.py') script_fpath.write_text(code) @@ -211,48 +212,3 @@ def func4(a): assert output_fpath.exists() assert (temp_dpath / 'custom_output.lprof').exists() shutil.rmtree(temp_dpath) - - -class ChDir: - """ - Context manager that changes the current working directory and then - returns you to where you were. - - This is nearly the same as the stdlib :func:`contextlib.chdir`, with the - exception that it will do nothing if the input path is None (i.e. the user - did not want to change directories). - - Args: - dpath (str | PathLike | None): - The new directory to work in. - If None, then the context manager is disabled. - - SeeAlso: - :func:`contextlib.chdir` - """ - def __init__(self, dpath): - self._context_dpath = dpath - self._orig_dpath = None - - def __enter__(self): - """ - Returns: - ChDir: self - """ - if self._context_dpath is not None: - self._orig_dpath = os.getcwd() - os.chdir(self._context_dpath) - return self - - def __exit__(self, ex_type, ex_value, ex_traceback): - """ - Args: - ex_type (Type[BaseException] | None): - ex_value (BaseException | None): - ex_traceback (TracebackType | None): - - Returns: - bool | None - """ - if self._context_dpath is not None: - os.chdir(self._orig_dpath) From 81d411076f6829f84d41b8a2203f760edbc911af Mon Sep 17 00:00:00 2001 From: joncrall Date: Sat, 12 Aug 2023 19:40:49 -0400 Subject: [PATCH 4/7] Update docs and tests --- README.rst | 16 +++- docs/source/conf.py | 1 + kernprof.py | 9 ++ line_profiler/__init__.py | 133 +++++++++++++++++++---------- line_profiler/_line_profiler.pyx | 6 ++ line_profiler/explicit_profiler.py | 36 ++++++-- line_profiler/ipython_extension.py | 7 +- line_profiler/line_profiler.py | 24 +++++- tests/complex_example.py | 2 +- tests/test_complex_case.py | 123 +++++++++++++------------- 10 files changed, 241 insertions(+), 116 deletions(-) diff --git a/README.rst b/README.rst index ff014366..53020d3b 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,7 @@ line_profiler and kernprof -------------------------- -|Pypi| |Downloads| |CircleCI| |GithubActions| |Codecov| +|Pypi| |ReadTheDocs| |Downloads| |CircleCI| |GithubActions| |Codecov| NOTICE: This is the official ``line_profiler`` repository. The most recent @@ -11,6 +11,14 @@ points to this repo. The original `@rkern `_ is unmaintained. This fork is the official continuation of the project. ++---------------+--------------------------------------------+ +| Github | https://github.com/pyutils/line_profiler | ++---------------+--------------------------------------------+ +| Pypi | https://pypi.org/project/line_profiler | ++---------------+--------------------------------------------+ +| ReadTheDocs | https://kernprof.readthedocs.io/en/latest/ | ++---------------+--------------------------------------------+ + ---- @@ -184,6 +192,9 @@ item to the extensions list:: 'line_profiler', ] +Or explicitly call:: + + %load_ext line_profiler To get usage help for %lprun, use the standard IPython help mechanism:: @@ -440,4 +451,5 @@ See `CHANGELOG`_. :target: https://pypistats.org/packages/line_profiler .. |GithubActions| image:: https://github.com/pyutils/line_profiler/actions/workflows/tests.yml/badge.svg?branch=main :target: https://github.com/pyutils/line_profiler/actions?query=branch%3Amain - +.. |ReadTheDocs| image:: https://readthedocs.org/projects/ubelt/badge/?version=latest + :target: http://ubelt.readthedocs.io/en/latest/ diff --git a/docs/source/conf.py b/docs/source/conf.py index 40141f6f..73c34f09 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -164,6 +164,7 @@ def visit_Assign(self, node): 'networkx': ('https://networkx.org/documentation/stable/', None), 'scriptconfig': ('https://scriptconfig.readthedocs.io/en/latest/', None), 'xdev': ('https://xdev.readthedocs.io/en/latest/', None), + 'rich': ('https://rich.readthedocs.io/en/latest/', None), } __dev_note__ = """ diff --git a/kernprof.py b/kernprof.py index 563c1622..13d5636f 100755 --- a/kernprof.py +++ b/kernprof.py @@ -203,6 +203,8 @@ def positive_float(value): help='Code to execute before the code to profile') parser.add_argument('-v', '--view', action='store_true', help='View the results of the profile in addition to saving it') + parser.add_argument('-r', '--rich', action='store_true', + help='Use rich formatting if viewing output') parser.add_argument('-u', '--unit', default='1e-6', type=positive_float, help='Output unit (in seconds) in which the timing info is ' @@ -286,7 +288,14 @@ def positive_float(value): else: prof.print_stats(output_unit=options.unit, stripzeros=options.skip_zero, + rich=options.rich, stream=original_stdout) + else: + print('Inspect results with:') + if isinstance(prof, ContextualProfile): + print(f'{sys.executable} -m pstats "{options.outfile}"') + else: + print(f'{sys.executable} -m line_profiler -rmt "{options.outfile}"') if __name__ == '__main__': diff --git a/line_profiler/__init__.py b/line_profiler/__init__.py index 18db6f5d..5f554b51 100644 --- a/line_profiler/__init__.py +++ b/line_profiler/__init__.py @@ -4,11 +4,13 @@ The line_profiler module for doing line-by-line profiling of functions -+---------------+-------------------------------------------+ -| Github | https://github.com/pyutils/line_profiler | -+---------------+-------------------------------------------+ -| Pypi | https://pypi.org/project/line_profiler | -+---------------+-------------------------------------------+ ++---------------+--------------------------------------------+ +| Github | https://github.com/pyutils/line_profiler | ++---------------+--------------------------------------------+ +| Pypi | https://pypi.org/project/line_profiler | ++---------------+--------------------------------------------+ +| ReadTheDocs | https://kernprof.readthedocs.io/en/latest/ | ++---------------+--------------------------------------------+ Installation @@ -21,70 +23,113 @@ pip install line_profiler +Basic Usage +=========== +To demonstrate line profiling, we first need to generate a Python script to +profile. Write the following code to a file called ``demo_primes.py``. -Demo ----- +.. code:: python -The following code gives a demonstration: + from line_profiler import profile -First we generate a demo python script: -.. code:: bash + @profile + def is_prime(n): + ''' + Check if the number "n" is prime, with n > 1. + + Returns a boolean, True if n is prime. + ''' + max_val = n ** 0.5 + stop = int(max_val + 1) + for i in range(2, stop): + if n % i == 0: + return False + return True + + + @profile + def find_primes(size): + primes = [] + for n in range(size): + flag = is_prime(n) + if flag: + primes.append(n) + return primes + + + @profile + def main(): + print('start calculating') + primes = find_primes(100000) + print(f'done calculating. Found {len(primes)} primes.') + + main() + + +In this script we explicitly import the ``profile`` function from +``line_profiler``, and then we decorate function of interest with ``@profile``. - python -c "if 1: - import textwrap +By default nothing is profiled when running the script. - # Define the script - text = textwrap.dedent( - ''' - from line_profiler import profile +.. code:: bash - @profile - def plus(a, b): - return a + b + python demo_primes.py - @profile - def fib(n): - a, b = 0, 1 - while a < n: - a, b = b, plus(a, b) - @profile - def main(): - import math - import time - start = time.time() +The output will be - print('start calculating') - while time.time() - start < 1: - fib(10) - math.factorial(1000) - print('done calculating') +.. code:: - main() - ''' - ).strip() + start calculating + done calculating. Found 9594 primes. - # Write the script to disk - with open('demo_script.py', 'w') as file: - file.write(text) - " +The quickest way to enable profiling is to set the environment variable +``LINE_PROFILE=1`` and running your script as normal. -Run the script with kernprof .. code:: bash - python -m kernprof demo_script.py + LINE_PROFILE=1 python demo_primes.py + +This will output 3 files: profile_output.txt, profile_output_.txt, +and profile_output.lprof and stdout will look something like: + + +.. code:: + + start calculating + done calculating. Found 9594 primes. + Timer unit: 1e-09 s + + 0.65 seconds - demo_primes.py:4 - is_prime + 1.47 seconds - demo_primes.py:19 - find_primes + 1.51 seconds - demo_primes.py:29 - main + Wrote profile results to profile_output.txt + Wrote profile results to profile_output_2023-08-12T193302.txt + Wrote profile results to profile_output.lprof + To view details run: + python -m line_profiler -rtmz profile_output.lprof + + +For more control over the outputs, run your script using :py:mod:`kernprof`. +The following invocation will run your script, dump results to +``demo_primes.py.lprof``, and display results. .. code:: bash - python -m pstats demo_script.py.prof + python -m kernprof -lvr demo_primes.py +Note: the ``-r`` flag will use "rich-output" if you have the :py:mod:`rich` +module installed. """ +# Note: there are better ways to generate primes +# https://github.com/Sylhare/nprime + __submodules__ = [ 'line_profiler', 'ipython_extension', diff --git a/line_profiler/_line_profiler.pyx b/line_profiler/_line_profiler.pyx index f2a23f4b..f83c5b82 100644 --- a/line_profiler/_line_profiler.pyx +++ b/line_profiler/_line_profiler.pyx @@ -1,4 +1,7 @@ #cython: language_level=3 +""" +This is the Cython backend used in :py:mod:`line_profiler.line_profiler`. +""" from .python25 cimport PyFrameObject, PyObject, PyStringObject from sys import byteorder cimport cython @@ -160,6 +163,9 @@ cdef class LineProfiler: """ Time the execution of lines of Python code. + This is the Cython base class for + :class:`line_profiler.line_profiler.LineProfiler`. + Example: >>> import copy >>> import line_profiler diff --git a/line_profiler/explicit_profiler.py b/line_profiler/explicit_profiler.py index 2d2bf972..240c7d5e 100644 --- a/line_profiler/explicit_profiler.py +++ b/line_profiler/explicit_profiler.py @@ -1,11 +1,16 @@ """ -The idea is that we are going to expose a top-level ``profile`` decorator which -will be disabled by default **unless** you are running with with line profiler -itself OR if the LINE_PROFILE environment variable is True. +New in ``line_profiler`` version 4.1.0, this module defines a top-level +``profile`` decorator which will be disabled by default **unless** a script is +being run with :mod:`kernprof`, if the environment variable ``LINE_PROFILE`` is +set, or if ``--line-profile`` is given on the command line. -This uses the :mod:`atexit` module to perform a profile dump at the end. +In the latter two cases, the :mod:`atexit` module is used to display and dump +line profiling results to disk when Python exits. -This work is ported from :mod:`xdev`. +If none of the enabling conditions are met, then +:py:obj:`line_profiler.profile` is a noop. This means you no longer have to add +and remove the implicit ``profile`` decorators required by previous version of +this library. Basic usage is to import line_profiler and decorate your function with line_profiler.profile. By default this does nothing, it's a no-op decorator. @@ -13,7 +18,8 @@ ``'--profile' in sys.argv'``, then it enables profiling and at the end of your script it will output the profile text. -Here is a minimal example: +Here is a minimal example that will write a script to disk and then run it +with profiling enabled or disabled by various methods: .. code:: bash @@ -71,7 +77,9 @@ def main(): LINE_PROFILE=1 python demo.py -An example with in-code enabling: +The explicit :py:attr:`line_profiler.profile` decorator can also be enabled and +configured in the Python code itself by calling +:func:`line_profiler.profile.enable`. The following example demonstrates this: .. code:: bash @@ -99,7 +107,10 @@ def fib(n): python demo.py -An example with in-code enabling and disabling: +Likewise there is a :func:`line_profiler.profile.disable` function that will +prevent any subsequent functions decorated with ``@profile`` from being +profiled. In the following example, profiling information will only be recorded +for ``func2`` and ``func4``. .. code:: bash @@ -145,7 +156,11 @@ def func4(): echo "---" echo "## Configuration handled inside the script" python demo.py + + # Running with --line-profile will also profile ``func1`` python demo.py --line-profile + +The core functionality in this module was ported from :mod:`xdev`. """ from .line_profiler import LineProfiler import sys @@ -160,6 +175,8 @@ class GlobalProfiler: """ Manages a profiler that will output on interpreter exit. + The :py:obj:`line_profile.profile` decorator is an instance of this object. + Attributes: setup_config (Dict[str, List[str]]): Determines how the implicit setup behaves by defining which @@ -317,7 +334,8 @@ def show(self): """ Write the managed profiler stats to enabled outputs. - If the implicit setup triggered, then this will be called by atexit. + If the implicit setup triggered, then this will be called by + :py:mod:`atexit`. """ import io import pathlib diff --git a/line_profiler/ipython_extension.py b/line_profiler/ipython_extension.py index 87c662ac..efa17789 100644 --- a/line_profiler/ipython_extension.py +++ b/line_profiler/ipython_extension.py @@ -1,3 +1,7 @@ +""" +This module defines the ``%lprun`` IPython magic. +""" + from io import StringIO from IPython.core.magic import Magics, magics_class, line_magic @@ -16,7 +20,8 @@ def lprun(self, parameter_s=""): line_profiler module. Usage: - %lprun -f func1 -f func2 + + %lprun -f func1 -f func2 The given statement (which doesn't require quote marks) is run via the LineProfiler. Profiling is enabled for the functions specified by the -f diff --git a/line_profiler/line_profiler.py b/line_profiler/line_profiler.py index 79c24eb7..3d48aeca 100755 --- a/line_profiler/line_profiler.py +++ b/line_profiler/line_profiler.py @@ -1,4 +1,9 @@ #!/usr/bin/env python +""" +This module defines the core :class:`LineProfiler` class as well as methods to +inspect its output. This depends on the :py:mod:`line_profiler._line_profiler` +Cython backend. +""" import pickle import functools import inspect @@ -46,7 +51,21 @@ def is_classmethod(f): class LineProfiler(CLineProfiler): - """ A profiler that records the execution times of individual lines. + """ + A profiler that records the execution times of individual lines. + + This provides the core line-profiler functionality. + + Example: + >>> import line_profiler + >>> profile = line_profiler.LineProfiler() + >>> @profile + >>> def func(): + >>> x1 = [_ for i in range(10)] + >>> x2 = [_ for i in range(100)] + >>> x3 = [_ for i in range(1000)] + >>> func() + >>> profile.print_stats() """ def __call__(self, func): @@ -457,6 +476,9 @@ def load_stats(filename): def main(): + """ + The line profiler CLI to view output from ``kernprof -l``. + """ def positive_float(value): val = float(value) if val <= 0: diff --git a/tests/complex_example.py b/tests/complex_example.py index 3a5a4707..b72810d4 100644 --- a/tests/complex_example.py +++ b/tests/complex_example.py @@ -25,7 +25,7 @@ # Explicit decorator with line kernprof -# NOTE: again, multiprocessing breaks when using kernprof (does this happen in older verions?) +# NOTE: again, multiprocessing breaks when using kernprof PROFILE_TYPE=explicit python -m kernprof -l complex_example.py --process_size=0 python -m line_profiler complex_example.py.lprof diff --git a/tests/test_complex_case.py b/tests/test_complex_case.py index 53a9f833..03abcf0d 100644 --- a/tests/test_complex_case.py +++ b/tests/test_complex_case.py @@ -1,5 +1,5 @@ -# TODO: use tempdir to run each command in isolation import ubelt as ub +import tempfile def get_complex_example_fpath(): @@ -13,16 +13,24 @@ def get_complex_example_fpath(): def test_complex_example_python_none(): + """ + Make sure the complex example script works without any profiling + """ complex_fpath = get_complex_example_fpath() info = ub.cmd(f'PROFILE_TYPE=none python {complex_fpath}', shell=True, verbose=3) - assert info.stdout() == '' + assert info.stdout == '' info.check_returncode() def test_varied_complex_invocations(): + """ + Tests variations of running the complex example: + with / without kernprof + with cProfile / LineProfiler backends + with / without explicit profiler + """ import sys import os - # import tempfile # Enumerate valid cases to test cases = [] @@ -56,62 +64,61 @@ def test_varied_complex_invocations(): }) complex_fpath = get_complex_example_fpath() - # os.environ.copy() results = [] for item in cases: - # temp_dpath = tempfile.mkdtemp() - - env = {} - - outpath = item['outpath'] - if outpath: - outpath = ub.Path(outpath) - - # Construct the invocation for each case - if item['runner'] == 'kernprof': - kern_flags = item['kern_flags'] - # Note: kernprof doesn't seem to play well with multiprocessing - prog_flags = ' --process_size=0' - runner = f'{sys.executable} -m kernprof {kern_flags}' - else: - env['LINE_PROFILE'] = item["env_line_profile"] - runner = f'{sys.executable}' - prog_flags = '' - env['PROFILE_TYPE'] = item["profile_type"] - command = f'{runner} {complex_fpath}' + prog_flags - - HAS_SHELL = 1 - if HAS_SHELL: - # Use shell because it gives a indication of what is happening - environ_prefix = ' '.join([f'{k}={v}' for k, v in env.items()]) - info = ub.cmd(environ_prefix + ' ' + command, shell=True, verbose=3) - else: - env = ub.udict(os.environ) | env - info = ub.cmd(command, env=env, verbose=3) - - info.check_returncode() - - result = item.copy() - if outpath: - result['outsize'] = outpath.stat().st_size - else: - result['outsize'] = None - results.append(result) - - if outpath: - assert outpath.exists() - assert outpath.is_file() - outpath.delete() - - if 0: - import pandas as pd - import rich - table = pd.DataFrame(results) - rich.print(table) - - # Ensure the scripts that produced output produced non-trivial output - for result in results: - if result['outpath'] is not None: - assert result['outsize'] > 100 + temp_dpath = tempfile.mkdtemp() + with ub.ChDir(temp_dpath): + env = {} + + outpath = item['outpath'] + if outpath: + outpath = ub.Path(outpath) + + # Construct the invocation for each case + if item['runner'] == 'kernprof': + kern_flags = item['kern_flags'] + # Note: kernprof doesn't seem to play well with multiprocessing + prog_flags = ' --process_size=0' + runner = f'{sys.executable} -m kernprof {kern_flags}' + else: + env['LINE_PROFILE'] = item["env_line_profile"] + runner = f'{sys.executable}' + prog_flags = '' + env['PROFILE_TYPE'] = item["profile_type"] + command = f'{runner} {complex_fpath}' + prog_flags + + HAS_SHELL = 1 + if HAS_SHELL: + # Use shell because it gives a indication of what is happening + environ_prefix = ' '.join([f'{k}={v}' for k, v in env.items()]) + info = ub.cmd(environ_prefix + ' ' + command, shell=True, verbose=3) + else: + env = ub.udict(os.environ) | env + info = ub.cmd(command, env=env, verbose=3) + + info.check_returncode() + + result = item.copy() + if outpath: + result['outsize'] = outpath.stat().st_size + else: + result['outsize'] = None + results.append(result) + + if outpath: + assert outpath.exists() + assert outpath.is_file() + outpath.delete() + + if 0: + import pandas as pd + import rich + table = pd.DataFrame(results) + rich.print(table) + + # Ensure the scripts that produced output produced non-trivial output + for result in results: + if result['outpath'] is not None: + assert result['outsize'] > 100 From d6839252a09acd3602df7b67d8f454cf31a81e3b Mon Sep 17 00:00:00 2001 From: joncrall Date: Sat, 12 Aug 2023 19:44:57 -0400 Subject: [PATCH 5/7] wip --- tests/test_complex_case.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_complex_case.py b/tests/test_complex_case.py index 03abcf0d..b4445c2c 100644 --- a/tests/test_complex_case.py +++ b/tests/test_complex_case.py @@ -32,6 +32,8 @@ def test_varied_complex_invocations(): import sys import os + LINUX = sys.platform.startswith('linux') + # Enumerate valid cases to test cases = [] for runner in ['python', 'kernprof']: @@ -89,7 +91,7 @@ def test_varied_complex_invocations(): env['PROFILE_TYPE'] = item["profile_type"] command = f'{runner} {complex_fpath}' + prog_flags - HAS_SHELL = 1 + HAS_SHELL = LINUX if HAS_SHELL: # Use shell because it gives a indication of what is happening environ_prefix = ' '.join([f'{k}={v}' for k, v in env.items()]) From 6e7b4c7c540f3e7d57b92c09c8b7d87ef3260e86 Mon Sep 17 00:00:00 2001 From: joncrall Date: Sat, 12 Aug 2023 20:23:51 -0400 Subject: [PATCH 6/7] wip --- README.rst | 2 +- tests/test_complex_case.py | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index 53020d3b..fd275acd 100644 --- a/README.rst +++ b/README.rst @@ -451,5 +451,5 @@ See `CHANGELOG`_. :target: https://pypistats.org/packages/line_profiler .. |GithubActions| image:: https://github.com/pyutils/line_profiler/actions/workflows/tests.yml/badge.svg?branch=main :target: https://github.com/pyutils/line_profiler/actions?query=branch%3Amain -.. |ReadTheDocs| image:: https://readthedocs.org/projects/ubelt/badge/?version=latest +.. |ReadTheDocs| image:: https://readthedocs.org/projects/kernprof/badge/?version=latest :target: http://ubelt.readthedocs.io/en/latest/ diff --git a/tests/test_complex_case.py b/tests/test_complex_case.py index b4445c2c..7745f1b2 100644 --- a/tests/test_complex_case.py +++ b/tests/test_complex_case.py @@ -1,5 +1,8 @@ -import ubelt as ub +import os +import sys import tempfile +import ubelt as ub +LINUX = sys.platform.startswith('linux') def get_complex_example_fpath(): @@ -17,7 +20,7 @@ def test_complex_example_python_none(): Make sure the complex example script works without any profiling """ complex_fpath = get_complex_example_fpath() - info = ub.cmd(f'PROFILE_TYPE=none python {complex_fpath}', shell=True, verbose=3) + info = ub.cmd(f'python {complex_fpath}', shell=True, verbose=3, env=ub.udict(os.environ) | {'PROFILE_TYPE': 'none'}) assert info.stdout == '' info.check_returncode() @@ -29,10 +32,6 @@ def test_varied_complex_invocations(): with cProfile / LineProfiler backends with / without explicit profiler """ - import sys - import os - - LINUX = sys.platform.startswith('linux') # Enumerate valid cases to test cases = [] From 480633fcb72aed970b4b272c3c3bddcd04624eec Mon Sep 17 00:00:00 2001 From: joncrall Date: Sat, 12 Aug 2023 21:09:21 -0400 Subject: [PATCH 7/7] wip --- line_profiler/line_profiler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/line_profiler/line_profiler.py b/line_profiler/line_profiler.py index 3d48aeca..dc5188cb 100755 --- a/line_profiler/line_profiler.py +++ b/line_profiler/line_profiler.py @@ -61,9 +61,9 @@ class LineProfiler(CLineProfiler): >>> profile = line_profiler.LineProfiler() >>> @profile >>> def func(): - >>> x1 = [_ for i in range(10)] - >>> x2 = [_ for i in range(100)] - >>> x3 = [_ for i in range(1000)] + >>> x1 = list(range(10)) + >>> x2 = list(range(100)) + >>> x3 = list(range(1000)) >>> func() >>> profile.print_stats() """