Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
  • Loading branch information
@nedbat
nedbat committed Sep 4, 2024
1 parent 39d3111 commit 17b2baf
Show file tree
Hide file tree
Showing 4 changed files with 95 additions and 104 deletions.
7 changes: 7 additions & 0 deletions coverage/core.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,13 @@ def __init__(self, warn: TWarnFn, config: CoverageConfig, metacov: bool) -> None
warn("sys.monitoring isn't available, using default core", slug="no-sysmon")
core_name = None

if core_name == "sysmon" and config.branch and not env.PYBEHAVIOR.branch_taken:
warn(
"sys.monitoring can't yet measure branches well, using default core",
slug="no-sysmon",
)
core_name = None

if not core_name:
# Once we're comfortable with sysmon as a default:
# if env.PYBEHAVIOR.pep669 and self.should_start_context is None:
Expand Down
2 changes: 1 addition & 1 deletion coverage/env.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ class PYBEHAVIOR:
# Does sys.monitoring support BRANCH_TAKEN?
branch_taken = (
pep669 and
hasattr(sys.monitoring.events, "BRANCH_TAKEN") # type:ignore[attr-defined]
hasattr(sys.monitoring.events, "BRANCH_TAKEN") # type:ignore[attr-defined,unused-ignore]
)


Expand Down
182 changes: 80 additions & 102 deletions coverage/sysmon.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,13 @@
import traceback

from dataclasses import dataclass
from types import CodeType, FrameType
from types import CodeType
from typing import (
Any,
Callable,
NewType,
Optional,
Set,
TYPE_CHECKING,
cast,
)

Expand All @@ -40,17 +41,16 @@

# pylint: disable=unused-argument

LOG = False
# $set_env.py: COVERAGE_LOG_SYSMON - Log sys.monitoring activity
LOG = bool(int(os.getenv("COVERAGE_LOG_SYSMON", 0)))

# This module will be imported in all versions of Python, but only used in 3.12+
# It will be type-checked for 3.12, but not for earlier versions.
sys_monitoring = getattr(sys, "monitoring", None)

if TYPE_CHECKING:
assert sys_monitoring is not None
# I want to say this but it's not allowed:
# MonitorReturn = Literal[sys.monitoring.DISABLE] | None
MonitorReturn = Any
DISABLE_TYPE = NewType("DISABLE_TYPE", object)
MonitorReturn = Optional[DISABLE_TYPE]
DISABLE = cast(MonitorReturn, getattr(sys_monitoring, "DISABLE", None))


if LOG: # pragma: debugging
Expand All @@ -73,7 +73,10 @@ def _wrapped(*args: Any, **kwargs: Any) -> Any:
assert sys_monitoring is not None

short_stack = functools.partial(
short_stack, full=True, short_filenames=True, frame_ids=True,
short_stack,
full=True,
short_filenames=True,
frame_ids=True,
)
seen_threads: set[int] = set()

Expand Down Expand Up @@ -127,7 +130,9 @@ def _wrapped(self: Any, *args: Any) -> Any:
return ret
except Exception as exc:
log(f"!!{exc.__class__.__name__}: {exc}")
log("".join(traceback.format_exception(exc))) # pylint: disable=[no-value-for-parameter]
if 1:
# pylint: disable=no-value-for-parameter
log("".join(traceback.format_exception(exc)))
try:
assert sys_monitoring is not None
sys_monitoring.set_events(sys.monitoring.COVERAGE_ID, 0)
Expand Down Expand Up @@ -201,7 +206,6 @@ def __init__(self, tool_id: int) -> None:
# A list of code_objects, just to keep them alive so that id's are
# useful as identity.
self.code_objects: list[CodeType] = []
self.last_lines: dict[FrameType, int] = {}
# Map id(code_object) -> code_object
self.local_event_codes: dict[int, CodeType] = {}
self.sysmon_on = False
Expand All @@ -227,20 +231,20 @@ def start(self) -> None:
assert sys_monitoring is not None
sys_monitoring.use_tool_id(self.myid, "coverage.py")
register = functools.partial(sys_monitoring.register_callback, self.myid)
events = sys_monitoring.events
events = sys.monitoring.events
import contextlib

with open("/tmp/foo.out", "a") as f:
with contextlib.redirect_stdout(f):
print(f"{events = }")
sys_monitoring.set_events(self.myid, events.PY_START)
register(events.PY_START, self.sysmon_py_start)
if self.trace_arcs:
sys_monitoring.set_events(
self.myid,
events.PY_START | events.PY_UNWIND,
)
register(events.PY_START, self.sysmon_py_start)
register(events.PY_RESUME, self.sysmon_py_resume_arcs)
register(events.PY_RETURN, self.sysmon_py_return_arcs)
register(events.PY_UNWIND, self.sysmon_py_unwind_arcs)
register(events.PY_RETURN, self.sysmon_py_return)
register(events.LINE, self.sysmon_line_arcs)
register(events.BRANCH_TAKEN, self.sysmon_branch_taken)
register(events.BRANCH_NOT_TAKEN, self.sysmon_branch_not_taken)
else:
sys_monitoring.set_events(self.myid, events.PY_START)
register(events.PY_START, self.sysmon_py_start)
register(events.LINE, self.sysmon_line_lines)
sys_monitoring.restart_events()
self.sysmon_on = True
Expand Down Expand Up @@ -278,23 +282,10 @@ def get_stats(self) -> dict[str, int] | None:
"""Return a dictionary of statistics, or None."""
return None

# The number of frames in callers_frame takes @panopticon into account.
if LOG:

def callers_frame(self) -> FrameType:
"""Get the frame of the Python code we're monitoring."""
return (
inspect.currentframe().f_back.f_back.f_back # type: ignore[union-attr,return-value]
)

else:

def callers_frame(self) -> FrameType:
"""Get the frame of the Python code we're monitoring."""
return inspect.currentframe().f_back.f_back # type: ignore[union-attr,return-value]

@panopticon("code", "@")
def sysmon_py_start(self, code: CodeType, instruction_offset: int) -> MonitorReturn:
def sysmon_py_start( # pylint: disable=useless-return
self, code: CodeType, instruction_offset: int
) -> MonitorReturn:
"""Handle sys.monitoring.events.PY_START events."""
# Entering a new frame. Decide if we should trace in this file.
self._activity = True
Expand Down Expand Up @@ -346,91 +337,78 @@ def sysmon_py_start(self, code: CodeType, instruction_offset: int) -> MonitorRet
with self.lock:
if self.sysmon_on:
assert sys_monitoring is not None
sys_monitoring.set_local_events(
self.myid,
code,
events.PY_RETURN
#
| events.PY_RESUME
# | events.PY_YIELD
| events.LINE
| events.BRANCH_TAKEN
| events.BRANCH_NOT_TAKEN
# | events.JUMP
)
local_events = events.PY_RETURN | events.PY_RESUME | events.LINE
if self.trace_arcs:
assert env.PYBEHAVIOR.branch_taken
local_events |= (
events.BRANCH_TAKEN | events.BRANCH_NOT_TAKEN
)
sys_monitoring.set_local_events(self.myid, code, local_events)
self.local_event_codes[id(code)] = code

if tracing_code and self.trace_arcs:
frame = self.callers_frame()
self.last_lines[frame] = -code.co_firstlineno
return None
else:
return sys.monitoring.DISABLE

@panopticon("code", "@")
def sysmon_py_resume_arcs(
self, code: CodeType, instruction_offset: int,
) -> MonitorReturn:
"""Handle sys.monitoring.events.PY_RESUME events for branch coverage."""
frame = self.callers_frame()
self.last_lines[frame] = frame.f_lineno
return None

@panopticon("code", "@", None)
def sysmon_py_return_arcs(
self, code: CodeType, instruction_offset: int, retval: object,
def sysmon_py_return( # pylint: disable=useless-return
self,
code: CodeType,
instruction_offset: int,
retval: object,
) -> MonitorReturn:
"""Handle sys.monitoring.events.PY_RETURN events for branch coverage."""
frame = self.callers_frame()
code_info = self.code_infos.get(id(code))
if code_info is not None and code_info.file_data is not None:
last_line = self.last_lines.get(frame)
assert code_info.byte_to_line is not None
last_line = code_info.byte_to_line[instruction_offset]
if last_line is not None:
arc = (last_line, -code.co_firstlineno)
# log(f"adding {arc=}")
cast(Set[TArc], code_info.file_data).add(arc)

# Leaving this function, no need for the frame any more.
self.last_lines.pop(frame, None)

@panopticon("code", "@", "exc")
def sysmon_py_unwind_arcs(
self, code: CodeType, instruction_offset: int, exception: BaseException,
) -> MonitorReturn:
"""Handle sys.monitoring.events.PY_UNWIND events for branch coverage."""
frame = self.callers_frame()
# Leaving this function.
last_line = self.last_lines.pop(frame, None)
if isinstance(exception, GeneratorExit):
# We don't want to count generator exits as arcs.
return
code_info = self.code_infos.get(id(code))
if code_info is not None and code_info.file_data is not None:
if last_line is not None:
arc = (last_line, -code.co_firstlineno)
# log(f"adding {arc=}")
cast(Set[TArc], code_info.file_data).add(arc)

log(f"adding {arc=}")
return None

@panopticon("code", "line")
def sysmon_line_lines(self, code: CodeType, line_number: int) -> MonitorReturn:
"""Handle sys.monitoring.events.LINE events for line coverage."""
code_info = self.code_infos[id(code)]
if code_info.file_data is not None:
cast(Set[TLineNo], code_info.file_data).add(line_number)
# log(f"adding {line_number=}")
return sys.monitoring.DISABLE
log(f"adding {line_number=}")
return DISABLE

@panopticon("code", "line")
def sysmon_line_arcs(self, code: CodeType, line_number: int) -> MonitorReturn:
"""Handle sys.monitoring.events.LINE events for branch coverage."""
code_info = self.code_infos[id(code)]
ret = None
if code_info.file_data is not None:
frame = self.callers_frame()
last_line = self.last_lines.get(frame)
if last_line is not None:
arc = (last_line, line_number)
cast(Set[TArc], code_info.file_data).add(arc)
# log(f"adding {arc=}")
self.last_lines[frame] = line_number
return ret
arc = (line_number, line_number)
cast(Set[TArc], code_info.file_data).add(arc)
log(f"adding {arc=}")
return DISABLE

@panopticon("code", "@", "@")
def sysmon_branch_taken(
self, code: CodeType, instruction_offset: int, destination_offset: int
) -> MonitorReturn:
"""Handed BRANCH_TAKEN and BRANCH_NOT_TAKEN events."""
code_info = self.code_infos[id(code)]
if code_info.file_data is not None:
b2l = code_info.byte_to_line
assert b2l is not None
arc = (b2l[instruction_offset], b2l[destination_offset])
cast(Set[TArc], code_info.file_data).add(arc)
log(f"adding {arc=}")
return DISABLE

@panopticon("code", "@", "@")
def sysmon_branch_not_taken(
self, code: CodeType, instruction_offset: int, destination_offset: int
) -> MonitorReturn:
"""Handed BRANCH_TAKEN and BRANCH_NOT_TAKEN events."""
code_info = self.code_infos[id(code)]
if code_info.file_data is not None:
b2l = code_info.byte_to_line
assert b2l is not None
arc = (b2l[instruction_offset], b2l[destination_offset])
cast(Set[TArc], code_info.file_data).add(arc)
log(f"adding {arc=}")
return DISABLE
8 changes: 7 additions & 1 deletion tests/test_concurrency.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -554,13 +554,19 @@ def test_multiprocessing_with_branching(self, start_method: str) -> None:
code = (SQUARE_OR_CUBE_WORK + MULTI_CODE).format(NPROCS=nprocs, UPTO=upto)
total = sum(x*x if x%2 else x*x*x for x in range(upto))
expected_out = f"{nprocs} pids, total = {total}"
expect_warn = (
env.PYBEHAVIOR.pep669
and (not env.PYBEHAVIOR.branch_taken)
and testenv.SYS_MON
)
self.make_file("multi.py", code)
self.make_file("multi.rc", """\
[run]
concurrency = multiprocessing
branch = True
omit = */site-packages/*
""")
""" + ("disable_warnings = no-sysmon" if expect_warn else "")
)

out = self.run_command(f"coverage run --rcfile=multi.rc multi.py {start_method}")
assert out.rstrip() == expected_out
Expand Down

0 comments on commit 17b2baf

Please sign in to comment.