Skip to content

Commit

Permalink
Events can have a payload now
Browse files Browse the repository at this point in the history
  • Loading branch information
Zac-HD committed Nov 27, 2023
1 parent 63c5298 commit a3d9623
Show file tree
Hide file tree
Showing 15 changed files with 71 additions and 130 deletions.
8 changes: 5 additions & 3 deletions hypothesis-python/RELEASE.rst
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
RELEASE_TYPE: patch
RELEASE_TYPE: minor

This patch updates our vendored `list of top-level domains <https://www.iana.org/domains/root/db>`__,
which is used by the provisional :func:`~hypothesis.provisional.domains` strategy.
This release adds an optional ``payload`` argument to :func:`hypothesis.event`,
so that you can clearly express the difference between the label and the value
of an observation. :ref:`statistics` will still summarize it as a string, but
future observability options can preserve the distinction.
29 changes: 25 additions & 4 deletions hypothesis-python/src/hypothesis/control.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import math
from collections import defaultdict
from typing import NoReturn, Union
from weakref import WeakKeyDictionary

from hypothesis import Verbosity, settings
from hypothesis._settings import note_deprecation
Expand Down Expand Up @@ -168,18 +169,38 @@ def note(value: str) -> None:
report(value)


def event(value: str) -> None:
"""Record an event that occurred this test. Statistics on number of test
def event(value: str, payload: Union[str, int, float] = "") -> None:
"""Record an event that occurred during this test. Statistics on the number of test
runs with each event will be reported at the end if you run Hypothesis in
statistics reporting mode.
Events should be strings or convertible to them.
Event values should be strings or convertible to them. If an optional
payload is given, it will be included in the string for :ref:`statistics`.
"""
context = _current_build_context.value
if context is None:
raise InvalidArgument("Cannot make record events outside of a test")

context.data.note_event(value)
payload = _event_to_string(payload, (str, int, float))
context.data.events[_event_to_string(value)] = payload


_events_to_strings: WeakKeyDictionary = WeakKeyDictionary()


def _event_to_string(event, allowed_types=str):
if isinstance(event, allowed_types):
return event
try:
return _events_to_strings[event]
except (KeyError, TypeError):
pass
result = str(event)
try:
_events_to_strings[event] = result
except TypeError:
pass
return result


def target(observation: Union[int, float], *, label: str = "") -> Union[int, float]:
Expand Down
10 changes: 2 additions & 8 deletions hypothesis-python/src/hypothesis/internal/conjecture/data.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
Callable,
Dict,
FrozenSet,
Hashable,
Iterable,
Iterator,
List,
Expand Down Expand Up @@ -1367,7 +1366,7 @@ def __init__(
self.testcounter = global_test_counter
global_test_counter += 1
self.start_time = time.perf_counter()
self.events: "Union[Set[Hashable], FrozenSet[Hashable]]" = set()
self.events: Dict[str, Union[str, int, float]] = {}
self.forced_indices: "Set[int]" = set()
self.interesting_origin: Optional[InterestingOrigin] = None
self.draw_times: "List[float]" = []
Expand Down Expand Up @@ -1615,10 +1614,6 @@ def stop_example(self, *, discard: bool = False) -> None:

self.observer.kill_branch()

def note_event(self, event: Hashable) -> None:
assert isinstance(self.events, set)
self.events.add(event)

@property
def examples(self) -> Examples:
assert self.frozen
Expand All @@ -1643,7 +1638,6 @@ def freeze(self) -> None:
self.frozen = True

self.buffer = bytes(self.buffer)
self.events = frozenset(self.events)
self.observer.conclude_test(self.status, self.interesting_origin)

def draw_bits(self, n: int, *, forced: Optional[int] = None) -> int:
Expand Down Expand Up @@ -1729,7 +1723,7 @@ def mark_interesting(

def mark_invalid(self, why: Optional[str] = None) -> NoReturn:
if why is not None:
self.note_event(why)
self.events["invalid because"] = why
self.conclude_test(Status.INVALID)

def mark_overrun(self) -> NoReturn:
Expand Down
21 changes: 3 additions & 18 deletions hypothesis-python/src/hypothesis/internal/conjecture/engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
from datetime import timedelta
from enum import Enum
from random import Random, getrandbits
from weakref import WeakKeyDictionary

import attr

Expand Down Expand Up @@ -101,8 +100,6 @@ def __init__(
self.statistics = {}
self.stats_per_test_case = []

self.events_to_strings = WeakKeyDictionary()

self.interesting_examples = {}
# We use call_count because there may be few possible valid_examples.
self.first_bug_found_at = None
Expand Down Expand Up @@ -209,7 +206,9 @@ def test_function(self, data):
"status": data.status.name.lower(),
"runtime": data.finish_time - data.start_time,
"drawtime": math.fsum(data.draw_times),
"events": sorted({self.event_to_string(e) for e in data.events}),
"events": sorted(
k if v == "" else f"{k}: {v}" for k, v in data.events.items()
),
}
self.stats_per_test_case.append(call_stats)
self.__data_cache[data.buffer] = data.as_result()
Expand Down Expand Up @@ -1055,20 +1054,6 @@ def kill_branch(self):
self.__data_cache[buffer] = result
return result

def event_to_string(self, event):
if isinstance(event, str):
return event
try:
return self.events_to_strings[event]
except (KeyError, TypeError):
pass
result = str(event)
try:
self.events_to_strings[event] = result
except TypeError:
pass
return result

def passing_buffers(self, prefix=b""):
"""Return a collection of bytestrings which cause the test to pass.
Expand Down
6 changes: 3 additions & 3 deletions hypothesis-python/src/hypothesis/internal/escalation.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
import traceback
from inspect import getframeinfo
from pathlib import Path
from typing import Dict, NamedTuple, Type
from typing import Dict, NamedTuple, Optional, Type

import hypothesis
from hypothesis.errors import (
Expand Down Expand Up @@ -114,8 +114,8 @@ class InterestingOrigin(NamedTuple):
# blocks and understand the __cause__ (`raise x from y`) or __context__ that
# first raised an exception as well as PEP-654 exception groups.
exc_type: Type[BaseException]
filename: str
lineno: int
filename: Optional[str]
lineno: Optional[int]
context: "InterestingOrigin | tuple[()]"
group_elems: "tuple[InterestingOrigin, ...]"

Expand Down
33 changes: 0 additions & 33 deletions hypothesis-python/src/hypothesis/internal/lazyformat.py

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -163,8 +163,10 @@ def draw_naive_datetime_and_combine(self, data, tz):
try:
return replace_tzinfo(dt.datetime(**result), timezone=tz)
except (ValueError, OverflowError):
msg = "Failed to draw a datetime between %r and %r with timezone from %r."
data.mark_invalid(msg % (self.min_value, self.max_value, self.tz_strat))
data.mark_invalid(
f"Failed to draw a datetime between {self.min_value!r} and "
f"{self.max_value!r} with timezone from {self.tz_strat!r}."
)


@defines_strategy(force_reusable_values=True)
Expand Down
13 changes: 3 additions & 10 deletions hypothesis-python/src/hypothesis/strategies/_internal/recursive.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
from contextlib import contextmanager

from hypothesis.errors import InvalidArgument
from hypothesis.internal.lazyformat import lazyformat
from hypothesis.internal.reflection import get_pretty_function_description
from hypothesis.internal.validation import check_type
from hypothesis.strategies._internal.strategies import (
Expand Down Expand Up @@ -112,13 +111,7 @@ def do_draw(self, data):
with self.limited_base.capped(self.max_leaves):
return data.draw(self.strategy)
except LimitReached:
# Workaround for possible coverage bug - this branch is definitely
# covered but for some reason is showing up as not covered.
if count == 0: # pragma: no branch
data.note_event(
lazyformat(
"Draw for %r exceeded max_leaves and had to be retried",
self,
)
)
if count == 0:
msg = f"Draw for {self!r} exceeded max_leaves and had to be retried"
data.events[msg] = ""
count += 1
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@
combine_labels,
)
from hypothesis.internal.coverage import check_function
from hypothesis.internal.lazyformat import lazyformat
from hypothesis.internal.reflection import (
get_pretty_function_description,
is_identity_function,
Expand Down Expand Up @@ -550,7 +549,7 @@ def do_filtered_draw(self, data):
if element is not filter_not_satisfied:
return element
if not known_bad_indices:
FilteredStrategy.note_retried(self, data)
data.events[f"Retried draw from {self!r} to satisfy filter"] = ""
known_bad_indices.add(i)

# If we've tried all the possible elements, give up now.
Expand Down Expand Up @@ -940,9 +939,6 @@ def do_draw(self, data: ConjectureData) -> Ex:
data.mark_invalid(f"Aborted test because unable to satisfy {self!r}")
raise NotImplementedError("Unreachable, for Mypy")

def note_retried(self, data):
data.note_event(lazyformat("Retried draw from %r to satisfy filter", self))

def do_filtered_draw(self, data):
for i in range(3):
start_index = data.index
Expand All @@ -954,7 +950,7 @@ def do_filtered_draw(self, data):
else:
data.stop_example(discard=True)
if i == 0:
self.note_retried(data)
data.events[f"Retried draw from {self!r} to satisfy filter"] = ""
# This is to guard against the case where we consume no data.
# As long as we consume data, we'll eventually pass or raise.
# But if we don't this could be an infinite loop.
Expand Down
22 changes: 1 addition & 21 deletions hypothesis-python/tests/common/debug.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,8 @@
# v. 2.0. If a copy of the MPL was not distributed with this file, You can
# obtain one at https://mozilla.org/MPL/2.0/.

from hypothesis import (
HealthCheck,
Phase,
Verbosity,
given,
settings as Settings,
strategies as st,
)
from hypothesis import HealthCheck, Phase, Verbosity, given, settings as Settings
from hypothesis.errors import Found, NoSuchExample, Unsatisfiable
from hypothesis.internal.conjecture.data import ConjectureData, StopTest
from hypothesis.internal.reflection import get_pretty_function_description

from tests.common.utils import no_shrink
Expand Down Expand Up @@ -107,15 +99,3 @@ def assert_examples(s):
assert predicate(s), msg

assert_examples()


def assert_can_trigger_event(strategy, predicate):
def test(buf):
data = ConjectureData.for_buffer(buf)
try:
data.draw(strategy)
except StopTest:
pass
return any(predicate(e) for e in data.events)

find_any(st.binary(), test)
5 changes: 0 additions & 5 deletions hypothesis-python/tests/conjecture/test_engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -1586,8 +1586,3 @@ def test(data):
runner.cached_test_function([c])

assert runner.tree.is_exhausted


def test_can_convert_non_weakref_types_to_event_strings():
runner = ConjectureRunner(lambda data: None)
runner.event_to_string(())
4 changes: 2 additions & 2 deletions hypothesis-python/tests/conjecture/test_test_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ def test_can_mark_invalid_with_why():
x.mark_invalid("some reason")
assert x.frozen
assert x.status == Status.INVALID
assert x.events == {"some reason"}
assert x.events == {"invalid because": "some reason"}


class BoomStrategy(SearchStrategy):
Expand Down Expand Up @@ -416,7 +416,7 @@ def test_trivial_before_force_agrees_with_trivial_after():

def test_events_are_noted():
d = ConjectureData.for_buffer(())
d.note_event("hello")
d.events["hello"] = ""
assert "hello" in d.events


Expand Down
5 changes: 5 additions & 0 deletions hypothesis-python/tests/cover/test_control.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from hypothesis.control import (
BuildContext,
_current_build_context,
_event_to_string,
cleanup,
current_build_context,
currently_in_test_context,
Expand Down Expand Up @@ -191,3 +192,7 @@ def step(self):


test_currently_in_stateful_test = ContextMachine.TestCase


def test_can_convert_non_weakref_types_to_event_strings():
_event_to_string(())
Loading

0 comments on commit a3d9623

Please sign in to comment.