1# This file is part of Hypothesis, which may be found at
2# https://github.com/HypothesisWorks/hypothesis/
3#
4# Copyright the Hypothesis Authors.
5# Individual contributors are listed in AUTHORS.rst and the git log.
6#
7# This Source Code Form is subject to the terms of the Mozilla Public License,
8# v. 2.0. If a copy of the MPL was not distributed with this file, You can
9# obtain one at https://mozilla.org/MPL/2.0/.
10
11import math
12from collections import Counter
13
14from hypothesis._settings import Phase
15from hypothesis.utils.dynamicvariables import DynamicVariable
16
17collector = DynamicVariable(None)
18
19
20def note_statistics(stats_dict):
21 callback = collector.value
22 if callback is not None:
23 callback(stats_dict)
24
25
26def describe_targets(best_targets):
27 """Return a list of lines describing the results of `target`, if any."""
28 # These lines are included in the general statistics description below,
29 # but also printed immediately below failing examples to alleviate the
30 # "threshold problem" where shrinking can make severe bug look trivial.
31 # See https://github.com/HypothesisWorks/hypothesis/issues/2180
32 if not best_targets:
33 return []
34 elif len(best_targets) == 1:
35 label, score = next(iter(best_targets.items()))
36 return [f"Highest target score: {score:g} ({label=})"]
37 else:
38 lines = ["Highest target scores:"]
39 for label, score in sorted(best_targets.items(), key=lambda x: x[::-1]):
40 lines.append(f"{score:>16g} ({label=})")
41 return lines
42
43
44def format_ms(times):
45 """Format `times` into a string representing approximate milliseconds.
46
47 `times` is a collection of durations in seconds.
48 """
49 ordered = sorted(times)
50 n = len(ordered) - 1
51 if n < 0 or any(math.isnan(t) for t in ordered):
52 return "NaN ms"
53 lower = int(ordered[int(math.floor(n * 0.05))] * 1000)
54 upper = int(ordered[int(math.ceil(n * 0.95))] * 1000)
55 if upper == 0:
56 return "< 1ms"
57 elif lower == upper:
58 return f"~ {lower}ms"
59 else:
60 return f"~ {lower}-{upper} ms"
61
62
63def describe_statistics(stats_dict):
64 """Return a multi-line string describing the passed run statistics.
65
66 `stats_dict` must be a dictionary of data in the format collected by
67 `hypothesis.internal.conjecture.engine.ConjectureRunner.statistics`.
68
69 We DO NOT promise that this format will be stable or supported over
70 time, but do aim to make it reasonably useful for downstream users.
71 It's also meant to support benchmarking for research purposes.
72
73 This function is responsible for the report which is printed in the
74 terminal for our pytest --hypothesis-show-statistics option.
75 """
76 lines = [stats_dict["nodeid"] + ":\n"] if "nodeid" in stats_dict else []
77 prev_failures = 0
78 for phase in (p.name for p in list(Phase)[1:]):
79 d = stats_dict.get(phase + "-phase", {})
80 # Basic information we report for every phase
81 cases = d.get("test-cases", [])
82 if not cases:
83 continue
84 statuses = Counter(t["status"] for t in cases)
85 runtime_ms = format_ms(t["runtime"] for t in cases)
86 drawtime_ms = format_ms(t["drawtime"] for t in cases)
87 lines.append(
88 f" - during {phase} phase ({d['duration-seconds']:.2f} seconds):\n"
89 f" - Typical runtimes: {runtime_ms}, of which {drawtime_ms} in data generation\n"
90 f" - {statuses['valid']} passing examples, {statuses['interesting']} "
91 f"failing examples, {statuses['invalid'] + statuses['overrun']} invalid examples"
92 )
93 # If we've found new distinct failures in this phase, report them
94 distinct_failures = d["distinct-failures"] - prev_failures
95 if distinct_failures:
96 plural = distinct_failures > 1
97 lines.append(
98 " - Found {}{} distinct error{} in this phase".format(
99 distinct_failures, " more" * bool(prev_failures), "s" * plural
100 )
101 )
102 prev_failures = d["distinct-failures"]
103 # Report events during the generate phase, if there were any
104 if phase == "generate":
105 events = Counter(sum((t["events"] for t in cases), []))
106 if events:
107 lines.append(" - Events:")
108 lines += [
109 f" * {100 * v / len(cases):.2f}%, {k}"
110 for k, v in sorted(events.items(), key=lambda x: (-x[1], x[0]))
111 ]
112 # Some additional details on the shrinking phase
113 if phase == "shrink":
114 lines.append(
115 " - Tried {} shrinks of which {} were successful".format(
116 len(cases), d["shrinks-successful"]
117 )
118 )
119 lines.append("")
120
121 target_lines = describe_targets(stats_dict.get("targets", {}))
122 if target_lines:
123 lines.append(" - " + target_lines[0])
124 lines.extend(" " + l for l in target_lines[1:])
125 lines.append(" - Stopped because " + stats_dict["stopped-because"])
126 return "\n".join(lines)