Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/hypothesis/statistics.py: 20%

1# This file is part of Hypothesis, which may be found at 

2# https://github.com/HypothesisWorks/hypothesis/ 

3# 

4# Copyright the Hypothesis Authors. 

5# Individual contributors are listed in AUTHORS.rst and the git log. 

6# 

7# This Source Code Form is subject to the terms of the Mozilla Public License, 

8# v. 2.0. If a copy of the MPL was not distributed with this file, You can 

9# obtain one at https://mozilla.org/MPL/2.0/. 

10 

11import itertools 

12import math 

13from collections import Counter 

14from collections.abc import Iterable 

15from typing import TYPE_CHECKING, cast 

16 

17from hypothesis._settings import Phase 

18from hypothesis.utils.dynamicvariables import DynamicVariable 

19 

20if TYPE_CHECKING: 

21 from hypothesis.internal.conjecture.engine import PhaseStatistics, StatisticsDict 

22 

23collector = DynamicVariable(None) 

24 

25 

26def note_statistics(stats_dict: "StatisticsDict") -> None: 

27 callback = collector.value 

28 if callback is not None: 

29 callback(stats_dict) 

30 

31 

32def describe_targets(best_targets: dict[str, float]) -> list[str]: 

33 """Return a list of lines describing the results of `target`, if any.""" 

34 # These lines are included in the general statistics description below, 

35 # but also printed immediately below failing test cases to alleviate the 

36 # "threshold problem" where shrinking can make severe bug look trivial. 

37 # See https://github.com/HypothesisWorks/hypothesis/issues/2180 

38 if not best_targets: 

39 return [] 

40 elif len(best_targets) == 1: 

41 label, score = next(iter(best_targets.items())) 

42 return [f"Highest target score: {score:g} ({label=})"] 

43 else: 

44 lines = ["Highest target scores:"] 

45 for label, score in sorted(best_targets.items(), key=lambda x: x[::-1]): 

46 lines.append(f"{score:>16g} ({label=})") 

47 return lines 

48 

49 

50def format_ms(times: Iterable[float]) -> str: 

51 """Format `times` into a string representing approximate milliseconds. 

52 

53 `times` is a collection of durations in seconds. 

54 """ 

55 ordered = sorted(times) 

56 n = len(ordered) - 1 

57 if n < 0 or any(math.isnan(t) for t in ordered): # pragma: no cover 

58 return "NaN ms" 

59 lower = int(ordered[math.floor(n * 0.05)] * 1000) 

60 upper = int(ordered[math.ceil(n * 0.95)] * 1000) 

61 if upper == 0: 

62 return "< 1ms" 

63 elif lower == upper: 

64 return f"~ {lower}ms" 

65 else: 

66 return f"~ {lower}-{upper} ms" 

67 

68 

69def describe_statistics(stats_dict: "StatisticsDict") -> str: 

70 """Return a multi-line string describing the passed run statistics. 

71 

72 `stats_dict` must be a dictionary of data in the format collected by 

73 `hypothesis.internal.conjecture.engine.ConjectureRunner.statistics`. 

74 

75 We DO NOT promise that this format will be stable or supported over 

76 time, but do aim to make it reasonably useful for downstream users. 

77 It's also meant to support benchmarking for research purposes. 

78 

79 This function is responsible for the report which is printed in the 

80 terminal for our pytest --hypothesis-show-statistics option. 

81 """ 

82 lines = [stats_dict["nodeid"] + ":\n"] if "nodeid" in stats_dict else [] 

83 prev_failures = 0 

84 for phase in (p.name for p in list(Phase)[1:]): 

85 d = cast("PhaseStatistics", stats_dict.get(phase + "-phase", {})) 

86 # Basic information we report for every phase 

87 cases = d.get("test-cases", []) 

88 if not cases: 

89 continue 

90 statuses = Counter(t["status"] for t in cases) 

91 runtime_ms = format_ms(t["runtime"] for t in cases) 

92 drawtime_ms = format_ms(t["drawtime"] for t in cases) 

93 lines.append( 

94 f" - during {phase} phase ({d['duration-seconds']:.2f} seconds):\n" 

95 f" - Typical runtimes: {runtime_ms}, of which {drawtime_ms} in data generation\n" 

96 f" - {statuses['valid']} passing, {statuses['interesting']} " 

97 f"failing, and {statuses['invalid'] + statuses['overrun']} " 

98 "invalid test cases" 

99 ) 

100 # If we've found new distinct failures in this phase, report them 

101 distinct_failures = d["distinct-failures"] - prev_failures 

102 if distinct_failures: 

103 plural = distinct_failures > 1 

104 lines.append( 

105 " - Found {}{} distinct error{} in this phase".format( 

106 distinct_failures, " more" * bool(prev_failures), "s" * plural 

107 ) 

108 ) 

109 prev_failures = d["distinct-failures"] 

110 # Report events during the generate phase, if there were any 

111 if phase == "generate": 

112 events = Counter(itertools.chain.from_iterable(t["events"] for t in cases)) 

113 if events: 

114 lines.append(" - Events:") 

115 lines += [ 

116 f" * {100 * v / len(cases):.2f}%, {k}" 

117 for k, v in sorted(events.items(), key=lambda x: (-x[1], x[0])) 

118 ] 

119 # Some additional details on the shrinking phase 

120 if phase == "shrink": 

121 lines.append( 

122 " - Tried {} shrinks of which {} were successful".format( 

123 len(cases), d["shrinks-successful"] 

124 ) 

125 ) 

126 lines.append("") 

127 

128 target_lines = describe_targets(stats_dict.get("targets", {})) 

129 if target_lines: 

130 lines.append(" - " + target_lines[0]) 

131 lines.extend(" " + l for l in target_lines[1:]) 

132 lines.append(" - Stopped because " + stats_dict["stopped-because"]) 

133 return "\n".join(lines)