Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/gprof2dot.py: 15%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#!/usr/bin/env python3
2#
3# Copyright 2008-2023 Jose Fonseca
4#
5# This program is free software: you can redistribute it and/or modify it
6# under the terms of the GNU Lesser General Public License as published
7# by the Free Software Foundation, either version 3 of the License, or
8# (at your option) any later version.
9#
10# This program is distributed in the hope that it will be useful,
11# but WITHOUT ANY WARRANTY; without even the implied warranty of
12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13# GNU Lesser General Public License for more details.
14#
15# You should have received a copy of the GNU Lesser General Public License
16# along with this program. If not, see <http://www.gnu.org/licenses/>.
17#
19"""Generate a dot graph from the output of several profilers."""
21__author__ = "Jose Fonseca et al"
24import sys
25import math
26import os.path
27import re
28import textwrap
29import argparse
30import xml.parsers.expat
31import collections
32import locale
33import json
34import fnmatch
35import codecs
36import io
37import hashlib
39assert sys.version_info[0] >= 3
42########################################################################
43# Model
46MULTIPLICATION_SIGN = chr(0xd7)
47timeFormat = "%.7g"
50def times(x):
51 return "%u%s" % (x, MULTIPLICATION_SIGN)
53def percentage(p):
54 return "%.02f%%" % (p*100.0,)
56def fmttime(t):
57 return timeFormat % t
59def add(a, b):
60 return a + b
62def fail(a, b):
63 assert False
65# To enhance readability, labels are rounded to the number of decimal
66# places corresponding to the tolerance value.
67def round_difference(difference, tolerance):
68 n = -math.floor(math.log10(tolerance))
69 return round(difference, n)
72def rescale_difference(x, min_val, max_val):
73 return (x - min_val) / (max_val - min_val)
76def min_max_difference(profile1, profile2):
77 f1_events = [f1[TOTAL_TIME_RATIO] for _, f1 in sorted_iteritems(profile1.functions)]
78 f2_events = [f2[TOTAL_TIME_RATIO] for _, f2 in sorted_iteritems(profile2.functions)]
79 differences = []
80 for i in range(len(f1_events)):
81 try:
82 differences.append(abs(f1_events[i] - f2_events[i]) * 100)
83 except IndexError:
84 differences.append(0)
86 return min(differences), max(differences)
89tol = 2 ** -23
91def ratio(numerator, denominator):
92 try:
93 ratio = float(numerator)/float(denominator)
94 except ZeroDivisionError:
95 # 0/0 is undefined, but 1.0 yields more useful results
96 return 1.0
97 if ratio < 0.0:
98 if ratio < -tol:
99 sys.stderr.write('warning: negative ratio (%s/%s)\n' % (numerator, denominator))
100 return 0.0
101 if ratio > 1.0:
102 if ratio > 1.0 + tol:
103 sys.stderr.write('warning: ratio greater than one (%s/%s)\n' % (numerator, denominator))
104 return 1.0
105 return ratio
108class UndefinedEvent(Exception):
109 """Raised when attempting to get an event which is undefined."""
111 def __init__(self, event):
112 Exception.__init__(self)
113 self.event = event
115 def __str__(self):
116 return 'unspecified event %s' % self.event.name
119class Event:
120 """Describe a kind of event, and its basic operations."""
122 def __init__(self, name, null, aggregator, formatter = str):
123 self.name = name
124 self._null = null
125 self._aggregator = aggregator
126 self._formatter = formatter
128 def __repr__(self):
129 return self.name
131 def null(self):
132 return self._null
134 def aggregate(self, val1, val2):
135 """Aggregate two event values."""
136 assert val1 is not None
137 assert val2 is not None
138 return self._aggregator(val1, val2)
140 def format(self, val):
141 """Format an event value."""
142 assert val is not None
143 return self._formatter(val)
146CALLS = Event("Calls", 0, add, times)
147SAMPLES = Event("Samples", 0, add, times)
148SAMPLES2 = Event("Samples", 0, add, times)
150# Count of samples where a given function was either executing or on the stack.
151# This is used to calculate the total time ratio according to the
152# straightforward method described in Mike Dunlavey's answer to
153# stackoverflow.com/questions/1777556/alternatives-to-gprof, item 4 (the myth
154# "that recursion is a tricky confusing issue"), last edited 2012-08-30: it's
155# just the ratio of TOTAL_SAMPLES over the number of samples in the profile.
156#
157# Used only when totalMethod == callstacks
158TOTAL_SAMPLES = Event("Samples", 0, add, times)
160TIME = Event("Time", 0.0, add, lambda x: '(' + fmttime(x) + ')')
161TIME_RATIO = Event("Time ratio", 0.0, add, lambda x: '(' + percentage(x) + ')')
162TOTAL_TIME = Event("Total time", 0.0, fail, fmttime)
163TOTAL_TIME_RATIO = Event("Total time ratio", 0.0, fail, percentage)
165labels = {
166 'self-time': TIME,
167 'self-time-percentage': TIME_RATIO,
168 'total-time': TOTAL_TIME,
169 'total-time-percentage': TOTAL_TIME_RATIO,
170}
171defaultLabelNames = ['total-time-percentage', 'self-time-percentage']
173totalMethod = 'callratios'
176class Object:
177 """Base class for all objects in profile which can store events."""
179 def __init__(self, events=None):
180 if events is None:
181 self.events = {}
182 else:
183 self.events = events
185 def __lt__(self, other):
186 return id(self) < id(other)
188 def __contains__(self, event):
189 return event in self.events
191 def __getitem__(self, event):
192 try:
193 return self.events[event]
194 except KeyError:
195 raise UndefinedEvent(event)
197 def __setitem__(self, event, value):
198 if value is None:
199 if event in self.events:
200 del self.events[event]
201 else:
202 self.events[event] = value
205class Call(Object):
206 """A call between functions.
208 There should be at most one call object for every pair of functions.
209 """
211 def __init__(self, callee_id):
212 Object.__init__(self)
213 self.callee_id = callee_id
214 self.ratio = None
215 self.weight = None
218class Function(Object):
219 """A function."""
221 def __init__(self, id, name):
222 Object.__init__(self)
223 self.id = id
224 self.name = name
225 self.module = None
226 self.process = None
227 self.calls = {}
228 self.called = None
229 self.weight = None
230 self.cycle = None
231 self.filename = None
233 def add_call(self, call):
234 if call.callee_id in self.calls:
235 sys.stderr.write('warning: overwriting call from function %s to %s\n' % (str(self.id), str(call.callee_id)))
236 self.calls[call.callee_id] = call
238 def get_call(self, callee_id):
239 if not callee_id in self.calls:
240 call = Call(callee_id)
241 call[SAMPLES] = 0
242 call[SAMPLES2] = 0
243 call[CALLS] = 0
244 self.calls[callee_id] = call
245 return self.calls[callee_id]
247 _parenthesis_re = re.compile(r'\([^()]*\)')
248 _angles_re = re.compile(r'<[^<>]*>')
249 _const_re = re.compile(r'\s+const$')
251 def stripped_name(self):
252 """Remove extraneous information from C++ demangled function names."""
254 name = self.name
256 # Strip function parameters from name by recursively removing paired parenthesis
257 while True:
258 name, n = self._parenthesis_re.subn('', name)
259 if not n:
260 break
262 # Strip const qualifier
263 name = self._const_re.sub('', name)
265 # Strip template parameters from name by recursively removing paired angles
266 while True:
267 name, n = self._angles_re.subn('', name)
268 if not n:
269 break
271 return name
273 # TODO: write utility functions
275 def __repr__(self):
276 return self.name
278 def dump(self, sep1=",\n\t", sep2=":=", sep3="\n"):
279 """ Returns as a string all information available in this Function object
280 separators sep1:between entries
281 sep2:between attribute name and value,
282 sep3: inserted at end
283 """
284 return sep1.join(sep2.join([k,str(v)]) for (k,v) in sorted(self.__dict__.items())) + sep3
287class Cycle(Object):
288 """A cycle made from recursive function calls."""
290 def __init__(self):
291 Object.__init__(self)
292 self.functions = set()
294 def add_function(self, function):
295 assert function not in self.functions
296 self.functions.add(function)
297 if function.cycle is not None:
298 for other in function.cycle.functions:
299 if function not in self.functions:
300 self.add_function(other)
301 function.cycle = self
304class Profile(Object):
305 """The whole profile."""
307 def __init__(self):
308 Object.__init__(self)
309 self.functions = {}
310 self.cycles = []
312 def add_function(self, function):
313 if function.id in self.functions:
314 sys.stderr.write('warning: overwriting function %s (id %s)\n' % (function.name, str(function.id)))
315 self.functions[function.id] = function
317 def add_cycle(self, cycle):
318 self.cycles.append(cycle)
320 def validate(self):
321 """Validate the edges."""
323 for function in self.functions.values():
324 for callee_id in list(function.calls.keys()):
325 assert function.calls[callee_id].callee_id == callee_id
326 if callee_id not in self.functions:
327 sys.stderr.write('warning: call to undefined function %s from function %s\n' % (str(callee_id), function.name))
328 del function.calls[callee_id]
330 def find_cycles(self):
331 """Find cycles using Tarjan's strongly connected components algorithm."""
333 # Apply the Tarjan's algorithm successively until all functions are visited
334 stack = []
335 data = {}
336 order = 0
337 for function in self.functions.values():
338 order = self._tarjan(function, order, stack, data)
339 cycles = []
340 for function in self.functions.values():
341 if function.cycle is not None and function.cycle not in cycles:
342 cycles.append(function.cycle)
343 self.cycles = cycles
344 if 0:
345 for cycle in cycles:
346 sys.stderr.write("Cycle:\n")
347 for member in cycle.functions:
348 sys.stderr.write("\tFunction %s\n" % member.name)
350 def prune_root(self, roots, depth=-1):
351 visited = set()
352 frontier = set([(root_node, depth) for root_node in roots])
353 while len(frontier) > 0:
354 node, node_depth = frontier.pop()
355 visited.add(node)
356 if node_depth == 0:
357 continue
358 f = self.functions[node]
359 newNodes = set(f.calls.keys()) - visited
360 frontier = frontier.union({(new_node, node_depth - 1) for new_node in newNodes})
361 subtreeFunctions = {}
362 for n in visited:
363 f = self.functions[n]
364 newCalls = {}
365 for c in f.calls.keys():
366 if c in visited:
367 newCalls[c] = f.calls[c]
368 f.calls = newCalls
369 subtreeFunctions[n] = f
370 self.functions = subtreeFunctions
372 def prune_leaf(self, leafs, depth=-1):
373 edgesUp = collections.defaultdict(set)
374 for f in self.functions.keys():
375 for n in self.functions[f].calls.keys():
376 edgesUp[n].add(f)
377 # build the tree up
378 visited = set()
379 frontier = set([(leaf_node, depth) for leaf_node in leafs])
380 while len(frontier) > 0:
381 node, node_depth = frontier.pop()
382 visited.add(node)
383 if node_depth == 0:
384 continue
385 newNodes = edgesUp[node] - visited
386 frontier = frontier.union({(new_node, node_depth - 1) for new_node in newNodes})
387 downTree = set(self.functions.keys())
388 upTree = visited
389 path = downTree.intersection(upTree)
390 pathFunctions = {}
391 for n in path:
392 f = self.functions[n]
393 newCalls = {}
394 for c in f.calls.keys():
395 if c in path:
396 newCalls[c] = f.calls[c]
397 f.calls = newCalls
398 pathFunctions[n] = f
399 self.functions = pathFunctions
401 def getFunctionIds(self, funcName):
402 function_names = {v.name: k for (k, v) in self.functions.items()}
403 return [function_names[name] for name in fnmatch.filter(function_names.keys(), funcName)]
405 def getFunctionId(self, funcName):
406 for f in self.functions:
407 if self.functions[f].name == funcName:
408 return f
409 return False
411 def printFunctionIds(self, selector=None, file=sys.stderr):
412 """ Print to file function entries selected by fnmatch.fnmatch like in
413 method getFunctionIds, with following extensions:
414 - selector starts with "%": dump all information available
415 - selector is '+' or '-': select all function entries
416 """
417 if selector is None or selector in ("+", "*"):
418 v = ",\n".join(("%s:\t%s" % (kf,self.functions[kf].name)
419 for kf in self.functions.keys()))
420 else:
421 if selector[0]=="%":
422 selector=selector[1:]
423 function_info={k:v for (k,v)
424 in self.functions.items()
425 if fnmatch.fnmatch(v.name,selector)}
426 v = ",\n".join( ("%s\t({k})\t(%s)::\n\t%s" % (v.name,type(v),v.dump())
427 for (k,v) in function_info.items()
428 ))
430 else:
431 function_names = (v.name for v in self.functions.values())
432 v = ",\n".join( ( nm for nm in fnmatch.filter(function_names,selector )))
434 file.write(v+"\n")
435 file.flush()
437 class _TarjanData:
438 def __init__(self, order):
439 self.order = order
440 self.lowlink = order
441 self.onstack = False
443 def _tarjan(self, function, order, stack, data):
444 """Tarjan's strongly connected components algorithm.
446 See also:
447 - http://en.wikipedia.org/wiki/Tarjan's_strongly_connected_components_algorithm
448 """
450 try:
451 func_data = data[function.id]
452 return order
453 except KeyError:
454 func_data = self._TarjanData(order)
455 data[function.id] = func_data
456 order += 1
457 pos = len(stack)
458 stack.append(function)
459 func_data.onstack = True
460 for call in function.calls.values():
461 try:
462 callee_data = data[call.callee_id]
463 if callee_data.onstack:
464 func_data.lowlink = min(func_data.lowlink, callee_data.order)
465 except KeyError:
466 callee = self.functions[call.callee_id]
467 order = self._tarjan(callee, order, stack, data)
468 callee_data = data[call.callee_id]
469 func_data.lowlink = min(func_data.lowlink, callee_data.lowlink)
470 if func_data.lowlink == func_data.order:
471 # Strongly connected component found
472 members = stack[pos:]
473 del stack[pos:]
474 if len(members) > 1:
475 cycle = Cycle()
476 for member in members:
477 cycle.add_function(member)
478 data[member.id].onstack = False
479 else:
480 for member in members:
481 data[member.id].onstack = False
482 return order
484 def call_ratios(self, event):
485 # Aggregate for incoming calls
486 cycle_totals = {}
487 for cycle in self.cycles:
488 cycle_totals[cycle] = 0.0
489 function_totals = {}
490 for function in self.functions.values():
491 function_totals[function] = 0.0
493 # Pass 1: function_total gets the sum of call[event] for all
494 # incoming arrows. Same for cycle_total for all arrows
495 # that are coming into the *cycle* but are not part of it.
496 for function in self.functions.values():
497 for call in function.calls.values():
498 if call.callee_id != function.id:
499 callee = self.functions[call.callee_id]
500 if event in call.events:
501 function_totals[callee] += call[event]
502 if callee.cycle is not None and callee.cycle is not function.cycle:
503 cycle_totals[callee.cycle] += call[event]
504 else:
505 sys.stderr.write("call_ratios: No data for " + function.name + " call to " + callee.name + "\n")
507 # Pass 2: Compute the ratios. Each call[event] is scaled by the
508 # function_total of the callee. Calls into cycles use the
509 # cycle_total, but not calls within cycles.
510 for function in self.functions.values():
511 for call in function.calls.values():
512 assert call.ratio is None
513 if call.callee_id != function.id:
514 callee = self.functions[call.callee_id]
515 if event in call.events:
516 if callee.cycle is not None and callee.cycle is not function.cycle:
517 total = cycle_totals[callee.cycle]
518 else:
519 total = function_totals[callee]
520 call.ratio = ratio(call[event], total)
521 else:
522 # Warnings here would only repeat those issued above.
523 call.ratio = 0.0
525 def integrate(self, outevent, inevent):
526 """Propagate function time ratio along the function calls.
528 Must be called after finding the cycles.
530 See also:
531 - http://citeseer.ist.psu.edu/graham82gprof.html
532 """
534 # Sanity checking
535 assert outevent not in self
536 for function in self.functions.values():
537 assert outevent not in function
538 assert inevent in function
539 for call in function.calls.values():
540 assert outevent not in call
541 if call.callee_id != function.id:
542 assert call.ratio is not None
544 # Aggregate the input for each cycle
545 for cycle in self.cycles:
546 total = inevent.null()
547 for function in self.functions.values():
548 total = inevent.aggregate(total, function[inevent])
549 self[inevent] = total
551 # Integrate along the edges
552 total = inevent.null()
553 for function in self.functions.values():
554 total = inevent.aggregate(total, function[inevent])
555 self._integrate_function(function, outevent, inevent)
556 self[outevent] = total
558 def _integrate_function(self, function, outevent, inevent):
559 if function.cycle is not None:
560 return self._integrate_cycle(function.cycle, outevent, inevent)
561 else:
562 if outevent not in function:
563 total = function[inevent]
564 for call in function.calls.values():
565 if call.callee_id != function.id:
566 total += self._integrate_call(call, outevent, inevent)
567 function[outevent] = total
568 return function[outevent]
570 def _integrate_call(self, call, outevent, inevent):
571 assert outevent not in call
572 assert call.ratio is not None
573 callee = self.functions[call.callee_id]
574 subtotal = call.ratio *self._integrate_function(callee, outevent, inevent)
575 call[outevent] = subtotal
576 return subtotal
578 def _integrate_cycle(self, cycle, outevent, inevent):
579 if outevent not in cycle:
581 # Compute the outevent for the whole cycle
582 total = inevent.null()
583 for member in cycle.functions:
584 subtotal = member[inevent]
585 for call in member.calls.values():
586 callee = self.functions[call.callee_id]
587 if callee.cycle is not cycle:
588 subtotal += self._integrate_call(call, outevent, inevent)
589 total += subtotal
590 cycle[outevent] = total
592 # Compute the time propagated to callers of this cycle
593 callees = {}
594 for function in self.functions.values():
595 if function.cycle is not cycle:
596 for call in function.calls.values():
597 callee = self.functions[call.callee_id]
598 if callee.cycle is cycle:
599 try:
600 callees[callee] += call.ratio
601 except KeyError:
602 callees[callee] = call.ratio
604 for member in cycle.functions:
605 member[outevent] = outevent.null()
607 for callee, call_ratio in callees.items():
608 ranks = {}
609 call_ratios = {}
610 partials = {}
611 self._rank_cycle_function(cycle, callee, ranks)
612 self._call_ratios_cycle(cycle, callee, ranks, call_ratios, set())
613 partial = self._integrate_cycle_function(cycle, callee, call_ratio, partials, ranks, call_ratios, outevent, inevent)
615 # Ensure `partial == max(partials.values())`, but with round-off tolerance
616 max_partial = max(partials.values())
617 assert abs(partial - max_partial) <= 1e-7*max_partial
619 assert abs(call_ratio*total - partial) <= 0.001*call_ratio*total
621 return cycle[outevent]
623 def _rank_cycle_function(self, cycle, function, ranks):
624 """Dijkstra's shortest paths algorithm.
626 See also:
627 - http://en.wikipedia.org/wiki/Dijkstra's_algorithm
628 """
630 import heapq
631 Q = []
632 Qd = {}
633 p = {}
634 visited = set([function])
636 ranks[function] = 0
637 for call in function.calls.values():
638 if call.callee_id != function.id:
639 callee = self.functions[call.callee_id]
640 if callee.cycle is cycle:
641 ranks[callee] = 1
642 item = [ranks[callee], function, callee]
643 heapq.heappush(Q, item)
644 Qd[callee] = item
646 while Q:
647 cost, parent, member = heapq.heappop(Q)
648 if member not in visited:
649 p[member]= parent
650 visited.add(member)
651 for call in member.calls.values():
652 if call.callee_id != member.id:
653 callee = self.functions[call.callee_id]
654 if callee.cycle is cycle:
655 member_rank = ranks[member]
656 rank = ranks.get(callee)
657 if rank is not None:
658 if rank > 1 + member_rank:
659 rank = 1 + member_rank
660 ranks[callee] = rank
661 Qd_callee = Qd[callee]
662 Qd_callee[0] = rank
663 Qd_callee[1] = member
664 heapq._siftdown(Q, 0, Q.index(Qd_callee))
665 else:
666 rank = 1 + member_rank
667 ranks[callee] = rank
668 item = [rank, member, callee]
669 heapq.heappush(Q, item)
670 Qd[callee] = item
672 def _call_ratios_cycle(self, cycle, function, ranks, call_ratios, visited):
673 if function not in visited:
674 visited.add(function)
675 for call in function.calls.values():
676 if call.callee_id != function.id:
677 callee = self.functions[call.callee_id]
678 if callee.cycle is cycle:
679 if ranks[callee] > ranks[function]:
680 call_ratios[callee] = call_ratios.get(callee, 0.0) + call.ratio
681 self._call_ratios_cycle(cycle, callee, ranks, call_ratios, visited)
683 def _integrate_cycle_function(self, cycle, function, partial_ratio, partials, ranks, call_ratios, outevent, inevent):
684 if function not in partials:
685 partial = partial_ratio*function[inevent]
686 for call in function.calls.values():
687 if call.callee_id != function.id:
688 callee = self.functions[call.callee_id]
689 if callee.cycle is not cycle:
690 assert outevent in call
691 partial += partial_ratio*call[outevent]
692 else:
693 if ranks[callee] > ranks[function]:
694 callee_partial = self._integrate_cycle_function(cycle, callee, partial_ratio, partials, ranks, call_ratios, outevent, inevent)
695 call_ratio = ratio(call.ratio, call_ratios[callee])
696 call_partial = call_ratio*callee_partial
697 try:
698 call[outevent] += call_partial
699 except UndefinedEvent:
700 call[outevent] = call_partial
701 partial += call_partial
702 partials[function] = partial
703 try:
704 function[outevent] += partial
705 except UndefinedEvent:
706 function[outevent] = partial
707 return partials[function]
709 def aggregate(self, event):
710 """Aggregate an event for the whole profile."""
712 total = event.null()
713 for function in self.functions.values():
714 try:
715 total = event.aggregate(total, function[event])
716 except UndefinedEvent:
717 return
718 self[event] = total
720 def ratio(self, outevent, inevent):
721 assert outevent not in self
722 assert inevent in self
723 for function in self.functions.values():
724 assert outevent not in function
725 assert inevent in function
726 function[outevent] = ratio(function[inevent], self[inevent])
727 for call in function.calls.values():
728 assert outevent not in call
729 if inevent in call:
730 call[outevent] = ratio(call[inevent], self[inevent])
731 self[outevent] = 1.0
733 def prune(self, node_thres, edge_thres, paths, color_nodes_by_selftime):
734 """Prune the profile"""
736 # compute the prune ratios
737 for function in self.functions.values():
738 try:
739 function.weight = function[TOTAL_TIME_RATIO]
740 except UndefinedEvent:
741 pass
743 for call in function.calls.values():
744 callee = self.functions[call.callee_id]
746 if TOTAL_TIME_RATIO in call:
747 # handle exact cases first
748 call.weight = call[TOTAL_TIME_RATIO]
749 else:
750 try:
751 # make a safe estimate
752 call.weight = min(function[TOTAL_TIME_RATIO], callee[TOTAL_TIME_RATIO])
753 except UndefinedEvent:
754 pass
756 # prune the nodes
757 for function_id in list(self.functions.keys()):
758 function = self.functions[function_id]
759 if function.weight is not None:
760 if function.weight < node_thres:
761 del self.functions[function_id]
763 # prune file paths
764 for function_id in list(self.functions.keys()):
765 function = self.functions[function_id]
766 if paths and function.filename and not any(function.filename.startswith(path) for path in paths):
767 del self.functions[function_id]
768 elif paths and function.module and not any((function.module.find(path)>-1) for path in paths):
769 del self.functions[function_id]
771 # prune the edges
772 for function in self.functions.values():
773 for callee_id in list(function.calls.keys()):
774 call = function.calls[callee_id]
775 if callee_id not in self.functions or call.weight is not None and call.weight < edge_thres:
776 del function.calls[callee_id]
778 if color_nodes_by_selftime:
779 weights = []
780 for function in self.functions.values():
781 try:
782 weights.append(function[TIME_RATIO])
783 except UndefinedEvent:
784 pass
785 max_ratio = max(weights or [1])
787 # apply rescaled weights for coloriung
788 for function in self.functions.values():
789 try:
790 function.weight = function[TIME_RATIO] / max_ratio
791 except (ZeroDivisionError, UndefinedEvent):
792 pass
794 def dump(self):
795 for function in self.functions.values():
796 sys.stderr.write('Function %s:\n' % (function.name,))
797 self._dump_events(function.events)
798 for call in function.calls.values():
799 callee = self.functions[call.callee_id]
800 sys.stderr.write(' Call %s:\n' % (callee.name,))
801 self._dump_events(call.events)
802 for cycle in self.cycles:
803 sys.stderr.write('Cycle:\n')
804 self._dump_events(cycle.events)
805 for function in cycle.functions:
806 sys.stderr.write(' Function %s\n' % (function.name,))
808 def _dump_events(self, events):
809 for event, value in events.items():
810 sys.stderr.write(' %s: %s\n' % (event.name, event.format(value)))
814########################################################################
815# Parsers
818class Struct:
819 """Masquerade a dictionary with a structure-like behavior."""
821 def __init__(self, attrs = None):
822 if attrs is None:
823 attrs = {}
824 self.__dict__['_attrs'] = attrs
826 def __getattr__(self, name):
827 try:
828 return self._attrs[name]
829 except KeyError:
830 raise AttributeError(name)
832 def __setattr__(self, name, value):
833 self._attrs[name] = value
835 def __str__(self):
836 return str(self._attrs)
838 def __repr__(self):
839 return repr(self._attrs)
842class ParseError(Exception):
843 """Raised when parsing to signal mismatches."""
845 def __init__(self, msg, line):
846 Exception.__init__(self)
847 self.msg = msg
848 # TODO: store more source line information
849 self.line = line
851 def __str__(self):
852 return '%s: %r' % (self.msg, self.line)
855class Parser:
856 """Parser interface."""
858 stdinInput = True
859 multipleInput = False
861 def __init__(self):
862 pass
864 def parse(self):
865 raise NotImplementedError
868class JsonParser(Parser):
869 """Parser for a custom JSON representation of profile data.
871 See schema.json for details.
872 """
875 def __init__(self, stream):
876 Parser.__init__(self)
877 self.stream = stream
879 @staticmethod
880 def _reject_constant(token):
881 # NaN/Infinity are not valid JSON (schema.json defines costs as finite
882 # numbers), but Python's json module accepts them as an extension. A
883 # non-finite cost poisons the SAMPLES total and every derived ratio, so
884 # refuse the tokens instead of accepting them.
885 raise ValueError('invalid JSON token %r' % token)
887 def parse(self):
889 try:
890 obj = json.load(self.stream, parse_constant=self._reject_constant)
891 except ValueError as ex:
892 sys.stderr.write('error: %s\n' % ex)
893 sys.exit(1)
895 assert obj['version'] == 0
897 profile = Profile()
898 profile[SAMPLES] = 0
900 fns = obj['functions']
902 for functionIndex in range(len(fns)):
903 fn = fns[functionIndex]
904 function = Function(functionIndex, fn['name'])
905 try:
906 function.module = fn['module']
907 except KeyError:
908 pass
909 try:
910 function.process = fn['process']
911 except KeyError:
912 pass
913 function[SAMPLES] = 0
914 function.called = 0
915 profile.add_function(function)
917 for event in obj['events']:
918 callchain = []
920 for functionIndex in event['callchain']:
921 function = profile.functions[functionIndex]
922 callchain.append(function)
924 # increment the call count of the first in the callchain
925 function = profile.functions[event['callchain'][0]]
926 function.called = function.called + 1
928 cost = event['cost'][0]
930 callee = callchain[0]
931 callee[SAMPLES] += cost
932 profile[SAMPLES] += cost
934 for caller in callchain[1:]:
935 try:
936 call = caller.calls[callee.id]
937 except KeyError:
938 call = Call(callee.id)
939 call[SAMPLES2] = cost
940 caller.add_call(call)
941 else:
942 call[SAMPLES2] += cost
944 callee = caller
946 if False:
947 profile.dump()
949 # compute derived data
950 profile.validate()
951 profile.find_cycles()
952 profile.ratio(TIME_RATIO, SAMPLES)
953 profile.call_ratios(SAMPLES2)
954 profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
956 return profile
959class LineParser(Parser):
960 """Base class for parsers that read line-based formats."""
962 def __init__(self, stream):
963 Parser.__init__(self)
964 self._stream = stream
965 self.__line = None
966 self.__eof = False
967 self.line_no = 0
969 def readline(self):
970 line = self._stream.readline()
971 if not line:
972 self.__line = ''
973 self.__eof = True
974 else:
975 self.line_no += 1
976 line = line.rstrip('\r\n')
977 self.__line = line
979 def lookahead(self):
980 assert self.__line is not None
981 return self.__line
983 def consume(self):
984 assert self.__line is not None
985 line = self.__line
986 self.readline()
987 return line
989 def eof(self):
990 assert self.__line is not None
991 return self.__eof
994XML_ELEMENT_START, XML_ELEMENT_END, XML_CHARACTER_DATA, XML_EOF = range(4)
997class XmlToken:
999 def __init__(self, type, name_or_data, attrs = None, line = None, column = None):
1000 assert type in (XML_ELEMENT_START, XML_ELEMENT_END, XML_CHARACTER_DATA, XML_EOF)
1001 self.type = type
1002 self.name_or_data = name_or_data
1003 self.attrs = attrs
1004 self.line = line
1005 self.column = column
1007 def __str__(self):
1008 if self.type == XML_ELEMENT_START:
1009 return '<' + self.name_or_data + ' ...>'
1010 if self.type == XML_ELEMENT_END:
1011 return '</' + self.name_or_data + '>'
1012 if self.type == XML_CHARACTER_DATA:
1013 return self.name_or_data
1014 if self.type == XML_EOF:
1015 return 'end of file'
1016 assert 0
1019class XmlTokenizer:
1020 """Expat based XML tokenizer."""
1022 def __init__(self, fp, skip_ws = True):
1023 self.fp = fp
1024 self.tokens = []
1025 self.index = 0
1026 self.final = False
1027 self.skip_ws = skip_ws
1029 self.character_pos = 0, 0
1030 self.character_data = ''
1032 self.parser = xml.parsers.expat.ParserCreate()
1033 self.parser.StartElementHandler = self.handle_element_start
1034 self.parser.EndElementHandler = self.handle_element_end
1035 self.parser.CharacterDataHandler = self.handle_character_data
1037 def handle_element_start(self, name, attributes):
1038 self.finish_character_data()
1039 line, column = self.pos()
1040 token = XmlToken(XML_ELEMENT_START, name, attributes, line, column)
1041 self.tokens.append(token)
1043 def handle_element_end(self, name):
1044 self.finish_character_data()
1045 line, column = self.pos()
1046 token = XmlToken(XML_ELEMENT_END, name, None, line, column)
1047 self.tokens.append(token)
1049 def handle_character_data(self, data):
1050 if not self.character_data:
1051 self.character_pos = self.pos()
1052 self.character_data += data
1054 def finish_character_data(self):
1055 if self.character_data:
1056 if not self.skip_ws or not self.character_data.isspace():
1057 line, column = self.character_pos
1058 token = XmlToken(XML_CHARACTER_DATA, self.character_data, None, line, column)
1059 self.tokens.append(token)
1060 self.character_data = ''
1062 def next(self):
1063 size = 16*1024
1064 while self.index >= len(self.tokens) and not self.final:
1065 self.tokens = []
1066 self.index = 0
1067 data = self.fp.read(size)
1068 self.final = len(data) < size
1069 self.parser.Parse(data, self.final)
1070 if self.index >= len(self.tokens):
1071 line, column = self.pos()
1072 token = XmlToken(XML_EOF, None, None, line, column)
1073 else:
1074 token = self.tokens[self.index]
1075 self.index += 1
1076 return token
1078 def pos(self):
1079 return self.parser.CurrentLineNumber, self.parser.CurrentColumnNumber
1082class XmlTokenMismatch(Exception):
1084 def __init__(self, expected, found):
1085 Exception.__init__(self)
1086 self.expected = expected
1087 self.found = found
1089 def __str__(self):
1090 return '%u:%u: %s expected, %s found' % (self.found.line, self.found.column, str(self.expected), str(self.found))
1093class XmlParser(Parser):
1094 """Base XML document parser."""
1096 def __init__(self, fp):
1097 Parser.__init__(self)
1098 self.tokenizer = XmlTokenizer(fp)
1099 self.consume()
1101 def consume(self):
1102 self.token = self.tokenizer.next()
1104 def match_element_start(self, name):
1105 return self.token.type == XML_ELEMENT_START and self.token.name_or_data == name
1107 def match_element_end(self, name):
1108 return self.token.type == XML_ELEMENT_END and self.token.name_or_data == name
1110 def element_start(self, name):
1111 while self.token.type == XML_CHARACTER_DATA:
1112 self.consume()
1113 if self.token.type != XML_ELEMENT_START:
1114 raise XmlTokenMismatch(XmlToken(XML_ELEMENT_START, name), self.token)
1115 if self.token.name_or_data != name:
1116 raise XmlTokenMismatch(XmlToken(XML_ELEMENT_START, name), self.token)
1117 attrs = self.token.attrs
1118 self.consume()
1119 return attrs
1121 def element_end(self, name):
1122 while self.token.type == XML_CHARACTER_DATA:
1123 self.consume()
1124 if self.token.type != XML_ELEMENT_END:
1125 raise XmlTokenMismatch(XmlToken(XML_ELEMENT_END, name), self.token)
1126 if self.token.name_or_data != name:
1127 raise XmlTokenMismatch(XmlToken(XML_ELEMENT_END, name), self.token)
1128 self.consume()
1130 def character_data(self, strip = True):
1131 data = ''
1132 while self.token.type == XML_CHARACTER_DATA:
1133 data += self.token.name_or_data
1134 self.consume()
1135 if strip:
1136 data = data.strip()
1137 return data
1140class GprofParser(Parser):
1141 """Parser for GNU gprof output.
1143 See also:
1144 - Chapter "Interpreting gprof's Output" from the GNU gprof manual
1145 http://sourceware.org/binutils/docs-2.18/gprof/Call-Graph.html#Call-Graph
1146 - File "cg_print.c" from the GNU gprof source code
1147 http://sourceware.org/cgi-bin/cvsweb.cgi/~checkout~/src/gprof/cg_print.c?rev=1.12&cvsroot=src
1148 """
1150 def __init__(self, fp):
1151 Parser.__init__(self)
1152 self.fp = fp
1153 self.functions = {}
1154 self.cycles = {}
1156 def readline(self):
1157 line = self.fp.readline()
1158 if not line:
1159 sys.stderr.write('error: unexpected end of file\n')
1160 sys.exit(1)
1161 line = line.rstrip('\r\n')
1162 return line
1164 _int_re = re.compile(r'^\d+$')
1165 _float_re = re.compile(r'^\d+\.\d+$')
1167 def translate(self, mo):
1168 """Extract a structure from a match object, while translating the types in the process."""
1169 attrs = {}
1170 groupdict = mo.groupdict()
1171 for name, value in groupdict.items():
1172 if value is None:
1173 value = None
1174 elif self._int_re.match(value):
1175 value = int(value)
1176 elif self._float_re.match(value):
1177 value = float(value)
1178 attrs[name] = (value)
1179 return Struct(attrs)
1181 _cg_header_re = re.compile(
1182 # original gprof header
1183 r'^\s+called/total\s+parents\s*$|' +
1184 r'^index\s+%time\s+self\s+descendents\s+called\+self\s+name\s+index\s*$|' +
1185 r'^\s+called/total\s+children\s*$|' +
1186 # GNU gprof header
1187 r'^index\s+%\s+(time\s+)?self\s+children\s+called\s+name\s*$'
1188 )
1190 _cg_ignore_re = re.compile(
1191 # spontaneous
1192 r'^\s+<spontaneous>\s*$|'
1193 # internal calls (such as "mcount")
1194 r'^.*\((\d+)\)$'
1195 )
1197 _cg_primary_re = re.compile(
1198 r'^\[(?P<index>\d+)\]?' +
1199 r'\s+(?P<percentage_time>\d+\.\d+)' +
1200 r'\s+(?P<self>\d+\.\d+)' +
1201 r'\s+(?P<descendants>\d+\.\d+)' +
1202 r'\s+(?:(?P<called>\d+)(?:\+(?P<called_self>\d+))?)?' +
1203 r'\s+(?P<name>\S.*?)' +
1204 r'(?:\s+<cycle\s(?P<cycle>\d+)>)?' +
1205 r'\s\[(\d+)\]$'
1206 )
1208 _cg_parent_re = re.compile(
1209 r'^\s+(?P<self>\d+\.\d+)?' +
1210 r'\s+(?P<descendants>\d+\.\d+)?' +
1211 r'\s+(?P<called>\d+)(?:/(?P<called_total>\d+))?' +
1212 r'\s+(?P<name>\S.*?)' +
1213 r'(?:\s+<cycle\s(?P<cycle>\d+)>)?' +
1214 r'\s\[(?P<index>\d+)\]$'
1215 )
1217 _cg_child_re = _cg_parent_re
1219 _cg_cycle_header_re = re.compile(
1220 r'^\[(?P<index>\d+)\]?' +
1221 r'\s+(?P<percentage_time>\d+\.\d+)' +
1222 r'\s+(?P<self>\d+\.\d+)' +
1223 r'\s+(?P<descendants>\d+\.\d+)' +
1224 r'\s+(?:(?P<called>\d+)(?:\+(?P<called_self>\d+))?)?' +
1225 r'\s+<cycle\s(?P<cycle>\d+)\sas\sa\swhole>' +
1226 r'\s\[(\d+)\]$'
1227 )
1229 _cg_cycle_member_re = re.compile(
1230 r'^\s+(?P<self>\d+\.\d+)?' +
1231 r'\s+(?P<descendants>\d+\.\d+)?' +
1232 r'\s+(?P<called>\d+)(?:\+(?P<called_self>\d+))?' +
1233 r'\s+(?P<name>\S.*?)' +
1234 r'(?:\s+<cycle\s(?P<cycle>\d+)>)?' +
1235 r'\s\[(?P<index>\d+)\]$'
1236 )
1238 _cg_sep_re = re.compile(r'^--+$')
1240 def parse_function_entry(self, lines):
1241 parents = []
1242 children = []
1244 while True:
1245 if not lines:
1246 sys.stderr.write('warning: unexpected end of entry\n')
1247 line = lines.pop(0)
1248 if line.startswith('['):
1249 break
1251 # read function parent line
1252 mo = self._cg_parent_re.match(line)
1253 if not mo:
1254 if self._cg_ignore_re.match(line):
1255 continue
1256 sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line)
1257 else:
1258 parent = self.translate(mo)
1259 parents.append(parent)
1261 # read primary line
1262 mo = self._cg_primary_re.match(line)
1263 if not mo:
1264 sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line)
1265 return
1266 else:
1267 function = self.translate(mo)
1269 while lines:
1270 line = lines.pop(0)
1272 # read function subroutine line
1273 mo = self._cg_child_re.match(line)
1274 if not mo:
1275 if self._cg_ignore_re.match(line):
1276 continue
1277 sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line)
1278 else:
1279 child = self.translate(mo)
1280 children.append(child)
1282 function.parents = parents
1283 function.children = children
1285 self.functions[function.index] = function
1287 def parse_cycle_entry(self, lines):
1289 # read cycle header line
1290 line = lines[0]
1291 mo = self._cg_cycle_header_re.match(line)
1292 if not mo:
1293 sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line)
1294 return
1295 cycle = self.translate(mo)
1297 # read cycle member lines
1298 cycle.functions = []
1299 for line in lines[1:]:
1300 mo = self._cg_cycle_member_re.match(line)
1301 if not mo:
1302 sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line)
1303 continue
1304 call = self.translate(mo)
1305 cycle.functions.append(call)
1307 self.cycles[cycle.cycle] = cycle
1309 def parse_cg_entry(self, lines):
1310 if lines[0].startswith("["):
1311 self.parse_cycle_entry(lines)
1312 else:
1313 self.parse_function_entry(lines)
1315 def parse_cg(self):
1316 """Parse the call graph."""
1318 # skip call graph header
1319 while not self._cg_header_re.match(self.readline()):
1320 pass
1321 line = self.readline()
1322 while self._cg_header_re.match(line):
1323 line = self.readline()
1325 # process call graph entries
1326 entry_lines = []
1327 while line != '\014': # form feed
1328 if line and not line.isspace():
1329 if self._cg_sep_re.match(line):
1330 self.parse_cg_entry(entry_lines)
1331 entry_lines = []
1332 else:
1333 entry_lines.append(line)
1334 line = self.readline()
1336 def parse(self):
1337 self.parse_cg()
1338 self.fp.close()
1340 profile = Profile()
1341 profile[TIME] = 0.0
1343 cycles = {}
1344 for index in self.cycles:
1345 cycles[index] = Cycle()
1347 for entry in self.functions.values():
1348 # populate the function
1349 function = Function(entry.index, entry.name)
1350 function[TIME] = entry.self
1351 if entry.called is not None:
1352 function.called = entry.called
1353 if entry.called_self is not None:
1354 call = Call(entry.index)
1355 call[CALLS] = entry.called_self
1356 function.called += entry.called_self
1358 # populate the function calls
1359 for child in entry.children:
1360 call = Call(child.index)
1362 assert child.called is not None
1363 call[CALLS] = child.called
1365 if child.index not in self.functions:
1366 # NOTE: functions that were never called but were discovered by gprof's
1367 # static call graph analysis dont have a call graph entry so we need
1368 # to add them here
1369 missing = Function(child.index, child.name)
1370 missing[TIME] = 0.0
1371 missing.called = 0
1372 profile.add_function(missing)
1374 function.add_call(call)
1376 profile.add_function(function)
1378 if entry.cycle is not None:
1379 try:
1380 cycle = cycles[entry.cycle]
1381 except KeyError:
1382 sys.stderr.write('warning: <cycle %u as a whole> entry missing\n' % entry.cycle)
1383 cycle = Cycle()
1384 cycles[entry.cycle] = cycle
1385 cycle.add_function(function)
1387 profile[TIME] = profile[TIME] + function[TIME]
1389 for cycle in cycles.values():
1390 profile.add_cycle(cycle)
1392 # Compute derived events
1393 profile.validate()
1394 profile.ratio(TIME_RATIO, TIME)
1395 profile.call_ratios(CALLS)
1396 profile.integrate(TOTAL_TIME, TIME)
1397 profile.ratio(TOTAL_TIME_RATIO, TOTAL_TIME)
1399 return profile
1402# Clone&hack of GprofParser for VTune Amplifier XE 2013 gprof-cc output.
1403# Tested only with AXE 2013 for Windows.
1404# - Use total times as reported by AXE.
1405# - In the absence of call counts, call ratios are faked from the relative
1406# proportions of total time. This affects only the weighting of the calls.
1407# - Different header, separator, and end marker.
1408# - Extra whitespace after function names.
1409# - You get a full entry for <spontaneous>, which does not have parents.
1410# - Cycles do have parents. These are saved but unused (as they are
1411# for functions).
1412# - Disambiguated "unrecognized call graph entry" error messages.
1413# Notes:
1414# - Total time of functions as reported by AXE passes the val3 test.
1415# - CPU Time:Children in the input is sometimes a negative number. This
1416# value goes to the variable descendants, which is unused.
1417# - The format of gprof-cc reports is unaffected by the use of
1418# -knob enable-call-counts=true (no call counts, ever), or
1419# -show-as=samples (results are quoted in seconds regardless).
1420class AXEParser(Parser):
1421 "Parser for VTune Amplifier XE 2013 gprof-cc report output."
1423 def __init__(self, fp):
1424 Parser.__init__(self)
1425 self.fp = fp
1426 self.functions = {}
1427 self.cycles = {}
1429 def readline(self):
1430 line = self.fp.readline()
1431 if not line:
1432 sys.stderr.write('error: unexpected end of file\n')
1433 sys.exit(1)
1434 line = line.rstrip('\r\n')
1435 return line
1437 _int_re = re.compile(r'^\d+$')
1438 _float_re = re.compile(r'^\d+\.\d+$')
1440 def translate(self, mo):
1441 """Extract a structure from a match object, while translating the types in the process."""
1442 attrs = {}
1443 groupdict = mo.groupdict()
1444 for name, value in groupdict.items():
1445 if value is None:
1446 value = None
1447 elif self._int_re.match(value):
1448 value = int(value)
1449 elif self._float_re.match(value):
1450 value = float(value)
1451 attrs[name] = (value)
1452 return Struct(attrs)
1454 _cg_header_re = re.compile(
1455 '^Index |'
1456 '^-----+ '
1457 )
1459 _cg_footer_re = re.compile(r'^Index\s+Function\s*$')
1461 _cg_primary_re = re.compile(
1462 r'^\[(?P<index>\d+)\]?' +
1463 r'\s+(?P<percentage_time>\d+\.\d+)' +
1464 r'\s+(?P<self>\d+\.\d+)' +
1465 r'\s+(?P<descendants>\d+\.\d+)' +
1466 r'\s+(?P<name>\S.*?)' +
1467 r'(?:\s+<cycle\s(?P<cycle>\d+)>)?' +
1468 r'\s+\[(\d+)\]' +
1469 r'\s*$'
1470 )
1472 _cg_parent_re = re.compile(
1473 r'^\s+(?P<self>\d+\.\d+)?' +
1474 r'\s+(?P<descendants>\d+\.\d+)?' +
1475 r'\s+(?P<name>\S.*?)' +
1476 r'(?:\s+<cycle\s(?P<cycle>\d+)>)?' +
1477 r'(?:\s+\[(?P<index>\d+)\]\s*)?' +
1478 r'\s*$'
1479 )
1481 _cg_child_re = _cg_parent_re
1483 _cg_cycle_header_re = re.compile(
1484 r'^\[(?P<index>\d+)\]?' +
1485 r'\s+(?P<percentage_time>\d+\.\d+)' +
1486 r'\s+(?P<self>\d+\.\d+)' +
1487 r'\s+(?P<descendants>\d+\.\d+)' +
1488 r'\s+<cycle\s(?P<cycle>\d+)\sas\sa\swhole>' +
1489 r'\s+\[(\d+)\]' +
1490 r'\s*$'
1491 )
1493 _cg_cycle_member_re = re.compile(
1494 r'^\s+(?P<self>\d+\.\d+)?' +
1495 r'\s+(?P<descendants>\d+\.\d+)?' +
1496 r'\s+(?P<name>\S.*?)' +
1497 r'(?:\s+<cycle\s(?P<cycle>\d+)>)?' +
1498 r'\s+\[(?P<index>\d+)\]' +
1499 r'\s*$'
1500 )
1502 def parse_function_entry(self, lines):
1503 parents = []
1504 children = []
1506 while True:
1507 if not lines:
1508 sys.stderr.write('warning: unexpected end of entry\n')
1509 return
1510 line = lines.pop(0)
1511 if line.startswith('['):
1512 break
1514 # read function parent line
1515 mo = self._cg_parent_re.match(line)
1516 if not mo:
1517 sys.stderr.write('warning: unrecognized call graph entry (1): %r\n' % line)
1518 else:
1519 parent = self.translate(mo)
1520 if parent.name != '<spontaneous>':
1521 parents.append(parent)
1523 # read primary line
1524 mo = self._cg_primary_re.match(line)
1525 if not mo:
1526 sys.stderr.write('warning: unrecognized call graph entry (2): %r\n' % line)
1527 return
1528 else:
1529 function = self.translate(mo)
1531 while lines:
1532 line = lines.pop(0)
1534 # read function subroutine line
1535 mo = self._cg_child_re.match(line)
1536 if not mo:
1537 sys.stderr.write('warning: unrecognized call graph entry (3): %r\n' % line)
1538 else:
1539 child = self.translate(mo)
1540 if child.name != '<spontaneous>':
1541 children.append(child)
1543 if function.name != '<spontaneous>':
1544 function.parents = parents
1545 function.children = children
1547 self.functions[function.index] = function
1549 def parse_cycle_entry(self, lines):
1551 # Process the parents that were not there in gprof format.
1552 parents = []
1553 while True:
1554 if not lines:
1555 sys.stderr.write('warning: unexpected end of cycle entry\n')
1556 return
1557 line = lines.pop(0)
1558 if line.startswith('['):
1559 break
1560 mo = self._cg_parent_re.match(line)
1561 if not mo:
1562 sys.stderr.write('warning: unrecognized call graph entry (6): %r\n' % line)
1563 else:
1564 parent = self.translate(mo)
1565 if parent.name != '<spontaneous>':
1566 parents.append(parent)
1568 # read cycle header line
1569 mo = self._cg_cycle_header_re.match(line)
1570 if not mo:
1571 sys.stderr.write('warning: unrecognized call graph entry (4): %r\n' % line)
1572 return
1573 cycle = self.translate(mo)
1575 # read cycle member lines
1576 cycle.functions = []
1577 for line in lines[1:]:
1578 mo = self._cg_cycle_member_re.match(line)
1579 if not mo:
1580 sys.stderr.write('warning: unrecognized call graph entry (5): %r\n' % line)
1581 continue
1582 call = self.translate(mo)
1583 cycle.functions.append(call)
1585 cycle.parents = parents
1586 self.cycles[cycle.cycle] = cycle
1588 def parse_cg_entry(self, lines):
1589 if any("as a whole" in linelooper for linelooper in lines):
1590 self.parse_cycle_entry(lines)
1591 else:
1592 self.parse_function_entry(lines)
1594 def parse_cg(self):
1595 """Parse the call graph."""
1597 # skip call graph header
1598 line = self.readline()
1599 while self._cg_header_re.match(line):
1600 line = self.readline()
1602 # process call graph entries
1603 entry_lines = []
1604 # An EOF in readline terminates the program without returning.
1605 while not self._cg_footer_re.match(line):
1606 if line.isspace():
1607 self.parse_cg_entry(entry_lines)
1608 entry_lines = []
1609 else:
1610 entry_lines.append(line)
1611 line = self.readline()
1613 def parse(self):
1614 sys.stderr.write('warning: for axe format, edge weights are unreliable estimates derived from function total times.\n')
1615 self.parse_cg()
1616 self.fp.close()
1618 profile = Profile()
1619 profile[TIME] = 0.0
1621 cycles = {}
1622 for index in self.cycles:
1623 cycles[index] = Cycle()
1625 for entry in self.functions.values():
1626 # populate the function
1627 function = Function(entry.index, entry.name)
1628 function[TIME] = entry.self
1629 function[TOTAL_TIME_RATIO] = entry.percentage_time / 100.0
1631 # populate the function calls
1632 for child in entry.children:
1633 call = Call(child.index)
1634 # The following bogus value affects only the weighting of
1635 # the calls.
1636 call[TOTAL_TIME_RATIO] = function[TOTAL_TIME_RATIO]
1638 if child.index not in self.functions:
1639 # NOTE: functions that were never called but were discovered by gprof's
1640 # static call graph analysis dont have a call graph entry so we need
1641 # to add them here
1642 # FIXME: Is this applicable?
1643 missing = Function(child.index, child.name)
1644 missing[TIME] = 0.0
1645 missing[TOTAL_TIME_RATIO] = 0.0
1646 profile.add_function(missing)
1648 function.add_call(call)
1650 profile.add_function(function)
1652 if entry.cycle is not None:
1653 try:
1654 cycle = cycles[entry.cycle]
1655 except KeyError:
1656 sys.stderr.write('warning: <cycle %u as a whole> entry missing\n' % entry.cycle)
1657 cycle = Cycle()
1658 cycles[entry.cycle] = cycle
1659 cycle.add_function(function)
1661 profile[TIME] = profile[TIME] + function[TIME]
1663 for cycle in cycles.values():
1664 profile.add_cycle(cycle)
1666 # Compute derived events.
1667 profile.validate()
1668 profile.ratio(TIME_RATIO, TIME)
1669 # Lacking call counts, fake call ratios based on total times.
1670 profile.call_ratios(TOTAL_TIME_RATIO)
1671 # The TOTAL_TIME_RATIO of functions is already set. Propagate that
1672 # total time to the calls. (TOTAL_TIME is neither set nor used.)
1673 for function in profile.functions.values():
1674 for call in function.calls.values():
1675 if call.ratio is not None:
1676 callee = profile.functions[call.callee_id]
1677 call[TOTAL_TIME_RATIO] = call.ratio * callee[TOTAL_TIME_RATIO]
1679 return profile
1682class CallgrindParser(LineParser):
1683 """Parser for valgrind's callgrind tool.
1685 See also:
1686 - https://valgrind.org/docs/manual/cl-format.html
1687 """
1689 _call_re = re.compile(r'^calls=\s*(\d+)\s+((\d+|\+\d+|-\d+|\*)\s+)+$')
1691 def __init__(self, infile):
1692 LineParser.__init__(self, infile)
1694 # Textual positions
1695 self.position_ids = {}
1696 self.positions = {}
1698 # Numeric positions
1699 self.num_positions = 1
1700 self.cost_positions = ['line']
1701 self.last_positions = [0]
1703 # Events
1704 self.num_events = 0
1705 self.cost_events = []
1707 self.profile = Profile()
1708 self.profile[SAMPLES] = 0
1710 def parse(self):
1711 # read lookahead
1712 self.readline()
1714 self.parse_key('version')
1715 self.parse_key('creator')
1716 while self.parse_part():
1717 pass
1718 if not self.eof():
1719 sys.stderr.write('warning: line %u: unexpected line\n' % self.line_no)
1720 sys.stderr.write('%s\n' % self.lookahead())
1722 # compute derived data
1723 self.profile.validate()
1724 self.profile.find_cycles()
1725 self.profile.ratio(TIME_RATIO, SAMPLES)
1726 self.profile.call_ratios(SAMPLES2)
1727 self.profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
1729 return self.profile
1731 def parse_part(self):
1732 if not self.parse_header_line():
1733 return False
1734 while self.parse_header_line():
1735 pass
1736 if not self.parse_body_line():
1737 return False
1738 while self.parse_body_line():
1739 pass
1740 return True
1742 def parse_header_line(self):
1743 return \
1744 self.parse_empty() or \
1745 self.parse_comment() or \
1746 self.parse_part_detail() or \
1747 self.parse_description() or \
1748 self.parse_event_specification() or \
1749 self.parse_cost_line_def() or \
1750 self.parse_cost_summary()
1752 _detail_keys = set(('cmd', 'pid', 'thread', 'part'))
1754 def parse_part_detail(self):
1755 return self.parse_keys(self._detail_keys)
1757 def parse_description(self):
1758 return self.parse_key('desc') is not None
1760 def parse_event_specification(self):
1761 event = self.parse_key('event')
1762 if event is None:
1763 return False
1764 return True
1766 def parse_cost_line_def(self):
1767 pair = self.parse_keys(('events', 'positions'))
1768 if pair is None:
1769 return False
1770 key, value = pair
1771 items = value.split()
1772 if key == 'events':
1773 self.num_events = len(items)
1774 self.cost_events = items
1775 if key == 'positions':
1776 self.num_positions = len(items)
1777 self.cost_positions = items
1778 self.last_positions = [0]*self.num_positions
1779 return True
1781 def parse_cost_summary(self):
1782 pair = self.parse_keys(('summary', 'totals'))
1783 if pair is None:
1784 return False
1785 return True
1787 def parse_body_line(self):
1788 return \
1789 self.parse_empty() or \
1790 self.parse_comment() or \
1791 self.parse_cost_line() or \
1792 self.parse_position_spec() or \
1793 self.parse_association_spec()
1795 __subpos_re = r'(0x[0-9a-fA-F]+|\d+|\+\d+|-\d+|\*)'
1796 _cost_re = re.compile(r'^' +
1797 __subpos_re + r'( +' + __subpos_re + r')*' +
1798 r'( +\d+)*' +
1799 '$')
1801 def parse_cost_line(self, calls=None):
1802 line = self.lookahead().rstrip()
1803 mo = self._cost_re.match(line)
1804 if not mo:
1805 return False
1807 function = self.get_function()
1809 if calls is None:
1810 # Unlike other aspects, call object (cob) is relative not to the
1811 # last call object, but to the caller's object (ob), so try to
1812 # update it when processing a functions cost line
1813 try:
1814 self.positions['cob'] = self.positions['ob']
1815 except KeyError:
1816 pass
1818 values = line.split()
1819 assert len(values) <= self.num_positions + self.num_events
1821 positions = values[0 : self.num_positions]
1822 events = values[self.num_positions : ]
1823 events += ['0']*(self.num_events - len(events))
1825 for i in range(self.num_positions):
1826 position = positions[i]
1827 if position == '*':
1828 position = self.last_positions[i]
1829 elif position[0] in '-+':
1830 position = self.last_positions[i] + int(position)
1831 elif position.startswith('0x'):
1832 position = int(position, 16)
1833 else:
1834 position = int(position)
1835 self.last_positions[i] = position
1837 events = [float(event) for event in events]
1839 if calls is None:
1840 function[SAMPLES] += events[0]
1841 self.profile[SAMPLES] += events[0]
1842 else:
1843 callee = self.get_callee()
1844 callee.called += calls
1846 try:
1847 call = function.calls[callee.id]
1848 except KeyError:
1849 call = Call(callee.id)
1850 call[CALLS] = calls
1851 call[SAMPLES2] = events[0]
1852 function.add_call(call)
1853 else:
1854 call[CALLS] += calls
1855 call[SAMPLES2] += events[0]
1857 self.consume()
1858 return True
1860 def parse_association_spec(self):
1861 line = self.lookahead()
1862 if not line.startswith('calls='):
1863 return False
1865 _, values = line.split('=', 1)
1866 values = values.strip().split()
1867 calls = int(values[0])
1868 call_position = values[1:]
1869 self.consume()
1871 self.parse_cost_line(calls)
1873 return True
1875 _position_re = re.compile(r'^(?P<position>[cj]?(?:ob|fl|fi|fe|fn))=\s*(?:\((?P<id>\d+)\))?(?:\s*(?P<name>.+))?')
1877 _position_table_map = {
1878 'ob': 'ob',
1879 'fl': 'fl',
1880 'fi': 'fl',
1881 'fe': 'fl',
1882 'fn': 'fn',
1883 'cob': 'ob',
1884 'cfl': 'fl',
1885 'cfi': 'fl',
1886 'cfe': 'fl',
1887 'cfn': 'fn',
1888 'jfi': 'fl',
1889 }
1891 _position_map = {
1892 'ob': 'ob',
1893 'fl': 'fl',
1894 'fi': 'fl',
1895 'fe': 'fl',
1896 'fn': 'fn',
1897 'cob': 'cob',
1898 'cfl': 'cfl',
1899 'cfi': 'cfl',
1900 'cfe': 'cfl',
1901 'cfn': 'cfn',
1902 'jfi': 'jfi',
1903 }
1905 def parse_position_spec(self):
1906 line = self.lookahead()
1908 if line.startswith('jump=') or line.startswith('jcnd='):
1909 self.consume()
1910 return True
1912 mo = self._position_re.match(line)
1913 if not mo:
1914 return False
1916 position, id, name = mo.groups()
1917 if id:
1918 table = self._position_table_map[position]
1919 if name:
1920 self.position_ids[(table, id)] = name
1921 else:
1922 name = self.position_ids.get((table, id), '')
1923 self.positions[self._position_map[position]] = name
1925 self.consume()
1926 return True
1928 def parse_empty(self):
1929 if self.eof():
1930 return False
1931 line = self.lookahead()
1932 if line.strip():
1933 return False
1934 self.consume()
1935 return True
1937 def parse_comment(self):
1938 line = self.lookahead()
1939 if not line.startswith('#'):
1940 return False
1941 self.consume()
1942 return True
1944 _key_re = re.compile(r'^(\w+):')
1946 def parse_key(self, key):
1947 pair = self.parse_keys((key,))
1948 if not pair:
1949 return None
1950 key, value = pair
1951 return value
1953 def parse_keys(self, keys):
1954 line = self.lookahead()
1955 mo = self._key_re.match(line)
1956 if not mo:
1957 return None
1958 key, value = line.split(':', 1)
1959 if key not in keys:
1960 return None
1961 value = value.strip()
1962 self.consume()
1963 return key, value
1965 def make_function(self, module, filename, name):
1966 # FIXME: module and filename are not being tracked reliably
1967 #id = '|'.join((module, filename, name))
1968 id = name
1969 try:
1970 function = self.profile.functions[id]
1971 except KeyError:
1972 function = Function(id, name)
1973 if module:
1974 function.module = os.path.basename(module)
1975 function[SAMPLES] = 0
1976 function.called = 0
1977 self.profile.add_function(function)
1978 return function
1980 def get_function(self):
1981 module = self.positions.get('ob', '')
1982 filename = self.positions.get('fl', '')
1983 function = self.positions.get('fn', '')
1984 return self.make_function(module, filename, function)
1986 def get_callee(self):
1987 module = self.positions.get('cob', '')
1988 filename = self.positions.get('cfi', '')
1989 function = self.positions.get('cfn', '')
1990 return self.make_function(module, filename, function)
1992 def readline(self):
1993 # Override LineParser.readline to ignore comment lines
1994 while True:
1995 LineParser.readline(self)
1996 if self.eof() or not self.lookahead().startswith('#'):
1997 break
2000class PerfParser(LineParser):
2001 """Parser for linux perf callgraph output.
2003 It expects output generated with
2005 perf record -g
2006 perf script | gprof2dot.py --format=perf
2007 """
2009 def __init__(self, infile):
2010 LineParser.__init__(self, infile)
2011 self.profile = Profile()
2013 def readline(self):
2014 # Override LineParser.readline to ignore comment lines
2015 while True:
2016 LineParser.readline(self)
2017 if self.eof() or not self.lookahead().startswith('#'):
2018 break
2020 def parse(self):
2021 # read lookahead
2022 self.readline()
2024 profile = self.profile
2025 profile[SAMPLES] = 0
2026 while not self.eof():
2027 self.parse_event()
2029 # compute derived data
2030 profile.validate()
2031 profile.find_cycles()
2032 profile.ratio(TIME_RATIO, SAMPLES)
2033 profile.call_ratios(SAMPLES2)
2034 if totalMethod == "callratios":
2035 # Heuristic approach. TOTAL_SAMPLES is unused.
2036 profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
2037 elif totalMethod == "callstacks":
2038 # Use the actual call chains for functions.
2039 profile[TOTAL_SAMPLES] = profile[SAMPLES]
2040 profile.ratio(TOTAL_TIME_RATIO, TOTAL_SAMPLES)
2041 # Then propagate that total time to the calls.
2042 for function in profile.functions.values():
2043 for call in function.calls.values():
2044 if call.ratio is not None:
2045 callee = profile.functions[call.callee_id]
2046 call[TOTAL_TIME_RATIO] = call.ratio * callee[TOTAL_TIME_RATIO]
2047 else:
2048 assert False
2050 return profile
2052 def parse_event(self):
2053 if self.eof():
2054 return
2056 line = self.consume()
2057 assert line
2059 callchain = self.parse_callchain()
2060 if not callchain:
2061 return
2063 callee = callchain[0]
2064 callee[SAMPLES] += 1
2065 self.profile[SAMPLES] += 1
2067 for caller in callchain[1:]:
2068 try:
2069 call = caller.calls[callee.id]
2070 except KeyError:
2071 call = Call(callee.id)
2072 call[SAMPLES2] = 1
2073 caller.add_call(call)
2074 else:
2075 call[SAMPLES2] += 1
2077 callee = caller
2079 # Increment TOTAL_SAMPLES only once on each function.
2080 stack = set(callchain)
2081 for function in stack:
2082 function[TOTAL_SAMPLES] += 1
2084 def parse_callchain(self):
2085 callchain = []
2086 while self.lookahead():
2087 function = self.parse_call()
2088 if function is None:
2089 break
2090 callchain.append(function)
2091 if self.lookahead() == '':
2092 self.consume()
2093 return callchain
2095 call_re = re.compile(r'^\s+(?P<address>[0-9a-fA-F]+)\s+(?P<symbol>.*)\s+\((?P<module>.*)\)$')
2096 addr2_re = re.compile(r'\+0x[0-9a-fA-F]+$')
2098 def parse_call(self):
2099 line = self.consume()
2100 mo = self.call_re.match(line)
2101 assert mo
2102 if not mo:
2103 return None
2105 function_name = mo.group('symbol')
2107 # If present, amputate program counter from function name.
2108 if function_name:
2109 function_name = re.sub(self.addr2_re, '', function_name)
2111 if not function_name or function_name == '[unknown]':
2112 function_name = mo.group('address')
2114 module = mo.group('module')
2116 function_id = function_name + ':' + module
2118 try:
2119 function = self.profile.functions[function_id]
2120 except KeyError:
2121 function = Function(function_id, function_name)
2122 function.module = os.path.basename(module)
2123 function[SAMPLES] = 0
2124 function[TOTAL_SAMPLES] = 0
2125 self.profile.add_function(function)
2127 return function
2130class HProfParser(LineParser):
2131 """Parser for java hprof output
2133 See also:
2134 - http://java.sun.com/developer/technicalArticles/Programming/HPROF.html
2135 """
2137 trace_re = re.compile(r'\t(.*)\((.*):(.*)\)')
2138 trace_id_re = re.compile(r'^TRACE (\d+):$')
2140 def __init__(self, infile):
2141 LineParser.__init__(self, infile)
2142 self.traces = {}
2143 self.samples = {}
2145 def parse(self):
2146 # read lookahead
2147 self.readline()
2149 while not self.lookahead().startswith('------'): self.consume()
2150 while not self.lookahead().startswith('TRACE '): self.consume()
2152 self.parse_traces()
2154 while not self.lookahead().startswith('CPU'):
2155 self.consume()
2157 self.parse_samples()
2159 # populate the profile
2160 profile = Profile()
2161 profile[SAMPLES] = 0
2163 functions = {}
2165 # build up callgraph
2166 for id, trace in self.traces.items():
2167 if not id in self.samples: continue
2168 mtime = self.samples[id][0]
2169 last = None
2171 for func, file, line in trace:
2172 if not func in functions:
2173 function = Function(func, func)
2174 function[SAMPLES] = 0
2175 profile.add_function(function)
2176 functions[func] = function
2178 function = functions[func]
2179 # allocate time to the deepest method in the trace
2180 if not last:
2181 function[SAMPLES] += mtime
2182 profile[SAMPLES] += mtime
2183 else:
2184 c = function.get_call(last)
2185 c[SAMPLES2] += mtime
2187 last = func
2189 # compute derived data
2190 profile.validate()
2191 profile.find_cycles()
2192 profile.ratio(TIME_RATIO, SAMPLES)
2193 profile.call_ratios(SAMPLES2)
2194 profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
2196 return profile
2198 def parse_traces(self):
2199 while self.lookahead().startswith('TRACE '):
2200 self.parse_trace()
2202 def parse_trace(self):
2203 l = self.consume()
2204 mo = self.trace_id_re.match(l)
2205 tid = mo.group(1)
2206 last = None
2207 trace = []
2209 while self.lookahead().startswith('\t'):
2210 l = self.consume()
2211 match = self.trace_re.search(l)
2212 if not match:
2213 #sys.stderr.write('Invalid line: %s\n' % l)
2214 break
2215 else:
2216 function_name, file, line = match.groups()
2217 trace += [(function_name, file, line)]
2219 self.traces[int(tid)] = trace
2221 def parse_samples(self):
2222 self.consume()
2223 self.consume()
2225 while not self.lookahead().startswith('CPU'):
2226 rank, percent_self, percent_accum, count, traceid, method = self.lookahead().split()
2227 self.samples[int(traceid)] = (int(count), method)
2228 self.consume()
2231class SysprofParser(XmlParser):
2233 def __init__(self, stream):
2234 XmlParser.__init__(self, stream)
2236 def parse(self):
2237 objects = {}
2238 nodes = {}
2240 self.element_start('profile')
2241 while self.token.type == XML_ELEMENT_START:
2242 if self.token.name_or_data == 'objects':
2243 assert not objects
2244 objects = self.parse_items('objects')
2245 elif self.token.name_or_data == 'nodes':
2246 assert not nodes
2247 nodes = self.parse_items('nodes')
2248 else:
2249 self.parse_value(self.token.name_or_data)
2250 self.element_end('profile')
2252 return self.build_profile(objects, nodes)
2254 def parse_items(self, name):
2255 assert name[-1] == 's'
2256 items = {}
2257 self.element_start(name)
2258 while self.token.type == XML_ELEMENT_START:
2259 id, values = self.parse_item(name[:-1])
2260 assert id not in items
2261 items[id] = values
2262 self.element_end(name)
2263 return items
2265 def parse_item(self, name):
2266 attrs = self.element_start(name)
2267 id = int(attrs['id'])
2268 values = self.parse_values()
2269 self.element_end(name)
2270 return id, values
2272 def parse_values(self):
2273 values = {}
2274 while self.token.type == XML_ELEMENT_START:
2275 name = self.token.name_or_data
2276 value = self.parse_value(name)
2277 assert name not in values
2278 values[name] = value
2279 return values
2281 def parse_value(self, tag):
2282 self.element_start(tag)
2283 value = self.character_data()
2284 self.element_end(tag)
2285 if value.isdigit():
2286 return int(value)
2287 if value.startswith('"') and value.endswith('"'):
2288 return value[1:-1]
2289 return value
2291 def build_profile(self, objects, nodes):
2292 profile = Profile()
2294 profile[SAMPLES] = 0
2295 for id, object in objects.items():
2296 # Ignore fake objects (process names, modules, "Everything", "kernel", etc.)
2297 if object['self'] == 0:
2298 continue
2300 function = Function(id, object['name'])
2301 function[SAMPLES] = object['self']
2302 profile.add_function(function)
2303 profile[SAMPLES] += function[SAMPLES]
2305 for id, node in nodes.items():
2306 # Ignore fake calls
2307 if node['self'] == 0:
2308 continue
2310 # Find a non-ignored parent
2311 parent_id = node['parent']
2312 while parent_id != 0:
2313 parent = nodes[parent_id]
2314 caller_id = parent['object']
2315 if objects[caller_id]['self'] != 0:
2316 break
2317 parent_id = parent['parent']
2318 if parent_id == 0:
2319 continue
2321 callee_id = node['object']
2323 assert objects[caller_id]['self']
2324 assert objects[callee_id]['self']
2326 function = profile.functions[caller_id]
2328 samples = node['self']
2329 try:
2330 call = function.calls[callee_id]
2331 except KeyError:
2332 call = Call(callee_id)
2333 call[SAMPLES2] = samples
2334 function.add_call(call)
2335 else:
2336 call[SAMPLES2] += samples
2338 # Compute derived events
2339 profile.validate()
2340 profile.find_cycles()
2341 profile.ratio(TIME_RATIO, SAMPLES)
2342 profile.call_ratios(SAMPLES2)
2343 profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
2345 return profile
2348class XPerfParser(Parser):
2349 """Parser for CSVs generated by XPerf, from Microsoft Windows Performance Tools.
2350 """
2352 def __init__(self, stream):
2353 Parser.__init__(self)
2354 self.stream = stream
2355 self.profile = Profile()
2356 self.profile[SAMPLES] = 0
2357 self.column = {}
2359 def parse(self):
2360 import csv
2361 reader = csv.reader(
2362 self.stream,
2363 delimiter = ',',
2364 quotechar = None,
2365 escapechar = None,
2366 doublequote = False,
2367 skipinitialspace = True,
2368 lineterminator = '\r\n',
2369 quoting = csv.QUOTE_NONE)
2370 header = True
2371 for row in reader:
2372 if header:
2373 self.parse_header(row)
2374 header = False
2375 else:
2376 self.parse_row(row)
2378 # compute derived data
2379 self.profile.validate()
2380 self.profile.find_cycles()
2381 self.profile.ratio(TIME_RATIO, SAMPLES)
2382 self.profile.call_ratios(SAMPLES2)
2383 self.profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
2385 return self.profile
2387 def parse_header(self, row):
2388 for column in range(len(row)):
2389 name = row[column]
2390 assert name not in self.column
2391 self.column[name] = column
2393 def parse_row(self, row):
2394 fields = {}
2395 for name, column in self.column.items():
2396 value = row[column]
2397 for factory in int, float:
2398 try:
2399 value = factory(value)
2400 except ValueError:
2401 pass
2402 else:
2403 break
2404 fields[name] = value
2406 process = fields['Process Name']
2407 symbol = fields['Module'] + '!' + fields['Function']
2408 weight = fields['Weight']
2409 count = fields['Count']
2411 if process == 'Idle':
2412 return
2414 function = self.get_function(process, symbol)
2415 function[SAMPLES] += weight * count
2416 self.profile[SAMPLES] += weight * count
2418 stack = fields['Stack']
2419 if stack != '?':
2420 stack = stack.split('/')
2421 assert stack[0] == '[Root]'
2422 if stack[-1] != symbol:
2423 # XXX: some cases the sampled function does not appear in the stack
2424 stack.append(symbol)
2425 caller = None
2426 for symbol in stack[1:]:
2427 callee = self.get_function(process, symbol)
2428 if caller is not None:
2429 try:
2430 call = caller.calls[callee.id]
2431 except KeyError:
2432 call = Call(callee.id)
2433 call[SAMPLES2] = count
2434 caller.add_call(call)
2435 else:
2436 call[SAMPLES2] += count
2437 caller = callee
2439 def get_function(self, process, symbol):
2440 function_id = process + '!' + symbol
2442 try:
2443 function = self.profile.functions[function_id]
2444 except KeyError:
2445 module, name = symbol.split('!', 1)
2446 function = Function(function_id, name)
2447 function.process = process
2448 function.module = module
2449 function[SAMPLES] = 0
2450 self.profile.add_function(function)
2452 return function
2455class SleepyParser(Parser):
2456 """Parser for GNU gprof output.
2458 See also:
2459 - http://www.codersnotes.com/sleepy/
2460 - http://sleepygraph.sourceforge.net/
2461 """
2463 stdinInput = False
2465 def __init__(self, filename):
2466 Parser.__init__(self)
2468 from zipfile import ZipFile
2470 self.database = ZipFile(filename)
2472 self.symbols = {}
2473 self.calls = {}
2475 self.profile = Profile()
2477 _symbol_re = re.compile(
2478 r'^(?P<id>\w+)' +
2479 r'\s+"(?P<module>[^"]*)"' +
2480 r'\s+"(?P<procname>[^"]*)"' +
2481 r'\s+"(?P<sourcefile>[^"]*)"' +
2482 r'\s+(?P<sourceline>\d+)$'
2483 )
2485 def openEntry(self, name):
2486 # Some versions of verysleepy use lowercase filenames
2487 for database_name in self.database.namelist():
2488 if name.lower() == database_name.lower():
2489 name = database_name
2490 break
2492 return self.database.open(name, 'r')
2494 def parse_symbols(self):
2495 for line in self.openEntry('Symbols.txt'):
2496 line = line.decode('UTF-8').rstrip('\r\n')
2498 mo = self._symbol_re.match(line)
2499 if mo:
2500 symbol_id, module, procname, sourcefile, sourceline = mo.groups()
2502 function_id = ':'.join([module, procname])
2504 try:
2505 function = self.profile.functions[function_id]
2506 except KeyError:
2507 function = Function(function_id, procname)
2508 function.module = module
2509 function[SAMPLES] = 0
2510 self.profile.add_function(function)
2512 self.symbols[symbol_id] = function
2514 def parse_callstacks(self):
2515 for line in self.openEntry('Callstacks.txt'):
2516 line = line.decode('UTF-8').rstrip('\r\n')
2518 fields = line.split()
2519 samples = float(fields[0])
2520 callstack = fields[1:]
2522 callstack = [self.symbols[symbol_id] for symbol_id in callstack]
2524 callee = callstack[0]
2526 callee[SAMPLES] += samples
2527 self.profile[SAMPLES] += samples
2529 for caller in callstack[1:]:
2530 try:
2531 call = caller.calls[callee.id]
2532 except KeyError:
2533 call = Call(callee.id)
2534 call[SAMPLES2] = samples
2535 caller.add_call(call)
2536 else:
2537 call[SAMPLES2] += samples
2539 callee = caller
2541 def parse(self):
2542 profile = self.profile
2543 profile[SAMPLES] = 0
2545 self.parse_symbols()
2546 self.parse_callstacks()
2548 # Compute derived events
2549 profile.validate()
2550 profile.find_cycles()
2551 profile.ratio(TIME_RATIO, SAMPLES)
2552 profile.call_ratios(SAMPLES2)
2553 profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
2555 return profile
2558class PstatsParser:
2559 """Parser python profiling statistics saved with te pstats module."""
2561 stdinInput = False
2562 multipleInput = True
2564 def __init__(self, *filename):
2565 import pstats
2566 try:
2567 self.stats = pstats.Stats(*filename)
2568 except ValueError:
2569 sys.stderr.write('error: failed to load %s, maybe they are generated by different python version?\n' % ', '.join(filename))
2570 sys.exit(1)
2571 self.profile = Profile()
2572 self.function_ids = {}
2574 def get_function_name(self, key):
2575 filename, line, name = key
2576 module = os.path.splitext(filename)[0]
2577 module = os.path.basename(module)
2578 return "%s:%d:%s" % (module, line, name)
2580 def get_function(self, key):
2581 try:
2582 id = self.function_ids[key]
2583 except KeyError:
2584 id = len(self.function_ids)
2585 name = self.get_function_name(key)
2586 function = Function(id, name)
2587 function.filename = key[0]
2588 self.profile.functions[id] = function
2589 self.function_ids[key] = id
2590 else:
2591 function = self.profile.functions[id]
2592 return function
2594 def parse(self):
2595 self.profile[TIME] = 0.0
2596 self.profile[TOTAL_TIME] = self.stats.total_tt
2597 for fn, (cc, nc, tt, ct, callers) in self.stats.stats.items():
2598 callee = self.get_function(fn)
2599 callee.called = nc
2600 callee[TOTAL_TIME] = ct
2601 callee[TIME] = tt
2602 self.profile[TIME] += tt
2603 self.profile[TOTAL_TIME] = max(self.profile[TOTAL_TIME], ct)
2604 for fn, value in callers.items():
2605 caller = self.get_function(fn)
2606 call = Call(callee.id)
2607 if isinstance(value, tuple):
2608 for i in range(0, len(value), 4):
2609 nc, cc, tt, ct = value[i:i+4]
2610 if CALLS in call:
2611 call[CALLS] += cc
2612 else:
2613 call[CALLS] = cc
2615 if TOTAL_TIME in call:
2616 call[TOTAL_TIME] += ct
2617 else:
2618 call[TOTAL_TIME] = ct
2620 else:
2621 call[CALLS] = value
2622 call[TOTAL_TIME] = ratio(value, nc)*ct
2624 caller.add_call(call)
2626 if False:
2627 self.stats.print_stats()
2628 self.stats.print_callees()
2630 # Compute derived events
2631 self.profile.validate()
2632 self.profile.ratio(TIME_RATIO, TIME)
2633 self.profile.ratio(TOTAL_TIME_RATIO, TOTAL_TIME)
2635 return self.profile
2637class DtraceParser(LineParser):
2638 """Parser for linux perf callgraph output.
2640 It expects output generated with
2642 # Refer to https://github.com/brendangregg/FlameGraph#dtrace
2643 # 60 seconds of user-level stacks, including time spent in-kernel, for PID 12345 at 97 Hertz
2644 sudo dtrace -x ustackframes=100 -n 'profile-97 /pid == 12345/ { @[ustack()] = count(); } tick-60s { exit(0); }' -o out.user_stacks
2646 # The dtrace output
2647 gprof2dot.py -f dtrace out.user_stacks
2649 # Notice: sometimes, the dtrace outputs format may be latin-1, and gprof2dot will fail to parse it.
2650 # To solve this problem, you should use iconv to convert to UTF-8 explicitly.
2651 # TODO: add an encoding flag to tell gprof2dot how to decode the profile file.
2652 iconv -f ISO-8859-1 -t UTF-8 out.user_stacks | gprof2dot.py -f dtrace
2653 """
2655 def __init__(self, infile):
2656 LineParser.__init__(self, infile)
2657 self.profile = Profile()
2659 def readline(self):
2660 # Override LineParser.readline to ignore comment lines
2661 while True:
2662 LineParser.readline(self)
2663 if self.eof():
2664 break
2666 line = self.lookahead().strip()
2667 if line.startswith('CPU'):
2668 # The format likes:
2669 # CPU ID FUNCTION:NAME
2670 # 1 29684 :tick-60s
2671 # Skip next line
2672 LineParser.readline(self)
2673 elif not line == '':
2674 break
2677 def parse(self):
2678 # read lookahead
2679 self.readline()
2681 profile = self.profile
2682 profile[SAMPLES] = 0
2683 while not self.eof():
2684 self.parse_event()
2686 # compute derived data
2687 profile.validate()
2688 profile.find_cycles()
2689 profile.ratio(TIME_RATIO, SAMPLES)
2690 profile.call_ratios(SAMPLES2)
2691 if totalMethod == "callratios":
2692 # Heuristic approach. TOTAL_SAMPLES is unused.
2693 profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
2694 elif totalMethod == "callstacks":
2695 # Use the actual call chains for functions.
2696 profile[TOTAL_SAMPLES] = profile[SAMPLES]
2697 profile.ratio(TOTAL_TIME_RATIO, TOTAL_SAMPLES)
2698 # Then propagate that total time to the calls.
2699 for function in profile.functions.values():
2700 for call in function.calls.values():
2701 if call.ratio is not None:
2702 callee = profile.functions[call.callee_id]
2703 call[TOTAL_TIME_RATIO] = call.ratio * callee[TOTAL_TIME_RATIO]
2704 else:
2705 assert False
2707 return profile
2709 def parse_event(self):
2710 if self.eof():
2711 return
2713 callchain, count = self.parse_callchain()
2714 if not callchain:
2715 return
2717 callee = callchain[0]
2718 callee[SAMPLES] += count
2719 self.profile[SAMPLES] += count
2721 for caller in callchain[1:]:
2722 try:
2723 call = caller.calls[callee.id]
2724 except KeyError:
2725 call = Call(callee.id)
2726 call[SAMPLES2] = count
2727 caller.add_call(call)
2728 else:
2729 call[SAMPLES2] += count
2731 callee = caller
2733 # Increment TOTAL_SAMPLES only once on each function.
2734 stack = set(callchain)
2735 for function in stack:
2736 function[TOTAL_SAMPLES] += count
2739 def parse_callchain(self):
2740 callchain = []
2741 count = 0
2742 while self.lookahead():
2743 function, count = self.parse_call()
2744 if function is None:
2745 break
2746 callchain.append(function)
2747 return callchain, count
2749 call_re = re.compile(r'^\s+(?P<module>.*)`(?P<symbol>.*)')
2750 addr2_re = re.compile(r'\+0x[0-9a-fA-F]+$')
2752 def parse_call(self):
2753 line = self.consume()
2754 mo = self.call_re.match(line)
2755 if not mo:
2756 # The line must be the stack count
2757 return None, int(line.strip())
2759 function_name = mo.group('symbol')
2761 # If present, amputate program counter from function name.
2762 if function_name:
2763 function_name = re.sub(self.addr2_re, '', function_name)
2765 # if not function_name or function_name == '[unknown]':
2766 # function_name = mo.group('address')
2768 module = mo.group('module')
2770 function_id = function_name + ':' + module
2772 try:
2773 function = self.profile.functions[function_id]
2774 except KeyError:
2775 function = Function(function_id, function_name)
2776 function.module = os.path.basename(module)
2777 function[SAMPLES] = 0
2778 function[TOTAL_SAMPLES] = 0
2779 self.profile.add_function(function)
2781 return function, None
2784class CollapseParser(LineParser):
2785 """Parser for the output of stackcollapse
2787 (from https://github.com/brendangregg/FlameGraph)
2788 """
2790 def __init__(self, infile):
2791 LineParser.__init__(self, infile)
2792 self.profile = Profile()
2794 def parse(self):
2795 profile = self.profile
2796 profile[SAMPLES] = 0
2798 self.readline()
2799 while not self.eof():
2800 self.parse_event()
2802 # compute derived data
2803 profile.validate()
2804 profile.find_cycles()
2805 profile.ratio(TIME_RATIO, SAMPLES)
2806 profile.call_ratios(SAMPLES2)
2807 if totalMethod == "callratios":
2808 # Heuristic approach. TOTAL_SAMPLES is unused.
2809 profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)
2810 elif totalMethod == "callstacks":
2811 # Use the actual call chains for functions.
2812 profile[TOTAL_SAMPLES] = profile[SAMPLES]
2813 profile.ratio(TOTAL_TIME_RATIO, TOTAL_SAMPLES)
2814 # Then propagate that total time to the calls.
2815 for function in compat_itervalues(profile.functions):
2816 for call in compat_itervalues(function.calls):
2817 if call.ratio is not None:
2818 callee = profile.functions[call.callee_id]
2819 call[TOTAL_TIME_RATIO] = call.ratio * callee[TOTAL_TIME_RATIO]
2820 else:
2821 assert False
2823 return profile
2825 def parse_event(self):
2826 line = self.consume()
2828 stack, count = line.rsplit(' ',maxsplit=1)
2829 count=int(count)
2830 self.profile[SAMPLES] += count
2832 calls = stack.split(';')
2833 functions = [self._make_function(call) for call in calls]
2835 functions[-1][SAMPLES] += count
2837 # TOTAL_SAMPLES excludes loops
2838 for func in set(functions):
2839 func[TOTAL_SAMPLES] += count
2841 # add call data
2842 callee = functions[-1]
2843 for caller in reversed(functions[:-1]):
2844 call = caller.calls.get(callee.id)
2845 if call is None:
2846 call = Call(callee.id)
2847 call[SAMPLES2] = 0
2848 caller.add_call(call)
2849 call[SAMPLES2] += count
2850 callee = caller
2852 call_re = re.compile(r'^(?P<func>[^ ]+) \((?P<file>.*):(?P<line>[0-9]+)\)$')
2854 def _make_function(self, call):
2855 """turn a call str into a Function
2857 takes a call site, as found between semicolons in the input, and returns
2858 a Function definition corresponding to that call site.
2859 """
2860 mo = self.call_re.match(call)
2861 if mo:
2862 name, module, line = mo.groups()
2863 func_id = "%s:%s" % (module, name)
2864 else:
2865 name = func_id = call
2866 module = None
2868 func = self.profile.functions.get(func_id)
2869 if func is None:
2870 func = Function(func_id, name)
2871 func.module = module
2872 func[SAMPLES] = 0
2873 func[TOTAL_SAMPLES] = 0
2874 self.profile.add_function(func)
2875 return func
2878formats = {
2879 "axe": AXEParser,
2880 "callgrind": CallgrindParser,
2881 "collapse": CollapseParser,
2882 "hprof": HProfParser,
2883 "json": JsonParser,
2884 "perf": PerfParser,
2885 "prof": GprofParser,
2886 "pstats": PstatsParser,
2887 "sleepy": SleepyParser,
2888 "sysprof": SysprofParser,
2889 "xperf": XPerfParser,
2890 "dtrace": DtraceParser,
2891}
2894########################################################################
2895# Output
2898class Theme:
2900 def __init__(self,
2901 bgcolor = (0.0, 0.0, 1.0),
2902 mincolor = (0.0, 0.0, 0.0),
2903 maxcolor = (0.0, 0.0, 1.0),
2904 fontname = "Arial",
2905 fontcolor = "white",
2906 nodestyle = "filled",
2907 minfontsize = 10.0,
2908 maxfontsize = 10.0,
2909 minpenwidth = 0.5,
2910 maxpenwidth = 4.0,
2911 gamma = 2.2,
2912 skew = 1.0):
2913 self.bgcolor = bgcolor
2914 self.mincolor = mincolor
2915 self.maxcolor = maxcolor
2916 self.fontname = fontname
2917 self.fontcolor = fontcolor
2918 self.nodestyle = nodestyle
2919 self.minfontsize = minfontsize
2920 self.maxfontsize = maxfontsize
2921 self.minpenwidth = minpenwidth
2922 self.maxpenwidth = maxpenwidth
2923 self.gamma = gamma
2924 self.skew = skew
2926 def graph_bgcolor(self):
2927 return self.hsl_to_rgb(*self.bgcolor)
2929 def graph_fontname(self):
2930 return self.fontname
2932 def graph_fontcolor(self):
2933 return self.fontcolor
2935 def node_bgcolor(self, weight):
2936 return self.color(weight)
2938 def node_fgcolor(self, weight):
2939 if self.nodestyle == "filled":
2940 return self.graph_bgcolor()
2941 else:
2942 return self.color(weight)
2944 def node_fontsize(self, weight):
2945 return self.fontsize(weight)
2947 def node_style(self):
2948 return self.nodestyle
2950 def edge_color(self, weight):
2951 return self.color(weight)
2953 def edge_fontsize(self, weight):
2954 return self.fontsize(weight)
2956 def edge_penwidth(self, weight):
2957 return max(weight*self.maxpenwidth, self.minpenwidth)
2959 def edge_arrowsize(self, weight):
2960 return 0.5 * math.sqrt(self.edge_penwidth(weight))
2962 def fontsize(self, weight):
2963 return max(weight**2 * self.maxfontsize, self.minfontsize)
2965 def color(self, weight):
2966 weight = min(max(weight, 0.0), 1.0)
2968 hmin, smin, lmin = self.mincolor
2969 hmax, smax, lmax = self.maxcolor
2971 if self.skew < 0:
2972 raise ValueError("Skew must be greater than 0")
2973 elif self.skew == 1.0:
2974 h = hmin + weight*(hmax - hmin)
2975 s = smin + weight*(smax - smin)
2976 l = lmin + weight*(lmax - lmin)
2977 else:
2978 base = self.skew
2979 h = hmin + ((hmax-hmin)*(-1.0 + (base ** weight)) / (base - 1.0))
2980 s = smin + ((smax-smin)*(-1.0 + (base ** weight)) / (base - 1.0))
2981 l = lmin + ((lmax-lmin)*(-1.0 + (base ** weight)) / (base - 1.0))
2983 return self.hsl_to_rgb(h, s, l)
2985 def hsl_to_rgb(self, h, s, l):
2986 """Convert a color from HSL color-model to RGB.
2988 See also:
2989 - http://www.w3.org/TR/css3-color/#hsl-color
2990 """
2992 h = h % 1.0
2993 s = min(max(s, 0.0), 1.0)
2994 l = min(max(l, 0.0), 1.0)
2996 if l <= 0.5:
2997 m2 = l*(s + 1.0)
2998 else:
2999 m2 = l + s - l*s
3000 m1 = l*2.0 - m2
3001 r = self._hue_to_rgb(m1, m2, h + 1.0/3.0)
3002 g = self._hue_to_rgb(m1, m2, h)
3003 b = self._hue_to_rgb(m1, m2, h - 1.0/3.0)
3005 # Apply gamma correction
3006 r **= self.gamma
3007 g **= self.gamma
3008 b **= self.gamma
3010 return (r, g, b)
3012 def _hue_to_rgb(self, m1, m2, h):
3013 if h < 0.0:
3014 h += 1.0
3015 elif h > 1.0:
3016 h -= 1.0
3017 if h*6 < 1.0:
3018 return m1 + (m2 - m1)*h*6.0
3019 elif h*2 < 1.0:
3020 return m2
3021 elif h*3 < 2.0:
3022 return m1 + (m2 - m1)*(2.0/3.0 - h)*6.0
3023 else:
3024 return m1
3027TEMPERATURE_COLORMAP = Theme(
3028 mincolor = (2.0/3.0, 0.80, 0.25), # dark blue
3029 maxcolor = (0.0, 1.0, 0.5), # satured red
3030 gamma = 1.0
3031)
3033PINK_COLORMAP = Theme(
3034 mincolor = (0.0, 1.0, 0.90), # pink
3035 maxcolor = (0.0, 1.0, 0.5), # satured red
3036)
3038GRAY_COLORMAP = Theme(
3039 mincolor = (0.0, 0.0, 0.85), # light gray
3040 maxcolor = (0.0, 0.0, 0.0), # black
3041)
3043BW_COLORMAP = Theme(
3044 minfontsize = 8.0,
3045 maxfontsize = 24.0,
3046 mincolor = (0.0, 0.0, 0.0), # black
3047 maxcolor = (0.0, 0.0, 0.0), # black
3048 minpenwidth = 0.1,
3049 maxpenwidth = 8.0,
3050)
3052PRINT_COLORMAP = Theme(
3053 minfontsize = 18.0,
3054 maxfontsize = 30.0,
3055 fontcolor = "black",
3056 nodestyle = "solid",
3057 mincolor = (0.0, 0.0, 0.0), # black
3058 maxcolor = (0.0, 0.0, 0.0), # black
3059 minpenwidth = 0.1,
3060 maxpenwidth = 8.0,
3061)
3064themes = {
3065 "color": TEMPERATURE_COLORMAP,
3066 "pink": PINK_COLORMAP,
3067 "gray": GRAY_COLORMAP,
3068 "bw": BW_COLORMAP,
3069 "print": PRINT_COLORMAP,
3070}
3073def sorted_iteritems(d):
3074 # Used mostly for result reproducibility (while testing.)
3075 keys = list(d.keys())
3076 keys.sort()
3077 for key in keys:
3078 value = d[key]
3079 yield key, value
3082class DotWriter:
3083 """Writer for the DOT language.
3085 See also:
3086 - "The DOT Language" specification
3087 http://www.graphviz.org/doc/info/lang.html
3088 """
3090 strip = False
3091 wrap = False
3093 def __init__(self, fp):
3094 self.fp = fp
3096 def wrap_function_name(self, name):
3097 """Split the function name on multiple lines."""
3099 if len(name) > 32:
3100 ratio = 2.0/3.0
3101 height = max(int(len(name)/(1.0 - ratio) + 0.5), 1)
3102 width = max(len(name)/height, 32)
3103 # TODO: break lines in symbols
3104 name = textwrap.fill(name, width, break_long_words=False)
3106 # Take away spaces
3107 name = name.replace(", ", ",")
3108 name = name.replace("> >", ">>")
3109 name = name.replace("> >", ">>") # catch consecutive
3111 return name
3113 show_function_events = [TOTAL_TIME_RATIO, TIME_RATIO]
3114 show_edge_events = [TOTAL_TIME_RATIO, CALLS]
3116 def graphs_compare(self, profile1, profile2, theme, options):
3117 self.begin_graph()
3119 fontname = theme.graph_fontname()
3120 fontcolor = theme.graph_fontcolor()
3121 nodestyle = theme.node_style()
3123 tolerance, only_slower, only_faster, color_by_difference = (
3124 options.tolerance, options.only_slower, options.only_faster, options.color_by_difference)
3125 self.attr('graph', fontname=fontname, ranksep=0.25, nodesep=0.125)
3126 self.attr('node', fontname=fontname, style=nodestyle, fontcolor=fontcolor, width=0, height=0)
3127 self.attr('edge', fontname=fontname)
3129 functions2 = {function.name: function for _, function in sorted_iteritems(profile2.functions)}
3131 for _, function1 in sorted_iteritems(profile1.functions):
3132 labels = []
3133 function2 = None
3135 name = function1.name
3136 try:
3137 function2 = functions2[name]
3138 if self.wrap:
3139 name = self.wrap_function_name(name)
3140 if color_by_difference:
3141 min_diff, max_diff = min_max_difference(profile1, profile2)
3142 labels.append(name)
3143 weight_difference = 0
3144 shape = 'box'
3145 orientation = '0'
3146 for event in self.show_function_events:
3147 if event in function1.events:
3148 event1 = function1[event]
3149 event2 = function2[event]
3151 difference = abs(event1 - event2) * 100
3153 if event == TOTAL_TIME_RATIO:
3154 weight_difference = difference
3155 if difference >= tolerance:
3156 if event2 > event1 and not only_faster:
3157 shape = 'cds'
3158 label = (f'{event.format(event1)} +'
3159 f' {round_difference(difference, tolerance)}%')
3160 elif event2 < event1 and not only_slower:
3161 orientation = "90"
3162 shape = 'cds'
3163 label = (f'{event.format(event1)} - '
3164 f'{round_difference(difference, tolerance)}%')
3165 else:
3166 # protection to not color by difference if we choose to show only_faster/only_slower
3167 weight_difference = 0
3168 label = event.format(function1[event])
3169 else:
3170 weight_difference = 0
3171 label = event.format(function1[event])
3172 else:
3173 if difference >= tolerance:
3174 if event2 > event1:
3175 label = (f'{event.format(event1)} +'
3176 f' {round_difference(difference, tolerance)}%')
3177 elif event2 < event1:
3178 label = (f'{event.format(event1)} - '
3179 f'{round_difference(difference, tolerance)}%')
3180 else:
3181 label = event.format(function1[event])
3183 labels.append(label)
3184 if function1.called is not None:
3185 labels.append(f"{function1.called} {MULTIPLICATION_SIGN}/ {function2.called} {MULTIPLICATION_SIGN}")
3187 except KeyError:
3188 shape = 'box'
3189 orientation = '0'
3190 weight_difference = 0
3191 if function1.process is not None:
3192 labels.append(function1.process)
3193 if function1.module is not None:
3194 labels.append(function1.module)
3196 if self.strip:
3197 function_name = function1.stripped_name()
3198 else:
3199 function_name = function1.name
3200 if color_by_difference:
3201 min_diff, max_diff = 0, 0
3203 # dot can't parse quoted strings longer than YY_BUF_SIZE, which
3204 # defaults to 16K. But some annotated C++ functions (e.g., boost,
3205 # https://github.com/jrfonseca/gprof2dot/issues/30) can exceed that
3206 MAX_FUNCTION_NAME = 4096
3207 if len(function_name) >= MAX_FUNCTION_NAME:
3208 sys.stderr.write('warning: truncating function name with %u chars (%s)\n' % (len(function_name), function_name[:32] + '...'))
3209 function_name = function_name[:MAX_FUNCTION_NAME - 1] + chr(0x2026)
3211 if self.wrap:
3212 function_name = self.wrap_function_name(function_name)
3213 labels.append(function_name)
3215 for event in self.show_function_events:
3216 if event in function1.events:
3217 label = event.format(function1[event])
3218 labels.append(label)
3219 if function1.called is not None:
3220 labels.append("%u%s" % (function1.called, MULTIPLICATION_SIGN))
3222 if color_by_difference and weight_difference:
3223 # min and max is calculated whe color_by_difference is true
3224 weight = rescale_difference(weight_difference, min_diff, max_diff)
3226 elif function1.weight is not None and not color_by_difference:
3227 weight = function1.weight
3228 else:
3229 weight = 0.0
3231 label = '\n'.join(labels)
3233 self.node(function1.id,
3234 label=label,
3235 orientation=orientation,
3236 color=self.color(theme.node_bgcolor(weight)),
3237 shape=shape,
3238 fontcolor=self.color(theme.node_fgcolor(weight)),
3239 fontsize="%f" % theme.node_fontsize(weight),
3240 tooltip=function1.filename,
3241 )
3243 if function2 is not None:
3244 calls2 = {call.callee_id: call for _, call in sorted_iteritems(function2.calls)}
3245 else:
3246 calls2 = {}
3247 functions_by_id1 = {function.id: function for _, function in sorted_iteritems(profile1.functions)}
3249 for _, call1 in sorted_iteritems(function1.calls):
3250 labels = []
3251 try:
3252 # if profiles do not have identical setups, callee_id will not be identical either
3253 call_id1 = call1.callee_id
3254 call_name = functions_by_id1[call_id1].name
3255 call_id2 = functions2[call_name].id
3256 call2 = calls2[call_id2]
3257 for event in self.show_edge_events:
3258 if event in call1.events:
3259 label = f'{event.format(call1[event])} / {event.format(call2[event])}'
3260 labels.append(label)
3261 except KeyError:
3262 for event in self.show_edge_events:
3263 if event in call1.events:
3264 label = f'{event.format(call1[event])}'
3265 labels.append(label)
3267 weight = 0 if color_by_difference else call1.weight
3268 label = '\n'.join(labels)
3269 self.edge(function1.id, call1.callee_id,
3270 label=label,
3271 color=self.color(theme.edge_color(weight)),
3272 fontcolor=self.color(theme.edge_color(weight)),
3273 fontsize="%.2f" % theme.edge_fontsize(weight),
3274 penwidth="%.2f" % theme.edge_penwidth(weight),
3275 labeldistance="%.2f" % theme.edge_penwidth(weight),
3276 arrowsize="%.2f" % theme.edge_arrowsize(weight),
3277 )
3278 self.end_graph()
3280 def graph(self, profile, theme):
3281 self.begin_graph()
3283 fontname = theme.graph_fontname()
3284 fontcolor = theme.graph_fontcolor()
3285 nodestyle = theme.node_style()
3287 self.attr('graph', fontname=fontname, ranksep=0.25, nodesep=0.125)
3288 self.attr('node', fontname=fontname, shape="box", style=nodestyle, fontcolor=fontcolor, width=0, height=0)
3289 self.attr('edge', fontname=fontname)
3291 for _, function in sorted_iteritems(profile.functions):
3292 labels = []
3293 if function.process is not None:
3294 labels.append(function.process)
3295 if function.module is not None:
3296 labels.append(function.module)
3298 if self.strip:
3299 function_name = function.stripped_name()
3300 else:
3301 function_name = function.name
3303 # dot can't parse quoted strings longer than YY_BUF_SIZE, which
3304 # defaults to 16K. But some annotated C++ functions (e.g., boost,
3305 # https://github.com/jrfonseca/gprof2dot/issues/30) can exceed that
3306 MAX_FUNCTION_NAME = 4096
3307 if len(function_name) >= MAX_FUNCTION_NAME:
3308 sys.stderr.write('warning: truncating function name with %u chars (%s)\n' % (len(function_name), function_name[:32] + '...'))
3309 function_name = function_name[:MAX_FUNCTION_NAME - 1] + chr(0x2026)
3311 if self.wrap:
3312 function_name = self.wrap_function_name(function_name)
3313 labels.append(function_name)
3315 for event in self.show_function_events:
3316 if event in function.events:
3317 label = event.format(function[event])
3318 labels.append(label)
3319 if function.called is not None:
3320 labels.append("%u%s" % (function.called, MULTIPLICATION_SIGN))
3322 if function.weight is not None:
3323 weight = function.weight
3324 else:
3325 weight = 0.0
3327 label = '\n'.join(labels)
3328 self.node(function.id,
3329 label = label,
3330 color = self.color(theme.node_bgcolor(weight)),
3331 fontcolor = self.color(theme.node_fgcolor(weight)),
3332 fontsize = "%.2f" % theme.node_fontsize(weight),
3333 tooltip = function.filename,
3334 )
3336 for _, call in sorted_iteritems(function.calls):
3337 callee = profile.functions[call.callee_id]
3339 labels = []
3340 for event in self.show_edge_events:
3341 if event in call.events:
3342 label = event.format(call[event])
3343 labels.append(label)
3345 if call.weight is not None:
3346 weight = call.weight
3347 elif callee.weight is not None:
3348 weight = callee.weight
3349 else:
3350 weight = 0.0
3352 label = '\n'.join(labels)
3354 self.edge(function.id, call.callee_id,
3355 label = label,
3356 color = self.color(theme.edge_color(weight)),
3357 fontcolor = self.color(theme.edge_color(weight)),
3358 fontsize = "%.2f" % theme.edge_fontsize(weight),
3359 penwidth = "%.2f" % theme.edge_penwidth(weight),
3360 labeldistance = "%.2f" % theme.edge_penwidth(weight),
3361 arrowsize = "%.2f" % theme.edge_arrowsize(weight),
3362 )
3364 self.end_graph()
3366 def begin_graph(self):
3367 self.write('digraph {\n')
3368 # Work-around graphviz bug[1]: unnamed graphs have "%3" tooltip in SVG
3369 # output. The bug was fixed upstream, but graphviz shipped in recent
3370 # Linux distros (for example, Ubuntu 24.04) still has the bug.
3371 # [1] https://gitlab.com/graphviz/graphviz/-/issues/1376
3372 self.write('\ttooltip=" "\n')
3374 def end_graph(self):
3375 self.write('}\n')
3377 def attr(self, what, **attrs):
3378 self.write("\t")
3379 self.write(what)
3380 self.attr_list(attrs)
3381 self.write(";\n")
3383 def node(self, node, **attrs):
3384 self.write("\t")
3385 self.node_id(node)
3386 self.attr_list(attrs)
3387 self.write(";\n")
3389 def edge(self, src, dst, **attrs):
3390 self.write("\t")
3391 self.node_id(src)
3392 self.write(" -> ")
3393 self.node_id(dst)
3394 self.attr_list(attrs)
3395 self.write(";\n")
3397 def attr_list(self, attrs):
3398 if not attrs:
3399 return
3400 self.write(' [')
3401 first = True
3402 for name, value in sorted_iteritems(attrs):
3403 if value is None:
3404 continue
3405 if first:
3406 first = False
3407 else:
3408 self.write(", ")
3409 assert isinstance(name, str)
3410 assert name.isidentifier()
3411 self.write(name)
3412 self.write('=')
3413 self.id(value)
3414 self.write(']')
3416 def node_id(self, id):
3417 # Node IDs need to be unique (can't be truncated) but dot doesn't allow
3418 # IDs longer than 16384 characters, so use an hash instead for the huge
3419 # C++ symbols that can arise, as seen in
3420 # https://github.com/jrfonseca/gprof2dot/issues/99
3421 if isinstance(id, str) and len(id) > 1024:
3422 id = '_' + hashlib.sha1(id.encode('utf-8'), usedforsecurity=False).hexdigest()
3423 self.id(id)
3425 # dot keywords, which must be quoted when used as ordinary identifiers.
3426 _keywords = frozenset(('node', 'edge', 'graph', 'digraph', 'subgraph', 'strict'))
3428 def id(self, id):
3429 if isinstance(id, (int, float)):
3430 s = str(id)
3431 elif isinstance(id, str):
3432 if id.isalnum() and not id[0].isdigit() and id.lower() not in self._keywords:
3433 s = id
3434 else:
3435 s = self.escape(id)
3436 else:
3437 raise TypeError
3438 self.write(s)
3440 def color(self, rgb):
3441 r, g, b = rgb
3443 def float2int(f):
3444 if f <= 0.0:
3445 return 0
3446 if f >= 1.0:
3447 return 255
3448 return int(255.0*f + 0.5)
3450 return "#" + "".join(["%02x" % float2int(c) for c in (r, g, b)])
3452 def escape(self, s):
3453 s = s.replace('\\', r'\\')
3454 s = s.replace('\n', r'\n')
3455 s = s.replace('\t', r'\t')
3456 s = s.replace('"', r'\"')
3457 return '"' + s + '"'
3459 def write(self, s):
3460 self.fp.write(s)
3464########################################################################
3465# Main program
3468def naturalJoin(values):
3469 if len(values) >= 2:
3470 return ', '.join(values[:-1]) + ' or ' + values[-1]
3472 else:
3473 return ''.join(values)
3476def openInput(path):
3477 """Open a text profile, consuming the byte order mark when there is one."""
3479 fp = open(path, 'rb')
3480 bom = fp.read(2)
3481 if bom in (codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE):
3482 # Default on Windows PowerShell (https://github.com/jrfonseca/gprof2dot/issues/88)
3483 encoding = 'utf-16'
3484 else:
3485 encoding = 'utf-8-sig'
3486 fp.seek(0)
3487 return io.TextIOWrapper(fp, encoding=encoding)
3490def main(argv=sys.argv[1:]):
3491 """Main program."""
3493 global totalMethod, timeFormat
3495 formatNames = list(formats.keys())
3496 formatNames.sort()
3498 themeNames = list(themes.keys())
3499 themeNames.sort()
3501 labelNames = list(labels.keys())
3502 labelNames.sort()
3504 argparser = argparse.ArgumentParser(
3505 usage="\n %(prog)s [options] [INPUT] ...")
3506 argparser.add_argument(
3507 '-o', '--output', metavar='OUTPUT',
3508 dest="output",
3509 help="output filename [stdout]")
3510 argparser.add_argument(
3511 '-n', '--node-thres', metavar='PERCENTAGE',
3512 type=float, dest="node_thres", default=0.5,
3513 help="eliminate nodes below this threshold [default: %(default)s]")
3514 argparser.add_argument(
3515 '-e', '--edge-thres', metavar='PERCENTAGE',
3516 type=float, dest="edge_thres", default=0.1,
3517 help="eliminate edges below this threshold [default: %(default)s]")
3518 argparser.add_argument(
3519 '-f', '--format',
3520 choices=formatNames,
3521 dest="format", default="prof",
3522 help="profile format: %s [default: %%(default)s]" % naturalJoin(formatNames))
3523 argparser.add_argument(
3524 '--total',
3525 choices=('callratios', 'callstacks'),
3526 dest="totalMethod", default=totalMethod,
3527 help="preferred method of calculating total time: callratios or callstacks (currently affects only perf format) [default: %(default)s]")
3528 argparser.add_argument(
3529 '-c', '--colormap',
3530 choices=themeNames,
3531 dest="theme", default="color",
3532 help="color map: %s [default: %%(default)s]" % naturalJoin(themeNames))
3533 argparser.add_argument(
3534 '-s', '--strip',
3535 action="store_true",
3536 dest="strip", default=False,
3537 help="strip function parameters, template parameters, and const modifiers from demangled C++ function names")
3538 argparser.add_argument(
3539 '--color-nodes-by-selftime',
3540 action="store_true",
3541 dest="color_nodes_by_selftime", default=False,
3542 help="color nodes by self time, rather than by total time (sum of self and descendants)")
3543 argparser.add_argument(
3544 '--colour-nodes-by-selftime',
3545 action="store_true",
3546 dest="color_nodes_by_selftime",
3547 help=argparse.SUPPRESS)
3548 argparser.add_argument(
3549 '-w', '--wrap',
3550 action="store_true",
3551 dest="wrap", default=False,
3552 help="wrap function names")
3553 argparser.add_argument(
3554 '--show-samples',
3555 action="store_true",
3556 dest="show_samples", default=False,
3557 help="show function samples")
3558 argparser.add_argument(
3559 '--time-format',
3560 default=timeFormat,
3561 help="format to use for showing time values [default: %(default)s]")
3562 argparser.add_argument(
3563 '--node-label', metavar='MEASURE',
3564 choices=labelNames,
3565 action='append',
3566 dest='node_labels',
3567 help="measurements to on show the node (can be specified multiple times): %s [default: %s]" % (
3568 naturalJoin(labelNames), ', '.join(defaultLabelNames)))
3569 # add option to show information on available entries ()
3570 argparser.add_argument(
3571 '--list-functions',
3572 dest="list_functions", default=None,
3573 help="""\
3574list functions available for selection in -z or -l, requires selector argument
3575( use '+' to select all).
3576Recall that the selector argument is used with Unix/Bash globbing/pattern matching,
3577and that entries are formatted '<pkg>:<linenum>:<function>'. When argument starts
3578with '%%', a dump of all available information is performed for selected entries,
3579 after removal of leading '%%'.
3580""")
3581 # add option to create subtree or show paths
3582 argparser.add_argument(
3583 '-z', '--root',
3584 dest="root", default="",
3585 help="prune call graph to show only descendants of specified root function")
3586 argparser.add_argument(
3587 '-l', '--leaf',
3588 dest="leaf", default="",
3589 help="prune call graph to show only ancestors of specified leaf function")
3590 argparser.add_argument(
3591 '--depth',
3592 type=int,
3593 dest="depth", default=-1,
3594 help="prune call graph to show only descendants or ancestors until specified depth")
3595 # add a new option to control skew of the colorization curve
3596 argparser.add_argument(
3597 '--skew',
3598 type=float, dest="theme_skew", default=1.0,
3599 help="skew the colorization curve. Values < 1.0 give more variety to lower percentages. Values > 1.0 give less variety to lower percentages")
3600 # add option for filtering by file path
3601 argparser.add_argument(
3602 '-p', '--path', action="append",
3603 dest="filter_paths",
3604 help="filter all modules not in a specified path")
3605 argparser.add_argument(
3606 '--compare',
3607 action="store_true",
3608 dest="compare", default=False,
3609 help="compare two graphs with almost identical structure. With this option two files should be provided."
3610 "gprof2dot.py [options] --compare [file1] [file2] ...")
3611 argparser.add_argument(
3612 '--compare-tolerance',
3613 type=float, dest="tolerance", default=0.001,
3614 help="tolerance threshold for node difference (default=0.001%%)."
3615 "If the difference is below this value the nodes are considered identical.")
3616 argparser.add_argument(
3617 '--compare-only-slower',
3618 action="store_true",
3619 dest="only_slower", default=False,
3620 help="display comparison only for function which are slower in second graph.")
3621 argparser.add_argument(
3622 '--compare-only-faster',
3623 action="store_true",
3624 dest="only_faster", default=False,
3625 help="display comparison only for function which are faster in second graph.")
3626 argparser.add_argument(
3627 '--compare-color-by-difference',
3628 action="store_true",
3629 dest="color_by_difference", default=False,
3630 help="color nodes based on the value of the difference. "
3631 "Nodes with the largest differences represent the hot spots.")
3632 argparser.add_argument('input', nargs='*', metavar='INPUT', help='input stats')
3633 options = argparser.parse_args(argv)
3634 args = options.input
3636 if len(args) > 1 and options.format != 'pstats' and not options.compare:
3637 argparser.error('incorrect number of arguments')
3639 try:
3640 theme = themes[options.theme]
3641 except KeyError:
3642 argparser.error('invalid colormap \'%s\'' % options.theme)
3644 # set skew on the theme now that it has been picked.
3645 if options.theme_skew:
3646 theme.skew = options.theme_skew
3648 totalMethod = options.totalMethod
3649 timeFormat = options.time_format
3651 try:
3652 Format = formats[options.format]
3653 except KeyError:
3654 argparser.error('invalid format \'%s\'' % options.format)
3656 if Format.stdinInput:
3657 if not args:
3658 fp = sys.stdin
3659 parser = Format(fp)
3660 elif options.compare:
3661 fp1 = openInput(args[0])
3662 fp2 = openInput(args[1])
3663 parser1 = Format(fp1)
3664 parser2 = Format(fp2)
3665 else:
3666 fp = openInput(args[0])
3667 parser = Format(fp)
3668 elif Format.multipleInput:
3669 if not args:
3670 argparser.error('at least a file must be specified for %s input' % options.format)
3671 if options.compare:
3672 parser1 = Format(args[-2])
3673 parser2 = Format(args[-1])
3674 else:
3675 parser = Format(*args)
3676 else:
3677 if len(args) != 1:
3678 argparser.error('exactly one file must be specified for %s input' % options.format)
3679 parser = Format(args[0])
3681 if options.compare:
3682 profile1 = parser1.parse()
3683 profile2 = parser2.parse()
3684 else:
3685 profile = parser.parse()
3687 if options.output is None:
3688 output = open(sys.stdout.fileno(), mode='wt', encoding='UTF-8', closefd=False)
3689 else:
3690 output = open(options.output, 'wt', encoding='UTF-8')
3692 dot = DotWriter(output)
3693 dot.strip = options.strip
3694 dot.wrap = options.wrap
3696 labelNames = options.node_labels or defaultLabelNames
3697 dot.show_function_events = [labels[l] for l in labelNames]
3698 if options.show_samples:
3699 dot.show_function_events.append(SAMPLES)
3701 if options.compare:
3702 profile1.prune(options.node_thres/100.0, options.edge_thres/100.0, options.filter_paths,
3703 options.color_nodes_by_selftime)
3704 profile2.prune(options.node_thres/100.0, options.edge_thres/100.0, options.filter_paths,
3705 options.color_nodes_by_selftime)
3707 if options.root:
3708 profile1.prune_root(profile1.getFunctionIds(options.root), options.depth)
3709 profile2.prune_root(profile2.getFunctionIds(options.root), options.depth)
3710 else:
3711 profile.prune(options.node_thres/100.0, options.edge_thres/100.0, options.filter_paths,
3712 options.color_nodes_by_selftime)
3713 if options.root:
3714 rootIds = profile.getFunctionIds(options.root)
3715 if not rootIds:
3716 sys.stderr.write('root node ' + options.root + ' not found (might already be pruned : try -e0 -n0 flags)\n')
3717 sys.exit(1)
3718 profile.prune_root(rootIds, options.depth)
3720 if options.list_functions:
3721 profile.printFunctionIds(selector=options.list_functions)
3722 sys.exit(0)
3724 if options.leaf:
3725 leafIds = profile.getFunctionIds(options.leaf)
3726 if not leafIds:
3727 sys.stderr.write('leaf node ' + options.leaf + ' not found (maybe already pruned : try -e0 -n0 flags)\n')
3728 sys.exit(1)
3729 profile.prune_leaf(leafIds, options.depth)
3731 if options.compare:
3732 dot.graphs_compare(profile1, profile2, theme, options)
3733 else:
3734 dot.graph(profile, theme)
3737if __name__ == '__main__':
3738 main()