Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/gprof2dot.py: 15%

1#!/usr/bin/env python3

5# This program is free software: you can redistribute it and/or modify it

6# under the terms of the GNU Lesser General Public License as published

7# by the Free Software Foundation, either version 3 of the License, or

8# (at your option) any later version.

10# This program is distributed in the hope that it will be useful,

11# but WITHOUT ANY WARRANTY; without even the implied warranty of

12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

13# GNU Lesser General Public License for more details.

14#

15# You should have received a copy of the GNU Lesser General Public License

16# along with this program. If not, see <http://www.gnu.org/licenses/>.

17#

19"""Generate a dot graph from the output of several profilers."""

21__author__ = "Jose Fonseca et al"

24import sys

25import math

26import os.path

27import re

28import textwrap

29import argparse

30import xml.parsers.expat

31import collections

32import locale

33import json

34import fnmatch

35import codecs

36import io

37import hashlib

39assert sys.version_info[0] >= 3

42########################################################################

43# Model

46MULTIPLICATION_SIGN = chr(0xd7)

47timeFormat = "%.7g"

50def times(x):

51 return "%u%s" % (x, MULTIPLICATION_SIGN)

53def percentage(p):

54 return "%.02f%%" % (p*100.0,)

56def fmttime(t):

57 return timeFormat % t

59def add(a, b):

60 return a + b

62def fail(a, b):

63 assert False

65# To enhance readability, labels are rounded to the number of decimal

66# places corresponding to the tolerance value.

67def round_difference(difference, tolerance):

68 n = -math.floor(math.log10(tolerance))

69 return round(difference, n)

72def rescale_difference(x, min_val, max_val):

73 return (x - min_val) / (max_val - min_val)

76def min_max_difference(profile1, profile2):

77 f1_events = [f1[TOTAL_TIME_RATIO] for _, f1 in sorted_iteritems(profile1.functions)]

78 f2_events = [f2[TOTAL_TIME_RATIO] for _, f2 in sorted_iteritems(profile2.functions)]

79 differences = []

80 for i in range(len(f1_events)):

81 try:

82 differences.append(abs(f1_events[i] - f2_events[i]) * 100)

83 except IndexError:

84 differences.append(0)

86 return min(differences), max(differences)

89tol = 2 ** -23

91def ratio(numerator, denominator):

92 try:

93 ratio = float(numerator)/float(denominator)

94 except ZeroDivisionError:

95 # 0/0 is undefined, but 1.0 yields more useful results

96 return 1.0

97 if ratio < 0.0:

98 if ratio < -tol:

99 sys.stderr.write('warning: negative ratio (%s/%s)\n' % (numerator, denominator))

100 return 0.0

101 if ratio > 1.0:

102 if ratio > 1.0 + tol:

103 sys.stderr.write('warning: ratio greater than one (%s/%s)\n' % (numerator, denominator))

104 return 1.0

105 return ratio

106

107

108class UndefinedEvent(Exception):

109 """Raised when attempting to get an event which is undefined."""

110

111 def __init__(self, event):

112 Exception.__init__(self)

113 self.event = event

114

115 def __str__(self):

116 return 'unspecified event %s' % self.event.name

117

118

119class Event:

120 """Describe a kind of event, and its basic operations."""

121

122 def __init__(self, name, null, aggregator, formatter = str):

123 self.name = name

124 self._null = null

125 self._aggregator = aggregator

126 self._formatter = formatter

127

128 def __repr__(self):

129 return self.name

130

131 def null(self):

132 return self._null

133

134 def aggregate(self, val1, val2):

135 """Aggregate two event values."""

136 assert val1 is not None

137 assert val2 is not None

138 return self._aggregator(val1, val2)

139

140 def format(self, val):

141 """Format an event value."""

142 assert val is not None

143 return self._formatter(val)

144

145

146CALLS = Event("Calls", 0, add, times)

147SAMPLES = Event("Samples", 0, add, times)

148SAMPLES2 = Event("Samples", 0, add, times)

149

150# Count of samples where a given function was either executing or on the stack.

151# This is used to calculate the total time ratio according to the

152# straightforward method described in Mike Dunlavey's answer to

153# stackoverflow.com/questions/1777556/alternatives-to-gprof, item 4 (the myth

154# "that recursion is a tricky confusing issue"), last edited 2012-08-30: it's

155# just the ratio of TOTAL_SAMPLES over the number of samples in the profile.

156#

157# Used only when totalMethod == callstacks

158TOTAL_SAMPLES = Event("Samples", 0, add, times)

159

160TIME = Event("Time", 0.0, add, lambda x: '(' + fmttime(x) + ')')

161TIME_RATIO = Event("Time ratio", 0.0, add, lambda x: '(' + percentage(x) + ')')

162TOTAL_TIME = Event("Total time", 0.0, fail, fmttime)

163TOTAL_TIME_RATIO = Event("Total time ratio", 0.0, fail, percentage)

164

165labels = {

166 'self-time': TIME,

167 'self-time-percentage': TIME_RATIO,

168 'total-time': TOTAL_TIME,

169 'total-time-percentage': TOTAL_TIME_RATIO,

170}

171defaultLabelNames = ['total-time-percentage', 'self-time-percentage']

172

173totalMethod = 'callratios'

174

175

176class Object:

177 """Base class for all objects in profile which can store events."""

178

179 def __init__(self, events=None):

180 if events is None:

181 self.events = {}

182 else:

183 self.events = events

184

185 def __lt__(self, other):

186 return id(self) < id(other)

187

188 def __contains__(self, event):

189 return event in self.events

190

191 def __getitem__(self, event):

192 try:

193 return self.events[event]

194 except KeyError:

195 raise UndefinedEvent(event)

196

197 def __setitem__(self, event, value):

198 if value is None:

199 if event in self.events:

200 del self.events[event]

201 else:

202 self.events[event] = value

203

204

205class Call(Object):

206 """A call between functions.

207

208 There should be at most one call object for every pair of functions.

209 """

210

211 def __init__(self, callee_id):

212 Object.__init__(self)

213 self.callee_id = callee_id

214 self.ratio = None

215 self.weight = None

216

217

218class Function(Object):

219 """A function."""

220

221 def __init__(self, id, name):

222 Object.__init__(self)

223 self.id = id

224 self.name = name

225 self.module = None

226 self.process = None

227 self.calls = {}

228 self.called = None

229 self.weight = None

230 self.cycle = None

231 self.filename = None

232

233 def add_call(self, call):

234 if call.callee_id in self.calls:

235 sys.stderr.write('warning: overwriting call from function %s to %s\n' % (str(self.id), str(call.callee_id)))

236 self.calls[call.callee_id] = call

237

238 def get_call(self, callee_id):

239 if not callee_id in self.calls:

240 call = Call(callee_id)

241 call[SAMPLES] = 0

242 call[SAMPLES2] = 0

243 call[CALLS] = 0

244 self.calls[callee_id] = call

245 return self.calls[callee_id]

246

247 _parenthesis_re = re.compile(r'$[^()]*$')

248 _angles_re = re.compile(r'<[^<>]*>')

249 _const_re = re.compile(r'\s+const$')

250

251 def stripped_name(self):

252 """Remove extraneous information from C++ demangled function names."""

253

254 name = self.name

255

256 # Strip function parameters from name by recursively removing paired parenthesis

257 while True:

258 name, n = self._parenthesis_re.subn('', name)

259 if not n:

260 break

261

262 # Strip const qualifier

263 name = self._const_re.sub('', name)

264

265 # Strip template parameters from name by recursively removing paired angles

266 while True:

267 name, n = self._angles_re.subn('', name)

268 if not n:

269 break

270

271 return name

272

273 # TODO: write utility functions

274

275 def __repr__(self):

276 return self.name

277

278 def dump(self, sep1=",\n\t", sep2=":=", sep3="\n"):

279 """ Returns as a string all information available in this Function object

280 separators sep1:between entries

281 sep2:between attribute name and value,

282 sep3: inserted at end

283 """

284 return sep1.join(sep2.join([k,str(v)]) for (k,v) in sorted(self.__dict__.items())) + sep3

285

286

287class Cycle(Object):

288 """A cycle made from recursive function calls."""

289

290 def __init__(self):

291 Object.__init__(self)

292 self.functions = set()

293

294 def add_function(self, function):

295 assert function not in self.functions

296 self.functions.add(function)

297 if function.cycle is not None:

298 for other in function.cycle.functions:

299 if function not in self.functions:

300 self.add_function(other)

301 function.cycle = self

302

303

304class Profile(Object):

305 """The whole profile."""

306

307 def __init__(self):

308 Object.__init__(self)

309 self.functions = {}

310 self.cycles = []

311

312 def add_function(self, function):

313 if function.id in self.functions:

314 sys.stderr.write('warning: overwriting function %s (id %s)\n' % (function.name, str(function.id)))

315 self.functions[function.id] = function

316

317 def add_cycle(self, cycle):

318 self.cycles.append(cycle)

319

320 def validate(self):

321 """Validate the edges."""

322

323 for function in self.functions.values():

324 for callee_id in list(function.calls.keys()):

325 assert function.calls[callee_id].callee_id == callee_id

326 if callee_id not in self.functions:

327 sys.stderr.write('warning: call to undefined function %s from function %s\n' % (str(callee_id), function.name))

328 del function.calls[callee_id]

329

330 def find_cycles(self):

331 """Find cycles using Tarjan's strongly connected components algorithm."""

332

333 # Apply the Tarjan's algorithm successively until all functions are visited

334 stack = []

335 data = {}

336 order = 0

337 for function in self.functions.values():

338 order = self._tarjan(function, order, stack, data)

339 cycles = []

340 for function in self.functions.values():

341 if function.cycle is not None and function.cycle not in cycles:

342 cycles.append(function.cycle)

343 self.cycles = cycles

344 if 0:

345 for cycle in cycles:

346 sys.stderr.write("Cycle:\n")

347 for member in cycle.functions:

348 sys.stderr.write("\tFunction %s\n" % member.name)

349

350 def prune_root(self, roots, depth=-1):

351 visited = set()

352 frontier = set([(root_node, depth) for root_node in roots])

353 while len(frontier) > 0:

354 node, node_depth = frontier.pop()

355 visited.add(node)

356 if node_depth == 0:

357 continue

358 f = self.functions[node]

359 newNodes = set(f.calls.keys()) - visited

360 frontier = frontier.union({(new_node, node_depth - 1) for new_node in newNodes})

361 subtreeFunctions = {}

362 for n in visited:

363 f = self.functions[n]

364 newCalls = {}

365 for c in f.calls.keys():

366 if c in visited:

367 newCalls[c] = f.calls[c]

368 f.calls = newCalls

369 subtreeFunctions[n] = f

370 self.functions = subtreeFunctions

371

372 def prune_leaf(self, leafs, depth=-1):

373 edgesUp = collections.defaultdict(set)

374 for f in self.functions.keys():

375 for n in self.functions[f].calls.keys():

376 edgesUp[n].add(f)

377 # build the tree up

378 visited = set()

379 frontier = set([(leaf_node, depth) for leaf_node in leafs])

380 while len(frontier) > 0:

381 node, node_depth = frontier.pop()

382 visited.add(node)

383 if node_depth == 0:

384 continue

385 newNodes = edgesUp[node] - visited

386 frontier = frontier.union({(new_node, node_depth - 1) for new_node in newNodes})

387 downTree = set(self.functions.keys())

388 upTree = visited

389 path = downTree.intersection(upTree)

390 pathFunctions = {}

391 for n in path:

392 f = self.functions[n]

393 newCalls = {}

394 for c in f.calls.keys():

395 if c in path:

396 newCalls[c] = f.calls[c]

397 f.calls = newCalls

398 pathFunctions[n] = f

399 self.functions = pathFunctions

400

401 def getFunctionIds(self, funcName):

402 function_names = {v.name: k for (k, v) in self.functions.items()}

403 return [function_names[name] for name in fnmatch.filter(function_names.keys(), funcName)]

404

405 def getFunctionId(self, funcName):

406 for f in self.functions:

407 if self.functions[f].name == funcName:

408 return f

409 return False

410

411 def printFunctionIds(self, selector=None, file=sys.stderr):

412 """ Print to file function entries selected by fnmatch.fnmatch like in

413 method getFunctionIds, with following extensions:

414 - selector starts with "%": dump all information available

415 - selector is '+' or '-': select all function entries

416 """

417 if selector is None or selector in ("+", "*"):

418 v = ",\n".join(("%s:\t%s" % (kf,self.functions[kf].name)

419 for kf in self.functions.keys()))

420 else:

421 if selector[0]=="%":

422 selector=selector[1:]

423 function_info={k:v for (k,v)

424 in self.functions.items()

425 if fnmatch.fnmatch(v.name,selector)}

426 v = ",\n".join( ("%s\t({k})\t(%s)::\n\t%s" % (v.name,type(v),v.dump())

427 for (k,v) in function_info.items()

428 ))

429

430 else:

431 function_names = (v.name for v in self.functions.values())

432 v = ",\n".join( ( nm for nm in fnmatch.filter(function_names,selector )))

433

434 file.write(v+"\n")

435 file.flush()

436

437 class _TarjanData:

438 def __init__(self, order):

439 self.order = order

440 self.lowlink = order

441 self.onstack = False

442

443 def _tarjan(self, function, order, stack, data):

444 """Tarjan's strongly connected components algorithm.

445

446 See also:

447 - http://en.wikipedia.org/wiki/Tarjan's_strongly_connected_components_algorithm

448 """

449

450 try:

451 func_data = data[function.id]

452 return order

453 except KeyError:

454 func_data = self._TarjanData(order)

455 data[function.id] = func_data

456 order += 1

457 pos = len(stack)

458 stack.append(function)

459 func_data.onstack = True

460 for call in function.calls.values():

461 try:

462 callee_data = data[call.callee_id]

463 if callee_data.onstack:

464 func_data.lowlink = min(func_data.lowlink, callee_data.order)

465 except KeyError:

466 callee = self.functions[call.callee_id]

467 order = self._tarjan(callee, order, stack, data)

468 callee_data = data[call.callee_id]

469 func_data.lowlink = min(func_data.lowlink, callee_data.lowlink)

470 if func_data.lowlink == func_data.order:

471 # Strongly connected component found

472 members = stack[pos:]

473 del stack[pos:]

474 if len(members) > 1:

475 cycle = Cycle()

476 for member in members:

477 cycle.add_function(member)

478 data[member.id].onstack = False

479 else:

480 for member in members:

481 data[member.id].onstack = False

482 return order

483

484 def call_ratios(self, event):

485 # Aggregate for incoming calls

486 cycle_totals = {}

487 for cycle in self.cycles:

488 cycle_totals[cycle] = 0.0

489 function_totals = {}

490 for function in self.functions.values():

491 function_totals[function] = 0.0

492

493 # Pass 1: function_total gets the sum of call[event] for all

494 # incoming arrows. Same for cycle_total for all arrows

495 # that are coming into the *cycle* but are not part of it.

496 for function in self.functions.values():

497 for call in function.calls.values():

498 if call.callee_id != function.id:

499 callee = self.functions[call.callee_id]

500 if event in call.events:

501 function_totals[callee] += call[event]

502 if callee.cycle is not None and callee.cycle is not function.cycle:

503 cycle_totals[callee.cycle] += call[event]

504 else:

505 sys.stderr.write("call_ratios: No data for " + function.name + " call to " + callee.name + "\n")

506

507 # Pass 2: Compute the ratios. Each call[event] is scaled by the

508 # function_total of the callee. Calls into cycles use the

509 # cycle_total, but not calls within cycles.

510 for function in self.functions.values():

511 for call in function.calls.values():

512 assert call.ratio is None

513 if call.callee_id != function.id:

514 callee = self.functions[call.callee_id]

515 if event in call.events:

516 if callee.cycle is not None and callee.cycle is not function.cycle:

517 total = cycle_totals[callee.cycle]

518 else:

519 total = function_totals[callee]

520 call.ratio = ratio(call[event], total)

521 else:

522 # Warnings here would only repeat those issued above.

523 call.ratio = 0.0

524

525 def integrate(self, outevent, inevent):

526 """Propagate function time ratio along the function calls.

527

528 Must be called after finding the cycles.

529

530 See also:

531 - http://citeseer.ist.psu.edu/graham82gprof.html

532 """

533

534 # Sanity checking

535 assert outevent not in self

536 for function in self.functions.values():

537 assert outevent not in function

538 assert inevent in function

539 for call in function.calls.values():

540 assert outevent not in call

541 if call.callee_id != function.id:

542 assert call.ratio is not None

543

544 # Aggregate the input for each cycle

545 for cycle in self.cycles:

546 total = inevent.null()

547 for function in self.functions.values():

548 total = inevent.aggregate(total, function[inevent])

549 self[inevent] = total

550

551 # Integrate along the edges

552 total = inevent.null()

553 for function in self.functions.values():

554 total = inevent.aggregate(total, function[inevent])

555 self._integrate_function(function, outevent, inevent)

556 self[outevent] = total

557

558 def _integrate_function(self, function, outevent, inevent):

559 if function.cycle is not None:

560 return self._integrate_cycle(function.cycle, outevent, inevent)

561 else:

562 if outevent not in function:

563 total = function[inevent]

564 for call in function.calls.values():

565 if call.callee_id != function.id:

566 total += self._integrate_call(call, outevent, inevent)

567 function[outevent] = total

568 return function[outevent]

569

570 def _integrate_call(self, call, outevent, inevent):

571 assert outevent not in call

572 assert call.ratio is not None

573 callee = self.functions[call.callee_id]

574 subtotal = call.ratio *self._integrate_function(callee, outevent, inevent)

575 call[outevent] = subtotal

576 return subtotal

577

578 def _integrate_cycle(self, cycle, outevent, inevent):

579 if outevent not in cycle:

580

581 # Compute the outevent for the whole cycle

582 total = inevent.null()

583 for member in cycle.functions:

584 subtotal = member[inevent]

585 for call in member.calls.values():

586 callee = self.functions[call.callee_id]

587 if callee.cycle is not cycle:

588 subtotal += self._integrate_call(call, outevent, inevent)

589 total += subtotal

590 cycle[outevent] = total

591

592 # Compute the time propagated to callers of this cycle

593 callees = {}

594 for function in self.functions.values():

595 if function.cycle is not cycle:

596 for call in function.calls.values():

597 callee = self.functions[call.callee_id]

598 if callee.cycle is cycle:

599 try:

600 callees[callee] += call.ratio

601 except KeyError:

602 callees[callee] = call.ratio

603

604 for member in cycle.functions:

605 member[outevent] = outevent.null()

606

607 for callee, call_ratio in callees.items():

608 ranks = {}

609 call_ratios = {}

610 partials = {}

611 self._rank_cycle_function(cycle, callee, ranks)

612 self._call_ratios_cycle(cycle, callee, ranks, call_ratios, set())

613 partial = self._integrate_cycle_function(cycle, callee, call_ratio, partials, ranks, call_ratios, outevent, inevent)

614

615 # Ensure `partial == max(partials.values())`, but with round-off tolerance

616 max_partial = max(partials.values())

617 assert abs(partial - max_partial) <= 1e-7*max_partial

618

619 assert abs(call_ratio*total - partial) <= 0.001*call_ratio*total

620

621 return cycle[outevent]

622

623 def _rank_cycle_function(self, cycle, function, ranks):

624 """Dijkstra's shortest paths algorithm.

625

626 See also:

627 - http://en.wikipedia.org/wiki/Dijkstra's_algorithm

628 """

629

630 import heapq

631 Q = []

632 Qd = {}

633 p = {}

634 visited = set([function])

635

636 ranks[function] = 0

637 for call in function.calls.values():

638 if call.callee_id != function.id:

639 callee = self.functions[call.callee_id]

640 if callee.cycle is cycle:

641 ranks[callee] = 1

642 item = [ranks[callee], function, callee]

643 heapq.heappush(Q, item)

644 Qd[callee] = item

645

646 while Q:

647 cost, parent, member = heapq.heappop(Q)

648 if member not in visited:

649 p[member]= parent

650 visited.add(member)

651 for call in member.calls.values():

652 if call.callee_id != member.id:

653 callee = self.functions[call.callee_id]

654 if callee.cycle is cycle:

655 member_rank = ranks[member]

656 rank = ranks.get(callee)

657 if rank is not None:

658 if rank > 1 + member_rank:

659 rank = 1 + member_rank

660 ranks[callee] = rank

661 Qd_callee = Qd[callee]

662 Qd_callee[0] = rank

663 Qd_callee[1] = member

664 heapq._siftdown(Q, 0, Q.index(Qd_callee))

665 else:

666 rank = 1 + member_rank

667 ranks[callee] = rank

668 item = [rank, member, callee]

669 heapq.heappush(Q, item)

670 Qd[callee] = item

671

672 def _call_ratios_cycle(self, cycle, function, ranks, call_ratios, visited):

673 if function not in visited:

674 visited.add(function)

675 for call in function.calls.values():

676 if call.callee_id != function.id:

677 callee = self.functions[call.callee_id]

678 if callee.cycle is cycle:

679 if ranks[callee] > ranks[function]:

680 call_ratios[callee] = call_ratios.get(callee, 0.0) + call.ratio

681 self._call_ratios_cycle(cycle, callee, ranks, call_ratios, visited)

682

683 def _integrate_cycle_function(self, cycle, function, partial_ratio, partials, ranks, call_ratios, outevent, inevent):

684 if function not in partials:

685 partial = partial_ratio*function[inevent]

686 for call in function.calls.values():

687 if call.callee_id != function.id:

688 callee = self.functions[call.callee_id]

689 if callee.cycle is not cycle:

690 assert outevent in call

691 partial += partial_ratio*call[outevent]

692 else:

693 if ranks[callee] > ranks[function]:

694 callee_partial = self._integrate_cycle_function(cycle, callee, partial_ratio, partials, ranks, call_ratios, outevent, inevent)

695 call_ratio = ratio(call.ratio, call_ratios[callee])

696 call_partial = call_ratio*callee_partial

697 try:

698 call[outevent] += call_partial

699 except UndefinedEvent:

700 call[outevent] = call_partial

701 partial += call_partial

702 partials[function] = partial

703 try:

704 function[outevent] += partial

705 except UndefinedEvent:

706 function[outevent] = partial

707 return partials[function]

708

709 def aggregate(self, event):

710 """Aggregate an event for the whole profile."""

711

712 total = event.null()

713 for function in self.functions.values():

714 try:

715 total = event.aggregate(total, function[event])

716 except UndefinedEvent:

717 return

718 self[event] = total

719

720 def ratio(self, outevent, inevent):

721 assert outevent not in self

722 assert inevent in self

723 for function in self.functions.values():

724 assert outevent not in function

725 assert inevent in function

726 function[outevent] = ratio(function[inevent], self[inevent])

727 for call in function.calls.values():

728 assert outevent not in call

729 if inevent in call:

730 call[outevent] = ratio(call[inevent], self[inevent])

731 self[outevent] = 1.0

732

733 def prune(self, node_thres, edge_thres, paths, color_nodes_by_selftime):

734 """Prune the profile"""

735

736 # compute the prune ratios

737 for function in self.functions.values():

738 try:

739 function.weight = function[TOTAL_TIME_RATIO]

740 except UndefinedEvent:

741 pass

742

743 for call in function.calls.values():

744 callee = self.functions[call.callee_id]

745

746 if TOTAL_TIME_RATIO in call:

747 # handle exact cases first

748 call.weight = call[TOTAL_TIME_RATIO]

749 else:

750 try:

751 # make a safe estimate

752 call.weight = min(function[TOTAL_TIME_RATIO], callee[TOTAL_TIME_RATIO])

753 except UndefinedEvent:

754 pass

755

756 # prune the nodes

757 for function_id in list(self.functions.keys()):

758 function = self.functions[function_id]

759 if function.weight is not None:

760 if function.weight < node_thres:

761 del self.functions[function_id]

762

763 # prune file paths

764 for function_id in list(self.functions.keys()):

765 function = self.functions[function_id]

766 if paths and function.filename and not any(function.filename.startswith(path) for path in paths):

767 del self.functions[function_id]

768 elif paths and function.module and not any((function.module.find(path)>-1) for path in paths):

769 del self.functions[function_id]

770

771 # prune the edges

772 for function in self.functions.values():

773 for callee_id in list(function.calls.keys()):

774 call = function.calls[callee_id]

775 if callee_id not in self.functions or call.weight is not None and call.weight < edge_thres:

776 del function.calls[callee_id]

777

778 if color_nodes_by_selftime:

779 weights = []

780 for function in self.functions.values():

781 try:

782 weights.append(function[TIME_RATIO])

783 except UndefinedEvent:

784 pass

785 max_ratio = max(weights or [1])

786

787 # apply rescaled weights for coloriung

788 for function in self.functions.values():

789 try:

790 function.weight = function[TIME_RATIO] / max_ratio

791 except (ZeroDivisionError, UndefinedEvent):

792 pass

793

794 def dump(self):

795 for function in self.functions.values():

796 sys.stderr.write('Function %s:\n' % (function.name,))

797 self._dump_events(function.events)

798 for call in function.calls.values():

799 callee = self.functions[call.callee_id]

800 sys.stderr.write(' Call %s:\n' % (callee.name,))

801 self._dump_events(call.events)

802 for cycle in self.cycles:

803 sys.stderr.write('Cycle:\n')

804 self._dump_events(cycle.events)

805 for function in cycle.functions:

806 sys.stderr.write(' Function %s\n' % (function.name,))

807

808 def _dump_events(self, events):

809 for event, value in events.items():

810 sys.stderr.write(' %s: %s\n' % (event.name, event.format(value)))

811

812

813

814########################################################################

815# Parsers

816

817

818class Struct:

819 """Masquerade a dictionary with a structure-like behavior."""

820

821 def __init__(self, attrs = None):

822 if attrs is None:

823 attrs = {}

824 self.__dict__['_attrs'] = attrs

825

826 def __getattr__(self, name):

827 try:

828 return self._attrs[name]

829 except KeyError:

830 raise AttributeError(name)

831

832 def __setattr__(self, name, value):

833 self._attrs[name] = value

834

835 def __str__(self):

836 return str(self._attrs)

837

838 def __repr__(self):

839 return repr(self._attrs)

840

841

842class ParseError(Exception):

843 """Raised when parsing to signal mismatches."""

844

845 def __init__(self, msg, line):

846 Exception.__init__(self)

847 self.msg = msg

848 # TODO: store more source line information

849 self.line = line

850

851 def __str__(self):

852 return '%s: %r' % (self.msg, self.line)

853

854

855class Parser:

856 """Parser interface."""

857

858 stdinInput = True

859 multipleInput = False

860

861 def __init__(self):

862 pass

863

864 def parse(self):

865 raise NotImplementedError

866

867

868class JsonParser(Parser):

869 """Parser for a custom JSON representation of profile data.

870

871 See schema.json for details.

872 """

873

874

875 def __init__(self, stream):

876 Parser.__init__(self)

877 self.stream = stream

878

879 @staticmethod

880 def _reject_constant(token):

881 # NaN/Infinity are not valid JSON (schema.json defines costs as finite

882 # numbers), but Python's json module accepts them as an extension. A

883 # non-finite cost poisons the SAMPLES total and every derived ratio, so

884 # refuse the tokens instead of accepting them.

885 raise ValueError('invalid JSON token %r' % token)

886

887 def parse(self):

888

889 try:

890 obj = json.load(self.stream, parse_constant=self._reject_constant)

891 except ValueError as ex:

892 sys.stderr.write('error: %s\n' % ex)

893 sys.exit(1)

894

895 assert obj['version'] == 0

896

897 profile = Profile()

898 profile[SAMPLES] = 0

899

900 fns = obj['functions']

901

902 for functionIndex in range(len(fns)):

903 fn = fns[functionIndex]

904 function = Function(functionIndex, fn['name'])

905 try:

906 function.module = fn['module']

907 except KeyError:

908 pass

909 try:

910 function.process = fn['process']

911 except KeyError:

912 pass

913 function[SAMPLES] = 0

914 function.called = 0

915 profile.add_function(function)

916

917 for event in obj['events']:

918 callchain = []

919

920 for functionIndex in event['callchain']:

921 function = profile.functions[functionIndex]

922 callchain.append(function)

923

924 # increment the call count of the first in the callchain

925 function = profile.functions[event['callchain'][0]]

926 function.called = function.called + 1

927

928 cost = event['cost'][0]

929

930 callee = callchain[0]

931 callee[SAMPLES] += cost

932 profile[SAMPLES] += cost

933

934 for caller in callchain[1:]:

935 try:

936 call = caller.calls[callee.id]

937 except KeyError:

938 call = Call(callee.id)

939 call[SAMPLES2] = cost

940 caller.add_call(call)

941 else:

942 call[SAMPLES2] += cost

943

944 callee = caller

945

946 if False:

947 profile.dump()

948

949 # compute derived data

950 profile.validate()

951 profile.find_cycles()

952 profile.ratio(TIME_RATIO, SAMPLES)

953 profile.call_ratios(SAMPLES2)

954 profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)

955

956 return profile

957

958

959class LineParser(Parser):

960 """Base class for parsers that read line-based formats."""

961

962 def __init__(self, stream):

963 Parser.__init__(self)

964 self._stream = stream

965 self.__line = None

966 self.__eof = False

967 self.line_no = 0

968

969 def readline(self):

970 line = self._stream.readline()

971 if not line:

972 self.__line = ''

973 self.__eof = True

974 else:

975 self.line_no += 1

976 line = line.rstrip('\r\n')

977 self.__line = line

978

979 def lookahead(self):

980 assert self.__line is not None

981 return self.__line

982

983 def consume(self):

984 assert self.__line is not None

985 line = self.__line

986 self.readline()

987 return line

988

989 def eof(self):

990 assert self.__line is not None

991 return self.__eof

992

993

994XML_ELEMENT_START, XML_ELEMENT_END, XML_CHARACTER_DATA, XML_EOF = range(4)

995

996

997class XmlToken:

998

999 def __init__(self, type, name_or_data, attrs = None, line = None, column = None):

1000 assert type in (XML_ELEMENT_START, XML_ELEMENT_END, XML_CHARACTER_DATA, XML_EOF)

1001 self.type = type

1002 self.name_or_data = name_or_data

1003 self.attrs = attrs

1004 self.line = line

1005 self.column = column

1006

1007 def __str__(self):

1008 if self.type == XML_ELEMENT_START:

1009 return '<' + self.name_or_data + ' ...>'

1010 if self.type == XML_ELEMENT_END:

1011 return '</' + self.name_or_data + '>'

1012 if self.type == XML_CHARACTER_DATA:

1013 return self.name_or_data

1014 if self.type == XML_EOF:

1015 return 'end of file'

1016 assert 0

1017

1018

1019class XmlTokenizer:

1020 """Expat based XML tokenizer."""

1021

1022 def __init__(self, fp, skip_ws = True):

1023 self.fp = fp

1024 self.tokens = []

1025 self.index = 0

1026 self.final = False

1027 self.skip_ws = skip_ws

1028

1029 self.character_pos = 0, 0

1030 self.character_data = ''

1031

1032 self.parser = xml.parsers.expat.ParserCreate()

1033 self.parser.StartElementHandler = self.handle_element_start

1034 self.parser.EndElementHandler = self.handle_element_end

1035 self.parser.CharacterDataHandler = self.handle_character_data

1036

1037 def handle_element_start(self, name, attributes):

1038 self.finish_character_data()

1039 line, column = self.pos()

1040 token = XmlToken(XML_ELEMENT_START, name, attributes, line, column)

1041 self.tokens.append(token)

1042

1043 def handle_element_end(self, name):

1044 self.finish_character_data()

1045 line, column = self.pos()

1046 token = XmlToken(XML_ELEMENT_END, name, None, line, column)

1047 self.tokens.append(token)

1048

1049 def handle_character_data(self, data):

1050 if not self.character_data:

1051 self.character_pos = self.pos()

1052 self.character_data += data

1053

1054 def finish_character_data(self):

1055 if self.character_data:

1056 if not self.skip_ws or not self.character_data.isspace():

1057 line, column = self.character_pos

1058 token = XmlToken(XML_CHARACTER_DATA, self.character_data, None, line, column)

1059 self.tokens.append(token)

1060 self.character_data = ''

1061

1062 def next(self):

1063 size = 16*1024

1064 while self.index >= len(self.tokens) and not self.final:

1065 self.tokens = []

1066 self.index = 0

1067 data = self.fp.read(size)

1068 self.final = len(data) < size

1069 self.parser.Parse(data, self.final)

1070 if self.index >= len(self.tokens):

1071 line, column = self.pos()

1072 token = XmlToken(XML_EOF, None, None, line, column)

1073 else:

1074 token = self.tokens[self.index]

1075 self.index += 1

1076 return token

1077

1078 def pos(self):

1079 return self.parser.CurrentLineNumber, self.parser.CurrentColumnNumber

1080

1081

1082class XmlTokenMismatch(Exception):

1083

1084 def __init__(self, expected, found):

1085 Exception.__init__(self)

1086 self.expected = expected

1087 self.found = found

1088

1089 def __str__(self):

1090 return '%u:%u: %s expected, %s found' % (self.found.line, self.found.column, str(self.expected), str(self.found))

1091

1092

1093class XmlParser(Parser):

1094 """Base XML document parser."""

1095

1096 def __init__(self, fp):

1097 Parser.__init__(self)

1098 self.tokenizer = XmlTokenizer(fp)

1099 self.consume()

1100

1101 def consume(self):

1102 self.token = self.tokenizer.next()

1103

1104 def match_element_start(self, name):

1105 return self.token.type == XML_ELEMENT_START and self.token.name_or_data == name

1106

1107 def match_element_end(self, name):

1108 return self.token.type == XML_ELEMENT_END and self.token.name_or_data == name

1109

1110 def element_start(self, name):

1111 while self.token.type == XML_CHARACTER_DATA:

1112 self.consume()

1113 if self.token.type != XML_ELEMENT_START:

1114 raise XmlTokenMismatch(XmlToken(XML_ELEMENT_START, name), self.token)

1115 if self.token.name_or_data != name:

1116 raise XmlTokenMismatch(XmlToken(XML_ELEMENT_START, name), self.token)

1117 attrs = self.token.attrs

1118 self.consume()

1119 return attrs

1120

1121 def element_end(self, name):

1122 while self.token.type == XML_CHARACTER_DATA:

1123 self.consume()

1124 if self.token.type != XML_ELEMENT_END:

1125 raise XmlTokenMismatch(XmlToken(XML_ELEMENT_END, name), self.token)

1126 if self.token.name_or_data != name:

1127 raise XmlTokenMismatch(XmlToken(XML_ELEMENT_END, name), self.token)

1128 self.consume()

1129

1130 def character_data(self, strip = True):

1131 data = ''

1132 while self.token.type == XML_CHARACTER_DATA:

1133 data += self.token.name_or_data

1134 self.consume()

1135 if strip:

1136 data = data.strip()

1137 return data

1138

1139

1140class GprofParser(Parser):

1141 """Parser for GNU gprof output.

1142

1143 See also:

1144 - Chapter "Interpreting gprof's Output" from the GNU gprof manual

1145 http://sourceware.org/binutils/docs-2.18/gprof/Call-Graph.html#Call-Graph

1146 - File "cg_print.c" from the GNU gprof source code

1147 http://sourceware.org/cgi-bin/cvsweb.cgi/~checkout~/src/gprof/cg_print.c?rev=1.12&cvsroot=src

1148 """

1149

1150 def __init__(self, fp):

1151 Parser.__init__(self)

1152 self.fp = fp

1153 self.functions = {}

1154 self.cycles = {}

1155

1156 def readline(self):

1157 line = self.fp.readline()

1158 if not line:

1159 sys.stderr.write('error: unexpected end of file\n')

1160 sys.exit(1)

1161 line = line.rstrip('\r\n')

1162 return line

1163

1164 _int_re = re.compile(r'^\d+$')

1165 _float_re = re.compile(r'^\d+\.\d+$')

1166

1167 def translate(self, mo):

1168 """Extract a structure from a match object, while translating the types in the process."""

1169 attrs = {}

1170 groupdict = mo.groupdict()

1171 for name, value in groupdict.items():

1172 if value is None:

1173 value = None

1174 elif self._int_re.match(value):

1175 value = int(value)

1176 elif self._float_re.match(value):

1177 value = float(value)

1178 attrs[name] = (value)

1179 return Struct(attrs)

1180

1181 _cg_header_re = re.compile(

1182 # original gprof header

1183 r'^\s+called/total\s+parents\s*$|' +

1184 r'^index\s+%time\s+self\s+descendents\s+called\+self\s+name\s+index\s*$|' +

1185 r'^\s+called/total\s+children\s*$|' +

1186 # GNU gprof header

1187 r'^index\s+%\s+(time\s+)?self\s+children\s+called\s+name\s*$'

1188 )

1189

1190 _cg_ignore_re = re.compile(

1191 # spontaneous

1192 r'^\s+<spontaneous>\s*$|'

1193 # internal calls (such as "mcount")

1194 r'^.*$(\d+)$$'

1195 )

1196

1197 _cg_primary_re = re.compile(

1198 r'^\[(?P<index>\d+)\]?' +

1199 r'\s+(?P<percentage_time>\d+\.\d+)' +

1200 r'\s+(?P<self>\d+\.\d+)' +

1201 r'\s+(?P<descendants>\d+\.\d+)' +

1202 r'\s+(?:(?P<called>\d+)(?:\+(?P<called_self>\d+))?)?' +

1203 r'\s+(?P<name>\S.*?)' +

1204 r'(?:\s+<cycle\s(?P<cycle>\d+)>)?' +

1205 r'\s\[(\d+)\]$'

1206 )

1207

1208 _cg_parent_re = re.compile(

1209 r'^\s+(?P<self>\d+\.\d+)?' +

1210 r'\s+(?P<descendants>\d+\.\d+)?' +

1211 r'\s+(?P<called>\d+)(?:/(?P<called_total>\d+))?' +

1212 r'\s+(?P<name>\S.*?)' +

1213 r'(?:\s+<cycle\s(?P<cycle>\d+)>)?' +

1214 r'\s\[(?P<index>\d+)\]$'

1215 )

1216

1217 _cg_child_re = _cg_parent_re

1218

1219 _cg_cycle_header_re = re.compile(

1220 r'^\[(?P<index>\d+)\]?' +

1221 r'\s+(?P<percentage_time>\d+\.\d+)' +

1222 r'\s+(?P<self>\d+\.\d+)' +

1223 r'\s+(?P<descendants>\d+\.\d+)' +

1224 r'\s+(?:(?P<called>\d+)(?:\+(?P<called_self>\d+))?)?' +

1225 r'\s+<cycle\s(?P<cycle>\d+)\sas\sa\swhole>' +

1226 r'\s\[(\d+)\]$'

1227 )

1228

1229 _cg_cycle_member_re = re.compile(

1230 r'^\s+(?P<self>\d+\.\d+)?' +

1231 r'\s+(?P<descendants>\d+\.\d+)?' +

1232 r'\s+(?P<called>\d+)(?:\+(?P<called_self>\d+))?' +

1233 r'\s+(?P<name>\S.*?)' +

1234 r'(?:\s+<cycle\s(?P<cycle>\d+)>)?' +

1235 r'\s\[(?P<index>\d+)\]$'

1236 )

1237

1238 _cg_sep_re = re.compile(r'^--+$')

1239

1240 def parse_function_entry(self, lines):

1241 parents = []

1242 children = []

1243

1244 while True:

1245 if not lines:

1246 sys.stderr.write('warning: unexpected end of entry\n')

1247 line = lines.pop(0)

1248 if line.startswith('['):

1249 break

1250

1251 # read function parent line

1252 mo = self._cg_parent_re.match(line)

1253 if not mo:

1254 if self._cg_ignore_re.match(line):

1255 continue

1256 sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line)

1257 else:

1258 parent = self.translate(mo)

1259 parents.append(parent)

1260

1261 # read primary line

1262 mo = self._cg_primary_re.match(line)

1263 if not mo:

1264 sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line)

1265 return

1266 else:

1267 function = self.translate(mo)

1268

1269 while lines:

1270 line = lines.pop(0)

1271

1272 # read function subroutine line

1273 mo = self._cg_child_re.match(line)

1274 if not mo:

1275 if self._cg_ignore_re.match(line):

1276 continue

1277 sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line)

1278 else:

1279 child = self.translate(mo)

1280 children.append(child)

1281

1282 function.parents = parents

1283 function.children = children

1284

1285 self.functions[function.index] = function

1286

1287 def parse_cycle_entry(self, lines):

1288

1289 # read cycle header line

1290 line = lines[0]

1291 mo = self._cg_cycle_header_re.match(line)

1292 if not mo:

1293 sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line)

1294 return

1295 cycle = self.translate(mo)

1296

1297 # read cycle member lines

1298 cycle.functions = []

1299 for line in lines[1:]:

1300 mo = self._cg_cycle_member_re.match(line)

1301 if not mo:

1302 sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line)

1303 continue

1304 call = self.translate(mo)

1305 cycle.functions.append(call)

1306

1307 self.cycles[cycle.cycle] = cycle

1308

1309 def parse_cg_entry(self, lines):

1310 if lines[0].startswith("["):

1311 self.parse_cycle_entry(lines)

1312 else:

1313 self.parse_function_entry(lines)

1314

1315 def parse_cg(self):

1316 """Parse the call graph."""

1317

1318 # skip call graph header

1319 while not self._cg_header_re.match(self.readline()):

1320 pass

1321 line = self.readline()

1322 while self._cg_header_re.match(line):

1323 line = self.readline()

1324

1325 # process call graph entries

1326 entry_lines = []

1327 while line != '\014': # form feed

1328 if line and not line.isspace():

1329 if self._cg_sep_re.match(line):

1330 self.parse_cg_entry(entry_lines)

1331 entry_lines = []

1332 else:

1333 entry_lines.append(line)

1334 line = self.readline()

1335

1336 def parse(self):

1337 self.parse_cg()

1338 self.fp.close()

1339

1340 profile = Profile()

1341 profile[TIME] = 0.0

1342

1343 cycles = {}

1344 for index in self.cycles:

1345 cycles[index] = Cycle()

1346

1347 for entry in self.functions.values():

1348 # populate the function

1349 function = Function(entry.index, entry.name)

1350 function[TIME] = entry.self

1351 if entry.called is not None:

1352 function.called = entry.called

1353 if entry.called_self is not None:

1354 call = Call(entry.index)

1355 call[CALLS] = entry.called_self

1356 function.called += entry.called_self

1357

1358 # populate the function calls

1359 for child in entry.children:

1360 call = Call(child.index)

1361

1362 assert child.called is not None

1363 call[CALLS] = child.called

1364

1365 if child.index not in self.functions:

1366 # NOTE: functions that were never called but were discovered by gprof's

1367 # static call graph analysis dont have a call graph entry so we need

1368 # to add them here

1369 missing = Function(child.index, child.name)

1370 function[TIME] = 0.0

1371 function.called = 0

1372 profile.add_function(missing)

1373

1374 function.add_call(call)

1375

1376 profile.add_function(function)

1377

1378 if entry.cycle is not None:

1379 try:

1380 cycle = cycles[entry.cycle]

1381 except KeyError:

1382 sys.stderr.write('warning: <cycle %u as a whole> entry missing\n' % entry.cycle)

1383 cycle = Cycle()

1384 cycles[entry.cycle] = cycle

1385 cycle.add_function(function)

1386

1387 profile[TIME] = profile[TIME] + function[TIME]

1388

1389 for cycle in cycles.values():

1390 profile.add_cycle(cycle)

1391

1392 # Compute derived events

1393 profile.validate()

1394 profile.ratio(TIME_RATIO, TIME)

1395 profile.call_ratios(CALLS)

1396 profile.integrate(TOTAL_TIME, TIME)

1397 profile.ratio(TOTAL_TIME_RATIO, TOTAL_TIME)

1398

1399 return profile

1400

1401

1402# Clone&hack of GprofParser for VTune Amplifier XE 2013 gprof-cc output.

1403# Tested only with AXE 2013 for Windows.

1404# - Use total times as reported by AXE.

1405# - In the absence of call counts, call ratios are faked from the relative

1406# proportions of total time. This affects only the weighting of the calls.

1407# - Different header, separator, and end marker.

1408# - Extra whitespace after function names.

1409# - You get a full entry for <spontaneous>, which does not have parents.

1410# - Cycles do have parents. These are saved but unused (as they are

1411# for functions).

1412# - Disambiguated "unrecognized call graph entry" error messages.

1413# Notes:

1414# - Total time of functions as reported by AXE passes the val3 test.

1415# - CPU Time:Children in the input is sometimes a negative number. This

1416# value goes to the variable descendants, which is unused.

1417# - The format of gprof-cc reports is unaffected by the use of

1418# -knob enable-call-counts=true (no call counts, ever), or

1419# -show-as=samples (results are quoted in seconds regardless).

1420class AXEParser(Parser):

1421 "Parser for VTune Amplifier XE 2013 gprof-cc report output."

1422

1423 def __init__(self, fp):

1424 Parser.__init__(self)

1425 self.fp = fp

1426 self.functions = {}

1427 self.cycles = {}

1428

1429 def readline(self):

1430 line = self.fp.readline()

1431 if not line:

1432 sys.stderr.write('error: unexpected end of file\n')

1433 sys.exit(1)

1434 line = line.rstrip('\r\n')

1435 return line

1436

1437 _int_re = re.compile(r'^\d+$')

1438 _float_re = re.compile(r'^\d+\.\d+$')

1439

1440 def translate(self, mo):

1441 """Extract a structure from a match object, while translating the types in the process."""

1442 attrs = {}

1443 groupdict = mo.groupdict()

1444 for name, value in groupdict.items():

1445 if value is None:

1446 value = None

1447 elif self._int_re.match(value):

1448 value = int(value)

1449 elif self._float_re.match(value):

1450 value = float(value)

1451 attrs[name] = (value)

1452 return Struct(attrs)

1453

1454 _cg_header_re = re.compile(

1455 '^Index |'

1456 '^-----+ '

1457 )

1458

1459 _cg_footer_re = re.compile(r'^Index\s+Function\s*$')

1460

1461 _cg_primary_re = re.compile(

1462 r'^\[(?P<index>\d+)\]?' +

1463 r'\s+(?P<percentage_time>\d+\.\d+)' +

1464 r'\s+(?P<self>\d+\.\d+)' +

1465 r'\s+(?P<descendants>\d+\.\d+)' +

1466 r'\s+(?P<name>\S.*?)' +

1467 r'(?:\s+<cycle\s(?P<cycle>\d+)>)?' +

1468 r'\s+\[(\d+)\]' +

1469 r'\s*$'

1470 )

1471

1472 _cg_parent_re = re.compile(

1473 r'^\s+(?P<self>\d+\.\d+)?' +

1474 r'\s+(?P<descendants>\d+\.\d+)?' +

1475 r'\s+(?P<name>\S.*?)' +

1476 r'(?:\s+<cycle\s(?P<cycle>\d+)>)?' +

1477 r'(?:\s+\[(?P<index>\d+)\]\s*)?' +

1478 r'\s*$'

1479 )

1480

1481 _cg_child_re = _cg_parent_re

1482

1483 _cg_cycle_header_re = re.compile(

1484 r'^\[(?P<index>\d+)\]?' +

1485 r'\s+(?P<percentage_time>\d+\.\d+)' +

1486 r'\s+(?P<self>\d+\.\d+)' +

1487 r'\s+(?P<descendants>\d+\.\d+)' +

1488 r'\s+<cycle\s(?P<cycle>\d+)\sas\sa\swhole>' +

1489 r'\s+\[(\d+)\]' +

1490 r'\s*$'

1491 )

1492

1493 _cg_cycle_member_re = re.compile(

1494 r'^\s+(?P<self>\d+\.\d+)?' +

1495 r'\s+(?P<descendants>\d+\.\d+)?' +

1496 r'\s+(?P<name>\S.*?)' +

1497 r'(?:\s+<cycle\s(?P<cycle>\d+)>)?' +

1498 r'\s+\[(?P<index>\d+)\]' +

1499 r'\s*$'

1500 )

1501

1502 def parse_function_entry(self, lines):

1503 parents = []

1504 children = []

1505

1506 while True:

1507 if not lines:

1508 sys.stderr.write('warning: unexpected end of entry\n')

1509 return

1510 line = lines.pop(0)

1511 if line.startswith('['):

1512 break

1513

1514 # read function parent line

1515 mo = self._cg_parent_re.match(line)

1516 if not mo:

1517 sys.stderr.write('warning: unrecognized call graph entry (1): %r\n' % line)

1518 else:

1519 parent = self.translate(mo)

1520 if parent.name != '<spontaneous>':

1521 parents.append(parent)

1522

1523 # read primary line

1524 mo = self._cg_primary_re.match(line)

1525 if not mo:

1526 sys.stderr.write('warning: unrecognized call graph entry (2): %r\n' % line)

1527 return

1528 else:

1529 function = self.translate(mo)

1530

1531 while lines:

1532 line = lines.pop(0)

1533

1534 # read function subroutine line

1535 mo = self._cg_child_re.match(line)

1536 if not mo:

1537 sys.stderr.write('warning: unrecognized call graph entry (3): %r\n' % line)

1538 else:

1539 child = self.translate(mo)

1540 if child.name != '<spontaneous>':

1541 children.append(child)

1542

1543 if function.name != '<spontaneous>':

1544 function.parents = parents

1545 function.children = children

1546

1547 self.functions[function.index] = function

1548

1549 def parse_cycle_entry(self, lines):

1550

1551 # Process the parents that were not there in gprof format.

1552 parents = []

1553 while True:

1554 if not lines:

1555 sys.stderr.write('warning: unexpected end of cycle entry\n')

1556 return

1557 line = lines.pop(0)

1558 if line.startswith('['):

1559 break

1560 mo = self._cg_parent_re.match(line)

1561 if not mo:

1562 sys.stderr.write('warning: unrecognized call graph entry (6): %r\n' % line)

1563 else:

1564 parent = self.translate(mo)

1565 if parent.name != '<spontaneous>':

1566 parents.append(parent)

1567

1568 # read cycle header line

1569 mo = self._cg_cycle_header_re.match(line)

1570 if not mo:

1571 sys.stderr.write('warning: unrecognized call graph entry (4): %r\n' % line)

1572 return

1573 cycle = self.translate(mo)

1574

1575 # read cycle member lines

1576 cycle.functions = []

1577 for line in lines[1:]:

1578 mo = self._cg_cycle_member_re.match(line)

1579 if not mo:

1580 sys.stderr.write('warning: unrecognized call graph entry (5): %r\n' % line)

1581 continue

1582 call = self.translate(mo)

1583 cycle.functions.append(call)

1584

1585 cycle.parents = parents

1586 self.cycles[cycle.cycle] = cycle

1587

1588 def parse_cg_entry(self, lines):

1589 if any("as a whole" in linelooper for linelooper in lines):

1590 self.parse_cycle_entry(lines)

1591 else:

1592 self.parse_function_entry(lines)

1593

1594 def parse_cg(self):

1595 """Parse the call graph."""

1596

1597 # skip call graph header

1598 line = self.readline()

1599 while self._cg_header_re.match(line):

1600 line = self.readline()

1601

1602 # process call graph entries

1603 entry_lines = []

1604 # An EOF in readline terminates the program without returning.

1605 while not self._cg_footer_re.match(line):

1606 if line.isspace():

1607 self.parse_cg_entry(entry_lines)

1608 entry_lines = []

1609 else:

1610 entry_lines.append(line)

1611 line = self.readline()

1612

1613 def parse(self):

1614 sys.stderr.write('warning: for axe format, edge weights are unreliable estimates derived from function total times.\n')

1615 self.parse_cg()

1616 self.fp.close()

1617

1618 profile = Profile()

1619 profile[TIME] = 0.0

1620

1621 cycles = {}

1622 for index in self.cycles:

1623 cycles[index] = Cycle()

1624

1625 for entry in self.functions.values():

1626 # populate the function

1627 function = Function(entry.index, entry.name)

1628 function[TIME] = entry.self

1629 function[TOTAL_TIME_RATIO] = entry.percentage_time / 100.0

1630

1631 # populate the function calls

1632 for child in entry.children:

1633 call = Call(child.index)

1634 # The following bogus value affects only the weighting of

1635 # the calls.

1636 call[TOTAL_TIME_RATIO] = function[TOTAL_TIME_RATIO]

1637

1638 if child.index not in self.functions:

1639 # NOTE: functions that were never called but were discovered by gprof's

1640 # static call graph analysis dont have a call graph entry so we need

1641 # to add them here

1642 # FIXME: Is this applicable?

1643 missing = Function(child.index, child.name)

1644 function[TIME] = 0.0

1645 profile.add_function(missing)

1646

1647 function.add_call(call)

1648

1649 profile.add_function(function)

1650

1651 if entry.cycle is not None:

1652 try:

1653 cycle = cycles[entry.cycle]

1654 except KeyError:

1655 sys.stderr.write('warning: <cycle %u as a whole> entry missing\n' % entry.cycle)

1656 cycle = Cycle()

1657 cycles[entry.cycle] = cycle

1658 cycle.add_function(function)

1659

1660 profile[TIME] = profile[TIME] + function[TIME]

1661

1662 for cycle in cycles.values():

1663 profile.add_cycle(cycle)

1664

1665 # Compute derived events.

1666 profile.validate()

1667 profile.ratio(TIME_RATIO, TIME)

1668 # Lacking call counts, fake call ratios based on total times.

1669 profile.call_ratios(TOTAL_TIME_RATIO)

1670 # The TOTAL_TIME_RATIO of functions is already set. Propagate that

1671 # total time to the calls. (TOTAL_TIME is neither set nor used.)

1672 for function in profile.functions.values():

1673 for call in function.calls.values():

1674 if call.ratio is not None:

1675 callee = profile.functions[call.callee_id]

1676 call[TOTAL_TIME_RATIO] = call.ratio * callee[TOTAL_TIME_RATIO]

1677

1678 return profile

1679

1680

1681class CallgrindParser(LineParser):

1682 """Parser for valgrind's callgrind tool.

1683

1684 See also:

1685 - https://valgrind.org/docs/manual/cl-format.html

1686 """

1687

1688 _call_re = re.compile(r'^calls=\s*(\d+)\s+((\d+|\+\d+|-\d+|\*)\s+)+$')

1689

1690 def __init__(self, infile):

1691 LineParser.__init__(self, infile)

1692

1693 # Textual positions

1694 self.position_ids = {}

1695 self.positions = {}

1696

1697 # Numeric positions

1698 self.num_positions = 1

1699 self.cost_positions = ['line']

1700 self.last_positions = [0]

1701

1702 # Events

1703 self.num_events = 0

1704 self.cost_events = []

1705

1706 self.profile = Profile()

1707 self.profile[SAMPLES] = 0

1708

1709 def parse(self):

1710 # read lookahead

1711 self.readline()

1712

1713 self.parse_key('version')

1714 self.parse_key('creator')

1715 while self.parse_part():

1716 pass

1717 if not self.eof():

1718 sys.stderr.write('warning: line %u: unexpected line\n' % self.line_no)

1719 sys.stderr.write('%s\n' % self.lookahead())

1720

1721 # compute derived data

1722 self.profile.validate()

1723 self.profile.find_cycles()

1724 self.profile.ratio(TIME_RATIO, SAMPLES)

1725 self.profile.call_ratios(SAMPLES2)

1726 self.profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)

1727

1728 return self.profile

1729

1730 def parse_part(self):

1731 if not self.parse_header_line():

1732 return False

1733 while self.parse_header_line():

1734 pass

1735 if not self.parse_body_line():

1736 return False

1737 while self.parse_body_line():

1738 pass

1739 return True

1740

1741 def parse_header_line(self):

1742 return \

1743 self.parse_empty() or \

1744 self.parse_comment() or \

1745 self.parse_part_detail() or \

1746 self.parse_description() or \

1747 self.parse_event_specification() or \

1748 self.parse_cost_line_def() or \

1749 self.parse_cost_summary()

1750

1751 _detail_keys = set(('cmd', 'pid', 'thread', 'part'))

1752

1753 def parse_part_detail(self):

1754 return self.parse_keys(self._detail_keys)

1755

1756 def parse_description(self):

1757 return self.parse_key('desc') is not None

1758

1759 def parse_event_specification(self):

1760 event = self.parse_key('event')

1761 if event is None:

1762 return False

1763 return True

1764

1765 def parse_cost_line_def(self):

1766 pair = self.parse_keys(('events', 'positions'))

1767 if pair is None:

1768 return False

1769 key, value = pair

1770 items = value.split()

1771 if key == 'events':

1772 self.num_events = len(items)

1773 self.cost_events = items

1774 if key == 'positions':

1775 self.num_positions = len(items)

1776 self.cost_positions = items

1777 self.last_positions = [0]*self.num_positions

1778 return True

1779

1780 def parse_cost_summary(self):

1781 pair = self.parse_keys(('summary', 'totals'))

1782 if pair is None:

1783 return False

1784 return True

1785

1786 def parse_body_line(self):

1787 return \

1788 self.parse_empty() or \

1789 self.parse_comment() or \

1790 self.parse_cost_line() or \

1791 self.parse_position_spec() or \

1792 self.parse_association_spec()

1793

1794 __subpos_re = r'(0x[0-9a-fA-F]+|\d+|\+\d+|-\d+|\*)'

1795 _cost_re = re.compile(r'^' +

1796 __subpos_re + r'( +' + __subpos_re + r')*' +

1797 r'( +\d+)*' +

1798 '$')

1799

1800 def parse_cost_line(self, calls=None):

1801 line = self.lookahead().rstrip()

1802 mo = self._cost_re.match(line)

1803 if not mo:

1804 return False

1805

1806 function = self.get_function()

1807

1808 if calls is None:

1809 # Unlike other aspects, call object (cob) is relative not to the

1810 # last call object, but to the caller's object (ob), so try to

1811 # update it when processing a functions cost line

1812 try:

1813 self.positions['cob'] = self.positions['ob']

1814 except KeyError:

1815 pass

1816

1817 values = line.split()

1818 assert len(values) <= self.num_positions + self.num_events

1819

1820 positions = values[0 : self.num_positions]

1821 events = values[self.num_positions : ]

1822 events += ['0']*(self.num_events - len(events))

1823

1824 for i in range(self.num_positions):

1825 position = positions[i]

1826 if position == '*':

1827 position = self.last_positions[i]

1828 elif position[0] in '-+':

1829 position = self.last_positions[i] + int(position)

1830 elif position.startswith('0x'):

1831 position = int(position, 16)

1832 else:

1833 position = int(position)

1834 self.last_positions[i] = position

1835

1836 events = [float(event) for event in events]

1837

1838 if calls is None:

1839 function[SAMPLES] += events[0]

1840 self.profile[SAMPLES] += events[0]

1841 else:

1842 callee = self.get_callee()

1843 callee.called += calls

1844

1845 try:

1846 call = function.calls[callee.id]

1847 except KeyError:

1848 call = Call(callee.id)

1849 call[CALLS] = calls

1850 call[SAMPLES2] = events[0]

1851 function.add_call(call)

1852 else:

1853 call[CALLS] += calls

1854 call[SAMPLES2] += events[0]

1855

1856 self.consume()

1857 return True

1858

1859 def parse_association_spec(self):

1860 line = self.lookahead()

1861 if not line.startswith('calls='):

1862 return False

1863

1864 _, values = line.split('=', 1)

1865 values = values.strip().split()

1866 calls = int(values[0])

1867 call_position = values[1:]

1868 self.consume()

1869

1870 self.parse_cost_line(calls)

1871

1872 return True

1873

1874 _position_re = re.compile(r'^(?P<position>[cj]?(?:ob|fl|fi|fe|fn))=\s*(?:$(?P<id>\d+)$)?(?:\s*(?P<name>.+))?')

1875

1876 _position_table_map = {

1877 'ob': 'ob',

1878 'fl': 'fl',

1879 'fi': 'fl',

1880 'fe': 'fl',

1881 'fn': 'fn',

1882 'cob': 'ob',

1883 'cfl': 'fl',

1884 'cfi': 'fl',

1885 'cfe': 'fl',

1886 'cfn': 'fn',

1887 'jfi': 'fl',

1888 }

1889

1890 _position_map = {

1891 'ob': 'ob',

1892 'fl': 'fl',

1893 'fi': 'fl',

1894 'fe': 'fl',

1895 'fn': 'fn',

1896 'cob': 'cob',

1897 'cfl': 'cfl',

1898 'cfi': 'cfl',

1899 'cfe': 'cfl',

1900 'cfn': 'cfn',

1901 'jfi': 'jfi',

1902 }

1903

1904 def parse_position_spec(self):

1905 line = self.lookahead()

1906

1907 if line.startswith('jump=') or line.startswith('jcnd='):

1908 self.consume()

1909 return True

1910

1911 mo = self._position_re.match(line)

1912 if not mo:

1913 return False

1914

1915 position, id, name = mo.groups()

1916 if id:

1917 table = self._position_table_map[position]

1918 if name:

1919 self.position_ids[(table, id)] = name

1920 else:

1921 name = self.position_ids.get((table, id), '')

1922 self.positions[self._position_map[position]] = name

1923

1924 self.consume()

1925 return True

1926

1927 def parse_empty(self):

1928 if self.eof():

1929 return False

1930 line = self.lookahead()

1931 if line.strip():

1932 return False

1933 self.consume()

1934 return True

1935

1936 def parse_comment(self):

1937 line = self.lookahead()

1938 if not line.startswith('#'):

1939 return False

1940 self.consume()

1941 return True

1942

1943 _key_re = re.compile(r'^(\w+):')

1944

1945 def parse_key(self, key):

1946 pair = self.parse_keys((key,))

1947 if not pair:

1948 return None

1949 key, value = pair

1950 return value

1951

1952 def parse_keys(self, keys):

1953 line = self.lookahead()

1954 mo = self._key_re.match(line)

1955 if not mo:

1956 return None

1957 key, value = line.split(':', 1)

1958 if key not in keys:

1959 return None

1960 value = value.strip()

1961 self.consume()

1962 return key, value

1963

1964 def make_function(self, module, filename, name):

1965 # FIXME: module and filename are not being tracked reliably

1966 #id = '|'.join((module, filename, name))

1967 id = name

1968 try:

1969 function = self.profile.functions[id]

1970 except KeyError:

1971 function = Function(id, name)

1972 if module:

1973 function.module = os.path.basename(module)

1974 function[SAMPLES] = 0

1975 function.called = 0

1976 self.profile.add_function(function)

1977 return function

1978

1979 def get_function(self):

1980 module = self.positions.get('ob', '')

1981 filename = self.positions.get('fl', '')

1982 function = self.positions.get('fn', '')

1983 return self.make_function(module, filename, function)

1984

1985 def get_callee(self):

1986 module = self.positions.get('cob', '')

1987 filename = self.positions.get('cfi', '')

1988 function = self.positions.get('cfn', '')

1989 return self.make_function(module, filename, function)

1990

1991 def readline(self):

1992 # Override LineParser.readline to ignore comment lines

1993 while True:

1994 LineParser.readline(self)

1995 if self.eof() or not self.lookahead().startswith('#'):

1996 break

1997

1998

1999class PerfParser(LineParser):

2000 """Parser for linux perf callgraph output.

2001

2002 It expects output generated with

2003

2004 perf record -g

2005 perf script | gprof2dot.py --format=perf

2006 """

2007

2008 def __init__(self, infile):

2009 LineParser.__init__(self, infile)

2010 self.profile = Profile()

2011

2012 def readline(self):

2013 # Override LineParser.readline to ignore comment lines

2014 while True:

2015 LineParser.readline(self)

2016 if self.eof() or not self.lookahead().startswith('#'):

2017 break

2018

2019 def parse(self):

2020 # read lookahead

2021 self.readline()

2022

2023 profile = self.profile

2024 profile[SAMPLES] = 0

2025 while not self.eof():

2026 self.parse_event()

2027

2028 # compute derived data

2029 profile.validate()

2030 profile.find_cycles()

2031 profile.ratio(TIME_RATIO, SAMPLES)

2032 profile.call_ratios(SAMPLES2)

2033 if totalMethod == "callratios":

2034 # Heuristic approach. TOTAL_SAMPLES is unused.

2035 profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)

2036 elif totalMethod == "callstacks":

2037 # Use the actual call chains for functions.

2038 profile[TOTAL_SAMPLES] = profile[SAMPLES]

2039 profile.ratio(TOTAL_TIME_RATIO, TOTAL_SAMPLES)

2040 # Then propagate that total time to the calls.

2041 for function in profile.functions.values():

2042 for call in function.calls.values():

2043 if call.ratio is not None:

2044 callee = profile.functions[call.callee_id]

2045 call[TOTAL_TIME_RATIO] = call.ratio * callee[TOTAL_TIME_RATIO]

2046 else:

2047 assert False

2048

2049 return profile

2050

2051 def parse_event(self):

2052 if self.eof():

2053 return

2054

2055 line = self.consume()

2056 assert line

2057

2058 callchain = self.parse_callchain()

2059 if not callchain:

2060 return

2061

2062 callee = callchain[0]

2063 callee[SAMPLES] += 1

2064 self.profile[SAMPLES] += 1

2065

2066 for caller in callchain[1:]:

2067 try:

2068 call = caller.calls[callee.id]

2069 except KeyError:

2070 call = Call(callee.id)

2071 call[SAMPLES2] = 1

2072 caller.add_call(call)

2073 else:

2074 call[SAMPLES2] += 1

2075

2076 callee = caller

2077

2078 # Increment TOTAL_SAMPLES only once on each function.

2079 stack = set(callchain)

2080 for function in stack:

2081 function[TOTAL_SAMPLES] += 1

2082

2083 def parse_callchain(self):

2084 callchain = []

2085 while self.lookahead():

2086 function = self.parse_call()

2087 if function is None:

2088 break

2089 callchain.append(function)

2090 if self.lookahead() == '':

2091 self.consume()

2092 return callchain

2093

2094 call_re = re.compile(r'^\s+(?P<address>[0-9a-fA-F]+)\s+(?P<symbol>.*)\s+$(?P<module>.*)$$')

2095 addr2_re = re.compile(r'\+0x[0-9a-fA-F]+$')

2096

2097 def parse_call(self):

2098 line = self.consume()

2099 mo = self.call_re.match(line)

2100 assert mo

2101 if not mo:

2102 return None

2103

2104 function_name = mo.group('symbol')

2105

2106 # If present, amputate program counter from function name.

2107 if function_name:

2108 function_name = re.sub(self.addr2_re, '', function_name)

2109

2110 if not function_name or function_name == '[unknown]':

2111 function_name = mo.group('address')

2112

2113 module = mo.group('module')

2114

2115 function_id = function_name + ':' + module

2116

2117 try:

2118 function = self.profile.functions[function_id]

2119 except KeyError:

2120 function = Function(function_id, function_name)

2121 function.module = os.path.basename(module)

2122 function[SAMPLES] = 0

2123 function[TOTAL_SAMPLES] = 0

2124 self.profile.add_function(function)

2125

2126 return function

2127

2128

2129class HProfParser(LineParser):

2130 """Parser for java hprof output

2131

2132 See also:

2133 - http://java.sun.com/developer/technicalArticles/Programming/HPROF.html

2134 """

2135

2136 trace_re = re.compile(r'\t(.*)$(.*):(.*)$')

2137 trace_id_re = re.compile(r'^TRACE (\d+):$')

2138

2139 def __init__(self, infile):

2140 LineParser.__init__(self, infile)

2141 self.traces = {}

2142 self.samples = {}

2143

2144 def parse(self):

2145 # read lookahead

2146 self.readline()

2147

2148 while not self.lookahead().startswith('------'): self.consume()

2149 while not self.lookahead().startswith('TRACE '): self.consume()

2150

2151 self.parse_traces()

2152

2153 while not self.lookahead().startswith('CPU'):

2154 self.consume()

2155

2156 self.parse_samples()

2157

2158 # populate the profile

2159 profile = Profile()

2160 profile[SAMPLES] = 0

2161

2162 functions = {}

2163

2164 # build up callgraph

2165 for id, trace in self.traces.items():

2166 if not id in self.samples: continue

2167 mtime = self.samples[id][0]

2168 last = None

2169

2170 for func, file, line in trace:

2171 if not func in functions:

2172 function = Function(func, func)

2173 function[SAMPLES] = 0

2174 profile.add_function(function)

2175 functions[func] = function

2176

2177 function = functions[func]

2178 # allocate time to the deepest method in the trace

2179 if not last:

2180 function[SAMPLES] += mtime

2181 profile[SAMPLES] += mtime

2182 else:

2183 c = function.get_call(last)

2184 c[SAMPLES2] += mtime

2185

2186 last = func

2187

2188 # compute derived data

2189 profile.validate()

2190 profile.find_cycles()

2191 profile.ratio(TIME_RATIO, SAMPLES)

2192 profile.call_ratios(SAMPLES2)

2193 profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)

2194

2195 return profile

2196

2197 def parse_traces(self):

2198 while self.lookahead().startswith('TRACE '):

2199 self.parse_trace()

2200

2201 def parse_trace(self):

2202 l = self.consume()

2203 mo = self.trace_id_re.match(l)

2204 tid = mo.group(1)

2205 last = None

2206 trace = []

2207

2208 while self.lookahead().startswith('\t'):

2209 l = self.consume()

2210 match = self.trace_re.search(l)

2211 if not match:

2212 #sys.stderr.write('Invalid line: %s\n' % l)

2213 break

2214 else:

2215 function_name, file, line = match.groups()

2216 trace += [(function_name, file, line)]

2217

2218 self.traces[int(tid)] = trace

2219

2220 def parse_samples(self):

2221 self.consume()

2222 self.consume()

2223

2224 while not self.lookahead().startswith('CPU'):

2225 rank, percent_self, percent_accum, count, traceid, method = self.lookahead().split()

2226 self.samples[int(traceid)] = (int(count), method)

2227 self.consume()

2228

2229

2230class SysprofParser(XmlParser):

2231

2232 def __init__(self, stream):

2233 XmlParser.__init__(self, stream)

2234

2235 def parse(self):

2236 objects = {}

2237 nodes = {}

2238

2239 self.element_start('profile')

2240 while self.token.type == XML_ELEMENT_START:

2241 if self.token.name_or_data == 'objects':

2242 assert not objects

2243 objects = self.parse_items('objects')

2244 elif self.token.name_or_data == 'nodes':

2245 assert not nodes

2246 nodes = self.parse_items('nodes')

2247 else:

2248 self.parse_value(self.token.name_or_data)

2249 self.element_end('profile')

2250

2251 return self.build_profile(objects, nodes)

2252

2253 def parse_items(self, name):

2254 assert name[-1] == 's'

2255 items = {}

2256 self.element_start(name)

2257 while self.token.type == XML_ELEMENT_START:

2258 id, values = self.parse_item(name[:-1])

2259 assert id not in items

2260 items[id] = values

2261 self.element_end(name)

2262 return items

2263

2264 def parse_item(self, name):

2265 attrs = self.element_start(name)

2266 id = int(attrs['id'])

2267 values = self.parse_values()

2268 self.element_end(name)

2269 return id, values

2270

2271 def parse_values(self):

2272 values = {}

2273 while self.token.type == XML_ELEMENT_START:

2274 name = self.token.name_or_data

2275 value = self.parse_value(name)

2276 assert name not in values

2277 values[name] = value

2278 return values

2279

2280 def parse_value(self, tag):

2281 self.element_start(tag)

2282 value = self.character_data()

2283 self.element_end(tag)

2284 if value.isdigit():

2285 return int(value)

2286 if value.startswith('"') and value.endswith('"'):

2287 return value[1:-1]

2288 return value

2289

2290 def build_profile(self, objects, nodes):

2291 profile = Profile()

2292

2293 profile[SAMPLES] = 0

2294 for id, object in objects.items():

2295 # Ignore fake objects (process names, modules, "Everything", "kernel", etc.)

2296 if object['self'] == 0:

2297 continue

2298

2299 function = Function(id, object['name'])

2300 function[SAMPLES] = object['self']

2301 profile.add_function(function)

2302 profile[SAMPLES] += function[SAMPLES]

2303

2304 for id, node in nodes.items():

2305 # Ignore fake calls

2306 if node['self'] == 0:

2307 continue

2308

2309 # Find a non-ignored parent

2310 parent_id = node['parent']

2311 while parent_id != 0:

2312 parent = nodes[parent_id]

2313 caller_id = parent['object']

2314 if objects[caller_id]['self'] != 0:

2315 break

2316 parent_id = parent['parent']

2317 if parent_id == 0:

2318 continue

2319

2320 callee_id = node['object']

2321

2322 assert objects[caller_id]['self']

2323 assert objects[callee_id]['self']

2324

2325 function = profile.functions[caller_id]

2326

2327 samples = node['self']

2328 try:

2329 call = function.calls[callee_id]

2330 except KeyError:

2331 call = Call(callee_id)

2332 call[SAMPLES2] = samples

2333 function.add_call(call)

2334 else:

2335 call[SAMPLES2] += samples

2336

2337 # Compute derived events

2338 profile.validate()

2339 profile.find_cycles()

2340 profile.ratio(TIME_RATIO, SAMPLES)

2341 profile.call_ratios(SAMPLES2)

2342 profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)

2343

2344 return profile

2345

2346

2347class XPerfParser(Parser):

2348 """Parser for CSVs generated by XPerf, from Microsoft Windows Performance Tools.

2349 """

2350

2351 def __init__(self, stream):

2352 Parser.__init__(self)

2353 self.stream = stream

2354 self.profile = Profile()

2355 self.profile[SAMPLES] = 0

2356 self.column = {}

2357

2358 def parse(self):

2359 import csv

2360 reader = csv.reader(

2361 self.stream,

2362 delimiter = ',',

2363 quotechar = None,

2364 escapechar = None,

2365 doublequote = False,

2366 skipinitialspace = True,

2367 lineterminator = '\r\n',

2368 quoting = csv.QUOTE_NONE)

2369 header = True

2370 for row in reader:

2371 if header:

2372 self.parse_header(row)

2373 header = False

2374 else:

2375 self.parse_row(row)

2376

2377 # compute derived data

2378 self.profile.validate()

2379 self.profile.find_cycles()

2380 self.profile.ratio(TIME_RATIO, SAMPLES)

2381 self.profile.call_ratios(SAMPLES2)

2382 self.profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)

2383

2384 return self.profile

2385

2386 def parse_header(self, row):

2387 for column in range(len(row)):

2388 name = row[column]

2389 assert name not in self.column

2390 self.column[name] = column

2391

2392 def parse_row(self, row):

2393 fields = {}

2394 for name, column in self.column.items():

2395 value = row[column]

2396 for factory in int, float:

2397 try:

2398 value = factory(value)

2399 except ValueError:

2400 pass

2401 else:

2402 break

2403 fields[name] = value

2404

2405 process = fields['Process Name']

2406 symbol = fields['Module'] + '!' + fields['Function']

2407 weight = fields['Weight']

2408 count = fields['Count']

2409

2410 if process == 'Idle':

2411 return

2412

2413 function = self.get_function(process, symbol)

2414 function[SAMPLES] += weight * count

2415 self.profile[SAMPLES] += weight * count

2416

2417 stack = fields['Stack']

2418 if stack != '?':

2419 stack = stack.split('/')

2420 assert stack[0] == '[Root]'

2421 if stack[-1] != symbol:

2422 # XXX: some cases the sampled function does not appear in the stack

2423 stack.append(symbol)

2424 caller = None

2425 for symbol in stack[1:]:

2426 callee = self.get_function(process, symbol)

2427 if caller is not None:

2428 try:

2429 call = caller.calls[callee.id]

2430 except KeyError:

2431 call = Call(callee.id)

2432 call[SAMPLES2] = count

2433 caller.add_call(call)

2434 else:

2435 call[SAMPLES2] += count

2436 caller = callee

2437

2438 def get_function(self, process, symbol):

2439 function_id = process + '!' + symbol

2440

2441 try:

2442 function = self.profile.functions[function_id]

2443 except KeyError:

2444 module, name = symbol.split('!', 1)

2445 function = Function(function_id, name)

2446 function.process = process

2447 function.module = module

2448 function[SAMPLES] = 0

2449 self.profile.add_function(function)

2450

2451 return function

2452

2453

2454class SleepyParser(Parser):

2455 """Parser for GNU gprof output.

2456

2457 See also:

2458 - http://www.codersnotes.com/sleepy/

2459 - http://sleepygraph.sourceforge.net/

2460 """

2461

2462 stdinInput = False

2463

2464 def __init__(self, filename):

2465 Parser.__init__(self)

2466

2467 from zipfile import ZipFile

2468

2469 self.database = ZipFile(filename)

2470

2471 self.symbols = {}

2472 self.calls = {}

2473

2474 self.profile = Profile()

2475

2476 _symbol_re = re.compile(

2477 r'^(?P<id>\w+)' +

2478 r'\s+"(?P<module>[^"]*)"' +

2479 r'\s+"(?P<procname>[^"]*)"' +

2480 r'\s+"(?P<sourcefile>[^"]*)"' +

2481 r'\s+(?P<sourceline>\d+)$'

2482 )

2483

2484 def openEntry(self, name):

2485 # Some versions of verysleepy use lowercase filenames

2486 for database_name in self.database.namelist():

2487 if name.lower() == database_name.lower():

2488 name = database_name

2489 break

2490

2491 return self.database.open(name, 'r')

2492

2493 def parse_symbols(self):

2494 for line in self.openEntry('Symbols.txt'):

2495 line = line.decode('UTF-8').rstrip('\r\n')

2496

2497 mo = self._symbol_re.match(line)

2498 if mo:

2499 symbol_id, module, procname, sourcefile, sourceline = mo.groups()

2500

2501 function_id = ':'.join([module, procname])

2502

2503 try:

2504 function = self.profile.functions[function_id]

2505 except KeyError:

2506 function = Function(function_id, procname)

2507 function.module = module

2508 function[SAMPLES] = 0

2509 self.profile.add_function(function)

2510

2511 self.symbols[symbol_id] = function

2512

2513 def parse_callstacks(self):

2514 for line in self.openEntry('Callstacks.txt'):

2515 line = line.decode('UTF-8').rstrip('\r\n')

2516

2517 fields = line.split()

2518 samples = float(fields[0])

2519 callstack = fields[1:]

2520

2521 callstack = [self.symbols[symbol_id] for symbol_id in callstack]

2522

2523 callee = callstack[0]

2524

2525 callee[SAMPLES] += samples

2526 self.profile[SAMPLES] += samples

2527

2528 for caller in callstack[1:]:

2529 try:

2530 call = caller.calls[callee.id]

2531 except KeyError:

2532 call = Call(callee.id)

2533 call[SAMPLES2] = samples

2534 caller.add_call(call)

2535 else:

2536 call[SAMPLES2] += samples

2537

2538 callee = caller

2539

2540 def parse(self):

2541 profile = self.profile

2542 profile[SAMPLES] = 0

2543

2544 self.parse_symbols()

2545 self.parse_callstacks()

2546

2547 # Compute derived events

2548 profile.validate()

2549 profile.find_cycles()

2550 profile.ratio(TIME_RATIO, SAMPLES)

2551 profile.call_ratios(SAMPLES2)

2552 profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)

2553

2554 return profile

2555

2556

2557class PstatsParser:

2558 """Parser python profiling statistics saved with te pstats module."""

2559

2560 stdinInput = False

2561 multipleInput = True

2562

2563 def __init__(self, *filename):

2564 import pstats

2565 try:

2566 self.stats = pstats.Stats(*filename)

2567 except ValueError:

2568 sys.stderr.write('error: failed to load %s, maybe they are generated by different python version?\n' % ', '.join(filename))

2569 sys.exit(1)

2570 self.profile = Profile()

2571 self.function_ids = {}

2572

2573 def get_function_name(self, key):

2574 filename, line, name = key

2575 module = os.path.splitext(filename)[0]

2576 module = os.path.basename(module)

2577 return "%s:%d:%s" % (module, line, name)

2578

2579 def get_function(self, key):

2580 try:

2581 id = self.function_ids[key]

2582 except KeyError:

2583 id = len(self.function_ids)

2584 name = self.get_function_name(key)

2585 function = Function(id, name)

2586 function.filename = key[0]

2587 self.profile.functions[id] = function

2588 self.function_ids[key] = id

2589 else:

2590 function = self.profile.functions[id]

2591 return function

2592

2593 def parse(self):

2594 self.profile[TIME] = 0.0

2595 self.profile[TOTAL_TIME] = self.stats.total_tt

2596 for fn, (cc, nc, tt, ct, callers) in self.stats.stats.items():

2597 callee = self.get_function(fn)

2598 callee.called = nc

2599 callee[TOTAL_TIME] = ct

2600 callee[TIME] = tt

2601 self.profile[TIME] += tt

2602 self.profile[TOTAL_TIME] = max(self.profile[TOTAL_TIME], ct)

2603 for fn, value in callers.items():

2604 caller = self.get_function(fn)

2605 call = Call(callee.id)

2606 if isinstance(value, tuple):

2607 for i in range(0, len(value), 4):

2608 nc, cc, tt, ct = value[i:i+4]

2609 if CALLS in call:

2610 call[CALLS] += cc

2611 else:

2612 call[CALLS] = cc

2613

2614 if TOTAL_TIME in call:

2615 call[TOTAL_TIME] += ct

2616 else:

2617 call[TOTAL_TIME] = ct

2618

2619 else:

2620 call[CALLS] = value

2621 call[TOTAL_TIME] = ratio(value, nc)*ct

2622

2623 caller.add_call(call)

2624

2625 if False:

2626 self.stats.print_stats()

2627 self.stats.print_callees()

2628

2629 # Compute derived events

2630 self.profile.validate()

2631 self.profile.ratio(TIME_RATIO, TIME)

2632 self.profile.ratio(TOTAL_TIME_RATIO, TOTAL_TIME)

2633

2634 return self.profile

2635

2636class DtraceParser(LineParser):

2637 """Parser for linux perf callgraph output.

2638

2639 It expects output generated with

2640

2641 # Refer to https://github.com/brendangregg/FlameGraph#dtrace

2642 # 60 seconds of user-level stacks, including time spent in-kernel, for PID 12345 at 97 Hertz

2643 sudo dtrace -x ustackframes=100 -n 'profile-97 /pid == 12345/ { @[ustack()] = count(); } tick-60s { exit(0); }' -o out.user_stacks

2644

2645 # The dtrace output

2646 gprof2dot.py -f dtrace out.user_stacks

2647

2648 # Notice: sometimes, the dtrace outputs format may be latin-1, and gprof2dot will fail to parse it.

2649 # To solve this problem, you should use iconv to convert to UTF-8 explicitly.

2650 # TODO: add an encoding flag to tell gprof2dot how to decode the profile file.

2651 iconv -f ISO-8859-1 -t UTF-8 out.user_stacks | gprof2dot.py -f dtrace

2652 """

2653

2654 def __init__(self, infile):

2655 LineParser.__init__(self, infile)

2656 self.profile = Profile()

2657

2658 def readline(self):

2659 # Override LineParser.readline to ignore comment lines

2660 while True:

2661 LineParser.readline(self)

2662 if self.eof():

2663 break

2664

2665 line = self.lookahead().strip()

2666 if line.startswith('CPU'):

2667 # The format likes:

2668 # CPU ID FUNCTION:NAME

2669 # 1 29684 :tick-60s

2670 # Skip next line

2671 LineParser.readline(self)

2672 elif not line == '':

2673 break

2674

2675

2676 def parse(self):

2677 # read lookahead

2678 self.readline()

2679

2680 profile = self.profile

2681 profile[SAMPLES] = 0

2682 while not self.eof():

2683 self.parse_event()

2684

2685 # compute derived data

2686 profile.validate()

2687 profile.find_cycles()

2688 profile.ratio(TIME_RATIO, SAMPLES)

2689 profile.call_ratios(SAMPLES2)

2690 if totalMethod == "callratios":

2691 # Heuristic approach. TOTAL_SAMPLES is unused.

2692 profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)

2693 elif totalMethod == "callstacks":

2694 # Use the actual call chains for functions.

2695 profile[TOTAL_SAMPLES] = profile[SAMPLES]

2696 profile.ratio(TOTAL_TIME_RATIO, TOTAL_SAMPLES)

2697 # Then propagate that total time to the calls.

2698 for function in profile.functions.values():

2699 for call in function.calls.values():

2700 if call.ratio is not None:

2701 callee = profile.functions[call.callee_id]

2702 call[TOTAL_TIME_RATIO] = call.ratio * callee[TOTAL_TIME_RATIO]

2703 else:

2704 assert False

2705

2706 return profile

2707

2708 def parse_event(self):

2709 if self.eof():

2710 return

2711

2712 callchain, count = self.parse_callchain()

2713 if not callchain:

2714 return

2715

2716 callee = callchain[0]

2717 callee[SAMPLES] += count

2718 self.profile[SAMPLES] += count

2719

2720 for caller in callchain[1:]:

2721 try:

2722 call = caller.calls[callee.id]

2723 except KeyError:

2724 call = Call(callee.id)

2725 call[SAMPLES2] = count

2726 caller.add_call(call)

2727 else:

2728 call[SAMPLES2] += count

2729

2730 callee = caller

2731

2732 # Increment TOTAL_SAMPLES only once on each function.

2733 stack = set(callchain)

2734 for function in stack:

2735 function[TOTAL_SAMPLES] += count

2736

2737

2738 def parse_callchain(self):

2739 callchain = []

2740 count = 0

2741 while self.lookahead():

2742 function, count = self.parse_call()

2743 if function is None:

2744 break

2745 callchain.append(function)

2746 return callchain, count

2747

2748 call_re = re.compile(r'^\s+(?P<module>.*)`(?P<symbol>.*)')

2749 addr2_re = re.compile(r'\+0x[0-9a-fA-F]+$')

2750

2751 def parse_call(self):

2752 line = self.consume()

2753 mo = self.call_re.match(line)

2754 if not mo:

2755 # The line must be the stack count

2756 return None, int(line.strip())

2757

2758 function_name = mo.group('symbol')

2759

2760 # If present, amputate program counter from function name.

2761 if function_name:

2762 function_name = re.sub(self.addr2_re, '', function_name)

2763

2764 # if not function_name or function_name == '[unknown]':

2765 # function_name = mo.group('address')

2766

2767 module = mo.group('module')

2768

2769 function_id = function_name + ':' + module

2770

2771 try:

2772 function = self.profile.functions[function_id]

2773 except KeyError:

2774 function = Function(function_id, function_name)

2775 function.module = os.path.basename(module)

2776 function[SAMPLES] = 0

2777 function[TOTAL_SAMPLES] = 0

2778 self.profile.add_function(function)

2779

2780 return function, None

2781

2782

2783class CollapseParser(LineParser):

2784 """Parser for the output of stackcollapse

2785

2786 (from https://github.com/brendangregg/FlameGraph)

2787 """

2788

2789 def __init__(self, infile):

2790 LineParser.__init__(self, infile)

2791 self.profile = Profile()

2792

2793 def parse(self):

2794 profile = self.profile

2795 profile[SAMPLES] = 0

2796

2797 self.readline()

2798 while not self.eof():

2799 self.parse_event()

2800

2801 # compute derived data

2802 profile.validate()

2803 profile.find_cycles()

2804 profile.ratio(TIME_RATIO, SAMPLES)

2805 profile.call_ratios(SAMPLES2)

2806 if totalMethod == "callratios":

2807 # Heuristic approach. TOTAL_SAMPLES is unused.

2808 profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO)

2809 elif totalMethod == "callstacks":

2810 # Use the actual call chains for functions.

2811 profile[TOTAL_SAMPLES] = profile[SAMPLES]

2812 profile.ratio(TOTAL_TIME_RATIO, TOTAL_SAMPLES)

2813 # Then propagate that total time to the calls.

2814 for function in compat_itervalues(profile.functions):

2815 for call in compat_itervalues(function.calls):

2816 if call.ratio is not None:

2817 callee = profile.functions[call.callee_id]

2818 call[TOTAL_TIME_RATIO] = call.ratio * callee[TOTAL_TIME_RATIO]

2819 else:

2820 assert False

2821

2822 return profile

2823

2824 def parse_event(self):

2825 line = self.consume()

2826

2827 stack, count = line.rsplit(' ',maxsplit=1)

2828 count=int(count)

2829 self.profile[SAMPLES] += count

2830

2831 calls = stack.split(';')

2832 functions = [self._make_function(call) for call in calls]

2833

2834 functions[-1][SAMPLES] += count

2835

2836 # TOTAL_SAMPLES excludes loops

2837 for func in set(functions):

2838 func[TOTAL_SAMPLES] += count

2839

2840 # add call data

2841 callee = functions[-1]

2842 for caller in reversed(functions[:-1]):

2843 call = caller.calls.get(callee.id)

2844 if call is None:

2845 call = Call(callee.id)

2846 call[SAMPLES2] = 0

2847 caller.add_call(call)

2848 call[SAMPLES2] += count

2849 callee = caller

2850

2851 call_re = re.compile(r'^(?P<func>[^ ]+) $(?P<file>.*):(?P<line>[0-9]+)$$')

2852

2853 def _make_function(self, call):

2854 """turn a call str into a Function

2855

2856 takes a call site, as found between semicolons in the input, and returns

2857 a Function definition corresponding to that call site.

2858 """

2859 mo = self.call_re.match(call)

2860 if mo:

2861 name, module, line = mo.groups()

2862 func_id = "%s:%s" % (module, name)

2863 else:

2864 name = func_id = call

2865 module = None

2866

2867 func = self.profile.functions.get(func_id)

2868 if func is None:

2869 func = Function(func_id, name)

2870 func.module = module

2871 func[SAMPLES] = 0

2872 func[TOTAL_SAMPLES] = 0

2873 self.profile.add_function(func)

2874 return func

2875

2876

2877formats = {

2878 "axe": AXEParser,

2879 "callgrind": CallgrindParser,

2880 "collapse": CollapseParser,

2881 "hprof": HProfParser,

2882 "json": JsonParser,

2883 "perf": PerfParser,

2884 "prof": GprofParser,

2885 "pstats": PstatsParser,

2886 "sleepy": SleepyParser,

2887 "sysprof": SysprofParser,

2888 "xperf": XPerfParser,

2889 "dtrace": DtraceParser,

2890}

2891

2892

2893########################################################################

2894# Output

2895

2896

2897class Theme:

2898

2899 def __init__(self,

2900 bgcolor = (0.0, 0.0, 1.0),

2901 mincolor = (0.0, 0.0, 0.0),

2902 maxcolor = (0.0, 0.0, 1.0),

2903 fontname = "Arial",

2904 fontcolor = "white",

2905 nodestyle = "filled",

2906 minfontsize = 10.0,

2907 maxfontsize = 10.0,

2908 minpenwidth = 0.5,

2909 maxpenwidth = 4.0,

2910 gamma = 2.2,

2911 skew = 1.0):

2912 self.bgcolor = bgcolor

2913 self.mincolor = mincolor

2914 self.maxcolor = maxcolor

2915 self.fontname = fontname

2916 self.fontcolor = fontcolor

2917 self.nodestyle = nodestyle

2918 self.minfontsize = minfontsize

2919 self.maxfontsize = maxfontsize

2920 self.minpenwidth = minpenwidth

2921 self.maxpenwidth = maxpenwidth

2922 self.gamma = gamma

2923 self.skew = skew

2924

2925 def graph_bgcolor(self):

2926 return self.hsl_to_rgb(*self.bgcolor)

2927

2928 def graph_fontname(self):

2929 return self.fontname

2930

2931 def graph_fontcolor(self):

2932 return self.fontcolor

2933

2934 def node_bgcolor(self, weight):

2935 return self.color(weight)

2936

2937 def node_fgcolor(self, weight):

2938 if self.nodestyle == "filled":

2939 return self.graph_bgcolor()

2940 else:

2941 return self.color(weight)

2942

2943 def node_fontsize(self, weight):

2944 return self.fontsize(weight)

2945

2946 def node_style(self):

2947 return self.nodestyle

2948

2949 def edge_color(self, weight):

2950 return self.color(weight)

2951

2952 def edge_fontsize(self, weight):

2953 return self.fontsize(weight)

2954

2955 def edge_penwidth(self, weight):

2956 return max(weight*self.maxpenwidth, self.minpenwidth)

2957

2958 def edge_arrowsize(self, weight):

2959 return 0.5 * math.sqrt(self.edge_penwidth(weight))

2960

2961 def fontsize(self, weight):

2962 return max(weight**2 * self.maxfontsize, self.minfontsize)

2963

2964 def color(self, weight):

2965 weight = min(max(weight, 0.0), 1.0)

2966

2967 hmin, smin, lmin = self.mincolor

2968 hmax, smax, lmax = self.maxcolor

2969

2970 if self.skew < 0:

2971 raise ValueError("Skew must be greater than 0")

2972 elif self.skew == 1.0:

2973 h = hmin + weight*(hmax - hmin)

2974 s = smin + weight*(smax - smin)

2975 l = lmin + weight*(lmax - lmin)

2976 else:

2977 base = self.skew

2978 h = hmin + ((hmax-hmin)*(-1.0 + (base ** weight)) / (base - 1.0))

2979 s = smin + ((smax-smin)*(-1.0 + (base ** weight)) / (base - 1.0))

2980 l = lmin + ((lmax-lmin)*(-1.0 + (base ** weight)) / (base - 1.0))

2981

2982 return self.hsl_to_rgb(h, s, l)

2983

2984 def hsl_to_rgb(self, h, s, l):

2985 """Convert a color from HSL color-model to RGB.

2986

2987 See also:

2988 - http://www.w3.org/TR/css3-color/#hsl-color

2989 """

2990

2991 h = h % 1.0

2992 s = min(max(s, 0.0), 1.0)

2993 l = min(max(l, 0.0), 1.0)

2994

2995 if l <= 0.5:

2996 m2 = l*(s + 1.0)

2997 else:

2998 m2 = l + s - l*s

2999 m1 = l*2.0 - m2

3000 r = self._hue_to_rgb(m1, m2, h + 1.0/3.0)

3001 g = self._hue_to_rgb(m1, m2, h)

3002 b = self._hue_to_rgb(m1, m2, h - 1.0/3.0)

3003

3004 # Apply gamma correction

3005 r **= self.gamma

3006 g **= self.gamma

3007 b **= self.gamma

3008

3009 return (r, g, b)

3010

3011 def _hue_to_rgb(self, m1, m2, h):

3012 if h < 0.0:

3013 h += 1.0

3014 elif h > 1.0:

3015 h -= 1.0

3016 if h*6 < 1.0:

3017 return m1 + (m2 - m1)*h*6.0

3018 elif h*2 < 1.0:

3019 return m2

3020 elif h*3 < 2.0:

3021 return m1 + (m2 - m1)*(2.0/3.0 - h)*6.0

3022 else:

3023 return m1

3024

3025

3026TEMPERATURE_COLORMAP = Theme(

3027 mincolor = (2.0/3.0, 0.80, 0.25), # dark blue

3028 maxcolor = (0.0, 1.0, 0.5), # satured red

3029 gamma = 1.0

3030)

3031

3032PINK_COLORMAP = Theme(

3033 mincolor = (0.0, 1.0, 0.90), # pink

3034 maxcolor = (0.0, 1.0, 0.5), # satured red

3035)

3036

3037GRAY_COLORMAP = Theme(

3038 mincolor = (0.0, 0.0, 0.85), # light gray

3039 maxcolor = (0.0, 0.0, 0.0), # black

3040)

3041

3042BW_COLORMAP = Theme(

3043 minfontsize = 8.0,

3044 maxfontsize = 24.0,

3045 mincolor = (0.0, 0.0, 0.0), # black

3046 maxcolor = (0.0, 0.0, 0.0), # black

3047 minpenwidth = 0.1,

3048 maxpenwidth = 8.0,

3049)

3050

3051PRINT_COLORMAP = Theme(

3052 minfontsize = 18.0,

3053 maxfontsize = 30.0,

3054 fontcolor = "black",

3055 nodestyle = "solid",

3056 mincolor = (0.0, 0.0, 0.0), # black

3057 maxcolor = (0.0, 0.0, 0.0), # black

3058 minpenwidth = 0.1,

3059 maxpenwidth = 8.0,

3060)

3061

3062

3063themes = {

3064 "color": TEMPERATURE_COLORMAP,

3065 "pink": PINK_COLORMAP,

3066 "gray": GRAY_COLORMAP,

3067 "bw": BW_COLORMAP,

3068 "print": PRINT_COLORMAP,

3069}

3070

3071

3072def sorted_iteritems(d):

3073 # Used mostly for result reproducibility (while testing.)

3074 keys = list(d.keys())

3075 keys.sort()

3076 for key in keys:

3077 value = d[key]

3078 yield key, value

3079

3080

3081class DotWriter:

3082 """Writer for the DOT language.

3083

3084 See also:

3085 - "The DOT Language" specification

3086 http://www.graphviz.org/doc/info/lang.html

3087 """

3088

3089 strip = False

3090 wrap = False

3091

3092 def __init__(self, fp):

3093 self.fp = fp

3094

3095 def wrap_function_name(self, name):

3096 """Split the function name on multiple lines."""

3097

3098 if len(name) > 32:

3099 ratio = 2.0/3.0

3100 height = max(int(len(name)/(1.0 - ratio) + 0.5), 1)

3101 width = max(len(name)/height, 32)

3102 # TODO: break lines in symbols

3103 name = textwrap.fill(name, width, break_long_words=False)

3104

3105 # Take away spaces

3106 name = name.replace(", ", ",")

3107 name = name.replace("> >", ">>")

3108 name = name.replace("> >", ">>") # catch consecutive

3109

3110 return name

3111

3112 show_function_events = [TOTAL_TIME_RATIO, TIME_RATIO]

3113 show_edge_events = [TOTAL_TIME_RATIO, CALLS]

3114

3115 def graphs_compare(self, profile1, profile2, theme, options):

3116 self.begin_graph()

3117

3118 fontname = theme.graph_fontname()

3119 fontcolor = theme.graph_fontcolor()

3120 nodestyle = theme.node_style()

3121

3122 tolerance, only_slower, only_faster, color_by_difference = (

3123 options.tolerance, options.only_slower, options.only_faster, options.color_by_difference)

3124 self.attr('graph', fontname=fontname, ranksep=0.25, nodesep=0.125)

3125 self.attr('node', fontname=fontname, style=nodestyle, fontcolor=fontcolor, width=0, height=0)

3126 self.attr('edge', fontname=fontname)

3127

3128 functions2 = {function.name: function for _, function in sorted_iteritems(profile2.functions)}

3129

3130 for _, function1 in sorted_iteritems(profile1.functions):

3131 labels = []

3132

3133 name = function1.name

3134 try:

3135 function2 = functions2[name]

3136 if self.wrap:

3137 name = self.wrap_function_name(name)

3138 if color_by_difference:

3139 min_diff, max_diff = min_max_difference(profile1, profile2)

3140 labels.append(name)

3141 weight_difference = 0

3142 shape = 'box'

3143 orientation = '0'

3144 for event in self.show_function_events:

3145 if event in function1.events:

3146 event1 = function1[event]

3147 event2 = function2[event]

3148

3149 difference = abs(event1 - event2) * 100

3150

3151 if event == TOTAL_TIME_RATIO:

3152 weight_difference = difference

3153 if difference >= tolerance:

3154 if event2 > event1 and not only_faster:

3155 shape = 'cds'

3156 label = (f'{event.format(event1)} +'

3157 f' {round_difference(difference, tolerance)}%')

3158 elif event2 < event1 and not only_slower:

3159 orientation = "90"

3160 shape = 'cds'

3161 label = (f'{event.format(event1)} - '

3162 f'{round_difference(difference, tolerance)}%')

3163 else:

3164 # protection to not color by difference if we choose to show only_faster/only_slower

3165 weight_difference = 0

3166 label = event.format(function1[event])

3167 else:

3168 weight_difference = 0

3169 label = event.format(function1[event])

3170 else:

3171 if difference >= tolerance:

3172 if event2 > event1:

3173 label = (f'{event.format(event1)} +'

3174 f' {round_difference(difference, tolerance)}%')

3175 elif event2 < event1:

3176 label = (f'{event.format(event1)} - '

3177 f'{round_difference(difference, tolerance)}%')

3178 else:

3179 label = event.format(function1[event])

3180

3181 labels.append(label)

3182 if function1.called is not None:

3183 labels.append(f"{function1.called} {MULTIPLICATION_SIGN}/ {function2.called} {MULTIPLICATION_SIGN}")

3184

3185 except KeyError:

3186 shape = 'box'

3187 orientation = '0'

3188 weight_difference = 0

3189 if function1.process is not None:

3190 labels.append(function1.process)

3191 if function1.module is not None:

3192 labels.append(function1.module)

3193

3194 if self.strip:

3195 function_name = function1.stripped_name()

3196 else:

3197 function_name = function1.name

3198 if color_by_difference:

3199 min_diff, max_diff = 0, 0

3200

3201 # dot can't parse quoted strings longer than YY_BUF_SIZE, which

3202 # defaults to 16K. But some annotated C++ functions (e.g., boost,

3203 # https://github.com/jrfonseca/gprof2dot/issues/30) can exceed that

3204 MAX_FUNCTION_NAME = 4096

3205 if len(function_name) >= MAX_FUNCTION_NAME:

3206 sys.stderr.write('warning: truncating function name with %u chars (%s)\n' % (len(function_name), function_name[:32] + '...'))

3207 function_name = function_name[:MAX_FUNCTION_NAME - 1] + chr(0x2026)

3208

3209 if self.wrap:

3210 function_name = self.wrap_function_name(function_name)

3211 labels.append(function_name)

3212

3213 for event in self.show_function_events:

3214 if event in function1.events:

3215 label = event.format(function1[event])

3216 labels.append(label)

3217 if function1.called is not None:

3218 labels.append("%u%s" % (function1.called, MULTIPLICATION_SIGN))

3219

3220 if color_by_difference and weight_difference:

3221 # min and max is calculated whe color_by_difference is true

3222 weight = rescale_difference(weight_difference, min_diff, max_diff)

3223

3224 elif function1.weight is not None and not color_by_difference:

3225 weight = function1.weight

3226 else:

3227 weight = 0.0

3228

3229 label = '\n'.join(labels)

3230

3231 self.node(function1.id,

3232 label=label,

3233 orientation=orientation,

3234 color=self.color(theme.node_bgcolor(weight)),

3235 shape=shape,

3236 fontcolor=self.color(theme.node_fgcolor(weight)),

3237 fontsize="%f" % theme.node_fontsize(weight),

3238 tooltip=function1.filename,

3239 )

3240

3241 calls2 = {call.callee_id: call for _, call in sorted_iteritems(function2.calls)}

3242 functions_by_id1 = {function.id: function for _, function in sorted_iteritems(profile1.functions)}

3243

3244 for _, call1 in sorted_iteritems(function1.calls):

3245 labels = []

3246 try:

3247 # if profiles do not have identical setups, callee_id will not be identical either

3248 call_id1 = call1.callee_id

3249 call_name = functions_by_id1[call_id1].name

3250 call_id2 = functions2[call_name].id

3251 call2 = calls2[call_id2]

3252 for event in self.show_edge_events:

3253 if event in call1.events:

3254 label = f'{event.format(call1[event])} / {event.format(call2[event])}'

3255 labels.append(label)

3256 except KeyError:

3257 for event in self.show_edge_events:

3258 if event in call1.events:

3259 label = f'{event.format(call1[event])}'

3260 labels.append(label)

3261

3262 weight = 0 if color_by_difference else call1.weight

3263 label = '\n'.join(labels)

3264 self.edge(function1.id, call1.callee_id,

3265 label=label,

3266 color=self.color(theme.edge_color(weight)),

3267 fontcolor=self.color(theme.edge_color(weight)),

3268 fontsize="%.2f" % theme.edge_fontsize(weight),

3269 penwidth="%.2f" % theme.edge_penwidth(weight),

3270 labeldistance="%.2f" % theme.edge_penwidth(weight),

3271 arrowsize="%.2f" % theme.edge_arrowsize(weight),

3272 )

3273 self.end_graph()

3274

3275 def graph(self, profile, theme):

3276 self.begin_graph()

3277

3278 fontname = theme.graph_fontname()

3279 fontcolor = theme.graph_fontcolor()

3280 nodestyle = theme.node_style()

3281

3282 self.attr('graph', fontname=fontname, ranksep=0.25, nodesep=0.125)

3283 self.attr('node', fontname=fontname, shape="box", style=nodestyle, fontcolor=fontcolor, width=0, height=0)

3284 self.attr('edge', fontname=fontname)

3285

3286 for _, function in sorted_iteritems(profile.functions):

3287 labels = []

3288 if function.process is not None:

3289 labels.append(function.process)

3290 if function.module is not None:

3291 labels.append(function.module)

3292

3293 if self.strip:

3294 function_name = function.stripped_name()

3295 else:

3296 function_name = function.name

3297

3298 # dot can't parse quoted strings longer than YY_BUF_SIZE, which

3299 # defaults to 16K. But some annotated C++ functions (e.g., boost,

3300 # https://github.com/jrfonseca/gprof2dot/issues/30) can exceed that

3301 MAX_FUNCTION_NAME = 4096

3302 if len(function_name) >= MAX_FUNCTION_NAME:

3303 sys.stderr.write('warning: truncating function name with %u chars (%s)\n' % (len(function_name), function_name[:32] + '...'))

3304 function_name = function_name[:MAX_FUNCTION_NAME - 1] + chr(0x2026)

3305

3306 if self.wrap:

3307 function_name = self.wrap_function_name(function_name)

3308 labels.append(function_name)

3309

3310 for event in self.show_function_events:

3311 if event in function.events:

3312 label = event.format(function[event])

3313 labels.append(label)

3314 if function.called is not None:

3315 labels.append("%u%s" % (function.called, MULTIPLICATION_SIGN))

3316

3317 if function.weight is not None:

3318 weight = function.weight

3319 else:

3320 weight = 0.0

3321

3322 label = '\n'.join(labels)

3323 self.node(function.id,

3324 label = label,

3325 color = self.color(theme.node_bgcolor(weight)),

3326 fontcolor = self.color(theme.node_fgcolor(weight)),

3327 fontsize = "%.2f" % theme.node_fontsize(weight),

3328 tooltip = function.filename,

3329 )

3330

3331 for _, call in sorted_iteritems(function.calls):

3332 callee = profile.functions[call.callee_id]

3333

3334 labels = []

3335 for event in self.show_edge_events:

3336 if event in call.events:

3337 label = event.format(call[event])

3338 labels.append(label)

3339

3340 if call.weight is not None:

3341 weight = call.weight

3342 elif callee.weight is not None:

3343 weight = callee.weight

3344 else:

3345 weight = 0.0

3346

3347 label = '\n'.join(labels)

3348

3349 self.edge(function.id, call.callee_id,

3350 label = label,

3351 color = self.color(theme.edge_color(weight)),

3352 fontcolor = self.color(theme.edge_color(weight)),

3353 fontsize = "%.2f" % theme.edge_fontsize(weight),

3354 penwidth = "%.2f" % theme.edge_penwidth(weight),

3355 labeldistance = "%.2f" % theme.edge_penwidth(weight),

3356 arrowsize = "%.2f" % theme.edge_arrowsize(weight),

3357 )

3358

3359 self.end_graph()

3360

3361 def begin_graph(self):

3362 self.write('digraph {\n')

3363 # Work-around graphviz bug[1]: unnamed graphs have "%3" tooltip in SVG

3364 # output. The bug was fixed upstream, but graphviz shipped in recent

3365 # Linux distros (for example, Ubuntu 24.04) still has the bug.

3366 # [1] https://gitlab.com/graphviz/graphviz/-/issues/1376

3367 self.write('\ttooltip=" "\n')

3368

3369 def end_graph(self):

3370 self.write('}\n')

3371

3372 def attr(self, what, **attrs):

3373 self.write("\t")

3374 self.write(what)

3375 self.attr_list(attrs)

3376 self.write(";\n")

3377

3378 def node(self, node, **attrs):

3379 self.write("\t")

3380 self.node_id(node)

3381 self.attr_list(attrs)

3382 self.write(";\n")

3383

3384 def edge(self, src, dst, **attrs):

3385 self.write("\t")

3386 self.node_id(src)

3387 self.write(" -> ")

3388 self.node_id(dst)

3389 self.attr_list(attrs)

3390 self.write(";\n")

3391

3392 def attr_list(self, attrs):

3393 if not attrs:

3394 return

3395 self.write(' [')

3396 first = True

3397 for name, value in sorted_iteritems(attrs):

3398 if value is None:

3399 continue

3400 if first:

3401 first = False

3402 else:

3403 self.write(", ")

3404 assert isinstance(name, str)

3405 assert name.isidentifier()

3406 self.write(name)

3407 self.write('=')

3408 self.id(value)

3409 self.write(']')

3410

3411 def node_id(self, id):

3412 # Node IDs need to be unique (can't be truncated) but dot doesn't allow

3413 # IDs longer than 16384 characters, so use an hash instead for the huge

3414 # C++ symbols that can arise, as seen in

3415 # https://github.com/jrfonseca/gprof2dot/issues/99

3416 if isinstance(id, str) and len(id) > 1024:

3417 id = '_' + hashlib.sha1(id.encode('utf-8'), usedforsecurity=False).hexdigest()

3418 self.id(id)

3419

3420 # dot keywords, which must be quoted when used as ordinary identifiers.

3421 _keywords = frozenset(('node', 'edge', 'graph', 'digraph', 'subgraph', 'strict'))

3422

3423 def id(self, id):

3424 if isinstance(id, (int, float)):

3425 s = str(id)

3426 elif isinstance(id, str):

3427 if id.isalnum() and not id.startswith('0x') and id.lower() not in self._keywords:

3428 s = id

3429 else:

3430 s = self.escape(id)

3431 else:

3432 raise TypeError

3433 self.write(s)

3434

3435 def color(self, rgb):

3436 r, g, b = rgb

3437

3438 def float2int(f):

3439 if f <= 0.0:

3440 return 0

3441 if f >= 1.0:

3442 return 255

3443 return int(255.0*f + 0.5)

3444

3445 return "#" + "".join(["%02x" % float2int(c) for c in (r, g, b)])

3446

3447 def escape(self, s):

3448 s = s.replace('\\', r'\\')

3449 s = s.replace('\n', r'\n')

3450 s = s.replace('\t', r'\t')

3451 s = s.replace('"', r'\"')

3452 return '"' + s + '"'

3453

3454 def write(self, s):

3455 self.fp.write(s)

3456

3457

3458

3459########################################################################

3460# Main program

3461

3462

3463def naturalJoin(values):

3464 if len(values) >= 2:

3465 return ', '.join(values[:-1]) + ' or ' + values[-1]

3466

3467 else:

3468 return ''.join(values)

3469

3470

3471def main(argv=sys.argv[1:]):

3472 """Main program."""

3473

3474 global totalMethod, timeFormat

3475

3476 formatNames = list(formats.keys())

3477 formatNames.sort()

3478

3479 themeNames = list(themes.keys())

3480 themeNames.sort()

3481

3482 labelNames = list(labels.keys())

3483 labelNames.sort()

3484

3485 argparser = argparse.ArgumentParser(

3486 usage="\n %(prog)s [options] [INPUT] ...")

3487 argparser.add_argument(

3488 '-o', '--output', metavar='OUTPUT',

3489 dest="output",

3490 help="output filename [stdout]")

3491 argparser.add_argument(

3492 '-n', '--node-thres', metavar='PERCENTAGE',

3493 type=float, dest="node_thres", default=0.5,

3494 help="eliminate nodes below this threshold [default: %(default)s]")

3495 argparser.add_argument(

3496 '-e', '--edge-thres', metavar='PERCENTAGE',

3497 type=float, dest="edge_thres", default=0.1,

3498 help="eliminate edges below this threshold [default: %(default)s]")

3499 argparser.add_argument(

3500 '-f', '--format',

3501 choices=formatNames,

3502 dest="format", default="prof",

3503 help="profile format: %s [default: %%(default)s]" % naturalJoin(formatNames))

3504 argparser.add_argument(

3505 '--total',

3506 choices=('callratios', 'callstacks'),

3507 dest="totalMethod", default=totalMethod,

3508 help="preferred method of calculating total time: callratios or callstacks (currently affects only perf format) [default: %(default)s]")

3509 argparser.add_argument(

3510 '-c', '--colormap',

3511 choices=themeNames,

3512 dest="theme", default="color",

3513 help="color map: %s [default: %%(default)s]" % naturalJoin(themeNames))

3514 argparser.add_argument(

3515 '-s', '--strip',

3516 action="store_true",

3517 dest="strip", default=False,

3518 help="strip function parameters, template parameters, and const modifiers from demangled C++ function names")

3519 argparser.add_argument(

3520 '--color-nodes-by-selftime',

3521 action="store_true",

3522 dest="color_nodes_by_selftime", default=False,

3523 help="color nodes by self time, rather than by total time (sum of self and descendants)")

3524 argparser.add_argument(

3525 '--colour-nodes-by-selftime',

3526 action="store_true",

3527 dest="color_nodes_by_selftime",

3528 help=argparse.SUPPRESS)

3529 argparser.add_argument(

3530 '-w', '--wrap',

3531 action="store_true",

3532 dest="wrap", default=False,

3533 help="wrap function names")

3534 argparser.add_argument(

3535 '--show-samples',

3536 action="store_true",

3537 dest="show_samples", default=False,

3538 help="show function samples")

3539 argparser.add_argument(

3540 '--time-format',

3541 default=timeFormat,

3542 help="format to use for showing time values [default: %(default)s]")

3543 argparser.add_argument(

3544 '--node-label', metavar='MEASURE',

3545 choices=labelNames,

3546 action='append',

3547 dest='node_labels',

3548 help="measurements to on show the node (can be specified multiple times): %s [default: %s]" % (

3549 naturalJoin(labelNames), ', '.join(defaultLabelNames)))

3550 # add option to show information on available entries ()

3551 argparser.add_argument(

3552 '--list-functions',

3553 dest="list_functions", default=None,

3554 help="""\

3555list functions available for selection in -z or -l, requires selector argument

3556( use '+' to select all).

3557Recall that the selector argument is used with Unix/Bash globbing/pattern matching,

3558and that entries are formatted '<pkg>:<linenum>:<function>'. When argument starts

3559with '%%', a dump of all available information is performed for selected entries,

3560 after removal of leading '%%'.

3561""")

3562 # add option to create subtree or show paths

3563 argparser.add_argument(

3564 '-z', '--root',

3565 dest="root", default="",

3566 help="prune call graph to show only descendants of specified root function")

3567 argparser.add_argument(

3568 '-l', '--leaf',

3569 dest="leaf", default="",

3570 help="prune call graph to show only ancestors of specified leaf function")

3571 argparser.add_argument(

3572 '--depth',

3573 type=int,

3574 dest="depth", default=-1,

3575 help="prune call graph to show only descendants or ancestors until specified depth")

3576 # add a new option to control skew of the colorization curve

3577 argparser.add_argument(

3578 '--skew',

3579 type=float, dest="theme_skew", default=1.0,

3580 help="skew the colorization curve. Values < 1.0 give more variety to lower percentages. Values > 1.0 give less variety to lower percentages")

3581 # add option for filtering by file path

3582 argparser.add_argument(

3583 '-p', '--path', action="append",

3584 dest="filter_paths",

3585 help="filter all modules not in a specified path")

3586 argparser.add_argument(

3587 '--compare',

3588 action="store_true",

3589 dest="compare", default=False,

3590 help="compare two graphs with almost identical structure. With this option two files should be provided."

3591 "gprof2dot.py [options] --compare [file1] [file2] ...")

3592 argparser.add_argument(

3593 '--compare-tolerance',

3594 type=float, dest="tolerance", default=0.001,

3595 help="tolerance threshold for node difference (default=0.001%%)."

3596 "If the difference is below this value the nodes are considered identical.")

3597 argparser.add_argument(

3598 '--compare-only-slower',

3599 action="store_true",

3600 dest="only_slower", default=False,

3601 help="display comparison only for function which are slower in second graph.")

3602 argparser.add_argument(

3603 '--compare-only-faster',

3604 action="store_true",

3605 dest="only_faster", default=False,

3606 help="display comparison only for function which are faster in second graph.")

3607 argparser.add_argument(

3608 '--compare-color-by-difference',

3609 action="store_true",

3610 dest="color_by_difference", default=False,

3611 help="color nodes based on the value of the difference. "

3612 "Nodes with the largest differences represent the hot spots.")

3613 argparser.add_argument('input', nargs='*', metavar='INPUT', help='input stats')

3614 options = argparser.parse_args(argv)

3615 args = options.input

3616

3617 if len(args) > 1 and options.format != 'pstats' and not options.compare:

3618 argparser.error('incorrect number of arguments')

3619

3620 try:

3621 theme = themes[options.theme]

3622 except KeyError:

3623 argparser.error('invalid colormap \'%s\'' % options.theme)

3624

3625 # set skew on the theme now that it has been picked.

3626 if options.theme_skew:

3627 theme.skew = options.theme_skew

3628

3629 totalMethod = options.totalMethod

3630 timeFormat = options.time_format

3631

3632 try:

3633 Format = formats[options.format]

3634 except KeyError:

3635 argparser.error('invalid format \'%s\'' % options.format)

3636

3637 if Format.stdinInput:

3638 if not args:

3639 fp = sys.stdin

3640 parser = Format(fp)

3641 elif options.compare:

3642 fp1 = open(args[0], 'rt', encoding='UTF-8')

3643 fp2 = open(args[1], 'rt', encoding='UTF-8')

3644 parser1 = Format(fp1)

3645 parser2 = Format(fp2)

3646 else:

3647 fp = open(args[0], 'rb')

3648 bom = fp.read(2)

3649 if bom == codecs.BOM_UTF16_LE:

3650 # Default on Windows PowerShell (https://github.com/jrfonseca/gprof2dot/issues/88)

3651 encoding = 'utf-16le'

3652 else:

3653 encoding = 'utf-8'

3654 fp.seek(0)

3655 fp = io.TextIOWrapper(fp, encoding=encoding)

3656 parser = Format(fp)

3657 elif Format.multipleInput:

3658 if not args:

3659 argparser.error('at least a file must be specified for %s input' % options.format)

3660 if options.compare:

3661 parser1 = Format(args[-2])

3662 parser2 = Format(args[-1])

3663 else:

3664 parser = Format(*args)

3665 else:

3666 if len(args) != 1:

3667 argparser.error('exactly one file must be specified for %s input' % options.format)

3668 parser = Format(args[0])

3669

3670 if options.compare:

3671 profile1 = parser1.parse()

3672 profile2 = parser2.parse()

3673 else:

3674 profile = parser.parse()

3675

3676 if options.output is None:

3677 output = open(sys.stdout.fileno(), mode='wt', encoding='UTF-8', closefd=False)

3678 else:

3679 output = open(options.output, 'wt', encoding='UTF-8')

3680

3681 dot = DotWriter(output)

3682 dot.strip = options.strip

3683 dot.wrap = options.wrap

3684

3685 labelNames = options.node_labels or defaultLabelNames

3686 dot.show_function_events = [labels[l] for l in labelNames]

3687 if options.show_samples:

3688 dot.show_function_events.append(SAMPLES)

3689

3690 if options.compare:

3691 profile1.prune(options.node_thres/100.0, options.edge_thres/100.0, options.filter_paths,

3692 options.color_nodes_by_selftime)

3693 profile2.prune(options.node_thres/100.0, options.edge_thres/100.0, options.filter_paths,

3694 options.color_nodes_by_selftime)

3695

3696 if options.root:

3697 profile1.prune_root(profile1.getFunctionIds(options.root), options.depth)

3698 profile2.prune_root(profile2.getFunctionIds(options.root), options.depth)

3699 else:

3700 profile.prune(options.node_thres/100.0, options.edge_thres/100.0, options.filter_paths,

3701 options.color_nodes_by_selftime)

3702 if options.root:

3703 rootIds = profile.getFunctionIds(options.root)

3704 if not rootIds:

3705 sys.stderr.write('root node ' + options.root + ' not found (might already be pruned : try -e0 -n0 flags)\n')

3706 sys.exit(1)

3707 profile.prune_root(rootIds, options.depth)

3708

3709 if options.list_functions:

3710 profile.printFunctionIds(selector=options.list_functions)

3711 sys.exit(0)

3712

3713 if options.leaf:

3714 leafIds = profile.getFunctionIds(options.leaf)

3715 if not leafIds:

3716 sys.stderr.write('leaf node ' + options.leaf + ' not found (maybe already pruned : try -e0 -n0 flags)\n')

3717 sys.exit(1)

3718 profile.prune_leaf(leafIds, options.depth)

3719

3720 if options.compare:

3721 dot.graphs_compare(profile1, profile2, theme, options)

3722 else:

3723 dot.graph(profile, theme)

3724

3725

3726if __name__ == '__main__':

3727 main()