Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/tensorflow/python/debug/cli/analyzer_cli.py: 10%
577 statements
« prev ^ index » next coverage.py v7.4.0, created at 2024-01-03 07:57 +0000
1# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ==============================================================================
15"""CLI Backend for the Analyzer Part of the Debugger.
17The analyzer performs post hoc analysis of dumped intermediate tensors and
18graph structure information from debugged Session.run() calls.
19"""
20import argparse
21import copy
22import re
25from tensorflow.python.debug.cli import cli_config
26from tensorflow.python.debug.cli import cli_shared
27from tensorflow.python.debug.cli import command_parser
28from tensorflow.python.debug.cli import debugger_cli_common
29from tensorflow.python.debug.cli import evaluator
30from tensorflow.python.debug.cli import ui_factory
31from tensorflow.python.debug.lib import debug_graphs
32from tensorflow.python.debug.lib import source_utils
34RL = debugger_cli_common.RichLine
36# String constants for the depth-dependent hanging indent at the beginning
37# of each line.
38HANG_UNFINISHED = "| " # Used for unfinished recursion depths.
39HANG_FINISHED = " "
40HANG_SUFFIX = "|- "
42# String constant for displaying depth and op type.
43DEPTH_TEMPLATE = "(%d) "
44OP_TYPE_TEMPLATE = "[%s] "
46# String constants for control inputs/outputs, etc.
47CTRL_LABEL = "(Ctrl) "
48ELLIPSIS = "..."
50SORT_TENSORS_BY_TIMESTAMP = "timestamp"
51SORT_TENSORS_BY_DUMP_SIZE = "dump_size"
52SORT_TENSORS_BY_OP_TYPE = "op_type"
53SORT_TENSORS_BY_TENSOR_NAME = "tensor_name"
56def _add_main_menu(output,
57 node_name=None,
58 enable_list_tensors=True,
59 enable_node_info=True,
60 enable_print_tensor=True,
61 enable_list_inputs=True,
62 enable_list_outputs=True):
63 """Generate main menu for the screen output from a command.
65 Args:
66 output: (debugger_cli_common.RichTextLines) the output object to modify.
67 node_name: (str or None) name of the node involved (if any). If None,
68 the menu items node_info, list_inputs and list_outputs will be
69 automatically disabled, overriding the values of arguments
70 enable_node_info, enable_list_inputs and enable_list_outputs.
71 enable_list_tensors: (bool) whether the list_tensor menu item will be
72 enabled.
73 enable_node_info: (bool) whether the node_info item will be enabled.
74 enable_print_tensor: (bool) whether the print_tensor item will be enabled.
75 enable_list_inputs: (bool) whether the item list_inputs will be enabled.
76 enable_list_outputs: (bool) whether the item list_outputs will be enabled.
77 """
79 menu = debugger_cli_common.Menu()
81 menu.append(
82 debugger_cli_common.MenuItem(
83 "list_tensors", "list_tensors", enabled=enable_list_tensors))
85 if node_name:
86 menu.append(
87 debugger_cli_common.MenuItem(
88 "node_info",
89 "node_info -a -d -t %s" % node_name,
90 enabled=enable_node_info))
91 menu.append(
92 debugger_cli_common.MenuItem(
93 "print_tensor",
94 "print_tensor %s" % node_name,
95 enabled=enable_print_tensor))
96 menu.append(
97 debugger_cli_common.MenuItem(
98 "list_inputs",
99 "list_inputs -c -r %s" % node_name,
100 enabled=enable_list_inputs))
101 menu.append(
102 debugger_cli_common.MenuItem(
103 "list_outputs",
104 "list_outputs -c -r %s" % node_name,
105 enabled=enable_list_outputs))
106 else:
107 menu.append(
108 debugger_cli_common.MenuItem(
109 "node_info", None, enabled=False))
110 menu.append(
111 debugger_cli_common.MenuItem("print_tensor", None, enabled=False))
112 menu.append(
113 debugger_cli_common.MenuItem("list_inputs", None, enabled=False))
114 menu.append(
115 debugger_cli_common.MenuItem("list_outputs", None, enabled=False))
117 menu.append(
118 debugger_cli_common.MenuItem("run_info", "run_info"))
119 menu.append(
120 debugger_cli_common.MenuItem("help", "help"))
122 output.annotations[debugger_cli_common.MAIN_MENU_KEY] = menu
125class DebugAnalyzer(object):
126 """Analyzer for debug data from dump directories."""
128 _TIMESTAMP_COLUMN_HEAD = "t (ms)"
129 _DUMP_SIZE_COLUMN_HEAD = "Size (B)"
130 _OP_TYPE_COLUMN_HEAD = "Op type"
131 _TENSOR_NAME_COLUMN_HEAD = "Tensor name"
133 # Op types to be omitted when generating descriptions of graph structure.
134 _GRAPH_STRUCT_OP_TYPE_DENYLIST = ("_Send", "_Recv", "_HostSend", "_HostRecv",
135 "_Retval")
137 def __init__(self, debug_dump, config):
138 """DebugAnalyzer constructor.
140 Args:
141 debug_dump: A DebugDumpDir object.
142 config: A `cli_config.CLIConfig` object that carries user-facing
143 configurations.
144 """
146 self._debug_dump = debug_dump
147 self._evaluator = evaluator.ExpressionEvaluator(self._debug_dump)
149 # Initialize tensor filters state.
150 self._tensor_filters = {}
152 self._build_argument_parsers(config)
153 config.set_callback("graph_recursion_depth",
154 self._build_argument_parsers)
156 # TODO(cais): Implement list_nodes.
158 def _build_argument_parsers(self, config):
159 """Build argument parsers for DebugAnalayzer.
161 Args:
162 config: A `cli_config.CLIConfig` object.
164 Returns:
165 A dict mapping command handler name to `ArgumentParser` instance.
166 """
167 # Argument parsers for command handlers.
168 self._arg_parsers = {}
170 # Parser for list_tensors.
171 ap = argparse.ArgumentParser(
172 description="List dumped intermediate tensors.",
173 usage=argparse.SUPPRESS)
174 ap.add_argument(
175 "-f",
176 "--tensor_filter",
177 dest="tensor_filter",
178 type=str,
179 default="",
180 help="List only Tensors passing the filter of the specified name")
181 ap.add_argument(
182 "-fenn",
183 "--filter_exclude_node_names",
184 dest="filter_exclude_node_names",
185 type=str,
186 default="",
187 help="When applying the tensor filter, exclude node with names "
188 "matching the regular expression. Applicable only if --tensor_filter "
189 "or -f is used.")
190 ap.add_argument(
191 "-n",
192 "--node_name_filter",
193 dest="node_name_filter",
194 type=str,
195 default="",
196 help="filter node name by regex.")
197 ap.add_argument(
198 "-t",
199 "--op_type_filter",
200 dest="op_type_filter",
201 type=str,
202 default="",
203 help="filter op type by regex.")
204 ap.add_argument(
205 "-s",
206 "--sort_by",
207 dest="sort_by",
208 type=str,
209 default=SORT_TENSORS_BY_TIMESTAMP,
210 help=("the field to sort the data by: (%s | %s | %s | %s)" %
211 (SORT_TENSORS_BY_TIMESTAMP, SORT_TENSORS_BY_DUMP_SIZE,
212 SORT_TENSORS_BY_OP_TYPE, SORT_TENSORS_BY_TENSOR_NAME)))
213 ap.add_argument(
214 "-r",
215 "--reverse",
216 dest="reverse",
217 action="store_true",
218 help="sort the data in reverse (descending) order")
219 self._arg_parsers["list_tensors"] = ap
221 # Parser for node_info.
222 ap = argparse.ArgumentParser(
223 description="Show information about a node.", usage=argparse.SUPPRESS)
224 ap.add_argument(
225 "node_name",
226 type=str,
227 help="Name of the node or an associated tensor, e.g., "
228 "hidden1/Wx_plus_b/MatMul, hidden1/Wx_plus_b/MatMul:0")
229 ap.add_argument(
230 "-a",
231 "--attributes",
232 dest="attributes",
233 action="store_true",
234 help="Also list attributes of the node.")
235 ap.add_argument(
236 "-d",
237 "--dumps",
238 dest="dumps",
239 action="store_true",
240 help="Also list dumps available from the node.")
241 ap.add_argument(
242 "-t",
243 "--traceback",
244 dest="traceback",
245 action="store_true",
246 help="Also include the traceback of the node's creation "
247 "(if available in Python).")
248 self._arg_parsers["node_info"] = ap
250 # Parser for list_inputs.
251 ap = argparse.ArgumentParser(
252 description="Show inputs to a node.", usage=argparse.SUPPRESS)
253 ap.add_argument(
254 "node_name",
255 type=str,
256 help="Name of the node or an output tensor from the node, e.g., "
257 "hidden1/Wx_plus_b/MatMul, hidden1/Wx_plus_b/MatMul:0")
258 ap.add_argument(
259 "-c", "--control", action="store_true", help="Include control inputs.")
260 ap.add_argument(
261 "-d",
262 "--depth",
263 dest="depth",
264 type=int,
265 default=config.get("graph_recursion_depth"),
266 help="Maximum depth of recursion used when showing the input tree.")
267 ap.add_argument(
268 "-r",
269 "--recursive",
270 dest="recursive",
271 action="store_true",
272 help="Show inputs to the node recursively, i.e., the input tree.")
273 ap.add_argument(
274 "-t",
275 "--op_type",
276 action="store_true",
277 help="Show op types of input nodes.")
278 self._arg_parsers["list_inputs"] = ap
280 # Parser for list_outputs.
281 ap = argparse.ArgumentParser(
282 description="Show the nodes that receive the outputs of given node.",
283 usage=argparse.SUPPRESS)
284 ap.add_argument(
285 "node_name",
286 type=str,
287 help="Name of the node or an output tensor from the node, e.g., "
288 "hidden1/Wx_plus_b/MatMul, hidden1/Wx_plus_b/MatMul:0")
289 ap.add_argument(
290 "-c", "--control", action="store_true", help="Include control inputs.")
291 ap.add_argument(
292 "-d",
293 "--depth",
294 dest="depth",
295 type=int,
296 default=config.get("graph_recursion_depth"),
297 help="Maximum depth of recursion used when showing the output tree.")
298 ap.add_argument(
299 "-r",
300 "--recursive",
301 dest="recursive",
302 action="store_true",
303 help="Show recipients of the node recursively, i.e., the output "
304 "tree.")
305 ap.add_argument(
306 "-t",
307 "--op_type",
308 action="store_true",
309 help="Show op types of recipient nodes.")
310 self._arg_parsers["list_outputs"] = ap
312 # Parser for print_tensor.
313 self._arg_parsers["print_tensor"] = (
314 command_parser.get_print_tensor_argparser(
315 "Print the value of a dumped tensor."))
317 # Parser for print_source.
318 ap = argparse.ArgumentParser(
319 description="Print a Python source file with overlaid debug "
320 "information, including the nodes (ops) or Tensors created at the "
321 "source lines.",
322 usage=argparse.SUPPRESS)
323 ap.add_argument(
324 "source_file_path",
325 type=str,
326 help="Path to the source file.")
327 ap.add_argument(
328 "-t",
329 "--tensors",
330 dest="tensors",
331 action="store_true",
332 help="Label lines with dumped Tensors, instead of ops.")
333 ap.add_argument(
334 "-m",
335 "--max_elements_per_line",
336 type=int,
337 default=10,
338 help="Maximum number of elements (ops or Tensors) to show per source "
339 "line.")
340 ap.add_argument(
341 "-b",
342 "--line_begin",
343 type=int,
344 default=1,
345 help="Print source beginning at line number (1-based.)")
346 self._arg_parsers["print_source"] = ap
348 # Parser for list_source.
349 ap = argparse.ArgumentParser(
350 description="List source files responsible for constructing nodes and "
351 "tensors present in the run().",
352 usage=argparse.SUPPRESS)
353 ap.add_argument(
354 "-p",
355 "--path_filter",
356 type=str,
357 default="",
358 help="Regular expression filter for file path.")
359 ap.add_argument(
360 "-n",
361 "--node_name_filter",
362 type=str,
363 default="",
364 help="Regular expression filter for node name.")
365 self._arg_parsers["list_source"] = ap
367 # Parser for eval.
368 ap = argparse.ArgumentParser(
369 description="""Evaluate an arbitrary expression. Can use tensor values
370 from the current debug dump. The debug tensor names should be enclosed
371 in pairs of backticks. Expressions with spaces should be enclosed in
372 a pair of double quotes or a pair of single quotes. By default, numpy
373 is imported as np and can be used in the expressions. E.g.,
374 1) eval np.argmax(`Softmax:0`),
375 2) eval 'np.sum(`Softmax:0`, axis=1)',
376 3) eval "np.matmul((`output/Identity:0`/`Softmax:0`).T, `Softmax:0`)".
377 """,
378 usage=argparse.SUPPRESS)
379 ap.add_argument(
380 "expression",
381 type=str,
382 help="""Expression to be evaluated.
383 1) in the simplest case, use <node_name>:<output_slot>, e.g.,
384 hidden_0/MatMul:0.
386 2) if the default debug op "DebugIdentity" is to be overridden, use
387 <node_name>:<output_slot>:<debug_op>, e.g.,
388 hidden_0/MatMul:0:DebugNumericSummary.
390 3) if the tensor of the same name exists on more than one device, use
391 <device_name>:<node_name>:<output_slot>[:<debug_op>], e.g.,
392 /job:worker/replica:0/task:0/gpu:0:hidden_0/MatMul:0
393 /job:worker/replica:0/task:2/cpu:0:hidden_0/MatMul:0:DebugNanCount.
395 4) if the tensor is executed multiple times in a given `Session.run`
396 call, specify the execution index with a 0-based integer enclose in a
397 pair of brackets at the end, e.g.,
398 RNN/tanh:0[0]
399 /job:worker/replica:0/task:0/gpu:0:RNN/tanh:0[0].""")
400 ap.add_argument(
401 "-a",
402 "--all",
403 dest="print_all",
404 action="store_true",
405 help="Print the tensor in its entirety, i.e., do not use ellipses "
406 "(may be slow for large results).")
407 ap.add_argument(
408 "-w",
409 "--write_path",
410 default="",
411 help="Path of the numpy file to write the evaluation result to, "
412 "using numpy.save()")
413 self._arg_parsers["eval"] = ap
415 def add_tensor_filter(self, filter_name, filter_callable):
416 """Add a tensor filter.
418 A tensor filter is a named callable of the signature:
419 filter_callable(dump_datum, tensor),
421 wherein dump_datum is an instance of debug_data.DebugTensorDatum carrying
422 metadata about the dumped tensor, including tensor name, timestamps, etc.
423 tensor is the value of the dumped tensor as an numpy.ndarray object.
424 The return value of the function is a bool.
425 This is the same signature as the input argument to
426 debug_data.DebugDumpDir.find().
428 Args:
429 filter_name: (str) name of the filter. Cannot be empty.
430 filter_callable: (callable) a filter function of the signature described
431 as above.
433 Raises:
434 ValueError: If filter_name is an empty str.
435 TypeError: If filter_name is not a str.
436 Or if filter_callable is not callable.
437 """
439 if not isinstance(filter_name, str):
440 raise TypeError("Input argument filter_name is expected to be str, "
441 "but is not.")
443 # Check that filter_name is not an empty str.
444 if not filter_name:
445 raise ValueError("Input argument filter_name cannot be empty.")
447 # Check that filter_callable is callable.
448 if not callable(filter_callable):
449 raise TypeError(
450 "Input argument filter_callable is expected to be callable, "
451 "but is not.")
453 self._tensor_filters[filter_name] = filter_callable
455 def get_tensor_filter(self, filter_name):
456 """Retrieve filter function by name.
458 Args:
459 filter_name: Name of the filter set during add_tensor_filter() call.
461 Returns:
462 The callable associated with the filter name.
464 Raises:
465 ValueError: If there is no tensor filter of the specified filter name.
466 """
468 if filter_name not in self._tensor_filters:
469 raise ValueError("There is no tensor filter named \"%s\"" % filter_name)
471 return self._tensor_filters[filter_name]
473 def get_help(self, handler_name):
474 return self._arg_parsers[handler_name].format_help()
476 def list_tensors(self, args, screen_info=None):
477 """Command handler for list_tensors.
479 List tensors dumped during debugged Session.run() call.
481 Args:
482 args: Command-line arguments, excluding the command prefix, as a list of
483 str.
484 screen_info: Optional dict input containing screen information such as
485 cols.
487 Returns:
488 Output text lines as a RichTextLines object.
490 Raises:
491 ValueError: If `--filter_exclude_node_names` is used without `-f` or
492 `--tensor_filter` being used.
493 """
495 # TODO(cais): Add annotations of substrings for dumped tensor names, to
496 # facilitate on-screen highlighting/selection of node names.
497 _ = screen_info
499 parsed = self._arg_parsers["list_tensors"].parse_args(args)
501 output = []
503 filter_strs = []
504 if parsed.op_type_filter:
505 op_type_regex = re.compile(parsed.op_type_filter)
506 filter_strs.append("Op type regex filter: \"%s\"" % parsed.op_type_filter)
507 else:
508 op_type_regex = None
510 if parsed.node_name_filter:
511 node_name_regex = re.compile(parsed.node_name_filter)
512 filter_strs.append("Node name regex filter: \"%s\"" %
513 parsed.node_name_filter)
514 else:
515 node_name_regex = None
517 output = debugger_cli_common.RichTextLines(filter_strs)
518 output.append("")
520 if parsed.tensor_filter:
521 try:
522 filter_callable = self.get_tensor_filter(parsed.tensor_filter)
523 except ValueError:
524 output = cli_shared.error("There is no tensor filter named \"%s\"." %
525 parsed.tensor_filter)
526 _add_main_menu(output, node_name=None, enable_list_tensors=False)
527 return output
529 data_to_show = self._debug_dump.find(
530 filter_callable,
531 exclude_node_names=parsed.filter_exclude_node_names)
532 else:
533 if parsed.filter_exclude_node_names:
534 raise ValueError(
535 "The flag --filter_exclude_node_names is valid only when "
536 "the flag -f or --tensor_filter is used.")
538 data_to_show = self._debug_dump.dumped_tensor_data
540 # TODO(cais): Implement filter by lambda on tensor value.
542 max_timestamp_width, max_dump_size_width, max_op_type_width = (
543 self._measure_tensor_list_column_widths(data_to_show))
545 # Sort the data.
546 data_to_show = self._sort_dump_data_by(
547 data_to_show, parsed.sort_by, parsed.reverse)
549 output.extend(
550 self._tensor_list_column_heads(parsed, max_timestamp_width,
551 max_dump_size_width, max_op_type_width))
553 dump_count = 0
554 for dump in data_to_show:
555 if node_name_regex and not node_name_regex.match(dump.node_name):
556 continue
558 if op_type_regex:
559 op_type = self._debug_dump.node_op_type(dump.node_name)
560 if not op_type_regex.match(op_type):
561 continue
563 rel_time = (dump.timestamp - self._debug_dump.t0) / 1000.0
564 dump_size_str = cli_shared.bytes_to_readable_str(dump.dump_size_bytes)
565 dumped_tensor_name = "%s:%d" % (dump.node_name, dump.output_slot)
566 op_type = self._debug_dump.node_op_type(dump.node_name)
568 line = "[%.3f]" % rel_time
569 line += " " * (max_timestamp_width - len(line))
570 line += dump_size_str
571 line += " " * (max_timestamp_width + max_dump_size_width - len(line))
572 line += op_type
573 line += " " * (max_timestamp_width + max_dump_size_width +
574 max_op_type_width - len(line))
575 line += dumped_tensor_name
577 output.append(
578 line,
579 font_attr_segs=[(
580 len(line) - len(dumped_tensor_name), len(line),
581 debugger_cli_common.MenuItem("", "pt %s" % dumped_tensor_name))])
582 dump_count += 1
584 if parsed.tensor_filter:
585 output.prepend([
586 "%d dumped tensor(s) passing filter \"%s\":" %
587 (dump_count, parsed.tensor_filter)
588 ])
589 else:
590 output.prepend(["%d dumped tensor(s):" % dump_count])
592 _add_main_menu(output, node_name=None, enable_list_tensors=False)
593 return output
595 def _measure_tensor_list_column_widths(self, data):
596 """Determine the maximum widths of the timestamp and op-type column.
598 This method assumes that data is sorted in the default order, i.e.,
599 by ascending timestamps.
601 Args:
602 data: (list of DebugTensorDaum) the data based on which the maximum
603 column widths will be determined.
605 Returns:
606 (int) maximum width of the timestamp column. 0 if data is empty.
607 (int) maximum width of the dump size column. 0 if data is empty.
608 (int) maximum width of the op type column. 0 if data is empty.
609 """
611 max_timestamp_width = 0
612 if data:
613 max_rel_time_ms = (data[-1].timestamp - self._debug_dump.t0) / 1000.0
614 max_timestamp_width = len("[%.3f] " % max_rel_time_ms) + 1
615 max_timestamp_width = max(max_timestamp_width,
616 len(self._TIMESTAMP_COLUMN_HEAD) + 1)
618 max_dump_size_width = 0
619 for dump in data:
620 dump_size_str = cli_shared.bytes_to_readable_str(dump.dump_size_bytes)
621 if len(dump_size_str) + 1 > max_dump_size_width:
622 max_dump_size_width = len(dump_size_str) + 1
623 max_dump_size_width = max(max_dump_size_width,
624 len(self._DUMP_SIZE_COLUMN_HEAD) + 1)
626 max_op_type_width = 0
627 for dump in data:
628 op_type = self._debug_dump.node_op_type(dump.node_name)
629 if len(op_type) + 1 > max_op_type_width:
630 max_op_type_width = len(op_type) + 1
631 max_op_type_width = max(max_op_type_width,
632 len(self._OP_TYPE_COLUMN_HEAD) + 1)
634 return max_timestamp_width, max_dump_size_width, max_op_type_width
636 def _sort_dump_data_by(self, data, sort_by, reverse):
637 """Sort a list of DebugTensorDatum in specified order.
639 Args:
640 data: (list of DebugTensorDatum) the data to be sorted.
641 sort_by: The field to sort data by.
642 reverse: (bool) Whether to use reversed (descending) order.
644 Returns:
645 (list of DebugTensorDatum) in sorted order.
647 Raises:
648 ValueError: given an invalid value of sort_by.
649 """
651 if sort_by == SORT_TENSORS_BY_TIMESTAMP:
652 return sorted(
653 data,
654 reverse=reverse,
655 key=lambda x: x.timestamp)
656 elif sort_by == SORT_TENSORS_BY_DUMP_SIZE:
657 return sorted(data, reverse=reverse, key=lambda x: x.dump_size_bytes)
658 elif sort_by == SORT_TENSORS_BY_OP_TYPE:
659 return sorted(
660 data,
661 reverse=reverse,
662 key=lambda x: self._debug_dump.node_op_type(x.node_name))
663 elif sort_by == SORT_TENSORS_BY_TENSOR_NAME:
664 return sorted(
665 data,
666 reverse=reverse,
667 key=lambda x: "%s:%d" % (x.node_name, x.output_slot))
668 else:
669 raise ValueError("Unsupported key to sort tensors by: %s" % sort_by)
671 def _tensor_list_column_heads(self, parsed, max_timestamp_width,
672 max_dump_size_width, max_op_type_width):
673 """Generate a line containing the column heads of the tensor list.
675 Args:
676 parsed: Parsed arguments (by argparse) of the list_tensors command.
677 max_timestamp_width: (int) maximum width of the timestamp column.
678 max_dump_size_width: (int) maximum width of the dump size column.
679 max_op_type_width: (int) maximum width of the op type column.
681 Returns:
682 A RichTextLines object.
683 """
685 base_command = "list_tensors"
686 if parsed.tensor_filter:
687 base_command += " -f %s" % parsed.tensor_filter
688 if parsed.op_type_filter:
689 base_command += " -t %s" % parsed.op_type_filter
690 if parsed.node_name_filter:
691 base_command += " -n %s" % parsed.node_name_filter
693 attr_segs = {0: []}
694 row = self._TIMESTAMP_COLUMN_HEAD
695 command = "%s -s %s" % (base_command, SORT_TENSORS_BY_TIMESTAMP)
696 if parsed.sort_by == SORT_TENSORS_BY_TIMESTAMP and not parsed.reverse:
697 command += " -r"
698 attr_segs[0].append(
699 (0, len(row), [debugger_cli_common.MenuItem(None, command), "bold"]))
700 row += " " * (max_timestamp_width - len(row))
702 prev_len = len(row)
703 row += self._DUMP_SIZE_COLUMN_HEAD
704 command = "%s -s %s" % (base_command, SORT_TENSORS_BY_DUMP_SIZE)
705 if parsed.sort_by == SORT_TENSORS_BY_DUMP_SIZE and not parsed.reverse:
706 command += " -r"
707 attr_segs[0].append((prev_len, len(row),
708 [debugger_cli_common.MenuItem(None, command), "bold"]))
709 row += " " * (max_dump_size_width + max_timestamp_width - len(row))
711 prev_len = len(row)
712 row += self._OP_TYPE_COLUMN_HEAD
713 command = "%s -s %s" % (base_command, SORT_TENSORS_BY_OP_TYPE)
714 if parsed.sort_by == SORT_TENSORS_BY_OP_TYPE and not parsed.reverse:
715 command += " -r"
716 attr_segs[0].append((prev_len, len(row),
717 [debugger_cli_common.MenuItem(None, command), "bold"]))
718 row += " " * (
719 max_op_type_width + max_dump_size_width + max_timestamp_width - len(row)
720 )
722 prev_len = len(row)
723 row += self._TENSOR_NAME_COLUMN_HEAD
724 command = "%s -s %s" % (base_command, SORT_TENSORS_BY_TENSOR_NAME)
725 if parsed.sort_by == SORT_TENSORS_BY_TENSOR_NAME and not parsed.reverse:
726 command += " -r"
727 attr_segs[0].append((prev_len, len(row),
728 [debugger_cli_common.MenuItem("", command), "bold"]))
729 row += " " * (
730 max_op_type_width + max_dump_size_width + max_timestamp_width - len(row)
731 )
733 return debugger_cli_common.RichTextLines([row], font_attr_segs=attr_segs)
735 def node_info(self, args, screen_info=None):
736 """Command handler for node_info.
738 Query information about a given node.
740 Args:
741 args: Command-line arguments, excluding the command prefix, as a list of
742 str.
743 screen_info: Optional dict input containing screen information such as
744 cols.
746 Returns:
747 Output text lines as a RichTextLines object.
748 """
750 # TODO(cais): Add annotation of substrings for node names, to facilitate
751 # on-screen highlighting/selection of node names.
752 _ = screen_info
754 parsed = self._arg_parsers["node_info"].parse_args(args)
756 # Get a node name, regardless of whether the input is a node name (without
757 # output slot attached) or a tensor name (with output slot attached).
758 node_name, unused_slot = debug_graphs.parse_node_or_tensor_name(
759 parsed.node_name)
761 if not self._debug_dump.node_exists(node_name):
762 output = cli_shared.error(
763 "There is no node named \"%s\" in the partition graphs" % node_name)
764 _add_main_menu(
765 output,
766 node_name=None,
767 enable_list_tensors=True,
768 enable_node_info=False,
769 enable_list_inputs=False,
770 enable_list_outputs=False)
771 return output
773 # TODO(cais): Provide UI glossary feature to explain to users what the
774 # term "partition graph" means and how it is related to TF graph objects
775 # in Python. The information can be along the line of:
776 # "A tensorflow graph defined in Python is stripped of unused ops
777 # according to the feeds and fetches and divided into a number of
778 # partition graphs that may be distributed among multiple devices and
779 # hosts. The partition graphs are what's actually executed by the C++
780 # runtime during a run() call."
782 lines = ["Node %s" % node_name]
783 font_attr_segs = {
784 0: [(len(lines[-1]) - len(node_name), len(lines[-1]), "bold")]
785 }
786 lines.append("")
787 lines.append(" Op: %s" % self._debug_dump.node_op_type(node_name))
788 lines.append(" Device: %s" % self._debug_dump.node_device(node_name))
789 output = debugger_cli_common.RichTextLines(
790 lines, font_attr_segs=font_attr_segs)
792 # List node inputs (non-control and control).
793 inputs = self._exclude_denylisted_ops(
794 self._debug_dump.node_inputs(node_name))
795 ctrl_inputs = self._exclude_denylisted_ops(
796 self._debug_dump.node_inputs(node_name, is_control=True))
797 output.extend(self._format_neighbors("input", inputs, ctrl_inputs))
799 # List node output recipients (non-control and control).
800 recs = self._exclude_denylisted_ops(
801 self._debug_dump.node_recipients(node_name))
802 ctrl_recs = self._exclude_denylisted_ops(
803 self._debug_dump.node_recipients(node_name, is_control=True))
804 output.extend(self._format_neighbors("recipient", recs, ctrl_recs))
806 # Optional: List attributes of the node.
807 if parsed.attributes:
808 output.extend(self._list_node_attributes(node_name))
810 # Optional: List dumps available from the node.
811 if parsed.dumps:
812 output.extend(self._list_node_dumps(node_name))
814 if parsed.traceback:
815 output.extend(self._render_node_traceback(node_name))
817 _add_main_menu(output, node_name=node_name, enable_node_info=False)
818 return output
820 def _exclude_denylisted_ops(self, node_names):
821 """Exclude all nodes whose op types are in _GRAPH_STRUCT_OP_TYPE_DENYLIST.
823 Args:
824 node_names: An iterable of node or graph element names.
826 Returns:
827 A list of node names that are not denylisted.
828 """
829 return [
830 node_name for node_name in node_names
831 if self._debug_dump.node_op_type(debug_graphs.get_node_name(node_name))
832 not in self._GRAPH_STRUCT_OP_TYPE_DENYLIST
833 ]
835 def _render_node_traceback(self, node_name):
836 """Render traceback of a node's creation in Python, if available.
838 Args:
839 node_name: (str) name of the node.
841 Returns:
842 A RichTextLines object containing the stack trace of the node's
843 construction.
844 """
846 lines = [RL(""), RL(""), RL("Traceback of node construction:", "bold")]
848 try:
849 node_stack = self._debug_dump.node_traceback(node_name)
850 for depth, (file_path, line, function_name, text) in enumerate(
851 node_stack):
852 lines.append("%d: %s" % (depth, file_path))
854 attribute = debugger_cli_common.MenuItem(
855 "", "ps %s -b %d" % (file_path, line)) if text else None
856 line_number_line = RL(" ")
857 line_number_line += RL("Line: %d" % line, attribute)
858 lines.append(line_number_line)
860 lines.append(" Function: %s" % function_name)
861 lines.append(" Text: " + (("\"%s\"" % text) if text else "None"))
862 lines.append("")
863 except KeyError:
864 lines.append("(Node unavailable in the loaded Python graph)")
865 except LookupError:
866 lines.append("(Unavailable because no Python graph has been loaded)")
868 return debugger_cli_common.rich_text_lines_from_rich_line_list(lines)
870 def list_inputs(self, args, screen_info=None):
871 """Command handler for inputs.
873 Show inputs to a given node.
875 Args:
876 args: Command-line arguments, excluding the command prefix, as a list of
877 str.
878 screen_info: Optional dict input containing screen information such as
879 cols.
881 Returns:
882 Output text lines as a RichTextLines object.
883 """
885 # Screen info not currently used by this handler. Include this line to
886 # mute pylint.
887 _ = screen_info
888 # TODO(cais): Use screen info to format the output lines more prettily,
889 # e.g., hanging indent of long node names.
891 parsed = self._arg_parsers["list_inputs"].parse_args(args)
893 output = self._list_inputs_or_outputs(
894 parsed.recursive,
895 parsed.node_name,
896 parsed.depth,
897 parsed.control,
898 parsed.op_type,
899 do_outputs=False)
901 node_name = debug_graphs.get_node_name(parsed.node_name)
902 _add_main_menu(output, node_name=node_name, enable_list_inputs=False)
904 return output
906 def print_tensor(self, args, screen_info=None):
907 """Command handler for print_tensor.
909 Print value of a given dumped tensor.
911 Args:
912 args: Command-line arguments, excluding the command prefix, as a list of
913 str.
914 screen_info: Optional dict input containing screen information such as
915 cols.
917 Returns:
918 Output text lines as a RichTextLines object.
919 """
921 parsed = self._arg_parsers["print_tensor"].parse_args(args)
923 np_printoptions = cli_shared.numpy_printoptions_from_screen_info(
924 screen_info)
926 # Determine if any range-highlighting is required.
927 highlight_options = cli_shared.parse_ranges_highlight(parsed.ranges)
929 tensor_name, tensor_slicing = (
930 command_parser.parse_tensor_name_with_slicing(parsed.tensor_name))
932 node_name, output_slot = debug_graphs.parse_node_or_tensor_name(tensor_name)
933 if (self._debug_dump.loaded_partition_graphs() and
934 not self._debug_dump.node_exists(node_name)):
935 output = cli_shared.error(
936 "Node \"%s\" does not exist in partition graphs" % node_name)
937 _add_main_menu(
938 output,
939 node_name=None,
940 enable_list_tensors=True,
941 enable_print_tensor=False)
942 return output
944 watch_keys = self._debug_dump.debug_watch_keys(node_name)
945 if output_slot is None:
946 output_slots = set()
947 for watch_key in watch_keys:
948 output_slots.add(int(watch_key.split(":")[1]))
950 if len(output_slots) == 1:
951 # There is only one dumped tensor from this node, so there is no
952 # ambiguity. Proceed to show the only dumped tensor.
953 output_slot = list(output_slots)[0]
954 else:
955 # There are more than one dumped tensors from this node. Indicate as
956 # such.
957 # TODO(cais): Provide an output screen with command links for
958 # convenience.
959 lines = [
960 "Node \"%s\" generated debug dumps from %s output slots:" %
961 (node_name, len(output_slots)),
962 "Please specify the output slot: %s:x." % node_name
963 ]
964 output = debugger_cli_common.RichTextLines(lines)
965 _add_main_menu(
966 output,
967 node_name=node_name,
968 enable_list_tensors=True,
969 enable_print_tensor=False)
970 return output
972 # Find debug dump data that match the tensor name (node name + output
973 # slot).
974 matching_data = []
975 for watch_key in watch_keys:
976 debug_tensor_data = self._debug_dump.watch_key_to_data(watch_key)
977 for datum in debug_tensor_data:
978 if datum.output_slot == output_slot:
979 matching_data.append(datum)
981 if not matching_data:
982 # No dump for this tensor.
983 output = cli_shared.error("Tensor \"%s\" did not generate any dumps." %
984 parsed.tensor_name)
985 elif len(matching_data) == 1:
986 # There is only one dump for this tensor.
987 if parsed.number <= 0:
988 output = cli_shared.format_tensor(
989 matching_data[0].get_tensor(),
990 matching_data[0].watch_key,
991 np_printoptions,
992 print_all=parsed.print_all,
993 tensor_slicing=tensor_slicing,
994 highlight_options=highlight_options,
995 include_numeric_summary=parsed.numeric_summary,
996 write_path=parsed.write_path)
997 else:
998 output = cli_shared.error(
999 "Invalid number (%d) for tensor %s, which generated one dump." %
1000 (parsed.number, parsed.tensor_name))
1002 _add_main_menu(output, node_name=node_name, enable_print_tensor=False)
1003 else:
1004 # There are more than one dumps for this tensor.
1005 if parsed.number < 0:
1006 lines = [
1007 "Tensor \"%s\" generated %d dumps:" % (parsed.tensor_name,
1008 len(matching_data))
1009 ]
1010 font_attr_segs = {}
1012 for i, datum in enumerate(matching_data):
1013 rel_time = (datum.timestamp - self._debug_dump.t0) / 1000.0
1014 lines.append("#%d [%.3f ms] %s" % (i, rel_time, datum.watch_key))
1015 command = "print_tensor %s -n %d" % (parsed.tensor_name, i)
1016 font_attr_segs[len(lines) - 1] = [(
1017 len(lines[-1]) - len(datum.watch_key), len(lines[-1]),
1018 debugger_cli_common.MenuItem(None, command))]
1020 lines.append("")
1021 lines.append(
1022 "You can use the -n (--number) flag to specify which dump to "
1023 "print.")
1024 lines.append("For example:")
1025 lines.append(" print_tensor %s -n 0" % parsed.tensor_name)
1027 output = debugger_cli_common.RichTextLines(
1028 lines, font_attr_segs=font_attr_segs)
1029 elif parsed.number >= len(matching_data):
1030 output = cli_shared.error(
1031 "Specified number (%d) exceeds the number of available dumps "
1032 "(%d) for tensor %s" %
1033 (parsed.number, len(matching_data), parsed.tensor_name))
1034 else:
1035 output = cli_shared.format_tensor(
1036 matching_data[parsed.number].get_tensor(),
1037 matching_data[parsed.number].watch_key + " (dump #%d)" %
1038 parsed.number,
1039 np_printoptions,
1040 print_all=parsed.print_all,
1041 tensor_slicing=tensor_slicing,
1042 highlight_options=highlight_options,
1043 write_path=parsed.write_path)
1044 _add_main_menu(output, node_name=node_name, enable_print_tensor=False)
1046 return output
1048 def list_outputs(self, args, screen_info=None):
1049 """Command handler for inputs.
1051 Show inputs to a given node.
1053 Args:
1054 args: Command-line arguments, excluding the command prefix, as a list of
1055 str.
1056 screen_info: Optional dict input containing screen information such as
1057 cols.
1059 Returns:
1060 Output text lines as a RichTextLines object.
1061 """
1063 # Screen info not currently used by this handler. Include this line to
1064 # mute pylint.
1065 _ = screen_info
1066 # TODO(cais): Use screen info to format the output lines more prettily,
1067 # e.g., hanging indent of long node names.
1069 parsed = self._arg_parsers["list_outputs"].parse_args(args)
1071 output = self._list_inputs_or_outputs(
1072 parsed.recursive,
1073 parsed.node_name,
1074 parsed.depth,
1075 parsed.control,
1076 parsed.op_type,
1077 do_outputs=True)
1079 node_name = debug_graphs.get_node_name(parsed.node_name)
1080 _add_main_menu(output, node_name=node_name, enable_list_outputs=False)
1082 return output
1084 def evaluate_expression(self, args, screen_info=None):
1085 parsed = self._arg_parsers["eval"].parse_args(args)
1087 eval_res = self._evaluator.evaluate(parsed.expression)
1089 np_printoptions = cli_shared.numpy_printoptions_from_screen_info(
1090 screen_info)
1091 return cli_shared.format_tensor(
1092 eval_res,
1093 "from eval of expression '%s'" % parsed.expression,
1094 np_printoptions,
1095 print_all=parsed.print_all,
1096 include_numeric_summary=True,
1097 write_path=parsed.write_path)
1099 def _reconstruct_print_source_command(self,
1100 parsed,
1101 line_begin,
1102 max_elements_per_line_increase=0):
1103 return "ps %s %s -b %d -m %d" % (
1104 parsed.source_file_path, "-t" if parsed.tensors else "", line_begin,
1105 parsed.max_elements_per_line + max_elements_per_line_increase)
1107 def print_source(self, args, screen_info=None):
1108 """Print the content of a source file."""
1109 del screen_info # Unused.
1111 parsed = self._arg_parsers["print_source"].parse_args(args)
1113 source_annotation = source_utils.annotate_source(
1114 self._debug_dump,
1115 parsed.source_file_path,
1116 do_dumped_tensors=parsed.tensors)
1118 source_lines, line_num_width = source_utils.load_source(
1119 parsed.source_file_path)
1121 labeled_source_lines = []
1122 actual_initial_scroll_target = 0
1123 for i, line in enumerate(source_lines):
1124 annotated_line = RL("L%d" % (i + 1), cli_shared.COLOR_YELLOW)
1125 annotated_line += " " * (line_num_width - len(annotated_line))
1126 annotated_line += line
1127 labeled_source_lines.append(annotated_line)
1129 if i + 1 == parsed.line_begin:
1130 actual_initial_scroll_target = len(labeled_source_lines) - 1
1132 if i + 1 in source_annotation:
1133 sorted_elements = sorted(source_annotation[i + 1])
1134 for k, element in enumerate(sorted_elements):
1135 if k >= parsed.max_elements_per_line:
1136 omitted_info_line = RL(" (... Omitted %d of %d %s ...) " % (
1137 len(sorted_elements) - parsed.max_elements_per_line,
1138 len(sorted_elements),
1139 "tensor(s)" if parsed.tensors else "op(s)"))
1140 omitted_info_line += RL(
1141 "+5",
1142 debugger_cli_common.MenuItem(
1143 None,
1144 self._reconstruct_print_source_command(
1145 parsed, i + 1, max_elements_per_line_increase=5)))
1146 labeled_source_lines.append(omitted_info_line)
1147 break
1149 label = RL(" " * 4)
1150 if self._debug_dump.debug_watch_keys(
1151 debug_graphs.get_node_name(element)):
1152 attribute = debugger_cli_common.MenuItem("", "pt %s" % element)
1153 else:
1154 attribute = cli_shared.COLOR_BLUE
1156 label += RL(element, attribute)
1157 labeled_source_lines.append(label)
1159 output = debugger_cli_common.rich_text_lines_from_rich_line_list(
1160 labeled_source_lines,
1161 annotations={debugger_cli_common.INIT_SCROLL_POS_KEY:
1162 actual_initial_scroll_target})
1163 _add_main_menu(output, node_name=None)
1164 return output
1166 def _make_source_table(self, source_list, is_tf_py_library):
1167 """Make a table summarizing the source files that create nodes and tensors.
1169 Args:
1170 source_list: List of source files and related information as a list of
1171 tuples (file_path, is_tf_library, num_nodes, num_tensors, num_dumps,
1172 first_line).
1173 is_tf_py_library: (`bool`) whether this table is for files that belong
1174 to the TensorFlow Python library.
1176 Returns:
1177 The table as a `debugger_cli_common.RichTextLines` object.
1178 """
1179 path_head = "Source file path"
1180 num_nodes_head = "#(nodes)"
1181 num_tensors_head = "#(tensors)"
1182 num_dumps_head = "#(tensor dumps)"
1184 if is_tf_py_library:
1185 # Use color to mark files that are guessed to belong to TensorFlow Python
1186 # library.
1187 color = cli_shared.COLOR_GRAY
1188 lines = [RL("TensorFlow Python library file(s):", color)]
1189 else:
1190 color = cli_shared.COLOR_WHITE
1191 lines = [RL("File(s) outside TensorFlow Python library:", color)]
1193 if not source_list:
1194 lines.append(RL("[No files.]"))
1195 lines.append(RL())
1196 return debugger_cli_common.rich_text_lines_from_rich_line_list(lines)
1198 path_column_width = max(
1199 max(len(item[0]) for item in source_list), len(path_head)) + 1
1200 num_nodes_column_width = max(
1201 max(len(str(item[2])) for item in source_list),
1202 len(num_nodes_head)) + 1
1203 num_tensors_column_width = max(
1204 max(len(str(item[3])) for item in source_list),
1205 len(num_tensors_head)) + 1
1207 head = RL(path_head + " " * (path_column_width - len(path_head)), color)
1208 head += RL(num_nodes_head + " " * (
1209 num_nodes_column_width - len(num_nodes_head)), color)
1210 head += RL(num_tensors_head + " " * (
1211 num_tensors_column_width - len(num_tensors_head)), color)
1212 head += RL(num_dumps_head, color)
1214 lines.append(head)
1216 for (file_path, _, num_nodes, num_tensors, num_dumps,
1217 first_line_num) in source_list:
1218 path_attributes = [color]
1219 if source_utils.is_extension_uncompiled_python_source(file_path):
1220 path_attributes.append(
1221 debugger_cli_common.MenuItem(None, "ps %s -b %d" %
1222 (file_path, first_line_num)))
1224 line = RL(file_path, path_attributes)
1225 line += " " * (path_column_width - len(line))
1226 line += RL(
1227 str(num_nodes) + " " * (num_nodes_column_width - len(str(num_nodes))),
1228 color)
1229 line += RL(
1230 str(num_tensors) + " " *
1231 (num_tensors_column_width - len(str(num_tensors))), color)
1232 line += RL(str(num_dumps), color)
1233 lines.append(line)
1234 lines.append(RL())
1236 return debugger_cli_common.rich_text_lines_from_rich_line_list(lines)
1238 def list_source(self, args, screen_info=None):
1239 """List Python source files that constructed nodes and tensors."""
1240 del screen_info # Unused.
1242 parsed = self._arg_parsers["list_source"].parse_args(args)
1243 source_list = source_utils.list_source_files_against_dump(
1244 self._debug_dump,
1245 path_regex_allowlist=parsed.path_filter,
1246 node_name_regex_allowlist=parsed.node_name_filter)
1248 top_lines = [
1249 RL("List of source files that created nodes in this run", "bold")]
1250 if parsed.path_filter:
1251 top_lines.append(
1252 RL("File path regex filter: \"%s\"" % parsed.path_filter))
1253 if parsed.node_name_filter:
1254 top_lines.append(
1255 RL("Node name regex filter: \"%s\"" % parsed.node_name_filter))
1256 top_lines.append(RL())
1257 output = debugger_cli_common.rich_text_lines_from_rich_line_list(top_lines)
1258 if not source_list:
1259 output.append("[No source file information.]")
1260 return output
1262 output.extend(self._make_source_table(
1263 [item for item in source_list if not item[1]], False))
1264 output.extend(self._make_source_table(
1265 [item for item in source_list if item[1]], True))
1266 _add_main_menu(output, node_name=None)
1267 return output
1269 def _list_inputs_or_outputs(self,
1270 recursive,
1271 node_name,
1272 depth,
1273 control,
1274 op_type,
1275 do_outputs=False):
1276 """Helper function used by list_inputs and list_outputs.
1278 Format a list of lines to display the inputs or output recipients of a
1279 given node.
1281 Args:
1282 recursive: Whether the listing is to be done recursively, as a boolean.
1283 node_name: The name of the node in question, as a str.
1284 depth: Maximum recursion depth, applies only if recursive == True, as an
1285 int.
1286 control: Whether control inputs or control recipients are included, as a
1287 boolean.
1288 op_type: Whether the op types of the nodes are to be included, as a
1289 boolean.
1290 do_outputs: Whether recipients, instead of input nodes are to be
1291 listed, as a boolean.
1293 Returns:
1294 Input or recipient tree formatted as a RichTextLines object.
1295 """
1297 if do_outputs:
1298 tracker = self._debug_dump.node_recipients
1299 type_str = "Recipients of"
1300 short_type_str = "recipients"
1301 else:
1302 tracker = self._debug_dump.node_inputs
1303 type_str = "Inputs to"
1304 short_type_str = "inputs"
1306 lines = []
1307 font_attr_segs = {}
1309 # Check if this is a tensor name, instead of a node name.
1310 node_name, _ = debug_graphs.parse_node_or_tensor_name(node_name)
1312 # Check if node exists.
1313 if not self._debug_dump.node_exists(node_name):
1314 return cli_shared.error(
1315 "There is no node named \"%s\" in the partition graphs" % node_name)
1317 if recursive:
1318 max_depth = depth
1319 else:
1320 max_depth = 1
1322 if control:
1323 include_ctrls_str = ", control %s included" % short_type_str
1324 else:
1325 include_ctrls_str = ""
1327 line = "%s node \"%s\"" % (type_str, node_name)
1328 font_attr_segs[0] = [(len(line) - 1 - len(node_name), len(line) - 1, "bold")
1329 ]
1330 lines.append(line + " (Depth limit = %d%s):" % (max_depth, include_ctrls_str
1331 ))
1333 command_template = "lo -c -r %s" if do_outputs else "li -c -r %s"
1334 self._dfs_from_node(
1335 lines,
1336 font_attr_segs,
1337 node_name,
1338 tracker,
1339 max_depth,
1340 1, [],
1341 control,
1342 op_type,
1343 command_template=command_template)
1345 # Include legend.
1346 lines.append("")
1347 lines.append("Legend:")
1348 lines.append(" (d): recursion depth = d.")
1350 if control:
1351 lines.append(" (Ctrl): Control input.")
1352 if op_type:
1353 lines.append(" [Op]: Input node has op type Op.")
1355 # TODO(cais): Consider appending ":0" at the end of 1st outputs of nodes.
1357 return debugger_cli_common.RichTextLines(
1358 lines, font_attr_segs=font_attr_segs)
1360 def _dfs_from_node(self,
1361 lines,
1362 attr_segs,
1363 node_name,
1364 tracker,
1365 max_depth,
1366 depth,
1367 unfinished,
1368 include_control=False,
1369 show_op_type=False,
1370 command_template=None):
1371 """Perform depth-first search (DFS) traversal of a node's input tree.
1373 It recursively tracks the inputs (or output recipients) of the node called
1374 node_name, and append these inputs (or output recipients) to a list of text
1375 lines (lines) with proper indentation that reflects the recursion depth,
1376 together with some formatting attributes (to attr_segs). The formatting
1377 attributes can include command shortcuts, for example.
1379 Args:
1380 lines: Text lines to append to, as a list of str.
1381 attr_segs: (dict) Attribute segments dictionary to append to.
1382 node_name: Name of the node, as a str. This arg is updated during the
1383 recursion.
1384 tracker: A callable that takes one str as the node name input and
1385 returns a list of str as the inputs/outputs.
1386 This makes it this function general enough to be used with both
1387 node-input and node-output tracking.
1388 max_depth: Maximum recursion depth, as an int.
1389 depth: Current recursion depth. This arg is updated during the
1390 recursion.
1391 unfinished: A stack of unfinished recursion depths, as a list of int.
1392 include_control: Whether control dependencies are to be included as
1393 inputs (and marked as such).
1394 show_op_type: Whether op type of the input nodes are to be displayed
1395 alongside the nodes' names.
1396 command_template: (str) Template for command shortcut of the node names.
1397 """
1399 # Make a shallow copy of the list because it may be extended later.
1400 all_inputs = self._exclude_denylisted_ops(
1401 copy.copy(tracker(node_name, is_control=False)))
1402 is_ctrl = [False] * len(all_inputs)
1403 if include_control:
1404 # Sort control inputs or recipients in alphabetical order of the node
1405 # names.
1406 ctrl_inputs = self._exclude_denylisted_ops(
1407 sorted(tracker(node_name, is_control=True)))
1408 all_inputs.extend(ctrl_inputs)
1409 is_ctrl.extend([True] * len(ctrl_inputs))
1411 if not all_inputs:
1412 if depth == 1:
1413 lines.append(" [None]")
1415 return
1417 unfinished.append(depth)
1419 # Create depth-dependent hanging indent for the line.
1420 hang = ""
1421 for k in range(depth):
1422 if k < depth - 1:
1423 if k + 1 in unfinished:
1424 hang += HANG_UNFINISHED
1425 else:
1426 hang += HANG_FINISHED
1427 else:
1428 hang += HANG_SUFFIX
1430 if all_inputs and depth > max_depth:
1431 lines.append(hang + ELLIPSIS)
1432 unfinished.pop()
1433 return
1435 hang += DEPTH_TEMPLATE % depth
1437 for i, inp in enumerate(all_inputs):
1438 op_type = self._debug_dump.node_op_type(debug_graphs.get_node_name(inp))
1439 if op_type in self._GRAPH_STRUCT_OP_TYPE_DENYLIST:
1440 continue
1442 if is_ctrl[i]:
1443 ctrl_str = CTRL_LABEL
1444 else:
1445 ctrl_str = ""
1447 op_type_str = ""
1448 if show_op_type:
1449 op_type_str = OP_TYPE_TEMPLATE % op_type
1451 if i == len(all_inputs) - 1:
1452 unfinished.pop()
1454 line = hang + ctrl_str + op_type_str + inp
1455 lines.append(line)
1456 if command_template:
1457 attr_segs[len(lines) - 1] = [(
1458 len(line) - len(inp), len(line),
1459 debugger_cli_common.MenuItem(None, command_template % inp))]
1461 # Recursive call.
1462 # The input's/output's name can be a tensor name, in the case of node
1463 # with >1 output slots.
1464 inp_node_name, _ = debug_graphs.parse_node_or_tensor_name(inp)
1465 self._dfs_from_node(
1466 lines,
1467 attr_segs,
1468 inp_node_name,
1469 tracker,
1470 max_depth,
1471 depth + 1,
1472 unfinished,
1473 include_control=include_control,
1474 show_op_type=show_op_type,
1475 command_template=command_template)
1477 def _format_neighbors(self, neighbor_type, non_ctrls, ctrls):
1478 """List neighbors (inputs or recipients) of a node.
1480 Args:
1481 neighbor_type: ("input" | "recipient")
1482 non_ctrls: Non-control neighbor node names, as a list of str.
1483 ctrls: Control neighbor node names, as a list of str.
1485 Returns:
1486 A RichTextLines object.
1487 """
1489 # TODO(cais): Return RichTextLines instead, to allow annotation of node
1490 # names.
1491 lines = []
1492 font_attr_segs = {}
1494 lines.append("")
1495 lines.append(" %d %s(s) + %d control %s(s):" %
1496 (len(non_ctrls), neighbor_type, len(ctrls), neighbor_type))
1497 lines.append(" %d %s(s):" % (len(non_ctrls), neighbor_type))
1498 for non_ctrl in non_ctrls:
1499 line = " [%s] %s" % (self._debug_dump.node_op_type(non_ctrl),
1500 non_ctrl)
1501 lines.append(line)
1502 font_attr_segs[len(lines) - 1] = [(
1503 len(line) - len(non_ctrl), len(line),
1504 debugger_cli_common.MenuItem(None, "ni -a -d -t %s" % non_ctrl))]
1506 if ctrls:
1507 lines.append("")
1508 lines.append(" %d control %s(s):" % (len(ctrls), neighbor_type))
1509 for ctrl in ctrls:
1510 line = " [%s] %s" % (self._debug_dump.node_op_type(ctrl), ctrl)
1511 lines.append(line)
1512 font_attr_segs[len(lines) - 1] = [(
1513 len(line) - len(ctrl), len(line),
1514 debugger_cli_common.MenuItem(None, "ni -a -d -t %s" % ctrl))]
1516 return debugger_cli_common.RichTextLines(
1517 lines, font_attr_segs=font_attr_segs)
1519 def _list_node_attributes(self, node_name):
1520 """List neighbors (inputs or recipients) of a node.
1522 Args:
1523 node_name: Name of the node of which the attributes are to be listed.
1525 Returns:
1526 A RichTextLines object.
1527 """
1529 lines = []
1530 lines.append("")
1531 lines.append("Node attributes:")
1533 attrs = self._debug_dump.node_attributes(node_name)
1534 for attr_key in attrs:
1535 lines.append(" %s:" % attr_key)
1536 attr_val_str = repr(attrs[attr_key]).strip().replace("\n", " ")
1537 lines.append(" %s" % attr_val_str)
1538 lines.append("")
1540 return debugger_cli_common.RichTextLines(lines)
1542 def _list_node_dumps(self, node_name):
1543 """List dumped tensor data from a node.
1545 Args:
1546 node_name: Name of the node of which the attributes are to be listed.
1548 Returns:
1549 A RichTextLines object.
1550 """
1552 lines = []
1553 font_attr_segs = {}
1555 watch_keys = self._debug_dump.debug_watch_keys(node_name)
1557 dump_count = 0
1558 for watch_key in watch_keys:
1559 debug_tensor_data = self._debug_dump.watch_key_to_data(watch_key)
1560 for datum in debug_tensor_data:
1561 line = " Slot %d @ %s @ %.3f ms" % (
1562 datum.output_slot, datum.debug_op,
1563 (datum.timestamp - self._debug_dump.t0) / 1000.0)
1564 lines.append(line)
1565 command = "pt %s:%d -n %d" % (node_name, datum.output_slot, dump_count)
1566 font_attr_segs[len(lines) - 1] = [(
1567 2, len(line), debugger_cli_common.MenuItem(None, command))]
1568 dump_count += 1
1570 output = debugger_cli_common.RichTextLines(
1571 lines, font_attr_segs=font_attr_segs)
1572 output_with_header = debugger_cli_common.RichTextLines(
1573 ["%d dumped tensor(s):" % dump_count, ""])
1574 output_with_header.extend(output)
1575 return output_with_header
1578def create_analyzer_ui(debug_dump,
1579 tensor_filters=None,
1580 ui_type="curses",
1581 on_ui_exit=None,
1582 config=None):
1583 """Create an instance of CursesUI based on a DebugDumpDir object.
1585 Args:
1586 debug_dump: (debug_data.DebugDumpDir) The debug dump to use.
1587 tensor_filters: (dict) A dict mapping tensor filter name (str) to tensor
1588 filter (Callable).
1589 ui_type: (str) requested UI type, e.g., "curses", "readline".
1590 on_ui_exit: (`Callable`) the callback to be called when the UI exits.
1591 config: A `cli_config.CLIConfig` object.
1593 Returns:
1594 (base_ui.BaseUI) A BaseUI subtype object with a set of standard analyzer
1595 commands and tab-completions registered.
1596 """
1597 if config is None:
1598 config = cli_config.CLIConfig()
1600 analyzer = DebugAnalyzer(debug_dump, config=config)
1601 if tensor_filters:
1602 for tensor_filter_name in tensor_filters:
1603 analyzer.add_tensor_filter(
1604 tensor_filter_name, tensor_filters[tensor_filter_name])
1606 cli = ui_factory.get_ui(ui_type, on_ui_exit=on_ui_exit, config=config)
1607 cli.register_command_handler(
1608 "list_tensors",
1609 analyzer.list_tensors,
1610 analyzer.get_help("list_tensors"),
1611 prefix_aliases=["lt"])
1612 cli.register_command_handler(
1613 "node_info",
1614 analyzer.node_info,
1615 analyzer.get_help("node_info"),
1616 prefix_aliases=["ni"])
1617 cli.register_command_handler(
1618 "list_inputs",
1619 analyzer.list_inputs,
1620 analyzer.get_help("list_inputs"),
1621 prefix_aliases=["li"])
1622 cli.register_command_handler(
1623 "list_outputs",
1624 analyzer.list_outputs,
1625 analyzer.get_help("list_outputs"),
1626 prefix_aliases=["lo"])
1627 cli.register_command_handler(
1628 "print_tensor",
1629 analyzer.print_tensor,
1630 analyzer.get_help("print_tensor"),
1631 prefix_aliases=["pt"])
1632 cli.register_command_handler(
1633 "print_source",
1634 analyzer.print_source,
1635 analyzer.get_help("print_source"),
1636 prefix_aliases=["ps"])
1637 cli.register_command_handler(
1638 "list_source",
1639 analyzer.list_source,
1640 analyzer.get_help("list_source"),
1641 prefix_aliases=["ls"])
1642 cli.register_command_handler(
1643 "eval",
1644 analyzer.evaluate_expression,
1645 analyzer.get_help("eval"),
1646 prefix_aliases=["ev"])
1648 dumped_tensor_names = []
1649 for datum in debug_dump.dumped_tensor_data:
1650 dumped_tensor_names.append("%s:%d" % (datum.node_name, datum.output_slot))
1652 # Tab completions for command "print_tensors".
1653 cli.register_tab_comp_context(["print_tensor", "pt"], dumped_tensor_names)
1655 return cli