Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/tensorflow/python/summary/writer/writer.py: 31%
130 statements
« prev ^ index » next coverage.py v7.4.0, created at 2024-01-03 07:57 +0000
1# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ==============================================================================
15"""Provides an API for generating Event protocol buffers."""
17import os.path
18import time
19import warnings
21from tensorflow.core.framework import graph_pb2
22from tensorflow.core.framework import summary_pb2
23from tensorflow.core.protobuf import meta_graph_pb2
24from tensorflow.core.util import event_pb2
25from tensorflow.python.eager import context
26from tensorflow.python.framework import meta_graph
27from tensorflow.python.framework import ops
28from tensorflow.python.platform import gfile
29from tensorflow.python.platform import tf_logging as logging
30from tensorflow.python.summary import plugin_asset
31from tensorflow.python.summary.writer.event_file_writer import EventFileWriter
32from tensorflow.python.summary.writer.event_file_writer_v2 import EventFileWriterV2
33from tensorflow.python.util.tf_export import tf_export
35_PLUGINS_DIR = "plugins"
38class SummaryToEventTransformer(object):
39 """Abstractly implements the SummaryWriter API.
41 This API basically implements a number of endpoints (add_summary,
42 add_session_log, etc). The endpoints all generate an event protobuf, which is
43 passed to the contained event_writer.
44 """
46 def __init__(self, event_writer, graph=None, graph_def=None):
47 """Creates a `SummaryWriter` and an event file.
49 On construction the summary writer creates a new event file in `logdir`.
50 This event file will contain `Event` protocol buffers constructed when you
51 call one of the following functions: `add_summary()`, `add_session_log()`,
52 `add_event()`, or `add_graph()`.
54 If you pass a `Graph` to the constructor it is added to
55 the event file. (This is equivalent to calling `add_graph()` later).
57 TensorBoard will pick the graph from the file and display it graphically so
58 you can interactively explore the graph you built. You will usually pass
59 the graph from the session in which you launched it:
61 ```python
62 ...create a graph...
63 # Launch the graph in a session.
64 sess = tf.compat.v1.Session()
65 # Create a summary writer, add the 'graph' to the event file.
66 writer = tf.compat.v1.summary.FileWriter(<some-directory>, sess.graph)
67 ```
70 Args:
71 event_writer: An EventWriter. Implements add_event and get_logdir.
72 graph: A `Graph` object, such as `sess.graph`.
73 graph_def: DEPRECATED: Use the `graph` argument instead.
74 """
75 self.event_writer = event_writer
76 # For storing used tags for session.run() outputs.
77 self._session_run_tags = {}
78 if graph is not None or graph_def is not None:
79 # Calling it with both graph and graph_def for backward compatibility.
80 self.add_graph(graph=graph, graph_def=graph_def)
81 # Also export the meta_graph_def in this case.
82 # graph may itself be a graph_def due to positional arguments
83 maybe_graph_as_def = (graph.as_graph_def(add_shapes=True)
84 if isinstance(graph, ops.Graph) else graph)
85 self.add_meta_graph(
86 meta_graph.create_meta_graph_def(graph_def=graph_def or
87 maybe_graph_as_def))
89 # This set contains tags of Summary Values that have been encountered
90 # already. The motivation here is that the SummaryWriter only keeps the
91 # metadata property (which is a SummaryMetadata proto) of the first Summary
92 # Value encountered for each tag. The SummaryWriter strips away the
93 # SummaryMetadata for all subsequent Summary Values with tags seen
94 # previously. This saves space.
95 self._seen_summary_tags = set()
97 def add_summary(self, summary, global_step=None):
98 """Adds a `Summary` protocol buffer to the event file.
100 This method wraps the provided summary in an `Event` protocol buffer
101 and adds it to the event file.
103 You can pass the result of evaluating any summary op, using
104 `tf.Session.run` or
105 `tf.Tensor.eval`, to this
106 function. Alternatively, you can pass a `tf.compat.v1.Summary` protocol
107 buffer that you populate with your own data. The latter is
108 commonly done to report evaluation results in event files.
110 Args:
111 summary: A `Summary` protocol buffer, optionally serialized as a string.
112 global_step: Number. Optional global step value to record with the
113 summary.
114 """
115 if isinstance(summary, bytes):
116 summ = summary_pb2.Summary()
117 summ.ParseFromString(summary)
118 summary = summ
120 # We strip metadata from values with tags that we have seen before in order
121 # to save space - we just store the metadata on the first value with a
122 # specific tag.
123 for value in summary.value:
124 if not value.metadata:
125 continue
127 if value.tag in self._seen_summary_tags:
128 # This tag has been encountered before. Strip the metadata.
129 value.ClearField("metadata")
130 continue
132 # We encounter a value with a tag we have not encountered previously. And
133 # it has metadata. Remember to strip metadata from future values with this
134 # tag string.
135 self._seen_summary_tags.add(value.tag)
137 event = event_pb2.Event(summary=summary)
138 self._add_event(event, global_step)
140 def add_session_log(self, session_log, global_step=None):
141 """Adds a `SessionLog` protocol buffer to the event file.
143 This method wraps the provided session in an `Event` protocol buffer
144 and adds it to the event file.
146 Args:
147 session_log: A `SessionLog` protocol buffer.
148 global_step: Number. Optional global step value to record with the
149 summary.
150 """
151 event = event_pb2.Event(session_log=session_log)
152 self._add_event(event, global_step)
154 def _add_graph_def(self, graph_def, global_step=None):
155 graph_bytes = graph_def.SerializeToString()
156 event = event_pb2.Event(graph_def=graph_bytes)
157 self._add_event(event, global_step)
159 def add_graph(self, graph, global_step=None, graph_def=None):
160 """Adds a `Graph` to the event file.
162 The graph described by the protocol buffer will be displayed by
163 TensorBoard. Most users pass a graph in the constructor instead.
165 Args:
166 graph: A `Graph` object, such as `sess.graph`.
167 global_step: Number. Optional global step counter to record with the
168 graph.
169 graph_def: DEPRECATED. Use the `graph` parameter instead.
171 Raises:
172 ValueError: If both graph and graph_def are passed to the method.
173 """
175 if graph is not None and graph_def is not None:
176 raise ValueError("Please pass only graph, or graph_def (deprecated), "
177 "but not both.")
179 if isinstance(graph, ops.Graph) or isinstance(graph_def, ops.Graph):
180 # The user passed a `Graph`.
182 # Check if the user passed it via the graph or the graph_def argument and
183 # correct for that.
184 if not isinstance(graph, ops.Graph):
185 logging.warning("When passing a `Graph` object, please use the `graph`"
186 " named argument instead of `graph_def`.")
187 graph = graph_def
189 # Serialize the graph with additional info.
190 true_graph_def = graph.as_graph_def(add_shapes=True)
191 self._write_plugin_assets(graph)
192 elif (isinstance(graph, graph_pb2.GraphDef) or
193 isinstance(graph_def, graph_pb2.GraphDef)):
194 # The user passed a `GraphDef`.
195 logging.warning("Passing a `GraphDef` to the SummaryWriter is deprecated."
196 " Pass a `Graph` object instead, such as `sess.graph`.")
198 # Check if the user passed it via the graph or the graph_def argument and
199 # correct for that.
200 if isinstance(graph, graph_pb2.GraphDef):
201 true_graph_def = graph
202 else:
203 true_graph_def = graph_def
205 else:
206 # The user passed neither `Graph`, nor `GraphDef`.
207 raise TypeError("The passed graph must be an instance of `Graph` "
208 "or the deprecated `GraphDef`")
209 # Finally, add the graph_def to the summary writer.
210 self._add_graph_def(true_graph_def, global_step)
212 def _write_plugin_assets(self, graph):
213 plugin_assets = plugin_asset.get_all_plugin_assets(graph)
214 logdir = self.event_writer.get_logdir()
215 for asset_container in plugin_assets:
216 plugin_name = asset_container.plugin_name
217 plugin_dir = os.path.join(logdir, _PLUGINS_DIR, plugin_name)
218 gfile.MakeDirs(plugin_dir)
219 assets = asset_container.assets()
220 for (asset_name, content) in assets.items():
221 asset_path = os.path.join(plugin_dir, asset_name)
222 with gfile.Open(asset_path, "w") as f:
223 f.write(content)
225 def add_meta_graph(self, meta_graph_def, global_step=None):
226 """Adds a `MetaGraphDef` to the event file.
228 The `MetaGraphDef` allows running the given graph via
229 `saver.import_meta_graph()`.
231 Args:
232 meta_graph_def: A `MetaGraphDef` object, often as returned by
233 `saver.export_meta_graph()`.
234 global_step: Number. Optional global step counter to record with the
235 graph.
237 Raises:
238 TypeError: If both `meta_graph_def` is not an instance of `MetaGraphDef`.
239 """
240 if not isinstance(meta_graph_def, meta_graph_pb2.MetaGraphDef):
241 raise TypeError("meta_graph_def must be type MetaGraphDef, saw type: %s" %
242 type(meta_graph_def))
243 meta_graph_bytes = meta_graph_def.SerializeToString()
244 event = event_pb2.Event(meta_graph_def=meta_graph_bytes)
245 self._add_event(event, global_step)
247 def add_run_metadata(self, run_metadata, tag, global_step=None):
248 """Adds a metadata information for a single session.run() call.
250 Args:
251 run_metadata: A `RunMetadata` protobuf object.
252 tag: The tag name for this metadata.
253 global_step: Number. Optional global step counter to record with the
254 StepStats.
256 Raises:
257 ValueError: If the provided tag was already used for this type of event.
258 """
259 if tag in self._session_run_tags:
260 raise ValueError("The provided tag was already used for this event type")
261 self._session_run_tags[tag] = True
263 tagged_metadata = event_pb2.TaggedRunMetadata()
264 tagged_metadata.tag = tag
265 # Store the `RunMetadata` object as bytes in order to have postponed
266 # (lazy) deserialization when used later.
267 tagged_metadata.run_metadata = run_metadata.SerializeToString()
268 event = event_pb2.Event(tagged_run_metadata=tagged_metadata)
269 self._add_event(event, global_step)
271 def _add_event(self, event, step):
272 event.wall_time = time.time()
273 if step is not None:
274 event.step = int(step)
275 self.event_writer.add_event(event)
278@tf_export(v1=["summary.FileWriter"])
279class FileWriter(SummaryToEventTransformer):
280 """Writes `Summary` protocol buffers to event files.
282 The `FileWriter` class provides a mechanism to create an event file in a
283 given directory and add summaries and events to it. The class updates the
284 file contents asynchronously. This allows a training program to call methods
285 to add data to the file directly from the training loop, without slowing down
286 training.
288 When constructed with a `tf.compat.v1.Session` parameter, a `FileWriter`
289 instead forms a compatibility layer over new graph-based summaries
290 to facilitate the use of new summary writing with
291 pre-existing code that expects a `FileWriter` instance.
293 This class is not thread-safe.
295 @compatibility(TF2)
296 This API is not compatible with eager execution or `tf.function`. To migrate
297 to TF2, please use `tf.summary.create_file_writer` instead for summary
298 management. To specify the summary step, you can manage the context with
299 `tf.summary.SummaryWriter`, which is returned by
300 `tf.summary.create_file_writer()`. Or, you can also use the `step` argument
301 of summary functions such as `tf.summary.histogram`.
302 See the usage example shown below.
304 For a comprehensive `tf.summary` migration guide, please follow
305 [Migrating tf.summary usage to
306 TF 2.0](https://www.tensorflow.org/tensorboard/migrate#in_tf_1x).
308 #### How to Map Arguments
310 | TF1 Arg Name | TF2 Arg Name | Note |
311 | :---------------- | :---------------- | :-------------------------------- |
312 | `logdir` | `logdir` | - |
313 | `graph` | Not supported | - |
314 | `max_queue` | `max_queue` | - |
315 | `flush_secs` | `flush_millis` | The unit of time is changed |
316 : : : from seconds to milliseconds. :
317 | `graph_def` | Not supported | - |
318 | `filename_suffix` | `filename_suffix` | - |
319 | `name` | `name` | - |
321 #### TF1 & TF2 Usage Example
323 TF1:
325 ```python
326 dist = tf.compat.v1.placeholder(tf.float32, [100])
327 tf.compat.v1.summary.histogram(name="distribution", values=dist)
328 writer = tf.compat.v1.summary.FileWriter("/tmp/tf1_summary_example")
329 summaries = tf.compat.v1.summary.merge_all()
331 sess = tf.compat.v1.Session()
332 for step in range(100):
333 mean_moving_normal = np.random.normal(loc=step, scale=1, size=[100])
334 summ = sess.run(summaries, feed_dict={dist: mean_moving_normal})
335 writer.add_summary(summ, global_step=step)
336 ```
338 TF2:
340 ```python
341 writer = tf.summary.create_file_writer("/tmp/tf2_summary_example")
342 for step in range(100):
343 mean_moving_normal = np.random.normal(loc=step, scale=1, size=[100])
344 with writer.as_default(step=step):
345 tf.summary.histogram(name='distribution', data=mean_moving_normal)
346 ```
348 @end_compatibility
349 """
351 def __init__(self,
352 logdir,
353 graph=None,
354 max_queue=10,
355 flush_secs=120,
356 graph_def=None,
357 filename_suffix=None,
358 session=None):
359 """Creates a `FileWriter`, optionally shared within the given session.
361 Typically, constructing a file writer creates a new event file in `logdir`.
362 This event file will contain `Event` protocol buffers constructed when you
363 call one of the following functions: `add_summary()`, `add_session_log()`,
364 `add_event()`, or `add_graph()`.
366 If you pass a `Graph` to the constructor it is added to
367 the event file. (This is equivalent to calling `add_graph()` later).
369 TensorBoard will pick the graph from the file and display it graphically so
370 you can interactively explore the graph you built. You will usually pass
371 the graph from the session in which you launched it:
373 ```python
374 ...create a graph...
375 # Launch the graph in a session.
376 sess = tf.compat.v1.Session()
377 # Create a summary writer, add the 'graph' to the event file.
378 writer = tf.compat.v1.summary.FileWriter(<some-directory>, sess.graph)
379 ```
381 The `session` argument to the constructor makes the returned `FileWriter` a
382 compatibility layer over new graph-based summaries (`tf.summary`).
383 Crucially, this means the underlying writer resource and events file will
384 be shared with any other `FileWriter` using the same `session` and `logdir`.
385 In either case, ops will be added to `session.graph` to control the
386 underlying file writer resource.
388 Args:
389 logdir: A string. Directory where event file will be written.
390 graph: A `Graph` object, such as `sess.graph`.
391 max_queue: Integer. Size of the queue for pending events and summaries.
392 flush_secs: Number. How often, in seconds, to flush the
393 pending events and summaries to disk.
394 graph_def: DEPRECATED: Use the `graph` argument instead.
395 filename_suffix: A string. Every event file's name is suffixed with
396 `suffix`.
397 session: A `tf.compat.v1.Session` object. See details above.
399 Raises:
400 RuntimeError: If called with eager execution enabled.
402 @compatibility(eager)
403 `v1.summary.FileWriter` is not compatible with eager execution.
404 To write TensorBoard summaries under eager execution,
405 use `tf.summary.create_file_writer` or
406 a `with v1.Graph().as_default():` context.
407 @end_compatibility
408 """
409 if context.executing_eagerly():
410 raise RuntimeError(
411 "v1.summary.FileWriter is not compatible with eager execution. "
412 "Use `tf.summary.create_file_writer`,"
413 "or a `with v1.Graph().as_default():` context")
414 if session is not None:
415 event_writer = EventFileWriterV2(
416 session, logdir, max_queue, flush_secs, filename_suffix)
417 else:
418 event_writer = EventFileWriter(logdir, max_queue, flush_secs,
419 filename_suffix)
421 self._closed = False
422 super(FileWriter, self).__init__(event_writer, graph, graph_def)
424 def __enter__(self):
425 """Make usable with "with" statement."""
426 return self
428 def __exit__(self, unused_type, unused_value, unused_traceback):
429 """Make usable with "with" statement."""
430 self.close()
432 def get_logdir(self):
433 """Returns the directory where event file will be written."""
434 return self.event_writer.get_logdir()
436 def _warn_if_event_writer_is_closed(self):
437 if self._closed:
438 warnings.warn("Attempting to use a closed FileWriter. "
439 "The operation will be a noop unless the FileWriter "
440 "is explicitly reopened.")
442 def _add_event(self, event, step):
443 self._warn_if_event_writer_is_closed()
444 super(FileWriter, self)._add_event(event, step)
446 def add_event(self, event):
447 """Adds an event to the event file.
449 Args:
450 event: An `Event` protocol buffer.
451 """
452 self._warn_if_event_writer_is_closed()
453 self.event_writer.add_event(event)
455 def flush(self):
456 """Flushes the event file to disk.
458 Call this method to make sure that all pending events have been written to
459 disk.
460 """
461 # Flushing a closed EventFileWriterV2 raises an exception. It is,
462 # however, a noop for EventFileWriter.
463 self._warn_if_event_writer_is_closed()
464 self.event_writer.flush()
466 def close(self):
467 """Flushes the event file to disk and close the file.
469 Call this method when you do not need the summary writer anymore.
470 """
471 self.event_writer.close()
472 self._closed = True
474 def reopen(self):
475 """Reopens the EventFileWriter.
477 Can be called after `close()` to add more events in the same directory.
478 The events will go into a new events file.
480 Does nothing if the EventFileWriter was not closed.
481 """
482 self.event_writer.reopen()
483 self._closed = False