Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/tensorflow/python/ops/gen_decode_proto_ops.py: 21%
104 statements
« prev ^ index » next coverage.py v7.4.0, created at 2024-01-03 07:57 +0000
1"""Python wrappers around TensorFlow ops.
3This file is MACHINE GENERATED! Do not edit.
4"""
6import collections
8from tensorflow.python import pywrap_tfe as pywrap_tfe
9from tensorflow.python.eager import context as _context
10from tensorflow.python.eager import core as _core
11from tensorflow.python.eager import execute as _execute
12from tensorflow.python.framework import dtypes as _dtypes
13from tensorflow.security.fuzzing.py import annotation_types as _atypes
15from tensorflow.python.framework import op_def_registry as _op_def_registry
16from tensorflow.python.framework import ops as _ops
17from tensorflow.python.framework import op_def_library as _op_def_library
18from tensorflow.python.util.deprecation import deprecated_endpoints
19from tensorflow.python.util import dispatch as _dispatch
20from tensorflow.python.util.tf_export import tf_export
22from typing import TypeVar
23_DecodeProtoV2Output = collections.namedtuple(
24 "DecodeProtoV2",
25 ["sizes", "values"])
28@_dispatch.add_fallback_dispatch_list
29@_dispatch.add_type_based_api_dispatcher
30@tf_export('io.decode_proto')
31def decode_proto_v2(bytes, message_type, field_names, output_types, descriptor_source="local://", message_format="binary", sanitize=False, name=None):
32 r"""The op extracts fields from a serialized protocol buffers message into tensors.
34 Note: This API is designed for orthogonality rather than human-friendliness. It
35 can be used to parse input protos by hand, but it is intended for use in
36 generated code.
38 The `decode_proto` op extracts fields from a serialized protocol buffers
39 message into tensors. The fields in `field_names` are decoded and converted
40 to the corresponding `output_types` if possible.
42 A `message_type` name must be provided to give context for the field names.
43 The actual message descriptor can be looked up either in the linked-in
44 descriptor pool or a filename provided by the caller using the
45 `descriptor_source` attribute.
47 Each output tensor is a dense tensor. This means that it is padded to hold
48 the largest number of repeated elements seen in the input minibatch. (The
49 shape is also padded by one to prevent zero-sized dimensions). The actual
50 repeat counts for each example in the minibatch can be found in the `sizes`
51 output. In many cases the output of `decode_proto` is fed immediately into
52 tf.squeeze if missing values are not a concern. When using tf.squeeze, always
53 pass the squeeze dimension explicitly to avoid surprises.
55 For the most part, the mapping between Proto field types and TensorFlow dtypes
56 is straightforward. However, there are a few special cases:
58 - A proto field that contains a submessage or group can only be converted
59 to `DT_STRING` (the serialized submessage). This is to reduce the complexity
60 of the API. The resulting string can be used as input to another instance of
61 the decode_proto op.
63 - TensorFlow lacks support for unsigned integers. The ops represent uint64
64 types as a `DT_INT64` with the same twos-complement bit pattern (the obvious
65 way). Unsigned int32 values can be represented exactly by specifying type
66 `DT_INT64`, or using twos-complement if the caller specifies `DT_INT32` in
67 the `output_types` attribute.
69 - `map` fields are not directly decoded. They are treated as `repeated` fields,
70 of the appropriate entry type. The proto-compiler defines entry types for each
71 map field. The type-name is the field name, converted to "CamelCase" with
72 "Entry" appended. The `tf.train.Features.FeatureEntry` message is an example of
73 one of these implicit `Entry` types.
75 - `enum` fields should be read as int32.
77 Both binary and text proto serializations are supported, and can be
78 chosen using the `format` attribute.
80 The `descriptor_source` attribute selects the source of protocol
81 descriptors to consult when looking up `message_type`. This may be:
83 - An empty string or "local://", in which case protocol descriptors are
84 created for C++ (not Python) proto definitions linked to the binary.
86 - A file, in which case protocol descriptors are created from the file,
87 which is expected to contain a `FileDescriptorSet` serialized as a string.
88 NOTE: You can build a `descriptor_source` file using the `--descriptor_set_out`
89 and `--include_imports` options to the protocol compiler `protoc`.
91 - A "bytes://<bytes>", in which protocol descriptors are created from `<bytes>`,
92 which is expected to be a `FileDescriptorSet` serialized as a string.
94 Here is an example:
96 The, internal, `Summary.Value` proto contains a
97 `oneof {float simple_value; Image image; ...}`
99 >>> from google.protobuf import text_format
100 >>>
101 >>> # A Summary.Value contains: oneof {float simple_value; Image image}
102 >>> values = [
103 ... "simple_value: 2.2",
104 ... "simple_value: 1.2",
105 ... "image { height: 128 width: 512 }",
106 ... "image { height: 256 width: 256 }",]
107 >>> values = [
108 ... text_format.Parse(v, tf.compat.v1.Summary.Value()).SerializeToString()
109 ... for v in values]
111 The following can decode both fields from the serialized strings:
113 >>> sizes, [simple_value, image] = tf.io.decode_proto(
114 ... values,
115 ... tf.compat.v1.Summary.Value.DESCRIPTOR.full_name,
116 ... field_names=['simple_value', 'image'],
117 ... output_types=[tf.float32, tf.string])
119 The `sizes` has the same shape as the input, with an additional axis across the
120 fields that were decoded. Here the first column of `sizes` is the size of the
121 decoded `simple_value` field:
123 >>> print(sizes)
124 tf.Tensor(
125 [[1 0]
126 [1 0]
127 [0 1]
128 [0 1]], shape=(4, 2), dtype=int32)
130 The result tensors each have one more index than the input byte-strings.
131 The valid elements of each result tensor are indicated by
132 the appropriate column of `sizes`. The invalid elements are padded with a
133 default value:
135 >>> print(simple_value)
136 tf.Tensor(
137 [[2.2]
138 [1.2]
139 [0. ]
140 [0. ]], shape=(4, 1), dtype=float32)
142 Nested protos are extracted as string tensors:
144 >>> print(image.dtype)
145 <dtype: 'string'>
146 >>> print(image.shape.as_list())
147 [4, 1]
149 To convert to a `tf.RaggedTensor` representation use:
151 >>> tf.RaggedTensor.from_tensor(simple_value, lengths=sizes[:, 0]).to_list()
152 [[2.2], [1.2], [], []]
154 Args:
155 bytes: A `Tensor` of type `string`.
156 Tensor of serialized protos with shape `batch_shape`.
157 message_type: A `string`. Name of the proto message type to decode.
158 field_names: A list of `strings`.
159 List of strings containing proto field names. An extension field can be decoded
160 by using its full name, e.g. EXT_PACKAGE.EXT_FIELD_NAME.
161 output_types: A list of `tf.DTypes`.
162 List of TF types to use for the respective field in field_names.
163 descriptor_source: An optional `string`. Defaults to `"local://"`.
164 Either the special value `local://` or a path to a file containing
165 a serialized `FileDescriptorSet`.
166 message_format: An optional `string`. Defaults to `"binary"`.
167 Either `binary` or `text`.
168 sanitize: An optional `bool`. Defaults to `False`.
169 Whether to sanitize the result or not.
170 name: A name for the operation (optional).
172 Returns:
173 A tuple of `Tensor` objects (sizes, values).
175 sizes: A `Tensor` of type `int32`.
176 values: A list of `Tensor` objects of type `output_types`.
177 """
178 _ctx = _context._context or _context.context()
179 tld = _ctx._thread_local_data
180 if tld.is_eager:
181 try:
182 _result = pywrap_tfe.TFE_Py_FastPathExecute(
183 _ctx, "DecodeProtoV2", name, bytes, "message_type", message_type,
184 "field_names", field_names, "output_types", output_types,
185 "descriptor_source", descriptor_source, "message_format",
186 message_format, "sanitize", sanitize)
187 _result = _DecodeProtoV2Output._make(_result)
188 return _result
189 except _core._NotOkStatusException as e:
190 _ops.raise_from_not_ok_status(e, name)
191 except _core._FallbackException:
192 pass
193 try:
194 _result = _dispatcher_for_decode_proto_v2(
195 (bytes, message_type, field_names, output_types, descriptor_source,
196 message_format, sanitize, name,), None)
197 if _result is not NotImplemented:
198 return _result
199 return decode_proto_v2_eager_fallback(
200 bytes, message_type=message_type, field_names=field_names,
201 output_types=output_types, descriptor_source=descriptor_source,
202 message_format=message_format, sanitize=sanitize, name=name,
203 ctx=_ctx)
204 except _core._SymbolicException:
205 pass # Add nodes to the TensorFlow graph.
206 except (TypeError, ValueError):
207 _result = _dispatch.dispatch(
208 decode_proto_v2, (), dict(bytes=bytes, message_type=message_type,
209 field_names=field_names,
210 output_types=output_types,
211 descriptor_source=descriptor_source,
212 message_format=message_format,
213 sanitize=sanitize, name=name)
214 )
215 if _result is not _dispatch.OpDispatcher.NOT_SUPPORTED:
216 return _result
217 raise
218 else:
219 _result = _dispatcher_for_decode_proto_v2(
220 (bytes, message_type, field_names, output_types, descriptor_source,
221 message_format, sanitize, name,), None)
222 if _result is not NotImplemented:
223 return _result
224 # Add nodes to the TensorFlow graph.
225 message_type = _execute.make_str(message_type, "message_type")
226 if not isinstance(field_names, (list, tuple)):
227 raise TypeError(
228 "Expected list for 'field_names' argument to "
229 "'decode_proto_v2' Op, not %r." % field_names)
230 field_names = [_execute.make_str(_s, "field_names") for _s in field_names]
231 if not isinstance(output_types, (list, tuple)):
232 raise TypeError(
233 "Expected list for 'output_types' argument to "
234 "'decode_proto_v2' Op, not %r." % output_types)
235 output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
236 if descriptor_source is None:
237 descriptor_source = "local://"
238 descriptor_source = _execute.make_str(descriptor_source, "descriptor_source")
239 if message_format is None:
240 message_format = "binary"
241 message_format = _execute.make_str(message_format, "message_format")
242 if sanitize is None:
243 sanitize = False
244 sanitize = _execute.make_bool(sanitize, "sanitize")
245 try:
246 _, _, _op, _outputs = _op_def_library._apply_op_helper(
247 "DecodeProtoV2", bytes=bytes, message_type=message_type,
248 field_names=field_names, output_types=output_types,
249 descriptor_source=descriptor_source,
250 message_format=message_format, sanitize=sanitize,
251 name=name)
252 except (TypeError, ValueError):
253 _result = _dispatch.dispatch(
254 decode_proto_v2, (), dict(bytes=bytes, message_type=message_type,
255 field_names=field_names,
256 output_types=output_types,
257 descriptor_source=descriptor_source,
258 message_format=message_format,
259 sanitize=sanitize, name=name)
260 )
261 if _result is not _dispatch.OpDispatcher.NOT_SUPPORTED:
262 return _result
263 raise
264 _result = _outputs[:]
265 if _execute.must_record_gradient():
266 _attrs = ("message_type", _op.get_attr("message_type"), "field_names",
267 _op.get_attr("field_names"), "output_types",
268 _op.get_attr("output_types"), "descriptor_source",
269 _op.get_attr("descriptor_source"), "message_format",
270 _op.get_attr("message_format"), "sanitize",
271 _op._get_attr_bool("sanitize"))
272 _inputs_flat = _op.inputs
273 _execute.record_gradient(
274 "DecodeProtoV2", _inputs_flat, _attrs, _result)
275 _result = _result[:1] + [_result[1:]]
276 _result = _DecodeProtoV2Output._make(_result)
277 return _result
279DecodeProtoV2 = tf_export("raw_ops.DecodeProtoV2")(_ops.to_raw_op(decode_proto_v2))
280_dispatcher_for_decode_proto_v2 = decode_proto_v2._tf_type_based_dispatcher.Dispatch
283def decode_proto_v2_eager_fallback(bytes, message_type, field_names, output_types, descriptor_source, message_format, sanitize, name, ctx):
284 message_type = _execute.make_str(message_type, "message_type")
285 if not isinstance(field_names, (list, tuple)):
286 raise TypeError(
287 "Expected list for 'field_names' argument to "
288 "'decode_proto_v2' Op, not %r." % field_names)
289 field_names = [_execute.make_str(_s, "field_names") for _s in field_names]
290 if not isinstance(output_types, (list, tuple)):
291 raise TypeError(
292 "Expected list for 'output_types' argument to "
293 "'decode_proto_v2' Op, not %r." % output_types)
294 output_types = [_execute.make_type(_t, "output_types") for _t in output_types]
295 if descriptor_source is None:
296 descriptor_source = "local://"
297 descriptor_source = _execute.make_str(descriptor_source, "descriptor_source")
298 if message_format is None:
299 message_format = "binary"
300 message_format = _execute.make_str(message_format, "message_format")
301 if sanitize is None:
302 sanitize = False
303 sanitize = _execute.make_bool(sanitize, "sanitize")
304 bytes = _ops.convert_to_tensor(bytes, _dtypes.string)
305 _inputs_flat = [bytes]
306 _attrs = ("message_type", message_type, "field_names", field_names,
307 "output_types", output_types, "descriptor_source", descriptor_source,
308 "message_format", message_format, "sanitize", sanitize)
309 _result = _execute.execute(b"DecodeProtoV2", len(output_types) + 1,
310 inputs=_inputs_flat, attrs=_attrs, ctx=ctx,
311 name=name)
312 if _execute.must_record_gradient():
313 _execute.record_gradient(
314 "DecodeProtoV2", _inputs_flat, _attrs, _result)
315 _result = _result[:1] + [_result[1:]]
316 _result = _DecodeProtoV2Output._make(_result)
317 return _result