Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/tensorboard/compat/tensorflow_stub/pywrap_tensorflow.py: 28%
95 statements
« prev ^ index » next coverage.py v7.4.0, created at 2024-01-03 07:57 +0000
1# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# =============================================================================
15"""A wrapper for TensorFlow SWIG-generated bindings."""
18import array
19import struct
21from . import errors
22from .io import gfile
25TFE_DEVICE_PLACEMENT_WARN = 0
26TFE_DEVICE_PLACEMENT_SILENT_FOR_INT32 = 0
27TFE_DEVICE_PLACEMENT_SILENT = 0
28TFE_DEVICE_PLACEMENT_EXPLICIT = 0
31def __getattr__(attr):
32 return 0
35def TF_bfloat16_type():
36 return 0
39def masked_crc32c(data):
40 x = u32(crc32c(data))
41 return u32(((x >> 15) | u32(x << 17)) + 0xA282EAD8)
44def u32(x):
45 return x & 0xFFFFFFFF
48# fmt: off
49CRC_TABLE = (
50 0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4,
51 0xC79A971F, 0x35F1141C, 0x26A1E7E8, 0xD4CA64EB,
52 0x8AD958CF, 0x78B2DBCC, 0x6BE22838, 0x9989AB3B,
53 0x4D43CFD0, 0xBF284CD3, 0xAC78BF27, 0x5E133C24,
54 0x105EC76F, 0xE235446C, 0xF165B798, 0x030E349B,
55 0xD7C45070, 0x25AFD373, 0x36FF2087, 0xC494A384,
56 0x9A879FA0, 0x68EC1CA3, 0x7BBCEF57, 0x89D76C54,
57 0x5D1D08BF, 0xAF768BBC, 0xBC267848, 0x4E4DFB4B,
58 0x20BD8EDE, 0xD2D60DDD, 0xC186FE29, 0x33ED7D2A,
59 0xE72719C1, 0x154C9AC2, 0x061C6936, 0xF477EA35,
60 0xAA64D611, 0x580F5512, 0x4B5FA6E6, 0xB93425E5,
61 0x6DFE410E, 0x9F95C20D, 0x8CC531F9, 0x7EAEB2FA,
62 0x30E349B1, 0xC288CAB2, 0xD1D83946, 0x23B3BA45,
63 0xF779DEAE, 0x05125DAD, 0x1642AE59, 0xE4292D5A,
64 0xBA3A117E, 0x4851927D, 0x5B016189, 0xA96AE28A,
65 0x7DA08661, 0x8FCB0562, 0x9C9BF696, 0x6EF07595,
66 0x417B1DBC, 0xB3109EBF, 0xA0406D4B, 0x522BEE48,
67 0x86E18AA3, 0x748A09A0, 0x67DAFA54, 0x95B17957,
68 0xCBA24573, 0x39C9C670, 0x2A993584, 0xD8F2B687,
69 0x0C38D26C, 0xFE53516F, 0xED03A29B, 0x1F682198,
70 0x5125DAD3, 0xA34E59D0, 0xB01EAA24, 0x42752927,
71 0x96BF4DCC, 0x64D4CECF, 0x77843D3B, 0x85EFBE38,
72 0xDBFC821C, 0x2997011F, 0x3AC7F2EB, 0xC8AC71E8,
73 0x1C661503, 0xEE0D9600, 0xFD5D65F4, 0x0F36E6F7,
74 0x61C69362, 0x93AD1061, 0x80FDE395, 0x72966096,
75 0xA65C047D, 0x5437877E, 0x4767748A, 0xB50CF789,
76 0xEB1FCBAD, 0x197448AE, 0x0A24BB5A, 0xF84F3859,
77 0x2C855CB2, 0xDEEEDFB1, 0xCDBE2C45, 0x3FD5AF46,
78 0x7198540D, 0x83F3D70E, 0x90A324FA, 0x62C8A7F9,
79 0xB602C312, 0x44694011, 0x5739B3E5, 0xA55230E6,
80 0xFB410CC2, 0x092A8FC1, 0x1A7A7C35, 0xE811FF36,
81 0x3CDB9BDD, 0xCEB018DE, 0xDDE0EB2A, 0x2F8B6829,
82 0x82F63B78, 0x709DB87B, 0x63CD4B8F, 0x91A6C88C,
83 0x456CAC67, 0xB7072F64, 0xA457DC90, 0x563C5F93,
84 0x082F63B7, 0xFA44E0B4, 0xE9141340, 0x1B7F9043,
85 0xCFB5F4A8, 0x3DDE77AB, 0x2E8E845F, 0xDCE5075C,
86 0x92A8FC17, 0x60C37F14, 0x73938CE0, 0x81F80FE3,
87 0x55326B08, 0xA759E80B, 0xB4091BFF, 0x466298FC,
88 0x1871A4D8, 0xEA1A27DB, 0xF94AD42F, 0x0B21572C,
89 0xDFEB33C7, 0x2D80B0C4, 0x3ED04330, 0xCCBBC033,
90 0xA24BB5A6, 0x502036A5, 0x4370C551, 0xB11B4652,
91 0x65D122B9, 0x97BAA1BA, 0x84EA524E, 0x7681D14D,
92 0x2892ED69, 0xDAF96E6A, 0xC9A99D9E, 0x3BC21E9D,
93 0xEF087A76, 0x1D63F975, 0x0E330A81, 0xFC588982,
94 0xB21572C9, 0x407EF1CA, 0x532E023E, 0xA145813D,
95 0x758FE5D6, 0x87E466D5, 0x94B49521, 0x66DF1622,
96 0x38CC2A06, 0xCAA7A905, 0xD9F75AF1, 0x2B9CD9F2,
97 0xFF56BD19, 0x0D3D3E1A, 0x1E6DCDEE, 0xEC064EED,
98 0xC38D26C4, 0x31E6A5C7, 0x22B65633, 0xD0DDD530,
99 0x0417B1DB, 0xF67C32D8, 0xE52CC12C, 0x1747422F,
100 0x49547E0B, 0xBB3FFD08, 0xA86F0EFC, 0x5A048DFF,
101 0x8ECEE914, 0x7CA56A17, 0x6FF599E3, 0x9D9E1AE0,
102 0xD3D3E1AB, 0x21B862A8, 0x32E8915C, 0xC083125F,
103 0x144976B4, 0xE622F5B7, 0xF5720643, 0x07198540,
104 0x590AB964, 0xAB613A67, 0xB831C993, 0x4A5A4A90,
105 0x9E902E7B, 0x6CFBAD78, 0x7FAB5E8C, 0x8DC0DD8F,
106 0xE330A81A, 0x115B2B19, 0x020BD8ED, 0xF0605BEE,
107 0x24AA3F05, 0xD6C1BC06, 0xC5914FF2, 0x37FACCF1,
108 0x69E9F0D5, 0x9B8273D6, 0x88D28022, 0x7AB90321,
109 0xAE7367CA, 0x5C18E4C9, 0x4F48173D, 0xBD23943E,
110 0xF36E6F75, 0x0105EC76, 0x12551F82, 0xE03E9C81,
111 0x34F4F86A, 0xC69F7B69, 0xD5CF889D, 0x27A40B9E,
112 0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E,
113 0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351,
114)
115# fmt: on
118CRC_INIT = 0
120_MASK = 0xFFFFFFFF
123def crc_update(crc, data):
124 """Update CRC-32C checksum with data.
126 Args:
127 crc: 32-bit checksum to update as long.
128 data: byte array, string or iterable over bytes.
129 Returns:
130 32-bit updated CRC-32C as long.
131 """
133 if type(data) != array.array or data.itemsize != 1:
134 buf = array.array("B", data)
135 else:
136 buf = data
138 crc ^= _MASK
139 for b in buf:
140 table_index = (crc ^ b) & 0xFF
141 crc = (CRC_TABLE[table_index] ^ (crc >> 8)) & _MASK
142 return crc ^ _MASK
145def crc_finalize(crc):
146 """Finalize CRC-32C checksum.
148 This function should be called as last step of crc calculation.
149 Args:
150 crc: 32-bit checksum as long.
151 Returns:
152 finalized 32-bit checksum as long
153 """
154 return crc & _MASK
157def crc32c(data):
158 """Compute CRC-32C checksum of the data.
160 Args:
161 data: byte array, string or iterable over bytes.
162 Returns:
163 32-bit CRC-32C checksum of data as long.
164 """
165 return crc_finalize(crc_update(CRC_INIT, data))
168class PyRecordReader_New:
169 def __init__(
170 self, filename=None, start_offset=0, compression_type=None, status=None
171 ):
172 if filename is None:
173 raise errors.NotFoundError(
174 None, None, "No filename provided, cannot read Events"
175 )
176 if not gfile.exists(filename):
177 raise errors.NotFoundError(
178 None,
179 None,
180 "{} does not point to valid Events file".format(filename),
181 )
182 if start_offset:
183 raise errors.UnimplementedError(
184 None, None, "start offset not supported by compat reader"
185 )
186 if compression_type:
187 # TODO: Handle gzip and zlib compressed files
188 raise errors.UnimplementedError(
189 None, None, "compression not supported by compat reader"
190 )
191 self.filename = filename
192 self.start_offset = start_offset
193 self.compression_type = compression_type
194 self.status = status
195 self.curr_event = None
196 self.file_handle = gfile.GFile(self.filename, "rb")
197 # Maintain a buffer of partially read records, so we can recover from
198 # truncated records upon a retry.
199 self._buffer = b""
200 self._buffer_pos = 0
202 def GetNext(self):
203 # Each new read should start at the beginning of any partial record.
204 self._buffer_pos = 0
205 # Read the header
206 self.curr_event = None
207 header_str = self._read(8)
208 if not header_str:
209 # Hit EOF so raise and exit
210 raise errors.OutOfRangeError(None, None, "No more events to read")
211 if len(header_str) < 8:
212 raise self._truncation_error("header")
213 header = struct.unpack("<Q", header_str)
215 # Read the crc32, which is 4 bytes, and check it against
216 # the crc32 of the header
217 crc_header_str = self._read(4)
218 if len(crc_header_str) < 4:
219 raise self._truncation_error("header crc")
220 crc_header = struct.unpack("<I", crc_header_str)
221 header_crc_calc = masked_crc32c(header_str)
222 if header_crc_calc != crc_header[0]:
223 raise errors.DataLossError(
224 None, None, "{} failed header crc32 check".format(self.filename)
225 )
227 # The length of the header tells us how many bytes the Event
228 # string takes
229 header_len = int(header[0])
230 event_str = self._read(header_len)
231 if len(event_str) < header_len:
232 raise self._truncation_error("data")
234 event_crc_calc = masked_crc32c(event_str)
236 # The next 4 bytes contain the crc32 of the Event string,
237 # which we check for integrity.
238 crc_event_str = self._read(4)
239 if len(crc_event_str) < 4:
240 raise self._truncation_error("data crc")
241 crc_event = struct.unpack("<I", crc_event_str)
242 if event_crc_calc != crc_event[0]:
243 raise errors.DataLossError(
244 None,
245 None,
246 "{} failed event crc32 check".format(self.filename),
247 )
249 # Set the current event to be read later by record() call
250 self.curr_event = event_str
251 # Clear the buffered partial record since we're done reading it.
252 self._buffer = b""
254 def _read(self, n):
255 """Read up to n bytes from the underlying file, with buffering.
257 Reads are satisfied from a buffer of previous data read starting at
258 `self._buffer_pos` until the buffer is exhausted, and then from the
259 actual underlying file. Any new data is added to the buffer, and
260 `self._buffer_pos` is advanced to the point in the buffer past all
261 data returned as part of this read.
263 Args:
264 n: non-negative number of bytes to read
266 Returns:
267 bytestring of data read, up to n bytes
268 """
269 result = self._buffer[self._buffer_pos : self._buffer_pos + n]
270 self._buffer_pos += len(result)
271 n -= len(result)
272 if n > 0:
273 new_data = self.file_handle.read(n)
274 result += new_data
275 self._buffer += new_data
276 self._buffer_pos += len(new_data)
277 return result
279 def _truncation_error(self, section):
280 return errors.DataLossError(
281 None,
282 None,
283 "{} has truncated record in {}".format(self.filename, section),
284 )
286 def record(self):
287 return self.curr_event