Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/tables/tests/common.py: 24%

1"""Utilities for PyTables' test suites.""" 

2 

3import os 

4import re 

5import sys 

6import locale 

7import platform 

8import tempfile 

9from pathlib import Path 

10from time import perf_counter as clock 

11from packaging.version import Version 

12 

13import unittest 

14 

15import numexpr as ne 

16import numpy as np 

17 

18import tables as tb 

19 

20hdf5_version = Version(tb.hdf5_version) 

21blosc_version = Version(tb.which_lib_version("blosc")[1]) 

22blosc2_version = Version(tb.which_lib_version("blosc2")[1]) 

23 

24 

25verbose = os.environ.get("VERBOSE", "FALSE") == "TRUE" 

26"""Show detailed output of the testing process.""" 

27 

28heavy = False 

29"""Run all tests even when they take long to complete.""" 

30 

31show_memory = False 

32"""Show the progress of memory consumption.""" 

33 

34 

35def parse_argv(argv): 

36 global verbose, heavy 

37 

38 if 'verbose' in argv: 

39 verbose = True 

40 argv.remove('verbose') 

41 

42 if 'silent' in argv: # take care of old flag, just in case 

43 verbose = False 

44 argv.remove('silent') 

45 

46 if '--heavy' in argv: 

47 heavy = True 

48 argv.remove('--heavy') 

49 

50 return argv 

51 

52 

53zlib_avail = tb.which_lib_version("zlib") is not None 

54lzo_avail = tb.which_lib_version("lzo") is not None 

55bzip2_avail = tb.which_lib_version("bzip2") is not None 

56blosc_avail = tb.which_lib_version("blosc") is not None 

57blosc2_avail = tb.which_lib_version("blosc2") is not None 

58 

59 

60def print_heavy(heavy): 

61 if heavy: 

62 print("""Performing the complete test suite!""") 

63 else: 

64 print("""\ 

65Performing only a light (yet comprehensive) subset of the test suite. 

66If you want a more complete test, try passing the --heavy flag to this script 

67(or set the 'heavy' parameter in case you are using tables.test() call). 

68The whole suite will take more than 4 hours to complete on a relatively 

69modern CPU and around 512 MB of main memory.""") 

70 print('-=' * 38) 

71 

72 

73def print_versions(): 

74 """Print all the versions of software that PyTables relies on.""" 

75 

76 print('-=' * 38) 

77 print("PyTables version: %s" % tb.__version__) 

78 print("HDF5 version: %s" % tb.which_lib_version("hdf5")[1]) 

79 print("NumPy version: %s" % np.__version__) 

80 tinfo = tb.which_lib_version("zlib") 

81 if ne.use_vml: 

82 # Get only the main version number and strip out all the rest 

83 vml_version = ne.get_vml_version() 

84 vml_version = re.findall("[0-9.]+", vml_version)[0] 

85 vml_avail = "using VML/MKL %s" % vml_version 

86 else: 

87 vml_avail = "not using Intel's VML/MKL" 

88 print(f"Numexpr version: {ne.__version__} ({vml_avail})") 

89 if tinfo is not None: 

90 print(f"Zlib version: {tinfo[1]} (in Python interpreter)") 

91 tinfo = tb.which_lib_version("lzo") 

92 if tinfo is not None: 

93 print("LZO version: {} ({})".format(tinfo[1], tinfo[2])) 

94 tinfo = tb.which_lib_version("bzip2") 

95 if tinfo is not None: 

96 print("BZIP2 version: {} ({})".format(tinfo[1], tinfo[2])) 

97 tinfo = tb.which_lib_version("blosc") 

98 if tinfo is not None: 

99 blosc_date = tinfo[2].split()[1] 

100 print("Blosc version: {} ({})".format(tinfo[1], blosc_date)) 

101 blosc_cinfo = tb.blosc_get_complib_info() 

102 blosc_cinfo = [ 

103 "{} ({})".format(k, v[1]) for k, v in sorted(blosc_cinfo.items()) 

104 ] 

105 print("Blosc compressors: %s" % ', '.join(blosc_cinfo)) 

106 blosc_finfo = ['shuffle', 'bitshuffle'] 

107 print("Blosc filters: %s" % ', '.join(blosc_finfo)) 

108 tinfo = tb.which_lib_version("blosc2") 

109 if tinfo is not None: 

110 blosc2_date = tinfo[2].split()[1] 

111 print("Blosc2 version: {} ({})".format(tinfo[1], blosc2_date)) 

112 blosc2_cinfo = tb.blosc2_get_complib_info() 

113 blosc2_cinfo = [ 

114 "{} ({})".format(k, v[1]) for k, v in sorted(blosc2_cinfo.items()) 

115 ] 

116 print("Blosc2 compressors: %s" % ', '.join(blosc2_cinfo)) 

117 blosc2_finfo = ['shuffle', 'bitshuffle'] 

118 print("Blosc2 filters: %s" % ', '.join(blosc2_finfo)) 

119 try: 

120 from Cython import __version__ as cython_version 

121 print('Cython version: %s' % cython_version) 

122 except Exception: 

123 pass 

124 print('Python version: %s' % sys.version) 

125 print('Platform: %s' % platform.platform()) 

126 # if os.name == 'posix': 

127 # (sysname, nodename, release, version, machine) = os.uname() 

128 # print('Platform: %s-%s' % (sys.platform, machine)) 

129 print('Byte-ordering: %s' % sys.byteorder) 

130 print('Detected cores: %s' % tb.utils.detect_number_of_cores()) 

131 print('Default encoding: %s' % sys.getdefaultencoding()) 

132 print('Default FS encoding: %s' % sys.getfilesystemencoding()) 

133 print('Default locale: (%s, %s)' % locale.getdefaultlocale()) 

134 print('-=' * 38) 

135 

136 # This should improve readability whan tests are run by CI tools 

137 sys.stdout.flush() 

138 

139 

140def test_filename(filename): 

141 from pkg_resources import resource_filename 

142 return resource_filename('tables.tests', filename) 

143 

144 

145def verbosePrint(string, nonl=False): 

146 """Print out the `string` if verbose output is enabled.""" 

147 if not verbose: 

148 return 

149 if nonl: 

150 print(string, end=' ') 

151 else: 

152 print(string) 

153 

154 

155def allequal(a, b, flavor="numpy"): 

156 """Checks if two numerical objects are equal.""" 

157 

158 # print("a-->", repr(a)) 

159 # print("b-->", repr(b)) 

160 if not hasattr(b, "shape"): 

161 # Scalar case 

162 return a == b 

163 

164 if ((not hasattr(a, "shape") or a.shape == ()) and 

165 (not hasattr(b, "shape") or b.shape == ())): 

166 return a == b 

167 

168 if a.shape != b.shape: 

169 if verbose: 

170 print("Shape is not equal:", a.shape, "!=", b.shape) 

171 return 0 

172 

173 # Way to check the type equality without byteorder considerations 

174 if hasattr(b, "dtype") and a.dtype.str[1:] != b.dtype.str[1:]: 

175 if verbose: 

176 print("dtype is not equal:", a.dtype, "!=", b.dtype) 

177 return 0 

178 

179 # Rank-0 case 

180 if len(a.shape) == 0: 

181 if a[()] == b[()]: 

182 return 1 

183 else: 

184 if verbose: 

185 print("Shape is not equal:", a.shape, "!=", b.shape) 

186 return 0 

187 

188 # null arrays 

189 if a.size == 0: # len(a) is not correct for generic shapes 

190 if b.size == 0: 

191 return 1 

192 else: 

193 if verbose: 

194 print("length is not equal") 

195 print("len(a.data) ==>", len(a.data)) 

196 print("len(b.data) ==>", len(b.data)) 

197 return 0 

198 

199 # Multidimensional case 

200 result = (a == b) 

201 result = np.all(result) 

202 if not result and verbose: 

203 print("Some of the elements in arrays are not equal") 

204 

205 return result 

206 

207 

208def areArraysEqual(arr1, arr2): 

209 """Are both `arr1` and `arr2` equal arrays? 

210 

211 Arguments can be regular NumPy arrays, chararray arrays or 

212 structured arrays (including structured record arrays). They are 

213 checked for type and value equality. 

214 

215 """ 

216 

217 t1 = type(arr1) 

218 t2 = type(arr2) 

219 

220 if not ((hasattr(arr1, 'dtype') and arr1.dtype == arr2.dtype) or 

221 issubclass(t1, t2) or issubclass(t2, t1)): 

222 return False 

223 

224 return np.all(arr1 == arr2) 

225 

226 

227class PyTablesTestCase(unittest.TestCase): 

228 def tearDown(self): 

229 super().tearDown() 

230 for key in self.__dict__: 

231 if self.__dict__[key].__class__.__name__ != 'instancemethod': 

232 self.__dict__[key] = None 

233 

234 def _getName(self): 

235 """Get the name of this test case.""" 

236 return self.id().split('.')[-2] 

237 

238 def _getMethodName(self): 

239 """Get the name of the method currently running in the test case.""" 

240 return self.id().split('.')[-1] 

241 

242 def _verboseHeader(self): 

243 """Print a nice header for the current test method if verbose.""" 

244 

245 if verbose: 

246 name = self._getName() 

247 methodName = self._getMethodName() 

248 

249 title = f"Running {name}.{methodName}" 

250 print('{}\n{}'.format(title, '-' * len(title))) 

251 

252 def _checkEqualityGroup(self, node1, node2, hardlink=False): 

253 if verbose: 

254 print("Group 1:", node1) 

255 print("Group 2:", node2) 

256 if hardlink: 

257 self.assertTrue( 

258 node1._v_pathname != node2._v_pathname, 

259 "node1 and node2 have the same pathnames.") 

260 else: 

261 self.assertTrue( 

262 node1._v_pathname == node2._v_pathname, 

263 "node1 and node2 does not have the same pathnames.") 

264 self.assertTrue( 

265 node1._v_children == node2._v_children, 

266 "node1 and node2 does not have the same children.") 

267 

268 def _checkEqualityLeaf(self, node1, node2, hardlink=False): 

269 if verbose: 

270 print("Leaf 1:", node1) 

271 print("Leaf 2:", node2) 

272 if hardlink: 

273 self.assertTrue( 

274 node1._v_pathname != node2._v_pathname, 

275 "node1 and node2 have the same pathnames.") 

276 else: 

277 self.assertTrue( 

278 node1._v_pathname == node2._v_pathname, 

279 "node1 and node2 does not have the same pathnames.") 

280 self.assertTrue( 

281 areArraysEqual(node1[:], node2[:]), 

282 "node1 and node2 does not have the same values.") 

283 

284 

285class TestFileMixin: 

286 h5fname = None 

287 open_kwargs = {} 

288 

289 def setUp(self): 

290 super().setUp() 

291 self.h5file = tb.open_file( 

292 self.h5fname, title=self._getName(), **self.open_kwargs) 

293 

294 def tearDown(self): 

295 """Close ``h5file``.""" 

296 

297 self.h5file.close() 

298 super().tearDown() 

299 

300 

301class TempFileMixin: 

302 open_mode = 'w' 

303 open_kwargs = {} 

304 

305 def _getTempFileName(self): 

306 return tempfile.mktemp(prefix=self._getName(), suffix='.h5') 

307 

308 def setUp(self): 

309 """Set ``h5file`` and ``h5fname`` instance attributes. 

310 

311 * ``h5fname``: the name of the temporary HDF5 file. 

312 * ``h5file``: the writable, empty, temporary HDF5 file. 

313 

314 """ 

315 

316 super().setUp() 

317 self.h5fname = self._getTempFileName() 

318 self.h5file = tb.open_file( 

319 self.h5fname, self.open_mode, title=self._getName(), 

320 **self.open_kwargs) 

321 

322 def tearDown(self): 

323 """Close ``h5file`` and remove ``h5fname``.""" 

324 

325 self.h5file.close() 

326 self.h5file = None 

327 Path(self.h5fname).unlink() # comment this for debug only 

328 super().tearDown() 

329 

330 def _reopen(self, mode='r', **kwargs): 

331 """Reopen ``h5file`` in the specified ``mode``. 

332 

333 Returns a true or false value depending on whether the file was 

334 reopenend or not. If not, nothing is changed. 

335 

336 """ 

337 

338 self.h5file.close() 

339 self.h5file = tb.open_file(self.h5fname, mode, **kwargs) 

340 return True 

341 

342 

343class ShowMemTime(PyTablesTestCase): 

344 tref = clock() 

345 """Test for showing memory and time consumption.""" 

346 

347 def test00(self): 

348 """Showing memory and time consumption.""" 

349 

350 # Obtain memory info (only for Linux 2.6.x) 

351 for line in Path("/proc/self/status").read_text().splitlines(): 

352 if line.startswith("VmSize:"): 

353 vmsize = int(line.split()[1]) 

354 elif line.startswith("VmRSS:"): 

355 vmrss = int(line.split()[1]) 

356 elif line.startswith("VmData:"): 

357 vmdata = int(line.split()[1]) 

358 elif line.startswith("VmStk:"): 

359 vmstk = int(line.split()[1]) 

360 elif line.startswith("VmExe:"): 

361 vmexe = int(line.split()[1]) 

362 elif line.startswith("VmLib:"): 

363 vmlib = int(line.split()[1]) 

364 print("\nWallClock time:", clock() - self.tref) 

365 print("Memory usage: ******* %s *******" % self._getName()) 

366 print(f"VmSize: {vmsize:>7} kB\tVmRSS: {vmrss:>7} kB") 

367 print(f"VmData: {vmdata:>7} kB\tVmStk: {vmstk:>7} kB") 

368 print(f"VmExe: {vmexe:>7} kB\tVmLib: {vmlib:>7} kB")