Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/numexpr-2.8.7.dev1-py3.8-linux-x86_64.egg/numexpr/utils.py: 47%

1################################################################### 

2# Numexpr - Fast numerical array expression evaluator for NumPy. 

3# 

4# License: MIT 

5# Author: See AUTHORS.txt 

6# 

7# See LICENSE.txt and LICENSES/*.txt for details about copyright and 

8# rights to use. 

9#################################################################### 

10 

11import logging 

12log = logging.getLogger(__name__) 

13 

14import os 

15import subprocess 

16import platform 

17 

18from numexpr.interpreter import _set_num_threads, _get_num_threads, MAX_THREADS 

19from numexpr import use_vml 

20from . import version 

21 

22if use_vml: 

23 from numexpr.interpreter import ( 

24 _get_vml_version, _set_vml_accuracy_mode, _set_vml_num_threads, 

25 _get_vml_num_threads) 

26 

27 

28def get_vml_version(): 

29 """ 

30 Get the VML/MKL library version. 

31 """ 

32 if use_vml: 

33 return _get_vml_version() 

34 else: 

35 return None 

36 

37 

38def set_vml_accuracy_mode(mode): 

39 """ 

40 Set the accuracy mode for VML operations. 

41 

42 The `mode` parameter can take the values: 

43 - 'high': high accuracy mode (HA), <1 least significant bit 

44 - 'low': low accuracy mode (LA), typically 1-2 least significant bits 

45 - 'fast': enhanced performance mode (EP) 

46 - None: mode settings are ignored 

47 

48 This call is equivalent to the `vmlSetMode()` in the VML library. 

49 See: 

50 

51 http://www.intel.com/software/products/mkl/docs/webhelp/vml/vml_DataTypesAccuracyModes.html 

52 

53 for more info on the accuracy modes. 

54 

55 Returns old accuracy settings. 

56 """ 

57 if use_vml: 

58 acc_dict = {None: 0, 'low': 1, 'high': 2, 'fast': 3} 

59 acc_reverse_dict = {1: 'low', 2: 'high', 3: 'fast'} 

60 if mode not in list(acc_dict.keys()): 

61 raise ValueError( 

62 "mode argument must be one of: None, 'high', 'low', 'fast'") 

63 retval = _set_vml_accuracy_mode(acc_dict.get(mode, 0)) 

64 return acc_reverse_dict.get(retval) 

65 else: 

66 return None 

67 

68 

69def set_vml_num_threads(nthreads): 

70 """ 

71 Suggests a maximum number of threads to be used in VML operations. 

72 

73 This function is equivalent to the call 

74 `mkl_domain_set_num_threads(nthreads, MKL_DOMAIN_VML)` in the MKL 

75 library. See: 

76 

77 http://www.intel.com/software/products/mkl/docs/webhelp/support/functn_mkl_domain_set_num_threads.html 

78 

79 for more info about it. 

80 """ 

81 if use_vml: 

82 _set_vml_num_threads(nthreads) 

83 pass 

84 

85def get_vml_num_threads(): 

86 """ 

87 Gets the maximum number of threads to be used in VML operations. 

88 

89 This function is equivalent to the call 

90 `mkl_domain_get_max_threads (MKL_DOMAIN_VML)` in the MKL 

91 library. See: 

92 

93 http://software.intel.com/en-us/node/522118 

94 

95 for more info about it. 

96 """ 

97 if use_vml: 

98 return _get_vml_num_threads() 

99 return None 

100 

101def set_num_threads(nthreads): 

102 """ 

103 Sets a number of threads to be used in operations. 

104 

105 DEPRECATED: returns the previous setting for the number of threads. 

106 

107 During initialization time NumExpr sets this number to the number 

108 of detected cores in the system (see `detect_number_of_cores()`). 

109 """ 

110 old_nthreads = _set_num_threads(nthreads) 

111 return old_nthreads 

112 

113def get_num_threads(): 

114 """ 

115 Gets the number of threads currently in use for operations. 

116 """ 

117 return _get_num_threads() 

118 

119def _init_num_threads(): 

120 """ 

121 Detects the environment variable 'NUMEXPR_MAX_THREADS' to set the threadpool  

122 size, and if necessary the slightly redundant 'NUMEXPR_NUM_THREADS' or  

123 'OMP_NUM_THREADS' env vars to set the initial number of threads used by  

124 the virtual machine. 

125 """ 

126 # Any platform-specific short-circuits 

127 if 'sparc' in version.platform_machine: 

128 log.warning('The number of threads have been set to 1 because problems related ' 

129 'to threading have been reported on some sparc machine. ' 

130 'The number of threads can be changed using the "set_num_threads" ' 

131 'function.') 

132 set_num_threads(1) 

133 return 1 

134 

135 env_configured = False 

136 n_cores = detect_number_of_cores() 

137 if 'NUMEXPR_MAX_THREADS' in os.environ: 

138 # The user has configured NumExpr in the expected way, so suppress logs. 

139 env_configured = True 

140 n_cores = MAX_THREADS 

141 else: 

142 # The use has not set 'NUMEXPR_MAX_THREADS', so likely they have not  

143 # configured NumExpr as desired, so we emit info logs. 

144 if n_cores > MAX_THREADS: 

145 log.info('Note: detected %d virtual cores but NumExpr set to maximum of %d, check "NUMEXPR_MAX_THREADS" environment variable.'%(n_cores, MAX_THREADS)) 

146 if n_cores > 8: 

147 # The historical 'safety' limit. 

148 log.info('Note: NumExpr detected %d cores but "NUMEXPR_MAX_THREADS" not set, so enforcing safe limit of 8.'%n_cores) 

149 n_cores = 8 

150 

151 # Now we check for 'NUMEXPR_NUM_THREADS' or 'OMP_NUM_THREADS' to set the  

152 # actual number of threads used. 

153 if 'NUMEXPR_NUM_THREADS' in os.environ: 

154 requested_threads = int(os.environ['NUMEXPR_NUM_THREADS']) 

155 elif 'OMP_NUM_THREADS' in os.environ: 

156 requested_threads = int(os.environ['OMP_NUM_THREADS']) 

157 else: 

158 requested_threads = n_cores 

159 if not env_configured: 

160 log.info('NumExpr defaulting to %d threads.'%n_cores) 

161 

162 # The C-extension function performs its own checks against `MAX_THREADS` 

163 set_num_threads(requested_threads) 

164 return requested_threads 

165 

166 

167def detect_number_of_cores(): 

168 """ 

169 Detects the number of cores on a system. Cribbed from pp. 

170 """ 

171 # Linux, Unix and MacOS: 

172 if hasattr(os, "sysconf"): 

173 if "SC_NPROCESSORS_ONLN" in os.sysconf_names: 

174 # Linux & Unix: 

175 ncpus = os.sysconf("SC_NPROCESSORS_ONLN") 

176 if isinstance(ncpus, int) and ncpus > 0: 

177 return ncpus 

178 else: # OSX: 

179 return int(subprocess.check_output(["sysctl", "-n", "hw.ncpu"])) 

180 # Windows: 

181 try: 

182 ncpus = int(os.environ.get("NUMBER_OF_PROCESSORS", "")) 

183 if ncpus > 0: 

184 return ncpus 

185 except ValueError: 

186 pass 

187 return 1 # Default 

188 

189 

190def detect_number_of_threads(): 

191 """ 

192 DEPRECATED: use `_init_num_threads` instead. 

193 If this is modified, please update the note in: https://github.com/pydata/numexpr/wiki/Numexpr-Users-Guide 

194 """ 

195 log.warning('Deprecated, use `init_num_threads` instead.') 

196 try: 

197 nthreads = int(os.environ.get('NUMEXPR_NUM_THREADS', '')) 

198 except ValueError: 

199 try: 

200 nthreads = int(os.environ.get('OMP_NUM_THREADS', '')) 

201 except ValueError: 

202 nthreads = detect_number_of_cores() 

203 

204 # Check that we don't surpass the MAX_THREADS in interpreter.cpp 

205 if nthreads > MAX_THREADS: 

206 nthreads = MAX_THREADS 

207 return nthreads 

208 

209 

210class CacheDict(dict): 

211 """ 

212 A dictionary that prevents itself from growing too much. 

213 """ 

214 

215 def __init__(self, maxentries): 

216 self.maxentries = maxentries 

217 super(CacheDict, self).__init__(self) 

218 

219 def __setitem__(self, key, value): 

220 # Protection against growing the cache too much 

221 if len(self) > self.maxentries: 

222 # Remove a 10% of (arbitrary) elements from the cache 

223 entries_to_remove = self.maxentries // 10 

224 for k in list(self.keys())[:entries_to_remove]: 

225 super(CacheDict, self).__delitem__(k) 

226 super(CacheDict, self).__setitem__(key, value) 

227