Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/joblib/hashing.py: 26%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Fast cryptographic hash of Python objects, with a special case for fast
3hashing of numpy arrays.
4"""
6# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
7# Copyright (c) 2009 Gael Varoquaux
8# License: BSD Style, 3 clauses.
10import decimal
11import hashlib
12import io
13import pickle
14import sys
15import types
17Pickler = pickle._Pickler
20class _ConsistentSet(object):
21 """Class used to ensure the hash of Sets is preserved
22 whatever the order of its items.
23 """
25 def __init__(self, set_sequence):
26 seq_type = type(set_sequence)
27 if seq_type is not set:
28 self.type = seq_type
29 # Forces order of elements in set to ensure consistent hash.
30 try:
31 # Trying first to order the set assuming the type of elements is
32 # consistent and orderable.
33 # This fails on python 3 when elements are unorderable
34 # but we keep it in a try as it's faster.
35 self._sequence = sorted(set_sequence)
36 except (TypeError, decimal.InvalidOperation):
37 # If elements are unorderable, sorting them using their hash.
38 # This is slower but works in any case.
39 self._sequence = sorted((hash(e) for e in set_sequence))
42class _MyHash(object):
43 """Class used to hash objects that won't normally pickle"""
45 def __init__(self, *args):
46 self.args = args
49class Hasher(Pickler):
50 """A subclass of pickler, to do cryptographic hashing, rather than
51 pickling. This is used to produce a unique hash of the given
52 Python object that is not necessarily cryptographically secure.
53 """
55 def __init__(self, hash_name="md5"):
56 self.stream = io.BytesIO()
57 # By default we want a pickle protocol that only changes with
58 # the major python version and not the minor one
59 protocol = 3
60 Pickler.__init__(self, self.stream, protocol=protocol)
61 # Initialise the hash obj
62 self._hash = hashlib.new(hash_name, usedforsecurity=False)
64 def hash(self, obj, return_digest=True):
65 try:
66 self.dump(obj)
67 except pickle.PicklingError as e:
68 e.args += ("PicklingError while hashing %r: %r" % (obj, e),)
69 raise
70 dumps = self.stream.getvalue()
71 self._hash.update(dumps)
72 if return_digest:
73 return self._hash.hexdigest()
75 def save(self, obj):
76 if isinstance(obj, (types.MethodType, types.BuiltinMethodType)):
77 # the Pickler cannot pickle instance methods; here we decompose
78 # them into components that make them uniquely identifiable
79 if hasattr(obj, "__func__"):
80 func_name = obj.__func__.__name__
81 else:
82 func_name = obj.__name__
83 inst = obj.__self__
84 if type(inst) is type(pickle):
85 obj = _MyHash(func_name, inst.__name__)
86 elif inst is None:
87 # type(None) or type(module) do not pickle
88 obj = _MyHash(func_name, inst)
89 else:
90 cls = obj.__self__.__class__
91 obj = _MyHash(func_name, inst, cls)
92 Pickler.save(self, obj)
94 def memoize(self, obj):
95 # We want hashing to be sensitive to value instead of reference.
96 # For example we want ['aa', 'aa'] and ['aa', 'aaZ'[:2]]
97 # to hash to the same value and that's why we disable memoization
98 # for strings
99 if isinstance(obj, (bytes, str)):
100 return
101 Pickler.memoize(self, obj)
103 # The dispatch table of the pickler is not accessible in Python
104 # 3, as these lines are only bugware for IPython, we skip them.
105 def save_global(self, obj, name=None):
106 # We have to override this method in order to deal with objects
107 # defined interactively in IPython that are not injected in
108 # __main__
109 try:
110 Pickler.save_global(self, obj, name=name)
111 except pickle.PicklingError:
112 Pickler.save_global(self, obj, name=name)
113 module = getattr(obj, "__module__", None)
114 if module == "__main__":
115 my_name = name
116 if my_name is None:
117 my_name = obj.__name__
118 mod = sys.modules[module]
119 if not hasattr(mod, my_name):
120 # IPython doesn't inject the variables define
121 # interactively in __main__
122 setattr(mod, my_name, obj)
124 dispatch = Pickler.dispatch.copy()
125 # builtin
126 dispatch[type(len)] = save_global
128 # We use *args in _batch_setitems signature because _batch_setitems has an
129 # additional 'obj' argument in Python 3.14
130 def _batch_setitems(self, items, *args):
131 # forces order of keys in dict to ensure consistent hash.
132 try:
133 # Trying first to compare dict assuming the type of keys is
134 # consistent and orderable.
135 # This fails on python 3 when keys are unorderable
136 # but we keep it in a try as it's faster.
137 Pickler._batch_setitems(self, iter(sorted(items)), *args)
138 except TypeError:
139 # If keys are unorderable, sorting them using their hash. This is
140 # slower but works in any case.
141 Pickler._batch_setitems(
142 self, iter(sorted((hash(k), v) for k, v in items)), *args
143 )
145 def save_set(self, set_items):
146 # forces order of items in Set to ensure consistent hash
147 Pickler.save(self, _ConsistentSet(set_items))
149 dispatch[set] = save_set
150 dispatch[frozenset] = save_set
152 def save_code(self, code):
153 """Digest the contents of a code object.
154 The fields below mirror CPython's own ``code_hash()``:
155 https://github.com/python/cpython/blob/5918085bb6f4a3a48193cacb9bb99b044d4e0452/Objects/codeobject.c#L2608-L2624
156 co_localsplusnames is spelled out as varnames/cellvars/freevars. Only the
157 position information, co_firstlineno and co_linetable, is deliberately left
158 out: joblib compares source *text* and tracks the line number separately,
159 so moving a function without editing it must not invalidate its cache. For
160 the same reason co_filename is not included, which would key a notebook
161 function to the cell number it last ran in.
162 """
163 obj = _MyHash(
164 code.co_name,
165 code.co_consts,
166 code.co_names,
167 code.co_varnames,
168 code.co_cellvars,
169 code.co_freevars,
170 code.co_exceptiontable,
171 code.co_argcount,
172 code.co_posonlyargcount,
173 code.co_kwonlyargcount,
174 code.co_flags,
175 # co_code de-specializes adaptive bytecode, like _Py_GetBaseCodeUnit
176 code.co_code,
177 )
178 Pickler.save(self, obj)
180 dispatch[type(save_code.__code__)] = save_code
183class NumpyHasher(Hasher):
184 """Special case the hasher for when numpy is loaded."""
186 def __init__(self, hash_name="md5", coerce_mmap=False):
187 """
188 Parameters
189 ----------
190 hash_name: string
191 The hash algorithm to be used
192 coerce_mmap: boolean
193 Make no difference between np.memmap and np.ndarray
194 objects.
195 """
196 self.coerce_mmap = coerce_mmap
197 Hasher.__init__(self, hash_name=hash_name)
198 # delayed import of numpy, to avoid tight coupling
199 import numpy as np
201 self.np = np
202 if hasattr(np, "getbuffer"):
203 self._getbuffer = np.getbuffer
204 else:
205 self._getbuffer = memoryview
207 def save(self, obj):
208 """Subclass the save method, to hash ndarray subclass, rather
209 than pickling them. Off course, this is a total abuse of
210 the Pickler class.
211 """
212 if isinstance(obj, self.np.ndarray) and not obj.dtype.hasobject:
213 # Compute a hash of the object
214 # The update function of the hash requires a c_contiguous buffer.
215 if obj.shape == ():
216 # 0d arrays need to be flattened because viewing them as bytes
217 # raises a ValueError exception.
218 obj_c_contiguous = obj.flatten()
219 elif obj.flags.c_contiguous:
220 obj_c_contiguous = obj
221 elif obj.flags.f_contiguous:
222 obj_c_contiguous = obj.T
223 else:
224 # Cater for non-single-segment arrays: this creates a
225 # copy, and thus alleviates this issue.
226 # XXX: There might be a more efficient way of doing this
227 obj_c_contiguous = obj.flatten()
229 # memoryview is not supported for some dtypes, e.g. datetime64, see
230 # https://github.com/numpy/numpy/issues/4983. The
231 # workaround is to view the array as bytes before
232 # taking the memoryview.
233 self._hash.update(self._getbuffer(obj_c_contiguous.view(self.np.uint8)))
235 # We store the class, to be able to distinguish between
236 # Objects with the same binary content, but different
237 # classes.
238 if self.coerce_mmap and isinstance(obj, self.np.memmap):
239 # We don't make the difference between memmap and
240 # normal ndarrays, to be able to reload previously
241 # computed results with memmap.
242 klass = self.np.ndarray
243 else:
244 klass = obj.__class__
245 # We also return the dtype and the shape, to distinguish
246 # different views on the same data with different dtypes.
248 # The object will be pickled by the pickler hashed at the end.
249 obj = (klass, ("HASHED", obj.dtype, obj.shape, obj.strides))
250 elif isinstance(obj, self.np.dtype):
251 # numpy.dtype consistent hashing is tricky to get right. This comes
252 # from the fact that atomic np.dtype objects are interned:
253 # ``np.dtype('f4') is np.dtype('f4')``. The situation is
254 # complicated by the fact that this interning does not resist a
255 # simple pickle.load/dump roundtrip:
256 # ``pickle.loads(pickle.dumps(np.dtype('f4'))) is not
257 # np.dtype('f4') Because pickle relies on memoization during
258 # pickling, it is easy to
259 # produce different hashes for seemingly identical objects, such as
260 # ``[np.dtype('f4'), np.dtype('f4')]``
261 # and ``[np.dtype('f4'), pickle.loads(pickle.dumps('f4'))]``.
262 # To prevent memoization from interfering with hashing, we isolate
263 # the serialization (and thus the pickle memoization) of each dtype
264 # using each time a different ``pickle.dumps`` call unrelated to
265 # the current Hasher instance.
266 self._hash.update("_HASHED_DTYPE".encode("utf-8"))
267 self._hash.update(pickle.dumps(obj))
268 return
269 Hasher.save(self, obj)
272def hash(obj, hash_name="md5", coerce_mmap=False):
273 """Quick calculation of a hash to identify uniquely Python objects
274 containing numpy arrays.
276 Parameters
277 ----------
278 hash_name: 'md5' or 'sha1'
279 Hashing algorithm used. sha1 is supposedly safer, but md5 is
280 faster.
281 coerce_mmap: boolean
282 Make no difference between np.memmap and np.ndarray
283 """
284 valid_hash_names = ("md5", "sha1")
285 if hash_name not in valid_hash_names:
286 raise ValueError(
287 "Valid options for 'hash_name' are {}. Got hash_name={!r} instead.".format(
288 valid_hash_names, hash_name
289 )
290 )
291 if "numpy" in sys.modules:
292 hasher = NumpyHasher(hash_name=hash_name, coerce_mmap=coerce_mmap)
293 else:
294 hasher = Hasher(hash_name=hash_name)
295 return hasher.hash(obj)