Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/joblib/memory.py: 24%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2A context object for caching a function's return value each time it
3is called with the same input arguments.
5"""
7# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
8# Copyright (c) 2009 Gael Varoquaux
9# License: BSD Style, 3 clauses.
11import asyncio
12import datetime
13import functools
14import inspect
15import logging
16import os
17import pathlib
18import pydoc
19import re
20import textwrap
21import time
22import tokenize
23import traceback
24import warnings
25import weakref
27from . import hashing
28from ._store_backends import (
29 CacheWarning, # noqa
30 FileSystemStoreBackend,
31 StoreBackendBase,
32)
33from .func_inspect import (
34 filter_args,
35 format_call,
36 format_signature,
37 get_func_code,
38 get_func_name,
39)
40from .logger import Logger, format_time, pformat
42FIRST_LINE_TEXT = "# first line:"
44# TODO: The following object should have a data store object as a sub
45# object, and the interface to persist and query should be separated in
46# the data store.
47#
48# This would enable creating 'Memory' objects with a different logic for
49# pickling that would simply span a MemorizedFunc with the same
50# store (or do we want to copy it to avoid cross-talks?), for instance to
51# implement HDF5 pickling.
53# TODO: Same remark for the logger, and probably use the Python logging
54# mechanism.
57def extract_first_line(func_code):
58 """Extract the first line information from the function code
59 text if available.
60 """
61 if func_code.startswith(FIRST_LINE_TEXT):
62 func_code = func_code.split("\n")
63 first_line = int(func_code[0][len(FIRST_LINE_TEXT) :])
64 func_code = "\n".join(func_code[1:])
65 else:
66 first_line = -1
67 return func_code, first_line
70class JobLibCollisionWarning(UserWarning):
71 """Warn that there might be a collision between names of functions."""
74_STORE_BACKENDS = {"local": FileSystemStoreBackend}
77def register_store_backend(backend_name, backend):
78 """Extend available store backends.
80 The Memory, MemorizeResult and MemorizeFunc objects are designed to be
81 agnostic to the type of store used behind. By default, the local file
82 system is used but this function gives the possibility to extend joblib's
83 memory pattern with other types of storage such as cloud storage (S3, GCS,
84 OpenStack, HadoopFS, etc) or blob DBs.
86 Parameters
87 ----------
88 backend_name: str
89 The name identifying the store backend being registered. For example,
90 'local' is used with FileSystemStoreBackend.
91 backend: StoreBackendBase subclass
92 The name of a class that implements the StoreBackendBase interface.
94 """
95 if not isinstance(backend_name, str):
96 raise ValueError(
97 "Store backend name should be a string, '{0}' given.".format(backend_name)
98 )
99 if backend is None or not issubclass(backend, StoreBackendBase):
100 raise ValueError(
101 "Store backend should inherit StoreBackendBase, '{0}' given.".format(
102 backend
103 )
104 )
106 _STORE_BACKENDS[backend_name] = backend
109def _store_backend_factory(backend, location, verbose=0, backend_options=None):
110 """Return the correct store object for the given location."""
111 if backend_options is None:
112 backend_options = {}
114 if isinstance(location, pathlib.Path):
115 location = str(location)
117 if isinstance(location, StoreBackendBase):
118 return location
119 elif isinstance(location, str):
120 obj = None
121 location = os.path.expanduser(location)
122 # The location is not a local file system, we look in the
123 # registered backends if there's one matching the given backend
124 # name.
125 for backend_key, backend_obj in _STORE_BACKENDS.items():
126 if backend == backend_key:
127 obj = backend_obj()
129 # By default, we assume the FileSystemStoreBackend can be used if no
130 # matching backend could be found.
131 if obj is None:
132 raise TypeError(
133 "Unknown location {0} or backend {1}".format(location, backend)
134 )
136 # The store backend is configured with the extra named parameters,
137 # some of them are specific to the underlying store backend.
138 obj.configure(location, verbose=verbose, backend_options=backend_options)
139 return obj
140 elif location is not None:
141 warnings.warn(
142 "Instantiating a backend using a {} as a location is not "
143 "supported by joblib. Returning None instead.".format(
144 location.__class__.__name__
145 ),
146 UserWarning,
147 )
149 return None
152def _build_func_identifier(func):
153 """Build a roughly unique identifier for the cached function."""
154 modules, funcname = get_func_name(func)
155 # We reuse historical fs-like way of building a function identifier
156 return os.path.join(*modules, funcname)
159# An in-memory store to avoid looking at the disk-based function
160# source code to check if a function definition has changed
161_FUNCTION_HASHES = weakref.WeakKeyDictionary()
164###############################################################################
165# class `MemorizedResult`
166###############################################################################
167class MemorizedResult(Logger):
168 """Object representing a cached value.
170 Attributes
171 ----------
172 location: str
173 The location of joblib cache. Depends on the store backend used.
175 call_id: tuple of str
176 Function identifier and hash of the function arguments identifying the
177 cached value. Instances can be reconstructed from their representation
178 (namely eval(repr(memorized_instance)) works).
180 backend: str
181 Type of store backend for reading/writing cache files.
182 Default is 'local'.
184 mmap_mode: {None, 'r+', 'r', 'w+', 'c'}
185 The memmapping mode used when loading from cache numpy arrays. See
186 numpy.load for the meaning of the different values.
188 verbose: int
189 verbosity level (0 means no message).
191 timestamp, metadata: string
192 for internal use only.
193 """
195 def __init__(
196 self,
197 location,
198 call_id,
199 backend="local",
200 mmap_mode=None,
201 verbose=0,
202 timestamp=None,
203 metadata=None,
204 ):
205 Logger.__init__(self)
206 self._call_id = call_id
207 self.store_backend = _store_backend_factory(
208 backend,
209 location,
210 verbose=verbose,
211 backend_options=dict(mmap_mode=mmap_mode),
212 )
213 self.mmap_mode = mmap_mode
215 if metadata is not None:
216 self.metadata = metadata
217 else:
218 self.metadata = self.store_backend.get_metadata(self._call_id)
220 self.duration = self.metadata.get("duration", None)
221 self.verbose = verbose
222 self.timestamp = timestamp
224 @property
225 def func(self):
226 return self.func_id
228 @property
229 def func_id(self):
230 return self._call_id[0]
232 @property
233 def args_id(self):
234 return self._call_id[1]
236 def get(self):
237 """Read value from cache and return it."""
238 try:
239 return self.store_backend.load_item(
240 self._call_id,
241 timestamp=self.timestamp,
242 metadata=self.metadata,
243 verbose=self.verbose,
244 )
245 except ValueError as exc:
246 new_exc = KeyError(
247 "Error while trying to load a MemorizedResult's value. "
248 "It seems that this folder is corrupted : {}".format(
249 os.path.join(self.store_backend.location, *self._call_id)
250 )
251 )
252 raise new_exc from exc
254 def clear(self):
255 """Clear value from cache"""
256 self.store_backend.clear_item(self._call_id)
258 def __repr__(self):
259 return "{}(location={!r}, call_id={!r})".format(
260 self.__class__.__name__, self.store_backend.location, self._call_id
261 )
263 def __getstate__(self):
264 state = self.__dict__.copy()
265 state["timestamp"] = None
266 return state
269class NotMemorizedResult(object):
270 """Class representing an arbitrary value.
272 This class is a replacement for MemorizedResult when there is no cache.
273 """
275 __slots__ = ("value", "valid")
277 def __init__(self, value):
278 self.value = value
279 self.valid = True
281 def get(self):
282 if self.valid:
283 return self.value
284 else:
285 raise KeyError("No value stored.")
287 def clear(self):
288 self.valid = False
289 self.value = None
291 def __repr__(self):
292 if self.valid:
293 return "{class_name}({value})".format(
294 class_name=self.__class__.__name__, value=pformat(self.value)
295 )
296 else:
297 return self.__class__.__name__ + " with no value"
299 # __getstate__ and __setstate__ are required because of __slots__
300 def __getstate__(self):
301 return {"valid": self.valid, "value": self.value}
303 def __setstate__(self, state):
304 self.valid = state["valid"]
305 self.value = state["value"]
308###############################################################################
309# class `NotMemorizedFunc`
310###############################################################################
311class NotMemorizedFunc(object):
312 """No-op object decorating a function.
314 This class replaces MemorizedFunc when there is no cache. It provides an
315 identical API but does not write anything on disk.
317 Attributes
318 ----------
319 func: callable
320 Original undecorated function.
321 """
323 # Should be a light as possible (for speed)
324 def __init__(self, func):
325 self.func = func
327 def __call__(self, *args, **kwargs):
328 return self.func(*args, **kwargs)
330 def call_and_shelve(self, *args, **kwargs):
331 return NotMemorizedResult(self.func(*args, **kwargs))
333 def __repr__(self):
334 return "{0}(func={1})".format(self.__class__.__name__, self.func)
336 def clear(self, warn=True):
337 # Argument "warn" is for compatibility with MemorizedFunc.clear
338 pass
340 def call(self, *args, **kwargs):
341 return self.func(*args, **kwargs), {}
343 def check_call_in_cache(self, *args, **kwargs):
344 return False
347###############################################################################
348# class `AsyncNotMemorizedFunc`
349###############################################################################
350class AsyncNotMemorizedFunc(NotMemorizedFunc):
351 async def call(self, *args, **kwargs):
352 return await self.func(*args, **kwargs), {}
354 async def call_and_shelve(self, *args, **kwargs):
355 return NotMemorizedResult(await self.func(*args, **kwargs))
358###############################################################################
359# class `MemorizedFunc`
360###############################################################################
361class MemorizedFunc(Logger):
362 """Callable object decorating a function for caching its return value
363 each time it is called.
365 Methods are provided to inspect the cache or clean it.
367 Attributes
368 ----------
369 func: callable
370 The original, undecorated, function.
372 location: string
373 The location of joblib cache. Depends on the store backend used.
375 backend: str
376 Type of store backend for reading/writing cache files.
377 Default is 'local', in which case the location is the path to a
378 disk storage.
380 ignore: list or None
381 List of variable names to ignore when choosing whether to
382 recompute.
384 mmap_mode: {None, 'r+', 'r', 'w+', 'c'}
385 The memmapping mode used when loading from cache
386 numpy arrays. See numpy.load for the meaning of the different
387 values.
389 compress: boolean, integer or 2-tuple, optional
390 Whether to zip the stored data on disk. If an integer is
391 given, it should be between 1 and 9, and sets the amount
392 of compression. Note that compressed arrays cannot be
393 read by memmapping. If a 2-tuple is given, the first
394 element must be a string corresponding to a compression
395 method (e.g 'zlib', 'gzip', 'bz2', 'lzma' or 'xz') and
396 the second element must be an integer from 0 to 9,
397 corresponding to the compression level.
399 verbose: int, optional
400 The verbosity flag, controls messages that are issued as
401 the function is evaluated.
403 cache_validation_callback: callable, optional
404 Callable to check if a result in cache is valid or is to be recomputed.
405 When the function is called with arguments for which a cache exists,
406 the callback is called with the cache entry's metadata as its sole
407 argument. If it returns True, the cached result is returned, else the
408 cache for these arguments is cleared and the result is recomputed.
409 """
411 # ------------------------------------------------------------------------
412 # Public interface
413 # ------------------------------------------------------------------------
415 def __init__(
416 self,
417 func,
418 location,
419 backend="local",
420 ignore=None,
421 mmap_mode=None,
422 compress=False,
423 verbose=1,
424 timestamp=None,
425 cache_validation_callback=None,
426 ):
427 Logger.__init__(self)
428 self.mmap_mode = mmap_mode
429 self.compress = compress
430 self.func = func
431 self.cache_validation_callback = cache_validation_callback
432 self.func_id = _build_func_identifier(func)
433 self.ignore = ignore if ignore is not None else []
434 self._verbose = verbose
436 # retrieve store object from backend type and location.
437 self.store_backend = _store_backend_factory(
438 backend,
439 location,
440 verbose=verbose,
441 backend_options=dict(compress=compress, mmap_mode=mmap_mode),
442 )
443 if self.store_backend is not None:
444 # Create func directory on demand.
445 self.store_backend.store_cached_func_code([self.func_id])
447 self.timestamp = timestamp if timestamp is not None else time.time()
448 try:
449 functools.update_wrapper(self, func)
450 except Exception:
451 pass # Objects like ufunc don't like that
452 if inspect.isfunction(func):
453 doc = pydoc.TextDoc().document(func)
454 # Remove blank line
455 doc = doc.replace("\n", "\n\n", 1)
456 # Strip backspace-overprints for compatibility with autodoc
457 doc = re.sub("\x08.", "", doc)
458 else:
459 # Pydoc does a poor job on other objects
460 doc = func.__doc__
461 self.__doc__ = "Memoized version of %s" % doc
463 self._func_code_info = None
464 self._func_code_id = None
466 def _is_in_cache_and_valid(self, call_id):
467 """Check if the function call is cached and valid for given arguments.
469 - Compare the function code with the one from the cached function,
470 asserting if it has changed.
471 - Check if the function call is present in the cache.
472 - Call `cache_validation_callback` for user define cache validation.
474 Returns True if the function call is in cache and can be used, and
475 returns False otherwise.
476 """
477 # Check if the code of the function has changed
478 if not self._check_previous_func_code(stacklevel=4):
479 return False
481 # Check if this specific call is in the cache
482 if not self.store_backend.contains_item(call_id):
483 return False
485 # Call the user defined cache validation callback
486 metadata = self.store_backend.get_metadata(call_id)
487 if (
488 self.cache_validation_callback is not None
489 and not self.cache_validation_callback(metadata)
490 ):
491 self.store_backend.clear_item(call_id)
492 return False
494 return True
496 def _cached_call(self, args, kwargs, shelving):
497 """Call wrapped function and cache result, or read cache if available.
499 This function returns the wrapped function output or a reference to
500 the cached result.
502 Arguments:
503 ----------
505 args, kwargs: list and dict
506 input arguments for wrapped function
508 shelving: bool
509 True when called via the call_and_shelve function.
512 Returns
513 -------
514 output: Output of the wrapped function if shelving is false, or a
515 MemorizedResult reference to the value if shelving is true.
516 metadata: dict containing the metadata associated with the call.
517 """
518 args_id = self._get_args_id(*args, **kwargs)
519 call_id = (self.func_id, args_id)
520 _, func_name = get_func_name(self.func)
521 func_info = self.store_backend.get_cached_func_info([self.func_id])
522 location = func_info["location"]
524 if self._verbose >= 20:
525 logging.basicConfig(level=logging.INFO)
526 _, signature = format_signature(self.func, *args, **kwargs)
527 self.info(
528 textwrap.dedent(
529 f"""
530 Querying {func_name} with signature
531 {signature}.
533 (argument hash {args_id})
535 The store location is {location}.
536 """
537 )
538 )
540 # Compare the function code with the previous to see if the
541 # function code has changed and check if the results are present in
542 # the cache.
543 if self._is_in_cache_and_valid(call_id):
544 if shelving:
545 return self._get_memorized_result(call_id), {}
547 try:
548 start_time = time.time()
549 output = self._load_item(call_id)
550 if self._verbose > 4:
551 self._print_duration(
552 time.time() - start_time, context="cache loaded "
553 )
554 return output, {}
555 except Exception:
556 # XXX: Should use an exception logger
557 _, signature = format_signature(self.func, *args, **kwargs)
558 self.warn(
559 "Exception while loading results for {}\n {}".format(
560 signature, traceback.format_exc()
561 )
562 )
564 if self._verbose > 10:
565 self.warn(
566 f"Computing func {func_name}, argument hash {args_id} "
567 f"in location {location}"
568 )
570 # Returns the output but not the metadata
571 return self._call(call_id, args, kwargs, shelving)
573 @property
574 def func_code_info(self):
575 # 3-tuple property containing: the function source code, source file,
576 # and first line of the code inside the source file
577 if hasattr(self.func, "__code__"):
578 if self._func_code_id is None:
579 self._func_code_id = id(self.func.__code__)
580 elif id(self.func.__code__) != self._func_code_id:
581 # Be robust to dynamic reassignments of self.func.__code__
582 self._func_code_info = None
584 if self._func_code_info is None:
585 # Cache the source code of self.func . Provided that get_func_code
586 # (which should be called once on self) gets called in the process
587 # in which self.func was defined, this caching mechanism prevents
588 # undesired cache clearing when the cached function is called in
589 # an environment where the introspection utilities get_func_code
590 # relies on do not work (typically, in joblib child processes).
591 # See #1035 for more info
592 # TODO (pierreglaser): do the same with get_func_name?
593 self._func_code_info = get_func_code(self.func)
594 return self._func_code_info
596 def call_and_shelve(self, *args, **kwargs):
597 """Call wrapped function, cache result and return a reference.
599 This method returns a reference to the cached result instead of the
600 result itself. The reference object is small and picklable, allowing
601 to send or store it easily. Call .get() on reference object to get
602 result.
604 Returns
605 -------
606 cached_result: MemorizedResult or NotMemorizedResult
607 reference to the value returned by the wrapped function. The
608 class "NotMemorizedResult" is used when there is no cache
609 activated (e.g. location=None in Memory).
610 """
611 # Return the wrapped output, without the metadata
612 return self._cached_call(args, kwargs, shelving=True)[0]
614 def __call__(self, *args, **kwargs):
615 # Return the output, without the metadata
616 return self._cached_call(args, kwargs, shelving=False)[0]
618 def __getstate__(self):
619 # Make sure self.func's source is introspected prior to being pickled -
620 # code introspection utilities typically do not work inside child
621 # processes
622 _ = self.func_code_info
624 # We don't store the timestamp when pickling, to avoid the hash
625 # depending from it.
626 state = self.__dict__.copy()
627 state["timestamp"] = None
629 # Invalidate the code id as id(obj) will be different in the child
630 state["_func_code_id"] = None
632 return state
634 def check_call_in_cache(self, *args, **kwargs):
635 """Check if the function call is cached and valid for given arguments.
637 Does not call the function or do any work besides function inspection
638 and argument hashing.
640 - Compare the function code with the one from the cached function,
641 asserting if it has changed.
642 - Check if the function call is present in the cache.
643 - Call `cache_validation_callback` for user define cache validation.
645 Returns
646 -------
647 is_call_in_cache: bool
648 Whether or not the function call is in cache and can be used.
649 """
650 call_id = (self.func_id, self._get_args_id(*args, **kwargs))
651 return self._is_in_cache_and_valid(call_id)
653 # ------------------------------------------------------------------------
654 # Private interface
655 # ------------------------------------------------------------------------
657 def _get_args_id(self, *args, **kwargs):
658 """Return the input parameter hash of a result."""
659 return hashing.hash(
660 filter_args(self.func, self.ignore, args, kwargs),
661 coerce_mmap=self.mmap_mode is not None,
662 )
664 def _hash_func(self):
665 """Hash a function to key the online cache"""
666 func_code_h = hash(getattr(self.func, "__code__", None))
667 return id(self.func), hash(self.func), func_code_h
669 def _write_func_code(self, func_code, first_line):
670 """Write the function code and the filename to a file."""
671 # We store the first line because the filename and the function
672 # name is not always enough to identify a function: people
673 # sometimes have several functions named the same way in a
674 # file. This is bad practice, but joblib should be robust to bad
675 # practice.
676 func_code = "%s %i\n%s" % (FIRST_LINE_TEXT, first_line, func_code)
677 self.store_backend.store_cached_func_code([self.func_id], func_code)
679 # Also store in the in-memory store of function hashes
680 is_named_callable = (
681 hasattr(self.func, "__name__") and self.func.__name__ != "<lambda>"
682 )
683 if is_named_callable:
684 # Don't do this for lambda functions or strange callable
685 # objects, as it ends up being too fragile
686 func_hash = self._hash_func()
687 try:
688 _FUNCTION_HASHES[self.func] = func_hash
689 except TypeError:
690 # Some callable are not hashable
691 pass
693 def _check_previous_func_code(self, stacklevel=2):
694 """
695 stacklevel is the depth a which this function is called, to
696 issue useful warnings to the user.
697 """
698 # First check if our function is in the in-memory store.
699 # Using the in-memory store not only makes things faster, but it
700 # also renders us robust to variations of the files when the
701 # in-memory version of the code does not vary
702 try:
703 func_hash = _FUNCTION_HASHES.get(self.func)
704 # We use as an identifier the id of the function and its
705 # hash. This is more likely to falsely change than have hash
706 # collisions, thus we are on the safe side.
707 if func_hash is not None and func_hash == self._hash_func():
708 return True
709 except TypeError:
710 # Some callables are not hashable
711 pass
713 # Here, we go through some effort to be robust to dynamically
714 # changing code and collision. We cannot inspect.getsource
715 # because it is not reliable when using IPython's magic "%run".
716 func_code, source_file, first_line = self.func_code_info
717 try:
718 old_func_code, old_first_line = extract_first_line(
719 self.store_backend.get_cached_func_code([self.func_id])
720 )
721 except (IOError, OSError): # some backend can also raise OSError
722 self._write_func_code(func_code, first_line)
723 return False
724 if old_func_code == func_code:
725 return True
727 # We have differing code, is this because we are referring to
728 # different functions, or because the function we are referring to has
729 # changed?
731 _, func_name = get_func_name(
732 self.func, resolv_alias=False, win_characters=False
733 )
734 if old_first_line == first_line == -1 or func_name == "<lambda>":
735 if not first_line == -1:
736 func_description = "{0} ({1}:{2})".format(
737 func_name, source_file, first_line
738 )
739 else:
740 func_description = func_name
741 warnings.warn(
742 JobLibCollisionWarning(
743 "Cannot detect name collisions for function '{0}'".format(
744 func_description
745 )
746 ),
747 stacklevel=stacklevel,
748 )
750 # Fetch the code at the old location and compare it. If it is the
751 # same than the code store, we have a collision: the code in the
752 # file has not changed, but the name we have is pointing to a new
753 # code block.
754 if not old_first_line == first_line and source_file is not None:
755 if os.path.exists(source_file):
756 _, func_name = get_func_name(self.func, resolv_alias=False)
757 num_lines = len(func_code.split("\n"))
758 with tokenize.open(source_file) as f:
759 on_disk_func_code = f.readlines()[
760 old_first_line - 1 : old_first_line - 1 + num_lines - 1
761 ]
762 on_disk_func_code = "".join(on_disk_func_code)
763 possible_collision = (
764 on_disk_func_code.rstrip() == old_func_code.rstrip()
765 )
766 else:
767 possible_collision = source_file.startswith("<doctest ")
768 if possible_collision:
769 warnings.warn(
770 JobLibCollisionWarning(
771 "Possible name collisions between functions "
772 "'%s' (%s:%i) and '%s' (%s:%i)"
773 % (
774 func_name,
775 source_file,
776 old_first_line,
777 func_name,
778 source_file,
779 first_line,
780 )
781 ),
782 stacklevel=stacklevel,
783 )
785 # The function has changed, wipe the cache directory.
786 # XXX: Should be using warnings, and giving stacklevel
787 if self._verbose > 10:
788 _, func_name = get_func_name(self.func, resolv_alias=False)
789 self.warn(
790 "Function {0} (identified by {1}) has changed.".format(
791 func_name, self.func_id
792 )
793 )
794 self.clear(warn=True)
795 return False
797 def clear(self, warn=True):
798 """Empty the function's cache."""
799 func_id = self.func_id
800 if self._verbose > 0 and warn:
801 self.warn("Clearing function cache identified by %s" % func_id)
802 self.store_backend.clear_path(
803 [
804 func_id,
805 ]
806 )
808 func_code, _, first_line = self.func_code_info
809 self._write_func_code(func_code, first_line)
811 def call(self, *args, **kwargs):
812 """Force the execution of the function with the given arguments.
814 The output values will be persisted, i.e., the cache will be updated
815 with any new values.
817 Parameters
818 ----------
819 *args: arguments
820 The arguments.
821 **kwargs: keyword arguments
822 Keyword arguments.
824 Returns
825 -------
826 output : object
827 The output of the function call.
828 metadata : dict
829 The metadata associated with the call.
830 """
831 call_id = (self.func_id, self._get_args_id(*args, **kwargs))
833 # Register the function code and invalidate old results before saving
834 # the new result, even when this is the first call.
835 self._check_previous_func_code(stacklevel=3)
837 # Return the output and the metadata
838 return self._call(call_id, args, kwargs)
840 def _call(self, call_id, args, kwargs, shelving=False):
841 # Return the output and the metadata
842 self._before_call(args, kwargs)
843 start_time = time.time()
844 output = self.func(*args, **kwargs)
845 return self._after_call(call_id, args, kwargs, shelving, output, start_time)
847 def _before_call(self, args, kwargs):
848 if self._verbose > 0:
849 print(format_call(self.func, args, kwargs))
851 def _after_call(self, call_id, args, kwargs, shelving, output, start_time):
852 self.store_backend.dump_item(call_id, output, verbose=self._verbose)
853 duration = time.time() - start_time
854 if self._verbose > 0:
855 self._print_duration(duration)
856 metadata = self._persist_input(duration, call_id, args, kwargs)
857 if shelving:
858 return self._get_memorized_result(call_id, metadata), metadata
860 if self.mmap_mode is not None:
861 # Memmap the output at the first call to be consistent with
862 # later calls
863 output = self._load_item(call_id, metadata)
864 return output, metadata
866 def _persist_input(self, duration, call_id, args, kwargs, this_duration_limit=0.5):
867 """Save a small summary of the call using json format in the
868 output directory.
870 output_dir: string
871 directory where to write metadata.
873 duration: float
874 time taken by hashing input arguments, calling the wrapped
875 function and persisting its output.
877 args, kwargs: list and dict
878 input arguments for wrapped function
880 this_duration_limit: float
881 Max execution time for this function before issuing a warning.
882 """
883 start_time = time.time()
884 argument_dict = filter_args(self.func, self.ignore, args, kwargs)
886 input_repr = dict((k, repr(v)) for k, v in argument_dict.items())
887 # This can fail due to race-conditions with multiple
888 # concurrent joblibs removing the file or the directory
889 metadata = {
890 "duration": duration,
891 "input_args": input_repr,
892 "time": start_time,
893 }
895 self.store_backend.store_metadata(call_id, metadata)
897 this_duration = time.time() - start_time
898 if this_duration > this_duration_limit:
899 # This persistence should be fast. It will not be if repr() takes
900 # time and its output is large, because json.dump will have to
901 # write a large file. This should not be an issue with numpy arrays
902 # for which repr() always output a short representation, but can
903 # be with complex dictionaries. Fixing the problem should be a
904 # matter of replacing repr() above by something smarter.
905 warnings.warn(
906 "Persisting input arguments took %.2fs to run."
907 "If this happens often in your code, it can cause "
908 "performance problems "
909 "(results will be correct in all cases). "
910 "The reason for this is probably some large input "
911 "arguments for a wrapped function." % this_duration,
912 stacklevel=5,
913 )
914 return metadata
916 def _get_memorized_result(self, call_id, metadata=None):
917 return MemorizedResult(
918 self.store_backend,
919 call_id,
920 metadata=metadata,
921 timestamp=self.timestamp,
922 verbose=self._verbose - 1,
923 )
925 def _load_item(self, call_id, metadata=None):
926 return self.store_backend.load_item(
927 call_id, metadata=metadata, timestamp=self.timestamp, verbose=self._verbose
928 )
930 def _print_duration(self, duration, context=""):
931 _, name = get_func_name(self.func)
932 msg = f"{name} {context}- {format_time(duration)}"
933 print(max(0, (80 - len(msg))) * "_" + msg)
935 # ------------------------------------------------------------------------
936 # Private `object` interface
937 # ------------------------------------------------------------------------
939 def __repr__(self):
940 return "{class_name}(func={func}, location={location})".format(
941 class_name=self.__class__.__name__,
942 func=self.func,
943 location=self.store_backend.location,
944 )
947###############################################################################
948# class `AsyncMemorizedFunc`
949###############################################################################
950class AsyncMemorizedFunc(MemorizedFunc):
951 async def __call__(self, *args, **kwargs):
952 out = self._cached_call(args, kwargs, shelving=False)
953 out = await out if asyncio.iscoroutine(out) else out
954 return out[0] # Don't return metadata
956 async def call_and_shelve(self, *args, **kwargs):
957 out = self._cached_call(args, kwargs, shelving=True)
958 out = await out if asyncio.iscoroutine(out) else out
959 return out[0] # Don't return metadata
961 async def call(self, *args, **kwargs):
962 out = super().call(*args, **kwargs)
963 return await out if asyncio.iscoroutine(out) else out
965 async def _call(self, call_id, args, kwargs, shelving=False):
966 self._before_call(args, kwargs)
967 start_time = time.time()
968 output = await self.func(*args, **kwargs)
969 return self._after_call(call_id, args, kwargs, shelving, output, start_time)
972###############################################################################
973# class `Memory`
974###############################################################################
975class Memory(Logger):
976 """A context object for caching a function's return value each time it
977 is called with the same input arguments.
979 All values are cached on the filesystem, in a deep directory
980 structure.
982 Read more in the :ref:`User Guide <memory>`.
984 Parameters
985 ----------
986 location: str, pathlib.Path or None
987 The path of the base directory to use as a data store
988 or None. If None is given, no caching is done and
989 the Memory object is completely transparent. This option
990 replaces cachedir since version 0.12.
992 backend: str, optional, default='local'
993 Type of store backend for reading/writing cache files.
994 The 'local' backend is using regular filesystem operations to
995 manipulate data (open, mv, etc) in the backend.
997 mmap_mode: {None, 'r+', 'r', 'w+', 'c'}, optional
998 The memmapping mode used when loading from cache
999 numpy arrays. See numpy.load for the meaning of the
1000 arguments.
1002 compress: boolean, integer or 2-tuple, optional
1003 Whether to zip the stored data on disk. If an integer is
1004 given, it should be between 1 and 9, and sets the amount
1005 of compression. Note that compressed arrays cannot be
1006 read by memmapping. If a 2-tuple is given, the first
1007 element must be a string corresponding to a compression
1008 method (e.g 'zlib', 'gzip', 'bz2', 'lzma' or 'xz') and
1009 the second element must be an integer from 0 to 9,
1010 corresponding to the compression level.
1012 verbose: int, optional
1013 Verbosity flag, controls the debug messages that are issued
1014 as functions are evaluated.
1016 backend_options: dict, optional
1017 Contains a dictionary of named parameters used to configure
1018 the store backend.
1019 """
1021 # ------------------------------------------------------------------------
1022 # Public interface
1023 # ------------------------------------------------------------------------
1025 def __init__(
1026 self,
1027 location=None,
1028 backend="local",
1029 mmap_mode=None,
1030 compress=False,
1031 verbose=1,
1032 backend_options=None,
1033 ):
1034 Logger.__init__(self)
1035 self._verbose = verbose
1036 self.mmap_mode = mmap_mode
1037 self.timestamp = time.time()
1038 self.backend = backend
1039 self.compress = compress
1040 if backend_options is None:
1041 backend_options = {}
1042 self.backend_options = backend_options
1044 if compress and mmap_mode is not None:
1045 warnings.warn("Compressed results cannot be memmapped", stacklevel=2)
1047 self.location = location
1048 if isinstance(location, str):
1049 location = os.path.join(location, "joblib")
1051 self.store_backend = _store_backend_factory(
1052 backend,
1053 location,
1054 verbose=self._verbose,
1055 backend_options=dict(
1056 compress=compress, mmap_mode=mmap_mode, **backend_options
1057 ),
1058 )
1060 def cache(
1061 self,
1062 func=None,
1063 ignore=None,
1064 verbose=None,
1065 mmap_mode=False,
1066 cache_validation_callback=None,
1067 ):
1068 """Decorates the given function func to only compute its return
1069 value for input arguments not cached on disk.
1071 Parameters
1072 ----------
1073 func: callable, optional
1074 The function to be decorated
1075 ignore: list of strings
1076 A list of arguments name to ignore in the hashing
1077 verbose: integer, optional
1078 The verbosity mode of the function. By default that
1079 of the memory object is used.
1080 mmap_mode: {None, 'r+', 'r', 'w+', 'c'}, optional
1081 The memmapping mode used when loading from cache
1082 numpy arrays. See numpy.load for the meaning of the
1083 arguments. By default that of the memory object is used.
1084 cache_validation_callback: callable, optional
1085 Callable to validate whether or not the cache is valid. When
1086 the cached function is called with arguments for which a cache
1087 exists, this callable is called with the metadata of the cached
1088 result as its sole argument. If it returns True, then the
1089 cached result is returned, else the cache for these arguments
1090 is cleared and recomputed.
1092 Returns
1093 -------
1094 decorated_func: MemorizedFunc object
1095 The returned object is a MemorizedFunc object, that is
1096 callable (behaves like a function), but offers extra
1097 methods for cache lookup and management. See the
1098 documentation for :class:`joblib.memory.MemorizedFunc`.
1099 """
1100 if cache_validation_callback is not None and not callable(
1101 cache_validation_callback
1102 ):
1103 raise ValueError(
1104 "cache_validation_callback needs to be callable. "
1105 f"Got {cache_validation_callback}."
1106 )
1107 if func is None:
1108 # Partial application, to be able to specify extra keyword
1109 # arguments in decorators
1110 return functools.partial(
1111 self.cache,
1112 ignore=ignore,
1113 mmap_mode=mmap_mode,
1114 verbose=verbose,
1115 cache_validation_callback=cache_validation_callback,
1116 )
1117 if self.store_backend is None:
1118 cls = (
1119 AsyncNotMemorizedFunc
1120 if inspect.iscoroutinefunction(func)
1121 else NotMemorizedFunc
1122 )
1123 return cls(func)
1124 if verbose is None:
1125 verbose = self._verbose
1126 if mmap_mode is False:
1127 mmap_mode = self.mmap_mode
1128 if isinstance(func, MemorizedFunc):
1129 func = func.func
1130 cls = AsyncMemorizedFunc if inspect.iscoroutinefunction(func) else MemorizedFunc
1132 # To reuse backend_options which is not an argument of MemorizedFunc
1133 # we recreate the store_backend here
1134 store_backend = self.store_backend
1135 if (verbose, mmap_mode) != (self._verbose, self.mmap_mode):
1136 location = self.location
1137 if isinstance(location, str):
1138 location = os.path.join(location, "joblib")
1139 store_backend = _store_backend_factory(
1140 self.backend,
1141 location,
1142 verbose=verbose,
1143 backend_options=dict(
1144 **self.backend_options,
1145 compress=self.compress,
1146 mmap_mode=mmap_mode,
1147 ),
1148 )
1150 return cls(
1151 func,
1152 location=store_backend,
1153 ignore=ignore,
1154 mmap_mode=mmap_mode,
1155 compress=self.compress,
1156 verbose=verbose,
1157 timestamp=self.timestamp,
1158 cache_validation_callback=cache_validation_callback,
1159 )
1161 def clear(self, warn=True):
1162 """Erase the complete cache directory."""
1163 if warn:
1164 self.warn("Flushing completely the cache")
1165 if self.store_backend is not None:
1166 self.store_backend.clear()
1168 # As the cache is completely clear, make sure the _FUNCTION_HASHES
1169 # cache is also reset. Else, for a function that is present in this
1170 # table, results cached after this clear will be have cache miss
1171 # as the function code is not re-written.
1172 _FUNCTION_HASHES.clear()
1174 def reduce_size(self, bytes_limit=None, items_limit=None, age_limit=None):
1175 """Remove cache elements to make the cache fit its limits.
1177 The limitation can impose that the cache size fits in ``bytes_limit``,
1178 that the number of cache items is no more than ``items_limit``, and
1179 that all files in cache are not older than ``age_limit``.
1181 Parameters
1182 ----------
1183 bytes_limit: int | str, optional
1184 Limit in bytes of the size of the cache. By default, the size of
1185 the cache is unlimited. When reducing the size of the cache,
1186 ``joblib`` keeps the most recently accessed items first. If a
1187 str is passed, it is converted to a number of bytes using units
1188 { K | M | G} for kilo, mega, giga.
1190 items_limit: int, optional
1191 Number of items to limit the cache to. By default, the number of
1192 items in the cache is unlimited. When reducing the size of the
1193 cache, ``joblib`` keeps the most recently accessed items first.
1195 age_limit: datetime.timedelta, optional
1196 Maximum age of items to limit the cache to. When reducing the size
1197 of the cache, any items last accessed more than the given length of
1198 time ago are deleted. Example: to remove files older than 5 days,
1199 use datetime.timedelta(days=5). Negative timedelta are not
1200 accepted.
1201 """
1202 if self.store_backend is None:
1203 # No cached results, this function does nothing.
1204 return
1206 if bytes_limit is None and items_limit is None and age_limit is None:
1207 # No limitation to impose, returning
1208 return
1210 # Defers the actual limits enforcing to the store backend.
1211 self.store_backend.enforce_store_limits(bytes_limit, items_limit, age_limit)
1213 def eval(self, func, *args, **kwargs):
1214 """Eval function func with arguments `*args` and `**kwargs`,
1215 in the context of the memory.
1217 This method works similarly to the builtin `apply`, except
1218 that the function is called only if the cache is not
1219 up to date.
1221 """
1222 if self.store_backend is None:
1223 return func(*args, **kwargs)
1224 return self.cache(func)(*args, **kwargs)
1226 def update_cache_tree(self):
1227 """Update the cache tree to a more efficient system
1228 when using backend 'local'.
1229 """
1230 if isinstance(self.store_backend, FileSystemStoreBackend):
1231 self.store_backend.update_cache_tree()
1232 else:
1233 warnings.warn(
1234 "Memory.update_cache_tree() does nothing when using a backend "
1235 "other than FileSystemStoreBackend"
1236 )
1238 # ------------------------------------------------------------------------
1239 # Private `object` interface
1240 # ------------------------------------------------------------------------
1242 def __repr__(self):
1243 return "{class_name}(location={location})".format(
1244 class_name=self.__class__.__name__,
1245 location=(
1246 None if self.store_backend is None else self.store_backend.location
1247 ),
1248 )
1250 def __getstate__(self):
1251 """We don't store the timestamp when pickling, to avoid the hash
1252 depending from it.
1253 """
1254 state = self.__dict__.copy()
1255 state["timestamp"] = None
1256 return state
1259###############################################################################
1260# cache_validation_callback helpers
1261###############################################################################
1264def expires_after(
1265 days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0, hours=0, weeks=0
1266):
1267 """Helper cache_validation_callback to force recompute after a duration.
1269 Parameters
1270 ----------
1271 days, seconds, microseconds, milliseconds, minutes, hours, weeks: numbers
1272 argument passed to a timedelta.
1273 """
1274 delta = datetime.timedelta(
1275 days=days,
1276 seconds=seconds,
1277 microseconds=microseconds,
1278 milliseconds=milliseconds,
1279 minutes=minutes,
1280 hours=hours,
1281 weeks=weeks,
1282 )
1284 def cache_validation_callback(metadata):
1285 computation_age = time.time() - metadata["time"]
1286 return computation_age < delta.total_seconds()
1288 return cache_validation_callback