Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/fsspec/callbacks.py: 42%
69 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-12-08 06:40 +0000
1class Callback:
2 """
3 Base class and interface for callback mechanism
5 This class can be used directly for monitoring file transfers by
6 providing ``callback=Callback(hooks=...)`` (see the ``hooks`` argument,
7 below), or subclassed for more specialised behaviour.
9 Parameters
10 ----------
11 size: int (optional)
12 Nominal quantity for the value that corresponds to a complete
13 transfer, e.g., total number of tiles or total number of
14 bytes
15 value: int (0)
16 Starting internal counter value
17 hooks: dict or None
18 A dict of named functions to be called on each update. The signature
19 of these must be ``f(size, value, **kwargs)``
20 """
22 def __init__(self, size=None, value=0, hooks=None, **kwargs):
23 self.size = size
24 self.value = value
25 self.hooks = hooks or {}
26 self.kw = kwargs
28 def set_size(self, size):
29 """
30 Set the internal maximum size attribute
32 Usually called if not initially set at instantiation. Note that this
33 triggers a ``call()``.
35 Parameters
36 ----------
37 size: int
38 """
39 self.size = size
40 self.call()
42 def absolute_update(self, value):
43 """
44 Set the internal value state
46 Triggers ``call()``
48 Parameters
49 ----------
50 value: int
51 """
52 self.value = value
53 self.call()
55 def relative_update(self, inc=1):
56 """
57 Delta increment the internal counter
59 Triggers ``call()``
61 Parameters
62 ----------
63 inc: int
64 """
65 self.value += inc
66 self.call()
68 def call(self, hook_name=None, **kwargs):
69 """
70 Execute hook(s) with current state
72 Each function is passed the internal size and current value
74 Parameters
75 ----------
76 hook_name: str or None
77 If given, execute on this hook
78 kwargs: passed on to (all) hook(s)
79 """
80 if not self.hooks:
81 return
82 kw = self.kw.copy()
83 kw.update(kwargs)
84 if hook_name:
85 if hook_name not in self.hooks:
86 return
87 return self.hooks[hook_name](self.size, self.value, **kw)
88 for hook in self.hooks.values() or []:
89 hook(self.size, self.value, **kw)
91 def wrap(self, iterable):
92 """
93 Wrap an iterable to call ``relative_update`` on each iterations
95 Parameters
96 ----------
97 iterable: Iterable
98 The iterable that is being wrapped
99 """
100 for item in iterable:
101 self.relative_update()
102 yield item
104 def branch(self, path_1, path_2, kwargs):
105 """
106 Set callbacks for child transfers
108 If this callback is operating at a higher level, e.g., put, which may
109 trigger transfers that can also be monitored. The passed kwargs are
110 to be *mutated* to add ``callback=``, if this class supports branching
111 to children.
113 Parameters
114 ----------
115 path_1: str
116 Child's source path
117 path_2: str
118 Child's destination path
119 kwargs: dict
120 arguments passed to child method, e.g., put_file.
122 Returns
123 -------
125 """
126 return None
128 def no_op(self, *_, **__):
129 pass
131 def __getattr__(self, item):
132 """
133 If undefined methods are called on this class, nothing happens
134 """
135 return self.no_op
137 @classmethod
138 def as_callback(cls, maybe_callback=None):
139 """Transform callback=... into Callback instance
141 For the special value of ``None``, return the global instance of
142 ``NoOpCallback``. This is an alternative to including
143 ``callback=_DEFAULT_CALLBACK`` directly in a method signature.
144 """
145 if maybe_callback is None:
146 return _DEFAULT_CALLBACK
147 return maybe_callback
150class NoOpCallback(Callback):
151 """
152 This implementation of Callback does exactly nothing
153 """
155 def call(self, *args, **kwargs):
156 return None
159class DotPrinterCallback(Callback):
160 """
161 Simple example Callback implementation
163 Almost identical to Callback with a hook that prints a char; here we
164 demonstrate how the outer layer may print "#" and the inner layer "."
165 """
167 def __init__(self, chr_to_print="#", **kwargs):
168 self.chr = chr_to_print
169 super().__init__(**kwargs)
171 def branch(self, path_1, path_2, kwargs):
172 """Mutate kwargs to add new instance with different print char"""
173 kwargs["callback"] = DotPrinterCallback(".")
175 def call(self, **kwargs):
176 """Just outputs a character"""
177 print(self.chr, end="")
180class TqdmCallback(Callback):
181 """
182 A callback to display a progress bar using tqdm
184 Parameters
185 ----------
186 tqdm_kwargs : dict, (optional)
187 Any argument accepted by the tqdm constructor.
188 See the `tqdm doc <https://tqdm.github.io/docs/tqdm/#__init__>`_.
189 Will be forwarded to tqdm.
191 Examples
192 --------
193 >>> import fsspec
194 >>> from fsspec.callbacks import TqdmCallback
195 >>> fs = fsspec.filesystem("memory")
196 >>> path2distant_data = "/your-path"
197 >>> fs.upload(
198 ".",
199 path2distant_data,
200 recursive=True,
201 callback=TqdmCallback(),
202 )
204 You can forward args to tqdm using the ``tqdm_kwargs`` parameter.
206 >>> fs.upload(
207 ".",
208 path2distant_data,
209 recursive=True,
210 callback=TqdmCallback(tqdm_kwargs={"desc": "Your tqdm description"}),
211 )
212 """
214 def __init__(self, tqdm_kwargs=None, *args, **kwargs):
215 try:
216 import tqdm
218 self._tqdm = tqdm
219 except ImportError as exce:
220 raise ImportError(
221 "Using TqdmCallback requires tqdm to be installed"
222 ) from exce
224 self._tqdm_kwargs = tqdm_kwargs or {}
225 super().__init__(*args, **kwargs)
227 def set_size(self, size):
228 self.tqdm = self._tqdm.tqdm(total=size, **self._tqdm_kwargs)
230 def relative_update(self, inc=1):
231 self.tqdm.update(inc)
233 def __del__(self):
234 self.tqdm.close()
235 self.tqdm = None
238_DEFAULT_CALLBACK = NoOpCallback()