Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/fsspec/mapping.py: 27%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import array
2import logging
3import posixpath
4import uuid
5import warnings
6from collections.abc import MutableMapping
7from functools import cached_property
9from fsspec.core import url_to_fs
11logger = logging.getLogger("fsspec.mapping")
14class FSMap(MutableMapping):
15 """Wrap a FileSystem instance as a mutable wrapping.
17 The keys of the mapping become files under the given root, and the
18 values (which must be bytes) the contents of those files.
20 Parameters
21 ----------
22 root: string
23 prefix for all the files
24 fs: FileSystem instance
25 check: bool (=True)
26 performs a touch at the location, to check for write access.
28 Examples
29 --------
30 >>> fs = FileSystem(**parameters) # doctest: +SKIP
31 >>> d = FSMap('my-data/path/', fs) # doctest: +SKIP
32 or, more likely
33 >>> d = fs.get_mapper('my-data/path/')
35 >>> d['loc1'] = b'Hello World' # doctest: +SKIP
36 >>> list(d.keys()) # doctest: +SKIP
37 ['loc1']
38 >>> d['loc1'] # doctest: +SKIP
39 b'Hello World'
40 """
42 def __init__(self, root, fs, check=False, create=False, missing_exceptions=None):
43 self.fs = fs
44 self.root = fs._strip_protocol(root)
45 self._root_key_to_str = fs._strip_protocol(posixpath.join(root, "x"))[:-1]
46 if missing_exceptions is None:
47 missing_exceptions = (
48 FileNotFoundError,
49 IsADirectoryError,
50 NotADirectoryError,
51 )
52 self.missing_exceptions = missing_exceptions
53 self.check = check
54 self.create = create
55 if create:
56 if not self.fs.exists(root):
57 self.fs.mkdir(root)
58 if check:
59 if not self.fs.exists(root):
60 raise ValueError(
61 f"Path {root} does not exist. Create "
62 f" with the ``create=True`` keyword"
63 )
64 check_path = f"{root}/{uuid.uuid4().hex}"
65 self.fs.touch(check_path)
66 self.fs.rm(check_path)
68 @cached_property
69 def dirfs(self):
70 """dirfs instance that can be used with the same keys as the mapper"""
71 from .implementations.dirfs import DirFileSystem
73 return DirFileSystem(path=self._root_key_to_str, fs=self.fs)
75 def clear(self):
76 """Remove all keys below root - empties out mapping"""
77 logger.info("Clear mapping at %s", self.root)
78 try:
79 self.fs.rm(self.root, True)
80 self.fs.mkdir(self.root)
81 except: # noqa: E722
82 pass
84 def getitems(self, keys, on_error="raise"):
85 """Fetch multiple items from the store
87 If the backend is async-able, this might proceed concurrently
89 Parameters
90 ----------
91 keys: list(str)
92 They keys to be fetched
93 on_error : "raise", "omit", "return"
94 If raise, an underlying exception will be raised (converted to KeyError
95 if the type is in self.missing_exceptions); if omit, keys with exception
96 will simply not be included in the output; if "return", all keys are
97 included in the output, but the value will be bytes or an exception
98 instance.
100 Returns
101 -------
102 dict(key, bytes|exception)
103 """
104 keys2 = [self._key_to_str(k) for k in keys]
105 oe = on_error if on_error == "raise" else "return"
106 try:
107 out = self.fs.cat(keys2, on_error=oe)
108 if isinstance(out, bytes):
109 out = {keys2[0]: out}
110 except self.missing_exceptions as e:
111 raise KeyError from e
112 out = {
113 k: (KeyError() if isinstance(v, self.missing_exceptions) else v)
114 for k, v in out.items()
115 }
116 return {
117 key: out[k2] if on_error == "raise" else out.get(k2, KeyError(k2))
118 for key, k2 in zip(keys, keys2)
119 if on_error == "return" or not isinstance(out[k2], BaseException)
120 }
122 def setitems(self, values_dict):
123 """Set the values of multiple items in the store
125 Parameters
126 ----------
127 values_dict: dict(str, bytes)
128 """
129 values = {self._key_to_str(k): maybe_convert(v) for k, v in values_dict.items()}
130 self.fs.pipe(values)
132 def delitems(self, keys):
133 """Remove multiple keys from the store"""
134 self.fs.rm([self._key_to_str(k) for k in keys])
136 def _key_to_str(self, key):
137 """Generate full path for the key"""
138 if not isinstance(key, str):
139 # raise TypeError("key must be of type `str`, got `{type(key).__name__}`"
140 warnings.warn(
141 "from fsspec 2023.5 onward FSMap non-str keys will raise TypeError",
142 DeprecationWarning,
143 )
144 if isinstance(key, list):
145 key = tuple(key)
146 key = str(key)
147 return f"{self._root_key_to_str}{key}".rstrip("/")
149 def _str_to_key(self, s):
150 """Strip path of to leave key name"""
151 return s[len(self.root) :].lstrip("/")
153 def __getitem__(self, key, default=None):
154 """Retrieve data"""
155 k = self._key_to_str(key)
156 try:
157 result = self.fs.cat(k)
158 except self.missing_exceptions as exc:
159 if default is not None:
160 return default
161 raise KeyError(key) from exc
162 return result
164 def pop(self, key, default=None):
165 """Pop data"""
166 result = self.__getitem__(key, default)
167 try:
168 del self[key]
169 except KeyError:
170 pass
171 return result
173 def __setitem__(self, key, value):
174 """Store value in key"""
175 key = self._key_to_str(key)
176 self.fs.mkdirs(self.fs._parent(key), exist_ok=True)
177 self.fs.pipe_file(key, maybe_convert(value))
179 def __iter__(self):
180 return (self._str_to_key(x) for x in self.fs.find(self.root))
182 def __len__(self):
183 return len(self.fs.find(self.root))
185 def __delitem__(self, key):
186 """Remove key"""
187 try:
188 self.fs.rm(self._key_to_str(key))
189 except Exception as exc:
190 raise KeyError from exc
192 def __contains__(self, key):
193 """Does key exist in mapping?"""
194 path = self._key_to_str(key)
195 return self.fs.isfile(path)
197 def __reduce__(self):
198 return FSMap, (self.root, self.fs, False, False, self.missing_exceptions)
201def maybe_convert(value):
202 if isinstance(value, array.array) or hasattr(value, "__array__"):
203 # bytes-like things
204 if hasattr(value, "dtype") and value.dtype.kind in "Mm":
205 # The buffer interface doesn't support datetime64/timdelta64 numpy
206 # arrays
207 value = value.view("int64")
208 value = bytes(memoryview(value))
209 return value
212def get_mapper(
213 url="",
214 check=False,
215 create=False,
216 missing_exceptions=None,
217 alternate_root=None,
218 **kwargs,
219):
220 """Create key-value interface for given URL and options
222 The URL will be of the form "protocol://location" and point to the root
223 of the mapper required. All keys will be file-names below this location,
224 and their values the contents of each key.
226 Also accepts compound URLs like zip::s3://bucket/file.zip , see ``fsspec.open``.
228 Parameters
229 ----------
230 url: str
231 Root URL of mapping
232 check: bool
233 Whether to attempt to read from the location before instantiation, to
234 check that the mapping does exist
235 create: bool
236 Whether to make the directory corresponding to the root before
237 instantiating
238 missing_exceptions: None or tuple
239 If given, these exception types will be regarded as missing keys and
240 return KeyError when trying to read data. By default, you get
241 (FileNotFoundError, IsADirectoryError, NotADirectoryError)
242 alternate_root: None or str
243 In cases of complex URLs, the parser may fail to pick the correct part
244 for the mapper root, so this arg can override
246 Returns
247 -------
248 ``FSMap`` instance, the dict-like key-value store.
249 """
250 # Removing protocol here - could defer to each open() on the backend
251 fs, urlpath = url_to_fs(url, **kwargs)
252 root = alternate_root if alternate_root is not None else urlpath
253 return FSMap(root, fs, check, create, missing_exceptions=missing_exceptions)