Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/fsspec/mapping.py: 26%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import array
2import logging
3import posixpath
4import uuid
5import warnings
6from collections.abc import MutableMapping
7from functools import cached_property
9from fsspec.core import url_to_fs
11logger = logging.getLogger("fsspec.mapping")
12_MISSING = object()
15class FSMap(MutableMapping):
16 """Wrap a FileSystem instance as a mutable wrapping.
18 The keys of the mapping become files under the given root, and the
19 values (which must be bytes) the contents of those files.
21 Parameters
22 ----------
23 root: string
24 prefix for all the files
25 fs: FileSystem instance
26 check: bool (=True)
27 performs a touch at the location, to check for write access.
29 Examples
30 --------
31 >>> fs = FileSystem(**parameters) # doctest: +SKIP
32 >>> d = FSMap('my-data/path/', fs) # doctest: +SKIP
33 or, more likely
34 >>> d = fs.get_mapper('my-data/path/')
36 >>> d['loc1'] = b'Hello World' # doctest: +SKIP
37 >>> list(d.keys()) # doctest: +SKIP
38 ['loc1']
39 >>> d['loc1'] # doctest: +SKIP
40 b'Hello World'
41 """
43 def __init__(self, root, fs, check=False, create=False, missing_exceptions=None):
44 self.fs = fs
45 self.root = fs._strip_protocol(root)
46 self._root_key_to_str = fs._strip_protocol(posixpath.join(root, "x"))[:-1]
47 if missing_exceptions is None:
48 missing_exceptions = (
49 FileNotFoundError,
50 IsADirectoryError,
51 NotADirectoryError,
52 )
53 self.missing_exceptions = missing_exceptions
54 self.check = check
55 self.create = create
56 if create:
57 if not self.fs.exists(root):
58 self.fs.mkdir(root)
59 if check:
60 if not self.fs.exists(root):
61 raise ValueError(
62 f"Path {root} does not exist. Create "
63 f" with the ``create=True`` keyword"
64 )
65 check_path = f"{root}/{uuid.uuid4().hex}"
66 self.fs.touch(check_path)
67 self.fs.rm(check_path)
69 @cached_property
70 def dirfs(self):
71 """dirfs instance that can be used with the same keys as the mapper"""
72 from .implementations.dirfs import DirFileSystem
74 return DirFileSystem(path=self._root_key_to_str, fs=self.fs)
76 def clear(self):
77 """Remove all keys below root - empties out mapping"""
78 logger.info("Clear mapping at %s", self.root)
79 try:
80 self.fs.rm(self.root, True)
81 self.fs.mkdir(self.root)
82 except: # noqa: E722
83 pass
85 def getitems(self, keys, on_error="raise"):
86 """Fetch multiple items from the store
88 If the backend is async-able, this might proceed concurrently
90 Parameters
91 ----------
92 keys: list(str)
93 They keys to be fetched
94 on_error : "raise", "omit", "return"
95 If raise, an underlying exception will be raised (converted to KeyError
96 if the type is in self.missing_exceptions); if omit, keys with exception
97 will simply not be included in the output; if "return", all keys are
98 included in the output, but the value will be bytes or an exception
99 instance.
101 Returns
102 -------
103 dict(key, bytes|exception)
104 """
105 keys2 = [self._key_to_str(k) for k in keys]
106 oe = on_error if on_error == "raise" else "return"
107 try:
108 out = self.fs.cat(keys2, on_error=oe)
109 if isinstance(out, bytes):
110 out = {keys2[0]: out}
111 except self.missing_exceptions as e:
112 raise KeyError from e
113 out = {
114 k: (KeyError() if isinstance(v, self.missing_exceptions) else v)
115 for k, v in out.items()
116 }
117 return {
118 key: out[k2] if on_error == "raise" else out.get(k2, KeyError(k2))
119 for key, k2 in zip(keys, keys2)
120 if on_error == "return" or not isinstance(out[k2], BaseException)
121 }
123 def setitems(self, values_dict):
124 """Set the values of multiple items in the store
126 Parameters
127 ----------
128 values_dict: dict(str, bytes)
129 """
130 values = {self._key_to_str(k): maybe_convert(v) for k, v in values_dict.items()}
131 self.fs.pipe(values)
133 def delitems(self, keys):
134 """Remove multiple keys from the store"""
135 self.fs.rm([self._key_to_str(k) for k in keys])
137 def _key_to_str(self, key):
138 """Generate full path for the key"""
139 if not isinstance(key, str):
140 # raise TypeError("key must be of type `str`, got `{type(key).__name__}`"
141 warnings.warn(
142 "from fsspec 2023.5 onward FSMap non-str keys will raise TypeError",
143 DeprecationWarning,
144 )
145 if isinstance(key, list):
146 key = tuple(key)
147 key = str(key)
148 return f"{self._root_key_to_str}{key}".rstrip("/")
150 def _str_to_key(self, s):
151 """Strip path of to leave key name"""
152 return s[len(self.root) :].lstrip("/")
154 def __getitem__(self, key, default=None):
155 """Retrieve data"""
156 k = self._key_to_str(key)
157 try:
158 result = self.fs.cat(k)
159 except self.missing_exceptions as exc:
160 if default is not None:
161 return default
162 raise KeyError(key) from exc
163 return result
165 def pop(self, key, default=_MISSING):
166 """Pop data"""
167 try:
168 result = self[key]
169 except KeyError:
170 if default is _MISSING:
171 raise
172 return default
173 try:
174 del self[key]
175 except KeyError:
176 pass
177 return result
179 def __setitem__(self, key, value):
180 """Store value in key"""
181 key = self._key_to_str(key)
182 self.fs.mkdirs(self.fs._parent(key), exist_ok=True)
183 self.fs.pipe_file(key, maybe_convert(value))
185 def __iter__(self):
186 return (self._str_to_key(x) for x in self.fs.find(self.root))
188 def __len__(self):
189 return len(self.fs.find(self.root))
191 def __delitem__(self, key):
192 """Remove key"""
193 try:
194 self.fs.rm(self._key_to_str(key))
195 except Exception as exc:
196 raise KeyError from exc
198 def __contains__(self, key):
199 """Does key exist in mapping?"""
200 path = self._key_to_str(key)
201 return self.fs.isfile(path)
203 def __reduce__(self):
204 return FSMap, (self.root, self.fs, False, False, self.missing_exceptions)
207def maybe_convert(value):
208 if isinstance(value, array.array) or hasattr(value, "__array__"):
209 # bytes-like things
210 if hasattr(value, "dtype") and value.dtype.kind in "Mm":
211 # The buffer interface doesn't support datetime64/timdelta64 numpy
212 # arrays
213 value = value.view("int64")
214 value = bytes(memoryview(value))
215 return value
218def get_mapper(
219 url="",
220 check=False,
221 create=False,
222 missing_exceptions=None,
223 alternate_root=None,
224 **kwargs,
225):
226 """Create key-value interface for given URL and options
228 The URL will be of the form "protocol://location" and point to the root
229 of the mapper required. All keys will be file-names below this location,
230 and their values the contents of each key.
232 Also accepts compound URLs like zip::s3://bucket/file.zip , see ``fsspec.open``.
234 Parameters
235 ----------
236 url: str
237 Root URL of mapping
238 check: bool
239 Whether to attempt to read from the location before instantiation, to
240 check that the mapping does exist
241 create: bool
242 Whether to make the directory corresponding to the root before
243 instantiating
244 missing_exceptions: None or tuple
245 If given, these exception types will be regarded as missing keys and
246 return KeyError when trying to read data. By default, you get
247 (FileNotFoundError, IsADirectoryError, NotADirectoryError)
248 alternate_root: None or str
249 In cases of complex URLs, the parser may fail to pick the correct part
250 for the mapper root, so this arg can override
252 Returns
253 -------
254 ``FSMap`` instance, the dict-like key-value store.
255 """
256 # Removing protocol here - could defer to each open() on the backend
257 fs, urlpath = url_to_fs(url, **kwargs)
258 root = alternate_root if alternate_root is not None else urlpath
259 return FSMap(root, fs, check, create, missing_exceptions=missing_exceptions)