Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/fsspec/mapping.py: 23%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import array
2import logging
3import posixpath
4import uuid
5import warnings
6from collections.abc import MutableMapping
7from functools import cached_property
9from fsspec.core import url_to_fs
11logger = logging.getLogger("fsspec.mapping")
12_MISSING = object()
15class FSMap(MutableMapping):
16 """Wrap a FileSystem instance as a mutable wrapping.
18 The keys of the mapping become files under the given root, and the
19 values (which must be bytes) the contents of those files.
21 Parameters
22 ----------
23 root: string
24 prefix for all the files
25 fs: FileSystem instance
26 check: bool (=True)
27 performs a touch at the location, to check for write access.
29 Examples
30 --------
31 >>> fs = FileSystem(**parameters) # doctest: +SKIP
32 >>> d = FSMap('my-data/path/', fs) # doctest: +SKIP
33 or, more likely
34 >>> d = fs.get_mapper('my-data/path/')
36 >>> d['loc1'] = b'Hello World' # doctest: +SKIP
37 >>> list(d.keys()) # doctest: +SKIP
38 ['loc1']
39 >>> d['loc1'] # doctest: +SKIP
40 b'Hello World'
41 """
43 def __init__(self, root, fs, check=False, create=False, missing_exceptions=None):
44 self.fs = fs
45 self.root = fs._strip_protocol(root)
46 self._root_key_to_str = fs._strip_protocol(posixpath.join(root, "x"))[:-1]
47 if missing_exceptions is None:
48 missing_exceptions = (
49 FileNotFoundError,
50 IsADirectoryError,
51 NotADirectoryError,
52 )
53 self.missing_exceptions = missing_exceptions
54 self.check = check
55 self.create = create
56 if create:
57 if not self.fs.exists(root):
58 self.fs.mkdir(root)
59 if check:
60 if not self.fs.exists(root):
61 raise ValueError(
62 f"Path {root} does not exist. Create "
63 f" with the ``create=True`` keyword"
64 )
65 check_path = f"{root}/{uuid.uuid4().hex}"
66 self.fs.touch(check_path)
67 self.fs.rm(check_path)
69 @cached_property
70 def dirfs(self):
71 """dirfs instance that can be used with the same keys as the mapper"""
72 from .implementations.dirfs import DirFileSystem
74 return DirFileSystem(path=self._root_key_to_str, fs=self.fs)
76 def clear(self):
77 """Remove all keys below root - empties out mapping"""
78 logger.info("Clear mapping at %s", self.root)
79 try:
80 self.fs.rm(self.root, True)
81 except self.missing_exceptions:
82 return
83 try:
84 self.fs.mkdir(self.root)
85 except FileExistsError:
86 if not self.fs.isdir(self.root):
87 raise
89 def getitems(self, keys, on_error="raise"):
90 """Fetch multiple items from the store
92 If the backend is async-able, this might proceed concurrently
94 Parameters
95 ----------
96 keys: list(str)
97 They keys to be fetched
98 on_error : "raise", "omit", "return"
99 If raise, an underlying exception will be raised (converted to KeyError
100 if the type is in self.missing_exceptions); if omit, keys with exception
101 will simply not be included in the output; if "return", all keys are
102 included in the output, but the value will be bytes or an exception
103 instance.
105 Returns
106 -------
107 dict(key, bytes|exception)
108 """
109 keys2 = [self._key_to_str(k) for k in keys]
110 oe = on_error if on_error == "raise" else "return"
111 try:
112 out = self.fs.cat(keys2, on_error=oe)
113 if isinstance(out, bytes):
114 out = {keys2[0]: out}
115 except self.missing_exceptions as e:
116 raise KeyError from e
117 out = {
118 k: (KeyError() if isinstance(v, self.missing_exceptions) else v)
119 for k, v in out.items()
120 }
121 return {
122 key: out[k2] if on_error == "raise" else out.get(k2, KeyError(k2))
123 for key, k2 in zip(keys, keys2)
124 if on_error == "return" or not isinstance(out[k2], BaseException)
125 }
127 def setitems(self, values_dict):
128 """Set the values of multiple items in the store
130 Parameters
131 ----------
132 values_dict: dict(str, bytes)
133 """
134 values = {self._key_to_str(k): maybe_convert(v) for k, v in values_dict.items()}
135 for parent in dict.fromkeys(self.fs._parent(k) for k in values):
136 self.fs.mkdirs(parent, exist_ok=True)
137 self.fs.pipe(values)
139 def delitems(self, keys):
140 """Remove multiple keys from the store"""
141 self.fs.rm([self._key_to_str(k) for k in keys])
143 def _key_to_str(self, key):
144 """Generate full path for the key"""
145 if not isinstance(key, str):
146 # raise TypeError("key must be of type `str`, got `{type(key).__name__}`"
147 warnings.warn(
148 "from fsspec 2023.5 onward FSMap non-str keys will raise TypeError",
149 DeprecationWarning,
150 )
151 if isinstance(key, list):
152 key = tuple(key)
153 key = str(key)
154 return f"{self._root_key_to_str}{key}".rstrip("/")
156 def _str_to_key(self, s):
157 """Strip path of to leave key name"""
158 return s[len(self.root) :].lstrip("/")
160 def __getitem__(self, key, default=None):
161 """Retrieve data"""
162 k = self._key_to_str(key)
163 try:
164 result = self.fs.cat(k)
165 except self.missing_exceptions as exc:
166 if default is not None:
167 return default
168 raise KeyError(key) from exc
169 return result
171 def pop(self, key, default=_MISSING):
172 """Pop data"""
173 try:
174 result = self[key]
175 except KeyError:
176 if default is _MISSING:
177 raise
178 return default
179 try:
180 del self[key]
181 except KeyError:
182 pass
183 return result
185 def __setitem__(self, key, value):
186 """Store value in key"""
187 key = self._key_to_str(key)
188 self.fs.mkdirs(self.fs._parent(key), exist_ok=True)
189 self.fs.pipe_file(key, maybe_convert(value))
191 def __iter__(self):
192 return (self._str_to_key(x) for x in self.fs.find(self.root))
194 def __len__(self):
195 return len(self.fs.find(self.root))
197 def __delitem__(self, key):
198 """Remove key"""
199 path = self._key_to_str(key)
200 try:
201 self.fs.rm(path)
202 except self.missing_exceptions as exc:
203 raise KeyError(key) from exc
204 except Exception as exc:
205 try:
206 is_directory = self.fs.info(path)["type"] == "directory"
207 except Exception:
208 # Keep the removal error if classifying the path also fails.
209 is_directory = False
210 if is_directory:
211 raise KeyError(key) from exc
212 raise
214 def __contains__(self, key):
215 """Does key exist in mapping?"""
216 path = self._key_to_str(key)
217 return self.fs.isfile(path)
219 def __reduce__(self):
220 return FSMap, (self.root, self.fs, False, False, self.missing_exceptions)
223def maybe_convert(value):
224 if isinstance(value, array.array) or hasattr(value, "__array__"):
225 # bytes-like things
226 if hasattr(value, "dtype") and value.dtype.kind in "Mm":
227 # The buffer interface doesn't support datetime64/timdelta64 numpy
228 # arrays
229 value = value.view("int64")
230 value = bytes(memoryview(value))
231 return value
234def get_mapper(
235 url="",
236 check=False,
237 create=False,
238 missing_exceptions=None,
239 alternate_root=None,
240 **kwargs,
241):
242 """Create key-value interface for given URL and options
244 The URL will be of the form "protocol://location" and point to the root
245 of the mapper required. All keys will be file-names below this location,
246 and their values the contents of each key.
248 Also accepts compound URLs like zip::s3://bucket/file.zip , see ``fsspec.open``.
250 Parameters
251 ----------
252 url: str
253 Root URL of mapping
254 check: bool
255 Whether to attempt to read from the location before instantiation, to
256 check that the mapping does exist
257 create: bool
258 Whether to make the directory corresponding to the root before
259 instantiating
260 missing_exceptions: None or tuple
261 If given, these exception types will be regarded as missing keys and
262 return KeyError when trying to read data. By default, you get
263 (FileNotFoundError, IsADirectoryError, NotADirectoryError)
264 alternate_root: None or str
265 In cases of complex URLs, the parser may fail to pick the correct part
266 for the mapper root, so this arg can override
268 Returns
269 -------
270 ``FSMap`` instance, the dict-like key-value store.
271 """
272 # Removing protocol here - could defer to each open() on the backend
273 fs, urlpath = url_to_fs(url, **kwargs)
274 root = alternate_root if alternate_root is not None else urlpath
275 return FSMap(root, fs, check, create, missing_exceptions=missing_exceptions)