Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/fsspec/mapping.py: 23%
98 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-07 06:56 +0000
1import array
2import posixpath
3import warnings
4from collections.abc import MutableMapping
6from .core import url_to_fs
9class FSMap(MutableMapping):
10 """Wrap a FileSystem instance as a mutable wrapping.
12 The keys of the mapping become files under the given root, and the
13 values (which must be bytes) the contents of those files.
15 Parameters
16 ----------
17 root: string
18 prefix for all the files
19 fs: FileSystem instance
20 check: bool (=True)
21 performs a touch at the location, to check for write access.
23 Examples
24 --------
25 >>> fs = FileSystem(**parameters) # doctest: +SKIP
26 >>> d = FSMap('my-data/path/', fs) # doctest: +SKIP
27 or, more likely
28 >>> d = fs.get_mapper('my-data/path/')
30 >>> d['loc1'] = b'Hello World' # doctest: +SKIP
31 >>> list(d.keys()) # doctest: +SKIP
32 ['loc1']
33 >>> d['loc1'] # doctest: +SKIP
34 b'Hello World'
35 """
37 def __init__(self, root, fs, check=False, create=False, missing_exceptions=None):
38 self.fs = fs
39 self.root = fs._strip_protocol(root).rstrip("/")
40 self._root_key_to_str = fs._strip_protocol(posixpath.join(root, "x"))[:-1]
41 if missing_exceptions is None:
42 missing_exceptions = (
43 FileNotFoundError,
44 IsADirectoryError,
45 NotADirectoryError,
46 )
47 self.missing_exceptions = missing_exceptions
48 self.check = check
49 self.create = create
50 if create:
51 if not self.fs.exists(root):
52 self.fs.mkdir(root)
53 if check:
54 if not self.fs.exists(root):
55 raise ValueError(
56 "Path %s does not exist. Create "
57 " with the ``create=True`` keyword" % root
58 )
59 self.fs.touch(root + "/a")
60 self.fs.rm(root + "/a")
62 def clear(self):
63 """Remove all keys below root - empties out mapping"""
64 try:
65 self.fs.rm(self.root, True)
66 self.fs.mkdir(self.root)
67 except: # noqa: E722
68 pass
70 def getitems(self, keys, on_error="raise"):
71 """Fetch multiple items from the store
73 If the backend is async-able, this might proceed concurrently
75 Parameters
76 ----------
77 keys: list(str)
78 They keys to be fetched
79 on_error : "raise", "omit", "return"
80 If raise, an underlying exception will be raised (converted to KeyError
81 if the type is in self.missing_exceptions); if omit, keys with exception
82 will simply not be included in the output; if "return", all keys are
83 included in the output, but the value will be bytes or an exception
84 instance.
86 Returns
87 -------
88 dict(key, bytes|exception)
89 """
90 keys2 = [self._key_to_str(k) for k in keys]
91 oe = on_error if on_error == "raise" else "return"
92 try:
93 out = self.fs.cat(keys2, on_error=oe)
94 if isinstance(out, bytes):
95 out = {keys2[0]: out}
96 except self.missing_exceptions as e:
97 raise KeyError from e
98 out = {
99 k: (KeyError() if isinstance(v, self.missing_exceptions) else v)
100 for k, v in out.items()
101 }
102 return {
103 key: out[k2]
104 for key, k2 in zip(keys, keys2)
105 if on_error == "return" or not isinstance(out[k2], BaseException)
106 }
108 def setitems(self, values_dict):
109 """Set the values of multiple items in the store
111 Parameters
112 ----------
113 values_dict: dict(str, bytes)
114 """
115 values = {self._key_to_str(k): maybe_convert(v) for k, v in values_dict.items()}
116 self.fs.pipe(values)
118 def delitems(self, keys):
119 """Remove multiple keys from the store"""
120 self.fs.rm([self._key_to_str(k) for k in keys])
122 def _key_to_str(self, key):
123 """Generate full path for the key"""
124 if not isinstance(key, str):
125 # raise TypeError("key must be of type `str`, got `{type(key).__name__}`"
126 warnings.warn(
127 "from fsspec 2023.5 onward FSMap non-str keys will raise TypeError",
128 DeprecationWarning,
129 )
130 if isinstance(key, list):
131 key = tuple(key)
132 key = str(key)
133 return f"{self._root_key_to_str}{key}"
135 def _str_to_key(self, s):
136 """Strip path of to leave key name"""
137 return s[len(self.root) :].lstrip("/")
139 def __getitem__(self, key, default=None):
140 """Retrieve data"""
141 k = self._key_to_str(key)
142 try:
143 result = self.fs.cat(k)
144 except self.missing_exceptions:
145 if default is not None:
146 return default
147 raise KeyError(key)
148 return result
150 def pop(self, key, default=None):
151 """Pop data"""
152 result = self.__getitem__(key, default)
153 try:
154 del self[key]
155 except KeyError:
156 pass
157 return result
159 def __setitem__(self, key, value):
160 """Store value in key"""
161 key = self._key_to_str(key)
162 self.fs.mkdirs(self.fs._parent(key), exist_ok=True)
163 self.fs.pipe_file(key, maybe_convert(value))
165 def __iter__(self):
166 return (self._str_to_key(x) for x in self.fs.find(self.root))
168 def __len__(self):
169 return len(self.fs.find(self.root))
171 def __delitem__(self, key):
172 """Remove key"""
173 try:
174 self.fs.rm(self._key_to_str(key))
175 except: # noqa: E722
176 raise KeyError
178 def __contains__(self, key):
179 """Does key exist in mapping?"""
180 path = self._key_to_str(key)
181 return self.fs.exists(path) and self.fs.isfile(path)
183 def __reduce__(self):
184 return FSMap, (self.root, self.fs, False, False, self.missing_exceptions)
187def maybe_convert(value):
188 if isinstance(value, array.array) or hasattr(value, "__array__"):
189 # bytes-like things
190 if hasattr(value, "dtype") and value.dtype.kind in "Mm":
191 # The buffer interface doesn't support datetime64/timdelta64 numpy
192 # arrays
193 value = value.view("int64")
194 value = bytes(memoryview(value))
195 return value
198def get_mapper(
199 url="",
200 check=False,
201 create=False,
202 missing_exceptions=None,
203 alternate_root=None,
204 **kwargs,
205):
206 """Create key-value interface for given URL and options
208 The URL will be of the form "protocol://location" and point to the root
209 of the mapper required. All keys will be file-names below this location,
210 and their values the contents of each key.
212 Also accepts compound URLs like zip::s3://bucket/file.zip , see ``fsspec.open``.
214 Parameters
215 ----------
216 url: str
217 Root URL of mapping
218 check: bool
219 Whether to attempt to read from the location before instantiation, to
220 check that the mapping does exist
221 create: bool
222 Whether to make the directory corresponding to the root before
223 instantiating
224 missing_exceptions: None or tuple
225 If given, these exception types will be regarded as missing keys and
226 return KeyError when trying to read data. By default, you get
227 (FileNotFoundError, IsADirectoryError, NotADirectoryError)
228 alternate_root: None or str
229 In cases of complex URLs, the parser may fail to pick the correct part
230 for the mapper root, so this arg can override
232 Returns
233 -------
234 ``FSMap`` instance, the dict-like key-value store.
235 """
236 # Removing protocol here - could defer to each open() on the backend
237 fs, urlpath = url_to_fs(url, **kwargs)
238 root = alternate_root if alternate_root is not None else urlpath
239 return FSMap(root, fs, check, create, missing_exceptions=missing_exceptions)