Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2<Program Name>
3 storage.py
5<Author>
6 Joshua Lock <jlock@vmware.com>
8<Started>
9 April 9, 2020
11<Copyright>
12 See LICENSE for licensing information.
14<Purpose>
15 Provides an interface for filesystem interactions, StorageBackendInterface.
16"""
18from __future__ import annotations
20import errno
21import logging
22import os
23import shutil
24import stat
25from abc import ABCMeta, abstractmethod
26from collections.abc import Iterator
27from contextlib import contextmanager
28from typing import IO, Any, BinaryIO
30from securesystemslib import exceptions
32logger = logging.getLogger(__name__)
35class StorageBackendInterface(metaclass=ABCMeta):
36 """
37 <Purpose>
38 Defines an interface for abstract storage operations which can be implemented
39 for a variety of storage solutions, such as remote and local filesystems.
40 """
42 @abstractmethod
43 @contextmanager
44 def get(self, filepath: str) -> Iterator[BinaryIO]:
45 """
46 <Purpose>
47 A context manager for 'with' statements that is used for retrieving files
48 from a storage backend and cleans up the files upon exit.
50 with storage_backend.get('/path/to/file') as file_object:
51 # operations
52 # file is now closed
54 <Arguments>
55 filepath:
56 The full path of the file to be retrieved.
58 <Exceptions>
59 securesystemslib.exceptions.StorageError, if the file does not exist or is
60 no accessible.
62 <Returns>
63 A ContextManager object that emits a file-like object for the file at
64 'filepath'.
65 """
66 raise NotImplementedError # pragma: no cover
68 @abstractmethod
69 def put(self, fileobj: IO, filepath: str, restrict: bool | None = False) -> None:
70 """
71 <Purpose>
72 Store a file-like object in the storage backend.
73 The file-like object is read from the beginning, not its current
74 offset (if any).
76 <Arguments>
77 fileobj:
78 The file-like object to be stored.
80 filepath:
81 The full path to the location where 'fileobj' will be stored.
83 restrict:
84 Whether the file should be created with restricted permissions.
85 What counts as restricted is backend-specific. For a filesystem on a
86 UNIX-like operating system, that may mean read/write permissions only
87 for the user (octal mode 0o600). For a cloud storage system, that
88 likely means Cloud provider specific ACL restrictions.
90 <Exceptions>
91 securesystemslib.exceptions.StorageError, if the file can not be stored.
93 <Returns>
94 None
95 """
96 raise NotImplementedError # pragma: no cover
98 @abstractmethod
99 def remove(self, filepath: str) -> None:
100 """
101 <Purpose>
102 Remove the file at 'filepath' from the storage.
104 <Arguments>
105 filepath:
106 The full path to the file.
108 <Exceptions>
109 securesystemslib.exceptions.StorageError, if the file can not be removed.
111 <Returns>
112 None
113 """
114 raise NotImplementedError # pragma: no cover
116 @abstractmethod
117 def getsize(self, filepath: str) -> int:
118 """
119 <Purpose>
120 Retrieve the size, in bytes, of the file at 'filepath'.
122 <Arguments>
123 filepath:
124 The full path to the file.
126 <Exceptions>
127 securesystemslib.exceptions.StorageError, if the file does not exist or is
128 not accessible.
130 <Returns>
131 The size in bytes of the file at 'filepath'.
132 """
133 raise NotImplementedError # pragma: no cover
135 @abstractmethod
136 def create_folder(self, filepath: str) -> None:
137 """
138 <Purpose>
139 Create a folder at filepath and ensure all intermediate components of the
140 path exist.
141 Passing an empty string for filepath does nothing and does not raise an
142 exception.
144 <Arguments>
145 filepath:
146 The full path of the folder to be created.
148 <Exceptions>
149 securesystemslib.exceptions.StorageError, if the folder can not be
150 created.
152 <Returns>
153 None
154 """
155 raise NotImplementedError # pragma: no cover
157 @abstractmethod
158 def list_folder(self, filepath: str) -> list[str]:
159 """
160 <Purpose>
161 List the contents of the folder at 'filepath'.
163 <Arguments>
164 filepath:
165 The full path of the folder to be listed.
167 <Exceptions>
168 securesystemslib.exceptions.StorageError, if the file does not exist or is
169 not accessible.
171 <Returns>
172 A list containing the names of the files in the folder. May be an empty
173 list.
174 """
175 raise NotImplementedError # pragma: no cover
178class FilesystemBackend(StorageBackendInterface):
179 """
180 <Purpose>
181 A concrete implementation of StorageBackendInterface which interacts with
182 local filesystems using Python standard library functions.
183 """
185 # As FilesystemBackend is effectively a stateless wrapper around various
186 # standard library operations, we only ever need a single instance of it.
187 # That single instance is safe to be (re-)used by all callers. Therefore
188 # implement the singleton pattern to avoid uneccesarily creating multiple
189 # objects.
190 _instance = None
192 def __new__(cls, *args: Any, **kwargs: Any) -> FilesystemBackend:
193 if cls._instance is None:
194 cls._instance = object.__new__(cls, *args, **kwargs)
195 return cls._instance
197 @contextmanager
198 def get(self, filepath: str) -> Iterator[BinaryIO]:
199 file_object = None
200 try:
201 file_object = open(filepath, "rb")
202 yield file_object
203 except OSError:
204 raise exceptions.StorageError(f"Can't open {filepath}")
205 finally:
206 if file_object is not None:
207 file_object.close()
209 def put(self, fileobj: IO, filepath: str, restrict: bool | None = False) -> None:
210 # If we are passed an open file, seek to the beginning such that we are
211 # copying the entire contents
212 if not fileobj.closed:
213 fileobj.seek(0)
215 # If a file with the same name already exists, the new permissions
216 # may not be applied.
217 try:
218 os.remove(filepath)
219 except OSError:
220 pass
222 try:
223 if restrict:
224 # On UNIX-based systems restricted files are created with read and
225 # write permissions for the user only (octal value 0o600).
226 fd = os.open(
227 filepath,
228 os.O_WRONLY | os.O_CREAT,
229 stat.S_IRUSR | stat.S_IWUSR,
230 )
231 else:
232 # Non-restricted files use the default 'mode' argument of os.open()
233 # granting read, write, and execute for all users (octal mode 0o777).
234 # NOTE: mode may be modified by the user's file mode creation mask
235 # (umask) or on Windows limited to the smaller set of OS supported
236 # permisssions.
237 fd = os.open(filepath, os.O_WRONLY | os.O_CREAT)
239 with os.fdopen(fd, "wb") as destination_file:
240 shutil.copyfileobj(fileobj, destination_file)
241 # Force the destination file to be written to disk
242 # from Python's internal and the operating system's buffers.
243 # os.fsync() should follow flush().
244 destination_file.flush()
245 os.fsync(destination_file.fileno())
246 except OSError:
247 raise exceptions.StorageError(f"Can't write file {filepath}")
249 def remove(self, filepath: str) -> None:
250 try:
251 os.remove(filepath)
252 except (
253 FileNotFoundError,
254 PermissionError,
255 OSError,
256 ): # pragma: no cover
257 raise exceptions.StorageError(f"Can't remove file {filepath}")
259 def getsize(self, filepath: str) -> int:
260 try:
261 return os.path.getsize(filepath)
262 except OSError:
263 raise exceptions.StorageError(f"Can't access file {filepath}")
265 def create_folder(self, filepath: str) -> None:
266 try:
267 os.makedirs(filepath)
268 except OSError as e:
269 # 'OSError' raised if the leaf directory already exists or cannot be
270 # created. Check for case where 'filepath' has already been created and
271 # silently ignore.
272 if e.errno == errno.EEXIST:
273 pass
274 elif e.errno == errno.ENOENT and not filepath:
275 raise exceptions.StorageError(
276 "Can't create a folder with an empty filepath!"
277 )
278 else:
279 raise exceptions.StorageError(f"Can't create folder at {filepath}")
281 def list_folder(self, filepath: str) -> list[str]:
282 try:
283 return os.listdir(filepath)
284 except FileNotFoundError:
285 raise exceptions.StorageError(f"Can't list folder at {filepath}")