Line data Source code
1 : // Copyright 2012 The LevelDB-Go and Pebble Authors. All rights reserved. Use
2 : // of this source code is governed by a BSD-style license that can be found in
3 : // the LICENSE file.
4 :
5 : package vfs
6 :
7 : import (
8 : "fmt"
9 : "io"
10 : "os"
11 : "path/filepath"
12 : "syscall"
13 :
14 : "github.com/cockroachdb/errors"
15 : "github.com/cockroachdb/errors/oserror"
16 : )
17 :
18 : // File is a readable, writable sequence of bytes.
19 : //
20 : // Typically, it will be an *os.File, but test code may choose to substitute
21 : // memory-backed implementations.
22 : //
23 : // Write-oriented operations (Write, Sync) must be called sequentially: At most
24 : // 1 call to Write or Sync may be executed at any given time.
25 : type File interface {
26 : io.Closer
27 : io.Reader
28 : io.ReaderAt
29 : // Unlike the specification for io.Writer.Write(), the vfs.File.Write()
30 : // method *is* allowed to modify the slice passed in, whether temporarily
31 : // or permanently. Callers of Write() need to take this into account.
32 : io.Writer
33 : // WriteAt() is only supported for files that were opened with FS.OpenReadWrite.
34 : io.WriterAt
35 :
36 : // Preallocate optionally preallocates storage for `length` at `offset`
37 : // within the file. Implementations may choose to do nothing.
38 : Preallocate(offset, length int64) error
39 : Stat() (FileInfo, error)
40 : Sync() error
41 :
42 : // SyncTo requests that a prefix of the file's data be synced to stable
43 : // storage. The caller passes provides a `length`, indicating how many bytes
44 : // to sync from the beginning of the file. SyncTo is a no-op for
45 : // directories, and therefore always returns false.
46 : //
47 : // SyncTo returns a fullSync return value, indicating one of two possible
48 : // outcomes.
49 : //
50 : // If fullSync is false, the first `length` bytes of the file was queued to
51 : // be synced to stable storage. The syncing of the file prefix may happen
52 : // asynchronously. No persistence guarantee is provided.
53 : //
54 : // If fullSync is true, the entirety of the file's contents were
55 : // synchronously synced to stable storage, and a persistence guarantee is
56 : // provided. In this outcome, any modified metadata for the file is not
57 : // guaranteed to be synced unless that metadata is needed in order to allow
58 : // a subsequent data retrieval to be correctly handled.
59 : SyncTo(length int64) (fullSync bool, err error)
60 :
61 : // SyncData requires that all written data be persisted. File metadata is
62 : // not required to be synced. Unsophisticated implementations may call Sync.
63 : SyncData() error
64 :
65 : // Prefetch signals the OS (on supported platforms) to fetch the next length
66 : // bytes in file (as returned by os.File.Fd()) after offset into cache. Any
67 : // subsequent reads in that range will not issue disk IO.
68 : Prefetch(offset int64, length int64) error
69 :
70 : // Fd returns the raw file descriptor when a File is backed by an *os.File.
71 : // It can be used for specific functionality like Prefetch.
72 : // Returns InvalidFd if not supported.
73 : Fd() uintptr
74 : }
75 :
76 : // InvalidFd is a special value returned by File.Fd() when the file is not
77 : // backed by an OS descriptor.
78 : // Note: the special value is consistent with what os.File implementation
79 : // returns on a nil receiver.
80 : const InvalidFd uintptr = ^(uintptr(0))
81 :
82 : // OpenOption provide an interface to do work on file handles in the Open()
83 : // call.
84 : type OpenOption interface {
85 : // Apply is called on the file handle after it's opened.
86 : Apply(File)
87 : }
88 :
89 : // FS is a namespace for files.
90 : //
91 : // The names are filepath names: they may be / separated or \ separated,
92 : // depending on the underlying operating system.
93 : type FS interface {
94 : // Create creates the named file for reading and writing. If a file
95 : // already exists at the provided name, it's removed first ensuring the
96 : // resulting file descriptor points to a new inode.
97 : Create(name string, category DiskWriteCategory) (File, error)
98 :
99 : // Link creates newname as a hard link to the oldname file.
100 : Link(oldname, newname string) error
101 :
102 : // Open opens the named file for reading. openOptions provides
103 : Open(name string, opts ...OpenOption) (File, error)
104 :
105 : // OpenReadWrite opens the named file for reading and writing. If the file
106 : // does not exist, it is created.
107 : OpenReadWrite(name string, category DiskWriteCategory, opts ...OpenOption) (File, error)
108 :
109 : // OpenDir opens the named directory for syncing.
110 : OpenDir(name string) (File, error)
111 :
112 : // Remove removes the named file or directory.
113 : Remove(name string) error
114 :
115 : // Remove removes the named file or directory and any children it
116 : // contains. It removes everything it can but returns the first error it
117 : // encounters.
118 : RemoveAll(name string) error
119 :
120 : // Rename renames a file. It overwrites the file at newname if one exists,
121 : // the same as os.Rename.
122 : Rename(oldname, newname string) error
123 :
124 : // ReuseForWrite attempts to reuse the file with oldname by renaming it to newname and opening
125 : // it for writing without truncation. It is acceptable for the implementation to choose not
126 : // to reuse oldname, and simply create the file with newname -- in this case the implementation
127 : // should delete oldname. If the caller calls this function with an oldname that does not exist,
128 : // the implementation may return an error.
129 : ReuseForWrite(oldname, newname string, category DiskWriteCategory) (File, error)
130 :
131 : // MkdirAll creates a directory and all necessary parents. The permission
132 : // bits perm have the same semantics as in os.MkdirAll. If the directory
133 : // already exists, MkdirAll does nothing and returns nil.
134 : MkdirAll(dir string, perm os.FileMode) error
135 :
136 : // Lock locks the given file, creating the file if necessary, and
137 : // truncating the file if it already exists. The lock is an exclusive lock
138 : // (a write lock), but locked files should neither be read from nor written
139 : // to. Such files should have zero size and only exist to co-ordinate
140 : // ownership across processes.
141 : //
142 : // A nil Closer is returned if an error occurred. Otherwise, close that
143 : // Closer to release the lock.
144 : //
145 : // On Linux and OSX, a lock has the same semantics as fcntl(2)'s advisory
146 : // locks. In particular, closing any other file descriptor for the same
147 : // file will release the lock prematurely.
148 : //
149 : // Attempting to lock a file that is already locked by the current process
150 : // returns an error and leaves the existing lock untouched.
151 : //
152 : // Lock is not yet implemented on other operating systems, and calling it
153 : // will return an error.
154 : Lock(name string) (io.Closer, error)
155 :
156 : // List returns a listing of the given directory. The names returned are
157 : // relative to dir.
158 : List(dir string) ([]string, error)
159 :
160 : // Stat returns an FileInfo describing the named file.
161 : Stat(name string) (FileInfo, error)
162 :
163 : // PathBase returns the last element of path. Trailing path separators are
164 : // removed before extracting the last element. If the path is empty, PathBase
165 : // returns ".". If the path consists entirely of separators, PathBase returns a
166 : // single separator.
167 : PathBase(path string) string
168 :
169 : // PathJoin joins any number of path elements into a single path, adding a
170 : // separator if necessary.
171 : PathJoin(elem ...string) string
172 :
173 : // PathDir returns all but the last element of path, typically the path's directory.
174 : PathDir(path string) string
175 :
176 : // GetDiskUsage returns disk space statistics for the filesystem where
177 : // path is any file or directory within that filesystem.
178 : GetDiskUsage(path string) (DiskUsage, error)
179 :
180 : // Unwrap is implemented by "wrapping" filesystems (those that add some
181 : // functionality on top of an underlying FS); it returns the wrapped FS.
182 : // It is used by vfs.Root.
183 : //
184 : // Returns nil if this is not a wrapping filesystem.
185 : Unwrap() FS
186 : }
187 :
188 : // A DeviceID uniquely identifies a block device on which filesystem data is
189 : // persisted.
190 : type DeviceID struct {
191 : major uint32
192 : minor uint32
193 : }
194 :
195 : // String returns the string representation of the device ID.
196 0 : func (d DeviceID) String() string {
197 0 : return fmt.Sprintf("%d:%d", d.major, d.minor)
198 0 : }
199 :
200 : // FileInfo describes a file.
201 : type FileInfo interface {
202 : os.FileInfo
203 : // DeviceID returns the ID of the device on which the file resides.
204 : DeviceID() DeviceID
205 : }
206 :
207 0 : func maybeWrapFileInfo(fi os.FileInfo, err error) (FileInfo, error) {
208 0 : if err != nil {
209 0 : return nil, err
210 0 : }
211 0 : return defaultFileInfo{FileInfo: fi}, nil
212 : }
213 :
214 : type defaultFileInfo struct {
215 : os.FileInfo
216 : }
217 :
218 : // DeviceID returns the ID of the device on which the file resides.
219 0 : func (fi defaultFileInfo) DeviceID() DeviceID {
220 0 : return deviceIDFromFileInfo(fi.FileInfo)
221 0 : }
222 :
223 : // DiskUsage summarizes disk space usage on a filesystem.
224 : type DiskUsage struct {
225 : // Total disk space available to the current process in bytes.
226 : AvailBytes uint64
227 : // Total disk space in bytes.
228 : TotalBytes uint64
229 : // Used disk space in bytes.
230 : UsedBytes uint64
231 : }
232 :
233 : // Default is a FS implementation backed by the underlying operating system's
234 : // file system.
235 : var Default FS = defaultFS{}
236 :
237 : type defaultFS struct{}
238 :
239 : // wrapOSFile takes a standard library OS file and returns a vfs.File. f may be
240 : // nil, in which case wrapOSFile must not panic. In such cases, it's okay if the
241 : // returned vfs.File may panic if used.
242 1 : func wrapOSFile(f *os.File) File {
243 1 : // See the implementations in default_{linux,unix,windows}.go.
244 1 : return wrapOSFileImpl(f)
245 1 : }
246 :
247 1 : func (defaultFS) Create(name string, category DiskWriteCategory) (File, error) {
248 1 : const openFlags = os.O_RDWR | os.O_CREATE | os.O_EXCL | syscall.O_CLOEXEC
249 1 :
250 1 : osFile, err := os.OpenFile(name, openFlags, 0666)
251 1 : // If the file already exists, remove it and try again.
252 1 : //
253 1 : // NB: We choose to remove the file instead of truncating it, despite the
254 1 : // fact that we can't do so atomically, because it's more resistant to
255 1 : // misuse when using hard links.
256 1 :
257 1 : // We must loop in case another goroutine/thread/process is also
258 1 : // attempting to create the a file at the same path.
259 1 : for oserror.IsExist(err) {
260 1 : if removeErr := os.Remove(name); removeErr != nil && !oserror.IsNotExist(removeErr) {
261 0 : return wrapOSFile(osFile), errors.WithStack(removeErr)
262 0 : }
263 1 : osFile, err = os.OpenFile(name, openFlags, 0666)
264 : }
265 1 : return wrapOSFile(osFile), errors.WithStack(err)
266 : }
267 :
268 1 : func (defaultFS) Link(oldname, newname string) error {
269 1 : return errors.WithStack(os.Link(oldname, newname))
270 1 : }
271 :
272 1 : func (defaultFS) Open(name string, opts ...OpenOption) (File, error) {
273 1 : osFile, err := os.OpenFile(name, os.O_RDONLY|syscall.O_CLOEXEC, 0)
274 1 : if err != nil {
275 0 : return nil, errors.WithStack(err)
276 0 : }
277 1 : file := wrapOSFile(osFile)
278 1 : for _, opt := range opts {
279 0 : opt.Apply(file)
280 0 : }
281 1 : return file, nil
282 : }
283 :
284 : func (defaultFS) OpenReadWrite(
285 : name string, category DiskWriteCategory, opts ...OpenOption,
286 0 : ) (File, error) {
287 0 : osFile, err := os.OpenFile(name, os.O_RDWR|syscall.O_CLOEXEC|os.O_CREATE, 0666)
288 0 : if err != nil {
289 0 : return nil, errors.WithStack(err)
290 0 : }
291 0 : file := wrapOSFile(osFile)
292 0 : for _, opt := range opts {
293 0 : opt.Apply(file)
294 0 : }
295 0 : return file, nil
296 : }
297 :
298 1 : func (defaultFS) Remove(name string) error {
299 1 : return errors.WithStack(os.Remove(name))
300 1 : }
301 :
302 0 : func (defaultFS) RemoveAll(name string) error {
303 0 : return errors.WithStack(os.RemoveAll(name))
304 0 : }
305 :
306 1 : func (defaultFS) Rename(oldname, newname string) error {
307 1 : return errors.WithStack(os.Rename(oldname, newname))
308 1 : }
309 :
310 : func (fs defaultFS) ReuseForWrite(
311 : oldname, newname string, category DiskWriteCategory,
312 0 : ) (File, error) {
313 0 : if err := fs.Rename(oldname, newname); err != nil {
314 0 : return nil, errors.WithStack(err)
315 0 : }
316 0 : f, err := os.OpenFile(newname, os.O_RDWR|os.O_CREATE|syscall.O_CLOEXEC, 0666)
317 0 : return wrapOSFile(f), errors.WithStack(err)
318 : }
319 :
320 1 : func (defaultFS) MkdirAll(dir string, perm os.FileMode) error {
321 1 : return errors.WithStack(os.MkdirAll(dir, perm))
322 1 : }
323 :
324 1 : func (defaultFS) List(dir string) ([]string, error) {
325 1 : f, err := os.Open(dir)
326 1 : if err != nil {
327 0 : return nil, err
328 0 : }
329 1 : defer f.Close()
330 1 : dirnames, err := f.Readdirnames(-1)
331 1 : return dirnames, errors.WithStack(err)
332 : }
333 :
334 1 : func (defaultFS) Stat(name string) (FileInfo, error) {
335 1 : finfo, err := os.Stat(name)
336 1 : if err != nil {
337 1 : return nil, errors.WithStack(err)
338 1 : }
339 1 : return defaultFileInfo{finfo}, nil
340 : }
341 :
342 1 : func (defaultFS) PathBase(path string) string {
343 1 : return filepath.Base(path)
344 1 : }
345 :
346 1 : func (defaultFS) PathJoin(elem ...string) string {
347 1 : return filepath.Join(elem...)
348 1 : }
349 :
350 1 : func (defaultFS) PathDir(path string) string {
351 1 : return filepath.Dir(path)
352 1 : }
353 :
354 0 : func (defaultFS) Unwrap() FS { return nil }
355 :
356 : type randomReadsOption struct{}
357 :
358 : // RandomReadsOption is an OpenOption that optimizes opened file handle for
359 : // random reads, by calling fadvise() with POSIX_FADV_RANDOM on Linux systems
360 : // to disable readahead.
361 : var RandomReadsOption OpenOption = &randomReadsOption{}
362 :
363 : // Apply implements the OpenOption interface.
364 1 : func (randomReadsOption) Apply(f File) {
365 1 : if fd := f.Fd(); fd != InvalidFd {
366 1 : _ = fadviseRandom(fd)
367 1 : }
368 : }
369 :
370 : type sequentialReadsOption struct{}
371 :
372 : // SequentialReadsOption is an OpenOption that optimizes opened file handle for
373 : // sequential reads, by calling fadvise() with POSIX_FADV_SEQUENTIAL on Linux
374 : // systems to enable readahead.
375 : var SequentialReadsOption OpenOption = &sequentialReadsOption{}
376 :
377 : // Apply implements the OpenOption interface.
378 1 : func (sequentialReadsOption) Apply(f File) {
379 1 : if fd := f.Fd(); fd != InvalidFd {
380 1 : _ = fadviseSequential(fd)
381 1 : }
382 : }
383 :
384 : // Copy copies the contents of oldname to newname. If newname exists, it will
385 : // be overwritten.
386 0 : func Copy(fs FS, oldname, newname string) error {
387 0 : return CopyAcrossFS(fs, oldname, fs, newname)
388 0 : }
389 :
390 : // CopyAcrossFS copies the contents of oldname on srcFS to newname dstFS. If
391 : // newname exists, it will be overwritten.
392 1 : func CopyAcrossFS(srcFS FS, oldname string, dstFS FS, newname string) error {
393 1 : src, err := srcFS.Open(oldname, SequentialReadsOption)
394 1 : if err != nil {
395 0 : return err
396 0 : }
397 1 : defer src.Close()
398 1 :
399 1 : dst, err := dstFS.Create(newname, WriteCategoryUnspecified)
400 1 : if err != nil {
401 0 : return err
402 0 : }
403 1 : defer dst.Close()
404 1 :
405 1 : if _, err := io.Copy(dst, src); err != nil {
406 0 : return err
407 0 : }
408 1 : return dst.Sync()
409 : }
410 :
411 : // LimitedCopy copies up to maxBytes from oldname to newname. If newname
412 : // exists, it will be overwritten.
413 0 : func LimitedCopy(fs FS, oldname, newname string, maxBytes int64) error {
414 0 : src, err := fs.Open(oldname, SequentialReadsOption)
415 0 : if err != nil {
416 0 : return err
417 0 : }
418 0 : defer src.Close()
419 0 :
420 0 : dst, err := fs.Create(newname, WriteCategoryUnspecified)
421 0 : if err != nil {
422 0 : return err
423 0 : }
424 0 : defer dst.Close()
425 0 :
426 0 : if _, err := io.Copy(dst, &io.LimitedReader{R: src, N: maxBytes}); err != nil {
427 0 : return err
428 0 : }
429 0 : return dst.Sync()
430 : }
431 :
432 : // LinkOrCopy creates newname as a hard link to the oldname file. If creating
433 : // the hard link fails, LinkOrCopy falls back to copying the file (which may
434 : // also fail if oldname doesn't exist or newname already exists).
435 1 : func LinkOrCopy(fs FS, oldname, newname string) error {
436 1 : err := fs.Link(oldname, newname)
437 1 : if err == nil {
438 1 : return nil
439 1 : }
440 : // Permit a handful of errors which we know won't be fixed by copying the
441 : // file. Note that we don't check for the specifics of the error code as it
442 : // isn't easy to do so in a portable manner. On Unix we'd have to check for
443 : // LinkError.Err == syscall.EXDEV. On Windows we'd have to check for
444 : // ERROR_NOT_SAME_DEVICE, ERROR_INVALID_FUNCTION, and
445 : // ERROR_INVALID_PARAMETER. Rather that such OS specific checks, we fall back
446 : // to always trying to copy if hard-linking failed.
447 0 : if oserror.IsExist(err) || oserror.IsNotExist(err) || oserror.IsPermission(err) {
448 0 : return err
449 0 : }
450 0 : return Copy(fs, oldname, newname)
451 : }
452 :
453 : // Root returns the base FS implementation, unwrapping all nested FSs that
454 : // expose an Unwrap method.
455 1 : func Root(fs FS) FS {
456 1 : for {
457 1 : n := fs.Unwrap()
458 1 : if n == nil {
459 1 : return fs
460 1 : }
461 1 : fs = n
462 : }
463 : }
464 :
465 : // ErrUnsupported may be returned a FS when it does not support an operation.
466 : var ErrUnsupported = errors.New("pebble: not supported")
|
