Line data Source code
1 : // Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 : // of this source code is governed by a BSD-style license that can be found in 3 : // the LICENSE file. 4 : 5 : package remote 6 : 7 : import ( 8 : "context" 9 : "io" 10 : 11 : "github.com/cockroachdb/redact" 12 : ) 13 : 14 : // Locator is an opaque string identifying a remote.Storage implementation. 15 : // 16 : // The Locator must not contain secrets (like authentication keys). Locators are 17 : // stored on disk in the shared object catalog and are passed around as part of 18 : // RemoteObjectBacking; they can also appear in error messages. 19 : type Locator string 20 : 21 : // SafeFormat implements redact.SafeFormatter. 22 0 : func (l Locator) SafeFormat(w redact.SafePrinter, _ rune) { 23 0 : w.Printf("%s", redact.SafeString(l)) 24 0 : } 25 : 26 : // StorageFactory is used to return Storage implementations based on locators. A 27 : // Pebble store that uses shared storage is configured with a StorageFactory. 28 : type StorageFactory interface { 29 : CreateStorage(locator Locator) (Storage, error) 30 : } 31 : 32 : // SharedLevelsStart denotes the highest (i.e. lowest numbered) level that will 33 : // have sstables shared across Pebble instances when doing skip-shared 34 : // iteration (see db.ScanInternal) or shared file ingestion (see 35 : // db.IngestAndExcise). 36 : const SharedLevelsStart = 5 37 : 38 : // CreateOnSharedStrategy specifies what table files should be created on shared 39 : // storage. For use with CreateOnShared in options. 40 : type CreateOnSharedStrategy int 41 : 42 : const ( 43 : // CreateOnSharedNone denotes no files being created on shared storage. 44 : CreateOnSharedNone CreateOnSharedStrategy = iota 45 : // CreateOnSharedLower denotes the creation of files in lower levels of the 46 : // LSM (specifically, L5 and L6 as they're below SharedLevelsStart) on 47 : // shared storage, and higher levels on local storage. 48 : CreateOnSharedLower 49 : // CreateOnSharedAll denotes the creation of all sstables on shared storage. 50 : CreateOnSharedAll 51 : ) 52 : 53 : // ShouldCreateShared returns whether new table files at the specified level 54 : // should be created on shared storage. 55 1 : func ShouldCreateShared(strategy CreateOnSharedStrategy, level int) bool { 56 1 : switch strategy { 57 1 : case CreateOnSharedAll: 58 1 : return true 59 1 : case CreateOnSharedNone: 60 1 : return false 61 1 : case CreateOnSharedLower: 62 1 : return level >= SharedLevelsStart 63 0 : default: 64 0 : panic("unexpected CreateOnSharedStrategy value") 65 : } 66 : } 67 : 68 : // Storage is an interface for a blob storage driver. This is lower-level 69 : // than an FS-like interface, however FS/File-like abstractions can be built on 70 : // top of these methods. 71 : // 72 : // TODO(bilal): Consider pushing shared file obsoletion as well as path 73 : // generation behind this interface. 74 : type Storage interface { 75 : io.Closer 76 : 77 : // ReadObject returns an ObjectReader that can be used to perform reads on an 78 : // object, along with the total size of the object. 79 : ReadObject(ctx context.Context, objName string) (_ ObjectReader, objSize int64, _ error) 80 : 81 : // CreateObject returns a writer for the object at the request name. A new 82 : // empty object is created if CreateObject is called on an existing object. 83 : // 84 : // A Writer *must* be closed via either Close, and if closing returns a 85 : // non-nil error, that error should be handled or reported to the user -- an 86 : // implementation may buffer written data until Close and only then return 87 : // an error, or Write may return an opaque io.EOF with the underlying cause 88 : // returned by the subsequent Close(). 89 : // 90 : // TODO(radu): if we encounter some unrelated error while writing to the 91 : // WriteCloser, we'd want to abort the whole thing rather than letting Close 92 : // finalize the upload. 93 : CreateObject(objName string) (io.WriteCloser, error) 94 : 95 : // List enumerates files within the supplied prefix, returning a list of 96 : // objects within that prefix. If delimiter is non-empty, names which have the 97 : // same prefix, prior to the delimiter but after the prefix, are grouped into a 98 : // single result which is that prefix. The order that results are returned is 99 : // undefined. If a prefix is specified, the prefix is trimmed from the result 100 : // list. 101 : // 102 : // An example would be, if the storage contains objects a, b/4, b/5 and b/6, 103 : // these would be the return values: 104 : // List("", "") -> ["a", "b/4", "b/5", "b/6"] 105 : // List("", "/") -> ["a", "b"] 106 : // List("b", "/") -> ["4", "5", "6"] 107 : // List("b", "") -> ["/4", "/5", "/6"] 108 : List(prefix, delimiter string) ([]string, error) 109 : 110 : // Delete removes the named object from the store. 111 : Delete(objName string) error 112 : 113 : // Size returns the length of the named object in bytesWritten. 114 : Size(objName string) (int64, error) 115 : 116 : // IsNotExistError returns true if the given error (returned by a method in 117 : // this interface) indicates that the object does not exist. 118 : IsNotExistError(err error) bool 119 : } 120 : 121 : // ObjectReader is used to perform reads on an object. 122 : type ObjectReader interface { 123 : // ReadAt reads len(p) bytes into p starting at offset off. 124 : // 125 : // Does not return partial results; if offset + len(p) is past the end of the 126 : // object, an error is returned. 127 : // 128 : // Clients of ReadAt can execute parallel ReadAt calls on the same 129 : // ObjectReader. 130 : ReadAt(ctx context.Context, p []byte, offset int64) error 131 : 132 : Close() error 133 : } 134 : 135 : // ObjectKey is a (locator, object name) pair which uniquely identifies a remote 136 : // object and can be used as a map key. 137 : type ObjectKey struct { 138 : Locator Locator 139 : ObjectName string 140 : } 141 : 142 : // MakeObjectKey is a convenience constructor for ObjectKey. 143 1 : func MakeObjectKey(locator Locator, objectName string) ObjectKey { 144 1 : return ObjectKey{ 145 1 : Locator: locator, 146 1 : ObjectName: objectName, 147 1 : } 148 1 : }