Line data    Source code

       1             : // Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
       2             : // of this source code is governed by a BSD-style license that can be found in
       3             : // the LICENSE file.
       4             : 
       5             : package remote
       6             : 
       7             : import (
       8             :         "context"
       9             :         "io"
      10             : 
      11             :         "github.com/cockroachdb/redact"
      12             : )
      13             : 
      14             : // Locator is an opaque string identifying a remote.Storage implementation.
      15             : //
      16             : // The Locator must not contain secrets (like authentication keys). Locators are
      17             : // stored on disk in the shared object catalog and are passed around as part of
      18             : // RemoteObjectBacking; they can also appear in error messages.
      19             : type Locator string
      20             : 
      21             : // SafeFormat implements redact.SafeFormatter.
      22           0 : func (l Locator) SafeFormat(w redact.SafePrinter, _ rune) {
      23           0 :         w.Printf("%s", redact.SafeString(l))
      24           0 : }
      25             : 
      26             : // StorageFactory is used to return Storage implementations based on locators. A
      27             : // Pebble store that uses shared storage is configured with a StorageFactory.
      28             : type StorageFactory interface {
      29             :         CreateStorage(locator Locator) (Storage, error)
      30             : }
      31             : 
      32             : // SharedLevelsStart denotes the highest (i.e. lowest numbered) level that will
      33             : // have sstables shared across Pebble instances when doing skip-shared
      34             : // iteration (see db.ScanInternal) or shared file ingestion (see
      35             : // db.IngestAndExcise).
      36             : const SharedLevelsStart = 5
      37             : 
      38             : // CreateOnSharedStrategy specifies what table files should be created on shared
      39             : // storage. For use with CreateOnShared in options.
      40             : type CreateOnSharedStrategy int
      41             : 
      42             : const (
      43             :         // CreateOnSharedNone denotes no files being created on shared storage.
      44             :         CreateOnSharedNone CreateOnSharedStrategy = iota
      45             :         // CreateOnSharedLower denotes the creation of files in lower levels of the
      46             :         // LSM (specifically, L5 and L6 as they're below SharedLevelsStart) on
      47             :         // shared storage, and higher levels on local storage.
      48             :         CreateOnSharedLower
      49             :         // CreateOnSharedAll denotes the creation of all sstables on shared storage.
      50             :         CreateOnSharedAll
      51             : )
      52             : 
      53             : // ShouldCreateShared returns whether new table files at the specified level
      54             : // should be created on shared storage.
      55           2 : func ShouldCreateShared(strategy CreateOnSharedStrategy, level int) bool {
      56           2 :         switch strategy {
      57           2 :         case CreateOnSharedAll:
      58           2 :                 return true
      59           2 :         case CreateOnSharedNone:
      60           2 :                 return false
      61           2 :         case CreateOnSharedLower:
      62           2 :                 return level >= SharedLevelsStart
      63           0 :         default:
      64           0 :                 panic("unexpected CreateOnSharedStrategy value")
      65             :         }
      66             : }
      67             : 
      68             : // Storage is an interface for a blob storage driver. This is lower-level
      69             : // than an FS-like interface, however FS/File-like abstractions can be built on
      70             : // top of these methods.
      71             : //
      72             : // TODO(bilal): Consider pushing shared file obsoletion as well as path
      73             : // generation behind this interface.
      74             : type Storage interface {
      75             :         io.Closer
      76             : 
      77             :         // ReadObject returns an ObjectReader that can be used to perform reads on an
      78             :         // object, along with the total size of the object.
      79             :         ReadObject(ctx context.Context, objName string) (_ ObjectReader, objSize int64, _ error)
      80             : 
      81             :         // CreateObject returns a writer for the object at the request name. A new
      82             :         // empty object is created if CreateObject is called on an existing object.
      83             :         //
      84             :         // A Writer *must* be closed via either Close, and if closing returns a
      85             :         // non-nil error, that error should be handled or reported to the user -- an
      86             :         // implementation may buffer written data until Close and only then return
      87             :         // an error, or Write may return an opaque io.EOF with the underlying cause
      88             :         // returned by the subsequent Close().
      89             :         //
      90             :         // TODO(radu): if we encounter some unrelated error while writing to the
      91             :         // WriteCloser, we'd want to abort the whole thing rather than letting Close
      92             :         // finalize the upload.
      93             :         CreateObject(objName string) (io.WriteCloser, error)
      94             : 
      95             :         // List enumerates files within the supplied prefix, returning a list of
      96             :         // objects within that prefix. If delimiter is non-empty, names which have the
      97             :         // same prefix, prior to the delimiter but after the prefix, are grouped into a
      98             :         // single result which is that prefix. The order that results are returned is
      99             :         // undefined. If a prefix is specified, the prefix is trimmed from the result
     100             :         // list.
     101             :         //
     102             :         // An example would be, if the storage contains objects a, b/4, b/5 and b/6,
     103             :         // these would be the return values:
     104             :         //   List("", "") -> ["a", "b/4", "b/5", "b/6"]
     105             :         //   List("", "/") -> ["a", "b"]
     106             :         //   List("b", "/") -> ["4", "5", "6"]
     107             :         //   List("b", "") -> ["/4", "/5", "/6"]
     108             :         List(prefix, delimiter string) ([]string, error)
     109             : 
     110             :         // Delete removes the named object from the store.
     111             :         Delete(objName string) error
     112             : 
     113             :         // Size returns the length of the named object in bytesWritten.
     114             :         Size(objName string) (int64, error)
     115             : 
     116             :         // IsNotExistError returns true if the given error (returned by a method in
     117             :         // this interface) indicates that the object does not exist.
     118             :         IsNotExistError(err error) bool
     119             : }
     120             : 
     121             : // ObjectReader is used to perform reads on an object.
     122             : type ObjectReader interface {
     123             :         // ReadAt reads len(p) bytes into p starting at offset off.
     124             :         //
     125             :         // Does not return partial results; if offset + len(p) is past the end of the
     126             :         // object, an error is returned.
     127             :         //
     128             :         // Clients of ReadAt can execute parallel ReadAt calls on the same
     129             :         // ObjectReader.
     130             :         ReadAt(ctx context.Context, p []byte, offset int64) error
     131             : 
     132             :         Close() error
     133             : }
     134             : 
     135             : // ObjectKey is a (locator, object name) pair which uniquely identifies a remote
     136             : // object and can be used as a map key.
     137             : type ObjectKey struct {
     138             :         Locator    Locator
     139             :         ObjectName string
     140             : }
     141             : 
     142             : // MakeObjectKey is a convenience constructor for ObjectKey.
     143           2 : func MakeObjectKey(locator Locator, objectName string) ObjectKey {
     144           2 :         return ObjectKey{
     145           2 :                 Locator:    locator,
     146           2 :                 ObjectName: objectName,
     147           2 :         }
     148           2 : }