Line data Source code
1 : // Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use 2 : // of this source code is governed by a BSD-style license that can be found in 3 : // the LICENSE file. 4 : 5 : package sstable 6 : 7 : import ( 8 : "github.com/cockroachdb/pebble/internal/base" 9 : "github.com/cockroachdb/pebble/internal/cache" 10 : ) 11 : 12 : // Compression is the per-block compression algorithm to use. 13 : type Compression int 14 : 15 : // The available compression types. 16 : const ( 17 : DefaultCompression Compression = iota 18 : NoCompression 19 : SnappyCompression 20 : ZstdCompression 21 : NCompression 22 : ) 23 : 24 : var ignoredInternalProperties = map[string]struct{}{ 25 : "rocksdb.column.family.id": {}, 26 : "rocksdb.fixed.key.length": {}, 27 : "rocksdb.index.key.is.user.key": {}, 28 : "rocksdb.index.value.is.delta.encoded": {}, 29 : "rocksdb.oldest.key.time": {}, 30 : "rocksdb.creation.time": {}, 31 : "rocksdb.file.creation.time": {}, 32 : "rocksdb.format.version": {}, 33 : } 34 : 35 1 : func (c Compression) String() string { 36 1 : switch c { 37 0 : case DefaultCompression: 38 0 : return "Default" 39 1 : case NoCompression: 40 1 : return "NoCompression" 41 1 : case SnappyCompression: 42 1 : return "Snappy" 43 1 : case ZstdCompression: 44 1 : return "ZSTD" 45 0 : default: 46 0 : return "Unknown" 47 : } 48 : } 49 : 50 : // FilterType exports the base.FilterType type. 51 : type FilterType = base.FilterType 52 : 53 : // Exported TableFilter constants. 54 : const ( 55 : TableFilter = base.TableFilter 56 : ) 57 : 58 : // FilterWriter exports the base.FilterWriter type. 59 : type FilterWriter = base.FilterWriter 60 : 61 : // FilterPolicy exports the base.FilterPolicy type. 62 : type FilterPolicy = base.FilterPolicy 63 : 64 : // ReaderOptions holds the parameters needed for reading an sstable. 65 : type ReaderOptions struct { 66 : // Cache is used to cache uncompressed blocks from sstables. 67 : // 68 : // The default cache size is a zero-size cache. 69 : Cache *cache.Cache 70 : 71 : // User properties specified in this map will not be added to sst.Properties.UserProperties. 72 : DeniedUserProperties map[string]struct{} 73 : 74 : // Comparer defines a total ordering over the space of []byte keys: a 'less 75 : // than' relationship. The same comparison algorithm must be used for reads 76 : // and writes over the lifetime of the DB. 77 : // 78 : // The default value uses the same ordering as bytes.Compare. 79 : Comparer *Comparer 80 : 81 : // Merge defines the Merge function in use for this keyspace. 82 : Merge base.Merge 83 : 84 : // Filters is a map from filter policy name to filter policy. It is used for 85 : // debugging tools which may be used on multiple databases configured with 86 : // different filter policies. It is not necessary to populate this filters 87 : // map during normal usage of a DB. 88 : Filters map[string]FilterPolicy 89 : 90 : // Merger defines the associative merge operation to use for merging values 91 : // written with {Batch,DB}.Merge. The MergerName is checked for consistency 92 : // with the value stored in the sstable when it was written. 93 : MergerName string 94 : 95 : // Logger is an optional logger and tracer. 96 : LoggerAndTracer base.LoggerAndTracer 97 : } 98 : 99 1 : func (o ReaderOptions) ensureDefaults() ReaderOptions { 100 1 : if o.Comparer == nil { 101 1 : o.Comparer = base.DefaultComparer 102 1 : } 103 1 : if o.Merge == nil { 104 1 : o.Merge = base.DefaultMerger.Merge 105 1 : } 106 1 : if o.MergerName == "" { 107 1 : o.MergerName = base.DefaultMerger.Name 108 1 : } 109 1 : if o.LoggerAndTracer == nil { 110 1 : o.LoggerAndTracer = base.NoopLoggerAndTracer{} 111 1 : } 112 1 : if o.DeniedUserProperties == nil { 113 1 : o.DeniedUserProperties = ignoredInternalProperties 114 1 : } 115 1 : return o 116 : } 117 : 118 : // WriterOptions holds the parameters used to control building an sstable. 119 : type WriterOptions struct { 120 : // BlockRestartInterval is the number of keys between restart points 121 : // for delta encoding of keys. 122 : // 123 : // The default value is 16. 124 : BlockRestartInterval int 125 : 126 : // BlockSize is the target uncompressed size in bytes of each table block. 127 : // 128 : // The default value is 4096. 129 : BlockSize int 130 : 131 : // BlockSizeThreshold finishes a block if the block size is larger than the 132 : // specified percentage of the target block size and adding the next entry 133 : // would cause the block to be larger than the target block size. 134 : // 135 : // The default value is 90. 136 : BlockSizeThreshold int 137 : 138 : // SizeClassAwareThreshold imposes a minimum block size restriction for blocks 139 : // to be flushed, that is computed as the percentage of the target block size. 140 : // Note that this threshold takes precedence over BlockSizeThreshold when 141 : // valid AllocatorSizeClasses are specified. 142 : // 143 : // The default value is 60. 144 : SizeClassAwareThreshold int 145 : 146 : // Cache is used to cache uncompressed blocks from sstables. 147 : // 148 : // The default is a nil cache. 149 : Cache *cache.Cache 150 : 151 : // Comparer defines a total ordering over the space of []byte keys: a 'less 152 : // than' relationship. The same comparison algorithm must be used for reads 153 : // and writes over the lifetime of the DB. 154 : // 155 : // The default value uses the same ordering as bytes.Compare. 156 : Comparer *Comparer 157 : 158 : // Compression defines the per-block compression to use. 159 : // 160 : // The default value (DefaultCompression) uses snappy compression. 161 : Compression Compression 162 : 163 : // FilterPolicy defines a filter algorithm (such as a Bloom filter) that can 164 : // reduce disk reads for Get calls. 165 : // 166 : // One such implementation is bloom.FilterPolicy(10) from the pebble/bloom 167 : // package. 168 : // 169 : // The default value means to use no filter. 170 : FilterPolicy FilterPolicy 171 : 172 : // FilterType defines whether an existing filter policy is applied at a 173 : // block-level or table-level. Block-level filters use less memory to create, 174 : // but are slower to access as a check for the key in the index must first be 175 : // performed to locate the filter block. A table-level filter will require 176 : // memory proportional to the number of keys in an sstable to create, but 177 : // avoids the index lookup when determining if a key is present. Table-level 178 : // filters should be preferred except under constrained memory situations. 179 : FilterType FilterType 180 : 181 : // IndexBlockSize is the target uncompressed size in bytes of each index 182 : // block. When the index block size is larger than this target, two-level 183 : // indexes are automatically enabled. Setting this option to a large value 184 : // (such as math.MaxInt32) disables the automatic creation of two-level 185 : // indexes. 186 : // 187 : // The default value is the value of BlockSize. 188 : IndexBlockSize int 189 : 190 : // Merger defines the associative merge operation to use for merging values 191 : // written with {Batch,DB}.Merge. The MergerName is checked for consistency 192 : // with the value stored in the sstable when it was written. 193 : MergerName string 194 : 195 : // TableFormat specifies the format version for writing sstables. The default 196 : // is TableFormatMinSupported. 197 : TableFormat TableFormat 198 : 199 : // IsStrictObsolete is only relevant for >= TableFormatPebblev4. See comment 200 : // in format.go. Must be false if format < TableFormatPebblev4. 201 : // 202 : // TODO(bilal): set this when writing shared ssts. 203 : IsStrictObsolete bool 204 : 205 : // WritingToLowestLevel is only relevant for >= TableFormatPebblev4. It is 206 : // used to set the obsolete bit on DEL/DELSIZED/SINGLEDEL if they are the 207 : // youngest for a userkey. 208 : WritingToLowestLevel bool 209 : 210 : // BlockPropertyCollectors is a list of BlockPropertyCollector creation 211 : // functions. A new BlockPropertyCollector is created for each sstable 212 : // built and lives for the lifetime of writing that table. 213 : BlockPropertyCollectors []func() BlockPropertyCollector 214 : 215 : // Checksum specifies which checksum to use. 216 : Checksum ChecksumType 217 : 218 : // Parallelism is used to indicate that the sstable Writer is allowed to 219 : // compress data blocks and write datablocks to disk in parallel with the 220 : // Writer client goroutine. 221 : Parallelism bool 222 : 223 : // ShortAttributeExtractor mirrors 224 : // Options.Experimental.ShortAttributeExtractor. 225 : ShortAttributeExtractor base.ShortAttributeExtractor 226 : 227 : // RequiredInPlaceValueBound mirrors 228 : // Options.Experimental.RequiredInPlaceValueBound. 229 : RequiredInPlaceValueBound UserKeyPrefixBound 230 : 231 : // DisableValueBlocks is only used for TableFormat >= TableFormatPebblev3, 232 : // and if set to true, does not write any values to value blocks. This is 233 : // only intended for cases where the in-memory buffering of all value blocks 234 : // while writing a sstable is too expensive and likely to cause an OOM. It 235 : // is never set to true by a Pebble DB, and can be set to true when some 236 : // external code is directly generating huge sstables using Pebble's 237 : // sstable.Writer (for example, CockroachDB backups can sometimes write 238 : // 750MB sstables -- see 239 : // https://github.com/cockroachdb/cockroach/issues/117113). 240 : DisableValueBlocks bool 241 : 242 : // AllocatorSizeClasses provides a sorted list containing the supported size 243 : // classes of the underlying memory allocator. This provides hints to the 244 : // writer's flushing policy to select block sizes that preemptively reduce 245 : // internal fragmentation when loaded into the block cache. 246 : AllocatorSizeClasses []int 247 : } 248 : 249 1 : func (o WriterOptions) ensureDefaults() WriterOptions { 250 1 : if o.BlockRestartInterval <= 0 { 251 1 : o.BlockRestartInterval = base.DefaultBlockRestartInterval 252 1 : } 253 1 : if o.BlockSize <= 0 { 254 1 : o.BlockSize = base.DefaultBlockSize 255 1 : } 256 1 : if o.BlockSizeThreshold <= 0 { 257 1 : o.BlockSizeThreshold = base.DefaultBlockSizeThreshold 258 1 : } 259 1 : if o.SizeClassAwareThreshold <= 0 { 260 1 : o.SizeClassAwareThreshold = base.SizeClassAwareBlockSizeThreshold 261 1 : } 262 1 : if o.Comparer == nil { 263 1 : o.Comparer = base.DefaultComparer 264 1 : } 265 1 : if o.Compression <= DefaultCompression || o.Compression >= NCompression { 266 1 : o.Compression = SnappyCompression 267 1 : } 268 1 : if o.IndexBlockSize <= 0 { 269 1 : o.IndexBlockSize = o.BlockSize 270 1 : } 271 1 : if o.MergerName == "" { 272 1 : o.MergerName = base.DefaultMerger.Name 273 1 : } 274 1 : if o.Checksum == ChecksumTypeNone { 275 1 : o.Checksum = ChecksumTypeCRC32c 276 1 : } 277 : // By default, if the table format is not specified, fall back to using the 278 : // most compatible format that is supported by Pebble. 279 1 : if o.TableFormat == TableFormatUnspecified { 280 1 : o.TableFormat = TableFormatMinSupported 281 1 : } 282 1 : return o 283 : }