Line data Source code
1 : // Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
2 : // of this source code is governed by a BSD-style license that can be found in
3 : // the LICENSE file.
4 :
5 : package metamorphic
6 :
7 : import (
8 : "bytes"
9 : "fmt"
10 : "math"
11 : "math/rand/v2"
12 : "os"
13 : "path/filepath"
14 : "runtime"
15 : "strconv"
16 : "strings"
17 : "sync/atomic"
18 : "time"
19 :
20 : "github.com/cockroachdb/errors"
21 : "github.com/cockroachdb/pebble"
22 : "github.com/cockroachdb/pebble/bloom"
23 : "github.com/cockroachdb/pebble/internal/base"
24 : "github.com/cockroachdb/pebble/internal/cache"
25 : "github.com/cockroachdb/pebble/objstorage/remote"
26 : "github.com/cockroachdb/pebble/sstable"
27 : "github.com/cockroachdb/pebble/sstable/block"
28 : "github.com/cockroachdb/pebble/vfs"
29 : "github.com/cockroachdb/pebble/wal"
30 : )
31 :
32 : const (
33 : minimumFormatMajorVersion = pebble.FormatMinSupported
34 : // The format major version to use in the default options configurations. We
35 : // default to the minimum supported format so we exercise the runtime version
36 : // ratcheting that a cluster upgrading would experience. The randomized
37 : // options may still use format major versions that are less than
38 : // defaultFormatMajorVersion but are at least minimumFormatMajorVersion.
39 : defaultFormatMajorVersion = pebble.FormatMinSupported
40 : // newestFormatMajorVersionToTest is the most recent format major version
41 : // the metamorphic tests should use. This may be greater than
42 : // pebble.FormatNewest when some format major versions are marked as
43 : // experimental.
44 : newestFormatMajorVersionToTest = pebble.FormatNewest
45 : )
46 :
47 : func parseOptions(
48 : opts *TestOptions, data string, customOptionParsers map[string]func(string) (CustomOption, bool),
49 2 : ) error {
50 2 : hooks := &pebble.ParseHooks{
51 2 : NewCache: pebble.NewCache,
52 2 : NewFilterPolicy: filterPolicyFromName,
53 2 : SkipUnknown: func(name, value string) bool {
54 2 : switch name {
55 0 : case "TestOptions":
56 0 : return true
57 2 : case "TestOptions.strictfs":
58 2 : opts.strictFS = true
59 2 : opts.Opts.FS = vfs.NewCrashableMem()
60 2 : return true
61 2 : case "TestOptions.key_format":
62 2 : opts.KeyFormat = keyFormatsByName[value]
63 2 : return true
64 2 : case "TestOptions.ingest_using_apply":
65 2 : opts.ingestUsingApply = true
66 2 : return true
67 2 : case "TestOptions.delete_sized":
68 2 : opts.deleteSized = true
69 2 : return true
70 2 : case "TestOptions.replace_single_delete":
71 2 : opts.replaceSingleDelete = true
72 2 : return true
73 2 : case "TestOptions.use_disk":
74 2 : opts.useDisk = true
75 2 : opts.Opts.FS = vfs.Default
76 2 : return true
77 0 : case "TestOptions.initial_state_desc":
78 0 : opts.initialStateDesc = value
79 0 : return true
80 0 : case "TestOptions.initial_state_path":
81 0 : opts.initialStatePath = value
82 0 : return true
83 2 : case "TestOptions.threads":
84 2 : v, err := strconv.Atoi(value)
85 2 : if err != nil {
86 0 : panic(err)
87 : }
88 2 : opts.Threads = v
89 2 : return true
90 2 : case "TestOptions.disable_block_property_collector":
91 2 : v, err := strconv.ParseBool(value)
92 2 : if err != nil {
93 0 : panic(err)
94 : }
95 2 : opts.disableBlockPropertyCollector = v
96 2 : if v {
97 2 : opts.Opts.BlockPropertyCollectors = nil
98 2 : }
99 2 : return true
100 2 : case "TestOptions.enable_value_blocks":
101 2 : opts.enableValueBlocks = true
102 2 : opts.Opts.Experimental.EnableValueBlocks = func() bool { return true }
103 2 : return true
104 2 : case "TestOptions.disable_value_blocks_for_ingest_sstables":
105 2 : opts.disableValueBlocksForIngestSSTables = true
106 2 : return true
107 2 : case "TestOptions.async_apply_to_db":
108 2 : opts.asyncApplyToDB = true
109 2 : return true
110 2 : case "TestOptions.shared_storage_enabled":
111 2 : opts.sharedStorageEnabled = true
112 2 : opts.sharedStorageFS = remote.NewInMem()
113 2 : if opts.Opts.Experimental.CreateOnShared == remote.CreateOnSharedNone {
114 1 : opts.Opts.Experimental.CreateOnShared = remote.CreateOnSharedAll
115 1 : }
116 2 : return true
117 2 : case "TestOptions.external_storage_enabled":
118 2 : opts.externalStorageEnabled = true
119 2 : opts.externalStorageFS = remote.NewInMem()
120 2 : return true
121 2 : case "TestOptions.secondary_cache_enabled":
122 2 : opts.secondaryCacheEnabled = true
123 2 : opts.Opts.Experimental.SecondaryCacheSizeBytes = 1024 * 1024 * 32 // 32 MBs
124 2 : return true
125 2 : case "TestOptions.seed_efos":
126 2 : v, err := strconv.ParseUint(value, 10, 64)
127 2 : if err != nil {
128 0 : panic(err)
129 : }
130 2 : opts.seedEFOS = v
131 2 : return true
132 2 : case "TestOptions.io_latency_mean":
133 2 : v, err := time.ParseDuration(value)
134 2 : if err != nil {
135 0 : panic(err)
136 : }
137 2 : opts.ioLatencyMean = v
138 2 : return true
139 2 : case "TestOptions.io_latency_probability":
140 2 : v, err := strconv.ParseFloat(value, 64)
141 2 : if err != nil {
142 0 : panic(err)
143 : }
144 2 : opts.ioLatencyProbability = v
145 2 : return true
146 2 : case "TestOptions.io_latency_seed":
147 2 : v, err := strconv.ParseInt(value, 10, 64)
148 2 : if err != nil {
149 0 : panic(err)
150 : }
151 2 : opts.ioLatencySeed = v
152 2 : return true
153 2 : case "TestOptions.ingest_split":
154 2 : opts.ingestSplit = true
155 2 : opts.Opts.Experimental.IngestSplit = func() bool {
156 2 : return true
157 2 : }
158 2 : return true
159 2 : case "TestOptions.use_shared_replicate":
160 2 : opts.useSharedReplicate = true
161 2 : return true
162 0 : case "TestOptions.use_external_replicate":
163 0 : opts.useExternalReplicate = true
164 0 : return true
165 2 : case "TestOptions.use_excise":
166 2 : opts.useExcise = true
167 2 : return true
168 2 : case "TestOptions.use_delete_only_compaction_excises":
169 2 : opts.useDeleteOnlyCompactionExcises = true
170 2 : opts.Opts.Experimental.EnableDeleteOnlyCompactionExcises = func() bool {
171 1 : return opts.useDeleteOnlyCompactionExcises
172 1 : }
173 2 : return true
174 2 : case "TestOptions.disable_downloads":
175 2 : opts.disableDownloads = true
176 2 : return true
177 2 : case "TestOptions.use_jemalloc_size_classes":
178 2 : opts.Opts.AllocatorSizeClasses = pebble.JemallocSizeClasses
179 2 : return true
180 1 : default:
181 1 : if customOptionParsers == nil {
182 0 : return false
183 0 : }
184 1 : name = strings.TrimPrefix(name, "TestOptions.")
185 1 : if p, ok := customOptionParsers[name]; ok {
186 1 : if customOpt, ok := p(value); ok {
187 1 : opts.CustomOpts = append(opts.CustomOpts, customOpt)
188 1 : return true
189 1 : }
190 : }
191 0 : return false
192 : }
193 : },
194 : }
195 2 : err := opts.Opts.Parse(data, hooks)
196 2 : // Ensure that the WAL failover FS agrees with the primary FS. They're
197 2 : // separate options, but in the metamorphic tests we keep them in sync.
198 2 : if opts.Opts.WALFailover != nil {
199 2 : opts.Opts.WALFailover.Secondary.FS = opts.Opts.FS
200 2 : }
201 2 : opts.InitRemoteStorageFactory()
202 2 : opts.Opts.EnsureDefaults()
203 2 : return err
204 : }
205 :
206 1 : func optionsToString(opts *TestOptions) string {
207 1 : var buf bytes.Buffer
208 1 : fmt.Fprintf(&buf, " key_format=%s\n", opts.KeyFormat.Name)
209 1 : if opts.strictFS {
210 1 : fmt.Fprint(&buf, " strictfs=true\n")
211 1 : }
212 1 : if opts.ingestUsingApply {
213 1 : fmt.Fprint(&buf, " ingest_using_apply=true\n")
214 1 : }
215 1 : if opts.deleteSized {
216 1 : fmt.Fprint(&buf, " delete_sized=true\n")
217 1 : }
218 1 : if opts.replaceSingleDelete {
219 1 : fmt.Fprint(&buf, " replace_single_delete=true\n")
220 1 : }
221 1 : if opts.useDisk {
222 1 : fmt.Fprint(&buf, " use_disk=true\n")
223 1 : }
224 1 : if opts.initialStatePath != "" {
225 0 : fmt.Fprintf(&buf, " initial_state_path=%s\n", opts.initialStatePath)
226 0 : }
227 1 : if opts.initialStateDesc != "" {
228 0 : fmt.Fprintf(&buf, " initial_state_desc=%s\n", opts.initialStateDesc)
229 0 : }
230 1 : if opts.Threads != 0 {
231 1 : fmt.Fprintf(&buf, " threads=%d\n", opts.Threads)
232 1 : }
233 1 : if opts.disableBlockPropertyCollector {
234 1 : fmt.Fprintf(&buf, " disable_block_property_collector=%t\n", opts.disableBlockPropertyCollector)
235 1 : }
236 1 : if opts.enableValueBlocks {
237 1 : fmt.Fprintf(&buf, " enable_value_blocks=%t\n", opts.enableValueBlocks)
238 1 : }
239 1 : if opts.disableValueBlocksForIngestSSTables {
240 1 : fmt.Fprintf(&buf, " disable_value_blocks_for_ingest_sstables=%t\n", opts.disableValueBlocksForIngestSSTables)
241 1 : }
242 1 : if opts.asyncApplyToDB {
243 1 : fmt.Fprint(&buf, " async_apply_to_db=true\n")
244 1 : }
245 1 : if opts.sharedStorageEnabled {
246 1 : fmt.Fprint(&buf, " shared_storage_enabled=true\n")
247 1 : }
248 1 : if opts.externalStorageEnabled {
249 1 : fmt.Fprint(&buf, " external_storage_enabled=true\n")
250 1 : }
251 1 : if opts.secondaryCacheEnabled {
252 1 : fmt.Fprint(&buf, " secondary_cache_enabled=true\n")
253 1 : }
254 1 : if opts.seedEFOS != 0 {
255 1 : fmt.Fprintf(&buf, " seed_efos=%d\n", opts.seedEFOS)
256 1 : }
257 1 : if opts.ingestSplit {
258 1 : fmt.Fprintf(&buf, " ingest_split=%v\n", opts.ingestSplit)
259 1 : }
260 1 : if opts.ioLatencyProbability > 0 {
261 1 : fmt.Fprintf(&buf, " io_latency_mean=%s\n", opts.ioLatencyMean)
262 1 : fmt.Fprintf(&buf, " io_latency_probability=%.10f\n", opts.ioLatencyProbability)
263 1 : fmt.Fprintf(&buf, " io_latency_seed=%d\n", opts.ioLatencySeed)
264 1 : }
265 1 : if opts.useSharedReplicate {
266 1 : fmt.Fprintf(&buf, " use_shared_replicate=%v\n", opts.useSharedReplicate)
267 1 : }
268 1 : if opts.useExternalReplicate {
269 0 : fmt.Fprintf(&buf, " use_external_replicate=%v\n", opts.useExternalReplicate)
270 0 : }
271 1 : if opts.useExcise {
272 1 : fmt.Fprintf(&buf, " use_excise=%v\n", opts.useExcise)
273 1 : }
274 1 : if opts.useDeleteOnlyCompactionExcises {
275 1 : fmt.Fprintf(&buf, " use_delete_only_compaction_excises=%v\n", opts.useDeleteOnlyCompactionExcises)
276 1 : }
277 1 : if opts.disableDownloads {
278 1 : fmt.Fprintf(&buf, " disable_downloads=%v\n", opts.disableDownloads)
279 1 : }
280 1 : if opts.Opts.AllocatorSizeClasses != nil {
281 1 : if fmt.Sprint(opts.Opts.AllocatorSizeClasses) != fmt.Sprint(pebble.JemallocSizeClasses) {
282 0 : panic(fmt.Sprintf("unexpected AllocatorSizeClasses %v", opts.Opts.AllocatorSizeClasses))
283 : }
284 1 : fmt.Fprint(&buf, " use_jemalloc_size_classes=true\n")
285 : }
286 1 : for _, customOpt := range opts.CustomOpts {
287 1 : fmt.Fprintf(&buf, " %s=%s\n", customOpt.Name(), customOpt.Value())
288 1 : }
289 :
290 1 : s := opts.Opts.String()
291 1 : if buf.Len() == 0 {
292 0 : return s
293 0 : }
294 1 : return s + "\n[TestOptions]\n" + buf.String()
295 : }
296 :
297 2 : func defaultTestOptions() *TestOptions {
298 2 : kf := TestkeysKeyFormat
299 2 : return &TestOptions{
300 2 : Opts: defaultOptions(kf),
301 2 : Threads: 16,
302 2 : RetryPolicy: NeverRetry,
303 2 : KeyFormat: kf,
304 2 : }
305 2 : }
306 :
307 2 : func defaultOptions(kf KeyFormat) *pebble.Options {
308 2 : opts := &pebble.Options{
309 2 : // Use an archive cleaner to ease post-mortem debugging.
310 2 : Cleaner: base.ArchiveCleaner{},
311 2 : // Always use our custom comparer which provides a Split method,
312 2 : // splitting keys at the trailing '@'.
313 2 : Comparer: kf.Comparer,
314 2 : KeySchema: kf.KeySchema.Name,
315 2 : KeySchemas: sstable.MakeKeySchemas(kf.KeySchema),
316 2 : FS: vfs.NewMem(),
317 2 : FormatMajorVersion: defaultFormatMajorVersion,
318 2 : Levels: []pebble.LevelOptions{{
319 2 : FilterPolicy: bloom.FilterPolicy(10),
320 2 : }},
321 2 : BlockPropertyCollectors: blockPropertyCollectorConstructors,
322 2 : }
323 2 : opts.Experimental.EnableColumnarBlocks = func() bool { return true }
324 :
325 : // We don't want to run the level checker every time because it can slow down
326 : // downloads and background compactions too much.
327 : //
328 : // We aim to run it once every 500ms (on average). To do this with some
329 : // randomization, each time we get a callback we see how much time passed
330 : // since the last call and run the check with a proportional probability.
331 2 : const meanTimeBetweenChecks = 500 * time.Millisecond
332 2 : startTime := time.Now()
333 2 : // lastCallTime stores the time of the last DebugCheck call, as the duration
334 2 : // since startTime.
335 2 : var lastCallTime atomic.Uint64
336 2 : opts.DebugCheck = func(db *pebble.DB) error {
337 2 : now := time.Since(startTime)
338 2 : last := time.Duration(lastCallTime.Swap(uint64(now)))
339 2 : // Run the check with probability equal to the time (as a fraction of
340 2 : // meanTimeBetweenChecks) passed since the last time we had a chance, as a
341 2 : // fraction of meanTimeBetweenChecks.
342 2 : if rand.Float64() < float64(now-last)/float64(meanTimeBetweenChecks) {
343 2 : return pebble.DebugCheckLevels(db)
344 2 : }
345 2 : return nil
346 : }
347 :
348 2 : return opts
349 : }
350 :
351 : // TestOptions describes the options configuring an individual run of the
352 : // metamorphic tests.
353 : type TestOptions struct {
354 : // Opts holds the *pebble.Options for the test.
355 : Opts *pebble.Options
356 : // Threads configures the parallelism of the test. Each thread will run in
357 : // an independent goroutine and be responsible for executing operations
358 : // against an independent set of objects. The outcome of any individual
359 : // operation will still be deterministic, with the metamorphic test
360 : // inserting synchronization where necessary.
361 : Threads int
362 : // RetryPolicy configures which errors should be retried.
363 : RetryPolicy RetryPolicy
364 : // KeyFormat defines the format of keys used within the test.
365 : KeyFormat KeyFormat
366 : // CustomOptions holds custom test options that are defined outside of this
367 : // package.
368 : CustomOpts []CustomOption
369 :
370 : // internal
371 :
372 : useDisk bool
373 : strictFS bool
374 : // Use Batch.Apply rather than DB.Ingest.
375 : ingestUsingApply bool
376 : // Use Batch.DeleteSized rather than Batch.Delete.
377 : deleteSized bool
378 : // Replace a SINGLEDEL with a DELETE.
379 : replaceSingleDelete bool
380 : // The path on the local filesystem where the initial state of the database
381 : // exists. Empty if the test run begins from an empty database state.
382 : initialStatePath string
383 : // A human-readable string describing the initial state of the database.
384 : // Empty if the test run begins from an empty database state.
385 : initialStateDesc string
386 : // Disable the block property collector, which may be used by block property
387 : // filters.
388 : disableBlockPropertyCollector bool
389 : // Enable the use of value blocks.
390 : enableValueBlocks bool
391 : // Disables value blocks in the sstables written for ingest.
392 : disableValueBlocksForIngestSSTables bool
393 : // Use DB.ApplyNoSyncWait for applies that want to sync the WAL.
394 : asyncApplyToDB bool
395 : // Enable the use of shared storage.
396 : sharedStorageEnabled bool
397 : sharedStorageFS remote.Storage
398 : // Enable the use of shared storage for external file ingestion.
399 : externalStorageEnabled bool
400 : externalStorageFS remote.Storage
401 : // Enables the use of shared replication in TestOptions.
402 : useSharedReplicate bool
403 : // Enables the use of external replication in TestOptions.
404 : useExternalReplicate bool
405 : // Enable the secondary cache. Only effective if sharedStorageEnabled is
406 : // also true.
407 : secondaryCacheEnabled bool
408 : // If nonzero, enables the use of EventuallyFileOnlySnapshots for
409 : // newSnapshotOps that are keyspan-bounded. The set of which newSnapshotOps
410 : // are actually created as EventuallyFileOnlySnapshots is deterministically
411 : // derived from the seed and the operation index.
412 : seedEFOS uint64
413 : // If nonzero, enables the injection of random IO latency. The mechanics of
414 : // a Pebble operation can be very timing dependent, so artificial latency
415 : // can ensure a wide variety of mechanics are exercised. Additionally,
416 : // exercising some mechanics such as WAL failover require IO latency.
417 : ioLatencyProbability float64
418 : ioLatencySeed int64
419 : ioLatencyMean time.Duration
420 : // Enables ingest splits. Saved here for serialization as Options does not
421 : // serialize this.
422 : ingestSplit bool
423 : // Enables operations that do excises. Note that a false value for this does
424 : // not guarantee the lack of excises, as useSharedReplicate can also cause
425 : // excises. However !useExcise && !useSharedReplicate can be used to guarantee
426 : // lack of excises.
427 : useExcise bool
428 : // useDeleteOnlyCompactionExcises turns on the ability for delete-only compactions
429 : // to do excises. Note that this can be true even when useExcise is false.
430 : useDeleteOnlyCompactionExcises bool
431 : // disableDownloads, if true, makes downloadOp a no-op.
432 : disableDownloads bool
433 : }
434 :
435 : // InitRemoteStorageFactory initializes Opts.Experimental.RemoteStorage.
436 2 : func (testOpts *TestOptions) InitRemoteStorageFactory() {
437 2 : if testOpts.sharedStorageEnabled || testOpts.externalStorageEnabled {
438 2 : m := make(map[remote.Locator]remote.Storage)
439 2 : if testOpts.sharedStorageEnabled {
440 2 : m[""] = testOpts.sharedStorageFS
441 2 : }
442 2 : if testOpts.externalStorageEnabled {
443 2 : m["external"] = testOpts.externalStorageFS
444 2 : }
445 2 : testOpts.Opts.Experimental.RemoteStorage = remote.MakeSimpleFactory(m)
446 : }
447 : }
448 :
449 : // CustomOption defines a custom option that configures the behavior of an
450 : // individual test run. Like all test options, custom options are serialized to
451 : // the OPTIONS file even if they're not options ordinarily understood by Pebble.
452 : type CustomOption interface {
453 : // Name returns the name of the custom option. This is the key under which
454 : // the option appears in the OPTIONS file, within the [TestOptions] stanza.
455 : Name() string
456 : // Value returns the value of the custom option, serialized as it should
457 : // appear within the OPTIONS file.
458 : Value() string
459 : // Close is run after the test database has been closed at the end of the
460 : // test as well as during restart operations within the test sequence. It's
461 : // passed a copy of the *pebble.Options. If the custom options hold on to
462 : // any resources outside, Close should release them.
463 : Close(*pebble.Options) error
464 : // Open is run before the test runs and during a restart operation after the
465 : // test database has been closed and Close has been called. It's passed a
466 : // copy of the *pebble.Options. If the custom options must acquire any
467 : // resources before the test continues, it should reacquire them.
468 : Open(*pebble.Options) error
469 :
470 : // TODO(jackson): provide additional hooks for custom options changing the
471 : // behavior of a run.
472 : }
473 :
474 1 : func standardOptions() []*TestOptions {
475 1 : // The index labels are not strictly necessary, but they make it easier to
476 1 : // find which options correspond to a failure.
477 1 : stdOpts := []string{
478 1 : 0: "", // default options
479 1 : 1: `
480 1 : [Options]
481 1 : cache_size=1
482 1 : `,
483 1 : 2: `
484 1 : [Options]
485 1 : disable_wal=true
486 1 : [TestOptions]
487 1 : use_jemalloc_size_classes=true
488 1 : `,
489 1 : 3: `
490 1 : [Options]
491 1 : l0_compaction_threshold=1
492 1 : `,
493 1 : 4: `
494 1 : [Options]
495 1 : l0_compaction_threshold=1
496 1 : l0_stop_writes_threshold=1
497 1 : `,
498 1 : 5: `
499 1 : [Options]
500 1 : lbase_max_bytes=1
501 1 : `,
502 1 : 6: `
503 1 : [Options]
504 1 : max_manifest_file_size=1
505 1 : `,
506 1 : 7: `
507 1 : [Options]
508 1 : max_open_files=1
509 1 : `,
510 1 : 8: `
511 1 : [Options]
512 1 : mem_table_size=2000
513 1 : `,
514 1 : 9: `
515 1 : [Options]
516 1 : mem_table_stop_writes_threshold=2
517 1 : `,
518 1 : 10: `
519 1 : [Options]
520 1 : wal_dir=data/wal
521 1 : `,
522 1 : 11: `
523 1 : [Level "0"]
524 1 : block_restart_interval=1
525 1 : `,
526 1 : 12: `
527 1 : [Level "0"]
528 1 : block_size=1
529 1 : `,
530 1 : 13: `
531 1 : [Level "0"]
532 1 : compression=NoCompression
533 1 : `,
534 1 : 14: `
535 1 : [Level "0"]
536 1 : index_block_size=1
537 1 : `,
538 1 : 15: `
539 1 : [Options]
540 1 : l0_stop_writes_threshold=100
541 1 : [Level "0"]
542 1 : target_file_size=12
543 1 : `,
544 1 : 16: `
545 1 : [Level "0"]
546 1 : filter_policy=none
547 1 : `,
548 1 : // 1GB
549 1 : 17: `
550 1 : [Options]
551 1 : bytes_per_sync=1073741824
552 1 : [TestOptions]
553 1 : strictfs=true
554 1 : `,
555 1 : 18: `
556 1 : [Options]
557 1 : max_concurrent_compactions=2
558 1 : `,
559 1 : 19: `
560 1 : [TestOptions]
561 1 : ingest_using_apply=true
562 1 : `,
563 1 : 20: `
564 1 : [TestOptions]
565 1 : replace_single_delete=true
566 1 : `,
567 1 : 21: `
568 1 : [TestOptions]
569 1 : use_disk=true
570 1 : use_jemalloc_size_classes=true
571 1 : `,
572 1 : 22: `
573 1 : [Options]
574 1 : max_writer_concurrency=2
575 1 : force_writer_parallelism=true
576 1 : `,
577 1 : 23: `
578 1 : [TestOptions]
579 1 : disable_block_property_collector=true
580 1 : `,
581 1 : 24: `
582 1 : [TestOptions]
583 1 : threads=1
584 1 : `,
585 1 : 25: `
586 1 : [TestOptions]
587 1 : enable_value_blocks=true
588 1 : use_jemalloc_size_classes=true
589 1 : `,
590 1 : 26: fmt.Sprintf(`
591 1 : [Options]
592 1 : format_major_version=%s
593 1 : `, newestFormatMajorVersionToTest),
594 1 : 27: fmt.Sprintf(`
595 1 : [Options]
596 1 : format_major_version=%s
597 1 : [TestOptions]
598 1 : shared_storage_enabled=true
599 1 : secondary_cache_enabled=true
600 1 : `, pebble.FormatMinForSharedObjects),
601 1 : 28: fmt.Sprintf(`
602 1 : [Options]
603 1 : format_major_version=%s
604 1 : [TestOptions]
605 1 : external_storage_enabled=true
606 1 : `, pebble.FormatSyntheticPrefixSuffix),
607 1 : 29: fmt.Sprintf(`
608 1 : [Options]
609 1 : format_major_version=%s
610 1 : max_concurrent_downloads=2
611 1 : [TestOptions]
612 1 : shared_storage_enabled=true
613 1 : external_storage_enabled=true
614 1 : secondary_cache_enabled=false
615 1 : `, pebble.FormatSyntheticPrefixSuffix),
616 1 : }
617 1 :
618 1 : opts := make([]*TestOptions, len(stdOpts))
619 1 : for i := range opts {
620 1 : opts[i] = defaultTestOptions()
621 1 : // NB: The standard options by definition can never include custom
622 1 : // options, so no need to propagate custom option parsers.
623 1 : if err := parseOptions(opts[i], stdOpts[i], nil /* custom option parsers */); err != nil {
624 0 : panic(err)
625 : }
626 : }
627 1 : return opts
628 : }
629 :
630 : // RandomOptions generates a random set of operations, drawing randomness from
631 : // rng.
632 : func RandomOptions(
633 : rng *rand.Rand, customOptionParsers map[string]func(string) (CustomOption, bool),
634 1 : ) *TestOptions {
635 1 : testOpts := defaultTestOptions()
636 1 : opts := testOpts.Opts
637 1 :
638 1 : // There are some private options, which we don't want users to fiddle with.
639 1 : // There's no way to set it through the public interface. The only method is
640 1 : // through Parse.
641 1 : {
642 1 : var privateOpts bytes.Buffer
643 1 : fmt.Fprintln(&privateOpts, `[Options]`)
644 1 : if rng.IntN(3) == 0 /* 33% */ {
645 1 : fmt.Fprintln(&privateOpts, ` disable_delete_only_compactions=true`)
646 1 : }
647 1 : if rng.IntN(3) == 0 /* 33% */ {
648 1 : fmt.Fprintln(&privateOpts, ` disable_elision_only_compactions=true`)
649 1 : }
650 1 : if rng.IntN(5) == 0 /* 20% */ {
651 1 : fmt.Fprintln(&privateOpts, ` disable_lazy_combined_iteration=true`)
652 1 : }
653 1 : if privateOptsStr := privateOpts.String(); privateOptsStr != `[Options]\n` {
654 1 : parseOptions(testOpts, privateOptsStr, customOptionParsers)
655 1 : }
656 : }
657 :
658 1 : opts.BytesPerSync = 1 << uint(rng.IntN(28)) // 1B - 256MB
659 1 : opts.Cache = cache.New(1 << uint(rng.IntN(30))) // 1B - 1GB
660 1 : opts.DisableWAL = rng.IntN(2) == 0
661 1 : opts.FlushDelayDeleteRange = time.Millisecond * time.Duration(5*rng.IntN(245)) // 5-250ms
662 1 : opts.FlushDelayRangeKey = time.Millisecond * time.Duration(5*rng.IntN(245)) // 5-250ms
663 1 : opts.FlushSplitBytes = 1 << rng.IntN(20) // 1B - 1MB
664 1 : opts.FormatMajorVersion = minimumFormatMajorVersion
665 1 : n := int(newestFormatMajorVersionToTest - opts.FormatMajorVersion)
666 1 : opts.FormatMajorVersion += pebble.FormatMajorVersion(rng.IntN(n + 1))
667 1 : opts.Experimental.L0CompactionConcurrency = 1 + rng.IntN(4) // 1-4
668 1 : opts.Experimental.LevelMultiplier = 5 << rng.IntN(7) // 5 - 320
669 1 : opts.TargetByteDeletionRate = 1 << uint(20+rng.IntN(10)) // 1MB - 1GB
670 1 : opts.Experimental.ValidateOnIngest = rng.IntN(2) != 0
671 1 : opts.L0CompactionThreshold = 1 + rng.IntN(100) // 1 - 100
672 1 : opts.L0CompactionFileThreshold = 1 << rng.IntN(11) // 1 - 1024
673 1 : opts.L0StopWritesThreshold = 50 + rng.IntN(100) // 50 - 150
674 1 : if opts.L0StopWritesThreshold < 2*opts.L0CompactionThreshold {
675 1 : opts.L0StopWritesThreshold = 2 * opts.L0CompactionThreshold
676 1 : }
677 1 : opts.LBaseMaxBytes = 1 << uint(rng.IntN(30)) // 1B - 1GB
678 1 : maxConcurrentCompactions := rng.IntN(3) + 1 // 1-3
679 1 : opts.MaxConcurrentCompactions = func() int {
680 1 : return maxConcurrentCompactions
681 1 : }
682 1 : maxConcurrentDownloads := rng.IntN(3) + 1 // 1-3
683 1 : opts.MaxConcurrentDownloads = func() int {
684 1 : return maxConcurrentDownloads
685 1 : }
686 1 : opts.MaxManifestFileSize = 1 << uint(rng.IntN(30)) // 1B - 1GB
687 1 : opts.MemTableSize = 2 << (10 + uint(rng.IntN(16))) // 2KB - 256MB
688 1 : opts.MemTableStopWritesThreshold = 2 + rng.IntN(5) // 2 - 5
689 1 : if rng.IntN(2) == 0 {
690 1 : opts.WALDir = "data/wal"
691 1 : }
692 :
693 : // Half the time enable WAL failover.
694 1 : if rng.IntN(2) == 0 {
695 1 : // Use 10x longer durations when writing directly to FS; we don't want
696 1 : // WAL failover to trigger excessively frequently.
697 1 : referenceDur := time.Millisecond
698 1 : if testOpts.useDisk {
699 0 : referenceDur *= 10
700 0 : }
701 :
702 1 : scaleDuration := func(d time.Duration, minFactor, maxFactor float64) time.Duration {
703 1 : return time.Duration(float64(d) * (minFactor + rng.Float64()*(maxFactor-minFactor)))
704 1 : }
705 1 : probeInterval := scaleDuration(referenceDur, 0.10, 0.50) // Between 10-50% of the reference duration.
706 1 : unhealthyThreshold := expRandDuration(rng, 3*referenceDur, time.Second)
707 1 : healthyThreshold := expRandDuration(rng, 3*referenceDur, time.Second)
708 1 : // Use a healthy interval between 1-119x the probe interval. The probe
709 1 : // maintains a maximum history 120 entries, so the healthy interval
710 1 : // must not exceed 119x the probe interval.
711 1 : healthyInterval := scaleDuration(probeInterval, 1.0, 119.0)
712 1 : opts.WALFailover = &pebble.WALFailoverOptions{
713 1 : Secondary: wal.Dir{FS: vfs.Default, Dirname: "data/wal_secondary"},
714 1 : FailoverOptions: wal.FailoverOptions{
715 1 : PrimaryDirProbeInterval: probeInterval,
716 1 : HealthyProbeLatencyThreshold: healthyThreshold,
717 1 : HealthyInterval: healthyInterval,
718 1 : UnhealthySamplingInterval: scaleDuration(unhealthyThreshold, 0.10, 0.50), // Between 10-50% of the unhealthy threshold
719 1 : UnhealthyOperationLatencyThreshold: func() (time.Duration, bool) {
720 1 : return unhealthyThreshold, true
721 1 : },
722 : ElevatedWriteStallThresholdLag: expRandDuration(rng, 5*referenceDur, 2*time.Second),
723 : },
724 : }
725 : }
726 1 : if rng.IntN(4) == 0 {
727 1 : // Enable Writer parallelism for 25% of the random options. Setting
728 1 : // MaxWriterConcurrency to any value greater than or equal to 1 has the
729 1 : // same effect currently.
730 1 : opts.Experimental.MaxWriterConcurrency = 2
731 1 : opts.Experimental.ForceWriterParallelism = true
732 1 : }
733 1 : if rng.IntN(2) == 0 {
734 1 : opts.Experimental.DisableIngestAsFlushable = func() bool { return true }
735 : }
736 :
737 : // We either use no multilevel compactions, multilevel compactions with the
738 : // default (zero) additional propensity, or multilevel compactions with an
739 : // additional propensity to encourage more multilevel compactions than we
740 : // ohterwise would.
741 1 : switch rng.IntN(3) {
742 1 : case 0:
743 1 : opts.Experimental.MultiLevelCompactionHeuristic = pebble.NoMultiLevel{}
744 1 : case 1:
745 1 : opts.Experimental.MultiLevelCompactionHeuristic = pebble.WriteAmpHeuristic{}
746 1 : default:
747 1 : opts.Experimental.MultiLevelCompactionHeuristic = pebble.WriteAmpHeuristic{
748 1 : AddPropensity: rng.Float64() * float64(rng.IntN(3)), // [0,3.0)
749 1 : AllowL0: rng.IntN(4) == 1, // 25% of the time
750 1 : }
751 : }
752 1 : if rng.IntN(2) == 0 {
753 1 : opts.AllocatorSizeClasses = pebble.JemallocSizeClasses
754 1 : }
755 :
756 1 : var lopts pebble.LevelOptions
757 1 : lopts.BlockRestartInterval = 1 + rng.IntN(64) // 1 - 64
758 1 : lopts.BlockSize = 1 << uint(rng.IntN(24)) // 1 - 16MB
759 1 : lopts.BlockSizeThreshold = 50 + rng.IntN(50) // 50 - 100
760 1 : lopts.IndexBlockSize = 1 << uint(rng.IntN(24)) // 1 - 16MB
761 1 : lopts.TargetFileSize = 1 << uint(rng.IntN(28)) // 1 - 256MB
762 1 : if lopts.TargetFileSize < 1<<12 {
763 1 : // We will generate a lot of files, which will slow down compactions.
764 1 : // Increase L0StopWritesThreshold to reduce the number of write stalls
765 1 : // that could cause operation timeouts.
766 1 : opts.L0StopWritesThreshold = 100
767 1 : }
768 : // The EstimatedSize of an empty table writer is 8 bytes. We want something a
769 : // little bigger than that as the minimum target.
770 1 : lopts.TargetFileSize = max(lopts.TargetFileSize, 12)
771 1 :
772 1 : // We either use no bloom filter, the default filter, or a filter with
773 1 : // randomized bits-per-key setting. We zero out the Filters map. It'll get
774 1 : // repopulated on EnsureDefaults accordingly.
775 1 : opts.Filters = nil
776 1 : switch rng.IntN(3) {
777 1 : case 0:
778 1 : lopts.FilterPolicy = nil
779 1 : case 1:
780 1 : lopts.FilterPolicy = bloom.FilterPolicy(10)
781 1 : default:
782 1 : lopts.FilterPolicy = newTestingFilterPolicy(1 << rng.IntN(5))
783 : }
784 :
785 : // We use either no compression, snappy compression or zstd compression.
786 1 : switch rng.IntN(3) {
787 1 : case 0:
788 1 : lopts.Compression = func() block.Compression { return pebble.NoCompression }
789 1 : case 1:
790 1 : lopts.Compression = func() block.Compression { return pebble.ZstdCompression }
791 1 : default:
792 1 : lopts.Compression = func() block.Compression { return pebble.SnappyCompression }
793 : }
794 1 : opts.Levels = []pebble.LevelOptions{lopts}
795 1 :
796 1 : // Explicitly disable disk-backed FS's for the random configurations. The
797 1 : // single standard test configuration that uses a disk-backed FS is
798 1 : // sufficient.
799 1 : testOpts.useDisk = false
800 1 : testOpts.strictFS = rng.IntN(2) != 0 // Only relevant for MemFS.
801 1 : // 50% of the time, enable IO latency injection.
802 1 : if rng.IntN(2) == 0 {
803 1 : // Note: we want ioLatencyProbability to be at least 1e-10, otherwise it
804 1 : // might print as 0 when we stringify options.
805 1 : testOpts.ioLatencyProbability = 1e-10 + 0.01*rng.Float64() // 0-1%
806 1 : testOpts.ioLatencyMean = expRandDuration(rng, 3*time.Millisecond, time.Second)
807 1 : testOpts.ioLatencySeed = rng.Int64()
808 1 : }
809 1 : testOpts.Threads = rng.IntN(runtime.GOMAXPROCS(0)) + 1
810 1 : if testOpts.strictFS {
811 1 : opts.DisableWAL = false
812 1 : opts.FS = vfs.NewCrashableMem()
813 1 : } else if !testOpts.useDisk {
814 1 : opts.FS = vfs.NewMem()
815 1 : }
816 : // Update the WALFailover's secondary to use the same FS. This isn't
817 : // strictly necessary (the WALFailover could use a separate FS), but it
818 : // ensures when we save a copy of the test state to disk, we include the
819 : // secondary's WALs.
820 1 : if opts.WALFailover != nil {
821 1 : opts.WALFailover.Secondary.FS = opts.FS
822 1 : }
823 1 : testOpts.ingestUsingApply = rng.IntN(2) != 0
824 1 : testOpts.deleteSized = rng.IntN(2) != 0
825 1 : testOpts.replaceSingleDelete = rng.IntN(2) != 0
826 1 : testOpts.disableBlockPropertyCollector = rng.IntN(2) == 1
827 1 : if testOpts.disableBlockPropertyCollector {
828 1 : testOpts.Opts.BlockPropertyCollectors = nil
829 1 : }
830 1 : testOpts.enableValueBlocks = rng.IntN(2) != 0
831 1 : if testOpts.enableValueBlocks {
832 1 : testOpts.Opts.Experimental.EnableValueBlocks = func() bool { return true }
833 : }
834 1 : testOpts.disableValueBlocksForIngestSSTables = rng.IntN(2) == 0
835 1 : testOpts.asyncApplyToDB = rng.IntN(2) != 0
836 1 : // 20% of time, enable shared storage.
837 1 : if rng.IntN(5) == 0 {
838 1 : testOpts.sharedStorageEnabled = true
839 1 : if testOpts.Opts.FormatMajorVersion < pebble.FormatMinForSharedObjects {
840 1 : testOpts.Opts.FormatMajorVersion = pebble.FormatMinForSharedObjects
841 1 : }
842 1 : testOpts.sharedStorageFS = remote.NewInMem()
843 1 : // If shared storage is enabled, pick between writing all files on shared
844 1 : // vs. lower levels only, 50% of the time.
845 1 : testOpts.Opts.Experimental.CreateOnShared = remote.CreateOnSharedAll
846 1 : if rng.IntN(2) == 0 {
847 1 : testOpts.Opts.Experimental.CreateOnShared = remote.CreateOnSharedLower
848 1 : }
849 : // If shared storage is enabled, enable secondary cache 50% of time.
850 1 : if rng.IntN(2) == 0 {
851 1 : testOpts.secondaryCacheEnabled = true
852 1 : // TODO(josh): Randomize various secondary cache settings.
853 1 : testOpts.Opts.Experimental.SecondaryCacheSizeBytes = 1024 * 1024 * 32 // 32 MBs
854 1 : }
855 : // 50% of the time, enable shared replication.
856 1 : testOpts.useSharedReplicate = rng.IntN(2) == 0
857 : }
858 :
859 : // 50% of time, enable external storage.
860 1 : if rng.IntN(2) == 0 {
861 1 : testOpts.externalStorageEnabled = true
862 1 : if testOpts.Opts.FormatMajorVersion < pebble.FormatSyntheticPrefixSuffix {
863 1 : testOpts.Opts.FormatMajorVersion = pebble.FormatSyntheticPrefixSuffix
864 1 : }
865 1 : testOpts.externalStorageFS = remote.NewInMem()
866 : }
867 :
868 1 : testOpts.seedEFOS = rng.Uint64()
869 1 : testOpts.ingestSplit = rng.IntN(2) == 0
870 1 : opts.Experimental.IngestSplit = func() bool { return testOpts.ingestSplit }
871 1 : testOpts.useExcise = rng.IntN(2) == 0
872 1 : if testOpts.useExcise {
873 1 : if testOpts.Opts.FormatMajorVersion < pebble.FormatVirtualSSTables {
874 1 : testOpts.Opts.FormatMajorVersion = pebble.FormatVirtualSSTables
875 1 : }
876 : }
877 1 : testOpts.useDeleteOnlyCompactionExcises = rng.IntN(2) == 0
878 1 : opts.Experimental.EnableDeleteOnlyCompactionExcises = func() bool {
879 1 : return testOpts.useDeleteOnlyCompactionExcises
880 1 : }
881 1 : testOpts.disableDownloads = rng.IntN(2) == 0
882 1 : testOpts.InitRemoteStorageFactory()
883 1 : testOpts.Opts.EnsureDefaults()
884 1 : return testOpts
885 : }
886 :
887 1 : func expRandDuration(rng *rand.Rand, meanDur, maxDur time.Duration) time.Duration {
888 1 : return min(maxDur, time.Duration(math.Round(rng.ExpFloat64()*float64(meanDur))))
889 1 : }
890 :
891 1 : func setupInitialState(dataDir string, testOpts *TestOptions) error {
892 1 : // Copy (vfs.Default,<initialStatePath>/data) to (testOpts.opts.FS,<dataDir>).
893 1 : ok, err := vfs.Clone(
894 1 : vfs.Default,
895 1 : testOpts.Opts.FS,
896 1 : vfs.Default.PathJoin(testOpts.initialStatePath, "data"),
897 1 : dataDir,
898 1 : vfs.CloneSync,
899 1 : vfs.CloneSkip(func(filename string) bool {
900 1 : // Skip the archive of historical files, any checkpoints created by
901 1 : // operations and files staged for ingest in tmp.
902 1 : b := filepath.Base(filename)
903 1 : return b == "archive" || b == "checkpoints" || b == "tmp"
904 1 : }))
905 1 : if err != nil {
906 0 : return err
907 1 : } else if !ok {
908 0 : return os.ErrNotExist
909 0 : }
910 :
911 : // Tests with wal_dir set store their WALs in a `wal` directory. The source
912 : // database (initialStatePath) could've had wal_dir set, or the current test
913 : // options (testOpts) could have wal_dir set, or both.
914 : //
915 : // If the test opts are not configured to use a WAL dir, we add the WAL dir
916 : // as a 'WAL recovery dir' so that we'll read any WALs in the directory in
917 : // Open.
918 1 : walRecoveryPath := testOpts.Opts.FS.PathJoin(dataDir, "wal")
919 1 : if testOpts.Opts.WALDir != "" {
920 0 : // If the test opts are configured to use a WAL dir, we add the data
921 0 : // directory itself as a 'WAL recovery dir' so that we'll read any WALs if
922 0 : // the previous test was writing them to the data directory.
923 0 : walRecoveryPath = dataDir
924 0 : }
925 1 : testOpts.Opts.WALRecoveryDirs = append(testOpts.Opts.WALRecoveryDirs, wal.Dir{
926 1 : FS: testOpts.Opts.FS,
927 1 : Dirname: walRecoveryPath,
928 1 : })
929 1 :
930 1 : // If the failover dir exists and the test opts are not configured to use
931 1 : // WAL failover, add the failover directory as a 'WAL recovery dir' in case
932 1 : // the previous test was configured to use failover.
933 1 : failoverDir := testOpts.Opts.FS.PathJoin(dataDir, "wal_secondary")
934 1 : if _, err := testOpts.Opts.FS.Stat(failoverDir); err == nil && testOpts.Opts.WALFailover == nil {
935 0 : testOpts.Opts.WALRecoveryDirs = append(testOpts.Opts.WALRecoveryDirs, wal.Dir{
936 0 : FS: testOpts.Opts.FS,
937 0 : Dirname: failoverDir,
938 0 : })
939 0 : }
940 1 : return nil
941 : }
942 :
943 : var blockPropertyCollectorConstructors = []func() pebble.BlockPropertyCollector{
944 : sstable.NewTestKeysBlockPropertyCollector,
945 : }
946 :
947 : // testingFilterPolicy is used to allow bloom filter policies with non-default
948 : // bits-per-key setting. It is necessary because the name of the production
949 : // filter policy is fixed (see bloom.FilterPolicy.Name()); we need to output a
950 : // custom policy name to the OPTIONS file that the test can then parse.
951 : type testingFilterPolicy struct {
952 : bloom.FilterPolicy
953 : }
954 :
955 : var _ pebble.FilterPolicy = (*testingFilterPolicy)(nil)
956 :
957 2 : func newTestingFilterPolicy(bitsPerKey int) *testingFilterPolicy {
958 2 : return &testingFilterPolicy{
959 2 : FilterPolicy: bloom.FilterPolicy(bitsPerKey),
960 2 : }
961 2 : }
962 :
963 : const testingFilterPolicyFmt = "testing_bloom_filter/bits_per_key=%d"
964 :
965 : // Name implements the pebble.FilterPolicy interface.
966 2 : func (t *testingFilterPolicy) Name() string {
967 2 : if t.FilterPolicy == 10 {
968 0 : return "rocksdb.BuiltinBloomFilter"
969 0 : }
970 2 : return fmt.Sprintf(testingFilterPolicyFmt, t.FilterPolicy)
971 : }
972 :
973 2 : func filterPolicyFromName(name string) (pebble.FilterPolicy, error) {
974 2 : switch name {
975 2 : case "none":
976 2 : return nil, nil
977 2 : case "rocksdb.BuiltinBloomFilter":
978 2 : return bloom.FilterPolicy(10), nil
979 : }
980 2 : var bitsPerKey int
981 2 : if _, err := fmt.Sscanf(name, testingFilterPolicyFmt, &bitsPerKey); err != nil {
982 0 : return nil, errors.Errorf("Invalid filter policy name '%s'", name)
983 0 : }
984 2 : return newTestingFilterPolicy(bitsPerKey), nil
985 : }
|
