Line data Source code
1 : // Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
2 : // of this source code is governed by a BSD-style license that can be found in
3 : // the LICENSE file.
4 :
5 : package metamorphic
6 :
7 : import (
8 : "bytes"
9 : "fmt"
10 : "math"
11 : "math/rand/v2"
12 : "os"
13 : "path/filepath"
14 : "runtime"
15 : "strconv"
16 : "strings"
17 : "sync/atomic"
18 : "time"
19 :
20 : "github.com/cockroachdb/errors"
21 : "github.com/cockroachdb/pebble"
22 : "github.com/cockroachdb/pebble/bloom"
23 : "github.com/cockroachdb/pebble/internal/base"
24 : "github.com/cockroachdb/pebble/internal/cache"
25 : "github.com/cockroachdb/pebble/internal/testkeys"
26 : "github.com/cockroachdb/pebble/objstorage/remote"
27 : "github.com/cockroachdb/pebble/sstable"
28 : "github.com/cockroachdb/pebble/sstable/block"
29 : "github.com/cockroachdb/pebble/vfs"
30 : "github.com/cockroachdb/pebble/wal"
31 : )
32 :
33 : const (
34 : minimumFormatMajorVersion = pebble.FormatMinSupported
35 : // The format major version to use in the default options configurations. We
36 : // default to the minimum supported format so we exercise the runtime version
37 : // ratcheting that a cluster upgrading would experience. The randomized
38 : // options may still use format major versions that are less than
39 : // defaultFormatMajorVersion but are at least minimumFormatMajorVersion.
40 : defaultFormatMajorVersion = pebble.FormatMinSupported
41 : // newestFormatMajorVersionToTest is the most recent format major version
42 : // the metamorphic tests should use. This may be greater than
43 : // pebble.FormatNewest when some format major versions are marked as
44 : // experimental.
45 : newestFormatMajorVersionToTest = pebble.FormatNewest
46 : )
47 :
48 : func parseOptions(
49 : opts *TestOptions, data string, customOptionParsers map[string]func(string) (CustomOption, bool),
50 1 : ) error {
51 1 : hooks := &pebble.ParseHooks{
52 1 : NewCache: pebble.NewCache,
53 1 : NewFilterPolicy: filterPolicyFromName,
54 1 : SkipUnknown: func(name, value string) bool {
55 1 : switch name {
56 0 : case "TestOptions":
57 0 : return true
58 1 : case "TestOptions.strictfs":
59 1 : opts.strictFS = true
60 1 : opts.Opts.FS = vfs.NewCrashableMem()
61 1 : return true
62 1 : case "TestOptions.ingest_using_apply":
63 1 : opts.ingestUsingApply = true
64 1 : return true
65 1 : case "TestOptions.delete_sized":
66 1 : opts.deleteSized = true
67 1 : return true
68 1 : case "TestOptions.replace_single_delete":
69 1 : opts.replaceSingleDelete = true
70 1 : return true
71 1 : case "TestOptions.use_disk":
72 1 : opts.useDisk = true
73 1 : opts.Opts.FS = vfs.Default
74 1 : return true
75 0 : case "TestOptions.initial_state_desc":
76 0 : opts.initialStateDesc = value
77 0 : return true
78 0 : case "TestOptions.initial_state_path":
79 0 : opts.initialStatePath = value
80 0 : return true
81 1 : case "TestOptions.threads":
82 1 : v, err := strconv.Atoi(value)
83 1 : if err != nil {
84 0 : panic(err)
85 : }
86 1 : opts.Threads = v
87 1 : return true
88 1 : case "TestOptions.disable_block_property_collector":
89 1 : v, err := strconv.ParseBool(value)
90 1 : if err != nil {
91 0 : panic(err)
92 : }
93 1 : opts.disableBlockPropertyCollector = v
94 1 : if v {
95 1 : opts.Opts.BlockPropertyCollectors = nil
96 1 : }
97 1 : return true
98 1 : case "TestOptions.enable_value_blocks":
99 1 : opts.enableValueBlocks = true
100 1 : opts.Opts.Experimental.EnableValueBlocks = func() bool { return true }
101 1 : return true
102 1 : case "TestOptions.disable_value_blocks_for_ingest_sstables":
103 1 : opts.disableValueBlocksForIngestSSTables = true
104 1 : return true
105 1 : case "TestOptions.async_apply_to_db":
106 1 : opts.asyncApplyToDB = true
107 1 : return true
108 1 : case "TestOptions.shared_storage_enabled":
109 1 : opts.sharedStorageEnabled = true
110 1 : opts.sharedStorageFS = remote.NewInMem()
111 1 : if opts.Opts.Experimental.CreateOnShared == remote.CreateOnSharedNone {
112 0 : opts.Opts.Experimental.CreateOnShared = remote.CreateOnSharedAll
113 0 : }
114 1 : return true
115 1 : case "TestOptions.external_storage_enabled":
116 1 : opts.externalStorageEnabled = true
117 1 : opts.externalStorageFS = remote.NewInMem()
118 1 : return true
119 1 : case "TestOptions.secondary_cache_enabled":
120 1 : opts.secondaryCacheEnabled = true
121 1 : opts.Opts.Experimental.SecondaryCacheSizeBytes = 1024 * 1024 * 32 // 32 MBs
122 1 : return true
123 1 : case "TestOptions.seed_efos":
124 1 : v, err := strconv.ParseUint(value, 10, 64)
125 1 : if err != nil {
126 0 : panic(err)
127 : }
128 1 : opts.seedEFOS = v
129 1 : return true
130 1 : case "TestOptions.io_latency_mean":
131 1 : v, err := time.ParseDuration(value)
132 1 : if err != nil {
133 0 : panic(err)
134 : }
135 1 : opts.ioLatencyMean = v
136 1 : return true
137 1 : case "TestOptions.io_latency_probability":
138 1 : v, err := strconv.ParseFloat(value, 64)
139 1 : if err != nil {
140 0 : panic(err)
141 : }
142 1 : opts.ioLatencyProbability = v
143 1 : return true
144 1 : case "TestOptions.io_latency_seed":
145 1 : v, err := strconv.ParseInt(value, 10, 64)
146 1 : if err != nil {
147 0 : panic(err)
148 : }
149 1 : opts.ioLatencySeed = v
150 1 : return true
151 1 : case "TestOptions.ingest_split":
152 1 : opts.ingestSplit = true
153 1 : opts.Opts.Experimental.IngestSplit = func() bool {
154 1 : return true
155 1 : }
156 1 : return true
157 1 : case "TestOptions.use_shared_replicate":
158 1 : opts.useSharedReplicate = true
159 1 : return true
160 0 : case "TestOptions.use_external_replicate":
161 0 : opts.useExternalReplicate = true
162 0 : return true
163 1 : case "TestOptions.use_excise":
164 1 : opts.useExcise = true
165 1 : return true
166 1 : case "TestOptions.use_delete_only_compaction_excises":
167 1 : opts.useDeleteOnlyCompactionExcises = true
168 1 : opts.Opts.Experimental.EnableDeleteOnlyCompactionExcises = func() bool {
169 1 : return opts.useDeleteOnlyCompactionExcises
170 1 : }
171 1 : return true
172 1 : case "TestOptions.use_jemalloc_size_classes":
173 1 : opts.Opts.AllocatorSizeClasses = pebble.JemallocSizeClasses
174 1 : return true
175 0 : default:
176 0 : if customOptionParsers == nil {
177 0 : return false
178 0 : }
179 0 : name = strings.TrimPrefix(name, "TestOptions.")
180 0 : if p, ok := customOptionParsers[name]; ok {
181 0 : if customOpt, ok := p(value); ok {
182 0 : opts.CustomOpts = append(opts.CustomOpts, customOpt)
183 0 : return true
184 0 : }
185 : }
186 0 : return false
187 : }
188 : },
189 : }
190 1 : err := opts.Opts.Parse(data, hooks)
191 1 : // Ensure that the WAL failover FS agrees with the primary FS. They're
192 1 : // separate options, but in the metamorphic tests we keep them in sync.
193 1 : if opts.Opts.WALFailover != nil {
194 1 : opts.Opts.WALFailover.Secondary.FS = opts.Opts.FS
195 1 : }
196 1 : opts.InitRemoteStorageFactory()
197 1 : opts.Opts.EnsureDefaults()
198 1 : return err
199 : }
200 :
201 0 : func optionsToString(opts *TestOptions) string {
202 0 : var buf bytes.Buffer
203 0 : if opts.strictFS {
204 0 : fmt.Fprint(&buf, " strictfs=true\n")
205 0 : }
206 0 : if opts.ingestUsingApply {
207 0 : fmt.Fprint(&buf, " ingest_using_apply=true\n")
208 0 : }
209 0 : if opts.deleteSized {
210 0 : fmt.Fprint(&buf, " delete_sized=true\n")
211 0 : }
212 0 : if opts.replaceSingleDelete {
213 0 : fmt.Fprint(&buf, " replace_single_delete=true\n")
214 0 : }
215 0 : if opts.useDisk {
216 0 : fmt.Fprint(&buf, " use_disk=true\n")
217 0 : }
218 0 : if opts.initialStatePath != "" {
219 0 : fmt.Fprintf(&buf, " initial_state_path=%s\n", opts.initialStatePath)
220 0 : }
221 0 : if opts.initialStateDesc != "" {
222 0 : fmt.Fprintf(&buf, " initial_state_desc=%s\n", opts.initialStateDesc)
223 0 : }
224 0 : if opts.Threads != 0 {
225 0 : fmt.Fprintf(&buf, " threads=%d\n", opts.Threads)
226 0 : }
227 0 : if opts.disableBlockPropertyCollector {
228 0 : fmt.Fprintf(&buf, " disable_block_property_collector=%t\n", opts.disableBlockPropertyCollector)
229 0 : }
230 0 : if opts.enableValueBlocks {
231 0 : fmt.Fprintf(&buf, " enable_value_blocks=%t\n", opts.enableValueBlocks)
232 0 : }
233 0 : if opts.disableValueBlocksForIngestSSTables {
234 0 : fmt.Fprintf(&buf, " disable_value_blocks_for_ingest_sstables=%t\n", opts.disableValueBlocksForIngestSSTables)
235 0 : }
236 0 : if opts.asyncApplyToDB {
237 0 : fmt.Fprint(&buf, " async_apply_to_db=true\n")
238 0 : }
239 0 : if opts.sharedStorageEnabled {
240 0 : fmt.Fprint(&buf, " shared_storage_enabled=true\n")
241 0 : }
242 0 : if opts.externalStorageEnabled {
243 0 : fmt.Fprint(&buf, " external_storage_enabled=true\n")
244 0 : }
245 0 : if opts.secondaryCacheEnabled {
246 0 : fmt.Fprint(&buf, " secondary_cache_enabled=true\n")
247 0 : }
248 0 : if opts.seedEFOS != 0 {
249 0 : fmt.Fprintf(&buf, " seed_efos=%d\n", opts.seedEFOS)
250 0 : }
251 0 : if opts.ingestSplit {
252 0 : fmt.Fprintf(&buf, " ingest_split=%v\n", opts.ingestSplit)
253 0 : }
254 0 : if opts.ioLatencyProbability > 0 {
255 0 : fmt.Fprintf(&buf, " io_latency_mean=%s\n", opts.ioLatencyMean)
256 0 : fmt.Fprintf(&buf, " io_latency_probability=%.10f\n", opts.ioLatencyProbability)
257 0 : fmt.Fprintf(&buf, " io_latency_seed=%d\n", opts.ioLatencySeed)
258 0 : }
259 0 : if opts.useSharedReplicate {
260 0 : fmt.Fprintf(&buf, " use_shared_replicate=%v\n", opts.useSharedReplicate)
261 0 : }
262 0 : if opts.useExternalReplicate {
263 0 : fmt.Fprintf(&buf, " use_external_replicate=%v\n", opts.useExternalReplicate)
264 0 : }
265 0 : if opts.useExcise {
266 0 : fmt.Fprintf(&buf, " use_excise=%v\n", opts.useExcise)
267 0 : }
268 0 : if opts.useDeleteOnlyCompactionExcises {
269 0 : fmt.Fprintf(&buf, " use_delete_only_compaction_excises=%v\n", opts.useDeleteOnlyCompactionExcises)
270 0 : }
271 0 : if opts.Opts.AllocatorSizeClasses != nil {
272 0 : if fmt.Sprint(opts.Opts.AllocatorSizeClasses) != fmt.Sprint(pebble.JemallocSizeClasses) {
273 0 : panic(fmt.Sprintf("unexpected AllocatorSizeClasses %v", opts.Opts.AllocatorSizeClasses))
274 : }
275 0 : fmt.Fprint(&buf, " use_jemalloc_size_classes=true\n")
276 : }
277 0 : for _, customOpt := range opts.CustomOpts {
278 0 : fmt.Fprintf(&buf, " %s=%s\n", customOpt.Name(), customOpt.Value())
279 0 : }
280 :
281 0 : s := opts.Opts.String()
282 0 : if buf.Len() == 0 {
283 0 : return s
284 0 : }
285 0 : return s + "\n[TestOptions]\n" + buf.String()
286 : }
287 :
288 1 : func defaultTestOptions() *TestOptions {
289 1 : return &TestOptions{
290 1 : Opts: defaultOptions(),
291 1 : Threads: 16,
292 1 : RetryPolicy: NeverRetry,
293 1 : }
294 1 : }
295 :
296 1 : func defaultOptions() *pebble.Options {
297 1 : opts := &pebble.Options{
298 1 : // Use an archive cleaner to ease post-mortem debugging.
299 1 : Cleaner: base.ArchiveCleaner{},
300 1 : // Always use our custom comparer which provides a Split method,
301 1 : // splitting keys at the trailing '@'.
302 1 : Comparer: testkeys.Comparer,
303 1 : FS: vfs.NewMem(),
304 1 : FormatMajorVersion: defaultFormatMajorVersion,
305 1 : Levels: []pebble.LevelOptions{{
306 1 : FilterPolicy: bloom.FilterPolicy(10),
307 1 : }},
308 1 : BlockPropertyCollectors: blockPropertyCollectorConstructors,
309 1 : }
310 1 : opts.Experimental.EnableColumnarBlocks = func() bool { return true }
311 :
312 : // We don't want to run the level checker every time because it can slow down
313 : // downloads and background compactions too much.
314 : //
315 : // We aim to run it once every 500ms (on average). To do this with some
316 : // randomization, each time we get a callback we see how much time passed
317 : // since the last call and run the check with a proportional probability.
318 1 : const meanTimeBetweenChecks = 500 * time.Millisecond
319 1 : startTime := time.Now()
320 1 : // lastCallTime stores the time of the last DebugCheck call, as the duration
321 1 : // since startTime.
322 1 : var lastCallTime atomic.Uint64
323 1 : opts.DebugCheck = func(db *pebble.DB) error {
324 1 : now := time.Since(startTime)
325 1 : last := time.Duration(lastCallTime.Swap(uint64(now)))
326 1 : // Run the check with probability equal to the time (as a fraction of
327 1 : // meanTimeBetweenChecks) passed since the last time we had a chance, as a
328 1 : // fraction of meanTimeBetweenChecks.
329 1 : if rand.Float64() < float64(now-last)/float64(meanTimeBetweenChecks) {
330 1 : return pebble.DebugCheckLevels(db)
331 1 : }
332 1 : return nil
333 : }
334 :
335 1 : return opts
336 : }
337 :
338 : // TestOptions describes the options configuring an individual run of the
339 : // metamorphic tests.
340 : type TestOptions struct {
341 : // Opts holds the *pebble.Options for the test.
342 : Opts *pebble.Options
343 : // Threads configures the parallelism of the test. Each thread will run in
344 : // an independent goroutine and be responsible for executing operations
345 : // against an independent set of objects. The outcome of any individual
346 : // operation will still be deterministic, with the metamorphic test
347 : // inserting synchronization where necessary.
348 : Threads int
349 : // RetryPolicy configures which errors should be retried.
350 : RetryPolicy RetryPolicy
351 : // CustomOptions holds custom test options that are defined outside of this
352 : // package.
353 : CustomOpts []CustomOption
354 :
355 : // internal
356 :
357 : useDisk bool
358 : strictFS bool
359 : // Use Batch.Apply rather than DB.Ingest.
360 : ingestUsingApply bool
361 : // Use Batch.DeleteSized rather than Batch.Delete.
362 : deleteSized bool
363 : // Replace a SINGLEDEL with a DELETE.
364 : replaceSingleDelete bool
365 : // The path on the local filesystem where the initial state of the database
366 : // exists. Empty if the test run begins from an empty database state.
367 : initialStatePath string
368 : // A human-readable string describing the initial state of the database.
369 : // Empty if the test run begins from an empty database state.
370 : initialStateDesc string
371 : // Disable the block property collector, which may be used by block property
372 : // filters.
373 : disableBlockPropertyCollector bool
374 : // Enable the use of value blocks.
375 : enableValueBlocks bool
376 : // Disables value blocks in the sstables written for ingest.
377 : disableValueBlocksForIngestSSTables bool
378 : // Use DB.ApplyNoSyncWait for applies that want to sync the WAL.
379 : asyncApplyToDB bool
380 : // Enable the use of shared storage.
381 : sharedStorageEnabled bool
382 : sharedStorageFS remote.Storage
383 : // Enable the use of shared storage for external file ingestion.
384 : externalStorageEnabled bool
385 : externalStorageFS remote.Storage
386 : // Enables the use of shared replication in TestOptions.
387 : useSharedReplicate bool
388 : // Enables the use of external replication in TestOptions.
389 : useExternalReplicate bool
390 : // Enable the secondary cache. Only effective if sharedStorageEnabled is
391 : // also true.
392 : secondaryCacheEnabled bool
393 : // If nonzero, enables the use of EventuallyFileOnlySnapshots for
394 : // newSnapshotOps that are keyspan-bounded. The set of which newSnapshotOps
395 : // are actually created as EventuallyFileOnlySnapshots is deterministically
396 : // derived from the seed and the operation index.
397 : seedEFOS uint64
398 : // If nonzero, enables the injection of random IO latency. The mechanics of
399 : // a Pebble operation can be very timing dependent, so artificial latency
400 : // can ensure a wide variety of mechanics are exercised. Additionally,
401 : // exercising some mechanics such as WAL failover require IO latency.
402 : ioLatencyProbability float64
403 : ioLatencySeed int64
404 : ioLatencyMean time.Duration
405 : // Enables ingest splits. Saved here for serialization as Options does not
406 : // serialize this.
407 : ingestSplit bool
408 : // Enables operations that do excises. Note that a false value for this does
409 : // not guarantee the lack of excises, as useSharedReplicate can also cause
410 : // excises. However !useExcise && !useSharedReplicate can be used to guarantee
411 : // lack of excises.
412 : useExcise bool
413 : // useDeleteOnlyCompactionExcises turns on the ability for delete-only compactions
414 : // to do excises. Note that this can be true even when useExcise is false.
415 : useDeleteOnlyCompactionExcises bool
416 : }
417 :
418 : // InitRemoteStorageFactory initializes Opts.Experimental.RemoteStorage.
419 1 : func (testOpts *TestOptions) InitRemoteStorageFactory() {
420 1 : if testOpts.sharedStorageEnabled || testOpts.externalStorageEnabled {
421 1 : m := make(map[remote.Locator]remote.Storage)
422 1 : if testOpts.sharedStorageEnabled {
423 1 : m[""] = testOpts.sharedStorageFS
424 1 : }
425 1 : if testOpts.externalStorageEnabled {
426 1 : m["external"] = testOpts.externalStorageFS
427 1 : }
428 1 : testOpts.Opts.Experimental.RemoteStorage = remote.MakeSimpleFactory(m)
429 : }
430 : }
431 :
432 : // CustomOption defines a custom option that configures the behavior of an
433 : // individual test run. Like all test options, custom options are serialized to
434 : // the OPTIONS file even if they're not options ordinarily understood by Pebble.
435 : type CustomOption interface {
436 : // Name returns the name of the custom option. This is the key under which
437 : // the option appears in the OPTIONS file, within the [TestOptions] stanza.
438 : Name() string
439 : // Value returns the value of the custom option, serialized as it should
440 : // appear within the OPTIONS file.
441 : Value() string
442 : // Close is run after the test database has been closed at the end of the
443 : // test as well as during restart operations within the test sequence. It's
444 : // passed a copy of the *pebble.Options. If the custom options hold on to
445 : // any resources outside, Close should release them.
446 : Close(*pebble.Options) error
447 : // Open is run before the test runs and during a restart operation after the
448 : // test database has been closed and Close has been called. It's passed a
449 : // copy of the *pebble.Options. If the custom options must acquire any
450 : // resources before the test continues, it should reacquire them.
451 : Open(*pebble.Options) error
452 :
453 : // TODO(jackson): provide additional hooks for custom options changing the
454 : // behavior of a run.
455 : }
456 :
457 0 : func standardOptions() []*TestOptions {
458 0 : // The index labels are not strictly necessary, but they make it easier to
459 0 : // find which options correspond to a failure.
460 0 : stdOpts := []string{
461 0 : 0: "", // default options
462 0 : 1: `
463 0 : [Options]
464 0 : cache_size=1
465 0 : `,
466 0 : 2: `
467 0 : [Options]
468 0 : disable_wal=true
469 0 : [TestOptions]
470 0 : use_jemalloc_size_classes=true
471 0 : `,
472 0 : 3: `
473 0 : [Options]
474 0 : l0_compaction_threshold=1
475 0 : `,
476 0 : 4: `
477 0 : [Options]
478 0 : l0_compaction_threshold=1
479 0 : l0_stop_writes_threshold=1
480 0 : `,
481 0 : 5: `
482 0 : [Options]
483 0 : lbase_max_bytes=1
484 0 : `,
485 0 : 6: `
486 0 : [Options]
487 0 : max_manifest_file_size=1
488 0 : `,
489 0 : 7: `
490 0 : [Options]
491 0 : max_open_files=1
492 0 : `,
493 0 : 8: `
494 0 : [Options]
495 0 : mem_table_size=2000
496 0 : `,
497 0 : 9: `
498 0 : [Options]
499 0 : mem_table_stop_writes_threshold=2
500 0 : `,
501 0 : 10: `
502 0 : [Options]
503 0 : wal_dir=data/wal
504 0 : `,
505 0 : 11: `
506 0 : [Level "0"]
507 0 : block_restart_interval=1
508 0 : `,
509 0 : 12: `
510 0 : [Level "0"]
511 0 : block_size=1
512 0 : `,
513 0 : 13: `
514 0 : [Level "0"]
515 0 : compression=NoCompression
516 0 : `,
517 0 : 14: `
518 0 : [Level "0"]
519 0 : index_block_size=1
520 0 : `,
521 0 : 15: `
522 0 : [Level "0"]
523 0 : target_file_size=12
524 0 : `,
525 0 : 16: `
526 0 : [Level "0"]
527 0 : filter_policy=none
528 0 : `,
529 0 : // 1GB
530 0 : 17: `
531 0 : [Options]
532 0 : bytes_per_sync=1073741824
533 0 : [TestOptions]
534 0 : strictfs=true
535 0 : `,
536 0 : 18: `
537 0 : [Options]
538 0 : max_concurrent_compactions=2
539 0 : `,
540 0 : 19: `
541 0 : [TestOptions]
542 0 : ingest_using_apply=true
543 0 : `,
544 0 : 20: `
545 0 : [TestOptions]
546 0 : replace_single_delete=true
547 0 : `,
548 0 : 21: `
549 0 : [TestOptions]
550 0 : use_disk=true
551 0 : use_jemalloc_size_classes=true
552 0 : `,
553 0 : 22: `
554 0 : [Options]
555 0 : max_writer_concurrency=2
556 0 : force_writer_parallelism=true
557 0 : `,
558 0 : 23: `
559 0 : [TestOptions]
560 0 : disable_block_property_collector=true
561 0 : `,
562 0 : 24: `
563 0 : [TestOptions]
564 0 : threads=1
565 0 : `,
566 0 : 25: `
567 0 : [TestOptions]
568 0 : enable_value_blocks=true
569 0 : use_jemalloc_size_classes=true
570 0 : `,
571 0 : 26: fmt.Sprintf(`
572 0 : [Options]
573 0 : format_major_version=%s
574 0 : `, newestFormatMajorVersionToTest),
575 0 : 27: fmt.Sprintf(`
576 0 : [Options]
577 0 : format_major_version=%s
578 0 : [TestOptions]
579 0 : shared_storage_enabled=true
580 0 : secondary_cache_enabled=true
581 0 : `, pebble.FormatMinForSharedObjects),
582 0 : 28: fmt.Sprintf(`
583 0 : [Options]
584 0 : format_major_version=%s
585 0 : [TestOptions]
586 0 : external_storage_enabled=true
587 0 : `, pebble.FormatSyntheticPrefixSuffix),
588 0 : 29: fmt.Sprintf(`
589 0 : [Options]
590 0 : format_major_version=%s
591 0 : max_concurrent_downloads=2
592 0 : [TestOptions]
593 0 : shared_storage_enabled=true
594 0 : external_storage_enabled=true
595 0 : secondary_cache_enabled=false
596 0 : `, pebble.FormatSyntheticPrefixSuffix),
597 0 : }
598 0 :
599 0 : opts := make([]*TestOptions, len(stdOpts))
600 0 : for i := range opts {
601 0 : opts[i] = defaultTestOptions()
602 0 : // NB: The standard options by definition can never include custom
603 0 : // options, so no need to propagate custom option parsers.
604 0 : if err := parseOptions(opts[i], stdOpts[i], nil /* custom option parsers */); err != nil {
605 0 : panic(err)
606 : }
607 : }
608 0 : return opts
609 : }
610 :
611 : // RandomOptions generates a random set of operations, drawing randomness from
612 : // rng.
613 : func RandomOptions(
614 : rng *rand.Rand, customOptionParsers map[string]func(string) (CustomOption, bool),
615 0 : ) *TestOptions {
616 0 : testOpts := defaultTestOptions()
617 0 : opts := testOpts.Opts
618 0 :
619 0 : // There are some private options, which we don't want users to fiddle with.
620 0 : // There's no way to set it through the public interface. The only method is
621 0 : // through Parse.
622 0 : {
623 0 : var privateOpts bytes.Buffer
624 0 : fmt.Fprintln(&privateOpts, `[Options]`)
625 0 : if rng.IntN(3) == 0 /* 33% */ {
626 0 : fmt.Fprintln(&privateOpts, ` disable_delete_only_compactions=true`)
627 0 : }
628 0 : if rng.IntN(3) == 0 /* 33% */ {
629 0 : fmt.Fprintln(&privateOpts, ` disable_elision_only_compactions=true`)
630 0 : }
631 0 : if rng.IntN(5) == 0 /* 20% */ {
632 0 : fmt.Fprintln(&privateOpts, ` disable_lazy_combined_iteration=true`)
633 0 : }
634 0 : if privateOptsStr := privateOpts.String(); privateOptsStr != `[Options]\n` {
635 0 : parseOptions(testOpts, privateOptsStr, customOptionParsers)
636 0 : }
637 : }
638 :
639 0 : opts.BytesPerSync = 1 << uint(rng.IntN(28)) // 1B - 256MB
640 0 : opts.Cache = cache.New(1 << uint(rng.IntN(30))) // 1B - 1GB
641 0 : opts.DisableWAL = rng.IntN(2) == 0
642 0 : opts.FlushDelayDeleteRange = time.Millisecond * time.Duration(5*rng.IntN(245)) // 5-250ms
643 0 : opts.FlushDelayRangeKey = time.Millisecond * time.Duration(5*rng.IntN(245)) // 5-250ms
644 0 : opts.FlushSplitBytes = 1 << rng.IntN(20) // 1B - 1MB
645 0 : opts.FormatMajorVersion = minimumFormatMajorVersion
646 0 : n := int(newestFormatMajorVersionToTest - opts.FormatMajorVersion)
647 0 : opts.FormatMajorVersion += pebble.FormatMajorVersion(rng.IntN(n + 1))
648 0 : opts.Experimental.L0CompactionConcurrency = 1 + rng.IntN(4) // 1-4
649 0 : opts.Experimental.LevelMultiplier = 5 << rng.IntN(7) // 5 - 320
650 0 : opts.TargetByteDeletionRate = 1 << uint(20+rng.IntN(10)) // 1MB - 1GB
651 0 : opts.Experimental.ValidateOnIngest = rng.IntN(2) != 0
652 0 : opts.L0CompactionThreshold = 1 + rng.IntN(100) // 1 - 100
653 0 : opts.L0CompactionFileThreshold = 1 << rng.IntN(11) // 1 - 1024
654 0 : opts.L0StopWritesThreshold = 50 + rng.IntN(100) // 50 - 150
655 0 : if opts.L0StopWritesThreshold < 2*opts.L0CompactionThreshold {
656 0 : opts.L0StopWritesThreshold = 2 * opts.L0CompactionThreshold
657 0 : }
658 0 : opts.LBaseMaxBytes = 1 << uint(rng.IntN(30)) // 1B - 1GB
659 0 : maxConcurrentCompactions := rng.IntN(3) + 1 // 1-3
660 0 : opts.MaxConcurrentCompactions = func() int {
661 0 : return maxConcurrentCompactions
662 0 : }
663 0 : maxConcurrentDownloads := rng.IntN(3) + 1 // 1-3
664 0 : opts.MaxConcurrentDownloads = func() int {
665 0 : return maxConcurrentDownloads
666 0 : }
667 0 : opts.MaxManifestFileSize = 1 << uint(rng.IntN(30)) // 1B - 1GB
668 0 : opts.MemTableSize = 2 << (10 + uint(rng.IntN(16))) // 2KB - 256MB
669 0 : opts.MemTableStopWritesThreshold = 2 + rng.IntN(5) // 2 - 5
670 0 : if rng.IntN(2) == 0 {
671 0 : opts.WALDir = "data/wal"
672 0 : }
673 :
674 : // Half the time enable WAL failover.
675 0 : if rng.IntN(2) == 0 {
676 0 : // Use 10x longer durations when writing directly to FS; we don't want
677 0 : // WAL failover to trigger excessively frequently.
678 0 : referenceDur := time.Millisecond
679 0 : if testOpts.useDisk {
680 0 : referenceDur *= 10
681 0 : }
682 :
683 0 : scaleDuration := func(d time.Duration, minFactor, maxFactor float64) time.Duration {
684 0 : return time.Duration(float64(d) * (minFactor + rng.Float64()*(maxFactor-minFactor)))
685 0 : }
686 0 : probeInterval := scaleDuration(referenceDur, 0.10, 0.50) // Between 10-50% of the reference duration.
687 0 : unhealthyThreshold := expRandDuration(rng, 3*referenceDur, time.Second)
688 0 : healthyThreshold := expRandDuration(rng, 3*referenceDur, time.Second)
689 0 : // Use a healthy interval between 1-119x the probe interval. The probe
690 0 : // maintains a maximum history 120 entries, so the healthy interval
691 0 : // must not exceed 119x the probe interval.
692 0 : healthyInterval := scaleDuration(probeInterval, 1.0, 119.0)
693 0 : opts.WALFailover = &pebble.WALFailoverOptions{
694 0 : Secondary: wal.Dir{FS: vfs.Default, Dirname: "data/wal_secondary"},
695 0 : FailoverOptions: wal.FailoverOptions{
696 0 : PrimaryDirProbeInterval: probeInterval,
697 0 : HealthyProbeLatencyThreshold: healthyThreshold,
698 0 : HealthyInterval: healthyInterval,
699 0 : UnhealthySamplingInterval: scaleDuration(unhealthyThreshold, 0.10, 0.50), // Between 10-50% of the unhealthy threshold
700 0 : UnhealthyOperationLatencyThreshold: func() (time.Duration, bool) {
701 0 : return unhealthyThreshold, true
702 0 : },
703 : ElevatedWriteStallThresholdLag: expRandDuration(rng, 5*referenceDur, 2*time.Second),
704 : },
705 : }
706 : }
707 0 : if rng.IntN(4) == 0 {
708 0 : // Enable Writer parallelism for 25% of the random options. Setting
709 0 : // MaxWriterConcurrency to any value greater than or equal to 1 has the
710 0 : // same effect currently.
711 0 : opts.Experimental.MaxWriterConcurrency = 2
712 0 : opts.Experimental.ForceWriterParallelism = true
713 0 : }
714 0 : if rng.IntN(2) == 0 {
715 0 : opts.Experimental.DisableIngestAsFlushable = func() bool { return true }
716 : }
717 :
718 : // We either use no multilevel compactions, multilevel compactions with the
719 : // default (zero) additional propensity, or multilevel compactions with an
720 : // additional propensity to encourage more multilevel compactions than we
721 : // ohterwise would.
722 0 : switch rng.IntN(3) {
723 0 : case 0:
724 0 : opts.Experimental.MultiLevelCompactionHeuristic = pebble.NoMultiLevel{}
725 0 : case 1:
726 0 : opts.Experimental.MultiLevelCompactionHeuristic = pebble.WriteAmpHeuristic{}
727 0 : default:
728 0 : opts.Experimental.MultiLevelCompactionHeuristic = pebble.WriteAmpHeuristic{
729 0 : AddPropensity: rng.Float64() * float64(rng.IntN(3)), // [0,3.0)
730 0 : AllowL0: rng.IntN(4) == 1, // 25% of the time
731 0 : }
732 : }
733 0 : if rng.IntN(2) == 0 {
734 0 : opts.AllocatorSizeClasses = pebble.JemallocSizeClasses
735 0 : }
736 :
737 0 : var lopts pebble.LevelOptions
738 0 : lopts.BlockRestartInterval = 1 + rng.IntN(64) // 1 - 64
739 0 : lopts.BlockSize = 1 << uint(rng.IntN(24)) // 1 - 16MB
740 0 : lopts.BlockSizeThreshold = 50 + rng.IntN(50) // 50 - 100
741 0 : lopts.IndexBlockSize = 1 << uint(rng.IntN(24)) // 1 - 16MB
742 0 : lopts.TargetFileSize = 1 << uint(rng.IntN(28)) // 1 - 256MB
743 0 : // The EstimatedSize of an empty table writer is 8 bytes. We want something a
744 0 : // little bigger than that as the minimum target.
745 0 : lopts.TargetFileSize = max(lopts.TargetFileSize, 12)
746 0 :
747 0 : // We either use no bloom filter, the default filter, or a filter with
748 0 : // randomized bits-per-key setting. We zero out the Filters map. It'll get
749 0 : // repopulated on EnsureDefaults accordingly.
750 0 : opts.Filters = nil
751 0 : switch rng.IntN(3) {
752 0 : case 0:
753 0 : lopts.FilterPolicy = nil
754 0 : case 1:
755 0 : lopts.FilterPolicy = bloom.FilterPolicy(10)
756 0 : default:
757 0 : lopts.FilterPolicy = newTestingFilterPolicy(1 << rng.IntN(5))
758 : }
759 :
760 : // We use either no compression, snappy compression or zstd compression.
761 0 : switch rng.IntN(3) {
762 0 : case 0:
763 0 : lopts.Compression = func() block.Compression { return pebble.NoCompression }
764 0 : case 1:
765 0 : lopts.Compression = func() block.Compression { return pebble.ZstdCompression }
766 0 : default:
767 0 : lopts.Compression = func() block.Compression { return pebble.SnappyCompression }
768 : }
769 0 : opts.Levels = []pebble.LevelOptions{lopts}
770 0 :
771 0 : // Explicitly disable disk-backed FS's for the random configurations. The
772 0 : // single standard test configuration that uses a disk-backed FS is
773 0 : // sufficient.
774 0 : testOpts.useDisk = false
775 0 : testOpts.strictFS = rng.IntN(2) != 0 // Only relevant for MemFS.
776 0 : // 50% of the time, enable IO latency injection.
777 0 : if rng.IntN(2) == 0 {
778 0 : // Note: we want ioLatencyProbability to be at least 1e-10, otherwise it
779 0 : // might print as 0 when we stringify options.
780 0 : testOpts.ioLatencyProbability = 1e-10 + 0.01*rng.Float64() // 0-1%
781 0 : testOpts.ioLatencyMean = expRandDuration(rng, 3*time.Millisecond, time.Second)
782 0 : testOpts.ioLatencySeed = rng.Int64()
783 0 : }
784 0 : testOpts.Threads = rng.IntN(runtime.GOMAXPROCS(0)) + 1
785 0 : if testOpts.strictFS {
786 0 : opts.DisableWAL = false
787 0 : opts.FS = vfs.NewCrashableMem()
788 0 : } else if !testOpts.useDisk {
789 0 : opts.FS = vfs.NewMem()
790 0 : }
791 : // Update the WALFailover's secondary to use the same FS. This isn't
792 : // strictly necessary (the WALFailover could use a separate FS), but it
793 : // ensures when we save a copy of the test state to disk, we include the
794 : // secondary's WALs.
795 0 : if opts.WALFailover != nil {
796 0 : opts.WALFailover.Secondary.FS = opts.FS
797 0 : }
798 0 : testOpts.ingestUsingApply = rng.IntN(2) != 0
799 0 : testOpts.deleteSized = rng.IntN(2) != 0
800 0 : testOpts.replaceSingleDelete = rng.IntN(2) != 0
801 0 : testOpts.disableBlockPropertyCollector = rng.IntN(2) == 1
802 0 : if testOpts.disableBlockPropertyCollector {
803 0 : testOpts.Opts.BlockPropertyCollectors = nil
804 0 : }
805 0 : testOpts.enableValueBlocks = rng.IntN(2) != 0
806 0 : if testOpts.enableValueBlocks {
807 0 : testOpts.Opts.Experimental.EnableValueBlocks = func() bool { return true }
808 : }
809 0 : testOpts.disableValueBlocksForIngestSSTables = rng.IntN(2) == 0
810 0 : testOpts.asyncApplyToDB = rng.IntN(2) != 0
811 0 : // 20% of time, enable shared storage.
812 0 : if rng.IntN(5) == 0 {
813 0 : testOpts.sharedStorageEnabled = true
814 0 : if testOpts.Opts.FormatMajorVersion < pebble.FormatMinForSharedObjects {
815 0 : testOpts.Opts.FormatMajorVersion = pebble.FormatMinForSharedObjects
816 0 : }
817 0 : testOpts.sharedStorageFS = remote.NewInMem()
818 0 : // If shared storage is enabled, pick between writing all files on shared
819 0 : // vs. lower levels only, 50% of the time.
820 0 : testOpts.Opts.Experimental.CreateOnShared = remote.CreateOnSharedAll
821 0 : if rng.IntN(2) == 0 {
822 0 : testOpts.Opts.Experimental.CreateOnShared = remote.CreateOnSharedLower
823 0 : }
824 : // If shared storage is enabled, enable secondary cache 50% of time.
825 0 : if rng.IntN(2) == 0 {
826 0 : testOpts.secondaryCacheEnabled = true
827 0 : // TODO(josh): Randomize various secondary cache settings.
828 0 : testOpts.Opts.Experimental.SecondaryCacheSizeBytes = 1024 * 1024 * 32 // 32 MBs
829 0 : }
830 : // 50% of the time, enable shared replication.
831 0 : testOpts.useSharedReplicate = rng.IntN(2) == 0
832 : }
833 :
834 : // 50% of time, enable external storage.
835 0 : if rng.IntN(2) == 0 {
836 0 : testOpts.externalStorageEnabled = true
837 0 : if testOpts.Opts.FormatMajorVersion < pebble.FormatSyntheticPrefixSuffix {
838 0 : testOpts.Opts.FormatMajorVersion = pebble.FormatSyntheticPrefixSuffix
839 0 : }
840 0 : testOpts.externalStorageFS = remote.NewInMem()
841 : }
842 :
843 0 : testOpts.seedEFOS = rng.Uint64()
844 0 : testOpts.ingestSplit = rng.IntN(2) == 0
845 0 : opts.Experimental.IngestSplit = func() bool { return testOpts.ingestSplit }
846 0 : testOpts.useExcise = rng.IntN(2) == 0
847 0 : if testOpts.useExcise {
848 0 : if testOpts.Opts.FormatMajorVersion < pebble.FormatVirtualSSTables {
849 0 : testOpts.Opts.FormatMajorVersion = pebble.FormatVirtualSSTables
850 0 : }
851 : }
852 0 : testOpts.useDeleteOnlyCompactionExcises = rng.IntN(2) == 0
853 0 : opts.Experimental.EnableDeleteOnlyCompactionExcises = func() bool {
854 0 : return testOpts.useDeleteOnlyCompactionExcises
855 0 : }
856 0 : testOpts.InitRemoteStorageFactory()
857 0 : testOpts.Opts.EnsureDefaults()
858 0 : return testOpts
859 : }
860 :
861 0 : func expRandDuration(rng *rand.Rand, meanDur, maxDur time.Duration) time.Duration {
862 0 : return min(maxDur, time.Duration(math.Round(rng.ExpFloat64()*float64(meanDur))))
863 0 : }
864 :
865 0 : func setupInitialState(dataDir string, testOpts *TestOptions) error {
866 0 : // Copy (vfs.Default,<initialStatePath>/data) to (testOpts.opts.FS,<dataDir>).
867 0 : ok, err := vfs.Clone(
868 0 : vfs.Default,
869 0 : testOpts.Opts.FS,
870 0 : vfs.Default.PathJoin(testOpts.initialStatePath, "data"),
871 0 : dataDir,
872 0 : vfs.CloneSync,
873 0 : vfs.CloneSkip(func(filename string) bool {
874 0 : // Skip the archive of historical files, any checkpoints created by
875 0 : // operations and files staged for ingest in tmp.
876 0 : b := filepath.Base(filename)
877 0 : return b == "archive" || b == "checkpoints" || b == "tmp"
878 0 : }))
879 0 : if err != nil {
880 0 : return err
881 0 : } else if !ok {
882 0 : return os.ErrNotExist
883 0 : }
884 :
885 : // Tests with wal_dir set store their WALs in a `wal` directory. The source
886 : // database (initialStatePath) could've had wal_dir set, or the current test
887 : // options (testOpts) could have wal_dir set, or both.
888 : //
889 : // If the test opts are not configured to use a WAL dir, we add the WAL dir
890 : // as a 'WAL recovery dir' so that we'll read any WALs in the directory in
891 : // Open.
892 0 : walRecoveryPath := testOpts.Opts.FS.PathJoin(dataDir, "wal")
893 0 : if testOpts.Opts.WALDir != "" {
894 0 : // If the test opts are configured to use a WAL dir, we add the data
895 0 : // directory itself as a 'WAL recovery dir' so that we'll read any WALs if
896 0 : // the previous test was writing them to the data directory.
897 0 : walRecoveryPath = dataDir
898 0 : }
899 0 : testOpts.Opts.WALRecoveryDirs = append(testOpts.Opts.WALRecoveryDirs, wal.Dir{
900 0 : FS: testOpts.Opts.FS,
901 0 : Dirname: walRecoveryPath,
902 0 : })
903 0 :
904 0 : // If the failover dir exists and the test opts are not configured to use
905 0 : // WAL failover, add the failover directory as a 'WAL recovery dir' in case
906 0 : // the previous test was configured to use failover.
907 0 : failoverDir := testOpts.Opts.FS.PathJoin(dataDir, "wal_secondary")
908 0 : if _, err := testOpts.Opts.FS.Stat(failoverDir); err == nil && testOpts.Opts.WALFailover == nil {
909 0 : testOpts.Opts.WALRecoveryDirs = append(testOpts.Opts.WALRecoveryDirs, wal.Dir{
910 0 : FS: testOpts.Opts.FS,
911 0 : Dirname: failoverDir,
912 0 : })
913 0 : }
914 0 : return nil
915 : }
916 :
917 : var blockPropertyCollectorConstructors = []func() pebble.BlockPropertyCollector{
918 : sstable.NewTestKeysBlockPropertyCollector,
919 : }
920 :
921 : // testingFilterPolicy is used to allow bloom filter policies with non-default
922 : // bits-per-key setting. It is necessary because the name of the production
923 : // filter policy is fixed (see bloom.FilterPolicy.Name()); we need to output a
924 : // custom policy name to the OPTIONS file that the test can then parse.
925 : type testingFilterPolicy struct {
926 : bloom.FilterPolicy
927 : }
928 :
929 : var _ pebble.FilterPolicy = (*testingFilterPolicy)(nil)
930 :
931 1 : func newTestingFilterPolicy(bitsPerKey int) *testingFilterPolicy {
932 1 : return &testingFilterPolicy{
933 1 : FilterPolicy: bloom.FilterPolicy(bitsPerKey),
934 1 : }
935 1 : }
936 :
937 : const testingFilterPolicyFmt = "testing_bloom_filter/bits_per_key=%d"
938 :
939 : // Name implements the pebble.FilterPolicy interface.
940 1 : func (t *testingFilterPolicy) Name() string {
941 1 : if t.FilterPolicy == 10 {
942 0 : return "rocksdb.BuiltinBloomFilter"
943 0 : }
944 1 : return fmt.Sprintf(testingFilterPolicyFmt, t.FilterPolicy)
945 : }
946 :
947 1 : func filterPolicyFromName(name string) (pebble.FilterPolicy, error) {
948 1 : switch name {
949 1 : case "none":
950 1 : return nil, nil
951 1 : case "rocksdb.BuiltinBloomFilter":
952 1 : return bloom.FilterPolicy(10), nil
953 : }
954 1 : var bitsPerKey int
955 1 : if _, err := fmt.Sscanf(name, testingFilterPolicyFmt, &bitsPerKey); err != nil {
956 0 : return nil, errors.Errorf("Invalid filter policy name '%s'", name)
957 0 : }
958 1 : return newTestingFilterPolicy(bitsPerKey), nil
959 : }
|
