Line data Source code
1 : // Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
2 : // of this source code is governed by a BSD-style license that can be found in
3 : // the LICENSE file.
4 :
5 : // Package metamorphic provides a testing framework for running randomized tests
6 : // over multiple Pebble databases with varying configurations. Logically
7 : // equivalent operations should result in equivalent output across all
8 : // configurations.
9 : package metamorphic
10 :
11 : import (
12 : "context"
13 : "fmt"
14 : "io"
15 : "os"
16 : "os/exec"
17 : "path"
18 : "path/filepath"
19 : "regexp"
20 : "sort"
21 : "testing"
22 : "time"
23 :
24 : "github.com/cockroachdb/pebble"
25 : "github.com/cockroachdb/pebble/internal/base"
26 : "github.com/cockroachdb/pebble/internal/randvar"
27 : "github.com/cockroachdb/pebble/internal/testkeys"
28 : "github.com/cockroachdb/pebble/vfs"
29 : "github.com/cockroachdb/pebble/vfs/errorfs"
30 : "github.com/pmezard/go-difflib/difflib"
31 : "github.com/stretchr/testify/require"
32 : "golang.org/x/exp/rand"
33 : "golang.org/x/sync/errgroup"
34 : )
35 :
36 : type runAndCompareOptions struct {
37 : seed uint64
38 : ops randvar.Static
39 : previousOpsPath string
40 : initialStatePath string
41 : initialStateDesc string
42 : traceFile string
43 : innerBinary string
44 : mutateTestOptions []func(*TestOptions)
45 : customRuns map[string]string
46 : runOnceOptions
47 : }
48 :
49 : // A RunOption configures the behavior of RunAndCompare.
50 : type RunOption interface {
51 : apply(*runAndCompareOptions)
52 : }
53 :
54 : // Seed configures generation to use the provided seed. Seed may be used to
55 : // deterministically reproduce the same run.
56 : type Seed uint64
57 :
58 1 : func (s Seed) apply(ro *runAndCompareOptions) { ro.seed = uint64(s) }
59 :
60 : // ExtendPreviousRun configures RunAndCompare to use the output of a previous
61 : // metamorphic test run to seed the this run. It's used in the crossversion
62 : // metamorphic tests, in which a data directory is upgraded through multiple
63 : // versions of Pebble, exercising upgrade code paths and cross-version
64 : // compatibility.
65 : //
66 : // The opsPath should be the filesystem path to the ops file containing the
67 : // operations run within the previous iteration of the metamorphic test. It's
68 : // used to inform operation generation to prefer using keys used in the previous
69 : // run, which are therefore more likely to be "interesting."
70 : //
71 : // The initialStatePath argument should be the filesystem path to the data
72 : // directory containing the database where the previous run of the metamorphic
73 : // test left off.
74 : //
75 : // The initialStateDesc argument is presentational and should hold a
76 : // human-readable description of the initial state.
77 0 : func ExtendPreviousRun(opsPath, initialStatePath, initialStateDesc string) RunOption {
78 0 : return closureOpt(func(ro *runAndCompareOptions) {
79 0 : ro.previousOpsPath = opsPath
80 0 : ro.initialStatePath = initialStatePath
81 0 : ro.initialStateDesc = initialStateDesc
82 0 : })
83 : }
84 :
85 : var (
86 : // UseDisk configures RunAndCompare to use the physical filesystem for all
87 : // generated runs.
88 0 : UseDisk = closureOpt(func(ro *runAndCompareOptions) {
89 0 : ro.mutateTestOptions = append(ro.mutateTestOptions, func(to *TestOptions) { to.useDisk = true })
90 : })
91 : // UseInMemory configures RunAndCompare to use an in-memory virtual
92 : // filesystem for all generated runs.
93 0 : UseInMemory = closureOpt(func(ro *runAndCompareOptions) {
94 0 : ro.mutateTestOptions = append(ro.mutateTestOptions, func(to *TestOptions) { to.useDisk = false })
95 : })
96 : )
97 :
98 : // OpCount configures the random variable for the number of operations to
99 : // generate.
100 1 : func OpCount(rv randvar.Static) RunOption {
101 1 : return closureOpt(func(ro *runAndCompareOptions) { ro.ops = rv })
102 : }
103 :
104 : // RuntimeTrace configures each test run to collect a runtime trace and output
105 : // it with the provided filename.
106 0 : func RuntimeTrace(name string) RunOption {
107 0 : return closureOpt(func(ro *runAndCompareOptions) { ro.traceFile = name })
108 : }
109 :
110 : // InnerBinary configures the binary that is called for each run. If not
111 : // specified, this binary (os.Args[0]) is called.
112 0 : func InnerBinary(path string) RunOption {
113 0 : return closureOpt(func(ro *runAndCompareOptions) { ro.innerBinary = path })
114 : }
115 :
116 : // ParseCustomTestOption adds support for parsing the provided CustomOption from
117 : // OPTIONS files serialized by the metamorphic tests. This RunOption alone does
118 : // not cause the metamorphic tests to run with any variant of the provided
119 : // CustomOption set.
120 0 : func ParseCustomTestOption(name string, parseFn func(value string) (CustomOption, bool)) RunOption {
121 0 : return closureOpt(func(ro *runAndCompareOptions) { ro.customOptionParsers[name] = parseFn })
122 : }
123 :
124 : // AddCustomRun adds an additional run of the metamorphic tests, using the
125 : // provided OPTIONS file contents. The default options will be used, except
126 : // those options that are overriden by the provided OPTIONS string.
127 0 : func AddCustomRun(name string, serializedOptions string) RunOption {
128 0 : return closureOpt(func(ro *runAndCompareOptions) { ro.customRuns[name] = serializedOptions })
129 : }
130 :
131 : type closureOpt func(*runAndCompareOptions)
132 :
133 1 : func (f closureOpt) apply(ro *runAndCompareOptions) { f(ro) }
134 :
135 : // RunAndCompare runs the metamorphic tests, using the provided root directory
136 : // to hold test data.
137 1 : func RunAndCompare(t *testing.T, rootDir string, rOpts ...RunOption) {
138 1 : runOpts := runAndCompareOptions{
139 1 : ops: randvar.NewUniform(1000, 10000),
140 1 : customRuns: map[string]string{},
141 1 : runOnceOptions: runOnceOptions{
142 1 : customOptionParsers: map[string]func(string) (CustomOption, bool){},
143 1 : },
144 1 : }
145 1 : for _, o := range rOpts {
146 1 : o.apply(&runOpts)
147 1 : }
148 1 : if runOpts.seed == 0 {
149 1 : runOpts.seed = uint64(time.Now().UnixNano())
150 1 : }
151 :
152 1 : require.NoError(t, os.MkdirAll(rootDir, 0755))
153 1 : metaDir, err := os.MkdirTemp(rootDir, time.Now().Format("060102-150405.000"))
154 1 : require.NoError(t, err)
155 1 : require.NoError(t, os.MkdirAll(metaDir, 0755))
156 1 : defer func() {
157 1 : if !t.Failed() && !runOpts.keep {
158 1 : _ = os.RemoveAll(metaDir)
159 1 : }
160 : }()
161 :
162 1 : rng := rand.New(rand.NewSource(runOpts.seed))
163 1 : opCount := runOpts.ops.Uint64(rng)
164 1 :
165 1 : // Generate a new set of random ops, writing them to <dir>/ops. These will be
166 1 : // read by the child processes when performing a test run.
167 1 : km := newKeyManager()
168 1 : cfg := presetConfigs[rng.Intn(len(presetConfigs))]
169 1 : if runOpts.previousOpsPath != "" {
170 0 : // During cross-version testing, we load keys from an `ops` file
171 0 : // produced by a metamorphic test run of an earlier Pebble version.
172 0 : // Seeding the keys ensure we generate interesting operations, including
173 0 : // ones with key shadowing, merging, etc.
174 0 : opsPath := filepath.Join(filepath.Dir(filepath.Clean(runOpts.previousOpsPath)), "ops")
175 0 : opsData, err := os.ReadFile(opsPath)
176 0 : require.NoError(t, err)
177 0 : ops, err := parse(opsData)
178 0 : require.NoError(t, err)
179 0 : loadPrecedingKeys(t, ops, &cfg, km)
180 0 : }
181 1 : ops := generate(rng, opCount, cfg, km)
182 1 : opsPath := filepath.Join(metaDir, "ops")
183 1 : formattedOps := formatOps(ops)
184 1 : require.NoError(t, os.WriteFile(opsPath, []byte(formattedOps), 0644))
185 1 :
186 1 : // runOptions performs a particular test run with the specified options. The
187 1 : // options are written to <run-dir>/OPTIONS and a child process is created to
188 1 : // actually execute the test.
189 1 : runOptions := func(t *testing.T, opts *TestOptions) {
190 1 : if opts.Opts.Cache != nil {
191 1 : defer opts.Opts.Cache.Unref()
192 1 : }
193 1 : for _, fn := range runOpts.mutateTestOptions {
194 0 : fn(opts)
195 0 : }
196 1 : runDir := filepath.Join(metaDir, path.Base(t.Name()))
197 1 : require.NoError(t, os.MkdirAll(runDir, 0755))
198 1 :
199 1 : optionsPath := filepath.Join(runDir, "OPTIONS")
200 1 : optionsStr := optionsToString(opts)
201 1 : require.NoError(t, os.WriteFile(optionsPath, []byte(optionsStr), 0644))
202 1 :
203 1 : args := []string{
204 1 : "-keep=" + fmt.Sprint(runOpts.keep),
205 1 : "-run-dir=" + runDir,
206 1 : "-test.run=" + t.Name() + "$",
207 1 : }
208 1 : if runOpts.traceFile != "" {
209 0 : args = append(args, "-test.trace="+filepath.Join(runDir, runOpts.traceFile))
210 0 : }
211 :
212 1 : binary := os.Args[0]
213 1 : if runOpts.innerBinary != "" {
214 0 : binary = runOpts.innerBinary
215 0 : }
216 1 : cmd := exec.Command(binary, args...)
217 1 : out, err := cmd.CombinedOutput()
218 1 : if err != nil {
219 0 : t.Fatalf(`
220 0 : ===== SEED =====
221 0 : %d
222 0 : ===== ERR =====
223 0 : %v
224 0 : ===== OUT =====
225 0 : %s
226 0 : ===== OPTIONS =====
227 0 : %s
228 0 : ===== OPS =====
229 0 : %s
230 0 : ===== HISTORY =====
231 0 : %s`, runOpts.seed, err, out, optionsStr, formattedOps, readFile(filepath.Join(runDir, "history")))
232 0 : }
233 : }
234 :
235 1 : var names []string
236 1 : options := map[string]*TestOptions{}
237 1 :
238 1 : // Create the standard options.
239 1 : for i, opts := range standardOptions() {
240 1 : name := fmt.Sprintf("standard-%03d", i)
241 1 : names = append(names, name)
242 1 : options[name] = opts
243 1 : }
244 :
245 : // Create the custom option runs, if any.
246 1 : for name, customOptsStr := range runOpts.customRuns {
247 0 : options[name] = defaultTestOptions()
248 0 : if err := parseOptions(options[name], customOptsStr, runOpts.customOptionParsers); err != nil {
249 0 : t.Fatalf("custom opts %q: %s", name, err)
250 0 : }
251 : }
252 : // Sort the custom options names for determinism (they're currently in
253 : // random order from map iteration).
254 1 : sort.Strings(names[len(names)-len(runOpts.customRuns):])
255 1 :
256 1 : // Create random options. We make an arbitrary choice to run with as many
257 1 : // random options as we have standard options.
258 1 : nOpts := len(options)
259 1 : for i := 0; i < nOpts; i++ {
260 1 : name := fmt.Sprintf("random-%03d", i)
261 1 : names = append(names, name)
262 1 : opts := randomOptions(rng, runOpts.customOptionParsers)
263 1 : options[name] = opts
264 1 : }
265 :
266 : // If the user provided the path to an initial database state to use, update
267 : // all the options to pull from it.
268 1 : if runOpts.initialStatePath != "" {
269 0 : for _, o := range options {
270 0 : var err error
271 0 : o.initialStatePath, err = filepath.Abs(runOpts.initialStatePath)
272 0 : require.NoError(t, err)
273 0 : o.initialStateDesc = runOpts.initialStateDesc
274 0 : }
275 : }
276 :
277 : // Run the options.
278 1 : t.Run("execution", func(t *testing.T) {
279 1 : for _, name := range names {
280 1 : name := name
281 1 : t.Run(name, func(t *testing.T) {
282 1 : t.Parallel()
283 1 : runOptions(t, options[name])
284 1 : })
285 : }
286 : })
287 : // NB: The above 'execution' subtest will not complete until all of the
288 : // individual execution/ subtests have completed. The grouping within the
289 : // `execution` subtest ensures all the histories are available when we
290 : // proceed to comparing against the base history.
291 :
292 : // Don't bother comparing output if we've already failed.
293 1 : if t.Failed() {
294 0 : return
295 0 : }
296 :
297 1 : t.Run("compare", func(t *testing.T) {
298 1 : getHistoryPath := func(name string) string {
299 1 : return filepath.Join(metaDir, name, "history")
300 1 : }
301 :
302 1 : base := readHistory(t, getHistoryPath(names[0]))
303 1 : base = reorderHistory(base)
304 1 : for i := 1; i < len(names); i++ {
305 1 : t.Run(names[i], func(t *testing.T) {
306 1 : lines := readHistory(t, getHistoryPath(names[i]))
307 1 : lines = reorderHistory(lines)
308 1 : diff := difflib.UnifiedDiff{
309 1 : A: base,
310 1 : B: lines,
311 1 : Context: 5,
312 1 : }
313 1 : text, err := difflib.GetUnifiedDiffString(diff)
314 1 : require.NoError(t, err)
315 1 : if text != "" {
316 0 : // NB: We force an exit rather than using t.Fatal because the latter
317 0 : // will run another instance of the test if -count is specified, while
318 0 : // we're happy to exit on the first failure.
319 0 : optionsStrA := optionsToString(options[names[0]])
320 0 : optionsStrB := optionsToString(options[names[i]])
321 0 :
322 0 : fmt.Printf(`
323 0 : ===== SEED =====
324 0 : %d
325 0 : ===== DIFF =====
326 0 : %s/{%s,%s}
327 0 : %s
328 0 : ===== OPTIONS %s =====
329 0 : %s
330 0 : ===== OPTIONS %s =====
331 0 : %s
332 0 : ===== OPS =====
333 0 : %s
334 0 : `, runOpts.seed, metaDir, names[0], names[i], text, names[0], optionsStrA, names[i], optionsStrB, formattedOps)
335 0 : os.Exit(1)
336 0 : }
337 : })
338 : }
339 : })
340 : }
341 :
342 : type runOnceOptions struct {
343 : keep bool
344 : maxThreads int
345 : errorRate float64
346 : failRegexp *regexp.Regexp
347 : customOptionParsers map[string]func(string) (CustomOption, bool)
348 : }
349 :
350 : // A RunOnceOption configures the behavior of a single run of the metamorphic
351 : // tests.
352 : type RunOnceOption interface {
353 : applyOnce(*runOnceOptions)
354 : }
355 :
356 : // KeepData keeps the database directory, even on successful runs. If the test
357 : // used an in-memory filesystem, the in-memory filesystem will be persisted to
358 : // the run directory.
359 : type KeepData struct{}
360 :
361 0 : func (KeepData) apply(ro *runAndCompareOptions) { ro.keep = true }
362 0 : func (KeepData) applyOnce(ro *runOnceOptions) { ro.keep = true }
363 :
364 : // InjectErrorsRate configures the run to inject errors into read-only
365 : // filesystem operations and retry injected errors.
366 : type InjectErrorsRate float64
367 :
368 0 : func (r InjectErrorsRate) apply(ro *runAndCompareOptions) { ro.errorRate = float64(r) }
369 0 : func (r InjectErrorsRate) applyOnce(ro *runOnceOptions) { ro.errorRate = float64(r) }
370 :
371 : // MaxThreads sets an upper bound on the number of parallel execution threads
372 : // during replay.
373 : type MaxThreads int
374 :
375 1 : func (m MaxThreads) apply(ro *runAndCompareOptions) { ro.maxThreads = int(m) }
376 0 : func (m MaxThreads) applyOnce(ro *runOnceOptions) { ro.maxThreads = int(m) }
377 :
378 : // FailOnMatch configures the run to fail immediately if the history matches the
379 : // provided regular expression.
380 : type FailOnMatch struct {
381 : *regexp.Regexp
382 : }
383 :
384 0 : func (f FailOnMatch) apply(ro *runAndCompareOptions) { ro.failRegexp = f.Regexp }
385 0 : func (f FailOnMatch) applyOnce(ro *runOnceOptions) { ro.failRegexp = f.Regexp }
386 :
387 : // RunOnce performs one run of the metamorphic tests. RunOnce expects the
388 : // directory named by `runDir` to already exist and contain an `OPTIONS` file
389 : // containing the test run's configuration. The history of the run is persisted
390 : // to a file at the path `historyPath`.
391 : //
392 : // The `seed` parameter is not functional; it's used for context in logging.
393 0 : func RunOnce(t TestingT, runDir string, seed uint64, historyPath string, rOpts ...RunOnceOption) {
394 0 : runOpts := runOnceOptions{
395 0 : customOptionParsers: map[string]func(string) (CustomOption, bool){},
396 0 : }
397 0 : for _, o := range rOpts {
398 0 : o.applyOnce(&runOpts)
399 0 : }
400 :
401 0 : opsPath := filepath.Join(filepath.Dir(filepath.Clean(runDir)), "ops")
402 0 : opsData, err := os.ReadFile(opsPath)
403 0 : require.NoError(t, err)
404 0 :
405 0 : ops, err := parse(opsData)
406 0 : require.NoError(t, err)
407 0 : _ = ops
408 0 :
409 0 : optionsPath := filepath.Join(runDir, "OPTIONS")
410 0 : optionsData, err := os.ReadFile(optionsPath)
411 0 : require.NoError(t, err)
412 0 :
413 0 : opts := &pebble.Options{}
414 0 : testOpts := &TestOptions{Opts: opts}
415 0 : require.NoError(t, parseOptions(testOpts, string(optionsData), runOpts.customOptionParsers))
416 0 :
417 0 : // Always use our custom comparer which provides a Split method, splitting
418 0 : // keys at the trailing '@'.
419 0 : opts.Comparer = testkeys.Comparer
420 0 : // Use an archive cleaner to ease post-mortem debugging.
421 0 : opts.Cleaner = base.ArchiveCleaner{}
422 0 :
423 0 : // Set up the filesystem to use for the test. Note that by default we use an
424 0 : // in-memory FS.
425 0 : if testOpts.useDisk {
426 0 : opts.FS = vfs.Default
427 0 : require.NoError(t, os.RemoveAll(opts.FS.PathJoin(runDir, "data")))
428 0 : } else {
429 0 : opts.Cleaner = base.ArchiveCleaner{}
430 0 : if testOpts.strictFS {
431 0 : opts.FS = vfs.NewStrictMem()
432 0 : } else {
433 0 : opts.FS = vfs.NewMem()
434 0 : }
435 : }
436 0 : opts.WithFSDefaults()
437 0 :
438 0 : threads := testOpts.threads
439 0 : if runOpts.maxThreads < threads {
440 0 : threads = runOpts.maxThreads
441 0 : }
442 :
443 0 : dir := opts.FS.PathJoin(runDir, "data")
444 0 : // Set up the initial database state if configured to start from a non-empty
445 0 : // database. By default tests start from an empty database, but split
446 0 : // version testing may configure a previous metamorphic tests's database
447 0 : // state as the initial state.
448 0 : if testOpts.initialStatePath != "" {
449 0 : require.NoError(t, setupInitialState(dir, testOpts))
450 0 : }
451 :
452 : // Wrap the filesystem with one that will inject errors into read
453 : // operations with *errorRate probability.
454 0 : opts.FS = errorfs.Wrap(opts.FS, errorfs.WithProbability(errorfs.OpKindRead, runOpts.errorRate))
455 0 :
456 0 : if opts.WALDir != "" {
457 0 : opts.WALDir = opts.FS.PathJoin(runDir, opts.WALDir)
458 0 : }
459 :
460 0 : historyFile, err := os.Create(historyPath)
461 0 : require.NoError(t, err)
462 0 : defer historyFile.Close()
463 0 : writers := []io.Writer{historyFile}
464 0 :
465 0 : if testing.Verbose() {
466 0 : writers = append(writers, os.Stdout)
467 0 : }
468 0 : h := newHistory(runOpts.failRegexp, writers...)
469 0 :
470 0 : m := newTest(ops)
471 0 : require.NoError(t, m.init(h, dir, testOpts))
472 0 :
473 0 : if threads <= 1 {
474 0 : for m.step(h) {
475 0 : if err := h.Error(); err != nil {
476 0 : fmt.Fprintf(os.Stderr, "Seed: %d\n", seed)
477 0 : fmt.Fprintln(os.Stderr, err)
478 0 : m.maybeSaveData()
479 0 : os.Exit(1)
480 0 : }
481 : }
482 0 : } else {
483 0 : eg, ctx := errgroup.WithContext(context.Background())
484 0 : for t := 0; t < threads; t++ {
485 0 : t := t // bind loop var to scope
486 0 : eg.Go(func() error {
487 0 : for idx := 0; idx < len(m.ops); idx++ {
488 0 : // Skip any operations whose receiver object hashes to a
489 0 : // different thread. All operations with the same receiver
490 0 : // are performed from the same thread. This goroutine is
491 0 : // only responsible for executing operations that hash to
492 0 : // `t`.
493 0 : if hashThread(m.ops[idx].receiver(), threads) != t {
494 0 : continue
495 : }
496 :
497 : // Some operations have additional synchronization
498 : // dependencies. If this operation has any, wait for its
499 : // dependencies to complete before executing.
500 0 : for _, waitOnIdx := range m.opsWaitOn[idx] {
501 0 : select {
502 0 : case <-ctx.Done():
503 0 : // Exit if some other thread already errored out.
504 0 : return ctx.Err()
505 0 : case <-m.opsDone[waitOnIdx]:
506 : }
507 : }
508 :
509 0 : m.ops[idx].run(m, h.recorder(t, idx))
510 0 :
511 0 : // If this operation has a done channel, close it so that
512 0 : // other operations that synchronize on this operation know
513 0 : // that it's been completed.
514 0 : if ch := m.opsDone[idx]; ch != nil {
515 0 : close(ch)
516 0 : }
517 :
518 0 : if err := h.Error(); err != nil {
519 0 : return err
520 0 : }
521 : }
522 0 : return nil
523 : })
524 : }
525 0 : if err := eg.Wait(); err != nil {
526 0 : fmt.Fprintf(os.Stderr, "Seed: %d\n", seed)
527 0 : fmt.Fprintln(os.Stderr, err)
528 0 : m.maybeSaveData()
529 0 : os.Exit(1)
530 0 : }
531 : }
532 :
533 0 : if runOpts.keep && !testOpts.useDisk {
534 0 : m.maybeSaveData()
535 0 : }
536 : }
537 :
538 0 : func hashThread(objID objID, numThreads int) int {
539 0 : // Fibonacci hash https://probablydance.com/2018/06/16/fibonacci-hashing-the-optimization-that-the-world-forgot-or-a-better-alternative-to-integer-modulo/
540 0 : return int((11400714819323198485 * uint64(objID)) % uint64(numThreads))
541 0 : }
542 :
543 : // Compare runs the metamorphic tests in the provided runDirs and compares their
544 : // histories.
545 0 : func Compare(t TestingT, rootDir string, seed uint64, runDirs []string, rOpts ...RunOnceOption) {
546 0 : historyPaths := make([]string, len(runDirs))
547 0 : for i := 0; i < len(runDirs); i++ {
548 0 : historyPath := filepath.Join(rootDir, runDirs[i]+"-"+time.Now().Format("060102-150405.000"))
549 0 : runDirs[i] = filepath.Join(rootDir, runDirs[i])
550 0 : _ = os.Remove(historyPath)
551 0 : historyPaths[i] = historyPath
552 0 : }
553 0 : defer func() {
554 0 : for _, path := range historyPaths {
555 0 : _ = os.Remove(path)
556 0 : }
557 : }()
558 :
559 0 : for i, runDir := range runDirs {
560 0 : RunOnce(t, runDir, seed, historyPaths[i], rOpts...)
561 0 : }
562 :
563 0 : if t.Failed() {
564 0 : return
565 0 : }
566 :
567 0 : i, diff := CompareHistories(t, historyPaths)
568 0 : if i != 0 {
569 0 : fmt.Printf(`
570 0 : ===== DIFF =====
571 0 : %s/{%s,%s}
572 0 : %s
573 0 : `, rootDir, runDirs[0], runDirs[i], diff)
574 0 : os.Exit(1)
575 0 : }
576 : }
577 :
578 : // TestingT is an interface wrapper around *testing.T
579 : type TestingT interface {
580 : require.TestingT
581 : Failed() bool
582 : }
583 :
584 0 : func readFile(path string) string {
585 0 : history, err := os.ReadFile(path)
586 0 : if err != nil {
587 0 : return fmt.Sprintf("err: %v", err)
588 0 : }
589 :
590 0 : return string(history)
591 : }
|
