Line data Source code
1 : // Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
2 : // of this source code is governed by a BSD-style license that can be found in
3 : // the LICENSE file.
4 :
5 : // Package metaflags defines command-line flags for the metamorphic tests and
6 : // provides functionality to construct the respective
7 : // metamorphic.RunOptions/RunOnceOptions.
8 : package metaflags
9 :
10 : import (
11 : "flag"
12 : "fmt"
13 : "math"
14 : "os"
15 : "regexp"
16 : "strings"
17 : "time"
18 :
19 : "github.com/cockroachdb/pebble/internal/invariants"
20 : "github.com/cockroachdb/pebble/internal/randvar"
21 : "github.com/cockroachdb/pebble/metamorphic"
22 : )
23 :
24 : // CommonFlags contains flags that apply to both metamorphic.Run and
25 : // metamorphic.RunOnce/Compare.
26 : type CommonFlags struct {
27 : // Dir is the directory storing test state. See "dir" flag below.
28 : Dir string
29 : // Seed for generation of random operations. See "seed" flag below.
30 : Seed uint64
31 : // ErrorRate is the rate of injected filesystem errors. See "error-rate" flag
32 : // below.
33 : ErrorRate float64
34 : // FailRE causes the test to fail if the output matches this regex. See "fail"
35 : // flag below.
36 : FailRE string
37 : // Keep determines if the DB directory is kept on successful runs. See "keep"
38 : // flag below.
39 : Keep bool
40 : // MaxThreads used by a single run. See "max-threads" flag below.
41 : MaxThreads int
42 : // NumInstances is the number of Pebble instances to create in one run. See
43 : // "num-instances" flag below.
44 : NumInstances int
45 : // OpTimeout is a per-operation timeout.
46 : OpTimeout time.Duration
47 : }
48 :
49 2 : func initCommonFlags() *CommonFlags {
50 2 : c := &CommonFlags{}
51 2 : flag.StringVar(&c.Dir, "dir", "_meta",
52 2 : "the directory storing test state")
53 2 :
54 2 : flag.Uint64Var(&c.Seed, "seed", 0,
55 2 : "a pseudorandom number generator seed")
56 2 :
57 2 : // TODO: default error rate to a non-zero value. Currently, retrying is
58 2 : // non-deterministic because of the Ierator.*WithLimit() methods since
59 2 : // they may say that the Iterator is not valid, but be positioned at a
60 2 : // certain key that can be returned in the future if the limit is changed.
61 2 : // Since that key is hidden from clients of Iterator, the retryableIter
62 2 : // using SeekGE will not necessarily position the Iterator that saw an
63 2 : // injected error at the same place as an Iterator that did not see that
64 2 : // error.
65 2 : flag.Float64Var(&c.ErrorRate, "error-rate", 0.0,
66 2 : "rate of errors injected into filesystem operations (0 ≤ r < 1)")
67 2 :
68 2 : flag.StringVar(&c.FailRE, "fail", "",
69 2 : "fail the test if the supplied regular expression matches the output")
70 2 :
71 2 : flag.BoolVar(&c.Keep, "keep", false,
72 2 : "keep the DB directory even on successful runs")
73 2 :
74 2 : flag.IntVar(&c.MaxThreads, "max-threads", math.MaxInt,
75 2 : "limit execution of a single run to the provided number of threads; must be ≥ 1")
76 2 :
77 2 : flag.IntVar(&c.NumInstances, "num-instances", 1, "number of pebble instances to create (default: 1)")
78 2 :
79 2 : defaultOpTimeout := 1 * time.Minute
80 2 : if invariants.RaceEnabled {
81 0 : defaultOpTimeout *= 5
82 0 : }
83 2 : flag.DurationVar(&c.OpTimeout, "op-timeout", defaultOpTimeout, "per-op timeout")
84 2 :
85 2 : return c
86 : }
87 :
88 : // RunOnceFlags contains flags that apply only to metamorphic.RunOnce/Compare.
89 : type RunOnceFlags struct {
90 : *CommonFlags
91 : // RunDir applies to metamorphic.RunOnce and contains the specific
92 : // configuration of the run. See "run-dir" flag below.
93 : RunDir string
94 : // Compare applies to metamorphic.Compare. See "compare" flag below.
95 : Compare string
96 : // TryToReduce enables a mode where we try to find a minimal subset of
97 : // operations that reproduce a problem during a test run (e.g. panic or
98 : // internal error).
99 : TryToReduce bool
100 : }
101 :
102 2 : func initRunOnceFlags(c *CommonFlags) *RunOnceFlags {
103 2 : ro := &RunOnceFlags{CommonFlags: c}
104 2 : flag.StringVar(&ro.RunDir, "run-dir", "",
105 2 : `directory containing the specific configuration to (re-)run (used for post-mortem debugging).
106 2 : Example: --run-dir _meta/231220-164251.3552792807512/random-003`)
107 2 :
108 2 : flag.StringVar(&ro.Compare, "compare", "",
109 2 : `runs to compare, in the format _meta/test-root-dir/{run1,run2,...}. The result
110 2 : of each run is compared with the result of the first run.
111 2 : Example, --compare '_meta/231220-164251.3552792807512/{standard-000,random-025}'`)
112 2 :
113 2 : flag.BoolVar(&ro.TryToReduce, "try-to-reduce", false,
114 2 : `if set, we will try to reduce the number of operations that cause a failure. The
115 2 : verbose flag should be used with this flag.`)
116 2 :
117 2 : return ro
118 2 : }
119 :
120 : // RunFlags contains flags that apply only to metamorphic.Run.
121 : type RunFlags struct {
122 : *CommonFlags
123 : // FS controls the type of filesystems to use. See "fs" flag below.
124 : FS string
125 : // TraceFile for execution tracing. See "trace-file" flag below.
126 : TraceFile string
127 : // Ops describes how the total number of operations is generated. See "ops" flags below.
128 : Ops randvar.Flag
129 : // InnerBinary is the binary to invoke for a single run. See "inner-binary"
130 : // flag below.
131 : InnerBinary string
132 : // PreviousOps is the path to the ops file of a previous run. See the
133 : // "previous-ops" flag below.
134 : PreviousOps string
135 : // InitialStatePath is the path to a database data directory from a previous
136 : // run. See the "initial-state" flag below.
137 : InitialStatePath string
138 : // InitialStateDesc is a human-readable description of the initial database
139 : // state. See "initial-state-desc" flag below.
140 : InitialStateDesc string
141 : }
142 :
143 1 : func initRunFlags(c *CommonFlags) *RunFlags {
144 1 : r := &RunFlags{CommonFlags: c}
145 1 : flag.StringVar(&r.FS, "fs", "rand",
146 1 : `force the tests to use either memory or disk-backed filesystems (valid: "mem", "disk", "rand")`)
147 1 :
148 1 : flag.StringVar(&r.TraceFile, "trace-file", "",
149 1 : "write an execution trace to `<run-dir>/file`")
150 1 :
151 1 : if err := r.Ops.Set("uniform:5000-10000"); err != nil {
152 0 : panic(err)
153 : }
154 1 : flag.Var(&r.Ops, "ops", "uniform:5000-10000")
155 1 :
156 1 : flag.StringVar(&r.InnerBinary, "inner-binary", "",
157 1 : `binary to run for each instance of the test (this same binary by default); cannot be used
158 1 : with --run-dir or --compare`)
159 1 :
160 1 : // The following options may be used for split-version metamorphic testing.
161 1 : // To perform split-version testing, the client runs the metamorphic tests
162 1 : // on an earlier Pebble SHA passing the `--keep` flag. The client then
163 1 : // switches to the later Pebble SHA, setting the below options to point to
164 1 : // the `ops` file and one of the previous run's data directories.
165 1 :
166 1 : flag.StringVar(&r.PreviousOps, "previous-ops", "",
167 1 : "path to an ops file, used to prepopulate the set of keys operations draw from")
168 1 :
169 1 : flag.StringVar(&r.InitialStatePath, "initial-state", "",
170 1 : "path to a database's data directory, used to prepopulate the test run's databases")
171 1 :
172 1 : flag.StringVar(&r.InitialStateDesc, "initial-state-desc", "",
173 1 : `a human-readable description of the initial database state.
174 1 : If set this parameter is written to the OPTIONS to aid in
175 1 : debugging. It's intended to describe the lineage of a
176 1 : database's state, including sufficient information for
177 1 : reproduction (eg, SHA, prng seed, etc).`)
178 1 : return r
179 : }
180 :
181 : // InitRunOnceFlags initializes the flags that are used for a single run of the
182 : // metamorphic test.
183 1 : func InitRunOnceFlags() *RunOnceFlags {
184 1 : return initRunOnceFlags(initCommonFlags())
185 1 : }
186 :
187 : // InitAllFlags initializes all metamorphic test flags: those used for a
188 : // single run, and those used for a "top level" run.
189 1 : func InitAllFlags() (*RunOnceFlags, *RunFlags) {
190 1 : c := initCommonFlags()
191 1 : return initRunOnceFlags(c), initRunFlags(c)
192 1 : }
193 :
194 : // MakeRunOnceOptions constructs RunOnceOptions based on the flags.
195 1 : func (ro *RunOnceFlags) MakeRunOnceOptions() []metamorphic.RunOnceOption {
196 1 : onceOpts := []metamorphic.RunOnceOption{
197 1 : metamorphic.MaxThreads(ro.MaxThreads),
198 1 : metamorphic.OpTimeout(ro.OpTimeout),
199 1 : }
200 1 : if ro.Keep {
201 0 : onceOpts = append(onceOpts, metamorphic.KeepData{})
202 0 : }
203 1 : if ro.FailRE != "" {
204 0 : onceOpts = append(onceOpts, metamorphic.FailOnMatch{Regexp: regexp.MustCompile(ro.FailRE)})
205 0 : }
206 1 : if ro.ErrorRate > 0 {
207 0 : onceOpts = append(onceOpts, metamorphic.InjectErrorsRate(ro.ErrorRate))
208 0 : }
209 1 : if ro.NumInstances > 1 {
210 1 : onceOpts = append(onceOpts, metamorphic.MultiInstance(ro.NumInstances))
211 1 : }
212 1 : return onceOpts
213 : }
214 :
215 : // ParseCompare parses the value of the compare flag, in format
216 : // "test-root-dir/{run1,run2,...}". Exits if the value is not valid.
217 : //
218 : // Returns the common test root dir (e.g. "test-root-dir") and a list of
219 : // subdirectories (e.g. {"run1", "run2"}).
220 0 : func (ro *RunOnceFlags) ParseCompare() (testRootDir string, runSubdirs []string) {
221 0 : testRootDir, runSubdirs, ok := ro.tryParseCompare()
222 0 : if !ok {
223 0 : fmt.Fprintf(os.Stderr, `cannot parse compare flag value %q; format is "test-root-dir/{run1,run2,..}"`, ro.Compare)
224 0 : os.Exit(1)
225 0 : }
226 0 : return testRootDir, runSubdirs
227 : }
228 :
229 0 : func (ro *RunOnceFlags) tryParseCompare() (testRootDir string, runSubdirs []string, ok bool) {
230 0 : value := ro.Compare
231 0 : brace := strings.Index(value, "{")
232 0 : if brace == -1 || !strings.HasSuffix(value, "}") {
233 0 : return "", nil, false
234 0 : }
235 :
236 0 : testRootDir = value[:brace]
237 0 : runSubdirs = strings.Split(value[brace+1:len(value)-1], ",")
238 0 : if len(runSubdirs) < 2 {
239 0 : return "", nil, false
240 0 : }
241 0 : return testRootDir, runSubdirs, true
242 : }
243 :
244 : // MakeRunOptions constructs RunOptions based on the flags.
245 1 : func (r *RunFlags) MakeRunOptions() []metamorphic.RunOption {
246 1 : opts := []metamorphic.RunOption{
247 1 : metamorphic.Seed(r.Seed),
248 1 : metamorphic.OpCount(r.Ops.Static),
249 1 : metamorphic.MaxThreads(r.MaxThreads),
250 1 : metamorphic.OpTimeout(r.OpTimeout),
251 1 : }
252 1 : if r.Keep {
253 0 : opts = append(opts, metamorphic.KeepData{})
254 0 : }
255 1 : if r.FailRE != "" {
256 0 : opts = append(opts, metamorphic.FailOnMatch{Regexp: regexp.MustCompile(r.FailRE)})
257 0 : }
258 1 : if r.ErrorRate > 0 {
259 0 : opts = append(opts, metamorphic.InjectErrorsRate(r.ErrorRate))
260 0 : }
261 1 : if r.TraceFile != "" {
262 0 : opts = append(opts, metamorphic.RuntimeTrace(r.TraceFile))
263 0 : }
264 1 : if r.PreviousOps != "" {
265 0 : opts = append(opts, metamorphic.ExtendPreviousRun(r.PreviousOps, r.InitialStatePath, r.InitialStateDesc))
266 0 : }
267 1 : if r.NumInstances > 1 {
268 0 : opts = append(opts, metamorphic.MultiInstance(r.NumInstances))
269 0 : }
270 :
271 : // If the filesystem type was forced, all tests will use that value.
272 1 : switch r.FS {
273 1 : case "", "rand", "default":
274 : // No-op. Use the generated value for the filesystem.
275 0 : case "disk":
276 0 : opts = append(opts, metamorphic.UseDisk)
277 0 : case "mem":
278 0 : opts = append(opts, metamorphic.UseInMemory)
279 0 : default:
280 0 : panic(fmt.Sprintf("unknown forced filesystem type: %q", r.FS))
281 : }
282 1 : if r.InnerBinary != "" {
283 0 : opts = append(opts, metamorphic.InnerBinary(r.InnerBinary))
284 0 : }
285 1 : return opts
286 : }
|
