Line data Source code
1 : // Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
2 : // of this source code is governed by a BSD-style license that can be found in
3 : // the LICENSE file.
4 :
5 : // Package metaflags defines command-line flags for the metamorphic tests and
6 : // provides functionality to construct the respective
7 : // metamorphic.RunOptions/RunOnceOptions.
8 : package metaflags
9 :
10 : import (
11 : "flag"
12 : "fmt"
13 : "math"
14 : "os"
15 : "regexp"
16 : "strings"
17 : "time"
18 :
19 : "github.com/cockroachdb/pebble/internal/invariants"
20 : "github.com/cockroachdb/pebble/internal/randvar"
21 : "github.com/cockroachdb/pebble/metamorphic"
22 : )
23 :
24 : // CommonFlags contains flags that apply to both metamorphic.Run and
25 : // metamorphic.RunOnce/Compare.
26 : type CommonFlags struct {
27 : // Dir is the directory storing test state. See "dir" flag below.
28 : Dir string
29 : // Seed for generation of random operations. See "seed" flag below.
30 : Seed uint64
31 : // ErrorRate is the rate of injected filesystem errors. See "error-rate" flag
32 : // below.
33 : ErrorRate float64
34 : // FailRE causes the test to fail if the output matches this regex. See "fail"
35 : // flag below.
36 : FailRE string
37 : // Keep determines if the DB directory is kept on successful runs. See "keep"
38 : // flag below.
39 : Keep bool
40 : // MaxThreads used by a single run. See "max-threads" flag below.
41 : MaxThreads int
42 : // NumInstances is the number of Pebble instances to create in one run. See
43 : // "num-instances" flag below.
44 : NumInstances int
45 : // OpTimeout is a per-operation timeout.
46 : OpTimeout time.Duration
47 : }
48 :
49 1 : func initCommonFlags() *CommonFlags {
50 1 : c := &CommonFlags{}
51 1 : flag.StringVar(&c.Dir, "dir", "_meta",
52 1 : "the directory storing test state")
53 1 :
54 1 : flag.Uint64Var(&c.Seed, "seed", 0,
55 1 : "a pseudorandom number generator seed")
56 1 :
57 1 : // TODO: default error rate to a non-zero value. Currently, retrying is
58 1 : // non-deterministic because of the Ierator.*WithLimit() methods since
59 1 : // they may say that the Iterator is not valid, but be positioned at a
60 1 : // certain key that can be returned in the future if the limit is changed.
61 1 : // Since that key is hidden from clients of Iterator, the retryableIter
62 1 : // using SeekGE will not necessarily position the Iterator that saw an
63 1 : // injected error at the same place as an Iterator that did not see that
64 1 : // error.
65 1 : flag.Float64Var(&c.ErrorRate, "error-rate", 0.0,
66 1 : "rate of errors injected into filesystem operations (0 ≤ r < 1)")
67 1 :
68 1 : flag.StringVar(&c.FailRE, "fail", "",
69 1 : "fail the test if the supplied regular expression matches the output")
70 1 :
71 1 : flag.BoolVar(&c.Keep, "keep", false,
72 1 : "keep the DB directory even on successful runs")
73 1 :
74 1 : flag.IntVar(&c.MaxThreads, "max-threads", math.MaxInt,
75 1 : "limit execution of a single run to the provided number of threads; must be ≥ 1")
76 1 :
77 1 : flag.IntVar(&c.NumInstances, "num-instances", 1, "number of pebble instances to create (default: 1)")
78 1 :
79 1 : defaultOpTimeout := 2 * time.Minute
80 1 : if invariants.RaceEnabled {
81 0 : defaultOpTimeout *= 5
82 0 : }
83 1 : flag.DurationVar(&c.OpTimeout, "op-timeout", defaultOpTimeout, "per-op timeout")
84 1 :
85 1 : return c
86 : }
87 :
88 : // RunOnceFlags contains flags that apply only to metamorphic.RunOnce/Compare.
89 : type RunOnceFlags struct {
90 : *CommonFlags
91 : // RunDir applies to metamorphic.RunOnce and contains the specific
92 : // configuration of the run. See "run-dir" flag below.
93 : RunDir string
94 : // Compare applies to metamorphic.Compare. See "compare" flag below.
95 : Compare string
96 : // TryToReduce enables a mode where we try to find a minimal subset of
97 : // operations that reproduce a problem during a test run (e.g. panic or
98 : // internal error).
99 : TryToReduce bool
100 : // ReduceAttempts is the number of attempts to reduce (for each op removal
101 : // probability).
102 : ReduceAttempts int
103 : }
104 :
105 1 : func initRunOnceFlags(c *CommonFlags) *RunOnceFlags {
106 1 : ro := &RunOnceFlags{CommonFlags: c}
107 1 : flag.StringVar(&ro.RunDir, "run-dir", "",
108 1 : `directory containing the specific configuration to (re-)run (used for post-mortem debugging).
109 1 : Example: --run-dir _meta/231220-164251.3552792807512/random-003`)
110 1 :
111 1 : flag.StringVar(&ro.Compare, "compare", "",
112 1 : `runs to compare, in the format _meta/test-root-dir/{run1,run2,...}. The result
113 1 : of each run is compared with the result of the first run.
114 1 : Example, --compare '_meta/231220-164251.3552792807512/{standard-000,random-025}'`)
115 1 :
116 1 : flag.BoolVar(&ro.TryToReduce, "try-to-reduce", false,
117 1 : `if set, we will try to reduce the number of operations that cause a failure. The
118 1 : verbose flag should be used with this flag.`)
119 1 :
120 1 : flag.IntVar(&ro.ReduceAttempts, "reduce-attempts", 100,
121 1 : `the number of attempts to reduce, for each probability; only used with --try-to-reduce.`)
122 1 :
123 1 : return ro
124 1 : }
125 :
126 : // RunFlags contains flags that apply only to metamorphic.Run.
127 : type RunFlags struct {
128 : *CommonFlags
129 : // FS controls the type of filesystems to use. See "fs" flag below.
130 : FS string
131 : // TraceFile for execution tracing. See "trace-file" flag below.
132 : TraceFile string
133 : // Ops describes how the total number of operations is generated. See "ops" flags below.
134 : Ops randvar.Flag
135 : // InnerBinary is the binary to invoke for a single run. See "inner-binary"
136 : // flag below.
137 : InnerBinary string
138 : // PreviousOps is the path to the ops file of a previous run. See the
139 : // "previous-ops" flag below.
140 : PreviousOps string
141 : // InitialStatePath is the path to a database data directory from a previous
142 : // run. See the "initial-state" flag below.
143 : InitialStatePath string
144 : // InitialStateDesc is a human-readable description of the initial database
145 : // state. See "initial-state-desc" flag below.
146 : InitialStateDesc string
147 : }
148 :
149 0 : func initRunFlags(c *CommonFlags) *RunFlags {
150 0 : r := &RunFlags{CommonFlags: c}
151 0 : flag.StringVar(&r.FS, "fs", "rand",
152 0 : `force the tests to use either memory or disk-backed filesystems (valid: "mem", "disk", "rand")`)
153 0 :
154 0 : flag.StringVar(&r.TraceFile, "trace-file", "",
155 0 : "write an execution trace to `<run-dir>/file`")
156 0 :
157 0 : ops := "uniform:5000-10000"
158 0 : if invariants.RaceEnabled {
159 0 : // Reduce the size of the test under race (to avoid timeouts).
160 0 : ops = "uniform:1000-2000"
161 0 : }
162 0 : if err := r.Ops.Set(ops); err != nil {
163 0 : panic(err)
164 : }
165 0 : flag.Var(&r.Ops, "ops", "uniform:min-max")
166 0 :
167 0 : flag.StringVar(&r.InnerBinary, "inner-binary", "",
168 0 : `binary to run for each instance of the test (this same binary by default); cannot be used
169 0 : with --run-dir or --compare`)
170 0 :
171 0 : // The following options may be used for split-version metamorphic testing.
172 0 : // To perform split-version testing, the client runs the metamorphic tests
173 0 : // on an earlier Pebble SHA passing the `--keep` flag. The client then
174 0 : // switches to the later Pebble SHA, setting the below options to point to
175 0 : // the `ops` file and one of the previous run's data directories.
176 0 :
177 0 : flag.StringVar(&r.PreviousOps, "previous-ops", "",
178 0 : "path to an ops file, used to prepopulate the set of keys operations draw from")
179 0 :
180 0 : flag.StringVar(&r.InitialStatePath, "initial-state", "",
181 0 : "path to a database's data directory, used to prepopulate the test run's databases")
182 0 :
183 0 : flag.StringVar(&r.InitialStateDesc, "initial-state-desc", "",
184 0 : `a human-readable description of the initial database state.
185 0 : If set this parameter is written to the OPTIONS to aid in
186 0 : debugging. It's intended to describe the lineage of a
187 0 : database's state, including sufficient information for
188 0 : reproduction (eg, SHA, prng seed, etc).`)
189 0 : return r
190 : }
191 :
192 : // InitRunOnceFlags initializes the flags that are used for a single run of the
193 : // metamorphic test.
194 1 : func InitRunOnceFlags() *RunOnceFlags {
195 1 : return initRunOnceFlags(initCommonFlags())
196 1 : }
197 :
198 : // InitAllFlags initializes all metamorphic test flags: those used for a
199 : // single run, and those used for a "top level" run.
200 0 : func InitAllFlags() (*RunOnceFlags, *RunFlags) {
201 0 : c := initCommonFlags()
202 0 : return initRunOnceFlags(c), initRunFlags(c)
203 0 : }
204 :
205 : // MakeRunOnceOptions constructs RunOnceOptions based on the flags.
206 1 : func (ro *RunOnceFlags) MakeRunOnceOptions() []metamorphic.RunOnceOption {
207 1 : onceOpts := []metamorphic.RunOnceOption{
208 1 : metamorphic.MaxThreads(ro.MaxThreads),
209 1 : metamorphic.OpTimeout(ro.OpTimeout),
210 1 : }
211 1 : if ro.Keep {
212 0 : onceOpts = append(onceOpts, metamorphic.KeepData{})
213 0 : }
214 1 : if ro.FailRE != "" {
215 0 : onceOpts = append(onceOpts, metamorphic.FailOnMatch{Regexp: regexp.MustCompile(ro.FailRE)})
216 0 : }
217 1 : if ro.ErrorRate > 0 {
218 0 : onceOpts = append(onceOpts, metamorphic.InjectErrorsRate(ro.ErrorRate))
219 0 : }
220 1 : if ro.NumInstances > 1 {
221 1 : onceOpts = append(onceOpts, metamorphic.MultiInstance(ro.NumInstances))
222 1 : }
223 1 : return onceOpts
224 : }
225 :
226 : // ParseCompare parses the value of the compare flag, in format
227 : // "test-root-dir/{run1,run2,...}". Exits if the value is not valid.
228 : //
229 : // Returns the common test root dir (e.g. "test-root-dir") and a list of
230 : // subdirectories (e.g. {"run1", "run2"}).
231 0 : func (ro *RunOnceFlags) ParseCompare() (testRootDir string, runSubdirs []string) {
232 0 : testRootDir, runSubdirs, ok := ro.tryParseCompare()
233 0 : if !ok {
234 0 : fmt.Fprintf(os.Stderr, `cannot parse compare flag value %q; format is "test-root-dir/{run1,run2,..}"`, ro.Compare)
235 0 : os.Exit(1)
236 0 : }
237 0 : return testRootDir, runSubdirs
238 : }
239 :
240 0 : func (ro *RunOnceFlags) tryParseCompare() (testRootDir string, runSubdirs []string, ok bool) {
241 0 : value := ro.Compare
242 0 : brace := strings.Index(value, "{")
243 0 : if brace == -1 || !strings.HasSuffix(value, "}") {
244 0 : return "", nil, false
245 0 : }
246 :
247 0 : testRootDir = value[:brace]
248 0 : runSubdirs = strings.Split(value[brace+1:len(value)-1], ",")
249 0 : if len(runSubdirs) < 2 {
250 0 : return "", nil, false
251 0 : }
252 0 : return testRootDir, runSubdirs, true
253 : }
254 :
255 : // MakeRunOptions constructs RunOptions based on the flags.
256 0 : func (r *RunFlags) MakeRunOptions() []metamorphic.RunOption {
257 0 : opts := []metamorphic.RunOption{
258 0 : metamorphic.Seed(r.Seed),
259 0 : metamorphic.OpCount(r.Ops.Static),
260 0 : metamorphic.MaxThreads(r.MaxThreads),
261 0 : metamorphic.OpTimeout(r.OpTimeout),
262 0 : }
263 0 : if r.Keep {
264 0 : opts = append(opts, metamorphic.KeepData{})
265 0 : }
266 0 : if r.FailRE != "" {
267 0 : opts = append(opts, metamorphic.FailOnMatch{Regexp: regexp.MustCompile(r.FailRE)})
268 0 : }
269 0 : if r.ErrorRate > 0 {
270 0 : opts = append(opts, metamorphic.InjectErrorsRate(r.ErrorRate))
271 0 : }
272 0 : if r.TraceFile != "" {
273 0 : opts = append(opts, metamorphic.RuntimeTrace(r.TraceFile))
274 0 : }
275 0 : if r.PreviousOps != "" {
276 0 : opts = append(opts, metamorphic.ExtendPreviousRun(r.PreviousOps, r.InitialStatePath, r.InitialStateDesc))
277 0 : }
278 0 : if r.NumInstances > 1 {
279 0 : opts = append(opts, metamorphic.MultiInstance(r.NumInstances))
280 0 : }
281 :
282 : // If the filesystem type was forced, all tests will use that value.
283 0 : switch r.FS {
284 0 : case "", "rand", "default":
285 : // No-op. Use the generated value for the filesystem.
286 0 : case "disk":
287 0 : opts = append(opts, metamorphic.UseDisk)
288 0 : case "mem":
289 0 : opts = append(opts, metamorphic.UseInMemory)
290 0 : default:
291 0 : panic(fmt.Sprintf("unknown forced filesystem type: %q", r.FS))
292 : }
293 0 : if r.InnerBinary != "" {
294 0 : opts = append(opts, metamorphic.InnerBinary(r.InnerBinary))
295 0 : }
296 0 : return opts
297 : }
|
