Line data Source code
1 : // Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
2 : // of this source code is governed by a BSD-style license that can be found in
3 : // the LICENSE file.
4 :
5 : // Package metaflags defines command-line flags for the metamorphic tests and
6 : // provides functionality to construct the respective
7 : // metamorphic.RunOptions/RunOnceOptions.
8 : package metaflags
9 :
10 : import (
11 : "flag"
12 : "fmt"
13 : "math"
14 : "os"
15 : "regexp"
16 : "strings"
17 : "time"
18 :
19 : "github.com/cockroachdb/pebble/internal/invariants"
20 : "github.com/cockroachdb/pebble/internal/randvar"
21 : "github.com/cockroachdb/pebble/metamorphic"
22 : )
23 :
24 : // CommonFlags contains flags that apply to both metamorphic.Run and
25 : // metamorphic.RunOnce/Compare.
26 : type CommonFlags struct {
27 : // Dir is the directory storing test state. See "dir" flag below.
28 : Dir string
29 : // Seed for generation of random operations. See "seed" flag below.
30 : Seed uint64
31 : // ErrorRate is the rate of injected filesystem errors. See "error-rate" flag
32 : // below.
33 : ErrorRate float64
34 : // FailRE causes the test to fail if the output matches this regex. See "fail"
35 : // flag below.
36 : FailRE string
37 : // Keep determines if the DB directory is kept on successful runs. See "keep"
38 : // flag below.
39 : Keep bool
40 : // MaxThreads used by a single run. See "max-threads" flag below.
41 : MaxThreads int
42 : // NumInstances is the number of Pebble instances to create in one run. See
43 : // "num-instances" flag below.
44 : NumInstances int
45 : // OpTimeout is a per-operation timeout.
46 : OpTimeout time.Duration
47 : }
48 :
49 2 : func initCommonFlags() *CommonFlags {
50 2 : c := &CommonFlags{}
51 2 : flag.StringVar(&c.Dir, "dir", "_meta",
52 2 : "the directory storing test state")
53 2 :
54 2 : flag.Uint64Var(&c.Seed, "seed", 0,
55 2 : "a pseudorandom number generator seed")
56 2 :
57 2 : // TODO: default error rate to a non-zero value. Currently, retrying is
58 2 : // non-deterministic because of the Ierator.*WithLimit() methods since
59 2 : // they may say that the Iterator is not valid, but be positioned at a
60 2 : // certain key that can be returned in the future if the limit is changed.
61 2 : // Since that key is hidden from clients of Iterator, the retryableIter
62 2 : // using SeekGE will not necessarily position the Iterator that saw an
63 2 : // injected error at the same place as an Iterator that did not see that
64 2 : // error.
65 2 : flag.Float64Var(&c.ErrorRate, "error-rate", 0.0,
66 2 : "rate of errors injected into filesystem operations (0 ≤ r < 1)")
67 2 :
68 2 : flag.StringVar(&c.FailRE, "fail", "",
69 2 : "fail the test if the supplied regular expression matches the output")
70 2 :
71 2 : flag.BoolVar(&c.Keep, "keep", false,
72 2 : "keep the DB directory even on successful runs")
73 2 :
74 2 : flag.IntVar(&c.MaxThreads, "max-threads", math.MaxInt,
75 2 : "limit execution of a single run to the provided number of threads; must be ≥ 1")
76 2 :
77 2 : flag.IntVar(&c.NumInstances, "num-instances", 1, "number of pebble instances to create (default: 1)")
78 2 :
79 2 : defaultOpTimeout := 2 * time.Minute
80 2 : if invariants.RaceEnabled {
81 0 : defaultOpTimeout *= 5
82 0 : }
83 2 : flag.DurationVar(&c.OpTimeout, "op-timeout", defaultOpTimeout, "per-op timeout")
84 2 :
85 2 : return c
86 : }
87 :
88 : // RunOnceFlags contains flags that apply only to metamorphic.RunOnce/Compare.
89 : type RunOnceFlags struct {
90 : *CommonFlags
91 : // RunDir applies to metamorphic.RunOnce and contains the specific
92 : // configuration of the run. See "run-dir" flag below.
93 : RunDir string
94 : // Compare applies to metamorphic.Compare. See "compare" flag below.
95 : Compare string
96 : // TryToReduce enables a mode where we try to find a minimal subset of
97 : // operations that reproduce a problem during a test run (e.g. panic or
98 : // internal error).
99 : TryToReduce bool
100 : // ReduceAttempts is the number of attempts to reduce (for each op removal
101 : // probability).
102 : ReduceAttempts int
103 : }
104 :
105 2 : func initRunOnceFlags(c *CommonFlags) *RunOnceFlags {
106 2 : ro := &RunOnceFlags{CommonFlags: c}
107 2 : flag.StringVar(&ro.RunDir, "run-dir", "",
108 2 : `directory containing the specific configuration to (re-)run (used for post-mortem debugging).
109 2 : Example: --run-dir _meta/231220-164251.3552792807512/random-003`)
110 2 :
111 2 : flag.StringVar(&ro.Compare, "compare", "",
112 2 : `runs to compare, in the format _meta/test-root-dir/{run1,run2,...}. The result
113 2 : of each run is compared with the result of the first run.
114 2 : Example, --compare '_meta/231220-164251.3552792807512/{standard-000,random-025}'`)
115 2 :
116 2 : flag.BoolVar(&ro.TryToReduce, "try-to-reduce", false,
117 2 : `if set, we will try to reduce the number of operations that cause a failure. The
118 2 : verbose flag should be used with this flag.`)
119 2 :
120 2 : flag.IntVar(&ro.ReduceAttempts, "reduce-attempts", 100,
121 2 : `the number of attempts to reduce, for each probability; only used with --try-to-reduce.`)
122 2 :
123 2 : return ro
124 2 : }
125 :
126 : // RunFlags contains flags that apply only to metamorphic.Run.
127 : type RunFlags struct {
128 : *CommonFlags
129 : // FS controls the type of filesystems to use. See "fs" flag below.
130 : FS string
131 : // TraceFile for execution tracing. See "trace-file" flag below.
132 : TraceFile string
133 : // Ops describes how the total number of operations is generated. See "ops" flags below.
134 : Ops randvar.Flag
135 : // InnerBinary is the binary to invoke for a single run. See "inner-binary"
136 : // flag below.
137 : InnerBinary string
138 : // PreviousOps is the path to the ops file of a previous run. See the
139 : // "previous-ops" flag below.
140 : PreviousOps string
141 : // InitialStatePath is the path to a database data directory from a previous
142 : // run. See the "initial-state" flag below.
143 : InitialStatePath string
144 : // InitialStateDesc is a human-readable description of the initial database
145 : // state. See "initial-state-desc" flag below.
146 : InitialStateDesc string
147 : }
148 :
149 1 : func initRunFlags(c *CommonFlags) *RunFlags {
150 1 : r := &RunFlags{CommonFlags: c}
151 1 : flag.StringVar(&r.FS, "fs", "rand",
152 1 : `force the tests to use either memory or disk-backed filesystems (valid: "mem", "disk", "rand")`)
153 1 :
154 1 : flag.StringVar(&r.TraceFile, "trace-file", "",
155 1 : "write an execution trace to `<run-dir>/file`")
156 1 :
157 1 : ops := "uniform:5000-10000"
158 1 : if invariants.RaceEnabled {
159 0 : // Reduce the size of the test under race (to avoid timeouts).
160 0 : ops = "uniform:1000-2000"
161 0 : }
162 1 : if err := r.Ops.Set(ops); err != nil {
163 0 : panic(err)
164 : }
165 1 : flag.Var(&r.Ops, "ops", "uniform:min-max")
166 1 :
167 1 : flag.StringVar(&r.InnerBinary, "inner-binary", "",
168 1 : `binary to run for each instance of the test (this same binary by default); cannot be used
169 1 : with --run-dir or --compare`)
170 1 :
171 1 : // The following options may be used for split-version metamorphic testing.
172 1 : // To perform split-version testing, the client runs the metamorphic tests
173 1 : // on an earlier Pebble SHA passing the `--keep` flag. The client then
174 1 : // switches to the later Pebble SHA, setting the below options to point to
175 1 : // the `ops` file and one of the previous run's data directories.
176 1 :
177 1 : flag.StringVar(&r.PreviousOps, "previous-ops", "",
178 1 : "path to an ops file, used to prepopulate the set of keys operations draw from")
179 1 :
180 1 : flag.StringVar(&r.InitialStatePath, "initial-state", "",
181 1 : "path to a database's data directory, used to prepopulate the test run's databases")
182 1 :
183 1 : flag.StringVar(&r.InitialStateDesc, "initial-state-desc", "",
184 1 : `a human-readable description of the initial database state.
185 1 : If set this parameter is written to the OPTIONS to aid in
186 1 : debugging. It's intended to describe the lineage of a
187 1 : database's state, including sufficient information for
188 1 : reproduction (eg, SHA, prng seed, etc).`)
189 1 : return r
190 : }
191 :
192 : // InitRunOnceFlags initializes the flags that are used for a single run of the
193 : // metamorphic test.
194 1 : func InitRunOnceFlags() *RunOnceFlags {
195 1 : return initRunOnceFlags(initCommonFlags())
196 1 : }
197 :
198 : // InitAllFlags initializes all metamorphic test flags: those used for a
199 : // single run, and those used for a "top level" run.
200 1 : func InitAllFlags() (*RunOnceFlags, *RunFlags) {
201 1 : c := initCommonFlags()
202 1 : return initRunOnceFlags(c), initRunFlags(c)
203 1 : }
204 :
205 : // MakeRunOnceOptions constructs RunOnceOptions based on the flags.
206 1 : func (ro *RunOnceFlags) MakeRunOnceOptions() []metamorphic.RunOnceOption {
207 1 : onceOpts := []metamorphic.RunOnceOption{
208 1 : metamorphic.MaxThreads(ro.MaxThreads),
209 1 : metamorphic.OpTimeout(ro.OpTimeout),
210 1 : }
211 1 : if ro.Keep {
212 0 : onceOpts = append(onceOpts, metamorphic.KeepData{})
213 0 : }
214 1 : if ro.FailRE != "" {
215 0 : onceOpts = append(onceOpts, metamorphic.FailOnMatch{Regexp: regexp.MustCompile(ro.FailRE)})
216 0 : }
217 1 : if ro.ErrorRate > 0 {
218 0 : onceOpts = append(onceOpts, metamorphic.InjectErrorsRate(ro.ErrorRate))
219 0 : }
220 1 : if ro.NumInstances > 1 {
221 1 : onceOpts = append(onceOpts, metamorphic.MultiInstance(ro.NumInstances))
222 1 : }
223 1 : return onceOpts
224 : }
225 :
226 : // ParseCompare parses the value of the compare flag, in format
227 : // "test-root-dir/{run1,run2,...}". Exits if the value is not valid.
228 : //
229 : // Returns the common test root dir (e.g. "test-root-dir") and a list of
230 : // subdirectories (e.g. {"run1", "run2"}).
231 0 : func (ro *RunOnceFlags) ParseCompare() (testRootDir string, runSubdirs []string) {
232 0 : testRootDir, runSubdirs, ok := ro.tryParseCompare()
233 0 : if !ok {
234 0 : fmt.Fprintf(os.Stderr, `cannot parse compare flag value %q; format is "test-root-dir/{run1,run2,..}"`, ro.Compare)
235 0 : os.Exit(1)
236 0 : }
237 0 : return testRootDir, runSubdirs
238 : }
239 :
240 0 : func (ro *RunOnceFlags) tryParseCompare() (testRootDir string, runSubdirs []string, ok bool) {
241 0 : value := ro.Compare
242 0 : brace := strings.Index(value, "{")
243 0 : if brace == -1 || !strings.HasSuffix(value, "}") {
244 0 : return "", nil, false
245 0 : }
246 :
247 0 : testRootDir = value[:brace]
248 0 : runSubdirs = strings.Split(value[brace+1:len(value)-1], ",")
249 0 : if len(runSubdirs) < 2 {
250 0 : return "", nil, false
251 0 : }
252 0 : return testRootDir, runSubdirs, true
253 : }
254 :
255 : // MakeRunOptions constructs RunOptions based on the flags.
256 1 : func (r *RunFlags) MakeRunOptions() []metamorphic.RunOption {
257 1 : opts := []metamorphic.RunOption{
258 1 : metamorphic.Seed(r.Seed),
259 1 : metamorphic.OpCount(r.Ops.Static),
260 1 : metamorphic.MaxThreads(r.MaxThreads),
261 1 : metamorphic.OpTimeout(r.OpTimeout),
262 1 : }
263 1 : if r.Keep {
264 0 : opts = append(opts, metamorphic.KeepData{})
265 0 : }
266 1 : if r.FailRE != "" {
267 0 : opts = append(opts, metamorphic.FailOnMatch{Regexp: regexp.MustCompile(r.FailRE)})
268 0 : }
269 1 : if r.ErrorRate > 0 {
270 0 : opts = append(opts, metamorphic.InjectErrorsRate(r.ErrorRate))
271 0 : }
272 1 : if r.TraceFile != "" {
273 0 : opts = append(opts, metamorphic.RuntimeTrace(r.TraceFile))
274 0 : }
275 1 : if r.PreviousOps != "" {
276 0 : opts = append(opts, metamorphic.ExtendPreviousRun(r.PreviousOps, r.InitialStatePath, r.InitialStateDesc))
277 0 : }
278 1 : if r.NumInstances > 1 {
279 0 : opts = append(opts, metamorphic.MultiInstance(r.NumInstances))
280 0 : }
281 :
282 : // If the filesystem type was forced, all tests will use that value.
283 1 : switch r.FS {
284 1 : case "", "rand", "default":
285 : // No-op. Use the generated value for the filesystem.
286 0 : case "disk":
287 0 : opts = append(opts, metamorphic.UseDisk)
288 0 : case "mem":
289 0 : opts = append(opts, metamorphic.UseInMemory)
290 0 : default:
291 0 : panic(fmt.Sprintf("unknown forced filesystem type: %q", r.FS))
292 : }
293 1 : if r.InnerBinary != "" {
294 0 : opts = append(opts, metamorphic.InnerBinary(r.InnerBinary))
295 0 : }
296 1 : return opts
297 : }
|
