Line data Source code
1 : // Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
2 : // of this source code is governed by a BSD-style license that can be found in
3 : // the LICENSE file.
4 :
5 : // Package metaflags defines command-line flags for the metamorphic tests and
6 : // provides functionality to construct the respective
7 : // metamorphic.RunOptions/RunOnceOptions.
8 : package metaflags
9 :
10 : import (
11 : "flag"
12 : "fmt"
13 : "math"
14 : "os"
15 : "regexp"
16 : "strings"
17 :
18 : "github.com/cockroachdb/pebble/internal/randvar"
19 : "github.com/cockroachdb/pebble/metamorphic"
20 : )
21 :
22 : // CommonFlags contains flags that apply to both metamorphic.Run and
23 : // metamorphic.RunOnce/Compare.
24 : type CommonFlags struct {
25 : // Dir is the directory storing test state. See "dir" flag below.
26 : Dir string
27 : // Seed for generation of random operations. See "seed" flag below.
28 : Seed uint64
29 : // ErrorRate is the rate of injected filesystem errors. See "error-rate" flag
30 : // below.
31 : ErrorRate float64
32 : // FailRE causes the test to fail if the output matches this regex. See "fail"
33 : // flag below.
34 : FailRE string
35 : // Keep determines if the DB directory is kept on successful runs. See "keep"
36 : // flag below.
37 : Keep bool
38 : // MaxThreads used by a single run. See "max-threads" flag below.
39 : MaxThreads int
40 : // NumInstances is the number of Pebble instances to create in one run. See
41 : // "num-instances" flag below.
42 : NumInstances int
43 : }
44 :
45 1 : func initCommonFlags() *CommonFlags {
46 1 : c := &CommonFlags{}
47 1 : flag.StringVar(&c.Dir, "dir", "_meta",
48 1 : "the directory storing test state")
49 1 :
50 1 : flag.Uint64Var(&c.Seed, "seed", 0,
51 1 : "a pseudorandom number generator seed")
52 1 :
53 1 : // TODO: default error rate to a non-zero value. Currently, retrying is
54 1 : // non-deterministic because of the Ierator.*WithLimit() methods since
55 1 : // they may say that the Iterator is not valid, but be positioned at a
56 1 : // certain key that can be returned in the future if the limit is changed.
57 1 : // Since that key is hidden from clients of Iterator, the retryableIter
58 1 : // using SeekGE will not necessarily position the Iterator that saw an
59 1 : // injected error at the same place as an Iterator that did not see that
60 1 : // error.
61 1 : flag.Float64Var(&c.ErrorRate, "error-rate", 0.0,
62 1 : "rate of errors injected into filesystem operations (0 ≤ r < 1)")
63 1 :
64 1 : flag.StringVar(&c.FailRE, "fail", "",
65 1 : "fail the test if the supplied regular expression matches the output")
66 1 :
67 1 : flag.BoolVar(&c.Keep, "keep", false,
68 1 : "keep the DB directory even on successful runs")
69 1 :
70 1 : flag.IntVar(&c.MaxThreads, "max-threads", math.MaxInt,
71 1 : "limit execution of a single run to the provided number of threads; must be ≥ 1")
72 1 :
73 1 : flag.IntVar(&c.NumInstances, "num-instances", 1, "number of pebble instances to create (default: 1)")
74 1 :
75 1 : return c
76 1 : }
77 :
78 : // RunOnceFlags contains flags that apply only to metamorphic.RunOnce/Compare.
79 : type RunOnceFlags struct {
80 : *CommonFlags
81 : // RunDir applies to metamorphic.RunOnce and contains the specific
82 : // configuration of the run. See "run-dir" flag below.
83 : RunDir string
84 : // Compare applies to metamorphic.Compare. See "compare" flag below.
85 : Compare string
86 : // TryToReduce enables a mode where we try to find a minimal subset of
87 : // operations that reproduce a problem during a test run (e.g. panic or
88 : // internal error).
89 : TryToReduce bool
90 : }
91 :
92 1 : func initRunOnceFlags(c *CommonFlags) *RunOnceFlags {
93 1 : ro := &RunOnceFlags{CommonFlags: c}
94 1 : flag.StringVar(&ro.RunDir, "run-dir", "",
95 1 : `directory containing the specific configuration to (re-)run (used for post-mortem debugging).
96 1 : Example: --run-dir _meta/231220-164251.3552792807512/random-003`)
97 1 :
98 1 : flag.StringVar(&ro.Compare, "compare", "",
99 1 : `runs to compare, in the format _meta/test-root-dir/{run1,run2,...}. The result
100 1 : of each run is compared with the result of the first run.
101 1 : Example, --compare '_meta/231220-164251.3552792807512/{standard-000,random-025}'`)
102 1 :
103 1 : flag.BoolVar(&ro.TryToReduce, "try-to-reduce", false,
104 1 : `if set, we will try to reduce the number of operations that cause a failure. The
105 1 : verbose flag should be used with this flag.`)
106 1 :
107 1 : return ro
108 1 : }
109 :
110 : // RunFlags contains flags that apply only to metamorphic.Run.
111 : type RunFlags struct {
112 : *CommonFlags
113 : // FS controls the type of filesystems to use. See "fs" flag below.
114 : FS string
115 : // TraceFile for execution tracing. See "trace-file" flag below.
116 : TraceFile string
117 : // Ops describes how the total number of operations is generated. See "ops" flags below.
118 : Ops randvar.Flag
119 : // InnerBinary is the binary to invoke for a single run. See "inner-binary"
120 : // flag below.
121 : InnerBinary string
122 : // PreviousOps is the path to the ops file of a previous run. See the
123 : // "previous-ops" flag below.
124 : PreviousOps string
125 : // InitialStatePath is the path to a database data directory from a previous
126 : // run. See the "initial-state" flag below.
127 : InitialStatePath string
128 : // InitialStateDesc is a human-readable description of the initial database
129 : // state. See "initial-state-desc" flag below.
130 : InitialStateDesc string
131 : }
132 :
133 1 : func initRunFlags(c *CommonFlags) *RunFlags {
134 1 : r := &RunFlags{CommonFlags: c}
135 1 : flag.StringVar(&r.FS, "fs", "rand",
136 1 : `force the tests to use either memory or disk-backed filesystems (valid: "mem", "disk", "rand")`)
137 1 :
138 1 : flag.StringVar(&r.TraceFile, "trace-file", "",
139 1 : "write an execution trace to `<run-dir>/file`")
140 1 :
141 1 : if err := r.Ops.Set("uniform:5000-10000"); err != nil {
142 0 : panic(err)
143 : }
144 1 : flag.Var(&r.Ops, "ops", "uniform:5000-10000")
145 1 :
146 1 : flag.StringVar(&r.InnerBinary, "inner-binary", "",
147 1 : `binary to run for each instance of the test (this same binary by default); cannot be used
148 1 : with --run-dir or --compare`)
149 1 :
150 1 : // The following options may be used for split-version metamorphic testing.
151 1 : // To perform split-version testing, the client runs the metamorphic tests
152 1 : // on an earlier Pebble SHA passing the `--keep` flag. The client then
153 1 : // switches to the later Pebble SHA, setting the below options to point to
154 1 : // the `ops` file and one of the previous run's data directories.
155 1 :
156 1 : flag.StringVar(&r.PreviousOps, "previous-ops", "",
157 1 : "path to an ops file, used to prepopulate the set of keys operations draw from")
158 1 :
159 1 : flag.StringVar(&r.InitialStatePath, "initial-state", "",
160 1 : "path to a database's data directory, used to prepopulate the test run's databases")
161 1 :
162 1 : flag.StringVar(&r.InitialStateDesc, "initial-state-desc", "",
163 1 : `a human-readable description of the initial database state.
164 1 : If set this parameter is written to the OPTIONS to aid in
165 1 : debugging. It's intended to describe the lineage of a
166 1 : database's state, including sufficient information for
167 1 : reproduction (eg, SHA, prng seed, etc).`)
168 1 : return r
169 : }
170 :
171 : // InitRunOnceFlags initializes the flags that are used for a single run of the
172 : // metamorphic test.
173 0 : func InitRunOnceFlags() *RunOnceFlags {
174 0 : return initRunOnceFlags(initCommonFlags())
175 0 : }
176 :
177 : // InitAllFlags initializes all metamorphic test flags: those used for a
178 : // single run, and those used for a "top level" run.
179 1 : func InitAllFlags() (*RunOnceFlags, *RunFlags) {
180 1 : c := initCommonFlags()
181 1 : return initRunOnceFlags(c), initRunFlags(c)
182 1 : }
183 :
184 : // MakeRunOnceOptions constructs RunOnceOptions based on the flags.
185 0 : func (ro *RunOnceFlags) MakeRunOnceOptions() []metamorphic.RunOnceOption {
186 0 : onceOpts := []metamorphic.RunOnceOption{
187 0 : metamorphic.MaxThreads(ro.MaxThreads),
188 0 : }
189 0 : if ro.Keep {
190 0 : onceOpts = append(onceOpts, metamorphic.KeepData{})
191 0 : }
192 0 : if ro.FailRE != "" {
193 0 : onceOpts = append(onceOpts, metamorphic.FailOnMatch{Regexp: regexp.MustCompile(ro.FailRE)})
194 0 : }
195 0 : if ro.ErrorRate > 0 {
196 0 : onceOpts = append(onceOpts, metamorphic.InjectErrorsRate(ro.ErrorRate))
197 0 : }
198 0 : if ro.NumInstances > 1 {
199 0 : onceOpts = append(onceOpts, metamorphic.MultiInstance(ro.NumInstances))
200 0 : }
201 0 : return onceOpts
202 : }
203 :
204 : // ParseCompare parses the value of the compare flag, in format
205 : // "test-root-dir/{run1,run2,...}". Exits if the value is not valid.
206 : //
207 : // Returns the common test root dir (e.g. "test-root-dir") and a list of
208 : // subdirectories (e.g. {"run1", "run2"}).
209 0 : func (ro *RunOnceFlags) ParseCompare() (testRootDir string, runSubdirs []string) {
210 0 : testRootDir, runSubdirs, ok := ro.tryParseCompare()
211 0 : if !ok {
212 0 : fmt.Fprintf(os.Stderr, `cannot parse compare flag value %q; format is "test-root-dir/{run1,run2,..}"`, ro.Compare)
213 0 : os.Exit(1)
214 0 : }
215 0 : return testRootDir, runSubdirs
216 : }
217 :
218 0 : func (ro *RunOnceFlags) tryParseCompare() (testRootDir string, runSubdirs []string, ok bool) {
219 0 : value := ro.Compare
220 0 : brace := strings.Index(value, "{")
221 0 : if brace == -1 || !strings.HasSuffix(value, "}") {
222 0 : return "", nil, false
223 0 : }
224 :
225 0 : testRootDir = value[:brace]
226 0 : runSubdirs = strings.Split(value[brace+1:len(value)-1], ",")
227 0 : if len(runSubdirs) < 2 {
228 0 : return "", nil, false
229 0 : }
230 0 : return testRootDir, runSubdirs, true
231 : }
232 :
233 : // MakeRunOptions constructs RunOptions based on the flags.
234 1 : func (r *RunFlags) MakeRunOptions() []metamorphic.RunOption {
235 1 : opts := []metamorphic.RunOption{
236 1 : metamorphic.Seed(r.Seed),
237 1 : metamorphic.OpCount(r.Ops.Static),
238 1 : metamorphic.MaxThreads(r.MaxThreads),
239 1 : }
240 1 : if r.Keep {
241 0 : opts = append(opts, metamorphic.KeepData{})
242 0 : }
243 1 : if r.FailRE != "" {
244 0 : opts = append(opts, metamorphic.FailOnMatch{Regexp: regexp.MustCompile(r.FailRE)})
245 0 : }
246 1 : if r.ErrorRate > 0 {
247 0 : opts = append(opts, metamorphic.InjectErrorsRate(r.ErrorRate))
248 0 : }
249 1 : if r.TraceFile != "" {
250 0 : opts = append(opts, metamorphic.RuntimeTrace(r.TraceFile))
251 0 : }
252 1 : if r.PreviousOps != "" {
253 0 : opts = append(opts, metamorphic.ExtendPreviousRun(r.PreviousOps, r.InitialStatePath, r.InitialStateDesc))
254 0 : }
255 1 : if r.NumInstances > 1 {
256 0 : opts = append(opts, metamorphic.MultiInstance(r.NumInstances))
257 0 : }
258 :
259 : // If the filesystem type was forced, all tests will use that value.
260 1 : switch r.FS {
261 1 : case "", "rand", "default":
262 : // No-op. Use the generated value for the filesystem.
263 0 : case "disk":
264 0 : opts = append(opts, metamorphic.UseDisk)
265 0 : case "mem":
266 0 : opts = append(opts, metamorphic.UseInMemory)
267 0 : default:
268 0 : panic(fmt.Sprintf("unknown forced filesystem type: %q", r.FS))
269 : }
270 1 : if r.InnerBinary != "" {
271 0 : opts = append(opts, metamorphic.InnerBinary(r.InnerBinary))
272 0 : }
273 1 : return opts
274 : }
|
