Line data Source code
1 : // Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
2 : // of this source code is governed by a BSD-style license that can be found in
3 : // the LICENSE file.
4 :
5 : package tool
6 :
7 : import (
8 : "bufio"
9 : "context"
10 : "fmt"
11 : "io"
12 : "math/rand"
13 : "strings"
14 : "text/tabwriter"
15 :
16 : "github.com/cockroachdb/errors"
17 : "github.com/cockroachdb/errors/oserror"
18 : "github.com/cockroachdb/pebble"
19 : "github.com/cockroachdb/pebble/internal/base"
20 : "github.com/cockroachdb/pebble/internal/humanize"
21 : "github.com/cockroachdb/pebble/internal/manifest"
22 : "github.com/cockroachdb/pebble/internal/sstableinternal"
23 : "github.com/cockroachdb/pebble/objstorage"
24 : "github.com/cockroachdb/pebble/objstorage/objstorageprovider"
25 : "github.com/cockroachdb/pebble/record"
26 : "github.com/cockroachdb/pebble/sstable"
27 : "github.com/cockroachdb/pebble/tool/logs"
28 : "github.com/cockroachdb/pebble/vfs"
29 : "github.com/spf13/cobra"
30 : )
31 :
32 : // dbT implements db-level tools, including both configuration state and the
33 : // commands themselves.
34 : type dbT struct {
35 : Root *cobra.Command
36 : Check *cobra.Command
37 : Checkpoint *cobra.Command
38 : Get *cobra.Command
39 : Logs *cobra.Command
40 : LSM *cobra.Command
41 : Properties *cobra.Command
42 : Scan *cobra.Command
43 : Set *cobra.Command
44 : Space *cobra.Command
45 : IOBench *cobra.Command
46 : Excise *cobra.Command
47 :
48 : // Configuration.
49 : opts *pebble.Options
50 : comparers sstable.Comparers
51 : mergers sstable.Mergers
52 : openErrEnhancer func(error) error
53 : openOptions []OpenOption
54 : exciseSpanFn DBExciseSpanFn
55 :
56 : // Flags.
57 : comparerName string
58 : mergerName string
59 : fmtKey keyFormatter
60 : fmtValue valueFormatter
61 : start key
62 : end key
63 : count int64
64 : allLevels bool
65 : ioCount int
66 : ioParallelism int
67 : ioSizes string
68 : verbose bool
69 : bypassPrompt bool
70 : lsmURL bool
71 : }
72 :
73 : func newDB(
74 : opts *pebble.Options,
75 : comparers sstable.Comparers,
76 : mergers sstable.Mergers,
77 : openErrEnhancer func(error) error,
78 : openOptions []OpenOption,
79 : exciseSpanFn DBExciseSpanFn,
80 1 : ) *dbT {
81 1 : d := &dbT{
82 1 : opts: opts,
83 1 : comparers: comparers,
84 1 : mergers: mergers,
85 1 : openErrEnhancer: openErrEnhancer,
86 1 : openOptions: openOptions,
87 1 : exciseSpanFn: exciseSpanFn,
88 1 : }
89 1 : d.fmtKey.mustSet("quoted")
90 1 : d.fmtValue.mustSet("[%x]")
91 1 :
92 1 : d.Root = &cobra.Command{
93 1 : Use: "db",
94 1 : Short: "DB introspection tools",
95 1 : }
96 1 : d.Check = &cobra.Command{
97 1 : Use: "check <dir>",
98 1 : Short: "verify checksums and metadata",
99 1 : Long: `
100 1 : Verify sstable, manifest, and WAL checksums. Requires that the specified
101 1 : database not be in use by another process.
102 1 : `,
103 1 : Args: cobra.ExactArgs(1),
104 1 : Run: d.runCheck,
105 1 : }
106 1 : d.Checkpoint = &cobra.Command{
107 1 : Use: "checkpoint <src-dir> <dest-dir>",
108 1 : Short: "create a checkpoint",
109 1 : Long: `
110 1 : Creates a Pebble checkpoint in the specified destination directory. A checkpoint
111 1 : is a point-in-time snapshot of DB state. Requires that the specified
112 1 : database not be in use by another process.
113 1 : `,
114 1 : Args: cobra.ExactArgs(2),
115 1 : Run: d.runCheckpoint,
116 1 : }
117 1 : d.Get = &cobra.Command{
118 1 : Use: "get <dir> <key>",
119 1 : Short: "get value for a key",
120 1 : Long: `
121 1 : Gets a value for a key, if it exists in DB. Prints a "not found" error if key
122 1 : does not exist. Requires that the specified database not be in use by another
123 1 : process.
124 1 : `,
125 1 : Args: cobra.ExactArgs(2),
126 1 : Run: d.runGet,
127 1 : }
128 1 : d.Logs = logs.NewCmd()
129 1 : d.LSM = &cobra.Command{
130 1 : Use: "lsm <dir>",
131 1 : Short: "print LSM structure",
132 1 : Long: `
133 1 : Print the structure of the LSM tree. Requires that the specified database not
134 1 : be in use by another process.
135 1 : `,
136 1 : Args: cobra.ExactArgs(1),
137 1 : Run: d.runLSM,
138 1 : }
139 1 : d.Properties = &cobra.Command{
140 1 : Use: "properties <dir>",
141 1 : Short: "print aggregated sstable properties",
142 1 : Long: `
143 1 : Print SSTable properties, aggregated per level of the LSM.
144 1 : `,
145 1 : Args: cobra.ExactArgs(1),
146 1 : Run: d.runProperties,
147 1 : }
148 1 : d.Scan = &cobra.Command{
149 1 : Use: "scan <dir>",
150 1 : Short: "print db records",
151 1 : Long: `
152 1 : Print the records in the DB. Requires that the specified database not be in use
153 1 : by another process.
154 1 : `,
155 1 : Args: cobra.ExactArgs(1),
156 1 : Run: d.runScan,
157 1 : }
158 1 : d.Set = &cobra.Command{
159 1 : Use: "set <dir> <key> <value>",
160 1 : Short: "set a value for a key",
161 1 : Long: `
162 1 : Adds a new key/value to the DB. Requires that the specified database
163 1 : not be in use by another process.
164 1 : `,
165 1 : Args: cobra.ExactArgs(3),
166 1 : Run: d.runSet,
167 1 : }
168 1 : d.Space = &cobra.Command{
169 1 : Use: "space <dir>",
170 1 : Short: "print filesystem space used",
171 1 : Long: `
172 1 : Print the estimated filesystem space usage for the inclusive-inclusive range
173 1 : specified by --start and --end. Requires that the specified database not be in
174 1 : use by another process.
175 1 : `,
176 1 : Args: cobra.ExactArgs(1),
177 1 : Run: d.runSpace,
178 1 : }
179 1 : d.Excise = &cobra.Command{
180 1 : Use: "excise <dir>",
181 1 : Short: "excise a key range",
182 1 : Long: `
183 1 : Excise a key range, removing all SSTs inside the range and virtualizing any SSTs
184 1 : that partially overlap the range.
185 1 : `,
186 1 : Args: cobra.ExactArgs(1),
187 1 : Run: d.runExcise,
188 1 : }
189 1 : d.IOBench = &cobra.Command{
190 1 : Use: "io-bench <dir>",
191 1 : Short: "perform sstable IO benchmark",
192 1 : Long: `
193 1 : Run a random IO workload with various IO sizes against the sstables in the
194 1 : specified database.
195 1 : `,
196 1 : Args: cobra.ExactArgs(1),
197 1 : Run: d.runIOBench,
198 1 : }
199 1 :
200 1 : d.Root.AddCommand(d.Check, d.Checkpoint, d.Get, d.Logs, d.LSM, d.Properties, d.Scan, d.Set, d.Space, d.Excise, d.IOBench)
201 1 : d.Root.PersistentFlags().BoolVarP(&d.verbose, "verbose", "v", false, "verbose output")
202 1 :
203 1 : for _, cmd := range []*cobra.Command{d.Check, d.Checkpoint, d.Get, d.LSM, d.Properties, d.Scan, d.Set, d.Space, d.Excise} {
204 1 : cmd.Flags().StringVar(
205 1 : &d.comparerName, "comparer", "", "comparer name (use default if empty)")
206 1 : cmd.Flags().StringVar(
207 1 : &d.mergerName, "merger", "", "merger name (use default if empty)")
208 1 : }
209 :
210 1 : for _, cmd := range []*cobra.Command{d.Scan, d.Get} {
211 1 : cmd.Flags().Var(
212 1 : &d.fmtValue, "value", "value formatter")
213 1 : }
214 :
215 1 : d.LSM.Flags().BoolVar(
216 1 : &d.lsmURL, "url", false, "generate LSM viewer URL")
217 1 :
218 1 : d.Space.Flags().Var(
219 1 : &d.start, "start", "start key for the range")
220 1 : d.Space.Flags().Var(
221 1 : &d.end, "end", "inclusive end key for the range")
222 1 :
223 1 : d.Scan.Flags().Var(
224 1 : &d.fmtKey, "key", "key formatter")
225 1 : d.Scan.Flags().Var(
226 1 : &d.start, "start", "start key for the range")
227 1 : d.Scan.Flags().Var(
228 1 : &d.end, "end", "exclusive end key for the range")
229 1 : d.Scan.Flags().Int64Var(
230 1 : &d.count, "count", 0, "key count for scan (0 is unlimited)")
231 1 :
232 1 : d.Excise.Flags().Var(
233 1 : &d.start, "start", "start key for the excised range")
234 1 : d.Excise.Flags().Var(
235 1 : &d.end, "end", "exclusive end key for the excised range")
236 1 : d.Excise.Flags().BoolVar(
237 1 : &d.bypassPrompt, "yes", false, "bypass prompt")
238 1 :
239 1 : d.IOBench.Flags().BoolVar(
240 1 : &d.allLevels, "all-levels", false, "if set, benchmark all levels (default is only L5/L6)")
241 1 : d.IOBench.Flags().IntVar(
242 1 : &d.ioCount, "io-count", 10000, "number of IOs (per IO size) to benchmark")
243 1 : d.IOBench.Flags().IntVar(
244 1 : &d.ioParallelism, "io-parallelism", 16, "number of goroutines issuing IO")
245 1 : d.IOBench.Flags().StringVar(
246 1 : &d.ioSizes, "io-sizes-kb", "4,16,64,128,256,512,1024", "comma separated list of IO sizes in KB")
247 1 :
248 1 : return d
249 : }
250 :
251 1 : func (d *dbT) loadOptions(dir string) error {
252 1 : ls, err := d.opts.FS.List(dir)
253 1 : if err != nil || len(ls) == 0 {
254 1 : // NB: We don't return the error here as we prefer to return the error from
255 1 : // pebble.Open. Another way to put this is that a non-existent directory is
256 1 : // not a failure in loading the options.
257 1 : return nil
258 1 : }
259 :
260 1 : hooks := &pebble.ParseHooks{
261 1 : NewComparer: func(name string) (*pebble.Comparer, error) {
262 1 : if c := d.comparers[name]; c != nil {
263 1 : return c, nil
264 1 : }
265 0 : return nil, errors.Errorf("unknown comparer %q", errors.Safe(name))
266 : },
267 1 : NewMerger: func(name string) (*pebble.Merger, error) {
268 1 : if m := d.mergers[name]; m != nil {
269 1 : return m, nil
270 1 : }
271 0 : return nil, errors.Errorf("unknown merger %q", errors.Safe(name))
272 : },
273 0 : SkipUnknown: func(name, value string) bool {
274 0 : return true
275 0 : },
276 : }
277 :
278 : // TODO(peter): RocksDB sometimes leaves multiple OPTIONS files in
279 : // existence. We parse all of them as the comparer and merger shouldn't be
280 : // changing. We could parse only the first or the latest. Not clear if this
281 : // matters.
282 1 : var dbOpts pebble.Options
283 1 : for _, filename := range ls {
284 1 : ft, _, ok := base.ParseFilename(d.opts.FS, filename)
285 1 : if !ok {
286 1 : continue
287 : }
288 1 : switch ft {
289 1 : case base.FileTypeOptions:
290 1 : err := func() error {
291 1 : f, err := d.opts.FS.Open(d.opts.FS.PathJoin(dir, filename))
292 1 : if err != nil {
293 0 : return err
294 0 : }
295 1 : defer f.Close()
296 1 :
297 1 : data, err := io.ReadAll(f)
298 1 : if err != nil {
299 0 : return err
300 0 : }
301 :
302 1 : if err := dbOpts.Parse(string(data), hooks); err != nil {
303 1 : return err
304 1 : }
305 1 : return nil
306 : }()
307 1 : if err != nil {
308 1 : return err
309 1 : }
310 : }
311 : }
312 :
313 1 : if dbOpts.Comparer != nil {
314 1 : d.opts.Comparer = dbOpts.Comparer
315 1 : }
316 1 : if dbOpts.Merger != nil {
317 1 : d.opts.Merger = dbOpts.Merger
318 1 : }
319 1 : return nil
320 : }
321 :
322 : // OpenOption is an option that may be applied to the *pebble.Options before
323 : // calling pebble.Open.
324 : type OpenOption interface {
325 : Apply(dirname string, opts *pebble.Options)
326 : }
327 :
328 1 : func (d *dbT) openDB(dir string, openOptions ...OpenOption) (*pebble.DB, error) {
329 1 : db, err := d.openDBInternal(dir, openOptions...)
330 1 : if err != nil {
331 1 : if d.openErrEnhancer != nil {
332 1 : err = d.openErrEnhancer(err)
333 1 : }
334 1 : return nil, err
335 : }
336 1 : return db, nil
337 : }
338 :
339 1 : func (d *dbT) openDBInternal(dir string, openOptions ...OpenOption) (*pebble.DB, error) {
340 1 : if err := d.loadOptions(dir); err != nil {
341 1 : return nil, errors.Wrap(err, "error loading options")
342 1 : }
343 1 : if d.comparerName != "" {
344 1 : d.opts.Comparer = d.comparers[d.comparerName]
345 1 : if d.opts.Comparer == nil {
346 1 : return nil, errors.Errorf("unknown comparer %q", errors.Safe(d.comparerName))
347 1 : }
348 : }
349 1 : if d.mergerName != "" {
350 1 : d.opts.Merger = d.mergers[d.mergerName]
351 1 : if d.opts.Merger == nil {
352 1 : return nil, errors.Errorf("unknown merger %q", errors.Safe(d.mergerName))
353 1 : }
354 : }
355 1 : opts := *d.opts
356 1 : for _, opt := range openOptions {
357 1 : opt.Apply(dir, &opts)
358 1 : }
359 1 : for _, opt := range d.openOptions {
360 0 : opt.Apply(dir, &opts)
361 0 : }
362 1 : opts.Cache = pebble.NewCache(128 << 20 /* 128 MB */)
363 1 : defer opts.Cache.Unref()
364 1 : return pebble.Open(dir, &opts)
365 : }
366 :
367 1 : func (d *dbT) closeDB(stderr io.Writer, db *pebble.DB) {
368 1 : if err := db.Close(); err != nil {
369 0 : fmt.Fprintf(stderr, "%s\n", err)
370 0 : }
371 : }
372 :
373 1 : func (d *dbT) runCheck(cmd *cobra.Command, args []string) {
374 1 : stdout, stderr := cmd.OutOrStdout(), cmd.ErrOrStderr()
375 1 : db, err := d.openDB(args[0])
376 1 : if err != nil {
377 1 : fmt.Fprintf(stderr, "%s\n", err)
378 1 : return
379 1 : }
380 1 : defer d.closeDB(stderr, db)
381 1 :
382 1 : var stats pebble.CheckLevelsStats
383 1 : if err := db.CheckLevels(&stats); err != nil {
384 0 : fmt.Fprintf(stderr, "%s\n", err)
385 0 : }
386 1 : fmt.Fprintf(stdout, "checked %d %s and %d %s\n",
387 1 : stats.NumPoints, makePlural("point", stats.NumPoints), stats.NumTombstones, makePlural("tombstone", int64(stats.NumTombstones)))
388 : }
389 :
390 : type nonReadOnly struct{}
391 :
392 1 : func (n nonReadOnly) Apply(dirname string, opts *pebble.Options) {
393 1 : opts.ReadOnly = false
394 1 : // Increase the L0 compaction threshold to reduce the likelihood of an
395 1 : // unintended compaction changing test output.
396 1 : opts.L0CompactionThreshold = 10
397 1 : }
398 :
399 1 : func (d *dbT) runCheckpoint(cmd *cobra.Command, args []string) {
400 1 : stderr := cmd.ErrOrStderr()
401 1 : db, err := d.openDB(args[0], nonReadOnly{})
402 1 : if err != nil {
403 0 : fmt.Fprintf(stderr, "%s\n", err)
404 0 : return
405 0 : }
406 1 : defer d.closeDB(stderr, db)
407 1 : destDir := args[1]
408 1 :
409 1 : if err := db.Checkpoint(destDir); err != nil {
410 0 : fmt.Fprintf(stderr, "%s\n", err)
411 0 : }
412 : }
413 :
414 1 : func (d *dbT) runGet(cmd *cobra.Command, args []string) {
415 1 : stdout, stderr := cmd.OutOrStdout(), cmd.ErrOrStderr()
416 1 : db, err := d.openDB(args[0])
417 1 : if err != nil {
418 0 : fmt.Fprintf(stderr, "%s\n", err)
419 0 : return
420 0 : }
421 1 : defer d.closeDB(stderr, db)
422 1 : var k key
423 1 : if err := k.Set(args[1]); err != nil {
424 0 : fmt.Fprintf(stderr, "%s\n", err)
425 0 : return
426 0 : }
427 :
428 1 : val, closer, err := db.Get(k)
429 1 : if err != nil {
430 1 : fmt.Fprintf(stderr, "%s\n", err)
431 1 : return
432 1 : }
433 1 : defer func() {
434 1 : if closer != nil {
435 1 : closer.Close()
436 1 : }
437 : }()
438 1 : if val != nil {
439 1 : fmt.Fprintf(stdout, "%s\n", d.fmtValue.fn(k, val))
440 1 : }
441 : }
442 :
443 1 : func (d *dbT) runLSM(cmd *cobra.Command, args []string) {
444 1 : stdout, stderr := cmd.OutOrStdout(), cmd.ErrOrStderr()
445 1 : db, err := d.openDB(args[0])
446 1 : if err != nil {
447 1 : fmt.Fprintf(stderr, "%s\n", err)
448 1 : return
449 1 : }
450 1 : defer d.closeDB(stderr, db)
451 1 :
452 1 : fmt.Fprintf(stdout, "%s", db.Metrics())
453 1 : if d.lsmURL {
454 1 : fmt.Fprintf(stdout, "\nLSM viewer: %s\n", db.LSMViewURL())
455 1 : }
456 : }
457 :
458 1 : func (d *dbT) runScan(cmd *cobra.Command, args []string) {
459 1 : stdout, stderr := cmd.OutOrStdout(), cmd.ErrOrStderr()
460 1 : db, err := d.openDB(args[0])
461 1 : if err != nil {
462 1 : fmt.Fprintf(stderr, "%s\n", err)
463 1 : return
464 1 : }
465 1 : defer d.closeDB(stderr, db)
466 1 :
467 1 : // Update the internal formatter if this comparator has one specified.
468 1 : if d.opts.Comparer != nil {
469 1 : d.fmtKey.setForComparer(d.opts.Comparer.Name, d.comparers)
470 1 : d.fmtValue.setForComparer(d.opts.Comparer.Name, d.comparers)
471 1 : }
472 :
473 1 : start := timeNow()
474 1 : fmtKeys := d.fmtKey.spec != "null"
475 1 : fmtValues := d.fmtValue.spec != "null"
476 1 : var count int64
477 1 :
478 1 : iter, _ := db.NewIter(&pebble.IterOptions{
479 1 : UpperBound: d.end,
480 1 : })
481 1 : for valid := iter.SeekGE(d.start); valid; valid = iter.Next() {
482 1 : if fmtKeys || fmtValues {
483 1 : needDelimiter := false
484 1 : if fmtKeys {
485 1 : fmt.Fprintf(stdout, "%s", d.fmtKey.fn(iter.Key()))
486 1 : needDelimiter = true
487 1 : }
488 1 : if fmtValues {
489 1 : if needDelimiter {
490 1 : stdout.Write([]byte{' '})
491 1 : }
492 1 : fmt.Fprintf(stdout, "%s", d.fmtValue.fn(iter.Key(), iter.Value()))
493 : }
494 1 : stdout.Write([]byte{'\n'})
495 : }
496 :
497 1 : count++
498 1 : if d.count > 0 && count >= d.count {
499 1 : break
500 : }
501 : }
502 :
503 1 : if err := iter.Close(); err != nil {
504 0 : fmt.Fprintf(stderr, "%s\n", err)
505 0 : }
506 :
507 1 : elapsed := timeNow().Sub(start)
508 1 :
509 1 : fmt.Fprintf(stdout, "scanned %d %s in %0.1fs\n",
510 1 : count, makePlural("record", count), elapsed.Seconds())
511 : }
512 :
513 1 : func (d *dbT) runSpace(cmd *cobra.Command, args []string) {
514 1 : stdout, stderr := cmd.OutOrStdout(), cmd.ErrOrStderr()
515 1 : db, err := d.openDB(args[0])
516 1 : if err != nil {
517 0 : fmt.Fprintf(stderr, "%s\n", err)
518 0 : return
519 0 : }
520 1 : defer d.closeDB(stdout, db)
521 1 :
522 1 : bytes, err := db.EstimateDiskUsage(d.start, d.end)
523 1 : if err != nil {
524 0 : fmt.Fprintf(stderr, "%s\n", err)
525 0 : return
526 0 : }
527 1 : fmt.Fprintf(stdout, "%d\n", bytes)
528 : }
529 :
530 1 : func (d *dbT) getExciseSpan() (pebble.KeyRange, error) {
531 1 : // If a DBExciseSpanFn is specified, try to use it and see if it returns a
532 1 : // valid span.
533 1 : if d.exciseSpanFn != nil {
534 0 : span, err := d.exciseSpanFn()
535 0 : if err != nil {
536 0 : return pebble.KeyRange{}, err
537 0 : }
538 0 : if span.Valid() {
539 0 : if d.start != nil || d.end != nil {
540 0 : return pebble.KeyRange{}, errors.Errorf(
541 0 : "--start/--end cannot be used when span is specified by other methods.")
542 0 : }
543 0 : return span, nil
544 : }
545 : }
546 1 : if d.start == nil || d.end == nil {
547 1 : return pebble.KeyRange{}, errors.Errorf("excise range not specified.")
548 1 : }
549 1 : return pebble.KeyRange{
550 1 : Start: d.start,
551 1 : End: d.end,
552 1 : }, nil
553 : }
554 :
555 1 : func (d *dbT) runExcise(cmd *cobra.Command, args []string) {
556 1 : stdout, stderr := cmd.OutOrStdout(), cmd.ErrOrStderr()
557 1 :
558 1 : span, err := d.getExciseSpan()
559 1 : if err != nil {
560 1 : fmt.Fprintf(stderr, "Error: %v\n", err)
561 1 : return
562 1 : }
563 :
564 1 : dbOpts := d.opts.EnsureDefaults()
565 1 : // Disable all processes that would try to open tables: table stats,
566 1 : // consistency check, automatic compactions.
567 1 : dbOpts.DisableTableStats = true
568 1 : dbOpts.DisableConsistencyCheck = true
569 1 : dbOpts.DisableAutomaticCompactions = true
570 1 :
571 1 : dbDir := args[0]
572 1 : db, err := d.openDB(dbDir, nonReadOnly{})
573 1 : if err != nil {
574 0 : fmt.Fprintf(stderr, "%s\n", err)
575 0 : return
576 0 : }
577 1 : defer d.closeDB(stdout, db)
578 1 :
579 1 : // Update the internal formatter if this comparator has one specified.
580 1 : if d.opts.Comparer != nil {
581 1 : d.fmtKey.setForComparer(d.opts.Comparer.Name, d.comparers)
582 1 : }
583 :
584 1 : fmt.Fprintf(stdout, "Excising range:\n")
585 1 : fmt.Fprintf(stdout, " start: %s\n", d.fmtKey.fn(span.Start))
586 1 : fmt.Fprintf(stdout, " end: %s\n", d.fmtKey.fn(span.End))
587 1 :
588 1 : if !d.bypassPrompt {
589 0 : fmt.Fprintf(stdout, "WARNING!!!\n")
590 0 : fmt.Fprintf(stdout, "This command will remove all keys in this range!\n")
591 0 : reader := bufio.NewReader(cmd.InOrStdin())
592 0 : for {
593 0 : fmt.Fprintf(stdout, "Continue? [Y/N] ")
594 0 : answer, _ := reader.ReadString('\n')
595 0 : answer = strings.ToLower(strings.TrimSpace(answer))
596 0 : if answer == "y" || answer == "yes" {
597 0 : break
598 : }
599 :
600 0 : if answer == "n" || answer == "no" {
601 0 : fmt.Fprintf(stderr, "Aborting\n")
602 0 : return
603 0 : }
604 : }
605 : }
606 :
607 : // Write a temporary sst that only has excise tombstones. We write it inside
608 : // the database directory so that the command works against any FS.
609 : // TODO(radu): remove this if we add a separate DB.Excise method.
610 1 : path := dbOpts.FS.PathJoin(dbDir, fmt.Sprintf("excise-%0x.sst", rand.Uint32()))
611 1 : defer dbOpts.FS.Remove(path)
612 1 : f, err := dbOpts.FS.Create(path, vfs.WriteCategoryUnspecified)
613 1 : if err != nil {
614 0 : fmt.Fprintf(stderr, "Error creating temporary sst file %q: %s\n", path, err)
615 0 : return
616 0 : }
617 1 : writable := objstorageprovider.NewFileWritable(f)
618 1 : writerOpts := dbOpts.MakeWriterOptions(0, db.FormatMajorVersion().MaxTableFormat())
619 1 : w := sstable.NewWriter(writable, writerOpts)
620 1 : err = w.DeleteRange(span.Start, span.End)
621 1 : err = errors.CombineErrors(err, w.RangeKeyDelete(span.Start, span.End))
622 1 : err = errors.CombineErrors(err, w.Close())
623 1 : if err != nil {
624 0 : fmt.Fprintf(stderr, "Error writing temporary sst file %q: %s\n", path, err)
625 0 : return
626 0 : }
627 :
628 1 : _, err = db.IngestAndExcise(context.Background(), []string{path}, nil, nil, span, true /* sstsContainExciseTombstone */)
629 1 : if err != nil {
630 0 : fmt.Fprintf(stderr, "Error excising: %s\n", err)
631 0 : return
632 0 : }
633 1 : fmt.Fprintf(stdout, "Excise complete.\n")
634 : }
635 :
636 1 : func (d *dbT) runProperties(cmd *cobra.Command, args []string) {
637 1 : stdout, stderr := cmd.OutOrStdout(), cmd.ErrOrStderr()
638 1 : dirname := args[0]
639 1 : err := func() error {
640 1 : desc, err := pebble.Peek(dirname, d.opts.FS)
641 1 : if err != nil {
642 1 : return err
643 1 : } else if !desc.Exists {
644 0 : return oserror.ErrNotExist
645 0 : }
646 1 : manifestFilename := d.opts.FS.PathBase(desc.ManifestFilename)
647 1 :
648 1 : // Replay the manifest to get the current version.
649 1 : f, err := d.opts.FS.Open(desc.ManifestFilename)
650 1 : if err != nil {
651 0 : return errors.Wrapf(err, "pebble: could not open MANIFEST file %q", manifestFilename)
652 0 : }
653 1 : defer f.Close()
654 1 :
655 1 : cmp := base.DefaultComparer
656 1 : var bve manifest.BulkVersionEdit
657 1 : bve.AddedByFileNum = make(map[base.FileNum]*manifest.FileMetadata)
658 1 : rr := record.NewReader(f, 0 /* logNum */)
659 1 : for {
660 1 : r, err := rr.Next()
661 1 : if err == io.EOF {
662 1 : break
663 : }
664 1 : if err != nil {
665 0 : return errors.Wrapf(err, "pebble: reading manifest %q", manifestFilename)
666 0 : }
667 1 : var ve manifest.VersionEdit
668 1 : err = ve.Decode(r)
669 1 : if err != nil {
670 0 : return err
671 0 : }
672 1 : if err := bve.Accumulate(&ve); err != nil {
673 0 : return err
674 0 : }
675 1 : if ve.ComparerName != "" {
676 1 : cmp = d.comparers[ve.ComparerName]
677 1 : d.fmtKey.setForComparer(ve.ComparerName, d.comparers)
678 1 : d.fmtValue.setForComparer(ve.ComparerName, d.comparers)
679 1 : }
680 : }
681 1 : v, err := bve.Apply(
682 1 : nil /* version */, cmp, d.opts.FlushSplitBytes,
683 1 : d.opts.Experimental.ReadCompactionRate,
684 1 : )
685 1 : if err != nil {
686 0 : return err
687 0 : }
688 :
689 1 : objProvider, err := objstorageprovider.Open(objstorageprovider.DefaultSettings(d.opts.FS, dirname))
690 1 : if err != nil {
691 0 : return err
692 0 : }
693 1 : defer objProvider.Close()
694 1 :
695 1 : // Load and aggregate sstable properties.
696 1 : tw := tabwriter.NewWriter(stdout, 2, 1, 4, ' ', 0)
697 1 : var total props
698 1 : var all []props
699 1 : for _, l := range v.Levels {
700 1 : iter := l.Iter()
701 1 : var level props
702 1 : for t := iter.First(); t != nil; t = iter.Next() {
703 1 : if t.Virtual {
704 0 : // TODO(bananabrick): Handle virtual sstables here. We don't
705 0 : // really have any stats or properties at this point. Maybe
706 0 : // we could approximate some of these properties for virtual
707 0 : // sstables by first grabbing properties for the backing
708 0 : // physical sstable, and then extrapolating.
709 0 : continue
710 : }
711 1 : err := d.addProps(objProvider, t.PhysicalMeta(), &level)
712 1 : if err != nil {
713 0 : return err
714 0 : }
715 : }
716 1 : all = append(all, level)
717 1 : total.update(level)
718 : }
719 1 : all = append(all, total)
720 1 :
721 1 : fmt.Fprintln(tw, "\tL0\tL1\tL2\tL3\tL4\tL5\tL6\tTOTAL")
722 1 :
723 1 : fmt.Fprintf(tw, "count\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\n",
724 1 : propArgs(all, func(p *props) interface{} { return p.Count })...)
725 :
726 1 : fmt.Fprintln(tw, "seq num\t\t\t\t\t\t\t\t")
727 1 : fmt.Fprintf(tw, " smallest\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\n",
728 1 : propArgs(all, func(p *props) interface{} { return p.SmallestSeqNum })...)
729 1 : fmt.Fprintf(tw, " largest\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\n",
730 1 : propArgs(all, func(p *props) interface{} { return p.LargestSeqNum })...)
731 :
732 1 : fmt.Fprintln(tw, "size\t\t\t\t\t\t\t\t")
733 1 : fmt.Fprintf(tw, " data\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
734 1 : propArgs(all, func(p *props) interface{} { return humanize.Bytes.Uint64(p.DataSize) })...)
735 1 : fmt.Fprintf(tw, " blocks\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\n",
736 1 : propArgs(all, func(p *props) interface{} { return p.NumDataBlocks })...)
737 1 : fmt.Fprintf(tw, " index\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
738 1 : propArgs(all, func(p *props) interface{} { return humanize.Bytes.Uint64(p.IndexSize) })...)
739 1 : fmt.Fprintf(tw, " blocks\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\n",
740 1 : propArgs(all, func(p *props) interface{} { return p.NumIndexBlocks })...)
741 1 : fmt.Fprintf(tw, " top-level\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
742 1 : propArgs(all, func(p *props) interface{} { return humanize.Bytes.Uint64(p.TopLevelIndexSize) })...)
743 1 : fmt.Fprintf(tw, " filter\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
744 1 : propArgs(all, func(p *props) interface{} { return humanize.Bytes.Uint64(p.FilterSize) })...)
745 1 : fmt.Fprintf(tw, " raw-key\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
746 1 : propArgs(all, func(p *props) interface{} { return humanize.Bytes.Uint64(p.RawKeySize) })...)
747 1 : fmt.Fprintf(tw, " raw-value\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
748 1 : propArgs(all, func(p *props) interface{} { return humanize.Bytes.Uint64(p.RawValueSize) })...)
749 1 : fmt.Fprintf(tw, " pinned-key\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
750 1 : propArgs(all, func(p *props) interface{} { return humanize.Bytes.Uint64(p.SnapshotPinnedKeySize) })...)
751 1 : fmt.Fprintf(tw, " pinned-value\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
752 1 : propArgs(all, func(p *props) interface{} { return humanize.Bytes.Uint64(p.SnapshotPinnedValueSize) })...)
753 1 : fmt.Fprintf(tw, " point-del-key-size\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
754 1 : propArgs(all, func(p *props) interface{} { return humanize.Bytes.Uint64(p.RawPointTombstoneKeySize) })...)
755 1 : fmt.Fprintf(tw, " point-del-value-size\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
756 1 : propArgs(all, func(p *props) interface{} { return humanize.Bytes.Uint64(p.RawPointTombstoneValueSize) })...)
757 :
758 1 : fmt.Fprintln(tw, "records\t\t\t\t\t\t\t\t")
759 1 : fmt.Fprintf(tw, " set\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
760 1 : propArgs(all, func(p *props) interface{} {
761 1 : return humanize.Count.Uint64(p.NumEntries - p.NumDeletions - p.NumMergeOperands)
762 1 : })...)
763 1 : fmt.Fprintf(tw, " delete\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
764 1 : propArgs(all, func(p *props) interface{} { return humanize.Count.Uint64(p.NumDeletions - p.NumRangeDeletions) })...)
765 1 : fmt.Fprintf(tw, " delete-sized\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
766 1 : propArgs(all, func(p *props) interface{} { return humanize.Count.Uint64(p.NumSizedDeletions) })...)
767 1 : fmt.Fprintf(tw, " range-delete\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
768 1 : propArgs(all, func(p *props) interface{} { return humanize.Count.Uint64(p.NumRangeDeletions) })...)
769 1 : fmt.Fprintf(tw, " range-key-sets\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
770 1 : propArgs(all, func(p *props) interface{} { return humanize.Count.Uint64(p.NumRangeKeySets) })...)
771 1 : fmt.Fprintf(tw, " range-key-unsets\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
772 1 : propArgs(all, func(p *props) interface{} { return humanize.Count.Uint64(p.NumRangeKeyUnSets) })...)
773 1 : fmt.Fprintf(tw, " range-key-deletes\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
774 1 : propArgs(all, func(p *props) interface{} { return humanize.Count.Uint64(p.NumRangeKeyDeletes) })...)
775 1 : fmt.Fprintf(tw, " merge\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
776 1 : propArgs(all, func(p *props) interface{} { return humanize.Count.Uint64(p.NumMergeOperands) })...)
777 1 : fmt.Fprintf(tw, " pinned\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
778 1 : propArgs(all, func(p *props) interface{} { return humanize.Count.Uint64(p.SnapshotPinnedKeys) })...)
779 :
780 1 : if err := tw.Flush(); err != nil {
781 0 : return err
782 0 : }
783 1 : return nil
784 : }()
785 1 : if err != nil {
786 1 : fmt.Fprintln(stderr, err)
787 1 : }
788 : }
789 :
790 1 : func (d *dbT) runSet(cmd *cobra.Command, args []string) {
791 1 : stderr := cmd.ErrOrStderr()
792 1 : db, err := d.openDB(args[0], nonReadOnly{})
793 1 : if err != nil {
794 0 : fmt.Fprintf(stderr, "%s\n", err)
795 0 : return
796 0 : }
797 1 : defer d.closeDB(stderr, db)
798 1 : var k, v key
799 1 : if err := k.Set(args[1]); err != nil {
800 0 : fmt.Fprintf(stderr, "%s\n", err)
801 0 : return
802 0 : }
803 1 : if err := v.Set(args[2]); err != nil {
804 0 : fmt.Fprintf(stderr, "%s\n", err)
805 0 : return
806 0 : }
807 :
808 1 : if err := db.Set(k, v, nil); err != nil {
809 0 : fmt.Fprintf(stderr, "%s\n", err)
810 0 : }
811 : }
812 :
813 1 : func propArgs(props []props, getProp func(*props) interface{}) []interface{} {
814 1 : args := make([]interface{}, 0, len(props))
815 1 : for _, p := range props {
816 1 : args = append(args, getProp(&p))
817 1 : }
818 1 : return args
819 : }
820 :
821 : type props struct {
822 : Count uint64
823 : SmallestSeqNum base.SeqNum
824 : LargestSeqNum base.SeqNum
825 : DataSize uint64
826 : FilterSize uint64
827 : IndexSize uint64
828 : NumDataBlocks uint64
829 : NumIndexBlocks uint64
830 : NumDeletions uint64
831 : NumSizedDeletions uint64
832 : NumEntries uint64
833 : NumMergeOperands uint64
834 : NumRangeDeletions uint64
835 : NumRangeKeySets uint64
836 : NumRangeKeyUnSets uint64
837 : NumRangeKeyDeletes uint64
838 : RawKeySize uint64
839 : RawPointTombstoneKeySize uint64
840 : RawPointTombstoneValueSize uint64
841 : RawValueSize uint64
842 : SnapshotPinnedKeys uint64
843 : SnapshotPinnedKeySize uint64
844 : SnapshotPinnedValueSize uint64
845 : TopLevelIndexSize uint64
846 : }
847 :
848 1 : func (p *props) update(o props) {
849 1 : p.Count += o.Count
850 1 : if o.SmallestSeqNum != 0 && (o.SmallestSeqNum < p.SmallestSeqNum || p.SmallestSeqNum == 0) {
851 1 : p.SmallestSeqNum = o.SmallestSeqNum
852 1 : }
853 1 : if o.LargestSeqNum > p.LargestSeqNum {
854 1 : p.LargestSeqNum = o.LargestSeqNum
855 1 : }
856 1 : p.DataSize += o.DataSize
857 1 : p.FilterSize += o.FilterSize
858 1 : p.IndexSize += o.IndexSize
859 1 : p.NumDataBlocks += o.NumDataBlocks
860 1 : p.NumIndexBlocks += o.NumIndexBlocks
861 1 : p.NumDeletions += o.NumDeletions
862 1 : p.NumSizedDeletions += o.NumSizedDeletions
863 1 : p.NumEntries += o.NumEntries
864 1 : p.NumMergeOperands += o.NumMergeOperands
865 1 : p.NumRangeDeletions += o.NumRangeDeletions
866 1 : p.NumRangeKeySets += o.NumRangeKeySets
867 1 : p.NumRangeKeyUnSets += o.NumRangeKeyUnSets
868 1 : p.NumRangeKeyDeletes += o.NumRangeKeyDeletes
869 1 : p.RawKeySize += o.RawKeySize
870 1 : p.RawPointTombstoneKeySize += o.RawPointTombstoneKeySize
871 1 : p.RawPointTombstoneValueSize += o.RawPointTombstoneValueSize
872 1 : p.RawValueSize += o.RawValueSize
873 1 : p.SnapshotPinnedKeySize += o.SnapshotPinnedKeySize
874 1 : p.SnapshotPinnedValueSize += o.SnapshotPinnedValueSize
875 1 : p.SnapshotPinnedKeys += o.SnapshotPinnedKeys
876 1 : p.TopLevelIndexSize += o.TopLevelIndexSize
877 : }
878 :
879 : func (d *dbT) addProps(
880 : objProvider objstorage.Provider, m manifest.PhysicalFileMeta, p *props,
881 1 : ) error {
882 1 : ctx := context.Background()
883 1 : f, err := objProvider.OpenForReading(ctx, base.FileTypeTable, m.FileBacking.DiskFileNum, objstorage.OpenOptions{})
884 1 : if err != nil {
885 0 : return err
886 0 : }
887 1 : opts := sstable.ReaderOptions{
888 1 : Mergers: d.mergers,
889 1 : Comparers: d.comparers,
890 1 : }
891 1 : opts.SetInternal(sstableinternal.ReaderOptions{
892 1 : CacheOpts: sstableinternal.CacheOptions{
893 1 : FileNum: m.FileBacking.DiskFileNum,
894 1 : },
895 1 : })
896 1 : r, err := sstable.NewReader(ctx, f, sstable.ReaderOptions{
897 1 : Mergers: d.mergers,
898 1 : Comparers: d.comparers,
899 1 : })
900 1 : if err != nil {
901 0 : _ = f.Close()
902 0 : return err
903 0 : }
904 1 : p.update(props{
905 1 : Count: 1,
906 1 : SmallestSeqNum: m.SmallestSeqNum,
907 1 : LargestSeqNum: m.LargestSeqNum,
908 1 : DataSize: r.Properties.DataSize,
909 1 : FilterSize: r.Properties.FilterSize,
910 1 : IndexSize: r.Properties.IndexSize,
911 1 : NumDataBlocks: r.Properties.NumDataBlocks,
912 1 : NumIndexBlocks: 1 + r.Properties.IndexPartitions,
913 1 : NumDeletions: r.Properties.NumDeletions,
914 1 : NumSizedDeletions: r.Properties.NumSizedDeletions,
915 1 : NumEntries: r.Properties.NumEntries,
916 1 : NumMergeOperands: r.Properties.NumMergeOperands,
917 1 : NumRangeDeletions: r.Properties.NumRangeDeletions,
918 1 : NumRangeKeySets: r.Properties.NumRangeKeySets,
919 1 : NumRangeKeyUnSets: r.Properties.NumRangeKeyUnsets,
920 1 : NumRangeKeyDeletes: r.Properties.NumRangeKeyDels,
921 1 : RawKeySize: r.Properties.RawKeySize,
922 1 : RawPointTombstoneKeySize: r.Properties.RawPointTombstoneKeySize,
923 1 : RawPointTombstoneValueSize: r.Properties.RawPointTombstoneValueSize,
924 1 : RawValueSize: r.Properties.RawValueSize,
925 1 : SnapshotPinnedKeySize: r.Properties.SnapshotPinnedKeySize,
926 1 : SnapshotPinnedValueSize: r.Properties.SnapshotPinnedValueSize,
927 1 : SnapshotPinnedKeys: r.Properties.SnapshotPinnedKeys,
928 1 : TopLevelIndexSize: r.Properties.TopLevelIndexSize,
929 1 : })
930 1 : return r.Close()
931 : }
932 :
933 1 : func makePlural(singular string, count int64) string {
934 1 : if count > 1 {
935 1 : return fmt.Sprintf("%ss", singular)
936 1 : }
937 1 : return singular
938 : }
|
