Line data Source code
1 : // Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
2 : // of this source code is governed by a BSD-style license that can be found in
3 : // the LICENSE file.
4 :
5 : package tool
6 :
7 : import (
8 : "bufio"
9 : "context"
10 : "fmt"
11 : "io"
12 : "math/rand"
13 : "strings"
14 : "text/tabwriter"
15 :
16 : "github.com/cockroachdb/errors"
17 : "github.com/cockroachdb/errors/oserror"
18 : "github.com/cockroachdb/pebble"
19 : "github.com/cockroachdb/pebble/internal/base"
20 : "github.com/cockroachdb/pebble/internal/humanize"
21 : "github.com/cockroachdb/pebble/internal/manifest"
22 : "github.com/cockroachdb/pebble/objstorage"
23 : "github.com/cockroachdb/pebble/objstorage/objstorageprovider"
24 : "github.com/cockroachdb/pebble/record"
25 : "github.com/cockroachdb/pebble/sstable"
26 : "github.com/cockroachdb/pebble/tool/logs"
27 : "github.com/cockroachdb/pebble/vfs"
28 : "github.com/spf13/cobra"
29 : )
30 :
31 : // dbT implements db-level tools, including both configuration state and the
32 : // commands themselves.
33 : type dbT struct {
34 : Root *cobra.Command
35 : Check *cobra.Command
36 : Checkpoint *cobra.Command
37 : Get *cobra.Command
38 : Logs *cobra.Command
39 : LSM *cobra.Command
40 : Properties *cobra.Command
41 : Scan *cobra.Command
42 : Set *cobra.Command
43 : Space *cobra.Command
44 : IOBench *cobra.Command
45 : Excise *cobra.Command
46 :
47 : // Configuration.
48 : opts *pebble.Options
49 : comparers sstable.Comparers
50 : mergers sstable.Mergers
51 : openErrEnhancer func(error) error
52 : openOptions []OpenOption
53 : exciseSpanFn DBExciseSpanFn
54 :
55 : // Flags.
56 : comparerName string
57 : mergerName string
58 : fmtKey keyFormatter
59 : fmtValue valueFormatter
60 : start key
61 : end key
62 : count int64
63 : allLevels bool
64 : ioCount int
65 : ioParallelism int
66 : ioSizes string
67 : verbose bool
68 : bypassPrompt bool
69 : }
70 :
71 : func newDB(
72 : opts *pebble.Options,
73 : comparers sstable.Comparers,
74 : mergers sstable.Mergers,
75 : openErrEnhancer func(error) error,
76 : openOptions []OpenOption,
77 : exciseSpanFn DBExciseSpanFn,
78 1 : ) *dbT {
79 1 : d := &dbT{
80 1 : opts: opts,
81 1 : comparers: comparers,
82 1 : mergers: mergers,
83 1 : openErrEnhancer: openErrEnhancer,
84 1 : openOptions: openOptions,
85 1 : exciseSpanFn: exciseSpanFn,
86 1 : }
87 1 : d.fmtKey.mustSet("quoted")
88 1 : d.fmtValue.mustSet("[%x]")
89 1 :
90 1 : d.Root = &cobra.Command{
91 1 : Use: "db",
92 1 : Short: "DB introspection tools",
93 1 : }
94 1 : d.Check = &cobra.Command{
95 1 : Use: "check <dir>",
96 1 : Short: "verify checksums and metadata",
97 1 : Long: `
98 1 : Verify sstable, manifest, and WAL checksums. Requires that the specified
99 1 : database not be in use by another process.
100 1 : `,
101 1 : Args: cobra.ExactArgs(1),
102 1 : Run: d.runCheck,
103 1 : }
104 1 : d.Checkpoint = &cobra.Command{
105 1 : Use: "checkpoint <src-dir> <dest-dir>",
106 1 : Short: "create a checkpoint",
107 1 : Long: `
108 1 : Creates a Pebble checkpoint in the specified destination directory. A checkpoint
109 1 : is a point-in-time snapshot of DB state. Requires that the specified
110 1 : database not be in use by another process.
111 1 : `,
112 1 : Args: cobra.ExactArgs(2),
113 1 : Run: d.runCheckpoint,
114 1 : }
115 1 : d.Get = &cobra.Command{
116 1 : Use: "get <dir> <key>",
117 1 : Short: "get value for a key",
118 1 : Long: `
119 1 : Gets a value for a key, if it exists in DB. Prints a "not found" error if key
120 1 : does not exist. Requires that the specified database not be in use by another
121 1 : process.
122 1 : `,
123 1 : Args: cobra.ExactArgs(2),
124 1 : Run: d.runGet,
125 1 : }
126 1 : d.Logs = logs.NewCmd()
127 1 : d.LSM = &cobra.Command{
128 1 : Use: "lsm <dir>",
129 1 : Short: "print LSM structure",
130 1 : Long: `
131 1 : Print the structure of the LSM tree. Requires that the specified database not
132 1 : be in use by another process.
133 1 : `,
134 1 : Args: cobra.ExactArgs(1),
135 1 : Run: d.runLSM,
136 1 : }
137 1 : d.Properties = &cobra.Command{
138 1 : Use: "properties <dir>",
139 1 : Short: "print aggregated sstable properties",
140 1 : Long: `
141 1 : Print SSTable properties, aggregated per level of the LSM.
142 1 : `,
143 1 : Args: cobra.ExactArgs(1),
144 1 : Run: d.runProperties,
145 1 : }
146 1 : d.Scan = &cobra.Command{
147 1 : Use: "scan <dir>",
148 1 : Short: "print db records",
149 1 : Long: `
150 1 : Print the records in the DB. Requires that the specified database not be in use
151 1 : by another process.
152 1 : `,
153 1 : Args: cobra.ExactArgs(1),
154 1 : Run: d.runScan,
155 1 : }
156 1 : d.Set = &cobra.Command{
157 1 : Use: "set <dir> <key> <value>",
158 1 : Short: "set a value for a key",
159 1 : Long: `
160 1 : Adds a new key/value to the DB. Requires that the specified database
161 1 : not be in use by another process.
162 1 : `,
163 1 : Args: cobra.ExactArgs(3),
164 1 : Run: d.runSet,
165 1 : }
166 1 : d.Space = &cobra.Command{
167 1 : Use: "space <dir>",
168 1 : Short: "print filesystem space used",
169 1 : Long: `
170 1 : Print the estimated filesystem space usage for the inclusive-inclusive range
171 1 : specified by --start and --end. Requires that the specified database not be in
172 1 : use by another process.
173 1 : `,
174 1 : Args: cobra.ExactArgs(1),
175 1 : Run: d.runSpace,
176 1 : }
177 1 : d.Excise = &cobra.Command{
178 1 : Use: "excise <dir>",
179 1 : Short: "excise a key range",
180 1 : Long: `
181 1 : Excise a key range, removing all SSTs inside the range and virtualizing any SSTs
182 1 : that partially overlap the range.
183 1 : `,
184 1 : Args: cobra.ExactArgs(1),
185 1 : Run: d.runExcise,
186 1 : }
187 1 : d.IOBench = &cobra.Command{
188 1 : Use: "io-bench <dir>",
189 1 : Short: "perform sstable IO benchmark",
190 1 : Long: `
191 1 : Run a random IO workload with various IO sizes against the sstables in the
192 1 : specified database.
193 1 : `,
194 1 : Args: cobra.ExactArgs(1),
195 1 : Run: d.runIOBench,
196 1 : }
197 1 :
198 1 : d.Root.AddCommand(d.Check, d.Checkpoint, d.Get, d.Logs, d.LSM, d.Properties, d.Scan, d.Set, d.Space, d.Excise, d.IOBench)
199 1 : d.Root.PersistentFlags().BoolVarP(&d.verbose, "verbose", "v", false, "verbose output")
200 1 :
201 1 : for _, cmd := range []*cobra.Command{d.Check, d.Checkpoint, d.Get, d.LSM, d.Properties, d.Scan, d.Set, d.Space, d.Excise} {
202 1 : cmd.Flags().StringVar(
203 1 : &d.comparerName, "comparer", "", "comparer name (use default if empty)")
204 1 : cmd.Flags().StringVar(
205 1 : &d.mergerName, "merger", "", "merger name (use default if empty)")
206 1 : }
207 :
208 1 : for _, cmd := range []*cobra.Command{d.Scan, d.Get} {
209 1 : cmd.Flags().Var(
210 1 : &d.fmtValue, "value", "value formatter")
211 1 : }
212 :
213 1 : d.Space.Flags().Var(
214 1 : &d.start, "start", "start key for the range")
215 1 : d.Space.Flags().Var(
216 1 : &d.end, "end", "inclusive end key for the range")
217 1 :
218 1 : d.Scan.Flags().Var(
219 1 : &d.fmtKey, "key", "key formatter")
220 1 : d.Scan.Flags().Var(
221 1 : &d.start, "start", "start key for the range")
222 1 : d.Scan.Flags().Var(
223 1 : &d.end, "end", "exclusive end key for the range")
224 1 : d.Scan.Flags().Int64Var(
225 1 : &d.count, "count", 0, "key count for scan (0 is unlimited)")
226 1 :
227 1 : d.Excise.Flags().Var(
228 1 : &d.start, "start", "start key for the excised range")
229 1 : d.Excise.Flags().Var(
230 1 : &d.end, "end", "exclusive end key for the excised range")
231 1 : d.Excise.Flags().BoolVar(
232 1 : &d.bypassPrompt, "yes", false, "bypass prompt")
233 1 :
234 1 : d.IOBench.Flags().BoolVar(
235 1 : &d.allLevels, "all-levels", false, "if set, benchmark all levels (default is only L5/L6)")
236 1 : d.IOBench.Flags().IntVar(
237 1 : &d.ioCount, "io-count", 10000, "number of IOs (per IO size) to benchmark")
238 1 : d.IOBench.Flags().IntVar(
239 1 : &d.ioParallelism, "io-parallelism", 16, "number of goroutines issuing IO")
240 1 : d.IOBench.Flags().StringVar(
241 1 : &d.ioSizes, "io-sizes-kb", "4,16,64,128,256,512,1024", "comma separated list of IO sizes in KB")
242 1 :
243 1 : return d
244 : }
245 :
246 1 : func (d *dbT) loadOptions(dir string) error {
247 1 : ls, err := d.opts.FS.List(dir)
248 1 : if err != nil || len(ls) == 0 {
249 1 : // NB: We don't return the error here as we prefer to return the error from
250 1 : // pebble.Open. Another way to put this is that a non-existent directory is
251 1 : // not a failure in loading the options.
252 1 : return nil
253 1 : }
254 :
255 1 : hooks := &pebble.ParseHooks{
256 1 : NewComparer: func(name string) (*pebble.Comparer, error) {
257 1 : if c := d.comparers[name]; c != nil {
258 1 : return c, nil
259 1 : }
260 0 : return nil, errors.Errorf("unknown comparer %q", errors.Safe(name))
261 : },
262 1 : NewMerger: func(name string) (*pebble.Merger, error) {
263 1 : if m := d.mergers[name]; m != nil {
264 1 : return m, nil
265 1 : }
266 0 : return nil, errors.Errorf("unknown merger %q", errors.Safe(name))
267 : },
268 0 : SkipUnknown: func(name, value string) bool {
269 0 : return true
270 0 : },
271 : }
272 :
273 : // TODO(peter): RocksDB sometimes leaves multiple OPTIONS files in
274 : // existence. We parse all of them as the comparer and merger shouldn't be
275 : // changing. We could parse only the first or the latest. Not clear if this
276 : // matters.
277 1 : var dbOpts pebble.Options
278 1 : for _, filename := range ls {
279 1 : ft, _, ok := base.ParseFilename(d.opts.FS, filename)
280 1 : if !ok {
281 1 : continue
282 : }
283 1 : switch ft {
284 1 : case base.FileTypeOptions:
285 1 : err := func() error {
286 1 : f, err := d.opts.FS.Open(d.opts.FS.PathJoin(dir, filename))
287 1 : if err != nil {
288 0 : return err
289 0 : }
290 1 : defer f.Close()
291 1 :
292 1 : data, err := io.ReadAll(f)
293 1 : if err != nil {
294 0 : return err
295 0 : }
296 :
297 1 : if err := dbOpts.Parse(string(data), hooks); err != nil {
298 1 : return err
299 1 : }
300 1 : return nil
301 : }()
302 1 : if err != nil {
303 1 : return err
304 1 : }
305 : }
306 : }
307 :
308 1 : if dbOpts.Comparer != nil {
309 1 : d.opts.Comparer = dbOpts.Comparer
310 1 : }
311 1 : if dbOpts.Merger != nil {
312 1 : d.opts.Merger = dbOpts.Merger
313 1 : }
314 1 : return nil
315 : }
316 :
317 : // OpenOption is an option that may be applied to the *pebble.Options before
318 : // calling pebble.Open.
319 : type OpenOption interface {
320 : Apply(dirname string, opts *pebble.Options)
321 : }
322 :
323 1 : func (d *dbT) openDB(dir string, openOptions ...OpenOption) (*pebble.DB, error) {
324 1 : db, err := d.openDBInternal(dir, openOptions...)
325 1 : if err != nil {
326 1 : if d.openErrEnhancer != nil {
327 1 : err = d.openErrEnhancer(err)
328 1 : }
329 1 : return nil, err
330 : }
331 1 : return db, nil
332 : }
333 :
334 1 : func (d *dbT) openDBInternal(dir string, openOptions ...OpenOption) (*pebble.DB, error) {
335 1 : if err := d.loadOptions(dir); err != nil {
336 1 : return nil, errors.Wrap(err, "error loading options")
337 1 : }
338 1 : if d.comparerName != "" {
339 1 : d.opts.Comparer = d.comparers[d.comparerName]
340 1 : if d.opts.Comparer == nil {
341 1 : return nil, errors.Errorf("unknown comparer %q", errors.Safe(d.comparerName))
342 1 : }
343 : }
344 1 : if d.mergerName != "" {
345 1 : d.opts.Merger = d.mergers[d.mergerName]
346 1 : if d.opts.Merger == nil {
347 1 : return nil, errors.Errorf("unknown merger %q", errors.Safe(d.mergerName))
348 1 : }
349 : }
350 1 : opts := *d.opts
351 1 : for _, opt := range openOptions {
352 1 : opt.Apply(dir, &opts)
353 1 : }
354 1 : for _, opt := range d.openOptions {
355 0 : opt.Apply(dir, &opts)
356 0 : }
357 1 : opts.Cache = pebble.NewCache(128 << 20 /* 128 MB */)
358 1 : defer opts.Cache.Unref()
359 1 : return pebble.Open(dir, &opts)
360 : }
361 :
362 1 : func (d *dbT) closeDB(stderr io.Writer, db *pebble.DB) {
363 1 : if err := db.Close(); err != nil {
364 0 : fmt.Fprintf(stderr, "%s\n", err)
365 0 : }
366 : }
367 :
368 1 : func (d *dbT) runCheck(cmd *cobra.Command, args []string) {
369 1 : stdout, stderr := cmd.OutOrStdout(), cmd.ErrOrStderr()
370 1 : db, err := d.openDB(args[0])
371 1 : if err != nil {
372 1 : fmt.Fprintf(stderr, "%s\n", err)
373 1 : return
374 1 : }
375 1 : defer d.closeDB(stderr, db)
376 1 :
377 1 : var stats pebble.CheckLevelsStats
378 1 : if err := db.CheckLevels(&stats); err != nil {
379 0 : fmt.Fprintf(stderr, "%s\n", err)
380 0 : }
381 1 : fmt.Fprintf(stdout, "checked %d %s and %d %s\n",
382 1 : stats.NumPoints, makePlural("point", stats.NumPoints), stats.NumTombstones, makePlural("tombstone", int64(stats.NumTombstones)))
383 : }
384 :
385 : type nonReadOnly struct{}
386 :
387 1 : func (n nonReadOnly) Apply(dirname string, opts *pebble.Options) {
388 1 : opts.ReadOnly = false
389 1 : // Increase the L0 compaction threshold to reduce the likelihood of an
390 1 : // unintended compaction changing test output.
391 1 : opts.L0CompactionThreshold = 10
392 1 : }
393 :
394 1 : func (d *dbT) runCheckpoint(cmd *cobra.Command, args []string) {
395 1 : stderr := cmd.ErrOrStderr()
396 1 : db, err := d.openDB(args[0], nonReadOnly{})
397 1 : if err != nil {
398 0 : fmt.Fprintf(stderr, "%s\n", err)
399 0 : return
400 0 : }
401 1 : defer d.closeDB(stderr, db)
402 1 : destDir := args[1]
403 1 :
404 1 : if err := db.Checkpoint(destDir); err != nil {
405 0 : fmt.Fprintf(stderr, "%s\n", err)
406 0 : }
407 : }
408 :
409 1 : func (d *dbT) runGet(cmd *cobra.Command, args []string) {
410 1 : stdout, stderr := cmd.OutOrStdout(), cmd.ErrOrStderr()
411 1 : db, err := d.openDB(args[0])
412 1 : if err != nil {
413 0 : fmt.Fprintf(stderr, "%s\n", err)
414 0 : return
415 0 : }
416 1 : defer d.closeDB(stderr, db)
417 1 : var k key
418 1 : if err := k.Set(args[1]); err != nil {
419 0 : fmt.Fprintf(stderr, "%s\n", err)
420 0 : return
421 0 : }
422 :
423 1 : val, closer, err := db.Get(k)
424 1 : if err != nil {
425 1 : fmt.Fprintf(stderr, "%s\n", err)
426 1 : return
427 1 : }
428 1 : defer func() {
429 1 : if closer != nil {
430 1 : closer.Close()
431 1 : }
432 : }()
433 1 : if val != nil {
434 1 : fmt.Fprintf(stdout, "%s\n", d.fmtValue.fn(k, val))
435 1 : }
436 : }
437 :
438 1 : func (d *dbT) runLSM(cmd *cobra.Command, args []string) {
439 1 : stdout, stderr := cmd.OutOrStdout(), cmd.ErrOrStderr()
440 1 : db, err := d.openDB(args[0])
441 1 : if err != nil {
442 1 : fmt.Fprintf(stderr, "%s\n", err)
443 1 : return
444 1 : }
445 1 : defer d.closeDB(stderr, db)
446 1 :
447 1 : fmt.Fprintf(stdout, "%s", db.Metrics())
448 : }
449 :
450 1 : func (d *dbT) runScan(cmd *cobra.Command, args []string) {
451 1 : stdout, stderr := cmd.OutOrStdout(), cmd.ErrOrStderr()
452 1 : db, err := d.openDB(args[0])
453 1 : if err != nil {
454 1 : fmt.Fprintf(stderr, "%s\n", err)
455 1 : return
456 1 : }
457 1 : defer d.closeDB(stderr, db)
458 1 :
459 1 : // Update the internal formatter if this comparator has one specified.
460 1 : if d.opts.Comparer != nil {
461 1 : d.fmtKey.setForComparer(d.opts.Comparer.Name, d.comparers)
462 1 : d.fmtValue.setForComparer(d.opts.Comparer.Name, d.comparers)
463 1 : }
464 :
465 1 : start := timeNow()
466 1 : fmtKeys := d.fmtKey.spec != "null"
467 1 : fmtValues := d.fmtValue.spec != "null"
468 1 : var count int64
469 1 :
470 1 : iter, _ := db.NewIter(&pebble.IterOptions{
471 1 : UpperBound: d.end,
472 1 : })
473 1 : for valid := iter.SeekGE(d.start); valid; valid = iter.Next() {
474 1 : if fmtKeys || fmtValues {
475 1 : needDelimiter := false
476 1 : if fmtKeys {
477 1 : fmt.Fprintf(stdout, "%s", d.fmtKey.fn(iter.Key()))
478 1 : needDelimiter = true
479 1 : }
480 1 : if fmtValues {
481 1 : if needDelimiter {
482 1 : stdout.Write([]byte{' '})
483 1 : }
484 1 : fmt.Fprintf(stdout, "%s", d.fmtValue.fn(iter.Key(), iter.Value()))
485 : }
486 1 : stdout.Write([]byte{'\n'})
487 : }
488 :
489 1 : count++
490 1 : if d.count > 0 && count >= d.count {
491 1 : break
492 : }
493 : }
494 :
495 1 : if err := iter.Close(); err != nil {
496 0 : fmt.Fprintf(stderr, "%s\n", err)
497 0 : }
498 :
499 1 : elapsed := timeNow().Sub(start)
500 1 :
501 1 : fmt.Fprintf(stdout, "scanned %d %s in %0.1fs\n",
502 1 : count, makePlural("record", count), elapsed.Seconds())
503 : }
504 :
505 1 : func (d *dbT) runSpace(cmd *cobra.Command, args []string) {
506 1 : stdout, stderr := cmd.OutOrStdout(), cmd.ErrOrStderr()
507 1 : db, err := d.openDB(args[0])
508 1 : if err != nil {
509 0 : fmt.Fprintf(stderr, "%s\n", err)
510 0 : return
511 0 : }
512 1 : defer d.closeDB(stdout, db)
513 1 :
514 1 : bytes, err := db.EstimateDiskUsage(d.start, d.end)
515 1 : if err != nil {
516 0 : fmt.Fprintf(stderr, "%s\n", err)
517 0 : return
518 0 : }
519 1 : fmt.Fprintf(stdout, "%d\n", bytes)
520 : }
521 :
522 1 : func (d *dbT) getExciseSpan() (pebble.KeyRange, error) {
523 1 : // If a DBExciseSpanFn is specified, try to use it and see if it returns a
524 1 : // valid span.
525 1 : if d.exciseSpanFn != nil {
526 0 : span, err := d.exciseSpanFn()
527 0 : if err != nil {
528 0 : return pebble.KeyRange{}, err
529 0 : }
530 0 : if span.Valid() {
531 0 : if d.start != nil || d.end != nil {
532 0 : return pebble.KeyRange{}, errors.Errorf(
533 0 : "--start/--end cannot be used when span is specified by other methods.")
534 0 : }
535 0 : return span, nil
536 : }
537 : }
538 1 : if d.start == nil || d.end == nil {
539 1 : return pebble.KeyRange{}, errors.Errorf("excise range not specified.")
540 1 : }
541 1 : return pebble.KeyRange{
542 1 : Start: d.start,
543 1 : End: d.end,
544 1 : }, nil
545 : }
546 :
547 1 : func (d *dbT) runExcise(cmd *cobra.Command, args []string) {
548 1 : stdout, stderr := cmd.OutOrStdout(), cmd.ErrOrStderr()
549 1 :
550 1 : span, err := d.getExciseSpan()
551 1 : if err != nil {
552 1 : fmt.Fprintf(stderr, "Error: %v\n", err)
553 1 : return
554 1 : }
555 :
556 1 : dbOpts := d.opts.EnsureDefaults()
557 1 : // Disable all processes that would try to open tables: table stats,
558 1 : // consistency check, automatic compactions.
559 1 : dbOpts.DisableTableStats = true
560 1 : dbOpts.DisableConsistencyCheck = true
561 1 : dbOpts.DisableAutomaticCompactions = true
562 1 :
563 1 : dbDir := args[0]
564 1 : db, err := d.openDB(dbDir, nonReadOnly{})
565 1 : if err != nil {
566 0 : fmt.Fprintf(stderr, "%s\n", err)
567 0 : return
568 0 : }
569 1 : defer d.closeDB(stdout, db)
570 1 :
571 1 : // Update the internal formatter if this comparator has one specified.
572 1 : if d.opts.Comparer != nil {
573 1 : d.fmtKey.setForComparer(d.opts.Comparer.Name, d.comparers)
574 1 : }
575 :
576 1 : fmt.Fprintf(stdout, "Excising range:\n")
577 1 : fmt.Fprintf(stdout, " start: %s\n", d.fmtKey.fn(span.Start))
578 1 : fmt.Fprintf(stdout, " end: %s\n", d.fmtKey.fn(span.End))
579 1 :
580 1 : if !d.bypassPrompt {
581 0 : fmt.Fprintf(stdout, "WARNING!!!\n")
582 0 : fmt.Fprintf(stdout, "This command will remove all keys in this range!\n")
583 0 : reader := bufio.NewReader(cmd.InOrStdin())
584 0 : for {
585 0 : fmt.Fprintf(stdout, "Continue? [Y/N] ")
586 0 : answer, _ := reader.ReadString('\n')
587 0 : answer = strings.ToLower(strings.TrimSpace(answer))
588 0 : if answer == "y" || answer == "yes" {
589 0 : break
590 : }
591 :
592 0 : if answer == "n" || answer == "no" {
593 0 : fmt.Fprintf(stderr, "Aborting\n")
594 0 : return
595 0 : }
596 : }
597 : }
598 :
599 : // Write a temporary sst that only has excise tombstones. We write it inside
600 : // the database directory so that the command works against any FS.
601 : // TODO(radu): remove this if we add a separate DB.Excise method.
602 1 : path := dbOpts.FS.PathJoin(dbDir, fmt.Sprintf("excise-%0x.sst", rand.Uint32()))
603 1 : defer dbOpts.FS.Remove(path)
604 1 : f, err := dbOpts.FS.Create(path, vfs.WriteCategoryUnspecified)
605 1 : if err != nil {
606 0 : fmt.Fprintf(stderr, "Error creating temporary sst file %q: %s\n", path, err)
607 0 : return
608 0 : }
609 1 : writable := objstorageprovider.NewFileWritable(f)
610 1 : writerOpts := dbOpts.MakeWriterOptions(0, db.FormatMajorVersion().MaxTableFormat())
611 1 : w := sstable.NewWriter(writable, writerOpts)
612 1 : err = w.DeleteRange(span.Start, span.End)
613 1 : err = errors.CombineErrors(err, w.RangeKeyDelete(span.Start, span.End))
614 1 : err = errors.CombineErrors(err, w.Close())
615 1 : if err != nil {
616 0 : fmt.Fprintf(stderr, "Error writing temporary sst file %q: %s\n", path, err)
617 0 : return
618 0 : }
619 :
620 1 : _, err = db.IngestAndExcise([]string{path}, nil, nil, span, true /* sstsContainExciseTombstone */)
621 1 : if err != nil {
622 0 : fmt.Fprintf(stderr, "Error excising: %s\n", err)
623 0 : return
624 0 : }
625 1 : fmt.Fprintf(stdout, "Excise complete.\n")
626 : }
627 :
628 1 : func (d *dbT) runProperties(cmd *cobra.Command, args []string) {
629 1 : stdout, stderr := cmd.OutOrStdout(), cmd.ErrOrStderr()
630 1 : dirname := args[0]
631 1 : err := func() error {
632 1 : desc, err := pebble.Peek(dirname, d.opts.FS)
633 1 : if err != nil {
634 1 : return err
635 1 : } else if !desc.Exists {
636 0 : return oserror.ErrNotExist
637 0 : }
638 1 : manifestFilename := d.opts.FS.PathBase(desc.ManifestFilename)
639 1 :
640 1 : // Replay the manifest to get the current version.
641 1 : f, err := d.opts.FS.Open(desc.ManifestFilename)
642 1 : if err != nil {
643 0 : return errors.Wrapf(err, "pebble: could not open MANIFEST file %q", manifestFilename)
644 0 : }
645 1 : defer f.Close()
646 1 :
647 1 : cmp := base.DefaultComparer
648 1 : var bve manifest.BulkVersionEdit
649 1 : bve.AddedByFileNum = make(map[base.FileNum]*manifest.FileMetadata)
650 1 : rr := record.NewReader(f, 0 /* logNum */)
651 1 : for {
652 1 : r, err := rr.Next()
653 1 : if err == io.EOF {
654 1 : break
655 : }
656 1 : if err != nil {
657 0 : return errors.Wrapf(err, "pebble: reading manifest %q", manifestFilename)
658 0 : }
659 1 : var ve manifest.VersionEdit
660 1 : err = ve.Decode(r)
661 1 : if err != nil {
662 0 : return err
663 0 : }
664 1 : if err := bve.Accumulate(&ve); err != nil {
665 0 : return err
666 0 : }
667 1 : if ve.ComparerName != "" {
668 1 : cmp = d.comparers[ve.ComparerName]
669 1 : d.fmtKey.setForComparer(ve.ComparerName, d.comparers)
670 1 : d.fmtValue.setForComparer(ve.ComparerName, d.comparers)
671 1 : }
672 : }
673 1 : v, err := bve.Apply(
674 1 : nil /* version */, cmp, d.opts.FlushSplitBytes,
675 1 : d.opts.Experimental.ReadCompactionRate,
676 1 : )
677 1 : if err != nil {
678 0 : return err
679 0 : }
680 :
681 1 : objProvider, err := objstorageprovider.Open(objstorageprovider.DefaultSettings(d.opts.FS, dirname))
682 1 : if err != nil {
683 0 : return err
684 0 : }
685 1 : defer objProvider.Close()
686 1 :
687 1 : // Load and aggregate sstable properties.
688 1 : tw := tabwriter.NewWriter(stdout, 2, 1, 4, ' ', 0)
689 1 : var total props
690 1 : var all []props
691 1 : for _, l := range v.Levels {
692 1 : iter := l.Iter()
693 1 : var level props
694 1 : for t := iter.First(); t != nil; t = iter.Next() {
695 1 : if t.Virtual {
696 0 : // TODO(bananabrick): Handle virtual sstables here. We don't
697 0 : // really have any stats or properties at this point. Maybe
698 0 : // we could approximate some of these properties for virtual
699 0 : // sstables by first grabbing properties for the backing
700 0 : // physical sstable, and then extrapolating.
701 0 : continue
702 : }
703 1 : err := d.addProps(objProvider, t.PhysicalMeta(), &level)
704 1 : if err != nil {
705 0 : return err
706 0 : }
707 : }
708 1 : all = append(all, level)
709 1 : total.update(level)
710 : }
711 1 : all = append(all, total)
712 1 :
713 1 : fmt.Fprintln(tw, "\tL0\tL1\tL2\tL3\tL4\tL5\tL6\tTOTAL")
714 1 :
715 1 : fmt.Fprintf(tw, "count\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\n",
716 1 : propArgs(all, func(p *props) interface{} { return p.Count })...)
717 :
718 1 : fmt.Fprintln(tw, "seq num\t\t\t\t\t\t\t\t")
719 1 : fmt.Fprintf(tw, " smallest\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\n",
720 1 : propArgs(all, func(p *props) interface{} { return p.SmallestSeqNum })...)
721 1 : fmt.Fprintf(tw, " largest\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\n",
722 1 : propArgs(all, func(p *props) interface{} { return p.LargestSeqNum })...)
723 :
724 1 : fmt.Fprintln(tw, "size\t\t\t\t\t\t\t\t")
725 1 : fmt.Fprintf(tw, " data\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
726 1 : propArgs(all, func(p *props) interface{} { return humanize.Bytes.Uint64(p.DataSize) })...)
727 1 : fmt.Fprintf(tw, " blocks\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\n",
728 1 : propArgs(all, func(p *props) interface{} { return p.NumDataBlocks })...)
729 1 : fmt.Fprintf(tw, " index\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
730 1 : propArgs(all, func(p *props) interface{} { return humanize.Bytes.Uint64(p.IndexSize) })...)
731 1 : fmt.Fprintf(tw, " blocks\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\n",
732 1 : propArgs(all, func(p *props) interface{} { return p.NumIndexBlocks })...)
733 1 : fmt.Fprintf(tw, " top-level\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
734 1 : propArgs(all, func(p *props) interface{} { return humanize.Bytes.Uint64(p.TopLevelIndexSize) })...)
735 1 : fmt.Fprintf(tw, " filter\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
736 1 : propArgs(all, func(p *props) interface{} { return humanize.Bytes.Uint64(p.FilterSize) })...)
737 1 : fmt.Fprintf(tw, " raw-key\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
738 1 : propArgs(all, func(p *props) interface{} { return humanize.Bytes.Uint64(p.RawKeySize) })...)
739 1 : fmt.Fprintf(tw, " raw-value\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
740 1 : propArgs(all, func(p *props) interface{} { return humanize.Bytes.Uint64(p.RawValueSize) })...)
741 1 : fmt.Fprintf(tw, " pinned-key\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
742 1 : propArgs(all, func(p *props) interface{} { return humanize.Bytes.Uint64(p.SnapshotPinnedKeySize) })...)
743 1 : fmt.Fprintf(tw, " pinned-value\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
744 1 : propArgs(all, func(p *props) interface{} { return humanize.Bytes.Uint64(p.SnapshotPinnedValueSize) })...)
745 1 : fmt.Fprintf(tw, " point-del-key-size\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
746 1 : propArgs(all, func(p *props) interface{} { return humanize.Bytes.Uint64(p.RawPointTombstoneKeySize) })...)
747 1 : fmt.Fprintf(tw, " point-del-value-size\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
748 1 : propArgs(all, func(p *props) interface{} { return humanize.Bytes.Uint64(p.RawPointTombstoneValueSize) })...)
749 :
750 1 : fmt.Fprintln(tw, "records\t\t\t\t\t\t\t\t")
751 1 : fmt.Fprintf(tw, " set\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
752 1 : propArgs(all, func(p *props) interface{} {
753 1 : return humanize.Count.Uint64(p.NumEntries - p.NumDeletions - p.NumMergeOperands)
754 1 : })...)
755 1 : fmt.Fprintf(tw, " delete\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
756 1 : propArgs(all, func(p *props) interface{} { return humanize.Count.Uint64(p.NumDeletions - p.NumRangeDeletions) })...)
757 1 : fmt.Fprintf(tw, " delete-sized\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
758 1 : propArgs(all, func(p *props) interface{} { return humanize.Count.Uint64(p.NumSizedDeletions) })...)
759 1 : fmt.Fprintf(tw, " range-delete\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
760 1 : propArgs(all, func(p *props) interface{} { return humanize.Count.Uint64(p.NumRangeDeletions) })...)
761 1 : fmt.Fprintf(tw, " range-key-sets\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
762 1 : propArgs(all, func(p *props) interface{} { return humanize.Count.Uint64(p.NumRangeKeySets) })...)
763 1 : fmt.Fprintf(tw, " range-key-unsets\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
764 1 : propArgs(all, func(p *props) interface{} { return humanize.Count.Uint64(p.NumRangeKeyUnSets) })...)
765 1 : fmt.Fprintf(tw, " range-key-deletes\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
766 1 : propArgs(all, func(p *props) interface{} { return humanize.Count.Uint64(p.NumRangeKeyDeletes) })...)
767 1 : fmt.Fprintf(tw, " merge\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
768 1 : propArgs(all, func(p *props) interface{} { return humanize.Count.Uint64(p.NumMergeOperands) })...)
769 1 : fmt.Fprintf(tw, " pinned\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
770 1 : propArgs(all, func(p *props) interface{} { return humanize.Count.Uint64(p.SnapshotPinnedKeys) })...)
771 :
772 1 : if err := tw.Flush(); err != nil {
773 0 : return err
774 0 : }
775 1 : return nil
776 : }()
777 1 : if err != nil {
778 1 : fmt.Fprintln(stderr, err)
779 1 : }
780 : }
781 :
782 1 : func (d *dbT) runSet(cmd *cobra.Command, args []string) {
783 1 : stderr := cmd.ErrOrStderr()
784 1 : db, err := d.openDB(args[0], nonReadOnly{})
785 1 : if err != nil {
786 0 : fmt.Fprintf(stderr, "%s\n", err)
787 0 : return
788 0 : }
789 1 : defer d.closeDB(stderr, db)
790 1 : var k, v key
791 1 : if err := k.Set(args[1]); err != nil {
792 0 : fmt.Fprintf(stderr, "%s\n", err)
793 0 : return
794 0 : }
795 1 : if err := v.Set(args[2]); err != nil {
796 0 : fmt.Fprintf(stderr, "%s\n", err)
797 0 : return
798 0 : }
799 :
800 1 : if err := db.Set(k, v, nil); err != nil {
801 0 : fmt.Fprintf(stderr, "%s\n", err)
802 0 : }
803 : }
804 :
805 1 : func propArgs(props []props, getProp func(*props) interface{}) []interface{} {
806 1 : args := make([]interface{}, 0, len(props))
807 1 : for _, p := range props {
808 1 : args = append(args, getProp(&p))
809 1 : }
810 1 : return args
811 : }
812 :
813 : type props struct {
814 : Count uint64
815 : SmallestSeqNum uint64
816 : LargestSeqNum uint64
817 : DataSize uint64
818 : FilterSize uint64
819 : IndexSize uint64
820 : NumDataBlocks uint64
821 : NumIndexBlocks uint64
822 : NumDeletions uint64
823 : NumSizedDeletions uint64
824 : NumEntries uint64
825 : NumMergeOperands uint64
826 : NumRangeDeletions uint64
827 : NumRangeKeySets uint64
828 : NumRangeKeyUnSets uint64
829 : NumRangeKeyDeletes uint64
830 : RawKeySize uint64
831 : RawPointTombstoneKeySize uint64
832 : RawPointTombstoneValueSize uint64
833 : RawValueSize uint64
834 : SnapshotPinnedKeys uint64
835 : SnapshotPinnedKeySize uint64
836 : SnapshotPinnedValueSize uint64
837 : TopLevelIndexSize uint64
838 : }
839 :
840 1 : func (p *props) update(o props) {
841 1 : p.Count += o.Count
842 1 : if o.SmallestSeqNum != 0 && (o.SmallestSeqNum < p.SmallestSeqNum || p.SmallestSeqNum == 0) {
843 1 : p.SmallestSeqNum = o.SmallestSeqNum
844 1 : }
845 1 : if o.LargestSeqNum > p.LargestSeqNum {
846 1 : p.LargestSeqNum = o.LargestSeqNum
847 1 : }
848 1 : p.DataSize += o.DataSize
849 1 : p.FilterSize += o.FilterSize
850 1 : p.IndexSize += o.IndexSize
851 1 : p.NumDataBlocks += o.NumDataBlocks
852 1 : p.NumIndexBlocks += o.NumIndexBlocks
853 1 : p.NumDeletions += o.NumDeletions
854 1 : p.NumSizedDeletions += o.NumSizedDeletions
855 1 : p.NumEntries += o.NumEntries
856 1 : p.NumMergeOperands += o.NumMergeOperands
857 1 : p.NumRangeDeletions += o.NumRangeDeletions
858 1 : p.NumRangeKeySets += o.NumRangeKeySets
859 1 : p.NumRangeKeyUnSets += o.NumRangeKeyUnSets
860 1 : p.NumRangeKeyDeletes += o.NumRangeKeyDeletes
861 1 : p.RawKeySize += o.RawKeySize
862 1 : p.RawPointTombstoneKeySize += o.RawPointTombstoneKeySize
863 1 : p.RawPointTombstoneValueSize += o.RawPointTombstoneValueSize
864 1 : p.RawValueSize += o.RawValueSize
865 1 : p.SnapshotPinnedKeySize += o.SnapshotPinnedKeySize
866 1 : p.SnapshotPinnedValueSize += o.SnapshotPinnedValueSize
867 1 : p.SnapshotPinnedKeys += o.SnapshotPinnedKeys
868 1 : p.TopLevelIndexSize += o.TopLevelIndexSize
869 : }
870 :
871 : func (d *dbT) addProps(
872 : objProvider objstorage.Provider, m manifest.PhysicalFileMeta, p *props,
873 1 : ) error {
874 1 : ctx := context.Background()
875 1 : f, err := objProvider.OpenForReading(ctx, base.FileTypeTable, m.FileBacking.DiskFileNum, objstorage.OpenOptions{})
876 1 : if err != nil {
877 0 : return err
878 0 : }
879 1 : r, err := sstable.NewReader(f, sstable.ReaderOptions{}, d.mergers, d.comparers)
880 1 : if err != nil {
881 0 : _ = f.Close()
882 0 : return err
883 0 : }
884 1 : p.update(props{
885 1 : Count: 1,
886 1 : SmallestSeqNum: m.SmallestSeqNum,
887 1 : LargestSeqNum: m.LargestSeqNum,
888 1 : DataSize: r.Properties.DataSize,
889 1 : FilterSize: r.Properties.FilterSize,
890 1 : IndexSize: r.Properties.IndexSize,
891 1 : NumDataBlocks: r.Properties.NumDataBlocks,
892 1 : NumIndexBlocks: 1 + r.Properties.IndexPartitions,
893 1 : NumDeletions: r.Properties.NumDeletions,
894 1 : NumSizedDeletions: r.Properties.NumSizedDeletions,
895 1 : NumEntries: r.Properties.NumEntries,
896 1 : NumMergeOperands: r.Properties.NumMergeOperands,
897 1 : NumRangeDeletions: r.Properties.NumRangeDeletions,
898 1 : NumRangeKeySets: r.Properties.NumRangeKeySets,
899 1 : NumRangeKeyUnSets: r.Properties.NumRangeKeyUnsets,
900 1 : NumRangeKeyDeletes: r.Properties.NumRangeKeyDels,
901 1 : RawKeySize: r.Properties.RawKeySize,
902 1 : RawPointTombstoneKeySize: r.Properties.RawPointTombstoneKeySize,
903 1 : RawPointTombstoneValueSize: r.Properties.RawPointTombstoneValueSize,
904 1 : RawValueSize: r.Properties.RawValueSize,
905 1 : SnapshotPinnedKeySize: r.Properties.SnapshotPinnedKeySize,
906 1 : SnapshotPinnedValueSize: r.Properties.SnapshotPinnedValueSize,
907 1 : SnapshotPinnedKeys: r.Properties.SnapshotPinnedKeys,
908 1 : TopLevelIndexSize: r.Properties.TopLevelIndexSize,
909 1 : })
910 1 : return r.Close()
911 : }
912 :
913 1 : func makePlural(singular string, count int64) string {
914 1 : if count > 1 {
915 1 : return fmt.Sprintf("%ss", singular)
916 1 : }
917 1 : return singular
918 : }
|
