Line data Source code
1 : // Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use
2 : // of this source code is governed by a BSD-style license that can be found in
3 : // the LICENSE file.
4 :
5 : package tool
6 :
7 : import (
8 : "context"
9 : "fmt"
10 : "io"
11 : "text/tabwriter"
12 :
13 : "github.com/cockroachdb/errors"
14 : "github.com/cockroachdb/errors/oserror"
15 : "github.com/cockroachdb/pebble"
16 : "github.com/cockroachdb/pebble/internal/base"
17 : "github.com/cockroachdb/pebble/internal/humanize"
18 : "github.com/cockroachdb/pebble/internal/manifest"
19 : "github.com/cockroachdb/pebble/objstorage"
20 : "github.com/cockroachdb/pebble/objstorage/objstorageprovider"
21 : "github.com/cockroachdb/pebble/record"
22 : "github.com/cockroachdb/pebble/sstable"
23 : "github.com/cockroachdb/pebble/tool/logs"
24 : "github.com/spf13/cobra"
25 : )
26 :
27 : // dbT implements db-level tools, including both configuration state and the
28 : // commands themselves.
29 : type dbT struct {
30 : Root *cobra.Command
31 : Check *cobra.Command
32 : Checkpoint *cobra.Command
33 : Get *cobra.Command
34 : Logs *cobra.Command
35 : LSM *cobra.Command
36 : Properties *cobra.Command
37 : Scan *cobra.Command
38 : Set *cobra.Command
39 : Space *cobra.Command
40 : IOBench *cobra.Command
41 :
42 : // Configuration.
43 : opts *pebble.Options
44 : comparers sstable.Comparers
45 : mergers sstable.Mergers
46 : openErrEnhancer func(error) error
47 : openOptions []OpenOption
48 :
49 : // Flags.
50 : comparerName string
51 : mergerName string
52 : fmtKey keyFormatter
53 : fmtValue valueFormatter
54 : start key
55 : end key
56 : count int64
57 : allLevels bool
58 : ioCount int
59 : ioParallelism int
60 : ioSizes string
61 : verbose bool
62 : }
63 :
64 : func newDB(
65 : opts *pebble.Options,
66 : comparers sstable.Comparers,
67 : mergers sstable.Mergers,
68 : openErrEnhancer func(error) error,
69 : openOptions []OpenOption,
70 1 : ) *dbT {
71 1 : d := &dbT{
72 1 : opts: opts,
73 1 : comparers: comparers,
74 1 : mergers: mergers,
75 1 : openErrEnhancer: openErrEnhancer,
76 1 : openOptions: openOptions,
77 1 : }
78 1 : d.fmtKey.mustSet("quoted")
79 1 : d.fmtValue.mustSet("[%x]")
80 1 :
81 1 : d.Root = &cobra.Command{
82 1 : Use: "db",
83 1 : Short: "DB introspection tools",
84 1 : }
85 1 : d.Check = &cobra.Command{
86 1 : Use: "check <dir>",
87 1 : Short: "verify checksums and metadata",
88 1 : Long: `
89 1 : Verify sstable, manifest, and WAL checksums. Requires that the specified
90 1 : database not be in use by another process.
91 1 : `,
92 1 : Args: cobra.ExactArgs(1),
93 1 : Run: d.runCheck,
94 1 : }
95 1 : d.Checkpoint = &cobra.Command{
96 1 : Use: "checkpoint <src-dir> <dest-dir>",
97 1 : Short: "create a checkpoint",
98 1 : Long: `
99 1 : Creates a Pebble checkpoint in the specified destination directory. A checkpoint
100 1 : is a point-in-time snapshot of DB state. Requires that the specified
101 1 : database not be in use by another process.
102 1 : `,
103 1 : Args: cobra.ExactArgs(2),
104 1 : Run: d.runCheckpoint,
105 1 : }
106 1 : d.Get = &cobra.Command{
107 1 : Use: "get <dir> <key>",
108 1 : Short: "get value for a key",
109 1 : Long: `
110 1 : Gets a value for a key, if it exists in DB. Prints a "not found" error if key
111 1 : does not exist. Requires that the specified database not be in use by another
112 1 : process.
113 1 : `,
114 1 : Args: cobra.ExactArgs(2),
115 1 : Run: d.runGet,
116 1 : }
117 1 : d.Logs = logs.NewCmd()
118 1 : d.LSM = &cobra.Command{
119 1 : Use: "lsm <dir>",
120 1 : Short: "print LSM structure",
121 1 : Long: `
122 1 : Print the structure of the LSM tree. Requires that the specified database not
123 1 : be in use by another process.
124 1 : `,
125 1 : Args: cobra.ExactArgs(1),
126 1 : Run: d.runLSM,
127 1 : }
128 1 : d.Properties = &cobra.Command{
129 1 : Use: "properties <dir>",
130 1 : Short: "print aggregated sstable properties",
131 1 : Long: `
132 1 : Print SSTable properties, aggregated per level of the LSM.
133 1 : `,
134 1 : Args: cobra.ExactArgs(1),
135 1 : Run: d.runProperties,
136 1 : }
137 1 : d.Scan = &cobra.Command{
138 1 : Use: "scan <dir>",
139 1 : Short: "print db records",
140 1 : Long: `
141 1 : Print the records in the DB. Requires that the specified database not be in use
142 1 : by another process.
143 1 : `,
144 1 : Args: cobra.ExactArgs(1),
145 1 : Run: d.runScan,
146 1 : }
147 1 : d.Set = &cobra.Command{
148 1 : Use: "set <dir> <key> <value>",
149 1 : Short: "set a value for a key",
150 1 : Long: `
151 1 : Adds a new key/value to the DB. Requires that the specified database
152 1 : not be in use by another process.
153 1 : `,
154 1 : Args: cobra.ExactArgs(3),
155 1 : Run: d.runSet,
156 1 : }
157 1 : d.Space = &cobra.Command{
158 1 : Use: "space <dir>",
159 1 : Short: "print filesystem space used",
160 1 : Long: `
161 1 : Print the estimated filesystem space usage for the inclusive-inclusive range
162 1 : specified by --start and --end. Requires that the specified database not be in
163 1 : use by another process.
164 1 : `,
165 1 : Args: cobra.ExactArgs(1),
166 1 : Run: d.runSpace,
167 1 : }
168 1 : d.IOBench = &cobra.Command{
169 1 : Use: "io-bench <dir>",
170 1 : Short: "perform sstable IO benchmark",
171 1 : Long: `
172 1 : Run a random IO workload with various IO sizes against the sstables in the
173 1 : specified database.
174 1 : `,
175 1 : Args: cobra.ExactArgs(1),
176 1 : Run: d.runIOBench,
177 1 : }
178 1 :
179 1 : d.Root.AddCommand(d.Check, d.Checkpoint, d.Get, d.Logs, d.LSM, d.Properties, d.Scan, d.Set, d.Space, d.IOBench)
180 1 : d.Root.PersistentFlags().BoolVarP(&d.verbose, "verbose", "v", false, "verbose output")
181 1 :
182 1 : for _, cmd := range []*cobra.Command{d.Check, d.Checkpoint, d.Get, d.LSM, d.Properties, d.Scan, d.Set, d.Space} {
183 1 : cmd.Flags().StringVar(
184 1 : &d.comparerName, "comparer", "", "comparer name (use default if empty)")
185 1 : cmd.Flags().StringVar(
186 1 : &d.mergerName, "merger", "", "merger name (use default if empty)")
187 1 : }
188 :
189 1 : for _, cmd := range []*cobra.Command{d.Scan, d.Space} {
190 1 : cmd.Flags().Var(
191 1 : &d.start, "start", "start key for the range")
192 1 : cmd.Flags().Var(
193 1 : &d.end, "end", "end key for the range")
194 1 : }
195 :
196 1 : d.Scan.Flags().Var(
197 1 : &d.fmtKey, "key", "key formatter")
198 1 : for _, cmd := range []*cobra.Command{d.Scan, d.Get} {
199 1 : cmd.Flags().Var(
200 1 : &d.fmtValue, "value", "value formatter")
201 1 : }
202 :
203 1 : d.Scan.Flags().Int64Var(
204 1 : &d.count, "count", 0, "key count for scan (0 is unlimited)")
205 1 :
206 1 : d.IOBench.Flags().BoolVar(
207 1 : &d.allLevels, "all-levels", false, "if set, benchmark all levels (default is only L5/L6)")
208 1 : d.IOBench.Flags().IntVar(
209 1 : &d.ioCount, "io-count", 10000, "number of IOs (per IO size) to benchmark")
210 1 : d.IOBench.Flags().IntVar(
211 1 : &d.ioParallelism, "io-parallelism", 16, "number of goroutines issuing IO")
212 1 : d.IOBench.Flags().StringVar(
213 1 : &d.ioSizes, "io-sizes-kb", "4,16,64,128,256,512,1024", "comma separated list of IO sizes in KB")
214 1 :
215 1 : return d
216 : }
217 :
218 1 : func (d *dbT) loadOptions(dir string) error {
219 1 : ls, err := d.opts.FS.List(dir)
220 1 : if err != nil || len(ls) == 0 {
221 1 : // NB: We don't return the error here as we prefer to return the error from
222 1 : // pebble.Open. Another way to put this is that a non-existent directory is
223 1 : // not a failure in loading the options.
224 1 : return nil
225 1 : }
226 :
227 1 : hooks := &pebble.ParseHooks{
228 1 : NewComparer: func(name string) (*pebble.Comparer, error) {
229 0 : if c := d.comparers[name]; c != nil {
230 0 : return c, nil
231 0 : }
232 0 : return nil, errors.Errorf("unknown comparer %q", errors.Safe(name))
233 : },
234 0 : NewMerger: func(name string) (*pebble.Merger, error) {
235 0 : if m := d.mergers[name]; m != nil {
236 0 : return m, nil
237 0 : }
238 0 : return nil, errors.Errorf("unknown merger %q", errors.Safe(name))
239 : },
240 0 : SkipUnknown: func(name, value string) bool {
241 0 : return true
242 0 : },
243 : }
244 :
245 : // TODO(peter): RocksDB sometimes leaves multiple OPTIONS files in
246 : // existence. We parse all of them as the comparer and merger shouldn't be
247 : // changing. We could parse only the first or the latest. Not clear if this
248 : // matters.
249 1 : var dbOpts pebble.Options
250 1 : for _, filename := range ls {
251 1 : ft, _, ok := base.ParseFilename(d.opts.FS, filename)
252 1 : if !ok {
253 1 : continue
254 : }
255 1 : switch ft {
256 1 : case base.FileTypeOptions:
257 1 : err := func() error {
258 1 : f, err := d.opts.FS.Open(d.opts.FS.PathJoin(dir, filename))
259 1 : if err != nil {
260 0 : return err
261 0 : }
262 1 : defer f.Close()
263 1 :
264 1 : data, err := io.ReadAll(f)
265 1 : if err != nil {
266 0 : return err
267 0 : }
268 :
269 1 : if err := dbOpts.Parse(string(data), hooks); err != nil {
270 1 : return err
271 1 : }
272 1 : return nil
273 : }()
274 1 : if err != nil {
275 1 : return err
276 1 : }
277 : }
278 : }
279 :
280 1 : if dbOpts.Comparer != nil {
281 1 : d.opts.Comparer = dbOpts.Comparer
282 1 : }
283 1 : if dbOpts.Merger != nil {
284 1 : d.opts.Merger = dbOpts.Merger
285 1 : }
286 1 : return nil
287 : }
288 :
289 : // OpenOption is an option that may be applied to the *pebble.Options before
290 : // calling pebble.Open.
291 : type OpenOption interface {
292 : Apply(dirname string, opts *pebble.Options)
293 : }
294 :
295 1 : func (d *dbT) openDB(dir string, openOptions ...OpenOption) (*pebble.DB, error) {
296 1 : db, err := d.openDBInternal(dir, openOptions...)
297 1 : if err != nil {
298 1 : if d.openErrEnhancer != nil {
299 1 : err = d.openErrEnhancer(err)
300 1 : }
301 1 : return nil, err
302 : }
303 1 : return db, nil
304 : }
305 :
306 1 : func (d *dbT) openDBInternal(dir string, openOptions ...OpenOption) (*pebble.DB, error) {
307 1 : if err := d.loadOptions(dir); err != nil {
308 1 : return nil, errors.Wrap(err, "error loading options")
309 1 : }
310 1 : if d.comparerName != "" {
311 1 : d.opts.Comparer = d.comparers[d.comparerName]
312 1 : if d.opts.Comparer == nil {
313 1 : return nil, errors.Errorf("unknown comparer %q", errors.Safe(d.comparerName))
314 1 : }
315 : }
316 1 : if d.mergerName != "" {
317 1 : d.opts.Merger = d.mergers[d.mergerName]
318 1 : if d.opts.Merger == nil {
319 1 : return nil, errors.Errorf("unknown merger %q", errors.Safe(d.mergerName))
320 1 : }
321 : }
322 1 : opts := *d.opts
323 1 : for _, opt := range openOptions {
324 1 : opt.Apply(dir, &opts)
325 1 : }
326 1 : for _, opt := range d.openOptions {
327 0 : opt.Apply(dir, &opts)
328 0 : }
329 1 : opts.Cache = pebble.NewCache(128 << 20 /* 128 MB */)
330 1 : defer opts.Cache.Unref()
331 1 : return pebble.Open(dir, &opts)
332 : }
333 :
334 1 : func (d *dbT) closeDB(stderr io.Writer, db *pebble.DB) {
335 1 : if err := db.Close(); err != nil {
336 0 : fmt.Fprintf(stderr, "%s\n", err)
337 0 : }
338 : }
339 :
340 1 : func (d *dbT) runCheck(cmd *cobra.Command, args []string) {
341 1 : stdout, stderr := cmd.OutOrStdout(), cmd.ErrOrStderr()
342 1 : db, err := d.openDB(args[0])
343 1 : if err != nil {
344 1 : fmt.Fprintf(stderr, "%s\n", err)
345 1 : return
346 1 : }
347 1 : defer d.closeDB(stderr, db)
348 1 :
349 1 : var stats pebble.CheckLevelsStats
350 1 : if err := db.CheckLevels(&stats); err != nil {
351 0 : fmt.Fprintf(stderr, "%s\n", err)
352 0 : }
353 1 : fmt.Fprintf(stdout, "checked %d %s and %d %s\n",
354 1 : stats.NumPoints, makePlural("point", stats.NumPoints), stats.NumTombstones, makePlural("tombstone", int64(stats.NumTombstones)))
355 : }
356 :
357 : type nonReadOnly struct{}
358 :
359 1 : func (n nonReadOnly) Apply(dirname string, opts *pebble.Options) {
360 1 : opts.ReadOnly = false
361 1 : // Increase the L0 compaction threshold to reduce the likelihood of an
362 1 : // unintended compaction changing test output.
363 1 : opts.L0CompactionThreshold = 10
364 1 : }
365 :
366 1 : func (d *dbT) runCheckpoint(cmd *cobra.Command, args []string) {
367 1 : stderr := cmd.ErrOrStderr()
368 1 : db, err := d.openDB(args[0], nonReadOnly{})
369 1 : if err != nil {
370 0 : fmt.Fprintf(stderr, "%s\n", err)
371 0 : return
372 0 : }
373 1 : defer d.closeDB(stderr, db)
374 1 : destDir := args[1]
375 1 :
376 1 : if err := db.Checkpoint(destDir); err != nil {
377 0 : fmt.Fprintf(stderr, "%s\n", err)
378 0 : }
379 : }
380 :
381 1 : func (d *dbT) runGet(cmd *cobra.Command, args []string) {
382 1 : stdout, stderr := cmd.OutOrStdout(), cmd.ErrOrStderr()
383 1 : db, err := d.openDB(args[0])
384 1 : if err != nil {
385 0 : fmt.Fprintf(stderr, "%s\n", err)
386 0 : return
387 0 : }
388 1 : defer d.closeDB(stderr, db)
389 1 : var k key
390 1 : if err := k.Set(args[1]); err != nil {
391 0 : fmt.Fprintf(stderr, "%s\n", err)
392 0 : return
393 0 : }
394 :
395 1 : val, closer, err := db.Get(k)
396 1 : if err != nil {
397 1 : fmt.Fprintf(stderr, "%s\n", err)
398 1 : return
399 1 : }
400 1 : defer func() {
401 1 : if closer != nil {
402 1 : closer.Close()
403 1 : }
404 : }()
405 1 : if val != nil {
406 1 : fmt.Fprintf(stdout, "%s\n", d.fmtValue.fn(k, val))
407 1 : }
408 : }
409 :
410 1 : func (d *dbT) runLSM(cmd *cobra.Command, args []string) {
411 1 : stdout, stderr := cmd.OutOrStdout(), cmd.ErrOrStderr()
412 1 : db, err := d.openDB(args[0])
413 1 : if err != nil {
414 1 : fmt.Fprintf(stderr, "%s\n", err)
415 1 : return
416 1 : }
417 1 : defer d.closeDB(stderr, db)
418 1 :
419 1 : fmt.Fprintf(stdout, "%s", db.Metrics())
420 : }
421 :
422 1 : func (d *dbT) runScan(cmd *cobra.Command, args []string) {
423 1 : stdout, stderr := cmd.OutOrStdout(), cmd.ErrOrStderr()
424 1 : db, err := d.openDB(args[0])
425 1 : if err != nil {
426 1 : fmt.Fprintf(stderr, "%s\n", err)
427 1 : return
428 1 : }
429 1 : defer d.closeDB(stderr, db)
430 1 :
431 1 : // Update the internal formatter if this comparator has one specified.
432 1 : if d.opts.Comparer != nil {
433 1 : d.fmtKey.setForComparer(d.opts.Comparer.Name, d.comparers)
434 1 : d.fmtValue.setForComparer(d.opts.Comparer.Name, d.comparers)
435 1 : }
436 :
437 1 : start := timeNow()
438 1 : fmtKeys := d.fmtKey.spec != "null"
439 1 : fmtValues := d.fmtValue.spec != "null"
440 1 : var count int64
441 1 :
442 1 : iter, _ := db.NewIter(&pebble.IterOptions{
443 1 : UpperBound: d.end,
444 1 : })
445 1 : for valid := iter.SeekGE(d.start); valid; valid = iter.Next() {
446 1 : if fmtKeys || fmtValues {
447 1 : needDelimiter := false
448 1 : if fmtKeys {
449 1 : fmt.Fprintf(stdout, "%s", d.fmtKey.fn(iter.Key()))
450 1 : needDelimiter = true
451 1 : }
452 1 : if fmtValues {
453 1 : if needDelimiter {
454 1 : stdout.Write([]byte{' '})
455 1 : }
456 1 : fmt.Fprintf(stdout, "%s", d.fmtValue.fn(iter.Key(), iter.Value()))
457 : }
458 1 : stdout.Write([]byte{'\n'})
459 : }
460 :
461 1 : count++
462 1 : if d.count > 0 && count >= d.count {
463 1 : break
464 : }
465 : }
466 :
467 1 : if err := iter.Close(); err != nil {
468 0 : fmt.Fprintf(stderr, "%s\n", err)
469 0 : }
470 :
471 1 : elapsed := timeNow().Sub(start)
472 1 :
473 1 : fmt.Fprintf(stdout, "scanned %d %s in %0.1fs\n",
474 1 : count, makePlural("record", count), elapsed.Seconds())
475 : }
476 :
477 1 : func (d *dbT) runSpace(cmd *cobra.Command, args []string) {
478 1 : stdout, stderr := cmd.OutOrStdout(), cmd.ErrOrStderr()
479 1 : db, err := d.openDB(args[0])
480 1 : if err != nil {
481 0 : fmt.Fprintf(stderr, "%s\n", err)
482 0 : return
483 0 : }
484 1 : defer d.closeDB(stdout, db)
485 1 :
486 1 : bytes, err := db.EstimateDiskUsage(d.start, d.end)
487 1 : if err != nil {
488 0 : fmt.Fprintf(stderr, "%s\n", err)
489 0 : return
490 0 : }
491 1 : fmt.Fprintf(stdout, "%d\n", bytes)
492 : }
493 :
494 1 : func (d *dbT) runProperties(cmd *cobra.Command, args []string) {
495 1 : stdout, stderr := cmd.OutOrStdout(), cmd.ErrOrStderr()
496 1 : dirname := args[0]
497 1 : err := func() error {
498 1 : desc, err := pebble.Peek(dirname, d.opts.FS)
499 1 : if err != nil {
500 1 : return err
501 1 : } else if !desc.Exists {
502 0 : return oserror.ErrNotExist
503 0 : }
504 1 : manifestFilename := d.opts.FS.PathBase(desc.ManifestFilename)
505 1 :
506 1 : // Replay the manifest to get the current version.
507 1 : f, err := d.opts.FS.Open(desc.ManifestFilename)
508 1 : if err != nil {
509 0 : return errors.Wrapf(err, "pebble: could not open MANIFEST file %q", manifestFilename)
510 0 : }
511 1 : defer f.Close()
512 1 :
513 1 : cmp := base.DefaultComparer
514 1 : var bve manifest.BulkVersionEdit
515 1 : bve.AddedByFileNum = make(map[base.FileNum]*manifest.FileMetadata)
516 1 : rr := record.NewReader(f, 0 /* logNum */)
517 1 : for {
518 1 : r, err := rr.Next()
519 1 : if err == io.EOF {
520 1 : break
521 : }
522 1 : if err != nil {
523 0 : return errors.Wrapf(err, "pebble: reading manifest %q", manifestFilename)
524 0 : }
525 1 : var ve manifest.VersionEdit
526 1 : err = ve.Decode(r)
527 1 : if err != nil {
528 0 : return err
529 0 : }
530 1 : if err := bve.Accumulate(&ve); err != nil {
531 0 : return err
532 0 : }
533 1 : if ve.ComparerName != "" {
534 1 : cmp = d.comparers[ve.ComparerName]
535 1 : d.fmtKey.setForComparer(ve.ComparerName, d.comparers)
536 1 : d.fmtValue.setForComparer(ve.ComparerName, d.comparers)
537 1 : }
538 : }
539 1 : v, err := bve.Apply(
540 1 : nil /* version */, cmp, d.opts.FlushSplitBytes,
541 1 : d.opts.Experimental.ReadCompactionRate,
542 1 : )
543 1 : if err != nil {
544 0 : return err
545 0 : }
546 :
547 1 : objProvider, err := objstorageprovider.Open(objstorageprovider.DefaultSettings(d.opts.FS, dirname))
548 1 : if err != nil {
549 0 : return err
550 0 : }
551 1 : defer objProvider.Close()
552 1 :
553 1 : // Load and aggregate sstable properties.
554 1 : tw := tabwriter.NewWriter(stdout, 2, 1, 4, ' ', 0)
555 1 : var total props
556 1 : var all []props
557 1 : for _, l := range v.Levels {
558 1 : iter := l.Iter()
559 1 : var level props
560 1 : for t := iter.First(); t != nil; t = iter.Next() {
561 1 : if t.Virtual {
562 0 : // TODO(bananabrick): Handle virtual sstables here. We don't
563 0 : // really have any stats or properties at this point. Maybe
564 0 : // we could approximate some of these properties for virtual
565 0 : // sstables by first grabbing properties for the backing
566 0 : // physical sstable, and then extrapolating.
567 0 : continue
568 : }
569 1 : err := d.addProps(objProvider, t.PhysicalMeta(), &level)
570 1 : if err != nil {
571 0 : return err
572 0 : }
573 : }
574 1 : all = append(all, level)
575 1 : total.update(level)
576 : }
577 1 : all = append(all, total)
578 1 :
579 1 : fmt.Fprintln(tw, "\tL0\tL1\tL2\tL3\tL4\tL5\tL6\tTOTAL")
580 1 :
581 1 : fmt.Fprintf(tw, "count\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\n",
582 1 : propArgs(all, func(p *props) interface{} { return p.Count })...)
583 :
584 1 : fmt.Fprintln(tw, "seq num\t\t\t\t\t\t\t\t")
585 1 : fmt.Fprintf(tw, " smallest\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\n",
586 1 : propArgs(all, func(p *props) interface{} { return p.SmallestSeqNum })...)
587 1 : fmt.Fprintf(tw, " largest\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\n",
588 1 : propArgs(all, func(p *props) interface{} { return p.LargestSeqNum })...)
589 :
590 1 : fmt.Fprintln(tw, "size\t\t\t\t\t\t\t\t")
591 1 : fmt.Fprintf(tw, " data\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
592 1 : propArgs(all, func(p *props) interface{} { return humanize.Bytes.Uint64(p.DataSize) })...)
593 1 : fmt.Fprintf(tw, " blocks\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\n",
594 1 : propArgs(all, func(p *props) interface{} { return p.NumDataBlocks })...)
595 1 : fmt.Fprintf(tw, " index\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
596 1 : propArgs(all, func(p *props) interface{} { return humanize.Bytes.Uint64(p.IndexSize) })...)
597 1 : fmt.Fprintf(tw, " blocks\t%d\t%d\t%d\t%d\t%d\t%d\t%d\t%d\n",
598 1 : propArgs(all, func(p *props) interface{} { return p.NumIndexBlocks })...)
599 1 : fmt.Fprintf(tw, " top-level\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
600 1 : propArgs(all, func(p *props) interface{} { return humanize.Bytes.Uint64(p.TopLevelIndexSize) })...)
601 1 : fmt.Fprintf(tw, " filter\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
602 1 : propArgs(all, func(p *props) interface{} { return humanize.Bytes.Uint64(p.FilterSize) })...)
603 1 : fmt.Fprintf(tw, " raw-key\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
604 1 : propArgs(all, func(p *props) interface{} { return humanize.Bytes.Uint64(p.RawKeySize) })...)
605 1 : fmt.Fprintf(tw, " raw-value\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
606 1 : propArgs(all, func(p *props) interface{} { return humanize.Bytes.Uint64(p.RawValueSize) })...)
607 1 : fmt.Fprintf(tw, " pinned-key\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
608 1 : propArgs(all, func(p *props) interface{} { return humanize.Bytes.Uint64(p.SnapshotPinnedKeySize) })...)
609 1 : fmt.Fprintf(tw, " pinned-value\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
610 1 : propArgs(all, func(p *props) interface{} { return humanize.Bytes.Uint64(p.SnapshotPinnedValueSize) })...)
611 1 : fmt.Fprintf(tw, " point-del-key-size\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
612 1 : propArgs(all, func(p *props) interface{} { return humanize.Bytes.Uint64(p.RawPointTombstoneKeySize) })...)
613 1 : fmt.Fprintf(tw, " point-del-value-size\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
614 1 : propArgs(all, func(p *props) interface{} { return humanize.Bytes.Uint64(p.RawPointTombstoneValueSize) })...)
615 :
616 1 : fmt.Fprintln(tw, "records\t\t\t\t\t\t\t\t")
617 1 : fmt.Fprintf(tw, " set\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
618 1 : propArgs(all, func(p *props) interface{} {
619 1 : return humanize.Count.Uint64(p.NumEntries - p.NumDeletions - p.NumMergeOperands)
620 1 : })...)
621 1 : fmt.Fprintf(tw, " delete\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
622 1 : propArgs(all, func(p *props) interface{} { return humanize.Count.Uint64(p.NumDeletions - p.NumRangeDeletions) })...)
623 1 : fmt.Fprintf(tw, " delete-sized\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
624 1 : propArgs(all, func(p *props) interface{} { return humanize.Count.Uint64(p.NumSizedDeletions) })...)
625 1 : fmt.Fprintf(tw, " range-delete\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
626 1 : propArgs(all, func(p *props) interface{} { return humanize.Count.Uint64(p.NumRangeDeletions) })...)
627 1 : fmt.Fprintf(tw, " range-key-sets\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
628 1 : propArgs(all, func(p *props) interface{} { return humanize.Count.Uint64(p.NumRangeKeySets) })...)
629 1 : fmt.Fprintf(tw, " range-key-unsets\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
630 1 : propArgs(all, func(p *props) interface{} { return humanize.Count.Uint64(p.NumRangeKeyUnSets) })...)
631 1 : fmt.Fprintf(tw, " range-key-deletes\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
632 1 : propArgs(all, func(p *props) interface{} { return humanize.Count.Uint64(p.NumRangeKeyDeletes) })...)
633 1 : fmt.Fprintf(tw, " merge\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
634 1 : propArgs(all, func(p *props) interface{} { return humanize.Count.Uint64(p.NumMergeOperands) })...)
635 1 : fmt.Fprintf(tw, " pinned\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
636 1 : propArgs(all, func(p *props) interface{} { return humanize.Count.Uint64(p.SnapshotPinnedKeys) })...)
637 :
638 1 : if err := tw.Flush(); err != nil {
639 0 : return err
640 0 : }
641 1 : return nil
642 : }()
643 1 : if err != nil {
644 1 : fmt.Fprintln(stderr, err)
645 1 : }
646 : }
647 :
648 1 : func (d *dbT) runSet(cmd *cobra.Command, args []string) {
649 1 : stderr := cmd.ErrOrStderr()
650 1 : db, err := d.openDB(args[0], nonReadOnly{})
651 1 : if err != nil {
652 0 : fmt.Fprintf(stderr, "%s\n", err)
653 0 : return
654 0 : }
655 1 : defer d.closeDB(stderr, db)
656 1 : var k, v key
657 1 : if err := k.Set(args[1]); err != nil {
658 0 : fmt.Fprintf(stderr, "%s\n", err)
659 0 : return
660 0 : }
661 1 : if err := v.Set(args[2]); err != nil {
662 0 : fmt.Fprintf(stderr, "%s\n", err)
663 0 : return
664 0 : }
665 :
666 1 : if err := db.Set(k, v, nil); err != nil {
667 0 : fmt.Fprintf(stderr, "%s\n", err)
668 0 : }
669 : }
670 :
671 1 : func propArgs(props []props, getProp func(*props) interface{}) []interface{} {
672 1 : args := make([]interface{}, 0, len(props))
673 1 : for _, p := range props {
674 1 : args = append(args, getProp(&p))
675 1 : }
676 1 : return args
677 : }
678 :
679 : type props struct {
680 : Count uint64
681 : SmallestSeqNum uint64
682 : LargestSeqNum uint64
683 : DataSize uint64
684 : FilterSize uint64
685 : IndexSize uint64
686 : NumDataBlocks uint64
687 : NumIndexBlocks uint64
688 : NumDeletions uint64
689 : NumSizedDeletions uint64
690 : NumEntries uint64
691 : NumMergeOperands uint64
692 : NumRangeDeletions uint64
693 : NumRangeKeySets uint64
694 : NumRangeKeyUnSets uint64
695 : NumRangeKeyDeletes uint64
696 : RawKeySize uint64
697 : RawPointTombstoneKeySize uint64
698 : RawPointTombstoneValueSize uint64
699 : RawValueSize uint64
700 : SnapshotPinnedKeys uint64
701 : SnapshotPinnedKeySize uint64
702 : SnapshotPinnedValueSize uint64
703 : TopLevelIndexSize uint64
704 : }
705 :
706 1 : func (p *props) update(o props) {
707 1 : p.Count += o.Count
708 1 : if o.SmallestSeqNum != 0 && (o.SmallestSeqNum < p.SmallestSeqNum || p.SmallestSeqNum == 0) {
709 1 : p.SmallestSeqNum = o.SmallestSeqNum
710 1 : }
711 1 : if o.LargestSeqNum > p.LargestSeqNum {
712 1 : p.LargestSeqNum = o.LargestSeqNum
713 1 : }
714 1 : p.DataSize += o.DataSize
715 1 : p.FilterSize += o.FilterSize
716 1 : p.IndexSize += o.IndexSize
717 1 : p.NumDataBlocks += o.NumDataBlocks
718 1 : p.NumIndexBlocks += o.NumIndexBlocks
719 1 : p.NumDeletions += o.NumDeletions
720 1 : p.NumSizedDeletions += o.NumSizedDeletions
721 1 : p.NumEntries += o.NumEntries
722 1 : p.NumMergeOperands += o.NumMergeOperands
723 1 : p.NumRangeDeletions += o.NumRangeDeletions
724 1 : p.NumRangeKeySets += o.NumRangeKeySets
725 1 : p.NumRangeKeyUnSets += o.NumRangeKeyUnSets
726 1 : p.NumRangeKeyDeletes += o.NumRangeKeyDeletes
727 1 : p.RawKeySize += o.RawKeySize
728 1 : p.RawPointTombstoneKeySize += o.RawPointTombstoneKeySize
729 1 : p.RawPointTombstoneValueSize += o.RawPointTombstoneValueSize
730 1 : p.RawValueSize += o.RawValueSize
731 1 : p.SnapshotPinnedKeySize += o.SnapshotPinnedKeySize
732 1 : p.SnapshotPinnedValueSize += o.SnapshotPinnedValueSize
733 1 : p.SnapshotPinnedKeys += o.SnapshotPinnedKeys
734 1 : p.TopLevelIndexSize += o.TopLevelIndexSize
735 : }
736 :
737 : func (d *dbT) addProps(
738 : objProvider objstorage.Provider, m manifest.PhysicalFileMeta, p *props,
739 1 : ) error {
740 1 : ctx := context.Background()
741 1 : f, err := objProvider.OpenForReading(ctx, base.FileTypeTable, m.FileBacking.DiskFileNum, objstorage.OpenOptions{})
742 1 : if err != nil {
743 0 : return err
744 0 : }
745 1 : r, err := sstable.NewReader(f, sstable.ReaderOptions{}, d.mergers, d.comparers)
746 1 : if err != nil {
747 0 : _ = f.Close()
748 0 : return err
749 0 : }
750 1 : p.update(props{
751 1 : Count: 1,
752 1 : SmallestSeqNum: m.SmallestSeqNum,
753 1 : LargestSeqNum: m.LargestSeqNum,
754 1 : DataSize: r.Properties.DataSize,
755 1 : FilterSize: r.Properties.FilterSize,
756 1 : IndexSize: r.Properties.IndexSize,
757 1 : NumDataBlocks: r.Properties.NumDataBlocks,
758 1 : NumIndexBlocks: 1 + r.Properties.IndexPartitions,
759 1 : NumDeletions: r.Properties.NumDeletions,
760 1 : NumSizedDeletions: r.Properties.NumSizedDeletions,
761 1 : NumEntries: r.Properties.NumEntries,
762 1 : NumMergeOperands: r.Properties.NumMergeOperands,
763 1 : NumRangeDeletions: r.Properties.NumRangeDeletions,
764 1 : NumRangeKeySets: r.Properties.NumRangeKeySets,
765 1 : NumRangeKeyUnSets: r.Properties.NumRangeKeyUnsets,
766 1 : NumRangeKeyDeletes: r.Properties.NumRangeKeyDels,
767 1 : RawKeySize: r.Properties.RawKeySize,
768 1 : RawPointTombstoneKeySize: r.Properties.RawPointTombstoneKeySize,
769 1 : RawPointTombstoneValueSize: r.Properties.RawPointTombstoneValueSize,
770 1 : RawValueSize: r.Properties.RawValueSize,
771 1 : SnapshotPinnedKeySize: r.Properties.SnapshotPinnedKeySize,
772 1 : SnapshotPinnedValueSize: r.Properties.SnapshotPinnedValueSize,
773 1 : SnapshotPinnedKeys: r.Properties.SnapshotPinnedKeys,
774 1 : TopLevelIndexSize: r.Properties.TopLevelIndexSize,
775 1 : })
776 1 : return r.Close()
777 : }
778 :
779 1 : func makePlural(singular string, count int64) string {
780 1 : if count > 1 {
781 1 : return fmt.Sprintf("%ss", singular)
782 1 : }
783 1 : return singular
784 : }
|
