Coverage Report

Created: 2026-11-12 07:20

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/postgres/src/backend/statistics/mcv.c
Line
Count
Source
1
/*-------------------------------------------------------------------------
2
 *
3
 * mcv.c
4
 *    POSTGRES multivariate MCV lists
5
 *
6
 *
7
 * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
8
 * Portions Copyright (c) 1994, Regents of the University of California
9
 *
10
 * IDENTIFICATION
11
 *    src/backend/statistics/mcv.c
12
 *
13
 *-------------------------------------------------------------------------
14
 */
15
#include "postgres.h"
16
17
#include "access/htup_details.h"
18
#include "catalog/pg_statistic_ext.h"
19
#include "catalog/pg_statistic_ext_data.h"
20
#include "fmgr.h"
21
#include "funcapi.h"
22
#include "nodes/nodeFuncs.h"
23
#include "statistics/extended_stats_internal.h"
24
#include "statistics/statistics.h"
25
#include "utils/array.h"
26
#include "utils/builtins.h"
27
#include "utils/fmgrprotos.h"
28
#include "utils/lsyscache.h"
29
#include "utils/selfuncs.h"
30
#include "utils/syscache.h"
31
#include "utils/typcache.h"
32
33
/*
34
 * Computes size of a serialized MCV item, depending on the number of
35
 * dimensions (columns) the statistic is defined on. The datum values are
36
 * stored in a separate array (deduplicated, to minimize the size), and
37
 * so the serialized items only store uint16 indexes into that array.
38
 *
39
 * Each serialized item stores (in this order):
40
 *
41
 * - indexes to values    (ndim * sizeof(uint16))
42
 * - null flags       (ndim * sizeof(bool))
43
 * - frequency        (sizeof(double))
44
 * - base_frequency     (sizeof(double))
45
 *
46
 * There is no alignment padding within an MCV item.
47
 * So in total each MCV item requires this many bytes:
48
 *
49
 *   ndim * (sizeof(uint16) + sizeof(bool)) + 2 * sizeof(double)
50
 */
51
#define ITEM_SIZE(ndims)  \
52
0
  ((ndims) * (sizeof(uint16) + sizeof(bool)) + 2 * sizeof(double))
53
54
/*
55
 * Used to compute size of serialized MCV list representation.
56
 */
57
#define MinSizeOfMCVList    \
58
0
  (VARHDRSZ + sizeof(uint32) * 3 + sizeof(AttrNumber))
59
60
/*
61
 * Size of the serialized MCV list, excluding the space needed for
62
 * deduplicated per-dimension values. The macro is meant to be used
63
 * when it's not yet safe to access the serialized info about amount
64
 * of data for each column.
65
 */
66
#define SizeOfMCVList(ndims,nitems) \
67
0
  ((MinSizeOfMCVList + sizeof(Oid) * (ndims)) + \
68
0
   ((ndims) * sizeof(DimensionInfo)) + \
69
0
   ((nitems) * ITEM_SIZE(ndims)))
70
71
static MultiSortSupport build_mss(StatsBuildData *data);
72
73
static SortItem *build_distinct_groups(int numrows, SortItem *items,
74
                     MultiSortSupport mss, int *ndistinct);
75
76
static SortItem **build_column_frequencies(SortItem *groups, int ngroups,
77
                       MultiSortSupport mss, int *ncounts);
78
79
static int  count_distinct_groups(int numrows, SortItem *items,
80
                  MultiSortSupport mss);
81
82
/*
83
 * Compute new value for bitmap item, considering whether it's used for
84
 * clauses connected by AND/OR.
85
 */
86
#define RESULT_MERGE(value, is_or, match) \
87
0
  ((is_or) ? ((value) || (match)) : ((value) && (match)))
88
89
/*
90
 * When processing a list of clauses, the bitmap item may get set to a value
91
 * such that additional clauses can't change it. For example, when processing
92
 * a list of clauses connected to AND, as soon as the item gets set to 'false'
93
 * then it'll remain like that. Similarly clauses connected by OR and 'true'.
94
 *
95
 * Returns true when the value in the bitmap can't change no matter how the
96
 * remaining clauses are evaluated.
97
 */
98
0
#define RESULT_IS_FINAL(value, is_or) ((is_or) ? (value) : (!(value)))
99
100
/*
101
 * get_mincount_for_mcv_list
102
 *    Determine the minimum number of times a value needs to appear in
103
 *    the sample for it to be included in the MCV list.
104
 *
105
 * We want to keep only values that appear sufficiently often in the
106
 * sample that it is reasonable to extrapolate their sample frequencies to
107
 * the entire table.  We do this by placing an upper bound on the relative
108
 * standard error of the sample frequency, so that any estimates the
109
 * planner generates from the MCV statistics can be expected to be
110
 * reasonably accurate.
111
 *
112
 * Since we are sampling without replacement, the sample frequency of a
113
 * particular value is described by a hypergeometric distribution.  A
114
 * common rule of thumb when estimating errors in this situation is to
115
 * require at least 10 instances of the value in the sample, in which case
116
 * the distribution can be approximated by a normal distribution, and
117
 * standard error analysis techniques can be applied.  Given a sample size
118
 * of n, a population size of N, and a sample frequency of p=cnt/n, the
119
 * standard error of the proportion p is given by
120
 *    SE = sqrt(p*(1-p)/n) * sqrt((N-n)/(N-1))
121
 * where the second term is the finite population correction.  To get
122
 * reasonably accurate planner estimates, we impose an upper bound on the
123
 * relative standard error of 20% -- i.e., SE/p < 0.2.  This 20% relative
124
 * error bound is fairly arbitrary, but has been found empirically to work
125
 * well.  Rearranging this formula gives a lower bound on the number of
126
 * instances of the value seen:
127
 *    cnt > n*(N-n) / (N-n+0.04*n*(N-1))
128
 * This bound is at most 25, and approaches 0 as n approaches 0 or N. The
129
 * case where n approaches 0 cannot happen in practice, since the sample
130
 * size is at least 300.  The case where n approaches N corresponds to
131
 * sampling the whole table, in which case it is reasonable to keep
132
 * the whole MCV list (have no lower bound), so it makes sense to apply
133
 * this formula for all inputs, even though the above derivation is
134
 * technically only valid when the right hand side is at least around 10.
135
 *
136
 * An alternative way to look at this formula is as follows -- assume that
137
 * the number of instances of the value seen scales up to the entire
138
 * table, so that the population count is K=N*cnt/n. Then the distribution
139
 * in the sample is a hypergeometric distribution parameterised by N, n
140
 * and K, and the bound above is mathematically equivalent to demanding
141
 * that the standard deviation of that distribution is less than 20% of
142
 * its mean.  Thus the relative errors in any planner estimates produced
143
 * from the MCV statistics are likely to be not too large.
144
 */
145
static double
146
get_mincount_for_mcv_list(int samplerows, double totalrows)
147
0
{
148
0
  double    n = samplerows;
149
0
  double    N = totalrows;
150
0
  double    numer,
151
0
        denom;
152
153
0
  numer = n * (N - n);
154
0
  denom = N - n + 0.04 * n * (N - 1);
155
156
  /* Guard against division by zero (possible if n = N = 1) */
157
0
  if (denom == 0.0)
158
0
    return 0.0;
159
160
0
  return numer / denom;
161
0
}
162
163
/*
164
 * Builds MCV list from the set of sampled rows.
165
 *
166
 * The algorithm is quite simple:
167
 *
168
 *     (1) sort the data (default collation, '<' for the data type)
169
 *
170
 *     (2) count distinct groups, decide how many to keep
171
 *
172
 *     (3) build the MCV list using the threshold determined in (2)
173
 *
174
 *     (4) remove rows represented by the MCV from the sample
175
 *
176
 */
177
MCVList *
178
statext_mcv_build(StatsBuildData *data, double totalrows, int stattarget)
179
0
{
180
0
  int     i,
181
0
        numattrs,
182
0
        numrows,
183
0
        ngroups,
184
0
        nitems;
185
0
  double    mincount;
186
0
  SortItem   *items;
187
0
  SortItem   *groups;
188
0
  MCVList    *mcvlist = NULL;
189
0
  MultiSortSupport mss;
190
191
  /* comparator for all the columns */
192
0
  mss = build_mss(data);
193
194
  /* sort the rows */
195
0
  items = build_sorted_items(data, &nitems, mss,
196
0
                 data->nattnums, data->attnums);
197
198
0
  if (!items)
199
0
    return NULL;
200
201
  /* for convenience */
202
0
  numattrs = data->nattnums;
203
0
  numrows = data->numrows;
204
205
  /* transform the sorted rows into groups (sorted by frequency) */
206
0
  groups = build_distinct_groups(nitems, items, mss, &ngroups);
207
208
  /*
209
   * The maximum number of MCV items to store, based on the statistics
210
   * target we computed for the statistics object (from the target set for
211
   * the object itself, attributes and the system default). In any case, we
212
   * can't keep more groups than we have available.
213
   */
214
0
  nitems = stattarget;
215
0
  if (nitems > ngroups)
216
0
    nitems = ngroups;
217
218
  /*
219
   * Decide how many items to keep in the MCV list. We can't use the same
220
   * algorithm as per-column MCV lists, because that only considers the
221
   * actual group frequency - but we're primarily interested in how the
222
   * actual frequency differs from the base frequency (product of simple
223
   * per-column frequencies, as if the columns were independent).
224
   *
225
   * Using the same algorithm might exclude items that are close to the
226
   * "average" frequency of the sample. But that does not say whether the
227
   * observed frequency is close to the base frequency or not. We also need
228
   * to consider unexpectedly uncommon items (again, compared to the base
229
   * frequency), and the single-column algorithm does not have to.
230
   *
231
   * We simply decide how many items to keep by computing the minimum count
232
   * using get_mincount_for_mcv_list() and then keep all items that seem to
233
   * be more common than that.
234
   */
235
0
  mincount = get_mincount_for_mcv_list(numrows, totalrows);
236
237
  /*
238
   * Walk the groups until we find the first group with a count below the
239
   * mincount threshold (the index of that group is the number of groups we
240
   * want to keep).
241
   */
242
0
  for (i = 0; i < nitems; i++)
243
0
  {
244
0
    if (groups[i].count < mincount)
245
0
    {
246
0
      nitems = i;
247
0
      break;
248
0
    }
249
0
  }
250
251
  /*
252
   * At this point, we know the number of items for the MCV list. There
253
   * might be none (for uniform distribution with many groups), and in that
254
   * case, there will be no MCV list. Otherwise, construct the MCV list.
255
   */
256
0
  if (nitems > 0)
257
0
  {
258
0
    int     j;
259
0
    SortItem  key;
260
0
    MultiSortSupport tmp;
261
262
    /* frequencies for values in each attribute */
263
0
    SortItem  **freqs;
264
0
    int      *nfreqs;
265
266
    /* used to search values */
267
0
    tmp = (MultiSortSupport) palloc(offsetof(MultiSortSupportData, ssup)
268
0
                    + sizeof(SortSupportData));
269
270
    /* compute frequencies for values in each column */
271
0
    nfreqs = palloc0_array(int, numattrs);
272
0
    freqs = build_column_frequencies(groups, ngroups, mss, nfreqs);
273
274
    /*
275
     * Allocate the MCV list structure, set the global parameters.
276
     */
277
0
    mcvlist = (MCVList *) palloc0(offsetof(MCVList, items) +
278
0
                    sizeof(MCVItem) * nitems);
279
280
0
    mcvlist->magic = STATS_MCV_MAGIC;
281
0
    mcvlist->type = STATS_MCV_TYPE_BASIC;
282
0
    mcvlist->ndimensions = numattrs;
283
0
    mcvlist->nitems = nitems;
284
285
    /* store info about data type OIDs */
286
0
    for (i = 0; i < numattrs; i++)
287
0
      mcvlist->types[i] = data->stats[i]->attrtypid;
288
289
    /* Copy the first chunk of groups into the result. */
290
0
    for (i = 0; i < nitems; i++)
291
0
    {
292
      /* just point to the proper place in the list */
293
0
      MCVItem    *item = &mcvlist->items[i];
294
295
0
      item->values = palloc_array(Datum, numattrs);
296
0
      item->isnull = palloc_array(bool, numattrs);
297
298
      /* copy values for the group */
299
0
      memcpy(item->values, groups[i].values, sizeof(Datum) * numattrs);
300
0
      memcpy(item->isnull, groups[i].isnull, sizeof(bool) * numattrs);
301
302
      /* groups should be sorted by frequency in descending order */
303
0
      Assert((i == 0) || (groups[i - 1].count >= groups[i].count));
304
305
      /* group frequency */
306
0
      item->frequency = (double) groups[i].count / numrows;
307
308
      /* base frequency, if the attributes were independent */
309
0
      item->base_frequency = 1.0;
310
0
      for (j = 0; j < numattrs; j++)
311
0
      {
312
0
        SortItem   *freq;
313
314
        /* single dimension */
315
0
        tmp->ndims = 1;
316
0
        tmp->ssup[0] = mss->ssup[j];
317
318
        /* fill search key */
319
0
        key.values = &groups[i].values[j];
320
0
        key.isnull = &groups[i].isnull[j];
321
322
0
        freq = (SortItem *) bsearch_arg(&key, freqs[j], nfreqs[j],
323
0
                        sizeof(SortItem),
324
0
                        multi_sort_compare, tmp);
325
326
0
        item->base_frequency *= ((double) freq->count) / numrows;
327
0
      }
328
0
    }
329
330
0
    pfree(nfreqs);
331
0
    pfree(freqs);
332
0
  }
333
334
0
  pfree(items);
335
0
  pfree(groups);
336
337
0
  return mcvlist;
338
0
}
339
340
/*
341
 * build_mss
342
 *    Build a MultiSortSupport for the given StatsBuildData.
343
 */
344
static MultiSortSupport
345
build_mss(StatsBuildData *data)
346
0
{
347
0
  int     i;
348
0
  int     numattrs = data->nattnums;
349
350
  /* Sort by multiple columns (using array of SortSupport) */
351
0
  MultiSortSupport mss = multi_sort_init(numattrs);
352
353
  /* prepare the sort functions for all the attributes */
354
0
  for (i = 0; i < numattrs; i++)
355
0
  {
356
0
    VacAttrStats *colstat = data->stats[i];
357
0
    TypeCacheEntry *type;
358
359
0
    type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR);
360
0
    if (type->lt_opr == InvalidOid) /* shouldn't happen */
361
0
      elog(ERROR, "cache lookup failed for ordering operator for type %u",
362
0
         colstat->attrtypid);
363
364
0
    multi_sort_add_dimension(mss, i, type->lt_opr, colstat->attrcollid);
365
0
  }
366
367
0
  return mss;
368
0
}
369
370
/*
371
 * count_distinct_groups
372
 *    Count distinct combinations of SortItems in the array.
373
 *
374
 * The array is assumed to be sorted according to the MultiSortSupport.
375
 */
376
static int
377
count_distinct_groups(int numrows, SortItem *items, MultiSortSupport mss)
378
0
{
379
0
  int     i;
380
0
  int     ndistinct;
381
382
0
  ndistinct = 1;
383
0
  for (i = 1; i < numrows; i++)
384
0
  {
385
    /* make sure the array really is sorted */
386
0
    Assert(multi_sort_compare(&items[i], &items[i - 1], mss) >= 0);
387
388
0
    if (multi_sort_compare(&items[i], &items[i - 1], mss) != 0)
389
0
      ndistinct += 1;
390
0
  }
391
392
0
  return ndistinct;
393
0
}
394
395
/*
396
 * compare_sort_item_count
397
 *    Comparator for sorting items by count (frequencies) in descending
398
 *    order.
399
 */
400
static int
401
compare_sort_item_count(const void *a, const void *b, void *arg)
402
0
{
403
0
  const SortItem *ia = a;
404
0
  const SortItem *ib = b;
405
406
0
  if (ia->count == ib->count)
407
0
    return 0;
408
0
  else if (ia->count > ib->count)
409
0
    return -1;
410
411
0
  return 1;
412
0
}
413
414
/*
415
 * build_distinct_groups
416
 *    Build an array of SortItems for distinct groups and counts matching
417
 *    items.
418
 *
419
 * The 'items' array is assumed to be sorted.
420
 */
421
static SortItem *
422
build_distinct_groups(int numrows, SortItem *items, MultiSortSupport mss,
423
            int *ndistinct)
424
0
{
425
0
  int     i,
426
0
        j;
427
0
  int     ngroups = count_distinct_groups(numrows, items, mss);
428
429
0
  SortItem   *groups = palloc_array(SortItem, ngroups);
430
431
0
  j = 0;
432
0
  groups[0] = items[0];
433
0
  groups[0].count = 1;
434
435
0
  for (i = 1; i < numrows; i++)
436
0
  {
437
    /* Assume sorted in ascending order. */
438
0
    Assert(multi_sort_compare(&items[i], &items[i - 1], mss) >= 0);
439
440
    /* New distinct group detected. */
441
0
    if (multi_sort_compare(&items[i], &items[i - 1], mss) != 0)
442
0
    {
443
0
      groups[++j] = items[i];
444
0
      groups[j].count = 0;
445
0
    }
446
447
0
    groups[j].count++;
448
0
  }
449
450
  /* ensure we filled the expected number of distinct groups */
451
0
  Assert(j + 1 == ngroups);
452
453
  /* Sort the distinct groups by frequency (in descending order). */
454
0
  qsort_interruptible(groups, ngroups, sizeof(SortItem),
455
0
            compare_sort_item_count, NULL);
456
457
0
  *ndistinct = ngroups;
458
0
  return groups;
459
0
}
460
461
/* compare sort items (single dimension) */
462
static int
463
sort_item_compare(const void *a, const void *b, void *arg)
464
0
{
465
0
  SortSupport ssup = (SortSupport) arg;
466
0
  const SortItem *ia = a;
467
0
  const SortItem *ib = b;
468
469
0
  return ApplySortComparator(ia->values[0], ia->isnull[0],
470
0
                 ib->values[0], ib->isnull[0],
471
0
                 ssup);
472
0
}
473
474
/*
475
 * build_column_frequencies
476
 *    Compute frequencies of values in each column.
477
 *
478
 * This returns an array of SortItems for each attribute the MCV is built
479
 * on, with a frequency (number of occurrences) for each value. This is
480
 * then used to compute "base" frequency of MCV items.
481
 *
482
 * All the memory is allocated in a single chunk, so that a single pfree
483
 * is enough to release it. We do not allocate space for values/isnull
484
 * arrays in the SortItems, because we can simply point into the input
485
 * groups directly.
486
 */
487
static SortItem **
488
build_column_frequencies(SortItem *groups, int ngroups,
489
             MultiSortSupport mss, int *ncounts)
490
0
{
491
0
  int     i,
492
0
        dim;
493
0
  SortItem  **result;
494
0
  char     *ptr;
495
496
0
  Assert(groups);
497
0
  Assert(ncounts);
498
499
  /* allocate arrays for all columns as a single chunk */
500
0
  ptr = palloc(MAXALIGN(sizeof(SortItem *) * mss->ndims) +
501
0
         mss->ndims * MAXALIGN(sizeof(SortItem) * ngroups));
502
503
  /* initial array of pointers */
504
0
  result = (SortItem **) ptr;
505
0
  ptr += MAXALIGN(sizeof(SortItem *) * mss->ndims);
506
507
0
  for (dim = 0; dim < mss->ndims; dim++)
508
0
  {
509
0
    SortSupport ssup = &mss->ssup[dim];
510
511
    /* array of values for a single column */
512
0
    result[dim] = (SortItem *) ptr;
513
0
    ptr += MAXALIGN(sizeof(SortItem) * ngroups);
514
515
    /* extract data for the dimension */
516
0
    for (i = 0; i < ngroups; i++)
517
0
    {
518
      /* point into the input groups */
519
0
      result[dim][i].values = &groups[i].values[dim];
520
0
      result[dim][i].isnull = &groups[i].isnull[dim];
521
0
      result[dim][i].count = groups[i].count;
522
0
    }
523
524
    /* sort the values, deduplicate */
525
0
    qsort_interruptible(result[dim], ngroups, sizeof(SortItem),
526
0
              sort_item_compare, ssup);
527
528
    /*
529
     * Identify distinct values, compute frequency (there might be
530
     * multiple MCV items containing this value, so we need to sum counts
531
     * from all of them.
532
     */
533
0
    ncounts[dim] = 1;
534
0
    for (i = 1; i < ngroups; i++)
535
0
    {
536
0
      if (sort_item_compare(&result[dim][i - 1], &result[dim][i], ssup) == 0)
537
0
      {
538
0
        result[dim][ncounts[dim] - 1].count += result[dim][i].count;
539
0
        continue;
540
0
      }
541
542
0
      result[dim][ncounts[dim]] = result[dim][i];
543
544
0
      ncounts[dim]++;
545
0
    }
546
0
  }
547
548
0
  return result;
549
0
}
550
551
/*
552
 * statext_mcv_load
553
 *    Load the MCV list for the indicated pg_statistic_ext_data tuple.
554
 */
555
MCVList *
556
statext_mcv_load(Oid mvoid, bool inh)
557
0
{
558
0
  MCVList    *result;
559
0
  bool    isnull;
560
0
  Datum   mcvlist;
561
0
  HeapTuple htup = SearchSysCache2(STATEXTDATASTXOID,
562
0
                     ObjectIdGetDatum(mvoid), BoolGetDatum(inh));
563
564
0
  if (!HeapTupleIsValid(htup))
565
0
    elog(ERROR, "cache lookup failed for statistics object %u", mvoid);
566
567
0
  mcvlist = SysCacheGetAttr(STATEXTDATASTXOID, htup,
568
0
                Anum_pg_statistic_ext_data_stxdmcv, &isnull);
569
570
0
  if (isnull)
571
0
    elog(ERROR,
572
0
       "requested statistics kind \"%c\" is not yet built for statistics object %u",
573
0
       STATS_EXT_MCV, mvoid);
574
575
0
  result = statext_mcv_deserialize(DatumGetByteaP(mcvlist));
576
577
0
  ReleaseSysCache(htup);
578
579
0
  return result;
580
0
}
581
582
583
/*
584
 * statext_mcv_serialize
585
 *    Serialize MCV list into a pg_mcv_list value.
586
 *
587
 * The MCV items may include values of various data types, and it's reasonable
588
 * to expect redundancy (values for a given attribute, repeated for multiple
589
 * MCV list items). So we deduplicate the values into arrays, and then replace
590
 * the values by indexes into those arrays.
591
 *
592
 * The overall structure of the serialized representation looks like this:
593
 *
594
 * +---------------+----------------+---------------------+-------+
595
 * | header fields | dimension info | deduplicated values | items |
596
 * +---------------+----------------+---------------------+-------+
597
 *
598
 * Where dimension info stores information about the type of the K-th
599
 * attribute (e.g. typlen, typbyval and length of deduplicated values).
600
 * Deduplicated values store deduplicated values for each attribute.  And
601
 * items store the actual MCV list items, with values replaced by indexes into
602
 * the arrays.
603
 *
604
 * When serializing the items, we use uint16 indexes. The number of MCV items
605
 * is limited by the statistics target (which is capped to 10k at the moment).
606
 * We might increase this to 65k and still fit into uint16, so there's a bit of
607
 * slack. Furthermore, this limit is on the number of distinct values per column,
608
 * and we usually have few of those (and various combinations of them for the
609
 * those MCV list). So uint16 seems fine for now.
610
 *
611
 * We don't really expect the serialization to save as much space as for
612
 * histograms, as we are not doing any bucket splits (which is the source
613
 * of high redundancy in histograms).
614
 *
615
 * TODO: Consider packing boolean flags (NULL) for each item into a single char
616
 * (or a longer type) instead of using an array of bool items.
617
 */
618
bytea *
619
statext_mcv_serialize(MCVList *mcvlist, VacAttrStats **stats)
620
0
{
621
0
  int     dim;
622
0
  int     ndims = mcvlist->ndimensions;
623
624
0
  SortSupport ssup;
625
0
  DimensionInfo *info;
626
627
0
  Size    total_length;
628
629
  /* serialized items (indexes into arrays, etc.) */
630
0
  bytea    *raw;
631
0
  char     *ptr;
632
0
  char     *endptr PG_USED_FOR_ASSERTS_ONLY;
633
634
  /* values per dimension (and number of non-NULL values) */
635
0
  Datum   **values = palloc0_array(Datum *, ndims);
636
0
  int      *counts = palloc0_array(int, ndims);
637
638
  /*
639
   * We'll include some rudimentary information about the attribute types
640
   * (length, by-val flag), so that we don't have to look them up while
641
   * deserializing the MCV list (we already have the type OID in the
642
   * header).  This is safe because when changing the type of the attribute
643
   * the statistics gets dropped automatically.  We need to store the info
644
   * about the arrays of deduplicated values anyway.
645
   */
646
0
  info = palloc0_array(DimensionInfo, ndims);
647
648
  /* sort support data for all attributes included in the MCV list */
649
0
  ssup = palloc0_array(SortSupportData, ndims);
650
651
  /* collect and deduplicate values for each dimension (attribute) */
652
0
  for (dim = 0; dim < ndims; dim++)
653
0
  {
654
0
    int     ndistinct;
655
0
    TypeCacheEntry *typentry;
656
657
    /*
658
     * Lookup the LT operator (can't get it from stats extra_data, as we
659
     * don't know how to interpret that - scalar vs. array etc.).
660
     */
661
0
    typentry = lookup_type_cache(stats[dim]->attrtypid, TYPECACHE_LT_OPR);
662
663
    /* copy important info about the data type (length, by-value) */
664
0
    info[dim].typlen = stats[dim]->attrtype->typlen;
665
0
    info[dim].typbyval = stats[dim]->attrtype->typbyval;
666
667
    /* allocate space for values in the attribute and collect them */
668
0
    values[dim] = palloc0_array(Datum, mcvlist->nitems);
669
670
0
    for (uint32 i = 0; i < mcvlist->nitems; i++)
671
0
    {
672
      /* skip NULL values - we don't need to deduplicate those */
673
0
      if (mcvlist->items[i].isnull[dim])
674
0
        continue;
675
676
      /* append the value at the end */
677
0
      values[dim][counts[dim]] = mcvlist->items[i].values[dim];
678
0
      counts[dim] += 1;
679
0
    }
680
681
    /* if there are just NULL values in this dimension, we're done */
682
0
    if (counts[dim] == 0)
683
0
      continue;
684
685
    /* sort and deduplicate the data */
686
0
    ssup[dim].ssup_cxt = CurrentMemoryContext;
687
0
    ssup[dim].ssup_collation = stats[dim]->attrcollid;
688
0
    ssup[dim].ssup_nulls_first = false;
689
690
0
    PrepareSortSupportFromOrderingOp(typentry->lt_opr, &ssup[dim]);
691
692
0
    qsort_interruptible(values[dim], counts[dim], sizeof(Datum),
693
0
              compare_scalars_simple, &ssup[dim]);
694
695
    /*
696
     * Walk through the array and eliminate duplicate values, but keep the
697
     * ordering (so that we can do a binary search later). We know there's
698
     * at least one item as (counts[dim] != 0), so we can skip the first
699
     * element.
700
     */
701
0
    ndistinct = 1;      /* number of distinct values */
702
0
    for (int i = 1; i < counts[dim]; i++)
703
0
    {
704
      /* expect sorted array */
705
0
      Assert(compare_datums_simple(values[dim][i - 1], values[dim][i], &ssup[dim]) <= 0);
706
707
      /* if the value is the same as the previous one, we can skip it */
708
0
      if (!compare_datums_simple(values[dim][i - 1], values[dim][i], &ssup[dim]))
709
0
        continue;
710
711
0
      values[dim][ndistinct] = values[dim][i];
712
0
      ndistinct += 1;
713
0
    }
714
715
    /* we must not exceed PG_UINT16_MAX, as we use uint16 indexes */
716
0
    Assert(ndistinct <= PG_UINT16_MAX);
717
718
    /*
719
     * Store additional info about the attribute - number of deduplicated
720
     * values, and also size of the serialized data. For fixed-length data
721
     * types this is trivial to compute, for varwidth types we need to
722
     * actually walk the array and sum the sizes.
723
     */
724
0
    info[dim].nvalues = ndistinct;
725
726
0
    if (info[dim].typbyval) /* by-value data types */
727
0
    {
728
0
      info[dim].nbytes = info[dim].nvalues * info[dim].typlen;
729
730
      /*
731
       * We copy the data into the MCV item during deserialization, so
732
       * we don't need to allocate any extra space.
733
       */
734
0
      info[dim].nbytes_aligned = 0;
735
0
    }
736
0
    else if (info[dim].typlen > 0) /* fixed-length by-ref */
737
0
    {
738
      /*
739
       * We don't care about alignment in the serialized data, so we
740
       * pack the data as much as possible. But we also track how much
741
       * data will be needed after deserialization, and in that case we
742
       * need to account for alignment of each item.
743
       *
744
       * Note: As the items are fixed-length, we could easily compute
745
       * this during deserialization, but we do it here anyway.
746
       */
747
0
      info[dim].nbytes = info[dim].nvalues * info[dim].typlen;
748
0
      info[dim].nbytes_aligned = info[dim].nvalues * MAXALIGN(info[dim].typlen);
749
0
    }
750
0
    else if (info[dim].typlen == -1) /* varlena */
751
0
    {
752
0
      info[dim].nbytes = 0;
753
0
      info[dim].nbytes_aligned = 0;
754
0
      for (int i = 0; i < info[dim].nvalues; i++)
755
0
      {
756
0
        Size    len;
757
758
        /*
759
         * For varlena values, we detoast the values and store the
760
         * length and data separately. We don't bother with alignment
761
         * here, which means that during deserialization we need to
762
         * copy the fields and only access the copies.
763
         */
764
0
        values[dim][i] = PointerGetDatum(PG_DETOAST_DATUM(values[dim][i]));
765
766
        /* serialized length (uint32 length + data) */
767
0
        len = VARSIZE_ANY_EXHDR(DatumGetPointer(values[dim][i]));
768
0
        info[dim].nbytes += sizeof(uint32); /* length */
769
0
        info[dim].nbytes += len;  /* value (no header) */
770
771
        /*
772
         * During deserialization we'll build regular varlena values
773
         * with full headers, and we need to align them properly.
774
         */
775
0
        info[dim].nbytes_aligned += MAXALIGN(VARHDRSZ + len);
776
0
      }
777
0
    }
778
0
    else if (info[dim].typlen == -2) /* cstring */
779
0
    {
780
0
      info[dim].nbytes = 0;
781
0
      info[dim].nbytes_aligned = 0;
782
0
      for (int i = 0; i < info[dim].nvalues; i++)
783
0
      {
784
0
        Size    len;
785
786
        /*
787
         * cstring is handled similar to varlena - first we store the
788
         * length as uint32 and then the data. We don't care about
789
         * alignment, which means that during deserialization we need
790
         * to copy the fields and only access the copies.
791
         */
792
793
        /* c-strings include terminator, so +1 byte */
794
0
        len = strlen(DatumGetCString(values[dim][i])) + 1;
795
0
        info[dim].nbytes += sizeof(uint32); /* length */
796
0
        info[dim].nbytes += len;  /* value */
797
798
        /* space needed for properly aligned deserialized copies */
799
0
        info[dim].nbytes_aligned += MAXALIGN(len);
800
0
      }
801
0
    }
802
803
    /* we know (count>0) so there must be some data */
804
0
    Assert(info[dim].nbytes > 0);
805
0
  }
806
807
  /*
808
   * Now we can finally compute how much space we'll actually need for the
809
   * whole serialized MCV list (varlena header, MCV header, dimension info
810
   * for each attribute, deduplicated values and items).
811
   */
812
0
  total_length = (3 * sizeof(uint32)) /* magic + type + nitems */
813
0
    + sizeof(AttrNumber)  /* ndimensions */
814
0
    + (ndims * sizeof(Oid));  /* attribute types */
815
816
  /* dimension info */
817
0
  total_length += ndims * sizeof(DimensionInfo);
818
819
  /* add space for the arrays of deduplicated values */
820
0
  for (int i = 0; i < ndims; i++)
821
0
    total_length += info[i].nbytes;
822
823
  /*
824
   * And finally account for the items (those are fixed-length, thanks to
825
   * replacing values with uint16 indexes into the deduplicated arrays).
826
   */
827
0
  total_length += mcvlist->nitems * ITEM_SIZE(dim);
828
829
  /*
830
   * Allocate space for the whole serialized MCV list (we'll skip bytes, so
831
   * we set them to zero to make the result more compressible).
832
   */
833
0
  raw = (bytea *) palloc0(VARHDRSZ + total_length);
834
0
  SET_VARSIZE(raw, VARHDRSZ + total_length);
835
836
0
  ptr = VARDATA(raw);
837
0
  endptr = ptr + total_length;
838
839
  /* copy the MCV list header fields, one by one */
840
0
  memcpy(ptr, &mcvlist->magic, sizeof(uint32));
841
0
  ptr += sizeof(uint32);
842
843
0
  memcpy(ptr, &mcvlist->type, sizeof(uint32));
844
0
  ptr += sizeof(uint32);
845
846
0
  memcpy(ptr, &mcvlist->nitems, sizeof(uint32));
847
0
  ptr += sizeof(uint32);
848
849
0
  memcpy(ptr, &mcvlist->ndimensions, sizeof(AttrNumber));
850
0
  ptr += sizeof(AttrNumber);
851
852
0
  memcpy(ptr, mcvlist->types, sizeof(Oid) * ndims);
853
0
  ptr += (sizeof(Oid) * ndims);
854
855
  /* store information about the attributes (data amounts, ...) */
856
0
  memcpy(ptr, info, sizeof(DimensionInfo) * ndims);
857
0
  ptr += sizeof(DimensionInfo) * ndims;
858
859
  /* Copy the deduplicated values for all attributes to the output. */
860
0
  for (dim = 0; dim < ndims; dim++)
861
0
  {
862
    /* remember the starting point for Asserts later */
863
0
    char     *start PG_USED_FOR_ASSERTS_ONLY = ptr;
864
865
0
    for (int i = 0; i < info[dim].nvalues; i++)
866
0
    {
867
0
      Datum   value = values[dim][i];
868
869
0
      if (info[dim].typbyval) /* passed by value */
870
0
      {
871
0
        Datum   tmp;
872
873
        /*
874
         * For byval types, we need to copy just the significant bytes
875
         * - we can't use memcpy directly, as that assumes
876
         * little-endian behavior.  store_att_byval does almost what
877
         * we need, but it requires a properly aligned buffer - the
878
         * output buffer does not guarantee that. So we simply use a
879
         * local Datum variable (which guarantees proper alignment),
880
         * and then copy the value from it.
881
         */
882
0
        store_att_byval(&tmp, value, info[dim].typlen);
883
884
0
        memcpy(ptr, &tmp, info[dim].typlen);
885
0
        ptr += info[dim].typlen;
886
0
      }
887
0
      else if (info[dim].typlen > 0) /* passed by reference */
888
0
      {
889
        /* no special alignment needed, treated as char array */
890
0
        memcpy(ptr, DatumGetPointer(value), info[dim].typlen);
891
0
        ptr += info[dim].typlen;
892
0
      }
893
0
      else if (info[dim].typlen == -1) /* varlena */
894
0
      {
895
0
        uint32    len = VARSIZE_ANY_EXHDR(DatumGetPointer(value));
896
897
        /* copy the length */
898
0
        memcpy(ptr, &len, sizeof(uint32));
899
0
        ptr += sizeof(uint32);
900
901
        /* data from the varlena value (without the header) */
902
0
        memcpy(ptr, VARDATA_ANY(DatumGetPointer(value)), len);
903
0
        ptr += len;
904
0
      }
905
0
      else if (info[dim].typlen == -2) /* cstring */
906
0
      {
907
0
        uint32    len = (uint32) strlen(DatumGetCString(value)) + 1;
908
909
        /* copy the length */
910
0
        memcpy(ptr, &len, sizeof(uint32));
911
0
        ptr += sizeof(uint32);
912
913
        /* value */
914
0
        memcpy(ptr, DatumGetCString(value), len);
915
0
        ptr += len;
916
0
      }
917
918
      /* no underflows or overflows */
919
0
      Assert((ptr > start) && ((ptr - start) <= info[dim].nbytes));
920
0
    }
921
922
    /* we should get exactly nbytes of data for this dimension */
923
0
    Assert((ptr - start) == info[dim].nbytes);
924
0
  }
925
926
  /* Serialize the items, with uint16 indexes instead of the values. */
927
0
  for (uint32 i = 0; i < mcvlist->nitems; i++)
928
0
  {
929
0
    MCVItem    *mcvitem = &mcvlist->items[i];
930
931
    /* don't write beyond the allocated space */
932
0
    Assert(ptr <= (endptr - ITEM_SIZE(dim)));
933
934
    /* copy NULL and frequency flags into the serialized MCV */
935
0
    memcpy(ptr, mcvitem->isnull, sizeof(bool) * ndims);
936
0
    ptr += sizeof(bool) * ndims;
937
938
0
    memcpy(ptr, &mcvitem->frequency, sizeof(double));
939
0
    ptr += sizeof(double);
940
941
0
    memcpy(ptr, &mcvitem->base_frequency, sizeof(double));
942
0
    ptr += sizeof(double);
943
944
    /* store the indexes last */
945
0
    for (dim = 0; dim < ndims; dim++)
946
0
    {
947
0
      uint16    index = 0;
948
0
      Datum    *value;
949
950
      /* do the lookup only for non-NULL values */
951
0
      if (!mcvitem->isnull[dim])
952
0
      {
953
0
        value = (Datum *) bsearch_arg(&mcvitem->values[dim], values[dim],
954
0
                        info[dim].nvalues, sizeof(Datum),
955
0
                        compare_scalars_simple, &ssup[dim]);
956
957
0
        Assert(value != NULL);  /* serialization or deduplication
958
                     * error */
959
960
        /* compute index within the deduplicated array */
961
0
        index = (uint16) (value - values[dim]);
962
963
        /* check the index is within expected bounds */
964
0
        Assert(index < info[dim].nvalues);
965
0
      }
966
967
      /* copy the index into the serialized MCV */
968
0
      memcpy(ptr, &index, sizeof(uint16));
969
0
      ptr += sizeof(uint16);
970
0
    }
971
972
    /* make sure we don't overflow the allocated value */
973
0
    Assert(ptr <= endptr);
974
0
  }
975
976
  /* at this point we expect to match the total_length exactly */
977
0
  Assert(ptr == endptr);
978
979
0
  pfree(values);
980
0
  pfree(counts);
981
982
0
  return raw;
983
0
}
984
985
/*
986
 * statext_mcv_deserialize
987
 *    Reads serialized MCV list into MCVList structure.
988
 *
989
 * All the memory needed by the MCV list is allocated as a single chunk, so
990
 * it's possible to simply pfree() it at once.
991
 */
992
MCVList *
993
statext_mcv_deserialize(bytea *data)
994
0
{
995
0
  int     dim,
996
0
        i;
997
0
  Size    expected_size;
998
0
  MCVList    *mcvlist;
999
0
  char     *raw;
1000
0
  char     *ptr;
1001
0
  char     *endptr PG_USED_FOR_ASSERTS_ONLY;
1002
1003
0
  int     ndims,
1004
0
        nitems;
1005
0
  DimensionInfo *info = NULL;
1006
1007
  /* local allocation buffer (used only for deserialization) */
1008
0
  Datum   **map = NULL;
1009
1010
  /* MCV list */
1011
0
  Size    mcvlen;
1012
1013
  /* buffer used for the result */
1014
0
  Size    datalen;
1015
0
  char     *dataptr;
1016
0
  char     *valuesptr;
1017
0
  char     *isnullptr;
1018
1019
0
  if (data == NULL)
1020
0
    return NULL;
1021
1022
  /*
1023
   * We can't possibly deserialize a MCV list if there's not even a complete
1024
   * header. We need an explicit formula here, because we serialize the
1025
   * header fields one by one, so we need to ignore struct alignment.
1026
   */
1027
0
  if (VARSIZE_ANY(data) < MinSizeOfMCVList)
1028
0
    elog(ERROR, "invalid MCV size %zu (expected at least %zu)",
1029
0
       VARSIZE_ANY(data), MinSizeOfMCVList);
1030
1031
  /* read the MCV list header */
1032
0
  mcvlist = (MCVList *) palloc0(offsetof(MCVList, items));
1033
1034
  /* pointer to the data part (skip the varlena header) */
1035
0
  raw = (char *) data;
1036
0
  ptr = VARDATA_ANY(raw);
1037
0
  endptr = raw + VARSIZE_ANY(data);
1038
1039
  /* get the header and perform further sanity checks */
1040
0
  memcpy(&mcvlist->magic, ptr, sizeof(uint32));
1041
0
  ptr += sizeof(uint32);
1042
1043
0
  memcpy(&mcvlist->type, ptr, sizeof(uint32));
1044
0
  ptr += sizeof(uint32);
1045
1046
0
  memcpy(&mcvlist->nitems, ptr, sizeof(uint32));
1047
0
  ptr += sizeof(uint32);
1048
1049
0
  memcpy(&mcvlist->ndimensions, ptr, sizeof(AttrNumber));
1050
0
  ptr += sizeof(AttrNumber);
1051
1052
0
  if (mcvlist->magic != STATS_MCV_MAGIC)
1053
0
    elog(ERROR, "invalid MCV magic %u (expected %u)",
1054
0
       mcvlist->magic, STATS_MCV_MAGIC);
1055
1056
0
  if (mcvlist->type != STATS_MCV_TYPE_BASIC)
1057
0
    elog(ERROR, "invalid MCV type %u (expected %u)",
1058
0
       mcvlist->type, STATS_MCV_TYPE_BASIC);
1059
1060
0
  if (mcvlist->ndimensions == 0)
1061
0
    elog(ERROR, "invalid zero-length dimension array in MCVList");
1062
0
  else if ((mcvlist->ndimensions > STATS_MAX_DIMENSIONS) ||
1063
0
       (mcvlist->ndimensions < 0))
1064
0
    elog(ERROR, "invalid length (%d) dimension array in MCVList",
1065
0
       mcvlist->ndimensions);
1066
1067
0
  if (mcvlist->nitems == 0)
1068
0
    elog(ERROR, "invalid zero-length item array in MCVList");
1069
0
  else if (mcvlist->nitems > STATS_MCVLIST_MAX_ITEMS)
1070
0
    elog(ERROR, "invalid length (%u) item array in MCVList",
1071
0
       mcvlist->nitems);
1072
1073
0
  nitems = mcvlist->nitems;
1074
0
  ndims = mcvlist->ndimensions;
1075
1076
  /*
1077
   * Check amount of data including DimensionInfo for all dimensions and
1078
   * also the serialized items (including uint16 indexes). Also, walk
1079
   * through the dimension information and add it to the sum.
1080
   */
1081
0
  expected_size = SizeOfMCVList(ndims, nitems);
1082
1083
  /*
1084
   * Check that we have at least the dimension and info records, along with
1085
   * the items. We don't know the size of the serialized values yet. We need
1086
   * to do this check first, before accessing the dimension info.
1087
   */
1088
0
  if (VARSIZE_ANY(data) < expected_size)
1089
0
    elog(ERROR, "invalid MCV size %zu (expected %zu)",
1090
0
       VARSIZE_ANY(data), expected_size);
1091
1092
  /* Now copy the array of type Oids. */
1093
0
  memcpy(mcvlist->types, ptr, sizeof(Oid) * ndims);
1094
0
  ptr += (sizeof(Oid) * ndims);
1095
1096
  /* Now it's safe to access the dimension info. */
1097
0
  info = palloc_array(DimensionInfo, ndims);
1098
1099
0
  memcpy(info, ptr, ndims * sizeof(DimensionInfo));
1100
0
  ptr += (ndims * sizeof(DimensionInfo));
1101
1102
  /* account for the value arrays */
1103
0
  for (dim = 0; dim < ndims; dim++)
1104
0
  {
1105
    /*
1106
     * XXX I wonder if we can/should rely on asserts here. Maybe those
1107
     * checks should be done every time?
1108
     */
1109
0
    Assert(info[dim].nvalues >= 0);
1110
0
    Assert(info[dim].nbytes >= 0);
1111
1112
0
    expected_size += info[dim].nbytes;
1113
0
  }
1114
1115
  /*
1116
   * Now we know the total expected MCV size, including all the pieces
1117
   * (header, dimension info. items and deduplicated data). So do the final
1118
   * check on size.
1119
   */
1120
0
  if (VARSIZE_ANY(data) != expected_size)
1121
0
    elog(ERROR, "invalid MCV size %zu (expected %zu)",
1122
0
       VARSIZE_ANY(data), expected_size);
1123
1124
  /*
1125
   * We need an array of Datum values for each dimension, so that we can
1126
   * easily translate the uint16 indexes later. We also need a top-level
1127
   * array of pointers to those per-dimension arrays.
1128
   *
1129
   * While allocating the arrays for dimensions, compute how much space we
1130
   * need for a copy of the by-ref data, as we can't simply point to the
1131
   * original values (it might go away).
1132
   */
1133
0
  datalen = 0;        /* space for by-ref data */
1134
0
  map = palloc_array(Datum *, ndims);
1135
1136
0
  for (dim = 0; dim < ndims; dim++)
1137
0
  {
1138
0
    map[dim] = palloc_array(Datum, info[dim].nvalues);
1139
1140
    /* space needed for a copy of data for by-ref types */
1141
0
    datalen += info[dim].nbytes_aligned;
1142
0
  }
1143
1144
  /*
1145
   * Now resize the MCV list so that the allocation includes all the data.
1146
   *
1147
   * Allocate space for a copy of the data, as we can't simply reference the
1148
   * serialized data - it's not aligned properly, and it may disappear while
1149
   * we're still using the MCV list, e.g. due to catcache release.
1150
   *
1151
   * We do care about alignment here, because we will allocate all the
1152
   * pieces at once, but then use pointers to different parts.
1153
   */
1154
0
  mcvlen = MAXALIGN(offsetof(MCVList, items) + (sizeof(MCVItem) * nitems));
1155
1156
  /* arrays of values and isnull flags for all MCV items */
1157
0
  mcvlen += nitems * MAXALIGN(sizeof(Datum) * ndims);
1158
0
  mcvlen += nitems * MAXALIGN(sizeof(bool) * ndims);
1159
1160
  /* we don't quite need to align this, but it makes some asserts easier */
1161
0
  mcvlen += MAXALIGN(datalen);
1162
1163
  /* now resize the deserialized MCV list, and compute pointers to parts */
1164
0
  mcvlist = repalloc(mcvlist, mcvlen);
1165
1166
  /* pointer to the beginning of values/isnull arrays */
1167
0
  valuesptr = (char *) mcvlist
1168
0
    + MAXALIGN(offsetof(MCVList, items) + (sizeof(MCVItem) * nitems));
1169
1170
0
  isnullptr = valuesptr + (nitems * MAXALIGN(sizeof(Datum) * ndims));
1171
1172
0
  dataptr = isnullptr + (nitems * MAXALIGN(sizeof(bool) * ndims));
1173
1174
  /*
1175
   * Build mapping (index => value) for translating the serialized data into
1176
   * the in-memory representation.
1177
   */
1178
0
  for (dim = 0; dim < ndims; dim++)
1179
0
  {
1180
    /* remember start position in the input array */
1181
0
    char     *start PG_USED_FOR_ASSERTS_ONLY = ptr;
1182
1183
0
    if (info[dim].typbyval)
1184
0
    {
1185
      /* for by-val types we simply copy data into the mapping */
1186
0
      for (i = 0; i < info[dim].nvalues; i++)
1187
0
      {
1188
0
        Datum   v = 0;
1189
1190
0
        memcpy(&v, ptr, info[dim].typlen);
1191
0
        ptr += info[dim].typlen;
1192
1193
0
        map[dim][i] = fetch_att(&v, true, info[dim].typlen);
1194
1195
        /* no under/overflow of input array */
1196
0
        Assert(ptr <= (start + info[dim].nbytes));
1197
0
      }
1198
0
    }
1199
0
    else
1200
0
    {
1201
      /* for by-ref types we need to also make a copy of the data */
1202
1203
      /* passed by reference, but fixed length (name, tid, ...) */
1204
0
      if (info[dim].typlen > 0)
1205
0
      {
1206
0
        for (i = 0; i < info[dim].nvalues; i++)
1207
0
        {
1208
0
          memcpy(dataptr, ptr, info[dim].typlen);
1209
0
          ptr += info[dim].typlen;
1210
1211
          /* just point into the array */
1212
0
          map[dim][i] = PointerGetDatum(dataptr);
1213
0
          dataptr += MAXALIGN(info[dim].typlen);
1214
0
        }
1215
0
      }
1216
0
      else if (info[dim].typlen == -1)
1217
0
      {
1218
        /* varlena */
1219
0
        for (i = 0; i < info[dim].nvalues; i++)
1220
0
        {
1221
0
          uint32    len;
1222
1223
          /* read the uint32 length */
1224
0
          memcpy(&len, ptr, sizeof(uint32));
1225
0
          ptr += sizeof(uint32);
1226
1227
          /* the length is data-only */
1228
0
          SET_VARSIZE(dataptr, len + VARHDRSZ);
1229
0
          memcpy(VARDATA(dataptr), ptr, len);
1230
0
          ptr += len;
1231
1232
          /* just point into the array */
1233
0
          map[dim][i] = PointerGetDatum(dataptr);
1234
1235
          /* skip to place of the next deserialized value */
1236
0
          dataptr += MAXALIGN(len + VARHDRSZ);
1237
0
        }
1238
0
      }
1239
0
      else if (info[dim].typlen == -2)
1240
0
      {
1241
        /* cstring */
1242
0
        for (i = 0; i < info[dim].nvalues; i++)
1243
0
        {
1244
0
          uint32    len;
1245
1246
0
          memcpy(&len, ptr, sizeof(uint32));
1247
0
          ptr += sizeof(uint32);
1248
1249
0
          memcpy(dataptr, ptr, len);
1250
0
          ptr += len;
1251
1252
          /* just point into the array */
1253
0
          map[dim][i] = PointerGetDatum(dataptr);
1254
0
          dataptr += MAXALIGN(len);
1255
0
        }
1256
0
      }
1257
1258
      /* no under/overflow of input array */
1259
0
      Assert(ptr <= (start + info[dim].nbytes));
1260
1261
      /* no overflow of the output mcv value */
1262
0
      Assert(dataptr <= ((char *) mcvlist + mcvlen));
1263
0
    }
1264
1265
    /* check we consumed input data for this dimension exactly */
1266
0
    Assert(ptr == (start + info[dim].nbytes));
1267
0
  }
1268
1269
  /* we should have also filled the MCV list exactly */
1270
0
  Assert(dataptr == ((char *) mcvlist + mcvlen));
1271
1272
  /* deserialize the MCV items and translate the indexes to Datums */
1273
0
  for (i = 0; i < nitems; i++)
1274
0
  {
1275
0
    MCVItem    *item = &mcvlist->items[i];
1276
1277
0
    item->values = (Datum *) valuesptr;
1278
0
    valuesptr += MAXALIGN(sizeof(Datum) * ndims);
1279
1280
0
    item->isnull = (bool *) isnullptr;
1281
0
    isnullptr += MAXALIGN(sizeof(bool) * ndims);
1282
1283
0
    memcpy(item->isnull, ptr, sizeof(bool) * ndims);
1284
0
    ptr += sizeof(bool) * ndims;
1285
1286
0
    memcpy(&item->frequency, ptr, sizeof(double));
1287
0
    ptr += sizeof(double);
1288
1289
0
    memcpy(&item->base_frequency, ptr, sizeof(double));
1290
0
    ptr += sizeof(double);
1291
1292
    /* finally translate the indexes (for non-NULL only) */
1293
0
    for (dim = 0; dim < ndims; dim++)
1294
0
    {
1295
0
      uint16    index;
1296
1297
0
      memcpy(&index, ptr, sizeof(uint16));
1298
0
      ptr += sizeof(uint16);
1299
1300
0
      if (item->isnull[dim])
1301
0
        continue;
1302
1303
0
      item->values[dim] = map[dim][index];
1304
0
    }
1305
1306
    /* check we're not overflowing the input */
1307
0
    Assert(ptr <= endptr);
1308
0
  }
1309
1310
  /* check that we processed all the data */
1311
0
  Assert(ptr == endptr);
1312
1313
  /* release the buffers used for mapping */
1314
0
  for (dim = 0; dim < ndims; dim++)
1315
0
    pfree(map[dim]);
1316
1317
0
  pfree(map);
1318
1319
0
  return mcvlist;
1320
0
}
1321
1322
/*
1323
 * SRF with details about buckets of a histogram:
1324
 *
1325
 * - item ID (0...nitems)
1326
 * - values (string array)
1327
 * - nulls only (boolean array)
1328
 * - frequency (double precision)
1329
 * - base_frequency (double precision)
1330
 *
1331
 * The input is the OID of the statistics, and there are no rows returned if
1332
 * the statistics contains no histogram.
1333
 */
1334
Datum
1335
pg_stats_ext_mcvlist_items(PG_FUNCTION_ARGS)
1336
0
{
1337
0
  FuncCallContext *funcctx;
1338
1339
  /* stuff done only on the first call of the function */
1340
0
  if (SRF_IS_FIRSTCALL())
1341
0
  {
1342
0
    MemoryContext oldcontext;
1343
0
    MCVList    *mcvlist;
1344
0
    TupleDesc tupdesc;
1345
1346
    /* create a function context for cross-call persistence */
1347
0
    funcctx = SRF_FIRSTCALL_INIT();
1348
1349
    /* switch to memory context appropriate for multiple function calls */
1350
0
    oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
1351
1352
0
    mcvlist = statext_mcv_deserialize(PG_GETARG_BYTEA_P(0));
1353
1354
0
    funcctx->user_fctx = mcvlist;
1355
1356
    /* total number of tuples to be returned */
1357
0
    funcctx->max_calls = 0;
1358
0
    if (funcctx->user_fctx != NULL)
1359
0
      funcctx->max_calls = mcvlist->nitems;
1360
1361
    /* Build a tuple descriptor for our result type */
1362
0
    if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
1363
0
      ereport(ERROR,
1364
0
          (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1365
0
           errmsg("function returning record called in context "
1366
0
              "that cannot accept type record")));
1367
0
    tupdesc = BlessTupleDesc(tupdesc);
1368
1369
    /*
1370
     * generate attribute metadata needed later to produce tuples from raw
1371
     * C strings
1372
     */
1373
0
    funcctx->attinmeta = TupleDescGetAttInMetadata(tupdesc);
1374
1375
0
    MemoryContextSwitchTo(oldcontext);
1376
0
  }
1377
1378
  /* stuff done on every call of the function */
1379
0
  funcctx = SRF_PERCALL_SETUP();
1380
1381
0
  if (funcctx->call_cntr < funcctx->max_calls) /* do when there is more
1382
                           * left to send */
1383
0
  {
1384
0
    Datum   values[5];
1385
0
    bool    nulls[5];
1386
0
    HeapTuple tuple;
1387
0
    Datum   result;
1388
0
    ArrayBuildState *astate_values = NULL;
1389
0
    ArrayBuildState *astate_nulls = NULL;
1390
1391
0
    int     i;
1392
0
    MCVList    *mcvlist;
1393
0
    MCVItem    *item;
1394
1395
0
    mcvlist = (MCVList *) funcctx->user_fctx;
1396
1397
0
    Assert(funcctx->call_cntr < mcvlist->nitems);
1398
1399
0
    item = &mcvlist->items[funcctx->call_cntr];
1400
1401
0
    for (i = 0; i < mcvlist->ndimensions; i++)
1402
0
    {
1403
1404
0
      astate_nulls = accumArrayResult(astate_nulls,
1405
0
                      BoolGetDatum(item->isnull[i]),
1406
0
                      false,
1407
0
                      BOOLOID,
1408
0
                      CurrentMemoryContext);
1409
1410
0
      if (!item->isnull[i])
1411
0
      {
1412
0
        bool    isvarlena;
1413
0
        Oid     outfunc;
1414
0
        FmgrInfo  fmgrinfo;
1415
0
        Datum   val;
1416
0
        text     *txt;
1417
1418
        /* lookup output func for the type */
1419
0
        getTypeOutputInfo(mcvlist->types[i], &outfunc, &isvarlena);
1420
0
        fmgr_info(outfunc, &fmgrinfo);
1421
1422
0
        val = FunctionCall1(&fmgrinfo, item->values[i]);
1423
0
        txt = cstring_to_text(DatumGetPointer(val));
1424
1425
0
        astate_values = accumArrayResult(astate_values,
1426
0
                         PointerGetDatum(txt),
1427
0
                         false,
1428
0
                         TEXTOID,
1429
0
                         CurrentMemoryContext);
1430
0
      }
1431
0
      else
1432
0
        astate_values = accumArrayResult(astate_values,
1433
0
                         (Datum) 0,
1434
0
                         true,
1435
0
                         TEXTOID,
1436
0
                         CurrentMemoryContext);
1437
0
    }
1438
1439
0
    values[0] = Int32GetDatum(funcctx->call_cntr);
1440
0
    values[1] = makeArrayResult(astate_values, CurrentMemoryContext);
1441
0
    values[2] = makeArrayResult(astate_nulls, CurrentMemoryContext);
1442
0
    values[3] = Float8GetDatum(item->frequency);
1443
0
    values[4] = Float8GetDatum(item->base_frequency);
1444
1445
    /* no NULLs in the tuple */
1446
0
    memset(nulls, 0, sizeof(nulls));
1447
1448
    /* build a tuple */
1449
0
    tuple = heap_form_tuple(funcctx->attinmeta->tupdesc, values, nulls);
1450
1451
    /* make the tuple into a datum */
1452
0
    result = HeapTupleGetDatum(tuple);
1453
1454
0
    SRF_RETURN_NEXT(funcctx, result);
1455
0
  }
1456
0
  else            /* do when there is no more left */
1457
0
  {
1458
0
    SRF_RETURN_DONE(funcctx);
1459
0
  }
1460
0
}
1461
1462
/*
1463
 * pg_mcv_list_in   - input routine for type pg_mcv_list.
1464
 *
1465
 * pg_mcv_list is real enough to be a table column, but it has no operations
1466
 * of its own, and disallows input too
1467
 */
1468
Datum
1469
pg_mcv_list_in(PG_FUNCTION_ARGS)
1470
0
{
1471
  /*
1472
   * pg_mcv_list stores the data in binary form and parsing text input is
1473
   * not needed, so disallow this.
1474
   */
1475
0
  ereport(ERROR,
1476
0
      (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1477
0
       errmsg("cannot accept a value of type %s", "pg_mcv_list")));
1478
1479
0
  PG_RETURN_VOID();     /* keep compiler quiet */
1480
0
}
1481
1482
1483
/*
1484
 * pg_mcv_list_out    - output routine for type pg_mcv_list.
1485
 *
1486
 * MCV lists are serialized into a bytea value, so we simply call byteaout()
1487
 * to serialize the value into text. But it'd be nice to serialize that into
1488
 * a meaningful representation (e.g. for inspection by people).
1489
 *
1490
 * XXX This should probably return something meaningful, similar to what
1491
 * pg_dependencies_out does. Not sure how to deal with the deduplicated
1492
 * values, though - do we want to expand that or not?
1493
 */
1494
Datum
1495
pg_mcv_list_out(PG_FUNCTION_ARGS)
1496
0
{
1497
0
  return byteaout(fcinfo);
1498
0
}
1499
1500
/*
1501
 * pg_mcv_list_recv   - binary input routine for type pg_mcv_list.
1502
 */
1503
Datum
1504
pg_mcv_list_recv(PG_FUNCTION_ARGS)
1505
0
{
1506
0
  ereport(ERROR,
1507
0
      (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1508
0
       errmsg("cannot accept a value of type %s", "pg_mcv_list")));
1509
1510
0
  PG_RETURN_VOID();     /* keep compiler quiet */
1511
0
}
1512
1513
/*
1514
 * pg_mcv_list_send   - binary output routine for type pg_mcv_list.
1515
 *
1516
 * MCV lists are serialized in a bytea value (although the type is named
1517
 * differently), so let's just send that.
1518
 */
1519
Datum
1520
pg_mcv_list_send(PG_FUNCTION_ARGS)
1521
0
{
1522
0
  return byteasend(fcinfo);
1523
0
}
1524
1525
/*
1526
 * match the attribute/expression to a dimension of the statistic
1527
 *
1528
 * Returns the zero-based index of the matching statistics dimension.
1529
 * Optionally determines the collation.
1530
 */
1531
static int
1532
mcv_match_expression(Node *expr, Bitmapset *keys, List *exprs, Oid *collid)
1533
0
{
1534
0
  int     idx;
1535
1536
0
  if (IsA(expr, Var))
1537
0
  {
1538
    /* simple Var, so just lookup using varattno */
1539
0
    Var      *var = (Var *) expr;
1540
1541
0
    if (collid)
1542
0
      *collid = var->varcollid;
1543
1544
0
    idx = bms_member_index(keys, var->varattno);
1545
1546
0
    if (idx < 0)
1547
0
      elog(ERROR, "variable not found in statistics object");
1548
0
  }
1549
0
  else
1550
0
  {
1551
    /* expression - lookup in stats expressions */
1552
0
    ListCell   *lc;
1553
1554
0
    if (collid)
1555
0
      *collid = exprCollation(expr);
1556
1557
    /* expressions are stored after the simple columns */
1558
0
    idx = bms_num_members(keys);
1559
0
    foreach(lc, exprs)
1560
0
    {
1561
0
      Node     *stat_expr = (Node *) lfirst(lc);
1562
1563
0
      if (equal(expr, stat_expr))
1564
0
        break;
1565
1566
0
      idx++;
1567
0
    }
1568
1569
0
    if (lc == NULL)
1570
0
      elog(ERROR, "expression not found in statistics object");
1571
0
  }
1572
1573
0
  return idx;
1574
0
}
1575
1576
/*
1577
 * mcv_get_match_bitmap
1578
 *  Evaluate clauses using the MCV list, and update the match bitmap.
1579
 *
1580
 * A match bitmap keeps match/mismatch status for each MCV item, and we
1581
 * update it based on additional clauses. We also use it to skip items
1582
 * that can't possibly match (e.g. item marked as "mismatch" can't change
1583
 * to "match" when evaluating AND clause list).
1584
 *
1585
 * The function also returns a flag indicating whether there was an
1586
 * equality condition for all attributes, the minimum frequency in the MCV
1587
 * list, and a total MCV frequency (sum of frequencies for all items).
1588
 *
1589
 * XXX Currently the match bitmap uses a bool for each MCV item, which is
1590
 * somewhat wasteful as we could do with just a single bit, thus reducing
1591
 * the size to ~1/8. It would also allow us to combine bitmaps simply using
1592
 * & and |, which should be faster than min/max. The bitmaps are fairly
1593
 * small, though (thanks to the cap on the MCV list size).
1594
 */
1595
static bool *
1596
mcv_get_match_bitmap(PlannerInfo *root, List *clauses,
1597
           Bitmapset *keys, List *exprs,
1598
           MCVList *mcvlist, bool is_or)
1599
0
{
1600
0
  ListCell   *l;
1601
0
  bool     *matches;
1602
1603
  /* The bitmap may be partially built. */
1604
0
  Assert(clauses != NIL);
1605
0
  Assert(mcvlist != NULL);
1606
0
  Assert(mcvlist->nitems > 0);
1607
0
  Assert(mcvlist->nitems <= STATS_MCVLIST_MAX_ITEMS);
1608
1609
0
  matches = palloc_array(bool, mcvlist->nitems);
1610
0
  memset(matches, !is_or, sizeof(bool) * mcvlist->nitems);
1611
1612
  /*
1613
   * Loop through the list of clauses, and for each of them evaluate all the
1614
   * MCV items not yet eliminated by the preceding clauses.
1615
   */
1616
0
  foreach(l, clauses)
1617
0
  {
1618
0
    Node     *clause = (Node *) lfirst(l);
1619
1620
    /* if it's a RestrictInfo, then extract the clause */
1621
0
    if (IsA(clause, RestrictInfo))
1622
0
      clause = (Node *) ((RestrictInfo *) clause)->clause;
1623
1624
    /*
1625
     * Handle the various types of clauses - OpClause, NullTest and
1626
     * AND/OR/NOT
1627
     */
1628
0
    if (is_opclause(clause))
1629
0
    {
1630
0
      OpExpr     *expr = (OpExpr *) clause;
1631
0
      FmgrInfo  opproc;
1632
1633
      /* valid only after examine_opclause_args returns true */
1634
0
      Node     *clause_expr;
1635
0
      Const    *cst;
1636
0
      bool    expronleft;
1637
0
      int     idx;
1638
0
      Oid     collid;
1639
1640
0
      fmgr_info(get_opcode(expr->opno), &opproc);
1641
1642
      /* extract the var/expr and const from the expression */
1643
0
      if (!examine_opclause_args(expr->args, &clause_expr, &cst, &expronleft))
1644
0
        elog(ERROR, "incompatible clause");
1645
1646
      /* match the attribute/expression to a dimension of the statistic */
1647
0
      idx = mcv_match_expression(clause_expr, keys, exprs, &collid);
1648
1649
      /*
1650
       * Walk through the MCV items and evaluate the current clause. We
1651
       * can skip items that were already ruled out, and terminate if
1652
       * there are no remaining MCV items that might possibly match.
1653
       */
1654
0
      for (uint32 i = 0; i < mcvlist->nitems; i++)
1655
0
      {
1656
0
        bool    match = true;
1657
0
        MCVItem    *item = &mcvlist->items[i];
1658
1659
0
        Assert(idx >= 0);
1660
1661
        /*
1662
         * When the MCV item or the Const value is NULL we can treat
1663
         * this as a mismatch. We must not call the operator because
1664
         * of strictness.
1665
         */
1666
0
        if (item->isnull[idx] || cst->constisnull)
1667
0
        {
1668
0
          matches[i] = RESULT_MERGE(matches[i], is_or, false);
1669
0
          continue;
1670
0
        }
1671
1672
        /*
1673
         * Skip MCV items that can't change result in the bitmap. Once
1674
         * the value gets false for AND-lists, or true for OR-lists,
1675
         * we don't need to look at more clauses.
1676
         */
1677
0
        if (RESULT_IS_FINAL(matches[i], is_or))
1678
0
          continue;
1679
1680
        /*
1681
         * First check whether the constant is below the lower
1682
         * boundary (in that case we can skip the bucket, because
1683
         * there's no overlap).
1684
         *
1685
         * We don't store collations used to build the statistics, but
1686
         * we can use the collation for the attribute itself, as
1687
         * stored in varcollid. We do reset the statistics after a
1688
         * type change (including collation change), so this is OK.
1689
         * For expressions, we use the collation extracted from the
1690
         * expression itself.
1691
         */
1692
0
        if (expronleft)
1693
0
          match = DatumGetBool(FunctionCall2Coll(&opproc,
1694
0
                               collid,
1695
0
                               item->values[idx],
1696
0
                               cst->constvalue));
1697
0
        else
1698
0
          match = DatumGetBool(FunctionCall2Coll(&opproc,
1699
0
                               collid,
1700
0
                               cst->constvalue,
1701
0
                               item->values[idx]));
1702
1703
        /* update the match bitmap with the result */
1704
0
        matches[i] = RESULT_MERGE(matches[i], is_or, match);
1705
0
      }
1706
0
    }
1707
0
    else if (IsA(clause, ScalarArrayOpExpr))
1708
0
    {
1709
0
      ScalarArrayOpExpr *expr = (ScalarArrayOpExpr *) clause;
1710
0
      FmgrInfo  opproc;
1711
1712
      /* valid only after examine_opclause_args returns true */
1713
0
      Node     *clause_expr;
1714
0
      Const    *cst;
1715
0
      bool    expronleft;
1716
0
      Oid     collid;
1717
0
      int     idx;
1718
1719
      /* array evaluation */
1720
0
      ArrayType  *arrayval;
1721
0
      int16   elmlen;
1722
0
      bool    elmbyval;
1723
0
      char    elmalign;
1724
0
      int     num_elems;
1725
0
      Datum    *elem_values;
1726
0
      bool     *elem_nulls;
1727
1728
0
      fmgr_info(get_opcode(expr->opno), &opproc);
1729
1730
      /* extract the var/expr and const from the expression */
1731
0
      if (!examine_opclause_args(expr->args, &clause_expr, &cst, &expronleft))
1732
0
        elog(ERROR, "incompatible clause");
1733
1734
      /* We expect Var on left */
1735
0
      if (!expronleft)
1736
0
        elog(ERROR, "incompatible clause");
1737
1738
      /*
1739
       * Deconstruct the array constant, unless it's NULL (we'll cover
1740
       * that case below)
1741
       */
1742
0
      if (!cst->constisnull)
1743
0
      {
1744
0
        arrayval = DatumGetArrayTypeP(cst->constvalue);
1745
0
        get_typlenbyvalalign(ARR_ELEMTYPE(arrayval),
1746
0
                   &elmlen, &elmbyval, &elmalign);
1747
0
        deconstruct_array(arrayval,
1748
0
                  ARR_ELEMTYPE(arrayval),
1749
0
                  elmlen, elmbyval, elmalign,
1750
0
                  &elem_values, &elem_nulls, &num_elems);
1751
0
      }
1752
1753
      /* match the attribute/expression to a dimension of the statistic */
1754
0
      idx = mcv_match_expression(clause_expr, keys, exprs, &collid);
1755
1756
      /*
1757
       * Walk through the MCV items and evaluate the current clause. We
1758
       * can skip items that were already ruled out, and terminate if
1759
       * there are no remaining MCV items that might possibly match.
1760
       */
1761
0
      for (uint32 i = 0; i < mcvlist->nitems; i++)
1762
0
      {
1763
0
        int     j;
1764
0
        bool    match = !expr->useOr;
1765
0
        MCVItem    *item = &mcvlist->items[i];
1766
1767
        /*
1768
         * When the MCV item or the Const value is NULL we can treat
1769
         * this as a mismatch. We must not call the operator because
1770
         * of strictness.
1771
         */
1772
0
        if (item->isnull[idx] || cst->constisnull)
1773
0
        {
1774
0
          matches[i] = RESULT_MERGE(matches[i], is_or, false);
1775
0
          continue;
1776
0
        }
1777
1778
        /*
1779
         * Skip MCV items that can't change result in the bitmap. Once
1780
         * the value gets false for AND-lists, or true for OR-lists,
1781
         * we don't need to look at more clauses.
1782
         */
1783
0
        if (RESULT_IS_FINAL(matches[i], is_or))
1784
0
          continue;
1785
1786
0
        for (j = 0; j < num_elems; j++)
1787
0
        {
1788
0
          Datum   elem_value = elem_values[j];
1789
0
          bool    elem_isnull = elem_nulls[j];
1790
0
          bool    elem_match;
1791
1792
          /* NULL values always evaluate as not matching. */
1793
0
          if (elem_isnull)
1794
0
          {
1795
0
            match = RESULT_MERGE(match, expr->useOr, false);
1796
0
            continue;
1797
0
          }
1798
1799
          /*
1800
           * Stop evaluating the array elements once we reach a
1801
           * matching value that can't change - ALL() is the same as
1802
           * AND-list, ANY() is the same as OR-list.
1803
           */
1804
0
          if (RESULT_IS_FINAL(match, expr->useOr))
1805
0
            break;
1806
1807
0
          elem_match = DatumGetBool(FunctionCall2Coll(&opproc,
1808
0
                                collid,
1809
0
                                item->values[idx],
1810
0
                                elem_value));
1811
1812
0
          match = RESULT_MERGE(match, expr->useOr, elem_match);
1813
0
        }
1814
1815
        /* update the match bitmap with the result */
1816
0
        matches[i] = RESULT_MERGE(matches[i], is_or, match);
1817
0
      }
1818
0
    }
1819
0
    else if (IsA(clause, NullTest))
1820
0
    {
1821
0
      NullTest   *expr = (NullTest *) clause;
1822
0
      Node     *clause_expr = (Node *) (expr->arg);
1823
1824
      /* match the attribute/expression to a dimension of the statistic */
1825
0
      int     idx = mcv_match_expression(clause_expr, keys, exprs, NULL);
1826
1827
      /*
1828
       * Walk through the MCV items and evaluate the current clause. We
1829
       * can skip items that were already ruled out, and terminate if
1830
       * there are no remaining MCV items that might possibly match.
1831
       */
1832
0
      for (uint32 i = 0; i < mcvlist->nitems; i++)
1833
0
      {
1834
0
        bool    match = false;  /* assume mismatch */
1835
0
        MCVItem    *item = &mcvlist->items[i];
1836
1837
        /* if the clause mismatches the MCV item, update the bitmap */
1838
0
        switch (expr->nulltesttype)
1839
0
        {
1840
0
          case IS_NULL:
1841
0
            match = (item->isnull[idx]) ? true : match;
1842
0
            break;
1843
1844
0
          case IS_NOT_NULL:
1845
0
            match = (!item->isnull[idx]) ? true : match;
1846
0
            break;
1847
0
        }
1848
1849
        /* now, update the match bitmap, depending on OR/AND type */
1850
0
        matches[i] = RESULT_MERGE(matches[i], is_or, match);
1851
0
      }
1852
0
    }
1853
0
    else if (is_orclause(clause) || is_andclause(clause))
1854
0
    {
1855
      /* AND/OR clause, with all subclauses being compatible */
1856
1857
0
      BoolExpr   *bool_clause = ((BoolExpr *) clause);
1858
0
      List     *bool_clauses = bool_clause->args;
1859
1860
      /* match/mismatch bitmap for each MCV item */
1861
0
      bool     *bool_matches = NULL;
1862
1863
0
      Assert(bool_clauses != NIL);
1864
0
      Assert(list_length(bool_clauses) >= 2);
1865
1866
      /* build the match bitmap for the OR-clauses */
1867
0
      bool_matches = mcv_get_match_bitmap(root, bool_clauses, keys, exprs,
1868
0
                        mcvlist, is_orclause(clause));
1869
1870
      /*
1871
       * Merge the bitmap produced by mcv_get_match_bitmap into the
1872
       * current one. We need to consider if we're evaluating AND or OR
1873
       * condition when merging the results.
1874
       */
1875
0
      for (uint32 i = 0; i < mcvlist->nitems; i++)
1876
0
        matches[i] = RESULT_MERGE(matches[i], is_or, bool_matches[i]);
1877
1878
0
      pfree(bool_matches);
1879
0
    }
1880
0
    else if (is_notclause(clause))
1881
0
    {
1882
      /* NOT clause, with all subclauses compatible */
1883
1884
0
      BoolExpr   *not_clause = ((BoolExpr *) clause);
1885
0
      List     *not_args = not_clause->args;
1886
1887
      /* match/mismatch bitmap for each MCV item */
1888
0
      bool     *not_matches = NULL;
1889
1890
0
      Assert(not_args != NIL);
1891
0
      Assert(list_length(not_args) == 1);
1892
1893
      /* build the match bitmap for the NOT-clause */
1894
0
      not_matches = mcv_get_match_bitmap(root, not_args, keys, exprs,
1895
0
                         mcvlist, false);
1896
1897
      /*
1898
       * Merge the bitmap produced by mcv_get_match_bitmap into the
1899
       * current one. We're handling a NOT clause, so invert the result
1900
       * before merging it into the global bitmap.
1901
       */
1902
0
      for (uint32 i = 0; i < mcvlist->nitems; i++)
1903
0
        matches[i] = RESULT_MERGE(matches[i], is_or, !not_matches[i]);
1904
1905
0
      pfree(not_matches);
1906
0
    }
1907
0
    else if (IsA(clause, Var))
1908
0
    {
1909
      /* Var (has to be a boolean Var, possibly from below NOT) */
1910
1911
0
      Var      *var = (Var *) (clause);
1912
1913
      /* match the attribute to a dimension of the statistic */
1914
0
      int     idx = bms_member_index(keys, var->varattno);
1915
1916
0
      Assert(var->vartype == BOOLOID);
1917
1918
      /*
1919
       * Walk through the MCV items and evaluate the current clause. We
1920
       * can skip items that were already ruled out, and terminate if
1921
       * there are no remaining MCV items that might possibly match.
1922
       */
1923
0
      for (uint32 i = 0; i < mcvlist->nitems; i++)
1924
0
      {
1925
0
        MCVItem    *item = &mcvlist->items[i];
1926
0
        bool    match = false;
1927
1928
        /* if the item is NULL, it's a mismatch */
1929
0
        if (!item->isnull[idx] && DatumGetBool(item->values[idx]))
1930
0
          match = true;
1931
1932
        /* update the result bitmap */
1933
0
        matches[i] = RESULT_MERGE(matches[i], is_or, match);
1934
0
      }
1935
0
    }
1936
0
    else
1937
0
    {
1938
      /* Otherwise, it must be a bare boolean-returning expression */
1939
0
      int     idx;
1940
1941
      /* match the expression to a dimension of the statistic */
1942
0
      idx = mcv_match_expression(clause, keys, exprs, NULL);
1943
1944
      /*
1945
       * Walk through the MCV items and evaluate the current clause. We
1946
       * can skip items that were already ruled out, and terminate if
1947
       * there are no remaining MCV items that might possibly match.
1948
       */
1949
0
      for (uint32 i = 0; i < mcvlist->nitems; i++)
1950
0
      {
1951
0
        bool    match;
1952
0
        MCVItem    *item = &mcvlist->items[i];
1953
1954
        /* "match" just means it's bool TRUE */
1955
0
        match = !item->isnull[idx] && DatumGetBool(item->values[idx]);
1956
1957
        /* now, update the match bitmap, depending on OR/AND type */
1958
0
        matches[i] = RESULT_MERGE(matches[i], is_or, match);
1959
0
      }
1960
0
    }
1961
0
  }
1962
1963
0
  return matches;
1964
0
}
1965
1966
1967
/*
1968
 * mcv_combine_selectivities
1969
 *    Combine per-column and multi-column MCV selectivity estimates.
1970
 *
1971
 * simple_sel is a "simple" selectivity estimate (produced without using any
1972
 * extended statistics, essentially assuming independence of columns/clauses).
1973
 *
1974
 * mcv_sel and mcv_basesel are sums of the frequencies and base frequencies of
1975
 * all matching MCV items.  The difference (mcv_sel - mcv_basesel) is then
1976
 * essentially interpreted as a correction to be added to simple_sel, as
1977
 * described below.
1978
 *
1979
 * mcv_totalsel is the sum of the frequencies of all MCV items (not just the
1980
 * matching ones).  This is used as an upper bound on the portion of the
1981
 * selectivity estimates not covered by the MCV statistics.
1982
 *
1983
 * Note: While simple and base selectivities are defined in a quite similar
1984
 * way, the values are computed differently and are not therefore equal. The
1985
 * simple selectivity is computed as a product of per-clause estimates, while
1986
 * the base selectivity is computed by adding up base frequencies of matching
1987
 * items of the multi-column MCV list. So the values may differ for two main
1988
 * reasons - (a) the MCV list may not cover 100% of the data and (b) some of
1989
 * the MCV items did not match the estimated clauses.
1990
 *
1991
 * As both (a) and (b) reduce the base selectivity value, it generally holds
1992
 * that (simple_sel >= mcv_basesel). If the MCV list covers all the data, the
1993
 * values may be equal.
1994
 *
1995
 * So, other_sel = (simple_sel - mcv_basesel) is an estimate for the part not
1996
 * covered by the MCV list, and (mcv_sel - mcv_basesel) may be seen as a
1997
 * correction for the part covered by the MCV list. Those two statements are
1998
 * actually equivalent.
1999
 */
2000
Selectivity
2001
mcv_combine_selectivities(Selectivity simple_sel,
2002
              Selectivity mcv_sel,
2003
              Selectivity mcv_basesel,
2004
              Selectivity mcv_totalsel)
2005
0
{
2006
0
  Selectivity other_sel;
2007
0
  Selectivity sel;
2008
2009
  /* estimated selectivity of values not covered by MCV matches */
2010
0
  other_sel = simple_sel - mcv_basesel;
2011
0
  CLAMP_PROBABILITY(other_sel);
2012
2013
  /* this non-MCV selectivity cannot exceed 1 - mcv_totalsel */
2014
0
  if (other_sel > 1.0 - mcv_totalsel)
2015
0
    other_sel = 1.0 - mcv_totalsel;
2016
2017
  /* overall selectivity is the sum of the MCV and non-MCV parts */
2018
0
  sel = mcv_sel + other_sel;
2019
0
  CLAMP_PROBABILITY(sel);
2020
2021
0
  return sel;
2022
0
}
2023
2024
2025
/*
2026
 * mcv_clauselist_selectivity
2027
 *    Use MCV statistics to estimate the selectivity of an implicitly-ANDed
2028
 *    list of clauses.
2029
 *
2030
 * This determines which MCV items match every clause in the list and returns
2031
 * the sum of the frequencies of those items.
2032
 *
2033
 * In addition, it returns the sum of the base frequencies of each of those
2034
 * items (that is the sum of the selectivities that each item would have if
2035
 * the columns were independent of one another), and the total selectivity of
2036
 * all the MCV items (not just the matching ones).  These are expected to be
2037
 * used together with a "simple" selectivity estimate (one based only on
2038
 * per-column statistics) to produce an overall selectivity estimate that
2039
 * makes use of both per-column and multi-column statistics --- see
2040
 * mcv_combine_selectivities().
2041
 */
2042
Selectivity
2043
mcv_clauselist_selectivity(PlannerInfo *root, StatisticExtInfo *stat,
2044
               List *clauses, int varRelid,
2045
               JoinType jointype, SpecialJoinInfo *sjinfo,
2046
               RelOptInfo *rel,
2047
               Selectivity *basesel, Selectivity *totalsel)
2048
0
{
2049
0
  MCVList    *mcv;
2050
0
  Selectivity s = 0.0;
2051
0
  RangeTblEntry *rte = root->simple_rte_array[rel->relid];
2052
2053
  /* match/mismatch bitmap for each MCV item */
2054
0
  bool     *matches = NULL;
2055
2056
  /* load the MCV list stored in the statistics object */
2057
0
  mcv = statext_mcv_load(stat->statOid, rte->inh);
2058
2059
  /* build a match bitmap for the clauses */
2060
0
  matches = mcv_get_match_bitmap(root, clauses, stat->keys, stat->exprs,
2061
0
                   mcv, false);
2062
2063
  /* sum frequencies for all the matching MCV items */
2064
0
  *basesel = 0.0;
2065
0
  *totalsel = 0.0;
2066
0
  for (uint32 i = 0; i < mcv->nitems; i++)
2067
0
  {
2068
0
    *totalsel += mcv->items[i].frequency;
2069
2070
0
    if (matches[i] != false)
2071
0
    {
2072
0
      *basesel += mcv->items[i].base_frequency;
2073
0
      s += mcv->items[i].frequency;
2074
0
    }
2075
0
  }
2076
2077
0
  return s;
2078
0
}
2079
2080
2081
/*
2082
 * mcv_clause_selectivity_or
2083
 *    Use MCV statistics to estimate the selectivity of a clause that
2084
 *    appears in an ORed list of clauses.
2085
 *
2086
 * As with mcv_clauselist_selectivity() this determines which MCV items match
2087
 * the clause and returns both the sum of the frequencies and the sum of the
2088
 * base frequencies of those items, as well as the sum of the frequencies of
2089
 * all MCV items (not just the matching ones) so that this information can be
2090
 * used by mcv_combine_selectivities() to produce a selectivity estimate that
2091
 * makes use of both per-column and multi-column statistics.
2092
 *
2093
 * Additionally, we return information to help compute the overall selectivity
2094
 * of the ORed list of clauses assumed to contain this clause.  This function
2095
 * is intended to be called for each clause in the ORed list of clauses,
2096
 * allowing the overall selectivity to be computed using the following
2097
 * algorithm:
2098
 *
2099
 * Suppose P[n] = P(C[1] OR C[2] OR ... OR C[n]) is the combined selectivity
2100
 * of the first n clauses in the list.  Then the combined selectivity taking
2101
 * into account the next clause C[n+1] can be written as
2102
 *
2103
 *    P[n+1] = P[n] + P(C[n+1]) - P((C[1] OR ... OR C[n]) AND C[n+1])
2104
 *
2105
 * The final term above represents the overlap between the clauses examined so
2106
 * far and the (n+1)'th clause.  To estimate its selectivity, we track the
2107
 * match bitmap for the ORed list of clauses examined so far and examine its
2108
 * intersection with the match bitmap for the (n+1)'th clause.
2109
 *
2110
 * We then also return the sums of the MCV item frequencies and base
2111
 * frequencies for the match bitmap intersection corresponding to the overlap
2112
 * term above, so that they can be combined with a simple selectivity estimate
2113
 * for that term.
2114
 *
2115
 * The parameter "or_matches" is an in/out parameter tracking the match bitmap
2116
 * for the clauses examined so far.  The caller is expected to set it to NULL
2117
 * the first time it calls this function.
2118
 */
2119
Selectivity
2120
mcv_clause_selectivity_or(PlannerInfo *root, StatisticExtInfo *stat,
2121
              MCVList *mcv, Node *clause, bool **or_matches,
2122
              Selectivity *basesel, Selectivity *overlap_mcvsel,
2123
              Selectivity *overlap_basesel, Selectivity *totalsel)
2124
0
{
2125
0
  Selectivity s = 0.0;
2126
0
  bool     *new_matches;
2127
2128
  /* build the OR-matches bitmap, if not built already */
2129
0
  if (*or_matches == NULL)
2130
0
    *or_matches = palloc0_array(bool, mcv->nitems);
2131
2132
  /* build the match bitmap for the new clause */
2133
0
  new_matches = mcv_get_match_bitmap(root, list_make1(clause), stat->keys,
2134
0
                     stat->exprs, mcv, false);
2135
2136
  /*
2137
   * Sum the frequencies for all the MCV items matching this clause and also
2138
   * those matching the overlap between this clause and any of the preceding
2139
   * clauses as described above.
2140
   */
2141
0
  *basesel = 0.0;
2142
0
  *overlap_mcvsel = 0.0;
2143
0
  *overlap_basesel = 0.0;
2144
0
  *totalsel = 0.0;
2145
0
  for (uint32 i = 0; i < mcv->nitems; i++)
2146
0
  {
2147
0
    *totalsel += mcv->items[i].frequency;
2148
2149
0
    if (new_matches[i])
2150
0
    {
2151
0
      s += mcv->items[i].frequency;
2152
0
      *basesel += mcv->items[i].base_frequency;
2153
2154
0
      if ((*or_matches)[i])
2155
0
      {
2156
0
        *overlap_mcvsel += mcv->items[i].frequency;
2157
0
        *overlap_basesel += mcv->items[i].base_frequency;
2158
0
      }
2159
0
    }
2160
2161
    /* update the OR-matches bitmap for the next clause */
2162
0
    (*or_matches)[i] = (*or_matches)[i] || new_matches[i];
2163
0
  }
2164
2165
0
  pfree(new_matches);
2166
2167
0
  return s;
2168
0
}
2169
2170
/*
2171
 * Free allocations of a MCVList.
2172
 */
2173
void
2174
statext_mcv_free(MCVList *mcvlist)
2175
0
{
2176
0
  for (uint32 i = 0; i < mcvlist->nitems; i++)
2177
0
  {
2178
0
    MCVItem    *item = &mcvlist->items[i];
2179
2180
0
    pfree(item->values);
2181
0
    pfree(item->isnull);
2182
0
  }
2183
0
  pfree(mcvlist);
2184
0
}
2185
2186
/*
2187
 * Create the MCV composite datum, which is a serialization of an array of
2188
 * MCVItems.
2189
 *
2190
 * The inputs consist of four separate arrays of equal length "numitems"
2191
 * (mcv_elems, mcv_nulls, freqs and base_freqs) that form the basics of
2192
 * what is stored in the catalogs.  These form an array of composite
2193
 * records defined by the three atttypX arrays of equal length "numattrs".
2194
 *
2195
 * If any data element fails to convert to the input type specified for that
2196
 * attribute, then function will return a NULL Datum if elevel < ERROR.
2197
 */
2198
Datum
2199
statext_mcv_import(int elevel, int numattrs,
2200
           Oid *atttypids, int32 *atttypmods, Oid *atttypcolls,
2201
           int nitems, Datum *mcv_elems, bool *mcv_nulls,
2202
           float8 *freqs, float8 *base_freqs)
2203
0
{
2204
0
  MCVList    *mcvlist;
2205
0
  bytea    *bytes;
2206
0
  VacAttrStats **vastats;
2207
2208
  /*
2209
   * Allocate the MCV list structure, set the global parameters.
2210
   */
2211
0
  mcvlist = (MCVList *) palloc0(offsetof(MCVList, items) +
2212
0
                  (sizeof(MCVItem) * nitems));
2213
2214
0
  mcvlist->magic = STATS_MCV_MAGIC;
2215
0
  mcvlist->type = STATS_MCV_TYPE_BASIC;
2216
0
  mcvlist->ndimensions = numattrs;
2217
0
  mcvlist->nitems = nitems;
2218
2219
  /* Set the values for the 1-D arrays and allocate space for the 2-D arrays */
2220
0
  for (int i = 0; i < nitems; i++)
2221
0
  {
2222
0
    MCVItem    *item = &mcvlist->items[i];
2223
2224
0
    item->frequency = freqs[i];
2225
0
    item->base_frequency = base_freqs[i];
2226
0
    item->values = (Datum *) palloc0_array(Datum, numattrs);
2227
0
    item->isnull = (bool *) palloc0_array(bool, numattrs);
2228
0
  }
2229
2230
  /*
2231
   * Walk through each dimension, determine the input function for that
2232
   * type, and then attempt to convert all values in that column via that
2233
   * function.  We approach this column-wise because it is simpler to deal
2234
   * with one input function at time, and possibly more cache-friendly.
2235
   */
2236
0
  for (int j = 0; j < numattrs; j++)
2237
0
  {
2238
0
    FmgrInfo  finfo;
2239
0
    Oid     ioparam;
2240
0
    Oid     infunc;
2241
0
    int     index = j;
2242
2243
0
    getTypeInputInfo(atttypids[j], &infunc, &ioparam);
2244
0
    fmgr_info(infunc, &finfo);
2245
2246
    /* store info about data type OIDs */
2247
0
    mcvlist->types[j] = atttypids[j];
2248
2249
0
    for (int i = 0; i < nitems; i++)
2250
0
    {
2251
0
      MCVItem    *item = &mcvlist->items[i];
2252
2253
0
      if (mcv_nulls[index])
2254
0
      {
2255
        /* NULL value detected, hence no input to process */
2256
0
        item->values[j] = (Datum) 0;
2257
0
        item->isnull[j] = true;
2258
0
      }
2259
0
      else
2260
0
      {
2261
0
        char     *s = TextDatumGetCString(mcv_elems[index]);
2262
0
        ErrorSaveContext escontext = {T_ErrorSaveContext};
2263
2264
0
        if (!InputFunctionCallSafe(&finfo, s, ioparam, atttypmods[j],
2265
0
                       (Node *) &escontext, &item->values[j]))
2266
0
        {
2267
0
          ereport(elevel,
2268
0
              (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
2269
0
               errmsg("could not parse MCV element \"%s\": incorrect value", s)));
2270
0
          pfree(s);
2271
0
          goto error;
2272
0
        }
2273
2274
0
        pfree(s);
2275
0
      }
2276
2277
0
      index += numattrs;
2278
0
    }
2279
0
  }
2280
2281
  /*
2282
   * The function statext_mcv_serialize() requires an array of pointers to
2283
   * VacAttrStats records, but only a few fields within those records have
2284
   * to be filled out.
2285
   */
2286
0
  vastats = (VacAttrStats **) palloc0_array(VacAttrStats *, numattrs);
2287
2288
0
  for (int i = 0; i < numattrs; i++)
2289
0
  {
2290
0
    Oid     typid = atttypids[i];
2291
0
    HeapTuple typtuple;
2292
2293
0
    typtuple = SearchSysCacheCopy1(TYPEOID, ObjectIdGetDatum(typid));
2294
2295
0
    if (!HeapTupleIsValid(typtuple))
2296
0
      elog(ERROR, "cache lookup failed for type %u", typid);
2297
2298
0
    vastats[i] = palloc0_object(VacAttrStats);
2299
2300
0
    vastats[i]->attrtype = (Form_pg_type) GETSTRUCT(typtuple);
2301
0
    vastats[i]->attrtypid = typid;
2302
0
    vastats[i]->attrcollid = atttypcolls[i];
2303
0
  }
2304
2305
0
  bytes = statext_mcv_serialize(mcvlist, vastats);
2306
2307
0
  for (int i = 0; i < numattrs; i++)
2308
0
  {
2309
0
    pfree(vastats[i]);
2310
0
  }
2311
0
  pfree((void *) vastats);
2312
2313
0
  pfree(mcv_elems);
2314
0
  pfree(mcv_nulls);
2315
2316
0
  if (bytes == NULL)
2317
0
  {
2318
0
    ereport(elevel,
2319
0
        (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
2320
0
         errmsg("could not import MCV list")));
2321
0
    goto error;
2322
0
  }
2323
2324
0
  return PointerGetDatum(bytes);
2325
2326
0
error:
2327
0
  statext_mcv_free(mcvlist);
2328
0
  return (Datum) 0;
2329
0
}