/src/postgres/src/backend/statistics/mcv.c
Line | Count | Source |
1 | | /*------------------------------------------------------------------------- |
2 | | * |
3 | | * mcv.c |
4 | | * POSTGRES multivariate MCV lists |
5 | | * |
6 | | * |
7 | | * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group |
8 | | * Portions Copyright (c) 1994, Regents of the University of California |
9 | | * |
10 | | * IDENTIFICATION |
11 | | * src/backend/statistics/mcv.c |
12 | | * |
13 | | *------------------------------------------------------------------------- |
14 | | */ |
15 | | #include "postgres.h" |
16 | | |
17 | | #include "access/htup_details.h" |
18 | | #include "catalog/pg_statistic_ext.h" |
19 | | #include "catalog/pg_statistic_ext_data.h" |
20 | | #include "fmgr.h" |
21 | | #include "funcapi.h" |
22 | | #include "nodes/nodeFuncs.h" |
23 | | #include "statistics/extended_stats_internal.h" |
24 | | #include "statistics/statistics.h" |
25 | | #include "utils/array.h" |
26 | | #include "utils/builtins.h" |
27 | | #include "utils/fmgrprotos.h" |
28 | | #include "utils/lsyscache.h" |
29 | | #include "utils/selfuncs.h" |
30 | | #include "utils/syscache.h" |
31 | | #include "utils/typcache.h" |
32 | | |
33 | | /* |
34 | | * Computes size of a serialized MCV item, depending on the number of |
35 | | * dimensions (columns) the statistic is defined on. The datum values are |
36 | | * stored in a separate array (deduplicated, to minimize the size), and |
37 | | * so the serialized items only store uint16 indexes into that array. |
38 | | * |
39 | | * Each serialized item stores (in this order): |
40 | | * |
41 | | * - indexes to values (ndim * sizeof(uint16)) |
42 | | * - null flags (ndim * sizeof(bool)) |
43 | | * - frequency (sizeof(double)) |
44 | | * - base_frequency (sizeof(double)) |
45 | | * |
46 | | * There is no alignment padding within an MCV item. |
47 | | * So in total each MCV item requires this many bytes: |
48 | | * |
49 | | * ndim * (sizeof(uint16) + sizeof(bool)) + 2 * sizeof(double) |
50 | | */ |
51 | | #define ITEM_SIZE(ndims) \ |
52 | 0 | ((ndims) * (sizeof(uint16) + sizeof(bool)) + 2 * sizeof(double)) |
53 | | |
54 | | /* |
55 | | * Used to compute size of serialized MCV list representation. |
56 | | */ |
57 | | #define MinSizeOfMCVList \ |
58 | 0 | (VARHDRSZ + sizeof(uint32) * 3 + sizeof(AttrNumber)) |
59 | | |
60 | | /* |
61 | | * Size of the serialized MCV list, excluding the space needed for |
62 | | * deduplicated per-dimension values. The macro is meant to be used |
63 | | * when it's not yet safe to access the serialized info about amount |
64 | | * of data for each column. |
65 | | */ |
66 | | #define SizeOfMCVList(ndims,nitems) \ |
67 | 0 | ((MinSizeOfMCVList + sizeof(Oid) * (ndims)) + \ |
68 | 0 | ((ndims) * sizeof(DimensionInfo)) + \ |
69 | 0 | ((nitems) * ITEM_SIZE(ndims))) |
70 | | |
71 | | static MultiSortSupport build_mss(StatsBuildData *data); |
72 | | |
73 | | static SortItem *build_distinct_groups(int numrows, SortItem *items, |
74 | | MultiSortSupport mss, int *ndistinct); |
75 | | |
76 | | static SortItem **build_column_frequencies(SortItem *groups, int ngroups, |
77 | | MultiSortSupport mss, int *ncounts); |
78 | | |
79 | | static int count_distinct_groups(int numrows, SortItem *items, |
80 | | MultiSortSupport mss); |
81 | | |
82 | | /* |
83 | | * Compute new value for bitmap item, considering whether it's used for |
84 | | * clauses connected by AND/OR. |
85 | | */ |
86 | | #define RESULT_MERGE(value, is_or, match) \ |
87 | 0 | ((is_or) ? ((value) || (match)) : ((value) && (match))) |
88 | | |
89 | | /* |
90 | | * When processing a list of clauses, the bitmap item may get set to a value |
91 | | * such that additional clauses can't change it. For example, when processing |
92 | | * a list of clauses connected to AND, as soon as the item gets set to 'false' |
93 | | * then it'll remain like that. Similarly clauses connected by OR and 'true'. |
94 | | * |
95 | | * Returns true when the value in the bitmap can't change no matter how the |
96 | | * remaining clauses are evaluated. |
97 | | */ |
98 | 0 | #define RESULT_IS_FINAL(value, is_or) ((is_or) ? (value) : (!(value))) |
99 | | |
100 | | /* |
101 | | * get_mincount_for_mcv_list |
102 | | * Determine the minimum number of times a value needs to appear in |
103 | | * the sample for it to be included in the MCV list. |
104 | | * |
105 | | * We want to keep only values that appear sufficiently often in the |
106 | | * sample that it is reasonable to extrapolate their sample frequencies to |
107 | | * the entire table. We do this by placing an upper bound on the relative |
108 | | * standard error of the sample frequency, so that any estimates the |
109 | | * planner generates from the MCV statistics can be expected to be |
110 | | * reasonably accurate. |
111 | | * |
112 | | * Since we are sampling without replacement, the sample frequency of a |
113 | | * particular value is described by a hypergeometric distribution. A |
114 | | * common rule of thumb when estimating errors in this situation is to |
115 | | * require at least 10 instances of the value in the sample, in which case |
116 | | * the distribution can be approximated by a normal distribution, and |
117 | | * standard error analysis techniques can be applied. Given a sample size |
118 | | * of n, a population size of N, and a sample frequency of p=cnt/n, the |
119 | | * standard error of the proportion p is given by |
120 | | * SE = sqrt(p*(1-p)/n) * sqrt((N-n)/(N-1)) |
121 | | * where the second term is the finite population correction. To get |
122 | | * reasonably accurate planner estimates, we impose an upper bound on the |
123 | | * relative standard error of 20% -- i.e., SE/p < 0.2. This 20% relative |
124 | | * error bound is fairly arbitrary, but has been found empirically to work |
125 | | * well. Rearranging this formula gives a lower bound on the number of |
126 | | * instances of the value seen: |
127 | | * cnt > n*(N-n) / (N-n+0.04*n*(N-1)) |
128 | | * This bound is at most 25, and approaches 0 as n approaches 0 or N. The |
129 | | * case where n approaches 0 cannot happen in practice, since the sample |
130 | | * size is at least 300. The case where n approaches N corresponds to |
131 | | * sampling the whole table, in which case it is reasonable to keep |
132 | | * the whole MCV list (have no lower bound), so it makes sense to apply |
133 | | * this formula for all inputs, even though the above derivation is |
134 | | * technically only valid when the right hand side is at least around 10. |
135 | | * |
136 | | * An alternative way to look at this formula is as follows -- assume that |
137 | | * the number of instances of the value seen scales up to the entire |
138 | | * table, so that the population count is K=N*cnt/n. Then the distribution |
139 | | * in the sample is a hypergeometric distribution parameterised by N, n |
140 | | * and K, and the bound above is mathematically equivalent to demanding |
141 | | * that the standard deviation of that distribution is less than 20% of |
142 | | * its mean. Thus the relative errors in any planner estimates produced |
143 | | * from the MCV statistics are likely to be not too large. |
144 | | */ |
145 | | static double |
146 | | get_mincount_for_mcv_list(int samplerows, double totalrows) |
147 | 0 | { |
148 | 0 | double n = samplerows; |
149 | 0 | double N = totalrows; |
150 | 0 | double numer, |
151 | 0 | denom; |
152 | |
|
153 | 0 | numer = n * (N - n); |
154 | 0 | denom = N - n + 0.04 * n * (N - 1); |
155 | | |
156 | | /* Guard against division by zero (possible if n = N = 1) */ |
157 | 0 | if (denom == 0.0) |
158 | 0 | return 0.0; |
159 | | |
160 | 0 | return numer / denom; |
161 | 0 | } |
162 | | |
163 | | /* |
164 | | * Builds MCV list from the set of sampled rows. |
165 | | * |
166 | | * The algorithm is quite simple: |
167 | | * |
168 | | * (1) sort the data (default collation, '<' for the data type) |
169 | | * |
170 | | * (2) count distinct groups, decide how many to keep |
171 | | * |
172 | | * (3) build the MCV list using the threshold determined in (2) |
173 | | * |
174 | | * (4) remove rows represented by the MCV from the sample |
175 | | * |
176 | | */ |
177 | | MCVList * |
178 | | statext_mcv_build(StatsBuildData *data, double totalrows, int stattarget) |
179 | 0 | { |
180 | 0 | int i, |
181 | 0 | numattrs, |
182 | 0 | numrows, |
183 | 0 | ngroups, |
184 | 0 | nitems; |
185 | 0 | double mincount; |
186 | 0 | SortItem *items; |
187 | 0 | SortItem *groups; |
188 | 0 | MCVList *mcvlist = NULL; |
189 | 0 | MultiSortSupport mss; |
190 | | |
191 | | /* comparator for all the columns */ |
192 | 0 | mss = build_mss(data); |
193 | | |
194 | | /* sort the rows */ |
195 | 0 | items = build_sorted_items(data, &nitems, mss, |
196 | 0 | data->nattnums, data->attnums); |
197 | |
|
198 | 0 | if (!items) |
199 | 0 | return NULL; |
200 | | |
201 | | /* for convenience */ |
202 | 0 | numattrs = data->nattnums; |
203 | 0 | numrows = data->numrows; |
204 | | |
205 | | /* transform the sorted rows into groups (sorted by frequency) */ |
206 | 0 | groups = build_distinct_groups(nitems, items, mss, &ngroups); |
207 | | |
208 | | /* |
209 | | * The maximum number of MCV items to store, based on the statistics |
210 | | * target we computed for the statistics object (from the target set for |
211 | | * the object itself, attributes and the system default). In any case, we |
212 | | * can't keep more groups than we have available. |
213 | | */ |
214 | 0 | nitems = stattarget; |
215 | 0 | if (nitems > ngroups) |
216 | 0 | nitems = ngroups; |
217 | | |
218 | | /* |
219 | | * Decide how many items to keep in the MCV list. We can't use the same |
220 | | * algorithm as per-column MCV lists, because that only considers the |
221 | | * actual group frequency - but we're primarily interested in how the |
222 | | * actual frequency differs from the base frequency (product of simple |
223 | | * per-column frequencies, as if the columns were independent). |
224 | | * |
225 | | * Using the same algorithm might exclude items that are close to the |
226 | | * "average" frequency of the sample. But that does not say whether the |
227 | | * observed frequency is close to the base frequency or not. We also need |
228 | | * to consider unexpectedly uncommon items (again, compared to the base |
229 | | * frequency), and the single-column algorithm does not have to. |
230 | | * |
231 | | * We simply decide how many items to keep by computing the minimum count |
232 | | * using get_mincount_for_mcv_list() and then keep all items that seem to |
233 | | * be more common than that. |
234 | | */ |
235 | 0 | mincount = get_mincount_for_mcv_list(numrows, totalrows); |
236 | | |
237 | | /* |
238 | | * Walk the groups until we find the first group with a count below the |
239 | | * mincount threshold (the index of that group is the number of groups we |
240 | | * want to keep). |
241 | | */ |
242 | 0 | for (i = 0; i < nitems; i++) |
243 | 0 | { |
244 | 0 | if (groups[i].count < mincount) |
245 | 0 | { |
246 | 0 | nitems = i; |
247 | 0 | break; |
248 | 0 | } |
249 | 0 | } |
250 | | |
251 | | /* |
252 | | * At this point, we know the number of items for the MCV list. There |
253 | | * might be none (for uniform distribution with many groups), and in that |
254 | | * case, there will be no MCV list. Otherwise, construct the MCV list. |
255 | | */ |
256 | 0 | if (nitems > 0) |
257 | 0 | { |
258 | 0 | int j; |
259 | 0 | SortItem key; |
260 | 0 | MultiSortSupport tmp; |
261 | | |
262 | | /* frequencies for values in each attribute */ |
263 | 0 | SortItem **freqs; |
264 | 0 | int *nfreqs; |
265 | | |
266 | | /* used to search values */ |
267 | 0 | tmp = (MultiSortSupport) palloc(offsetof(MultiSortSupportData, ssup) |
268 | 0 | + sizeof(SortSupportData)); |
269 | | |
270 | | /* compute frequencies for values in each column */ |
271 | 0 | nfreqs = palloc0_array(int, numattrs); |
272 | 0 | freqs = build_column_frequencies(groups, ngroups, mss, nfreqs); |
273 | | |
274 | | /* |
275 | | * Allocate the MCV list structure, set the global parameters. |
276 | | */ |
277 | 0 | mcvlist = (MCVList *) palloc0(offsetof(MCVList, items) + |
278 | 0 | sizeof(MCVItem) * nitems); |
279 | |
|
280 | 0 | mcvlist->magic = STATS_MCV_MAGIC; |
281 | 0 | mcvlist->type = STATS_MCV_TYPE_BASIC; |
282 | 0 | mcvlist->ndimensions = numattrs; |
283 | 0 | mcvlist->nitems = nitems; |
284 | | |
285 | | /* store info about data type OIDs */ |
286 | 0 | for (i = 0; i < numattrs; i++) |
287 | 0 | mcvlist->types[i] = data->stats[i]->attrtypid; |
288 | | |
289 | | /* Copy the first chunk of groups into the result. */ |
290 | 0 | for (i = 0; i < nitems; i++) |
291 | 0 | { |
292 | | /* just point to the proper place in the list */ |
293 | 0 | MCVItem *item = &mcvlist->items[i]; |
294 | |
|
295 | 0 | item->values = palloc_array(Datum, numattrs); |
296 | 0 | item->isnull = palloc_array(bool, numattrs); |
297 | | |
298 | | /* copy values for the group */ |
299 | 0 | memcpy(item->values, groups[i].values, sizeof(Datum) * numattrs); |
300 | 0 | memcpy(item->isnull, groups[i].isnull, sizeof(bool) * numattrs); |
301 | | |
302 | | /* groups should be sorted by frequency in descending order */ |
303 | 0 | Assert((i == 0) || (groups[i - 1].count >= groups[i].count)); |
304 | | |
305 | | /* group frequency */ |
306 | 0 | item->frequency = (double) groups[i].count / numrows; |
307 | | |
308 | | /* base frequency, if the attributes were independent */ |
309 | 0 | item->base_frequency = 1.0; |
310 | 0 | for (j = 0; j < numattrs; j++) |
311 | 0 | { |
312 | 0 | SortItem *freq; |
313 | | |
314 | | /* single dimension */ |
315 | 0 | tmp->ndims = 1; |
316 | 0 | tmp->ssup[0] = mss->ssup[j]; |
317 | | |
318 | | /* fill search key */ |
319 | 0 | key.values = &groups[i].values[j]; |
320 | 0 | key.isnull = &groups[i].isnull[j]; |
321 | |
|
322 | 0 | freq = (SortItem *) bsearch_arg(&key, freqs[j], nfreqs[j], |
323 | 0 | sizeof(SortItem), |
324 | 0 | multi_sort_compare, tmp); |
325 | |
|
326 | 0 | item->base_frequency *= ((double) freq->count) / numrows; |
327 | 0 | } |
328 | 0 | } |
329 | |
|
330 | 0 | pfree(nfreqs); |
331 | 0 | pfree(freqs); |
332 | 0 | } |
333 | |
|
334 | 0 | pfree(items); |
335 | 0 | pfree(groups); |
336 | |
|
337 | 0 | return mcvlist; |
338 | 0 | } |
339 | | |
340 | | /* |
341 | | * build_mss |
342 | | * Build a MultiSortSupport for the given StatsBuildData. |
343 | | */ |
344 | | static MultiSortSupport |
345 | | build_mss(StatsBuildData *data) |
346 | 0 | { |
347 | 0 | int i; |
348 | 0 | int numattrs = data->nattnums; |
349 | | |
350 | | /* Sort by multiple columns (using array of SortSupport) */ |
351 | 0 | MultiSortSupport mss = multi_sort_init(numattrs); |
352 | | |
353 | | /* prepare the sort functions for all the attributes */ |
354 | 0 | for (i = 0; i < numattrs; i++) |
355 | 0 | { |
356 | 0 | VacAttrStats *colstat = data->stats[i]; |
357 | 0 | TypeCacheEntry *type; |
358 | |
|
359 | 0 | type = lookup_type_cache(colstat->attrtypid, TYPECACHE_LT_OPR); |
360 | 0 | if (type->lt_opr == InvalidOid) /* shouldn't happen */ |
361 | 0 | elog(ERROR, "cache lookup failed for ordering operator for type %u", |
362 | 0 | colstat->attrtypid); |
363 | | |
364 | 0 | multi_sort_add_dimension(mss, i, type->lt_opr, colstat->attrcollid); |
365 | 0 | } |
366 | | |
367 | 0 | return mss; |
368 | 0 | } |
369 | | |
370 | | /* |
371 | | * count_distinct_groups |
372 | | * Count distinct combinations of SortItems in the array. |
373 | | * |
374 | | * The array is assumed to be sorted according to the MultiSortSupport. |
375 | | */ |
376 | | static int |
377 | | count_distinct_groups(int numrows, SortItem *items, MultiSortSupport mss) |
378 | 0 | { |
379 | 0 | int i; |
380 | 0 | int ndistinct; |
381 | |
|
382 | 0 | ndistinct = 1; |
383 | 0 | for (i = 1; i < numrows; i++) |
384 | 0 | { |
385 | | /* make sure the array really is sorted */ |
386 | 0 | Assert(multi_sort_compare(&items[i], &items[i - 1], mss) >= 0); |
387 | |
|
388 | 0 | if (multi_sort_compare(&items[i], &items[i - 1], mss) != 0) |
389 | 0 | ndistinct += 1; |
390 | 0 | } |
391 | |
|
392 | 0 | return ndistinct; |
393 | 0 | } |
394 | | |
395 | | /* |
396 | | * compare_sort_item_count |
397 | | * Comparator for sorting items by count (frequencies) in descending |
398 | | * order. |
399 | | */ |
400 | | static int |
401 | | compare_sort_item_count(const void *a, const void *b, void *arg) |
402 | 0 | { |
403 | 0 | const SortItem *ia = a; |
404 | 0 | const SortItem *ib = b; |
405 | |
|
406 | 0 | if (ia->count == ib->count) |
407 | 0 | return 0; |
408 | 0 | else if (ia->count > ib->count) |
409 | 0 | return -1; |
410 | | |
411 | 0 | return 1; |
412 | 0 | } |
413 | | |
414 | | /* |
415 | | * build_distinct_groups |
416 | | * Build an array of SortItems for distinct groups and counts matching |
417 | | * items. |
418 | | * |
419 | | * The 'items' array is assumed to be sorted. |
420 | | */ |
421 | | static SortItem * |
422 | | build_distinct_groups(int numrows, SortItem *items, MultiSortSupport mss, |
423 | | int *ndistinct) |
424 | 0 | { |
425 | 0 | int i, |
426 | 0 | j; |
427 | 0 | int ngroups = count_distinct_groups(numrows, items, mss); |
428 | |
|
429 | 0 | SortItem *groups = palloc_array(SortItem, ngroups); |
430 | |
|
431 | 0 | j = 0; |
432 | 0 | groups[0] = items[0]; |
433 | 0 | groups[0].count = 1; |
434 | |
|
435 | 0 | for (i = 1; i < numrows; i++) |
436 | 0 | { |
437 | | /* Assume sorted in ascending order. */ |
438 | 0 | Assert(multi_sort_compare(&items[i], &items[i - 1], mss) >= 0); |
439 | | |
440 | | /* New distinct group detected. */ |
441 | 0 | if (multi_sort_compare(&items[i], &items[i - 1], mss) != 0) |
442 | 0 | { |
443 | 0 | groups[++j] = items[i]; |
444 | 0 | groups[j].count = 0; |
445 | 0 | } |
446 | |
|
447 | 0 | groups[j].count++; |
448 | 0 | } |
449 | | |
450 | | /* ensure we filled the expected number of distinct groups */ |
451 | 0 | Assert(j + 1 == ngroups); |
452 | | |
453 | | /* Sort the distinct groups by frequency (in descending order). */ |
454 | 0 | qsort_interruptible(groups, ngroups, sizeof(SortItem), |
455 | 0 | compare_sort_item_count, NULL); |
456 | |
|
457 | 0 | *ndistinct = ngroups; |
458 | 0 | return groups; |
459 | 0 | } |
460 | | |
461 | | /* compare sort items (single dimension) */ |
462 | | static int |
463 | | sort_item_compare(const void *a, const void *b, void *arg) |
464 | 0 | { |
465 | 0 | SortSupport ssup = (SortSupport) arg; |
466 | 0 | const SortItem *ia = a; |
467 | 0 | const SortItem *ib = b; |
468 | |
|
469 | 0 | return ApplySortComparator(ia->values[0], ia->isnull[0], |
470 | 0 | ib->values[0], ib->isnull[0], |
471 | 0 | ssup); |
472 | 0 | } |
473 | | |
474 | | /* |
475 | | * build_column_frequencies |
476 | | * Compute frequencies of values in each column. |
477 | | * |
478 | | * This returns an array of SortItems for each attribute the MCV is built |
479 | | * on, with a frequency (number of occurrences) for each value. This is |
480 | | * then used to compute "base" frequency of MCV items. |
481 | | * |
482 | | * All the memory is allocated in a single chunk, so that a single pfree |
483 | | * is enough to release it. We do not allocate space for values/isnull |
484 | | * arrays in the SortItems, because we can simply point into the input |
485 | | * groups directly. |
486 | | */ |
487 | | static SortItem ** |
488 | | build_column_frequencies(SortItem *groups, int ngroups, |
489 | | MultiSortSupport mss, int *ncounts) |
490 | 0 | { |
491 | 0 | int i, |
492 | 0 | dim; |
493 | 0 | SortItem **result; |
494 | 0 | char *ptr; |
495 | |
|
496 | 0 | Assert(groups); |
497 | 0 | Assert(ncounts); |
498 | | |
499 | | /* allocate arrays for all columns as a single chunk */ |
500 | 0 | ptr = palloc(MAXALIGN(sizeof(SortItem *) * mss->ndims) + |
501 | 0 | mss->ndims * MAXALIGN(sizeof(SortItem) * ngroups)); |
502 | | |
503 | | /* initial array of pointers */ |
504 | 0 | result = (SortItem **) ptr; |
505 | 0 | ptr += MAXALIGN(sizeof(SortItem *) * mss->ndims); |
506 | |
|
507 | 0 | for (dim = 0; dim < mss->ndims; dim++) |
508 | 0 | { |
509 | 0 | SortSupport ssup = &mss->ssup[dim]; |
510 | | |
511 | | /* array of values for a single column */ |
512 | 0 | result[dim] = (SortItem *) ptr; |
513 | 0 | ptr += MAXALIGN(sizeof(SortItem) * ngroups); |
514 | | |
515 | | /* extract data for the dimension */ |
516 | 0 | for (i = 0; i < ngroups; i++) |
517 | 0 | { |
518 | | /* point into the input groups */ |
519 | 0 | result[dim][i].values = &groups[i].values[dim]; |
520 | 0 | result[dim][i].isnull = &groups[i].isnull[dim]; |
521 | 0 | result[dim][i].count = groups[i].count; |
522 | 0 | } |
523 | | |
524 | | /* sort the values, deduplicate */ |
525 | 0 | qsort_interruptible(result[dim], ngroups, sizeof(SortItem), |
526 | 0 | sort_item_compare, ssup); |
527 | | |
528 | | /* |
529 | | * Identify distinct values, compute frequency (there might be |
530 | | * multiple MCV items containing this value, so we need to sum counts |
531 | | * from all of them. |
532 | | */ |
533 | 0 | ncounts[dim] = 1; |
534 | 0 | for (i = 1; i < ngroups; i++) |
535 | 0 | { |
536 | 0 | if (sort_item_compare(&result[dim][i - 1], &result[dim][i], ssup) == 0) |
537 | 0 | { |
538 | 0 | result[dim][ncounts[dim] - 1].count += result[dim][i].count; |
539 | 0 | continue; |
540 | 0 | } |
541 | | |
542 | 0 | result[dim][ncounts[dim]] = result[dim][i]; |
543 | |
|
544 | 0 | ncounts[dim]++; |
545 | 0 | } |
546 | 0 | } |
547 | |
|
548 | 0 | return result; |
549 | 0 | } |
550 | | |
551 | | /* |
552 | | * statext_mcv_load |
553 | | * Load the MCV list for the indicated pg_statistic_ext_data tuple. |
554 | | */ |
555 | | MCVList * |
556 | | statext_mcv_load(Oid mvoid, bool inh) |
557 | 0 | { |
558 | 0 | MCVList *result; |
559 | 0 | bool isnull; |
560 | 0 | Datum mcvlist; |
561 | 0 | HeapTuple htup = SearchSysCache2(STATEXTDATASTXOID, |
562 | 0 | ObjectIdGetDatum(mvoid), BoolGetDatum(inh)); |
563 | |
|
564 | 0 | if (!HeapTupleIsValid(htup)) |
565 | 0 | elog(ERROR, "cache lookup failed for statistics object %u", mvoid); |
566 | | |
567 | 0 | mcvlist = SysCacheGetAttr(STATEXTDATASTXOID, htup, |
568 | 0 | Anum_pg_statistic_ext_data_stxdmcv, &isnull); |
569 | |
|
570 | 0 | if (isnull) |
571 | 0 | elog(ERROR, |
572 | 0 | "requested statistics kind \"%c\" is not yet built for statistics object %u", |
573 | 0 | STATS_EXT_MCV, mvoid); |
574 | | |
575 | 0 | result = statext_mcv_deserialize(DatumGetByteaP(mcvlist)); |
576 | |
|
577 | 0 | ReleaseSysCache(htup); |
578 | |
|
579 | 0 | return result; |
580 | 0 | } |
581 | | |
582 | | |
583 | | /* |
584 | | * statext_mcv_serialize |
585 | | * Serialize MCV list into a pg_mcv_list value. |
586 | | * |
587 | | * The MCV items may include values of various data types, and it's reasonable |
588 | | * to expect redundancy (values for a given attribute, repeated for multiple |
589 | | * MCV list items). So we deduplicate the values into arrays, and then replace |
590 | | * the values by indexes into those arrays. |
591 | | * |
592 | | * The overall structure of the serialized representation looks like this: |
593 | | * |
594 | | * +---------------+----------------+---------------------+-------+ |
595 | | * | header fields | dimension info | deduplicated values | items | |
596 | | * +---------------+----------------+---------------------+-------+ |
597 | | * |
598 | | * Where dimension info stores information about the type of the K-th |
599 | | * attribute (e.g. typlen, typbyval and length of deduplicated values). |
600 | | * Deduplicated values store deduplicated values for each attribute. And |
601 | | * items store the actual MCV list items, with values replaced by indexes into |
602 | | * the arrays. |
603 | | * |
604 | | * When serializing the items, we use uint16 indexes. The number of MCV items |
605 | | * is limited by the statistics target (which is capped to 10k at the moment). |
606 | | * We might increase this to 65k and still fit into uint16, so there's a bit of |
607 | | * slack. Furthermore, this limit is on the number of distinct values per column, |
608 | | * and we usually have few of those (and various combinations of them for the |
609 | | * those MCV list). So uint16 seems fine for now. |
610 | | * |
611 | | * We don't really expect the serialization to save as much space as for |
612 | | * histograms, as we are not doing any bucket splits (which is the source |
613 | | * of high redundancy in histograms). |
614 | | * |
615 | | * TODO: Consider packing boolean flags (NULL) for each item into a single char |
616 | | * (or a longer type) instead of using an array of bool items. |
617 | | */ |
618 | | bytea * |
619 | | statext_mcv_serialize(MCVList *mcvlist, VacAttrStats **stats) |
620 | 0 | { |
621 | 0 | int dim; |
622 | 0 | int ndims = mcvlist->ndimensions; |
623 | |
|
624 | 0 | SortSupport ssup; |
625 | 0 | DimensionInfo *info; |
626 | |
|
627 | 0 | Size total_length; |
628 | | |
629 | | /* serialized items (indexes into arrays, etc.) */ |
630 | 0 | bytea *raw; |
631 | 0 | char *ptr; |
632 | 0 | char *endptr PG_USED_FOR_ASSERTS_ONLY; |
633 | | |
634 | | /* values per dimension (and number of non-NULL values) */ |
635 | 0 | Datum **values = palloc0_array(Datum *, ndims); |
636 | 0 | int *counts = palloc0_array(int, ndims); |
637 | | |
638 | | /* |
639 | | * We'll include some rudimentary information about the attribute types |
640 | | * (length, by-val flag), so that we don't have to look them up while |
641 | | * deserializing the MCV list (we already have the type OID in the |
642 | | * header). This is safe because when changing the type of the attribute |
643 | | * the statistics gets dropped automatically. We need to store the info |
644 | | * about the arrays of deduplicated values anyway. |
645 | | */ |
646 | 0 | info = palloc0_array(DimensionInfo, ndims); |
647 | | |
648 | | /* sort support data for all attributes included in the MCV list */ |
649 | 0 | ssup = palloc0_array(SortSupportData, ndims); |
650 | | |
651 | | /* collect and deduplicate values for each dimension (attribute) */ |
652 | 0 | for (dim = 0; dim < ndims; dim++) |
653 | 0 | { |
654 | 0 | int ndistinct; |
655 | 0 | TypeCacheEntry *typentry; |
656 | | |
657 | | /* |
658 | | * Lookup the LT operator (can't get it from stats extra_data, as we |
659 | | * don't know how to interpret that - scalar vs. array etc.). |
660 | | */ |
661 | 0 | typentry = lookup_type_cache(stats[dim]->attrtypid, TYPECACHE_LT_OPR); |
662 | | |
663 | | /* copy important info about the data type (length, by-value) */ |
664 | 0 | info[dim].typlen = stats[dim]->attrtype->typlen; |
665 | 0 | info[dim].typbyval = stats[dim]->attrtype->typbyval; |
666 | | |
667 | | /* allocate space for values in the attribute and collect them */ |
668 | 0 | values[dim] = palloc0_array(Datum, mcvlist->nitems); |
669 | |
|
670 | 0 | for (uint32 i = 0; i < mcvlist->nitems; i++) |
671 | 0 | { |
672 | | /* skip NULL values - we don't need to deduplicate those */ |
673 | 0 | if (mcvlist->items[i].isnull[dim]) |
674 | 0 | continue; |
675 | | |
676 | | /* append the value at the end */ |
677 | 0 | values[dim][counts[dim]] = mcvlist->items[i].values[dim]; |
678 | 0 | counts[dim] += 1; |
679 | 0 | } |
680 | | |
681 | | /* if there are just NULL values in this dimension, we're done */ |
682 | 0 | if (counts[dim] == 0) |
683 | 0 | continue; |
684 | | |
685 | | /* sort and deduplicate the data */ |
686 | 0 | ssup[dim].ssup_cxt = CurrentMemoryContext; |
687 | 0 | ssup[dim].ssup_collation = stats[dim]->attrcollid; |
688 | 0 | ssup[dim].ssup_nulls_first = false; |
689 | |
|
690 | 0 | PrepareSortSupportFromOrderingOp(typentry->lt_opr, &ssup[dim]); |
691 | |
|
692 | 0 | qsort_interruptible(values[dim], counts[dim], sizeof(Datum), |
693 | 0 | compare_scalars_simple, &ssup[dim]); |
694 | | |
695 | | /* |
696 | | * Walk through the array and eliminate duplicate values, but keep the |
697 | | * ordering (so that we can do a binary search later). We know there's |
698 | | * at least one item as (counts[dim] != 0), so we can skip the first |
699 | | * element. |
700 | | */ |
701 | 0 | ndistinct = 1; /* number of distinct values */ |
702 | 0 | for (int i = 1; i < counts[dim]; i++) |
703 | 0 | { |
704 | | /* expect sorted array */ |
705 | 0 | Assert(compare_datums_simple(values[dim][i - 1], values[dim][i], &ssup[dim]) <= 0); |
706 | | |
707 | | /* if the value is the same as the previous one, we can skip it */ |
708 | 0 | if (!compare_datums_simple(values[dim][i - 1], values[dim][i], &ssup[dim])) |
709 | 0 | continue; |
710 | | |
711 | 0 | values[dim][ndistinct] = values[dim][i]; |
712 | 0 | ndistinct += 1; |
713 | 0 | } |
714 | | |
715 | | /* we must not exceed PG_UINT16_MAX, as we use uint16 indexes */ |
716 | 0 | Assert(ndistinct <= PG_UINT16_MAX); |
717 | | |
718 | | /* |
719 | | * Store additional info about the attribute - number of deduplicated |
720 | | * values, and also size of the serialized data. For fixed-length data |
721 | | * types this is trivial to compute, for varwidth types we need to |
722 | | * actually walk the array and sum the sizes. |
723 | | */ |
724 | 0 | info[dim].nvalues = ndistinct; |
725 | |
|
726 | 0 | if (info[dim].typbyval) /* by-value data types */ |
727 | 0 | { |
728 | 0 | info[dim].nbytes = info[dim].nvalues * info[dim].typlen; |
729 | | |
730 | | /* |
731 | | * We copy the data into the MCV item during deserialization, so |
732 | | * we don't need to allocate any extra space. |
733 | | */ |
734 | 0 | info[dim].nbytes_aligned = 0; |
735 | 0 | } |
736 | 0 | else if (info[dim].typlen > 0) /* fixed-length by-ref */ |
737 | 0 | { |
738 | | /* |
739 | | * We don't care about alignment in the serialized data, so we |
740 | | * pack the data as much as possible. But we also track how much |
741 | | * data will be needed after deserialization, and in that case we |
742 | | * need to account for alignment of each item. |
743 | | * |
744 | | * Note: As the items are fixed-length, we could easily compute |
745 | | * this during deserialization, but we do it here anyway. |
746 | | */ |
747 | 0 | info[dim].nbytes = info[dim].nvalues * info[dim].typlen; |
748 | 0 | info[dim].nbytes_aligned = info[dim].nvalues * MAXALIGN(info[dim].typlen); |
749 | 0 | } |
750 | 0 | else if (info[dim].typlen == -1) /* varlena */ |
751 | 0 | { |
752 | 0 | info[dim].nbytes = 0; |
753 | 0 | info[dim].nbytes_aligned = 0; |
754 | 0 | for (int i = 0; i < info[dim].nvalues; i++) |
755 | 0 | { |
756 | 0 | Size len; |
757 | | |
758 | | /* |
759 | | * For varlena values, we detoast the values and store the |
760 | | * length and data separately. We don't bother with alignment |
761 | | * here, which means that during deserialization we need to |
762 | | * copy the fields and only access the copies. |
763 | | */ |
764 | 0 | values[dim][i] = PointerGetDatum(PG_DETOAST_DATUM(values[dim][i])); |
765 | | |
766 | | /* serialized length (uint32 length + data) */ |
767 | 0 | len = VARSIZE_ANY_EXHDR(DatumGetPointer(values[dim][i])); |
768 | 0 | info[dim].nbytes += sizeof(uint32); /* length */ |
769 | 0 | info[dim].nbytes += len; /* value (no header) */ |
770 | | |
771 | | /* |
772 | | * During deserialization we'll build regular varlena values |
773 | | * with full headers, and we need to align them properly. |
774 | | */ |
775 | 0 | info[dim].nbytes_aligned += MAXALIGN(VARHDRSZ + len); |
776 | 0 | } |
777 | 0 | } |
778 | 0 | else if (info[dim].typlen == -2) /* cstring */ |
779 | 0 | { |
780 | 0 | info[dim].nbytes = 0; |
781 | 0 | info[dim].nbytes_aligned = 0; |
782 | 0 | for (int i = 0; i < info[dim].nvalues; i++) |
783 | 0 | { |
784 | 0 | Size len; |
785 | | |
786 | | /* |
787 | | * cstring is handled similar to varlena - first we store the |
788 | | * length as uint32 and then the data. We don't care about |
789 | | * alignment, which means that during deserialization we need |
790 | | * to copy the fields and only access the copies. |
791 | | */ |
792 | | |
793 | | /* c-strings include terminator, so +1 byte */ |
794 | 0 | len = strlen(DatumGetCString(values[dim][i])) + 1; |
795 | 0 | info[dim].nbytes += sizeof(uint32); /* length */ |
796 | 0 | info[dim].nbytes += len; /* value */ |
797 | | |
798 | | /* space needed for properly aligned deserialized copies */ |
799 | 0 | info[dim].nbytes_aligned += MAXALIGN(len); |
800 | 0 | } |
801 | 0 | } |
802 | | |
803 | | /* we know (count>0) so there must be some data */ |
804 | 0 | Assert(info[dim].nbytes > 0); |
805 | 0 | } |
806 | | |
807 | | /* |
808 | | * Now we can finally compute how much space we'll actually need for the |
809 | | * whole serialized MCV list (varlena header, MCV header, dimension info |
810 | | * for each attribute, deduplicated values and items). |
811 | | */ |
812 | 0 | total_length = (3 * sizeof(uint32)) /* magic + type + nitems */ |
813 | 0 | + sizeof(AttrNumber) /* ndimensions */ |
814 | 0 | + (ndims * sizeof(Oid)); /* attribute types */ |
815 | | |
816 | | /* dimension info */ |
817 | 0 | total_length += ndims * sizeof(DimensionInfo); |
818 | | |
819 | | /* add space for the arrays of deduplicated values */ |
820 | 0 | for (int i = 0; i < ndims; i++) |
821 | 0 | total_length += info[i].nbytes; |
822 | | |
823 | | /* |
824 | | * And finally account for the items (those are fixed-length, thanks to |
825 | | * replacing values with uint16 indexes into the deduplicated arrays). |
826 | | */ |
827 | 0 | total_length += mcvlist->nitems * ITEM_SIZE(dim); |
828 | | |
829 | | /* |
830 | | * Allocate space for the whole serialized MCV list (we'll skip bytes, so |
831 | | * we set them to zero to make the result more compressible). |
832 | | */ |
833 | 0 | raw = (bytea *) palloc0(VARHDRSZ + total_length); |
834 | 0 | SET_VARSIZE(raw, VARHDRSZ + total_length); |
835 | |
|
836 | 0 | ptr = VARDATA(raw); |
837 | 0 | endptr = ptr + total_length; |
838 | | |
839 | | /* copy the MCV list header fields, one by one */ |
840 | 0 | memcpy(ptr, &mcvlist->magic, sizeof(uint32)); |
841 | 0 | ptr += sizeof(uint32); |
842 | |
|
843 | 0 | memcpy(ptr, &mcvlist->type, sizeof(uint32)); |
844 | 0 | ptr += sizeof(uint32); |
845 | |
|
846 | 0 | memcpy(ptr, &mcvlist->nitems, sizeof(uint32)); |
847 | 0 | ptr += sizeof(uint32); |
848 | |
|
849 | 0 | memcpy(ptr, &mcvlist->ndimensions, sizeof(AttrNumber)); |
850 | 0 | ptr += sizeof(AttrNumber); |
851 | |
|
852 | 0 | memcpy(ptr, mcvlist->types, sizeof(Oid) * ndims); |
853 | 0 | ptr += (sizeof(Oid) * ndims); |
854 | | |
855 | | /* store information about the attributes (data amounts, ...) */ |
856 | 0 | memcpy(ptr, info, sizeof(DimensionInfo) * ndims); |
857 | 0 | ptr += sizeof(DimensionInfo) * ndims; |
858 | | |
859 | | /* Copy the deduplicated values for all attributes to the output. */ |
860 | 0 | for (dim = 0; dim < ndims; dim++) |
861 | 0 | { |
862 | | /* remember the starting point for Asserts later */ |
863 | 0 | char *start PG_USED_FOR_ASSERTS_ONLY = ptr; |
864 | |
|
865 | 0 | for (int i = 0; i < info[dim].nvalues; i++) |
866 | 0 | { |
867 | 0 | Datum value = values[dim][i]; |
868 | |
|
869 | 0 | if (info[dim].typbyval) /* passed by value */ |
870 | 0 | { |
871 | 0 | Datum tmp; |
872 | | |
873 | | /* |
874 | | * For byval types, we need to copy just the significant bytes |
875 | | * - we can't use memcpy directly, as that assumes |
876 | | * little-endian behavior. store_att_byval does almost what |
877 | | * we need, but it requires a properly aligned buffer - the |
878 | | * output buffer does not guarantee that. So we simply use a |
879 | | * local Datum variable (which guarantees proper alignment), |
880 | | * and then copy the value from it. |
881 | | */ |
882 | 0 | store_att_byval(&tmp, value, info[dim].typlen); |
883 | |
|
884 | 0 | memcpy(ptr, &tmp, info[dim].typlen); |
885 | 0 | ptr += info[dim].typlen; |
886 | 0 | } |
887 | 0 | else if (info[dim].typlen > 0) /* passed by reference */ |
888 | 0 | { |
889 | | /* no special alignment needed, treated as char array */ |
890 | 0 | memcpy(ptr, DatumGetPointer(value), info[dim].typlen); |
891 | 0 | ptr += info[dim].typlen; |
892 | 0 | } |
893 | 0 | else if (info[dim].typlen == -1) /* varlena */ |
894 | 0 | { |
895 | 0 | uint32 len = VARSIZE_ANY_EXHDR(DatumGetPointer(value)); |
896 | | |
897 | | /* copy the length */ |
898 | 0 | memcpy(ptr, &len, sizeof(uint32)); |
899 | 0 | ptr += sizeof(uint32); |
900 | | |
901 | | /* data from the varlena value (without the header) */ |
902 | 0 | memcpy(ptr, VARDATA_ANY(DatumGetPointer(value)), len); |
903 | 0 | ptr += len; |
904 | 0 | } |
905 | 0 | else if (info[dim].typlen == -2) /* cstring */ |
906 | 0 | { |
907 | 0 | uint32 len = (uint32) strlen(DatumGetCString(value)) + 1; |
908 | | |
909 | | /* copy the length */ |
910 | 0 | memcpy(ptr, &len, sizeof(uint32)); |
911 | 0 | ptr += sizeof(uint32); |
912 | | |
913 | | /* value */ |
914 | 0 | memcpy(ptr, DatumGetCString(value), len); |
915 | 0 | ptr += len; |
916 | 0 | } |
917 | | |
918 | | /* no underflows or overflows */ |
919 | 0 | Assert((ptr > start) && ((ptr - start) <= info[dim].nbytes)); |
920 | 0 | } |
921 | | |
922 | | /* we should get exactly nbytes of data for this dimension */ |
923 | 0 | Assert((ptr - start) == info[dim].nbytes); |
924 | 0 | } |
925 | | |
926 | | /* Serialize the items, with uint16 indexes instead of the values. */ |
927 | 0 | for (uint32 i = 0; i < mcvlist->nitems; i++) |
928 | 0 | { |
929 | 0 | MCVItem *mcvitem = &mcvlist->items[i]; |
930 | | |
931 | | /* don't write beyond the allocated space */ |
932 | 0 | Assert(ptr <= (endptr - ITEM_SIZE(dim))); |
933 | | |
934 | | /* copy NULL and frequency flags into the serialized MCV */ |
935 | 0 | memcpy(ptr, mcvitem->isnull, sizeof(bool) * ndims); |
936 | 0 | ptr += sizeof(bool) * ndims; |
937 | |
|
938 | 0 | memcpy(ptr, &mcvitem->frequency, sizeof(double)); |
939 | 0 | ptr += sizeof(double); |
940 | |
|
941 | 0 | memcpy(ptr, &mcvitem->base_frequency, sizeof(double)); |
942 | 0 | ptr += sizeof(double); |
943 | | |
944 | | /* store the indexes last */ |
945 | 0 | for (dim = 0; dim < ndims; dim++) |
946 | 0 | { |
947 | 0 | uint16 index = 0; |
948 | 0 | Datum *value; |
949 | | |
950 | | /* do the lookup only for non-NULL values */ |
951 | 0 | if (!mcvitem->isnull[dim]) |
952 | 0 | { |
953 | 0 | value = (Datum *) bsearch_arg(&mcvitem->values[dim], values[dim], |
954 | 0 | info[dim].nvalues, sizeof(Datum), |
955 | 0 | compare_scalars_simple, &ssup[dim]); |
956 | |
|
957 | 0 | Assert(value != NULL); /* serialization or deduplication |
958 | | * error */ |
959 | | |
960 | | /* compute index within the deduplicated array */ |
961 | 0 | index = (uint16) (value - values[dim]); |
962 | | |
963 | | /* check the index is within expected bounds */ |
964 | 0 | Assert(index < info[dim].nvalues); |
965 | 0 | } |
966 | | |
967 | | /* copy the index into the serialized MCV */ |
968 | 0 | memcpy(ptr, &index, sizeof(uint16)); |
969 | 0 | ptr += sizeof(uint16); |
970 | 0 | } |
971 | | |
972 | | /* make sure we don't overflow the allocated value */ |
973 | 0 | Assert(ptr <= endptr); |
974 | 0 | } |
975 | | |
976 | | /* at this point we expect to match the total_length exactly */ |
977 | 0 | Assert(ptr == endptr); |
978 | |
|
979 | 0 | pfree(values); |
980 | 0 | pfree(counts); |
981 | |
|
982 | 0 | return raw; |
983 | 0 | } |
984 | | |
985 | | /* |
986 | | * statext_mcv_deserialize |
987 | | * Reads serialized MCV list into MCVList structure. |
988 | | * |
989 | | * All the memory needed by the MCV list is allocated as a single chunk, so |
990 | | * it's possible to simply pfree() it at once. |
991 | | */ |
992 | | MCVList * |
993 | | statext_mcv_deserialize(bytea *data) |
994 | 0 | { |
995 | 0 | int dim, |
996 | 0 | i; |
997 | 0 | Size expected_size; |
998 | 0 | MCVList *mcvlist; |
999 | 0 | char *raw; |
1000 | 0 | char *ptr; |
1001 | 0 | char *endptr PG_USED_FOR_ASSERTS_ONLY; |
1002 | |
|
1003 | 0 | int ndims, |
1004 | 0 | nitems; |
1005 | 0 | DimensionInfo *info = NULL; |
1006 | | |
1007 | | /* local allocation buffer (used only for deserialization) */ |
1008 | 0 | Datum **map = NULL; |
1009 | | |
1010 | | /* MCV list */ |
1011 | 0 | Size mcvlen; |
1012 | | |
1013 | | /* buffer used for the result */ |
1014 | 0 | Size datalen; |
1015 | 0 | char *dataptr; |
1016 | 0 | char *valuesptr; |
1017 | 0 | char *isnullptr; |
1018 | |
|
1019 | 0 | if (data == NULL) |
1020 | 0 | return NULL; |
1021 | | |
1022 | | /* |
1023 | | * We can't possibly deserialize a MCV list if there's not even a complete |
1024 | | * header. We need an explicit formula here, because we serialize the |
1025 | | * header fields one by one, so we need to ignore struct alignment. |
1026 | | */ |
1027 | 0 | if (VARSIZE_ANY(data) < MinSizeOfMCVList) |
1028 | 0 | elog(ERROR, "invalid MCV size %zu (expected at least %zu)", |
1029 | 0 | VARSIZE_ANY(data), MinSizeOfMCVList); |
1030 | | |
1031 | | /* read the MCV list header */ |
1032 | 0 | mcvlist = (MCVList *) palloc0(offsetof(MCVList, items)); |
1033 | | |
1034 | | /* pointer to the data part (skip the varlena header) */ |
1035 | 0 | raw = (char *) data; |
1036 | 0 | ptr = VARDATA_ANY(raw); |
1037 | 0 | endptr = raw + VARSIZE_ANY(data); |
1038 | | |
1039 | | /* get the header and perform further sanity checks */ |
1040 | 0 | memcpy(&mcvlist->magic, ptr, sizeof(uint32)); |
1041 | 0 | ptr += sizeof(uint32); |
1042 | |
|
1043 | 0 | memcpy(&mcvlist->type, ptr, sizeof(uint32)); |
1044 | 0 | ptr += sizeof(uint32); |
1045 | |
|
1046 | 0 | memcpy(&mcvlist->nitems, ptr, sizeof(uint32)); |
1047 | 0 | ptr += sizeof(uint32); |
1048 | |
|
1049 | 0 | memcpy(&mcvlist->ndimensions, ptr, sizeof(AttrNumber)); |
1050 | 0 | ptr += sizeof(AttrNumber); |
1051 | |
|
1052 | 0 | if (mcvlist->magic != STATS_MCV_MAGIC) |
1053 | 0 | elog(ERROR, "invalid MCV magic %u (expected %u)", |
1054 | 0 | mcvlist->magic, STATS_MCV_MAGIC); |
1055 | | |
1056 | 0 | if (mcvlist->type != STATS_MCV_TYPE_BASIC) |
1057 | 0 | elog(ERROR, "invalid MCV type %u (expected %u)", |
1058 | 0 | mcvlist->type, STATS_MCV_TYPE_BASIC); |
1059 | | |
1060 | 0 | if (mcvlist->ndimensions == 0) |
1061 | 0 | elog(ERROR, "invalid zero-length dimension array in MCVList"); |
1062 | 0 | else if ((mcvlist->ndimensions > STATS_MAX_DIMENSIONS) || |
1063 | 0 | (mcvlist->ndimensions < 0)) |
1064 | 0 | elog(ERROR, "invalid length (%d) dimension array in MCVList", |
1065 | 0 | mcvlist->ndimensions); |
1066 | | |
1067 | 0 | if (mcvlist->nitems == 0) |
1068 | 0 | elog(ERROR, "invalid zero-length item array in MCVList"); |
1069 | 0 | else if (mcvlist->nitems > STATS_MCVLIST_MAX_ITEMS) |
1070 | 0 | elog(ERROR, "invalid length (%u) item array in MCVList", |
1071 | 0 | mcvlist->nitems); |
1072 | | |
1073 | 0 | nitems = mcvlist->nitems; |
1074 | 0 | ndims = mcvlist->ndimensions; |
1075 | | |
1076 | | /* |
1077 | | * Check amount of data including DimensionInfo for all dimensions and |
1078 | | * also the serialized items (including uint16 indexes). Also, walk |
1079 | | * through the dimension information and add it to the sum. |
1080 | | */ |
1081 | 0 | expected_size = SizeOfMCVList(ndims, nitems); |
1082 | | |
1083 | | /* |
1084 | | * Check that we have at least the dimension and info records, along with |
1085 | | * the items. We don't know the size of the serialized values yet. We need |
1086 | | * to do this check first, before accessing the dimension info. |
1087 | | */ |
1088 | 0 | if (VARSIZE_ANY(data) < expected_size) |
1089 | 0 | elog(ERROR, "invalid MCV size %zu (expected %zu)", |
1090 | 0 | VARSIZE_ANY(data), expected_size); |
1091 | | |
1092 | | /* Now copy the array of type Oids. */ |
1093 | 0 | memcpy(mcvlist->types, ptr, sizeof(Oid) * ndims); |
1094 | 0 | ptr += (sizeof(Oid) * ndims); |
1095 | | |
1096 | | /* Now it's safe to access the dimension info. */ |
1097 | 0 | info = palloc_array(DimensionInfo, ndims); |
1098 | |
|
1099 | 0 | memcpy(info, ptr, ndims * sizeof(DimensionInfo)); |
1100 | 0 | ptr += (ndims * sizeof(DimensionInfo)); |
1101 | | |
1102 | | /* account for the value arrays */ |
1103 | 0 | for (dim = 0; dim < ndims; dim++) |
1104 | 0 | { |
1105 | | /* |
1106 | | * XXX I wonder if we can/should rely on asserts here. Maybe those |
1107 | | * checks should be done every time? |
1108 | | */ |
1109 | 0 | Assert(info[dim].nvalues >= 0); |
1110 | 0 | Assert(info[dim].nbytes >= 0); |
1111 | |
|
1112 | 0 | expected_size += info[dim].nbytes; |
1113 | 0 | } |
1114 | | |
1115 | | /* |
1116 | | * Now we know the total expected MCV size, including all the pieces |
1117 | | * (header, dimension info. items and deduplicated data). So do the final |
1118 | | * check on size. |
1119 | | */ |
1120 | 0 | if (VARSIZE_ANY(data) != expected_size) |
1121 | 0 | elog(ERROR, "invalid MCV size %zu (expected %zu)", |
1122 | 0 | VARSIZE_ANY(data), expected_size); |
1123 | | |
1124 | | /* |
1125 | | * We need an array of Datum values for each dimension, so that we can |
1126 | | * easily translate the uint16 indexes later. We also need a top-level |
1127 | | * array of pointers to those per-dimension arrays. |
1128 | | * |
1129 | | * While allocating the arrays for dimensions, compute how much space we |
1130 | | * need for a copy of the by-ref data, as we can't simply point to the |
1131 | | * original values (it might go away). |
1132 | | */ |
1133 | 0 | datalen = 0; /* space for by-ref data */ |
1134 | 0 | map = palloc_array(Datum *, ndims); |
1135 | |
|
1136 | 0 | for (dim = 0; dim < ndims; dim++) |
1137 | 0 | { |
1138 | 0 | map[dim] = palloc_array(Datum, info[dim].nvalues); |
1139 | | |
1140 | | /* space needed for a copy of data for by-ref types */ |
1141 | 0 | datalen += info[dim].nbytes_aligned; |
1142 | 0 | } |
1143 | | |
1144 | | /* |
1145 | | * Now resize the MCV list so that the allocation includes all the data. |
1146 | | * |
1147 | | * Allocate space for a copy of the data, as we can't simply reference the |
1148 | | * serialized data - it's not aligned properly, and it may disappear while |
1149 | | * we're still using the MCV list, e.g. due to catcache release. |
1150 | | * |
1151 | | * We do care about alignment here, because we will allocate all the |
1152 | | * pieces at once, but then use pointers to different parts. |
1153 | | */ |
1154 | 0 | mcvlen = MAXALIGN(offsetof(MCVList, items) + (sizeof(MCVItem) * nitems)); |
1155 | | |
1156 | | /* arrays of values and isnull flags for all MCV items */ |
1157 | 0 | mcvlen += nitems * MAXALIGN(sizeof(Datum) * ndims); |
1158 | 0 | mcvlen += nitems * MAXALIGN(sizeof(bool) * ndims); |
1159 | | |
1160 | | /* we don't quite need to align this, but it makes some asserts easier */ |
1161 | 0 | mcvlen += MAXALIGN(datalen); |
1162 | | |
1163 | | /* now resize the deserialized MCV list, and compute pointers to parts */ |
1164 | 0 | mcvlist = repalloc(mcvlist, mcvlen); |
1165 | | |
1166 | | /* pointer to the beginning of values/isnull arrays */ |
1167 | 0 | valuesptr = (char *) mcvlist |
1168 | 0 | + MAXALIGN(offsetof(MCVList, items) + (sizeof(MCVItem) * nitems)); |
1169 | |
|
1170 | 0 | isnullptr = valuesptr + (nitems * MAXALIGN(sizeof(Datum) * ndims)); |
1171 | |
|
1172 | 0 | dataptr = isnullptr + (nitems * MAXALIGN(sizeof(bool) * ndims)); |
1173 | | |
1174 | | /* |
1175 | | * Build mapping (index => value) for translating the serialized data into |
1176 | | * the in-memory representation. |
1177 | | */ |
1178 | 0 | for (dim = 0; dim < ndims; dim++) |
1179 | 0 | { |
1180 | | /* remember start position in the input array */ |
1181 | 0 | char *start PG_USED_FOR_ASSERTS_ONLY = ptr; |
1182 | |
|
1183 | 0 | if (info[dim].typbyval) |
1184 | 0 | { |
1185 | | /* for by-val types we simply copy data into the mapping */ |
1186 | 0 | for (i = 0; i < info[dim].nvalues; i++) |
1187 | 0 | { |
1188 | 0 | Datum v = 0; |
1189 | |
|
1190 | 0 | memcpy(&v, ptr, info[dim].typlen); |
1191 | 0 | ptr += info[dim].typlen; |
1192 | |
|
1193 | 0 | map[dim][i] = fetch_att(&v, true, info[dim].typlen); |
1194 | | |
1195 | | /* no under/overflow of input array */ |
1196 | 0 | Assert(ptr <= (start + info[dim].nbytes)); |
1197 | 0 | } |
1198 | 0 | } |
1199 | 0 | else |
1200 | 0 | { |
1201 | | /* for by-ref types we need to also make a copy of the data */ |
1202 | | |
1203 | | /* passed by reference, but fixed length (name, tid, ...) */ |
1204 | 0 | if (info[dim].typlen > 0) |
1205 | 0 | { |
1206 | 0 | for (i = 0; i < info[dim].nvalues; i++) |
1207 | 0 | { |
1208 | 0 | memcpy(dataptr, ptr, info[dim].typlen); |
1209 | 0 | ptr += info[dim].typlen; |
1210 | | |
1211 | | /* just point into the array */ |
1212 | 0 | map[dim][i] = PointerGetDatum(dataptr); |
1213 | 0 | dataptr += MAXALIGN(info[dim].typlen); |
1214 | 0 | } |
1215 | 0 | } |
1216 | 0 | else if (info[dim].typlen == -1) |
1217 | 0 | { |
1218 | | /* varlena */ |
1219 | 0 | for (i = 0; i < info[dim].nvalues; i++) |
1220 | 0 | { |
1221 | 0 | uint32 len; |
1222 | | |
1223 | | /* read the uint32 length */ |
1224 | 0 | memcpy(&len, ptr, sizeof(uint32)); |
1225 | 0 | ptr += sizeof(uint32); |
1226 | | |
1227 | | /* the length is data-only */ |
1228 | 0 | SET_VARSIZE(dataptr, len + VARHDRSZ); |
1229 | 0 | memcpy(VARDATA(dataptr), ptr, len); |
1230 | 0 | ptr += len; |
1231 | | |
1232 | | /* just point into the array */ |
1233 | 0 | map[dim][i] = PointerGetDatum(dataptr); |
1234 | | |
1235 | | /* skip to place of the next deserialized value */ |
1236 | 0 | dataptr += MAXALIGN(len + VARHDRSZ); |
1237 | 0 | } |
1238 | 0 | } |
1239 | 0 | else if (info[dim].typlen == -2) |
1240 | 0 | { |
1241 | | /* cstring */ |
1242 | 0 | for (i = 0; i < info[dim].nvalues; i++) |
1243 | 0 | { |
1244 | 0 | uint32 len; |
1245 | |
|
1246 | 0 | memcpy(&len, ptr, sizeof(uint32)); |
1247 | 0 | ptr += sizeof(uint32); |
1248 | |
|
1249 | 0 | memcpy(dataptr, ptr, len); |
1250 | 0 | ptr += len; |
1251 | | |
1252 | | /* just point into the array */ |
1253 | 0 | map[dim][i] = PointerGetDatum(dataptr); |
1254 | 0 | dataptr += MAXALIGN(len); |
1255 | 0 | } |
1256 | 0 | } |
1257 | | |
1258 | | /* no under/overflow of input array */ |
1259 | 0 | Assert(ptr <= (start + info[dim].nbytes)); |
1260 | | |
1261 | | /* no overflow of the output mcv value */ |
1262 | 0 | Assert(dataptr <= ((char *) mcvlist + mcvlen)); |
1263 | 0 | } |
1264 | | |
1265 | | /* check we consumed input data for this dimension exactly */ |
1266 | 0 | Assert(ptr == (start + info[dim].nbytes)); |
1267 | 0 | } |
1268 | | |
1269 | | /* we should have also filled the MCV list exactly */ |
1270 | 0 | Assert(dataptr == ((char *) mcvlist + mcvlen)); |
1271 | | |
1272 | | /* deserialize the MCV items and translate the indexes to Datums */ |
1273 | 0 | for (i = 0; i < nitems; i++) |
1274 | 0 | { |
1275 | 0 | MCVItem *item = &mcvlist->items[i]; |
1276 | |
|
1277 | 0 | item->values = (Datum *) valuesptr; |
1278 | 0 | valuesptr += MAXALIGN(sizeof(Datum) * ndims); |
1279 | |
|
1280 | 0 | item->isnull = (bool *) isnullptr; |
1281 | 0 | isnullptr += MAXALIGN(sizeof(bool) * ndims); |
1282 | |
|
1283 | 0 | memcpy(item->isnull, ptr, sizeof(bool) * ndims); |
1284 | 0 | ptr += sizeof(bool) * ndims; |
1285 | |
|
1286 | 0 | memcpy(&item->frequency, ptr, sizeof(double)); |
1287 | 0 | ptr += sizeof(double); |
1288 | |
|
1289 | 0 | memcpy(&item->base_frequency, ptr, sizeof(double)); |
1290 | 0 | ptr += sizeof(double); |
1291 | | |
1292 | | /* finally translate the indexes (for non-NULL only) */ |
1293 | 0 | for (dim = 0; dim < ndims; dim++) |
1294 | 0 | { |
1295 | 0 | uint16 index; |
1296 | |
|
1297 | 0 | memcpy(&index, ptr, sizeof(uint16)); |
1298 | 0 | ptr += sizeof(uint16); |
1299 | |
|
1300 | 0 | if (item->isnull[dim]) |
1301 | 0 | continue; |
1302 | | |
1303 | 0 | item->values[dim] = map[dim][index]; |
1304 | 0 | } |
1305 | | |
1306 | | /* check we're not overflowing the input */ |
1307 | 0 | Assert(ptr <= endptr); |
1308 | 0 | } |
1309 | | |
1310 | | /* check that we processed all the data */ |
1311 | 0 | Assert(ptr == endptr); |
1312 | | |
1313 | | /* release the buffers used for mapping */ |
1314 | 0 | for (dim = 0; dim < ndims; dim++) |
1315 | 0 | pfree(map[dim]); |
1316 | |
|
1317 | 0 | pfree(map); |
1318 | |
|
1319 | 0 | return mcvlist; |
1320 | 0 | } |
1321 | | |
1322 | | /* |
1323 | | * SRF with details about buckets of a histogram: |
1324 | | * |
1325 | | * - item ID (0...nitems) |
1326 | | * - values (string array) |
1327 | | * - nulls only (boolean array) |
1328 | | * - frequency (double precision) |
1329 | | * - base_frequency (double precision) |
1330 | | * |
1331 | | * The input is the OID of the statistics, and there are no rows returned if |
1332 | | * the statistics contains no histogram. |
1333 | | */ |
1334 | | Datum |
1335 | | pg_stats_ext_mcvlist_items(PG_FUNCTION_ARGS) |
1336 | 0 | { |
1337 | 0 | FuncCallContext *funcctx; |
1338 | | |
1339 | | /* stuff done only on the first call of the function */ |
1340 | 0 | if (SRF_IS_FIRSTCALL()) |
1341 | 0 | { |
1342 | 0 | MemoryContext oldcontext; |
1343 | 0 | MCVList *mcvlist; |
1344 | 0 | TupleDesc tupdesc; |
1345 | | |
1346 | | /* create a function context for cross-call persistence */ |
1347 | 0 | funcctx = SRF_FIRSTCALL_INIT(); |
1348 | | |
1349 | | /* switch to memory context appropriate for multiple function calls */ |
1350 | 0 | oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); |
1351 | |
|
1352 | 0 | mcvlist = statext_mcv_deserialize(PG_GETARG_BYTEA_P(0)); |
1353 | |
|
1354 | 0 | funcctx->user_fctx = mcvlist; |
1355 | | |
1356 | | /* total number of tuples to be returned */ |
1357 | 0 | funcctx->max_calls = 0; |
1358 | 0 | if (funcctx->user_fctx != NULL) |
1359 | 0 | funcctx->max_calls = mcvlist->nitems; |
1360 | | |
1361 | | /* Build a tuple descriptor for our result type */ |
1362 | 0 | if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) |
1363 | 0 | ereport(ERROR, |
1364 | 0 | (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), |
1365 | 0 | errmsg("function returning record called in context " |
1366 | 0 | "that cannot accept type record"))); |
1367 | 0 | tupdesc = BlessTupleDesc(tupdesc); |
1368 | | |
1369 | | /* |
1370 | | * generate attribute metadata needed later to produce tuples from raw |
1371 | | * C strings |
1372 | | */ |
1373 | 0 | funcctx->attinmeta = TupleDescGetAttInMetadata(tupdesc); |
1374 | |
|
1375 | 0 | MemoryContextSwitchTo(oldcontext); |
1376 | 0 | } |
1377 | | |
1378 | | /* stuff done on every call of the function */ |
1379 | 0 | funcctx = SRF_PERCALL_SETUP(); |
1380 | |
|
1381 | 0 | if (funcctx->call_cntr < funcctx->max_calls) /* do when there is more |
1382 | | * left to send */ |
1383 | 0 | { |
1384 | 0 | Datum values[5]; |
1385 | 0 | bool nulls[5]; |
1386 | 0 | HeapTuple tuple; |
1387 | 0 | Datum result; |
1388 | 0 | ArrayBuildState *astate_values = NULL; |
1389 | 0 | ArrayBuildState *astate_nulls = NULL; |
1390 | |
|
1391 | 0 | int i; |
1392 | 0 | MCVList *mcvlist; |
1393 | 0 | MCVItem *item; |
1394 | |
|
1395 | 0 | mcvlist = (MCVList *) funcctx->user_fctx; |
1396 | |
|
1397 | 0 | Assert(funcctx->call_cntr < mcvlist->nitems); |
1398 | |
|
1399 | 0 | item = &mcvlist->items[funcctx->call_cntr]; |
1400 | |
|
1401 | 0 | for (i = 0; i < mcvlist->ndimensions; i++) |
1402 | 0 | { |
1403 | |
|
1404 | 0 | astate_nulls = accumArrayResult(astate_nulls, |
1405 | 0 | BoolGetDatum(item->isnull[i]), |
1406 | 0 | false, |
1407 | 0 | BOOLOID, |
1408 | 0 | CurrentMemoryContext); |
1409 | |
|
1410 | 0 | if (!item->isnull[i]) |
1411 | 0 | { |
1412 | 0 | bool isvarlena; |
1413 | 0 | Oid outfunc; |
1414 | 0 | FmgrInfo fmgrinfo; |
1415 | 0 | Datum val; |
1416 | 0 | text *txt; |
1417 | | |
1418 | | /* lookup output func for the type */ |
1419 | 0 | getTypeOutputInfo(mcvlist->types[i], &outfunc, &isvarlena); |
1420 | 0 | fmgr_info(outfunc, &fmgrinfo); |
1421 | |
|
1422 | 0 | val = FunctionCall1(&fmgrinfo, item->values[i]); |
1423 | 0 | txt = cstring_to_text(DatumGetPointer(val)); |
1424 | |
|
1425 | 0 | astate_values = accumArrayResult(astate_values, |
1426 | 0 | PointerGetDatum(txt), |
1427 | 0 | false, |
1428 | 0 | TEXTOID, |
1429 | 0 | CurrentMemoryContext); |
1430 | 0 | } |
1431 | 0 | else |
1432 | 0 | astate_values = accumArrayResult(astate_values, |
1433 | 0 | (Datum) 0, |
1434 | 0 | true, |
1435 | 0 | TEXTOID, |
1436 | 0 | CurrentMemoryContext); |
1437 | 0 | } |
1438 | |
|
1439 | 0 | values[0] = Int32GetDatum(funcctx->call_cntr); |
1440 | 0 | values[1] = makeArrayResult(astate_values, CurrentMemoryContext); |
1441 | 0 | values[2] = makeArrayResult(astate_nulls, CurrentMemoryContext); |
1442 | 0 | values[3] = Float8GetDatum(item->frequency); |
1443 | 0 | values[4] = Float8GetDatum(item->base_frequency); |
1444 | | |
1445 | | /* no NULLs in the tuple */ |
1446 | 0 | memset(nulls, 0, sizeof(nulls)); |
1447 | | |
1448 | | /* build a tuple */ |
1449 | 0 | tuple = heap_form_tuple(funcctx->attinmeta->tupdesc, values, nulls); |
1450 | | |
1451 | | /* make the tuple into a datum */ |
1452 | 0 | result = HeapTupleGetDatum(tuple); |
1453 | |
|
1454 | 0 | SRF_RETURN_NEXT(funcctx, result); |
1455 | 0 | } |
1456 | 0 | else /* do when there is no more left */ |
1457 | 0 | { |
1458 | 0 | SRF_RETURN_DONE(funcctx); |
1459 | 0 | } |
1460 | 0 | } |
1461 | | |
1462 | | /* |
1463 | | * pg_mcv_list_in - input routine for type pg_mcv_list. |
1464 | | * |
1465 | | * pg_mcv_list is real enough to be a table column, but it has no operations |
1466 | | * of its own, and disallows input too |
1467 | | */ |
1468 | | Datum |
1469 | | pg_mcv_list_in(PG_FUNCTION_ARGS) |
1470 | 0 | { |
1471 | | /* |
1472 | | * pg_mcv_list stores the data in binary form and parsing text input is |
1473 | | * not needed, so disallow this. |
1474 | | */ |
1475 | 0 | ereport(ERROR, |
1476 | 0 | (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), |
1477 | 0 | errmsg("cannot accept a value of type %s", "pg_mcv_list"))); |
1478 | | |
1479 | 0 | PG_RETURN_VOID(); /* keep compiler quiet */ |
1480 | 0 | } |
1481 | | |
1482 | | |
1483 | | /* |
1484 | | * pg_mcv_list_out - output routine for type pg_mcv_list. |
1485 | | * |
1486 | | * MCV lists are serialized into a bytea value, so we simply call byteaout() |
1487 | | * to serialize the value into text. But it'd be nice to serialize that into |
1488 | | * a meaningful representation (e.g. for inspection by people). |
1489 | | * |
1490 | | * XXX This should probably return something meaningful, similar to what |
1491 | | * pg_dependencies_out does. Not sure how to deal with the deduplicated |
1492 | | * values, though - do we want to expand that or not? |
1493 | | */ |
1494 | | Datum |
1495 | | pg_mcv_list_out(PG_FUNCTION_ARGS) |
1496 | 0 | { |
1497 | 0 | return byteaout(fcinfo); |
1498 | 0 | } |
1499 | | |
1500 | | /* |
1501 | | * pg_mcv_list_recv - binary input routine for type pg_mcv_list. |
1502 | | */ |
1503 | | Datum |
1504 | | pg_mcv_list_recv(PG_FUNCTION_ARGS) |
1505 | 0 | { |
1506 | 0 | ereport(ERROR, |
1507 | 0 | (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), |
1508 | 0 | errmsg("cannot accept a value of type %s", "pg_mcv_list"))); |
1509 | | |
1510 | 0 | PG_RETURN_VOID(); /* keep compiler quiet */ |
1511 | 0 | } |
1512 | | |
1513 | | /* |
1514 | | * pg_mcv_list_send - binary output routine for type pg_mcv_list. |
1515 | | * |
1516 | | * MCV lists are serialized in a bytea value (although the type is named |
1517 | | * differently), so let's just send that. |
1518 | | */ |
1519 | | Datum |
1520 | | pg_mcv_list_send(PG_FUNCTION_ARGS) |
1521 | 0 | { |
1522 | 0 | return byteasend(fcinfo); |
1523 | 0 | } |
1524 | | |
1525 | | /* |
1526 | | * match the attribute/expression to a dimension of the statistic |
1527 | | * |
1528 | | * Returns the zero-based index of the matching statistics dimension. |
1529 | | * Optionally determines the collation. |
1530 | | */ |
1531 | | static int |
1532 | | mcv_match_expression(Node *expr, Bitmapset *keys, List *exprs, Oid *collid) |
1533 | 0 | { |
1534 | 0 | int idx; |
1535 | |
|
1536 | 0 | if (IsA(expr, Var)) |
1537 | 0 | { |
1538 | | /* simple Var, so just lookup using varattno */ |
1539 | 0 | Var *var = (Var *) expr; |
1540 | |
|
1541 | 0 | if (collid) |
1542 | 0 | *collid = var->varcollid; |
1543 | |
|
1544 | 0 | idx = bms_member_index(keys, var->varattno); |
1545 | |
|
1546 | 0 | if (idx < 0) |
1547 | 0 | elog(ERROR, "variable not found in statistics object"); |
1548 | 0 | } |
1549 | 0 | else |
1550 | 0 | { |
1551 | | /* expression - lookup in stats expressions */ |
1552 | 0 | ListCell *lc; |
1553 | |
|
1554 | 0 | if (collid) |
1555 | 0 | *collid = exprCollation(expr); |
1556 | | |
1557 | | /* expressions are stored after the simple columns */ |
1558 | 0 | idx = bms_num_members(keys); |
1559 | 0 | foreach(lc, exprs) |
1560 | 0 | { |
1561 | 0 | Node *stat_expr = (Node *) lfirst(lc); |
1562 | |
|
1563 | 0 | if (equal(expr, stat_expr)) |
1564 | 0 | break; |
1565 | | |
1566 | 0 | idx++; |
1567 | 0 | } |
1568 | |
|
1569 | 0 | if (lc == NULL) |
1570 | 0 | elog(ERROR, "expression not found in statistics object"); |
1571 | 0 | } |
1572 | | |
1573 | 0 | return idx; |
1574 | 0 | } |
1575 | | |
1576 | | /* |
1577 | | * mcv_get_match_bitmap |
1578 | | * Evaluate clauses using the MCV list, and update the match bitmap. |
1579 | | * |
1580 | | * A match bitmap keeps match/mismatch status for each MCV item, and we |
1581 | | * update it based on additional clauses. We also use it to skip items |
1582 | | * that can't possibly match (e.g. item marked as "mismatch" can't change |
1583 | | * to "match" when evaluating AND clause list). |
1584 | | * |
1585 | | * The function also returns a flag indicating whether there was an |
1586 | | * equality condition for all attributes, the minimum frequency in the MCV |
1587 | | * list, and a total MCV frequency (sum of frequencies for all items). |
1588 | | * |
1589 | | * XXX Currently the match bitmap uses a bool for each MCV item, which is |
1590 | | * somewhat wasteful as we could do with just a single bit, thus reducing |
1591 | | * the size to ~1/8. It would also allow us to combine bitmaps simply using |
1592 | | * & and |, which should be faster than min/max. The bitmaps are fairly |
1593 | | * small, though (thanks to the cap on the MCV list size). |
1594 | | */ |
1595 | | static bool * |
1596 | | mcv_get_match_bitmap(PlannerInfo *root, List *clauses, |
1597 | | Bitmapset *keys, List *exprs, |
1598 | | MCVList *mcvlist, bool is_or) |
1599 | 0 | { |
1600 | 0 | ListCell *l; |
1601 | 0 | bool *matches; |
1602 | | |
1603 | | /* The bitmap may be partially built. */ |
1604 | 0 | Assert(clauses != NIL); |
1605 | 0 | Assert(mcvlist != NULL); |
1606 | 0 | Assert(mcvlist->nitems > 0); |
1607 | 0 | Assert(mcvlist->nitems <= STATS_MCVLIST_MAX_ITEMS); |
1608 | |
|
1609 | 0 | matches = palloc_array(bool, mcvlist->nitems); |
1610 | 0 | memset(matches, !is_or, sizeof(bool) * mcvlist->nitems); |
1611 | | |
1612 | | /* |
1613 | | * Loop through the list of clauses, and for each of them evaluate all the |
1614 | | * MCV items not yet eliminated by the preceding clauses. |
1615 | | */ |
1616 | 0 | foreach(l, clauses) |
1617 | 0 | { |
1618 | 0 | Node *clause = (Node *) lfirst(l); |
1619 | | |
1620 | | /* if it's a RestrictInfo, then extract the clause */ |
1621 | 0 | if (IsA(clause, RestrictInfo)) |
1622 | 0 | clause = (Node *) ((RestrictInfo *) clause)->clause; |
1623 | | |
1624 | | /* |
1625 | | * Handle the various types of clauses - OpClause, NullTest and |
1626 | | * AND/OR/NOT |
1627 | | */ |
1628 | 0 | if (is_opclause(clause)) |
1629 | 0 | { |
1630 | 0 | OpExpr *expr = (OpExpr *) clause; |
1631 | 0 | FmgrInfo opproc; |
1632 | | |
1633 | | /* valid only after examine_opclause_args returns true */ |
1634 | 0 | Node *clause_expr; |
1635 | 0 | Const *cst; |
1636 | 0 | bool expronleft; |
1637 | 0 | int idx; |
1638 | 0 | Oid collid; |
1639 | |
|
1640 | 0 | fmgr_info(get_opcode(expr->opno), &opproc); |
1641 | | |
1642 | | /* extract the var/expr and const from the expression */ |
1643 | 0 | if (!examine_opclause_args(expr->args, &clause_expr, &cst, &expronleft)) |
1644 | 0 | elog(ERROR, "incompatible clause"); |
1645 | | |
1646 | | /* match the attribute/expression to a dimension of the statistic */ |
1647 | 0 | idx = mcv_match_expression(clause_expr, keys, exprs, &collid); |
1648 | | |
1649 | | /* |
1650 | | * Walk through the MCV items and evaluate the current clause. We |
1651 | | * can skip items that were already ruled out, and terminate if |
1652 | | * there are no remaining MCV items that might possibly match. |
1653 | | */ |
1654 | 0 | for (uint32 i = 0; i < mcvlist->nitems; i++) |
1655 | 0 | { |
1656 | 0 | bool match = true; |
1657 | 0 | MCVItem *item = &mcvlist->items[i]; |
1658 | |
|
1659 | 0 | Assert(idx >= 0); |
1660 | | |
1661 | | /* |
1662 | | * When the MCV item or the Const value is NULL we can treat |
1663 | | * this as a mismatch. We must not call the operator because |
1664 | | * of strictness. |
1665 | | */ |
1666 | 0 | if (item->isnull[idx] || cst->constisnull) |
1667 | 0 | { |
1668 | 0 | matches[i] = RESULT_MERGE(matches[i], is_or, false); |
1669 | 0 | continue; |
1670 | 0 | } |
1671 | | |
1672 | | /* |
1673 | | * Skip MCV items that can't change result in the bitmap. Once |
1674 | | * the value gets false for AND-lists, or true for OR-lists, |
1675 | | * we don't need to look at more clauses. |
1676 | | */ |
1677 | 0 | if (RESULT_IS_FINAL(matches[i], is_or)) |
1678 | 0 | continue; |
1679 | | |
1680 | | /* |
1681 | | * First check whether the constant is below the lower |
1682 | | * boundary (in that case we can skip the bucket, because |
1683 | | * there's no overlap). |
1684 | | * |
1685 | | * We don't store collations used to build the statistics, but |
1686 | | * we can use the collation for the attribute itself, as |
1687 | | * stored in varcollid. We do reset the statistics after a |
1688 | | * type change (including collation change), so this is OK. |
1689 | | * For expressions, we use the collation extracted from the |
1690 | | * expression itself. |
1691 | | */ |
1692 | 0 | if (expronleft) |
1693 | 0 | match = DatumGetBool(FunctionCall2Coll(&opproc, |
1694 | 0 | collid, |
1695 | 0 | item->values[idx], |
1696 | 0 | cst->constvalue)); |
1697 | 0 | else |
1698 | 0 | match = DatumGetBool(FunctionCall2Coll(&opproc, |
1699 | 0 | collid, |
1700 | 0 | cst->constvalue, |
1701 | 0 | item->values[idx])); |
1702 | | |
1703 | | /* update the match bitmap with the result */ |
1704 | 0 | matches[i] = RESULT_MERGE(matches[i], is_or, match); |
1705 | 0 | } |
1706 | 0 | } |
1707 | 0 | else if (IsA(clause, ScalarArrayOpExpr)) |
1708 | 0 | { |
1709 | 0 | ScalarArrayOpExpr *expr = (ScalarArrayOpExpr *) clause; |
1710 | 0 | FmgrInfo opproc; |
1711 | | |
1712 | | /* valid only after examine_opclause_args returns true */ |
1713 | 0 | Node *clause_expr; |
1714 | 0 | Const *cst; |
1715 | 0 | bool expronleft; |
1716 | 0 | Oid collid; |
1717 | 0 | int idx; |
1718 | | |
1719 | | /* array evaluation */ |
1720 | 0 | ArrayType *arrayval; |
1721 | 0 | int16 elmlen; |
1722 | 0 | bool elmbyval; |
1723 | 0 | char elmalign; |
1724 | 0 | int num_elems; |
1725 | 0 | Datum *elem_values; |
1726 | 0 | bool *elem_nulls; |
1727 | |
|
1728 | 0 | fmgr_info(get_opcode(expr->opno), &opproc); |
1729 | | |
1730 | | /* extract the var/expr and const from the expression */ |
1731 | 0 | if (!examine_opclause_args(expr->args, &clause_expr, &cst, &expronleft)) |
1732 | 0 | elog(ERROR, "incompatible clause"); |
1733 | | |
1734 | | /* We expect Var on left */ |
1735 | 0 | if (!expronleft) |
1736 | 0 | elog(ERROR, "incompatible clause"); |
1737 | | |
1738 | | /* |
1739 | | * Deconstruct the array constant, unless it's NULL (we'll cover |
1740 | | * that case below) |
1741 | | */ |
1742 | 0 | if (!cst->constisnull) |
1743 | 0 | { |
1744 | 0 | arrayval = DatumGetArrayTypeP(cst->constvalue); |
1745 | 0 | get_typlenbyvalalign(ARR_ELEMTYPE(arrayval), |
1746 | 0 | &elmlen, &elmbyval, &elmalign); |
1747 | 0 | deconstruct_array(arrayval, |
1748 | 0 | ARR_ELEMTYPE(arrayval), |
1749 | 0 | elmlen, elmbyval, elmalign, |
1750 | 0 | &elem_values, &elem_nulls, &num_elems); |
1751 | 0 | } |
1752 | | |
1753 | | /* match the attribute/expression to a dimension of the statistic */ |
1754 | 0 | idx = mcv_match_expression(clause_expr, keys, exprs, &collid); |
1755 | | |
1756 | | /* |
1757 | | * Walk through the MCV items and evaluate the current clause. We |
1758 | | * can skip items that were already ruled out, and terminate if |
1759 | | * there are no remaining MCV items that might possibly match. |
1760 | | */ |
1761 | 0 | for (uint32 i = 0; i < mcvlist->nitems; i++) |
1762 | 0 | { |
1763 | 0 | int j; |
1764 | 0 | bool match = !expr->useOr; |
1765 | 0 | MCVItem *item = &mcvlist->items[i]; |
1766 | | |
1767 | | /* |
1768 | | * When the MCV item or the Const value is NULL we can treat |
1769 | | * this as a mismatch. We must not call the operator because |
1770 | | * of strictness. |
1771 | | */ |
1772 | 0 | if (item->isnull[idx] || cst->constisnull) |
1773 | 0 | { |
1774 | 0 | matches[i] = RESULT_MERGE(matches[i], is_or, false); |
1775 | 0 | continue; |
1776 | 0 | } |
1777 | | |
1778 | | /* |
1779 | | * Skip MCV items that can't change result in the bitmap. Once |
1780 | | * the value gets false for AND-lists, or true for OR-lists, |
1781 | | * we don't need to look at more clauses. |
1782 | | */ |
1783 | 0 | if (RESULT_IS_FINAL(matches[i], is_or)) |
1784 | 0 | continue; |
1785 | | |
1786 | 0 | for (j = 0; j < num_elems; j++) |
1787 | 0 | { |
1788 | 0 | Datum elem_value = elem_values[j]; |
1789 | 0 | bool elem_isnull = elem_nulls[j]; |
1790 | 0 | bool elem_match; |
1791 | | |
1792 | | /* NULL values always evaluate as not matching. */ |
1793 | 0 | if (elem_isnull) |
1794 | 0 | { |
1795 | 0 | match = RESULT_MERGE(match, expr->useOr, false); |
1796 | 0 | continue; |
1797 | 0 | } |
1798 | | |
1799 | | /* |
1800 | | * Stop evaluating the array elements once we reach a |
1801 | | * matching value that can't change - ALL() is the same as |
1802 | | * AND-list, ANY() is the same as OR-list. |
1803 | | */ |
1804 | 0 | if (RESULT_IS_FINAL(match, expr->useOr)) |
1805 | 0 | break; |
1806 | | |
1807 | 0 | elem_match = DatumGetBool(FunctionCall2Coll(&opproc, |
1808 | 0 | collid, |
1809 | 0 | item->values[idx], |
1810 | 0 | elem_value)); |
1811 | |
|
1812 | 0 | match = RESULT_MERGE(match, expr->useOr, elem_match); |
1813 | 0 | } |
1814 | | |
1815 | | /* update the match bitmap with the result */ |
1816 | 0 | matches[i] = RESULT_MERGE(matches[i], is_or, match); |
1817 | 0 | } |
1818 | 0 | } |
1819 | 0 | else if (IsA(clause, NullTest)) |
1820 | 0 | { |
1821 | 0 | NullTest *expr = (NullTest *) clause; |
1822 | 0 | Node *clause_expr = (Node *) (expr->arg); |
1823 | | |
1824 | | /* match the attribute/expression to a dimension of the statistic */ |
1825 | 0 | int idx = mcv_match_expression(clause_expr, keys, exprs, NULL); |
1826 | | |
1827 | | /* |
1828 | | * Walk through the MCV items and evaluate the current clause. We |
1829 | | * can skip items that were already ruled out, and terminate if |
1830 | | * there are no remaining MCV items that might possibly match. |
1831 | | */ |
1832 | 0 | for (uint32 i = 0; i < mcvlist->nitems; i++) |
1833 | 0 | { |
1834 | 0 | bool match = false; /* assume mismatch */ |
1835 | 0 | MCVItem *item = &mcvlist->items[i]; |
1836 | | |
1837 | | /* if the clause mismatches the MCV item, update the bitmap */ |
1838 | 0 | switch (expr->nulltesttype) |
1839 | 0 | { |
1840 | 0 | case IS_NULL: |
1841 | 0 | match = (item->isnull[idx]) ? true : match; |
1842 | 0 | break; |
1843 | | |
1844 | 0 | case IS_NOT_NULL: |
1845 | 0 | match = (!item->isnull[idx]) ? true : match; |
1846 | 0 | break; |
1847 | 0 | } |
1848 | | |
1849 | | /* now, update the match bitmap, depending on OR/AND type */ |
1850 | 0 | matches[i] = RESULT_MERGE(matches[i], is_or, match); |
1851 | 0 | } |
1852 | 0 | } |
1853 | 0 | else if (is_orclause(clause) || is_andclause(clause)) |
1854 | 0 | { |
1855 | | /* AND/OR clause, with all subclauses being compatible */ |
1856 | |
|
1857 | 0 | BoolExpr *bool_clause = ((BoolExpr *) clause); |
1858 | 0 | List *bool_clauses = bool_clause->args; |
1859 | | |
1860 | | /* match/mismatch bitmap for each MCV item */ |
1861 | 0 | bool *bool_matches = NULL; |
1862 | |
|
1863 | 0 | Assert(bool_clauses != NIL); |
1864 | 0 | Assert(list_length(bool_clauses) >= 2); |
1865 | | |
1866 | | /* build the match bitmap for the OR-clauses */ |
1867 | 0 | bool_matches = mcv_get_match_bitmap(root, bool_clauses, keys, exprs, |
1868 | 0 | mcvlist, is_orclause(clause)); |
1869 | | |
1870 | | /* |
1871 | | * Merge the bitmap produced by mcv_get_match_bitmap into the |
1872 | | * current one. We need to consider if we're evaluating AND or OR |
1873 | | * condition when merging the results. |
1874 | | */ |
1875 | 0 | for (uint32 i = 0; i < mcvlist->nitems; i++) |
1876 | 0 | matches[i] = RESULT_MERGE(matches[i], is_or, bool_matches[i]); |
1877 | |
|
1878 | 0 | pfree(bool_matches); |
1879 | 0 | } |
1880 | 0 | else if (is_notclause(clause)) |
1881 | 0 | { |
1882 | | /* NOT clause, with all subclauses compatible */ |
1883 | |
|
1884 | 0 | BoolExpr *not_clause = ((BoolExpr *) clause); |
1885 | 0 | List *not_args = not_clause->args; |
1886 | | |
1887 | | /* match/mismatch bitmap for each MCV item */ |
1888 | 0 | bool *not_matches = NULL; |
1889 | |
|
1890 | 0 | Assert(not_args != NIL); |
1891 | 0 | Assert(list_length(not_args) == 1); |
1892 | | |
1893 | | /* build the match bitmap for the NOT-clause */ |
1894 | 0 | not_matches = mcv_get_match_bitmap(root, not_args, keys, exprs, |
1895 | 0 | mcvlist, false); |
1896 | | |
1897 | | /* |
1898 | | * Merge the bitmap produced by mcv_get_match_bitmap into the |
1899 | | * current one. We're handling a NOT clause, so invert the result |
1900 | | * before merging it into the global bitmap. |
1901 | | */ |
1902 | 0 | for (uint32 i = 0; i < mcvlist->nitems; i++) |
1903 | 0 | matches[i] = RESULT_MERGE(matches[i], is_or, !not_matches[i]); |
1904 | |
|
1905 | 0 | pfree(not_matches); |
1906 | 0 | } |
1907 | 0 | else if (IsA(clause, Var)) |
1908 | 0 | { |
1909 | | /* Var (has to be a boolean Var, possibly from below NOT) */ |
1910 | |
|
1911 | 0 | Var *var = (Var *) (clause); |
1912 | | |
1913 | | /* match the attribute to a dimension of the statistic */ |
1914 | 0 | int idx = bms_member_index(keys, var->varattno); |
1915 | |
|
1916 | 0 | Assert(var->vartype == BOOLOID); |
1917 | | |
1918 | | /* |
1919 | | * Walk through the MCV items and evaluate the current clause. We |
1920 | | * can skip items that were already ruled out, and terminate if |
1921 | | * there are no remaining MCV items that might possibly match. |
1922 | | */ |
1923 | 0 | for (uint32 i = 0; i < mcvlist->nitems; i++) |
1924 | 0 | { |
1925 | 0 | MCVItem *item = &mcvlist->items[i]; |
1926 | 0 | bool match = false; |
1927 | | |
1928 | | /* if the item is NULL, it's a mismatch */ |
1929 | 0 | if (!item->isnull[idx] && DatumGetBool(item->values[idx])) |
1930 | 0 | match = true; |
1931 | | |
1932 | | /* update the result bitmap */ |
1933 | 0 | matches[i] = RESULT_MERGE(matches[i], is_or, match); |
1934 | 0 | } |
1935 | 0 | } |
1936 | 0 | else |
1937 | 0 | { |
1938 | | /* Otherwise, it must be a bare boolean-returning expression */ |
1939 | 0 | int idx; |
1940 | | |
1941 | | /* match the expression to a dimension of the statistic */ |
1942 | 0 | idx = mcv_match_expression(clause, keys, exprs, NULL); |
1943 | | |
1944 | | /* |
1945 | | * Walk through the MCV items and evaluate the current clause. We |
1946 | | * can skip items that were already ruled out, and terminate if |
1947 | | * there are no remaining MCV items that might possibly match. |
1948 | | */ |
1949 | 0 | for (uint32 i = 0; i < mcvlist->nitems; i++) |
1950 | 0 | { |
1951 | 0 | bool match; |
1952 | 0 | MCVItem *item = &mcvlist->items[i]; |
1953 | | |
1954 | | /* "match" just means it's bool TRUE */ |
1955 | 0 | match = !item->isnull[idx] && DatumGetBool(item->values[idx]); |
1956 | | |
1957 | | /* now, update the match bitmap, depending on OR/AND type */ |
1958 | 0 | matches[i] = RESULT_MERGE(matches[i], is_or, match); |
1959 | 0 | } |
1960 | 0 | } |
1961 | 0 | } |
1962 | | |
1963 | 0 | return matches; |
1964 | 0 | } |
1965 | | |
1966 | | |
1967 | | /* |
1968 | | * mcv_combine_selectivities |
1969 | | * Combine per-column and multi-column MCV selectivity estimates. |
1970 | | * |
1971 | | * simple_sel is a "simple" selectivity estimate (produced without using any |
1972 | | * extended statistics, essentially assuming independence of columns/clauses). |
1973 | | * |
1974 | | * mcv_sel and mcv_basesel are sums of the frequencies and base frequencies of |
1975 | | * all matching MCV items. The difference (mcv_sel - mcv_basesel) is then |
1976 | | * essentially interpreted as a correction to be added to simple_sel, as |
1977 | | * described below. |
1978 | | * |
1979 | | * mcv_totalsel is the sum of the frequencies of all MCV items (not just the |
1980 | | * matching ones). This is used as an upper bound on the portion of the |
1981 | | * selectivity estimates not covered by the MCV statistics. |
1982 | | * |
1983 | | * Note: While simple and base selectivities are defined in a quite similar |
1984 | | * way, the values are computed differently and are not therefore equal. The |
1985 | | * simple selectivity is computed as a product of per-clause estimates, while |
1986 | | * the base selectivity is computed by adding up base frequencies of matching |
1987 | | * items of the multi-column MCV list. So the values may differ for two main |
1988 | | * reasons - (a) the MCV list may not cover 100% of the data and (b) some of |
1989 | | * the MCV items did not match the estimated clauses. |
1990 | | * |
1991 | | * As both (a) and (b) reduce the base selectivity value, it generally holds |
1992 | | * that (simple_sel >= mcv_basesel). If the MCV list covers all the data, the |
1993 | | * values may be equal. |
1994 | | * |
1995 | | * So, other_sel = (simple_sel - mcv_basesel) is an estimate for the part not |
1996 | | * covered by the MCV list, and (mcv_sel - mcv_basesel) may be seen as a |
1997 | | * correction for the part covered by the MCV list. Those two statements are |
1998 | | * actually equivalent. |
1999 | | */ |
2000 | | Selectivity |
2001 | | mcv_combine_selectivities(Selectivity simple_sel, |
2002 | | Selectivity mcv_sel, |
2003 | | Selectivity mcv_basesel, |
2004 | | Selectivity mcv_totalsel) |
2005 | 0 | { |
2006 | 0 | Selectivity other_sel; |
2007 | 0 | Selectivity sel; |
2008 | | |
2009 | | /* estimated selectivity of values not covered by MCV matches */ |
2010 | 0 | other_sel = simple_sel - mcv_basesel; |
2011 | 0 | CLAMP_PROBABILITY(other_sel); |
2012 | | |
2013 | | /* this non-MCV selectivity cannot exceed 1 - mcv_totalsel */ |
2014 | 0 | if (other_sel > 1.0 - mcv_totalsel) |
2015 | 0 | other_sel = 1.0 - mcv_totalsel; |
2016 | | |
2017 | | /* overall selectivity is the sum of the MCV and non-MCV parts */ |
2018 | 0 | sel = mcv_sel + other_sel; |
2019 | 0 | CLAMP_PROBABILITY(sel); |
2020 | |
|
2021 | 0 | return sel; |
2022 | 0 | } |
2023 | | |
2024 | | |
2025 | | /* |
2026 | | * mcv_clauselist_selectivity |
2027 | | * Use MCV statistics to estimate the selectivity of an implicitly-ANDed |
2028 | | * list of clauses. |
2029 | | * |
2030 | | * This determines which MCV items match every clause in the list and returns |
2031 | | * the sum of the frequencies of those items. |
2032 | | * |
2033 | | * In addition, it returns the sum of the base frequencies of each of those |
2034 | | * items (that is the sum of the selectivities that each item would have if |
2035 | | * the columns were independent of one another), and the total selectivity of |
2036 | | * all the MCV items (not just the matching ones). These are expected to be |
2037 | | * used together with a "simple" selectivity estimate (one based only on |
2038 | | * per-column statistics) to produce an overall selectivity estimate that |
2039 | | * makes use of both per-column and multi-column statistics --- see |
2040 | | * mcv_combine_selectivities(). |
2041 | | */ |
2042 | | Selectivity |
2043 | | mcv_clauselist_selectivity(PlannerInfo *root, StatisticExtInfo *stat, |
2044 | | List *clauses, int varRelid, |
2045 | | JoinType jointype, SpecialJoinInfo *sjinfo, |
2046 | | RelOptInfo *rel, |
2047 | | Selectivity *basesel, Selectivity *totalsel) |
2048 | 0 | { |
2049 | 0 | MCVList *mcv; |
2050 | 0 | Selectivity s = 0.0; |
2051 | 0 | RangeTblEntry *rte = root->simple_rte_array[rel->relid]; |
2052 | | |
2053 | | /* match/mismatch bitmap for each MCV item */ |
2054 | 0 | bool *matches = NULL; |
2055 | | |
2056 | | /* load the MCV list stored in the statistics object */ |
2057 | 0 | mcv = statext_mcv_load(stat->statOid, rte->inh); |
2058 | | |
2059 | | /* build a match bitmap for the clauses */ |
2060 | 0 | matches = mcv_get_match_bitmap(root, clauses, stat->keys, stat->exprs, |
2061 | 0 | mcv, false); |
2062 | | |
2063 | | /* sum frequencies for all the matching MCV items */ |
2064 | 0 | *basesel = 0.0; |
2065 | 0 | *totalsel = 0.0; |
2066 | 0 | for (uint32 i = 0; i < mcv->nitems; i++) |
2067 | 0 | { |
2068 | 0 | *totalsel += mcv->items[i].frequency; |
2069 | |
|
2070 | 0 | if (matches[i] != false) |
2071 | 0 | { |
2072 | 0 | *basesel += mcv->items[i].base_frequency; |
2073 | 0 | s += mcv->items[i].frequency; |
2074 | 0 | } |
2075 | 0 | } |
2076 | |
|
2077 | 0 | return s; |
2078 | 0 | } |
2079 | | |
2080 | | |
2081 | | /* |
2082 | | * mcv_clause_selectivity_or |
2083 | | * Use MCV statistics to estimate the selectivity of a clause that |
2084 | | * appears in an ORed list of clauses. |
2085 | | * |
2086 | | * As with mcv_clauselist_selectivity() this determines which MCV items match |
2087 | | * the clause and returns both the sum of the frequencies and the sum of the |
2088 | | * base frequencies of those items, as well as the sum of the frequencies of |
2089 | | * all MCV items (not just the matching ones) so that this information can be |
2090 | | * used by mcv_combine_selectivities() to produce a selectivity estimate that |
2091 | | * makes use of both per-column and multi-column statistics. |
2092 | | * |
2093 | | * Additionally, we return information to help compute the overall selectivity |
2094 | | * of the ORed list of clauses assumed to contain this clause. This function |
2095 | | * is intended to be called for each clause in the ORed list of clauses, |
2096 | | * allowing the overall selectivity to be computed using the following |
2097 | | * algorithm: |
2098 | | * |
2099 | | * Suppose P[n] = P(C[1] OR C[2] OR ... OR C[n]) is the combined selectivity |
2100 | | * of the first n clauses in the list. Then the combined selectivity taking |
2101 | | * into account the next clause C[n+1] can be written as |
2102 | | * |
2103 | | * P[n+1] = P[n] + P(C[n+1]) - P((C[1] OR ... OR C[n]) AND C[n+1]) |
2104 | | * |
2105 | | * The final term above represents the overlap between the clauses examined so |
2106 | | * far and the (n+1)'th clause. To estimate its selectivity, we track the |
2107 | | * match bitmap for the ORed list of clauses examined so far and examine its |
2108 | | * intersection with the match bitmap for the (n+1)'th clause. |
2109 | | * |
2110 | | * We then also return the sums of the MCV item frequencies and base |
2111 | | * frequencies for the match bitmap intersection corresponding to the overlap |
2112 | | * term above, so that they can be combined with a simple selectivity estimate |
2113 | | * for that term. |
2114 | | * |
2115 | | * The parameter "or_matches" is an in/out parameter tracking the match bitmap |
2116 | | * for the clauses examined so far. The caller is expected to set it to NULL |
2117 | | * the first time it calls this function. |
2118 | | */ |
2119 | | Selectivity |
2120 | | mcv_clause_selectivity_or(PlannerInfo *root, StatisticExtInfo *stat, |
2121 | | MCVList *mcv, Node *clause, bool **or_matches, |
2122 | | Selectivity *basesel, Selectivity *overlap_mcvsel, |
2123 | | Selectivity *overlap_basesel, Selectivity *totalsel) |
2124 | 0 | { |
2125 | 0 | Selectivity s = 0.0; |
2126 | 0 | bool *new_matches; |
2127 | | |
2128 | | /* build the OR-matches bitmap, if not built already */ |
2129 | 0 | if (*or_matches == NULL) |
2130 | 0 | *or_matches = palloc0_array(bool, mcv->nitems); |
2131 | | |
2132 | | /* build the match bitmap for the new clause */ |
2133 | 0 | new_matches = mcv_get_match_bitmap(root, list_make1(clause), stat->keys, |
2134 | 0 | stat->exprs, mcv, false); |
2135 | | |
2136 | | /* |
2137 | | * Sum the frequencies for all the MCV items matching this clause and also |
2138 | | * those matching the overlap between this clause and any of the preceding |
2139 | | * clauses as described above. |
2140 | | */ |
2141 | 0 | *basesel = 0.0; |
2142 | 0 | *overlap_mcvsel = 0.0; |
2143 | 0 | *overlap_basesel = 0.0; |
2144 | 0 | *totalsel = 0.0; |
2145 | 0 | for (uint32 i = 0; i < mcv->nitems; i++) |
2146 | 0 | { |
2147 | 0 | *totalsel += mcv->items[i].frequency; |
2148 | |
|
2149 | 0 | if (new_matches[i]) |
2150 | 0 | { |
2151 | 0 | s += mcv->items[i].frequency; |
2152 | 0 | *basesel += mcv->items[i].base_frequency; |
2153 | |
|
2154 | 0 | if ((*or_matches)[i]) |
2155 | 0 | { |
2156 | 0 | *overlap_mcvsel += mcv->items[i].frequency; |
2157 | 0 | *overlap_basesel += mcv->items[i].base_frequency; |
2158 | 0 | } |
2159 | 0 | } |
2160 | | |
2161 | | /* update the OR-matches bitmap for the next clause */ |
2162 | 0 | (*or_matches)[i] = (*or_matches)[i] || new_matches[i]; |
2163 | 0 | } |
2164 | |
|
2165 | 0 | pfree(new_matches); |
2166 | |
|
2167 | 0 | return s; |
2168 | 0 | } |
2169 | | |
2170 | | /* |
2171 | | * Free allocations of a MCVList. |
2172 | | */ |
2173 | | void |
2174 | | statext_mcv_free(MCVList *mcvlist) |
2175 | 0 | { |
2176 | 0 | for (uint32 i = 0; i < mcvlist->nitems; i++) |
2177 | 0 | { |
2178 | 0 | MCVItem *item = &mcvlist->items[i]; |
2179 | |
|
2180 | 0 | pfree(item->values); |
2181 | 0 | pfree(item->isnull); |
2182 | 0 | } |
2183 | 0 | pfree(mcvlist); |
2184 | 0 | } |
2185 | | |
2186 | | /* |
2187 | | * Create the MCV composite datum, which is a serialization of an array of |
2188 | | * MCVItems. |
2189 | | * |
2190 | | * The inputs consist of four separate arrays of equal length "numitems" |
2191 | | * (mcv_elems, mcv_nulls, freqs and base_freqs) that form the basics of |
2192 | | * what is stored in the catalogs. These form an array of composite |
2193 | | * records defined by the three atttypX arrays of equal length "numattrs". |
2194 | | * |
2195 | | * If any data element fails to convert to the input type specified for that |
2196 | | * attribute, then function will return a NULL Datum if elevel < ERROR. |
2197 | | */ |
2198 | | Datum |
2199 | | statext_mcv_import(int elevel, int numattrs, |
2200 | | Oid *atttypids, int32 *atttypmods, Oid *atttypcolls, |
2201 | | int nitems, Datum *mcv_elems, bool *mcv_nulls, |
2202 | | float8 *freqs, float8 *base_freqs) |
2203 | 0 | { |
2204 | 0 | MCVList *mcvlist; |
2205 | 0 | bytea *bytes; |
2206 | 0 | VacAttrStats **vastats; |
2207 | | |
2208 | | /* |
2209 | | * Allocate the MCV list structure, set the global parameters. |
2210 | | */ |
2211 | 0 | mcvlist = (MCVList *) palloc0(offsetof(MCVList, items) + |
2212 | 0 | (sizeof(MCVItem) * nitems)); |
2213 | |
|
2214 | 0 | mcvlist->magic = STATS_MCV_MAGIC; |
2215 | 0 | mcvlist->type = STATS_MCV_TYPE_BASIC; |
2216 | 0 | mcvlist->ndimensions = numattrs; |
2217 | 0 | mcvlist->nitems = nitems; |
2218 | | |
2219 | | /* Set the values for the 1-D arrays and allocate space for the 2-D arrays */ |
2220 | 0 | for (int i = 0; i < nitems; i++) |
2221 | 0 | { |
2222 | 0 | MCVItem *item = &mcvlist->items[i]; |
2223 | |
|
2224 | 0 | item->frequency = freqs[i]; |
2225 | 0 | item->base_frequency = base_freqs[i]; |
2226 | 0 | item->values = (Datum *) palloc0_array(Datum, numattrs); |
2227 | 0 | item->isnull = (bool *) palloc0_array(bool, numattrs); |
2228 | 0 | } |
2229 | | |
2230 | | /* |
2231 | | * Walk through each dimension, determine the input function for that |
2232 | | * type, and then attempt to convert all values in that column via that |
2233 | | * function. We approach this column-wise because it is simpler to deal |
2234 | | * with one input function at time, and possibly more cache-friendly. |
2235 | | */ |
2236 | 0 | for (int j = 0; j < numattrs; j++) |
2237 | 0 | { |
2238 | 0 | FmgrInfo finfo; |
2239 | 0 | Oid ioparam; |
2240 | 0 | Oid infunc; |
2241 | 0 | int index = j; |
2242 | |
|
2243 | 0 | getTypeInputInfo(atttypids[j], &infunc, &ioparam); |
2244 | 0 | fmgr_info(infunc, &finfo); |
2245 | | |
2246 | | /* store info about data type OIDs */ |
2247 | 0 | mcvlist->types[j] = atttypids[j]; |
2248 | |
|
2249 | 0 | for (int i = 0; i < nitems; i++) |
2250 | 0 | { |
2251 | 0 | MCVItem *item = &mcvlist->items[i]; |
2252 | |
|
2253 | 0 | if (mcv_nulls[index]) |
2254 | 0 | { |
2255 | | /* NULL value detected, hence no input to process */ |
2256 | 0 | item->values[j] = (Datum) 0; |
2257 | 0 | item->isnull[j] = true; |
2258 | 0 | } |
2259 | 0 | else |
2260 | 0 | { |
2261 | 0 | char *s = TextDatumGetCString(mcv_elems[index]); |
2262 | 0 | ErrorSaveContext escontext = {T_ErrorSaveContext}; |
2263 | |
|
2264 | 0 | if (!InputFunctionCallSafe(&finfo, s, ioparam, atttypmods[j], |
2265 | 0 | (Node *) &escontext, &item->values[j])) |
2266 | 0 | { |
2267 | 0 | ereport(elevel, |
2268 | 0 | (errcode(ERRCODE_INVALID_PARAMETER_VALUE), |
2269 | 0 | errmsg("could not parse MCV element \"%s\": incorrect value", s))); |
2270 | 0 | pfree(s); |
2271 | 0 | goto error; |
2272 | 0 | } |
2273 | | |
2274 | 0 | pfree(s); |
2275 | 0 | } |
2276 | | |
2277 | 0 | index += numattrs; |
2278 | 0 | } |
2279 | 0 | } |
2280 | | |
2281 | | /* |
2282 | | * The function statext_mcv_serialize() requires an array of pointers to |
2283 | | * VacAttrStats records, but only a few fields within those records have |
2284 | | * to be filled out. |
2285 | | */ |
2286 | 0 | vastats = (VacAttrStats **) palloc0_array(VacAttrStats *, numattrs); |
2287 | |
|
2288 | 0 | for (int i = 0; i < numattrs; i++) |
2289 | 0 | { |
2290 | 0 | Oid typid = atttypids[i]; |
2291 | 0 | HeapTuple typtuple; |
2292 | |
|
2293 | 0 | typtuple = SearchSysCacheCopy1(TYPEOID, ObjectIdGetDatum(typid)); |
2294 | |
|
2295 | 0 | if (!HeapTupleIsValid(typtuple)) |
2296 | 0 | elog(ERROR, "cache lookup failed for type %u", typid); |
2297 | | |
2298 | 0 | vastats[i] = palloc0_object(VacAttrStats); |
2299 | |
|
2300 | 0 | vastats[i]->attrtype = (Form_pg_type) GETSTRUCT(typtuple); |
2301 | 0 | vastats[i]->attrtypid = typid; |
2302 | 0 | vastats[i]->attrcollid = atttypcolls[i]; |
2303 | 0 | } |
2304 | | |
2305 | 0 | bytes = statext_mcv_serialize(mcvlist, vastats); |
2306 | |
|
2307 | 0 | for (int i = 0; i < numattrs; i++) |
2308 | 0 | { |
2309 | 0 | pfree(vastats[i]); |
2310 | 0 | } |
2311 | 0 | pfree((void *) vastats); |
2312 | |
|
2313 | 0 | pfree(mcv_elems); |
2314 | 0 | pfree(mcv_nulls); |
2315 | |
|
2316 | 0 | if (bytes == NULL) |
2317 | 0 | { |
2318 | 0 | ereport(elevel, |
2319 | 0 | (errcode(ERRCODE_INVALID_PARAMETER_VALUE), |
2320 | 0 | errmsg("could not import MCV list"))); |
2321 | 0 | goto error; |
2322 | 0 | } |
2323 | | |
2324 | 0 | return PointerGetDatum(bytes); |
2325 | | |
2326 | 0 | error: |
2327 | 0 | statext_mcv_free(mcvlist); |
2328 | 0 | return (Datum) 0; |
2329 | 0 | } |