/src/postgres/src/common/compression.c
Line | Count | Source |
1 | | /*------------------------------------------------------------------------- |
2 | | * |
3 | | * compression.c |
4 | | * |
5 | | * Shared code for compression methods and specifications. |
6 | | * |
7 | | * A compression specification specifies the parameters that should be used |
8 | | * when performing compression with a specific algorithm. The simplest |
9 | | * possible compression specification is an integer, which sets the |
10 | | * compression level. |
11 | | * |
12 | | * Otherwise, a compression specification is a comma-separated list of items, |
13 | | * each having the form keyword or keyword=value. |
14 | | * |
15 | | * Currently, the supported keywords are "level", "long", and "workers". |
16 | | * |
17 | | * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group |
18 | | * |
19 | | * IDENTIFICATION |
20 | | * src/common/compression.c |
21 | | *------------------------------------------------------------------------- |
22 | | */ |
23 | | |
24 | | #ifndef FRONTEND |
25 | | #include "postgres.h" |
26 | | #else |
27 | | #include "postgres_fe.h" |
28 | | #endif |
29 | | |
30 | | #ifdef USE_ZSTD |
31 | | #include <zstd.h> |
32 | | #endif |
33 | | #ifdef HAVE_LIBZ |
34 | | #include <zlib.h> |
35 | | #endif |
36 | | |
37 | | #include "common/compression.h" |
38 | | |
39 | | static int expect_integer_value(char *keyword, char *value, |
40 | | pg_compress_specification *result); |
41 | | static bool expect_boolean_value(char *keyword, char *value, |
42 | | pg_compress_specification *result); |
43 | | |
44 | | /* |
45 | | * Look up a compression algorithm by archive file extension. Sets *algorithm |
46 | | * and returns the length of the non-extension portion of the filename, or -1 |
47 | | * if the filename does not end with a recognized tar extension. |
48 | | */ |
49 | | int |
50 | | parse_tar_compress_algorithm(const char *fname, pg_compress_algorithm *algorithm) |
51 | 0 | { |
52 | 0 | int fname_len = strlen(fname); |
53 | |
|
54 | 0 | if (fname_len >= 4 && |
55 | 0 | strcmp(fname + fname_len - 4, ".tar") == 0) |
56 | 0 | { |
57 | 0 | *algorithm = PG_COMPRESSION_NONE; |
58 | 0 | return fname_len - 4; |
59 | 0 | } |
60 | 0 | else if (fname_len >= 4 && |
61 | 0 | strcmp(fname + fname_len - 4, ".tgz") == 0) |
62 | 0 | { |
63 | 0 | *algorithm = PG_COMPRESSION_GZIP; |
64 | 0 | return fname_len - 4; |
65 | 0 | } |
66 | 0 | else if (fname_len >= 7 && |
67 | 0 | strcmp(fname + fname_len - 7, ".tar.gz") == 0) |
68 | 0 | { |
69 | 0 | *algorithm = PG_COMPRESSION_GZIP; |
70 | 0 | return fname_len - 7; |
71 | 0 | } |
72 | 0 | else if (fname_len >= 8 && |
73 | 0 | strcmp(fname + fname_len - 8, ".tar.lz4") == 0) |
74 | 0 | { |
75 | 0 | *algorithm = PG_COMPRESSION_LZ4; |
76 | 0 | return fname_len - 8; |
77 | 0 | } |
78 | 0 | else if (fname_len >= 8 && |
79 | 0 | strcmp(fname + fname_len - 8, ".tar.zst") == 0) |
80 | 0 | { |
81 | 0 | *algorithm = PG_COMPRESSION_ZSTD; |
82 | 0 | return fname_len - 8; |
83 | 0 | } |
84 | | |
85 | 0 | return -1; |
86 | 0 | } |
87 | | |
88 | | /* |
89 | | * Look up a compression algorithm by name. Returns true and sets *algorithm |
90 | | * if the name is recognized. Otherwise returns false. |
91 | | */ |
92 | | bool |
93 | | parse_compress_algorithm(char *name, pg_compress_algorithm *algorithm) |
94 | 0 | { |
95 | 0 | if (strcmp(name, "none") == 0) |
96 | 0 | *algorithm = PG_COMPRESSION_NONE; |
97 | 0 | else if (strcmp(name, "gzip") == 0) |
98 | 0 | *algorithm = PG_COMPRESSION_GZIP; |
99 | 0 | else if (strcmp(name, "lz4") == 0) |
100 | 0 | *algorithm = PG_COMPRESSION_LZ4; |
101 | 0 | else if (strcmp(name, "zstd") == 0) |
102 | 0 | *algorithm = PG_COMPRESSION_ZSTD; |
103 | 0 | else |
104 | 0 | return false; |
105 | 0 | return true; |
106 | 0 | } |
107 | | |
108 | | /* |
109 | | * Get the human-readable name corresponding to a particular compression |
110 | | * algorithm. |
111 | | */ |
112 | | const char * |
113 | | get_compress_algorithm_name(pg_compress_algorithm algorithm) |
114 | 0 | { |
115 | 0 | switch (algorithm) |
116 | 0 | { |
117 | 0 | case PG_COMPRESSION_NONE: |
118 | 0 | return "none"; |
119 | 0 | case PG_COMPRESSION_GZIP: |
120 | 0 | return "gzip"; |
121 | 0 | case PG_COMPRESSION_LZ4: |
122 | 0 | return "lz4"; |
123 | 0 | case PG_COMPRESSION_ZSTD: |
124 | 0 | return "zstd"; |
125 | | /* no default, to provoke compiler warnings if values are added */ |
126 | 0 | } |
127 | 0 | Assert(false); |
128 | 0 | return "???"; /* placate compiler */ |
129 | 0 | } |
130 | | |
131 | | /* |
132 | | * Parse a compression specification for a specified algorithm. |
133 | | * |
134 | | * See the file header comments for a brief description of what a compression |
135 | | * specification is expected to look like. |
136 | | * |
137 | | * On return, all fields of the result object will be initialized. |
138 | | * In particular, result->parse_error will be NULL if no errors occurred |
139 | | * during parsing, and will otherwise contain an appropriate error message. |
140 | | * The caller may free this error message string using pfree, if desired. |
141 | | * Note, however, even if there's no parse error, the string might not make |
142 | | * sense: e.g. for gzip, level=12 is not sensible, but it does parse OK. |
143 | | * |
144 | | * The compression level is assigned by default if not directly specified |
145 | | * by the specification. |
146 | | * |
147 | | * Use validate_compress_specification() to find out whether a compression |
148 | | * specification is semantically sensible. |
149 | | */ |
150 | | void |
151 | | parse_compress_specification(pg_compress_algorithm algorithm, char *specification, |
152 | | pg_compress_specification *result) |
153 | 0 | { |
154 | 0 | int bare_level; |
155 | 0 | char *bare_level_endp; |
156 | | |
157 | | /* Initial setup of result object. */ |
158 | 0 | result->algorithm = algorithm; |
159 | 0 | result->options = 0; |
160 | 0 | result->parse_error = NULL; |
161 | | |
162 | | /* |
163 | | * Assign a default level depending on the compression method. This may |
164 | | * be enforced later. |
165 | | */ |
166 | 0 | switch (result->algorithm) |
167 | 0 | { |
168 | 0 | case PG_COMPRESSION_NONE: |
169 | 0 | result->level = 0; |
170 | 0 | break; |
171 | 0 | case PG_COMPRESSION_LZ4: |
172 | | #ifdef USE_LZ4 |
173 | | result->level = 0; /* fast compression mode */ |
174 | | #else |
175 | 0 | result->parse_error = |
176 | 0 | psprintf(_("this build does not support compression with %s"), |
177 | 0 | "LZ4"); |
178 | 0 | #endif |
179 | 0 | break; |
180 | 0 | case PG_COMPRESSION_ZSTD: |
181 | | #ifdef USE_ZSTD |
182 | | result->level = ZSTD_CLEVEL_DEFAULT; |
183 | | #else |
184 | 0 | result->parse_error = |
185 | 0 | psprintf(_("this build does not support compression with %s"), |
186 | 0 | "ZSTD"); |
187 | 0 | #endif |
188 | 0 | break; |
189 | 0 | case PG_COMPRESSION_GZIP: |
190 | 0 | #ifdef HAVE_LIBZ |
191 | 0 | result->level = Z_DEFAULT_COMPRESSION; |
192 | | #else |
193 | | result->parse_error = |
194 | | psprintf(_("this build does not support compression with %s"), |
195 | | "gzip"); |
196 | | #endif |
197 | 0 | break; |
198 | 0 | } |
199 | | |
200 | | /* If there is no specification, we're done already. */ |
201 | 0 | if (specification == NULL) |
202 | 0 | return; |
203 | | |
204 | | /* As a special case, the specification can be a bare integer. */ |
205 | 0 | bare_level = strtol(specification, &bare_level_endp, 10); |
206 | 0 | if (specification != bare_level_endp && *bare_level_endp == '\0') |
207 | 0 | { |
208 | 0 | result->level = bare_level; |
209 | 0 | return; |
210 | 0 | } |
211 | | |
212 | | /* Look for comma-separated keyword or keyword=value entries. */ |
213 | 0 | while (1) |
214 | 0 | { |
215 | 0 | char *kwstart; |
216 | 0 | char *kwend; |
217 | 0 | char *vstart; |
218 | 0 | char *vend; |
219 | 0 | int kwlen; |
220 | 0 | int vlen; |
221 | 0 | bool has_value; |
222 | 0 | char *keyword; |
223 | 0 | char *value; |
224 | | |
225 | | /* Figure start, end, and length of next keyword and any value. */ |
226 | 0 | kwstart = kwend = specification; |
227 | 0 | while (*kwend != '\0' && *kwend != ',' && *kwend != '=') |
228 | 0 | ++kwend; |
229 | 0 | kwlen = kwend - kwstart; |
230 | 0 | if (*kwend != '=') |
231 | 0 | { |
232 | 0 | vstart = vend = NULL; |
233 | 0 | vlen = 0; |
234 | 0 | has_value = false; |
235 | 0 | } |
236 | 0 | else |
237 | 0 | { |
238 | 0 | vstart = vend = kwend + 1; |
239 | 0 | while (*vend != '\0' && *vend != ',') |
240 | 0 | ++vend; |
241 | 0 | vlen = vend - vstart; |
242 | 0 | has_value = true; |
243 | 0 | } |
244 | | |
245 | | /* Reject empty keyword. */ |
246 | 0 | if (kwlen == 0) |
247 | 0 | { |
248 | 0 | result->parse_error = |
249 | 0 | pstrdup(_("found empty string where a compression option was expected")); |
250 | 0 | break; |
251 | 0 | } |
252 | | |
253 | | /* Extract keyword and value as separate C strings. */ |
254 | 0 | keyword = palloc(kwlen + 1); |
255 | 0 | memcpy(keyword, kwstart, kwlen); |
256 | 0 | keyword[kwlen] = '\0'; |
257 | 0 | if (!has_value) |
258 | 0 | value = NULL; |
259 | 0 | else |
260 | 0 | { |
261 | 0 | value = palloc(vlen + 1); |
262 | 0 | memcpy(value, vstart, vlen); |
263 | 0 | value[vlen] = '\0'; |
264 | 0 | } |
265 | | |
266 | | /* Handle whatever keyword we found. */ |
267 | 0 | if (strcmp(keyword, "level") == 0) |
268 | 0 | { |
269 | 0 | result->level = expect_integer_value(keyword, value, result); |
270 | | |
271 | | /* |
272 | | * No need to set a flag in "options", there is a default level |
273 | | * set at least thanks to the logic above. |
274 | | */ |
275 | 0 | } |
276 | 0 | else if (strcmp(keyword, "workers") == 0) |
277 | 0 | { |
278 | 0 | result->workers = expect_integer_value(keyword, value, result); |
279 | 0 | result->options |= PG_COMPRESSION_OPTION_WORKERS; |
280 | 0 | } |
281 | 0 | else if (strcmp(keyword, "long") == 0) |
282 | 0 | { |
283 | 0 | result->long_distance = expect_boolean_value(keyword, value, result); |
284 | 0 | result->options |= PG_COMPRESSION_OPTION_LONG_DISTANCE; |
285 | 0 | } |
286 | 0 | else |
287 | 0 | result->parse_error = |
288 | 0 | psprintf(_("unrecognized compression option: \"%s\""), keyword); |
289 | | |
290 | | /* Release memory, just to be tidy. */ |
291 | 0 | pfree(keyword); |
292 | 0 | if (value != NULL) |
293 | 0 | pfree(value); |
294 | | |
295 | | /* |
296 | | * If we got an error or have reached the end of the string, stop. |
297 | | * |
298 | | * If there is no value, then the end of the keyword might have been |
299 | | * the end of the string. If there is a value, then the end of the |
300 | | * keyword cannot have been the end of the string, but the end of the |
301 | | * value might have been. |
302 | | */ |
303 | 0 | if (result->parse_error != NULL || |
304 | 0 | (vend == NULL ? *kwend == '\0' : *vend == '\0')) |
305 | 0 | break; |
306 | | |
307 | | /* Advance to next entry and loop around. */ |
308 | 0 | specification = vend == NULL ? kwend + 1 : vend + 1; |
309 | 0 | } |
310 | 0 | } |
311 | | |
312 | | /* |
313 | | * Parse 'value' as an integer and return the result. |
314 | | * |
315 | | * If parsing fails, set result->parse_error to an appropriate message |
316 | | * and return -1. |
317 | | */ |
318 | | static int |
319 | | expect_integer_value(char *keyword, char *value, pg_compress_specification *result) |
320 | 0 | { |
321 | 0 | int ivalue; |
322 | 0 | char *ivalue_endp; |
323 | |
|
324 | 0 | if (value == NULL) |
325 | 0 | { |
326 | 0 | result->parse_error = |
327 | 0 | psprintf(_("compression option \"%s\" requires a value"), |
328 | 0 | keyword); |
329 | 0 | return -1; |
330 | 0 | } |
331 | | |
332 | 0 | ivalue = strtol(value, &ivalue_endp, 10); |
333 | 0 | if (ivalue_endp == value || *ivalue_endp != '\0') |
334 | 0 | { |
335 | 0 | result->parse_error = |
336 | 0 | psprintf(_("value for compression option \"%s\" must be an integer"), |
337 | 0 | keyword); |
338 | 0 | return -1; |
339 | 0 | } |
340 | 0 | return ivalue; |
341 | 0 | } |
342 | | |
343 | | /* |
344 | | * Parse 'value' as a boolean and return the result. |
345 | | * |
346 | | * If parsing fails, set result->parse_error to an appropriate message |
347 | | * and return -1. The caller must check result->parse_error to determine if |
348 | | * the call was successful. |
349 | | * |
350 | | * Valid values are: yes, no, on, off, 1, 0. |
351 | | * |
352 | | * Inspired by ParseVariableBool(). |
353 | | */ |
354 | | static bool |
355 | | expect_boolean_value(char *keyword, char *value, pg_compress_specification *result) |
356 | 0 | { |
357 | 0 | if (value == NULL) |
358 | 0 | return true; |
359 | | |
360 | 0 | if (pg_strcasecmp(value, "yes") == 0) |
361 | 0 | return true; |
362 | 0 | if (pg_strcasecmp(value, "on") == 0) |
363 | 0 | return true; |
364 | 0 | if (pg_strcasecmp(value, "1") == 0) |
365 | 0 | return true; |
366 | | |
367 | 0 | if (pg_strcasecmp(value, "no") == 0) |
368 | 0 | return false; |
369 | 0 | if (pg_strcasecmp(value, "off") == 0) |
370 | 0 | return false; |
371 | 0 | if (pg_strcasecmp(value, "0") == 0) |
372 | 0 | return false; |
373 | | |
374 | 0 | result->parse_error = |
375 | 0 | psprintf(_("value for compression option \"%s\" must be a Boolean value"), |
376 | 0 | keyword); |
377 | 0 | return false; |
378 | 0 | } |
379 | | |
380 | | /* |
381 | | * Returns NULL if the compression specification string was syntactically |
382 | | * valid and semantically sensible. Otherwise, returns an error message. |
383 | | * |
384 | | * Does not test whether this build of PostgreSQL supports the requested |
385 | | * compression method. |
386 | | */ |
387 | | char * |
388 | | validate_compress_specification(pg_compress_specification *spec) |
389 | 0 | { |
390 | 0 | int min_level = 1; |
391 | 0 | int max_level = 1; |
392 | 0 | int default_level = 0; |
393 | | |
394 | | /* If it didn't even parse OK, it's definitely no good. */ |
395 | 0 | if (spec->parse_error != NULL) |
396 | 0 | return spec->parse_error; |
397 | | |
398 | | /* |
399 | | * Check that the algorithm expects a compression level and it is within |
400 | | * the legal range for the algorithm. |
401 | | */ |
402 | 0 | switch (spec->algorithm) |
403 | 0 | { |
404 | 0 | case PG_COMPRESSION_GZIP: |
405 | 0 | max_level = 9; |
406 | 0 | #ifdef HAVE_LIBZ |
407 | 0 | default_level = Z_DEFAULT_COMPRESSION; |
408 | 0 | #endif |
409 | 0 | break; |
410 | 0 | case PG_COMPRESSION_LZ4: |
411 | 0 | max_level = 12; |
412 | 0 | default_level = 0; /* fast mode */ |
413 | 0 | break; |
414 | 0 | case PG_COMPRESSION_ZSTD: |
415 | | #ifdef USE_ZSTD |
416 | | max_level = ZSTD_maxCLevel(); |
417 | | min_level = ZSTD_minCLevel(); |
418 | | default_level = ZSTD_CLEVEL_DEFAULT; |
419 | | #endif |
420 | 0 | break; |
421 | 0 | case PG_COMPRESSION_NONE: |
422 | 0 | if (spec->level != 0) |
423 | 0 | return psprintf(_("compression algorithm \"%s\" does not accept a compression level"), |
424 | 0 | get_compress_algorithm_name(spec->algorithm)); |
425 | 0 | break; |
426 | 0 | } |
427 | | |
428 | 0 | if ((spec->level < min_level || spec->level > max_level) && |
429 | 0 | spec->level != default_level) |
430 | 0 | return psprintf(_("compression algorithm \"%s\" expects a compression level between %d and %d (default at %d)"), |
431 | 0 | get_compress_algorithm_name(spec->algorithm), |
432 | 0 | min_level, max_level, default_level); |
433 | | |
434 | | /* |
435 | | * Of the compression algorithms that we currently support, only zstd |
436 | | * allows parallel workers. |
437 | | */ |
438 | 0 | if ((spec->options & PG_COMPRESSION_OPTION_WORKERS) != 0 && |
439 | 0 | (spec->algorithm != PG_COMPRESSION_ZSTD)) |
440 | 0 | { |
441 | 0 | return psprintf(_("compression algorithm \"%s\" does not accept a worker count"), |
442 | 0 | get_compress_algorithm_name(spec->algorithm)); |
443 | 0 | } |
444 | | |
445 | | /* |
446 | | * Of the compression algorithms that we currently support, only zstd |
447 | | * supports long-distance mode. |
448 | | */ |
449 | 0 | if ((spec->options & PG_COMPRESSION_OPTION_LONG_DISTANCE) != 0 && |
450 | 0 | (spec->algorithm != PG_COMPRESSION_ZSTD)) |
451 | 0 | { |
452 | 0 | return psprintf(_("compression algorithm \"%s\" does not support long-distance mode"), |
453 | 0 | get_compress_algorithm_name(spec->algorithm)); |
454 | 0 | } |
455 | | |
456 | 0 | return NULL; |
457 | 0 | } |
458 | | |
459 | | #ifdef FRONTEND |
460 | | |
461 | | /* |
462 | | * Basic parsing of a value specified through a command-line option, commonly |
463 | | * -Z/--compress. |
464 | | * |
465 | | * The parsing consists of a METHOD:DETAIL string fed later to |
466 | | * parse_compress_specification(). This only extracts METHOD and DETAIL. |
467 | | * If only an integer is found, the method is implied by the value specified. |
468 | | */ |
469 | | void |
470 | | parse_compress_options(const char *option, char **algorithm, char **detail) |
471 | | { |
472 | | const char *sep; |
473 | | char *endp; |
474 | | long result; |
475 | | |
476 | | /* |
477 | | * Check whether the compression specification consists of a bare integer. |
478 | | * |
479 | | * For backward-compatibility, assume "none" if the integer found is zero |
480 | | * and "gzip" otherwise. |
481 | | */ |
482 | | result = strtol(option, &endp, 10); |
483 | | if (*endp == '\0') |
484 | | { |
485 | | if (result == 0) |
486 | | { |
487 | | *algorithm = pstrdup("none"); |
488 | | *detail = NULL; |
489 | | } |
490 | | else |
491 | | { |
492 | | *algorithm = pstrdup("gzip"); |
493 | | *detail = pstrdup(option); |
494 | | } |
495 | | return; |
496 | | } |
497 | | |
498 | | /* |
499 | | * Check whether there is a compression detail following the algorithm |
500 | | * name. |
501 | | */ |
502 | | sep = strchr(option, ':'); |
503 | | if (sep == NULL) |
504 | | { |
505 | | *algorithm = pstrdup(option); |
506 | | *detail = NULL; |
507 | | } |
508 | | else |
509 | | { |
510 | | char *alg; |
511 | | |
512 | | alg = palloc((sep - option) + 1); |
513 | | memcpy(alg, option, sep - option); |
514 | | alg[sep - option] = '\0'; |
515 | | |
516 | | *algorithm = alg; |
517 | | *detail = pstrdup(sep + 1); |
518 | | } |
519 | | } |
520 | | #endif /* FRONTEND */ |