/src/zstd/tests/fuzz/zstd_helpers.c
Line | Count | Source (jump to first uncovered line) |
1 | | /* |
2 | | * Copyright (c) Meta Platforms, Inc. and affiliates. |
3 | | * All rights reserved. |
4 | | * |
5 | | * This source code is licensed under both the BSD-style license (found in the |
6 | | * LICENSE file in the root directory of this source tree) and the GPLv2 (found |
7 | | * in the COPYING file in the root directory of this source tree). |
8 | | * You may select, at your option, one of the above-listed licenses. |
9 | | */ |
10 | | |
11 | | #define ZSTD_STATIC_LINKING_ONLY |
12 | | #define ZDICT_STATIC_LINKING_ONLY |
13 | | |
14 | | #include <string.h> |
15 | | |
16 | | #include "zstd_helpers.h" |
17 | | #include "fuzz_helpers.h" |
18 | | #include "zstd.h" |
19 | | #include "zdict.h" |
20 | | #include "sequence_producer.h" |
21 | | #include "fuzz_third_party_seq_prod.h" |
22 | | |
23 | | const int kMinClevel = -3; |
24 | | const int kMaxClevel = 19; |
25 | | |
26 | | void* FUZZ_seqProdState = NULL; |
27 | | |
28 | | static void set(ZSTD_CCtx *cctx, ZSTD_cParameter param, int value) |
29 | 0 | { |
30 | 0 | FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, param, value)); |
31 | 0 | } |
32 | | |
33 | | static unsigned produceParamValue(unsigned min, unsigned max, |
34 | 0 | FUZZ_dataProducer_t *producer) { |
35 | 0 | return FUZZ_dataProducer_uint32Range(producer, min, max); |
36 | 0 | } |
37 | | |
38 | | static void setRand(ZSTD_CCtx *cctx, ZSTD_cParameter param, unsigned min, |
39 | 0 | unsigned max, FUZZ_dataProducer_t *producer) { |
40 | 0 | unsigned const value = produceParamValue(min, max, producer); |
41 | 0 | set(cctx, param, value); |
42 | 0 | } |
43 | | |
44 | | ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, FUZZ_dataProducer_t *producer) |
45 | 0 | { |
46 | | /* Select compression parameters */ |
47 | 0 | ZSTD_compressionParameters cParams; |
48 | 0 | cParams.windowLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_WINDOWLOG_MIN, 15); |
49 | 0 | cParams.hashLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_HASHLOG_MIN, 15); |
50 | 0 | cParams.chainLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_CHAINLOG_MIN, 16); |
51 | 0 | cParams.searchLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_SEARCHLOG_MIN, 9); |
52 | 0 | cParams.minMatch = FUZZ_dataProducer_uint32Range(producer, ZSTD_MINMATCH_MIN, |
53 | 0 | ZSTD_MINMATCH_MAX); |
54 | 0 | cParams.targetLength = FUZZ_dataProducer_uint32Range(producer, 0, 512); |
55 | 0 | cParams.strategy = FUZZ_dataProducer_uint32Range(producer, ZSTD_STRATEGY_MIN, ZSTD_STRATEGY_MAX); |
56 | 0 | return ZSTD_adjustCParams(cParams, srcSize, 0); |
57 | 0 | } |
58 | | |
59 | | ZSTD_frameParameters FUZZ_randomFParams(FUZZ_dataProducer_t *producer) |
60 | 0 | { |
61 | | /* Select frame parameters */ |
62 | 0 | ZSTD_frameParameters fParams; |
63 | 0 | fParams.contentSizeFlag = FUZZ_dataProducer_uint32Range(producer, 0, 1); |
64 | 0 | fParams.checksumFlag = FUZZ_dataProducer_uint32Range(producer, 0, 1); |
65 | 0 | fParams.noDictIDFlag = FUZZ_dataProducer_uint32Range(producer, 0, 1); |
66 | 0 | return fParams; |
67 | 0 | } |
68 | | |
69 | | ZSTD_parameters FUZZ_randomParams(size_t srcSize, FUZZ_dataProducer_t *producer) |
70 | 0 | { |
71 | 0 | ZSTD_parameters params; |
72 | 0 | params.cParams = FUZZ_randomCParams(srcSize, producer); |
73 | 0 | params.fParams = FUZZ_randomFParams(producer); |
74 | 0 | return params; |
75 | 0 | } |
76 | | |
77 | 0 | static void setSequenceProducerParams(ZSTD_CCtx *cctx, FUZZ_dataProducer_t *producer) { |
78 | | #ifdef FUZZ_THIRD_PARTY_SEQ_PROD |
79 | | ZSTD_registerSequenceProducer( |
80 | | cctx, |
81 | | FUZZ_seqProdState, |
82 | | FUZZ_thirdPartySeqProd |
83 | | ); |
84 | | #else |
85 | 0 | ZSTD_registerSequenceProducer( |
86 | 0 | cctx, |
87 | 0 | NULL, |
88 | 0 | simpleSequenceProducer |
89 | 0 | ); |
90 | 0 | #endif |
91 | |
|
92 | | #ifdef FUZZ_THIRD_PARTY_SEQ_PROD |
93 | | FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableSeqProducerFallback, 1)); |
94 | | #else |
95 | 0 | setRand(cctx, ZSTD_c_enableSeqProducerFallback, 0, 1, producer); |
96 | 0 | #endif |
97 | 0 | FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 0)); |
98 | 0 | FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_disable)); |
99 | 0 | } |
100 | | |
101 | | void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer_t *producer) |
102 | 0 | { |
103 | 0 | ZSTD_compressionParameters cParams = FUZZ_randomCParams(srcSize, producer); |
104 | 0 | set(cctx, ZSTD_c_windowLog, cParams.windowLog); |
105 | 0 | set(cctx, ZSTD_c_hashLog, cParams.hashLog); |
106 | 0 | set(cctx, ZSTD_c_chainLog, cParams.chainLog); |
107 | 0 | set(cctx, ZSTD_c_searchLog, cParams.searchLog); |
108 | 0 | set(cctx, ZSTD_c_minMatch, cParams.minMatch); |
109 | 0 | set(cctx, ZSTD_c_targetLength, cParams.targetLength); |
110 | 0 | set(cctx, ZSTD_c_strategy, cParams.strategy); |
111 | | /* Select frame parameters */ |
112 | 0 | setRand(cctx, ZSTD_c_contentSizeFlag, 0, 1, producer); |
113 | 0 | setRand(cctx, ZSTD_c_checksumFlag, 0, 1, producer); |
114 | 0 | setRand(cctx, ZSTD_c_dictIDFlag, 0, 1, producer); |
115 | | /* Select long distance matching parameters */ |
116 | 0 | setRand(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_auto, ZSTD_ps_disable, producer); |
117 | 0 | setRand(cctx, ZSTD_c_ldmHashLog, ZSTD_HASHLOG_MIN, 16, producer); |
118 | 0 | setRand(cctx, ZSTD_c_ldmMinMatch, ZSTD_LDM_MINMATCH_MIN, |
119 | 0 | ZSTD_LDM_MINMATCH_MAX, producer); |
120 | 0 | setRand(cctx, ZSTD_c_ldmBucketSizeLog, 0, ZSTD_LDM_BUCKETSIZELOG_MAX, |
121 | 0 | producer); |
122 | 0 | setRand(cctx, ZSTD_c_ldmHashRateLog, ZSTD_LDM_HASHRATELOG_MIN, |
123 | 0 | ZSTD_LDM_HASHRATELOG_MAX, producer); |
124 | | /* Set misc parameters */ |
125 | | #ifndef ZSTD_MULTITHREAD |
126 | | // To reproduce with or without ZSTD_MULTITHREAD, we are going to use |
127 | | // the same amount of entropy. |
128 | | unsigned const nbWorkers_value = produceParamValue(0, 2, producer); |
129 | | unsigned const rsyncable_value = produceParamValue(0, 1, producer); |
130 | | (void)nbWorkers_value; |
131 | | (void)rsyncable_value; |
132 | | set(cctx, ZSTD_c_nbWorkers, 0); |
133 | | set(cctx, ZSTD_c_rsyncable, 0); |
134 | | #else |
135 | 0 | setRand(cctx, ZSTD_c_nbWorkers, 0, 2, producer); |
136 | 0 | setRand(cctx, ZSTD_c_rsyncable, 0, 1, producer); |
137 | 0 | #endif |
138 | 0 | setRand(cctx, ZSTD_c_useRowMatchFinder, 0, 2, producer); |
139 | 0 | setRand(cctx, ZSTD_c_enableDedicatedDictSearch, 0, 1, producer); |
140 | 0 | setRand(cctx, ZSTD_c_forceMaxWindow, 0, 1, producer); |
141 | 0 | setRand(cctx, ZSTD_c_literalCompressionMode, 0, 2, producer); |
142 | 0 | setRand(cctx, ZSTD_c_forceAttachDict, 0, 2, producer); |
143 | 0 | setRand(cctx, ZSTD_c_useBlockSplitter, 0, 2, producer); |
144 | 0 | setRand(cctx, ZSTD_c_deterministicRefPrefix, 0, 1, producer); |
145 | 0 | setRand(cctx, ZSTD_c_prefetchCDictTables, 0, 2, producer); |
146 | 0 | setRand(cctx, ZSTD_c_maxBlockSize, ZSTD_BLOCKSIZE_MAX_MIN, ZSTD_BLOCKSIZE_MAX, producer); |
147 | 0 | setRand(cctx, ZSTD_c_validateSequences, 0, 1, producer); |
148 | 0 | setRand(cctx, ZSTD_c_searchForExternalRepcodes, 0, 2, producer); |
149 | 0 | if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) { |
150 | 0 | setRand(cctx, ZSTD_c_srcSizeHint, ZSTD_SRCSIZEHINT_MIN, 2 * srcSize, producer); |
151 | 0 | } |
152 | 0 | if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) { |
153 | 0 | setRand(cctx, ZSTD_c_targetCBlockSize, ZSTD_TARGETCBLOCKSIZE_MIN, ZSTD_TARGETCBLOCKSIZE_MAX, producer); |
154 | 0 | } |
155 | |
|
156 | | #ifdef FUZZ_THIRD_PARTY_SEQ_PROD |
157 | | setSequenceProducerParams(cctx, producer); |
158 | | #else |
159 | 0 | if (FUZZ_dataProducer_uint32Range(producer, 0, 10) == 1) { |
160 | 0 | setSequenceProducerParams(cctx, producer); |
161 | 0 | } else { |
162 | 0 | ZSTD_registerSequenceProducer(cctx, NULL, NULL); |
163 | 0 | } |
164 | 0 | #endif |
165 | 0 | } |
166 | | |
167 | | FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, FUZZ_dataProducer_t *producer) |
168 | 0 | { |
169 | 0 | size_t const dictSize = MAX(srcSize / 8, 1024); |
170 | 0 | size_t const totalSampleSize = dictSize * 11; |
171 | 0 | FUZZ_dict_t dict = { FUZZ_malloc(dictSize), dictSize }; |
172 | 0 | char* const samples = (char*)FUZZ_malloc(totalSampleSize); |
173 | 0 | unsigned nbSamples = 100; |
174 | 0 | size_t* const samplesSizes = (size_t*)FUZZ_malloc(sizeof(size_t) * nbSamples); |
175 | 0 | size_t pos = 0; |
176 | 0 | size_t sample = 0; |
177 | 0 | ZDICT_fastCover_params_t params; |
178 | |
|
179 | 0 | for (sample = 0; sample < nbSamples; ++sample) { |
180 | 0 | size_t const remaining = totalSampleSize - pos; |
181 | 0 | size_t const offset = FUZZ_dataProducer_uint32Range(producer, 0, MAX(srcSize, 1) - 1); |
182 | 0 | size_t const limit = MIN(srcSize - offset, remaining); |
183 | 0 | size_t const toCopy = MIN(limit, remaining / (nbSamples - sample)); |
184 | 0 | memcpy(samples + pos, (const char*)src + offset, toCopy); |
185 | 0 | pos += toCopy; |
186 | 0 | samplesSizes[sample] = toCopy; |
187 | 0 | } |
188 | 0 | memset(samples + pos, 0, totalSampleSize - pos); |
189 | |
|
190 | 0 | memset(¶ms, 0, sizeof(params)); |
191 | 0 | params.accel = 5; |
192 | 0 | params.k = 40; |
193 | 0 | params.d = 8; |
194 | 0 | params.f = 14; |
195 | 0 | params.zParams.compressionLevel = 1; |
196 | 0 | dict.size = ZDICT_trainFromBuffer_fastCover(dict.buff, dictSize, |
197 | 0 | samples, samplesSizes, nbSamples, params); |
198 | 0 | if (ZSTD_isError(dict.size)) { |
199 | 0 | free(dict.buff); |
200 | 0 | memset(&dict, 0, sizeof(dict)); |
201 | 0 | } |
202 | |
|
203 | 0 | free(samplesSizes); |
204 | 0 | free(samples); |
205 | |
|
206 | 0 | return dict; |
207 | 0 | } |