Coverage Report

Created: 2024-06-18 06:08

/src/perfetto/buildtools/zstd/lib/decompress/huf_decompress.c
Line
Count
Source (jump to first uncovered line)
1
/* ******************************************************************
2
 * huff0 huffman decoder,
3
 * part of Finite State Entropy library
4
 * Copyright (c) Meta Platforms, Inc. and affiliates.
5
 *
6
 *  You can contact the author at :
7
 *  - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
8
 *
9
 * This source code is licensed under both the BSD-style license (found in the
10
 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
11
 * in the COPYING file in the root directory of this source tree).
12
 * You may select, at your option, one of the above-listed licenses.
13
****************************************************************** */
14
15
/* **************************************************************
16
*  Dependencies
17
****************************************************************/
18
#include "../common/zstd_deps.h"  /* ZSTD_memcpy, ZSTD_memset */
19
#include "../common/compiler.h"
20
#include "../common/bitstream.h"  /* BIT_* */
21
#include "../common/fse.h"        /* to compress headers */
22
#include "../common/huf.h"
23
#include "../common/error_private.h"
24
#include "../common/zstd_internal.h"
25
#include "../common/bits.h"       /* ZSTD_highbit32, ZSTD_countTrailingZeros64 */
26
27
/* **************************************************************
28
*  Constants
29
****************************************************************/
30
31
0
#define HUF_DECODER_FAST_TABLELOG 11
32
33
/* **************************************************************
34
*  Macros
35
****************************************************************/
36
37
/* These two optional macros force the use one way or another of the two
38
 * Huffman decompression implementations. You can't force in both directions
39
 * at the same time.
40
 */
41
#if defined(HUF_FORCE_DECOMPRESS_X1) && \
42
    defined(HUF_FORCE_DECOMPRESS_X2)
43
#error "Cannot force the use of the X1 and X2 decoders at the same time!"
44
#endif
45
46
/* When DYNAMIC_BMI2 is enabled, fast decoders are only called when bmi2 is
47
 * supported at runtime, so we can add the BMI2 target attribute.
48
 * When it is disabled, we will still get BMI2 if it is enabled statically.
49
 */
50
#if DYNAMIC_BMI2
51
# define HUF_FAST_BMI2_ATTRS BMI2_TARGET_ATTRIBUTE
52
#else
53
# define HUF_FAST_BMI2_ATTRS
54
#endif
55
56
#ifdef __cplusplus
57
# define HUF_EXTERN_C extern "C"
58
#else
59
# define HUF_EXTERN_C
60
#endif
61
#define HUF_ASM_DECL HUF_EXTERN_C
62
63
#if DYNAMIC_BMI2
64
# define HUF_NEED_BMI2_FUNCTION 1
65
#else
66
# define HUF_NEED_BMI2_FUNCTION 0
67
#endif
68
69
/* **************************************************************
70
*  Error Management
71
****************************************************************/
72
0
#define HUF_isError ERR_isError
73
74
75
/* **************************************************************
76
*  Byte alignment for workSpace management
77
****************************************************************/
78
#define HUF_ALIGN(x, a)         HUF_ALIGN_MASK((x), (a) - 1)
79
#define HUF_ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask))
80
81
82
/* **************************************************************
83
*  BMI2 Variant Wrappers
84
****************************************************************/
85
typedef size_t (*HUF_DecompressUsingDTableFn)(void *dst, size_t dstSize,
86
                                              const void *cSrc,
87
                                              size_t cSrcSize,
88
                                              const HUF_DTable *DTable);
89
90
#if DYNAMIC_BMI2
91
92
#define HUF_DGEN(fn)                                                        \
93
                                                                            \
94
    static size_t fn##_default(                                             \
95
                  void* dst,  size_t dstSize,                               \
96
            const void* cSrc, size_t cSrcSize,                              \
97
            const HUF_DTable* DTable)                                       \
98
    {                                                                       \
99
        return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable);             \
100
    }                                                                       \
101
                                                                            \
102
    static BMI2_TARGET_ATTRIBUTE size_t fn##_bmi2(                          \
103
                  void* dst,  size_t dstSize,                               \
104
            const void* cSrc, size_t cSrcSize,                              \
105
            const HUF_DTable* DTable)                                       \
106
    {                                                                       \
107
        return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable);             \
108
    }                                                                       \
109
                                                                            \
110
    static size_t fn(void* dst, size_t dstSize, void const* cSrc,           \
111
                     size_t cSrcSize, HUF_DTable const* DTable, int flags)  \
112
    {                                                                       \
113
        if (flags & HUF_flags_bmi2) {                                       \
114
            return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable);         \
115
        }                                                                   \
116
        return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable);          \
117
    }
118
119
#else
120
121
#define HUF_DGEN(fn)                                                        \
122
    static size_t fn(void* dst, size_t dstSize, void const* cSrc,           \
123
                     size_t cSrcSize, HUF_DTable const* DTable, int flags)  \
124
0
    {                                                                       \
125
0
        (void)flags;                                                        \
126
0
        return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable);             \
127
0
    }
Unexecuted instantiation: huf_decompress.c:HUF_decompress1X2_usingDTable_internal
Unexecuted instantiation: huf_decompress.c:HUF_decompress1X1_usingDTable_internal
128
129
#endif
130
131
132
/*-***************************/
133
/*  generic DTableDesc       */
134
/*-***************************/
135
typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc;
136
137
static DTableDesc HUF_getDTableDesc(const HUF_DTable* table)
138
0
{
139
0
    DTableDesc dtd;
140
0
    ZSTD_memcpy(&dtd, table, sizeof(dtd));
141
0
    return dtd;
142
0
}
143
144
0
static size_t HUF_initFastDStream(BYTE const* ip) {
145
0
    BYTE const lastByte = ip[7];
146
0
    size_t const bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0;
147
0
    size_t const value = MEM_readLEST(ip) | 1;
148
0
    assert(bitsConsumed <= 8);
149
0
    assert(sizeof(size_t) == 8);
150
0
    return value << bitsConsumed;
151
0
}
152
153
154
/**
155
 * The input/output arguments to the Huffman fast decoding loop:
156
 *
157
 * ip [in/out] - The input pointers, must be updated to reflect what is consumed.
158
 * op [in/out] - The output pointers, must be updated to reflect what is written.
159
 * bits [in/out] - The bitstream containers, must be updated to reflect the current state.
160
 * dt [in] - The decoding table.
161
 * ilimit [in] - The input limit, stop when any input pointer is below ilimit.
162
 * oend [in] - The end of the output stream. op[3] must not cross oend.
163
 * iend [in] - The end of each input stream. ip[i] may cross iend[i],
164
 *             as long as it is above ilimit, but that indicates corruption.
165
 */
166
typedef struct {
167
    BYTE const* ip[4];
168
    BYTE* op[4];
169
    U64 bits[4];
170
    void const* dt;
171
    BYTE const* ilimit;
172
    BYTE* oend;
173
    BYTE const* iend[4];
174
} HUF_DecompressFastArgs;
175
176
typedef void (*HUF_DecompressFastLoopFn)(HUF_DecompressFastArgs*);
177
178
/**
179
 * Initializes args for the fast decoding loop.
180
 * @returns 1 on success
181
 *          0 if the fallback implementation should be used.
182
 *          Or an error code on failure.
183
 */
184
static size_t HUF_DecompressFastArgs_init(HUF_DecompressFastArgs* args, void* dst, size_t dstSize, void const* src, size_t srcSize, const HUF_DTable* DTable)
185
0
{
186
0
    void const* dt = DTable + 1;
187
0
    U32 const dtLog = HUF_getDTableDesc(DTable).tableLog;
188
189
0
    const BYTE* const ilimit = (const BYTE*)src + 6 + 8;
190
191
0
    BYTE* const oend = (BYTE*)dst + dstSize;
192
193
    /* The fast decoding loop assumes 64-bit little-endian.
194
     * This condition is false on x32.
195
     */
196
0
    if (!MEM_isLittleEndian() || MEM_32bits())
197
0
        return 0;
198
199
    /* strict minimum : jump table + 1 byte per stream */
200
0
    if (srcSize < 10)
201
0
        return ERROR(corruption_detected);
202
203
    /* Must have at least 8 bytes per stream because we don't handle initializing smaller bit containers.
204
     * If table log is not correct at this point, fallback to the old decoder.
205
     * On small inputs we don't have enough data to trigger the fast loop, so use the old decoder.
206
     */
207
0
    if (dtLog != HUF_DECODER_FAST_TABLELOG)
208
0
        return 0;
209
210
    /* Read the jump table. */
211
0
    {
212
0
        const BYTE* const istart = (const BYTE*)src;
213
0
        size_t const length1 = MEM_readLE16(istart);
214
0
        size_t const length2 = MEM_readLE16(istart+2);
215
0
        size_t const length3 = MEM_readLE16(istart+4);
216
0
        size_t const length4 = srcSize - (length1 + length2 + length3 + 6);
217
0
        args->iend[0] = istart + 6;  /* jumpTable */
218
0
        args->iend[1] = args->iend[0] + length1;
219
0
        args->iend[2] = args->iend[1] + length2;
220
0
        args->iend[3] = args->iend[2] + length3;
221
222
        /* HUF_initFastDStream() requires this, and this small of an input
223
         * won't benefit from the ASM loop anyways.
224
         * length1 must be >= 16 so that ip[0] >= ilimit before the loop
225
         * starts.
226
         */
227
0
        if (length1 < 16 || length2 < 8 || length3 < 8 || length4 < 8)
228
0
            return 0;
229
0
        if (length4 > srcSize) return ERROR(corruption_detected);   /* overflow */
230
0
    }
231
    /* ip[] contains the position that is currently loaded into bits[]. */
232
0
    args->ip[0] = args->iend[1] - sizeof(U64);
233
0
    args->ip[1] = args->iend[2] - sizeof(U64);
234
0
    args->ip[2] = args->iend[3] - sizeof(U64);
235
0
    args->ip[3] = (BYTE const*)src + srcSize - sizeof(U64);
236
237
    /* op[] contains the output pointers. */
238
0
    args->op[0] = (BYTE*)dst;
239
0
    args->op[1] = args->op[0] + (dstSize+3)/4;
240
0
    args->op[2] = args->op[1] + (dstSize+3)/4;
241
0
    args->op[3] = args->op[2] + (dstSize+3)/4;
242
243
    /* No point to call the ASM loop for tiny outputs. */
244
0
    if (args->op[3] >= oend)
245
0
        return 0;
246
247
    /* bits[] is the bit container.
248
        * It is read from the MSB down to the LSB.
249
        * It is shifted left as it is read, and zeros are
250
        * shifted in. After the lowest valid bit a 1 is
251
        * set, so that CountTrailingZeros(bits[]) can be used
252
        * to count how many bits we've consumed.
253
        */
254
0
    args->bits[0] = HUF_initFastDStream(args->ip[0]);
255
0
    args->bits[1] = HUF_initFastDStream(args->ip[1]);
256
0
    args->bits[2] = HUF_initFastDStream(args->ip[2]);
257
0
    args->bits[3] = HUF_initFastDStream(args->ip[3]);
258
259
    /* If ip[] >= ilimit, it is guaranteed to be safe to
260
        * reload bits[]. It may be beyond its section, but is
261
        * guaranteed to be valid (>= istart).
262
        */
263
0
    args->ilimit = ilimit;
264
265
0
    args->oend = oend;
266
0
    args->dt = dt;
267
268
0
    return 1;
269
0
}
270
271
static size_t HUF_initRemainingDStream(BIT_DStream_t* bit, HUF_DecompressFastArgs const* args, int stream, BYTE* segmentEnd)
272
0
{
273
    /* Validate that we haven't overwritten. */
274
0
    if (args->op[stream] > segmentEnd)
275
0
        return ERROR(corruption_detected);
276
    /* Validate that we haven't read beyond iend[].
277
        * Note that ip[] may be < iend[] because the MSB is
278
        * the next bit to read, and we may have consumed 100%
279
        * of the stream, so down to iend[i] - 8 is valid.
280
        */
281
0
    if (args->ip[stream] < args->iend[stream] - 8)
282
0
        return ERROR(corruption_detected);
283
284
    /* Construct the BIT_DStream_t. */
285
0
    assert(sizeof(size_t) == 8);
286
0
    bit->bitContainer = MEM_readLEST(args->ip[stream]);
287
0
    bit->bitsConsumed = ZSTD_countTrailingZeros64(args->bits[stream]);
288
0
    bit->start = (const char*)args->iend[0];
289
0
    bit->limitPtr = bit->start + sizeof(size_t);
290
0
    bit->ptr = (const char*)args->ip[stream];
291
292
0
    return 0;
293
0
}
294
295
296
#ifndef HUF_FORCE_DECOMPRESS_X2
297
298
/*-***************************/
299
/*  single-symbol decoding   */
300
/*-***************************/
301
typedef struct { BYTE nbBits; BYTE byte; } HUF_DEltX1;   /* single-symbol decoding */
302
303
/**
304
 * Packs 4 HUF_DEltX1 structs into a U64. This is used to lay down 4 entries at
305
 * a time.
306
 */
307
0
static U64 HUF_DEltX1_set4(BYTE symbol, BYTE nbBits) {
308
0
    U64 D4;
309
0
    if (MEM_isLittleEndian()) {
310
0
        D4 = (U64)((symbol << 8) + nbBits);
311
0
    } else {
312
0
        D4 = (U64)(symbol + (nbBits << 8));
313
0
    }
314
0
    assert(D4 < (1U << 16));
315
0
    D4 *= 0x0001000100010001ULL;
316
0
    return D4;
317
0
}
318
319
/**
320
 * Increase the tableLog to targetTableLog and rescales the stats.
321
 * If tableLog > targetTableLog this is a no-op.
322
 * @returns New tableLog
323
 */
324
static U32 HUF_rescaleStats(BYTE* huffWeight, U32* rankVal, U32 nbSymbols, U32 tableLog, U32 targetTableLog)
325
0
{
326
0
    if (tableLog > targetTableLog)
327
0
        return tableLog;
328
0
    if (tableLog < targetTableLog) {
329
0
        U32 const scale = targetTableLog - tableLog;
330
0
        U32 s;
331
        /* Increase the weight for all non-zero probability symbols by scale. */
332
0
        for (s = 0; s < nbSymbols; ++s) {
333
0
            huffWeight[s] += (BYTE)((huffWeight[s] == 0) ? 0 : scale);
334
0
        }
335
        /* Update rankVal to reflect the new weights.
336
         * All weights except 0 get moved to weight + scale.
337
         * Weights [1, scale] are empty.
338
         */
339
0
        for (s = targetTableLog; s > scale; --s) {
340
0
            rankVal[s] = rankVal[s - scale];
341
0
        }
342
0
        for (s = scale; s > 0; --s) {
343
0
            rankVal[s] = 0;
344
0
        }
345
0
    }
346
0
    return targetTableLog;
347
0
}
348
349
typedef struct {
350
        U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1];
351
        U32 rankStart[HUF_TABLELOG_ABSOLUTEMAX + 1];
352
        U32 statsWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32];
353
        BYTE symbols[HUF_SYMBOLVALUE_MAX + 1];
354
        BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1];
355
} HUF_ReadDTableX1_Workspace;
356
357
size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags)
358
0
{
359
0
    U32 tableLog = 0;
360
0
    U32 nbSymbols = 0;
361
0
    size_t iSize;
362
0
    void* const dtPtr = DTable + 1;
363
0
    HUF_DEltX1* const dt = (HUF_DEltX1*)dtPtr;
364
0
    HUF_ReadDTableX1_Workspace* wksp = (HUF_ReadDTableX1_Workspace*)workSpace;
365
366
0
    DEBUG_STATIC_ASSERT(HUF_DECOMPRESS_WORKSPACE_SIZE >= sizeof(*wksp));
367
0
    if (sizeof(*wksp) > wkspSize) return ERROR(tableLog_tooLarge);
368
369
0
    DEBUG_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable));
370
    /* ZSTD_memset(huffWeight, 0, sizeof(huffWeight)); */   /* is not necessary, even though some analyzer complain ... */
371
372
0
    iSize = HUF_readStats_wksp(wksp->huffWeight, HUF_SYMBOLVALUE_MAX + 1, wksp->rankVal, &nbSymbols, &tableLog, src, srcSize, wksp->statsWksp, sizeof(wksp->statsWksp), flags);
373
0
    if (HUF_isError(iSize)) return iSize;
374
375
376
    /* Table header */
377
0
    {   DTableDesc dtd = HUF_getDTableDesc(DTable);
378
0
        U32 const maxTableLog = dtd.maxTableLog + 1;
379
0
        U32 const targetTableLog = MIN(maxTableLog, HUF_DECODER_FAST_TABLELOG);
380
0
        tableLog = HUF_rescaleStats(wksp->huffWeight, wksp->rankVal, nbSymbols, tableLog, targetTableLog);
381
0
        if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge);   /* DTable too small, Huffman tree cannot fit in */
382
0
        dtd.tableType = 0;
383
0
        dtd.tableLog = (BYTE)tableLog;
384
0
        ZSTD_memcpy(DTable, &dtd, sizeof(dtd));
385
0
    }
386
387
    /* Compute symbols and rankStart given rankVal:
388
     *
389
     * rankVal already contains the number of values of each weight.
390
     *
391
     * symbols contains the symbols ordered by weight. First are the rankVal[0]
392
     * weight 0 symbols, followed by the rankVal[1] weight 1 symbols, and so on.
393
     * symbols[0] is filled (but unused) to avoid a branch.
394
     *
395
     * rankStart contains the offset where each rank belongs in the DTable.
396
     * rankStart[0] is not filled because there are no entries in the table for
397
     * weight 0.
398
     */
399
0
    {   int n;
400
0
        U32 nextRankStart = 0;
401
0
        int const unroll = 4;
402
0
        int const nLimit = (int)nbSymbols - unroll + 1;
403
0
        for (n=0; n<(int)tableLog+1; n++) {
404
0
            U32 const curr = nextRankStart;
405
0
            nextRankStart += wksp->rankVal[n];
406
0
            wksp->rankStart[n] = curr;
407
0
        }
408
0
        for (n=0; n < nLimit; n += unroll) {
409
0
            int u;
410
0
            for (u=0; u < unroll; ++u) {
411
0
                size_t const w = wksp->huffWeight[n+u];
412
0
                wksp->symbols[wksp->rankStart[w]++] = (BYTE)(n+u);
413
0
            }
414
0
        }
415
0
        for (; n < (int)nbSymbols; ++n) {
416
0
            size_t const w = wksp->huffWeight[n];
417
0
            wksp->symbols[wksp->rankStart[w]++] = (BYTE)n;
418
0
        }
419
0
    }
420
421
    /* fill DTable
422
     * We fill all entries of each weight in order.
423
     * That way length is a constant for each iteration of the outer loop.
424
     * We can switch based on the length to a different inner loop which is
425
     * optimized for that particular case.
426
     */
427
0
    {   U32 w;
428
0
        int symbol = wksp->rankVal[0];
429
0
        int rankStart = 0;
430
0
        for (w=1; w<tableLog+1; ++w) {
431
0
            int const symbolCount = wksp->rankVal[w];
432
0
            int const length = (1 << w) >> 1;
433
0
            int uStart = rankStart;
434
0
            BYTE const nbBits = (BYTE)(tableLog + 1 - w);
435
0
            int s;
436
0
            int u;
437
0
            switch (length) {
438
0
            case 1:
439
0
                for (s=0; s<symbolCount; ++s) {
440
0
                    HUF_DEltX1 D;
441
0
                    D.byte = wksp->symbols[symbol + s];
442
0
                    D.nbBits = nbBits;
443
0
                    dt[uStart] = D;
444
0
                    uStart += 1;
445
0
                }
446
0
                break;
447
0
            case 2:
448
0
                for (s=0; s<symbolCount; ++s) {
449
0
                    HUF_DEltX1 D;
450
0
                    D.byte = wksp->symbols[symbol + s];
451
0
                    D.nbBits = nbBits;
452
0
                    dt[uStart+0] = D;
453
0
                    dt[uStart+1] = D;
454
0
                    uStart += 2;
455
0
                }
456
0
                break;
457
0
            case 4:
458
0
                for (s=0; s<symbolCount; ++s) {
459
0
                    U64 const D4 = HUF_DEltX1_set4(wksp->symbols[symbol + s], nbBits);
460
0
                    MEM_write64(dt + uStart, D4);
461
0
                    uStart += 4;
462
0
                }
463
0
                break;
464
0
            case 8:
465
0
                for (s=0; s<symbolCount; ++s) {
466
0
                    U64 const D4 = HUF_DEltX1_set4(wksp->symbols[symbol + s], nbBits);
467
0
                    MEM_write64(dt + uStart, D4);
468
0
                    MEM_write64(dt + uStart + 4, D4);
469
0
                    uStart += 8;
470
0
                }
471
0
                break;
472
0
            default:
473
0
                for (s=0; s<symbolCount; ++s) {
474
0
                    U64 const D4 = HUF_DEltX1_set4(wksp->symbols[symbol + s], nbBits);
475
0
                    for (u=0; u < length; u += 16) {
476
0
                        MEM_write64(dt + uStart + u + 0, D4);
477
0
                        MEM_write64(dt + uStart + u + 4, D4);
478
0
                        MEM_write64(dt + uStart + u + 8, D4);
479
0
                        MEM_write64(dt + uStart + u + 12, D4);
480
0
                    }
481
0
                    assert(u == length);
482
0
                    uStart += length;
483
0
                }
484
0
                break;
485
0
            }
486
0
            symbol += symbolCount;
487
0
            rankStart += symbolCount * length;
488
0
        }
489
0
    }
490
0
    return iSize;
491
0
}
492
493
FORCE_INLINE_TEMPLATE BYTE
494
HUF_decodeSymbolX1(BIT_DStream_t* Dstream, const HUF_DEltX1* dt, const U32 dtLog)
495
0
{
496
0
    size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */
497
0
    BYTE const c = dt[val].byte;
498
0
    BIT_skipBits(Dstream, dt[val].nbBits);
499
0
    return c;
500
0
}
501
502
#define HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) \
503
0
    *ptr++ = HUF_decodeSymbolX1(DStreamPtr, dt, dtLog)
504
505
#define HUF_DECODE_SYMBOLX1_1(ptr, DStreamPtr)  \
506
0
    if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \
507
0
        HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr)
508
509
#define HUF_DECODE_SYMBOLX1_2(ptr, DStreamPtr) \
510
0
    if (MEM_64bits()) \
511
0
        HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr)
512
513
HINT_INLINE size_t
514
HUF_decodeStreamX1(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX1* const dt, const U32 dtLog)
515
0
{
516
0
    BYTE* const pStart = p;
517
518
    /* up to 4 symbols at a time */
519
0
    if ((pEnd - p) > 3) {
520
0
        while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-3)) {
521
0
            HUF_DECODE_SYMBOLX1_2(p, bitDPtr);
522
0
            HUF_DECODE_SYMBOLX1_1(p, bitDPtr);
523
0
            HUF_DECODE_SYMBOLX1_2(p, bitDPtr);
524
0
            HUF_DECODE_SYMBOLX1_0(p, bitDPtr);
525
0
        }
526
0
    } else {
527
0
        BIT_reloadDStream(bitDPtr);
528
0
    }
529
530
    /* [0-3] symbols remaining */
531
0
    if (MEM_32bits())
532
0
        while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd))
533
0
            HUF_DECODE_SYMBOLX1_0(p, bitDPtr);
534
535
    /* no more data to retrieve from bitstream, no need to reload */
536
0
    while (p < pEnd)
537
0
        HUF_DECODE_SYMBOLX1_0(p, bitDPtr);
538
539
0
    return (size_t)(pEnd-pStart);
540
0
}
541
542
FORCE_INLINE_TEMPLATE size_t
543
HUF_decompress1X1_usingDTable_internal_body(
544
          void* dst,  size_t dstSize,
545
    const void* cSrc, size_t cSrcSize,
546
    const HUF_DTable* DTable)
547
0
{
548
0
    BYTE* op = (BYTE*)dst;
549
0
    BYTE* const oend = op + dstSize;
550
0
    const void* dtPtr = DTable + 1;
551
0
    const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr;
552
0
    BIT_DStream_t bitD;
553
0
    DTableDesc const dtd = HUF_getDTableDesc(DTable);
554
0
    U32 const dtLog = dtd.tableLog;
555
556
0
    CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) );
557
558
0
    HUF_decodeStreamX1(op, &bitD, oend, dt, dtLog);
559
560
0
    if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected);
561
562
0
    return dstSize;
563
0
}
564
565
/* HUF_decompress4X1_usingDTable_internal_body():
566
 * Conditions :
567
 * @dstSize >= 6
568
 */
569
FORCE_INLINE_TEMPLATE size_t
570
HUF_decompress4X1_usingDTable_internal_body(
571
          void* dst,  size_t dstSize,
572
    const void* cSrc, size_t cSrcSize,
573
    const HUF_DTable* DTable)
574
0
{
575
    /* Check */
576
0
    if (cSrcSize < 10) return ERROR(corruption_detected);  /* strict minimum : jump table + 1 byte per stream */
577
578
0
    {   const BYTE* const istart = (const BYTE*) cSrc;
579
0
        BYTE* const ostart = (BYTE*) dst;
580
0
        BYTE* const oend = ostart + dstSize;
581
0
        BYTE* const olimit = oend - 3;
582
0
        const void* const dtPtr = DTable + 1;
583
0
        const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr;
584
585
        /* Init */
586
0
        BIT_DStream_t bitD1;
587
0
        BIT_DStream_t bitD2;
588
0
        BIT_DStream_t bitD3;
589
0
        BIT_DStream_t bitD4;
590
0
        size_t const length1 = MEM_readLE16(istart);
591
0
        size_t const length2 = MEM_readLE16(istart+2);
592
0
        size_t const length3 = MEM_readLE16(istart+4);
593
0
        size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6);
594
0
        const BYTE* const istart1 = istart + 6;  /* jumpTable */
595
0
        const BYTE* const istart2 = istart1 + length1;
596
0
        const BYTE* const istart3 = istart2 + length2;
597
0
        const BYTE* const istart4 = istart3 + length3;
598
0
        const size_t segmentSize = (dstSize+3) / 4;
599
0
        BYTE* const opStart2 = ostart + segmentSize;
600
0
        BYTE* const opStart3 = opStart2 + segmentSize;
601
0
        BYTE* const opStart4 = opStart3 + segmentSize;
602
0
        BYTE* op1 = ostart;
603
0
        BYTE* op2 = opStart2;
604
0
        BYTE* op3 = opStart3;
605
0
        BYTE* op4 = opStart4;
606
0
        DTableDesc const dtd = HUF_getDTableDesc(DTable);
607
0
        U32 const dtLog = dtd.tableLog;
608
0
        U32 endSignal = 1;
609
610
0
        if (length4 > cSrcSize) return ERROR(corruption_detected);   /* overflow */
611
0
        if (opStart4 > oend) return ERROR(corruption_detected);      /* overflow */
612
0
        if (dstSize < 6) return ERROR(corruption_detected);         /* stream 4-split doesn't work */
613
0
        CHECK_F( BIT_initDStream(&bitD1, istart1, length1) );
614
0
        CHECK_F( BIT_initDStream(&bitD2, istart2, length2) );
615
0
        CHECK_F( BIT_initDStream(&bitD3, istart3, length3) );
616
0
        CHECK_F( BIT_initDStream(&bitD4, istart4, length4) );
617
618
        /* up to 16 symbols per loop (4 symbols per stream) in 64-bit mode */
619
0
        if ((size_t)(oend - op4) >= sizeof(size_t)) {
620
0
            for ( ; (endSignal) & (op4 < olimit) ; ) {
621
0
                HUF_DECODE_SYMBOLX1_2(op1, &bitD1);
622
0
                HUF_DECODE_SYMBOLX1_2(op2, &bitD2);
623
0
                HUF_DECODE_SYMBOLX1_2(op3, &bitD3);
624
0
                HUF_DECODE_SYMBOLX1_2(op4, &bitD4);
625
0
                HUF_DECODE_SYMBOLX1_1(op1, &bitD1);
626
0
                HUF_DECODE_SYMBOLX1_1(op2, &bitD2);
627
0
                HUF_DECODE_SYMBOLX1_1(op3, &bitD3);
628
0
                HUF_DECODE_SYMBOLX1_1(op4, &bitD4);
629
0
                HUF_DECODE_SYMBOLX1_2(op1, &bitD1);
630
0
                HUF_DECODE_SYMBOLX1_2(op2, &bitD2);
631
0
                HUF_DECODE_SYMBOLX1_2(op3, &bitD3);
632
0
                HUF_DECODE_SYMBOLX1_2(op4, &bitD4);
633
0
                HUF_DECODE_SYMBOLX1_0(op1, &bitD1);
634
0
                HUF_DECODE_SYMBOLX1_0(op2, &bitD2);
635
0
                HUF_DECODE_SYMBOLX1_0(op3, &bitD3);
636
0
                HUF_DECODE_SYMBOLX1_0(op4, &bitD4);
637
0
                endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished;
638
0
                endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished;
639
0
                endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished;
640
0
                endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished;
641
0
            }
642
0
        }
643
644
        /* check corruption */
645
        /* note : should not be necessary : op# advance in lock step, and we control op4.
646
         *        but curiously, binary generated by gcc 7.2 & 7.3 with -mbmi2 runs faster when >=1 test is present */
647
0
        if (op1 > opStart2) return ERROR(corruption_detected);
648
0
        if (op2 > opStart3) return ERROR(corruption_detected);
649
0
        if (op3 > opStart4) return ERROR(corruption_detected);
650
        /* note : op4 supposed already verified within main loop */
651
652
        /* finish bitStreams one by one */
653
0
        HUF_decodeStreamX1(op1, &bitD1, opStart2, dt, dtLog);
654
0
        HUF_decodeStreamX1(op2, &bitD2, opStart3, dt, dtLog);
655
0
        HUF_decodeStreamX1(op3, &bitD3, opStart4, dt, dtLog);
656
0
        HUF_decodeStreamX1(op4, &bitD4, oend,     dt, dtLog);
657
658
        /* check */
659
0
        { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
660
0
          if (!endCheck) return ERROR(corruption_detected); }
661
662
        /* decoded size */
663
0
        return dstSize;
664
0
    }
665
0
}
666
667
#if HUF_NEED_BMI2_FUNCTION
668
static BMI2_TARGET_ATTRIBUTE
669
size_t HUF_decompress4X1_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc,
670
                    size_t cSrcSize, HUF_DTable const* DTable) {
671
    return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable);
672
}
673
#endif
674
675
static
676
size_t HUF_decompress4X1_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc,
677
0
                    size_t cSrcSize, HUF_DTable const* DTable) {
678
0
    return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable);
679
0
}
680
681
#if ZSTD_ENABLE_ASM_X86_64_BMI2
682
683
HUF_ASM_DECL void HUF_decompress4X1_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN;
684
685
#endif
686
687
static HUF_FAST_BMI2_ATTRS
688
void HUF_decompress4X1_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args)
689
0
{
690
0
    U64 bits[4];
691
0
    BYTE const* ip[4];
692
0
    BYTE* op[4];
693
0
    U16 const* const dtable = (U16 const*)args->dt;
694
0
    BYTE* const oend = args->oend;
695
0
    BYTE const* const ilimit = args->ilimit;
696
697
    /* Copy the arguments to local variables */
698
0
    ZSTD_memcpy(&bits, &args->bits, sizeof(bits));
699
0
    ZSTD_memcpy((void*)(&ip), &args->ip, sizeof(ip));
700
0
    ZSTD_memcpy(&op, &args->op, sizeof(op));
701
702
0
    assert(MEM_isLittleEndian());
703
0
    assert(!MEM_32bits());
704
705
0
    for (;;) {
706
0
        BYTE* olimit;
707
0
        int stream;
708
0
        int symbol;
709
710
        /* Assert loop preconditions */
711
#ifndef NDEBUG
712
        for (stream = 0; stream < 4; ++stream) {
713
            assert(op[stream] <= (stream == 3 ? oend : op[stream + 1]));
714
            assert(ip[stream] >= ilimit);
715
        }
716
#endif
717
        /* Compute olimit */
718
0
        {
719
            /* Each iteration produces 5 output symbols per stream */
720
0
            size_t const oiters = (size_t)(oend - op[3]) / 5;
721
            /* Each iteration consumes up to 11 bits * 5 = 55 bits < 7 bytes
722
             * per stream.
723
             */
724
0
            size_t const iiters = (size_t)(ip[0] - ilimit) / 7;
725
            /* We can safely run iters iterations before running bounds checks */
726
0
            size_t const iters = MIN(oiters, iiters);
727
0
            size_t const symbols = iters * 5;
728
729
            /* We can simply check that op[3] < olimit, instead of checking all
730
             * of our bounds, since we can't hit the other bounds until we've run
731
             * iters iterations, which only happens when op[3] == olimit.
732
             */
733
0
            olimit = op[3] + symbols;
734
735
            /* Exit fast decoding loop once we get close to the end. */
736
0
            if (op[3] + 20 > olimit)
737
0
                break;
738
739
            /* Exit the decoding loop if any input pointer has crossed the
740
             * previous one. This indicates corruption, and a precondition
741
             * to our loop is that ip[i] >= ip[0].
742
             */
743
0
            for (stream = 1; stream < 4; ++stream) {
744
0
                if (ip[stream] < ip[stream - 1])
745
0
                    goto _out;
746
0
            }
747
0
        }
748
749
#ifndef NDEBUG
750
        for (stream = 1; stream < 4; ++stream) {
751
            assert(ip[stream] >= ip[stream - 1]);
752
        }
753
#endif
754
755
0
        do {
756
            /* Decode 5 symbols in each of the 4 streams */
757
0
            for (symbol = 0; symbol < 5; ++symbol) {
758
0
                for (stream = 0; stream < 4; ++stream) {
759
0
                    int const index = (int)(bits[stream] >> 53);
760
0
                    int const entry = (int)dtable[index];
761
0
                    bits[stream] <<= (entry & 63);
762
0
                    op[stream][symbol] = (BYTE)((entry >> 8) & 0xFF);
763
0
                }
764
0
            }
765
            /* Reload the bitstreams */
766
0
            for (stream = 0; stream < 4; ++stream) {
767
0
                int const ctz = ZSTD_countTrailingZeros64(bits[stream]);
768
0
                int const nbBits = ctz & 7;
769
0
                int const nbBytes = ctz >> 3;
770
0
                op[stream] += 5;
771
0
                ip[stream] -= nbBytes;
772
0
                bits[stream] = MEM_read64(ip[stream]) | 1;
773
0
                bits[stream] <<= nbBits;
774
0
            }
775
0
        } while (op[3] < olimit);
776
0
    }
777
778
0
_out:
779
780
    /* Save the final values of each of the state variables back to args. */
781
0
    ZSTD_memcpy(&args->bits, &bits, sizeof(bits));
782
0
    ZSTD_memcpy((void*)(&args->ip), &ip, sizeof(ip));
783
0
    ZSTD_memcpy(&args->op, &op, sizeof(op));
784
0
}
785
786
/**
787
 * @returns @p dstSize on success (>= 6)
788
 *          0 if the fallback implementation should be used
789
 *          An error if an error occurred
790
 */
791
static HUF_FAST_BMI2_ATTRS
792
size_t
793
HUF_decompress4X1_usingDTable_internal_fast(
794
          void* dst,  size_t dstSize,
795
    const void* cSrc, size_t cSrcSize,
796
    const HUF_DTable* DTable,
797
    HUF_DecompressFastLoopFn loopFn)
798
0
{
799
0
    void const* dt = DTable + 1;
800
0
    const BYTE* const iend = (const BYTE*)cSrc + 6;
801
0
    BYTE* const oend = (BYTE*)dst + dstSize;
802
0
    HUF_DecompressFastArgs args;
803
0
    {   size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable);
804
0
        FORWARD_IF_ERROR(ret, "Failed to init fast loop args");
805
0
        if (ret == 0)
806
0
            return 0;
807
0
    }
808
809
0
    assert(args.ip[0] >= args.ilimit);
810
0
    loopFn(&args);
811
812
    /* Our loop guarantees that ip[] >= ilimit and that we haven't
813
    * overwritten any op[].
814
    */
815
0
    assert(args.ip[0] >= iend);
816
0
    assert(args.ip[1] >= iend);
817
0
    assert(args.ip[2] >= iend);
818
0
    assert(args.ip[3] >= iend);
819
0
    assert(args.op[3] <= oend);
820
0
    (void)iend;
821
822
    /* finish bit streams one by one. */
823
0
    {   size_t const segmentSize = (dstSize+3) / 4;
824
0
        BYTE* segmentEnd = (BYTE*)dst;
825
0
        int i;
826
0
        for (i = 0; i < 4; ++i) {
827
0
            BIT_DStream_t bit;
828
0
            if (segmentSize <= (size_t)(oend - segmentEnd))
829
0
                segmentEnd += segmentSize;
830
0
            else
831
0
                segmentEnd = oend;
832
0
            FORWARD_IF_ERROR(HUF_initRemainingDStream(&bit, &args, i, segmentEnd), "corruption");
833
            /* Decompress and validate that we've produced exactly the expected length. */
834
0
            args.op[i] += HUF_decodeStreamX1(args.op[i], &bit, segmentEnd, (HUF_DEltX1 const*)dt, HUF_DECODER_FAST_TABLELOG);
835
0
            if (args.op[i] != segmentEnd) return ERROR(corruption_detected);
836
0
        }
837
0
    }
838
839
    /* decoded size */
840
0
    assert(dstSize != 0);
841
0
    return dstSize;
842
0
}
843
844
HUF_DGEN(HUF_decompress1X1_usingDTable_internal)
845
846
static size_t HUF_decompress4X1_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc,
847
                    size_t cSrcSize, HUF_DTable const* DTable, int flags)
848
0
{
849
0
    HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X1_usingDTable_internal_default;
850
0
    HUF_DecompressFastLoopFn loopFn = HUF_decompress4X1_usingDTable_internal_fast_c_loop;
851
852
#if DYNAMIC_BMI2
853
    if (flags & HUF_flags_bmi2) {
854
        fallbackFn = HUF_decompress4X1_usingDTable_internal_bmi2;
855
# if ZSTD_ENABLE_ASM_X86_64_BMI2
856
        if (!(flags & HUF_flags_disableAsm)) {
857
            loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop;
858
        }
859
# endif
860
    } else {
861
        return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable);
862
    }
863
#endif
864
865
#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)
866
    if (!(flags & HUF_flags_disableAsm)) {
867
        loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop;
868
    }
869
#endif
870
871
0
    if (!(flags & HUF_flags_disableFast)) {
872
0
        size_t const ret = HUF_decompress4X1_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn);
873
0
        if (ret != 0)
874
0
            return ret;
875
0
    }
876
0
    return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable);
877
0
}
878
879
static size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
880
                                   const void* cSrc, size_t cSrcSize,
881
                                   void* workSpace, size_t wkspSize, int flags)
882
0
{
883
0
    const BYTE* ip = (const BYTE*) cSrc;
884
885
0
    size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags);
886
0
    if (HUF_isError(hSize)) return hSize;
887
0
    if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
888
0
    ip += hSize; cSrcSize -= hSize;
889
890
0
    return HUF_decompress4X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags);
891
0
}
892
893
#endif /* HUF_FORCE_DECOMPRESS_X2 */
894
895
896
#ifndef HUF_FORCE_DECOMPRESS_X1
897
898
/* *************************/
899
/* double-symbols decoding */
900
/* *************************/
901
902
typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX2;  /* double-symbols decoding */
903
typedef struct { BYTE symbol; } sortedSymbol_t;
904
typedef U32 rankValCol_t[HUF_TABLELOG_MAX + 1];
905
typedef rankValCol_t rankVal_t[HUF_TABLELOG_MAX];
906
907
/**
908
 * Constructs a HUF_DEltX2 in a U32.
909
 */
910
static U32 HUF_buildDEltX2U32(U32 symbol, U32 nbBits, U32 baseSeq, int level)
911
0
{
912
0
    U32 seq;
913
0
    DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, sequence) == 0);
914
0
    DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, nbBits) == 2);
915
0
    DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, length) == 3);
916
0
    DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(U32));
917
0
    if (MEM_isLittleEndian()) {
918
0
        seq = level == 1 ? symbol : (baseSeq + (symbol << 8));
919
0
        return seq + (nbBits << 16) + ((U32)level << 24);
920
0
    } else {
921
0
        seq = level == 1 ? (symbol << 8) : ((baseSeq << 8) + symbol);
922
0
        return (seq << 16) + (nbBits << 8) + (U32)level;
923
0
    }
924
0
}
925
926
/**
927
 * Constructs a HUF_DEltX2.
928
 */
929
static HUF_DEltX2 HUF_buildDEltX2(U32 symbol, U32 nbBits, U32 baseSeq, int level)
930
0
{
931
0
    HUF_DEltX2 DElt;
932
0
    U32 const val = HUF_buildDEltX2U32(symbol, nbBits, baseSeq, level);
933
0
    DEBUG_STATIC_ASSERT(sizeof(DElt) == sizeof(val));
934
0
    ZSTD_memcpy(&DElt, &val, sizeof(val));
935
0
    return DElt;
936
0
}
937
938
/**
939
 * Constructs 2 HUF_DEltX2s and packs them into a U64.
940
 */
941
static U64 HUF_buildDEltX2U64(U32 symbol, U32 nbBits, U16 baseSeq, int level)
942
0
{
943
0
    U32 DElt = HUF_buildDEltX2U32(symbol, nbBits, baseSeq, level);
944
0
    return (U64)DElt + ((U64)DElt << 32);
945
0
}
946
947
/**
948
 * Fills the DTable rank with all the symbols from [begin, end) that are each
949
 * nbBits long.
950
 *
951
 * @param DTableRank The start of the rank in the DTable.
952
 * @param begin The first symbol to fill (inclusive).
953
 * @param end The last symbol to fill (exclusive).
954
 * @param nbBits Each symbol is nbBits long.
955
 * @param tableLog The table log.
956
 * @param baseSeq If level == 1 { 0 } else { the first level symbol }
957
 * @param level The level in the table. Must be 1 or 2.
958
 */
959
static void HUF_fillDTableX2ForWeight(
960
    HUF_DEltX2* DTableRank,
961
    sortedSymbol_t const* begin, sortedSymbol_t const* end,
962
    U32 nbBits, U32 tableLog,
963
    U16 baseSeq, int const level)
964
0
{
965
0
    U32 const length = 1U << ((tableLog - nbBits) & 0x1F /* quiet static-analyzer */);
966
0
    const sortedSymbol_t* ptr;
967
0
    assert(level >= 1 && level <= 2);
968
0
    switch (length) {
969
0
    case 1:
970
0
        for (ptr = begin; ptr != end; ++ptr) {
971
0
            HUF_DEltX2 const DElt = HUF_buildDEltX2(ptr->symbol, nbBits, baseSeq, level);
972
0
            *DTableRank++ = DElt;
973
0
        }
974
0
        break;
975
0
    case 2:
976
0
        for (ptr = begin; ptr != end; ++ptr) {
977
0
            HUF_DEltX2 const DElt = HUF_buildDEltX2(ptr->symbol, nbBits, baseSeq, level);
978
0
            DTableRank[0] = DElt;
979
0
            DTableRank[1] = DElt;
980
0
            DTableRank += 2;
981
0
        }
982
0
        break;
983
0
    case 4:
984
0
        for (ptr = begin; ptr != end; ++ptr) {
985
0
            U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level);
986
0
            ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2));
987
0
            ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2));
988
0
            DTableRank += 4;
989
0
        }
990
0
        break;
991
0
    case 8:
992
0
        for (ptr = begin; ptr != end; ++ptr) {
993
0
            U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level);
994
0
            ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2));
995
0
            ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2));
996
0
            ZSTD_memcpy(DTableRank + 4, &DEltX2, sizeof(DEltX2));
997
0
            ZSTD_memcpy(DTableRank + 6, &DEltX2, sizeof(DEltX2));
998
0
            DTableRank += 8;
999
0
        }
1000
0
        break;
1001
0
    default:
1002
0
        for (ptr = begin; ptr != end; ++ptr) {
1003
0
            U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level);
1004
0
            HUF_DEltX2* const DTableRankEnd = DTableRank + length;
1005
0
            for (; DTableRank != DTableRankEnd; DTableRank += 8) {
1006
0
                ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2));
1007
0
                ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2));
1008
0
                ZSTD_memcpy(DTableRank + 4, &DEltX2, sizeof(DEltX2));
1009
0
                ZSTD_memcpy(DTableRank + 6, &DEltX2, sizeof(DEltX2));
1010
0
            }
1011
0
        }
1012
0
        break;
1013
0
    }
1014
0
}
1015
1016
/* HUF_fillDTableX2Level2() :
1017
 * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */
1018
static void HUF_fillDTableX2Level2(HUF_DEltX2* DTable, U32 targetLog, const U32 consumedBits,
1019
                           const U32* rankVal, const int minWeight, const int maxWeight1,
1020
                           const sortedSymbol_t* sortedSymbols, U32 const* rankStart,
1021
                           U32 nbBitsBaseline, U16 baseSeq)
1022
0
{
1023
    /* Fill skipped values (all positions up to rankVal[minWeight]).
1024
     * These are positions only get a single symbol because the combined weight
1025
     * is too large.
1026
     */
1027
0
    if (minWeight>1) {
1028
0
        U32 const length = 1U << ((targetLog - consumedBits) & 0x1F /* quiet static-analyzer */);
1029
0
        U64 const DEltX2 = HUF_buildDEltX2U64(baseSeq, consumedBits, /* baseSeq */ 0, /* level */ 1);
1030
0
        int const skipSize = rankVal[minWeight];
1031
0
        assert(length > 1);
1032
0
        assert((U32)skipSize < length);
1033
0
        switch (length) {
1034
0
        case 2:
1035
0
            assert(skipSize == 1);
1036
0
            ZSTD_memcpy(DTable, &DEltX2, sizeof(DEltX2));
1037
0
            break;
1038
0
        case 4:
1039
0
            assert(skipSize <= 4);
1040
0
            ZSTD_memcpy(DTable + 0, &DEltX2, sizeof(DEltX2));
1041
0
            ZSTD_memcpy(DTable + 2, &DEltX2, sizeof(DEltX2));
1042
0
            break;
1043
0
        default:
1044
0
            {
1045
0
                int i;
1046
0
                for (i = 0; i < skipSize; i += 8) {
1047
0
                    ZSTD_memcpy(DTable + i + 0, &DEltX2, sizeof(DEltX2));
1048
0
                    ZSTD_memcpy(DTable + i + 2, &DEltX2, sizeof(DEltX2));
1049
0
                    ZSTD_memcpy(DTable + i + 4, &DEltX2, sizeof(DEltX2));
1050
0
                    ZSTD_memcpy(DTable + i + 6, &DEltX2, sizeof(DEltX2));
1051
0
                }
1052
0
            }
1053
0
        }
1054
0
    }
1055
1056
    /* Fill each of the second level symbols by weight. */
1057
0
    {
1058
0
        int w;
1059
0
        for (w = minWeight; w < maxWeight1; ++w) {
1060
0
            int const begin = rankStart[w];
1061
0
            int const end = rankStart[w+1];
1062
0
            U32 const nbBits = nbBitsBaseline - w;
1063
0
            U32 const totalBits = nbBits + consumedBits;
1064
0
            HUF_fillDTableX2ForWeight(
1065
0
                DTable + rankVal[w],
1066
0
                sortedSymbols + begin, sortedSymbols + end,
1067
0
                totalBits, targetLog,
1068
0
                baseSeq, /* level */ 2);
1069
0
        }
1070
0
    }
1071
0
}
1072
1073
static void HUF_fillDTableX2(HUF_DEltX2* DTable, const U32 targetLog,
1074
                           const sortedSymbol_t* sortedList,
1075
                           const U32* rankStart, rankValCol_t* rankValOrigin, const U32 maxWeight,
1076
                           const U32 nbBitsBaseline)
1077
0
{
1078
0
    U32* const rankVal = rankValOrigin[0];
1079
0
    const int scaleLog = nbBitsBaseline - targetLog;   /* note : targetLog >= srcLog, hence scaleLog <= 1 */
1080
0
    const U32 minBits  = nbBitsBaseline - maxWeight;
1081
0
    int w;
1082
0
    int const wEnd = (int)maxWeight + 1;
1083
1084
    /* Fill DTable in order of weight. */
1085
0
    for (w = 1; w < wEnd; ++w) {
1086
0
        int const begin = (int)rankStart[w];
1087
0
        int const end = (int)rankStart[w+1];
1088
0
        U32 const nbBits = nbBitsBaseline - w;
1089
1090
0
        if (targetLog-nbBits >= minBits) {
1091
            /* Enough room for a second symbol. */
1092
0
            int start = rankVal[w];
1093
0
            U32 const length = 1U << ((targetLog - nbBits) & 0x1F /* quiet static-analyzer */);
1094
0
            int minWeight = nbBits + scaleLog;
1095
0
            int s;
1096
0
            if (minWeight < 1) minWeight = 1;
1097
            /* Fill the DTable for every symbol of weight w.
1098
             * These symbols get at least 1 second symbol.
1099
             */
1100
0
            for (s = begin; s != end; ++s) {
1101
0
                HUF_fillDTableX2Level2(
1102
0
                    DTable + start, targetLog, nbBits,
1103
0
                    rankValOrigin[nbBits], minWeight, wEnd,
1104
0
                    sortedList, rankStart,
1105
0
                    nbBitsBaseline, sortedList[s].symbol);
1106
0
                start += length;
1107
0
            }
1108
0
        } else {
1109
            /* Only a single symbol. */
1110
0
            HUF_fillDTableX2ForWeight(
1111
0
                DTable + rankVal[w],
1112
0
                sortedList + begin, sortedList + end,
1113
0
                nbBits, targetLog,
1114
0
                /* baseSeq */ 0, /* level */ 1);
1115
0
        }
1116
0
    }
1117
0
}
1118
1119
typedef struct {
1120
    rankValCol_t rankVal[HUF_TABLELOG_MAX];
1121
    U32 rankStats[HUF_TABLELOG_MAX + 1];
1122
    U32 rankStart0[HUF_TABLELOG_MAX + 3];
1123
    sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1];
1124
    BYTE weightList[HUF_SYMBOLVALUE_MAX + 1];
1125
    U32 calleeWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32];
1126
} HUF_ReadDTableX2_Workspace;
1127
1128
size_t HUF_readDTableX2_wksp(HUF_DTable* DTable,
1129
                       const void* src, size_t srcSize,
1130
                             void* workSpace, size_t wkspSize, int flags)
1131
0
{
1132
0
    U32 tableLog, maxW, nbSymbols;
1133
0
    DTableDesc dtd = HUF_getDTableDesc(DTable);
1134
0
    U32 maxTableLog = dtd.maxTableLog;
1135
0
    size_t iSize;
1136
0
    void* dtPtr = DTable+1;   /* force compiler to avoid strict-aliasing */
1137
0
    HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr;
1138
0
    U32 *rankStart;
1139
1140
0
    HUF_ReadDTableX2_Workspace* const wksp = (HUF_ReadDTableX2_Workspace*)workSpace;
1141
1142
0
    if (sizeof(*wksp) > wkspSize) return ERROR(GENERIC);
1143
1144
0
    rankStart = wksp->rankStart0 + 1;
1145
0
    ZSTD_memset(wksp->rankStats, 0, sizeof(wksp->rankStats));
1146
0
    ZSTD_memset(wksp->rankStart0, 0, sizeof(wksp->rankStart0));
1147
1148
0
    DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(HUF_DTable));   /* if compiler fails here, assertion is wrong */
1149
0
    if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
1150
    /* ZSTD_memset(weightList, 0, sizeof(weightList)); */  /* is not necessary, even though some analyzer complain ... */
1151
1152
0
    iSize = HUF_readStats_wksp(wksp->weightList, HUF_SYMBOLVALUE_MAX + 1, wksp->rankStats, &nbSymbols, &tableLog, src, srcSize, wksp->calleeWksp, sizeof(wksp->calleeWksp), flags);
1153
0
    if (HUF_isError(iSize)) return iSize;
1154
1155
    /* check result */
1156
0
    if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge);   /* DTable can't fit code depth */
1157
0
    if (tableLog <= HUF_DECODER_FAST_TABLELOG && maxTableLog > HUF_DECODER_FAST_TABLELOG) maxTableLog = HUF_DECODER_FAST_TABLELOG;
1158
1159
    /* find maxWeight */
1160
0
    for (maxW = tableLog; wksp->rankStats[maxW]==0; maxW--) {}  /* necessarily finds a solution before 0 */
1161
1162
    /* Get start index of each weight */
1163
0
    {   U32 w, nextRankStart = 0;
1164
0
        for (w=1; w<maxW+1; w++) {
1165
0
            U32 curr = nextRankStart;
1166
0
            nextRankStart += wksp->rankStats[w];
1167
0
            rankStart[w] = curr;
1168
0
        }
1169
0
        rankStart[0] = nextRankStart;   /* put all 0w symbols at the end of sorted list*/
1170
0
        rankStart[maxW+1] = nextRankStart;
1171
0
    }
1172
1173
    /* sort symbols by weight */
1174
0
    {   U32 s;
1175
0
        for (s=0; s<nbSymbols; s++) {
1176
0
            U32 const w = wksp->weightList[s];
1177
0
            U32 const r = rankStart[w]++;
1178
0
            wksp->sortedSymbol[r].symbol = (BYTE)s;
1179
0
        }
1180
0
        rankStart[0] = 0;   /* forget 0w symbols; this is beginning of weight(1) */
1181
0
    }
1182
1183
    /* Build rankVal */
1184
0
    {   U32* const rankVal0 = wksp->rankVal[0];
1185
0
        {   int const rescale = (maxTableLog-tableLog) - 1;   /* tableLog <= maxTableLog */
1186
0
            U32 nextRankVal = 0;
1187
0
            U32 w;
1188
0
            for (w=1; w<maxW+1; w++) {
1189
0
                U32 curr = nextRankVal;
1190
0
                nextRankVal += wksp->rankStats[w] << (w+rescale);
1191
0
                rankVal0[w] = curr;
1192
0
        }   }
1193
0
        {   U32 const minBits = tableLog+1 - maxW;
1194
0
            U32 consumed;
1195
0
            for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) {
1196
0
                U32* const rankValPtr = wksp->rankVal[consumed];
1197
0
                U32 w;
1198
0
                for (w = 1; w < maxW+1; w++) {
1199
0
                    rankValPtr[w] = rankVal0[w] >> consumed;
1200
0
    }   }   }   }
1201
1202
0
    HUF_fillDTableX2(dt, maxTableLog,
1203
0
                   wksp->sortedSymbol,
1204
0
                   wksp->rankStart0, wksp->rankVal, maxW,
1205
0
                   tableLog+1);
1206
1207
0
    dtd.tableLog = (BYTE)maxTableLog;
1208
0
    dtd.tableType = 1;
1209
0
    ZSTD_memcpy(DTable, &dtd, sizeof(dtd));
1210
0
    return iSize;
1211
0
}
1212
1213
1214
FORCE_INLINE_TEMPLATE U32
1215
HUF_decodeSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog)
1216
0
{
1217
0
    size_t const val = BIT_lookBitsFast(DStream, dtLog);   /* note : dtLog >= 1 */
1218
0
    ZSTD_memcpy(op, &dt[val].sequence, 2);
1219
0
    BIT_skipBits(DStream, dt[val].nbBits);
1220
0
    return dt[val].length;
1221
0
}
1222
1223
FORCE_INLINE_TEMPLATE U32
1224
HUF_decodeLastSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog)
1225
0
{
1226
0
    size_t const val = BIT_lookBitsFast(DStream, dtLog);   /* note : dtLog >= 1 */
1227
0
    ZSTD_memcpy(op, &dt[val].sequence, 1);
1228
0
    if (dt[val].length==1) {
1229
0
        BIT_skipBits(DStream, dt[val].nbBits);
1230
0
    } else {
1231
0
        if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) {
1232
0
            BIT_skipBits(DStream, dt[val].nbBits);
1233
0
            if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8))
1234
                /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */
1235
0
                DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8);
1236
0
        }
1237
0
    }
1238
0
    return 1;
1239
0
}
1240
1241
#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \
1242
0
    ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog)
1243
1244
#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \
1245
0
    if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \
1246
0
        ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog)
1247
1248
#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \
1249
0
    if (MEM_64bits()) \
1250
0
        ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog)
1251
1252
HINT_INLINE size_t
1253
HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd,
1254
                const HUF_DEltX2* const dt, const U32 dtLog)
1255
0
{
1256
0
    BYTE* const pStart = p;
1257
1258
    /* up to 8 symbols at a time */
1259
0
    if ((size_t)(pEnd - p) >= sizeof(bitDPtr->bitContainer)) {
1260
0
        if (dtLog <= 11 && MEM_64bits()) {
1261
            /* up to 10 symbols at a time */
1262
0
            while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-9)) {
1263
0
                HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
1264
0
                HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
1265
0
                HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
1266
0
                HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
1267
0
                HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
1268
0
            }
1269
0
        } else {
1270
            /* up to 8 symbols at a time */
1271
0
            while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-(sizeof(bitDPtr->bitContainer)-1))) {
1272
0
                HUF_DECODE_SYMBOLX2_2(p, bitDPtr);
1273
0
                HUF_DECODE_SYMBOLX2_1(p, bitDPtr);
1274
0
                HUF_DECODE_SYMBOLX2_2(p, bitDPtr);
1275
0
                HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
1276
0
            }
1277
0
        }
1278
0
    } else {
1279
0
        BIT_reloadDStream(bitDPtr);
1280
0
    }
1281
1282
    /* closer to end : up to 2 symbols at a time */
1283
0
    if ((size_t)(pEnd - p) >= 2) {
1284
0
        while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd-2))
1285
0
            HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
1286
1287
0
        while (p <= pEnd-2)
1288
0
            HUF_DECODE_SYMBOLX2_0(p, bitDPtr);   /* no need to reload : reached the end of DStream */
1289
0
    }
1290
1291
0
    if (p < pEnd)
1292
0
        p += HUF_decodeLastSymbolX2(p, bitDPtr, dt, dtLog);
1293
1294
0
    return p-pStart;
1295
0
}
1296
1297
FORCE_INLINE_TEMPLATE size_t
1298
HUF_decompress1X2_usingDTable_internal_body(
1299
          void* dst,  size_t dstSize,
1300
    const void* cSrc, size_t cSrcSize,
1301
    const HUF_DTable* DTable)
1302
0
{
1303
0
    BIT_DStream_t bitD;
1304
1305
    /* Init */
1306
0
    CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) );
1307
1308
    /* decode */
1309
0
    {   BYTE* const ostart = (BYTE*) dst;
1310
0
        BYTE* const oend = ostart + dstSize;
1311
0
        const void* const dtPtr = DTable+1;   /* force compiler to not use strict-aliasing */
1312
0
        const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr;
1313
0
        DTableDesc const dtd = HUF_getDTableDesc(DTable);
1314
0
        HUF_decodeStreamX2(ostart, &bitD, oend, dt, dtd.tableLog);
1315
0
    }
1316
1317
    /* check */
1318
0
    if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected);
1319
1320
    /* decoded size */
1321
0
    return dstSize;
1322
0
}
1323
1324
/* HUF_decompress4X2_usingDTable_internal_body():
1325
 * Conditions:
1326
 * @dstSize >= 6
1327
 */
1328
FORCE_INLINE_TEMPLATE size_t
1329
HUF_decompress4X2_usingDTable_internal_body(
1330
          void* dst,  size_t dstSize,
1331
    const void* cSrc, size_t cSrcSize,
1332
    const HUF_DTable* DTable)
1333
0
{
1334
0
    if (cSrcSize < 10) return ERROR(corruption_detected);   /* strict minimum : jump table + 1 byte per stream */
1335
1336
0
    {   const BYTE* const istart = (const BYTE*) cSrc;
1337
0
        BYTE* const ostart = (BYTE*) dst;
1338
0
        BYTE* const oend = ostart + dstSize;
1339
0
        BYTE* const olimit = oend - (sizeof(size_t)-1);
1340
0
        const void* const dtPtr = DTable+1;
1341
0
        const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr;
1342
1343
        /* Init */
1344
0
        BIT_DStream_t bitD1;
1345
0
        BIT_DStream_t bitD2;
1346
0
        BIT_DStream_t bitD3;
1347
0
        BIT_DStream_t bitD4;
1348
0
        size_t const length1 = MEM_readLE16(istart);
1349
0
        size_t const length2 = MEM_readLE16(istart+2);
1350
0
        size_t const length3 = MEM_readLE16(istart+4);
1351
0
        size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6);
1352
0
        const BYTE* const istart1 = istart + 6;  /* jumpTable */
1353
0
        const BYTE* const istart2 = istart1 + length1;
1354
0
        const BYTE* const istart3 = istart2 + length2;
1355
0
        const BYTE* const istart4 = istart3 + length3;
1356
0
        size_t const segmentSize = (dstSize+3) / 4;
1357
0
        BYTE* const opStart2 = ostart + segmentSize;
1358
0
        BYTE* const opStart3 = opStart2 + segmentSize;
1359
0
        BYTE* const opStart4 = opStart3 + segmentSize;
1360
0
        BYTE* op1 = ostart;
1361
0
        BYTE* op2 = opStart2;
1362
0
        BYTE* op3 = opStart3;
1363
0
        BYTE* op4 = opStart4;
1364
0
        U32 endSignal = 1;
1365
0
        DTableDesc const dtd = HUF_getDTableDesc(DTable);
1366
0
        U32 const dtLog = dtd.tableLog;
1367
1368
0
        if (length4 > cSrcSize) return ERROR(corruption_detected);  /* overflow */
1369
0
        if (opStart4 > oend) return ERROR(corruption_detected);     /* overflow */
1370
0
        if (dstSize < 6) return ERROR(corruption_detected);         /* stream 4-split doesn't work */
1371
0
        CHECK_F( BIT_initDStream(&bitD1, istart1, length1) );
1372
0
        CHECK_F( BIT_initDStream(&bitD2, istart2, length2) );
1373
0
        CHECK_F( BIT_initDStream(&bitD3, istart3, length3) );
1374
0
        CHECK_F( BIT_initDStream(&bitD4, istart4, length4) );
1375
1376
        /* 16-32 symbols per loop (4-8 symbols per stream) */
1377
0
        if ((size_t)(oend - op4) >= sizeof(size_t)) {
1378
0
            for ( ; (endSignal) & (op4 < olimit); ) {
1379
0
#if defined(__clang__) && (defined(__x86_64__) || defined(__i386__))
1380
0
                HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
1381
0
                HUF_DECODE_SYMBOLX2_1(op1, &bitD1);
1382
0
                HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
1383
0
                HUF_DECODE_SYMBOLX2_0(op1, &bitD1);
1384
0
                HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
1385
0
                HUF_DECODE_SYMBOLX2_1(op2, &bitD2);
1386
0
                HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
1387
0
                HUF_DECODE_SYMBOLX2_0(op2, &bitD2);
1388
0
                endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished;
1389
0
                endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished;
1390
0
                HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
1391
0
                HUF_DECODE_SYMBOLX2_1(op3, &bitD3);
1392
0
                HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
1393
0
                HUF_DECODE_SYMBOLX2_0(op3, &bitD3);
1394
0
                HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
1395
0
                HUF_DECODE_SYMBOLX2_1(op4, &bitD4);
1396
0
                HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
1397
0
                HUF_DECODE_SYMBOLX2_0(op4, &bitD4);
1398
0
                endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished;
1399
0
                endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished;
1400
#else
1401
                HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
1402
                HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
1403
                HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
1404
                HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
1405
                HUF_DECODE_SYMBOLX2_1(op1, &bitD1);
1406
                HUF_DECODE_SYMBOLX2_1(op2, &bitD2);
1407
                HUF_DECODE_SYMBOLX2_1(op3, &bitD3);
1408
                HUF_DECODE_SYMBOLX2_1(op4, &bitD4);
1409
                HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
1410
                HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
1411
                HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
1412
                HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
1413
                HUF_DECODE_SYMBOLX2_0(op1, &bitD1);
1414
                HUF_DECODE_SYMBOLX2_0(op2, &bitD2);
1415
                HUF_DECODE_SYMBOLX2_0(op3, &bitD3);
1416
                HUF_DECODE_SYMBOLX2_0(op4, &bitD4);
1417
                endSignal = (U32)LIKELY((U32)
1418
                            (BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished)
1419
                        & (BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished)
1420
                        & (BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished)
1421
                        & (BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished));
1422
#endif
1423
0
            }
1424
0
        }
1425
1426
        /* check corruption */
1427
0
        if (op1 > opStart2) return ERROR(corruption_detected);
1428
0
        if (op2 > opStart3) return ERROR(corruption_detected);
1429
0
        if (op3 > opStart4) return ERROR(corruption_detected);
1430
        /* note : op4 already verified within main loop */
1431
1432
        /* finish bitStreams one by one */
1433
0
        HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog);
1434
0
        HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog);
1435
0
        HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog);
1436
0
        HUF_decodeStreamX2(op4, &bitD4, oend,     dt, dtLog);
1437
1438
        /* check */
1439
0
        { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
1440
0
          if (!endCheck) return ERROR(corruption_detected); }
1441
1442
        /* decoded size */
1443
0
        return dstSize;
1444
0
    }
1445
0
}
1446
1447
#if HUF_NEED_BMI2_FUNCTION
1448
static BMI2_TARGET_ATTRIBUTE
1449
size_t HUF_decompress4X2_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc,
1450
                    size_t cSrcSize, HUF_DTable const* DTable) {
1451
    return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable);
1452
}
1453
#endif
1454
1455
static
1456
size_t HUF_decompress4X2_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc,
1457
0
                    size_t cSrcSize, HUF_DTable const* DTable) {
1458
0
    return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable);
1459
0
}
1460
1461
#if ZSTD_ENABLE_ASM_X86_64_BMI2
1462
1463
HUF_ASM_DECL void HUF_decompress4X2_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN;
1464
1465
#endif
1466
1467
static HUF_FAST_BMI2_ATTRS
1468
void HUF_decompress4X2_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args)
1469
0
{
1470
0
    U64 bits[4];
1471
0
    BYTE const* ip[4];
1472
0
    BYTE* op[4];
1473
0
    BYTE* oend[4];
1474
0
    HUF_DEltX2 const* const dtable = (HUF_DEltX2 const*)args->dt;
1475
0
    BYTE const* const ilimit = args->ilimit;
1476
1477
    /* Copy the arguments to local registers. */
1478
0
    ZSTD_memcpy(&bits, &args->bits, sizeof(bits));
1479
0
    ZSTD_memcpy((void*)(&ip), &args->ip, sizeof(ip));
1480
0
    ZSTD_memcpy(&op, &args->op, sizeof(op));
1481
1482
0
    oend[0] = op[1];
1483
0
    oend[1] = op[2];
1484
0
    oend[2] = op[3];
1485
0
    oend[3] = args->oend;
1486
1487
0
    assert(MEM_isLittleEndian());
1488
0
    assert(!MEM_32bits());
1489
1490
0
    for (;;) {
1491
0
        BYTE* olimit;
1492
0
        int stream;
1493
0
        int symbol;
1494
1495
        /* Assert loop preconditions */
1496
#ifndef NDEBUG
1497
        for (stream = 0; stream < 4; ++stream) {
1498
            assert(op[stream] <= oend[stream]);
1499
            assert(ip[stream] >= ilimit);
1500
        }
1501
#endif
1502
        /* Compute olimit */
1503
0
        {
1504
            /* Each loop does 5 table lookups for each of the 4 streams.
1505
             * Each table lookup consumes up to 11 bits of input, and produces
1506
             * up to 2 bytes of output.
1507
             */
1508
            /* We can consume up to 7 bytes of input per iteration per stream.
1509
             * We also know that each input pointer is >= ip[0]. So we can run
1510
             * iters loops before running out of input.
1511
             */
1512
0
            size_t iters = (size_t)(ip[0] - ilimit) / 7;
1513
            /* Each iteration can produce up to 10 bytes of output per stream.
1514
             * Each output stream my advance at different rates. So take the
1515
             * minimum number of safe iterations among all the output streams.
1516
             */
1517
0
            for (stream = 0; stream < 4; ++stream) {
1518
0
                size_t const oiters = (size_t)(oend[stream] - op[stream]) / 10;
1519
0
                iters = MIN(iters, oiters);
1520
0
            }
1521
1522
            /* Each iteration produces at least 5 output symbols. So until
1523
             * op[3] crosses olimit, we know we haven't executed iters
1524
             * iterations yet. This saves us maintaining an iters counter,
1525
             * at the expense of computing the remaining # of iterations
1526
             * more frequently.
1527
             */
1528
0
            olimit = op[3] + (iters * 5);
1529
1530
            /* Exit the fast decoding loop if we are too close to the end. */
1531
0
            if (op[3] + 10 > olimit)
1532
0
                break;
1533
1534
            /* Exit the decoding loop if any input pointer has crossed the
1535
             * previous one. This indicates corruption, and a precondition
1536
             * to our loop is that ip[i] >= ip[0].
1537
             */
1538
0
            for (stream = 1; stream < 4; ++stream) {
1539
0
                if (ip[stream] < ip[stream - 1])
1540
0
                    goto _out;
1541
0
            }
1542
0
        }
1543
1544
#ifndef NDEBUG
1545
        for (stream = 1; stream < 4; ++stream) {
1546
            assert(ip[stream] >= ip[stream - 1]);
1547
        }
1548
#endif
1549
1550
0
        do {
1551
            /* Do 5 table lookups for each of the first 3 streams */
1552
0
            for (symbol = 0; symbol < 5; ++symbol) {
1553
0
                for (stream = 0; stream < 3; ++stream) {
1554
0
                    int const index = (int)(bits[stream] >> 53);
1555
0
                    HUF_DEltX2 const entry = dtable[index];
1556
0
                    MEM_write16(op[stream], entry.sequence);
1557
0
                    bits[stream] <<= (entry.nbBits);
1558
0
                    op[stream] += (entry.length);
1559
0
                }
1560
0
            }
1561
            /* Do 1 table lookup from the final stream */
1562
0
            {
1563
0
                int const index = (int)(bits[3] >> 53);
1564
0
                HUF_DEltX2 const entry = dtable[index];
1565
0
                MEM_write16(op[3], entry.sequence);
1566
0
                bits[3] <<= (entry.nbBits);
1567
0
                op[3] += (entry.length);
1568
0
            }
1569
            /* Do 4 table lookups from the final stream & reload bitstreams */
1570
0
            for (stream = 0; stream < 4; ++stream) {
1571
                /* Do a table lookup from the final stream.
1572
                 * This is interleaved with the reloading to reduce register
1573
                 * pressure. This shouldn't be necessary, but compilers can
1574
                 * struggle with codegen with high register pressure.
1575
                 */
1576
0
                {
1577
0
                    int const index = (int)(bits[3] >> 53);
1578
0
                    HUF_DEltX2 const entry = dtable[index];
1579
0
                    MEM_write16(op[3], entry.sequence);
1580
0
                    bits[3] <<= (entry.nbBits);
1581
0
                    op[3] += (entry.length);
1582
0
                }
1583
                /* Reload the bistreams. The final bitstream must be reloaded
1584
                 * after the 5th symbol was decoded.
1585
                 */
1586
0
                {
1587
0
                    int const ctz = ZSTD_countTrailingZeros64(bits[stream]);
1588
0
                    int const nbBits = ctz & 7;
1589
0
                    int const nbBytes = ctz >> 3;
1590
0
                    ip[stream] -= nbBytes;
1591
0
                    bits[stream] = MEM_read64(ip[stream]) | 1;
1592
0
                    bits[stream] <<= nbBits;
1593
0
                }
1594
0
            }
1595
0
        } while (op[3] < olimit);
1596
0
    }
1597
1598
0
_out:
1599
1600
    /* Save the final values of each of the state variables back to args. */
1601
0
    ZSTD_memcpy(&args->bits, &bits, sizeof(bits));
1602
0
    ZSTD_memcpy((void*)(&args->ip), &ip, sizeof(ip));
1603
0
    ZSTD_memcpy(&args->op, &op, sizeof(op));
1604
0
}
1605
1606
1607
static HUF_FAST_BMI2_ATTRS size_t
1608
HUF_decompress4X2_usingDTable_internal_fast(
1609
          void* dst,  size_t dstSize,
1610
    const void* cSrc, size_t cSrcSize,
1611
    const HUF_DTable* DTable,
1612
0
    HUF_DecompressFastLoopFn loopFn) {
1613
0
    void const* dt = DTable + 1;
1614
0
    const BYTE* const iend = (const BYTE*)cSrc + 6;
1615
0
    BYTE* const oend = (BYTE*)dst + dstSize;
1616
0
    HUF_DecompressFastArgs args;
1617
0
    {
1618
0
        size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable);
1619
0
        FORWARD_IF_ERROR(ret, "Failed to init asm args");
1620
0
        if (ret == 0)
1621
0
            return 0;
1622
0
    }
1623
1624
0
    assert(args.ip[0] >= args.ilimit);
1625
0
    loopFn(&args);
1626
1627
    /* note : op4 already verified within main loop */
1628
0
    assert(args.ip[0] >= iend);
1629
0
    assert(args.ip[1] >= iend);
1630
0
    assert(args.ip[2] >= iend);
1631
0
    assert(args.ip[3] >= iend);
1632
0
    assert(args.op[3] <= oend);
1633
0
    (void)iend;
1634
1635
    /* finish bitStreams one by one */
1636
0
    {
1637
0
        size_t const segmentSize = (dstSize+3) / 4;
1638
0
        BYTE* segmentEnd = (BYTE*)dst;
1639
0
        int i;
1640
0
        for (i = 0; i < 4; ++i) {
1641
0
            BIT_DStream_t bit;
1642
0
            if (segmentSize <= (size_t)(oend - segmentEnd))
1643
0
                segmentEnd += segmentSize;
1644
0
            else
1645
0
                segmentEnd = oend;
1646
0
            FORWARD_IF_ERROR(HUF_initRemainingDStream(&bit, &args, i, segmentEnd), "corruption");
1647
0
            args.op[i] += HUF_decodeStreamX2(args.op[i], &bit, segmentEnd, (HUF_DEltX2 const*)dt, HUF_DECODER_FAST_TABLELOG);
1648
0
            if (args.op[i] != segmentEnd)
1649
0
                return ERROR(corruption_detected);
1650
0
        }
1651
0
    }
1652
1653
    /* decoded size */
1654
0
    return dstSize;
1655
0
}
1656
1657
static size_t HUF_decompress4X2_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc,
1658
                    size_t cSrcSize, HUF_DTable const* DTable, int flags)
1659
0
{
1660
0
    HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X2_usingDTable_internal_default;
1661
0
    HUF_DecompressFastLoopFn loopFn = HUF_decompress4X2_usingDTable_internal_fast_c_loop;
1662
1663
#if DYNAMIC_BMI2
1664
    if (flags & HUF_flags_bmi2) {
1665
        fallbackFn = HUF_decompress4X2_usingDTable_internal_bmi2;
1666
# if ZSTD_ENABLE_ASM_X86_64_BMI2
1667
        if (!(flags & HUF_flags_disableAsm)) {
1668
            loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop;
1669
        }
1670
# endif
1671
    } else {
1672
        return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable);
1673
    }
1674
#endif
1675
1676
#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)
1677
    if (!(flags & HUF_flags_disableAsm)) {
1678
        loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop;
1679
    }
1680
#endif
1681
1682
0
    if (!(flags & HUF_flags_disableFast)) {
1683
0
        size_t const ret = HUF_decompress4X2_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn);
1684
0
        if (ret != 0)
1685
0
            return ret;
1686
0
    }
1687
0
    return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable);
1688
0
}
1689
1690
HUF_DGEN(HUF_decompress1X2_usingDTable_internal)
1691
1692
size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize,
1693
                                   const void* cSrc, size_t cSrcSize,
1694
                                   void* workSpace, size_t wkspSize, int flags)
1695
0
{
1696
0
    const BYTE* ip = (const BYTE*) cSrc;
1697
1698
0
    size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize,
1699
0
                                               workSpace, wkspSize, flags);
1700
0
    if (HUF_isError(hSize)) return hSize;
1701
0
    if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
1702
0
    ip += hSize; cSrcSize -= hSize;
1703
1704
0
    return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, flags);
1705
0
}
1706
1707
static size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
1708
                                   const void* cSrc, size_t cSrcSize,
1709
                                   void* workSpace, size_t wkspSize, int flags)
1710
0
{
1711
0
    const BYTE* ip = (const BYTE*) cSrc;
1712
1713
0
    size_t hSize = HUF_readDTableX2_wksp(dctx, cSrc, cSrcSize,
1714
0
                                         workSpace, wkspSize, flags);
1715
0
    if (HUF_isError(hSize)) return hSize;
1716
0
    if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
1717
0
    ip += hSize; cSrcSize -= hSize;
1718
1719
0
    return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags);
1720
0
}
1721
1722
#endif /* HUF_FORCE_DECOMPRESS_X1 */
1723
1724
1725
/* ***********************************/
1726
/* Universal decompression selectors */
1727
/* ***********************************/
1728
1729
1730
#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2)
1731
typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t;
1732
static const algo_time_t algoTime[16 /* Quantization */][2 /* single, double */] =
1733
{
1734
    /* single, double, quad */
1735
    {{0,0}, {1,1}},  /* Q==0 : impossible */
1736
    {{0,0}, {1,1}},  /* Q==1 : impossible */
1737
    {{ 150,216}, { 381,119}},   /* Q == 2 : 12-18% */
1738
    {{ 170,205}, { 514,112}},   /* Q == 3 : 18-25% */
1739
    {{ 177,199}, { 539,110}},   /* Q == 4 : 25-32% */
1740
    {{ 197,194}, { 644,107}},   /* Q == 5 : 32-38% */
1741
    {{ 221,192}, { 735,107}},   /* Q == 6 : 38-44% */
1742
    {{ 256,189}, { 881,106}},   /* Q == 7 : 44-50% */
1743
    {{ 359,188}, {1167,109}},   /* Q == 8 : 50-56% */
1744
    {{ 582,187}, {1570,114}},   /* Q == 9 : 56-62% */
1745
    {{ 688,187}, {1712,122}},   /* Q ==10 : 62-69% */
1746
    {{ 825,186}, {1965,136}},   /* Q ==11 : 69-75% */
1747
    {{ 976,185}, {2131,150}},   /* Q ==12 : 75-81% */
1748
    {{1180,186}, {2070,175}},   /* Q ==13 : 81-87% */
1749
    {{1377,185}, {1731,202}},   /* Q ==14 : 87-93% */
1750
    {{1412,185}, {1695,202}},   /* Q ==15 : 93-99% */
1751
};
1752
#endif
1753
1754
/** HUF_selectDecoder() :
1755
 *  Tells which decoder is likely to decode faster,
1756
 *  based on a set of pre-computed metrics.
1757
 * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 .
1758
 *  Assumption : 0 < dstSize <= 128 KB */
1759
U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize)
1760
0
{
1761
0
    assert(dstSize > 0);
1762
0
    assert(dstSize <= 128*1024);
1763
#if defined(HUF_FORCE_DECOMPRESS_X1)
1764
    (void)dstSize;
1765
    (void)cSrcSize;
1766
    return 0;
1767
#elif defined(HUF_FORCE_DECOMPRESS_X2)
1768
    (void)dstSize;
1769
    (void)cSrcSize;
1770
    return 1;
1771
#else
1772
    /* decoder timing evaluation */
1773
0
    {   U32 const Q = (cSrcSize >= dstSize) ? 15 : (U32)(cSrcSize * 16 / dstSize);   /* Q < 16 */
1774
0
        U32 const D256 = (U32)(dstSize >> 8);
1775
0
        U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256);
1776
0
        U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256);
1777
0
        DTime1 += DTime1 >> 5;  /* small advantage to algorithm using less memory, to reduce cache eviction */
1778
0
        return DTime1 < DTime0;
1779
0
    }
1780
0
#endif
1781
0
}
1782
1783
size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
1784
                                  const void* cSrc, size_t cSrcSize,
1785
                                  void* workSpace, size_t wkspSize, int flags)
1786
0
{
1787
    /* validation checks */
1788
0
    if (dstSize == 0) return ERROR(dstSize_tooSmall);
1789
0
    if (cSrcSize > dstSize) return ERROR(corruption_detected);   /* invalid */
1790
0
    if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; }   /* not compressed */
1791
0
    if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; }   /* RLE */
1792
1793
0
    {   U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
1794
#if defined(HUF_FORCE_DECOMPRESS_X1)
1795
        (void)algoNb;
1796
        assert(algoNb == 0);
1797
        return HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc,
1798
                                cSrcSize, workSpace, wkspSize, flags);
1799
#elif defined(HUF_FORCE_DECOMPRESS_X2)
1800
        (void)algoNb;
1801
        assert(algoNb == 1);
1802
        return HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
1803
                                cSrcSize, workSpace, wkspSize, flags);
1804
#else
1805
0
        return algoNb ? HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
1806
0
                                cSrcSize, workSpace, wkspSize, flags):
1807
0
                        HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc,
1808
0
                                cSrcSize, workSpace, wkspSize, flags);
1809
0
#endif
1810
0
    }
1811
0
}
1812
1813
1814
size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags)
1815
0
{
1816
0
    DTableDesc const dtd = HUF_getDTableDesc(DTable);
1817
#if defined(HUF_FORCE_DECOMPRESS_X1)
1818
    (void)dtd;
1819
    assert(dtd.tableType == 0);
1820
    return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
1821
#elif defined(HUF_FORCE_DECOMPRESS_X2)
1822
    (void)dtd;
1823
    assert(dtd.tableType == 1);
1824
    return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
1825
#else
1826
0
    return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) :
1827
0
                           HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
1828
0
#endif
1829
0
}
1830
1831
#ifndef HUF_FORCE_DECOMPRESS_X2
1832
size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags)
1833
0
{
1834
0
    const BYTE* ip = (const BYTE*) cSrc;
1835
1836
0
    size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags);
1837
0
    if (HUF_isError(hSize)) return hSize;
1838
0
    if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
1839
0
    ip += hSize; cSrcSize -= hSize;
1840
1841
0
    return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags);
1842
0
}
1843
#endif
1844
1845
size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags)
1846
0
{
1847
0
    DTableDesc const dtd = HUF_getDTableDesc(DTable);
1848
#if defined(HUF_FORCE_DECOMPRESS_X1)
1849
    (void)dtd;
1850
    assert(dtd.tableType == 0);
1851
    return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
1852
#elif defined(HUF_FORCE_DECOMPRESS_X2)
1853
    (void)dtd;
1854
    assert(dtd.tableType == 1);
1855
    return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
1856
#else
1857
0
    return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) :
1858
0
                           HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags);
1859
0
#endif
1860
0
}
1861
1862
size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags)
1863
0
{
1864
    /* validation checks */
1865
0
    if (dstSize == 0) return ERROR(dstSize_tooSmall);
1866
0
    if (cSrcSize == 0) return ERROR(corruption_detected);
1867
1868
0
    {   U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
1869
#if defined(HUF_FORCE_DECOMPRESS_X1)
1870
        (void)algoNb;
1871
        assert(algoNb == 0);
1872
        return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags);
1873
#elif defined(HUF_FORCE_DECOMPRESS_X2)
1874
        (void)algoNb;
1875
        assert(algoNb == 1);
1876
        return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags);
1877
#else
1878
0
        return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags) :
1879
0
                        HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags);
1880
0
#endif
1881
0
    }
1882
0
}