Coverage Report

Created: 2024-07-27 06:19

/src/c-blosc2/internal-complibs/zlib-ng-2.0.7/inffast.c
Line
Count
Source (jump to first uncovered line)
1
/* inffast.c -- fast decoding
2
 * Copyright (C) 1995-2017 Mark Adler
3
 * For conditions of distribution and use, see copyright notice in zlib.h
4
 */
5
6
#include "zbuild.h"
7
#include "zutil.h"
8
#include "inftrees.h"
9
#include "inflate.h"
10
#include "inffast.h"
11
#include "inflate_p.h"
12
#include "functable.h"
13
14
15
/* Load 64 bits from IN and place the bytes at offset BITS in the result. */
16
0
static inline uint64_t load_64_bits(const unsigned char *in, unsigned bits) {
17
0
    uint64_t chunk;
18
0
    memcpy(&chunk, in, sizeof(chunk));
19
20
0
#if BYTE_ORDER == LITTLE_ENDIAN
21
0
    return chunk << bits;
22
#else
23
    return ZSWAP64(chunk) << bits;
24
#endif
25
0
}
26
/*
27
   Decode literal, length, and distance codes and write out the resulting
28
   literal and match bytes until either not enough input or output is
29
   available, an end-of-block is encountered, or a data error is encountered.
30
   When large enough input and output buffers are supplied to inflate(), for
31
   example, a 16K input buffer and a 64K output buffer, more than 95% of the
32
   inflate execution time is spent in this routine.
33
34
   Entry assumptions:
35
36
        state->mode == LEN
37
        strm->avail_in >= INFLATE_FAST_MIN_HAVE
38
        strm->avail_out >= INFLATE_FAST_MIN_LEFT
39
        start >= strm->avail_out
40
        state->bits < 8
41
42
   On return, state->mode is one of:
43
44
        LEN -- ran out of enough output space or enough available input
45
        TYPE -- reached end of block code, inflate() to interpret next block
46
        BAD -- error in block data
47
48
   Notes:
49
50
    - The maximum input bits used by a length/distance pair is 15 bits for the
51
      length code, 5 bits for the length extra, 15 bits for the distance code,
52
      and 13 bits for the distance extra.  This totals 48 bits, or six bytes.
53
      Therefore if strm->avail_in >= 6, then there is enough input to avoid
54
      checking for available input while decoding.
55
56
    - On some architectures, it can be significantly faster (e.g. up to 1.2x
57
      faster on x86_64) to load from strm->next_in 64 bits, or 8 bytes, at a
58
      time, so INFLATE_FAST_MIN_HAVE == 8.
59
60
    - The maximum bytes that a single length/distance pair can output is 258
61
      bytes, which is the maximum length that can be coded.  inflate_fast()
62
      requires strm->avail_out >= 258 for each loop to avoid checking for
63
      output space.
64
 */
65
0
void Z_INTERNAL zng_inflate_fast(PREFIX3(stream) *strm, unsigned long start) {
66
    /* start: inflate()'s starting value for strm->avail_out */
67
0
    struct inflate_state *state;
68
0
    z_const unsigned char *in;  /* local strm->next_in */
69
0
    const unsigned char *last;  /* have enough input while in < last */
70
0
    unsigned char *out;         /* local strm->next_out */
71
0
    unsigned char *beg;         /* inflate()'s initial strm->next_out */
72
0
    unsigned char *end;         /* while out < end, enough space available */
73
0
    unsigned char *safe;        /* can use chunkcopy provided out < safe */
74
#ifdef INFLATE_STRICT
75
    unsigned dmax;              /* maximum distance from zlib header */
76
#endif
77
0
    unsigned wsize;             /* window size or zero if not using window */
78
0
    unsigned whave;             /* valid bytes in the window */
79
0
    unsigned wnext;             /* window write index */
80
0
    unsigned char *window;      /* allocated sliding window, if wsize != 0 */
81
82
    /* hold is a local copy of strm->hold. By default, hold satisfies the same
83
       invariants that strm->hold does, namely that (hold >> bits) == 0. This
84
       invariant is kept by loading bits into hold one byte at a time, like:
85
86
       hold |= next_byte_of_input << bits; in++; bits += 8;
87
88
       If we need to ensure that bits >= 15 then this code snippet is simply
89
       repeated. Over one iteration of the outermost do/while loop, this
90
       happens up to six times (48 bits of input), as described in the NOTES
91
       above.
92
93
       However, on some little endian architectures, it can be significantly
94
       faster to load 64 bits once instead of 8 bits six times:
95
96
       if (bits <= 16) {
97
         hold |= next_8_bytes_of_input << bits; in += 6; bits += 48;
98
       }
99
100
       Unlike the simpler one byte load, shifting the next_8_bytes_of_input
101
       by bits will overflow and lose those high bits, up to 2 bytes' worth.
102
       The conservative estimate is therefore that we have read only 6 bytes
103
       (48 bits). Again, as per the NOTES above, 48 bits is sufficient for the
104
       rest of the iteration, and we will not need to load another 8 bytes.
105
106
       Inside this function, we no longer satisfy (hold >> bits) == 0, but
107
       this is not problematic, even if that overflow does not land on an 8 bit
108
       byte boundary. Those excess bits will eventually shift down lower as the
109
       Huffman decoder consumes input, and when new input bits need to be loaded
110
       into the bits variable, the same input bits will be or'ed over those
111
       existing bits. A bitwise or is idempotent: (a | b | b) equals (a | b).
112
       Note that we therefore write that load operation as "hold |= etc" and not
113
       "hold += etc".
114
115
       Outside that loop, at the end of the function, hold is bitwise and'ed
116
       with (1<<bits)-1 to drop those excess bits so that, on function exit, we
117
       keep the invariant that (state->hold >> state->bits) == 0.
118
    */
119
0
    uint64_t hold;              /* local strm->hold */
120
0
    unsigned bits;              /* local strm->bits */
121
0
    code const *lcode;          /* local strm->lencode */
122
0
    code const *dcode;          /* local strm->distcode */
123
0
    unsigned lmask;             /* mask for first level of length codes */
124
0
    unsigned dmask;             /* mask for first level of distance codes */
125
0
    const code *here;           /* retrieved table entry */
126
0
    unsigned op;                /* code bits, operation, extra bits, or */
127
                                /*  window position, window bytes to copy */
128
0
    unsigned len;               /* match length, unused bytes */
129
0
    unsigned dist;              /* match distance */
130
0
    unsigned char *from;        /* where to copy match from */
131
0
    unsigned extra_safe;        /* copy chunks safely in all cases */
132
133
    /* copy state to local variables */
134
0
    state = (struct inflate_state *)strm->state;
135
0
    in = strm->next_in;
136
0
    last = in + (strm->avail_in - (INFLATE_FAST_MIN_HAVE - 1));
137
0
    out = strm->next_out;
138
0
    beg = out - (start - strm->avail_out);
139
0
    end = out + (strm->avail_out - (INFLATE_FAST_MIN_LEFT - 1));
140
0
    safe = out + strm->avail_out;
141
#ifdef INFLATE_STRICT
142
    dmax = state->dmax;
143
#endif
144
0
    wsize = state->wsize;
145
0
    whave = state->whave;
146
0
    wnext = state->wnext;
147
0
    window = state->window;
148
0
    hold = state->hold;
149
0
    bits = state->bits;
150
0
    lcode = state->lencode;
151
0
    dcode = state->distcode;
152
0
    lmask = (1U << state->lenbits) - 1;
153
0
    dmask = (1U << state->distbits) - 1;
154
155
    /* Detect if out and window point to the same memory allocation. In this instance it is
156
       necessary to use safe chunk copy functions to prevent overwriting the window. If the
157
       window is overwritten then future matches with far distances will fail to copy correctly. */
158
0
    extra_safe = (wsize != 0 && out >= window && out + INFLATE_FAST_MIN_LEFT <= window + wsize);
159
160
    /* decode literals and length/distances until end-of-block or not enough
161
       input data or output space */
162
0
    do {
163
0
        if (bits < 15) {
164
0
            hold |= load_64_bits(in, bits);
165
0
            in += 6;
166
0
            bits += 48;
167
0
        }
168
0
        here = lcode + (hold & lmask);
169
0
      dolen:
170
0
        DROPBITS(here->bits);
171
0
        op = here->op;
172
0
        if (op == 0) {                          /* literal */
173
0
            Tracevv((stderr, here->val >= 0x20 && here->val < 0x7f ?
174
0
                    "inflate:         literal '%c'\n" :
175
0
                    "inflate:         literal 0x%02x\n", here->val));
176
0
            *out++ = (unsigned char)(here->val);
177
0
        } else if (op & 16) {                     /* length base */
178
0
            len = here->val;
179
0
            op &= 15;                           /* number of extra bits */
180
0
            if (bits < op) {
181
0
                hold |= load_64_bits(in, bits);
182
0
                in += 6;
183
0
                bits += 48;
184
0
            }
185
0
            len += BITS(op);
186
0
            DROPBITS(op);
187
0
            Tracevv((stderr, "inflate:         length %u\n", len));
188
0
            if (bits < 15) {
189
0
                hold |= load_64_bits(in, bits);
190
0
                in += 6;
191
0
                bits += 48;
192
0
            }
193
0
            here = dcode + (hold & dmask);
194
0
          dodist:
195
0
            DROPBITS(here->bits);
196
0
            op = here->op;
197
0
            if (op & 16) {                      /* distance base */
198
0
                dist = here->val;
199
0
                op &= 15;                       /* number of extra bits */
200
0
                if (bits < op) {
201
0
                    hold |= load_64_bits(in, bits);
202
0
                    in += 6;
203
0
                    bits += 48;
204
0
                }
205
0
                dist += BITS(op);
206
#ifdef INFLATE_STRICT
207
                if (dist > dmax) {
208
                    SET_BAD("invalid distance too far back");
209
                    break;
210
                }
211
#endif
212
0
                DROPBITS(op);
213
0
                Tracevv((stderr, "inflate:         distance %u\n", dist));
214
0
                op = (unsigned)(out - beg);     /* max distance in output */
215
0
                if (dist > op) {                /* see if copy from window */
216
0
                    op = dist - op;             /* distance back in window */
217
0
                    if (op > whave) {
218
0
                        if (state->sane) {
219
0
                            SET_BAD("invalid distance too far back");
220
0
                            break;
221
0
                        }
222
#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR
223
                        if (len <= op - whave) {
224
                            do {
225
                                *out++ = 0;
226
                            } while (--len);
227
                            continue;
228
                        }
229
                        len -= op - whave;
230
                        do {
231
                            *out++ = 0;
232
                        } while (--op > whave);
233
                        if (op == 0) {
234
                            from = out - dist;
235
                            do {
236
                                *out++ = *from++;
237
                            } while (--len);
238
                            continue;
239
                        }
240
#endif
241
0
                    }
242
0
                    from = window;
243
0
                    if (wnext == 0) {           /* very common case */
244
0
                        from += wsize - op;
245
0
                    } else if (wnext >= op) {   /* contiguous in window */
246
0
                        from += wnext - op;
247
0
                    } else {                    /* wrap around window */
248
0
                        op -= wnext;
249
0
                        from += wsize - op;
250
0
                        if (op < len) {         /* some from end of window */
251
0
                            len -= op;
252
0
                            out = functable.chunkcopy_safe(out, from, op, safe);
253
0
                            from = window;      /* more from start of window */
254
0
                            op = wnext;
255
                            /* This (rare) case can create a situation where
256
                               the first chunkcopy below must be checked.
257
                             */
258
0
                        }
259
0
                    }
260
0
                    if (op < len) {             /* still need some from output */
261
0
                        len -= op;
262
0
                        out = functable.chunkcopy_safe(out, from, op, safe);
263
0
                        out = functable.chunkunroll(out, &dist, &len);
264
0
                        out = functable.chunkcopy_safe(out, out - dist, len, safe);
265
0
                    } else {
266
0
                        out = functable.chunkcopy_safe(out, from, len, safe);
267
0
                    }
268
0
                } else if (extra_safe) {
269
                    /* Whole reference is in range of current output. */
270
0
                    if (dist >= len || dist >= state->chunksize)
271
0
                        out = functable.chunkcopy_safe(out, out - dist, len, safe);
272
0
                    else
273
0
                        out = functable.chunkmemset_safe(out, dist, len, (unsigned)((safe - out) + 1));
274
0
                } else {
275
                    /* Whole reference is in range of current output.  No range checks are
276
                       necessary because we start with room for at least 258 bytes of output,
277
                       so unroll and roundoff operations can write beyond `out+len` so long
278
                       as they stay within 258 bytes of `out`.
279
                    */
280
0
                    if (dist >= len || dist >= state->chunksize)
281
0
                        out = functable.chunkcopy(out, out - dist, len);
282
0
                    else
283
0
                        out = functable.chunkmemset(out, dist, len);
284
0
                }
285
0
            } else if ((op & 64) == 0) {          /* 2nd level distance code */
286
0
                here = dcode + here->val + BITS(op);
287
0
                goto dodist;
288
0
            } else {
289
0
                SET_BAD("invalid distance code");
290
0
                break;
291
0
            }
292
0
        } else if ((op & 64) == 0) {              /* 2nd level length code */
293
0
            here = lcode + here->val + BITS(op);
294
0
            goto dolen;
295
0
        } else if (op & 32) {                     /* end-of-block */
296
0
            Tracevv((stderr, "inflate:         end of block\n"));
297
0
            state->mode = TYPE;
298
0
            break;
299
0
        } else {
300
0
            SET_BAD("invalid literal/length code");
301
0
            break;
302
0
        }
303
0
    } while (in < last && out < end);
304
305
    /* return unused bytes (on entry, bits < 8, so in won't go too far back) */
306
0
    len = bits >> 3;
307
0
    in -= len;
308
0
    bits -= len << 3;
309
0
    hold &= (UINT64_C(1) << bits) - 1;
310
311
    /* update state and return */
312
0
    strm->next_in = in;
313
0
    strm->next_out = out;
314
0
    strm->avail_in = (unsigned)(in < last ? (INFLATE_FAST_MIN_HAVE - 1) + (last - in)
315
0
                                          : (INFLATE_FAST_MIN_HAVE - 1) - (in - last));
316
0
    strm->avail_out = (unsigned)(out < end ? (INFLATE_FAST_MIN_LEFT - 1) + (end - out)
317
0
                                           : (INFLATE_FAST_MIN_LEFT - 1) - (out - end));
318
319
0
    Assert(bits <= 32, "Remaining bits greater than 32");
320
0
    state->hold = (uint32_t)hold;
321
0
    state->bits = bits;
322
0
    return;
323
0
}
324
325
/*
326
   inflate_fast() speedups that turned out slower (on a PowerPC G3 750CXe):
327
   - Using bit fields for code structure
328
   - Different op definition to avoid & for extra bits (do & for table bits)
329
   - Three separate decoding do-loops for direct, window, and wnext == 0
330
   - Special case for distance > 1 copies to do overlapped load and store copy
331
   - Explicit branch predictions (based on measured branch probabilities)
332
   - Deferring match copy and interspersed it with decoding subsequent codes
333
   - Swapping literal/length else
334
   - Swapping window/direct else
335
   - Larger unrolled copy loops (three is about right)
336
   - Moving len -= 3 statement into middle of loop
337
 */