Coverage Report

Created: 2026-11-19 07:17

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/c-blosc2/_deps/zlib_ng-src/inffast_tpl.h
Line
Count
Source
1
/* inffast.c -- fast decoding
2
 * Copyright (C) 1995-2017 Mark Adler
3
 * For conditions of distribution and use, see copyright notice in zlib.h
4
 */
5
6
#include "zbuild.h"
7
#include "zendian.h"
8
#include "zutil.h"
9
#include "inftrees.h"
10
#include "inflate.h"
11
#include "inflate_p.h"
12
#include "functable.h"
13
14
/*
15
   Decode literal, length, and distance codes and write out the resulting
16
   literal and match bytes until either not enough input or output is
17
   available, an end-of-block is encountered, or a data error is encountered.
18
   When large enough input and output buffers are supplied to inflate(), for
19
   example, a 16K input buffer and a 64K output buffer, more than 95% of the
20
   inflate execution time is spent in this routine.
21
22
   Entry assumptions:
23
24
        state->mode == LEN
25
        strm->avail_in >= INFLATE_FAST_MIN_HAVE
26
        strm->avail_out >= INFLATE_FAST_MIN_LEFT
27
        start >= strm->avail_out
28
        state->bits < 8
29
30
   On return, state->mode is one of:
31
32
        LEN -- ran out of enough output space or enough available input
33
        TYPE -- reached end of block code, inflate() to interpret next block
34
        BAD -- error in block data
35
36
   Notes:
37
38
    - The maximum input bits used by a length/distance pair is 15 bits for the
39
      length code, 5 bits for the length extra, 15 bits for the distance code,
40
      and 13 bits for the distance extra.  This totals 48 bits, or six bytes.
41
      Therefore if strm->avail_in >= 6, then there is enough input to avoid
42
      checking for available input while decoding.
43
44
    - On some architectures, it can be significantly faster (e.g. up to 1.2x
45
      faster on x86_64) to load from strm->next_in 64 bits, or 8 bytes, at a
46
      time, so INFLATE_FAST_MIN_HAVE == 8.
47
48
    - The maximum bytes that a single length/distance pair can output is 258
49
      bytes, which is the maximum length that can be coded.  inflate_fast()
50
      requires strm->avail_out >= 258 for each loop to avoid checking for
51
      output space.
52
 */
53
33.1k
void Z_INTERNAL INFLATE_FAST(PREFIX3(stream) *strm, uint32_t start) {
54
    /* start: inflate()'s starting value for strm->avail_out */
55
33.1k
    struct inflate_state *state;
56
33.1k
    z_const unsigned char *in;  /* local strm->next_in */
57
33.1k
    const unsigned char *last;  /* have enough input while in < last */
58
33.1k
    unsigned char *out;         /* local strm->next_out */
59
33.1k
    unsigned char *beg;         /* inflate()'s initial strm->next_out */
60
33.1k
    unsigned char *end;         /* while out < end, enough space available */
61
33.1k
    unsigned char *safe;        /* can use chunkcopy provided out < safe */
62
33.1k
    unsigned char *window;      /* allocated sliding window, if wsize != 0 */
63
33.1k
    unsigned wsize;             /* window size or zero if not using window */
64
33.1k
    unsigned whave;             /* valid bytes in the window */
65
33.1k
    unsigned wnext;             /* window write index */
66
67
    /* hold is a local copy of strm->hold. By default, hold satisfies the same
68
       invariants that strm->hold does, namely that (hold >> bits) == 0. This
69
       invariant is kept by loading bits into hold one byte at a time, like:
70
71
       hold |= next_byte_of_input << bits; in++; bits += 8;
72
73
       If we need to ensure that bits >= 15 then this code snippet is simply
74
       repeated. Over one iteration of the outermost do/while loop, this
75
       happens up to six times (48 bits of input), as described in the NOTES
76
       above.
77
78
       However, on some little endian architectures, it can be significantly
79
       faster to load 64 bits once instead of 8 bits six times:
80
81
       if (bits <= 16) {
82
         hold |= next_8_bytes_of_input << bits; in += 6; bits += 48;
83
       }
84
85
       Unlike the simpler one byte load, shifting the next_8_bytes_of_input
86
       by bits will overflow and lose those high bits, up to 2 bytes' worth.
87
       The conservative estimate is therefore that we have read only 6 bytes
88
       (48 bits). Again, as per the NOTES above, 48 bits is sufficient for the
89
       rest of the iteration, and we will not need to load another 8 bytes.
90
91
       Inside this function, we no longer satisfy (hold >> bits) == 0, but
92
       this is not problematic, even if that overflow does not land on an 8 bit
93
       byte boundary. Those excess bits will eventually shift down lower as the
94
       Huffman decoder consumes input, and when new input bits need to be loaded
95
       into the bits variable, the same input bits will be or'ed over those
96
       existing bits. A bitwise or is idempotent: (a | b | b) equals (a | b).
97
       Note that we therefore write that load operation as "hold |= etc" and not
98
       "hold += etc".
99
100
       Outside that loop, at the end of the function, hold is bitwise and'ed
101
       with (1<<bits)-1 to drop those excess bits so that, on function exit, we
102
       keep the invariant that (state->hold >> state->bits) == 0.
103
    */
104
33.1k
    unsigned bits;              /* local strm->bits */
105
33.1k
    uint64_t hold;              /* local strm->hold */
106
33.1k
    unsigned lmask;             /* mask for first level of length codes */
107
33.1k
    unsigned dmask;             /* mask for first level of distance codes */
108
33.1k
    code const *lcode;          /* local strm->lencode */
109
33.1k
    code const *dcode;          /* local strm->distcode */
110
33.1k
    const code *here;           /* retrieved table entry */
111
33.1k
    unsigned op;                /* code bits, operation, extra bits, or */
112
                                /*  window position, window bytes to copy */
113
33.1k
    unsigned len;               /* match length, unused bytes */
114
33.1k
    unsigned char *from;        /* where to copy match from */
115
33.1k
    unsigned dist;              /* match distance */
116
33.1k
    unsigned extra_safe;        /* copy chunks safely in all cases */
117
118
    /* copy state to local variables */
119
33.1k
    state = (struct inflate_state *)strm->state;
120
33.1k
    in = strm->next_in;
121
33.1k
    last = in + (strm->avail_in - (INFLATE_FAST_MIN_HAVE - 1));
122
33.1k
    out = strm->next_out;
123
33.1k
    beg = out - (start - strm->avail_out);
124
33.1k
    end = out + (strm->avail_out - (INFLATE_FAST_MIN_LEFT - 1));
125
33.1k
    safe = out + strm->avail_out;
126
33.1k
    wsize = state->wsize;
127
33.1k
    whave = state->whave;
128
33.1k
    wnext = state->wnext;
129
33.1k
    window = state->window;
130
33.1k
    hold = state->hold;
131
33.1k
    bits = state->bits;
132
33.1k
    lcode = state->lencode;
133
33.1k
    dcode = state->distcode;
134
33.1k
    lmask = (1U << state->lenbits) - 1;
135
33.1k
    dmask = (1U << state->distbits) - 1;
136
137
    /* Detect if out and window point to the same memory allocation. In this instance it is
138
       necessary to use safe chunk copy functions to prevent overwriting the window. If the
139
       window is overwritten then future matches with far distances will fail to copy correctly. */
140
33.1k
    extra_safe = (wsize != 0 && out >= window && out + INFLATE_FAST_MIN_LEFT <= window + state->wbufsize);
141
142
10.4M
#define REFILL() do { \
143
10.4M
        hold |= load_64_bits(in, bits); \
144
10.4M
        in += 7; \
145
10.4M
        in -= ((bits >> 3) & 7); \
146
10.4M
        bits |= 56; \
147
10.4M
    } while (0)
148
149
    /* decode literals and length/distances until end-of-block or not enough
150
       input data or output space */
151
10.4M
    do {
152
10.4M
        REFILL();
153
10.4M
        here = lcode + (hold & lmask);
154
10.4M
        if (here->op == 0) {
155
10.0M
            *out++ = (unsigned char)(here->val);
156
10.0M
            DROPBITS(here->bits);
157
10.0M
            here = lcode + (hold & lmask);
158
10.0M
            if (here->op == 0) {
159
9.93M
                *out++ = (unsigned char)(here->val);
160
9.93M
                DROPBITS(here->bits);
161
9.93M
                here = lcode + (hold & lmask);
162
9.93M
            }
163
10.0M
        }
164
10.6M
      dolen:
165
10.6M
        DROPBITS(here->bits);
166
10.6M
        op = here->op;
167
10.6M
        if (op == 0) {                          /* literal */
168
9.99M
            Tracevv((stderr, here->val >= 0x20 && here->val < 0x7f ?
169
9.99M
                    "inflate:         literal '%c'\n" :
170
9.99M
                    "inflate:         literal 0x%02x\n", here->val));
171
9.99M
            *out++ = (unsigned char)(here->val);
172
9.99M
        } else if (op & 16) {                     /* length base */
173
400k
            len = here->val;
174
400k
            op &= MAX_BITS;                       /* number of extra bits */
175
400k
            len += BITS(op);
176
400k
            DROPBITS(op);
177
400k
            Tracevv((stderr, "inflate:         length %u\n", len));
178
400k
            here = dcode + (hold & dmask);
179
400k
            if (bits < MAX_BITS + MAX_DIST_EXTRA_BITS) {
180
1.56k
                REFILL();
181
1.56k
            }
182
407k
          dodist:
183
407k
            DROPBITS(here->bits);
184
407k
            op = here->op;
185
407k
            if (op & 16) {                      /* distance base */
186
400k
                dist = here->val;
187
400k
                op &= MAX_BITS;                 /* number of extra bits */
188
400k
                dist += BITS(op);
189
#ifdef INFLATE_STRICT
190
                if (dist > state->dmax) {
191
                    SET_BAD("invalid distance too far back");
192
                    break;
193
                }
194
#endif
195
400k
                DROPBITS(op);
196
400k
                Tracevv((stderr, "inflate:         distance %u\n", dist));
197
400k
                op = (unsigned)(out - beg);     /* max distance in output */
198
400k
                if (dist > op) {                /* see if copy from window */
199
99
                    op = dist - op;             /* distance back in window */
200
99
                    if (op > whave) {
201
#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR
202
                        if (state->sane) {
203
                            SET_BAD("invalid distance too far back");
204
                            break;
205
                        }
206
                        if (len <= op - whave) {
207
                            do {
208
                                *out++ = 0;
209
                            } while (--len);
210
                            continue;
211
                        }
212
                        len -= op - whave;
213
                        do {
214
                            *out++ = 0;
215
                        } while (--op > whave);
216
                        if (op == 0) {
217
                            from = out - dist;
218
                            do {
219
                                *out++ = *from++;
220
                            } while (--len);
221
                            continue;
222
                        }
223
#else
224
99
                        SET_BAD("invalid distance too far back");
225
99
                        break;
226
99
#endif
227
99
                    }
228
0
                    from = window;
229
0
                    if (wnext == 0) {           /* very common case */
230
0
                        from += wsize - op;
231
0
                    } else if (wnext >= op) {   /* contiguous in window */
232
0
                        from += wnext - op;
233
0
                    } else {                    /* wrap around window */
234
0
                        op -= wnext;
235
0
                        from += wsize - op;
236
0
                        if (op < len) {         /* some from end of window */
237
0
                            len -= op;
238
0
                            out = CHUNKCOPY_SAFE(out, from, op, safe);
239
0
                            from = window;      /* more from start of window */
240
0
                            op = wnext;
241
                            /* This (rare) case can create a situation where
242
                               the first chunkcopy below must be checked.
243
                             */
244
0
                        }
245
0
                    }
246
0
                    if (op < len) {             /* still need some from output */
247
0
                        len -= op;
248
0
                        if (!extra_safe) {
249
0
                            out = CHUNKCOPY_SAFE(out, from, op, safe);
250
0
                            out = CHUNKUNROLL(out, &dist, &len);
251
0
                            out = CHUNKCOPY_SAFE(out, out - dist, len, safe);
252
0
                        } else {
253
0
                            out = chunkcopy_safe(out, from, op, safe);
254
0
                            out = chunkcopy_safe(out, out - dist, len, safe);
255
0
                        }
256
0
                    } else {
257
#ifndef HAVE_MASKED_READWRITE
258
0
                        if (extra_safe)
259
0
                            out = chunkcopy_safe(out, from, len, safe);
260
0
                        else
261
0
#endif
262
0
                            out = CHUNKCOPY_SAFE(out, from, len, safe);
263
0
                    }
264
#ifndef HAVE_MASKED_READWRITE
265
400k
                } else if (extra_safe) {
266
                    /* Whole reference is in range of current output. */
267
                        out = chunkcopy_safe(out, out - dist, len, safe);
268
#endif
269
400k
                } else {
270
                    /* Whole reference is in range of current output.  No range checks are
271
                       necessary because we start with room for at least 258 bytes of output,
272
                       so unroll and roundoff operations can write beyond `out+len` so long
273
                       as they stay within 258 bytes of `out`.
274
                    */
275
400k
                    if (dist >= len || dist >= CHUNKSIZE())
276
177k
                        out = CHUNKCOPY(out, out - dist, len);
277
222k
                    else
278
222k
                        out = CHUNKMEMSET(out, out - dist, len);
279
400k
                }
280
400k
            } else if ((op & 64) == 0) {          /* 2nd level distance code */
281
6.53k
                here = dcode + here->val + BITS(op);
282
6.53k
                goto dodist;
283
6.53k
            } else {
284
53
                SET_BAD("invalid distance code");
285
53
                break;
286
53
            }
287
407k
        } else if ((op & 64) == 0) {              /* 2nd level length code */
288
187k
            here = lcode + here->val + BITS(op);
289
187k
            goto dolen;
290
187k
        } else if (op & 32) {                     /* end-of-block */
291
23.0k
            Tracevv((stderr, "inflate:         end of block\n"));
292
23.0k
            state->mode = TYPE;
293
23.0k
            break;
294
23.0k
        } else {
295
40
            SET_BAD("invalid literal/length code");
296
40
            break;
297
40
        }
298
10.6M
    } while (in < last && out < end);
299
300
    /* return unused bytes (on entry, bits < 8, so in won't go too far back) */
301
33.1k
    len = bits >> 3;
302
33.1k
    in -= len;
303
33.1k
    bits -= len << 3;
304
33.1k
    hold &= (UINT64_C(1) << bits) - 1;
305
306
    /* update state and return */
307
33.1k
    strm->next_in = in;
308
33.1k
    strm->next_out = out;
309
33.1k
    strm->avail_in = (unsigned)(in < last ? (INFLATE_FAST_MIN_HAVE - 1) + (last - in)
310
33.1k
                                          : (INFLATE_FAST_MIN_HAVE - 1) - (in - last));
311
33.1k
    strm->avail_out = (unsigned)(out < end ? (INFLATE_FAST_MIN_LEFT - 1) + (end - out)
312
33.1k
                                           : (INFLATE_FAST_MIN_LEFT - 1) - (out - end));
313
314
33.1k
    Assert(bits <= 32, "Remaining bits greater than 32");
315
33.1k
    state->hold = (uint32_t)hold;
316
33.1k
    state->bits = bits;
317
33.1k
    return;
318
33.1k
}
Unexecuted instantiation: inflate_fast_sse2
Unexecuted instantiation: inflate_fast_ssse3
inflate_fast_avx2
Line
Count
Source
53
33.1k
void Z_INTERNAL INFLATE_FAST(PREFIX3(stream) *strm, uint32_t start) {
54
    /* start: inflate()'s starting value for strm->avail_out */
55
33.1k
    struct inflate_state *state;
56
33.1k
    z_const unsigned char *in;  /* local strm->next_in */
57
33.1k
    const unsigned char *last;  /* have enough input while in < last */
58
33.1k
    unsigned char *out;         /* local strm->next_out */
59
33.1k
    unsigned char *beg;         /* inflate()'s initial strm->next_out */
60
33.1k
    unsigned char *end;         /* while out < end, enough space available */
61
33.1k
    unsigned char *safe;        /* can use chunkcopy provided out < safe */
62
33.1k
    unsigned char *window;      /* allocated sliding window, if wsize != 0 */
63
33.1k
    unsigned wsize;             /* window size or zero if not using window */
64
33.1k
    unsigned whave;             /* valid bytes in the window */
65
33.1k
    unsigned wnext;             /* window write index */
66
67
    /* hold is a local copy of strm->hold. By default, hold satisfies the same
68
       invariants that strm->hold does, namely that (hold >> bits) == 0. This
69
       invariant is kept by loading bits into hold one byte at a time, like:
70
71
       hold |= next_byte_of_input << bits; in++; bits += 8;
72
73
       If we need to ensure that bits >= 15 then this code snippet is simply
74
       repeated. Over one iteration of the outermost do/while loop, this
75
       happens up to six times (48 bits of input), as described in the NOTES
76
       above.
77
78
       However, on some little endian architectures, it can be significantly
79
       faster to load 64 bits once instead of 8 bits six times:
80
81
       if (bits <= 16) {
82
         hold |= next_8_bytes_of_input << bits; in += 6; bits += 48;
83
       }
84
85
       Unlike the simpler one byte load, shifting the next_8_bytes_of_input
86
       by bits will overflow and lose those high bits, up to 2 bytes' worth.
87
       The conservative estimate is therefore that we have read only 6 bytes
88
       (48 bits). Again, as per the NOTES above, 48 bits is sufficient for the
89
       rest of the iteration, and we will not need to load another 8 bytes.
90
91
       Inside this function, we no longer satisfy (hold >> bits) == 0, but
92
       this is not problematic, even if that overflow does not land on an 8 bit
93
       byte boundary. Those excess bits will eventually shift down lower as the
94
       Huffman decoder consumes input, and when new input bits need to be loaded
95
       into the bits variable, the same input bits will be or'ed over those
96
       existing bits. A bitwise or is idempotent: (a | b | b) equals (a | b).
97
       Note that we therefore write that load operation as "hold |= etc" and not
98
       "hold += etc".
99
100
       Outside that loop, at the end of the function, hold is bitwise and'ed
101
       with (1<<bits)-1 to drop those excess bits so that, on function exit, we
102
       keep the invariant that (state->hold >> state->bits) == 0.
103
    */
104
33.1k
    unsigned bits;              /* local strm->bits */
105
33.1k
    uint64_t hold;              /* local strm->hold */
106
33.1k
    unsigned lmask;             /* mask for first level of length codes */
107
33.1k
    unsigned dmask;             /* mask for first level of distance codes */
108
33.1k
    code const *lcode;          /* local strm->lencode */
109
33.1k
    code const *dcode;          /* local strm->distcode */
110
33.1k
    const code *here;           /* retrieved table entry */
111
33.1k
    unsigned op;                /* code bits, operation, extra bits, or */
112
                                /*  window position, window bytes to copy */
113
33.1k
    unsigned len;               /* match length, unused bytes */
114
33.1k
    unsigned char *from;        /* where to copy match from */
115
33.1k
    unsigned dist;              /* match distance */
116
33.1k
    unsigned extra_safe;        /* copy chunks safely in all cases */
117
118
    /* copy state to local variables */
119
33.1k
    state = (struct inflate_state *)strm->state;
120
33.1k
    in = strm->next_in;
121
33.1k
    last = in + (strm->avail_in - (INFLATE_FAST_MIN_HAVE - 1));
122
33.1k
    out = strm->next_out;
123
33.1k
    beg = out - (start - strm->avail_out);
124
33.1k
    end = out + (strm->avail_out - (INFLATE_FAST_MIN_LEFT - 1));
125
33.1k
    safe = out + strm->avail_out;
126
33.1k
    wsize = state->wsize;
127
33.1k
    whave = state->whave;
128
33.1k
    wnext = state->wnext;
129
33.1k
    window = state->window;
130
33.1k
    hold = state->hold;
131
33.1k
    bits = state->bits;
132
33.1k
    lcode = state->lencode;
133
33.1k
    dcode = state->distcode;
134
33.1k
    lmask = (1U << state->lenbits) - 1;
135
33.1k
    dmask = (1U << state->distbits) - 1;
136
137
    /* Detect if out and window point to the same memory allocation. In this instance it is
138
       necessary to use safe chunk copy functions to prevent overwriting the window. If the
139
       window is overwritten then future matches with far distances will fail to copy correctly. */
140
33.1k
    extra_safe = (wsize != 0 && out >= window && out + INFLATE_FAST_MIN_LEFT <= window + state->wbufsize);
141
142
33.1k
#define REFILL() do { \
143
33.1k
        hold |= load_64_bits(in, bits); \
144
33.1k
        in += 7; \
145
33.1k
        in -= ((bits >> 3) & 7); \
146
33.1k
        bits |= 56; \
147
33.1k
    } while (0)
148
149
    /* decode literals and length/distances until end-of-block or not enough
150
       input data or output space */
151
10.4M
    do {
152
10.4M
        REFILL();
153
10.4M
        here = lcode + (hold & lmask);
154
10.4M
        if (here->op == 0) {
155
10.0M
            *out++ = (unsigned char)(here->val);
156
10.0M
            DROPBITS(here->bits);
157
10.0M
            here = lcode + (hold & lmask);
158
10.0M
            if (here->op == 0) {
159
9.93M
                *out++ = (unsigned char)(here->val);
160
9.93M
                DROPBITS(here->bits);
161
9.93M
                here = lcode + (hold & lmask);
162
9.93M
            }
163
10.0M
        }
164
10.6M
      dolen:
165
10.6M
        DROPBITS(here->bits);
166
10.6M
        op = here->op;
167
10.6M
        if (op == 0) {                          /* literal */
168
9.99M
            Tracevv((stderr, here->val >= 0x20 && here->val < 0x7f ?
169
9.99M
                    "inflate:         literal '%c'\n" :
170
9.99M
                    "inflate:         literal 0x%02x\n", here->val));
171
9.99M
            *out++ = (unsigned char)(here->val);
172
9.99M
        } else if (op & 16) {                     /* length base */
173
400k
            len = here->val;
174
400k
            op &= MAX_BITS;                       /* number of extra bits */
175
400k
            len += BITS(op);
176
400k
            DROPBITS(op);
177
400k
            Tracevv((stderr, "inflate:         length %u\n", len));
178
400k
            here = dcode + (hold & dmask);
179
400k
            if (bits < MAX_BITS + MAX_DIST_EXTRA_BITS) {
180
1.56k
                REFILL();
181
1.56k
            }
182
407k
          dodist:
183
407k
            DROPBITS(here->bits);
184
407k
            op = here->op;
185
407k
            if (op & 16) {                      /* distance base */
186
400k
                dist = here->val;
187
400k
                op &= MAX_BITS;                 /* number of extra bits */
188
400k
                dist += BITS(op);
189
#ifdef INFLATE_STRICT
190
                if (dist > state->dmax) {
191
                    SET_BAD("invalid distance too far back");
192
                    break;
193
                }
194
#endif
195
400k
                DROPBITS(op);
196
400k
                Tracevv((stderr, "inflate:         distance %u\n", dist));
197
400k
                op = (unsigned)(out - beg);     /* max distance in output */
198
400k
                if (dist > op) {                /* see if copy from window */
199
99
                    op = dist - op;             /* distance back in window */
200
99
                    if (op > whave) {
201
#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR
202
                        if (state->sane) {
203
                            SET_BAD("invalid distance too far back");
204
                            break;
205
                        }
206
                        if (len <= op - whave) {
207
                            do {
208
                                *out++ = 0;
209
                            } while (--len);
210
                            continue;
211
                        }
212
                        len -= op - whave;
213
                        do {
214
                            *out++ = 0;
215
                        } while (--op > whave);
216
                        if (op == 0) {
217
                            from = out - dist;
218
                            do {
219
                                *out++ = *from++;
220
                            } while (--len);
221
                            continue;
222
                        }
223
#else
224
99
                        SET_BAD("invalid distance too far back");
225
99
                        break;
226
99
#endif
227
99
                    }
228
0
                    from = window;
229
0
                    if (wnext == 0) {           /* very common case */
230
0
                        from += wsize - op;
231
0
                    } else if (wnext >= op) {   /* contiguous in window */
232
0
                        from += wnext - op;
233
0
                    } else {                    /* wrap around window */
234
0
                        op -= wnext;
235
0
                        from += wsize - op;
236
0
                        if (op < len) {         /* some from end of window */
237
0
                            len -= op;
238
0
                            out = CHUNKCOPY_SAFE(out, from, op, safe);
239
0
                            from = window;      /* more from start of window */
240
0
                            op = wnext;
241
                            /* This (rare) case can create a situation where
242
                               the first chunkcopy below must be checked.
243
                             */
244
0
                        }
245
0
                    }
246
0
                    if (op < len) {             /* still need some from output */
247
0
                        len -= op;
248
0
                        if (!extra_safe) {
249
0
                            out = CHUNKCOPY_SAFE(out, from, op, safe);
250
0
                            out = CHUNKUNROLL(out, &dist, &len);
251
0
                            out = CHUNKCOPY_SAFE(out, out - dist, len, safe);
252
0
                        } else {
253
0
                            out = chunkcopy_safe(out, from, op, safe);
254
0
                            out = chunkcopy_safe(out, out - dist, len, safe);
255
0
                        }
256
0
                    } else {
257
0
#ifndef HAVE_MASKED_READWRITE
258
0
                        if (extra_safe)
259
0
                            out = chunkcopy_safe(out, from, len, safe);
260
0
                        else
261
0
#endif
262
0
                            out = CHUNKCOPY_SAFE(out, from, len, safe);
263
0
                    }
264
0
#ifndef HAVE_MASKED_READWRITE
265
400k
                } else if (extra_safe) {
266
                    /* Whole reference is in range of current output. */
267
0
                        out = chunkcopy_safe(out, out - dist, len, safe);
268
0
#endif
269
400k
                } else {
270
                    /* Whole reference is in range of current output.  No range checks are
271
                       necessary because we start with room for at least 258 bytes of output,
272
                       so unroll and roundoff operations can write beyond `out+len` so long
273
                       as they stay within 258 bytes of `out`.
274
                    */
275
400k
                    if (dist >= len || dist >= CHUNKSIZE())
276
177k
                        out = CHUNKCOPY(out, out - dist, len);
277
222k
                    else
278
222k
                        out = CHUNKMEMSET(out, out - dist, len);
279
400k
                }
280
400k
            } else if ((op & 64) == 0) {          /* 2nd level distance code */
281
6.53k
                here = dcode + here->val + BITS(op);
282
6.53k
                goto dodist;
283
6.53k
            } else {
284
53
                SET_BAD("invalid distance code");
285
53
                break;
286
53
            }
287
407k
        } else if ((op & 64) == 0) {              /* 2nd level length code */
288
187k
            here = lcode + here->val + BITS(op);
289
187k
            goto dolen;
290
187k
        } else if (op & 32) {                     /* end-of-block */
291
23.0k
            Tracevv((stderr, "inflate:         end of block\n"));
292
23.0k
            state->mode = TYPE;
293
23.0k
            break;
294
23.0k
        } else {
295
40
            SET_BAD("invalid literal/length code");
296
40
            break;
297
40
        }
298
10.6M
    } while (in < last && out < end);
299
300
    /* return unused bytes (on entry, bits < 8, so in won't go too far back) */
301
33.1k
    len = bits >> 3;
302
33.1k
    in -= len;
303
33.1k
    bits -= len << 3;
304
33.1k
    hold &= (UINT64_C(1) << bits) - 1;
305
306
    /* update state and return */
307
33.1k
    strm->next_in = in;
308
33.1k
    strm->next_out = out;
309
33.1k
    strm->avail_in = (unsigned)(in < last ? (INFLATE_FAST_MIN_HAVE - 1) + (last - in)
310
33.1k
                                          : (INFLATE_FAST_MIN_HAVE - 1) - (in - last));
311
33.1k
    strm->avail_out = (unsigned)(out < end ? (INFLATE_FAST_MIN_LEFT - 1) + (end - out)
312
33.1k
                                           : (INFLATE_FAST_MIN_LEFT - 1) - (out - end));
313
314
33.1k
    Assert(bits <= 32, "Remaining bits greater than 32");
315
33.1k
    state->hold = (uint32_t)hold;
316
33.1k
    state->bits = bits;
317
33.1k
    return;
318
33.1k
}
Unexecuted instantiation: inflate_fast_avx512
319
320
/*
321
   inflate_fast() speedups that turned out slower (on a PowerPC G3 750CXe):
322
   - Using bit fields for code structure
323
   - Different op definition to avoid & for extra bits (do & for table bits)
324
   - Three separate decoding do-loops for direct, window, and wnext == 0
325
   - Special case for distance > 1 copies to do overlapped load and store copy
326
   - Explicit branch predictions (based on measured branch probabilities)
327
   - Deferring match copy and interspersed it with decoding subsequent codes
328
   - Swapping literal/length else
329
   - Swapping window/direct else
330
   - Larger unrolled copy loops (three is about right)
331
   - Moving len -= 3 statement into middle of loop
332
 */