/src/zlib-ng/inffast_tpl.h

Source
/* inffast.c -- fast decoding
 * Copyright (C) 1995-2017 Mark Adler
 * For conditions of distribution and use, see copyright notice in zlib.h
 */

#include "zbuild.h"
#include "zutil.h"
#include "inftrees.h"
#include "inflate.h"
#include "inflate_p.h"
#include "functable.h"

/*
   Decode literal, length, and distance codes and write out the resulting
   literal and match bytes until either not enough input or output is
   available, an end-of-block is encountered, or a data error is encountered.
   When large enough input and output buffers are supplied to inflate(), for
   example, a 16K input buffer and a 64K output buffer, more than 95% of the
   inflate execution time is spent in this routine.

   Entry assumptions:

        state->mode == LEN
        strm->avail_in >= INFLATE_FAST_MIN_HAVE
        strm->avail_out >= INFLATE_FAST_MIN_LEFT
        start >= strm->avail_out
        state->bits < 8

   On return, state->mode is one of:

        LEN -- ran out of enough output space or enough available input
        TYPE -- reached end of block code, inflate() to interpret next block
        BAD -- error in block data

   Notes:

    - The maximum input bits used by a length/distance pair is 15 bits for the
      length code, 5 bits for the length extra, 15 bits for the distance code,
      and 13 bits for the distance extra.  This totals 48 bits, or six bytes.
      Therefore if strm->avail_in >= 6, then there is enough input to avoid
      checking for available input while decoding.

    - On some architectures, it can be significantly faster (e.g. up to 1.2x
      faster on x86_64) to load from strm->next_in 64 bits, or 8 bytes, at a
      time, so INFLATE_FAST_MIN_HAVE == 8.

    - The maximum bytes that a single length/distance pair can output is 258
      bytes, which is the maximum length that can be coded.  inflate_fast()
      requires strm->avail_out >= 258 for each loop to avoid checking for
      output space.
 */
void Z_INTERNAL INFLATE_FAST(PREFIX3(stream) *strm, uint32_t start, int safe_mode) {
    /* start: inflate()'s starting value for strm->avail_out */
    struct inflate_state *state;
    z_const unsigned char *in;  /* local strm->next_in */
    const unsigned char *last;  /* have enough input while in < last */
    unsigned char *out;         /* local strm->next_out */
    unsigned char *beg;         /* inflate()'s initial strm->next_out */
    unsigned char *end;         /* while out < end, enough space available */
    unsigned char *safe;        /* can use chunkcopy provided out < safe */
    unsigned char *window;      /* allocated sliding window, if wsize != 0 */
    unsigned wsize;             /* window size or zero if not using window */
    unsigned whave;             /* valid bytes in the window */
    unsigned wnext;             /* window write index */

    /* hold is a local copy of strm->hold. By default, hold satisfies the same
       invariants that strm->hold does, namely that (hold >> bits) == 0. This
       invariant is kept by loading bits into hold one byte at a time, like:

       hold |= next_byte_of_input << bits; in++; bits += 8;

       If we need to ensure that bits >= 15 then this code snippet is simply
       repeated. Over one iteration of the outermost do/while loop, this
       happens up to six times (48 bits of input), as described in the NOTES
       above.

       However, on some little endian architectures, it can be significantly
       faster to load 64 bits once instead of 8 bits six times:

       if (bits <= 16) {
         hold |= next_8_bytes_of_input << bits; in += 6; bits += 48;
       }

       Unlike the simpler one byte load, shifting the next_8_bytes_of_input
       by bits will overflow and lose those high bits, up to 2 bytes' worth.
       The conservative estimate is therefore that we have read only 6 bytes
       (48 bits). Again, as per the NOTES above, 48 bits is sufficient for the
       rest of the iteration, and we will not need to load another 8 bytes.

       Inside this function, we no longer satisfy (hold >> bits) == 0, but
       this is not problematic, even if that overflow does not land on an 8 bit
       byte boundary. Those excess bits will eventually shift down lower as the
       Huffman decoder consumes input, and when new input bits need to be loaded
       into the bits variable, the same input bits will be or'ed over those
       existing bits. A bitwise or is idempotent: (a | b | b) equals (a | b).
       Note that we therefore write that load operation as "hold |= etc" and not
       "hold += etc".

       Outside that loop, at the end of the function, hold is bitwise and'ed
       with (1<<bits)-1 to drop those excess bits so that, on function exit, we
       keep the invariant that (state->hold >> state->bits) == 0.
    */
    bits_t bits;                /* local strm->bits */
    uint64_t hold;              /* local strm->hold */
    unsigned lmask;             /* mask for first level of length codes */
    unsigned dmask;             /* mask for first level of distance codes */
    code const *lcode;          /* local strm->lencode */
    code const *dcode;          /* local strm->distcode */
    code here;                  /* retrieved table entry */
    unsigned op;                /* code bits, operation, extra bits, or */
                                /*  window position, window bytes to copy */
    unsigned len;               /* match length, unused bytes */
    unsigned char *from;        /* where to copy match from */
    unsigned dist;              /* match distance */
    uint64_t old;               /* look-behind buffer for extra bits */

    /* copy state to local variables */
    state = (struct inflate_state *)strm->state;
    in = strm->next_in;
    last = in + (strm->avail_in - (INFLATE_FAST_MIN_HAVE - 1));
    out = strm->next_out;
    beg = out - (start - strm->avail_out);
    safe = out + strm->avail_out;
    end = safe - (safe_mode ? INFLATE_FAST_MIN_SAFE : INFLATE_FAST_MIN_LEFT) + 1;
    wsize = state->wsize;
    whave = state->whave;
    wnext = state->wnext;
    window = state->window;
    hold = state->hold;
    bits = (bits_t)state->bits;
    lcode = state->lencode;
    dcode = state->distcode;
    lmask = (1U << state->lenbits) - 1;
    dmask = (1U << state->distbits) - 1;

    /* decode literals and length/distances until end-of-block or not enough
       input data or output space */
    do {
        REFILL();
        here = lcode[hold & lmask];
        Z_TOUCH(here);
        old = hold;
        DROPBITS(here.bits);
        if (LIKELY(here.op == 0)) {
            TRACE_LITERAL(here.val);
            *out++ = (unsigned char)(here.val);
            here = lcode[hold & lmask];
            Z_TOUCH(here);
            old = hold;
            DROPBITS(here.bits);
            if (LIKELY(here.op == 0)) {
                TRACE_LITERAL(here.val);
                *out++ = (unsigned char)(here.val);
                here = lcode[hold & lmask];
                Z_TOUCH(here);
            dolen:
                old = hold;
                DROPBITS(here.bits);
                if (LIKELY(here.op == 0)) {
                    TRACE_LITERAL(here.val);
                    *out++ = (unsigned char)(here.val);
                    continue;
                }
            }
        }
        op = here.op;
        if (LIKELY(op & 16)) {                  /* length base */
            len = here.val + EXTRA_BITS(old, here, op);
            TRACE_LENGTH(len);
            here = dcode[hold & dmask];
            Z_TOUCH(here);
            if (UNLIKELY(bits < MAX_BITS + MAX_DIST_EXTRA_BITS)) {
                REFILL();
            }
          dodist:
            old = hold;
            DROPBITS(here.bits);
            op = here.op;
            if (LIKELY(op & 16)) {              /* distance base */
                dist = here.val + EXTRA_BITS(old, here, op);
#ifdef INFLATE_STRICT
                if (UNLIKELY(dist > state->dmax)) {
                    SET_BAD("invalid distance too far back");
                    break;
                }
#endif
                TRACE_DISTANCE(dist);

                /* In safe mode, if there isn't enough output space for the full copy,
                   bail to the slow path's MATCH state which handles partial copies. */
                if (UNLIKELY(safe_mode && len > (unsigned)(safe - out))) {
                    state->mode = MATCH;
                    state->length = len;
                    state->offset = dist;
                    break;
                }

                op = (unsigned)(out - beg);     /* max distance in output */
                if (UNLIKELY(dist > op)) {      /* see if copy from window */
                    op = dist - op;             /* distance back in window */
                    if (UNLIKELY(op > whave)) {
#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR
                        if (LIKELY(state->sane)) {
                            SET_BAD("invalid distance too far back");
                            break;
                        }
                        unsigned gap = op - whave;
                        unsigned zeros = MIN(len, gap);
                        memset(out, 0, zeros);  /* fill missing bytes with zeros */
                        out += zeros;
                        len -= zeros;
                        if (UNLIKELY(len == 0))
                            continue;
                        op = whave;
                        if (UNLIKELY(op == 0)) {/* copy from already-decoded output */
                            out = chunkcopy_safe(out, out - dist, len, safe);
                            continue;
                        }
#else
                        SET_BAD("invalid distance too far back");
                        break;
#endif
                    }
                    from = window;
                    if (LIKELY(wnext == 0)) {           /* very common case */
                        from += wsize - op;
                    } else if (LIKELY(wnext >= op)) {   /* contiguous in window */
                        from += wnext - op;
                    } else {                            /* wrap around window */
                        op -= wnext;
                        from += wsize - op;
                        if (UNLIKELY(op < len)) {       /* some from end of window */
                            len -= op;
                            out = CHUNKCOPY_SAFE(out, from, op, safe);
                            from = window;              /* more from start of window */
                            op = wnext;
                            /* This (rare) case can create a situation where
                               the first chunkcopy below must be checked.
                             */
                        }
                    }
                    if (UNLIKELY(op < len)) {           /* still need some from output */
                        len -= op;
                        if (LIKELY(!safe_mode)) {
                            out = CHUNKCOPY_SAFE(out, from, op, safe);
                            out = CHUNKUNROLL(out, &dist, &len);
                            out = CHUNKCOPY_SAFE(out, out - dist, len, safe);
                        } else {
#ifdef HAVE_MASKED_READWRITE
                            out = CHUNKCOPY_SAFE(out, from, op, safe);
                            out = CHUNKCOPY_SAFE(out, out - dist, len, safe);
#else
                            out = chunkcopy_safe(out, from, op, safe);
                            out = chunkcopy_safe(out, out - dist, len, safe);
#endif
                        }
                    } else {
#ifdef HAVE_MASKED_READWRITE
                        out = CHUNKCOPY_SAFE(out, from, len, safe);
#else
                        if (LIKELY(!safe_mode))
                            out = CHUNKCOPY_SAFE(out, from, len, safe);
                        else
                            out = chunkcopy_safe(out, from, len, safe);
#endif
                    }
                } else if (LIKELY(!safe_mode)) {
                    /* Whole reference is in range of current output.  No range checks are
                       necessary because we start with room for at least 258 bytes of output,
                       so unroll and roundoff operations can write beyond `out+len` so long
                       as they stay within 258 bytes of `out`.
                    */
                    if (LIKELY(dist >= len || dist >= CHUNKSIZE()))
                        out = CHUNKCOPY(out, out - dist, len);
                    else
                        out = CHUNKMEMSET(out, out - dist, len);
                } else {
#ifdef HAVE_MASKED_READWRITE
                    out = CHUNKCOPY_SAFE(out, out - dist, len, safe);
#else
                    out = chunkcopy_safe(out, out - dist, len, safe);
#endif
                }
            } else if (UNLIKELY((op & 64) == 0)) {          /* 2nd level distance code */
                here = dcode[here.val + BITS(op)];
                Z_TOUCH(here);
                goto dodist;
            } else {
                SET_BAD("invalid distance code");
                break;
            }
        } else if (UNLIKELY((op & 64) == 0)) {              /* 2nd level length code */
            here = lcode[here.val + BITS(op)];
            Z_TOUCH(here);
            goto dolen;
        } else if (UNLIKELY(op & 32)) {                     /* end-of-block */
            TRACE_END_OF_BLOCK();
            state->mode = TYPE;
            break;
        } else {
            SET_BAD("invalid literal/length code");
            break;
        }
    } while (in < last && out < end);

    /* return unused bytes (on entry, bits < 8, so in won't go too far back) */
    len = bits >> 3;
    in -= len;
    bits -= (bits_t)(len << 3);
    hold &= (UINT64_C(1) << bits) - 1;

    /* update state and return */
    strm->next_in = in;
    strm->next_out = out;
    strm->avail_in = (unsigned)(in < last ? (INFLATE_FAST_MIN_HAVE - 1) + (last - in)
                                          : (INFLATE_FAST_MIN_HAVE - 1) - (in - last));
    strm->avail_out = (unsigned)(safe - out);

    Assert(bits <= 32, "Remaining bits greater than 32");
    state->hold = (uint32_t)hold;
    state->bits = bits;
    return;
}

/*
   inflate_fast() speedups that turned out slower (on a PowerPC G3 750CXe):
   - Using bit fields for code structure
   - Different op definition to avoid & for extra bits (do & for table bits)
   - Three separate decoding do-loops for direct, window, and wnext == 0
   - Special case for distance > 1 copies to do overlapped load and store copy
   - Explicit branch predictions (based on measured branch probabilities)
   - Deferring match copy and interspersed it with decoding subsequent codes
   - Swapping literal/length else
   - Swapping window/direct else
   - Larger unrolled copy loops (three is about right)
   - Moving len -= 3 statement into middle of loop
 */

 // Cleanup
#undef CHUNKCOPY
#undef CHUNKMEMSET
#undef CHUNKMEMSET_SAFE
#undef CHUNKSIZE
#undef CHUNKUNROLL
#undef HAVE_CHUNKCOPY
#undef HAVE_CHUNKMEMSET_2
#undef HAVE_CHUNKMEMSET_4
#undef HAVE_CHUNKMEMSET_8
#undef HAVE_CHUNKMEMSET_16
#undef HAVE_CHUNK_MAG
#undef HAVE_HALFCHUNKCOPY
#undef HAVE_HALF_CHUNK
#undef HAVE_MASKED_READWRITE
#undef INFLATE_FAST

Coverage Report

Created: 2026-05-30 06:45

Line	Count	Source
1		/* inffast.c -- fast decoding
2		* Copyright (C) 1995-2017 Mark Adler
3		* For conditions of distribution and use, see copyright notice in zlib.h
4		*/
5
6		#include "zbuild.h"
7		#include "zutil.h"
8		#include "inftrees.h"
9		#include "inflate.h"
10		#include "inflate_p.h"
11		#include "functable.h"
12
13		/*
14		Decode literal, length, and distance codes and write out the resulting
15		literal and match bytes until either not enough input or output is
16		available, an end-of-block is encountered, or a data error is encountered.
17		When large enough input and output buffers are supplied to inflate(), for
18		example, a 16K input buffer and a 64K output buffer, more than 95% of the
19		inflate execution time is spent in this routine.
20
21		Entry assumptions:
22
23		state->mode == LEN
24		strm->avail_in >= INFLATE_FAST_MIN_HAVE
25		strm->avail_out >= INFLATE_FAST_MIN_LEFT
26		start >= strm->avail_out
27		state->bits < 8
28
29		On return, state->mode is one of:
30
31		LEN -- ran out of enough output space or enough available input
32		TYPE -- reached end of block code, inflate() to interpret next block
33		BAD -- error in block data
34
35		Notes:
36
37		- The maximum input bits used by a length/distance pair is 15 bits for the
38		length code, 5 bits for the length extra, 15 bits for the distance code,
39		and 13 bits for the distance extra. This totals 48 bits, or six bytes.
40		Therefore if strm->avail_in >= 6, then there is enough input to avoid
41		checking for available input while decoding.
42
43		- On some architectures, it can be significantly faster (e.g. up to 1.2x
44		faster on x86_64) to load from strm->next_in 64 bits, or 8 bytes, at a
45		time, so INFLATE_FAST_MIN_HAVE == 8.
46
47		- The maximum bytes that a single length/distance pair can output is 258
48		bytes, which is the maximum length that can be coded. inflate_fast()
49		requires strm->avail_out >= 258 for each loop to avoid checking for
50		output space.
51		*/
52	0	void Z_INTERNAL INFLATE_FAST(PREFIX3(stream) *strm, uint32_t start, int safe_mode) {
53		/* start: inflate()'s starting value for strm->avail_out */
54	0	struct inflate_state *state;
55	0	z_const unsigned char in; / local strm->next_in */
56	0	const unsigned char last; / have enough input while in < last */
57	0	unsigned char out; / local strm->next_out */
58	0	unsigned char beg; / inflate()'s initial strm->next_out */
59	0	unsigned char end; / while out < end, enough space available */
60	0	unsigned char safe; / can use chunkcopy provided out < safe */
61	0	unsigned char window; / allocated sliding window, if wsize != 0 */
62	0	unsigned wsize; /* window size or zero if not using window */
63	0	unsigned whave; /* valid bytes in the window */
64	0	unsigned wnext; /* window write index */
65
66		/* hold is a local copy of strm->hold. By default, hold satisfies the same
67		invariants that strm->hold does, namely that (hold >> bits) == 0. This
68		invariant is kept by loading bits into hold one byte at a time, like:
69
70		hold \|= next_byte_of_input << bits; in++; bits += 8;
71
72		If we need to ensure that bits >= 15 then this code snippet is simply
73		repeated. Over one iteration of the outermost do/while loop, this
74		happens up to six times (48 bits of input), as described in the NOTES
75		above.
76
77		However, on some little endian architectures, it can be significantly
78		faster to load 64 bits once instead of 8 bits six times:
79
80		if (bits <= 16) {
81		hold \|= next_8_bytes_of_input << bits; in += 6; bits += 48;
82		}
83
84		Unlike the simpler one byte load, shifting the next_8_bytes_of_input
85		by bits will overflow and lose those high bits, up to 2 bytes' worth.
86		The conservative estimate is therefore that we have read only 6 bytes
87		(48 bits). Again, as per the NOTES above, 48 bits is sufficient for the
88		rest of the iteration, and we will not need to load another 8 bytes.
89
90		Inside this function, we no longer satisfy (hold >> bits) == 0, but
91		this is not problematic, even if that overflow does not land on an 8 bit
92		byte boundary. Those excess bits will eventually shift down lower as the
93		Huffman decoder consumes input, and when new input bits need to be loaded
94		into the bits variable, the same input bits will be or'ed over those
95		existing bits. A bitwise or is idempotent: (a \| b \| b) equals (a \| b).
96		Note that we therefore write that load operation as "hold \|= etc" and not
97		"hold += etc".
98
99		Outside that loop, at the end of the function, hold is bitwise and'ed
100		with (1<<bits)-1 to drop those excess bits so that, on function exit, we
101		keep the invariant that (state->hold >> state->bits) == 0.
102		*/
103	0	bits_t bits; /* local strm->bits */
104	0	uint64_t hold; /* local strm->hold */
105	0	unsigned lmask; /* mask for first level of length codes */
106	0	unsigned dmask; /* mask for first level of distance codes */
107	0	code const lcode; / local strm->lencode */
108	0	code const dcode; / local strm->distcode */
109	0	code here; /* retrieved table entry */
110	0	unsigned op; /* code bits, operation, extra bits, or */
111		/* window position, window bytes to copy */
112	0	unsigned len; /* match length, unused bytes */
113	0	unsigned char from; / where to copy match from */
114	0	unsigned dist; /* match distance */
115	0	uint64_t old; /* look-behind buffer for extra bits */
116
117		/* copy state to local variables */
118	0	state = (struct inflate_state *)strm->state;
119	0	in = strm->next_in;
120	0	last = in + (strm->avail_in - (INFLATE_FAST_MIN_HAVE - 1));
121	0	out = strm->next_out;
122	0	beg = out - (start - strm->avail_out);
123	0	safe = out + strm->avail_out;
124	0	end = safe - (safe_mode ? INFLATE_FAST_MIN_SAFE : INFLATE_FAST_MIN_LEFT) + 1;
125	0	wsize = state->wsize;
126	0	whave = state->whave;
127	0	wnext = state->wnext;
128	0	window = state->window;
129	0	hold = state->hold;
130	0	bits = (bits_t)state->bits;
131	0	lcode = state->lencode;
132	0	dcode = state->distcode;
133	0	lmask = (1U << state->lenbits) - 1;
134	0	dmask = (1U << state->distbits) - 1;
135
136		/* decode literals and length/distances until end-of-block or not enough
137		input data or output space */
138	0	do {
139	0	REFILL();
140	0	here = lcode[hold & lmask];
141	0	Z_TOUCH(here);
142	0	old = hold;
143	0	DROPBITS(here.bits);
144	0	if (LIKELY(here.op == 0)) {
145	0	TRACE_LITERAL(here.val);
146	0	*out++ = (unsigned char)(here.val);
147	0	here = lcode[hold & lmask];
148	0	Z_TOUCH(here);
149	0	old = hold;
150	0	DROPBITS(here.bits);
151	0	if (LIKELY(here.op == 0)) {
152	0	TRACE_LITERAL(here.val);
153	0	*out++ = (unsigned char)(here.val);
154	0	here = lcode[hold & lmask];
155	0	Z_TOUCH(here);
156	0	dolen:
157	0	old = hold;
158	0	DROPBITS(here.bits);
159	0	if (LIKELY(here.op == 0)) {
160	0	TRACE_LITERAL(here.val);
161	0	*out++ = (unsigned char)(here.val);
162	0	continue;
163	0	}
164	0	}
165	0	}
166	0	op = here.op;
167	0	if (LIKELY(op & 16)) { /* length base */
168	0	len = here.val + EXTRA_BITS(old, here, op);
169	0	TRACE_LENGTH(len);
170	0	here = dcode[hold & dmask];
171	0	Z_TOUCH(here);
172	0	if (UNLIKELY(bits < MAX_BITS + MAX_DIST_EXTRA_BITS)) {
173	0	REFILL();
174	0	}
175	0	dodist:
176	0	old = hold;
177	0	DROPBITS(here.bits);
178	0	op = here.op;
179	0	if (LIKELY(op & 16)) { /* distance base */
180	0	dist = here.val + EXTRA_BITS(old, here, op);
181		#ifdef INFLATE_STRICT
182		if (UNLIKELY(dist > state->dmax)) {
183		SET_BAD("invalid distance too far back");
184		break;
185		}
186		#endif
187	0	TRACE_DISTANCE(dist);
188
189		/* In safe mode, if there isn't enough output space for the full copy,
190		bail to the slow path's MATCH state which handles partial copies. */
191	0	if (UNLIKELY(safe_mode && len > (unsigned)(safe - out))) {
192	0	state->mode = MATCH;
193	0	state->length = len;
194	0	state->offset = dist;
195	0	break;
196	0	}
197
198	0	op = (unsigned)(out - beg); /* max distance in output */
199	0	if (UNLIKELY(dist > op)) { /* see if copy from window */
200	0	op = dist - op; /* distance back in window */
201	0	if (UNLIKELY(op > whave)) {
202		#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR
203		if (LIKELY(state->sane)) {
204		SET_BAD("invalid distance too far back");
205		break;
206		}
207		unsigned gap = op - whave;
208		unsigned zeros = MIN(len, gap);
209		memset(out, 0, zeros); /* fill missing bytes with zeros */
210		out += zeros;
211		len -= zeros;
212		if (UNLIKELY(len == 0))
213		continue;
214		op = whave;
215		if (UNLIKELY(op == 0)) {/* copy from already-decoded output */
216		out = chunkcopy_safe(out, out - dist, len, safe);
217		continue;
218		}
219		#else
220	0	SET_BAD("invalid distance too far back");
221	0	break;
222	0	#endif
223	0	}
224	0	from = window;
225	0	if (LIKELY(wnext == 0)) { /* very common case */
226	0	from += wsize - op;
227	0	} else if (LIKELY(wnext >= op)) { /* contiguous in window */
228	0	from += wnext - op;
229	0	} else { /* wrap around window */
230	0	op -= wnext;
231	0	from += wsize - op;
232	0	if (UNLIKELY(op < len)) { /* some from end of window */
233	0	len -= op;
234	0	out = CHUNKCOPY_SAFE(out, from, op, safe);
235	0	from = window; /* more from start of window */
236	0	op = wnext;
237		/* This (rare) case can create a situation where
238		the first chunkcopy below must be checked.
239		*/
240	0	}
241	0	}
242	0	if (UNLIKELY(op < len)) { /* still need some from output */
243	0	len -= op;
244	0	if (LIKELY(!safe_mode)) {
245	0	out = CHUNKCOPY_SAFE(out, from, op, safe);
246	0	out = CHUNKUNROLL(out, &dist, &len);
247	0	out = CHUNKCOPY_SAFE(out, out - dist, len, safe);
248	0	} else {
249		#ifdef HAVE_MASKED_READWRITE
250		out = CHUNKCOPY_SAFE(out, from, op, safe);
251		out = CHUNKCOPY_SAFE(out, out - dist, len, safe);
252		#else
253		out = chunkcopy_safe(out, from, op, safe);
254		out = chunkcopy_safe(out, out - dist, len, safe);
255		#endif
256	0	}
257	0	} else {
258		#ifdef HAVE_MASKED_READWRITE
259		out = CHUNKCOPY_SAFE(out, from, len, safe);
260		#else
261	0	if (LIKELY(!safe_mode))
262	0	out = CHUNKCOPY_SAFE(out, from, len, safe);
263	0	else
264	0	out = chunkcopy_safe(out, from, len, safe);
265		#endif
266	0	}
267	0	} else if (LIKELY(!safe_mode)) {
268		/* Whole reference is in range of current output. No range checks are
269		necessary because we start with room for at least 258 bytes of output,
270		so unroll and roundoff operations can write beyond `out+len` so long
271		as they stay within 258 bytes of `out`.
272		*/
273	0	if (LIKELY(dist >= len \|\| dist >= CHUNKSIZE()))
274	0	out = CHUNKCOPY(out, out - dist, len);
275	0	else
276	0	out = CHUNKMEMSET(out, out - dist, len);
277	0	} else {
278		#ifdef HAVE_MASKED_READWRITE
279		out = CHUNKCOPY_SAFE(out, out - dist, len, safe);
280		#else
281		out = chunkcopy_safe(out, out - dist, len, safe);
282		#endif
283	0	}
284	0	} else if (UNLIKELY((op & 64) == 0)) { /* 2nd level distance code */
285	0	here = dcode[here.val + BITS(op)];
286	0	Z_TOUCH(here);
287	0	goto dodist;
288	0	} else {
289	0	SET_BAD("invalid distance code");
290	0	break;
291	0	}
292	0	} else if (UNLIKELY((op & 64) == 0)) { /* 2nd level length code */
293	0	here = lcode[here.val + BITS(op)];
294	0	Z_TOUCH(here);
295	0	goto dolen;
296	0	} else if (UNLIKELY(op & 32)) { /* end-of-block */
297	0	TRACE_END_OF_BLOCK();
298	0	state->mode = TYPE;
299	0	break;
300	0	} else {
301	0	SET_BAD("invalid literal/length code");
302	0	break;
303	0	}
304	0	} while (in < last && out < end);
305
306		/* return unused bytes (on entry, bits < 8, so in won't go too far back) */
307	0	len = bits >> 3;
308	0	in -= len;
309	0	bits -= (bits_t)(len << 3);
310	0	hold &= (UINT64_C(1) << bits) - 1;
311
312		/* update state and return */
313	0	strm->next_in = in;
314	0	strm->next_out = out;
315	0	strm->avail_in = (unsigned)(in < last ? (INFLATE_FAST_MIN_HAVE - 1) + (last - in)
316	0	: (INFLATE_FAST_MIN_HAVE - 1) - (in - last));
317	0	strm->avail_out = (unsigned)(safe - out);
318
319	0	Assert(bits <= 32, "Remaining bits greater than 32");
320	0	state->hold = (uint32_t)hold;
321	0	state->bits = bits;
322	0	return;
323	0	} Unexecuted instantiation: inflate_fast_sse2 Unexecuted instantiation: inflate_fast_ssse3 Unexecuted instantiation: inflate_fast_avx2 Unexecuted instantiation: inflate_fast_avx512
324
325		/*
326		inflate_fast() speedups that turned out slower (on a PowerPC G3 750CXe):
327		- Using bit fields for code structure
328		- Different op definition to avoid & for extra bits (do & for table bits)
329		- Three separate decoding do-loops for direct, window, and wnext == 0
330		- Special case for distance > 1 copies to do overlapped load and store copy
331		- Explicit branch predictions (based on measured branch probabilities)
332		- Deferring match copy and interspersed it with decoding subsequent codes
333		- Swapping literal/length else
334		- Swapping window/direct else
335		- Larger unrolled copy loops (three is about right)
336		- Moving len -= 3 statement into middle of loop
337		*/
338
339		// Cleanup
340		#undef CHUNKCOPY
341		#undef CHUNKMEMSET
342		#undef CHUNKMEMSET_SAFE
343		#undef CHUNKSIZE
344		#undef CHUNKUNROLL
345		#undef HAVE_CHUNKCOPY
346		#undef HAVE_CHUNKMEMSET_2
347		#undef HAVE_CHUNKMEMSET_4
348		#undef HAVE_CHUNKMEMSET_8
349		#undef HAVE_CHUNKMEMSET_16
350		#undef HAVE_CHUNK_MAG
351		#undef HAVE_HALFCHUNKCOPY
352		#undef HAVE_HALF_CHUNK
353		#undef HAVE_MASKED_READWRITE
354		#undef INFLATE_FAST