/src/qtbase/src/network/access/qdecompresshelper.cpp
Line | Count | Source |
1 | | // Copyright (C) 2020 The Qt Company Ltd. |
2 | | // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only |
3 | | // Qt-Security score:significant reason:default |
4 | | |
5 | | #include "qdecompresshelper_p.h" |
6 | | |
7 | | #include <QtCore/qiodevice.h> |
8 | | #include <QtCore/qcoreapplication.h> |
9 | | |
10 | | #include <limits> |
11 | | #include <zlib.h> |
12 | | |
13 | | #if QT_CONFIG(brotli) |
14 | | # include <brotli/decode.h> |
15 | | #endif |
16 | | |
17 | | #if QT_CONFIG(zstd) |
18 | | # include <zstd.h> |
19 | | #endif |
20 | | |
21 | | #include <array> |
22 | | |
23 | | QT_BEGIN_NAMESPACE |
24 | | |
25 | | using namespace Qt::StringLiterals; |
26 | | |
27 | | namespace { |
28 | | struct ContentEncodingMapping |
29 | | { |
30 | | QByteArrayView name; |
31 | | QDecompressHelper::ContentEncoding encoding; |
32 | | }; |
33 | | |
34 | | constexpr ContentEncodingMapping contentEncodingMapping[] { |
35 | | #if QT_CONFIG(zstd) |
36 | | { "zstd", QDecompressHelper::Zstandard }, |
37 | | #endif |
38 | | #if QT_CONFIG(brotli) |
39 | | { "br", QDecompressHelper::Brotli }, |
40 | | #endif |
41 | | { "gzip", QDecompressHelper::GZip }, |
42 | | { "deflate", QDecompressHelper::Deflate }, |
43 | | }; |
44 | | |
45 | | QDecompressHelper::ContentEncoding encodingFromByteArray(QByteArrayView ce) noexcept |
46 | 0 | { |
47 | 0 | for (const auto &mapping : contentEncodingMapping) { |
48 | 0 | if (ce.compare(mapping.name, Qt::CaseInsensitive) == 0) |
49 | 0 | return mapping.encoding; |
50 | 0 | } |
51 | 0 | return QDecompressHelper::None; |
52 | 0 | } |
53 | | |
54 | | z_stream *toZlibPointer(void *ptr) |
55 | 0 | { |
56 | 0 | return static_cast<z_stream_s *>(ptr); |
57 | 0 | } |
58 | | |
59 | | #if QT_CONFIG(brotli) |
60 | | BrotliDecoderState *toBrotliPointer(void *ptr) |
61 | | { |
62 | | return static_cast<BrotliDecoderState *>(ptr); |
63 | | } |
64 | | #endif |
65 | | |
66 | | #if QT_CONFIG(zstd) |
67 | | ZSTD_DStream *toZstandardPointer(void *ptr) |
68 | | { |
69 | | return static_cast<ZSTD_DStream *>(ptr); |
70 | | } |
71 | | #endif |
72 | | } |
73 | | |
74 | | bool QDecompressHelper::isSupportedEncoding(QByteArrayView encoding) |
75 | 0 | { |
76 | 0 | return encodingFromByteArray(encoding) != QDecompressHelper::None; |
77 | 0 | } |
78 | | |
79 | | QByteArrayList QDecompressHelper::acceptedEncoding() |
80 | 0 | { |
81 | 0 | QByteArrayList list; |
82 | 0 | list.reserve(std::size(contentEncodingMapping)); |
83 | 0 | for (const auto &mapping : contentEncodingMapping) { |
84 | 0 | list << mapping.name.toByteArray(); |
85 | 0 | } |
86 | 0 | return list; |
87 | 0 | } |
88 | | |
89 | | QDecompressHelper::~QDecompressHelper() |
90 | 0 | { |
91 | 0 | clear(); |
92 | 0 | } |
93 | | |
94 | | bool QDecompressHelper::setEncoding(QByteArrayView encoding) |
95 | 0 | { |
96 | 0 | Q_ASSERT(contentEncoding == QDecompressHelper::None); |
97 | 0 | if (contentEncoding != QDecompressHelper::None) { |
98 | 0 | qWarning("Encoding is already set."); |
99 | | // This isn't an error, so it doesn't set errorStr, it's just wrong usage. |
100 | 0 | return false; |
101 | 0 | } |
102 | 0 | ContentEncoding ce = encodingFromByteArray(encoding); |
103 | 0 | if (ce == None) { |
104 | 0 | errorStr = QCoreApplication::translate("QHttp", "Unsupported content encoding: %1") |
105 | 0 | .arg(QLatin1String(encoding)); |
106 | 0 | return false; |
107 | 0 | } |
108 | 0 | errorStr = QString(); // clear error |
109 | 0 | return setEncoding(ce); |
110 | 0 | } |
111 | | |
112 | | bool QDecompressHelper::setEncoding(ContentEncoding ce) |
113 | 0 | { |
114 | 0 | Q_ASSERT(contentEncoding == None); |
115 | 0 | contentEncoding = ce; |
116 | 0 | switch (contentEncoding) { |
117 | 0 | case None: |
118 | 0 | Q_UNREACHABLE(); |
119 | 0 | break; |
120 | 0 | case Deflate: |
121 | 0 | case GZip: { |
122 | 0 | z_stream *inflateStream = new z_stream; |
123 | 0 | memset(inflateStream, 0, sizeof(z_stream)); |
124 | | // "windowBits can also be greater than 15 for optional gzip decoding. |
125 | | // Add 32 to windowBits to enable zlib and gzip decoding with automatic header detection" |
126 | | // http://www.zlib.net/manual.html |
127 | 0 | if (inflateInit2(inflateStream, MAX_WBITS + 32) != Z_OK) { |
128 | 0 | delete inflateStream; |
129 | 0 | inflateStream = nullptr; |
130 | 0 | } |
131 | 0 | decoderPointer = inflateStream; |
132 | 0 | break; |
133 | 0 | } |
134 | 0 | case Brotli: |
135 | | #if QT_CONFIG(brotli) |
136 | | decoderPointer = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr); |
137 | | #else |
138 | 0 | Q_UNREACHABLE(); |
139 | 0 | #endif |
140 | 0 | break; |
141 | 0 | case Zstandard: |
142 | | #if QT_CONFIG(zstd) |
143 | | decoderPointer = ZSTD_createDStream(); |
144 | | #else |
145 | 0 | Q_UNREACHABLE(); |
146 | 0 | #endif |
147 | 0 | break; |
148 | 0 | } |
149 | 0 | if (!decoderPointer) { |
150 | 0 | errorStr = QCoreApplication::translate("QHttp", |
151 | 0 | "Failed to initialize the compression decoder."); |
152 | 0 | contentEncoding = QDecompressHelper::None; |
153 | 0 | return false; |
154 | 0 | } |
155 | 0 | return true; |
156 | 0 | } |
157 | | |
158 | | /*! |
159 | | \internal |
160 | | |
161 | | Returns true if the QDecompressHelper is measuring the |
162 | | size of the decompressed data. |
163 | | |
164 | | \sa setCountingBytesEnabled, uncompressedSize |
165 | | */ |
166 | | bool QDecompressHelper::isCountingBytes() const |
167 | 0 | { |
168 | 0 | return countDecompressed; |
169 | 0 | } |
170 | | |
171 | | /*! |
172 | | \internal |
173 | | |
174 | | Enable or disable counting the decompressed size of the data |
175 | | based on \a shouldCount. Enabling this means the data will be |
176 | | decompressed twice (once for counting and once when data is |
177 | | being read). |
178 | | |
179 | | \note Can only be called before contentEncoding is set and data |
180 | | is fed to the object. |
181 | | |
182 | | \sa isCountingBytes, uncompressedSize |
183 | | */ |
184 | | void QDecompressHelper::setCountingBytesEnabled(bool shouldCount) |
185 | 0 | { |
186 | | // These are a best-effort check to ensure that no data has already been processed before this |
187 | | // gets enabled |
188 | 0 | Q_ASSERT(compressedDataBuffer.byteAmount() == 0); |
189 | 0 | Q_ASSERT(contentEncoding == None); |
190 | 0 | countDecompressed = shouldCount; |
191 | 0 | } |
192 | | |
193 | | /*! |
194 | | \internal |
195 | | |
196 | | Returns the amount of uncompressed bytes left. |
197 | | |
198 | | \note Since this is only based on the data received |
199 | | so far the final size could be larger. |
200 | | |
201 | | \note It is only valid to call this if isCountingBytes() |
202 | | returns true |
203 | | |
204 | | \sa isCountingBytes, setCountBytes |
205 | | */ |
206 | | qint64 QDecompressHelper::uncompressedSize() const |
207 | 0 | { |
208 | 0 | Q_ASSERT(countDecompressed); |
209 | | // Use the 'totalUncompressedBytes' from the countHelper if it exceeds the amount of bytes |
210 | | // that we know about. |
211 | 0 | auto totalUncompressed = |
212 | 0 | countHelper && countHelper->totalUncompressedBytes > totalUncompressedBytes |
213 | 0 | ? countHelper->totalUncompressedBytes |
214 | 0 | : totalUncompressedBytes; |
215 | 0 | return totalUncompressed - totalBytesRead; |
216 | 0 | } |
217 | | |
218 | | /*! |
219 | | \internal |
220 | | \overload |
221 | | */ |
222 | | void QDecompressHelper::feed(const QByteArray &data) |
223 | 0 | { |
224 | 0 | return feed(QByteArray(data)); |
225 | 0 | } |
226 | | |
227 | | /*! |
228 | | \internal |
229 | | Give \a data to the QDecompressHelper which will be stored until |
230 | | a read is attempted. |
231 | | |
232 | | If \c isCountingBytes() is true then it will decompress immediately |
233 | | before discarding the data, but will count the uncompressed byte |
234 | | size. |
235 | | */ |
236 | | void QDecompressHelper::feed(QByteArray &&data) |
237 | 0 | { |
238 | 0 | Q_ASSERT(contentEncoding != None); |
239 | 0 | totalCompressedBytes += data.size(); |
240 | 0 | compressedDataBuffer.append(std::move(data)); |
241 | 0 | if (!countInternal(compressedDataBuffer[compressedDataBuffer.bufferCount() - 1])) { |
242 | 0 | if (errorStr.isEmpty() && countHelper) |
243 | 0 | errorStr = countHelper->errorString(); |
244 | 0 | clear(); // If our counting brother failed then so will we :| |
245 | 0 | } |
246 | 0 | } |
247 | | |
248 | | /*! |
249 | | \internal |
250 | | \overload |
251 | | */ |
252 | | void QDecompressHelper::feed(const QByteDataBuffer &buffer) |
253 | 0 | { |
254 | 0 | Q_ASSERT(contentEncoding != None); |
255 | 0 | totalCompressedBytes += buffer.byteAmount(); |
256 | 0 | compressedDataBuffer.append(buffer); |
257 | 0 | if (!countInternal(buffer)) { |
258 | 0 | if (errorStr.isEmpty() && countHelper) |
259 | 0 | errorStr = countHelper->errorString(); |
260 | 0 | clear(); // If our counting brother failed then so will we :| |
261 | 0 | } |
262 | 0 | } |
263 | | |
264 | | /*! |
265 | | \internal |
266 | | \overload |
267 | | */ |
268 | | void QDecompressHelper::feed(QByteDataBuffer &&buffer) |
269 | 0 | { |
270 | 0 | Q_ASSERT(contentEncoding != None); |
271 | 0 | totalCompressedBytes += buffer.byteAmount(); |
272 | 0 | const QByteDataBuffer copy(buffer); |
273 | 0 | compressedDataBuffer.append(std::move(buffer)); |
274 | 0 | if (!countInternal(copy)) { |
275 | 0 | if (errorStr.isEmpty() && countHelper) |
276 | 0 | errorStr = countHelper->errorString(); |
277 | 0 | clear(); // If our counting brother failed then so will we :| |
278 | 0 | } |
279 | 0 | } |
280 | | |
281 | | /*! |
282 | | \internal |
283 | | Decompress the data internally and immediately discard the |
284 | | uncompressed data, but count how many bytes were decoded. |
285 | | This lets us know the final size, unfortunately at the cost of |
286 | | increased computation. |
287 | | |
288 | | To save on some of the computation we will store the data until |
289 | | we reach \c MaxDecompressedDataBufferSize stored. In this case the |
290 | | "penalty" is completely removed from users who read the data on |
291 | | readyRead rather than waiting for it all to be received. And |
292 | | any file smaller than \c MaxDecompressedDataBufferSize will |
293 | | avoid this issue as well. |
294 | | */ |
295 | | bool QDecompressHelper::countInternal() |
296 | 0 | { |
297 | 0 | Q_ASSERT(countDecompressed); |
298 | 0 | while (hasDataInternal() |
299 | 0 | && decompressedDataBuffer.byteAmount() < MaxDecompressedDataBufferSize) { |
300 | 0 | const qsizetype toRead = 256 * 1024; |
301 | 0 | QByteArray buffer(toRead, Qt::Uninitialized); |
302 | 0 | qsizetype bytesRead = readInternal(buffer.data(), buffer.size()); |
303 | 0 | if (bytesRead == -1) |
304 | 0 | return false; |
305 | 0 | buffer.truncate(bytesRead); |
306 | 0 | decompressedDataBuffer.append(std::move(buffer)); |
307 | 0 | } |
308 | 0 | if (!hasDataInternal()) |
309 | 0 | return true; // handled all the data so far, just return |
310 | | |
311 | 0 | while (countHelper->hasData()) { |
312 | 0 | std::array<char, 1024> temp; |
313 | 0 | qsizetype bytesRead = countHelper->read(temp.data(), temp.size()); |
314 | 0 | if (bytesRead == -1) |
315 | 0 | return false; |
316 | 0 | } |
317 | 0 | return true; |
318 | 0 | } |
319 | | |
320 | | /*! |
321 | | \internal |
322 | | \overload |
323 | | */ |
324 | | bool QDecompressHelper::countInternal(const QByteArray &data) |
325 | 0 | { |
326 | 0 | if (countDecompressed) { |
327 | 0 | if (!countHelper) { |
328 | 0 | countHelper = std::make_unique<QDecompressHelper>(); |
329 | 0 | countHelper->setDecompressedSafetyCheckThreshold(archiveBombCheckThreshold); |
330 | 0 | countHelper->setEncoding(contentEncoding); |
331 | 0 | } |
332 | 0 | countHelper->feed(data); |
333 | 0 | return countInternal(); |
334 | 0 | } |
335 | 0 | return true; |
336 | 0 | } |
337 | | |
338 | | /*! |
339 | | \internal |
340 | | \overload |
341 | | */ |
342 | | bool QDecompressHelper::countInternal(const QByteDataBuffer &buffer) |
343 | 0 | { |
344 | 0 | if (countDecompressed) { |
345 | 0 | if (!countHelper) { |
346 | 0 | countHelper = std::make_unique<QDecompressHelper>(); |
347 | 0 | countHelper->setDecompressedSafetyCheckThreshold(archiveBombCheckThreshold); |
348 | 0 | countHelper->setEncoding(contentEncoding); |
349 | 0 | } |
350 | 0 | countHelper->feed(buffer); |
351 | 0 | return countInternal(); |
352 | 0 | } |
353 | 0 | return true; |
354 | 0 | } |
355 | | |
356 | | qsizetype QDecompressHelper::read(char *data, qsizetype maxSize) |
357 | 0 | { |
358 | 0 | if (maxSize <= 0) |
359 | 0 | return 0; |
360 | | |
361 | 0 | if (!isValid()) |
362 | 0 | return -1; |
363 | | |
364 | 0 | if (!hasData()) |
365 | 0 | return 0; |
366 | | |
367 | 0 | qsizetype cachedRead = 0; |
368 | 0 | if (!decompressedDataBuffer.isEmpty()) { |
369 | 0 | cachedRead = decompressedDataBuffer.read(data, maxSize); |
370 | 0 | data += cachedRead; |
371 | 0 | maxSize -= cachedRead; |
372 | 0 | } |
373 | |
|
374 | 0 | qsizetype bytesRead = readInternal(data, maxSize); |
375 | 0 | if (bytesRead == -1) |
376 | 0 | return -1; |
377 | 0 | totalBytesRead += bytesRead + cachedRead; |
378 | 0 | return bytesRead + cachedRead; |
379 | 0 | } |
380 | | |
381 | | /*! |
382 | | \internal |
383 | | Like read() but without attempting to read the |
384 | | cached/already-decompressed data. |
385 | | */ |
386 | | qsizetype QDecompressHelper::readInternal(char *data, qsizetype maxSize) |
387 | 0 | { |
388 | 0 | Q_ASSERT(isValid()); |
389 | |
|
390 | 0 | if (maxSize <= 0) |
391 | 0 | return 0; |
392 | | |
393 | 0 | if (!hasDataInternal()) |
394 | 0 | return 0; |
395 | | |
396 | 0 | qsizetype bytesRead = -1; |
397 | 0 | switch (contentEncoding) { |
398 | 0 | case None: |
399 | 0 | Q_UNREACHABLE(); |
400 | 0 | break; |
401 | 0 | case Deflate: |
402 | 0 | case GZip: |
403 | 0 | bytesRead = readZLib(data, maxSize); |
404 | 0 | break; |
405 | 0 | case Brotli: |
406 | 0 | bytesRead = readBrotli(data, maxSize); |
407 | 0 | break; |
408 | 0 | case Zstandard: |
409 | 0 | bytesRead = readZstandard(data, maxSize); |
410 | 0 | break; |
411 | 0 | } |
412 | 0 | if (bytesRead == -1) |
413 | 0 | clear(); |
414 | |
|
415 | 0 | totalUncompressedBytes += bytesRead; |
416 | 0 | if (isPotentialArchiveBomb()) { |
417 | 0 | errorStr = QCoreApplication::translate( |
418 | 0 | "QHttp", |
419 | 0 | "The decompressed output exceeds the limits specified by " |
420 | 0 | "QNetworkRequest::decompressedSafetyCheckThreshold()"); |
421 | 0 | return -1; |
422 | 0 | } |
423 | | |
424 | 0 | return bytesRead; |
425 | 0 | } |
426 | | |
427 | | /*! |
428 | | \internal |
429 | | Set the \a threshold required before the archive bomb detection kicks in. |
430 | | By default this is 10MB. Setting it to -1 is treated as disabling the |
431 | | feature. |
432 | | */ |
433 | | void QDecompressHelper::setDecompressedSafetyCheckThreshold(qint64 threshold) |
434 | 0 | { |
435 | 0 | if (threshold == -1) |
436 | 0 | threshold = std::numeric_limits<qint64>::max(); |
437 | 0 | archiveBombCheckThreshold = threshold; |
438 | 0 | } |
439 | | |
440 | | bool QDecompressHelper::isPotentialArchiveBomb() const |
441 | 0 | { |
442 | 0 | if (totalCompressedBytes == 0) |
443 | 0 | return false; |
444 | | |
445 | 0 | if (totalUncompressedBytes <= archiveBombCheckThreshold) |
446 | 0 | return false; |
447 | | |
448 | | // Some protection against malicious or corrupted compressed files that expand far more than |
449 | | // is reasonable. |
450 | 0 | double ratio = double(totalUncompressedBytes) / double(totalCompressedBytes); |
451 | 0 | switch (contentEncoding) { |
452 | 0 | case None: |
453 | 0 | Q_UNREACHABLE(); |
454 | 0 | break; |
455 | 0 | case Deflate: |
456 | 0 | case GZip: |
457 | | // This value is mentioned in docs for |
458 | | // QNetworkRequest::setMinimumArchiveBombSize, keep synchronized |
459 | 0 | if (ratio > 40) { |
460 | 0 | return true; |
461 | 0 | } |
462 | 0 | break; |
463 | 0 | case Brotli: |
464 | 0 | case Zstandard: |
465 | | // This value is mentioned in docs for |
466 | | // QNetworkRequest::setMinimumArchiveBombSize, keep synchronized |
467 | 0 | if (ratio > 100) { |
468 | 0 | return true; |
469 | 0 | } |
470 | 0 | break; |
471 | 0 | } |
472 | 0 | return false; |
473 | 0 | } |
474 | | |
475 | | /*! |
476 | | \internal |
477 | | Returns true if there are encoded bytes left or there is some |
478 | | indication that the decoder still has data left internally. |
479 | | |
480 | | \note Even if this returns true the next call to read() might |
481 | | read 0 bytes. This most likely means the decompression is done. |
482 | | */ |
483 | | bool QDecompressHelper::hasData() const |
484 | 0 | { |
485 | 0 | return hasDataInternal() || !decompressedDataBuffer.isEmpty(); |
486 | 0 | } |
487 | | |
488 | | /*! |
489 | | \internal |
490 | | Like hasData() but internally the buffer of decompressed data is |
491 | | not interesting. |
492 | | */ |
493 | | bool QDecompressHelper::hasDataInternal() const |
494 | 0 | { |
495 | 0 | return encodedBytesAvailable() || decoderHasData; |
496 | 0 | } |
497 | | |
498 | | qint64 QDecompressHelper::encodedBytesAvailable() const |
499 | 0 | { |
500 | 0 | return compressedDataBuffer.byteAmount(); |
501 | 0 | } |
502 | | |
503 | | /*! |
504 | | \internal |
505 | | Returns whether or not the object is valid. |
506 | | If it becomes invalid after an operation has been performed |
507 | | then an error has occurred. |
508 | | \sa errorString() |
509 | | */ |
510 | | bool QDecompressHelper::isValid() const |
511 | 0 | { |
512 | 0 | return contentEncoding != None; |
513 | 0 | } |
514 | | |
515 | | /*! |
516 | | \internal |
517 | | Returns a string describing the error that occurred or an empty |
518 | | string if no error occurred. |
519 | | \sa isValid() |
520 | | */ |
521 | | QString QDecompressHelper::errorString() const |
522 | 0 | { |
523 | 0 | return errorStr; |
524 | 0 | } |
525 | | |
526 | | void QDecompressHelper::clear() |
527 | 0 | { |
528 | 0 | switch (contentEncoding) { |
529 | 0 | case None: |
530 | 0 | break; |
531 | 0 | case Deflate: |
532 | 0 | case GZip: { |
533 | 0 | z_stream *inflateStream = toZlibPointer(decoderPointer); |
534 | 0 | if (inflateStream) |
535 | 0 | inflateEnd(inflateStream); |
536 | 0 | delete inflateStream; |
537 | 0 | break; |
538 | 0 | } |
539 | 0 | case Brotli: { |
540 | | #if QT_CONFIG(brotli) |
541 | | BrotliDecoderState *brotliDecoderState = toBrotliPointer(decoderPointer); |
542 | | if (brotliDecoderState) |
543 | | BrotliDecoderDestroyInstance(brotliDecoderState); |
544 | | #endif |
545 | 0 | break; |
546 | 0 | } |
547 | 0 | case Zstandard: { |
548 | | #if QT_CONFIG(zstd) |
549 | | ZSTD_DStream *zstdStream = toZstandardPointer(decoderPointer); |
550 | | if (zstdStream) |
551 | | ZSTD_freeDStream(zstdStream); |
552 | | #endif |
553 | 0 | break; |
554 | 0 | } |
555 | 0 | } |
556 | 0 | decoderPointer = nullptr; |
557 | 0 | contentEncoding = None; |
558 | |
|
559 | 0 | compressedDataBuffer.clear(); |
560 | 0 | decompressedDataBuffer.clear(); |
561 | 0 | decoderHasData = false; |
562 | |
|
563 | 0 | countDecompressed = false; |
564 | 0 | countHelper.reset(); |
565 | 0 | totalBytesRead = 0; |
566 | 0 | totalUncompressedBytes = 0; |
567 | 0 | totalCompressedBytes = 0; |
568 | 0 | } |
569 | | |
570 | | qsizetype QDecompressHelper::readZLib(char *data, const qsizetype maxSize) |
571 | 0 | { |
572 | 0 | bool triedRawDeflate = false; |
573 | |
|
574 | 0 | z_stream *inflateStream = toZlibPointer(decoderPointer); |
575 | 0 | static const size_t zlibMaxSize = |
576 | 0 | size_t(std::numeric_limits<decltype(inflateStream->avail_in)>::max()); |
577 | |
|
578 | 0 | QByteArrayView input = compressedDataBuffer.readPointer(); |
579 | 0 | if (size_t(input.size()) > zlibMaxSize) |
580 | 0 | input = input.sliced(zlibMaxSize); |
581 | |
|
582 | 0 | inflateStream->avail_in = input.size(); |
583 | 0 | inflateStream->next_in = reinterpret_cast<Bytef *>(const_cast<char *>(input.data())); |
584 | |
|
585 | 0 | bool bigMaxSize = (zlibMaxSize < size_t(maxSize)); |
586 | 0 | qsizetype adjustedAvailableOut = bigMaxSize ? qsizetype(zlibMaxSize) : maxSize; |
587 | 0 | inflateStream->avail_out = adjustedAvailableOut; |
588 | 0 | inflateStream->next_out = reinterpret_cast<Bytef *>(data); |
589 | |
|
590 | 0 | qsizetype bytesDecoded = 0; |
591 | 0 | do { |
592 | 0 | auto previous_avail_out = inflateStream->avail_out; |
593 | 0 | int ret = inflate(inflateStream, Z_NO_FLUSH); |
594 | | // All negative return codes are errors, in the context of HTTP compression, Z_NEED_DICT is |
595 | | // also an error. |
596 | | // in the case where we get Z_DATA_ERROR this could be because we received raw deflate |
597 | | // compressed data. |
598 | 0 | if (ret == Z_DATA_ERROR && !triedRawDeflate) { Q_UNLIKELY_BRANCH |
599 | 0 | inflateEnd(inflateStream); |
600 | 0 | triedRawDeflate = true; |
601 | 0 | inflateStream->zalloc = Z_NULL; |
602 | 0 | inflateStream->zfree = Z_NULL; |
603 | 0 | inflateStream->opaque = Z_NULL; |
604 | 0 | inflateStream->avail_in = 0; |
605 | 0 | inflateStream->next_in = Z_NULL; |
606 | 0 | int ret = inflateInit2(inflateStream, -MAX_WBITS); |
607 | 0 | if (ret != Z_OK) { |
608 | 0 | return -1; |
609 | 0 | } else { |
610 | 0 | inflateStream->avail_in = input.size(); |
611 | 0 | inflateStream->next_in = |
612 | 0 | reinterpret_cast<Bytef *>(const_cast<char *>(input.data())); |
613 | 0 | continue; |
614 | 0 | } |
615 | 0 | } else if (ret < 0 || ret == Z_NEED_DICT) { |
616 | 0 | return -1; |
617 | 0 | } |
618 | 0 | bytesDecoded += qsizetype(previous_avail_out - inflateStream->avail_out); |
619 | 0 | if (ret == Z_STREAM_END) { |
620 | | |
621 | | // If there's more data after the stream then this is probably composed of multiple |
622 | | // streams. |
623 | 0 | if (inflateStream->avail_in != 0) { |
624 | 0 | inflateEnd(inflateStream); |
625 | 0 | Bytef *next_in = inflateStream->next_in; |
626 | 0 | uInt avail_in = inflateStream->avail_in; |
627 | 0 | inflateStream->zalloc = Z_NULL; |
628 | 0 | inflateStream->zfree = Z_NULL; |
629 | 0 | inflateStream->opaque = Z_NULL; |
630 | 0 | if (inflateInit2(inflateStream, MAX_WBITS + 32) != Z_OK) { |
631 | 0 | delete inflateStream; |
632 | 0 | decoderPointer = nullptr; |
633 | | // Failed to reinitialize, so we'll just return what we have |
634 | 0 | compressedDataBuffer.advanceReadPointer(input.size() - avail_in); |
635 | 0 | return bytesDecoded; |
636 | 0 | } else { |
637 | 0 | inflateStream->next_in = next_in; |
638 | 0 | inflateStream->avail_in = avail_in; |
639 | | // Keep going to handle the other cases below |
640 | 0 | } |
641 | 0 | } else { |
642 | | // No extra data, stream is at the end. We're done. |
643 | 0 | compressedDataBuffer.advanceReadPointer(input.size()); |
644 | 0 | return bytesDecoded; |
645 | 0 | } |
646 | 0 | } |
647 | | |
648 | 0 | if (bigMaxSize && inflateStream->avail_out == 0) { |
649 | | // Need to adjust the next_out and avail_out parameters since we reached the end |
650 | | // of the current range |
651 | 0 | bigMaxSize = (zlibMaxSize < size_t(maxSize - bytesDecoded)); |
652 | 0 | inflateStream->avail_out = bigMaxSize ? qsizetype(zlibMaxSize) : maxSize - bytesDecoded; |
653 | 0 | inflateStream->next_out = reinterpret_cast<Bytef *>(data + bytesDecoded); |
654 | 0 | } |
655 | |
|
656 | 0 | if (inflateStream->avail_in == 0 && inflateStream->avail_out > 0) { |
657 | | // Grab the next input! |
658 | 0 | compressedDataBuffer.advanceReadPointer(input.size()); |
659 | 0 | input = compressedDataBuffer.readPointer(); |
660 | 0 | if (size_t(input.size()) > zlibMaxSize) |
661 | 0 | input = input.sliced(zlibMaxSize); |
662 | 0 | inflateStream->avail_in = input.size(); |
663 | 0 | inflateStream->next_in = |
664 | 0 | reinterpret_cast<Bytef *>(const_cast<char *>(input.data())); |
665 | 0 | } |
666 | 0 | } while (inflateStream->avail_out > 0 && inflateStream->avail_in > 0); |
667 | | |
668 | 0 | compressedDataBuffer.advanceReadPointer(input.size() - inflateStream->avail_in); |
669 | |
|
670 | 0 | return bytesDecoded; |
671 | 0 | } |
672 | | |
673 | | qsizetype QDecompressHelper::readBrotli(char *data, const qsizetype maxSize) |
674 | 0 | { |
675 | 0 | #if !QT_CONFIG(brotli) |
676 | 0 | Q_UNUSED(data); |
677 | 0 | Q_UNUSED(maxSize); |
678 | 0 | Q_UNREACHABLE(); |
679 | | #else |
680 | | qint64 bytesDecoded = 0; |
681 | | |
682 | | BrotliDecoderState *brotliDecoderState = toBrotliPointer(decoderPointer); |
683 | | |
684 | | while (decoderHasData && bytesDecoded < maxSize) { |
685 | | Q_ASSERT(brotliUnconsumedDataPtr || BrotliDecoderHasMoreOutput(brotliDecoderState)); |
686 | | if (brotliUnconsumedDataPtr) { |
687 | | Q_ASSERT(brotliUnconsumedAmount); |
688 | | size_t toRead = std::min(size_t(maxSize - bytesDecoded), brotliUnconsumedAmount); |
689 | | memcpy(data + bytesDecoded, brotliUnconsumedDataPtr, toRead); |
690 | | bytesDecoded += toRead; |
691 | | brotliUnconsumedAmount -= toRead; |
692 | | brotliUnconsumedDataPtr += toRead; |
693 | | if (brotliUnconsumedAmount == 0) { |
694 | | brotliUnconsumedDataPtr = nullptr; |
695 | | decoderHasData = false; |
696 | | } |
697 | | } |
698 | | if (BrotliDecoderHasMoreOutput(brotliDecoderState) == BROTLI_TRUE) { |
699 | | brotliUnconsumedDataPtr = |
700 | | BrotliDecoderTakeOutput(brotliDecoderState, &brotliUnconsumedAmount); |
701 | | decoderHasData = true; |
702 | | } |
703 | | } |
704 | | if (bytesDecoded == maxSize) |
705 | | return bytesDecoded; |
706 | | Q_ASSERT(bytesDecoded < maxSize); |
707 | | |
708 | | QByteArrayView input = compressedDataBuffer.readPointer(); |
709 | | const uint8_t *encodedPtr = reinterpret_cast<const uint8_t *>(input.data()); |
710 | | size_t encodedBytesRemaining = input.size(); |
711 | | |
712 | | uint8_t *decodedPtr = reinterpret_cast<uint8_t *>(data + bytesDecoded); |
713 | | size_t unusedDecodedSize = size_t(maxSize - bytesDecoded); |
714 | | while (unusedDecodedSize > 0) { |
715 | | auto previousUnusedDecodedSize = unusedDecodedSize; |
716 | | BrotliDecoderResult result = BrotliDecoderDecompressStream( |
717 | | brotliDecoderState, &encodedBytesRemaining, &encodedPtr, &unusedDecodedSize, |
718 | | &decodedPtr, nullptr); |
719 | | bytesDecoded += previousUnusedDecodedSize - unusedDecodedSize; |
720 | | |
721 | | switch (result) { |
722 | | Q_UNLIKELY_BRANCH |
723 | | case BROTLI_DECODER_RESULT_ERROR: |
724 | | //: Brotli (compression algorithm) decoding error, e.g. corrupted input or memory allocation problem. |
725 | | errorStr = QCoreApplication::translate("QHttp", "Brotli error: %1") |
726 | | .arg(QUtf8StringView{BrotliDecoderErrorString( |
727 | | BrotliDecoderGetErrorCode(brotliDecoderState))}); |
728 | | return -1; |
729 | | case BROTLI_DECODER_RESULT_SUCCESS: |
730 | | BrotliDecoderDestroyInstance(brotliDecoderState); |
731 | | decoderPointer = nullptr; |
732 | | compressedDataBuffer.clear(); |
733 | | return bytesDecoded; |
734 | | case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT: |
735 | | compressedDataBuffer.advanceReadPointer(input.size()); |
736 | | input = compressedDataBuffer.readPointer(); |
737 | | if (!input.isEmpty()) { |
738 | | encodedPtr = reinterpret_cast<const uint8_t *>(input.constData()); |
739 | | encodedBytesRemaining = input.size(); |
740 | | break; |
741 | | } |
742 | | return bytesDecoded; |
743 | | case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT: |
744 | | // Some data is leftover inside the brotli decoder, remember for next time |
745 | | decoderHasData = BrotliDecoderHasMoreOutput(brotliDecoderState); |
746 | | Q_ASSERT(unusedDecodedSize == 0); |
747 | | break; |
748 | | } |
749 | | } |
750 | | compressedDataBuffer.advanceReadPointer(input.size() - encodedBytesRemaining); |
751 | | return bytesDecoded; |
752 | | #endif |
753 | 0 | } |
754 | | |
755 | | qsizetype QDecompressHelper::readZstandard(char *data, const qsizetype maxSize) |
756 | 0 | { |
757 | 0 | #if !QT_CONFIG(zstd) |
758 | 0 | Q_UNUSED(data); |
759 | 0 | Q_UNUSED(maxSize); |
760 | 0 | Q_UNREACHABLE(); |
761 | | #else |
762 | | ZSTD_DStream *zstdStream = toZstandardPointer(decoderPointer); |
763 | | |
764 | | QByteArrayView input = compressedDataBuffer.readPointer(); |
765 | | ZSTD_inBuffer inBuf { input.data(), size_t(input.size()), 0 }; |
766 | | |
767 | | ZSTD_outBuffer outBuf { data, size_t(maxSize), 0 }; |
768 | | |
769 | | qsizetype bytesDecoded = 0; |
770 | | while (outBuf.pos < outBuf.size && (inBuf.pos < inBuf.size || decoderHasData)) { |
771 | | size_t retValue = ZSTD_decompressStream(zstdStream, &outBuf, &inBuf); |
772 | | if (ZSTD_isError(retValue)) { Q_UNLIKELY_BRANCH |
773 | | errorStr = QCoreApplication::translate("QHttp", "ZStandard error: %1") |
774 | | .arg(QUtf8StringView{ZSTD_getErrorName(retValue)}); |
775 | | return -1; |
776 | | } |
777 | | decoderHasData = false; |
778 | | bytesDecoded = outBuf.pos; |
779 | | // if pos == size then there may be data left over in internal buffers |
780 | | if (outBuf.pos == outBuf.size) { |
781 | | decoderHasData = true; |
782 | | } else if (inBuf.pos == inBuf.size) { |
783 | | compressedDataBuffer.advanceReadPointer(input.size()); |
784 | | input = compressedDataBuffer.readPointer(); |
785 | | inBuf = { input.constData(), size_t(input.size()), 0 }; |
786 | | } |
787 | | } |
788 | | compressedDataBuffer.advanceReadPointer(inBuf.pos); |
789 | | return bytesDecoded; |
790 | | #endif |
791 | 0 | } |
792 | | |
793 | | QT_END_NAMESPACE |