/src/libheif/libheif/codecs/decoder.cc
Line | Count | Source |
1 | | /* |
2 | | * HEIF codec. |
3 | | * Copyright (c) 2024 Dirk Farin <dirk.farin@gmail.com> |
4 | | * |
5 | | * This file is part of libheif. |
6 | | * |
7 | | * libheif is free software: you can redistribute it and/or modify |
8 | | * it under the terms of the GNU Lesser General Public License as |
9 | | * published by the Free Software Foundation, either version 3 of |
10 | | * the License, or (at your option) any later version. |
11 | | * |
12 | | * libheif is distributed in the hope that it will be useful, |
13 | | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
14 | | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
15 | | * GNU Lesser General Public License for more details. |
16 | | * |
17 | | * You should have received a copy of the GNU Lesser General Public License |
18 | | * along with libheif. If not, see <http://www.gnu.org/licenses/>. |
19 | | */ |
20 | | |
21 | | #include "codecs/decoder.h" |
22 | | |
23 | | #include <utility> |
24 | | #include "error.h" |
25 | | #include "context.h" |
26 | | #include "plugin_registry.h" |
27 | | #include "api_structs.h" |
28 | | |
29 | | #include "codecs/hevc_dec.h" |
30 | | #include "codecs/avif_dec.h" |
31 | | #include "codecs/avc_dec.h" |
32 | | #include "codecs/vvc_dec.h" |
33 | | #include "codecs/jpeg_dec.h" |
34 | | #include "codecs/jpeg2000_dec.h" |
35 | | #include "avc_boxes.h" |
36 | | #include "avif_boxes.h" |
37 | | #include "hevc_boxes.h" |
38 | | #include "vvc_boxes.h" |
39 | | #include "jpeg_boxes.h" |
40 | | #include "jpeg2000_boxes.h" |
41 | | |
42 | | #if WITH_UNCOMPRESSED_CODEC |
43 | | #include "codecs/uncompressed/unc_dec.h" |
44 | | #include "codecs/uncompressed/unc_boxes.h" |
45 | | #endif |
46 | | |
47 | | void DataExtent::set_from_image_item(std::shared_ptr<HeifFile> file, heif_item_id item) |
48 | 24.4k | { |
49 | 24.4k | m_file = std::move(file); |
50 | 24.4k | m_item_id = item; |
51 | 24.4k | m_source = Source::Image; |
52 | 24.4k | } |
53 | | |
54 | | |
55 | | void DataExtent::set_file_range(std::shared_ptr<HeifFile> file, uint64_t offset, uint32_t size) |
56 | 0 | { |
57 | 0 | m_file = std::move(file); |
58 | 0 | m_source = Source::FileRange; |
59 | 0 | m_offset = offset; |
60 | 0 | m_size = size; |
61 | 0 | } |
62 | | |
63 | | |
64 | | Result<std::vector<uint8_t>*> DataExtent::read_data() const |
65 | 10.3k | { |
66 | 10.3k | if (!m_raw.empty()) { |
67 | 0 | return &m_raw; |
68 | 0 | } |
69 | 10.3k | else if (m_source == Source::Image) { |
70 | 10.3k | assert(m_file); |
71 | | |
72 | | // image |
73 | 10.3k | Error err = m_file->append_data_from_iloc(m_item_id, m_raw); |
74 | 10.3k | if (err) { |
75 | 230 | return err; |
76 | 230 | } |
77 | 10.3k | } |
78 | 0 | else { |
79 | | // file range |
80 | 0 | Error err = m_file->append_data_from_file_range(m_raw, m_offset, m_size); |
81 | 0 | if (err) { |
82 | 0 | return err; |
83 | 0 | } |
84 | 0 | } |
85 | | |
86 | 10.1k | return &m_raw; |
87 | 10.3k | } |
88 | | |
89 | | |
90 | | Result<std::vector<uint8_t>> DataExtent::read_data(uint64_t offset, uint64_t size) const |
91 | 0 | { |
92 | 0 | std::vector<uint8_t> data; |
93 | |
|
94 | 0 | if (!m_raw.empty()) { |
95 | 0 | data.insert(data.begin(), m_raw.begin() + offset, m_raw.begin() + offset + size); |
96 | 0 | return data; |
97 | 0 | } |
98 | 0 | else if (m_source == Source::Image) { |
99 | | // TODO: cache data |
100 | | |
101 | | // image |
102 | 0 | Error err = m_file->append_data_from_iloc(m_item_id, data, offset, size); |
103 | 0 | if (err) { |
104 | 0 | return err; |
105 | 0 | } |
106 | 0 | return data; |
107 | 0 | } |
108 | 0 | else { |
109 | | // file range |
110 | 0 | Error err = m_file->append_data_from_file_range(data, m_offset, m_size); |
111 | 0 | if (err) { |
112 | 0 | return err; |
113 | 0 | } |
114 | | |
115 | 0 | return data; |
116 | 0 | } |
117 | 0 | } |
118 | | |
119 | | |
120 | | std::shared_ptr<Decoder> Decoder::alloc_for_infe_type(const ImageItem* item) |
121 | 0 | { |
122 | 0 | uint32_t format_4cc = item->get_infe_type(); |
123 | |
|
124 | 0 | switch (format_4cc) { |
125 | 0 | case fourcc("hvc1"): { |
126 | 0 | auto hvcC = item->get_property<Box_hvcC>(); |
127 | 0 | return std::make_shared<Decoder_HEVC>(hvcC); |
128 | 0 | } |
129 | 0 | case fourcc("av01"): { |
130 | 0 | auto av1C = item->get_property<Box_av1C>(); |
131 | 0 | return std::make_shared<Decoder_AVIF>(av1C); |
132 | 0 | } |
133 | 0 | case fourcc("avc1"): { |
134 | 0 | auto avcC = item->get_property<Box_avcC>(); |
135 | 0 | return std::make_shared<Decoder_AVC>(avcC); |
136 | 0 | } |
137 | 0 | case fourcc("j2k1"): { |
138 | 0 | auto j2kH = item->get_property<Box_j2kH>(); |
139 | 0 | return std::make_shared<Decoder_JPEG2000>(j2kH); |
140 | 0 | } |
141 | 0 | case fourcc("vvc1"): { |
142 | 0 | auto vvcC = item->get_property<Box_vvcC>(); |
143 | 0 | return std::make_shared<Decoder_VVC>(vvcC); |
144 | 0 | } |
145 | 0 | case fourcc("jpeg"): { |
146 | 0 | auto jpgC = item->get_property<Box_jpgC>(); |
147 | 0 | return std::make_shared<Decoder_JPEG>(jpgC); |
148 | 0 | } |
149 | | #if WITH_UNCOMPRESSED_CODEC |
150 | | case fourcc("unci"): { |
151 | | auto uncC = item->get_property<Box_uncC>(); |
152 | | auto cmpd = item->get_property<Box_cmpd>(); |
153 | | auto ispe = item->get_property<Box_ispe>(); |
154 | | return std::make_shared<Decoder_uncompressed>(uncC,cmpd,ispe); |
155 | | } |
156 | | #endif |
157 | 0 | case fourcc("mski"): { |
158 | 0 | return nullptr; // do we need a decoder for this? |
159 | 0 | } |
160 | 0 | default: |
161 | 0 | return nullptr; |
162 | 0 | } |
163 | 0 | } |
164 | | |
165 | | |
166 | | std::shared_ptr<Decoder> Decoder::alloc_for_sequence_sample_description_box(std::shared_ptr<const Box_VisualSampleEntry> sample_description_box) |
167 | 0 | { |
168 | 0 | std::string compressor = sample_description_box->get_VisualSampleEntry_const().compressorname; |
169 | 0 | uint32_t sampleType = sample_description_box->get_short_type(); |
170 | |
|
171 | 0 | switch (sampleType) { |
172 | 0 | case fourcc("hvc1"): { |
173 | 0 | auto hvcC = sample_description_box->get_child_box<Box_hvcC>(); |
174 | 0 | return std::make_shared<Decoder_HEVC>(hvcC); |
175 | 0 | } |
176 | | |
177 | 0 | case fourcc("av01"): { |
178 | 0 | auto av1C = sample_description_box->get_child_box<Box_av1C>(); |
179 | 0 | return std::make_shared<Decoder_AVIF>(av1C); |
180 | 0 | } |
181 | | |
182 | 0 | case fourcc("vvc1"): { |
183 | 0 | auto vvcC = sample_description_box->get_child_box<Box_vvcC>(); |
184 | 0 | return std::make_shared<Decoder_VVC>(vvcC); |
185 | 0 | } |
186 | | |
187 | 0 | case fourcc("avc1"): { |
188 | 0 | auto avcC = sample_description_box->get_child_box<Box_avcC>(); |
189 | 0 | return std::make_shared<Decoder_AVC>(avcC); |
190 | 0 | } |
191 | | |
192 | | #if WITH_UNCOMPRESSED_CODEC |
193 | | case fourcc("uncv"): { |
194 | | auto uncC = sample_description_box->get_child_box<Box_uncC>(); |
195 | | auto cmpd = sample_description_box->get_child_box<Box_cmpd>(); |
196 | | auto ispe = std::make_shared<Box_ispe>(); |
197 | | ispe->set_size(sample_description_box->get_VisualSampleEntry_const().width, |
198 | | sample_description_box->get_VisualSampleEntry_const().height); |
199 | | return std::make_shared<Decoder_uncompressed>(uncC, cmpd, ispe); |
200 | | } |
201 | | #endif |
202 | | |
203 | 0 | case fourcc("j2ki"): { |
204 | 0 | auto j2kH = sample_description_box->get_child_box<Box_j2kH>(); |
205 | 0 | return std::make_shared<Decoder_JPEG2000>(j2kH); |
206 | 0 | } |
207 | | |
208 | 0 | case fourcc("mjpg"): { |
209 | 0 | auto jpgC = sample_description_box->get_child_box<Box_jpgC>(); |
210 | 0 | return std::make_shared<Decoder_JPEG>(jpgC); |
211 | 0 | } |
212 | | |
213 | 0 | default: |
214 | 0 | return nullptr; |
215 | 0 | } |
216 | 0 | } |
217 | | |
218 | | |
219 | | Result<std::vector<uint8_t>> Decoder::get_compressed_data(bool with_configuration_NALs) const |
220 | 10.3k | { |
221 | | // --- get the compressed image data |
222 | | |
223 | 10.3k | if (with_configuration_NALs) { |
224 | | // data from configuration blocks |
225 | | |
226 | 10.3k | Result<std::vector<uint8_t>> confData = read_bitstream_configuration_data(); |
227 | 10.3k | if (!confData) { |
228 | 0 | return confData.error(); |
229 | 0 | } |
230 | | |
231 | 10.3k | std::vector<uint8_t> data = *confData; |
232 | | |
233 | | // append image data |
234 | | |
235 | 10.3k | auto dataResult = m_data_extent.read_data(); |
236 | 10.3k | if (!dataResult) { |
237 | 230 | return dataResult.error(); |
238 | 230 | } |
239 | | |
240 | 10.1k | data.insert(data.end(), (*dataResult)->begin(), (*dataResult)->end()); |
241 | | |
242 | 10.1k | return data; |
243 | 10.3k | } |
244 | 0 | else { |
245 | 0 | auto dataResult = m_data_extent.read_data(); |
246 | 0 | if (!dataResult) { |
247 | 0 | return dataResult.error(); |
248 | 0 | } |
249 | | |
250 | 0 | return {*(*dataResult)}; |
251 | 0 | } |
252 | 10.3k | } |
253 | | |
254 | | |
255 | | Decoder::~Decoder() |
256 | 14.1k | { |
257 | 14.1k | if (m_decoder) { |
258 | 10.1k | assert(m_decoder_plugin); |
259 | 10.1k | m_decoder_plugin->free_decoder(m_decoder); |
260 | 10.1k | } |
261 | | |
262 | | //std::unique_ptr<void, void (*)(void*)> decoderSmartPtr(m_decoder, m_decoder_plugin->free_decoder); |
263 | 14.1k | } |
264 | | |
265 | | |
266 | | Error Decoder::require_decoder_plugin(const heif_decoding_options& options) |
267 | 33.7k | { |
268 | 33.7k | if (!m_decoder_plugin) { |
269 | 10.1k | if (options.decoder_id && !has_decoder(get_compression_format(), options.decoder_id)) { |
270 | 0 | return { |
271 | 0 | heif_error_Plugin_loading_error, |
272 | 0 | heif_suberror_Unspecified, |
273 | 0 | "No decoder with that ID found." |
274 | 0 | }; |
275 | 0 | } |
276 | | |
277 | 10.1k | m_decoder_plugin = get_decoder(get_compression_format(), options.decoder_id); |
278 | 10.1k | if (!m_decoder_plugin) { |
279 | 0 | return Error(heif_error_Plugin_loading_error, heif_suberror_No_matching_decoder_installed); |
280 | 0 | } |
281 | | |
282 | 10.1k | if (m_decoder_plugin->plugin_api_version < 5) { |
283 | 0 | return Error{ |
284 | 0 | heif_error_Plugin_loading_error, heif_suberror_No_matching_decoder_installed, |
285 | 0 | "Decoder plugin needs to be at least version 5." |
286 | 0 | }; |
287 | 0 | } |
288 | 10.1k | } |
289 | | |
290 | 33.7k | return {}; |
291 | 33.7k | } |
292 | | |
293 | | |
294 | | Error Decoder::decode_sequence_frame_from_compressed_data(bool upload_configuration_NALs, |
295 | | const heif_decoding_options& options, |
296 | | uintptr_t user_data, |
297 | | const heif_security_limits* limits) |
298 | 10.3k | { |
299 | 10.3k | auto pluginErr = require_decoder_plugin(options); |
300 | 10.3k | if (pluginErr) { |
301 | 0 | return pluginErr; |
302 | 0 | } |
303 | | |
304 | | // --- decode image with the plugin |
305 | | |
306 | 10.3k | heif_error err; |
307 | | |
308 | 10.3k | if (!m_decoder) { |
309 | 10.1k | if (m_decoder_plugin->new_decoder == nullptr) { |
310 | 0 | return Error(heif_error_Plugin_loading_error, heif_suberror_No_matching_decoder_installed, |
311 | 0 | "Cannot decode with a dummy decoder plugin."); |
312 | 0 | } |
313 | | |
314 | 10.1k | if (m_decoder_plugin->plugin_api_version >= 5) { |
315 | 10.1k | heif_decoder_plugin_options plugin_options; |
316 | 10.1k | plugin_options.format = get_compression_format(); |
317 | 10.1k | plugin_options.num_threads = options.num_codec_threads; |
318 | 10.1k | plugin_options.strict_decoding = options.strict_decoding; |
319 | | |
320 | 10.1k | err = m_decoder_plugin->new_decoder2(&m_decoder, &plugin_options); |
321 | 10.1k | if (err.code != heif_error_Ok) { |
322 | 0 | return Error(err.code, err.subcode, err.message); |
323 | 0 | } |
324 | 10.1k | } |
325 | 0 | else { |
326 | 0 | err = m_decoder_plugin->new_decoder(&m_decoder); |
327 | 0 | if (err.code != heif_error_Ok) { |
328 | 0 | return Error(err.code, err.subcode, err.message); |
329 | 0 | } |
330 | | |
331 | | // automatically delete decoder plugin when we leave the scope |
332 | | //std::unique_ptr<void, void (*)(void*)> decoderSmartPtr(m_decoder, m_decoder_plugin->free_decoder); |
333 | | |
334 | 0 | if (m_decoder_plugin->plugin_api_version >= 2) { |
335 | 0 | if (m_decoder_plugin->set_strict_decoding) { |
336 | 0 | m_decoder_plugin->set_strict_decoding(m_decoder, options.strict_decoding); |
337 | 0 | } |
338 | 0 | } |
339 | 0 | } |
340 | 10.1k | } |
341 | | |
342 | 10.3k | auto dataResult = get_compressed_data(upload_configuration_NALs); |
343 | 10.3k | if (!dataResult) { |
344 | 224 | return dataResult.error(); |
345 | 224 | } |
346 | | |
347 | | // Check that we are pushing at least some data into the decoder. |
348 | | // Some decoders (e.g. aom) do not complain when the input data is empty and we might |
349 | | // get stuck in an endless decoding loop, waiting for the decompressed image. |
350 | | |
351 | 10.0k | if (dataResult->size() == 0) { |
352 | 4 | return Error{ |
353 | 4 | heif_error_Invalid_input, |
354 | 4 | heif_suberror_Unspecified, |
355 | 4 | "Input with empty data extent." |
356 | 4 | }; |
357 | 4 | } |
358 | | |
359 | | //std::cout << "Decoder::decode_sequence_frame_from_compressed_data push " << dataResult->size() << "\n"; |
360 | 10.0k | if (m_decoder_plugin->plugin_api_version >= 5 && m_decoder_plugin->push_data2) { |
361 | 10.0k | err = m_decoder_plugin->push_data2(m_decoder, dataResult->data(), dataResult->size(), user_data); |
362 | 10.0k | } |
363 | 0 | else { |
364 | 0 | err = m_decoder_plugin->push_data(m_decoder, dataResult->data(), dataResult->size()); |
365 | 0 | } |
366 | 10.0k | if (err.code != heif_error_Ok) { |
367 | 5.11k | return Error(err.code, err.subcode, err.message); |
368 | 5.11k | } |
369 | | |
370 | 4.97k | return {}; |
371 | 10.0k | } |
372 | | |
373 | | Error Decoder::flush_decoder() |
374 | 4.97k | { |
375 | 4.97k | assert(m_decoder_plugin); |
376 | | |
377 | 4.97k | if (m_decoder_plugin->plugin_api_version >= 5) { |
378 | 4.97k | heif_error err = m_decoder_plugin->flush_data(m_decoder); |
379 | 4.97k | return Error::from_heif_error(err); |
380 | 4.97k | } |
381 | | |
382 | 0 | return {}; |
383 | 4.97k | } |
384 | | |
385 | | Result<std::shared_ptr<HeifPixelImage> > Decoder::get_decoded_frame(const heif_decoding_options& options, |
386 | | uintptr_t* out_user_data, |
387 | | const heif_security_limits* limits) |
388 | 23.4k | { |
389 | 23.4k | auto pluginErr = require_decoder_plugin(options); |
390 | 23.4k | if (pluginErr) { |
391 | 0 | return pluginErr; |
392 | 0 | } |
393 | | |
394 | 23.4k | heif_image* decoded_img = nullptr; |
395 | | |
396 | 23.4k | heif_error err; |
397 | | |
398 | 23.4k | if (m_decoder_plugin->plugin_api_version >= 5 && |
399 | 23.4k | m_decoder_plugin->decode_next_image2 != nullptr) { |
400 | | |
401 | 23.4k | err = m_decoder_plugin->decode_next_image2(m_decoder, &decoded_img, out_user_data, limits); |
402 | 23.4k | if (err.code != heif_error_Ok) { |
403 | 397 | return Error::from_heif_error(err); |
404 | 397 | } |
405 | 23.4k | } |
406 | 0 | else if (m_decoder_plugin->plugin_api_version >= 4 && |
407 | 0 | m_decoder_plugin->decode_next_image != nullptr) { |
408 | |
|
409 | 0 | err = m_decoder_plugin->decode_next_image(m_decoder, &decoded_img, limits); |
410 | 0 | if (err.code != heif_error_Ok) { |
411 | 0 | return Error::from_heif_error(err); |
412 | 0 | } |
413 | 0 | } |
414 | 0 | else { |
415 | 0 | err = m_decoder_plugin->decode_image(m_decoder, &decoded_img); |
416 | 0 | if (err.code != heif_error_Ok) { |
417 | 0 | return Error::from_heif_error(err); |
418 | 0 | } |
419 | 0 | } |
420 | | |
421 | 23.0k | if (!decoded_img) { |
422 | 18.8k | return {nullptr}; |
423 | 18.8k | } |
424 | | |
425 | | // -- cleanup |
426 | | |
427 | 4.20k | std::shared_ptr<HeifPixelImage> img = std::move(decoded_img->image); |
428 | 4.20k | heif_image_release(decoded_img); |
429 | | |
430 | 4.20k | return img; |
431 | 23.0k | } |
432 | | |
433 | | |
434 | | Result<std::shared_ptr<HeifPixelImage>> |
435 | | Decoder::decode_single_frame_from_compressed_data(const heif_decoding_options& options, |
436 | | const heif_security_limits* limits) |
437 | 10.3k | { |
438 | 10.3k | Error decodeError = decode_sequence_frame_from_compressed_data(true, options, 0, limits); |
439 | 10.3k | if (decodeError) { |
440 | 5.34k | return decodeError; |
441 | 5.34k | } |
442 | | |
443 | 4.97k | flush_decoder(); |
444 | | |
445 | | // We might have to try several times to get an image out of the decoder. |
446 | | // However, we stop after a maximum number of tries because the decoder might not |
447 | | // give any image when the input data is incomplete. |
448 | 4.97k | const int max_decoding_tries = 50; // hardcoded value, should be large enough |
449 | | |
450 | 23.8k | for (int i = 0; i < max_decoding_tries; i++) { |
451 | 23.4k | Result<std::shared_ptr<HeifPixelImage>> imgResult; |
452 | 23.4k | imgResult = get_decoded_frame(options, nullptr, limits); |
453 | 23.4k | if (imgResult.error()) { |
454 | 397 | return imgResult.error(); |
455 | 397 | } |
456 | | |
457 | 23.0k | if (*imgResult != nullptr) { |
458 | 4.20k | return imgResult; |
459 | 4.20k | } |
460 | 23.0k | } |
461 | | |
462 | | // We did not receive and image from the decoder. We give up. |
463 | | |
464 | 373 | return Error{ |
465 | 373 | heif_error_Decoder_plugin_error, |
466 | 373 | heif_suberror_Unspecified, |
467 | 373 | "Decoding the input data did not give a decompressed image." |
468 | 373 | }; |
469 | 4.97k | } |