/src/libheif/libheif/plugins/decoder_vvdec.cc
Line | Count | Source |
1 | | /* |
2 | | * AVIF codec. |
3 | | * Copyright (c) 2019 Dirk Farin <dirk.farin@gmail.com> |
4 | | * |
5 | | * This file is part of libheif. |
6 | | * |
7 | | * libheif is free software: you can redistribute it and/or modify |
8 | | * it under the terms of the GNU Lesser General Public License as |
9 | | * published by the Free Software Foundation, either version 3 of |
10 | | * the License, or (at your option) any later version. |
11 | | * |
12 | | * libheif is distributed in the hope that it will be useful, |
13 | | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
14 | | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
15 | | * GNU Lesser General Public License for more details. |
16 | | * |
17 | | * You should have received a copy of the GNU Lesser General Public License |
18 | | * along with libheif. If not, see <http://www.gnu.org/licenses/>. |
19 | | */ |
20 | | |
21 | | #include "libheif/heif.h" |
22 | | #include "libheif/heif_plugin.h" |
23 | | #include "decoder_vvdec.h" |
24 | | #include <cstring> |
25 | | #include <cassert> |
26 | | #include <vector> |
27 | | #include <algorithm> |
28 | | #include <deque> |
29 | | #include <string> |
30 | | |
31 | | #include <vvdec/vvdec.h> |
32 | | #include <utility> |
33 | | |
34 | | #if 0 |
35 | | #include <iostream> |
36 | | #include <logging.h> |
37 | | #endif |
38 | | |
39 | | |
40 | | struct vvdec_decoder |
41 | | { |
42 | | vvdecDecoder* decoder = nullptr; |
43 | | vvdecAccessUnit* au = nullptr; |
44 | | |
45 | | bool strict_decoding = false; |
46 | | |
47 | | struct Packet |
48 | | { |
49 | | std::vector<uint8_t> data; |
50 | | uintptr_t user_data; |
51 | | }; |
52 | | std::deque<Packet> nalus; |
53 | | bool end_of_stream_reached = false; |
54 | | |
55 | | std::string error_message; |
56 | | }; |
57 | | |
58 | | static const char kEmptyString[] = ""; |
59 | | static const char kSuccess[] = "Success"; |
60 | | |
61 | | static const int VVDEC_PLUGIN_PRIORITY = 100; |
62 | | |
63 | | #define MAX_PLUGIN_NAME_LENGTH 80 |
64 | | |
65 | | static char plugin_name[MAX_PLUGIN_NAME_LENGTH]; |
66 | | |
67 | | |
68 | | static const char* vvdec_plugin_name() |
69 | 9 | { |
70 | 9 | const char* version = vvdec_get_version(); |
71 | | |
72 | 9 | if (strlen(version) < 60) { |
73 | 9 | sprintf(plugin_name, "VVCDEC decoder (%s)", version); |
74 | 9 | } |
75 | 0 | else { |
76 | 0 | strcpy(plugin_name, "VVDEC decoder"); |
77 | 0 | } |
78 | | |
79 | 9 | return plugin_name; |
80 | 9 | } |
81 | | |
82 | | |
83 | | static void vvdec_init_plugin() |
84 | 254 | { |
85 | 254 | } |
86 | | |
87 | | |
88 | | static void vvdec_deinit_plugin() |
89 | 0 | { |
90 | 0 | } |
91 | | |
92 | | |
93 | | static int vvdec_does_support_format(heif_compression_format format) |
94 | 48.5k | { |
95 | 48.5k | if (format == heif_compression_VVC) { |
96 | 422 | return VVDEC_PLUGIN_PRIORITY; |
97 | 422 | } |
98 | 48.1k | else { |
99 | 48.1k | return 0; |
100 | 48.1k | } |
101 | 48.5k | } |
102 | | |
103 | | |
104 | | static int vvdec_does_support_format2(const heif_decoder_plugin_compressed_format_description* format) |
105 | 0 | { |
106 | 0 | return vvdec_does_support_format(format->format); |
107 | 0 | } |
108 | | |
109 | | heif_error vvdec_new_decoder2(void** dec, const heif_decoder_plugin_options* options) |
110 | 404 | { |
111 | 404 | auto* decoder = new vvdec_decoder(); |
112 | | |
113 | 404 | vvdecParams params; |
114 | 404 | vvdec_params_default(¶ms); |
115 | 404 | params.logLevel = VVDEC_INFO; |
116 | | |
117 | 404 | if (options->num_threads) { |
118 | 0 | params.threads = options->num_threads; |
119 | 0 | } |
120 | 404 | decoder->decoder = vvdec_decoder_open(¶ms); |
121 | | |
122 | 404 | const int MaxNaluSize = 256 * 1024; |
123 | 404 | decoder->au = vvdec_accessUnit_alloc(); |
124 | 404 | vvdec_accessUnit_default(decoder->au); |
125 | 404 | vvdec_accessUnit_alloc_payload(decoder->au, MaxNaluSize); |
126 | | |
127 | 404 | *dec = decoder; |
128 | | |
129 | 404 | return {heif_error_Ok, heif_suberror_Unspecified, kSuccess}; |
130 | 404 | } |
131 | | |
132 | | |
133 | | heif_error vvdec_new_decoder(void** dec) |
134 | 0 | { |
135 | 0 | heif_decoder_plugin_options options{}; |
136 | 0 | options.format = heif_compression_VVC; |
137 | 0 | options.num_threads = 0; |
138 | 0 | options.strict_decoding = false; |
139 | |
|
140 | 0 | vvdec_new_decoder2(dec, &options); |
141 | |
|
142 | 0 | return {}; |
143 | 0 | } |
144 | | |
145 | | |
146 | | void vvdec_free_decoder(void* decoder_raw) |
147 | 404 | { |
148 | 404 | auto* decoder = (vvdec_decoder*) decoder_raw; |
149 | | |
150 | 404 | if (!decoder) { |
151 | 0 | return; |
152 | 0 | } |
153 | | |
154 | 404 | if (decoder->au) { |
155 | 404 | vvdec_accessUnit_free(decoder->au); |
156 | 404 | decoder->au = nullptr; |
157 | 404 | } |
158 | | |
159 | 404 | if (decoder->decoder) { |
160 | 404 | vvdec_decoder_close(decoder->decoder); |
161 | 404 | decoder->decoder = nullptr; |
162 | 404 | } |
163 | | |
164 | 404 | delete decoder; |
165 | 404 | } |
166 | | |
167 | | |
168 | | void vvdec_set_strict_decoding(void* decoder_raw, int flag) |
169 | 0 | { |
170 | 0 | auto* decoder = (vvdec_decoder*) decoder_raw; |
171 | |
|
172 | 0 | decoder->strict_decoding = flag; |
173 | 0 | } |
174 | | |
175 | | |
176 | | heif_error vvdec_push_data2(void* decoder_raw, const void* frame_data, size_t frame_size, |
177 | | uintptr_t user_data) |
178 | 401 | { |
179 | 401 | auto* decoder = (vvdec_decoder*) decoder_raw; |
180 | | |
181 | 401 | const auto* data = (const uint8_t*) frame_data; |
182 | | |
183 | 1.99k | while (frame_size > 0) { |
184 | 1.60k | if (frame_size < 4) { |
185 | 2 | return { |
186 | 2 | heif_error_Decoder_plugin_error, |
187 | 2 | heif_suberror_End_of_data, |
188 | 2 | kEmptyString |
189 | 2 | }; |
190 | 2 | } |
191 | | |
192 | 1.60k | uint32_t size = four_bytes_to_uint32(data[0], data[1], data[2], data[3]); |
193 | | |
194 | 1.60k | if (frame_size - 4 < size) { |
195 | 6 | return { |
196 | 6 | heif_error_Decoder_plugin_error, |
197 | 6 | heif_suberror_End_of_data, |
198 | 6 | kEmptyString |
199 | 6 | }; |
200 | 6 | } |
201 | | |
202 | 1.59k | data += 4; |
203 | 1.59k | frame_size -= 4; |
204 | | |
205 | 1.59k | std::vector<uint8_t> nalu; |
206 | 1.59k | nalu.push_back(0); |
207 | 1.59k | nalu.push_back(0); |
208 | 1.59k | nalu.push_back(1); |
209 | 1.59k | nalu.insert(nalu.end(), data, data + size); |
210 | | |
211 | 1.59k | decoder->nalus.push_back({std::move(nalu), user_data}); |
212 | 1.59k | data += size; |
213 | 1.59k | frame_size -= size; |
214 | 1.59k | } |
215 | | |
216 | 393 | return heif_error_ok; |
217 | 401 | } |
218 | | |
219 | | heif_error vvdec_push_data(void* decoder_raw, const void* frame_data, size_t frame_size) |
220 | 0 | { |
221 | 0 | return vvdec_push_data2(decoder_raw, frame_data, frame_size, 0); |
222 | 0 | } |
223 | | |
224 | | heif_error vvdec_decode_next_image2(void* decoder_raw, heif_image** out_img, |
225 | | uintptr_t* out_user_data, |
226 | | const heif_security_limits* limits) |
227 | 395 | { |
228 | 395 | auto* decoder = (vvdec_decoder*) decoder_raw; |
229 | | |
230 | 395 | vvdecFrame* frame = nullptr; |
231 | | |
232 | | // --- prepare AU payload with maximum NALU size |
233 | | |
234 | 395 | size_t max_payload_size = 0; |
235 | 1.56k | for (const auto& nalu : decoder->nalus) { |
236 | 1.56k | max_payload_size = std::max(max_payload_size, nalu.data.size()); |
237 | 1.56k | } |
238 | | |
239 | 395 | if (decoder->au == nullptr || max_payload_size > (size_t) decoder->au->payloadSize) { |
240 | 0 | if (decoder->au) { |
241 | 0 | vvdec_accessUnit_free(decoder->au); |
242 | 0 | } |
243 | |
|
244 | 0 | decoder->au = vvdec_accessUnit_alloc(); |
245 | 0 | vvdec_accessUnit_default(decoder->au); |
246 | 0 | vvdec_accessUnit_alloc_payload(decoder->au, (int)max_payload_size); |
247 | 0 | } |
248 | | |
249 | | // --- feed NALUs into decoder, flush when done |
250 | | |
251 | 1.64k | for (;;) { |
252 | 1.64k | int ret; |
253 | | |
254 | | // -> end of stream reached |
255 | 1.64k | if (decoder->nalus.empty() && decoder->end_of_stream_reached) { |
256 | 231 | ret = vvdec_flush(decoder->decoder, &frame); |
257 | 231 | } |
258 | | // -> not enough data to decode an image |
259 | 1.41k | else if (decoder->nalus.empty()) { |
260 | 0 | *out_img = nullptr; |
261 | 0 | return heif_error_ok; |
262 | 0 | } |
263 | | // -> push NALs from queue into decoder |
264 | 1.41k | else { |
265 | 1.41k | const auto& nalu = decoder->nalus.front(); |
266 | | |
267 | 1.41k | memcpy(decoder->au->payload, nalu.data.data(), nalu.data.size()); |
268 | 1.41k | decoder->au->payloadUsedSize = (int) nalu.data.size(); |
269 | 1.41k | decoder->au->cts = nalu.user_data; |
270 | 1.41k | decoder->au->ctsValid = true; |
271 | 1.41k | decoder->nalus.pop_front(); |
272 | 1.41k | ret = vvdec_decode(decoder->decoder, decoder->au, &frame); |
273 | 1.41k | } |
274 | | |
275 | 1.64k | if (ret == VVDEC_OK && frame) { |
276 | 3 | break; |
277 | 3 | } |
278 | | |
279 | 1.64k | if (ret == VVDEC_EOF) { |
280 | 2 | assert(!frame); |
281 | 2 | *out_img = nullptr; |
282 | 2 | return heif_error_ok; |
283 | 2 | } |
284 | | |
285 | 1.64k | if (ret != VVDEC_OK && ret != VVDEC_TRY_AGAIN) { |
286 | 390 | return {heif_error_Decoder_plugin_error, heif_suberror_Unspecified, "vvdec decoding error"}; |
287 | 390 | } |
288 | 1.64k | } |
289 | | |
290 | 3 | if (out_user_data) { |
291 | 0 | *out_user_data = frame->cts; |
292 | 0 | } |
293 | | |
294 | | |
295 | | // --- convert decoded frame to heif_image |
296 | | |
297 | 3 | heif_chroma chroma; |
298 | 3 | heif_colorspace colorspace; |
299 | | |
300 | 3 | if (frame->colorFormat == VVDEC_CF_YUV400_PLANAR) { |
301 | 0 | chroma = heif_chroma_monochrome; |
302 | 0 | colorspace = heif_colorspace_monochrome; |
303 | 0 | } |
304 | 3 | else { |
305 | 3 | if (frame->colorFormat == VVDEC_CF_YUV444_PLANAR) { |
306 | 0 | chroma = heif_chroma_444; |
307 | 0 | } |
308 | 3 | else if (frame->colorFormat == VVDEC_CF_YUV422_PLANAR) { |
309 | 0 | chroma = heif_chroma_422; |
310 | 0 | } |
311 | 3 | else { |
312 | 3 | chroma = heif_chroma_420; |
313 | 3 | } |
314 | 3 | colorspace = heif_colorspace_YCbCr; |
315 | 3 | } |
316 | | |
317 | 3 | heif_image* heif_img = nullptr; |
318 | 3 | heif_error err = heif_image_create((int) frame->width, |
319 | 3 | (int) frame->height, |
320 | 3 | colorspace, |
321 | 3 | chroma, |
322 | 3 | &heif_img); |
323 | 3 | if (err.code != heif_error_Ok) { |
324 | 0 | assert(heif_img == nullptr); |
325 | 0 | return err; |
326 | 0 | } |
327 | | |
328 | | |
329 | | // --- read nclx parameters from decoded AV1 bitstream |
330 | | |
331 | | #if 0 |
332 | | heif_color_profile_nclx nclx; |
333 | | nclx.version = 1; |
334 | | HEIF_WARN_OR_FAIL(decoder->strict_decoding, heif_img, heif_nclx_color_profile_set_color_primaries(&nclx, static_cast<uint16_t>(img->cp)), { heif_image_release(heif_img); }); |
335 | | HEIF_WARN_OR_FAIL(decoder->strict_decoding, heif_img, heif_nclx_color_profile_set_transfer_characteristics(&nclx, static_cast<uint16_t>(img->tc)), { heif_image_release(heif_img); }); |
336 | | HEIF_WARN_OR_FAIL(decoder->strict_decoding, heif_img, heif_nclx_color_profile_set_matrix_coefficients(&nclx, static_cast<uint16_t>(img->mc)), { heif_image_release(heif_img); }); |
337 | | nclx.full_range_flag = (img->range == AOM_CR_FULL_RANGE); |
338 | | heif_image_set_nclx_color_profile(heif_img, &nclx); |
339 | | #endif |
340 | | |
341 | | // --- transfer data from vvdecFrame to HeifPixelImage |
342 | | |
343 | 3 | heif_channel channel2plane[3] = { |
344 | 3 | heif_channel_Y, |
345 | 3 | heif_channel_Cb, |
346 | 3 | heif_channel_Cr |
347 | 3 | }; |
348 | | |
349 | | |
350 | 3 | int num_planes = (chroma == heif_chroma_monochrome ? 1 : 3); |
351 | | |
352 | 12 | for (int c = 0; c < num_planes; c++) { |
353 | 9 | int bpp = (int)frame->bitDepth; |
354 | | |
355 | 9 | const auto& plane = frame->planes[c]; |
356 | 9 | const uint8_t* data = plane.ptr; |
357 | 9 | int stride = (int)plane.stride; |
358 | | |
359 | 9 | int w = (int)plane.width; |
360 | 9 | int h = (int)plane.height; |
361 | | |
362 | 9 | err = heif_image_add_plane_safe(heif_img, channel2plane[c], w, h, bpp, limits); |
363 | 9 | if (err.code != heif_error_Ok) { |
364 | | // copy error message to decoder object because heif_image will be released |
365 | 0 | decoder->error_message = err.message; |
366 | 0 | err.message = decoder->error_message.c_str(); |
367 | |
|
368 | 0 | heif_image_release(heif_img); |
369 | 0 | return err; |
370 | 0 | } |
371 | | |
372 | 9 | size_t dst_stride; |
373 | 9 | uint8_t* dst_mem = heif_image_get_plane2(heif_img, channel2plane[c], &dst_stride); |
374 | | |
375 | 9 | int bytes_per_pixel = (bpp + 7) / 8; |
376 | | |
377 | 1.16k | for (int y = 0; y < h; y++) { |
378 | 1.15k | memcpy(dst_mem + y * dst_stride, data + static_cast<size_t>(y) * stride, static_cast<size_t>(w) * bytes_per_pixel); |
379 | 1.15k | } |
380 | | |
381 | | #if 0 |
382 | | std::cout << "DATA " << c << " " << w << " " << h << " bpp:" << bpp << "\n"; |
383 | | std::cout << write_raw_data_as_hex(dst_mem, w*h, {}, {}); |
384 | | std::cout << "---\n"; |
385 | | #endif |
386 | 9 | } |
387 | | |
388 | 3 | *out_img = heif_img; |
389 | | |
390 | 3 | vvdec_frame_unref(decoder->decoder, frame); |
391 | | |
392 | 3 | return err; |
393 | 3 | } |
394 | | |
395 | | heif_error vvdec_decode_next_image(void* decoder_raw, heif_image** out_img, |
396 | | const heif_security_limits* limits) |
397 | 0 | { |
398 | 0 | return vvdec_decode_next_image2(decoder_raw, out_img, nullptr, limits); |
399 | 0 | } |
400 | | |
401 | | heif_error vvdec_decode_image(void* decoder_raw, heif_image** out_img) |
402 | 0 | { |
403 | 0 | auto* limits = heif_get_global_security_limits(); |
404 | 0 | return vvdec_decode_next_image(decoder_raw, out_img, limits); |
405 | 0 | } |
406 | | |
407 | | heif_error vvdec_flush_data(void* decoder_raw) |
408 | 393 | { |
409 | 393 | auto* decoder = (vvdec_decoder*) decoder_raw; |
410 | 393 | decoder->end_of_stream_reached = true; |
411 | 393 | return heif_error_ok; |
412 | 393 | } |
413 | | |
414 | | static const heif_decoder_plugin decoder_vvdec |
415 | | { |
416 | | 5, |
417 | | vvdec_plugin_name, |
418 | | vvdec_init_plugin, |
419 | | vvdec_deinit_plugin, |
420 | | vvdec_does_support_format, |
421 | | vvdec_new_decoder, |
422 | | vvdec_free_decoder, |
423 | | vvdec_push_data, |
424 | | vvdec_decode_image, |
425 | | vvdec_set_strict_decoding, |
426 | | "vvdec", |
427 | | vvdec_decode_next_image, |
428 | | /* minimum_required_libheif_version */ LIBHEIF_MAKE_VERSION(1,21,0), |
429 | | vvdec_does_support_format2, |
430 | | vvdec_new_decoder2, |
431 | | vvdec_push_data2, |
432 | | vvdec_flush_data, |
433 | | vvdec_decode_next_image2 |
434 | | }; |
435 | | |
436 | | |
437 | | const heif_decoder_plugin* get_decoder_plugin_vvdec() |
438 | 254 | { |
439 | 254 | return &decoder_vvdec; |
440 | 254 | } |
441 | | |
442 | | |
443 | | #if PLUGIN_VVDEC |
444 | | heif_plugin_info plugin_info { |
445 | | 1, |
446 | | heif_plugin_type_decoder, |
447 | | &decoder_vvdec |
448 | | }; |
449 | | #endif |