/src/libheif/libheif/plugins/decoder_openh264.cc
Line | Count | Source |
1 | | /* |
2 | | * openh264 codec. |
3 | | * Copyright (c) 2024 Dirk Farin <dirk.farin@gmail.com> |
4 | | * |
5 | | * This file is part of libheif. |
6 | | * |
7 | | * libheif is free software: you can redistribute it and/or modify |
8 | | * it under the terms of the GNU Lesser General Public License as |
9 | | * published by the Free Software Foundation, either version 3 of |
10 | | * the License, or (at your option) any later version. |
11 | | * |
12 | | * libheif is distributed in the hope that it will be useful, |
13 | | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
14 | | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
15 | | * GNU Lesser General Public License for more details. |
16 | | * |
17 | | * You should have received a copy of the GNU Lesser General Public License |
18 | | * along with libheif. If not, see <http://www.gnu.org/licenses/>. |
19 | | */ |
20 | | |
21 | | #include "libheif/heif.h" |
22 | | #include "libheif/heif_plugin.h" |
23 | | #include "decoder_openh264.h" |
24 | | #include <cstring> |
25 | | #include <cassert> |
26 | | #include <vector> |
27 | | #include <cstdio> |
28 | | #include <deque> |
29 | | |
30 | | #include <wels/codec_api.h> |
31 | | #include <string> |
32 | | #include <utility> |
33 | | |
34 | | // TODO: the openh264 decoder seems to fail with some images. |
35 | | // I have not figured out yet which images are affected. |
36 | | // Maybe it has something to do with the image size. I got the error with large images. |
37 | | |
38 | | struct openh264_decoder |
39 | | { |
40 | | struct Packet |
41 | | { |
42 | | std::vector<uint8_t> data; |
43 | | uintptr_t pts; |
44 | | }; |
45 | | |
46 | | std::deque<Packet> input_data; |
47 | | bool m_eof_reached = false; |
48 | | |
49 | | std::string error_message; |
50 | | |
51 | | |
52 | | // --- decoder |
53 | | |
54 | | ISVCDecoder* decoder = nullptr; |
55 | | |
56 | | ~openh264_decoder() |
57 | 0 | { |
58 | 0 | if (decoder) { |
59 | | // Step 6:uninitialize the decoder and memory free |
60 | |
|
61 | 0 | decoder->Uninitialize(); // TODO: do we have to Uninitialize when an error is returned? |
62 | | |
63 | |
|
64 | 0 | WelsDestroyDecoder(decoder); |
65 | 0 | } |
66 | 0 | } |
67 | | }; |
68 | | |
69 | | static const char kSuccess[] = "Success"; |
70 | | |
71 | | // Reduced priority because OpenH264 cannot pass through user-data. |
72 | | // We need this feature for decoding sequences with SAI. |
73 | | // Prefer to use the FFMPEG plugin. |
74 | | static const int OpenH264_PLUGIN_PRIORITY = 70; |
75 | | |
76 | | #define MAX_PLUGIN_NAME_LENGTH 80 |
77 | | |
78 | | static char plugin_name[MAX_PLUGIN_NAME_LENGTH]; |
79 | | |
80 | | static heif_error kError_EOF = {heif_error_Decoder_plugin_error, heif_suberror_End_of_data, "Insufficient input data"}; |
81 | | static heif_error kError_invalid_data = {heif_error_Decoder_plugin_error, heif_suberror_Invalid_parameter_value, "Invalid input data"}; |
82 | | |
83 | | |
84 | | static const char* openh264_plugin_name() |
85 | 0 | { |
86 | 0 | OpenH264Version version = WelsGetCodecVersion(); |
87 | |
|
88 | 0 | sprintf(plugin_name, "OpenH264 %d.%d.%d", version.uMajor, version.uMinor, version.uRevision); |
89 | |
|
90 | 0 | return plugin_name; |
91 | 0 | } |
92 | | |
93 | | |
94 | | static void openh264_init_plugin() |
95 | 2 | { |
96 | 2 | } |
97 | | |
98 | | |
99 | | static void openh264_deinit_plugin() |
100 | 0 | { |
101 | 0 | } |
102 | | |
103 | | |
104 | | static int openh264_does_support_format(heif_compression_format format) |
105 | 0 | { |
106 | 0 | if (format == heif_compression_AVC) { |
107 | 0 | return OpenH264_PLUGIN_PRIORITY; |
108 | 0 | } |
109 | 0 | else { |
110 | 0 | return 0; |
111 | 0 | } |
112 | 0 | } |
113 | | |
114 | | static int openh264_does_support_format2(const heif_decoder_plugin_compressed_format_description* format) |
115 | 0 | { |
116 | 0 | return openh264_does_support_format(format->format); |
117 | 0 | } |
118 | | |
119 | | |
120 | | heif_error openh264_new_decoder2(void** dec, const heif_decoder_plugin_options* options) |
121 | 0 | { |
122 | 0 | auto* decoder = new openh264_decoder(); |
123 | 0 | *dec = decoder; |
124 | | |
125 | | // Step 2:decoder creation |
126 | 0 | WelsCreateDecoder(&decoder->decoder); |
127 | 0 | if (!decoder->decoder) { |
128 | 0 | return { |
129 | 0 | heif_error_Decoder_plugin_error, |
130 | 0 | heif_suberror_Unspecified, |
131 | 0 | "Cannot create OpenH264 decoder" |
132 | 0 | }; |
133 | 0 | } |
134 | | |
135 | | // Step 3:declare required parameter, used to differentiate Decoding only and Parsing only |
136 | 0 | SDecodingParam sDecParam{}; |
137 | 0 | sDecParam.sVideoProperty.eVideoBsType = VIDEO_BITSTREAM_AVC; |
138 | | |
139 | | //for Parsing only, the assignment is mandatory |
140 | | // sDecParam.bParseOnly = true; |
141 | | |
142 | | // Step 4:initialize the parameter and decoder context, allocate memory |
143 | 0 | decoder->decoder->Initialize(&sDecParam); |
144 | |
|
145 | 0 | return {heif_error_Ok, heif_suberror_Unspecified, kSuccess}; |
146 | 0 | } |
147 | | |
148 | | |
149 | | heif_error openh264_new_decoder(void** dec) |
150 | 0 | { |
151 | 0 | heif_decoder_plugin_options options{}; |
152 | 0 | options.format = heif_compression_AVC; |
153 | 0 | options.num_threads = 0; |
154 | 0 | options.strict_decoding = false; |
155 | |
|
156 | 0 | return openh264_new_decoder2(dec, &options); |
157 | 0 | } |
158 | | |
159 | | void openh264_free_decoder(void* decoder_raw) |
160 | 0 | { |
161 | 0 | auto* decoder = (openh264_decoder*) decoder_raw; |
162 | |
|
163 | 0 | if (!decoder) { |
164 | 0 | return; |
165 | 0 | } |
166 | | |
167 | 0 | delete decoder; |
168 | 0 | } |
169 | | |
170 | | |
171 | | void openh264_set_strict_decoding(void* decoder_raw, int flag) |
172 | 0 | { |
173 | | // auto* decoder = (openh264_decoder*) decoder_raw; |
174 | 0 | } |
175 | | |
176 | | |
177 | | heif_error openh264_push_data2(void* decoder_raw, const void* frame_data, size_t frame_size, uintptr_t user_data) |
178 | 0 | { |
179 | 0 | auto* decoder = (openh264_decoder*) decoder_raw; |
180 | |
|
181 | 0 | const auto* input_data = (const uint8_t*) frame_data; |
182 | |
|
183 | 0 | if (frame_size < 4) { |
184 | 0 | return kError_invalid_data; |
185 | 0 | } |
186 | | |
187 | 0 | openh264_decoder::Packet pkt; |
188 | 0 | pkt.data.insert(pkt.data.end(), input_data, input_data + frame_size); |
189 | 0 | pkt.pts = user_data; |
190 | 0 | decoder->input_data.push_back(std::move(pkt)); |
191 | |
|
192 | 0 | return {heif_error_Ok, heif_suberror_Unspecified, kSuccess}; |
193 | 0 | } |
194 | | |
195 | | heif_error openh264_push_data(void* decoder_raw, const void* frame_data, size_t frame_size) |
196 | 0 | { |
197 | 0 | return openh264_push_data2(decoder_raw, frame_data, frame_size, 0); |
198 | 0 | } |
199 | | |
200 | | |
201 | | heif_error openh264_decode_next_image2(void* decoder_raw, heif_image** out_img, |
202 | | uintptr_t* out_user_data, |
203 | | const heif_security_limits* limits) |
204 | 0 | { |
205 | 0 | auto* decoder = (openh264_decoder*) decoder_raw; |
206 | 0 | ISVCDecoder* pSvcDecoder = decoder->decoder; |
207 | | |
208 | | // --- Not enough data, but no EOF yet. |
209 | |
|
210 | 0 | if (decoder->input_data.empty() && !decoder->m_eof_reached) { |
211 | 0 | *out_img = nullptr; |
212 | 0 | return heif_error_ok; |
213 | 0 | } |
214 | | |
215 | | |
216 | | //output: [0~2] for Y,U,V buffer for Decoding only |
217 | 0 | unsigned char* pData[3] = {nullptr, nullptr, nullptr}; |
218 | | |
219 | | // in-out: for Decoding only: declare and initialize the output buffer info, this should never co-exist with Parsing only |
220 | |
|
221 | 0 | SBufferInfo sDstBufInfo; |
222 | 0 | memset(&sDstBufInfo, 0, sizeof(SBufferInfo)); |
223 | |
|
224 | 0 | int iRet; |
225 | |
|
226 | 0 | if (!decoder->input_data.empty()) { |
227 | 0 | sDstBufInfo.uiInBsTimeStamp = decoder->input_data.front().pts; |
228 | |
|
229 | 0 | const std::vector<uint8_t>& indata = decoder->input_data.front().data; |
230 | 0 | std::vector<uint8_t> scdata; |
231 | |
|
232 | 0 | size_t idx = 0; |
233 | 0 | while (idx < indata.size()) { |
234 | 0 | if (indata.size() - 4 < idx) { |
235 | 0 | return kError_EOF; |
236 | 0 | } |
237 | | |
238 | | // Compute in uint32_t. The bytes promote to 'int', so a leading byte >= 0x80 |
239 | | // shifted left by 24 lands in the sign bit and the result would be negative. |
240 | 0 | uint32_t size = (static_cast<uint32_t>(indata[idx]) << 24) | |
241 | 0 | (static_cast<uint32_t>(indata[idx + 1]) << 16) | |
242 | 0 | (static_cast<uint32_t>(indata[idx + 2]) << 8) | |
243 | 0 | static_cast<uint32_t>(indata[idx + 3]); |
244 | 0 | idx += 4; |
245 | |
|
246 | 0 | if (indata.size() < size || indata.size() - size < idx) { |
247 | 0 | return kError_EOF; |
248 | 0 | } |
249 | | |
250 | 0 | scdata.push_back(0); |
251 | 0 | scdata.push_back(0); |
252 | 0 | scdata.push_back(1); |
253 | | |
254 | | // check for need of start code emulation prevention |
255 | |
|
256 | 0 | bool do_start_code_emulation_check = true; |
257 | |
|
258 | 0 | while (do_start_code_emulation_check && size >= 3) { |
259 | 0 | bool found_start_code_emulation = false; |
260 | |
|
261 | 0 | for (size_t i = 0; i < size - 3; i++) { |
262 | 0 | if (indata[idx + 0] == 0 && |
263 | 0 | indata[idx + 1] == 0 && |
264 | 0 | (indata[idx + 2] >= 0 && indata[idx + 2] <= 3)) { |
265 | 0 | scdata.push_back(0); |
266 | 0 | scdata.push_back(0); |
267 | 0 | scdata.push_back(3); |
268 | |
|
269 | 0 | scdata.insert(scdata.end(), &indata[idx + 2], indata.data() + idx + i + 2); |
270 | 0 | idx += i + 2; |
271 | 0 | size -= (uint32_t) (i + 2); |
272 | 0 | found_start_code_emulation = true; |
273 | 0 | break; |
274 | 0 | } |
275 | 0 | } |
276 | |
|
277 | 0 | do_start_code_emulation_check = found_start_code_emulation; |
278 | 0 | } |
279 | |
|
280 | 0 | if (size == 0) { |
281 | 0 | return kError_invalid_data; |
282 | 0 | } |
283 | | // Note: we cannot write &indata[idx + size] since that would use the operator[] on an element beyond the vector range. |
284 | 0 | scdata.insert(scdata.end(), &indata[idx], indata.data() + idx + size); |
285 | |
|
286 | 0 | idx += size; |
287 | 0 | } |
288 | | |
289 | 0 | if (idx != indata.size()) { |
290 | 0 | decoder->input_data.pop_front(); |
291 | 0 | return kError_EOF; |
292 | 0 | } |
293 | | |
294 | 0 | decoder->input_data.pop_front(); |
295 | | |
296 | | // input: encoded bitstream start position; should include start code prefix |
297 | 0 | unsigned char* pBuf = scdata.data(); |
298 | | |
299 | | // input: encoded bit stream length; should include the size of start code prefix |
300 | 0 | int iSize = static_cast<int>(scdata.size()); |
301 | | |
302 | | |
303 | | // Step 5:do actual decoding process in slice level; this can be done in a loop until data ends |
304 | | |
305 | | |
306 | | //for Decoding only |
307 | 0 | iRet = pSvcDecoder->DecodeFrameNoDelay(pBuf, iSize, pData, &sDstBufInfo); |
308 | 0 | } |
309 | 0 | else { |
310 | 0 | iRet = pSvcDecoder->FlushFrame(pData, &sDstBufInfo); |
311 | 0 | } |
312 | | |
313 | 0 | if (iRet != 0) { |
314 | 0 | return { |
315 | 0 | heif_error_Decoder_plugin_error, |
316 | 0 | heif_suberror_Unspecified, |
317 | 0 | "OpenH264 decoder error" |
318 | 0 | }; |
319 | 0 | } |
320 | | |
321 | 0 | if (sDstBufInfo.iBufferStatus != 1) { |
322 | 0 | *out_img = nullptr; |
323 | 0 | return heif_error_ok; |
324 | 0 | } |
325 | | |
326 | 0 | if (out_user_data) { |
327 | 0 | *out_user_data = sDstBufInfo.uiOutYuvTimeStamp; |
328 | 0 | } |
329 | | |
330 | | /* |
331 | | // TODO: I receive an iBufferStatus==0, but the output image is still decoded |
332 | | if (sDstBufInfo.iBufferStatus == 0) { |
333 | | return {heif_error_Decoder_plugin_error, |
334 | | heif_suberror_Unspecified, |
335 | | "OpenH264 decoder did not output any image"}; |
336 | | } |
337 | | */ |
338 | |
|
339 | 0 | uint32_t width = sDstBufInfo.UsrData.sSystemBuffer.iWidth; |
340 | 0 | uint32_t height = sDstBufInfo.UsrData.sSystemBuffer.iHeight; |
341 | |
|
342 | 0 | heif_image* heif_img; |
343 | 0 | heif_error err{}; |
344 | |
|
345 | 0 | uint32_t cwidth, cheight; |
346 | |
|
347 | 0 | if (sDstBufInfo.UsrData.sSystemBuffer.iFormat == videoFormatI420) { |
348 | 0 | cwidth = (width + 1) / 2; |
349 | 0 | cheight = (height + 1) / 2; |
350 | |
|
351 | 0 | err = heif_image_create(width, height, |
352 | 0 | heif_colorspace_YCbCr, |
353 | 0 | heif_chroma_420, |
354 | 0 | &heif_img); |
355 | 0 | if (err.code != heif_error_Ok) { |
356 | 0 | assert(heif_img == nullptr); |
357 | 0 | return err; |
358 | 0 | } |
359 | | |
360 | 0 | *out_img = heif_img; |
361 | |
|
362 | 0 | err = heif_image_add_plane_safe(heif_img, heif_channel_Y, width, height, 8, limits); |
363 | 0 | if (err.code != heif_error_Ok) { |
364 | | // copy error message to decoder object because heif_image will be released |
365 | 0 | decoder->error_message = err.message; |
366 | 0 | err.message = decoder->error_message.c_str(); |
367 | |
|
368 | 0 | heif_image_release(heif_img); |
369 | 0 | return err; |
370 | 0 | } |
371 | | |
372 | 0 | err = heif_image_add_plane_safe(heif_img, heif_channel_Cb, cwidth, cheight, 8, limits); |
373 | 0 | if (err.code != heif_error_Ok) { |
374 | | // copy error message to decoder object because heif_image will be released |
375 | 0 | decoder->error_message = err.message; |
376 | 0 | err.message = decoder->error_message.c_str(); |
377 | |
|
378 | 0 | heif_image_release(heif_img); |
379 | 0 | return err; |
380 | 0 | } |
381 | | |
382 | 0 | err = heif_image_add_plane_safe(heif_img, heif_channel_Cr, cwidth, cheight, 8, limits); |
383 | 0 | if (err.code != heif_error_Ok) { |
384 | | // copy error message to decoder object because heif_image will be released |
385 | 0 | decoder->error_message = err.message; |
386 | 0 | err.message = decoder->error_message.c_str(); |
387 | |
|
388 | 0 | heif_image_release(heif_img); |
389 | 0 | return err; |
390 | 0 | } |
391 | | |
392 | 0 | size_t y_stride; |
393 | 0 | size_t cb_stride; |
394 | 0 | size_t cr_stride; |
395 | 0 | uint8_t* py = heif_image_get_plane2(heif_img, heif_channel_Y, &y_stride); |
396 | 0 | uint8_t* pcb = heif_image_get_plane2(heif_img, heif_channel_Cb, &cb_stride); |
397 | 0 | uint8_t* pcr = heif_image_get_plane2(heif_img, heif_channel_Cr, &cr_stride); |
398 | |
|
399 | 0 | int ystride = sDstBufInfo.UsrData.sSystemBuffer.iStride[0]; |
400 | 0 | int cstride = sDstBufInfo.UsrData.sSystemBuffer.iStride[1]; |
401 | |
|
402 | 0 | for (uint32_t y = 0; y < height; y++) { |
403 | 0 | memcpy(py + y * y_stride, sDstBufInfo.pDst[0] + y * ystride, width); |
404 | 0 | } |
405 | |
|
406 | 0 | for (uint32_t y = 0; y < (height + 1) / 2; y++) { |
407 | 0 | memcpy(pcb + y * cb_stride, sDstBufInfo.pDst[1] + y * cstride, (width + 1) / 2); |
408 | 0 | memcpy(pcr + y * cr_stride, sDstBufInfo.pDst[2] + y * cstride, (width + 1) / 2); |
409 | 0 | } |
410 | 0 | } |
411 | 0 | else { |
412 | 0 | return { |
413 | 0 | heif_error_Decoder_plugin_error, |
414 | 0 | heif_suberror_Unspecified, |
415 | 0 | "Unsupported image pixel format" |
416 | 0 | }; |
417 | 0 | } |
418 | | |
419 | | // decoder->data.clear(); |
420 | | |
421 | 0 | return heif_error_ok; |
422 | 0 | } |
423 | | |
424 | | |
425 | | heif_error openh264_decode_next_image(void* decoder_raw, heif_image** out_img, |
426 | | const heif_security_limits* limits) |
427 | 0 | { |
428 | 0 | return openh264_decode_next_image2(decoder_raw, out_img, nullptr, limits); |
429 | 0 | } |
430 | | |
431 | | heif_error openh264_decode_image(void* decoder_raw, heif_image** out_img) |
432 | 0 | { |
433 | 0 | auto* limits = heif_get_global_security_limits(); |
434 | 0 | return openh264_decode_next_image(decoder_raw, out_img, limits); |
435 | 0 | } |
436 | | |
437 | | |
438 | | heif_error openh264_flush_data(void* decoder_raw) |
439 | 0 | { |
440 | 0 | auto* decoder = (struct openh264_decoder*) decoder_raw; |
441 | |
|
442 | 0 | decoder->m_eof_reached = true; |
443 | |
|
444 | 0 | return heif_error_ok; |
445 | 0 | } |
446 | | |
447 | | |
448 | | static const heif_decoder_plugin decoder_openh264{ |
449 | | 5, |
450 | | openh264_plugin_name, |
451 | | openh264_init_plugin, |
452 | | openh264_deinit_plugin, |
453 | | openh264_does_support_format, |
454 | | openh264_new_decoder, |
455 | | openh264_free_decoder, |
456 | | openh264_push_data, |
457 | | openh264_decode_image, |
458 | | openh264_set_strict_decoding, |
459 | | "openh264", |
460 | | openh264_decode_next_image, |
461 | | /* minimum_required_libheif_version */ LIBHEIF_MAKE_VERSION(1,21,0), |
462 | | openh264_does_support_format2, |
463 | | openh264_new_decoder2, |
464 | | openh264_push_data2, |
465 | | openh264_flush_data, |
466 | | openh264_decode_next_image2 |
467 | | }; |
468 | | |
469 | | |
470 | | const heif_decoder_plugin* get_decoder_plugin_openh264() |
471 | 2 | { |
472 | 2 | return &decoder_openh264; |
473 | 2 | } |
474 | | |
475 | | |
476 | | #if PLUGIN_OpenH264_DECODER |
477 | | heif_plugin_info plugin_info { |
478 | | 1, |
479 | | heif_plugin_type_decoder, |
480 | | &decoder_openh264 |
481 | | }; |
482 | | #endif |