/src/libheif/libheif/plugins/decoder_openh264.cc
Line | Count | Source |
1 | | /* |
2 | | * openh264 codec. |
3 | | * Copyright (c) 2024 Dirk Farin <dirk.farin@gmail.com> |
4 | | * |
5 | | * This file is part of libheif. |
6 | | * |
7 | | * libheif is free software: you can redistribute it and/or modify |
8 | | * it under the terms of the GNU Lesser General Public License as |
9 | | * published by the Free Software Foundation, either version 3 of |
10 | | * the License, or (at your option) any later version. |
11 | | * |
12 | | * libheif is distributed in the hope that it will be useful, |
13 | | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
14 | | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
15 | | * GNU Lesser General Public License for more details. |
16 | | * |
17 | | * You should have received a copy of the GNU Lesser General Public License |
18 | | * along with libheif. If not, see <http://www.gnu.org/licenses/>. |
19 | | */ |
20 | | |
21 | | #include "libheif/heif.h" |
22 | | #include "libheif/heif_plugin.h" |
23 | | #include "decoder_openh264.h" |
24 | | #include <cstring> |
25 | | #include <cassert> |
26 | | #include <vector> |
27 | | #include <cstdio> |
28 | | #include <deque> |
29 | | |
30 | | #include <wels/codec_api.h> |
31 | | #include <string> |
32 | | #include <utility> |
33 | | |
34 | | // TODO: the openh264 decoder seems to fail with some images. |
35 | | // I have not figured out yet which images are affected. |
36 | | // Maybe it has something to do with the image size. I got the error with large images. |
37 | | |
38 | | struct openh264_decoder |
39 | | { |
40 | | struct Packet |
41 | | { |
42 | | std::vector<uint8_t> data; |
43 | | uintptr_t pts; |
44 | | }; |
45 | | |
46 | | std::deque<Packet> input_data; |
47 | | bool m_eof_reached = false; |
48 | | |
49 | | std::string error_message; |
50 | | |
51 | | |
52 | | // --- decoder |
53 | | |
54 | | ISVCDecoder* decoder = nullptr; |
55 | | |
56 | | ~openh264_decoder() |
57 | 770 | { |
58 | 770 | if (decoder) { |
59 | | // Step 6:uninitialize the decoder and memory free |
60 | | |
61 | 770 | decoder->Uninitialize(); // TODO: do we have to Uninitialize when an error is returned? |
62 | | |
63 | | |
64 | 770 | WelsDestroyDecoder(decoder); |
65 | 770 | } |
66 | 770 | } |
67 | | }; |
68 | | |
69 | | static const char kSuccess[] = "Success"; |
70 | | |
71 | | // Reduced priority because OpenH264 cannot pass through user-data. |
72 | | // We need this feature for decoding sequences with SAI. |
73 | | // Prefer to use the FFMPEG plugin. |
74 | | static const int OpenH264_PLUGIN_PRIORITY = 70; |
75 | | |
76 | | #define MAX_PLUGIN_NAME_LENGTH 80 |
77 | | |
78 | | static char plugin_name[MAX_PLUGIN_NAME_LENGTH]; |
79 | | |
80 | | static heif_error kError_EOF = {heif_error_Decoder_plugin_error, heif_suberror_End_of_data, "Insufficient input data"}; |
81 | | static heif_error kError_invalid_data = {heif_error_Decoder_plugin_error, heif_suberror_Invalid_parameter_value, "Invalid input data"}; |
82 | | |
83 | | |
84 | | static const char* openh264_plugin_name() |
85 | 9 | { |
86 | 9 | OpenH264Version version = WelsGetCodecVersion(); |
87 | | |
88 | 9 | sprintf(plugin_name, "OpenH264 %d.%d.%d", version.uMajor, version.uMinor, version.uRevision); |
89 | | |
90 | 9 | return plugin_name; |
91 | 9 | } |
92 | | |
93 | | |
94 | | static void openh264_init_plugin() |
95 | 254 | { |
96 | 254 | } |
97 | | |
98 | | |
99 | | static void openh264_deinit_plugin() |
100 | 0 | { |
101 | 0 | } |
102 | | |
103 | | |
104 | | static int openh264_does_support_format(heif_compression_format format) |
105 | 59.4k | { |
106 | 59.4k | if (format == heif_compression_AVC) { |
107 | 803 | return OpenH264_PLUGIN_PRIORITY; |
108 | 803 | } |
109 | 58.6k | else { |
110 | 58.6k | return 0; |
111 | 58.6k | } |
112 | 59.4k | } |
113 | | |
114 | | static int openh264_does_support_format2(const heif_decoder_plugin_compressed_format_description* format) |
115 | 0 | { |
116 | 0 | return openh264_does_support_format(format->format); |
117 | 0 | } |
118 | | |
119 | | |
120 | | heif_error openh264_new_decoder2(void** dec, const heif_decoder_plugin_options* options) |
121 | 770 | { |
122 | 770 | auto* decoder = new openh264_decoder(); |
123 | 770 | *dec = decoder; |
124 | | |
125 | | // Step 2:decoder creation |
126 | 770 | WelsCreateDecoder(&decoder->decoder); |
127 | 770 | if (!decoder->decoder) { |
128 | 0 | return { |
129 | 0 | heif_error_Decoder_plugin_error, |
130 | 0 | heif_suberror_Unspecified, |
131 | 0 | "Cannot create OpenH264 decoder" |
132 | 0 | }; |
133 | 0 | } |
134 | | |
135 | | // Step 3:declare required parameter, used to differentiate Decoding only and Parsing only |
136 | 770 | SDecodingParam sDecParam{}; |
137 | 770 | sDecParam.sVideoProperty.eVideoBsType = VIDEO_BITSTREAM_AVC; |
138 | | |
139 | | //for Parsing only, the assignment is mandatory |
140 | | // sDecParam.bParseOnly = true; |
141 | | |
142 | | // Step 4:initialize the parameter and decoder context, allocate memory |
143 | 770 | decoder->decoder->Initialize(&sDecParam); |
144 | | |
145 | 770 | return {heif_error_Ok, heif_suberror_Unspecified, kSuccess}; |
146 | 770 | } |
147 | | |
148 | | |
149 | | heif_error openh264_new_decoder(void** dec) |
150 | 0 | { |
151 | 0 | heif_decoder_plugin_options options{}; |
152 | 0 | options.format = heif_compression_AVC; |
153 | 0 | options.num_threads = 0; |
154 | 0 | options.strict_decoding = false; |
155 | |
|
156 | 0 | return openh264_new_decoder2(dec, &options); |
157 | 0 | } |
158 | | |
159 | | void openh264_free_decoder(void* decoder_raw) |
160 | 770 | { |
161 | 770 | auto* decoder = (openh264_decoder*) decoder_raw; |
162 | | |
163 | 770 | if (!decoder) { |
164 | 0 | return; |
165 | 0 | } |
166 | | |
167 | 770 | delete decoder; |
168 | 770 | } |
169 | | |
170 | | |
171 | | void openh264_set_strict_decoding(void* decoder_raw, int flag) |
172 | 0 | { |
173 | | // auto* decoder = (openh264_decoder*) decoder_raw; |
174 | 0 | } |
175 | | |
176 | | |
177 | | heif_error openh264_push_data2(void* decoder_raw, const void* frame_data, size_t frame_size, uintptr_t user_data) |
178 | 762 | { |
179 | 762 | auto* decoder = (openh264_decoder*) decoder_raw; |
180 | | |
181 | 762 | const auto* input_data = (const uint8_t*) frame_data; |
182 | | |
183 | 762 | if (frame_size < 4) { |
184 | 0 | return kError_invalid_data; |
185 | 0 | } |
186 | | |
187 | 762 | openh264_decoder::Packet pkt; |
188 | 762 | pkt.data.insert(pkt.data.end(), input_data, input_data + frame_size); |
189 | 762 | pkt.pts = user_data; |
190 | 762 | decoder->input_data.push_back(std::move(pkt)); |
191 | | |
192 | 762 | return {heif_error_Ok, heif_suberror_Unspecified, kSuccess}; |
193 | 762 | } |
194 | | |
195 | | heif_error openh264_push_data(void* decoder_raw, const void* frame_data, size_t frame_size) |
196 | 0 | { |
197 | 0 | return openh264_push_data2(decoder_raw, frame_data, frame_size, 0); |
198 | 0 | } |
199 | | |
200 | | |
201 | | heif_error openh264_decode_next_image2(void* decoder_raw, heif_image** out_img, |
202 | | uintptr_t* out_user_data, |
203 | | const heif_security_limits* limits) |
204 | 3.21k | { |
205 | 3.21k | auto* decoder = (openh264_decoder*) decoder_raw; |
206 | 3.21k | ISVCDecoder* pSvcDecoder = decoder->decoder; |
207 | | |
208 | | // --- Not enough data, but no EOF yet. |
209 | | |
210 | 3.21k | if (decoder->input_data.empty() && !decoder->m_eof_reached) { |
211 | 0 | *out_img = nullptr; |
212 | 0 | return heif_error_ok; |
213 | 0 | } |
214 | | |
215 | | |
216 | | //output: [0~2] for Y,U,V buffer for Decoding only |
217 | 3.21k | unsigned char* pData[3] = {nullptr, nullptr, nullptr}; |
218 | | |
219 | | // in-out: for Decoding only: declare and initialize the output buffer info, this should never co-exist with Parsing only |
220 | | |
221 | 3.21k | SBufferInfo sDstBufInfo; |
222 | 3.21k | memset(&sDstBufInfo, 0, sizeof(SBufferInfo)); |
223 | | |
224 | 3.21k | int iRet; |
225 | | |
226 | 3.21k | if (!decoder->input_data.empty()) { |
227 | 762 | sDstBufInfo.uiInBsTimeStamp = decoder->input_data.front().pts; |
228 | | |
229 | 762 | const std::vector<uint8_t>& indata = decoder->input_data.front().data; |
230 | 762 | std::vector<uint8_t> scdata; |
231 | | |
232 | 762 | size_t idx = 0; |
233 | 3.77k | while (idx < indata.size()) { |
234 | 3.04k | if (indata.size() - 4 < idx) { |
235 | 1 | return kError_EOF; |
236 | 1 | } |
237 | | |
238 | 3.04k | uint32_t size = ((indata[idx] << 24) | (indata[idx + 1] << 16) | (indata[idx + 2] << 8) | indata[idx + 3]); |
239 | 3.04k | idx += 4; |
240 | | |
241 | 3.04k | if (indata.size() < size || indata.size() - size < idx) { |
242 | 23 | return kError_EOF; |
243 | 23 | } |
244 | | |
245 | 3.01k | scdata.push_back(0); |
246 | 3.01k | scdata.push_back(0); |
247 | 3.01k | scdata.push_back(1); |
248 | | |
249 | | // check for need of start code emulation prevention |
250 | | |
251 | 3.01k | bool do_start_code_emulation_check = true; |
252 | | |
253 | 7.74k | while (do_start_code_emulation_check && size >= 3) { |
254 | 4.72k | bool found_start_code_emulation = false; |
255 | | |
256 | 1.57M | for (size_t i = 0; i < size - 3; i++) { |
257 | 1.57M | if (indata[idx + 0] == 0 && |
258 | 16.9k | indata[idx + 1] == 0 && |
259 | 9.67k | (indata[idx + 2] >= 0 && indata[idx + 2] <= 3)) { |
260 | 1.71k | scdata.push_back(0); |
261 | 1.71k | scdata.push_back(0); |
262 | 1.71k | scdata.push_back(3); |
263 | | |
264 | 1.71k | scdata.insert(scdata.end(), &indata[idx + 2], indata.data() + idx + i + 2); |
265 | 1.71k | idx += i + 2; |
266 | 1.71k | size -= (uint32_t) (i + 2); |
267 | 1.71k | found_start_code_emulation = true; |
268 | 1.71k | break; |
269 | 1.71k | } |
270 | 1.57M | } |
271 | | |
272 | 4.72k | do_start_code_emulation_check = found_start_code_emulation; |
273 | 4.72k | } |
274 | | |
275 | 3.01k | if (size == 0) { |
276 | 5 | return kError_invalid_data; |
277 | 5 | } |
278 | | // Note: we cannot write &indata[idx + size] since that would use the operator[] on an element beyond the vector range. |
279 | 3.01k | scdata.insert(scdata.end(), &indata[idx], indata.data() + idx + size); |
280 | | |
281 | 3.01k | idx += size; |
282 | 3.01k | } |
283 | | |
284 | 733 | if (idx != indata.size()) { |
285 | 0 | decoder->input_data.pop_front(); |
286 | 0 | return kError_EOF; |
287 | 0 | } |
288 | | |
289 | 733 | decoder->input_data.pop_front(); |
290 | | |
291 | | // input: encoded bitstream start position; should include start code prefix |
292 | 733 | unsigned char* pBuf = scdata.data(); |
293 | | |
294 | | // input: encoded bit stream length; should include the size of start code prefix |
295 | 733 | int iSize = static_cast<int>(scdata.size()); |
296 | | |
297 | | |
298 | | // Step 5:do actual decoding process in slice level; this can be done in a loop until data ends |
299 | | |
300 | | |
301 | | //for Decoding only |
302 | 733 | iRet = pSvcDecoder->DecodeFrameNoDelay(pBuf, iSize, pData, &sDstBufInfo); |
303 | 733 | } |
304 | 2.45k | else { |
305 | 2.45k | iRet = pSvcDecoder->FlushFrame(pData, &sDstBufInfo); |
306 | 2.45k | } |
307 | | |
308 | 3.18k | if (iRet != 0) { |
309 | 635 | return { |
310 | 635 | heif_error_Decoder_plugin_error, |
311 | 635 | heif_suberror_Unspecified, |
312 | 635 | "OpenH264 decoder error" |
313 | 635 | }; |
314 | 635 | } |
315 | | |
316 | 2.54k | if (sDstBufInfo.iBufferStatus != 1) { |
317 | 2.49k | *out_img = nullptr; |
318 | 2.49k | return heif_error_ok; |
319 | 2.49k | } |
320 | | |
321 | 49 | if (out_user_data) { |
322 | 0 | *out_user_data = sDstBufInfo.uiOutYuvTimeStamp; |
323 | 0 | } |
324 | | |
325 | | /* |
326 | | // TODO: I receive an iBufferStatus==0, but the output image is still decoded |
327 | | if (sDstBufInfo.iBufferStatus == 0) { |
328 | | return {heif_error_Decoder_plugin_error, |
329 | | heif_suberror_Unspecified, |
330 | | "OpenH264 decoder did not output any image"}; |
331 | | } |
332 | | */ |
333 | | |
334 | 49 | uint32_t width = sDstBufInfo.UsrData.sSystemBuffer.iWidth; |
335 | 49 | uint32_t height = sDstBufInfo.UsrData.sSystemBuffer.iHeight; |
336 | | |
337 | 49 | heif_image* heif_img; |
338 | 49 | heif_error err{}; |
339 | | |
340 | 49 | uint32_t cwidth, cheight; |
341 | | |
342 | 49 | if (sDstBufInfo.UsrData.sSystemBuffer.iFormat == videoFormatI420) { |
343 | 49 | cwidth = (width + 1) / 2; |
344 | 49 | cheight = (height + 1) / 2; |
345 | | |
346 | 49 | err = heif_image_create(width, height, |
347 | 49 | heif_colorspace_YCbCr, |
348 | 49 | heif_chroma_420, |
349 | 49 | &heif_img); |
350 | 49 | if (err.code != heif_error_Ok) { |
351 | 0 | assert(heif_img == nullptr); |
352 | 0 | return err; |
353 | 0 | } |
354 | | |
355 | 49 | *out_img = heif_img; |
356 | | |
357 | 49 | err = heif_image_add_plane_safe(heif_img, heif_channel_Y, width, height, 8, limits); |
358 | 49 | if (err.code != heif_error_Ok) { |
359 | | // copy error message to decoder object because heif_image will be released |
360 | 0 | decoder->error_message = err.message; |
361 | 0 | err.message = decoder->error_message.c_str(); |
362 | |
|
363 | 0 | heif_image_release(heif_img); |
364 | 0 | return err; |
365 | 0 | } |
366 | | |
367 | 49 | err = heif_image_add_plane_safe(heif_img, heif_channel_Cb, cwidth, cheight, 8, limits); |
368 | 49 | if (err.code != heif_error_Ok) { |
369 | | // copy error message to decoder object because heif_image will be released |
370 | 0 | decoder->error_message = err.message; |
371 | 0 | err.message = decoder->error_message.c_str(); |
372 | |
|
373 | 0 | heif_image_release(heif_img); |
374 | 0 | return err; |
375 | 0 | } |
376 | | |
377 | 49 | err = heif_image_add_plane_safe(heif_img, heif_channel_Cr, cwidth, cheight, 8, limits); |
378 | 49 | if (err.code != heif_error_Ok) { |
379 | | // copy error message to decoder object because heif_image will be released |
380 | 0 | decoder->error_message = err.message; |
381 | 0 | err.message = decoder->error_message.c_str(); |
382 | |
|
383 | 0 | heif_image_release(heif_img); |
384 | 0 | return err; |
385 | 0 | } |
386 | | |
387 | 49 | size_t y_stride; |
388 | 49 | size_t cb_stride; |
389 | 49 | size_t cr_stride; |
390 | 49 | uint8_t* py = heif_image_get_plane2(heif_img, heif_channel_Y, &y_stride); |
391 | 49 | uint8_t* pcb = heif_image_get_plane2(heif_img, heif_channel_Cb, &cb_stride); |
392 | 49 | uint8_t* pcr = heif_image_get_plane2(heif_img, heif_channel_Cr, &cr_stride); |
393 | | |
394 | 49 | int ystride = sDstBufInfo.UsrData.sSystemBuffer.iStride[0]; |
395 | 49 | int cstride = sDstBufInfo.UsrData.sSystemBuffer.iStride[1]; |
396 | | |
397 | 3.31k | for (uint32_t y = 0; y < height; y++) { |
398 | 3.26k | memcpy(py + y * y_stride, sDstBufInfo.pDst[0] + y * ystride, width); |
399 | 3.26k | } |
400 | | |
401 | 1.68k | for (uint32_t y = 0; y < (height + 1) / 2; y++) { |
402 | 1.63k | memcpy(pcb + y * cb_stride, sDstBufInfo.pDst[1] + y * cstride, (width + 1) / 2); |
403 | 1.63k | memcpy(pcr + y * cr_stride, sDstBufInfo.pDst[2] + y * cstride, (width + 1) / 2); |
404 | 1.63k | } |
405 | 49 | } |
406 | 0 | else { |
407 | 0 | return { |
408 | 0 | heif_error_Decoder_plugin_error, |
409 | 0 | heif_suberror_Unspecified, |
410 | 0 | "Unsupported image pixel format" |
411 | 0 | }; |
412 | 0 | } |
413 | | |
414 | | // decoder->data.clear(); |
415 | | |
416 | 49 | return heif_error_ok; |
417 | 49 | } |
418 | | |
419 | | |
420 | | heif_error openh264_decode_next_image(void* decoder_raw, heif_image** out_img, |
421 | | const heif_security_limits* limits) |
422 | 0 | { |
423 | 0 | return openh264_decode_next_image2(decoder_raw, out_img, nullptr, limits); |
424 | 0 | } |
425 | | |
426 | | heif_error openh264_decode_image(void* decoder_raw, heif_image** out_img) |
427 | 0 | { |
428 | 0 | auto* limits = heif_get_global_security_limits(); |
429 | 0 | return openh264_decode_next_image(decoder_raw, out_img, limits); |
430 | 0 | } |
431 | | |
432 | | |
433 | | heif_error openh264_flush_data(void* decoder_raw) |
434 | 762 | { |
435 | 762 | auto* decoder = (struct openh264_decoder*) decoder_raw; |
436 | | |
437 | 762 | decoder->m_eof_reached = true; |
438 | | |
439 | 762 | return heif_error_ok; |
440 | 762 | } |
441 | | |
442 | | |
443 | | static const heif_decoder_plugin decoder_openh264{ |
444 | | 5, |
445 | | openh264_plugin_name, |
446 | | openh264_init_plugin, |
447 | | openh264_deinit_plugin, |
448 | | openh264_does_support_format, |
449 | | openh264_new_decoder, |
450 | | openh264_free_decoder, |
451 | | openh264_push_data, |
452 | | openh264_decode_image, |
453 | | openh264_set_strict_decoding, |
454 | | "openh264", |
455 | | openh264_decode_next_image, |
456 | | /* minimum_required_libheif_version */ LIBHEIF_MAKE_VERSION(1,21,0), |
457 | | openh264_does_support_format2, |
458 | | openh264_new_decoder2, |
459 | | openh264_push_data2, |
460 | | openh264_flush_data, |
461 | | openh264_decode_next_image2 |
462 | | }; |
463 | | |
464 | | |
465 | | const heif_decoder_plugin* get_decoder_plugin_openh264() |
466 | 254 | { |
467 | 254 | return &decoder_openh264; |
468 | 254 | } |
469 | | |
470 | | |
471 | | #if PLUGIN_OpenH264_DECODER |
472 | | heif_plugin_info plugin_info { |
473 | | 1, |
474 | | heif_plugin_type_decoder, |
475 | | &decoder_openh264 |
476 | | }; |
477 | | #endif |