Coverage Report

Created: 2026-06-16 07:20

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/libheif/libheif/plugins/decoder_openh264.cc
Line
Count
Source
1
/*
2
 * openh264 codec.
3
 * Copyright (c) 2024 Dirk Farin <dirk.farin@gmail.com>
4
 *
5
 * This file is part of libheif.
6
 *
7
 * libheif is free software: you can redistribute it and/or modify
8
 * it under the terms of the GNU Lesser General Public License as
9
 * published by the Free Software Foundation, either version 3 of
10
 * the License, or (at your option) any later version.
11
 *
12
 * libheif is distributed in the hope that it will be useful,
13
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15
 * GNU Lesser General Public License for more details.
16
 *
17
 * You should have received a copy of the GNU Lesser General Public License
18
 * along with libheif.  If not, see <http://www.gnu.org/licenses/>.
19
 */
20
21
#include "libheif/heif.h"
22
#include "libheif/heif_plugin.h"
23
#include "decoder_openh264.h"
24
#include <cstring>
25
#include <cassert>
26
#include <vector>
27
#include <cstdio>
28
#include <deque>
29
30
#include <wels/codec_api.h>
31
#include <string>
32
#include <utility>
33
34
// TODO: the openh264 decoder seems to fail with some images.
35
//       I have not figured out yet which images are affected.
36
//       Maybe it has something to do with the image size. I got the error with large images.
37
38
struct openh264_decoder
39
{
40
  struct Packet
41
  {
42
    std::vector<uint8_t> data;
43
    uintptr_t pts;
44
  };
45
46
  std::deque<Packet> input_data;
47
  bool m_eof_reached = false;
48
49
  std::string error_message;
50
51
52
  // --- decoder
53
54
  ISVCDecoder* decoder = nullptr;
55
56
  ~openh264_decoder()
57
770
  {
58
770
    if (decoder) {
59
      // Step 6:uninitialize the decoder and memory free
60
61
770
      decoder->Uninitialize(); // TODO: do we have to Uninitialize when an error is returned?
62
63
64
770
      WelsDestroyDecoder(decoder);
65
770
    }
66
770
  }
67
};
68
69
static const char kSuccess[] = "Success";
70
71
// Reduced priority because OpenH264 cannot pass through user-data.
72
// We need this feature for decoding sequences with SAI.
73
// Prefer to use the FFMPEG plugin.
74
static const int OpenH264_PLUGIN_PRIORITY = 70;
75
76
#define MAX_PLUGIN_NAME_LENGTH 80
77
78
static char plugin_name[MAX_PLUGIN_NAME_LENGTH];
79
80
static heif_error kError_EOF = {heif_error_Decoder_plugin_error, heif_suberror_End_of_data, "Insufficient input data"};
81
static heif_error kError_invalid_data = {heif_error_Decoder_plugin_error, heif_suberror_Invalid_parameter_value, "Invalid input data"};
82
83
84
static const char* openh264_plugin_name()
85
9
{
86
9
  OpenH264Version version = WelsGetCodecVersion();
87
88
9
  sprintf(plugin_name, "OpenH264 %d.%d.%d", version.uMajor, version.uMinor, version.uRevision);
89
90
9
  return plugin_name;
91
9
}
92
93
94
static void openh264_init_plugin()
95
254
{
96
254
}
97
98
99
static void openh264_deinit_plugin()
100
0
{
101
0
}
102
103
104
static int openh264_does_support_format(heif_compression_format format)
105
59.4k
{
106
59.4k
  if (format == heif_compression_AVC) {
107
803
    return OpenH264_PLUGIN_PRIORITY;
108
803
  }
109
58.6k
  else {
110
58.6k
    return 0;
111
58.6k
  }
112
59.4k
}
113
114
static int openh264_does_support_format2(const heif_decoder_plugin_compressed_format_description* format)
115
0
{
116
0
  return openh264_does_support_format(format->format);
117
0
}
118
119
120
heif_error openh264_new_decoder2(void** dec, const heif_decoder_plugin_options* options)
121
770
{
122
770
  auto* decoder = new openh264_decoder();
123
770
  *dec = decoder;
124
125
  // Step 2:decoder creation
126
770
  WelsCreateDecoder(&decoder->decoder);
127
770
  if (!decoder->decoder) {
128
0
    return {
129
0
      heif_error_Decoder_plugin_error,
130
0
      heif_suberror_Unspecified,
131
0
      "Cannot create OpenH264 decoder"
132
0
    };
133
0
  }
134
135
  // Step 3:declare required parameter, used to differentiate Decoding only and Parsing only
136
770
  SDecodingParam sDecParam{};
137
770
  sDecParam.sVideoProperty.eVideoBsType = VIDEO_BITSTREAM_AVC;
138
139
  //for Parsing only, the assignment is mandatory
140
  // sDecParam.bParseOnly = true;
141
142
  // Step 4:initialize the parameter and decoder context, allocate memory
143
770
  decoder->decoder->Initialize(&sDecParam);
144
145
770
  return {heif_error_Ok, heif_suberror_Unspecified, kSuccess};
146
770
}
147
148
149
heif_error openh264_new_decoder(void** dec)
150
0
{
151
0
  heif_decoder_plugin_options options{};
152
0
  options.format = heif_compression_AVC;
153
0
  options.num_threads = 0;
154
0
  options.strict_decoding = false;
155
156
0
  return openh264_new_decoder2(dec, &options);
157
0
}
158
159
void openh264_free_decoder(void* decoder_raw)
160
770
{
161
770
  auto* decoder = (openh264_decoder*) decoder_raw;
162
163
770
  if (!decoder) {
164
0
    return;
165
0
  }
166
167
770
  delete decoder;
168
770
}
169
170
171
void openh264_set_strict_decoding(void* decoder_raw, int flag)
172
0
{
173
  //  auto* decoder = (openh264_decoder*) decoder_raw;
174
0
}
175
176
177
heif_error openh264_push_data2(void* decoder_raw, const void* frame_data, size_t frame_size, uintptr_t user_data)
178
762
{
179
762
  auto* decoder = (openh264_decoder*) decoder_raw;
180
181
762
  const auto* input_data = (const uint8_t*) frame_data;
182
183
762
  if (frame_size < 4) {
184
0
    return kError_invalid_data;
185
0
  }
186
187
762
  openh264_decoder::Packet pkt;
188
762
  pkt.data.insert(pkt.data.end(), input_data, input_data + frame_size);
189
762
  pkt.pts = user_data;
190
762
  decoder->input_data.push_back(std::move(pkt));
191
192
762
  return {heif_error_Ok, heif_suberror_Unspecified, kSuccess};
193
762
}
194
195
heif_error openh264_push_data(void* decoder_raw, const void* frame_data, size_t frame_size)
196
0
{
197
0
  return openh264_push_data2(decoder_raw, frame_data, frame_size, 0);
198
0
}
199
200
201
heif_error openh264_decode_next_image2(void* decoder_raw, heif_image** out_img,
202
                                       uintptr_t* out_user_data,
203
                                       const heif_security_limits* limits)
204
3.21k
{
205
3.21k
  auto* decoder = (openh264_decoder*) decoder_raw;
206
3.21k
  ISVCDecoder* pSvcDecoder = decoder->decoder;
207
208
  // --- Not enough data, but no EOF yet.
209
210
3.21k
  if (decoder->input_data.empty() && !decoder->m_eof_reached) {
211
0
    *out_img = nullptr;
212
0
    return heif_error_ok;
213
0
  }
214
215
216
  //output: [0~2] for Y,U,V buffer for Decoding only
217
3.21k
  unsigned char* pData[3] = {nullptr, nullptr, nullptr};
218
219
  // in-out: for Decoding only: declare and initialize the output buffer info, this should never co-exist with Parsing only
220
221
3.21k
  SBufferInfo sDstBufInfo;
222
3.21k
  memset(&sDstBufInfo, 0, sizeof(SBufferInfo));
223
224
3.21k
  int iRet;
225
226
3.21k
  if (!decoder->input_data.empty()) {
227
762
    sDstBufInfo.uiInBsTimeStamp = decoder->input_data.front().pts;
228
229
762
    const std::vector<uint8_t>& indata = decoder->input_data.front().data;
230
762
    std::vector<uint8_t> scdata;
231
232
762
    size_t idx = 0;
233
3.77k
    while (idx < indata.size()) {
234
3.04k
      if (indata.size() - 4 < idx) {
235
1
        return kError_EOF;
236
1
      }
237
238
3.04k
      uint32_t size = ((indata[idx] << 24) | (indata[idx + 1] << 16) | (indata[idx + 2] << 8) | indata[idx + 3]);
239
3.04k
      idx += 4;
240
241
3.04k
      if (indata.size() < size || indata.size() - size < idx) {
242
23
        return kError_EOF;
243
23
      }
244
245
3.01k
      scdata.push_back(0);
246
3.01k
      scdata.push_back(0);
247
3.01k
      scdata.push_back(1);
248
249
      // check for need of start code emulation prevention
250
251
3.01k
      bool do_start_code_emulation_check = true;
252
253
7.74k
      while (do_start_code_emulation_check && size >= 3) {
254
4.72k
        bool found_start_code_emulation = false;
255
256
1.57M
        for (size_t i = 0; i < size - 3; i++) {
257
1.57M
          if (indata[idx + 0] == 0 &&
258
16.9k
              indata[idx + 1] == 0 &&
259
9.67k
              (indata[idx + 2] >= 0 && indata[idx + 2] <= 3)) {
260
1.71k
            scdata.push_back(0);
261
1.71k
            scdata.push_back(0);
262
1.71k
            scdata.push_back(3);
263
264
1.71k
            scdata.insert(scdata.end(), &indata[idx + 2], indata.data() + idx + i + 2);
265
1.71k
            idx += i + 2;
266
1.71k
            size -= (uint32_t) (i + 2);
267
1.71k
            found_start_code_emulation = true;
268
1.71k
            break;
269
1.71k
          }
270
1.57M
        }
271
272
4.72k
        do_start_code_emulation_check = found_start_code_emulation;
273
4.72k
      }
274
275
3.01k
      if (size == 0) {
276
5
        return kError_invalid_data;
277
5
      }
278
      // Note: we cannot write &indata[idx + size] since that would use the operator[] on an element beyond the vector range.
279
3.01k
      scdata.insert(scdata.end(), &indata[idx], indata.data() + idx + size);
280
281
3.01k
      idx += size;
282
3.01k
    }
283
284
733
    if (idx != indata.size()) {
285
0
      decoder->input_data.pop_front();
286
0
      return kError_EOF;
287
0
    }
288
289
733
    decoder->input_data.pop_front();
290
291
    // input: encoded bitstream start position; should include start code prefix
292
733
    unsigned char* pBuf = scdata.data();
293
294
    // input: encoded bit stream length; should include the size of start code prefix
295
733
    int iSize = static_cast<int>(scdata.size());
296
297
298
    // Step 5:do actual decoding process in slice level; this can be done in a loop until data ends
299
300
301
    //for Decoding only
302
733
    iRet = pSvcDecoder->DecodeFrameNoDelay(pBuf, iSize, pData, &sDstBufInfo);
303
733
  }
304
2.45k
  else {
305
2.45k
    iRet = pSvcDecoder->FlushFrame(pData, &sDstBufInfo);
306
2.45k
  }
307
308
3.18k
  if (iRet != 0) {
309
635
    return {
310
635
      heif_error_Decoder_plugin_error,
311
635
      heif_suberror_Unspecified,
312
635
      "OpenH264 decoder error"
313
635
    };
314
635
  }
315
316
2.54k
  if (sDstBufInfo.iBufferStatus != 1) {
317
2.49k
    *out_img = nullptr;
318
2.49k
    return heif_error_ok;
319
2.49k
  }
320
321
49
  if (out_user_data) {
322
0
    *out_user_data = sDstBufInfo.uiOutYuvTimeStamp;
323
0
  }
324
325
  /*
326
  // TODO: I receive an iBufferStatus==0, but the output image is still decoded
327
  if (sDstBufInfo.iBufferStatus == 0) {
328
    return {heif_error_Decoder_plugin_error,
329
            heif_suberror_Unspecified,
330
            "OpenH264 decoder did not output any image"};
331
  }
332
  */
333
334
49
  uint32_t width = sDstBufInfo.UsrData.sSystemBuffer.iWidth;
335
49
  uint32_t height = sDstBufInfo.UsrData.sSystemBuffer.iHeight;
336
337
49
  heif_image* heif_img;
338
49
  heif_error err{};
339
340
49
  uint32_t cwidth, cheight;
341
342
49
  if (sDstBufInfo.UsrData.sSystemBuffer.iFormat == videoFormatI420) {
343
49
    cwidth = (width + 1) / 2;
344
49
    cheight = (height + 1) / 2;
345
346
49
    err = heif_image_create(width, height,
347
49
                            heif_colorspace_YCbCr,
348
49
                            heif_chroma_420,
349
49
                            &heif_img);
350
49
    if (err.code != heif_error_Ok) {
351
0
      assert(heif_img == nullptr);
352
0
      return err;
353
0
    }
354
355
49
    *out_img = heif_img;
356
357
49
    err = heif_image_add_plane_safe(heif_img, heif_channel_Y, width, height, 8, limits);
358
49
    if (err.code != heif_error_Ok) {
359
      // copy error message to decoder object because heif_image will be released
360
0
      decoder->error_message = err.message;
361
0
      err.message = decoder->error_message.c_str();
362
363
0
      heif_image_release(heif_img);
364
0
      return err;
365
0
    }
366
367
49
    err = heif_image_add_plane_safe(heif_img, heif_channel_Cb, cwidth, cheight, 8, limits);
368
49
    if (err.code != heif_error_Ok) {
369
      // copy error message to decoder object because heif_image will be released
370
0
      decoder->error_message = err.message;
371
0
      err.message = decoder->error_message.c_str();
372
373
0
      heif_image_release(heif_img);
374
0
      return err;
375
0
    }
376
377
49
    err = heif_image_add_plane_safe(heif_img, heif_channel_Cr, cwidth, cheight, 8, limits);
378
49
    if (err.code != heif_error_Ok) {
379
      // copy error message to decoder object because heif_image will be released
380
0
      decoder->error_message = err.message;
381
0
      err.message = decoder->error_message.c_str();
382
383
0
      heif_image_release(heif_img);
384
0
      return err;
385
0
    }
386
387
49
    size_t y_stride;
388
49
    size_t cb_stride;
389
49
    size_t cr_stride;
390
49
    uint8_t* py = heif_image_get_plane2(heif_img, heif_channel_Y, &y_stride);
391
49
    uint8_t* pcb = heif_image_get_plane2(heif_img, heif_channel_Cb, &cb_stride);
392
49
    uint8_t* pcr = heif_image_get_plane2(heif_img, heif_channel_Cr, &cr_stride);
393
394
49
    int ystride = sDstBufInfo.UsrData.sSystemBuffer.iStride[0];
395
49
    int cstride = sDstBufInfo.UsrData.sSystemBuffer.iStride[1];
396
397
3.31k
    for (uint32_t y = 0; y < height; y++) {
398
3.26k
      memcpy(py + y * y_stride, sDstBufInfo.pDst[0] + y * ystride, width);
399
3.26k
    }
400
401
1.68k
    for (uint32_t y = 0; y < (height + 1) / 2; y++) {
402
1.63k
      memcpy(pcb + y * cb_stride, sDstBufInfo.pDst[1] + y * cstride, (width + 1) / 2);
403
1.63k
      memcpy(pcr + y * cr_stride, sDstBufInfo.pDst[2] + y * cstride, (width + 1) / 2);
404
1.63k
    }
405
49
  }
406
0
  else {
407
0
    return {
408
0
      heif_error_Decoder_plugin_error,
409
0
      heif_suberror_Unspecified,
410
0
      "Unsupported image pixel format"
411
0
    };
412
0
  }
413
414
  // decoder->data.clear();
415
416
49
  return heif_error_ok;
417
49
}
418
419
420
heif_error openh264_decode_next_image(void* decoder_raw, heif_image** out_img,
421
                                      const heif_security_limits* limits)
422
0
{
423
0
  return openh264_decode_next_image2(decoder_raw, out_img, nullptr, limits);
424
0
}
425
426
heif_error openh264_decode_image(void* decoder_raw, heif_image** out_img)
427
0
{
428
0
  auto* limits = heif_get_global_security_limits();
429
0
  return openh264_decode_next_image(decoder_raw, out_img, limits);
430
0
}
431
432
433
heif_error openh264_flush_data(void* decoder_raw)
434
762
{
435
762
  auto* decoder = (struct openh264_decoder*) decoder_raw;
436
437
762
  decoder->m_eof_reached = true;
438
439
762
  return heif_error_ok;
440
762
}
441
442
443
static const heif_decoder_plugin decoder_openh264{
444
  5,
445
  openh264_plugin_name,
446
  openh264_init_plugin,
447
  openh264_deinit_plugin,
448
  openh264_does_support_format,
449
  openh264_new_decoder,
450
  openh264_free_decoder,
451
  openh264_push_data,
452
  openh264_decode_image,
453
  openh264_set_strict_decoding,
454
  "openh264",
455
  openh264_decode_next_image,
456
  /* minimum_required_libheif_version */ LIBHEIF_MAKE_VERSION(1,21,0),
457
  openh264_does_support_format2,
458
  openh264_new_decoder2,
459
  openh264_push_data2,
460
  openh264_flush_data,
461
  openh264_decode_next_image2
462
};
463
464
465
const heif_decoder_plugin* get_decoder_plugin_openh264()
466
254
{
467
254
  return &decoder_openh264;
468
254
}
469
470
471
#if PLUGIN_OpenH264_DECODER
472
heif_plugin_info plugin_info {
473
  1,
474
  heif_plugin_type_decoder,
475
  &decoder_openh264
476
};
477
#endif