Coverage Report

Created: 2026-09-28 06:47

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/libheif/libheif/sequences/track_visual.cc
Line
Count
Source
1
/*
2
 * HEIF image base codec.
3
 * Copyright (c) 2025 Dirk Farin <dirk.farin@gmail.com>
4
 *
5
 * This file is part of libheif.
6
 *
7
 * libheif is free software: you can redistribute it and/or modify
8
 * it under the terms of the GNU Lesser General Public License as
9
 * published by the Free Software Foundation, either version 3 of
10
 * the License, or (at your option) any later version.
11
 *
12
 * libheif is distributed in the hope that it will be useful,
13
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15
 * GNU Lesser General Public License for more details.
16
 *
17
 * You should have received a copy of the GNU Lesser General Public License
18
 * along with libheif.  If not, see <http://www.gnu.org/licenses/>.
19
 */
20
21
#include "track_visual.h"
22
23
#include <memory>
24
#include <utility>
25
#include "codecs/decoder.h"
26
#include "codecs/encoder.h"
27
#include "chunk.h"
28
#include "image/pixelimage.h"
29
#include "context.h"
30
#include "api_structs.h"
31
#include "codecs/hevc_boxes.h"
32
#include "codecs/uncompressed/unc_boxes.h"
33
34
35
Track_Visual::Track_Visual(HeifContext* ctx)
36
9
  : Track(ctx)
37
9
{
38
9
}
39
40
41
Track_Visual::~Track_Visual()
42
9
{
43
9
  for (auto& user_data : m_frame_user_data) {
44
0
    user_data.second.release();
45
0
  }
46
9
}
47
48
49
Error Track_Visual::load(const std::shared_ptr<Box_trak>& trak)
50
9
{
51
9
  Error parentLoadError = Track::load(trak);
52
9
  if (parentLoadError) {
53
9
    return parentLoadError;
54
9
  }
55
56
0
  const std::vector<uint32_t>& chunk_offsets = m_stco->get_offsets();
57
58
  // Find sequence resolution
59
60
0
  if (!chunk_offsets.empty()) {
61
0
    auto* s2c = m_stsc->get_chunk(1);
62
0
    if (!s2c) {
63
0
      return {
64
0
        heif_error_Invalid_input,
65
0
        heif_suberror_Unspecified,
66
0
        "Visual track has no chunk 1"
67
0
      };
68
0
    }
69
70
0
    Box_stsc::SampleToChunk sampleToChunk = *s2c;
71
72
0
    auto sample_description = m_stsd->get_sample_entry(sampleToChunk.sample_description_index - 1);
73
0
    if (!sample_description) {
74
0
      return {
75
0
        heif_error_Invalid_input,
76
0
        heif_suberror_Unspecified,
77
0
        "Visual track has sample description"
78
0
      };
79
0
    }
80
81
0
    auto visual_sample_description = std::dynamic_pointer_cast<const Box_VisualSampleEntry>(sample_description);
82
0
    if (!visual_sample_description) {
83
0
      return {
84
0
        heif_error_Invalid_input,
85
0
        heif_suberror_Unspecified,
86
0
        "Visual track sample description does not match visual track."
87
0
      };
88
0
    }
89
90
0
    m_width = visual_sample_description->get_VisualSampleEntry_const().width;
91
0
    m_height = visual_sample_description->get_VisualSampleEntry_const().height;
92
0
  }
93
94
0
  return {};
95
0
}
96
97
98
Error Track_Visual::initialize_after_parsing(HeifContext* ctx, const std::vector<std::shared_ptr<Track> >& all_tracks)
99
0
{
100
  // --- check whether there is an auxiliary alpha track assigned to this track
101
102
  // Only assign to image-sequence tracks (TODO: are there also alpha tracks allowed for video tracks 'heif_track_type_video'?)
103
104
0
  if (get_handler() == heif_track_type_image_sequence) {
105
0
    for (const auto& track : all_tracks) {
106
      // skip ourselves
107
0
      if (track->get_id() != get_id()) {
108
        // Is this an aux alpha track?
109
0
        auto h = fourcc_to_string(track->get_handler());
110
0
        if (track->get_handler() == heif_track_type_auxiliary &&
111
0
            track->get_auxiliary_info_type() == heif_auxiliary_track_info_type_alpha) {
112
          // Is it assigned to the current track
113
0
          auto tref = track->get_tref_box();
114
0
          if (!tref) {
115
0
            return {
116
0
              heif_error_Invalid_input,
117
0
              heif_suberror_Unspecified,
118
0
              "Auxiliary track without 'tref'"
119
0
            };
120
0
          }
121
122
0
          auto references = tref->get_references(fourcc("auxl"));
123
0
          if (std::any_of(references.begin(), references.end(), [this](uint32_t id) { return id == get_id(); })) {
124
            // Assign it
125
126
0
            m_aux_alpha_track = std::dynamic_pointer_cast<Track_Visual>(track);
127
0
          }
128
0
        }
129
0
      }
130
0
    }
131
0
  }
132
133
0
  return {};
134
0
}
135
136
137
Track_Visual::Track_Visual(HeifContext* ctx, uint32_t track_id, uint16_t width, uint16_t height,
138
                           const TrackOptions* options, uint32_t handler_type)
139
0
  : Track(ctx, track_id, options, handler_type)
140
0
{
141
0
  m_tkhd->set_resolution(width, height);
142
  //m_hdlr->set_handler_type(handler_type);  already done in Track()
143
144
0
  auto vmhd = std::make_shared<Box_vmhd>();
145
0
  m_minf->append_child_box(vmhd);
146
0
}
147
148
149
bool Track_Visual::has_alpha_channel() const
150
0
{
151
0
  if (m_aux_alpha_track != nullptr) {
152
0
    return true;
153
0
  }
154
155
  // --- special case: 'uncv' with alpha component
156
#if WITH_UNCOMPRESSED_CODEC
157
  if (m_stsd) {
158
    auto sampleEntry = m_stsd->get_sample_entry(0);
159
    if (sampleEntry) {
160
      if (auto box_uncv = std::dynamic_pointer_cast<const Box_uncv>(sampleEntry)) {
161
        if (auto cmpd = box_uncv->get_child_box<const Box_cmpd>()) {
162
          if (cmpd->has_component(heif_cmpd_component_type_alpha)) {
163
            return true;
164
          }
165
        }
166
      }
167
    }
168
  }
169
#endif
170
171
0
  return false;
172
0
}
173
174
175
Result<std::shared_ptr<HeifPixelImage> > Track_Visual::decode_next_image_sample(const heif_decoding_options& options)
176
0
{
177
  // --- If we ignore the editlist, we stop when we reached the end of the original samples.
178
179
0
  uint64_t num_output_samples = m_num_output_samples;
180
0
  if (options.ignore_sequence_editlist) {
181
0
    num_output_samples = m_num_samples;
182
0
  }
183
184
  // --- Did we reach the end of the sequence?
185
186
0
  if (m_next_sample_to_be_output >= num_output_samples) {
187
0
    return Error{
188
0
      heif_error_End_of_sequence,
189
0
      heif_suberror_Unspecified,
190
0
      "End of sequence"
191
0
    };
192
0
  }
193
194
195
0
  std::shared_ptr<HeifPixelImage> image;
196
197
0
  uint32_t sample_idx_in_chunk;
198
0
  uintptr_t decoded_sample_idx = 0; // TODO: map this to sample idx + chunk
199
200
0
  for (;;) {
201
0
    const SampleTiming& sampleTiming = m_presentation_timeline[m_next_sample_to_be_decoded % m_presentation_timeline.size()];
202
0
    sample_idx_in_chunk = sampleTiming.sampleIdx;
203
0
    uint32_t chunk_idx = sampleTiming.chunkIdx;
204
205
0
    const std::shared_ptr<Chunk>& chunk = m_chunks[chunk_idx];
206
207
0
    auto decoder = chunk->get_decoder();
208
0
    assert(decoder);
209
210
    // avoid calling get_decoded_frame() before starting the decoder.
211
0
    if (m_next_sample_to_be_decoded != 0) {
212
0
      Result<std::shared_ptr<HeifPixelImage> > getFrameResult = decoder->get_decoded_frame(options,
213
0
                                                                                           &decoded_sample_idx,
214
0
                                                                                           m_heif_context->get_security_limits());
215
0
      if (getFrameResult.error()) {
216
0
        return getFrameResult.error();
217
0
      }
218
219
220
      // We received a decoded frame. Exit "push data" / "decode" loop.
221
222
0
      if (*getFrameResult != nullptr) {
223
0
        image = *getFrameResult;
224
225
        // If this was the last frame in the EditList segment, reset the 'flushed' flag
226
        // in case we have to restart the decoder for another repetition of the segment.
227
0
        if ((m_next_sample_to_be_output + 1) % m_presentation_timeline.size() == 0) {
228
0
          m_decoder_is_flushed = false;
229
0
        }
230
231
0
        break;
232
0
      }
233
234
      // If the sequence has ended and the decoder was flushed, but we still did not receive
235
      // the image, we are waiting for, this is an error.
236
0
      if (m_decoder_is_flushed) {
237
0
        return Error(heif_error_Decoder_plugin_error,
238
0
                     heif_suberror_Unspecified,
239
0
                     "Did not decode all frames");
240
0
      }
241
0
    }
242
243
244
    // --- Push more data into the decoder (or send end-of-sequence).
245
246
0
    if (m_next_sample_to_be_decoded < m_num_output_samples) {
247
248
      // --- Find the data extent that stores the compressed frame data.
249
250
0
      DataExtent extent = chunk->get_data_extent_for_sample(sample_idx_in_chunk);
251
0
      decoder->set_data_extent(std::move(extent));
252
253
      // std::cout << "PUSH chunk " << chunk_idx << " sample " << sample_idx << " (" << extent.m_size << " bytes)\n";
254
255
      // advance decoding index to next in segment
256
0
      m_next_sample_to_be_decoded++;
257
258
      // Send the decoder configuration when we send the first sample of the chunk.
259
      // The configuration NALs might change for each chunk.
260
0
      const bool is_first_sample = (sample_idx_in_chunk == 0);
261
262
      // --- Push data into the decoder.
263
0
      Error decodingError = decoder->decode_sequence_frame_from_compressed_data(is_first_sample,
264
0
                                                                                options,
265
0
                                                                                sample_idx_in_chunk, // user data
266
0
                                                                                m_heif_context->get_security_limits());
267
0
      if (decodingError) {
268
0
        return decodingError;
269
0
      }
270
0
    }
271
0
    else {
272
      // --- End of sequence (Editlist segment) reached.
273
274
      // std::cout << "FLUSH\n";
275
0
      Error flushError = decoder->flush_decoder();
276
0
      if (flushError) {
277
0
        return flushError;
278
0
      }
279
280
0
      m_decoder_is_flushed = true;
281
0
    }
282
0
  }
283
284
285
  // --- We have received a new decoded image.
286
  //     Postprocess decoded image, attach metadata.
287
288
0
  if (m_stts) {
289
    // 'sample_idx_in_chunk' follows m_next_sample_to_be_decoded, which runs ahead of the output
290
    // by the decoder latency. Look the duration up by the output position instead, like
291
    // Track::get_next_sample_raw_data() does.
292
0
    const SampleTiming& outputTiming = m_presentation_timeline[m_next_sample_to_be_output % m_presentation_timeline.size()];
293
0
    image->set_sample_duration(m_stts->get_sample_duration(outputTiming.sampleIdx));
294
0
  }
295
296
  // --- assign alpha if we have an assigned alpha track
297
298
0
  if (m_aux_alpha_track) {
299
0
    auto alphaResult = m_aux_alpha_track->decode_next_image_sample(options);
300
0
    if (!alphaResult) {
301
0
      return alphaResult.error();
302
0
    }
303
304
0
    auto alphaImage = *alphaResult;
305
306
    // The alpha auxiliary track may have a different size than the main track. Nothing in the
307
    // standard forbids this. We scale it to the main image size so that downstream consumers can
308
    // rely on the alpha plane matching the color planes. This mirrors what the still-image decoder
309
    // does in ImageItem (see image_item.cc). As noted there, we may later add a decoding option to
310
    // turn this automatic scaling off, but that requires that libheif no longer assumes everywhere
311
    // that the alpha channel has the same resolution as the color channels.
312
0
    if (alphaImage->get_width() != image->get_width() ||
313
0
        alphaImage->get_height() != image->get_height()) {
314
0
      std::shared_ptr<HeifPixelImage> scaled_alpha;
315
0
      Error err = alphaImage->scale_nearest_neighbor(scaled_alpha, image->get_width(), image->get_height(),
316
0
                                                     m_heif_context->get_security_limits());
317
0
      if (err) {
318
0
        return err;
319
0
      }
320
0
      alphaImage = std::move(scaled_alpha);
321
0
    }
322
323
0
    if (Error err = image->transfer_channel_from_image_as(alphaImage, heif_channel_Y, heif_channel_Alpha)) {
324
0
      return err;
325
0
    }
326
0
  }
327
328
329
  // --- read sample auxiliary data
330
331
0
  if (m_aux_reader_content_ids) {
332
0
    auto readResult = m_aux_reader_content_ids->get_sample_info(get_file().get(), (uint32_t)decoded_sample_idx);
333
0
    if (!readResult) {
334
0
      return readResult.error();
335
0
    }
336
337
0
    Result<std::string> convResult = vector_to_string(*readResult);
338
0
    if (!convResult) {
339
0
      return convResult.error();
340
0
    }
341
342
0
    image->set_gimi_sample_content_id(*convResult);
343
0
  }
344
345
0
  if (m_aux_reader_tai_timestamps) {
346
0
    auto readResult = m_aux_reader_tai_timestamps->get_sample_info(get_file().get(), (uint32_t)decoded_sample_idx);
347
0
    if (!readResult) {
348
0
      return readResult.error();
349
0
    }
350
351
0
    std::vector<uint8_t>& tai_data = *readResult;
352
0
    if (!tai_data.empty()) {
353
0
      auto resultTai = Box_itai::decode_tai_from_vector(tai_data);
354
0
      if (!resultTai) {
355
0
        return resultTai.error();
356
0
      }
357
358
0
      image->set_tai_timestamp(&*resultTai);
359
0
    }
360
0
  }
361
362
0
  m_next_sample_to_be_output++;
363
364
0
  return image;
365
0
}
366
367
368
Error Track_Visual::encode_end_of_sequence(heif_encoder* h_encoder)
369
0
{
370
0
  auto encoder = m_chunks.back()->get_encoder();
371
372
0
  for (;;) {
373
0
    Error err = encoder->encode_sequence_flush(h_encoder);
374
0
    if (err) {
375
0
      return err;
376
0
    }
377
378
0
    Result<bool> processingResult = process_encoded_data(h_encoder);
379
0
    if (processingResult.is_error()) {
380
0
      return processingResult.error();
381
0
    }
382
383
0
    if (!*processingResult) {
384
0
      break;
385
0
    }
386
0
  }
387
388
389
  // --- also end alpha track
390
391
0
  if (m_aux_alpha_track) {
392
0
    auto err = m_aux_alpha_track->encode_end_of_sequence(m_alpha_track_encoder.get());
393
0
    if (err) {
394
0
      return err;
395
0
    }
396
0
  }
397
398
0
  return {};
399
0
}
400
401
402
Error Track_Visual::encode_image(const std::shared_ptr<HeifPixelImage>& image,
403
                                 heif_encoder* h_encoder,
404
                                 const heif_sequence_encoding_options* in_options,
405
                                 heif_image_input_class input_class)
406
0
{
407
0
  if (image->get_width() > 0xFFFF ||
408
0
      image->get_height() > 0xFFFF) {
409
0
    return {
410
0
      heif_error_Invalid_input,
411
0
      heif_suberror_Unspecified,
412
0
      "Input image resolution too high"
413
0
    };
414
0
  }
415
416
0
  m_image_class = input_class;
417
418
419
  // --- resolve the encoding options
420
421
  // Build an effective options struct so that the rest of this function never
422
  // has to look at the caller's pointer. 'in_options' may be NULL, which the API
423
  // documents as "use the default options", so every field has to come from a
424
  // fully populated struct instead. This also lets libheif resolve "auto" fields
425
  // (e.g. content_kind) to concrete values before passing them to the encoder
426
  // plugin. heif_sequence_encoding_options_copy is version-aware and ignores a
427
  // NULL source, so callers built against older headers won't be read past their
428
  // actual allocation and a NULL source simply leaves the defaults in place.
429
430
0
  std::unique_ptr<heif_sequence_encoding_options, void(*)(heif_sequence_encoding_options*)>
431
0
      effective_options(heif_sequence_encoding_options_alloc(),
432
0
                        heif_sequence_encoding_options_release);
433
0
  heif_sequence_encoding_options_copy(effective_options.get(), in_options);
434
435
  // Resolve content_kind=auto based on this track's handler type. For auxiliary
436
  // tracks (e.g. alpha) the parent track resolves the value before recursing in,
437
  // so we never reach this branch with handler=auxv.
438
  // TODO: in the future, "auto" could also factor in input frame rate or
439
  // frame-to-frame similarity.
440
0
  if (effective_options->content_kind == heif_sequence_content_kind_auto) {
441
0
    switch (get_handler()) {
442
0
      case heif_track_type_video:
443
0
        effective_options->content_kind = heif_sequence_content_kind_video;
444
0
        break;
445
0
      case heif_track_type_image_sequence:
446
0
      default:
447
0
        effective_options->content_kind = heif_sequence_content_kind_image_sequence;
448
0
        break;
449
0
    }
450
0
  }
451
452
453
  // --- If input has an alpha channel, add an alpha auxiliary track.
454
455
0
  if (effective_options->save_alpha_channel && image->has_alpha() && !m_aux_alpha_track &&
456
0
      h_encoder->plugin->compression_format != heif_compression_uncompressed) { // TODO: ask plugin
457
0
    if (m_active_encoder) {
458
0
      return {
459
0
        heif_error_Usage_error,
460
0
        heif_suberror_Unspecified,
461
0
        "Input images must all either have an alpha channel or none of them."
462
0
      };
463
0
    }
464
0
    else {
465
      // alpha track uses default options, same timescale as color track
466
467
0
      TrackOptions alphaOptions;
468
0
      alphaOptions.track_timescale = m_track_info.track_timescale;
469
470
0
      auto newAlphaTrackResult = m_heif_context->add_visual_sequence_track(&alphaOptions,
471
0
                                                                           heif_track_type_auxiliary,
472
0
                                                                           static_cast<uint16_t>(image->get_width()),
473
0
                                                                           static_cast<uint16_t>(image->get_height()));
474
475
0
      if (auto err = newAlphaTrackResult.error()) {
476
0
        return err;
477
0
      }
478
479
      // add a reference to the color track
480
481
0
      m_aux_alpha_track = *newAlphaTrackResult;
482
0
      m_aux_alpha_track->add_reference_to_track(fourcc("auxl"), m_id);
483
484
      // make a copy of the encoder from the color track for encoding the alpha track
485
486
0
      m_alpha_track_encoder = std::make_unique<heif_encoder>(h_encoder->plugin);
487
0
      heif_error err = m_alpha_track_encoder->alloc();
488
0
      if (err.code) {
489
0
        return {err.code, err.subcode, err.message};
490
0
      }
491
0
      m_alpha_track_encoder->copy_parameters_from(*h_encoder);
492
0
    }
493
0
  }
494
495
496
0
  if (!m_active_encoder) {
497
0
    m_active_encoder = h_encoder;
498
0
  }
499
0
  else if (m_active_encoder != h_encoder) {
500
0
    return {
501
0
      heif_error_Usage_error,
502
0
      heif_suberror_Unspecified,
503
0
      "You may not switch the heif_encoder while encoding a sequence."
504
0
    };
505
0
  }
506
507
0
  if (h_encoder->plugin->plugin_api_version < 4) {
508
0
    return Error{
509
0
      heif_error_Plugin_loading_error, heif_suberror_No_matching_decoder_installed,
510
0
      "Encoder plugin needs to be at least version 4."
511
0
    };
512
0
  }
513
514
  // === generate compressed image bitstream
515
516
  // generate new chunk for first image or when compression formats don't match
517
518
0
  if (m_chunks.empty() || m_chunks.back()->get_compression_format() != h_encoder->plugin->compression_format) {
519
0
    add_chunk(h_encoder->plugin->compression_format);
520
0
  }
521
522
  // --- check whether we have to convert the image color space
523
524
  // The reason for doing the color conversion here is that the input might be an RGBA image and the color conversion
525
  // will extract the alpha plane anyway. We can reuse that plane below instead of having to do a new conversion.
526
527
0
  auto encoder = m_chunks.back()->get_encoder();
528
529
0
  const heif_color_profile_nclx* output_nclx;
530
0
  heif_color_profile_nclx nclx;
531
532
0
  if (const auto* image_nclx = encoder->get_forced_output_nclx()) {
533
0
    output_nclx = const_cast<heif_color_profile_nclx*>(image_nclx);
534
0
  }
535
0
  else if (in_options) {
536
0
    output_nclx = in_options->output_nclx_profile;
537
0
  }
538
0
  else {
539
    // Note: this is the one place that still distinguishes "caller passed no
540
    // options" from "caller passed options with a NULL output_nclx_profile",
541
    // so it reads in_options rather than effective_options. Both cases mean
542
    // "keep the input image's parameters", but they express it differently: a
543
    // NULL profile lets compute_target_nclx_profile() derive them, while the
544
    // branch below builds the profile here. The two differ only for an image
545
    // that carries no NCLX at all, where the explicit undefined profile built
546
    // below makes nclx_profile_matches_spec() request a conversion that a NULL
547
    // profile would skip. Unifying them would change that behaviour, so it is
548
    // left alone here.
549
0
    if (image->has_nclx_color_profile()) {
550
0
      nclx_profile input_nclx = image->get_color_profile_nclx();
551
552
0
      nclx.version = 1;
553
0
      nclx.color_primaries = (enum heif_color_primaries) input_nclx.get_colour_primaries();
554
0
      nclx.transfer_characteristics = (enum heif_transfer_characteristics) input_nclx.get_transfer_characteristics();
555
0
      nclx.matrix_coefficients = (enum heif_matrix_coefficients) input_nclx.get_matrix_coefficients();
556
0
      nclx.full_range_flag = input_nclx.get_full_range_flag();
557
558
0
      output_nclx = &nclx;
559
0
    }
560
0
    else {
561
0
      nclx_profile undefined_nclx;
562
0
      undefined_nclx.copy_to_heif_color_profile_nclx(&nclx);
563
564
0
      output_nclx = &nclx;
565
0
    }
566
0
  }
567
568
0
  Result<std::shared_ptr<HeifPixelImage> > srcImageResult = encoder->convert_colorspace_for_encoding(image,
569
0
                                                                                                     h_encoder,
570
0
                                                                                                     output_nclx,
571
0
                                                                                                     &effective_options->color_conversion_options,
572
0
                                                                                                     m_heif_context->get_security_limits());
573
0
  if (!srcImageResult) {
574
0
    return srcImageResult.error();
575
0
  }
576
577
0
  std::shared_ptr<HeifPixelImage> colorConvertedImage = *srcImageResult;
578
579
  // integer range is checked at beginning of function.
580
0
  assert(colorConvertedImage->get_width() == image->get_width());
581
0
  assert(colorConvertedImage->get_height() == image->get_height());
582
0
  m_width = static_cast<uint16_t>(colorConvertedImage->get_width());
583
0
  m_height = static_cast<uint16_t>(colorConvertedImage->get_height());
584
585
  // --- encode image
586
587
0
  Error encodeError = encoder->encode_sequence_frame(colorConvertedImage, h_encoder,
588
0
                                                     *effective_options,
589
0
                                                     input_class,
590
0
                                                     colorConvertedImage->get_sample_duration(), get_timescale(),
591
0
                                                     m_current_frame_nr);
592
593
0
  if (encodeError) {
594
0
    return encodeError;
595
0
  }
596
597
0
  m_sample_duration = colorConvertedImage->get_sample_duration();
598
  // TODO heif_tai_timestamp_packet* tai = image->get_tai_timestamp();
599
  // TODO image->has_gimi_sample_content_id() ? image->get_gimi_sample_content_id() : std::string{});
600
601
602
  // store frame user data
603
604
0
  FrameUserData userData;
605
0
  userData.sample_duration = colorConvertedImage->get_sample_duration();
606
0
  if (image->has_gimi_sample_content_id()) {
607
0
    userData.gimi_content_id = image->get_gimi_sample_content_id();
608
0
  }
609
610
0
  if (const auto* tai = image->get_tai_timestamp()) {
611
0
    userData.tai_timestamp = heif_tai_timestamp_packet_alloc();
612
0
    heif_tai_timestamp_packet_copy(userData.tai_timestamp, tai);
613
0
  }
614
615
0
  m_frame_user_data[m_current_frame_nr] = userData;
616
617
0
  m_current_frame_nr++;
618
619
  // --- get compressed data from encoder
620
621
0
  Result<bool> processingResult = process_encoded_data(h_encoder);
622
0
  if (auto err = processingResult.error()) {
623
0
    return err;
624
0
  }
625
626
627
  // --- encode alpha channel into auxiliary track
628
629
0
  if (m_aux_alpha_track) {
630
0
    auto alphaImageResult = create_alpha_image_from_image_alpha_channel(colorConvertedImage,
631
0
                                                                        m_heif_context->get_security_limits());
632
0
    if (auto err = alphaImageResult.error()) {
633
0
      return err;
634
0
    }
635
636
0
    (*alphaImageResult)->set_sample_duration(colorConvertedImage->get_sample_duration());
637
638
    // Pass the resolved options (not the caller's in_options) so the alpha aux
639
    // track inherits the color track's content_kind instead of re-resolving
640
    // from its own 'auxv' handler.
641
0
    auto err = m_aux_alpha_track->encode_image(*alphaImageResult,
642
0
                                               m_alpha_track_encoder.get(),
643
0
                                               effective_options.get(),
644
0
                                               heif_image_input_class_alpha);
645
0
    if (err) {
646
0
      return err;
647
0
    }
648
0
  }
649
650
0
  return {};
651
0
}
652
653
654
Result<bool> Track_Visual::process_encoded_data(heif_encoder* h_encoder)
655
0
{
656
0
  auto encoder = m_chunks.back()->get_encoder();
657
658
0
  std::optional<Encoder::CodedImageData> encodingResult = encoder->encode_sequence_extract_data();
659
0
  if (!encodingResult) {
660
    // nothing pending in the encoder
661
0
    return {false};
662
0
  }
663
664
0
  const Encoder::CodedImageData& data = *encodingResult;
665
666
667
0
  if (data.bitstream.empty() &&
668
0
      data.properties.empty()) {
669
0
    return {false};
670
0
  }
671
672
  // --- generate SampleDescriptionBox
673
674
0
  if (!m_generated_sample_description_box) {
675
0
    auto sample_description_box = encoder->get_sample_description_box(data);
676
0
    if (sample_description_box) {
677
0
      VisualSampleEntry& visualSampleEntry = sample_description_box->get_VisualSampleEntry();
678
0
      visualSampleEntry.width = m_width;
679
0
      visualSampleEntry.height = m_height;
680
681
      // add Coding-Constraints box (ccst) only if we are generating an image sequence
682
683
      // TODO: does the alpha track also need a ccst box?
684
      //       ComplianceWarden says so (and it makes sense), but HEIF says that 'ccst' shall be present
685
      //       if the handler is 'pict'. However, the alpha track is 'auxv'.
686
0
      if (true) { // m_hdlr->get_handler_type() == heif_track_type_image_sequence) {
687
0
        auto ccst = std::make_shared<Box_ccst>();
688
0
        ccst->set_coding_constraints(data.codingConstraints);
689
0
        sample_description_box->append_child_box(ccst);
690
0
      }
691
692
0
      if (m_image_class == heif_image_input_class_alpha) {
693
0
        auto auxi_box = std::make_shared<Box_auxi>();
694
0
        auxi_box->set_aux_track_type_urn(get_track_auxiliary_info_type(h_encoder->plugin->compression_format));
695
0
        sample_description_box->append_child_box(auxi_box);
696
0
      }
697
698
0
      set_sample_description_box(sample_description_box);
699
0
      m_generated_sample_description_box = true;
700
0
    }
701
0
  }
702
703
0
  if (!data.bitstream.empty()) {
704
0
    uintptr_t frame_number = data.frame_nr;
705
706
0
    auto& user_data = m_frame_user_data[frame_number];
707
708
0
    int32_t decoding_time = static_cast<int32_t>(m_stsz->num_samples()) * m_sample_duration;
709
0
    int32_t composition_time = static_cast<int32_t>(frame_number) * m_sample_duration;
710
711
0
    Error err = write_sample_data(data.bitstream,
712
0
                                  user_data.sample_duration,
713
0
                                  composition_time - decoding_time,
714
0
                                  data.is_sync_frame,
715
0
                                  user_data.tai_timestamp,
716
0
                                  user_data.gimi_content_id);
717
718
0
    user_data.release();
719
0
    m_frame_user_data.erase(frame_number);
720
721
0
    if (err) {
722
0
      return err;
723
0
    }
724
0
  }
725
726
0
  return {true};
727
0
}
728
729
730
Error Track_Visual::finalize_track()
731
0
{
732
0
  if (m_active_encoder) {
733
0
    Error err = encode_end_of_sequence(m_active_encoder);
734
0
    if (err) {
735
0
      return err;
736
0
    }
737
0
  }
738
739
0
  return Track::finalize_track();
740
0
}
741
742
743
heif_brand2 Track_Visual::get_compatible_brand() const
744
0
{
745
0
  if (m_stsd->get_num_sample_entries() == 0) {
746
0
    return 0; // TODO: error ? Or can we assume at this point that there is at least one sample entry?
747
0
  }
748
749
0
  auto sampleEntry = m_stsd->get_sample_entry(0);
750
751
0
  uint32_t sample_entry_type = sampleEntry->get_short_type();
752
0
  switch (sample_entry_type) {
753
0
    case fourcc("hvc1"): {
754
0
      auto hvcC = sampleEntry->get_child_box<Box_hvcC>();
755
0
      if (!hvcC) { return 0; }
756
757
0
      const auto& config = hvcC->get_configuration();
758
0
      if (config.is_profile_compatible(HEVCDecoderConfigurationRecord::Profile_Main) ||
759
0
          config.is_profile_compatible(HEVCDecoderConfigurationRecord::Profile_MainStillPicture)) {
760
0
        return heif_brand2_hevc;
761
0
      }
762
0
      else {
763
0
        return heif_brand2_hevx;
764
0
      }
765
0
    }
766
767
0
    case fourcc("avc1"):
768
0
      return heif_brand2_avcs;
769
770
0
    case fourcc("av01"):
771
0
      return heif_brand2_avis;
772
773
0
    case fourcc("j2ki"):
774
0
      return heif_brand2_j2is;
775
776
0
    case fourcc("mjpg"):
777
0
      return heif_brand2_jpgs;
778
779
0
    case fourcc("vvc1"):
780
0
      return heif_brand2_vvis;
781
782
0
    default:
783
0
      return 0;
784
0
  }
785
0
}