/src/libheif/libheif/sequences/track_visual.cc
Line | Count | Source |
1 | | /* |
2 | | * HEIF image base codec. |
3 | | * Copyright (c) 2025 Dirk Farin <dirk.farin@gmail.com> |
4 | | * |
5 | | * This file is part of libheif. |
6 | | * |
7 | | * libheif is free software: you can redistribute it and/or modify |
8 | | * it under the terms of the GNU Lesser General Public License as |
9 | | * published by the Free Software Foundation, either version 3 of |
10 | | * the License, or (at your option) any later version. |
11 | | * |
12 | | * libheif is distributed in the hope that it will be useful, |
13 | | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
14 | | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
15 | | * GNU Lesser General Public License for more details. |
16 | | * |
17 | | * You should have received a copy of the GNU Lesser General Public License |
18 | | * along with libheif. If not, see <http://www.gnu.org/licenses/>. |
19 | | */ |
20 | | |
21 | | #include "track_visual.h" |
22 | | |
23 | | #include <memory> |
24 | | #include <utility> |
25 | | #include "codecs/decoder.h" |
26 | | #include "codecs/encoder.h" |
27 | | #include "chunk.h" |
28 | | #include "image/pixelimage.h" |
29 | | #include "context.h" |
30 | | #include "api_structs.h" |
31 | | #include "codecs/hevc_boxes.h" |
32 | | #include "codecs/uncompressed/unc_boxes.h" |
33 | | |
34 | | |
35 | | Track_Visual::Track_Visual(HeifContext* ctx) |
36 | 9 | : Track(ctx) |
37 | 9 | { |
38 | 9 | } |
39 | | |
40 | | |
41 | | Track_Visual::~Track_Visual() |
42 | 9 | { |
43 | 9 | for (auto& user_data : m_frame_user_data) { |
44 | 0 | user_data.second.release(); |
45 | 0 | } |
46 | 9 | } |
47 | | |
48 | | |
49 | | Error Track_Visual::load(const std::shared_ptr<Box_trak>& trak) |
50 | 9 | { |
51 | 9 | Error parentLoadError = Track::load(trak); |
52 | 9 | if (parentLoadError) { |
53 | 9 | return parentLoadError; |
54 | 9 | } |
55 | | |
56 | 0 | const std::vector<uint32_t>& chunk_offsets = m_stco->get_offsets(); |
57 | | |
58 | | // Find sequence resolution |
59 | |
|
60 | 0 | if (!chunk_offsets.empty()) { |
61 | 0 | auto* s2c = m_stsc->get_chunk(1); |
62 | 0 | if (!s2c) { |
63 | 0 | return { |
64 | 0 | heif_error_Invalid_input, |
65 | 0 | heif_suberror_Unspecified, |
66 | 0 | "Visual track has no chunk 1" |
67 | 0 | }; |
68 | 0 | } |
69 | | |
70 | 0 | Box_stsc::SampleToChunk sampleToChunk = *s2c; |
71 | |
|
72 | 0 | auto sample_description = m_stsd->get_sample_entry(sampleToChunk.sample_description_index - 1); |
73 | 0 | if (!sample_description) { |
74 | 0 | return { |
75 | 0 | heif_error_Invalid_input, |
76 | 0 | heif_suberror_Unspecified, |
77 | 0 | "Visual track has sample description" |
78 | 0 | }; |
79 | 0 | } |
80 | | |
81 | 0 | auto visual_sample_description = std::dynamic_pointer_cast<const Box_VisualSampleEntry>(sample_description); |
82 | 0 | if (!visual_sample_description) { |
83 | 0 | return { |
84 | 0 | heif_error_Invalid_input, |
85 | 0 | heif_suberror_Unspecified, |
86 | 0 | "Visual track sample description does not match visual track." |
87 | 0 | }; |
88 | 0 | } |
89 | | |
90 | 0 | m_width = visual_sample_description->get_VisualSampleEntry_const().width; |
91 | 0 | m_height = visual_sample_description->get_VisualSampleEntry_const().height; |
92 | 0 | } |
93 | | |
94 | 0 | return {}; |
95 | 0 | } |
96 | | |
97 | | |
98 | | Error Track_Visual::initialize_after_parsing(HeifContext* ctx, const std::vector<std::shared_ptr<Track> >& all_tracks) |
99 | 0 | { |
100 | | // --- check whether there is an auxiliary alpha track assigned to this track |
101 | | |
102 | | // Only assign to image-sequence tracks (TODO: are there also alpha tracks allowed for video tracks 'heif_track_type_video'?) |
103 | |
|
104 | 0 | if (get_handler() == heif_track_type_image_sequence) { |
105 | 0 | for (const auto& track : all_tracks) { |
106 | | // skip ourselves |
107 | 0 | if (track->get_id() != get_id()) { |
108 | | // Is this an aux alpha track? |
109 | 0 | auto h = fourcc_to_string(track->get_handler()); |
110 | 0 | if (track->get_handler() == heif_track_type_auxiliary && |
111 | 0 | track->get_auxiliary_info_type() == heif_auxiliary_track_info_type_alpha) { |
112 | | // Is it assigned to the current track |
113 | 0 | auto tref = track->get_tref_box(); |
114 | 0 | if (!tref) { |
115 | 0 | return { |
116 | 0 | heif_error_Invalid_input, |
117 | 0 | heif_suberror_Unspecified, |
118 | 0 | "Auxiliary track without 'tref'" |
119 | 0 | }; |
120 | 0 | } |
121 | | |
122 | 0 | auto references = tref->get_references(fourcc("auxl")); |
123 | 0 | if (std::any_of(references.begin(), references.end(), [this](uint32_t id) { return id == get_id(); })) { |
124 | | // Assign it |
125 | |
|
126 | 0 | m_aux_alpha_track = std::dynamic_pointer_cast<Track_Visual>(track); |
127 | 0 | } |
128 | 0 | } |
129 | 0 | } |
130 | 0 | } |
131 | 0 | } |
132 | | |
133 | 0 | return {}; |
134 | 0 | } |
135 | | |
136 | | |
137 | | Track_Visual::Track_Visual(HeifContext* ctx, uint32_t track_id, uint16_t width, uint16_t height, |
138 | | const TrackOptions* options, uint32_t handler_type) |
139 | 0 | : Track(ctx, track_id, options, handler_type) |
140 | 0 | { |
141 | 0 | m_tkhd->set_resolution(width, height); |
142 | | //m_hdlr->set_handler_type(handler_type); already done in Track() |
143 | |
|
144 | 0 | auto vmhd = std::make_shared<Box_vmhd>(); |
145 | 0 | m_minf->append_child_box(vmhd); |
146 | 0 | } |
147 | | |
148 | | |
149 | | bool Track_Visual::has_alpha_channel() const |
150 | 0 | { |
151 | 0 | if (m_aux_alpha_track != nullptr) { |
152 | 0 | return true; |
153 | 0 | } |
154 | | |
155 | | // --- special case: 'uncv' with alpha component |
156 | | #if WITH_UNCOMPRESSED_CODEC |
157 | | if (m_stsd) { |
158 | | auto sampleEntry = m_stsd->get_sample_entry(0); |
159 | | if (sampleEntry) { |
160 | | if (auto box_uncv = std::dynamic_pointer_cast<const Box_uncv>(sampleEntry)) { |
161 | | if (auto cmpd = box_uncv->get_child_box<const Box_cmpd>()) { |
162 | | if (cmpd->has_component(heif_cmpd_component_type_alpha)) { |
163 | | return true; |
164 | | } |
165 | | } |
166 | | } |
167 | | } |
168 | | } |
169 | | #endif |
170 | | |
171 | 0 | return false; |
172 | 0 | } |
173 | | |
174 | | |
175 | | Result<std::shared_ptr<HeifPixelImage> > Track_Visual::decode_next_image_sample(const heif_decoding_options& options) |
176 | 0 | { |
177 | | // --- If we ignore the editlist, we stop when we reached the end of the original samples. |
178 | |
|
179 | 0 | uint64_t num_output_samples = m_num_output_samples; |
180 | 0 | if (options.ignore_sequence_editlist) { |
181 | 0 | num_output_samples = m_num_samples; |
182 | 0 | } |
183 | | |
184 | | // --- Did we reach the end of the sequence? |
185 | |
|
186 | 0 | if (m_next_sample_to_be_output >= num_output_samples) { |
187 | 0 | return Error{ |
188 | 0 | heif_error_End_of_sequence, |
189 | 0 | heif_suberror_Unspecified, |
190 | 0 | "End of sequence" |
191 | 0 | }; |
192 | 0 | } |
193 | | |
194 | | |
195 | 0 | std::shared_ptr<HeifPixelImage> image; |
196 | |
|
197 | 0 | uint32_t sample_idx_in_chunk; |
198 | 0 | uintptr_t decoded_sample_idx = 0; // TODO: map this to sample idx + chunk |
199 | |
|
200 | 0 | for (;;) { |
201 | 0 | const SampleTiming& sampleTiming = m_presentation_timeline[m_next_sample_to_be_decoded % m_presentation_timeline.size()]; |
202 | 0 | sample_idx_in_chunk = sampleTiming.sampleIdx; |
203 | 0 | uint32_t chunk_idx = sampleTiming.chunkIdx; |
204 | |
|
205 | 0 | const std::shared_ptr<Chunk>& chunk = m_chunks[chunk_idx]; |
206 | |
|
207 | 0 | auto decoder = chunk->get_decoder(); |
208 | 0 | assert(decoder); |
209 | | |
210 | | // avoid calling get_decoded_frame() before starting the decoder. |
211 | 0 | if (m_next_sample_to_be_decoded != 0) { |
212 | 0 | Result<std::shared_ptr<HeifPixelImage> > getFrameResult = decoder->get_decoded_frame(options, |
213 | 0 | &decoded_sample_idx, |
214 | 0 | m_heif_context->get_security_limits()); |
215 | 0 | if (getFrameResult.error()) { |
216 | 0 | return getFrameResult.error(); |
217 | 0 | } |
218 | | |
219 | | |
220 | | // We received a decoded frame. Exit "push data" / "decode" loop. |
221 | | |
222 | 0 | if (*getFrameResult != nullptr) { |
223 | 0 | image = *getFrameResult; |
224 | | |
225 | | // If this was the last frame in the EditList segment, reset the 'flushed' flag |
226 | | // in case we have to restart the decoder for another repetition of the segment. |
227 | 0 | if ((m_next_sample_to_be_output + 1) % m_presentation_timeline.size() == 0) { |
228 | 0 | m_decoder_is_flushed = false; |
229 | 0 | } |
230 | |
|
231 | 0 | break; |
232 | 0 | } |
233 | | |
234 | | // If the sequence has ended and the decoder was flushed, but we still did not receive |
235 | | // the image, we are waiting for, this is an error. |
236 | 0 | if (m_decoder_is_flushed) { |
237 | 0 | return Error(heif_error_Decoder_plugin_error, |
238 | 0 | heif_suberror_Unspecified, |
239 | 0 | "Did not decode all frames"); |
240 | 0 | } |
241 | 0 | } |
242 | | |
243 | | |
244 | | // --- Push more data into the decoder (or send end-of-sequence). |
245 | | |
246 | 0 | if (m_next_sample_to_be_decoded < m_num_output_samples) { |
247 | | |
248 | | // --- Find the data extent that stores the compressed frame data. |
249 | |
|
250 | 0 | DataExtent extent = chunk->get_data_extent_for_sample(sample_idx_in_chunk); |
251 | 0 | decoder->set_data_extent(std::move(extent)); |
252 | | |
253 | | // std::cout << "PUSH chunk " << chunk_idx << " sample " << sample_idx << " (" << extent.m_size << " bytes)\n"; |
254 | | |
255 | | // advance decoding index to next in segment |
256 | 0 | m_next_sample_to_be_decoded++; |
257 | | |
258 | | // Send the decoder configuration when we send the first sample of the chunk. |
259 | | // The configuration NALs might change for each chunk. |
260 | 0 | const bool is_first_sample = (sample_idx_in_chunk == 0); |
261 | | |
262 | | // --- Push data into the decoder. |
263 | 0 | Error decodingError = decoder->decode_sequence_frame_from_compressed_data(is_first_sample, |
264 | 0 | options, |
265 | 0 | sample_idx_in_chunk, // user data |
266 | 0 | m_heif_context->get_security_limits()); |
267 | 0 | if (decodingError) { |
268 | 0 | return decodingError; |
269 | 0 | } |
270 | 0 | } |
271 | 0 | else { |
272 | | // --- End of sequence (Editlist segment) reached. |
273 | | |
274 | | // std::cout << "FLUSH\n"; |
275 | 0 | Error flushError = decoder->flush_decoder(); |
276 | 0 | if (flushError) { |
277 | 0 | return flushError; |
278 | 0 | } |
279 | | |
280 | 0 | m_decoder_is_flushed = true; |
281 | 0 | } |
282 | 0 | } |
283 | | |
284 | | |
285 | | // --- We have received a new decoded image. |
286 | | // Postprocess decoded image, attach metadata. |
287 | | |
288 | 0 | if (m_stts) { |
289 | | // 'sample_idx_in_chunk' follows m_next_sample_to_be_decoded, which runs ahead of the output |
290 | | // by the decoder latency. Look the duration up by the output position instead, like |
291 | | // Track::get_next_sample_raw_data() does. |
292 | 0 | const SampleTiming& outputTiming = m_presentation_timeline[m_next_sample_to_be_output % m_presentation_timeline.size()]; |
293 | 0 | image->set_sample_duration(m_stts->get_sample_duration(outputTiming.sampleIdx)); |
294 | 0 | } |
295 | | |
296 | | // --- assign alpha if we have an assigned alpha track |
297 | |
|
298 | 0 | if (m_aux_alpha_track) { |
299 | 0 | auto alphaResult = m_aux_alpha_track->decode_next_image_sample(options); |
300 | 0 | if (!alphaResult) { |
301 | 0 | return alphaResult.error(); |
302 | 0 | } |
303 | | |
304 | 0 | auto alphaImage = *alphaResult; |
305 | | |
306 | | // The alpha auxiliary track may have a different size than the main track. Nothing in the |
307 | | // standard forbids this. We scale it to the main image size so that downstream consumers can |
308 | | // rely on the alpha plane matching the color planes. This mirrors what the still-image decoder |
309 | | // does in ImageItem (see image_item.cc). As noted there, we may later add a decoding option to |
310 | | // turn this automatic scaling off, but that requires that libheif no longer assumes everywhere |
311 | | // that the alpha channel has the same resolution as the color channels. |
312 | 0 | if (alphaImage->get_width() != image->get_width() || |
313 | 0 | alphaImage->get_height() != image->get_height()) { |
314 | 0 | std::shared_ptr<HeifPixelImage> scaled_alpha; |
315 | 0 | Error err = alphaImage->scale_nearest_neighbor(scaled_alpha, image->get_width(), image->get_height(), |
316 | 0 | m_heif_context->get_security_limits()); |
317 | 0 | if (err) { |
318 | 0 | return err; |
319 | 0 | } |
320 | 0 | alphaImage = std::move(scaled_alpha); |
321 | 0 | } |
322 | | |
323 | 0 | if (Error err = image->transfer_channel_from_image_as(alphaImage, heif_channel_Y, heif_channel_Alpha)) { |
324 | 0 | return err; |
325 | 0 | } |
326 | 0 | } |
327 | | |
328 | | |
329 | | // --- read sample auxiliary data |
330 | | |
331 | 0 | if (m_aux_reader_content_ids) { |
332 | 0 | auto readResult = m_aux_reader_content_ids->get_sample_info(get_file().get(), (uint32_t)decoded_sample_idx); |
333 | 0 | if (!readResult) { |
334 | 0 | return readResult.error(); |
335 | 0 | } |
336 | | |
337 | 0 | Result<std::string> convResult = vector_to_string(*readResult); |
338 | 0 | if (!convResult) { |
339 | 0 | return convResult.error(); |
340 | 0 | } |
341 | | |
342 | 0 | image->set_gimi_sample_content_id(*convResult); |
343 | 0 | } |
344 | | |
345 | 0 | if (m_aux_reader_tai_timestamps) { |
346 | 0 | auto readResult = m_aux_reader_tai_timestamps->get_sample_info(get_file().get(), (uint32_t)decoded_sample_idx); |
347 | 0 | if (!readResult) { |
348 | 0 | return readResult.error(); |
349 | 0 | } |
350 | | |
351 | 0 | std::vector<uint8_t>& tai_data = *readResult; |
352 | 0 | if (!tai_data.empty()) { |
353 | 0 | auto resultTai = Box_itai::decode_tai_from_vector(tai_data); |
354 | 0 | if (!resultTai) { |
355 | 0 | return resultTai.error(); |
356 | 0 | } |
357 | | |
358 | 0 | image->set_tai_timestamp(&*resultTai); |
359 | 0 | } |
360 | 0 | } |
361 | | |
362 | 0 | m_next_sample_to_be_output++; |
363 | |
|
364 | 0 | return image; |
365 | 0 | } |
366 | | |
367 | | |
368 | | Error Track_Visual::encode_end_of_sequence(heif_encoder* h_encoder) |
369 | 0 | { |
370 | 0 | auto encoder = m_chunks.back()->get_encoder(); |
371 | |
|
372 | 0 | for (;;) { |
373 | 0 | Error err = encoder->encode_sequence_flush(h_encoder); |
374 | 0 | if (err) { |
375 | 0 | return err; |
376 | 0 | } |
377 | | |
378 | 0 | Result<bool> processingResult = process_encoded_data(h_encoder); |
379 | 0 | if (processingResult.is_error()) { |
380 | 0 | return processingResult.error(); |
381 | 0 | } |
382 | | |
383 | 0 | if (!*processingResult) { |
384 | 0 | break; |
385 | 0 | } |
386 | 0 | } |
387 | | |
388 | | |
389 | | // --- also end alpha track |
390 | | |
391 | 0 | if (m_aux_alpha_track) { |
392 | 0 | auto err = m_aux_alpha_track->encode_end_of_sequence(m_alpha_track_encoder.get()); |
393 | 0 | if (err) { |
394 | 0 | return err; |
395 | 0 | } |
396 | 0 | } |
397 | | |
398 | 0 | return {}; |
399 | 0 | } |
400 | | |
401 | | |
402 | | Error Track_Visual::encode_image(const std::shared_ptr<HeifPixelImage>& image, |
403 | | heif_encoder* h_encoder, |
404 | | const heif_sequence_encoding_options* in_options, |
405 | | heif_image_input_class input_class) |
406 | 0 | { |
407 | 0 | if (image->get_width() > 0xFFFF || |
408 | 0 | image->get_height() > 0xFFFF) { |
409 | 0 | return { |
410 | 0 | heif_error_Invalid_input, |
411 | 0 | heif_suberror_Unspecified, |
412 | 0 | "Input image resolution too high" |
413 | 0 | }; |
414 | 0 | } |
415 | | |
416 | 0 | m_image_class = input_class; |
417 | | |
418 | | |
419 | | // --- resolve the encoding options |
420 | | |
421 | | // Build an effective options struct so that the rest of this function never |
422 | | // has to look at the caller's pointer. 'in_options' may be NULL, which the API |
423 | | // documents as "use the default options", so every field has to come from a |
424 | | // fully populated struct instead. This also lets libheif resolve "auto" fields |
425 | | // (e.g. content_kind) to concrete values before passing them to the encoder |
426 | | // plugin. heif_sequence_encoding_options_copy is version-aware and ignores a |
427 | | // NULL source, so callers built against older headers won't be read past their |
428 | | // actual allocation and a NULL source simply leaves the defaults in place. |
429 | |
|
430 | 0 | std::unique_ptr<heif_sequence_encoding_options, void(*)(heif_sequence_encoding_options*)> |
431 | 0 | effective_options(heif_sequence_encoding_options_alloc(), |
432 | 0 | heif_sequence_encoding_options_release); |
433 | 0 | heif_sequence_encoding_options_copy(effective_options.get(), in_options); |
434 | | |
435 | | // Resolve content_kind=auto based on this track's handler type. For auxiliary |
436 | | // tracks (e.g. alpha) the parent track resolves the value before recursing in, |
437 | | // so we never reach this branch with handler=auxv. |
438 | | // TODO: in the future, "auto" could also factor in input frame rate or |
439 | | // frame-to-frame similarity. |
440 | 0 | if (effective_options->content_kind == heif_sequence_content_kind_auto) { |
441 | 0 | switch (get_handler()) { |
442 | 0 | case heif_track_type_video: |
443 | 0 | effective_options->content_kind = heif_sequence_content_kind_video; |
444 | 0 | break; |
445 | 0 | case heif_track_type_image_sequence: |
446 | 0 | default: |
447 | 0 | effective_options->content_kind = heif_sequence_content_kind_image_sequence; |
448 | 0 | break; |
449 | 0 | } |
450 | 0 | } |
451 | | |
452 | | |
453 | | // --- If input has an alpha channel, add an alpha auxiliary track. |
454 | | |
455 | 0 | if (effective_options->save_alpha_channel && image->has_alpha() && !m_aux_alpha_track && |
456 | 0 | h_encoder->plugin->compression_format != heif_compression_uncompressed) { // TODO: ask plugin |
457 | 0 | if (m_active_encoder) { |
458 | 0 | return { |
459 | 0 | heif_error_Usage_error, |
460 | 0 | heif_suberror_Unspecified, |
461 | 0 | "Input images must all either have an alpha channel or none of them." |
462 | 0 | }; |
463 | 0 | } |
464 | 0 | else { |
465 | | // alpha track uses default options, same timescale as color track |
466 | |
|
467 | 0 | TrackOptions alphaOptions; |
468 | 0 | alphaOptions.track_timescale = m_track_info.track_timescale; |
469 | |
|
470 | 0 | auto newAlphaTrackResult = m_heif_context->add_visual_sequence_track(&alphaOptions, |
471 | 0 | heif_track_type_auxiliary, |
472 | 0 | static_cast<uint16_t>(image->get_width()), |
473 | 0 | static_cast<uint16_t>(image->get_height())); |
474 | |
|
475 | 0 | if (auto err = newAlphaTrackResult.error()) { |
476 | 0 | return err; |
477 | 0 | } |
478 | | |
479 | | // add a reference to the color track |
480 | | |
481 | 0 | m_aux_alpha_track = *newAlphaTrackResult; |
482 | 0 | m_aux_alpha_track->add_reference_to_track(fourcc("auxl"), m_id); |
483 | | |
484 | | // make a copy of the encoder from the color track for encoding the alpha track |
485 | |
|
486 | 0 | m_alpha_track_encoder = std::make_unique<heif_encoder>(h_encoder->plugin); |
487 | 0 | heif_error err = m_alpha_track_encoder->alloc(); |
488 | 0 | if (err.code) { |
489 | 0 | return {err.code, err.subcode, err.message}; |
490 | 0 | } |
491 | 0 | m_alpha_track_encoder->copy_parameters_from(*h_encoder); |
492 | 0 | } |
493 | 0 | } |
494 | | |
495 | | |
496 | 0 | if (!m_active_encoder) { |
497 | 0 | m_active_encoder = h_encoder; |
498 | 0 | } |
499 | 0 | else if (m_active_encoder != h_encoder) { |
500 | 0 | return { |
501 | 0 | heif_error_Usage_error, |
502 | 0 | heif_suberror_Unspecified, |
503 | 0 | "You may not switch the heif_encoder while encoding a sequence." |
504 | 0 | }; |
505 | 0 | } |
506 | | |
507 | 0 | if (h_encoder->plugin->plugin_api_version < 4) { |
508 | 0 | return Error{ |
509 | 0 | heif_error_Plugin_loading_error, heif_suberror_No_matching_decoder_installed, |
510 | 0 | "Encoder plugin needs to be at least version 4." |
511 | 0 | }; |
512 | 0 | } |
513 | | |
514 | | // === generate compressed image bitstream |
515 | | |
516 | | // generate new chunk for first image or when compression formats don't match |
517 | | |
518 | 0 | if (m_chunks.empty() || m_chunks.back()->get_compression_format() != h_encoder->plugin->compression_format) { |
519 | 0 | add_chunk(h_encoder->plugin->compression_format); |
520 | 0 | } |
521 | | |
522 | | // --- check whether we have to convert the image color space |
523 | | |
524 | | // The reason for doing the color conversion here is that the input might be an RGBA image and the color conversion |
525 | | // will extract the alpha plane anyway. We can reuse that plane below instead of having to do a new conversion. |
526 | |
|
527 | 0 | auto encoder = m_chunks.back()->get_encoder(); |
528 | |
|
529 | 0 | const heif_color_profile_nclx* output_nclx; |
530 | 0 | heif_color_profile_nclx nclx; |
531 | |
|
532 | 0 | if (const auto* image_nclx = encoder->get_forced_output_nclx()) { |
533 | 0 | output_nclx = const_cast<heif_color_profile_nclx*>(image_nclx); |
534 | 0 | } |
535 | 0 | else if (in_options) { |
536 | 0 | output_nclx = in_options->output_nclx_profile; |
537 | 0 | } |
538 | 0 | else { |
539 | | // Note: this is the one place that still distinguishes "caller passed no |
540 | | // options" from "caller passed options with a NULL output_nclx_profile", |
541 | | // so it reads in_options rather than effective_options. Both cases mean |
542 | | // "keep the input image's parameters", but they express it differently: a |
543 | | // NULL profile lets compute_target_nclx_profile() derive them, while the |
544 | | // branch below builds the profile here. The two differ only for an image |
545 | | // that carries no NCLX at all, where the explicit undefined profile built |
546 | | // below makes nclx_profile_matches_spec() request a conversion that a NULL |
547 | | // profile would skip. Unifying them would change that behaviour, so it is |
548 | | // left alone here. |
549 | 0 | if (image->has_nclx_color_profile()) { |
550 | 0 | nclx_profile input_nclx = image->get_color_profile_nclx(); |
551 | |
|
552 | 0 | nclx.version = 1; |
553 | 0 | nclx.color_primaries = (enum heif_color_primaries) input_nclx.get_colour_primaries(); |
554 | 0 | nclx.transfer_characteristics = (enum heif_transfer_characteristics) input_nclx.get_transfer_characteristics(); |
555 | 0 | nclx.matrix_coefficients = (enum heif_matrix_coefficients) input_nclx.get_matrix_coefficients(); |
556 | 0 | nclx.full_range_flag = input_nclx.get_full_range_flag(); |
557 | |
|
558 | 0 | output_nclx = &nclx; |
559 | 0 | } |
560 | 0 | else { |
561 | 0 | nclx_profile undefined_nclx; |
562 | 0 | undefined_nclx.copy_to_heif_color_profile_nclx(&nclx); |
563 | |
|
564 | 0 | output_nclx = &nclx; |
565 | 0 | } |
566 | 0 | } |
567 | |
|
568 | 0 | Result<std::shared_ptr<HeifPixelImage> > srcImageResult = encoder->convert_colorspace_for_encoding(image, |
569 | 0 | h_encoder, |
570 | 0 | output_nclx, |
571 | 0 | &effective_options->color_conversion_options, |
572 | 0 | m_heif_context->get_security_limits()); |
573 | 0 | if (!srcImageResult) { |
574 | 0 | return srcImageResult.error(); |
575 | 0 | } |
576 | | |
577 | 0 | std::shared_ptr<HeifPixelImage> colorConvertedImage = *srcImageResult; |
578 | | |
579 | | // integer range is checked at beginning of function. |
580 | 0 | assert(colorConvertedImage->get_width() == image->get_width()); |
581 | 0 | assert(colorConvertedImage->get_height() == image->get_height()); |
582 | 0 | m_width = static_cast<uint16_t>(colorConvertedImage->get_width()); |
583 | 0 | m_height = static_cast<uint16_t>(colorConvertedImage->get_height()); |
584 | | |
585 | | // --- encode image |
586 | |
|
587 | 0 | Error encodeError = encoder->encode_sequence_frame(colorConvertedImage, h_encoder, |
588 | 0 | *effective_options, |
589 | 0 | input_class, |
590 | 0 | colorConvertedImage->get_sample_duration(), get_timescale(), |
591 | 0 | m_current_frame_nr); |
592 | |
|
593 | 0 | if (encodeError) { |
594 | 0 | return encodeError; |
595 | 0 | } |
596 | | |
597 | 0 | m_sample_duration = colorConvertedImage->get_sample_duration(); |
598 | | // TODO heif_tai_timestamp_packet* tai = image->get_tai_timestamp(); |
599 | | // TODO image->has_gimi_sample_content_id() ? image->get_gimi_sample_content_id() : std::string{}); |
600 | | |
601 | | |
602 | | // store frame user data |
603 | |
|
604 | 0 | FrameUserData userData; |
605 | 0 | userData.sample_duration = colorConvertedImage->get_sample_duration(); |
606 | 0 | if (image->has_gimi_sample_content_id()) { |
607 | 0 | userData.gimi_content_id = image->get_gimi_sample_content_id(); |
608 | 0 | } |
609 | |
|
610 | 0 | if (const auto* tai = image->get_tai_timestamp()) { |
611 | 0 | userData.tai_timestamp = heif_tai_timestamp_packet_alloc(); |
612 | 0 | heif_tai_timestamp_packet_copy(userData.tai_timestamp, tai); |
613 | 0 | } |
614 | |
|
615 | 0 | m_frame_user_data[m_current_frame_nr] = userData; |
616 | |
|
617 | 0 | m_current_frame_nr++; |
618 | | |
619 | | // --- get compressed data from encoder |
620 | |
|
621 | 0 | Result<bool> processingResult = process_encoded_data(h_encoder); |
622 | 0 | if (auto err = processingResult.error()) { |
623 | 0 | return err; |
624 | 0 | } |
625 | | |
626 | | |
627 | | // --- encode alpha channel into auxiliary track |
628 | | |
629 | 0 | if (m_aux_alpha_track) { |
630 | 0 | auto alphaImageResult = create_alpha_image_from_image_alpha_channel(colorConvertedImage, |
631 | 0 | m_heif_context->get_security_limits()); |
632 | 0 | if (auto err = alphaImageResult.error()) { |
633 | 0 | return err; |
634 | 0 | } |
635 | | |
636 | 0 | (*alphaImageResult)->set_sample_duration(colorConvertedImage->get_sample_duration()); |
637 | | |
638 | | // Pass the resolved options (not the caller's in_options) so the alpha aux |
639 | | // track inherits the color track's content_kind instead of re-resolving |
640 | | // from its own 'auxv' handler. |
641 | 0 | auto err = m_aux_alpha_track->encode_image(*alphaImageResult, |
642 | 0 | m_alpha_track_encoder.get(), |
643 | 0 | effective_options.get(), |
644 | 0 | heif_image_input_class_alpha); |
645 | 0 | if (err) { |
646 | 0 | return err; |
647 | 0 | } |
648 | 0 | } |
649 | | |
650 | 0 | return {}; |
651 | 0 | } |
652 | | |
653 | | |
654 | | Result<bool> Track_Visual::process_encoded_data(heif_encoder* h_encoder) |
655 | 0 | { |
656 | 0 | auto encoder = m_chunks.back()->get_encoder(); |
657 | |
|
658 | 0 | std::optional<Encoder::CodedImageData> encodingResult = encoder->encode_sequence_extract_data(); |
659 | 0 | if (!encodingResult) { |
660 | | // nothing pending in the encoder |
661 | 0 | return {false}; |
662 | 0 | } |
663 | | |
664 | 0 | const Encoder::CodedImageData& data = *encodingResult; |
665 | | |
666 | |
|
667 | 0 | if (data.bitstream.empty() && |
668 | 0 | data.properties.empty()) { |
669 | 0 | return {false}; |
670 | 0 | } |
671 | | |
672 | | // --- generate SampleDescriptionBox |
673 | | |
674 | 0 | if (!m_generated_sample_description_box) { |
675 | 0 | auto sample_description_box = encoder->get_sample_description_box(data); |
676 | 0 | if (sample_description_box) { |
677 | 0 | VisualSampleEntry& visualSampleEntry = sample_description_box->get_VisualSampleEntry(); |
678 | 0 | visualSampleEntry.width = m_width; |
679 | 0 | visualSampleEntry.height = m_height; |
680 | | |
681 | | // add Coding-Constraints box (ccst) only if we are generating an image sequence |
682 | | |
683 | | // TODO: does the alpha track also need a ccst box? |
684 | | // ComplianceWarden says so (and it makes sense), but HEIF says that 'ccst' shall be present |
685 | | // if the handler is 'pict'. However, the alpha track is 'auxv'. |
686 | 0 | if (true) { // m_hdlr->get_handler_type() == heif_track_type_image_sequence) { |
687 | 0 | auto ccst = std::make_shared<Box_ccst>(); |
688 | 0 | ccst->set_coding_constraints(data.codingConstraints); |
689 | 0 | sample_description_box->append_child_box(ccst); |
690 | 0 | } |
691 | |
|
692 | 0 | if (m_image_class == heif_image_input_class_alpha) { |
693 | 0 | auto auxi_box = std::make_shared<Box_auxi>(); |
694 | 0 | auxi_box->set_aux_track_type_urn(get_track_auxiliary_info_type(h_encoder->plugin->compression_format)); |
695 | 0 | sample_description_box->append_child_box(auxi_box); |
696 | 0 | } |
697 | |
|
698 | 0 | set_sample_description_box(sample_description_box); |
699 | 0 | m_generated_sample_description_box = true; |
700 | 0 | } |
701 | 0 | } |
702 | |
|
703 | 0 | if (!data.bitstream.empty()) { |
704 | 0 | uintptr_t frame_number = data.frame_nr; |
705 | |
|
706 | 0 | auto& user_data = m_frame_user_data[frame_number]; |
707 | |
|
708 | 0 | int32_t decoding_time = static_cast<int32_t>(m_stsz->num_samples()) * m_sample_duration; |
709 | 0 | int32_t composition_time = static_cast<int32_t>(frame_number) * m_sample_duration; |
710 | |
|
711 | 0 | Error err = write_sample_data(data.bitstream, |
712 | 0 | user_data.sample_duration, |
713 | 0 | composition_time - decoding_time, |
714 | 0 | data.is_sync_frame, |
715 | 0 | user_data.tai_timestamp, |
716 | 0 | user_data.gimi_content_id); |
717 | |
|
718 | 0 | user_data.release(); |
719 | 0 | m_frame_user_data.erase(frame_number); |
720 | |
|
721 | 0 | if (err) { |
722 | 0 | return err; |
723 | 0 | } |
724 | 0 | } |
725 | | |
726 | 0 | return {true}; |
727 | 0 | } |
728 | | |
729 | | |
730 | | Error Track_Visual::finalize_track() |
731 | 0 | { |
732 | 0 | if (m_active_encoder) { |
733 | 0 | Error err = encode_end_of_sequence(m_active_encoder); |
734 | 0 | if (err) { |
735 | 0 | return err; |
736 | 0 | } |
737 | 0 | } |
738 | | |
739 | 0 | return Track::finalize_track(); |
740 | 0 | } |
741 | | |
742 | | |
743 | | heif_brand2 Track_Visual::get_compatible_brand() const |
744 | 0 | { |
745 | 0 | if (m_stsd->get_num_sample_entries() == 0) { |
746 | 0 | return 0; // TODO: error ? Or can we assume at this point that there is at least one sample entry? |
747 | 0 | } |
748 | | |
749 | 0 | auto sampleEntry = m_stsd->get_sample_entry(0); |
750 | |
|
751 | 0 | uint32_t sample_entry_type = sampleEntry->get_short_type(); |
752 | 0 | switch (sample_entry_type) { |
753 | 0 | case fourcc("hvc1"): { |
754 | 0 | auto hvcC = sampleEntry->get_child_box<Box_hvcC>(); |
755 | 0 | if (!hvcC) { return 0; } |
756 | | |
757 | 0 | const auto& config = hvcC->get_configuration(); |
758 | 0 | if (config.is_profile_compatible(HEVCDecoderConfigurationRecord::Profile_Main) || |
759 | 0 | config.is_profile_compatible(HEVCDecoderConfigurationRecord::Profile_MainStillPicture)) { |
760 | 0 | return heif_brand2_hevc; |
761 | 0 | } |
762 | 0 | else { |
763 | 0 | return heif_brand2_hevx; |
764 | 0 | } |
765 | 0 | } |
766 | | |
767 | 0 | case fourcc("avc1"): |
768 | 0 | return heif_brand2_avcs; |
769 | | |
770 | 0 | case fourcc("av01"): |
771 | 0 | return heif_brand2_avis; |
772 | | |
773 | 0 | case fourcc("j2ki"): |
774 | 0 | return heif_brand2_j2is; |
775 | | |
776 | 0 | case fourcc("mjpg"): |
777 | 0 | return heif_brand2_jpgs; |
778 | | |
779 | 0 | case fourcc("vvc1"): |
780 | 0 | return heif_brand2_vvis; |
781 | | |
782 | 0 | default: |
783 | 0 | return 0; |
784 | 0 | } |
785 | 0 | } |