/src/libheif/libheif/sequences/track.h
Line | Count | Source |
1 | | /* |
2 | | * HEIF image base codec. |
3 | | * Copyright (c) 2024 Dirk Farin <dirk.farin@gmail.com> |
4 | | * |
5 | | * This file is part of libheif. |
6 | | * |
7 | | * libheif is free software: you can redistribute it and/or modify |
8 | | * it under the terms of the GNU Lesser General Public License as |
9 | | * published by the Free Software Foundation, either version 3 of |
10 | | * the License, or (at your option) any later version. |
11 | | * |
12 | | * libheif is distributed in the hope that it will be useful, |
13 | | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
14 | | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
15 | | * GNU Lesser General Public License for more details. |
16 | | * |
17 | | * You should have received a copy of the GNU Lesser General Public License |
18 | | * along with libheif. If not, see <http://www.gnu.org/licenses/>. |
19 | | */ |
20 | | |
21 | | #ifndef LIBHEIF_TRACK_H |
22 | | #define LIBHEIF_TRACK_H |
23 | | |
24 | | #include "error.h" |
25 | | #include "api_structs.h" |
26 | | #include "security_limits.h" |
27 | | #include "libheif/heif_plugin.h" |
28 | | #include "libheif/heif_sequences.h" |
29 | | #include <string> |
30 | | #include <memory> |
31 | | #include <utility> |
32 | | #include <vector> |
33 | | |
34 | | class HeifContext; |
35 | | |
36 | | class HeifPixelImage; |
37 | | |
38 | | class Chunk; |
39 | | |
40 | | class Box_trak; |
41 | | |
42 | | |
43 | | class SampleAuxInfoHelper |
44 | | { |
45 | | public: |
46 | | SampleAuxInfoHelper(bool interleaved = false); |
47 | | |
48 | | void set_aux_info_type(uint32_t aux_info_type, uint32_t aux_info_type_parameter = 0); |
49 | | |
50 | | Error add_sample_info(const std::vector<uint8_t>& data); |
51 | | |
52 | | void add_nonpresent_sample(); |
53 | | |
54 | | void write_interleaved(const std::shared_ptr<HeifFile>& file); |
55 | | |
56 | | void write_all(const std::shared_ptr<Box>& parent, const std::shared_ptr<HeifFile>& file); |
57 | | |
58 | | private: |
59 | | std::shared_ptr<Box_saiz> m_saiz; |
60 | | std::shared_ptr<Box_saio> m_saio; |
61 | | |
62 | | std::vector<uint8_t> m_data; |
63 | | |
64 | | bool m_interleaved; |
65 | | }; |
66 | | |
67 | | |
68 | | class SampleAuxInfoReader |
69 | | { |
70 | | public: |
71 | | SampleAuxInfoReader(const std::shared_ptr<Box_saiz>&, |
72 | | const std::shared_ptr<Box_saio>&, |
73 | | const std::vector<std::shared_ptr<Chunk>>& chunks); |
74 | | |
75 | | heif_sample_aux_info_type get_type() const; |
76 | | |
77 | | Result<std::vector<uint8_t>> get_sample_info(const HeifFile* file, uint32_t sample_idx); |
78 | | |
79 | | private: |
80 | | std::shared_ptr<Box_saiz> m_saiz; |
81 | | std::shared_ptr<Box_saio> m_saio; |
82 | | |
83 | | // If there is only one chunk and the SAI data sizes are constant, we do not need an offset table. |
84 | | // We just store the base offset and can directly calculate the sample offset from that. |
85 | | bool m_contiguous_and_constant_size=false; |
86 | | uint64_t m_singleChunk_offset=0; |
87 | | |
88 | | // For chunked data or non-constant sample sizes, we use a table with the offsets for all SAI samples. |
89 | | std::vector<uint64_t> m_sample_offsets; |
90 | | }; |
91 | | |
92 | | |
93 | | /** |
94 | | * This structure specifies what will be written in a track and how it will be laid out in the file. |
95 | | */ |
96 | | struct TrackOptions |
97 | | { |
98 | | ~TrackOptions() |
99 | 0 | { |
100 | 0 | heif_tai_clock_info_release(tai_clock_info); |
101 | 0 | } |
102 | | |
103 | | // Timescale (clock ticks per second) for this track. |
104 | | uint32_t track_timescale = 90000; |
105 | | |
106 | | // If 'true', the aux_info data blocks will be interleaved with the compressed image. |
107 | | // This has the advantage that the aux_info is localized near the image data. |
108 | | // |
109 | | // If 'false', all aux_info will be written as one block after the compressed image data. |
110 | | // This has the advantage that no aux_info offsets have to be written. |
111 | | bool write_sample_aux_infos_interleaved = false; |
112 | | |
113 | | |
114 | | // --- TAI timestamps for samples |
115 | | heif_sample_aux_info_presence with_sample_tai_timestamps = heif_sample_aux_info_presence_none; |
116 | | heif_tai_clock_info* tai_clock_info = nullptr; |
117 | | |
118 | | // --- GIMI content IDs for samples |
119 | | |
120 | | heif_sample_aux_info_presence with_sample_content_ids = heif_sample_aux_info_presence_none; |
121 | | |
122 | | // --- GIMI content ID for the track |
123 | | |
124 | | std::string gimi_track_content_id; |
125 | | |
126 | | TrackOptions& operator=(const TrackOptions&); |
127 | | }; |
128 | | |
129 | | |
130 | | const char* get_track_auxiliary_info_type(heif_compression_format format); |
131 | | |
132 | | |
133 | | class Track : public ErrorBuffer { |
134 | | public: |
135 | | //Track(HeifContext* ctx); |
136 | | |
137 | | Track(HeifContext* ctx, uint32_t track_id, const TrackOptions* info, uint32_t handler_type); |
138 | | |
139 | | Track(HeifContext* ctx); |
140 | | |
141 | 0 | virtual ~Track() = default; |
142 | | |
143 | | // Allocate a Track of the correct sub-class (visual or metadata). |
144 | | // For tracks with an unsupported handler type, heif_error_Unsupported_feature/heif_suberror_Unsupported_track_type is returned. |
145 | | static Result<std::shared_ptr<Track>> alloc_track(HeifContext*, const std::shared_ptr<Box_trak>&); |
146 | | |
147 | | // load track from file |
148 | | virtual Error load(const std::shared_ptr<Box_trak>&); |
149 | | |
150 | | // This is called after creating all Track objects when reading a HEIF file. |
151 | | // We can now do initializations that require access to all tracks. |
152 | 0 | [[nodiscard]] virtual Error initialize_after_parsing(HeifContext*, const std::vector<std::shared_ptr<Track>>& all_tracks) { return {}; } |
153 | | |
154 | 0 | heif_item_id get_id() const { return m_id; } |
155 | | |
156 | | std::shared_ptr<HeifFile> get_file() const; |
157 | | |
158 | 0 | uint32_t get_handler() const { return m_handler_type; } |
159 | | |
160 | | heif_auxiliary_track_info_type get_auxiliary_info_type() const; |
161 | | |
162 | 0 | std::string get_auxiliary_info_type_urn() const { return m_auxiliary_info_type; } |
163 | | |
164 | | void set_auxiliary_info_type(heif_auxiliary_track_info_type); |
165 | | |
166 | 0 | void set_auxiliary_info_type_urn(std::string t) { m_auxiliary_info_type = std::move(t); } |
167 | | |
168 | | bool is_visual_track() const; |
169 | | |
170 | 0 | virtual bool has_alpha_channel() const { return false; } |
171 | | |
172 | | uint32_t get_first_cluster_sample_entry_type() const; |
173 | | |
174 | | Result<std::string> get_first_cluster_urim_uri() const; |
175 | | |
176 | | uint64_t get_duration_in_media_units() const; |
177 | | |
178 | | uint32_t get_timescale() const; |
179 | | |
180 | | // The context will compute the duration in global movie units and set this. |
181 | | void set_track_duration_in_movie_units(uint64_t total_duration, uint64_t segment_duration); |
182 | | |
183 | | void enable_edit_list_repeat_mode(bool enable); |
184 | | |
185 | 0 | std::shared_ptr<Box_taic> get_first_cluster_taic() { return m_first_taic; } |
186 | | |
187 | | bool end_of_sequence_reached() const; |
188 | | |
189 | | // See m_num_repetitions for the meaning of the return value. |
190 | 0 | uint32_t get_number_of_repetitions() const { return m_num_repetitions; } |
191 | | |
192 | | // Compute some parameters after all frames have been encoded (for example: track duration). |
193 | | virtual Error finalize_track(); |
194 | | |
195 | 0 | const TrackOptions& get_track_info() const { return m_track_info; } |
196 | | |
197 | | void add_reference_to_track(uint32_t referenceType, uint32_t to_track_id); |
198 | | |
199 | 0 | std::shared_ptr<const Box_tref> get_tref_box() const { return m_tref; } |
200 | | |
201 | | Result<heif_raw_sequence_sample*> get_next_sample_raw_data(const heif_decoding_options* options); |
202 | | |
203 | | std::vector<heif_sample_aux_info_type> get_sample_aux_info_types() const; |
204 | | |
205 | | protected: |
206 | | HeifContext* m_heif_context = nullptr; |
207 | | uint32_t m_id = 0; |
208 | | uint32_t m_handler_type = 0; |
209 | | |
210 | | TrackOptions m_track_info; |
211 | | |
212 | | uint32_t m_num_samples = 0; |
213 | | |
214 | | struct SampleTiming { |
215 | | uint32_t sampleIdx = 0; |
216 | | uint32_t sampleInChunkIdx = 0; |
217 | | uint32_t chunkIdx = 0; |
218 | | uint64_t presentation_time = 0; // TODO |
219 | | uint64_t media_composition_time = 0; // TODO |
220 | | uint64_t media_decoding_time = 0; |
221 | | uint32_t sample_duration_media_time = 0; |
222 | | uint32_t sample_duration_presentation_time = 0; // TODO |
223 | | }; |
224 | | std::vector<SampleTiming> m_presentation_timeline; |
225 | | // Accounts m_presentation_timeline against max_total_memory. A small track can |
226 | | // declare millions of samples, so this vector (~48 bytes/sample) must be tracked |
227 | | // to bound multi-track accumulation (GHSA-xw34-mjcp-jqh8, variants V2/V3). |
228 | | MemoryHandle m_presentation_timeline_memory; |
229 | | uint64_t m_num_output_samples = 0; // Can be larger than the vector. It then repeats the playback. |
230 | | |
231 | | // How many times the media timeline is repeated. |
232 | | // 0 = editlist is present but its pattern is not understood (caller should assume a single playback). |
233 | | // 1 = no editlist: media plays exactly once. |
234 | | // UINT32_MAX = infinite (mvhd duration is the indefinite-sentinel and the editlist is in repeat mode). |
235 | | // N = the media segment is played N times. |
236 | | uint32_t m_num_repetitions = 1; |
237 | | |
238 | | // Continuous counting through all repetitions. You have to take the modulo operation to get the |
239 | | // index into m_presentation_timeline SampleTiming table. |
240 | | // (At 30 fps, this 32 bit integer will overflow in >4 years. I think this is acceptable.) |
241 | | uint32_t m_next_sample_to_be_decoded = 0; |
242 | | |
243 | | // Total sequence output index. |
244 | | uint32_t m_next_sample_to_be_output = 0; |
245 | | bool m_decoder_is_flushed = false; |
246 | | |
247 | | Error init_sample_timing_table(); |
248 | | |
249 | | std::vector<std::shared_ptr<Chunk>> m_chunks; |
250 | | // Accounts the per-chunk Chunk::m_sample_ranges tables against max_total_memory. |
251 | | // Their combined size over all chunks is num_samples * sizeof(SampleFileRange), |
252 | | // so a single reservation here bounds them all (GHSA-xw34-mjcp-jqh8, V2/V3). |
253 | | MemoryHandle m_chunk_sample_ranges_memory; |
254 | | std::vector<uint8_t> m_chunk_data; |
255 | | |
256 | | std::shared_ptr<Box_moov> m_moov; |
257 | | std::shared_ptr<Box_trak> m_trak; |
258 | | std::shared_ptr<Box_tkhd> m_tkhd; |
259 | | std::shared_ptr<Box_minf> m_minf; |
260 | | std::shared_ptr<Box_mdhd> m_mdhd; |
261 | | std::shared_ptr<Box_hdlr> m_hdlr; |
262 | | std::shared_ptr<Box_stbl> m_stbl; |
263 | | std::shared_ptr<Box_stsd> m_stsd; |
264 | | std::shared_ptr<Box_stsc> m_stsc; |
265 | | std::shared_ptr<Box_stco> m_stco; |
266 | | std::shared_ptr<Box_stts> m_stts; |
267 | | std::shared_ptr<Box_ctts> m_ctts; // optional box, TODO: add only if needed |
268 | | std::shared_ptr<Box_stss> m_stss; |
269 | | std::shared_ptr<Box_stsz> m_stsz; |
270 | | std::shared_ptr<Box_elst> m_elst; |
271 | | |
272 | | std::shared_ptr<class Box_tref> m_tref; // optional |
273 | | |
274 | | std::string m_auxiliary_info_type; // only for auxiliary tracks |
275 | | |
276 | | // --- sample auxiliary information |
277 | | |
278 | | std::unique_ptr<SampleAuxInfoHelper> m_aux_helper_tai_timestamps; |
279 | | std::unique_ptr<SampleAuxInfoHelper> m_aux_helper_content_ids; |
280 | | |
281 | | std::unique_ptr<SampleAuxInfoReader> m_aux_reader_tai_timestamps; |
282 | | std::unique_ptr<SampleAuxInfoReader> m_aux_reader_content_ids; |
283 | | |
284 | | std::shared_ptr<class Box_taic> m_first_taic; // the TAIC of the first chunk |
285 | | |
286 | | |
287 | | // --- Helper functions for writing samples. |
288 | | |
289 | | // Call when we begin a new chunk of samples, e.g. because the compression format changed |
290 | | void add_chunk(heif_compression_format format); |
291 | | |
292 | | // Call to set the sample_description_box for the last added chunk. |
293 | | // Has to be called when we call add_chunk(). |
294 | | // It is not merged with add_chunk() because the sample_description_box may need information from the |
295 | | // first encoded frame. |
296 | | void set_sample_description_box(const std::shared_ptr<Box>& sample_description_box); |
297 | | |
298 | | // Write the actual sample data. `tai` may be null and `gimi_contentID` may be empty. |
299 | | // In these cases, no timestamp or no contentID will be written, respectively. |
300 | | Error write_sample_data(const std::vector<uint8_t>& raw_data, |
301 | | uint32_t sample_duration, |
302 | | int32_t composition_time_offset, |
303 | | bool is_sync_sample, |
304 | | const heif_tai_timestamp_packet* tai, |
305 | | const std::optional<std::string>& gimi_contentID); |
306 | | }; |
307 | | |
308 | | |
309 | | #endif //LIBHEIF_TRACK_H |