/work/vvdec/source/Lib/DecoderLib/DecLib.cpp
Line | Count | Source |
1 | | /* ----------------------------------------------------------------------------- |
2 | | The copyright in this software is being made available under the Clear BSD |
3 | | License, included below. No patent rights, trademark rights and/or |
4 | | other Intellectual Property Rights other than the copyrights concerning |
5 | | the Software are granted under this license. |
6 | | |
7 | | The Clear BSD License |
8 | | |
9 | | Copyright (c) 2018-2026, Fraunhofer-Gesellschaft zur Förderung der angewandten Forschung e.V. & The VVdeC Authors. |
10 | | All rights reserved. |
11 | | |
12 | | Redistribution and use in source and binary forms, with or without modification, |
13 | | are permitted (subject to the limitations in the disclaimer below) provided that |
14 | | the following conditions are met: |
15 | | |
16 | | * Redistributions of source code must retain the above copyright notice, |
17 | | this list of conditions and the following disclaimer. |
18 | | |
19 | | * Redistributions in binary form must reproduce the above copyright |
20 | | notice, this list of conditions and the following disclaimer in the |
21 | | documentation and/or other materials provided with the distribution. |
22 | | |
23 | | * Neither the name of the copyright holder nor the names of its |
24 | | contributors may be used to endorse or promote products derived from this |
25 | | software without specific prior written permission. |
26 | | |
27 | | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY |
28 | | THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND |
29 | | CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
30 | | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A |
31 | | PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR |
32 | | CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, |
33 | | EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
34 | | PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR |
35 | | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER |
36 | | IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
37 | | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
38 | | POSSIBILITY OF SUCH DAMAGE. |
39 | | |
40 | | |
41 | | ------------------------------------------------------------------------------------------- */ |
42 | | |
43 | | /** \file DecLib.cpp |
44 | | \brief decoder wrapper class |
45 | | */ |
46 | | |
47 | | #include "DecLib.h" |
48 | | |
49 | | #include "CommonLib/dtrace_next.h" |
50 | | #include "CommonLib/dtrace_buffer.h" |
51 | | #include "CommonLib/TimeProfiler.h" |
52 | | |
53 | | #include "CommonLib/arm/CommonDefARM.h" |
54 | | #include "CommonLib/x86/CommonDefX86.h" |
55 | | |
56 | | #include "NALread.h" |
57 | | |
58 | | |
59 | | namespace vvdec |
60 | | { |
61 | | |
62 | | #ifdef TRACE_ENABLE_ITT |
63 | | std::vector<__itt_domain*> itt_domain_decInst; |
64 | | __itt_domain* itt_domain_dec = __itt_domain_create( "Decode" ); |
65 | | __itt_domain* itt_domain_prs = __itt_domain_create( "Parse" ); |
66 | | __itt_domain* itt_domain_oth = __itt_domain_create( "Other" ); |
67 | | |
68 | | __itt_string_handle* itt_handle_alf = __itt_string_handle_create( "ALF_CTU" ); |
69 | | __itt_string_handle* itt_handle_presao = __itt_string_handle_create( "PreSAO_Line" ); |
70 | | __itt_string_handle* itt_handle_sao = __itt_string_handle_create( "SAO_CTU" ); |
71 | | __itt_string_handle* itt_handle_lfl = __itt_string_handle_create( "LFL_CTU" ); |
72 | | __itt_string_handle* itt_handle_intra = __itt_string_handle_create( "Intra_CTU" ); |
73 | | __itt_string_handle* itt_handle_inter = __itt_string_handle_create( "Inter_CTU" ); |
74 | | __itt_string_handle* itt_handle_mider = __itt_string_handle_create( "MI-Der_CTU" ); |
75 | | __itt_string_handle* itt_handle_lfcl = __itt_string_handle_create( "Prep_ClearLF" ); |
76 | | __itt_string_handle* itt_handle_ext = __itt_string_handle_create( "Prep_ExtBrdr" ); |
77 | | __itt_string_handle* itt_handle_dmvr = __itt_string_handle_create( "MI-DMVR" ); |
78 | | __itt_string_handle* itt_handle_rsp = __itt_string_handle_create( "Reshape_CTU" ); |
79 | | |
80 | | __itt_string_handle* itt_handle_parse = __itt_string_handle_create( "Parse_Slice" ); |
81 | | |
82 | | __itt_string_handle* itt_handle_start = __itt_string_handle_create( "Start_Pic" ); |
83 | | __itt_string_handle* itt_handle_done = __itt_string_handle_create( "Pic_Done" ); |
84 | | __itt_string_handle* itt_handle_finish = __itt_string_handle_create( "Finish_Pic" ); |
85 | | |
86 | | __itt_string_handle* itt_handle_schedTasks = __itt_string_handle_create( "Scheduling_Tasks" ); |
87 | | __itt_string_handle* itt_handle_waitTasks = __itt_string_handle_create( "Wait_for_Dec_Tasks" ); |
88 | | |
89 | | |
90 | | // create global domain for DecLib |
91 | | __itt_domain* itt_domain_glb = __itt_domain_create ( "Global" ); |
92 | | // create a global counter |
93 | | __itt_counter itt_frame_counter = __itt_counter_create( "FrameNumber", "Global" ); |
94 | | |
95 | | #define ITT_TASKSTART( d, t ) __itt_task_begin( ( d ), __itt_null, __itt_null, ( t ) ) |
96 | | #define ITT_TASKEND( d, t ) __itt_task_end ( ( d ) ) |
97 | | #else |
98 | | #define ITT_TASKSTART( d, t ) |
99 | | #define ITT_TASKEND( d, t ) |
100 | | #endif |
101 | | |
102 | | //! \ingroup DecoderLib |
103 | | //! \{ |
104 | | |
105 | | DecLib::DecLib() |
106 | 378 | { |
107 | | #ifdef TRACE_ENABLE_ITT |
108 | | itt_domain_dec->flags = 1; |
109 | | itt_domain_prs->flags = 1; |
110 | | itt_domain_glb->flags = 1; |
111 | | itt_domain_oth->flags = 1; |
112 | | #endif |
113 | 378 | } |
114 | | |
115 | | void DecLib::create( int numDecThreads, int parserFrameDelay, const UserAllocator& userAllocator, ErrHandlingFlags errHandlingFlags ) |
116 | 189 | { |
117 | | // run constructor again to ensure all variables, especially in DecLibParser have been reset |
118 | 189 | this->~DecLib(); |
119 | 189 | new( this ) DecLib; |
120 | | |
121 | 189 | if( numDecThreads < 0 ) |
122 | 189 | { |
123 | 189 | numDecThreads = std::thread::hardware_concurrency(); |
124 | 189 | } |
125 | | |
126 | 189 | m_decodeThreadPool.reset( new ThreadPool( numDecThreads, "DecThread" ) ); |
127 | | |
128 | 189 | if( parserFrameDelay < 0 ) |
129 | 189 | { |
130 | 189 | CHECK_FATAL( numDecThreads < 0, "invalid number of threads" ); |
131 | 189 | parserFrameDelay = std::min<int>( ( numDecThreads * DEFAULT_PARSE_DELAY_FACTOR ) >> 4, DEFAULT_PARSE_DELAY_MAX ); |
132 | 189 | } |
133 | 189 | m_parseFrameDelay = parserFrameDelay; |
134 | | |
135 | 189 | bool upscalingEnabled = false; |
136 | 189 | m_picListManager.create( m_parseFrameDelay, (int) m_decLibRecon.size(), userAllocator ); |
137 | 189 | m_decLibParser.create ( m_decodeThreadPool.get(), m_parseFrameDelay, (int) m_decLibRecon.size(), numDecThreads, errHandlingFlags ); |
138 | | |
139 | 189 | int id=0; |
140 | 189 | for( auto &dec: m_decLibRecon ) |
141 | 378 | { |
142 | 378 | dec.create( m_decodeThreadPool.get(), id++, upscalingEnabled ); |
143 | 378 | } |
144 | | |
145 | 189 | std::stringstream cssCap; |
146 | 189 | cssCap << "THREADS=" << numDecThreads << "; " |
147 | 189 | << "PARSE_DELAY=" << parserFrameDelay << "; "; |
148 | 189 | #if ENABLE_SIMD_OPT |
149 | | # if defined( TARGET_SIMD_ARM ) |
150 | | cssCap << "SIMD=" << read_arm_extension_name(); |
151 | | # elif defined( TARGET_SIMD_X86 ) |
152 | | cssCap << "SIMD=" << read_x86_extension_name(); |
153 | | # else |
154 | 189 | cssCap << "SIMD=SCALAR"; |
155 | 189 | # endif |
156 | | #else |
157 | | cssCap << "SIMD=NONE"; |
158 | | #endif |
159 | | |
160 | 189 | m_sDecoderCapabilities = cssCap.str(); |
161 | | |
162 | 189 | DTRACE_UPDATE( g_trace_ctx, std::make_pair( "final", 1 ) ); |
163 | 189 | } |
164 | | |
165 | | void DecLib::destroy() |
166 | 189 | { |
167 | 189 | if( m_decodeThreadPool ) |
168 | 189 | { |
169 | 189 | m_decodeThreadPool->shutdown( true ); |
170 | 189 | m_decodeThreadPool.reset(); |
171 | 189 | } |
172 | | |
173 | 189 | m_decLibParser.destroy(); |
174 | 189 | for( auto &dec: m_decLibRecon ) |
175 | 378 | { |
176 | 378 | dec.destroy(); |
177 | 378 | } |
178 | | |
179 | 189 | m_picListManager.deleteBuffers(); |
180 | 189 | } |
181 | | |
182 | | Picture* DecLib::decode( InputNALUnit& nalu ) |
183 | 168 | { |
184 | 168 | PROFILER_SCOPE_AND_STAGE( 1, g_timeProfiler, P_NALU_SLICE_PIC_HL ); |
185 | | |
186 | 168 | bool newPic = false; |
187 | 168 | if( m_iMaxTemporalLayer < 0 || nalu.m_temporalId <= m_iMaxTemporalLayer ) |
188 | 168 | { |
189 | 168 | newPic = m_decLibParser.parse( nalu ); |
190 | 168 | } |
191 | | |
192 | 168 | if( newPic ) |
193 | 0 | { |
194 | 0 | Picture* pcParsedPic = m_decLibParser.getNextDecodablePicture(); |
195 | |
|
196 | 0 | while( pcParsedPic && ( pcParsedPic->error || pcParsedPic->wasLost || pcParsedPic->parseDone.hasException() ) ) |
197 | 0 | { |
198 | 0 | CHECK_FATAL( pcParsedPic->progress >= Picture::reconstructing, "The error picture shouldn't be in reconstructing state yet." ); |
199 | |
|
200 | 0 | sanitizeBrokenPicture( pcParsedPic ); |
201 | | |
202 | | // try again to get a picture, that we can reconstruct now |
203 | 0 | pcParsedPic = m_decLibParser.getNextDecodablePicture(); |
204 | 0 | } |
205 | | |
206 | 0 | if( pcParsedPic ) |
207 | 0 | { |
208 | 0 | reconPicture( pcParsedPic ); |
209 | 0 | } |
210 | 0 | } |
211 | | |
212 | 168 | if( newPic || nalu.m_nalUnitType == NAL_UNIT_EOS ) |
213 | 1 | { |
214 | 1 | Picture* outPic = getNextOutputPic( false ); |
215 | 1 | if( outPic ) |
216 | 0 | { |
217 | 0 | CHECK_WARN( outPic->progress < Picture::finished, "Picture should have been finished by now. Blocking and finishing..." ); |
218 | 0 | if( outPic->progress < Picture::finished ) |
219 | 0 | { |
220 | 0 | blockAndFinishPictures( outPic ); |
221 | |
|
222 | 0 | CHECK( outPic->progress < Picture::finished, "Picture still not finished. Something is really broken." ); |
223 | 0 | } |
224 | | |
225 | 0 | m_checkMissingOutput = true; |
226 | 0 | } |
227 | | |
228 | | // warn if we don't produce an output picture for every incoming picture |
229 | 1 | CHECK_WARN( m_checkMissingOutput && !outPic, "missing output picture" ); // this CHECK_FATAL is not needed in flushPic(), because in flushPic() the nullptr signals the end of the bitstream |
230 | 1 | return outPic; |
231 | 1 | } |
232 | | |
233 | 167 | return nullptr; |
234 | 168 | } |
235 | | |
236 | | Picture* DecLib::flushPic() |
237 | 188 | { |
238 | 188 | Picture* outPic = nullptr; |
239 | 188 | try |
240 | 188 | { |
241 | 188 | m_decLibParser.failIncompleteParsePic(); // no more slices are coming for the last picture |
242 | | |
243 | | // at end of file, fill the decompression queue and decode pictures until the next output-picture is finished |
244 | 188 | while( Picture* pcParsedPic = m_decLibParser.getNextDecodablePicture() ) |
245 | 0 | { |
246 | 0 | if( pcParsedPic->error || pcParsedPic->wasLost || pcParsedPic->parseDone.hasException() ) |
247 | 0 | { |
248 | 0 | CHECK_FATAL( pcParsedPic->progress >= Picture::reconstructing, "The error picture shouldn't be in reconstructing state yet." ); |
249 | |
|
250 | 0 | sanitizeBrokenPicture( pcParsedPic ); |
251 | | |
252 | | // try again to get a picture, that we can reconstruct now |
253 | 0 | continue; |
254 | 0 | } |
255 | | |
256 | | // reconPicture() blocks and finishes one picture on each call |
257 | 0 | reconPicture( pcParsedPic ); |
258 | |
|
259 | 0 | if( !outPic ) |
260 | 0 | { |
261 | 0 | outPic = getNextOutputPic( false ); |
262 | 0 | } |
263 | 0 | if( outPic && outPic->progress == Picture::finished ) |
264 | 0 | { |
265 | 0 | return outPic; |
266 | 0 | } |
267 | 0 | } |
268 | | |
269 | | // no more pictures to finish decoding, but we might still have finished pictures in the recon queue |
270 | 188 | if( !outPic ) |
271 | 188 | { |
272 | 188 | outPic = getNextOutputPic( false ); |
273 | 188 | } |
274 | 188 | if( outPic && outPic->progress == Picture::finished ) |
275 | 0 | { |
276 | 0 | return outPic; |
277 | 0 | } |
278 | | |
279 | | // if all pictures have been parsed, but not finished, iteratively wait for and finish next pictures |
280 | 188 | blockAndFinishPictures( outPic ); |
281 | 188 | if( !outPic ) |
282 | 188 | { |
283 | 188 | outPic = getNextOutputPic( false ); |
284 | 188 | } |
285 | 188 | if( outPic && outPic->progress == Picture::finished ) |
286 | 0 | { |
287 | 0 | return outPic; |
288 | 0 | } |
289 | | |
290 | 188 | CHECK( outPic, "we shouldn't be holding an output picture here" ); |
291 | | // flush remaining pictures without considering num reorder pics |
292 | 188 | outPic = getNextOutputPic( true ); |
293 | 188 | if( outPic ) |
294 | 0 | { |
295 | 0 | CHECK( outPic->progress != Picture::finished, "all pictures should have been finished by now" ); |
296 | | // outPic->referenced = false; |
297 | 0 | return outPic; |
298 | 0 | } |
299 | 188 | } |
300 | 188 | catch( ... ) |
301 | 188 | { |
302 | 0 | m_picListManager.releasePicture( outPic ); |
303 | 0 | throw; |
304 | 0 | } |
305 | | |
306 | | // At the very end reset parser state |
307 | 188 | InputNALUnit eosNAL; |
308 | 188 | eosNAL.m_nalUnitType = NAL_UNIT_EOS; |
309 | 188 | m_decLibParser.parse( eosNAL ); |
310 | 188 | m_checkMissingOutput = false; |
311 | | |
312 | 188 | return nullptr; |
313 | 188 | } |
314 | | |
315 | | void DecLib::sanitizeBrokenPicture( Picture* pcParsedPic ) |
316 | 0 | { |
317 | 0 | std::exception_ptr parsing_exception = pcParsedPic->parseDone.hasException() ? pcParsedPic->parseDone.getException() : nullptr; |
318 | 0 | if( parsing_exception ) |
319 | 0 | { // the exception has not been thrown out of the library. Do that after preparing this picture for referencing |
320 | 0 | pcParsedPic->error = true; |
321 | 0 | pcParsedPic->waitForAllTasks(); |
322 | 0 | pcParsedPic->parseDone.clearException(); |
323 | 0 | } |
324 | |
|
325 | 0 | pcParsedPic->waitForAllTasks(); |
326 | |
|
327 | 0 | if( pcParsedPic->progress < Picture::parsing ) |
328 | 0 | { |
329 | | // we don't know if all structures are there yet, so we init them |
330 | 0 | pcParsedPic->ensureUsableAsRef(); |
331 | 0 | } |
332 | 0 | pcParsedPic->fillGrey( m_decLibParser.getParameterSetManager().getFirstSPS() ); |
333 | | |
334 | | // need to finish picture here, because it won't go through declibRecon |
335 | 0 | finishPicture( pcParsedPic ); |
336 | | |
337 | | // this exception has not been thrown outside (error must have happened in slice parsing task) |
338 | 0 | if( parsing_exception ) |
339 | 0 | { |
340 | 0 | CHECK_FATAL( pcParsedPic->exceptionThrownOut, "The exception shouldn't have been thrown out already." ); |
341 | 0 | pcParsedPic->exceptionThrownOut = true; |
342 | 0 | std::rethrow_exception( parsing_exception ); |
343 | 0 | } |
344 | 0 | } |
345 | | |
346 | | #if JVET_R0270 |
347 | | int DecLib::finishPicture( Picture* pcPic, MsgLevel msgl, bool associatedWithNewClvs ) |
348 | | #else |
349 | | int DecLib::finishPicture( Picture* pcPic, MsgLevel msgl ) |
350 | | #endif |
351 | 0 | { |
352 | | #ifdef TRACE_ENABLE_ITT |
353 | | // increment Framecounter |
354 | | __itt_counter_inc( itt_frame_counter ); |
355 | | #endif |
356 | |
|
357 | 0 | Slice* pcSlice = pcPic->slices[0]; |
358 | 0 | if( pcPic->wasLost || pcPic->error || pcPic->reconDone.hasException() || pcPic->parseDone.hasException() ) |
359 | 0 | { |
360 | 0 | msg( msgl, "POC %4d LId: %2d TId: %1d %s\n", pcPic->poc, pcPic->layerId, pcSlice->getTLayer(), pcPic->wasLost ? "LOST" : "ERROR" ); |
361 | 0 | pcPic->fillGrey( pcPic->cs->sps.get() ); |
362 | 0 | pcPic->progress = Picture::finished; |
363 | | |
364 | | // if the picture has an exception set (originating from thread-pool tasks), don't return here, but rethrow the exception |
365 | 0 | try |
366 | 0 | { |
367 | 0 | pcPic->parseDone.checkAndRethrowException(); |
368 | 0 | pcPic->reconDone.checkAndRethrowException(); |
369 | 0 | } |
370 | 0 | catch( ... ) |
371 | 0 | { |
372 | 0 | pcPic->waitForAllTasks(); |
373 | | |
374 | | // need to clear exception so we can use it as reference picture |
375 | 0 | pcPic->reconDone.clearException(); |
376 | 0 | pcPic->reconDone.unlock(); |
377 | 0 | pcPic->error = true; |
378 | 0 | if( !pcPic->exceptionThrownOut ) |
379 | 0 | { |
380 | 0 | pcPic->exceptionThrownOut = true; |
381 | 0 | throw; |
382 | 0 | } |
383 | 0 | } |
384 | | |
385 | 0 | return pcPic->poc; |
386 | 0 | } |
387 | | |
388 | 0 | ITT_TASKSTART( itt_domain_oth, itt_handle_finish ); |
389 | |
|
390 | 0 | char c = ( pcSlice->isIntra() ? 'I' : pcSlice->isInterP() ? 'P' : 'B' ); |
391 | 0 | if( !pcPic->isReferencePic ) |
392 | 0 | { |
393 | 0 | c += 32; // tolower |
394 | 0 | } |
395 | | |
396 | | //-- For time output for each slice |
397 | 0 | msg( msgl, "POC %4d LId: %2d TId: %1d ( %c-SLICE, QP%3d ) ", pcPic->poc, pcPic->layerId, |
398 | 0 | pcSlice->getTLayer(), |
399 | 0 | c, |
400 | 0 | pcSlice->getSliceQp() ); |
401 | 0 | msg( msgl, "[DT %6.3f] ", pcPic->getProcessingTime() ); |
402 | |
|
403 | 0 | for (int iRefList = 0; iRefList < 2; iRefList++) |
404 | 0 | { |
405 | 0 | msg( msgl, "[L%d ", iRefList); |
406 | 0 | for (int iRefIndex = 0; iRefIndex < pcSlice->getNumRefIdx(RefPicList(iRefList)); iRefIndex++) |
407 | 0 | { |
408 | 0 | #if RPR_OUTPUT_MSG |
409 | 0 | const std::pair<int, int>& scaleRatio = pcSlice->getScalingRatio( RefPicList( iRefList ), iRefIndex ); |
410 | | |
411 | 0 | if( pcSlice->getPicHeader()->getEnableTMVPFlag() && pcSlice->getColFromL0Flag() == bool(1 - iRefList) && pcSlice->getColRefIdx() == iRefIndex ) |
412 | 0 | { |
413 | 0 | if( scaleRatio.first != 1 << SCALE_RATIO_BITS || scaleRatio.second != 1 << SCALE_RATIO_BITS ) |
414 | 0 | { |
415 | 0 | msg( msgl, "%dc(%1.2lfx, %1.2lfx) ", pcSlice->getRefPOC( RefPicList( iRefList ), iRefIndex ), double( scaleRatio.first ) / ( 1 << SCALE_RATIO_BITS ), double( scaleRatio.second ) / ( 1 << SCALE_RATIO_BITS ) ); |
416 | 0 | } |
417 | 0 | else |
418 | 0 | { |
419 | 0 | msg( msgl, "%dc ", pcSlice->getRefPOC( RefPicList( iRefList ), iRefIndex ) ); |
420 | 0 | } |
421 | 0 | } |
422 | 0 | else |
423 | 0 | { |
424 | 0 | if( scaleRatio.first != 1 << SCALE_RATIO_BITS || scaleRatio.second != 1 << SCALE_RATIO_BITS ) |
425 | 0 | { |
426 | 0 | msg( msgl, "%d(%1.2lfx, %1.2lfx) ", pcSlice->getRefPOC( RefPicList( iRefList ), iRefIndex ), double( scaleRatio.first ) / ( 1 << SCALE_RATIO_BITS ), double( scaleRatio.second ) / ( 1 << SCALE_RATIO_BITS ) ); |
427 | 0 | } |
428 | 0 | else |
429 | 0 | { |
430 | 0 | msg( msgl, "%d ", pcSlice->getRefPOC(RefPicList(iRefList), iRefIndex)); |
431 | 0 | } |
432 | 0 | } |
433 | | #else |
434 | | msg( msgl, "%d ", pcSlice->getRefPOC(RefPicList(iRefList), iRefIndex)); |
435 | | #endif |
436 | 0 | } |
437 | 0 | msg( msgl, "] "); |
438 | 0 | } |
439 | |
|
440 | 0 | msg( msgl, "\n"); |
441 | | |
442 | | // pcPic->neededForOutput = (pcSlice->getPicHeader()->getPicOutputFlag() ? true : false); |
443 | | #if JVET_R0270 |
444 | | if (associatedWithNewClvs && pcPic->neededForOutput) |
445 | | { |
446 | | if (!pcSlice->getPPS()->getMixedNaluTypesInPicFlag() && pcSlice->getNalUnitType() == NAL_UNIT_CODED_SLICE_RASL) |
447 | | { |
448 | | pcPic->neededForOutput = false; |
449 | | } |
450 | | else if (pcSlice->getPPS()->getMixedNaluTypesInPicFlag()) |
451 | | { |
452 | | bool isRaslPic = true; |
453 | | for (int i = 0; isRaslPic && i < pcPic->numSlices; i++) |
454 | | { |
455 | | if (!(pcSlice->getNalUnitType() == NAL_UNIT_CODED_SLICE_RASL || pcSlice->getNalUnitType() == NAL_UNIT_CODED_SLICE_RADL)) |
456 | | { |
457 | | isRaslPic = false; |
458 | | } |
459 | | } |
460 | | if (isRaslPic) |
461 | | { |
462 | | pcPic->neededForOutput = false; |
463 | | } |
464 | | } |
465 | | } |
466 | | #endif |
467 | |
|
468 | 0 | m_picListManager.markUnusedPicturesReusable(); |
469 | |
|
470 | 0 | if( m_parseFrameDelay > 0 ) |
471 | 0 | { |
472 | 0 | checkPictureHashSEI( pcPic ); |
473 | 0 | } |
474 | |
|
475 | 0 | ITT_TASKEND( itt_domain_oth, itt_handle_finish ); |
476 | |
|
477 | 0 | pcPic->progress = Picture::finished; |
478 | |
|
479 | 0 | return pcSlice->getPOC(); |
480 | 0 | } |
481 | | |
482 | | void DecLib::checkPictureHashSEI( Picture* pcPic ) |
483 | 0 | { |
484 | 0 | if( !m_decodedPictureHashSEIEnabled ) |
485 | 0 | { |
486 | 0 | return; |
487 | 0 | } |
488 | 0 | if( !pcPic->neededForOutput || pcPic->picCheckedDPH ) |
489 | 0 | { |
490 | 0 | return; |
491 | 0 | } |
492 | | |
493 | 0 | CHECK( pcPic->progress < Picture::reconstructed, "picture not reconstructed" ); |
494 | |
|
495 | 0 | seiMessages pictureHashes = SEI_internal::getSeisByType( pcPic->seiMessageList, VVDEC_DECODED_PICTURE_HASH ); |
496 | 0 | if( !pictureHashes.empty() ) |
497 | 0 | { |
498 | 0 | if( pictureHashes.size() > 1 ) |
499 | 0 | { |
500 | 0 | msg( WARNING, "Warning: Got multiple decoded picture hash SEI messages. Using first." ); |
501 | 0 | } |
502 | |
|
503 | 0 | const vvdecSEIDecodedPictureHash* hash = (vvdecSEIDecodedPictureHash*)pictureHashes.front()->payload; |
504 | |
|
505 | 0 | msg( INFO, " " ); |
506 | 0 | const int hashErrors = calcAndPrintHashStatus( pcPic->getRecoBuf(), hash, pcPic->cs->sps->getBitDepths(), INFO ); |
507 | 0 | m_numberOfChecksumErrorsDetected += hashErrors; |
508 | 0 | pcPic->dphMismatch = !!hashErrors; |
509 | 0 | pcPic->picCheckedDPH = true; |
510 | 0 | msg( INFO, "\n" ); |
511 | 0 | } |
512 | 0 | else |
513 | 0 | { |
514 | 0 | if( pcPic->subPictures.empty() ) |
515 | 0 | { |
516 | 0 | msg( WARNING, "Warning: missing decoded picture hash SEI message.\n" ); |
517 | 0 | return; |
518 | 0 | } |
519 | | |
520 | 0 | seiMessages scalableNestingSeis = SEI_internal::getSeisByType( pcPic->seiMessageList, VVDEC_SCALABLE_NESTING ); |
521 | 0 | for( auto* seiIt: scalableNestingSeis ) |
522 | 0 | { |
523 | 0 | CHECK( seiIt->payloadType != VVDEC_SCALABLE_NESTING, "expected nesting SEI" ); |
524 | |
|
525 | 0 | const vvdecSEIScalableNesting* nestingSei = (vvdecSEIScalableNesting*) seiIt->payload; |
526 | 0 | if( !nestingSei->snSubpicFlag ) |
527 | 0 | { |
528 | 0 | continue; |
529 | 0 | } |
530 | | |
531 | 0 | for( int i = 0; i < nestingSei->snNumSEIs; ++i ) |
532 | 0 | { |
533 | 0 | auto& nestedSei = nestingSei->nestedSEIs[i]; |
534 | 0 | CHECK( nestedSei == nullptr, "missing nested sei" ); |
535 | 0 | if( nestedSei && nestedSei->payloadType != VVDEC_DECODED_PICTURE_HASH ) |
536 | 0 | continue; |
537 | | |
538 | 0 | const vvdecSEIDecodedPictureHash* hash = (vvdecSEIDecodedPictureHash*) nestedSei->payload; |
539 | |
|
540 | 0 | if( pcPic->subpicsCheckedDPH.empty() ) |
541 | 0 | { |
542 | 0 | pcPic->subpicsCheckedDPH.resize( pcPic->subPictures.size(), false ); |
543 | 0 | } |
544 | 0 | else |
545 | 0 | { |
546 | 0 | CHECK( pcPic->subpicsCheckedDPH.size() != pcPic->subPictures.size(), "Picture::subpicsCheckedDPH not properly initialized" ); |
547 | 0 | } |
548 | | |
549 | 0 | for( int j = 0; j < nestingSei->snNumSubpics; ++j ) |
550 | 0 | { |
551 | 0 | uint32_t subpicId = nestingSei->snSubpicId[j]; |
552 | |
|
553 | 0 | for( auto& subPic: pcPic->subPictures ) |
554 | 0 | { |
555 | 0 | if( subPic.getSubPicID() != subpicId ) |
556 | 0 | continue; |
557 | | |
558 | 0 | auto subPicIdx = subPic.getSubPicIdx(); |
559 | 0 | if( pcPic->subpicsCheckedDPH[subPicIdx] ) |
560 | 0 | continue; |
561 | | |
562 | 0 | const UnitArea area = UnitArea( pcPic->chromaFormat, subPic.getLumaArea() ); |
563 | 0 | const int hashErrors = calcAndPrintHashStatus( pcPic->cs->getRecoBuf( area ), hash, pcPic->cs->sps->getBitDepths(), INFO ); |
564 | 0 | m_numberOfChecksumErrorsDetected += hashErrors; |
565 | 0 | pcPic->dphMismatch |= !!hashErrors; |
566 | 0 | pcPic->subpicsCheckedDPH[subPicIdx] = true; |
567 | 0 | msg( INFO, "\n" ); |
568 | 0 | } |
569 | 0 | } |
570 | 0 | } |
571 | 0 | } |
572 | | |
573 | 0 | size_t checkedSubpicCount = std::count( pcPic->subpicsCheckedDPH.cbegin(), pcPic->subpicsCheckedDPH.cend(), true ); |
574 | 0 | pcPic->picCheckedDPH = ( checkedSubpicCount == pcPic->subPictures.size() ); // mark when all subpics have been checked |
575 | |
|
576 | 0 | if( m_parseFrameDelay ) |
577 | 0 | { |
578 | | // this warning is only enabled, when running with parse delay enabled, because otherwise we don't know here if the last DPH Suffix-SEI has already been |
579 | | // parsed |
580 | 0 | if( checkedSubpicCount != pcPic->subPictures.size() ) |
581 | 0 | { |
582 | 0 | msg( WARNING, "Warning: missing decoded picture hash SEI message for SubPics (%u/%u).\n", checkedSubpicCount, pcPic->subPictures.size() ); |
583 | 0 | } |
584 | 0 | } |
585 | 0 | } |
586 | 0 | } |
587 | | |
588 | | Picture* DecLib::getNextOutputPic( bool bFlush ) |
589 | 565 | { |
590 | 565 | if( m_picListManager.getFrontPic() == nullptr ) |
591 | 565 | { |
592 | 565 | return nullptr; |
593 | 565 | } |
594 | | |
595 | 0 | const SPS* activeSPS = m_picListManager.getFrontPic()->cs->sps.get(); |
596 | 0 | const int maxNrSublayers = activeSPS->getMaxTLayers(); |
597 | |
|
598 | 0 | int numReorderPicsHighestTid; |
599 | 0 | int maxDecPicBufferingHighestTid; |
600 | 0 | if( m_iMaxTemporalLayer == -1 || m_iMaxTemporalLayer >= maxNrSublayers ) |
601 | 0 | { |
602 | 0 | numReorderPicsHighestTid = activeSPS->getNumReorderPics( maxNrSublayers - 1 ); |
603 | 0 | maxDecPicBufferingHighestTid = activeSPS->getMaxDecPicBuffering( maxNrSublayers - 1 ); |
604 | 0 | } |
605 | 0 | else |
606 | 0 | { |
607 | 0 | numReorderPicsHighestTid = activeSPS->getNumReorderPics( m_iMaxTemporalLayer ); |
608 | 0 | maxDecPicBufferingHighestTid = activeSPS->getMaxDecPicBuffering( m_iMaxTemporalLayer ); |
609 | 0 | } |
610 | |
|
611 | 0 | return m_picListManager.getNextOutputPic( numReorderPicsHighestTid, maxDecPicBufferingHighestTid, bFlush ); |
612 | 565 | } |
613 | | |
614 | | void DecLib::reconPicture( Picture* pcPic ) |
615 | 0 | { |
616 | 0 | CHECK_FATAL( std::any_of( m_decLibRecon.begin(), m_decLibRecon.end(), [=]( auto& rec ) { return rec.getCurrPic() == pcPic; } ), |
617 | 0 | "(Reused) Picture structure is still in progress in decLibRecon." ); |
618 | |
|
619 | 0 | DecLibRecon* reconInstance = &m_decLibRecon.front(); |
620 | 0 | move_to_end( m_decLibRecon.begin(), m_decLibRecon ); |
621 | |
|
622 | 0 | Picture* donePic = reconInstance->waitForPrevDecompressedPic(); |
623 | 0 | try |
624 | 0 | { |
625 | 0 | reconInstance->decompressPicture( pcPic ); |
626 | 0 | } |
627 | 0 | catch( ... ) |
628 | 0 | { |
629 | 0 | pcPic->reconDone.setException( std::current_exception() ); |
630 | 0 | pcPic->error = true; |
631 | 0 | } |
632 | |
|
633 | 0 | if( donePic ) |
634 | 0 | { |
635 | 0 | finishPicture( donePic ); |
636 | 0 | } |
637 | 0 | } |
638 | | |
639 | | void DecLib::blockAndFinishPictures( Picture* pcPic ) |
640 | 188 | { |
641 | | // find Recon instance where current picture (if not null) is active and ensure the picture gets finished |
642 | | // otherwise all pictures get finished |
643 | 188 | for( auto& recon: m_decLibRecon ) |
644 | 376 | { |
645 | 376 | if( pcPic && recon.getCurrPic() != pcPic ) |
646 | 0 | { |
647 | 0 | continue; |
648 | 0 | } |
649 | | |
650 | 376 | if( Picture* donePic = recon.waitForPrevDecompressedPic() ) |
651 | 0 | { |
652 | 0 | finishPicture( donePic ); |
653 | 0 | } |
654 | 376 | } |
655 | 188 | } |
656 | | |
657 | | void DecLib::checkNalUnitConstraints( uint32_t naluType ) |
658 | 0 | { |
659 | 0 | const ConstraintInfo *cInfo = NULL; |
660 | 0 | if (m_decLibParser.getParameterSetManager().getActiveSPS() != NULL && m_decLibParser.getParameterSetManager().getActiveSPS()->getProfileTierLevel() != NULL) |
661 | 0 | { |
662 | 0 | cInfo = m_decLibParser.getParameterSetManager().getActiveSPS()->getProfileTierLevel()->getConstraintInfo(); |
663 | 0 | if( cInfo != NULL ) |
664 | 0 | { |
665 | 0 | xCheckNalUnitConstraintFlags( cInfo, naluType ); |
666 | 0 | } |
667 | 0 | } |
668 | 0 | } |
669 | | |
670 | | void DecLib::xCheckNalUnitConstraintFlags( const ConstraintInfo *cInfo, uint32_t naluType ) |
671 | 0 | { |
672 | 0 | if( cInfo != NULL ) |
673 | 0 | { |
674 | 0 | CHECK( cInfo->getNoTrailConstraintFlag() && naluType == NAL_UNIT_CODED_SLICE_TRAIL, |
675 | 0 | "Non-conforming bitstream. no_trail_constraint_flag is equal to 1 but bitstream contains NAL unit of type TRAIL_NUT." ); |
676 | 0 | CHECK( cInfo->getNoStsaConstraintFlag() && naluType == NAL_UNIT_CODED_SLICE_STSA, |
677 | 0 | "Non-conforming bitstream. no_stsa_constraint_flag is equal to 1 but bitstream contains NAL unit of type STSA_NUT." ); |
678 | 0 | CHECK( cInfo->getNoRaslConstraintFlag() && naluType == NAL_UNIT_CODED_SLICE_RASL, |
679 | 0 | "Non-conforming bitstream. no_rasl_constraint_flag is equal to 1 but bitstream contains NAL unit of type RASL_NUT." ); |
680 | 0 | CHECK( cInfo->getNoRadlConstraintFlag() && naluType == NAL_UNIT_CODED_SLICE_RADL, |
681 | 0 | "Non-conforming bitstream. no_radl_constraint_flag is equal to 1 but bitstream contains NAL unit of type RADL_NUT." ); |
682 | 0 | CHECK( cInfo->getNoIdrConstraintFlag() && naluType == NAL_UNIT_CODED_SLICE_IDR_W_RADL, |
683 | 0 | "Non-conforming bitstream. no_idr_constraint_flag is equal to 1 but bitstream contains NAL unit of type IDR_W_RADL." ); |
684 | 0 | CHECK( cInfo->getNoIdrConstraintFlag() && naluType == NAL_UNIT_CODED_SLICE_IDR_N_LP, |
685 | 0 | "Non-conforming bitstream. no_idr_constraint_flag is equal to 1 but bitstream contains NAL unit of type IDR_N_LP." ); |
686 | 0 | CHECK( cInfo->getNoCraConstraintFlag() && naluType == NAL_UNIT_CODED_SLICE_CRA, |
687 | 0 | "Non-conforming bitstream. no_cra_constraint_flag is equal to 1 but bitstream contains NAL unit of type CRA_NUT." ); |
688 | 0 | CHECK( cInfo->getNoGdrConstraintFlag() && naluType == NAL_UNIT_CODED_SLICE_GDR, |
689 | 0 | "Non-conforming bitstream. no_gdr_constraint_flag is equal to 1 but bitstream contains NAL unit of type GDR_NUT." ); |
690 | 0 | CHECK( cInfo->getNoApsConstraintFlag() && naluType == NAL_UNIT_PREFIX_APS, |
691 | 0 | "Non-conforming bitstream. no_aps_constraint_flag is equal to 1 but bitstream contains NAL unit of type APS_PREFIX_NUT." ); |
692 | 0 | CHECK( cInfo->getNoApsConstraintFlag() && naluType == NAL_UNIT_SUFFIX_APS, |
693 | 0 | "Non-conforming bitstream. no_aps_constraint_flag is equal to 1 but bitstream contains NAL unit of type APS_SUFFIX_NUT." ); |
694 | 0 | } |
695 | 0 | } |
696 | | |
697 | 0 | #define SEI_REPETITION_CONSTRAINT_LIST_SIZE 21 |
698 | | |
699 | | /** |
700 | | - Count the number of identical SEI messages in the current picture |
701 | | */ |
702 | | void DecLib::checkSeiInPictureUnit() |
703 | 0 | { |
704 | 0 | std::vector<std::tuple<int, uint32_t, uint8_t*>> seiList; |
705 | | |
706 | | // payload types subject to constrained SEI repetition |
707 | 0 | int picUnitRepConSeiList[SEI_REPETITION_CONSTRAINT_LIST_SIZE] = { 0, 1, 19, 45, 129, 132, 133, 137, 144, 145, 147, 148, 149, 150, 153, 154, 155, 156, 168, 203, 204}; |
708 | | |
709 | | // extract SEI messages from NAL units |
710 | 0 | for( auto &sei : m_pictureSeiNalus ) |
711 | 0 | { |
712 | 0 | InputBitstream bs = sei.getBitstream(); |
713 | |
|
714 | 0 | do |
715 | 0 | { |
716 | 0 | int payloadType = 0; |
717 | 0 | uint32_t val = 0; |
718 | |
|
719 | 0 | do |
720 | 0 | { |
721 | 0 | val = bs.readByte(); |
722 | 0 | payloadType += val; |
723 | 0 | } while (val==0xFF); |
724 | |
|
725 | 0 | uint32_t payloadSize = 0; |
726 | 0 | do |
727 | 0 | { |
728 | 0 | val = bs.readByte(); |
729 | 0 | payloadSize += val; |
730 | 0 | } while (val==0xFF); |
731 | |
|
732 | 0 | uint8_t *payload = new uint8_t[payloadSize]; |
733 | 0 | for( uint32_t i = 0; i < payloadSize; i++ ) |
734 | 0 | { |
735 | 0 | val = bs.readByte(); |
736 | 0 | payload[i] = (uint8_t)val; |
737 | 0 | } |
738 | 0 | seiList.push_back(std::tuple<int, uint32_t, uint8_t*>(payloadType, payloadSize, payload)); |
739 | 0 | } |
740 | 0 | while( bs.getNumBitsLeft() > 8 ); |
741 | 0 | } |
742 | | |
743 | | // count repeated messages in list |
744 | 0 | for( uint32_t i = 0; i < seiList.size(); i++ ) |
745 | 0 | { |
746 | 0 | int k, count = 1; |
747 | 0 | int payloadType1 = std::get<0>(seiList[i]); |
748 | 0 | uint32_t payloadSize1 = std::get<1>(seiList[i]); |
749 | 0 | uint8_t *payload1 = std::get<2>(seiList[i]); |
750 | | |
751 | | // only consider SEI payload types in the PicUnitRepConSeiList |
752 | 0 | for( k=0; k<SEI_REPETITION_CONSTRAINT_LIST_SIZE; k++ ) |
753 | 0 | { |
754 | 0 | if( payloadType1 == picUnitRepConSeiList[k] ) |
755 | 0 | { |
756 | 0 | break; |
757 | 0 | } |
758 | 0 | } |
759 | 0 | if( k >= SEI_REPETITION_CONSTRAINT_LIST_SIZE ) |
760 | 0 | { |
761 | 0 | continue; |
762 | 0 | } |
763 | | |
764 | | // compare current SEI message with remaining messages in the list |
765 | 0 | for( uint32_t j = i+1; j < seiList.size(); j++ ) |
766 | 0 | { |
767 | 0 | int payloadType2 = std::get<0>(seiList[j]); |
768 | 0 | uint32_t payloadSize2 = std::get<1>(seiList[j]); |
769 | 0 | uint8_t *payload2 = std::get<2>(seiList[j]); |
770 | | |
771 | | // check for identical SEI type, size, and payload |
772 | 0 | if( payloadType1 == payloadType2 && payloadSize1 == payloadSize2 ) |
773 | 0 | { |
774 | 0 | if( memcmp(payload1, payload2, payloadSize1*sizeof(uint8_t)) == 0 ) |
775 | 0 | { |
776 | 0 | count++; |
777 | 0 | } |
778 | 0 | } |
779 | 0 | } |
780 | 0 | CHECK(count > 4, "There shall be less than or equal to 4 identical sei_payload( ) syntax structures within a picture unit."); |
781 | 0 | } |
782 | | |
783 | | // free SEI message list memory |
784 | 0 | for( uint32_t i = 0; i < seiList.size(); i++ ) |
785 | 0 | { |
786 | 0 | uint8_t *payload = std::get<2>(seiList[i]); |
787 | 0 | delete payload; |
788 | 0 | } |
789 | 0 | seiList.clear(); |
790 | 0 | } |
791 | | |
792 | | /** |
793 | | - Reset list of SEI NAL units from the current picture |
794 | | */ |
795 | | void DecLib::resetPictureSeiNalus() |
796 | 0 | { |
797 | 0 | m_pictureSeiNalus.clear(); |
798 | 0 | } |
799 | | |
800 | | void DecLib::checkAPSInPictureUnit() |
801 | 0 | { |
802 | 0 | bool firstVCLFound = false; |
803 | 0 | bool suffixAPSFound = false; |
804 | 0 | for (auto &nalu : m_pictureUnitNals) |
805 | 0 | { |
806 | 0 | if (NALUnit::isVclNalUnitType(nalu)) |
807 | 0 | { |
808 | 0 | firstVCLFound = true; |
809 | 0 | CHECK( suffixAPSFound, "When any suffix APS NAL units are present in a PU, they shall follow the last VCL unit of the PU" ); |
810 | 0 | } |
811 | 0 | else if (nalu == NAL_UNIT_PREFIX_APS) |
812 | 0 | { |
813 | 0 | CHECK( firstVCLFound, "When any prefix APS NAL units are present in a PU, they shall precede the first VCL unit of the PU"); |
814 | 0 | } |
815 | 0 | else if (nalu == NAL_UNIT_SUFFIX_APS) |
816 | 0 | { |
817 | 0 | suffixAPSFound = true; |
818 | 0 | } |
819 | 0 | } |
820 | 0 | } |
821 | | |
822 | | } |