/src/vvdec/source/Lib/CommonLib/TrQuant.cpp
Line | Count | Source |
1 | | /* ----------------------------------------------------------------------------- |
2 | | The copyright in this software is being made available under the Clear BSD |
3 | | License, included below. No patent rights, trademark rights and/or |
4 | | other Intellectual Property Rights other than the copyrights concerning |
5 | | the Software are granted under this license. |
6 | | |
7 | | The Clear BSD License |
8 | | |
9 | | Copyright (c) 2018-2026, Fraunhofer-Gesellschaft zur Förderung der angewandten Forschung e.V. & The VVdeC Authors. |
10 | | All rights reserved. |
11 | | |
12 | | Redistribution and use in source and binary forms, with or without modification, |
13 | | are permitted (subject to the limitations in the disclaimer below) provided that |
14 | | the following conditions are met: |
15 | | |
16 | | * Redistributions of source code must retain the above copyright notice, |
17 | | this list of conditions and the following disclaimer. |
18 | | |
19 | | * Redistributions in binary form must reproduce the above copyright |
20 | | notice, this list of conditions and the following disclaimer in the |
21 | | documentation and/or other materials provided with the distribution. |
22 | | |
23 | | * Neither the name of the copyright holder nor the names of its |
24 | | contributors may be used to endorse or promote products derived from this |
25 | | software without specific prior written permission. |
26 | | |
27 | | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY |
28 | | THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND |
29 | | CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
30 | | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A |
31 | | PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR |
32 | | CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, |
33 | | EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
34 | | PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR |
35 | | BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER |
36 | | IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
37 | | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
38 | | POSSIBILITY OF SUCH DAMAGE. |
39 | | |
40 | | |
41 | | ------------------------------------------------------------------------------------------- */ |
42 | | |
43 | | /** \file TrQuant.cpp |
44 | | \brief transform and quantization class |
45 | | */ |
46 | | |
47 | | #include "TrQuant.h" |
48 | | #include "TrQuant_EMT.h" |
49 | | |
50 | | #include "UnitTools.h" |
51 | | #include "ContextModelling.h" |
52 | | #include "CodingStructure.h" |
53 | | #include "TimeProfiler.h" |
54 | | #include "Quant.h" |
55 | | #include "InterPrediction.h" |
56 | | |
57 | | |
58 | | #include "dtrace_buffer.h" |
59 | | |
60 | | #include <stdlib.h> |
61 | | #include <limits> |
62 | | #include <memory.h> |
63 | | |
64 | | #include "Quant.h" |
65 | | |
66 | | namespace vvdec |
67 | | { |
68 | | |
69 | | InvTrans *fastInvTrans[NUM_TRANS_TYPE][g_numTransformMatrixSizes] = |
70 | | { |
71 | | { fastInverseDCT2_B2, fastInverseDCT2_B4, fastInverseDCT2_B8, fastInverseDCT2_B16, fastInverseDCT2_B32, fastInverseDCT2_B64 }, |
72 | | { nullptr, fastInverseDCT8_B4, fastInverseDCT8_B8, fastInverseDCT8_B16, fastInverseDCT8_B32, nullptr }, |
73 | | { nullptr, fastInverseDST7_B4, fastInverseDST7_B8, fastInverseDST7_B16, fastInverseDST7_B32, nullptr }, |
74 | | }; |
75 | | |
76 | | //! \ingroup CommonLib |
77 | | //! \{ |
78 | | |
79 | | static void invLfnstNxNCore( int* src, int* dst, const uint32_t mode, const uint32_t index, const uint32_t size, int zeroOutSize ) |
80 | 0 | { |
81 | 0 | int maxLog2TrDynamicRange = 15; |
82 | 0 | const TCoeff outputMinimum = -( 1 << maxLog2TrDynamicRange ); |
83 | 0 | const TCoeff outputMaximum = ( 1 << maxLog2TrDynamicRange ) - 1; |
84 | 0 | const int8_t* trMat = ( size > 4 ) ? g_lfnst8x8[ mode ][ index ][ 0 ] : g_lfnst4x4[ mode ][ index ][ 0 ]; |
85 | 0 | const int trSize = ( size > 4 ) ? 48 : 16; |
86 | 0 | int resi; |
87 | 0 | int* out = dst; |
88 | |
|
89 | 0 | CHECK( index > 2, "wrong" ); |
90 | |
|
91 | 0 | for( int j = 0; j < trSize; j++, trMat += 16 ) |
92 | 0 | { |
93 | 0 | resi = 0; |
94 | 0 | const int8_t* trMatTmp = trMat; |
95 | 0 | int* srcPtr = src; |
96 | |
|
97 | 0 | for( int i = 0; i < zeroOutSize; i++ ) |
98 | 0 | { |
99 | 0 | resi += *srcPtr++ * *trMatTmp++; |
100 | 0 | } |
101 | |
|
102 | 0 | *out++ = Clip3( outputMinimum, outputMaximum, ( int ) ( resi + 64 ) >> 7 ); |
103 | 0 | } |
104 | 0 | } |
105 | | |
106 | 0 | static inline int64_t square( const int d ) { return d * (int64_t)d; } |
107 | | |
108 | | template<int signedMode> void invTransformCbCr( PelBuf &resCb, PelBuf &resCr ) |
109 | 0 | { |
110 | 0 | Pel* cb = resCb.buf; |
111 | 0 | Pel* cr = resCr.buf; |
112 | 0 | for( SizeType y = 0; y < resCb.height; y++, cb += resCb.stride, cr += resCr.stride ) |
113 | 0 | { |
114 | 0 | for( SizeType x = 0; x < resCb.width; x++ ) |
115 | 0 | { |
116 | 0 | if ( signedMode == 1 ) { cr[x] = cb[x] >> 1; } |
117 | 0 | else if ( signedMode == -1 ) { cr[x] = -cb[x] >> 1; } |
118 | 0 | else if ( signedMode == 2 ) { cr[x] = cb[x]; } |
119 | 0 | else if ( signedMode == -2 ) { cr[x] = -cb[x]; } |
120 | 0 | else if ( signedMode == 3 ) { cb[x] = cr[x] >> 1; } |
121 | 0 | else if ( signedMode == -3 ) { cb[x] = -cr[x] >> 1; } |
122 | 0 | } |
123 | 0 | } |
124 | 0 | } Unexecuted instantiation: void vvdec::invTransformCbCr<0>(vvdec::AreaBuf<short>&, vvdec::AreaBuf<short>&) Unexecuted instantiation: void vvdec::invTransformCbCr<1>(vvdec::AreaBuf<short>&, vvdec::AreaBuf<short>&) Unexecuted instantiation: void vvdec::invTransformCbCr<-1>(vvdec::AreaBuf<short>&, vvdec::AreaBuf<short>&) Unexecuted instantiation: void vvdec::invTransformCbCr<2>(vvdec::AreaBuf<short>&, vvdec::AreaBuf<short>&) Unexecuted instantiation: void vvdec::invTransformCbCr<-2>(vvdec::AreaBuf<short>&, vvdec::AreaBuf<short>&) Unexecuted instantiation: void vvdec::invTransformCbCr<3>(vvdec::AreaBuf<short>&, vvdec::AreaBuf<short>&) Unexecuted instantiation: void vvdec::invTransformCbCr<-3>(vvdec::AreaBuf<short>&, vvdec::AreaBuf<short>&) |
125 | | |
126 | | // ==================================================================================================================== |
127 | | // TrQuant class member functions |
128 | | // ==================================================================================================================== |
129 | | |
130 | 0 | TrQuant::TrQuant( class InterPrediction* ip, const TrQuant* other ) : Quant( other ) |
131 | 0 | { |
132 | | // allocate temporary buffers |
133 | 0 | m_invICT = m_invICTMem + maxAbsIctMode; |
134 | 0 | m_invICT[ 0] = invTransformCbCr< 0>; |
135 | 0 | m_invICT[ 1] = invTransformCbCr< 1>; |
136 | 0 | m_invICT[-1] = invTransformCbCr<-1>; |
137 | 0 | m_invICT[ 2] = invTransformCbCr< 2>; |
138 | 0 | m_invICT[-2] = invTransformCbCr<-2>; |
139 | 0 | m_invICT[ 3] = invTransformCbCr< 3>; |
140 | 0 | m_invICT[-3] = invTransformCbCr<-3>; |
141 | |
|
142 | 0 | m_invLfnstNxN = invLfnstNxNCore; |
143 | |
|
144 | 0 | static_assert( sizeof( ip->m_acYuvPred[0] ) > sizeof( TCoeff ) * ( MAX_TU_SIZE_FOR_PROFILE * MAX_TU_SIZE_FOR_PROFILE + MEMORY_ALIGN_DEF_SIZE ), "Buffer to small to be reused!" ); |
145 | 0 | static_assert( sizeof( ip->m_acYuvPred[1] ) > sizeof( TCoeff ) * ( MAX_TU_SIZE_FOR_PROFILE * MAX_TU_SIZE_FOR_PROFILE + MEMORY_ALIGN_DEF_SIZE ), "Buffer to small to be reused!" ); |
146 | 0 | static_assert( sizeof( ip->m_acYuvPred[2] ) > sizeof( TCoeff ) * ( MAX_TU_SIZE_FOR_PROFILE * MAX_TU_SIZE_FOR_PROFILE + MEMORY_ALIGN_DEF_SIZE ), "Buffer to small to be reused!" ); |
147 | |
|
148 | 0 | char* tmp = ( char* ) ip->m_acYuvPred[0]; |
149 | 0 | char* blk = ( char* ) ip->m_acYuvPred[1]; |
150 | 0 | char* dqnt = ( char* ) ip->m_acYuvPred[2]; |
151 | |
|
152 | 0 | m_tmp = ( TCoeff* ) ( ( ptrdiff_t ) tmp + ( MEMORY_ALIGN_DEF_SIZE - ( ( ptrdiff_t ) tmp & ( MEMORY_ALIGN_DEF_SIZE - 1 ) ) ) ); |
153 | 0 | m_blk = ( TCoeff* ) ( ( ptrdiff_t ) blk + ( MEMORY_ALIGN_DEF_SIZE - ( ( ptrdiff_t ) blk & ( MEMORY_ALIGN_DEF_SIZE - 1 ) ) ) ); |
154 | 0 | m_dqnt = ( TCoeff* ) ( ( ptrdiff_t ) dqnt + ( MEMORY_ALIGN_DEF_SIZE - ( ( ptrdiff_t ) dqnt & ( MEMORY_ALIGN_DEF_SIZE - 1 ) ) ) ); |
155 | |
|
156 | 0 | #if defined( TARGET_SIMD_X86 ) && ENABLE_SIMD_TCOEFF_OPS |
157 | 0 | initTrQuantX86(); |
158 | 0 | #endif |
159 | 0 | } |
160 | | |
161 | | void TrQuant::xDeQuant(const TransformUnit &tu, |
162 | | CoeffBuf &dstCoeff, |
163 | | const ComponentID &compID, |
164 | | const QpParam &cQP) |
165 | 0 | { |
166 | 0 | PROFILER_SCOPE_AND_STAGE_EXT( 1, g_timeProfiler, P_PARSERESIDUALS, *tu.cu->cs, compID ); |
167 | 0 | dequant( tu, dstCoeff, compID, cQP ); |
168 | 0 | } |
169 | | |
170 | | void TrQuant::init( const Picture *pic ) |
171 | 0 | { |
172 | 0 | Quant::init( pic ); |
173 | 0 | } |
174 | | |
175 | | uint32_t TrQuant::getLFNSTIntraMode( int wideAngPredMode ) |
176 | 0 | { |
177 | 0 | uint32_t intraMode; |
178 | |
|
179 | 0 | if( wideAngPredMode < 0 ) |
180 | 0 | { |
181 | 0 | intraMode = ( uint32_t ) ( wideAngPredMode + ( NUM_EXT_LUMA_MODE >> 1 ) + NUM_LUMA_MODE ); |
182 | 0 | } |
183 | 0 | else if( wideAngPredMode >= NUM_LUMA_MODE ) |
184 | 0 | { |
185 | 0 | intraMode = ( uint32_t ) ( wideAngPredMode + ( NUM_EXT_LUMA_MODE >> 1 ) ); |
186 | 0 | } |
187 | 0 | else |
188 | 0 | { |
189 | 0 | intraMode = ( uint32_t ) wideAngPredMode; |
190 | 0 | } |
191 | |
|
192 | 0 | return intraMode; |
193 | 0 | } |
194 | | |
195 | | bool TrQuant::getTransposeFlag( uint32_t intraMode ) |
196 | 0 | { |
197 | 0 | return ( ( intraMode >= NUM_LUMA_MODE ) && ( intraMode >= ( NUM_LUMA_MODE + ( NUM_EXT_LUMA_MODE >> 1 ) ) ) ) || |
198 | 0 | ( ( intraMode < NUM_LUMA_MODE ) && ( intraMode > DIA_IDX ) ); |
199 | 0 | } |
200 | | |
201 | | void TrQuant::xInvLfnst( TransformUnit &tu, const ComponentID& compID ) |
202 | 0 | { |
203 | 0 | const CompArea& area = tu.blocks[ compID ]; |
204 | 0 | const uint32_t width = area.width; |
205 | 0 | const uint32_t height = area.height; |
206 | 0 | const uint32_t lfnstIdx = tu.cu->lfnstIdx(); |
207 | |
|
208 | 0 | if( lfnstIdx && tu.mtsIdx( compID ) != MTS_SKIP && ( CU::isSepTree( *tu.cu ) ? true : isLuma( compID ) ) ) |
209 | 0 | { |
210 | 0 | const bool whge3 = width >= 8 && height >= 8; |
211 | 0 | const uint16_t* scan = whge3 ? g_coefTopLeftDiagScan8x8[ g_sizeIdxInfo.idxFrom( width ) ] : g_scanOrder[ SCAN_GROUPED_4x4 ][ g_sizeIdxInfo.idxFrom( width ) ][ g_sizeIdxInfo.idxFrom( height ) ]; |
212 | 0 | uint32_t intraMode = 0; |
213 | |
|
214 | 0 | if( CU::isMIP( *tu.cu, toChannelType( compID ) ) ) |
215 | 0 | { |
216 | 0 | intraMode = PLANAR_IDX; |
217 | 0 | } |
218 | 0 | else |
219 | 0 | { |
220 | 0 | intraMode = PU::isLMCMode( tu.cu->intraDir[toChannelType( compID )] ) ? PU::getCoLocatedIntraLumaMode( *tu.cu ) : PU::getFinalIntraMode( *tu.cu, toChannelType( compID ) ); |
221 | 0 | } |
222 | |
|
223 | 0 | CHECKD( intraMode > NUM_INTRA_MODE, "Invalid intra mode" ); |
224 | 0 | CHECKD( lfnstIdx >= 3, "Invalid LFNST index" ); |
225 | |
|
226 | 0 | intraMode = getLFNSTIntraMode( PU::getWideAngIntraMode( tu, intraMode, compID ) ); |
227 | 0 | bool transposeFlag = getTransposeFlag( intraMode ); |
228 | 0 | const int sbSize = whge3 ? 8 : 4; |
229 | 0 | bool tu4x4Flag = ( width == 4 && height == 4 ); |
230 | 0 | bool tu8x8Flag = ( width == 8 && height == 8 ); |
231 | 0 | TCoeff* lfnstTemp; |
232 | 0 | TCoeff* coeffTemp; |
233 | 0 | int y; |
234 | 0 | lfnstTemp = m_tempInMatrix; // inverse spectral rearrangement |
235 | 0 | coeffTemp = m_dqnt; |
236 | 0 | TCoeff * dst = lfnstTemp; |
237 | 0 | for( y = 0; y < 16; y++ ) |
238 | 0 | { |
239 | 0 | *dst++ = coeffTemp[ scan[y] ]; |
240 | 0 | } |
241 | |
|
242 | 0 | m_invLfnstNxN( m_tempInMatrix, m_tempOutMatrix, g_lfnstLut[ intraMode ], lfnstIdx - 1, sbSize, ( tu4x4Flag || tu8x8Flag ) ? 8 : 16 ); |
243 | |
|
244 | 0 | lfnstTemp = m_tempOutMatrix; // inverse spectral rearrangement |
245 | |
|
246 | 0 | if( transposeFlag ) |
247 | 0 | { |
248 | 0 | if( sbSize == 4 ) |
249 | 0 | { |
250 | 0 | for( y = 0; y < 4; y++ ) |
251 | 0 | { |
252 | 0 | coeffTemp[ 0 ] = lfnstTemp[ 0 ]; coeffTemp[ 1 ] = lfnstTemp[ 4 ]; |
253 | 0 | coeffTemp[ 2 ] = lfnstTemp[ 8 ]; coeffTemp[ 3 ] = lfnstTemp[ 12 ]; |
254 | 0 | lfnstTemp++; |
255 | 0 | coeffTemp += width; |
256 | 0 | } |
257 | 0 | } |
258 | 0 | else // ( sbSize == 8 ) |
259 | 0 | { |
260 | 0 | for( y = 0; y < 8; y++ ) |
261 | 0 | { |
262 | 0 | coeffTemp[ 0 ] = lfnstTemp[ 0 ]; coeffTemp[ 1 ] = lfnstTemp[ 8 ]; |
263 | 0 | coeffTemp[ 2 ] = lfnstTemp[ 16 ]; coeffTemp[ 3 ] = lfnstTemp[ 24 ]; |
264 | 0 | if( y < 4 ) |
265 | 0 | { |
266 | 0 | coeffTemp[ 4 ] = lfnstTemp[ 32 ]; coeffTemp[ 5 ] = lfnstTemp[ 36 ]; |
267 | 0 | coeffTemp[ 6 ] = lfnstTemp[ 40 ]; coeffTemp[ 7 ] = lfnstTemp[ 44 ]; |
268 | 0 | } |
269 | 0 | lfnstTemp++; |
270 | 0 | coeffTemp += width; |
271 | 0 | } |
272 | 0 | } |
273 | 0 | } |
274 | 0 | else |
275 | 0 | { |
276 | 0 | for( y = 0; y < sbSize; y++ ) |
277 | 0 | { |
278 | 0 | uint32_t uiStride = ( y < 4 ) ? sbSize : 4; |
279 | 0 | ::memcpy( coeffTemp, lfnstTemp, uiStride * sizeof( TCoeff ) ); |
280 | 0 | lfnstTemp += uiStride; |
281 | 0 | coeffTemp += width; |
282 | 0 | } |
283 | 0 | } |
284 | |
|
285 | 0 | tu.maxScanPosX[compID] = std::max<int>( tu.maxScanPosX[compID], std::min<int>( width - 1, 7 ) ); |
286 | 0 | tu.maxScanPosY[compID] = std::max<int>( tu.maxScanPosY[compID], std::min<int>( height - 1, 7 ) ); |
287 | 0 | } |
288 | 0 | } |
289 | | |
290 | | void TrQuant::invTransformNxN( TransformUnit &tu, const ComponentID &compID, PelBuf &pResi, const QpParam &cQP ) |
291 | 0 | { |
292 | 0 | CompArea &area = tu.blocks[compID]; |
293 | 0 | uint32_t uiWidth = area.width; |
294 | 0 | uint32_t uiHeight = area.height; |
295 | |
|
296 | 0 | CoeffBuf coeff( m_dqnt, uiWidth, uiHeight ); |
297 | 0 | coeff.memset( 0 ); |
298 | |
|
299 | 0 | xDeQuant( tu, coeff, compID, cQP ); |
300 | |
|
301 | 0 | DTRACE_COEFF_BUF( D_TCOEFF, coeff, tu, tu.cu->predMode(), compID ); |
302 | |
|
303 | 0 | if( tu.cu->sps->getUseLFNST() ) |
304 | 0 | { |
305 | 0 | xInvLfnst( tu, compID ); |
306 | 0 | } |
307 | |
|
308 | 0 | if( tu.mtsIdx( compID )== 1 ) |
309 | 0 | { |
310 | 0 | xITransformSkip( coeff, pResi, tu, compID ); |
311 | 0 | } |
312 | 0 | else |
313 | 0 | { |
314 | 0 | xIT( tu, compID, coeff, pResi ); |
315 | 0 | } |
316 | |
|
317 | 0 | DTRACE_PEL_BUF( D_RESIDUALS, pResi, tu, tu.cu->predMode(), compID); |
318 | 0 | } |
319 | | |
320 | | void TrQuant::invTransformICT( const TransformUnit &tu, PelBuf &resCb, PelBuf &resCr ) |
321 | 0 | { |
322 | 0 | CHECKD( Size(resCb) != Size(resCr), "resCb and resCr have different sizes" ); |
323 | 0 | ( *m_invICT[TU::getICTMode( tu, tu.cu->cs->picHeader->getJointCbCrSignFlag() )] )( resCb, resCr ); |
324 | 0 | } |
325 | | |
326 | | // ------------------------------------------------------------------------------------------------ |
327 | | // Logical transform |
328 | | // ------------------------------------------------------------------------------------------------ |
329 | | |
330 | | void TrQuant::getTrTypes( const TransformUnit& tu, const ComponentID compID, int &trTypeHor, int &trTypeVer ) |
331 | 0 | { |
332 | 0 | const bool isCuIntra = CU::isIntra( *tu.cu ); |
333 | 0 | const bool isCompLuma = isLuma( compID ); |
334 | 0 | const bool isImplicitMTS = isCuIntra && isCompLuma && tu.cu->sps->getUseImplicitMTS() && tu.cu->lfnstIdx() == 0 && tu.cu->mipFlag() == 0; |
335 | 0 | const bool isISP = isCuIntra && isCompLuma && tu.cu->ispMode(); |
336 | |
|
337 | 0 | if( isISP && tu.cu->lfnstIdx() ) |
338 | 0 | { |
339 | 0 | return; |
340 | 0 | } |
341 | | |
342 | 0 | const int lwidth = tu.lwidth(); |
343 | 0 | const int lheight = tu.lheight(); |
344 | |
|
345 | 0 | if( !tu.cu->sps->getUseMTS() ) |
346 | 0 | return; |
347 | | |
348 | 0 | if( isImplicitMTS || isISP ) |
349 | 0 | { |
350 | 0 | bool widthDstOk = lwidth >= 4 && lwidth <= 16; |
351 | 0 | bool heightDstOk = lheight >= 4 && lheight <= 16; |
352 | |
|
353 | 0 | if( widthDstOk ) |
354 | 0 | trTypeHor = DST7; |
355 | 0 | if( heightDstOk ) |
356 | 0 | trTypeVer = DST7; |
357 | |
|
358 | 0 | return; |
359 | 0 | } |
360 | | |
361 | 0 | const bool isCuInter = CU::isInter( *tu.cu ) && isCompLuma; |
362 | 0 | const bool isExplicitMTS = isCuIntra ? tu.cu->sps->getUseIntraMTS() && isCompLuma : tu.cu->sps->getUseInterMTS() && isCuInter; |
363 | 0 | const bool isSBT = isCuInter && tu.cu->sbtInfo(); |
364 | |
|
365 | 0 | if( isSBT ) |
366 | 0 | { |
367 | 0 | const uint8_t sbtIdx = CU::getSbtIdx( *tu.cu ); |
368 | 0 | const uint8_t sbtPos = CU::getSbtPos( *tu.cu ); |
369 | |
|
370 | 0 | if( sbtIdx == SBT_VER_HALF || sbtIdx == SBT_VER_QUAD ) |
371 | 0 | { |
372 | 0 | CHECK( lwidth > MTS_INTER_MAX_CU_SIZE, "wrong" ); |
373 | 0 | if( lheight > MTS_INTER_MAX_CU_SIZE ) |
374 | 0 | { |
375 | 0 | trTypeHor = trTypeVer = DCT2; |
376 | 0 | } |
377 | 0 | else |
378 | 0 | { |
379 | 0 | if( sbtPos == SBT_POS0 ) { trTypeHor = DCT8; trTypeVer = DST7; } |
380 | 0 | else { trTypeHor = DST7; trTypeVer = DST7; } |
381 | 0 | } |
382 | 0 | } |
383 | 0 | else |
384 | 0 | { |
385 | 0 | CHECK( lheight > MTS_INTER_MAX_CU_SIZE, "wrong" ); |
386 | 0 | if( lwidth > MTS_INTER_MAX_CU_SIZE ) |
387 | 0 | { |
388 | 0 | trTypeHor = trTypeVer = DCT2; |
389 | 0 | } |
390 | 0 | else |
391 | 0 | { |
392 | 0 | if( sbtPos == SBT_POS0 ) { trTypeHor = DST7; trTypeVer = DCT8; } |
393 | 0 | else { trTypeHor = DST7; trTypeVer = DST7; } |
394 | 0 | } |
395 | 0 | } |
396 | 0 | return; |
397 | 0 | } |
398 | 0 | else if( isExplicitMTS ) |
399 | 0 | { |
400 | 0 | if (tu.mtsIdx( compID ) > MTS_SKIP) |
401 | 0 | { |
402 | 0 | int indHor = (tu.mtsIdx( compID ) - MTS_DST7_DST7) & 1; |
403 | 0 | int indVer = (tu.mtsIdx( compID ) - MTS_DST7_DST7) >> 1; |
404 | 0 | trTypeHor = indHor ? DCT8 : DST7; |
405 | 0 | trTypeVer = indVer ? DCT8 : DST7; |
406 | 0 | } |
407 | 0 | } |
408 | 0 | } |
409 | | |
410 | | void TrQuant::xIT( const TransformUnit &tu, const ComponentID &compID, const CCoeffBuf &pCoeff, PelBuf &pResidual ) |
411 | 0 | { |
412 | 0 | const int width = pCoeff.width; |
413 | 0 | const int height = pCoeff.height; |
414 | 0 | const unsigned maxLog2TrDynamicRange = tu.cu->sps->getMaxLog2TrDynamicRange( toChannelType( compID ) ); |
415 | 0 | const unsigned bitDepth = tu.cu->sps->getBitDepth(); |
416 | 0 | const int TRANSFORM_MATRIX_SHIFT = g_transformMatrixShift; |
417 | 0 | const TCoeff clipMinimum = -( 1 << maxLog2TrDynamicRange ); |
418 | 0 | const TCoeff clipMaximum = ( 1 << maxLog2TrDynamicRange ) - 1; |
419 | 0 | const uint32_t transformWidthIndex = getLog2(width ) - 1; // nLog2WidthMinus1, since transform start from 2-point |
420 | 0 | const uint32_t transformHeightIndex = getLog2(height) - 1; // nLog2HeightMinus1, since transform start from 2-point |
421 | |
|
422 | 0 | int trTypeHor = DCT2; |
423 | 0 | int trTypeVer = DCT2; |
424 | |
|
425 | 0 | getTrTypes( tu, compID, trTypeHor, trTypeVer ); |
426 | |
|
427 | 0 | if( tu.maxScanPosX[compID] == 0 && tu.maxScanPosY[compID] == 0 && trTypeHor == DCT2 && trTypeVer == DCT2 ) |
428 | 0 | { |
429 | 0 | int dcVal = 0; |
430 | |
|
431 | 0 | if( width > 1 && height > 1 ) |
432 | 0 | { |
433 | 0 | const int shift_1st = TRANSFORM_MATRIX_SHIFT + 1 + COM16_C806_TRANS_PREC; |
434 | 0 | const int shift_2nd = ( TRANSFORM_MATRIX_SHIFT + maxLog2TrDynamicRange - 1 ) - bitDepth + COM16_C806_TRANS_PREC; |
435 | |
|
436 | 0 | dcVal = ( ( pCoeff.buf[0] * ( 1 << TRANSFORM_MATRIX_SHIFT ) ) + ( 1 << ( shift_1st - 1 ) ) ) >> shift_1st; |
437 | 0 | dcVal = ( ( dcVal * ( 1 << TRANSFORM_MATRIX_SHIFT ) ) + ( 1 << ( shift_2nd - 1 ) ) ) >> shift_2nd; |
438 | 0 | } |
439 | 0 | else |
440 | 0 | { |
441 | 0 | const int shift = ( TRANSFORM_MATRIX_SHIFT + maxLog2TrDynamicRange ) - bitDepth + COM16_C806_TRANS_PREC; |
442 | 0 | dcVal = ( ( pCoeff.buf[0] * ( 1 << TRANSFORM_MATRIX_SHIFT ) ) + ( 1 << ( shift - 1 ) ) ) >> shift; |
443 | 0 | } |
444 | |
|
445 | 0 | pResidual.fill( dcVal ); |
446 | 0 | return; |
447 | 0 | } |
448 | | |
449 | 0 | const int skipWidth = std::max<int>( ( trTypeHor != DCT2 && width == 32 ) ? 16 : width > JVET_C0024_ZERO_OUT_TH ? width - JVET_C0024_ZERO_OUT_TH : 0, width - tu.maxScanPosX[compID] - 1 ); |
450 | 0 | const int skipHeight = std::max<int>( ( trTypeVer != DCT2 && height == 32 ) ? 16 : height > JVET_C0024_ZERO_OUT_TH ? height - JVET_C0024_ZERO_OUT_TH : 0, height - tu.maxScanPosY[compID] - 1 ); |
451 | |
|
452 | 0 | TCoeff *block = m_blk; |
453 | 0 | int shiftlast; |
454 | |
|
455 | 0 | if( width > 1 && height > 1 ) //2-D transform |
456 | 0 | { |
457 | 0 | const int shift_1st = TRANSFORM_MATRIX_SHIFT + 1 + COM16_C806_TRANS_PREC; // 1 has been added to shift_1st at the expense of shift_2nd |
458 | 0 | const int shift_2nd = ( TRANSFORM_MATRIX_SHIFT + maxLog2TrDynamicRange - 1 ) - bitDepth + COM16_C806_TRANS_PREC; |
459 | 0 | CHECK( shift_1st < 0, "Negative shift" ); |
460 | 0 | CHECK( shift_2nd < 0, "Negative shift" ); |
461 | 0 | TCoeff *tmp = m_tmp; |
462 | 0 | fastInvTrans[trTypeVer][transformHeightIndex](pCoeff.buf, tmp, shift_1st, width, skipWidth, skipHeight, true, clipMinimum, clipMaximum); |
463 | 0 | fastInvTrans[trTypeHor][transformWidthIndex] (tmp, block, shift_2nd, height, 0, skipWidth, false, clipMinimum, clipMaximum); |
464 | 0 | shiftlast = shift_2nd; |
465 | 0 | } |
466 | 0 | else if( width == 1 ) //1-D vertical transform |
467 | 0 | { |
468 | 0 | int shift = ( TRANSFORM_MATRIX_SHIFT + maxLog2TrDynamicRange - 1 ) - bitDepth + COM16_C806_TRANS_PREC; |
469 | 0 | CHECK( shift < 0, "Negative shift" ); |
470 | 0 | CHECK( ( transformHeightIndex < 0 ), "There is a problem with the height." ); |
471 | 0 | fastInvTrans[trTypeVer][transformHeightIndex]( pCoeff.buf, block, shift + 1, 1, 0, skipHeight, false, clipMinimum, clipMaximum ); |
472 | 0 | shiftlast = shift + 1; |
473 | 0 | } |
474 | 0 | else //if(iHeight == 1) //1-D horizontal transform |
475 | 0 | { |
476 | 0 | const int shift = ( TRANSFORM_MATRIX_SHIFT + maxLog2TrDynamicRange - 1 ) - bitDepth + COM16_C806_TRANS_PREC; |
477 | 0 | CHECK( shift < 0, "Negative shift" ); |
478 | 0 | CHECK( ( transformWidthIndex < 0 ), "There is a problem with the width." ); |
479 | 0 | fastInvTrans[trTypeHor][transformWidthIndex]( pCoeff.buf, block, shift + 1, 1, 0, skipWidth, false, clipMinimum, clipMaximum ); |
480 | 0 | shiftlast = shift + 1; |
481 | 0 | } |
482 | | |
483 | 0 | int round = 1 << ( shiftlast - 1 ); |
484 | 0 | g_tCoeffOps.cpyResiClip[getLog2( width )]( block, pResidual.buf, pResidual.stride, width, height, clipMinimum, clipMaximum, round, shiftlast ); |
485 | 0 | } |
486 | | |
487 | | /** Wrapper function between HM interface and core NxN transform skipping |
488 | | */ |
489 | | void TrQuant::xITransformSkip(const CCoeffBuf &pCoeff, |
490 | | PelBuf &pResidual, |
491 | | const TransformUnit &tu, |
492 | | const ComponentID &compID) |
493 | 0 | { |
494 | 0 | const CompArea &area = tu.blocks[compID]; |
495 | 0 | const int width = area.width; |
496 | 0 | const int height = area.height; |
497 | | |
498 | 0 | for( int y = 0; y < height; y++ ) |
499 | 0 | { |
500 | 0 | for( int x = 0; x < width; x++ ) |
501 | 0 | { |
502 | 0 | pResidual.at( x, y ) = Pel( pCoeff.at( x, y ) ); |
503 | 0 | } |
504 | 0 | } |
505 | 0 | } |
506 | | |
507 | | } |