Coverage Report

Created: 2026-06-16 07:20

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/vvdec/source/Lib/CommonLib/TrQuant.cpp
Line
Count
Source
1
/* -----------------------------------------------------------------------------
2
The copyright in this software is being made available under the Clear BSD
3
License, included below. No patent rights, trademark rights and/or 
4
other Intellectual Property Rights other than the copyrights concerning 
5
the Software are granted under this license.
6
7
The Clear BSD License
8
9
Copyright (c) 2018-2026, Fraunhofer-Gesellschaft zur Förderung der angewandten Forschung e.V. & The VVdeC Authors.
10
All rights reserved.
11
12
Redistribution and use in source and binary forms, with or without modification,
13
are permitted (subject to the limitations in the disclaimer below) provided that
14
the following conditions are met:
15
16
     * Redistributions of source code must retain the above copyright notice,
17
     this list of conditions and the following disclaimer.
18
19
     * Redistributions in binary form must reproduce the above copyright
20
     notice, this list of conditions and the following disclaimer in the
21
     documentation and/or other materials provided with the distribution.
22
23
     * Neither the name of the copyright holder nor the names of its
24
     contributors may be used to endorse or promote products derived from this
25
     software without specific prior written permission.
26
27
NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY
28
THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
29
CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
30
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
31
PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
32
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
33
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
34
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
35
BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
36
IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
37
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
38
POSSIBILITY OF SUCH DAMAGE.
39
40
41
------------------------------------------------------------------------------------------- */
42
43
/** \file     TrQuant.cpp
44
    \brief    transform and quantization class
45
*/
46
47
#include "TrQuant.h"
48
#include "TrQuant_EMT.h"
49
50
#include "UnitTools.h"
51
#include "ContextModelling.h"
52
#include "CodingStructure.h"
53
#include "TimeProfiler.h"
54
#include "Quant.h"
55
#include "InterPrediction.h"
56
57
58
#include "dtrace_buffer.h"
59
60
#include <stdlib.h>
61
#include <limits>
62
#include <memory.h>
63
64
#include "Quant.h"
65
66
namespace vvdec
67
{
68
69
InvTrans *fastInvTrans[NUM_TRANS_TYPE][g_numTransformMatrixSizes] =
70
{
71
  { fastInverseDCT2_B2, fastInverseDCT2_B4, fastInverseDCT2_B8, fastInverseDCT2_B16, fastInverseDCT2_B32, fastInverseDCT2_B64 },
72
  { nullptr,            fastInverseDCT8_B4, fastInverseDCT8_B8, fastInverseDCT8_B16, fastInverseDCT8_B32, nullptr },
73
  { nullptr,            fastInverseDST7_B4, fastInverseDST7_B8, fastInverseDST7_B16, fastInverseDST7_B32, nullptr },
74
};
75
76
//! \ingroup CommonLib
77
//! \{
78
79
static void invLfnstNxNCore( int* src, int* dst, const uint32_t mode, const uint32_t index, const uint32_t size, int zeroOutSize )
80
0
{
81
0
  int             maxLog2TrDynamicRange =  15;
82
0
  const TCoeff    outputMinimum         = -( 1 << maxLog2TrDynamicRange );
83
0
  const TCoeff    outputMaximum         =  ( 1 << maxLog2TrDynamicRange ) - 1;
84
0
  const int8_t*   trMat                 =  ( size > 4 ) ? g_lfnst8x8[ mode ][ index ][ 0 ] : g_lfnst4x4[ mode ][ index ][ 0 ];
85
0
  const int       trSize                =  ( size > 4 ) ? 48 : 16;
86
0
  int             resi;
87
0
  int*            out                   =  dst;
88
89
0
  CHECK( index > 2, "wrong" );
90
91
0
  for( int j = 0; j < trSize; j++, trMat += 16 )
92
0
  {
93
0
    resi = 0;
94
0
    const int8_t* trMatTmp = trMat;
95
0
    int*          srcPtr   = src;
96
97
0
    for( int i = 0; i < zeroOutSize; i++ )
98
0
    {
99
0
      resi += *srcPtr++ * *trMatTmp++;
100
0
    }
101
102
0
    *out++ = Clip3( outputMinimum, outputMaximum, ( int ) ( resi + 64 ) >> 7 );
103
0
  }
104
0
}
105
106
0
static inline int64_t square( const int d ) { return d * (int64_t)d; }
107
108
template<int signedMode> void invTransformCbCr( PelBuf &resCb, PelBuf &resCr )
109
7.26k
{
110
7.26k
  Pel*  cb  = resCb.buf;
111
7.26k
  Pel*  cr  = resCr.buf;
112
104k
  for( SizeType y = 0; y < resCb.height; y++, cb += resCb.stride, cr += resCr.stride )
113
96.8k
  {
114
1.94M
    for( SizeType x = 0; x < resCb.width; x++ )
115
1.84M
    {
116
1.84M
      if      ( signedMode ==  1 )  { cr[x] =  cb[x] >> 1;  }
117
1.04M
      else if ( signedMode == -1 )  { cr[x] = -cb[x] >> 1;  }
118
1.01M
      else if ( signedMode ==  2 )  { cr[x] =  cb[x]; }
119
348k
      else if ( signedMode == -2 )  { cr[x] = -cb[x]; }
120
334k
      else if ( signedMode ==  3 )  { cb[x] =  cr[x] >> 1; }
121
6.70k
      else if ( signedMode == -3 )  { cb[x] = -cr[x] >> 1; }
122
1.84M
    }
123
96.8k
  }
124
7.26k
}
Unexecuted instantiation: void vvdec::invTransformCbCr<0>(vvdec::AreaBuf<short>&, vvdec::AreaBuf<short>&)
void vvdec::invTransformCbCr<1>(vvdec::AreaBuf<short>&, vvdec::AreaBuf<short>&)
Line
Count
Source
109
3.24k
{
110
3.24k
  Pel*  cb  = resCb.buf;
111
3.24k
  Pel*  cr  = resCr.buf;
112
46.3k
  for( SizeType y = 0; y < resCb.height; y++, cb += resCb.stride, cr += resCr.stride )
113
43.0k
  {
114
844k
    for( SizeType x = 0; x < resCb.width; x++ )
115
801k
    {
116
801k
      if      ( signedMode ==  1 )  { cr[x] =  cb[x] >> 1;  }
117
1
      else if ( signedMode == -1 )  { cr[x] = -cb[x] >> 1;  }
118
1
      else if ( signedMode ==  2 )  { cr[x] =  cb[x]; }
119
1
      else if ( signedMode == -2 )  { cr[x] = -cb[x]; }
120
1
      else if ( signedMode ==  3 )  { cb[x] =  cr[x] >> 1; }
121
1
      else if ( signedMode == -3 )  { cb[x] = -cr[x] >> 1; }
122
801k
    }
123
43.0k
  }
124
3.24k
}
void vvdec::invTransformCbCr<-1>(vvdec::AreaBuf<short>&, vvdec::AreaBuf<short>&)
Line
Count
Source
109
104
{
110
104
  Pel*  cb  = resCb.buf;
111
104
  Pel*  cr  = resCr.buf;
112
1.48k
  for( SizeType y = 0; y < resCb.height; y++, cb += resCb.stride, cr += resCr.stride )
113
1.38k
  {
114
25.4k
    for( SizeType x = 0; x < resCb.width; x++ )
115
24.0k
    {
116
24.0k
      if      ( signedMode ==  1 )  { cr[x] =  cb[x] >> 1;  }
117
24.0k
      else if ( signedMode == -1 )  { cr[x] = -cb[x] >> 1;  }
118
0
      else if ( signedMode ==  2 )  { cr[x] =  cb[x]; }
119
0
      else if ( signedMode == -2 )  { cr[x] = -cb[x]; }
120
0
      else if ( signedMode ==  3 )  { cb[x] =  cr[x] >> 1; }
121
0
      else if ( signedMode == -3 )  { cb[x] = -cr[x] >> 1; }
122
24.0k
    }
123
1.38k
  }
124
104
}
void vvdec::invTransformCbCr<2>(vvdec::AreaBuf<short>&, vvdec::AreaBuf<short>&)
Line
Count
Source
109
2.65k
{
110
2.65k
  Pel*  cb  = resCb.buf;
111
2.65k
  Pel*  cr  = resCr.buf;
112
37.3k
  for( SizeType y = 0; y < resCb.height; y++, cb += resCb.stride, cr += resCr.stride )
113
34.6k
  {
114
705k
    for( SizeType x = 0; x < resCb.width; x++ )
115
670k
    {
116
670k
      if      ( signedMode ==  1 )  { cr[x] =  cb[x] >> 1;  }
117
670k
      else if ( signedMode == -1 )  { cr[x] = -cb[x] >> 1;  }
118
670k
      else if ( signedMode ==  2 )  { cr[x] =  cb[x]; }
119
0
      else if ( signedMode == -2 )  { cr[x] = -cb[x]; }
120
0
      else if ( signedMode ==  3 )  { cb[x] =  cr[x] >> 1; }
121
0
      else if ( signedMode == -3 )  { cb[x] = -cr[x] >> 1; }
122
670k
    }
123
34.6k
  }
124
2.65k
}
void vvdec::invTransformCbCr<-2>(vvdec::AreaBuf<short>&, vvdec::AreaBuf<short>&)
Line
Count
Source
109
64
{
110
64
  Pel*  cb  = resCb.buf;
111
64
  Pel*  cr  = resCr.buf;
112
862
  for( SizeType y = 0; y < resCb.height; y++, cb += resCb.stride, cr += resCr.stride )
113
798
  {
114
15.1k
    for( SizeType x = 0; x < resCb.width; x++ )
115
14.4k
    {
116
14.4k
      if      ( signedMode ==  1 )  { cr[x] =  cb[x] >> 1;  }
117
14.4k
      else if ( signedMode == -1 )  { cr[x] = -cb[x] >> 1;  }
118
14.4k
      else if ( signedMode ==  2 )  { cr[x] =  cb[x]; }
119
14.4k
      else if ( signedMode == -2 )  { cr[x] = -cb[x]; }
120
0
      else if ( signedMode ==  3 )  { cb[x] =  cr[x] >> 1; }
121
0
      else if ( signedMode == -3 )  { cb[x] = -cr[x] >> 1; }
122
14.4k
    }
123
798
  }
124
64
}
void vvdec::invTransformCbCr<3>(vvdec::AreaBuf<short>&, vvdec::AreaBuf<short>&)
Line
Count
Source
109
1.16k
{
110
1.16k
  Pel*  cb  = resCb.buf;
111
1.16k
  Pel*  cr  = resCr.buf;
112
17.6k
  for( SizeType y = 0; y < resCb.height; y++, cb += resCb.stride, cr += resCr.stride )
113
16.4k
  {
114
344k
    for( SizeType x = 0; x < resCb.width; x++ )
115
327k
    {
116
327k
      if      ( signedMode ==  1 )  { cr[x] =  cb[x] >> 1;  }
117
327k
      else if ( signedMode == -1 )  { cr[x] = -cb[x] >> 1;  }
118
327k
      else if ( signedMode ==  2 )  { cr[x] =  cb[x]; }
119
327k
      else if ( signedMode == -2 )  { cr[x] = -cb[x]; }
120
327k
      else if ( signedMode ==  3 )  { cb[x] =  cr[x] >> 1; }
121
3
      else if ( signedMode == -3 )  { cb[x] = -cr[x] >> 1; }
122
327k
    }
123
16.4k
  }
124
1.16k
}
void vvdec::invTransformCbCr<-3>(vvdec::AreaBuf<short>&, vvdec::AreaBuf<short>&)
Line
Count
Source
109
29
{
110
29
  Pel*  cb  = resCb.buf;
111
29
  Pel*  cr  = resCr.buf;
112
451
  for( SizeType y = 0; y < resCb.height; y++, cb += resCb.stride, cr += resCr.stride )
113
422
  {
114
7.12k
    for( SizeType x = 0; x < resCb.width; x++ )
115
6.70k
    {
116
6.70k
      if      ( signedMode ==  1 )  { cr[x] =  cb[x] >> 1;  }
117
6.70k
      else if ( signedMode == -1 )  { cr[x] = -cb[x] >> 1;  }
118
6.70k
      else if ( signedMode ==  2 )  { cr[x] =  cb[x]; }
119
6.70k
      else if ( signedMode == -2 )  { cr[x] = -cb[x]; }
120
6.70k
      else if ( signedMode ==  3 )  { cb[x] =  cr[x] >> 1; }
121
6.70k
      else if ( signedMode == -3 )  { cb[x] = -cr[x] >> 1; }
122
6.70k
    }
123
422
  }
124
29
}
125
126
// ====================================================================================================================
127
// TrQuant class member functions
128
// ====================================================================================================================
129
130
59.8k
TrQuant::TrQuant( class InterPrediction* ip, const TrQuant* other ) : Quant( other )
131
59.8k
{
132
  // allocate temporary buffers
133
59.8k
  m_invICT      = m_invICTMem + maxAbsIctMode;
134
59.8k
  m_invICT[ 0]  = invTransformCbCr< 0>;
135
59.8k
  m_invICT[ 1]  = invTransformCbCr< 1>;
136
59.8k
  m_invICT[-1]  = invTransformCbCr<-1>;
137
59.8k
  m_invICT[ 2]  = invTransformCbCr< 2>;
138
59.8k
  m_invICT[-2]  = invTransformCbCr<-2>;
139
59.8k
  m_invICT[ 3]  = invTransformCbCr< 3>;
140
59.8k
  m_invICT[-3]  = invTransformCbCr<-3>;
141
142
59.8k
  m_invLfnstNxN = invLfnstNxNCore;
143
144
59.8k
  static_assert( sizeof( ip->m_acYuvPred[0] ) > sizeof( TCoeff ) * ( MAX_TU_SIZE_FOR_PROFILE * MAX_TU_SIZE_FOR_PROFILE + MEMORY_ALIGN_DEF_SIZE ), "Buffer to small to be reused!" );
145
59.8k
  static_assert( sizeof( ip->m_acYuvPred[1] ) > sizeof( TCoeff ) * ( MAX_TU_SIZE_FOR_PROFILE * MAX_TU_SIZE_FOR_PROFILE + MEMORY_ALIGN_DEF_SIZE ), "Buffer to small to be reused!" );
146
59.8k
  static_assert( sizeof( ip->m_acYuvPred[2] ) > sizeof( TCoeff ) * ( MAX_TU_SIZE_FOR_PROFILE * MAX_TU_SIZE_FOR_PROFILE + MEMORY_ALIGN_DEF_SIZE ), "Buffer to small to be reused!" );
147
148
59.8k
  char* tmp  = ( char* ) ip->m_acYuvPred[0];
149
59.8k
  char* blk  = ( char* ) ip->m_acYuvPred[1];
150
59.8k
  char* dqnt = ( char* ) ip->m_acYuvPred[2];
151
152
59.8k
  m_tmp  = ( TCoeff* ) ( ( ptrdiff_t ) tmp  + ( MEMORY_ALIGN_DEF_SIZE - ( ( ptrdiff_t ) tmp  & ( MEMORY_ALIGN_DEF_SIZE - 1 ) ) ) );
153
59.8k
  m_blk  = ( TCoeff* ) ( ( ptrdiff_t ) blk  + ( MEMORY_ALIGN_DEF_SIZE - ( ( ptrdiff_t ) blk  & ( MEMORY_ALIGN_DEF_SIZE - 1 ) ) ) );
154
59.8k
  m_dqnt = ( TCoeff* ) ( ( ptrdiff_t ) dqnt + ( MEMORY_ALIGN_DEF_SIZE - ( ( ptrdiff_t ) dqnt & ( MEMORY_ALIGN_DEF_SIZE - 1 ) ) ) );
155
156
59.8k
#if defined( TARGET_SIMD_X86 ) && ENABLE_SIMD_TCOEFF_OPS
157
59.8k
  initTrQuantX86();
158
59.8k
#endif
159
59.8k
}
160
161
void TrQuant::xDeQuant(const TransformUnit &tu,
162
                             CoeffBuf      &dstCoeff,
163
                       const ComponentID   &compID,
164
                       const QpParam       &cQP)
165
79.8k
{
166
79.8k
  PROFILER_SCOPE_AND_STAGE_EXT( 1, g_timeProfiler, P_PARSERESIDUALS, *tu.cu->cs, compID );
167
79.8k
  dequant( tu, dstCoeff, compID, cQP );
168
79.8k
}
169
170
void TrQuant::init( const Picture *pic )
171
22.8k
{
172
22.8k
  Quant::init( pic );
173
22.8k
}
174
175
uint32_t TrQuant::getLFNSTIntraMode( int wideAngPredMode )
176
18.8k
{
177
18.8k
  uint32_t intraMode;
178
179
18.8k
  if( wideAngPredMode < 0 )
180
256
  {
181
256
    intraMode = ( uint32_t ) ( wideAngPredMode + ( NUM_EXT_LUMA_MODE >> 1 ) + NUM_LUMA_MODE );
182
256
  }
183
18.6k
  else if( wideAngPredMode >= NUM_LUMA_MODE )
184
219
  {
185
219
    intraMode = ( uint32_t ) ( wideAngPredMode + ( NUM_EXT_LUMA_MODE >> 1 ) );
186
219
  }
187
18.4k
  else
188
18.4k
  {
189
18.4k
    intraMode = ( uint32_t ) wideAngPredMode;
190
18.4k
  }
191
192
18.8k
  return intraMode;
193
18.8k
}
194
195
bool TrQuant::getTransposeFlag( uint32_t intraMode )
196
18.8k
{
197
18.8k
  return ( ( intraMode >= NUM_LUMA_MODE ) && ( intraMode >= ( NUM_LUMA_MODE + ( NUM_EXT_LUMA_MODE >> 1 ) ) ) ) ||
198
18.6k
         ( ( intraMode <  NUM_LUMA_MODE ) && ( intraMode >  DIA_IDX ) );
199
18.8k
}
200
201
void TrQuant::xInvLfnst( TransformUnit &tu, const ComponentID& compID )
202
79.9k
{
203
79.9k
  const CompArea& area     = tu.blocks[ compID ];
204
79.9k
  const uint32_t  width    = area.width;
205
79.9k
  const uint32_t  height   = area.height;
206
79.9k
  const uint32_t  lfnstIdx = tu.cu->lfnstIdx();
207
208
79.9k
  if( lfnstIdx && tu.mtsIdx( compID ) != MTS_SKIP && ( CU::isSepTree( *tu.cu ) ? true : isLuma( compID ) ) )
209
18.8k
  {
210
18.8k
    const bool whge3     = width >= 8 && height >= 8;
211
18.8k
    const uint16_t* scan = whge3 ? g_coefTopLeftDiagScan8x8[ g_sizeIdxInfo.idxFrom( width ) ] : g_scanOrder[ SCAN_GROUPED_4x4 ][ g_sizeIdxInfo.idxFrom( width ) ][ g_sizeIdxInfo.idxFrom( height ) ];
212
18.8k
    uint32_t intraMode   = 0;
213
214
18.8k
    if( CU::isMIP( *tu.cu, toChannelType( compID ) ) )
215
802
    {
216
802
      intraMode = PLANAR_IDX;
217
802
    }
218
18.0k
    else
219
18.0k
    {
220
18.0k
      intraMode = PU::isLMCMode( tu.cu->intraDir[toChannelType( compID )] ) ? PU::getCoLocatedIntraLumaMode( *tu.cu ) : PU::getFinalIntraMode( *tu.cu, toChannelType( compID ) );
221
18.0k
    }
222
223
18.8k
    CHECKD( intraMode > NUM_INTRA_MODE, "Invalid intra mode" );
224
18.8k
    CHECKD( lfnstIdx >= 3, "Invalid LFNST index" );
225
226
18.8k
    intraMode                     = getLFNSTIntraMode( PU::getWideAngIntraMode( tu, intraMode, compID ) );
227
18.8k
    bool          transposeFlag   = getTransposeFlag( intraMode );
228
18.8k
    const int     sbSize          = whge3 ? 8 : 4;
229
18.8k
    bool          tu4x4Flag       = ( width == 4 && height == 4 );
230
18.8k
    bool          tu8x8Flag       = ( width == 8 && height == 8 );
231
18.8k
    TCoeff*       lfnstTemp;
232
18.8k
    TCoeff*       coeffTemp;
233
18.8k
    int y;
234
18.8k
    lfnstTemp = m_tempInMatrix; // inverse spectral rearrangement
235
18.8k
    coeffTemp = m_dqnt;
236
18.8k
    TCoeff * dst = lfnstTemp;
237
320k
    for( y = 0; y < 16; y++ )
238
301k
    {
239
301k
      *dst++ = coeffTemp[ scan[y] ];
240
301k
    }
241
242
18.8k
    m_invLfnstNxN( m_tempInMatrix, m_tempOutMatrix, g_lfnstLut[ intraMode ], lfnstIdx - 1, sbSize, ( tu4x4Flag || tu8x8Flag ) ? 8 : 16 );
243
244
18.8k
    lfnstTemp = m_tempOutMatrix; // inverse spectral rearrangement
245
246
18.8k
    if( transposeFlag )
247
4.37k
    {
248
4.37k
      if( sbSize == 4 )
249
1.34k
      {
250
6.71k
        for( y = 0; y < 4; y++ )
251
5.37k
        {
252
5.37k
          coeffTemp[ 0 ] = lfnstTemp[ 0 ];  coeffTemp[ 1 ] = lfnstTemp[  4 ];
253
5.37k
          coeffTemp[ 2 ] = lfnstTemp[ 8 ];  coeffTemp[ 3 ] = lfnstTemp[ 12 ];
254
5.37k
          lfnstTemp++;
255
5.37k
          coeffTemp += width;
256
5.37k
        }
257
1.34k
      }
258
3.03k
      else // ( sbSize == 8 )
259
3.03k
      {
260
27.3k
        for( y = 0; y < 8; y++ )
261
24.2k
        {
262
24.2k
          coeffTemp[ 0 ] = lfnstTemp[  0 ];  coeffTemp[ 1 ] = lfnstTemp[  8 ];
263
24.2k
          coeffTemp[ 2 ] = lfnstTemp[ 16 ];  coeffTemp[ 3 ] = lfnstTemp[ 24 ];
264
24.2k
          if( y < 4 )
265
12.1k
          {
266
12.1k
            coeffTemp[ 4 ] = lfnstTemp[ 32 ];  coeffTemp[ 5 ] = lfnstTemp[ 36 ];
267
12.1k
            coeffTemp[ 6 ] = lfnstTemp[ 40 ];  coeffTemp[ 7 ] = lfnstTemp[ 44 ];
268
12.1k
          }
269
24.2k
          lfnstTemp++;
270
24.2k
          coeffTemp += width;
271
24.2k
        }
272
3.03k
      }
273
4.37k
    }
274
14.5k
    else
275
14.5k
    {
276
114k
      for( y = 0; y < sbSize; y++ )
277
99.9k
      {
278
99.9k
        uint32_t uiStride = ( y < 4 ) ? sbSize : 4;
279
99.9k
        ::memcpy( coeffTemp, lfnstTemp, uiStride * sizeof( TCoeff ) );
280
99.9k
        lfnstTemp += uiStride;
281
99.9k
        coeffTemp += width;
282
99.9k
      }
283
14.5k
    }
284
285
18.8k
    tu.maxScanPosX[compID] = std::max<int>( tu.maxScanPosX[compID], std::min<int>( width  - 1, 7 ) );
286
18.8k
    tu.maxScanPosY[compID] = std::max<int>( tu.maxScanPosY[compID], std::min<int>( height - 1, 7 ) );
287
18.8k
  }
288
79.9k
}
289
290
void TrQuant::invTransformNxN( TransformUnit &tu, const ComponentID &compID, PelBuf &pResi, const QpParam &cQP )
291
79.6k
{
292
79.6k
  CompArea &area    = tu.blocks[compID];
293
79.6k
  uint32_t uiWidth  = area.width;
294
79.6k
  uint32_t uiHeight = area.height;
295
296
79.6k
  CoeffBuf coeff( m_dqnt, uiWidth, uiHeight );
297
79.6k
  coeff.memset( 0 );
298
299
79.6k
  xDeQuant( tu, coeff, compID, cQP );
300
301
79.6k
  DTRACE_COEFF_BUF( D_TCOEFF, coeff, tu, tu.cu->predMode(), compID );
302
303
79.6k
  if( tu.cu->sps->getUseLFNST() )
304
79.9k
  {
305
79.9k
    xInvLfnst( tu, compID );
306
79.9k
  }
307
308
79.6k
  if( tu.mtsIdx( compID )== 1 )
309
3.70k
  {
310
3.70k
    xITransformSkip( coeff, pResi, tu, compID );
311
3.70k
  }
312
75.9k
  else
313
75.9k
  {
314
75.9k
    xIT( tu, compID, coeff, pResi );
315
75.9k
  }
316
317
79.6k
  DTRACE_PEL_BUF( D_RESIDUALS, pResi, tu, tu.cu->predMode(), compID);
318
79.6k
}
319
320
void TrQuant::invTransformICT( const TransformUnit &tu, PelBuf &resCb, PelBuf &resCr )
321
7.26k
{
322
7.26k
  CHECKD( Size(resCb) != Size(resCr), "resCb and resCr have different sizes" );
323
7.26k
  ( *m_invICT[TU::getICTMode( tu, tu.cu->cs->picHeader->getJointCbCrSignFlag() )] )( resCb, resCr );
324
7.26k
}
325
326
// ------------------------------------------------------------------------------------------------
327
// Logical transform
328
// ------------------------------------------------------------------------------------------------
329
330
void TrQuant::getTrTypes( const TransformUnit& tu, const ComponentID compID, int &trTypeHor, int &trTypeVer )
331
76.1k
{
332
76.1k
  const bool isCuIntra     = CU::isIntra( *tu.cu );
333
76.1k
  const bool isCompLuma    = isLuma( compID );
334
76.1k
  const bool isImplicitMTS = isCuIntra && isCompLuma && tu.cu->sps->getUseImplicitMTS() && tu.cu->lfnstIdx() == 0 && tu.cu->mipFlag() == 0;
335
76.1k
  const bool isISP         = isCuIntra && isCompLuma && tu.cu->ispMode();
336
337
76.1k
  if( isISP && tu.cu->lfnstIdx() )
338
2.05k
  {
339
2.05k
    return;
340
2.05k
  }
341
342
74.1k
  const int lwidth  = tu.lwidth();
343
74.1k
  const int lheight = tu.lheight();
344
345
74.1k
  if( !tu.cu->sps->getUseMTS() )
346
0
    return;
347
348
74.1k
  if( isImplicitMTS || isISP )
349
18.4k
  {
350
18.4k
    bool widthDstOk   = lwidth  >= 4 && lwidth  <= 16;
351
18.4k
    bool heightDstOk  = lheight >= 4 && lheight <= 16;
352
353
18.4k
    if( widthDstOk )
354
10.4k
      trTypeHor = DST7;
355
18.4k
    if( heightDstOk )
356
10.5k
      trTypeVer = DST7;
357
358
18.4k
    return;
359
18.4k
  }
360
361
55.6k
  const bool isCuInter     = CU::isInter( *tu.cu ) && isCompLuma;
362
55.6k
  const bool isExplicitMTS = isCuIntra ? tu.cu->sps->getUseIntraMTS() && isCompLuma : tu.cu->sps->getUseInterMTS() && isCuInter;
363
55.6k
  const bool isSBT         = isCuInter && tu.cu->sbtInfo();
364
365
55.6k
  if( isSBT )
366
0
  {
367
0
    const uint8_t sbtIdx = CU::getSbtIdx( *tu.cu );
368
0
    const uint8_t sbtPos = CU::getSbtPos( *tu.cu );
369
370
0
    if( sbtIdx == SBT_VER_HALF || sbtIdx == SBT_VER_QUAD )
371
0
    {
372
0
      CHECK( lwidth > MTS_INTER_MAX_CU_SIZE, "wrong" );
373
0
      if( lheight > MTS_INTER_MAX_CU_SIZE )
374
0
      {
375
0
        trTypeHor = trTypeVer = DCT2;
376
0
      }
377
0
      else
378
0
      {
379
0
        if( sbtPos == SBT_POS0 )  { trTypeHor = DCT8;  trTypeVer = DST7; }
380
0
        else                      { trTypeHor = DST7;  trTypeVer = DST7; }
381
0
      }
382
0
    }
383
0
    else
384
0
    {
385
0
      CHECK( lheight > MTS_INTER_MAX_CU_SIZE, "wrong" );
386
0
      if( lwidth > MTS_INTER_MAX_CU_SIZE )
387
0
      {
388
0
        trTypeHor = trTypeVer = DCT2;
389
0
      }
390
0
      else
391
0
      {
392
0
        if( sbtPos == SBT_POS0 )  { trTypeHor = DST7;  trTypeVer = DCT8; }
393
0
        else                      { trTypeHor = DST7;  trTypeVer = DST7; }
394
0
      }
395
0
    }
396
0
    return;
397
0
  }
398
55.6k
  else if( isExplicitMTS )
399
0
  {
400
0
    if (tu.mtsIdx( compID ) > MTS_SKIP)
401
0
    {
402
0
      int indHor = (tu.mtsIdx( compID ) - MTS_DST7_DST7) & 1;
403
0
      int indVer = (tu.mtsIdx( compID ) - MTS_DST7_DST7) >> 1;
404
0
      trTypeHor = indHor ? DCT8 : DST7;
405
0
      trTypeVer = indVer ? DCT8 : DST7;
406
0
    }
407
0
  }
408
55.6k
}
409
410
void TrQuant::xIT( const TransformUnit &tu, const ComponentID &compID, const CCoeffBuf &pCoeff, PelBuf &pResidual )
411
76.2k
{
412
76.2k
  const int      width                  = pCoeff.width;
413
76.2k
  const int      height                 = pCoeff.height;
414
76.2k
  const unsigned maxLog2TrDynamicRange  = tu.cu->sps->getMaxLog2TrDynamicRange( toChannelType( compID ) );
415
76.2k
  const unsigned bitDepth               = tu.cu->sps->getBitDepth();
416
76.2k
  const int      TRANSFORM_MATRIX_SHIFT = g_transformMatrixShift;
417
76.2k
  const TCoeff   clipMinimum            = -( 1 << maxLog2TrDynamicRange );
418
76.2k
  const TCoeff   clipMaximum            =  ( 1 << maxLog2TrDynamicRange ) - 1;
419
76.2k
  const uint32_t transformWidthIndex    = getLog2(width ) - 1;                                // nLog2WidthMinus1, since transform start from 2-point
420
76.2k
  const uint32_t transformHeightIndex   = getLog2(height) - 1;                                // nLog2HeightMinus1, since transform start from 2-point
421
422
76.2k
  int trTypeHor = DCT2;
423
76.2k
  int trTypeVer = DCT2;
424
425
76.2k
  getTrTypes( tu, compID, trTypeHor, trTypeVer );
426
427
76.2k
  if( tu.maxScanPosX[compID] == 0 && tu.maxScanPosY[compID] == 0 && trTypeHor == DCT2 && trTypeVer == DCT2 )
428
14.2k
  {
429
14.2k
    int dcVal = 0;
430
431
14.2k
    if( width > 1 && height > 1 )
432
14.1k
    {
433
14.1k
      const int shift_1st = TRANSFORM_MATRIX_SHIFT + 1 + COM16_C806_TRANS_PREC;
434
14.1k
      const int shift_2nd = ( TRANSFORM_MATRIX_SHIFT + maxLog2TrDynamicRange - 1 ) - bitDepth + COM16_C806_TRANS_PREC;
435
436
14.1k
      dcVal = ( ( pCoeff.buf[0] * ( 1 << TRANSFORM_MATRIX_SHIFT ) ) + ( 1 << ( shift_1st - 1 ) ) ) >> shift_1st;
437
14.1k
      dcVal = ( ( dcVal * ( 1 << TRANSFORM_MATRIX_SHIFT ) ) + ( 1 << ( shift_2nd - 1 ) ) ) >> shift_2nd;
438
14.1k
    }
439
75
    else
440
75
    {
441
75
      const int shift = ( TRANSFORM_MATRIX_SHIFT + maxLog2TrDynamicRange ) - bitDepth + COM16_C806_TRANS_PREC;
442
75
      dcVal = ( ( pCoeff.buf[0] * ( 1 << TRANSFORM_MATRIX_SHIFT ) ) + ( 1 << ( shift - 1 ) ) ) >> shift;
443
75
    }
444
445
14.2k
    pResidual.fill( dcVal );
446
14.2k
    return;
447
14.2k
  }
448
449
62.0k
  const int skipWidth  = std::max<int>( ( trTypeHor != DCT2 && width  == 32 ) ? 16 : width  > JVET_C0024_ZERO_OUT_TH ? width  - JVET_C0024_ZERO_OUT_TH : 0, width  - tu.maxScanPosX[compID] - 1 );
450
62.0k
  const int skipHeight = std::max<int>( ( trTypeVer != DCT2 && height == 32 ) ? 16 : height > JVET_C0024_ZERO_OUT_TH ? height - JVET_C0024_ZERO_OUT_TH : 0, height - tu.maxScanPosY[compID] - 1 );
451
452
62.0k
  TCoeff *block = m_blk;
453
62.0k
  int shiftlast;
454
455
62.0k
  if( width > 1 && height > 1 ) //2-D transform
456
61.7k
  {
457
61.7k
    const int      shift_1st              =   TRANSFORM_MATRIX_SHIFT + 1 + COM16_C806_TRANS_PREC; // 1 has been added to shift_1st at the expense of shift_2nd
458
61.7k
    const int      shift_2nd              = ( TRANSFORM_MATRIX_SHIFT + maxLog2TrDynamicRange - 1 ) - bitDepth + COM16_C806_TRANS_PREC;
459
61.7k
    CHECK( shift_1st < 0, "Negative shift" );
460
61.7k
    CHECK( shift_2nd < 0, "Negative shift" );
461
61.7k
    TCoeff *tmp   = m_tmp;
462
61.7k
    fastInvTrans[trTypeVer][transformHeightIndex](pCoeff.buf, tmp, shift_1st, width, skipWidth, skipHeight, true,  clipMinimum, clipMaximum);
463
61.7k
    fastInvTrans[trTypeHor][transformWidthIndex] (tmp,      block, shift_2nd, height,         0, skipWidth, false, clipMinimum, clipMaximum);
464
61.7k
    shiftlast = shift_2nd;
465
61.7k
  }
466
336
  else if( width == 1 ) //1-D vertical transform
467
129
  {
468
129
    int shift = ( TRANSFORM_MATRIX_SHIFT + maxLog2TrDynamicRange - 1 ) - bitDepth + COM16_C806_TRANS_PREC;
469
129
    CHECK( shift < 0, "Negative shift" );
470
129
    CHECK( ( transformHeightIndex < 0 ), "There is a problem with the height." );
471
129
    fastInvTrans[trTypeVer][transformHeightIndex]( pCoeff.buf, block, shift + 1, 1, 0, skipHeight, false, clipMinimum, clipMaximum );
472
129
    shiftlast = shift + 1;
473
129
  }
474
207
  else //if(iHeight == 1) //1-D horizontal transform
475
207
  {
476
207
    const int      shift              = ( TRANSFORM_MATRIX_SHIFT + maxLog2TrDynamicRange - 1 ) - bitDepth + COM16_C806_TRANS_PREC;
477
207
    CHECK( shift < 0, "Negative shift" );
478
207
    CHECK( ( transformWidthIndex < 0 ), "There is a problem with the width." );
479
207
    fastInvTrans[trTypeHor][transformWidthIndex]( pCoeff.buf, block, shift + 1, 1, 0, skipWidth, false, clipMinimum, clipMaximum );
480
207
    shiftlast = shift + 1;
481
207
  }
482
483
62.0k
  int round = 1 << ( shiftlast - 1 );
484
62.0k
  g_tCoeffOps.cpyResiClip[getLog2( width )]( block, pResidual.buf, pResidual.stride, width, height, clipMinimum, clipMaximum, round, shiftlast );
485
62.0k
}
486
487
/** Wrapper function between HM interface and core NxN transform skipping
488
 */
489
void TrQuant::xITransformSkip(const CCoeffBuf     &pCoeff,
490
                                    PelBuf        &pResidual,
491
                              const TransformUnit &tu,
492
                              const ComponentID   &compID)
493
3.70k
{
494
3.70k
  const CompArea &area  = tu.blocks[compID];
495
3.70k
  const int width       = area.width;
496
3.70k
  const int height      = area.height;
497
  
498
42.6k
  for( int y = 0; y < height; y++ )
499
38.9k
  {
500
489k
    for( int x = 0; x < width; x++ )
501
450k
    {
502
450k
      pResidual.at( x, y ) = Pel( pCoeff.at( x, y ) );
503
450k
    }
504
38.9k
  }
505
3.70k
}
506
507
}