Coverage Report

Created: 2026-07-25 07:06

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/vvdec/source/Lib/CommonLib/TrQuant.cpp
Line
Count
Source
1
/* -----------------------------------------------------------------------------
2
The copyright in this software is being made available under the Clear BSD
3
License, included below. No patent rights, trademark rights and/or 
4
other Intellectual Property Rights other than the copyrights concerning 
5
the Software are granted under this license.
6
7
The Clear BSD License
8
9
Copyright (c) 2018-2026, Fraunhofer-Gesellschaft zur Förderung der angewandten Forschung e.V. & The VVdeC Authors.
10
All rights reserved.
11
12
Redistribution and use in source and binary forms, with or without modification,
13
are permitted (subject to the limitations in the disclaimer below) provided that
14
the following conditions are met:
15
16
     * Redistributions of source code must retain the above copyright notice,
17
     this list of conditions and the following disclaimer.
18
19
     * Redistributions in binary form must reproduce the above copyright
20
     notice, this list of conditions and the following disclaimer in the
21
     documentation and/or other materials provided with the distribution.
22
23
     * Neither the name of the copyright holder nor the names of its
24
     contributors may be used to endorse or promote products derived from this
25
     software without specific prior written permission.
26
27
NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY
28
THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
29
CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
30
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
31
PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
32
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
33
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
34
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
35
BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
36
IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
37
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
38
POSSIBILITY OF SUCH DAMAGE.
39
40
41
------------------------------------------------------------------------------------------- */
42
43
/** \file     TrQuant.cpp
44
    \brief    transform and quantization class
45
*/
46
47
#include "TrQuant.h"
48
#include "TrQuant_EMT.h"
49
50
#include "UnitTools.h"
51
#include "ContextModelling.h"
52
#include "CodingStructure.h"
53
#include "TimeProfiler.h"
54
#include "Quant.h"
55
#include "InterPrediction.h"
56
57
58
#include "dtrace_buffer.h"
59
60
#include <stdlib.h>
61
#include <limits>
62
#include <memory.h>
63
64
#include "Quant.h"
65
66
namespace vvdec
67
{
68
69
InvTrans *fastInvTrans[NUM_TRANS_TYPE][g_numTransformMatrixSizes] =
70
{
71
  { fastInverseDCT2_B2, fastInverseDCT2_B4, fastInverseDCT2_B8, fastInverseDCT2_B16, fastInverseDCT2_B32, fastInverseDCT2_B64 },
72
  { nullptr,            fastInverseDCT8_B4, fastInverseDCT8_B8, fastInverseDCT8_B16, fastInverseDCT8_B32, nullptr },
73
  { nullptr,            fastInverseDST7_B4, fastInverseDST7_B8, fastInverseDST7_B16, fastInverseDST7_B32, nullptr },
74
};
75
76
//! \ingroup CommonLib
77
//! \{
78
79
static void invLfnstNxNCore( int* src, int* dst, const uint32_t mode, const uint32_t index, const uint32_t size, int zeroOutSize )
80
0
{
81
0
  int             maxLog2TrDynamicRange =  15;
82
0
  const TCoeff    outputMinimum         = -( 1 << maxLog2TrDynamicRange );
83
0
  const TCoeff    outputMaximum         =  ( 1 << maxLog2TrDynamicRange ) - 1;
84
0
  const int8_t*   trMat                 =  ( size > 4 ) ? g_lfnst8x8[ mode ][ index ][ 0 ] : g_lfnst4x4[ mode ][ index ][ 0 ];
85
0
  const int       trSize                =  ( size > 4 ) ? 48 : 16;
86
0
  int             resi;
87
0
  int*            out                   =  dst;
88
89
0
  CHECK( index > 2, "wrong" );
90
91
0
  for( int j = 0; j < trSize; j++, trMat += 16 )
92
0
  {
93
0
    resi = 0;
94
0
    const int8_t* trMatTmp = trMat;
95
0
    int*          srcPtr   = src;
96
97
0
    for( int i = 0; i < zeroOutSize; i++ )
98
0
    {
99
0
      resi += *srcPtr++ * *trMatTmp++;
100
0
    }
101
102
0
    *out++ = Clip3( outputMinimum, outputMaximum, ( int ) ( resi + 64 ) >> 7 );
103
0
  }
104
0
}
105
106
0
static inline int64_t square( const int d ) { return d * (int64_t)d; }
107
108
template<int signedMode> void invTransformCbCr( PelBuf &resCb, PelBuf &resCr )
109
7.05k
{
110
7.05k
  Pel*  cb  = resCb.buf;
111
7.05k
  Pel*  cr  = resCr.buf;
112
101k
  for( SizeType y = 0; y < resCb.height; y++, cb += resCb.stride, cr += resCr.stride )
113
94.4k
  {
114
1.89M
    for( SizeType x = 0; x < resCb.width; x++ )
115
1.80M
    {
116
1.80M
      if      ( signedMode ==  1 )  { cr[x] =  cb[x] >> 1;  }
117
1.02M
      else if ( signedMode == -1 )  { cr[x] = -cb[x] >> 1;  }
118
1.00M
      else if ( signedMode ==  2 )  { cr[x] =  cb[x]; }
119
341k
      else if ( signedMode == -2 )  { cr[x] = -cb[x]; }
120
330k
      else if ( signedMode ==  3 )  { cb[x] =  cr[x] >> 1; }
121
5.90k
      else if ( signedMode == -3 )  { cb[x] = -cr[x] >> 1; }
122
1.80M
    }
123
94.4k
  }
124
7.05k
}
Unexecuted instantiation: void vvdec::invTransformCbCr<0>(vvdec::AreaBuf<short>&, vvdec::AreaBuf<short>&)
void vvdec::invTransformCbCr<1>(vvdec::AreaBuf<short>&, vvdec::AreaBuf<short>&)
Line
Count
Source
109
3.14k
{
110
3.14k
  Pel*  cb  = resCb.buf;
111
3.14k
  Pel*  cr  = resCr.buf;
112
45.0k
  for( SizeType y = 0; y < resCb.height; y++, cb += resCb.stride, cr += resCr.stride )
113
41.9k
  {
114
820k
    for( SizeType x = 0; x < resCb.width; x++ )
115
779k
    {
116
779k
      if      ( signedMode ==  1 )  { cr[x] =  cb[x] >> 1;  }
117
4
      else if ( signedMode == -1 )  { cr[x] = -cb[x] >> 1;  }
118
4
      else if ( signedMode ==  2 )  { cr[x] =  cb[x]; }
119
4
      else if ( signedMode == -2 )  { cr[x] = -cb[x]; }
120
4
      else if ( signedMode ==  3 )  { cb[x] =  cr[x] >> 1; }
121
4
      else if ( signedMode == -3 )  { cb[x] = -cr[x] >> 1; }
122
779k
    }
123
41.9k
  }
124
3.14k
}
void vvdec::invTransformCbCr<-1>(vvdec::AreaBuf<short>&, vvdec::AreaBuf<short>&)
Line
Count
Source
109
87
{
110
87
  Pel*  cb  = resCb.buf;
111
87
  Pel*  cr  = resCr.buf;
112
1.27k
  for( SizeType y = 0; y < resCb.height; y++, cb += resCb.stride, cr += resCr.stride )
113
1.18k
  {
114
22.0k
    for( SizeType x = 0; x < resCb.width; x++ )
115
20.8k
    {
116
20.8k
      if      ( signedMode ==  1 )  { cr[x] =  cb[x] >> 1;  }
117
20.8k
      else if ( signedMode == -1 )  { cr[x] = -cb[x] >> 1;  }
118
18.4E
      else if ( signedMode ==  2 )  { cr[x] =  cb[x]; }
119
18.4E
      else if ( signedMode == -2 )  { cr[x] = -cb[x]; }
120
18.4E
      else if ( signedMode ==  3 )  { cb[x] =  cr[x] >> 1; }
121
18.4E
      else if ( signedMode == -3 )  { cb[x] = -cr[x] >> 1; }
122
20.8k
    }
123
1.18k
  }
124
87
}
void vvdec::invTransformCbCr<2>(vvdec::AreaBuf<short>&, vvdec::AreaBuf<short>&)
Line
Count
Source
109
2.61k
{
110
2.61k
  Pel*  cb  = resCb.buf;
111
2.61k
  Pel*  cr  = resCr.buf;
112
36.7k
  for( SizeType y = 0; y < resCb.height; y++, cb += resCb.stride, cr += resCr.stride )
113
34.1k
  {
114
695k
    for( SizeType x = 0; x < resCb.width; x++ )
115
660k
    {
116
660k
      if      ( signedMode ==  1 )  { cr[x] =  cb[x] >> 1;  }
117
660k
      else if ( signedMode == -1 )  { cr[x] = -cb[x] >> 1;  }
118
660k
      else if ( signedMode ==  2 )  { cr[x] =  cb[x]; }
119
34
      else if ( signedMode == -2 )  { cr[x] = -cb[x]; }
120
34
      else if ( signedMode ==  3 )  { cb[x] =  cr[x] >> 1; }
121
34
      else if ( signedMode == -3 )  { cb[x] = -cr[x] >> 1; }
122
660k
    }
123
34.1k
  }
124
2.61k
}
void vvdec::invTransformCbCr<-2>(vvdec::AreaBuf<short>&, vvdec::AreaBuf<short>&)
Line
Count
Source
109
38
{
110
38
  Pel*  cb  = resCb.buf;
111
38
  Pel*  cr  = resCr.buf;
112
600
  for( SizeType y = 0; y < resCb.height; y++, cb += resCb.stride, cr += resCr.stride )
113
562
  {
114
11.8k
    for( SizeType x = 0; x < resCb.width; x++ )
115
11.2k
    {
116
11.2k
      if      ( signedMode ==  1 )  { cr[x] =  cb[x] >> 1;  }
117
11.2k
      else if ( signedMode == -1 )  { cr[x] = -cb[x] >> 1;  }
118
11.2k
      else if ( signedMode ==  2 )  { cr[x] =  cb[x]; }
119
11.2k
      else if ( signedMode == -2 )  { cr[x] = -cb[x]; }
120
0
      else if ( signedMode ==  3 )  { cb[x] =  cr[x] >> 1; }
121
0
      else if ( signedMode == -3 )  { cb[x] = -cr[x] >> 1; }
122
11.2k
    }
123
562
  }
124
38
}
void vvdec::invTransformCbCr<3>(vvdec::AreaBuf<short>&, vvdec::AreaBuf<short>&)
Line
Count
Source
109
1.16k
{
110
1.16k
  Pel*  cb  = resCb.buf;
111
1.16k
  Pel*  cr  = resCr.buf;
112
17.4k
  for( SizeType y = 0; y < resCb.height; y++, cb += resCb.stride, cr += resCr.stride )
113
16.2k
  {
114
340k
    for( SizeType x = 0; x < resCb.width; x++ )
115
324k
    {
116
324k
      if      ( signedMode ==  1 )  { cr[x] =  cb[x] >> 1;  }
117
324k
      else if ( signedMode == -1 )  { cr[x] = -cb[x] >> 1;  }
118
324k
      else if ( signedMode ==  2 )  { cr[x] =  cb[x]; }
119
324k
      else if ( signedMode == -2 )  { cr[x] = -cb[x]; }
120
324k
      else if ( signedMode ==  3 )  { cb[x] =  cr[x] >> 1; }
121
0
      else if ( signedMode == -3 )  { cb[x] = -cr[x] >> 1; }
122
324k
    }
123
16.2k
  }
124
1.16k
}
void vvdec::invTransformCbCr<-3>(vvdec::AreaBuf<short>&, vvdec::AreaBuf<short>&)
Line
Count
Source
109
23
{
110
23
  Pel*  cb  = resCb.buf;
111
23
  Pel*  cr  = resCr.buf;
112
385
  for( SizeType y = 0; y < resCb.height; y++, cb += resCb.stride, cr += resCr.stride )
113
362
  {
114
6.23k
    for( SizeType x = 0; x < resCb.width; x++ )
115
5.87k
    {
116
5.87k
      if      ( signedMode ==  1 )  { cr[x] =  cb[x] >> 1;  }
117
5.87k
      else if ( signedMode == -1 )  { cr[x] = -cb[x] >> 1;  }
118
5.87k
      else if ( signedMode ==  2 )  { cr[x] =  cb[x]; }
119
5.87k
      else if ( signedMode == -2 )  { cr[x] = -cb[x]; }
120
5.87k
      else if ( signedMode ==  3 )  { cb[x] =  cr[x] >> 1; }
121
5.87k
      else if ( signedMode == -3 )  { cb[x] = -cr[x] >> 1; }
122
5.87k
    }
123
362
  }
124
23
}
125
126
// ====================================================================================================================
127
// TrQuant class member functions
128
// ====================================================================================================================
129
130
59.8k
TrQuant::TrQuant( class InterPrediction* ip, const TrQuant* other ) : Quant( other )
131
59.8k
{
132
  // allocate temporary buffers
133
59.8k
  m_invICT      = m_invICTMem + maxAbsIctMode;
134
59.8k
  m_invICT[ 0]  = invTransformCbCr< 0>;
135
59.8k
  m_invICT[ 1]  = invTransformCbCr< 1>;
136
59.8k
  m_invICT[-1]  = invTransformCbCr<-1>;
137
59.8k
  m_invICT[ 2]  = invTransformCbCr< 2>;
138
59.8k
  m_invICT[-2]  = invTransformCbCr<-2>;
139
59.8k
  m_invICT[ 3]  = invTransformCbCr< 3>;
140
59.8k
  m_invICT[-3]  = invTransformCbCr<-3>;
141
142
59.8k
  m_invLfnstNxN = invLfnstNxNCore;
143
144
59.8k
  static_assert( sizeof( ip->m_acYuvPred[0] ) > sizeof( TCoeff ) * ( MAX_TU_SIZE_FOR_PROFILE * MAX_TU_SIZE_FOR_PROFILE + MEMORY_ALIGN_DEF_SIZE ), "Buffer to small to be reused!" );
145
59.8k
  static_assert( sizeof( ip->m_acYuvPred[1] ) > sizeof( TCoeff ) * ( MAX_TU_SIZE_FOR_PROFILE * MAX_TU_SIZE_FOR_PROFILE + MEMORY_ALIGN_DEF_SIZE ), "Buffer to small to be reused!" );
146
59.8k
  static_assert( sizeof( ip->m_acYuvPred[2] ) > sizeof( TCoeff ) * ( MAX_TU_SIZE_FOR_PROFILE * MAX_TU_SIZE_FOR_PROFILE + MEMORY_ALIGN_DEF_SIZE ), "Buffer to small to be reused!" );
147
148
59.8k
  char* tmp  = ( char* ) ip->m_acYuvPred[0];
149
59.8k
  char* blk  = ( char* ) ip->m_acYuvPred[1];
150
59.8k
  char* dqnt = ( char* ) ip->m_acYuvPred[2];
151
152
59.8k
  m_tmp  = ( TCoeff* ) ( ( ptrdiff_t ) tmp  + ( MEMORY_ALIGN_DEF_SIZE - ( ( ptrdiff_t ) tmp  & ( MEMORY_ALIGN_DEF_SIZE - 1 ) ) ) );
153
59.8k
  m_blk  = ( TCoeff* ) ( ( ptrdiff_t ) blk  + ( MEMORY_ALIGN_DEF_SIZE - ( ( ptrdiff_t ) blk  & ( MEMORY_ALIGN_DEF_SIZE - 1 ) ) ) );
154
59.8k
  m_dqnt = ( TCoeff* ) ( ( ptrdiff_t ) dqnt + ( MEMORY_ALIGN_DEF_SIZE - ( ( ptrdiff_t ) dqnt & ( MEMORY_ALIGN_DEF_SIZE - 1 ) ) ) );
155
156
59.8k
#if defined( TARGET_SIMD_X86 ) && ENABLE_SIMD_TCOEFF_OPS
157
59.8k
  initTrQuantX86();
158
59.8k
#endif
159
59.8k
}
160
161
void TrQuant::xDeQuant(const TransformUnit &tu,
162
                             CoeffBuf      &dstCoeff,
163
                       const ComponentID   &compID,
164
                       const QpParam       &cQP)
165
78.2k
{
166
78.2k
  PROFILER_SCOPE_AND_STAGE_EXT( 1, g_timeProfiler, P_PARSERESIDUALS, *tu.cu->cs, compID );
167
78.2k
  dequant( tu, dstCoeff, compID, cQP );
168
78.2k
}
169
170
void TrQuant::init( const Picture *pic )
171
22.8k
{
172
22.8k
  Quant::init( pic );
173
22.8k
}
174
175
uint32_t TrQuant::getLFNSTIntraMode( int wideAngPredMode )
176
18.4k
{
177
18.4k
  uint32_t intraMode;
178
179
18.4k
  if( wideAngPredMode < 0 )
180
252
  {
181
252
    intraMode = ( uint32_t ) ( wideAngPredMode + ( NUM_EXT_LUMA_MODE >> 1 ) + NUM_LUMA_MODE );
182
252
  }
183
18.1k
  else if( wideAngPredMode >= NUM_LUMA_MODE )
184
226
  {
185
226
    intraMode = ( uint32_t ) ( wideAngPredMode + ( NUM_EXT_LUMA_MODE >> 1 ) );
186
226
  }
187
17.9k
  else
188
17.9k
  {
189
17.9k
    intraMode = ( uint32_t ) wideAngPredMode;
190
17.9k
  }
191
192
18.4k
  return intraMode;
193
18.4k
}
194
195
bool TrQuant::getTransposeFlag( uint32_t intraMode )
196
18.3k
{
197
18.3k
  return ( ( intraMode >= NUM_LUMA_MODE ) && ( intraMode >= ( NUM_LUMA_MODE + ( NUM_EXT_LUMA_MODE >> 1 ) ) ) ) ||
198
18.1k
         ( ( intraMode <  NUM_LUMA_MODE ) && ( intraMode >  DIA_IDX ) );
199
18.3k
}
200
201
void TrQuant::xInvLfnst( TransformUnit &tu, const ComponentID& compID )
202
78.3k
{
203
78.3k
  const CompArea& area     = tu.blocks[ compID ];
204
78.3k
  const uint32_t  width    = area.width;
205
78.3k
  const uint32_t  height   = area.height;
206
78.3k
  const uint32_t  lfnstIdx = tu.cu->lfnstIdx();
207
208
78.3k
  if( lfnstIdx && tu.mtsIdx( compID ) != MTS_SKIP && ( CU::isSepTree( *tu.cu ) ? true : isLuma( compID ) ) )
209
18.4k
  {
210
18.4k
    const bool whge3     = width >= 8 && height >= 8;
211
18.4k
    const uint16_t* scan = whge3 ? g_coefTopLeftDiagScan8x8[ g_sizeIdxInfo.idxFrom( width ) ] : g_scanOrder[ SCAN_GROUPED_4x4 ][ g_sizeIdxInfo.idxFrom( width ) ][ g_sizeIdxInfo.idxFrom( height ) ];
212
18.4k
    uint32_t intraMode   = 0;
213
214
18.4k
    if( CU::isMIP( *tu.cu, toChannelType( compID ) ) )
215
777
    {
216
777
      intraMode = PLANAR_IDX;
217
777
    }
218
17.6k
    else
219
17.6k
    {
220
17.6k
      intraMode = PU::isLMCMode( tu.cu->intraDir[toChannelType( compID )] ) ? PU::getCoLocatedIntraLumaMode( *tu.cu ) : PU::getFinalIntraMode( *tu.cu, toChannelType( compID ) );
221
17.6k
    }
222
223
18.4k
    CHECKD( intraMode > NUM_INTRA_MODE, "Invalid intra mode" );
224
18.4k
    CHECKD( lfnstIdx >= 3, "Invalid LFNST index" );
225
226
18.4k
    intraMode                     = getLFNSTIntraMode( PU::getWideAngIntraMode( tu, intraMode, compID ) );
227
18.4k
    bool          transposeFlag   = getTransposeFlag( intraMode );
228
18.4k
    const int     sbSize          = whge3 ? 8 : 4;
229
18.4k
    bool          tu4x4Flag       = ( width == 4 && height == 4 );
230
18.4k
    bool          tu8x8Flag       = ( width == 8 && height == 8 );
231
18.4k
    TCoeff*       lfnstTemp;
232
18.4k
    TCoeff*       coeffTemp;
233
18.4k
    int y;
234
18.4k
    lfnstTemp = m_tempInMatrix; // inverse spectral rearrangement
235
18.4k
    coeffTemp = m_dqnt;
236
18.4k
    TCoeff * dst = lfnstTemp;
237
312k
    for( y = 0; y < 16; y++ )
238
293k
    {
239
293k
      *dst++ = coeffTemp[ scan[y] ];
240
293k
    }
241
242
18.4k
    m_invLfnstNxN( m_tempInMatrix, m_tempOutMatrix, g_lfnstLut[ intraMode ], lfnstIdx - 1, sbSize, ( tu4x4Flag || tu8x8Flag ) ? 8 : 16 );
243
244
18.4k
    lfnstTemp = m_tempOutMatrix; // inverse spectral rearrangement
245
246
18.4k
    if( transposeFlag )
247
4.32k
    {
248
4.32k
      if( sbSize == 4 )
249
1.33k
      {
250
6.68k
        for( y = 0; y < 4; y++ )
251
5.34k
        {
252
5.34k
          coeffTemp[ 0 ] = lfnstTemp[ 0 ];  coeffTemp[ 1 ] = lfnstTemp[  4 ];
253
5.34k
          coeffTemp[ 2 ] = lfnstTemp[ 8 ];  coeffTemp[ 3 ] = lfnstTemp[ 12 ];
254
5.34k
          lfnstTemp++;
255
5.34k
          coeffTemp += width;
256
5.34k
        }
257
1.33k
      }
258
2.98k
      else // ( sbSize == 8 )
259
2.98k
      {
260
26.9k
        for( y = 0; y < 8; y++ )
261
23.9k
        {
262
23.9k
          coeffTemp[ 0 ] = lfnstTemp[  0 ];  coeffTemp[ 1 ] = lfnstTemp[  8 ];
263
23.9k
          coeffTemp[ 2 ] = lfnstTemp[ 16 ];  coeffTemp[ 3 ] = lfnstTemp[ 24 ];
264
23.9k
          if( y < 4 )
265
11.9k
          {
266
11.9k
            coeffTemp[ 4 ] = lfnstTemp[ 32 ];  coeffTemp[ 5 ] = lfnstTemp[ 36 ];
267
11.9k
            coeffTemp[ 6 ] = lfnstTemp[ 40 ];  coeffTemp[ 7 ] = lfnstTemp[ 44 ];
268
11.9k
          }
269
23.9k
          lfnstTemp++;
270
23.9k
          coeffTemp += width;
271
23.9k
        }
272
2.98k
      }
273
4.32k
    }
274
14.0k
    else
275
14.0k
    {
276
110k
      for( y = 0; y < sbSize; y++ )
277
96.8k
      {
278
96.8k
        uint32_t uiStride = ( y < 4 ) ? sbSize : 4;
279
96.8k
        ::memcpy( coeffTemp, lfnstTemp, uiStride * sizeof( TCoeff ) );
280
96.8k
        lfnstTemp += uiStride;
281
96.8k
        coeffTemp += width;
282
96.8k
      }
283
14.0k
    }
284
285
18.4k
    tu.maxScanPosX[compID] = std::max<int>( tu.maxScanPosX[compID], std::min<int>( width  - 1, 7 ) );
286
18.4k
    tu.maxScanPosY[compID] = std::max<int>( tu.maxScanPosY[compID], std::min<int>( height - 1, 7 ) );
287
18.4k
  }
288
78.3k
}
289
290
void TrQuant::invTransformNxN( TransformUnit &tu, const ComponentID &compID, PelBuf &pResi, const QpParam &cQP )
291
78.0k
{
292
78.0k
  CompArea &area    = tu.blocks[compID];
293
78.0k
  uint32_t uiWidth  = area.width;
294
78.0k
  uint32_t uiHeight = area.height;
295
296
78.0k
  CoeffBuf coeff( m_dqnt, uiWidth, uiHeight );
297
78.0k
  coeff.memset( 0 );
298
299
78.0k
  xDeQuant( tu, coeff, compID, cQP );
300
301
78.0k
  DTRACE_COEFF_BUF( D_TCOEFF, coeff, tu, tu.cu->predMode(), compID );
302
303
78.0k
  if( tu.cu->sps->getUseLFNST() )
304
78.3k
  {
305
78.3k
    xInvLfnst( tu, compID );
306
78.3k
  }
307
308
78.0k
  if( tu.mtsIdx( compID )== 1 )
309
3.67k
  {
310
3.67k
    xITransformSkip( coeff, pResi, tu, compID );
311
3.67k
  }
312
74.3k
  else
313
74.3k
  {
314
74.3k
    xIT( tu, compID, coeff, pResi );
315
74.3k
  }
316
317
78.0k
  DTRACE_PEL_BUF( D_RESIDUALS, pResi, tu, tu.cu->predMode(), compID);
318
78.0k
}
319
320
void TrQuant::invTransformICT( const TransformUnit &tu, PelBuf &resCb, PelBuf &resCr )
321
7.05k
{
322
7.05k
  CHECKD( Size(resCb) != Size(resCr), "resCb and resCr have different sizes" );
323
7.05k
  ( *m_invICT[TU::getICTMode( tu, tu.cu->cs->picHeader->getJointCbCrSignFlag() )] )( resCb, resCr );
324
7.05k
}
325
326
// ------------------------------------------------------------------------------------------------
327
// Logical transform
328
// ------------------------------------------------------------------------------------------------
329
330
void TrQuant::getTrTypes( const TransformUnit& tu, const ComponentID compID, int &trTypeHor, int &trTypeVer )
331
74.5k
{
332
74.5k
  const bool isCuIntra     = CU::isIntra( *tu.cu );
333
74.5k
  const bool isCompLuma    = isLuma( compID );
334
74.5k
  const bool isImplicitMTS = isCuIntra && isCompLuma && tu.cu->sps->getUseImplicitMTS() && tu.cu->lfnstIdx() == 0 && tu.cu->mipFlag() == 0;
335
74.5k
  const bool isISP         = isCuIntra && isCompLuma && tu.cu->ispMode();
336
337
74.5k
  if( isISP && tu.cu->lfnstIdx() )
338
1.98k
  {
339
1.98k
    return;
340
1.98k
  }
341
342
72.5k
  const int lwidth  = tu.lwidth();
343
72.5k
  const int lheight = tu.lheight();
344
345
72.5k
  if( !tu.cu->sps->getUseMTS() )
346
0
    return;
347
348
72.5k
  if( isImplicitMTS || isISP )
349
18.0k
  {
350
18.0k
    bool widthDstOk   = lwidth  >= 4 && lwidth  <= 16;
351
18.0k
    bool heightDstOk  = lheight >= 4 && lheight <= 16;
352
353
18.0k
    if( widthDstOk )
354
10.1k
      trTypeHor = DST7;
355
18.0k
    if( heightDstOk )
356
10.2k
      trTypeVer = DST7;
357
358
18.0k
    return;
359
18.0k
  }
360
361
54.5k
  const bool isCuInter     = CU::isInter( *tu.cu ) && isCompLuma;
362
54.5k
  const bool isExplicitMTS = isCuIntra ? tu.cu->sps->getUseIntraMTS() && isCompLuma : tu.cu->sps->getUseInterMTS() && isCuInter;
363
54.5k
  const bool isSBT         = isCuInter && tu.cu->sbtInfo();
364
365
54.5k
  if( isSBT )
366
0
  {
367
0
    const uint8_t sbtIdx = CU::getSbtIdx( *tu.cu );
368
0
    const uint8_t sbtPos = CU::getSbtPos( *tu.cu );
369
370
0
    if( sbtIdx == SBT_VER_HALF || sbtIdx == SBT_VER_QUAD )
371
0
    {
372
0
      CHECK( lwidth > MTS_INTER_MAX_CU_SIZE, "wrong" );
373
0
      if( lheight > MTS_INTER_MAX_CU_SIZE )
374
0
      {
375
0
        trTypeHor = trTypeVer = DCT2;
376
0
      }
377
0
      else
378
0
      {
379
0
        if( sbtPos == SBT_POS0 )  { trTypeHor = DCT8;  trTypeVer = DST7; }
380
0
        else                      { trTypeHor = DST7;  trTypeVer = DST7; }
381
0
      }
382
0
    }
383
0
    else
384
0
    {
385
0
      CHECK( lheight > MTS_INTER_MAX_CU_SIZE, "wrong" );
386
0
      if( lwidth > MTS_INTER_MAX_CU_SIZE )
387
0
      {
388
0
        trTypeHor = trTypeVer = DCT2;
389
0
      }
390
0
      else
391
0
      {
392
0
        if( sbtPos == SBT_POS0 )  { trTypeHor = DST7;  trTypeVer = DCT8; }
393
0
        else                      { trTypeHor = DST7;  trTypeVer = DST7; }
394
0
      }
395
0
    }
396
0
    return;
397
0
  }
398
54.5k
  else if( isExplicitMTS )
399
0
  {
400
0
    if (tu.mtsIdx( compID ) > MTS_SKIP)
401
0
    {
402
0
      int indHor = (tu.mtsIdx( compID ) - MTS_DST7_DST7) & 1;
403
0
      int indVer = (tu.mtsIdx( compID ) - MTS_DST7_DST7) >> 1;
404
0
      trTypeHor = indHor ? DCT8 : DST7;
405
0
      trTypeVer = indVer ? DCT8 : DST7;
406
0
    }
407
0
  }
408
54.5k
}
409
410
void TrQuant::xIT( const TransformUnit &tu, const ComponentID &compID, const CCoeffBuf &pCoeff, PelBuf &pResidual )
411
74.6k
{
412
74.6k
  const int      width                  = pCoeff.width;
413
74.6k
  const int      height                 = pCoeff.height;
414
74.6k
  const unsigned maxLog2TrDynamicRange  = tu.cu->sps->getMaxLog2TrDynamicRange( toChannelType( compID ) );
415
74.6k
  const unsigned bitDepth               = tu.cu->sps->getBitDepth();
416
74.6k
  const int      TRANSFORM_MATRIX_SHIFT = g_transformMatrixShift;
417
74.6k
  const TCoeff   clipMinimum            = -( 1 << maxLog2TrDynamicRange );
418
74.6k
  const TCoeff   clipMaximum            =  ( 1 << maxLog2TrDynamicRange ) - 1;
419
74.6k
  const uint32_t transformWidthIndex    = getLog2(width ) - 1;                                // nLog2WidthMinus1, since transform start from 2-point
420
74.6k
  const uint32_t transformHeightIndex   = getLog2(height) - 1;                                // nLog2HeightMinus1, since transform start from 2-point
421
422
74.6k
  int trTypeHor = DCT2;
423
74.6k
  int trTypeVer = DCT2;
424
425
74.6k
  getTrTypes( tu, compID, trTypeHor, trTypeVer );
426
427
74.6k
  if( tu.maxScanPosX[compID] == 0 && tu.maxScanPosY[compID] == 0 && trTypeHor == DCT2 && trTypeVer == DCT2 )
428
14.0k
  {
429
14.0k
    int dcVal = 0;
430
431
14.0k
    if( width > 1 && height > 1 )
432
13.9k
    {
433
13.9k
      const int shift_1st = TRANSFORM_MATRIX_SHIFT + 1 + COM16_C806_TRANS_PREC;
434
13.9k
      const int shift_2nd = ( TRANSFORM_MATRIX_SHIFT + maxLog2TrDynamicRange - 1 ) - bitDepth + COM16_C806_TRANS_PREC;
435
436
13.9k
      dcVal = ( ( pCoeff.buf[0] * ( 1 << TRANSFORM_MATRIX_SHIFT ) ) + ( 1 << ( shift_1st - 1 ) ) ) >> shift_1st;
437
13.9k
      dcVal = ( ( dcVal * ( 1 << TRANSFORM_MATRIX_SHIFT ) ) + ( 1 << ( shift_2nd - 1 ) ) ) >> shift_2nd;
438
13.9k
    }
439
74
    else
440
74
    {
441
74
      const int shift = ( TRANSFORM_MATRIX_SHIFT + maxLog2TrDynamicRange ) - bitDepth + COM16_C806_TRANS_PREC;
442
74
      dcVal = ( ( pCoeff.buf[0] * ( 1 << TRANSFORM_MATRIX_SHIFT ) ) + ( 1 << ( shift - 1 ) ) ) >> shift;
443
74
    }
444
445
14.0k
    pResidual.fill( dcVal );
446
14.0k
    return;
447
14.0k
  }
448
449
60.6k
  const int skipWidth  = std::max<int>( ( trTypeHor != DCT2 && width  == 32 ) ? 16 : width  > JVET_C0024_ZERO_OUT_TH ? width  - JVET_C0024_ZERO_OUT_TH : 0, width  - tu.maxScanPosX[compID] - 1 );
450
60.6k
  const int skipHeight = std::max<int>( ( trTypeVer != DCT2 && height == 32 ) ? 16 : height > JVET_C0024_ZERO_OUT_TH ? height - JVET_C0024_ZERO_OUT_TH : 0, height - tu.maxScanPosY[compID] - 1 );
451
452
60.6k
  TCoeff *block = m_blk;
453
60.6k
  int shiftlast;
454
455
60.6k
  if( width > 1 && height > 1 ) //2-D transform
456
60.3k
  {
457
60.3k
    const int      shift_1st              =   TRANSFORM_MATRIX_SHIFT + 1 + COM16_C806_TRANS_PREC; // 1 has been added to shift_1st at the expense of shift_2nd
458
60.3k
    const int      shift_2nd              = ( TRANSFORM_MATRIX_SHIFT + maxLog2TrDynamicRange - 1 ) - bitDepth + COM16_C806_TRANS_PREC;
459
60.3k
    CHECK( shift_1st < 0, "Negative shift" );
460
60.3k
    CHECK( shift_2nd < 0, "Negative shift" );
461
60.3k
    TCoeff *tmp   = m_tmp;
462
60.3k
    fastInvTrans[trTypeVer][transformHeightIndex](pCoeff.buf, tmp, shift_1st, width, skipWidth, skipHeight, true,  clipMinimum, clipMaximum);
463
60.3k
    fastInvTrans[trTypeHor][transformWidthIndex] (tmp,      block, shift_2nd, height,         0, skipWidth, false, clipMinimum, clipMaximum);
464
60.3k
    shiftlast = shift_2nd;
465
60.3k
  }
466
343
  else if( width == 1 ) //1-D vertical transform
467
126
  {
468
126
    int shift = ( TRANSFORM_MATRIX_SHIFT + maxLog2TrDynamicRange - 1 ) - bitDepth + COM16_C806_TRANS_PREC;
469
126
    CHECK( shift < 0, "Negative shift" );
470
126
    CHECK( ( transformHeightIndex < 0 ), "There is a problem with the height." );
471
126
    fastInvTrans[trTypeVer][transformHeightIndex]( pCoeff.buf, block, shift + 1, 1, 0, skipHeight, false, clipMinimum, clipMaximum );
472
126
    shiftlast = shift + 1;
473
126
  }
474
217
  else //if(iHeight == 1) //1-D horizontal transform
475
217
  {
476
217
    const int      shift              = ( TRANSFORM_MATRIX_SHIFT + maxLog2TrDynamicRange - 1 ) - bitDepth + COM16_C806_TRANS_PREC;
477
217
    CHECK( shift < 0, "Negative shift" );
478
217
    CHECK( ( transformWidthIndex < 0 ), "There is a problem with the width." );
479
217
    fastInvTrans[trTypeHor][transformWidthIndex]( pCoeff.buf, block, shift + 1, 1, 0, skipWidth, false, clipMinimum, clipMaximum );
480
217
    shiftlast = shift + 1;
481
217
  }
482
483
60.6k
  int round = 1 << ( shiftlast - 1 );
484
60.6k
  g_tCoeffOps.cpyResiClip[getLog2( width )]( block, pResidual.buf, pResidual.stride, width, height, clipMinimum, clipMaximum, round, shiftlast );
485
60.6k
}
486
487
/** Wrapper function between HM interface and core NxN transform skipping
488
 */
489
void TrQuant::xITransformSkip(const CCoeffBuf     &pCoeff,
490
                                    PelBuf        &pResidual,
491
                              const TransformUnit &tu,
492
                              const ComponentID   &compID)
493
3.67k
{
494
3.67k
  const CompArea &area  = tu.blocks[compID];
495
3.67k
  const int width       = area.width;
496
3.67k
  const int height      = area.height;
497
  
498
42.3k
  for( int y = 0; y < height; y++ )
499
38.6k
  {
500
483k
    for( int x = 0; x < width; x++ )
501
445k
    {
502
445k
      pResidual.at( x, y ) = Pel( pCoeff.at( x, y ) );
503
445k
    }
504
38.6k
  }
505
3.67k
}
506
507
}