Coverage Report

Created: 2026-10-17 06:56

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/work/vvenc/source/Lib/CommonLib/Quant.cpp
Line
Count
Source
1
/* -----------------------------------------------------------------------------
2
The copyright in this software is being made available under the Clear BSD
3
License, included below. No patent rights, trademark rights and/or 
4
other Intellectual Property Rights other than the copyrights concerning 
5
the Software are granted under this license.
6
7
The Clear BSD License
8
9
Copyright (c) 2019-2026, Fraunhofer-Gesellschaft zur Förderung der angewandten Forschung e.V. & The VVenC Authors.
10
All rights reserved.
11
12
Redistribution and use in source and binary forms, with or without modification,
13
are permitted (subject to the limitations in the disclaimer below) provided that
14
the following conditions are met:
15
16
     * Redistributions of source code must retain the above copyright notice,
17
     this list of conditions and the following disclaimer.
18
19
     * Redistributions in binary form must reproduce the above copyright
20
     notice, this list of conditions and the following disclaimer in the
21
     documentation and/or other materials provided with the distribution.
22
23
     * Neither the name of the copyright holder nor the names of its
24
     contributors may be used to endorse or promote products derived from this
25
     software without specific prior written permission.
26
27
NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY
28
THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
29
CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
30
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
31
PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
32
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
33
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
34
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
35
BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
36
IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
37
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
38
POSSIBILITY OF SUCH DAMAGE.
39
40
41
------------------------------------------------------------------------------------------- */
42
43
44
/** \file     Quant.cpp
45
    \brief    transform and quantization class
46
*/
47
48
#include "Quant.h"
49
#include "UnitTools.h"
50
#include "ContextModelling.h"
51
#include "CodingStructure.h"
52
#include "dtrace_buffer.h"
53
54
#include <stdlib.h>
55
#include <memory.h>
56
57
//! \ingroup CommonLib
58
//! \{
59
60
namespace vvenc {
61
62
// ====================================================================================================================
63
// Constants
64
// ====================================================================================================================
65
66
67
// ====================================================================================================================
68
// QpParam constructor
69
// ====================================================================================================================
70
 
71
QpParam::QpParam(const TransformUnit& tu, const ComponentID &compID, const bool allowACTQpoffset)
72
5.97M
{
73
5.97M
  const ChannelType chType = toChannelType( compID );
74
5.97M
  const SPS        &sps    = *tu.cu->cs->sps;
75
5.97M
  const int     qpBdOffset = sps.qpBDOffset[chType];
76
5.97M
  const bool useJQP        = isChroma( compID ) && abs( TU::getICTMode( tu ) ) == 2;
77
5.97M
  const ComponentID jCbCr  = useJQP ? COMP_JOINT_CbCr : compID;
78
  
79
5.97M
        int chromaQpOffset = 0;
80
81
5.97M
  if( isChroma( compID ) )
82
5.76M
  {
83
5.76M
    const PPS &pps  = *tu.cu->slice->pps;
84
5.76M
    chromaQpOffset  = pps.chromaQpOffset              [jCbCr];
85
5.76M
    chromaQpOffset += tu.cu->slice->sliceChromaQpDelta[jCbCr];
86
5.76M
    chromaQpOffset += pps.getChromaQpOffsetListEntry( tu.cu->chromaQpAdj ).u.offset[int( jCbCr ) - 1];
87
5.76M
  }
88
  
89
5.97M
  int baseQp;
90
5.97M
  int qpy        = tu.cu->qp;
91
  //bool skip      = tu.mtsIdx[compID] == MTS_SKIP;
92
93
5.97M
  if( isLuma( compID ) )
94
206k
  {
95
206k
    baseQp = tu.cu->qp + qpBdOffset;
96
206k
  }
97
5.76M
  else
98
5.76M
  {
99
5.76M
    int qpi = Clip3( -qpBdOffset, MAX_QP, qpy );
100
5.76M
    baseQp  = sps.chromaQpMappingTable.getMappedChromaQpValue( jCbCr, qpi );
101
5.76M
    baseQp  = Clip3( -qpBdOffset, MAX_QP, baseQp + chromaQpOffset ) + qpBdOffset;
102
5.76M
  }
103
104
5.97M
  if( allowACTQpoffset && tu.cu->colorTransform )
105
0
  {
106
0
    baseQp += DELTA_QP_ACT[jCbCr];
107
0
  }
108
109
5.97M
  baseQp = Clip3( 0, MAX_QP + qpBdOffset, baseQp );
110
111
  //if( !skip )
112
5.97M
  {
113
5.97M
    Qps [0] = baseQp;
114
5.97M
    pers[0] = baseQp / 6;
115
5.97M
    rems[0] = baseQp % 6;
116
5.97M
  }
117
  //else
118
5.97M
  {
119
5.97M
    int internalMinusInputBitDepth = sps.internalMinusInputBitDepth[chType];
120
5.97M
    int baseQpTS           = std::max( baseQp, 4 + 6 * internalMinusInputBitDepth );
121
122
5.97M
    Qps [1] = baseQpTS;
123
5.97M
    pers[1] = baseQpTS / 6;
124
5.97M
    rems[1] = baseQpTS % 6;
125
5.97M
  }
126
5.97M
}
127
128
129
// ====================================================================================================================
130
// Quant class member functions
131
// ====================================================================================================================
132
static void QuantCore(const TransformUnit tu, const ComponentID compID, const CCoeffBuf& piCoef,CoeffSigBuf piQCoef,TCoeff &uiAbsSum, int &lastScanPos,TCoeff *deltaU,const int defaultQuantisationCoefficient,const int iQBits,const int64_t iAdd,const TCoeff entropyCodingMinimum,const TCoeff entropyCodingMaximum,const bool signHiding, const TCoeff m_thrVal)
133
5.63k
{
134
5.63k
  CoeffCodingContext cctx( tu, compID, signHiding );
135
136
5.63k
  const CompArea &rect      = tu.blocks[compID];
137
5.63k
  const uint32_t uiWidth    = rect.width;
138
5.63k
  const uint32_t uiHeight   = rect.height;
139
140
  /* for 422 chroma blocks, the effective scaling applied during transformation is not a power of 2, hence it cannot be
141
  * implemented as a bit-shift (the quantised result will be sqrt(2) * larger than required). Alternatively, adjust the
142
  * uiLog2TrSize applied in iTransformShift, such that the result is 1/sqrt(2) the required result (i.e. smaller)
143
  * Then a QP+3 (sqrt(2)) or QP-3 (1/sqrt(2)) method could be used to get the required result
144
  */
145
146
5.63k
  const uint32_t log2CGSize         = cctx.log2CGSize();
147
148
5.63k
  uiAbsSum = 0;
149
150
5.63k
  const int iCGSize   = 1 << log2CGSize;
151
152
5.63k
  const uint32_t lfnstIdx = tu.cu->lfnstIdx;
153
5.63k
  const int iCGNum   = lfnstIdx > 0 ? 1 : std::min<int>(JVET_C0024_ZERO_OUT_TH, uiWidth) * std::min<int>(JVET_C0024_ZERO_OUT_TH, uiHeight) >> cctx.log2CGSize();
154
5.63k
  int       iScanPos = ( iCGNum << log2CGSize ) - 1;
155
156
5.63k
  if( lfnstIdx > 0 && ( ( uiWidth == 4 && uiHeight == 4 ) || ( uiWidth == 8 && uiHeight == 8 ) ) )
157
0
  {
158
0
    iScanPos = 7;
159
0
  }
160
161
  // Find first non-zero coeff
162
31.4k
  for( ; iScanPos > 0; iScanPos-- )
163
30.0k
  {
164
30.0k
    uint32_t uiBlkPos = cctx.blockPos( iScanPos );
165
30.0k
    if( piCoef.buf[uiBlkPos] )
166
4.24k
      break;
167
30.0k
  }
168
169
  //////////////////////////////////////////////////////////////////////////
170
  //  Loop over sub-sets (coefficient groups)
171
  //////////////////////////////////////////////////////////////////////////
172
  
173
5.63k
  TCoeff thres = 0, useThres = 0;
174
  
175
5.63k
  if( iQBits )
176
5.63k
    thres = TCoeff( ( int64_t( m_thrVal ) << ( iQBits - 1 ) ) );
177
0
  else
178
0
    thres = TCoeff( ( int64_t( m_thrVal >> 1 ) << iQBits ) );
179
180
5.63k
  useThres = thres / ( defaultQuantisationCoefficient << 2 );
181
182
5.63k
  const bool is4x4sbb = log2CGSize == 4 && cctx.log2CGWidth() == 2;
183
184
5.63k
  int subSetId = iScanPos >> log2CGSize;
185
6.52k
  for( ; subSetId >= 1; subSetId-- )
186
890
  {
187
890
    if( is4x4sbb && iScanPos >= 16 )
188
0
    {
189
0
      int  iScanPosinCG = iScanPos & ( iCGSize - 1 );
190
0
      bool allSmaller   = true;
191
192
0
      for( int xScanPosinCG = iScanPosinCG, xScanPos = iScanPos; allSmaller && xScanPosinCG >= 0; xScanPosinCG--, xScanPos-- )
193
0
      {
194
0
        const uint32_t uiBlkPos = cctx.blockPos( xScanPos );
195
0
        allSmaller &= abs( piCoef.buf[uiBlkPos] ) <= useThres;
196
0
      }
197
198
0
      if( allSmaller )
199
0
      {
200
0
        iScanPos    -= iScanPosinCG + 1;
201
0
        continue;
202
0
      }
203
0
      else
204
0
      {
205
0
        break;
206
0
      }
207
0
    }
208
890
  }
209
210
5.63k
  const int qBits8 = iQBits - 8;
211
5.63k
  piQCoef.memset( 0 );
212
213
89.1k
  for( int currPos = 0; currPos <= iScanPos; currPos++ )
214
83.5k
  {
215
83.5k
    const int uiBlockPos  = cctx.blockPos( currPos );
216
83.5k
    const TCoeff iLevel   = piCoef.buf[uiBlockPos];
217
83.5k
    const TCoeff iSign    = (iLevel < 0 ? -1: 1);
218
219
83.5k
    const int64_t  tmpLevel = (int64_t)abs(iLevel) * defaultQuantisationCoefficient;
220
83.5k
    const TCoeff quantisedMagnitude = TCoeff((tmpLevel + iAdd ) >> iQBits);
221
83.5k
    deltaU[uiBlockPos] = (TCoeff)((tmpLevel - ((int64_t)quantisedMagnitude<<iQBits) )>> qBits8);
222
223
83.5k
    uiAbsSum += quantisedMagnitude;
224
83.5k
    const TCoeff quantisedCoefficient = quantisedMagnitude * iSign;
225
226
83.5k
    piQCoef.buf[uiBlockPos] = Clip3<TCoeff>( entropyCodingMinimum, entropyCodingMaximum, quantisedCoefficient );
227
83.5k
  } // for n
228
229
5.63k
  lastScanPos = iScanPos;
230
5.63k
}
231
232
static void DeQuantCore(const int maxX,const int maxY,const int scale,const TCoeffSig* const piQCoef,const size_t piQCfStride,TCoeff   *const piCoef,const int rightShift,const int inputMaximum,const TCoeff transformMaximum)
233
46.2k
{
234
46.2k
  const int inputMinimum = -(inputMaximum+1);
235
46.2k
  const TCoeff transformMinimum = -(transformMaximum+1);
236
46.2k
  if (rightShift>0)
237
36.9k
  {
238
36.9k
    const Intermediate_Int iAdd = (Intermediate_Int) 1 << (rightShift - 1);
239
350k
    for( int y = 0, n = 0; y <= maxY; y++)
240
313k
    {
241
3.35M
      for( int x = 0; x <= maxX; x++, n++ )
242
3.03M
      {
243
3.03M
        const TCoeff           clipQCoef = TCoeff(Clip3<Intermediate_Int>(inputMinimum, inputMaximum, piQCoef[x + y * piQCfStride]));
244
3.03M
        Intermediate_Int iCoeffQ   = (Intermediate_Int(clipQCoef) * scale + iAdd) >> rightShift;
245
3.03M
        piCoef[n] = TCoeff(Clip3<Intermediate_Int>(transformMinimum,transformMaximum,iCoeffQ));
246
3.03M
      }
247
313k
    }
248
36.9k
  }
249
9.35k
  else  // rightshift <0
250
9.35k
  {
251
9.35k
    int leftShift = -rightShift;
252
112k
    for( int y = 0, n = 0; y <= maxY; y++)
253
102k
    {
254
1.23M
      for( int x = 0; x <= maxX; x++, n++ )
255
1.13M
      {
256
1.13M
        const TCoeff           clipQCoef = TCoeff(Clip3<Intermediate_Int>(inputMinimum, inputMaximum, piQCoef[x + y * piQCfStride]));
257
1.13M
        const Intermediate_Int iCoeffQ   = (Intermediate_Int(clipQCoef) * scale) * (1 << leftShift);
258
1.13M
        piCoef[n] = TCoeff(Clip3<Intermediate_Int>(transformMinimum,transformMaximum,iCoeffQ));
259
1.13M
      }
260
102k
    }
261
9.35k
  }
262
46.2k
}
263
264
static bool needRdoqCore( const TCoeff* pCoeff, size_t numCoeff, int quantCoeff, int64_t offset, int shift )
265
0
{
266
0
  for( int uiBlockPos = 0; uiBlockPos < numCoeff; uiBlockPos++ )
267
0
  {
268
0
    const TCoeff   iLevel = pCoeff[uiBlockPos];
269
0
    const int64_t  tmpLevel = ( int64_t ) std::abs( iLevel ) * quantCoeff;
270
0
    const TCoeff quantisedMagnitude = TCoeff( ( tmpLevel + offset ) >> shift );
271
272
0
    if( quantisedMagnitude != 0 )
273
0
    {
274
0
      return true;
275
0
    }
276
0
  } // for n
277
0
  return false;
278
0
}
279
280
281
4.62k
Quant::Quant( const Quant* other, bool useScalingLists, bool enableOpt ) : m_RDOQ( 0 ), m_useRDOQTS( false ), m_dLambda( 0.0 )
282
4.62k
{
283
4.62k
  xInitScalingList( other, useScalingLists );
284
4.62k
  xDeQuant  = DeQuantCore;
285
4.62k
  xQuant    = QuantCore;
286
4.62k
  xNeedRdoq = needRdoqCore;
287
4.62k
  if( enableOpt )
288
4.62k
  {
289
#if defined( TARGET_SIMD_X86 ) && ENABLE_SIMD_OPT_QUANT
290
    initQuantX86();
291
#endif
292
#if defined( TARGET_SIMD_ARM ) && ENABLE_SIMD_OPT_QUANT
293
    initQuantARM();
294
#endif
295
4.62k
  }
296
4.62k
}
297
298
Quant::~Quant()
299
4.62k
{
300
4.62k
  xDestroyScalingList();
301
4.62k
}
302
303
void invResDPCM( const TransformUnit& tu, const ComponentID compID, CoeffSigBuf& dstBuf )
304
46.2k
{
305
46.2k
  const CompArea&    rect   = tu.blocks[compID];
306
46.2k
  const int          wdt    = rect.width;
307
46.2k
  const int          hgt    = rect.height;
308
46.2k
  const CCoeffSigBuf coeffs = tu.getCoeffs(compID);
309
310
46.2k
  const int      maxLog2TrDynamicRange = tu.cs->sps->getMaxLog2TrDynamicRange();
311
46.2k
  const TCoeff   inputMinimum          = -(1 << maxLog2TrDynamicRange);
312
46.2k
  const TCoeff   inputMaximum          =  (1 << maxLog2TrDynamicRange) - 1;
313
314
46.2k
  const TCoeffSig* coef = &coeffs.buf[0];
315
46.2k
        TCoeffSig* dst  = &dstBuf.buf[0];
316
317
46.2k
  if ( tu.cu->bdpcmM[toChannelType(compID)] == 1)
318
518
  {
319
7.47k
    for( int y = 0; y < hgt; y++ )
320
6.96k
    {
321
6.96k
      dst[0] = coef[0];
322
111k
      for( int x = 1; x < wdt; x++ )
323
104k
      {
324
104k
        dst[x] = Clip3(inputMinimum, inputMaximum, TCoeff( dst[x - 1] ) + TCoeff( coef[x] ));
325
104k
      }
326
6.96k
      coef += coeffs.stride;
327
6.96k
      dst += dstBuf.stride;
328
6.96k
    }
329
518
  }
330
45.7k
  else
331
45.7k
  {
332
481k
    for( int x = 0; x < wdt; x++ )
333
435k
    {
334
435k
      dst[x] = coef[x];
335
435k
    }
336
409k
    for( int y = 0; y < hgt - 1; y++ )
337
363k
    {
338
3.98M
      for( int x = 0; x < wdt; x++ )
339
3.62M
      {
340
3.62M
        dst[dstBuf.stride + x] = Clip3(inputMinimum, inputMaximum, TCoeff( dst[x] ) + TCoeff( coef[coeffs.stride + x] ));
341
3.62M
      }
342
363k
      coef += coeffs.stride;
343
363k
      dst += dstBuf.stride;
344
363k
    }
345
45.7k
  }
346
46.2k
}
347
348
void fwdResDPCM( TransformUnit& tu, const ComponentID compID )
349
5.63k
{
350
5.63k
  const CompArea& rect   = tu.blocks[compID];
351
5.63k
  const int       wdt    = rect.width;
352
5.63k
  const int       hgt    = rect.height;
353
5.63k
  CoeffSigBuf     coeffs = tu.getCoeffs(compID);
354
355
5.63k
  TCoeffSig* coef = &coeffs.buf[0];
356
5.63k
  if (tu.cu->bdpcmM[toChannelType(compID)] == 1)
357
0
  {
358
0
    for( int y = 0; y < hgt; y++ )
359
0
    {
360
0
      for( int x = wdt - 1; x > 0; x-- )
361
0
      {
362
0
        coef[x] -= coef[x - 1];
363
0
      }
364
0
      coef += coeffs.stride;
365
0
    }
366
0
  }
367
5.63k
  else
368
5.63k
  {
369
5.63k
    coef += coeffs.stride * (hgt - 1);
370
11.2k
    for( int y = 0; y < hgt - 1; y++ )
371
5.63k
    {
372
60.3k
      for ( int x = 0; x < wdt; x++ )
373
54.6k
      {
374
54.6k
        coef[x] -= coef[x - coeffs.stride];
375
54.6k
      }
376
5.63k
      coef -= coeffs.stride;
377
5.63k
    }
378
5.63k
  }
379
5.63k
}
380
381
// To minimize the distortion only. No rate is considered.
382
void Quant::xSignBitHidingHDQ( TCoeffSig* pQCoef, const TCoeff* pCoef, TCoeff* deltaU, const CoeffCodingContext& cctx, int& lastScanPos, const int maxLog2TrDynamicRange )
383
0
{
384
0
  const uint32_t groupSize = 1 << cctx.log2CGSize();
385
386
0
  const TCoeff entropyCodingMinimum = -(1 << maxLog2TrDynamicRange);
387
0
  const TCoeff entropyCodingMaximum =  (1 << maxLog2TrDynamicRange) - 1;
388
389
0
  int lastCG = -1;
390
0
  int absSum = 0 ;
391
0
  int n ;
392
393
0
  for( int subSet = lastScanPos >> cctx.log2CGSize(); subSet >= 0; subSet-- )
394
0
  {
395
0
    int  subPos = subSet << cctx.log2CGSize();
396
0
    int  firstNZPosInCG=groupSize , lastNZPosInCG=-1 ;
397
0
    absSum = 0 ;
398
399
0
    for(n = groupSize-1; n >= 0; --n )
400
0
    {
401
0
      if( pQCoef[ cctx.blockPos( n + subPos ) ] )
402
0
      {
403
0
        lastNZPosInCG = n;
404
0
        break;
405
0
      }
406
0
    }
407
408
0
    for(n = 0; n <groupSize; n++ )
409
0
    {
410
0
      if( pQCoef[ cctx.blockPos( n + subPos ) ] )
411
0
      {
412
0
        firstNZPosInCG = n;
413
0
        break;
414
0
      }
415
0
    }
416
417
0
    for(n = firstNZPosInCG; n <=lastNZPosInCG; n++ )
418
0
    {
419
0
      absSum += int(pQCoef[ cctx.blockPos( n + subPos ) ]);
420
0
    }
421
422
0
    if(lastNZPosInCG>=0 && lastCG==-1)
423
0
    {
424
0
      lastCG = 1 ;
425
0
    }
426
427
0
    if( lastNZPosInCG-firstNZPosInCG>=SBH_THRESHOLD )
428
0
    {
429
0
      uint32_t signbit = (pQCoef[cctx.blockPos(subPos+firstNZPosInCG)]>0?0:1) ;
430
0
      if( signbit!=(absSum&0x1) )  //compare signbit with sum_parity
431
0
      {
432
0
        TCoeff curCost    = std::numeric_limits<TCoeff>::max();
433
0
        TCoeff minCostInc = std::numeric_limits<TCoeff>::max();
434
0
        int minPos =-1, finalChange=0, curChange=0, minScanPos = -1;
435
436
0
        for( n = (lastCG==1?lastNZPosInCG:groupSize-1) ; n >= 0; --n )
437
0
        {
438
0
          uint32_t blkPos   = cctx.blockPos( n+subPos );
439
0
          if(pQCoef[ blkPos ] != 0 )
440
0
          {
441
0
            if(deltaU[blkPos]>0)
442
0
            {
443
0
              curCost = - deltaU[blkPos];
444
0
              curChange=1 ;
445
0
            }
446
0
            else
447
0
            {
448
              //curChange =-1;
449
0
              if(n==firstNZPosInCG && abs(pQCoef[blkPos])==1)
450
0
              {
451
0
                curCost = std::numeric_limits<TCoeff>::max();
452
0
              }
453
0
              else
454
0
              {
455
0
                curCost = deltaU[blkPos];
456
0
                curChange =-1;
457
0
              }
458
0
            }
459
0
          }
460
0
          else
461
0
          {
462
0
            if(n<firstNZPosInCG)
463
0
            {
464
0
              uint32_t thisSignBit = (pCoef[blkPos]>=0?0:1);
465
0
              if(thisSignBit != signbit )
466
0
              {
467
0
                curCost = std::numeric_limits<TCoeff>::max();
468
0
              }
469
0
              else
470
0
              {
471
0
                curCost = - (deltaU[blkPos])  ;
472
0
                curChange = 1 ;
473
0
              }
474
0
            }
475
0
            else
476
0
            {
477
0
              curCost = - (deltaU[blkPos])  ;
478
0
              curChange = 1 ;
479
0
            }
480
0
          }
481
482
0
          if( curCost<minCostInc)
483
0
          {
484
0
            minCostInc = curCost ;
485
0
            finalChange = curChange ;
486
0
            minPos = blkPos;
487
0
            minScanPos = n + subPos;
488
0
          }
489
0
        } //CG loop
490
491
0
        if(pQCoef[minPos] == entropyCodingMaximum || pQCoef[minPos] == entropyCodingMinimum)
492
0
        {
493
0
          finalChange = -1;
494
0
        }
495
496
0
        if(pCoef[minPos]>=0)
497
0
        {
498
0
          pQCoef[minPos] += finalChange ;
499
0
        }
500
0
        else
501
0
        {
502
0
          pQCoef[minPos] -= finalChange ;
503
0
        }
504
505
        // if changing lastScanPos element to 0, move the pointer to the new lastScanPos element
506
0
        if( minScanPos == lastScanPos && pQCoef[minPos] == 0 )
507
0
        {
508
0
          for( ; lastScanPos >= 0 && pQCoef[cctx.blockPos( lastScanPos )] == 0; lastScanPos-- );
509
0
        }
510
0
        else if( minScanPos > lastScanPos && pQCoef[minPos] != 0 )
511
0
        {
512
0
          lastScanPos = minPos;
513
0
        }
514
0
      } // Hide
515
0
    }
516
0
    if(lastCG==1)
517
0
    {
518
0
      lastCG=0 ;
519
0
    }
520
0
  } // TU loop
521
522
0
  return;
523
0
}
524
525
void Quant::dequant(const TransformUnit& tu,
526
                          CoeffBuf&      dstCoeff,
527
                    const ComponentID    compID,
528
                    const QpParam&       cQP)
529
46.2k
{
530
46.2k
  const SPS       *sps                  = tu.cs->sps;
531
46.2k
  const CompArea  &area                 = tu.blocks[compID];
532
46.2k
  const uint32_t  uiWidth               = area.width;
533
46.2k
  const uint32_t  uiHeight              = area.height;
534
46.2k
  TCoeff *const   piCoef                = dstCoeff.buf;
535
46.2k
  const uint32_t  numSamplesInBlock     = uiWidth * uiHeight;
536
46.2k
  const int       maxLog2TrDynamicRange = sps->getMaxLog2TrDynamicRange();
537
46.2k
  const TCoeff    transformMinimum      = -(1 << maxLog2TrDynamicRange);
538
46.2k
  const TCoeff    transformMaximum      =  (1 << maxLog2TrDynamicRange) - 1;
539
46.2k
  const bool      isTransformSkip       = tu.mtsIdx[compID] == MTS_SKIP;
540
46.2k
  const bool      isLfnstApplied        = tu.cu->lfnstIdx > 0 && (CU::isSepTree(*tu.cu) ? true : isLuma(compID));
541
46.2k
  const bool      enableScalingLists    = getUseScalingList(uiWidth, uiHeight, isTransformSkip, isLfnstApplied);
542
46.2k
  const int       scalingListType       = getScalingListType(tu.cu->predMode, compID);
543
46.2k
  const int       channelBitDepth       = sps->bitDepths[toChannelType(compID)];
544
545
46.2k
  const TCoeffSig *coef     = tu.getCoeffs( compID ).buf;
546
46.2k
  const ptrdiff_t  piStride = tu.getCoeffs( compID ).stride;
547
548
46.2k
  if( tu.cu->bdpcmM[toChannelType( compID )] )
549
46.2k
  {
550
46.2k
    CoeffSigBuf coefBuf( m_tmpBdpcm, uiWidth, uiHeight );
551
46.2k
    invResDPCM( tu, compID, coefBuf );
552
46.2k
    coef      = m_tmpBdpcm;
553
46.2k
  }
554
555
46.2k
  const TCoeffSig  *const piQCoef = coef;
556
46.2k
  CHECK(scalingListType >= SCALING_LIST_NUM, "Invalid scaling list");
557
558
  // Represents scaling through forward transform
559
46.2k
  const int  originalTransformShift = getTransformShift(channelBitDepth, area.size(), maxLog2TrDynamicRange);
560
46.2k
  const bool needSqrtAdjustment     = TU::needsSqrt2Scale( tu, compID );
561
46.2k
  const int  iTransformShift        = originalTransformShift + (needSqrtAdjustment?-1:0);
562
563
46.2k
  const int QP_per = cQP.per(isTransformSkip);
564
46.2k
  const int QP_rem = cQP.rem(isTransformSkip);
565
566
46.2k
  const int  rightShift = (IQUANT_SHIFT - ((isTransformSkip ? 0 : iTransformShift) + QP_per)) + (enableScalingLists ? LOG2_SCALING_LIST_NEUTRAL_VALUE : 0);
567
568
46.2k
  if(enableScalingLists)
569
0
  {
570
    //from the dequantization equation:
571
    //iCoeffQ                         = ((Intermediate_Int(clipQCoef) * piDequantCoef[deQuantIdx]) + iAdd ) >> rightShift
572
    //(sizeof(Intermediate_Int) * 8)  =              inputBitDepth    +    dequantCoefBits                   - rightShift
573
0
    const uint32_t             dequantCoefBits     = 1 + IQUANT_SHIFT + SCALING_LIST_BITS;
574
0
    const uint32_t             targetInputBitDepth = std::min<uint32_t>((maxLog2TrDynamicRange + 1), (((sizeof(Intermediate_Int) * 8) + rightShift) - dequantCoefBits));
575
576
0
    const Intermediate_Int inputMinimum        = -(1 << (targetInputBitDepth - 1));
577
0
    const Intermediate_Int inputMaximum        =  (1 << (targetInputBitDepth - 1)) - 1;
578
579
0
    const uint32_t uiLog2TrWidth  = Log2(uiWidth);
580
0
    const uint32_t uiLog2TrHeight = Log2(uiHeight);
581
0
    int* piDequantCoef            = getDequantCoeff(scalingListType, QP_rem, uiLog2TrWidth, uiLog2TrHeight);
582
583
0
    if(rightShift > 0)
584
0
    {
585
0
      const Intermediate_Int iAdd = (Intermediate_Int) 1 << (rightShift - 1);
586
0
      for( int n = 0; n < numSamplesInBlock; n++ )
587
0
      {
588
0
        const TCoeff           clipQCoef = TCoeff(Clip3<Intermediate_Int>(inputMinimum, inputMaximum, piQCoef[n]));
589
0
        const Intermediate_Int iCoeffQ   = ((Intermediate_Int(clipQCoef) * piDequantCoef[n]) + iAdd ) >> rightShift;
590
0
        piCoef[n] = TCoeff(Clip3<Intermediate_Int>(transformMinimum,transformMaximum,iCoeffQ));
591
0
      }
592
0
    }
593
0
    else
594
0
    {
595
0
      const int leftShift = -rightShift;
596
0
      for( int n = 0; n < numSamplesInBlock; n++ )
597
0
      {
598
0
        const TCoeff           clipQCoef = TCoeff(Clip3<Intermediate_Int>(inputMinimum, inputMaximum, piQCoef[n]));
599
0
        const Intermediate_Int iCoeffQ   = (Intermediate_Int(clipQCoef) * piDequantCoef[n]) << leftShift;
600
0
        piCoef[n] = TCoeff(Clip3<Intermediate_Int>(transformMinimum,transformMaximum,iCoeffQ));
601
0
      }
602
0
    }
603
0
  }
604
46.2k
  else
605
46.2k
  {
606
46.2k
    const int scale     = g_invQuantScales[needSqrtAdjustment?1:0][QP_rem];
607
46.2k
    const int scaleBits = ( IQUANT_SHIFT + 1 );
608
    //from the dequantisation equation:
609
    //iCoeffQ                         = Intermediate_Int((int64_t(clipQCoef) * scale + iAdd) >> rightShift);
610
    //(sizeof(Intermediate_Int) * 8)  =                    inputBitDepth   + scaleBits      - rightShift
611
46.2k
    const uint32_t             targetInputBitDepth = std::min<uint32_t>((maxLog2TrDynamicRange + 1), (((sizeof(Intermediate_Int) * 8) + rightShift) - scaleBits));
612
46.2k
    const Intermediate_Int inputMaximum        =  (1 << (targetInputBitDepth - 1)) - 1;
613
46.2k
    xDeQuant(uiWidth-1,uiHeight-1,scale,piQCoef,piStride,piCoef,rightShift,inputMaximum,transformMaximum);
614
46.2k
  }
615
46.2k
}
616
617
void Quant::init( int rdoq, bool bUseRDOQTS, int thrVal )
618
4.62k
{
619
620
  // TODO: pass to init() a single variable containing (quantization) flags,
621
  //       instead of variables that don't have to do with this class
622
623
4.62k
  m_RDOQ             = rdoq;
624
4.62k
  m_useRDOQTS        = bUseRDOQTS;
625
4.62k
  m_thrVal           = thrVal;
626
4.62k
}
627
628
/** set flat matrix value to quantized coefficient
629
 */
630
void Quant::setFlatScalingList(const int maxLog2TrDynamicRange[MAX_NUM_CH], const BitDepths &bitDepths )
631
4.62k
{
632
4.62k
  if( !m_scalingListEnabled ) return;
633
634
0
  const int minimumQp = 0;
635
0
  const int maximumQp = SCALING_LIST_REM_NUM;
636
637
0
  for(uint32_t sizeX = 0; sizeX < SCALING_LIST_SIZE_NUM; sizeX++)
638
0
  {
639
0
    for(uint32_t sizeY = 0; sizeY < SCALING_LIST_SIZE_NUM; sizeY++)
640
0
    {
641
0
      for(uint32_t list = 0; list < SCALING_LIST_NUM; list++)
642
0
      {
643
0
        for(int qp = minimumQp; qp < maximumQp; qp++)
644
0
        {
645
0
          xSetFlatScalingList( list, sizeX, sizeY, qp );
646
0
        }
647
0
      }
648
0
    }
649
0
  }
650
0
}
651
652
/** set flat matrix value to quantized coefficient
653
 * \param list List ID
654
 * \param size size index
655
 * \param qp Quantization parameter
656
 * \param format chroma format
657
 */
658
void Quant::xSetFlatScalingList(uint32_t list, uint32_t sizeX, uint32_t sizeY, int qp )
659
0
{
660
0
  uint32_t i,num = g_scalingListSizeX[sizeX]*g_scalingListSizeX[sizeY];
661
0
  int *quantcoeff;
662
0
  int *dequantcoeff;
663
664
0
  const bool blockIsNotPowerOf4 = ((Log2(g_scalingListSizeX[sizeX] * g_scalingListSizeX[sizeY])) & 1) == 1;
665
0
  int quantScales    = g_quantScales   [blockIsNotPowerOf4?1:0][qp];
666
0
  int invQuantScales = g_invQuantScales[blockIsNotPowerOf4?1:0][qp] << 4;
667
668
0
  quantcoeff   = getQuantCoeff(list, qp, sizeX, sizeY);
669
0
  dequantcoeff = getDequantCoeff(list, qp, sizeX, sizeY);
670
671
0
  for(i=0;i<num;i++)
672
0
  {
673
0
    *quantcoeff++ = quantScales;
674
0
    *dequantcoeff++ = invQuantScales;
675
0
  }
676
0
}
677
678
679
/** initialization process of scaling list array
680
 */
681
void Quant::xInitScalingList( const Quant* other, bool useScalingLists )
682
4.62k
{
683
4.62k
  m_isScalingListOwner = other == nullptr;
684
4.62k
  m_scalingListEnabled = useScalingLists;
685
686
36.9k
  for(uint32_t sizeIdX = 0; sizeIdX < SCALING_LIST_SIZE_NUM; sizeIdX++)
687
32.3k
  {
688
258k
    for(uint32_t sizeIdY = 0; sizeIdY < SCALING_LIST_SIZE_NUM; sizeIdY++)
689
226k
    {
690
1.58M
      for(uint32_t qp = 0; qp < SCALING_LIST_REM_NUM; qp++)
691
1.35M
      {
692
9.51M
        for(uint32_t listId = 0; listId < SCALING_LIST_NUM; listId++)
693
8.15M
        {
694
8.15M
          if( m_isScalingListOwner )
695
2.03M
          {
696
2.03M
            const size_t scalingListSize = g_scalingListSizeX[sizeIdX] * g_scalingListSizeX[sizeIdY];
697
698
2.03M
            m_quantCoef   [sizeIdX][sizeIdY][listId][qp] = useScalingLists ? new int[scalingListSize] : nullptr;
699
2.03M
            m_dequantCoef [sizeIdX][sizeIdY][listId][qp] = useScalingLists ? new int[scalingListSize] : nullptr;
700
2.03M
          }
701
6.11M
          else
702
6.11M
          {
703
6.11M
            m_quantCoef   [sizeIdX][sizeIdY][listId][qp] = other->m_quantCoef   [sizeIdX][sizeIdY][listId][qp];
704
6.11M
            m_dequantCoef [sizeIdX][sizeIdY][listId][qp] = other->m_dequantCoef [sizeIdX][sizeIdY][listId][qp];
705
6.11M
          }
706
8.15M
        } // listID loop
707
1.35M
      }
708
226k
    }
709
32.3k
  }
710
4.62k
}
711
712
/** destroy quantization matrix array
713
 */
714
void Quant::xDestroyScalingList()
715
4.62k
{
716
4.62k
  if( !m_isScalingListOwner ) return;
717
718
9.24k
  for(uint32_t sizeIdX = 0; sizeIdX < SCALING_LIST_SIZE_NUM; sizeIdX++)
719
8.09k
  {
720
64.7k
    for(uint32_t sizeIdY = 0; sizeIdY < SCALING_LIST_SIZE_NUM; sizeIdY++)
721
56.6k
    {
722
396k
      for(uint32_t listId = 0; listId < SCALING_LIST_NUM; listId++)
723
339k
      {
724
2.37M
        for(uint32_t qp = 0; qp < SCALING_LIST_REM_NUM; qp++)
725
2.03M
        {
726
2.03M
          if(m_quantCoef[sizeIdX][sizeIdY][listId][qp])
727
0
          {
728
0
            delete [] m_quantCoef[sizeIdX][sizeIdY][listId][qp];
729
0
          }
730
2.03M
          if(m_dequantCoef[sizeIdX][sizeIdY][listId][qp])
731
0
          {
732
0
            delete [] m_dequantCoef[sizeIdX][sizeIdY][listId][qp];
733
0
          }
734
2.03M
        }
735
339k
      }
736
56.6k
    }
737
8.09k
  }
738
1.15k
}
739
740
void Quant::quant(TransformUnit& tu, const ComponentID compID, const CCoeffBuf& pSrc, TCoeff &uiAbsSum, const QpParam& cQP, const Ctx& ctx)
741
5.63k
{
742
5.63k
  const SPS &sps            = *tu.cs->sps;
743
5.63k
  const CompArea& rect      = tu.blocks[compID];
744
5.63k
  const uint32_t uiWidth    = rect.width;
745
5.63k
  const uint32_t uiHeight   = rect.height;
746
5.63k
  const int channelBitDepth = sps.bitDepths[toChannelType(compID)];
747
748
5.63k
  const CCoeffBuf&  piCoef  = pSrc;
749
5.63k
        CoeffSigBuf piQCoef = tu.getCoeffs(compID);
750
751
5.63k
  const bool useTransformSkip = tu.mtsIdx[compID] == MTS_SKIP;
752
5.63k
  const int  maxLog2TrDynamicRange = sps.getMaxLog2TrDynamicRange();
753
754
5.63k
  {
755
5.63k
    CoeffCodingContext cctx(tu, compID, tu.cs->slice->signDataHidingEnabled);
756
757
5.63k
    const TCoeff entropyCodingMinimum = -(1 << maxLog2TrDynamicRange);
758
5.63k
    const TCoeff entropyCodingMaximum =  (1 << maxLog2TrDynamicRange) - 1;
759
760
5.63k
    TCoeff deltaU[MAX_TB_SIZEY * MAX_TB_SIZEY];
761
5.63k
    int scalingListType           = getScalingListType(tu.cu->predMode, compID);
762
5.63k
    CHECK(scalingListType >= SCALING_LIST_NUM, "Invalid scaling list");
763
5.63k
    const uint32_t uiLog2TrWidth  = Log2(uiWidth);
764
5.63k
    const uint32_t uiLog2TrHeight = Log2(uiHeight);
765
5.63k
    int *piQuantCoeff             = getQuantCoeff(scalingListType, cQP.rem(useTransformSkip), uiLog2TrWidth, uiLog2TrHeight);
766
767
5.63k
    const bool isLfnstApplied     = tu.cu->lfnstIdx > 0 && (CU::isSepTree(*tu.cu) ? true : isLuma(compID));
768
5.63k
    const bool enableScalingLists = getUseScalingList(uiWidth, uiHeight, useTransformSkip, isLfnstApplied);
769
770
    // for blocks that where width*height != 4^N, the effective scaling applied during transformation cannot be
771
    // compensated by a bit-shift (the quantised result will be sqrt(2) * larger than required).
772
    // The quantScale table and shift is used to compensate for this.
773
5.63k
    const bool needSqrtAdjustment= TU::needsSqrt2Scale( tu, compID );
774
5.63k
    const int defaultQuantisationCoefficient    = g_quantScales[needSqrtAdjustment?1:0][cQP.rem(useTransformSkip)];
775
5.63k
    const int iTransformShift = getTransformShift(channelBitDepth, rect.size(), maxLog2TrDynamicRange) + ( needSqrtAdjustment?-1:0);
776
777
5.63k
    const int iQBits = QUANT_SHIFT + cQP.per(useTransformSkip) + (useTransformSkip ? 0 : iTransformShift);
778
    // QBits will be OK for any internal bit depth as the reduction in transform shift is balanced by an increase in Qp_per due to QpBDOffset
779
780
5.63k
    const int64_t iAdd = int64_t(tu.cs->slice->isIRAP() ? 171 : 85) << int64_t(iQBits - 9);
781
5.63k
    const int qBits8 = iQBits - 8;
782
783
5.63k
    int lastScanPos = -1;
784
785
5.63k
    if (!enableScalingLists)
786
5.63k
      xQuant(tu,compID,piCoef,piQCoef,uiAbsSum,lastScanPos,deltaU,defaultQuantisationCoefficient,iQBits,iAdd,entropyCodingMinimum,entropyCodingMaximum,cctx.signHiding(),m_thrVal);
787
0
    else
788
0
    {
789
0
      const uint32_t lfnstIdx = tu.cu->lfnstIdx;
790
0
      const int maxNumberOfCoeffs = lfnstIdx > 0 ? ( ( ( uiWidth == 4 && uiHeight == 4 ) || ( uiWidth == 8 && uiHeight == 8 ) ) ? 8 : 16 ) : piQCoef.area();
791
792
0
      piQCoef.memset( 0 );
793
0
      for (int uiScanPos = 0; uiScanPos < maxNumberOfCoeffs; uiScanPos++ )
794
0
      {
795
0
        const int uiBlockPos  = cctx.blockPos( uiScanPos );
796
0
        const TCoeff iLevel   = piCoef.buf[uiBlockPos];
797
0
        const TCoeff iSign    = (iLevel < 0 ? -1: 1);
798
799
0
        const int64_t  tmpLevel = (int64_t)abs(iLevel) * (enableScalingLists ? piQuantCoeff[uiBlockPos] : defaultQuantisationCoefficient);
800
0
        const TCoeff quantisedMagnitude = TCoeff((tmpLevel + iAdd ) >> iQBits);
801
0
        deltaU[uiBlockPos] = (TCoeff)((tmpLevel - ((int64_t)quantisedMagnitude<<iQBits) )>> qBits8);
802
803
0
        uiAbsSum += quantisedMagnitude;
804
0
        const TCoeff quantisedCoefficient = quantisedMagnitude * iSign;
805
806
0
        piQCoef.buf[uiBlockPos] = Clip3<TCoeff>( entropyCodingMinimum, entropyCodingMaximum, quantisedCoefficient );
807
0
      } // for n
808
0
    }
809
5.63k
    if (tu.cu->bdpcmM[toChannelType(compID)])
810
5.63k
    {
811
5.63k
      fwdResDPCM( tu, compID );
812
5.63k
    }
813
814
5.63k
    if( uiAbsSum )
815
3.38k
    {
816
6.76k
      for( int scanPos = lastScanPos; scanPos >= 0; scanPos-- )
817
6.76k
      {
818
6.76k
        unsigned blkPos = cctx.blockPos( scanPos );
819
6.76k
        if( piQCoef.buf[blkPos] )
820
3.38k
        {
821
3.38k
          lastScanPos = scanPos;
822
3.38k
          break;
823
3.38k
        }
824
6.76k
      }
825
826
3.38k
      if( cctx.signHiding() )
827
0
      {
828
0
        if( uiAbsSum >= 2 ) //this prevents TUs with only one coefficient of value 1 from being tested
829
0
        {
830
0
          xSignBitHidingHDQ( piQCoef.buf, piCoef.buf, deltaU, cctx, lastScanPos, maxLog2TrDynamicRange );
831
0
        }
832
0
      }
833
3.38k
    }
834
835
5.63k
    tu.lastPos[compID] = lastScanPos;
836
5.63k
  } //if RDOQ
837
  //return;
838
5.63k
}
839
840
bool Quant::xNeedRDOQ(TransformUnit& tu, const ComponentID compID, const CCoeffBuf& pSrc, const QpParam& cQP)
841
0
{
842
0
  const SPS &sps            = *tu.cs->sps;
843
0
  const CompArea& rect      = tu.blocks[compID];
844
0
  const uint32_t uiWidth    = rect.width;
845
0
  const uint32_t uiHeight   = rect.height;
846
0
  const uint32_t efHeight   = std::min<unsigned>( uiHeight, JVET_C0024_ZERO_OUT_TH );
847
0
  const uint32_t efArea     = uiWidth * efHeight;
848
0
  const int channelBitDepth = sps.bitDepths[toChannelType(compID)];
849
0
  const CCoeffBuf piCoef    = pSrc;
850
851
0
  const bool useTransformSkip      = tu.mtsIdx[compID] == MTS_SKIP;
852
0
  const int  maxLog2TrDynamicRange = sps.getMaxLog2TrDynamicRange();
853
854
0
  const int scalingListType     = getScalingListType( tu.cu->predMode, compID );
855
0
  CHECK( scalingListType >= SCALING_LIST_NUM, "Invalid scaling list" );
856
857
0
  const bool        isDq        = tu.cs->slice->depQuantEnabled && !useTransformSkip;
858
0
  const int         qpDQ        = isDq ? cQP.Qp( false ) + 1 : cQP.Qp( useTransformSkip );
859
0
  const int         qpPer       = isDq ? qpDQ / 6 : cQP.per( useTransformSkip );
860
0
  const int         qpRem       = isDq ? qpDQ - 6 * qpPer : cQP.rem( useTransformSkip );
861
862
0
  const uint32_t uiLog2TrWidth  = Log2( uiWidth );
863
0
  const uint32_t uiLog2TrHeight = Log2( uiHeight );
864
0
  int *piQuantCoeff             = getQuantCoeff( scalingListType, qpRem, uiLog2TrWidth, uiLog2TrHeight );
865
866
0
  const bool isLfnstApplied     = tu.cu->lfnstIdx > 0 && ( CU::isSepTree( *tu.cu ) ? true : isLuma( compID ) );
867
0
  const bool enableScalingLists = getUseScalingList( uiWidth, uiHeight, ( useTransformSkip != 0 ), isLfnstApplied );
868
869
0
  const bool needSqrtAdjustment = TU::needsSqrt2Scale( tu, compID );
870
0
  const int defaultQuantisationCoefficient
871
0
                                = g_quantScales[needSqrtAdjustment?1:0][qpRem];
872
0
  const int iTransformShift     = getTransformShift( channelBitDepth, rect.size(), maxLog2TrDynamicRange ) + ( needSqrtAdjustment ? -1 : 0 );
873
874
875
0
  const int iQBits              = QUANT_SHIFT + qpPer + iTransformShift;
876
877
  // QBits will be OK for any internal bit depth as the reduction in transform shift is balanced by an increase in Qp_per due to QpBDOffset
878
  // iAdd is different from the iAdd used in normal quantization
879
0
  const int64_t iAdd = int64_t( compID == COMP_Y ? 171 : 256 ) << ( iQBits - 9 );
880
881
0
  if( !enableScalingLists )
882
0
    return xNeedRdoq( piCoef.buf, efArea, defaultQuantisationCoefficient, iAdd, iQBits );
883
884
0
  for( int uiBlockPos = 0; uiBlockPos < efArea; uiBlockPos++ )
885
0
  {
886
0
    const TCoeff   iLevel           = piCoef.buf[uiBlockPos];
887
0
    const int64_t  tmpLevel         = ( int64_t ) std::abs( iLevel ) * piQuantCoeff[uiBlockPos];
888
0
    const TCoeff quantisedMagnitude = TCoeff( ( tmpLevel + iAdd ) >> iQBits );
889
890
0
    if( quantisedMagnitude != 0 )
891
0
    {
892
0
      return true;
893
0
    }
894
0
  } // for n
895
0
  return false;
896
0
}
897
898
} // namespace vvenc
899
900
//! \}
901