Coverage Report

Created: 2026-09-14 06:44

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/work/vvenc/source/Lib/CommonLib/Quant.cpp
Line
Count
Source
1
/* -----------------------------------------------------------------------------
2
The copyright in this software is being made available under the Clear BSD
3
License, included below. No patent rights, trademark rights and/or 
4
other Intellectual Property Rights other than the copyrights concerning 
5
the Software are granted under this license.
6
7
The Clear BSD License
8
9
Copyright (c) 2019-2026, Fraunhofer-Gesellschaft zur Förderung der angewandten Forschung e.V. & The VVenC Authors.
10
All rights reserved.
11
12
Redistribution and use in source and binary forms, with or without modification,
13
are permitted (subject to the limitations in the disclaimer below) provided that
14
the following conditions are met:
15
16
     * Redistributions of source code must retain the above copyright notice,
17
     this list of conditions and the following disclaimer.
18
19
     * Redistributions in binary form must reproduce the above copyright
20
     notice, this list of conditions and the following disclaimer in the
21
     documentation and/or other materials provided with the distribution.
22
23
     * Neither the name of the copyright holder nor the names of its
24
     contributors may be used to endorse or promote products derived from this
25
     software without specific prior written permission.
26
27
NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY
28
THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
29
CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
30
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
31
PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
32
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
33
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
34
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
35
BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
36
IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
37
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
38
POSSIBILITY OF SUCH DAMAGE.
39
40
41
------------------------------------------------------------------------------------------- */
42
43
44
/** \file     Quant.cpp
45
    \brief    transform and quantization class
46
*/
47
48
#include "Quant.h"
49
#include "UnitTools.h"
50
#include "ContextModelling.h"
51
#include "CodingStructure.h"
52
#include "dtrace_buffer.h"
53
54
#include <stdlib.h>
55
#include <memory.h>
56
57
//! \ingroup CommonLib
58
//! \{
59
60
namespace vvenc {
61
62
// ====================================================================================================================
63
// Constants
64
// ====================================================================================================================
65
66
67
// ====================================================================================================================
68
// QpParam constructor
69
// ====================================================================================================================
70
 
71
QpParam::QpParam(const TransformUnit& tu, const ComponentID &compID, const bool allowACTQpoffset)
72
6.33M
{
73
6.33M
  const ChannelType chType = toChannelType( compID );
74
6.33M
  const SPS        &sps    = *tu.cu->cs->sps;
75
6.33M
  const int     qpBdOffset = sps.qpBDOffset[chType];
76
6.33M
  const bool useJQP        = isChroma( compID ) && abs( TU::getICTMode( tu ) ) == 2;
77
6.33M
  const ComponentID jCbCr  = useJQP ? COMP_JOINT_CbCr : compID;
78
  
79
6.33M
        int chromaQpOffset = 0;
80
81
6.33M
  if( isChroma( compID ) )
82
6.12M
  {
83
6.12M
    const PPS &pps  = *tu.cu->slice->pps;
84
6.12M
    chromaQpOffset  = pps.chromaQpOffset              [jCbCr];
85
6.12M
    chromaQpOffset += tu.cu->slice->sliceChromaQpDelta[jCbCr];
86
6.12M
    chromaQpOffset += pps.getChromaQpOffsetListEntry( tu.cu->chromaQpAdj ).u.offset[int( jCbCr ) - 1];
87
6.12M
  }
88
  
89
6.33M
  int baseQp;
90
6.33M
  int qpy        = tu.cu->qp;
91
  //bool skip      = tu.mtsIdx[compID] == MTS_SKIP;
92
93
6.33M
  if( isLuma( compID ) )
94
216k
  {
95
216k
    baseQp = tu.cu->qp + qpBdOffset;
96
216k
  }
97
6.12M
  else
98
6.12M
  {
99
6.12M
    int qpi = Clip3( -qpBdOffset, MAX_QP, qpy );
100
6.12M
    baseQp  = sps.chromaQpMappingTable.getMappedChromaQpValue( jCbCr, qpi );
101
6.12M
    baseQp  = Clip3( -qpBdOffset, MAX_QP, baseQp + chromaQpOffset ) + qpBdOffset;
102
6.12M
  }
103
104
6.33M
  if( allowACTQpoffset && tu.cu->colorTransform )
105
0
  {
106
0
    baseQp += DELTA_QP_ACT[jCbCr];
107
0
  }
108
109
6.33M
  baseQp = Clip3( 0, MAX_QP + qpBdOffset, baseQp );
110
111
  //if( !skip )
112
6.33M
  {
113
6.33M
    Qps [0] = baseQp;
114
6.33M
    pers[0] = baseQp / 6;
115
6.33M
    rems[0] = baseQp % 6;
116
6.33M
  }
117
  //else
118
6.33M
  {
119
6.33M
    int internalMinusInputBitDepth = sps.internalMinusInputBitDepth[chType];
120
6.33M
    int baseQpTS           = std::max( baseQp, 4 + 6 * internalMinusInputBitDepth );
121
122
6.33M
    Qps [1] = baseQpTS;
123
6.33M
    pers[1] = baseQpTS / 6;
124
6.33M
    rems[1] = baseQpTS % 6;
125
6.33M
  }
126
6.33M
}
127
128
129
// ====================================================================================================================
130
// Quant class member functions
131
// ====================================================================================================================
132
static void QuantCore(const TransformUnit tu, const ComponentID compID, const CCoeffBuf& piCoef,CoeffSigBuf piQCoef,TCoeff &uiAbsSum, int &lastScanPos,TCoeff *deltaU,const int defaultQuantisationCoefficient,const int iQBits,const int64_t iAdd,const TCoeff entropyCodingMinimum,const TCoeff entropyCodingMaximum,const bool signHiding, const TCoeff m_thrVal)
133
5.66k
{
134
5.66k
  CoeffCodingContext cctx( tu, compID, signHiding );
135
136
5.66k
  const CompArea &rect      = tu.blocks[compID];
137
5.66k
  const uint32_t uiWidth    = rect.width;
138
5.66k
  const uint32_t uiHeight   = rect.height;
139
140
  /* for 422 chroma blocks, the effective scaling applied during transformation is not a power of 2, hence it cannot be
141
  * implemented as a bit-shift (the quantised result will be sqrt(2) * larger than required). Alternatively, adjust the
142
  * uiLog2TrSize applied in iTransformShift, such that the result is 1/sqrt(2) the required result (i.e. smaller)
143
  * Then a QP+3 (sqrt(2)) or QP-3 (1/sqrt(2)) method could be used to get the required result
144
  */
145
146
5.66k
  const uint32_t log2CGSize         = cctx.log2CGSize();
147
148
5.66k
  uiAbsSum = 0;
149
150
5.66k
  const int iCGSize   = 1 << log2CGSize;
151
152
5.66k
  const uint32_t lfnstIdx = tu.cu->lfnstIdx;
153
5.66k
  const int iCGNum   = lfnstIdx > 0 ? 1 : std::min<int>(JVET_C0024_ZERO_OUT_TH, uiWidth) * std::min<int>(JVET_C0024_ZERO_OUT_TH, uiHeight) >> cctx.log2CGSize();
154
5.66k
  int       iScanPos = ( iCGNum << log2CGSize ) - 1;
155
156
5.66k
  if( lfnstIdx > 0 && ( ( uiWidth == 4 && uiHeight == 4 ) || ( uiWidth == 8 && uiHeight == 8 ) ) )
157
0
  {
158
0
    iScanPos = 7;
159
0
  }
160
161
  // Find first non-zero coeff
162
31.9k
  for( ; iScanPos > 0; iScanPos-- )
163
30.5k
  {
164
30.5k
    uint32_t uiBlkPos = cctx.blockPos( iScanPos );
165
30.5k
    if( piCoef.buf[uiBlkPos] )
166
4.24k
      break;
167
30.5k
  }
168
169
  //////////////////////////////////////////////////////////////////////////
170
  //  Loop over sub-sets (coefficient groups)
171
  //////////////////////////////////////////////////////////////////////////
172
  
173
5.66k
  TCoeff thres = 0, useThres = 0;
174
  
175
5.66k
  if( iQBits )
176
5.66k
    thres = TCoeff( ( int64_t( m_thrVal ) << ( iQBits - 1 ) ) );
177
0
  else
178
0
    thres = TCoeff( ( int64_t( m_thrVal >> 1 ) << iQBits ) );
179
180
5.66k
  useThres = thres / ( defaultQuantisationCoefficient << 2 );
181
182
5.66k
  const bool is4x4sbb = log2CGSize == 4 && cctx.log2CGWidth() == 2;
183
184
5.66k
  int subSetId = iScanPos >> log2CGSize;
185
6.53k
  for( ; subSetId >= 1; subSetId-- )
186
872
  {
187
872
    if( is4x4sbb && iScanPos >= 16 )
188
0
    {
189
0
      int  iScanPosinCG = iScanPos & ( iCGSize - 1 );
190
0
      bool allSmaller   = true;
191
192
0
      for( int xScanPosinCG = iScanPosinCG, xScanPos = iScanPos; allSmaller && xScanPosinCG >= 0; xScanPosinCG--, xScanPos-- )
193
0
      {
194
0
        const uint32_t uiBlkPos = cctx.blockPos( xScanPos );
195
0
        allSmaller &= abs( piCoef.buf[uiBlkPos] ) <= useThres;
196
0
      }
197
198
0
      if( allSmaller )
199
0
      {
200
0
        iScanPos    -= iScanPosinCG + 1;
201
0
        continue;
202
0
      }
203
0
      else
204
0
      {
205
0
        break;
206
0
      }
207
0
    }
208
872
  }
209
210
5.66k
  const int qBits8 = iQBits - 8;
211
5.66k
  piQCoef.memset( 0 );
212
213
88.9k
  for( int currPos = 0; currPos <= iScanPos; currPos++ )
214
83.3k
  {
215
83.3k
    const int uiBlockPos  = cctx.blockPos( currPos );
216
83.3k
    const TCoeff iLevel   = piCoef.buf[uiBlockPos];
217
83.3k
    const TCoeff iSign    = (iLevel < 0 ? -1: 1);
218
219
83.3k
    const int64_t  tmpLevel = (int64_t)abs(iLevel) * defaultQuantisationCoefficient;
220
83.3k
    const TCoeff quantisedMagnitude = TCoeff((tmpLevel + iAdd ) >> iQBits);
221
83.3k
    deltaU[uiBlockPos] = (TCoeff)((tmpLevel - ((int64_t)quantisedMagnitude<<iQBits) )>> qBits8);
222
223
83.3k
    uiAbsSum += quantisedMagnitude;
224
83.3k
    const TCoeff quantisedCoefficient = quantisedMagnitude * iSign;
225
226
83.3k
    piQCoef.buf[uiBlockPos] = Clip3<TCoeff>( entropyCodingMinimum, entropyCodingMaximum, quantisedCoefficient );
227
83.3k
  } // for n
228
229
5.66k
  lastScanPos = iScanPos;
230
5.66k
}
231
232
static void DeQuantCore(const int maxX,const int maxY,const int scale,const TCoeffSig* const piQCoef,const size_t piQCfStride,TCoeff   *const piCoef,const int rightShift,const int inputMaximum,const TCoeff transformMaximum)
233
47.6k
{
234
47.6k
  const int inputMinimum = -(inputMaximum+1);
235
47.6k
  const TCoeff transformMinimum = -(transformMaximum+1);
236
47.6k
  if (rightShift>0)
237
37.6k
  {
238
37.6k
    const Intermediate_Int iAdd = (Intermediate_Int) 1 << (rightShift - 1);
239
358k
    for( int y = 0, n = 0; y <= maxY; y++)
240
320k
    {
241
3.44M
      for( int x = 0; x <= maxX; x++, n++ )
242
3.12M
      {
243
3.12M
        const TCoeff           clipQCoef = TCoeff(Clip3<Intermediate_Int>(inputMinimum, inputMaximum, piQCoef[x + y * piQCfStride]));
244
3.12M
        Intermediate_Int iCoeffQ   = (Intermediate_Int(clipQCoef) * scale + iAdd) >> rightShift;
245
3.12M
        piCoef[n] = TCoeff(Clip3<Intermediate_Int>(transformMinimum,transformMaximum,iCoeffQ));
246
3.12M
      }
247
320k
    }
248
37.6k
  }
249
10.0k
  else  // rightshift <0
250
10.0k
  {
251
10.0k
    int leftShift = -rightShift;
252
120k
    for( int y = 0, n = 0; y <= maxY; y++)
253
110k
    {
254
1.34M
      for( int x = 0; x <= maxX; x++, n++ )
255
1.23M
      {
256
1.23M
        const TCoeff           clipQCoef = TCoeff(Clip3<Intermediate_Int>(inputMinimum, inputMaximum, piQCoef[x + y * piQCfStride]));
257
1.23M
        const Intermediate_Int iCoeffQ   = (Intermediate_Int(clipQCoef) * scale) * (1 << leftShift);
258
1.23M
        piCoef[n] = TCoeff(Clip3<Intermediate_Int>(transformMinimum,transformMaximum,iCoeffQ));
259
1.23M
      }
260
110k
    }
261
10.0k
  }
262
47.6k
}
263
264
static bool needRdoqCore( const TCoeff* pCoeff, size_t numCoeff, int quantCoeff, int64_t offset, int shift )
265
0
{
266
0
  for( int uiBlockPos = 0; uiBlockPos < numCoeff; uiBlockPos++ )
267
0
  {
268
0
    const TCoeff   iLevel = pCoeff[uiBlockPos];
269
0
    const int64_t  tmpLevel = ( int64_t ) std::abs( iLevel ) * quantCoeff;
270
0
    const TCoeff quantisedMagnitude = TCoeff( ( tmpLevel + offset ) >> shift );
271
272
0
    if( quantisedMagnitude != 0 )
273
0
    {
274
0
      return true;
275
0
    }
276
0
  } // for n
277
0
  return false;
278
0
}
279
280
281
19.1k
Quant::Quant( const Quant* other, bool useScalingLists, bool enableOpt ) : m_RDOQ( 0 ), m_useRDOQTS( false ), m_dLambda( 0.0 )
282
19.1k
{
283
19.1k
  xInitScalingList( other, useScalingLists );
284
19.1k
  xDeQuant  = DeQuantCore;
285
19.1k
  xQuant    = QuantCore;
286
19.1k
  xNeedRdoq = needRdoqCore;
287
19.1k
  if( enableOpt )
288
19.1k
  {
289
#if defined( TARGET_SIMD_X86 ) && ENABLE_SIMD_OPT_QUANT
290
    initQuantX86();
291
#endif
292
#if defined( TARGET_SIMD_ARM ) && ENABLE_SIMD_OPT_QUANT
293
    initQuantARM();
294
#endif
295
19.1k
  }
296
19.1k
}
297
298
Quant::~Quant()
299
19.1k
{
300
19.1k
  xDestroyScalingList();
301
19.1k
}
302
303
void invResDPCM( const TransformUnit& tu, const ComponentID compID, CoeffSigBuf& dstBuf )
304
47.6k
{
305
47.6k
  const CompArea&    rect   = tu.blocks[compID];
306
47.6k
  const int          wdt    = rect.width;
307
47.6k
  const int          hgt    = rect.height;
308
47.6k
  const CCoeffSigBuf coeffs = tu.getCoeffs(compID);
309
310
47.6k
  const int      maxLog2TrDynamicRange = tu.cs->sps->getMaxLog2TrDynamicRange();
311
47.6k
  const TCoeff   inputMinimum          = -(1 << maxLog2TrDynamicRange);
312
47.6k
  const TCoeff   inputMaximum          =  (1 << maxLog2TrDynamicRange) - 1;
313
314
47.6k
  const TCoeffSig* coef = &coeffs.buf[0];
315
47.6k
        TCoeffSig* dst  = &dstBuf.buf[0];
316
317
47.6k
  if ( tu.cu->bdpcmM[toChannelType(compID)] == 1)
318
498
  {
319
7.27k
    for( int y = 0; y < hgt; y++ )
320
6.77k
    {
321
6.77k
      dst[0] = coef[0];
322
108k
      for( int x = 1; x < wdt; x++ )
323
101k
      {
324
101k
        dst[x] = Clip3(inputMinimum, inputMaximum, TCoeff( dst[x - 1] ) + TCoeff( coef[x] ));
325
101k
      }
326
6.77k
      coef += coeffs.stride;
327
6.77k
      dst += dstBuf.stride;
328
6.77k
    }
329
498
  }
330
47.1k
  else
331
47.1k
  {
332
498k
    for( int x = 0; x < wdt; x++ )
333
451k
    {
334
451k
      dst[x] = coef[x];
335
451k
    }
336
424k
    for( int y = 0; y < hgt - 1; y++ )
337
377k
    {
338
4.17M
      for( int x = 0; x < wdt; x++ )
339
3.79M
      {
340
3.79M
        dst[dstBuf.stride + x] = Clip3(inputMinimum, inputMaximum, TCoeff( dst[x] ) + TCoeff( coef[coeffs.stride + x] ));
341
3.79M
      }
342
377k
      coef += coeffs.stride;
343
377k
      dst += dstBuf.stride;
344
377k
    }
345
47.1k
  }
346
47.6k
}
347
348
void fwdResDPCM( TransformUnit& tu, const ComponentID compID )
349
5.66k
{
350
5.66k
  const CompArea& rect   = tu.blocks[compID];
351
5.66k
  const int       wdt    = rect.width;
352
5.66k
  const int       hgt    = rect.height;
353
5.66k
  CoeffSigBuf     coeffs = tu.getCoeffs(compID);
354
355
5.66k
  TCoeffSig* coef = &coeffs.buf[0];
356
5.66k
  if (tu.cu->bdpcmM[toChannelType(compID)] == 1)
357
0
  {
358
0
    for( int y = 0; y < hgt; y++ )
359
0
    {
360
0
      for( int x = wdt - 1; x > 0; x-- )
361
0
      {
362
0
        coef[x] -= coef[x - 1];
363
0
      }
364
0
      coef += coeffs.stride;
365
0
    }
366
0
  }
367
5.66k
  else
368
5.66k
  {
369
5.66k
    coef += coeffs.stride * (hgt - 1);
370
11.3k
    for( int y = 0; y < hgt - 1; y++ )
371
5.66k
    {
372
60.4k
      for ( int x = 0; x < wdt; x++ )
373
54.8k
      {
374
54.8k
        coef[x] -= coef[x - coeffs.stride];
375
54.8k
      }
376
5.66k
      coef -= coeffs.stride;
377
5.66k
    }
378
5.66k
  }
379
5.66k
}
380
381
// To minimize the distortion only. No rate is considered.
382
void Quant::xSignBitHidingHDQ( TCoeffSig* pQCoef, const TCoeff* pCoef, TCoeff* deltaU, const CoeffCodingContext& cctx, int& lastScanPos, const int maxLog2TrDynamicRange )
383
0
{
384
0
  const uint32_t groupSize = 1 << cctx.log2CGSize();
385
386
0
  const TCoeff entropyCodingMinimum = -(1 << maxLog2TrDynamicRange);
387
0
  const TCoeff entropyCodingMaximum =  (1 << maxLog2TrDynamicRange) - 1;
388
389
0
  int lastCG = -1;
390
0
  int absSum = 0 ;
391
0
  int n ;
392
393
0
  for( int subSet = lastScanPos >> cctx.log2CGSize(); subSet >= 0; subSet-- )
394
0
  {
395
0
    int  subPos = subSet << cctx.log2CGSize();
396
0
    int  firstNZPosInCG=groupSize , lastNZPosInCG=-1 ;
397
0
    absSum = 0 ;
398
399
0
    for(n = groupSize-1; n >= 0; --n )
400
0
    {
401
0
      if( pQCoef[ cctx.blockPos( n + subPos ) ] )
402
0
      {
403
0
        lastNZPosInCG = n;
404
0
        break;
405
0
      }
406
0
    }
407
408
0
    for(n = 0; n <groupSize; n++ )
409
0
    {
410
0
      if( pQCoef[ cctx.blockPos( n + subPos ) ] )
411
0
      {
412
0
        firstNZPosInCG = n;
413
0
        break;
414
0
      }
415
0
    }
416
417
0
    for(n = firstNZPosInCG; n <=lastNZPosInCG; n++ )
418
0
    {
419
0
      absSum += int(pQCoef[ cctx.blockPos( n + subPos ) ]);
420
0
    }
421
422
0
    if(lastNZPosInCG>=0 && lastCG==-1)
423
0
    {
424
0
      lastCG = 1 ;
425
0
    }
426
427
0
    if( lastNZPosInCG-firstNZPosInCG>=SBH_THRESHOLD )
428
0
    {
429
0
      uint32_t signbit = (pQCoef[cctx.blockPos(subPos+firstNZPosInCG)]>0?0:1) ;
430
0
      if( signbit!=(absSum&0x1) )  //compare signbit with sum_parity
431
0
      {
432
0
        TCoeff curCost    = std::numeric_limits<TCoeff>::max();
433
0
        TCoeff minCostInc = std::numeric_limits<TCoeff>::max();
434
0
        int minPos =-1, finalChange=0, curChange=0, minScanPos = -1;
435
436
0
        for( n = (lastCG==1?lastNZPosInCG:groupSize-1) ; n >= 0; --n )
437
0
        {
438
0
          uint32_t blkPos   = cctx.blockPos( n+subPos );
439
0
          if(pQCoef[ blkPos ] != 0 )
440
0
          {
441
0
            if(deltaU[blkPos]>0)
442
0
            {
443
0
              curCost = - deltaU[blkPos];
444
0
              curChange=1 ;
445
0
            }
446
0
            else
447
0
            {
448
              //curChange =-1;
449
0
              if(n==firstNZPosInCG && abs(pQCoef[blkPos])==1)
450
0
              {
451
0
                curCost = std::numeric_limits<TCoeff>::max();
452
0
              }
453
0
              else
454
0
              {
455
0
                curCost = deltaU[blkPos];
456
0
                curChange =-1;
457
0
              }
458
0
            }
459
0
          }
460
0
          else
461
0
          {
462
0
            if(n<firstNZPosInCG)
463
0
            {
464
0
              uint32_t thisSignBit = (pCoef[blkPos]>=0?0:1);
465
0
              if(thisSignBit != signbit )
466
0
              {
467
0
                curCost = std::numeric_limits<TCoeff>::max();
468
0
              }
469
0
              else
470
0
              {
471
0
                curCost = - (deltaU[blkPos])  ;
472
0
                curChange = 1 ;
473
0
              }
474
0
            }
475
0
            else
476
0
            {
477
0
              curCost = - (deltaU[blkPos])  ;
478
0
              curChange = 1 ;
479
0
            }
480
0
          }
481
482
0
          if( curCost<minCostInc)
483
0
          {
484
0
            minCostInc = curCost ;
485
0
            finalChange = curChange ;
486
0
            minPos = blkPos;
487
0
            minScanPos = n + subPos;
488
0
          }
489
0
        } //CG loop
490
491
0
        if(pQCoef[minPos] == entropyCodingMaximum || pQCoef[minPos] == entropyCodingMinimum)
492
0
        {
493
0
          finalChange = -1;
494
0
        }
495
496
0
        if(pCoef[minPos]>=0)
497
0
        {
498
0
          pQCoef[minPos] += finalChange ;
499
0
        }
500
0
        else
501
0
        {
502
0
          pQCoef[minPos] -= finalChange ;
503
0
        }
504
505
        // if changing lastScanPos element to 0, move the pointer to the new lastScanPos element
506
0
        if( minScanPos == lastScanPos && pQCoef[minPos] == 0 )
507
0
        {
508
0
          for( ; lastScanPos >= 0 && pQCoef[cctx.blockPos( lastScanPos )] == 0; lastScanPos-- );
509
0
        }
510
0
        else if( minScanPos > lastScanPos && pQCoef[minPos] != 0 )
511
0
        {
512
0
          lastScanPos = minPos;
513
0
        }
514
0
      } // Hide
515
0
    }
516
0
    if(lastCG==1)
517
0
    {
518
0
      lastCG=0 ;
519
0
    }
520
0
  } // TU loop
521
522
0
  return;
523
0
}
524
525
void Quant::dequant(const TransformUnit& tu,
526
                          CoeffBuf&      dstCoeff,
527
                    const ComponentID    compID,
528
                    const QpParam&       cQP)
529
47.6k
{
530
47.6k
  const SPS       *sps                  = tu.cs->sps;
531
47.6k
  const CompArea  &area                 = tu.blocks[compID];
532
47.6k
  const uint32_t  uiWidth               = area.width;
533
47.6k
  const uint32_t  uiHeight              = area.height;
534
47.6k
  TCoeff *const   piCoef                = dstCoeff.buf;
535
47.6k
  const uint32_t  numSamplesInBlock     = uiWidth * uiHeight;
536
47.6k
  const int       maxLog2TrDynamicRange = sps->getMaxLog2TrDynamicRange();
537
47.6k
  const TCoeff    transformMinimum      = -(1 << maxLog2TrDynamicRange);
538
47.6k
  const TCoeff    transformMaximum      =  (1 << maxLog2TrDynamicRange) - 1;
539
47.6k
  const bool      isTransformSkip       = tu.mtsIdx[compID] == MTS_SKIP;
540
47.6k
  const bool      isLfnstApplied        = tu.cu->lfnstIdx > 0 && (CU::isSepTree(*tu.cu) ? true : isLuma(compID));
541
47.6k
  const bool      enableScalingLists    = getUseScalingList(uiWidth, uiHeight, isTransformSkip, isLfnstApplied);
542
47.6k
  const int       scalingListType       = getScalingListType(tu.cu->predMode, compID);
543
47.6k
  const int       channelBitDepth       = sps->bitDepths[toChannelType(compID)];
544
545
47.6k
  const TCoeffSig *coef     = tu.getCoeffs( compID ).buf;
546
47.6k
  const ptrdiff_t  piStride = tu.getCoeffs( compID ).stride;
547
548
47.6k
  if( tu.cu->bdpcmM[toChannelType( compID )] )
549
47.6k
  {
550
47.6k
    CoeffSigBuf coefBuf( m_tmpBdpcm, uiWidth, uiHeight );
551
47.6k
    invResDPCM( tu, compID, coefBuf );
552
47.6k
    coef      = m_tmpBdpcm;
553
47.6k
  }
554
555
47.6k
  const TCoeffSig  *const piQCoef = coef;
556
47.6k
  CHECK(scalingListType >= SCALING_LIST_NUM, "Invalid scaling list");
557
558
  // Represents scaling through forward transform
559
47.6k
  const int  originalTransformShift = getTransformShift(channelBitDepth, area.size(), maxLog2TrDynamicRange);
560
47.6k
  const bool needSqrtAdjustment     = TU::needsSqrt2Scale( tu, compID );
561
47.6k
  const int  iTransformShift        = originalTransformShift + (needSqrtAdjustment?-1:0);
562
563
47.6k
  const int QP_per = cQP.per(isTransformSkip);
564
47.6k
  const int QP_rem = cQP.rem(isTransformSkip);
565
566
47.6k
  const int  rightShift = (IQUANT_SHIFT - ((isTransformSkip ? 0 : iTransformShift) + QP_per)) + (enableScalingLists ? LOG2_SCALING_LIST_NEUTRAL_VALUE : 0);
567
568
47.6k
  if(enableScalingLists)
569
0
  {
570
    //from the dequantization equation:
571
    //iCoeffQ                         = ((Intermediate_Int(clipQCoef) * piDequantCoef[deQuantIdx]) + iAdd ) >> rightShift
572
    //(sizeof(Intermediate_Int) * 8)  =              inputBitDepth    +    dequantCoefBits                   - rightShift
573
0
    const uint32_t             dequantCoefBits     = 1 + IQUANT_SHIFT + SCALING_LIST_BITS;
574
0
    const uint32_t             targetInputBitDepth = std::min<uint32_t>((maxLog2TrDynamicRange + 1), (((sizeof(Intermediate_Int) * 8) + rightShift) - dequantCoefBits));
575
576
0
    const Intermediate_Int inputMinimum        = -(1 << (targetInputBitDepth - 1));
577
0
    const Intermediate_Int inputMaximum        =  (1 << (targetInputBitDepth - 1)) - 1;
578
579
0
    const uint32_t uiLog2TrWidth  = Log2(uiWidth);
580
0
    const uint32_t uiLog2TrHeight = Log2(uiHeight);
581
0
    int* piDequantCoef            = getDequantCoeff(scalingListType, QP_rem, uiLog2TrWidth, uiLog2TrHeight);
582
583
0
    if(rightShift > 0)
584
0
    {
585
0
      const Intermediate_Int iAdd = (Intermediate_Int) 1 << (rightShift - 1);
586
0
      for( int n = 0; n < numSamplesInBlock; n++ )
587
0
      {
588
0
        const TCoeff           clipQCoef = TCoeff(Clip3<Intermediate_Int>(inputMinimum, inputMaximum, piQCoef[n]));
589
0
        const Intermediate_Int iCoeffQ   = ((Intermediate_Int(clipQCoef) * piDequantCoef[n]) + iAdd ) >> rightShift;
590
0
        piCoef[n] = TCoeff(Clip3<Intermediate_Int>(transformMinimum,transformMaximum,iCoeffQ));
591
0
      }
592
0
    }
593
0
    else
594
0
    {
595
0
      const int leftShift = -rightShift;
596
0
      for( int n = 0; n < numSamplesInBlock; n++ )
597
0
      {
598
0
        const TCoeff           clipQCoef = TCoeff(Clip3<Intermediate_Int>(inputMinimum, inputMaximum, piQCoef[n]));
599
0
        const Intermediate_Int iCoeffQ   = (Intermediate_Int(clipQCoef) * piDequantCoef[n]) << leftShift;
600
0
        piCoef[n] = TCoeff(Clip3<Intermediate_Int>(transformMinimum,transformMaximum,iCoeffQ));
601
0
      }
602
0
    }
603
0
  }
604
47.6k
  else
605
47.6k
  {
606
47.6k
    const int scale     = g_invQuantScales[needSqrtAdjustment?1:0][QP_rem];
607
47.6k
    const int scaleBits = ( IQUANT_SHIFT + 1 );
608
    //from the dequantisation equation:
609
    //iCoeffQ                         = Intermediate_Int((int64_t(clipQCoef) * scale + iAdd) >> rightShift);
610
    //(sizeof(Intermediate_Int) * 8)  =                    inputBitDepth   + scaleBits      - rightShift
611
47.6k
    const uint32_t             targetInputBitDepth = std::min<uint32_t>((maxLog2TrDynamicRange + 1), (((sizeof(Intermediate_Int) * 8) + rightShift) - scaleBits));
612
47.6k
    const Intermediate_Int inputMaximum        =  (1 << (targetInputBitDepth - 1)) - 1;
613
47.6k
    xDeQuant(uiWidth-1,uiHeight-1,scale,piQCoef,piStride,piCoef,rightShift,inputMaximum,transformMaximum);
614
47.6k
  }
615
47.6k
}
616
617
void Quant::init( int rdoq, bool bUseRDOQTS, int thrVal )
618
19.1k
{
619
620
  // TODO: pass to init() a single variable containing (quantization) flags,
621
  //       instead of variables that don't have to do with this class
622
623
19.1k
  m_RDOQ             = rdoq;
624
19.1k
  m_useRDOQTS        = bUseRDOQTS;
625
19.1k
  m_thrVal           = thrVal;
626
19.1k
}
627
628
/** set flat matrix value to quantized coefficient
629
 */
630
void Quant::setFlatScalingList(const int maxLog2TrDynamicRange[MAX_NUM_CH], const BitDepths &bitDepths )
631
19.1k
{
632
19.1k
  if( !m_scalingListEnabled ) return;
633
634
0
  const int minimumQp = 0;
635
0
  const int maximumQp = SCALING_LIST_REM_NUM;
636
637
0
  for(uint32_t sizeX = 0; sizeX < SCALING_LIST_SIZE_NUM; sizeX++)
638
0
  {
639
0
    for(uint32_t sizeY = 0; sizeY < SCALING_LIST_SIZE_NUM; sizeY++)
640
0
    {
641
0
      for(uint32_t list = 0; list < SCALING_LIST_NUM; list++)
642
0
      {
643
0
        for(int qp = minimumQp; qp < maximumQp; qp++)
644
0
        {
645
0
          xSetFlatScalingList( list, sizeX, sizeY, qp );
646
0
        }
647
0
      }
648
0
    }
649
0
  }
650
0
}
651
652
/** set flat matrix value to quantized coefficient
653
 * \param list List ID
654
 * \param size size index
655
 * \param qp Quantization parameter
656
 * \param format chroma format
657
 */
658
void Quant::xSetFlatScalingList(uint32_t list, uint32_t sizeX, uint32_t sizeY, int qp )
659
0
{
660
0
  uint32_t i,num = g_scalingListSizeX[sizeX]*g_scalingListSizeX[sizeY];
661
0
  int *quantcoeff;
662
0
  int *dequantcoeff;
663
664
0
  const bool blockIsNotPowerOf4 = ((Log2(g_scalingListSizeX[sizeX] * g_scalingListSizeX[sizeY])) & 1) == 1;
665
0
  int quantScales    = g_quantScales   [blockIsNotPowerOf4?1:0][qp];
666
0
  int invQuantScales = g_invQuantScales[blockIsNotPowerOf4?1:0][qp] << 4;
667
668
0
  quantcoeff   = getQuantCoeff(list, qp, sizeX, sizeY);
669
0
  dequantcoeff = getDequantCoeff(list, qp, sizeX, sizeY);
670
671
0
  for(i=0;i<num;i++)
672
0
  {
673
0
    *quantcoeff++ = quantScales;
674
0
    *dequantcoeff++ = invQuantScales;
675
0
  }
676
0
}
677
678
679
/** initialization process of scaling list array
680
 */
681
void Quant::xInitScalingList( const Quant* other, bool useScalingLists )
682
19.1k
{
683
19.1k
  m_isScalingListOwner = other == nullptr;
684
19.1k
  m_scalingListEnabled = useScalingLists;
685
686
153k
  for(uint32_t sizeIdX = 0; sizeIdX < SCALING_LIST_SIZE_NUM; sizeIdX++)
687
134k
  {
688
1.07M
    for(uint32_t sizeIdY = 0; sizeIdY < SCALING_LIST_SIZE_NUM; sizeIdY++)
689
940k
    {
690
6.58M
      for(uint32_t qp = 0; qp < SCALING_LIST_REM_NUM; qp++)
691
5.64M
      {
692
39.4M
        for(uint32_t listId = 0; listId < SCALING_LIST_NUM; listId++)
693
33.8M
        {
694
33.8M
          if( m_isScalingListOwner )
695
33.8M
          {
696
33.8M
            const size_t scalingListSize = g_scalingListSizeX[sizeIdX] * g_scalingListSizeX[sizeIdY];
697
698
33.8M
            m_quantCoef   [sizeIdX][sizeIdY][listId][qp] = useScalingLists ? new int[scalingListSize] : nullptr;
699
33.8M
            m_dequantCoef [sizeIdX][sizeIdY][listId][qp] = useScalingLists ? new int[scalingListSize] : nullptr;
700
33.8M
          }
701
0
          else
702
0
          {
703
0
            m_quantCoef   [sizeIdX][sizeIdY][listId][qp] = other->m_quantCoef   [sizeIdX][sizeIdY][listId][qp];
704
0
            m_dequantCoef [sizeIdX][sizeIdY][listId][qp] = other->m_dequantCoef [sizeIdX][sizeIdY][listId][qp];
705
0
          }
706
33.8M
        } // listID loop
707
5.64M
      }
708
940k
    }
709
134k
  }
710
19.1k
}
711
712
/** destroy quantization matrix array
713
 */
714
void Quant::xDestroyScalingList()
715
19.1k
{
716
19.1k
  if( !m_isScalingListOwner ) return;
717
718
153k
  for(uint32_t sizeIdX = 0; sizeIdX < SCALING_LIST_SIZE_NUM; sizeIdX++)
719
134k
  {
720
1.07M
    for(uint32_t sizeIdY = 0; sizeIdY < SCALING_LIST_SIZE_NUM; sizeIdY++)
721
940k
    {
722
6.58M
      for(uint32_t listId = 0; listId < SCALING_LIST_NUM; listId++)
723
5.64M
      {
724
39.4M
        for(uint32_t qp = 0; qp < SCALING_LIST_REM_NUM; qp++)
725
33.8M
        {
726
33.8M
          if(m_quantCoef[sizeIdX][sizeIdY][listId][qp])
727
0
          {
728
0
            delete [] m_quantCoef[sizeIdX][sizeIdY][listId][qp];
729
0
          }
730
33.8M
          if(m_dequantCoef[sizeIdX][sizeIdY][listId][qp])
731
0
          {
732
0
            delete [] m_dequantCoef[sizeIdX][sizeIdY][listId][qp];
733
0
          }
734
33.8M
        }
735
5.64M
      }
736
940k
    }
737
134k
  }
738
19.1k
}
739
740
void Quant::quant(TransformUnit& tu, const ComponentID compID, const CCoeffBuf& pSrc, TCoeff &uiAbsSum, const QpParam& cQP, const Ctx& ctx)
741
5.66k
{
742
5.66k
  const SPS &sps            = *tu.cs->sps;
743
5.66k
  const CompArea& rect      = tu.blocks[compID];
744
5.66k
  const uint32_t uiWidth    = rect.width;
745
5.66k
  const uint32_t uiHeight   = rect.height;
746
5.66k
  const int channelBitDepth = sps.bitDepths[toChannelType(compID)];
747
748
5.66k
  const CCoeffBuf&  piCoef  = pSrc;
749
5.66k
        CoeffSigBuf piQCoef = tu.getCoeffs(compID);
750
751
5.66k
  const bool useTransformSkip = tu.mtsIdx[compID] == MTS_SKIP;
752
5.66k
  const int  maxLog2TrDynamicRange = sps.getMaxLog2TrDynamicRange();
753
754
5.66k
  {
755
5.66k
    CoeffCodingContext cctx(tu, compID, tu.cs->slice->signDataHidingEnabled);
756
757
5.66k
    const TCoeff entropyCodingMinimum = -(1 << maxLog2TrDynamicRange);
758
5.66k
    const TCoeff entropyCodingMaximum =  (1 << maxLog2TrDynamicRange) - 1;
759
760
5.66k
    TCoeff deltaU[MAX_TB_SIZEY * MAX_TB_SIZEY];
761
5.66k
    int scalingListType           = getScalingListType(tu.cu->predMode, compID);
762
5.66k
    CHECK(scalingListType >= SCALING_LIST_NUM, "Invalid scaling list");
763
5.66k
    const uint32_t uiLog2TrWidth  = Log2(uiWidth);
764
5.66k
    const uint32_t uiLog2TrHeight = Log2(uiHeight);
765
5.66k
    int *piQuantCoeff             = getQuantCoeff(scalingListType, cQP.rem(useTransformSkip), uiLog2TrWidth, uiLog2TrHeight);
766
767
5.66k
    const bool isLfnstApplied     = tu.cu->lfnstIdx > 0 && (CU::isSepTree(*tu.cu) ? true : isLuma(compID));
768
5.66k
    const bool enableScalingLists = getUseScalingList(uiWidth, uiHeight, useTransformSkip, isLfnstApplied);
769
770
    // for blocks that where width*height != 4^N, the effective scaling applied during transformation cannot be
771
    // compensated by a bit-shift (the quantised result will be sqrt(2) * larger than required).
772
    // The quantScale table and shift is used to compensate for this.
773
5.66k
    const bool needSqrtAdjustment= TU::needsSqrt2Scale( tu, compID );
774
5.66k
    const int defaultQuantisationCoefficient    = g_quantScales[needSqrtAdjustment?1:0][cQP.rem(useTransformSkip)];
775
5.66k
    const int iTransformShift = getTransformShift(channelBitDepth, rect.size(), maxLog2TrDynamicRange) + ( needSqrtAdjustment?-1:0);
776
777
5.66k
    const int iQBits = QUANT_SHIFT + cQP.per(useTransformSkip) + (useTransformSkip ? 0 : iTransformShift);
778
    // QBits will be OK for any internal bit depth as the reduction in transform shift is balanced by an increase in Qp_per due to QpBDOffset
779
780
5.66k
    const int64_t iAdd = int64_t(tu.cs->slice->isIRAP() ? 171 : 85) << int64_t(iQBits - 9);
781
5.66k
    const int qBits8 = iQBits - 8;
782
783
5.66k
    int lastScanPos = -1;
784
785
5.66k
    if (!enableScalingLists)
786
5.66k
      xQuant(tu,compID,piCoef,piQCoef,uiAbsSum,lastScanPos,deltaU,defaultQuantisationCoefficient,iQBits,iAdd,entropyCodingMinimum,entropyCodingMaximum,cctx.signHiding(),m_thrVal);
787
0
    else
788
0
    {
789
0
      const uint32_t lfnstIdx = tu.cu->lfnstIdx;
790
0
      const int maxNumberOfCoeffs = lfnstIdx > 0 ? ( ( ( uiWidth == 4 && uiHeight == 4 ) || ( uiWidth == 8 && uiHeight == 8 ) ) ? 8 : 16 ) : piQCoef.area();
791
792
0
      piQCoef.memset( 0 );
793
0
      for (int uiScanPos = 0; uiScanPos < maxNumberOfCoeffs; uiScanPos++ )
794
0
      {
795
0
        const int uiBlockPos  = cctx.blockPos( uiScanPos );
796
0
        const TCoeff iLevel   = piCoef.buf[uiBlockPos];
797
0
        const TCoeff iSign    = (iLevel < 0 ? -1: 1);
798
799
0
        const int64_t  tmpLevel = (int64_t)abs(iLevel) * (enableScalingLists ? piQuantCoeff[uiBlockPos] : defaultQuantisationCoefficient);
800
0
        const TCoeff quantisedMagnitude = TCoeff((tmpLevel + iAdd ) >> iQBits);
801
0
        deltaU[uiBlockPos] = (TCoeff)((tmpLevel - ((int64_t)quantisedMagnitude<<iQBits) )>> qBits8);
802
803
0
        uiAbsSum += quantisedMagnitude;
804
0
        const TCoeff quantisedCoefficient = quantisedMagnitude * iSign;
805
806
0
        piQCoef.buf[uiBlockPos] = Clip3<TCoeff>( entropyCodingMinimum, entropyCodingMaximum, quantisedCoefficient );
807
0
      } // for n
808
0
    }
809
5.66k
    if (tu.cu->bdpcmM[toChannelType(compID)])
810
5.66k
    {
811
5.66k
      fwdResDPCM( tu, compID );
812
5.66k
    }
813
814
5.66k
    if( uiAbsSum )
815
3.39k
    {
816
6.79k
      for( int scanPos = lastScanPos; scanPos >= 0; scanPos-- )
817
6.79k
      {
818
6.79k
        unsigned blkPos = cctx.blockPos( scanPos );
819
6.79k
        if( piQCoef.buf[blkPos] )
820
3.39k
        {
821
3.39k
          lastScanPos = scanPos;
822
3.39k
          break;
823
3.39k
        }
824
6.79k
      }
825
826
3.39k
      if( cctx.signHiding() )
827
0
      {
828
0
        if( uiAbsSum >= 2 ) //this prevents TUs with only one coefficient of value 1 from being tested
829
0
        {
830
0
          xSignBitHidingHDQ( piQCoef.buf, piCoef.buf, deltaU, cctx, lastScanPos, maxLog2TrDynamicRange );
831
0
        }
832
0
      }
833
3.39k
    }
834
835
5.66k
    tu.lastPos[compID] = lastScanPos;
836
5.66k
  } //if RDOQ
837
  //return;
838
5.66k
}
839
840
bool Quant::xNeedRDOQ(TransformUnit& tu, const ComponentID compID, const CCoeffBuf& pSrc, const QpParam& cQP)
841
0
{
842
0
  const SPS &sps            = *tu.cs->sps;
843
0
  const CompArea& rect      = tu.blocks[compID];
844
0
  const uint32_t uiWidth    = rect.width;
845
0
  const uint32_t uiHeight   = rect.height;
846
0
  const uint32_t efHeight   = std::min<unsigned>( uiHeight, JVET_C0024_ZERO_OUT_TH );
847
0
  const uint32_t efArea     = uiWidth * efHeight;
848
0
  const int channelBitDepth = sps.bitDepths[toChannelType(compID)];
849
0
  const CCoeffBuf piCoef    = pSrc;
850
851
0
  const bool useTransformSkip      = tu.mtsIdx[compID] == MTS_SKIP;
852
0
  const int  maxLog2TrDynamicRange = sps.getMaxLog2TrDynamicRange();
853
854
0
  const int scalingListType     = getScalingListType( tu.cu->predMode, compID );
855
0
  CHECK( scalingListType >= SCALING_LIST_NUM, "Invalid scaling list" );
856
857
0
  const bool        isDq        = tu.cs->slice->depQuantEnabled && !useTransformSkip;
858
0
  const int         qpDQ        = isDq ? cQP.Qp( false ) + 1 : cQP.Qp( useTransformSkip );
859
0
  const int         qpPer       = isDq ? qpDQ / 6 : cQP.per( useTransformSkip );
860
0
  const int         qpRem       = isDq ? qpDQ - 6 * qpPer : cQP.rem( useTransformSkip );
861
862
0
  const uint32_t uiLog2TrWidth  = Log2( uiWidth );
863
0
  const uint32_t uiLog2TrHeight = Log2( uiHeight );
864
0
  int *piQuantCoeff             = getQuantCoeff( scalingListType, qpRem, uiLog2TrWidth, uiLog2TrHeight );
865
866
0
  const bool isLfnstApplied     = tu.cu->lfnstIdx > 0 && ( CU::isSepTree( *tu.cu ) ? true : isLuma( compID ) );
867
0
  const bool enableScalingLists = getUseScalingList( uiWidth, uiHeight, ( useTransformSkip != 0 ), isLfnstApplied );
868
869
0
  const bool needSqrtAdjustment = TU::needsSqrt2Scale( tu, compID );
870
0
  const int defaultQuantisationCoefficient
871
0
                                = g_quantScales[needSqrtAdjustment?1:0][qpRem];
872
0
  const int iTransformShift     = getTransformShift( channelBitDepth, rect.size(), maxLog2TrDynamicRange ) + ( needSqrtAdjustment ? -1 : 0 );
873
874
875
0
  const int iQBits              = QUANT_SHIFT + qpPer + iTransformShift;
876
877
  // QBits will be OK for any internal bit depth as the reduction in transform shift is balanced by an increase in Qp_per due to QpBDOffset
878
  // iAdd is different from the iAdd used in normal quantization
879
0
  const int64_t iAdd = int64_t( compID == COMP_Y ? 171 : 256 ) << ( iQBits - 9 );
880
881
0
  if( !enableScalingLists )
882
0
    return xNeedRdoq( piCoef.buf, efArea, defaultQuantisationCoefficient, iAdd, iQBits );
883
884
0
  for( int uiBlockPos = 0; uiBlockPos < efArea; uiBlockPos++ )
885
0
  {
886
0
    const TCoeff   iLevel           = piCoef.buf[uiBlockPos];
887
0
    const int64_t  tmpLevel         = ( int64_t ) std::abs( iLevel ) * piQuantCoeff[uiBlockPos];
888
0
    const TCoeff quantisedMagnitude = TCoeff( ( tmpLevel + iAdd ) >> iQBits );
889
890
0
    if( quantisedMagnitude != 0 )
891
0
    {
892
0
      return true;
893
0
    }
894
0
  } // for n
895
0
  return false;
896
0
}
897
898
} // namespace vvenc
899
900
//! \}
901