Coverage Report

Created: 2026-04-01 07:49

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/vvenc/source/Lib/CommonLib/Quant.cpp
Line
Count
Source
1
/* -----------------------------------------------------------------------------
2
The copyright in this software is being made available under the Clear BSD
3
License, included below. No patent rights, trademark rights and/or 
4
other Intellectual Property Rights other than the copyrights concerning 
5
the Software are granted under this license.
6
7
The Clear BSD License
8
9
Copyright (c) 2019-2026, Fraunhofer-Gesellschaft zur Förderung der angewandten Forschung e.V. & The VVenC Authors.
10
All rights reserved.
11
12
Redistribution and use in source and binary forms, with or without modification,
13
are permitted (subject to the limitations in the disclaimer below) provided that
14
the following conditions are met:
15
16
     * Redistributions of source code must retain the above copyright notice,
17
     this list of conditions and the following disclaimer.
18
19
     * Redistributions in binary form must reproduce the above copyright
20
     notice, this list of conditions and the following disclaimer in the
21
     documentation and/or other materials provided with the distribution.
22
23
     * Neither the name of the copyright holder nor the names of its
24
     contributors may be used to endorse or promote products derived from this
25
     software without specific prior written permission.
26
27
NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY
28
THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
29
CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
30
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
31
PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
32
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
33
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
34
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
35
BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
36
IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
37
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
38
POSSIBILITY OF SUCH DAMAGE.
39
40
41
------------------------------------------------------------------------------------------- */
42
43
44
/** \file     Quant.cpp
45
    \brief    transform and quantization class
46
*/
47
48
#include "Quant.h"
49
#include "UnitTools.h"
50
#include "ContextModelling.h"
51
#include "CodingStructure.h"
52
#include "dtrace_buffer.h"
53
54
#include <stdlib.h>
55
#include <memory.h>
56
57
//! \ingroup CommonLib
58
//! \{
59
60
namespace vvenc {
61
62
// ====================================================================================================================
63
// Constants
64
// ====================================================================================================================
65
66
67
// ====================================================================================================================
68
// QpParam constructor
69
// ====================================================================================================================
70
 
71
QpParam::QpParam(const TransformUnit& tu, const ComponentID &compID, const bool allowACTQpoffset)
72
0
{
73
0
  const ChannelType chType = toChannelType( compID );
74
0
  const SPS        &sps    = *tu.cu->cs->sps;
75
0
  const int     qpBdOffset = sps.qpBDOffset[chType];
76
0
  const bool useJQP        = isChroma( compID ) && abs( TU::getICTMode( tu ) ) == 2;
77
0
  const ComponentID jCbCr  = useJQP ? COMP_JOINT_CbCr : compID;
78
  
79
0
        int chromaQpOffset = 0;
80
81
0
  if( isChroma( compID ) )
82
0
  {
83
0
    const PPS &pps  = *tu.cu->slice->pps;
84
0
    chromaQpOffset  = pps.chromaQpOffset              [jCbCr];
85
0
    chromaQpOffset += tu.cu->slice->sliceChromaQpDelta[jCbCr];
86
0
    chromaQpOffset += pps.getChromaQpOffsetListEntry( tu.cu->chromaQpAdj ).u.offset[int( jCbCr ) - 1];
87
0
  }
88
  
89
0
  int baseQp;
90
0
  int qpy        = tu.cu->qp;
91
  //bool skip      = tu.mtsIdx[compID] == MTS_SKIP;
92
93
0
  if( isLuma( compID ) )
94
0
  {
95
0
    baseQp = tu.cu->qp + qpBdOffset;
96
0
  }
97
0
  else
98
0
  {
99
0
    int qpi = Clip3( -qpBdOffset, MAX_QP, qpy );
100
0
    baseQp  = sps.chromaQpMappingTable.getMappedChromaQpValue( jCbCr, qpi );
101
0
    baseQp  = Clip3( -qpBdOffset, MAX_QP, baseQp + chromaQpOffset ) + qpBdOffset;
102
0
  }
103
104
0
  if( allowACTQpoffset && tu.cu->colorTransform )
105
0
  {
106
0
    baseQp += DELTA_QP_ACT[jCbCr];
107
0
  }
108
109
0
  baseQp = Clip3( 0, MAX_QP + qpBdOffset, baseQp );
110
111
  //if( !skip )
112
0
  {
113
0
    Qps [0] = baseQp;
114
0
    pers[0] = baseQp / 6;
115
0
    rems[0] = baseQp % 6;
116
0
  }
117
  //else
118
0
  {
119
0
    int internalMinusInputBitDepth = sps.internalMinusInputBitDepth[chType];
120
0
    int baseQpTS           = std::max( baseQp, 4 + 6 * internalMinusInputBitDepth );
121
122
0
    Qps [1] = baseQpTS;
123
0
    pers[1] = baseQpTS / 6;
124
0
    rems[1] = baseQpTS % 6;
125
0
  }
126
0
}
127
128
129
// ====================================================================================================================
130
// Quant class member functions
131
// ====================================================================================================================
132
static void QuantCore(const TransformUnit tu, const ComponentID compID, const CCoeffBuf& piCoef,CoeffSigBuf piQCoef,TCoeff &uiAbsSum, int &lastScanPos,TCoeff *deltaU,const int defaultQuantisationCoefficient,const int iQBits,const int64_t iAdd,const TCoeff entropyCodingMinimum,const TCoeff entropyCodingMaximum,const bool signHiding, const TCoeff m_thrVal)
133
0
{
134
0
  CoeffCodingContext cctx( tu, compID, signHiding );
135
136
0
  const CompArea &rect      = tu.blocks[compID];
137
0
  const uint32_t uiWidth    = rect.width;
138
0
  const uint32_t uiHeight   = rect.height;
139
140
  /* for 422 chroma blocks, the effective scaling applied during transformation is not a power of 2, hence it cannot be
141
  * implemented as a bit-shift (the quantised result will be sqrt(2) * larger than required). Alternatively, adjust the
142
  * uiLog2TrSize applied in iTransformShift, such that the result is 1/sqrt(2) the required result (i.e. smaller)
143
  * Then a QP+3 (sqrt(2)) or QP-3 (1/sqrt(2)) method could be used to get the required result
144
  */
145
146
0
  const uint32_t log2CGSize         = cctx.log2CGSize();
147
148
0
  uiAbsSum = 0;
149
150
0
  const int iCGSize   = 1 << log2CGSize;
151
152
0
  const uint32_t lfnstIdx = tu.cu->lfnstIdx;
153
0
  const int iCGNum   = lfnstIdx > 0 ? 1 : std::min<int>(JVET_C0024_ZERO_OUT_TH, uiWidth) * std::min<int>(JVET_C0024_ZERO_OUT_TH, uiHeight) >> cctx.log2CGSize();
154
0
  int       iScanPos = ( iCGNum << log2CGSize ) - 1;
155
156
0
  if( lfnstIdx > 0 && ( ( uiWidth == 4 && uiHeight == 4 ) || ( uiWidth == 8 && uiHeight == 8 ) ) )
157
0
  {
158
0
    iScanPos = 7;
159
0
  }
160
161
  // Find first non-zero coeff
162
0
  for( ; iScanPos > 0; iScanPos-- )
163
0
  {
164
0
    uint32_t uiBlkPos = cctx.blockPos( iScanPos );
165
0
    if( piCoef.buf[uiBlkPos] )
166
0
      break;
167
0
  }
168
169
  //////////////////////////////////////////////////////////////////////////
170
  //  Loop over sub-sets (coefficient groups)
171
  //////////////////////////////////////////////////////////////////////////
172
  
173
0
  TCoeff thres = 0, useThres = 0;
174
  
175
0
  if( iQBits )
176
0
    thres = TCoeff( ( int64_t( m_thrVal ) << ( iQBits - 1 ) ) );
177
0
  else
178
0
    thres = TCoeff( ( int64_t( m_thrVal >> 1 ) << iQBits ) );
179
180
0
  useThres = thres / ( defaultQuantisationCoefficient << 2 );
181
182
0
  const bool is4x4sbb = log2CGSize == 4 && cctx.log2CGWidth() == 2;
183
184
0
  int subSetId = iScanPos >> log2CGSize;
185
0
  for( ; subSetId >= 1; subSetId-- )
186
0
  {
187
0
    if( is4x4sbb && iScanPos >= 16 )
188
0
    {
189
0
      int  iScanPosinCG = iScanPos & ( iCGSize - 1 );
190
0
      bool allSmaller   = true;
191
192
0
      for( int xScanPosinCG = iScanPosinCG, xScanPos = iScanPos; allSmaller && xScanPosinCG >= 0; xScanPosinCG--, xScanPos-- )
193
0
      {
194
0
        const uint32_t uiBlkPos = cctx.blockPos( xScanPos );
195
0
        allSmaller &= abs( piCoef.buf[uiBlkPos] ) <= useThres;
196
0
      }
197
198
0
      if( allSmaller )
199
0
      {
200
0
        iScanPos    -= iScanPosinCG + 1;
201
0
        continue;
202
0
      }
203
0
      else
204
0
      {
205
0
        break;
206
0
      }
207
0
    }
208
0
  }
209
210
0
  const int qBits8 = iQBits - 8;
211
0
  piQCoef.memset( 0 );
212
213
0
  for( int currPos = 0; currPos <= iScanPos; currPos++ )
214
0
  {
215
0
    const int uiBlockPos  = cctx.blockPos( currPos );
216
0
    const TCoeff iLevel   = piCoef.buf[uiBlockPos];
217
0
    const TCoeff iSign    = (iLevel < 0 ? -1: 1);
218
219
0
    const int64_t  tmpLevel = (int64_t)abs(iLevel) * defaultQuantisationCoefficient;
220
0
    const TCoeff quantisedMagnitude = TCoeff((tmpLevel + iAdd ) >> iQBits);
221
0
    deltaU[uiBlockPos] = (TCoeff)((tmpLevel - ((int64_t)quantisedMagnitude<<iQBits) )>> qBits8);
222
223
0
    uiAbsSum += quantisedMagnitude;
224
0
    const TCoeff quantisedCoefficient = quantisedMagnitude * iSign;
225
226
0
    piQCoef.buf[uiBlockPos] = Clip3<TCoeff>( entropyCodingMinimum, entropyCodingMaximum, quantisedCoefficient );
227
0
  } // for n
228
229
0
  lastScanPos = iScanPos;
230
0
}
231
232
static void DeQuantCore(const int maxX,const int maxY,const int scale,const TCoeffSig* const piQCoef,const size_t piQCfStride,TCoeff   *const piCoef,const int rightShift,const int inputMaximum,const TCoeff transformMaximum)
233
0
{
234
0
  const int inputMinimum = -(inputMaximum+1);
235
0
  const TCoeff transformMinimum = -(transformMaximum+1);
236
0
  if (rightShift>0)
237
0
  {
238
0
    const Intermediate_Int iAdd = (Intermediate_Int) 1 << (rightShift - 1);
239
0
    for( int y = 0, n = 0; y <= maxY; y++)
240
0
    {
241
0
      for( int x = 0; x <= maxX; x++, n++ )
242
0
      {
243
0
        const TCoeff           clipQCoef = TCoeff(Clip3<Intermediate_Int>(inputMinimum, inputMaximum, piQCoef[x + y * piQCfStride]));
244
0
        Intermediate_Int iCoeffQ   = (Intermediate_Int(clipQCoef) * scale + iAdd) >> rightShift;
245
0
        piCoef[n] = TCoeff(Clip3<Intermediate_Int>(transformMinimum,transformMaximum,iCoeffQ));
246
0
      }
247
0
    }
248
0
  }
249
0
  else  // rightshift <0
250
0
  {
251
0
    int leftShift = -rightShift;
252
0
    for( int y = 0, n = 0; y <= maxY; y++)
253
0
    {
254
0
      for( int x = 0; x <= maxX; x++, n++ )
255
0
      {
256
0
        const TCoeff           clipQCoef = TCoeff(Clip3<Intermediate_Int>(inputMinimum, inputMaximum, piQCoef[x + y * piQCfStride]));
257
0
        const Intermediate_Int iCoeffQ   = (Intermediate_Int(clipQCoef) * scale) * (1 << leftShift);
258
0
        piCoef[n] = TCoeff(Clip3<Intermediate_Int>(transformMinimum,transformMaximum,iCoeffQ));
259
0
      }
260
0
    }
261
0
  }
262
0
}
263
264
static bool needRdoqCore( const TCoeff* pCoeff, size_t numCoeff, int quantCoeff, int64_t offset, int shift )
265
0
{
266
0
  for( int uiBlockPos = 0; uiBlockPos < numCoeff; uiBlockPos++ )
267
0
  {
268
0
    const TCoeff   iLevel = pCoeff[uiBlockPos];
269
0
    const int64_t  tmpLevel = ( int64_t ) std::abs( iLevel ) * quantCoeff;
270
0
    const TCoeff quantisedMagnitude = TCoeff( ( tmpLevel + offset ) >> shift );
271
272
0
    if( quantisedMagnitude != 0 )
273
0
    {
274
0
      return true;
275
0
    }
276
0
  } // for n
277
0
  return false;
278
0
}
279
280
281
0
Quant::Quant( const Quant* other, bool useScalingLists ) : m_RDOQ( 0 ), m_useRDOQTS( false ), m_dLambda( 0.0 )
282
0
{
283
0
  xInitScalingList( other, useScalingLists );
284
0
  xDeQuant  = DeQuantCore;
285
0
  xQuant    = QuantCore;
286
0
  xNeedRdoq = needRdoqCore;
287
0
#if defined( TARGET_SIMD_X86 ) && ENABLE_SIMD_OPT_QUANT
288
0
  initQuantX86();
289
0
#endif
290
291
0
}
292
293
Quant::~Quant()
294
0
{
295
0
  xDestroyScalingList();
296
0
}
297
298
void invResDPCM( const TransformUnit& tu, const ComponentID compID, CoeffSigBuf& dstBuf )
299
0
{
300
0
  const CompArea&    rect   = tu.blocks[compID];
301
0
  const int          wdt    = rect.width;
302
0
  const int          hgt    = rect.height;
303
0
  const CCoeffSigBuf coeffs = tu.getCoeffs(compID);
304
305
0
  const int      maxLog2TrDynamicRange = tu.cs->sps->getMaxLog2TrDynamicRange();
306
0
  const TCoeff   inputMinimum          = -(1 << maxLog2TrDynamicRange);
307
0
  const TCoeff   inputMaximum          =  (1 << maxLog2TrDynamicRange) - 1;
308
309
0
  const TCoeffSig* coef = &coeffs.buf[0];
310
0
        TCoeffSig* dst  = &dstBuf.buf[0];
311
312
0
  if ( tu.cu->bdpcmM[toChannelType(compID)] == 1)
313
0
  {
314
0
    for( int y = 0; y < hgt; y++ )
315
0
    {
316
0
      dst[0] = coef[0];
317
0
      for( int x = 1; x < wdt; x++ )
318
0
      {
319
0
        dst[x] = Clip3(inputMinimum, inputMaximum, TCoeff( dst[x - 1] ) + TCoeff( coef[x] ));
320
0
      }
321
0
      coef += coeffs.stride;
322
0
      dst += dstBuf.stride;
323
0
    }
324
0
  }
325
0
  else
326
0
  {
327
0
    for( int x = 0; x < wdt; x++ )
328
0
    {
329
0
      dst[x] = coef[x];
330
0
    }
331
0
    for( int y = 0; y < hgt - 1; y++ )
332
0
    {
333
0
      for( int x = 0; x < wdt; x++ )
334
0
      {
335
0
        dst[dstBuf.stride + x] = Clip3(inputMinimum, inputMaximum, TCoeff( dst[x] ) + TCoeff( coef[coeffs.stride + x] ));
336
0
      }
337
0
      coef += coeffs.stride;
338
0
      dst += dstBuf.stride;
339
0
    }
340
0
  }
341
0
}
342
343
void fwdResDPCM( TransformUnit& tu, const ComponentID compID )
344
0
{
345
0
  const CompArea& rect   = tu.blocks[compID];
346
0
  const int       wdt    = rect.width;
347
0
  const int       hgt    = rect.height;
348
0
  CoeffSigBuf     coeffs = tu.getCoeffs(compID);
349
350
0
  TCoeffSig* coef = &coeffs.buf[0];
351
0
  if (tu.cu->bdpcmM[toChannelType(compID)] == 1)
352
0
  {
353
0
    for( int y = 0; y < hgt; y++ )
354
0
    {
355
0
      for( int x = wdt - 1; x > 0; x-- )
356
0
      {
357
0
        coef[x] -= coef[x - 1];
358
0
      }
359
0
      coef += coeffs.stride;
360
0
    }
361
0
  }
362
0
  else
363
0
  {
364
0
    coef += coeffs.stride * (hgt - 1);
365
0
    for( int y = 0; y < hgt - 1; y++ )
366
0
    {
367
0
      for ( int x = 0; x < wdt; x++ )
368
0
      {
369
0
        coef[x] -= coef[x - coeffs.stride];
370
0
      }
371
0
      coef -= coeffs.stride;
372
0
    }
373
0
  }
374
0
}
375
376
// To minimize the distortion only. No rate is considered.
377
void Quant::xSignBitHidingHDQ( TCoeffSig* pQCoef, const TCoeff* pCoef, TCoeff* deltaU, const CoeffCodingContext& cctx, int& lastScanPos, const int maxLog2TrDynamicRange )
378
0
{
379
0
  const uint32_t groupSize = 1 << cctx.log2CGSize();
380
381
0
  const TCoeff entropyCodingMinimum = -(1 << maxLog2TrDynamicRange);
382
0
  const TCoeff entropyCodingMaximum =  (1 << maxLog2TrDynamicRange) - 1;
383
384
0
  int lastCG = -1;
385
0
  int absSum = 0 ;
386
0
  int n ;
387
388
0
  for( int subSet = lastScanPos >> cctx.log2CGSize(); subSet >= 0; subSet-- )
389
0
  {
390
0
    int  subPos = subSet << cctx.log2CGSize();
391
0
    int  firstNZPosInCG=groupSize , lastNZPosInCG=-1 ;
392
0
    absSum = 0 ;
393
394
0
    for(n = groupSize-1; n >= 0; --n )
395
0
    {
396
0
      if( pQCoef[ cctx.blockPos( n + subPos ) ] )
397
0
      {
398
0
        lastNZPosInCG = n;
399
0
        break;
400
0
      }
401
0
    }
402
403
0
    for(n = 0; n <groupSize; n++ )
404
0
    {
405
0
      if( pQCoef[ cctx.blockPos( n + subPos ) ] )
406
0
      {
407
0
        firstNZPosInCG = n;
408
0
        break;
409
0
      }
410
0
    }
411
412
0
    for(n = firstNZPosInCG; n <=lastNZPosInCG; n++ )
413
0
    {
414
0
      absSum += int(pQCoef[ cctx.blockPos( n + subPos ) ]);
415
0
    }
416
417
0
    if(lastNZPosInCG>=0 && lastCG==-1)
418
0
    {
419
0
      lastCG = 1 ;
420
0
    }
421
422
0
    if( lastNZPosInCG-firstNZPosInCG>=SBH_THRESHOLD )
423
0
    {
424
0
      uint32_t signbit = (pQCoef[cctx.blockPos(subPos+firstNZPosInCG)]>0?0:1) ;
425
0
      if( signbit!=(absSum&0x1) )  //compare signbit with sum_parity
426
0
      {
427
0
        TCoeff curCost    = std::numeric_limits<TCoeff>::max();
428
0
        TCoeff minCostInc = std::numeric_limits<TCoeff>::max();
429
0
        int minPos =-1, finalChange=0, curChange=0, minScanPos = -1;
430
431
0
        for( n = (lastCG==1?lastNZPosInCG:groupSize-1) ; n >= 0; --n )
432
0
        {
433
0
          uint32_t blkPos   = cctx.blockPos( n+subPos );
434
0
          if(pQCoef[ blkPos ] != 0 )
435
0
          {
436
0
            if(deltaU[blkPos]>0)
437
0
            {
438
0
              curCost = - deltaU[blkPos];
439
0
              curChange=1 ;
440
0
            }
441
0
            else
442
0
            {
443
              //curChange =-1;
444
0
              if(n==firstNZPosInCG && abs(pQCoef[blkPos])==1)
445
0
              {
446
0
                curCost = std::numeric_limits<TCoeff>::max();
447
0
              }
448
0
              else
449
0
              {
450
0
                curCost = deltaU[blkPos];
451
0
                curChange =-1;
452
0
              }
453
0
            }
454
0
          }
455
0
          else
456
0
          {
457
0
            if(n<firstNZPosInCG)
458
0
            {
459
0
              uint32_t thisSignBit = (pCoef[blkPos]>=0?0:1);
460
0
              if(thisSignBit != signbit )
461
0
              {
462
0
                curCost = std::numeric_limits<TCoeff>::max();
463
0
              }
464
0
              else
465
0
              {
466
0
                curCost = - (deltaU[blkPos])  ;
467
0
                curChange = 1 ;
468
0
              }
469
0
            }
470
0
            else
471
0
            {
472
0
              curCost = - (deltaU[blkPos])  ;
473
0
              curChange = 1 ;
474
0
            }
475
0
          }
476
477
0
          if( curCost<minCostInc)
478
0
          {
479
0
            minCostInc = curCost ;
480
0
            finalChange = curChange ;
481
0
            minPos = blkPos;
482
0
            minScanPos = n + subPos;
483
0
          }
484
0
        } //CG loop
485
486
0
        if(pQCoef[minPos] == entropyCodingMaximum || pQCoef[minPos] == entropyCodingMinimum)
487
0
        {
488
0
          finalChange = -1;
489
0
        }
490
491
0
        if(pCoef[minPos]>=0)
492
0
        {
493
0
          pQCoef[minPos] += finalChange ;
494
0
        }
495
0
        else
496
0
        {
497
0
          pQCoef[minPos] -= finalChange ;
498
0
        }
499
500
        // if changing lastScanPos element to 0, move the pointer to the new lastScanPos element
501
0
        if( minScanPos == lastScanPos && pQCoef[minPos] == 0 )
502
0
        {
503
0
          for( ; lastScanPos >= 0 && pQCoef[cctx.blockPos( lastScanPos )] == 0; lastScanPos-- );
504
0
        }
505
0
        else if( minScanPos > lastScanPos && pQCoef[minPos] != 0 )
506
0
        {
507
0
          lastScanPos = minPos;
508
0
        }
509
0
      } // Hide
510
0
    }
511
0
    if(lastCG==1)
512
0
    {
513
0
      lastCG=0 ;
514
0
    }
515
0
  } // TU loop
516
517
0
  return;
518
0
}
519
520
void Quant::dequant(const TransformUnit& tu,
521
                          CoeffBuf&      dstCoeff,
522
                    const ComponentID    compID,
523
                    const QpParam&       cQP)
524
0
{
525
0
  const SPS       *sps                  = tu.cs->sps;
526
0
  const CompArea  &area                 = tu.blocks[compID];
527
0
  const uint32_t  uiWidth               = area.width;
528
0
  const uint32_t  uiHeight              = area.height;
529
0
  TCoeff *const   piCoef                = dstCoeff.buf;
530
0
  const uint32_t  numSamplesInBlock     = uiWidth * uiHeight;
531
0
  const int       maxLog2TrDynamicRange = sps->getMaxLog2TrDynamicRange();
532
0
  const TCoeff    transformMinimum      = -(1 << maxLog2TrDynamicRange);
533
0
  const TCoeff    transformMaximum      =  (1 << maxLog2TrDynamicRange) - 1;
534
0
  const bool      isTransformSkip       = tu.mtsIdx[compID] == MTS_SKIP;
535
0
  const bool      isLfnstApplied        = tu.cu->lfnstIdx > 0 && (CU::isSepTree(*tu.cu) ? true : isLuma(compID));
536
0
  const bool      enableScalingLists    = getUseScalingList(uiWidth, uiHeight, isTransformSkip, isLfnstApplied);
537
0
  const int       scalingListType       = getScalingListType(tu.cu->predMode, compID);
538
0
  const int       channelBitDepth       = sps->bitDepths[toChannelType(compID)];
539
540
0
  const TCoeffSig *coef     = tu.getCoeffs( compID ).buf;
541
0
  const ptrdiff_t  piStride = tu.getCoeffs( compID ).stride;
542
543
0
  if( tu.cu->bdpcmM[toChannelType( compID )] )
544
0
  {
545
0
    CoeffSigBuf coefBuf( m_tmpBdpcm, uiWidth, uiHeight );
546
0
    invResDPCM( tu, compID, coefBuf );
547
0
    coef      = m_tmpBdpcm;
548
0
  }
549
550
0
  const TCoeffSig  *const piQCoef = coef;
551
0
  CHECK(scalingListType >= SCALING_LIST_NUM, "Invalid scaling list");
552
553
  // Represents scaling through forward transform
554
0
  const int  originalTransformShift = getTransformShift(channelBitDepth, area.size(), maxLog2TrDynamicRange);
555
0
  const bool needSqrtAdjustment     = TU::needsSqrt2Scale( tu, compID );
556
0
  const int  iTransformShift        = originalTransformShift + (needSqrtAdjustment?-1:0);
557
558
0
  const int QP_per = cQP.per(isTransformSkip);
559
0
  const int QP_rem = cQP.rem(isTransformSkip);
560
561
0
  const int  rightShift = (IQUANT_SHIFT - ((isTransformSkip ? 0 : iTransformShift) + QP_per)) + (enableScalingLists ? LOG2_SCALING_LIST_NEUTRAL_VALUE : 0);
562
563
0
  if(enableScalingLists)
564
0
  {
565
    //from the dequantization equation:
566
    //iCoeffQ                         = ((Intermediate_Int(clipQCoef) * piDequantCoef[deQuantIdx]) + iAdd ) >> rightShift
567
    //(sizeof(Intermediate_Int) * 8)  =              inputBitDepth    +    dequantCoefBits                   - rightShift
568
0
    const uint32_t             dequantCoefBits     = 1 + IQUANT_SHIFT + SCALING_LIST_BITS;
569
0
    const uint32_t             targetInputBitDepth = std::min<uint32_t>((maxLog2TrDynamicRange + 1), (((sizeof(Intermediate_Int) * 8) + rightShift) - dequantCoefBits));
570
571
0
    const Intermediate_Int inputMinimum        = -(1 << (targetInputBitDepth - 1));
572
0
    const Intermediate_Int inputMaximum        =  (1 << (targetInputBitDepth - 1)) - 1;
573
574
0
    const uint32_t uiLog2TrWidth  = Log2(uiWidth);
575
0
    const uint32_t uiLog2TrHeight = Log2(uiHeight);
576
0
    int* piDequantCoef            = getDequantCoeff(scalingListType, QP_rem, uiLog2TrWidth, uiLog2TrHeight);
577
578
0
    if(rightShift > 0)
579
0
    {
580
0
      const Intermediate_Int iAdd = (Intermediate_Int) 1 << (rightShift - 1);
581
0
      for( int n = 0; n < numSamplesInBlock; n++ )
582
0
      {
583
0
        const TCoeff           clipQCoef = TCoeff(Clip3<Intermediate_Int>(inputMinimum, inputMaximum, piQCoef[n]));
584
0
        const Intermediate_Int iCoeffQ   = ((Intermediate_Int(clipQCoef) * piDequantCoef[n]) + iAdd ) >> rightShift;
585
0
        piCoef[n] = TCoeff(Clip3<Intermediate_Int>(transformMinimum,transformMaximum,iCoeffQ));
586
0
      }
587
0
    }
588
0
    else
589
0
    {
590
0
      const int leftShift = -rightShift;
591
0
      for( int n = 0; n < numSamplesInBlock; n++ )
592
0
      {
593
0
        const TCoeff           clipQCoef = TCoeff(Clip3<Intermediate_Int>(inputMinimum, inputMaximum, piQCoef[n]));
594
0
        const Intermediate_Int iCoeffQ   = (Intermediate_Int(clipQCoef) * piDequantCoef[n]) << leftShift;
595
0
        piCoef[n] = TCoeff(Clip3<Intermediate_Int>(transformMinimum,transformMaximum,iCoeffQ));
596
0
      }
597
0
    }
598
0
  }
599
0
  else
600
0
  {
601
0
    const int scale     = g_invQuantScales[needSqrtAdjustment?1:0][QP_rem];
602
0
    const int scaleBits = ( IQUANT_SHIFT + 1 );
603
    //from the dequantisation equation:
604
    //iCoeffQ                         = Intermediate_Int((int64_t(clipQCoef) * scale + iAdd) >> rightShift);
605
    //(sizeof(Intermediate_Int) * 8)  =                    inputBitDepth   + scaleBits      - rightShift
606
0
    const uint32_t             targetInputBitDepth = std::min<uint32_t>((maxLog2TrDynamicRange + 1), (((sizeof(Intermediate_Int) * 8) + rightShift) - scaleBits));
607
0
    const Intermediate_Int inputMaximum        =  (1 << (targetInputBitDepth - 1)) - 1;
608
0
    xDeQuant(uiWidth-1,uiHeight-1,scale,piQCoef,piStride,piCoef,rightShift,inputMaximum,transformMaximum);
609
0
  }
610
0
}
611
612
void Quant::init( int rdoq, bool bUseRDOQTS, int thrVal )
613
0
{
614
615
  // TODO: pass to init() a single variable containing (quantization) flags,
616
  //       instead of variables that don't have to do with this class
617
618
0
  m_RDOQ             = rdoq;
619
0
  m_useRDOQTS        = bUseRDOQTS;
620
0
  m_thrVal           = thrVal;
621
0
}
622
623
/** set flat matrix value to quantized coefficient
624
 */
625
void Quant::setFlatScalingList(const int maxLog2TrDynamicRange[MAX_NUM_CH], const BitDepths &bitDepths )
626
0
{
627
0
  if( !m_scalingListEnabled ) return;
628
629
0
  const int minimumQp = 0;
630
0
  const int maximumQp = SCALING_LIST_REM_NUM;
631
632
0
  for(uint32_t sizeX = 0; sizeX < SCALING_LIST_SIZE_NUM; sizeX++)
633
0
  {
634
0
    for(uint32_t sizeY = 0; sizeY < SCALING_LIST_SIZE_NUM; sizeY++)
635
0
    {
636
0
      for(uint32_t list = 0; list < SCALING_LIST_NUM; list++)
637
0
      {
638
0
        for(int qp = minimumQp; qp < maximumQp; qp++)
639
0
        {
640
0
          xSetFlatScalingList( list, sizeX, sizeY, qp );
641
0
        }
642
0
      }
643
0
    }
644
0
  }
645
0
}
646
647
/** set flat matrix value to quantized coefficient
648
 * \param list List ID
649
 * \param size size index
650
 * \param qp Quantization parameter
651
 * \param format chroma format
652
 */
653
void Quant::xSetFlatScalingList(uint32_t list, uint32_t sizeX, uint32_t sizeY, int qp )
654
0
{
655
0
  uint32_t i,num = g_scalingListSizeX[sizeX]*g_scalingListSizeX[sizeY];
656
0
  int *quantcoeff;
657
0
  int *dequantcoeff;
658
659
0
  const bool blockIsNotPowerOf4 = ((Log2(g_scalingListSizeX[sizeX] * g_scalingListSizeX[sizeY])) & 1) == 1;
660
0
  int quantScales    = g_quantScales   [blockIsNotPowerOf4?1:0][qp];
661
0
  int invQuantScales = g_invQuantScales[blockIsNotPowerOf4?1:0][qp] << 4;
662
663
0
  quantcoeff   = getQuantCoeff(list, qp, sizeX, sizeY);
664
0
  dequantcoeff = getDequantCoeff(list, qp, sizeX, sizeY);
665
666
0
  for(i=0;i<num;i++)
667
0
  {
668
0
    *quantcoeff++ = quantScales;
669
0
    *dequantcoeff++ = invQuantScales;
670
0
  }
671
0
}
672
673
674
/** initialization process of scaling list array
675
 */
676
void Quant::xInitScalingList( const Quant* other, bool useScalingLists )
677
0
{
678
0
  m_isScalingListOwner = other == nullptr;
679
0
  m_scalingListEnabled = useScalingLists;
680
681
0
  for(uint32_t sizeIdX = 0; sizeIdX < SCALING_LIST_SIZE_NUM; sizeIdX++)
682
0
  {
683
0
    for(uint32_t sizeIdY = 0; sizeIdY < SCALING_LIST_SIZE_NUM; sizeIdY++)
684
0
    {
685
0
      for(uint32_t qp = 0; qp < SCALING_LIST_REM_NUM; qp++)
686
0
      {
687
0
        for(uint32_t listId = 0; listId < SCALING_LIST_NUM; listId++)
688
0
        {
689
0
          if( m_isScalingListOwner )
690
0
          {
691
0
            const size_t scalingListSize = g_scalingListSizeX[sizeIdX] * g_scalingListSizeX[sizeIdY];
692
693
0
            m_quantCoef   [sizeIdX][sizeIdY][listId][qp] = useScalingLists ? new int[scalingListSize] : nullptr;
694
0
            m_dequantCoef [sizeIdX][sizeIdY][listId][qp] = useScalingLists ? new int[scalingListSize] : nullptr;
695
0
          }
696
0
          else
697
0
          {
698
0
            m_quantCoef   [sizeIdX][sizeIdY][listId][qp] = other->m_quantCoef   [sizeIdX][sizeIdY][listId][qp];
699
0
            m_dequantCoef [sizeIdX][sizeIdY][listId][qp] = other->m_dequantCoef [sizeIdX][sizeIdY][listId][qp];
700
0
          }
701
0
        } // listID loop
702
0
      }
703
0
    }
704
0
  }
705
0
}
706
707
/** destroy quantization matrix array
708
 */
709
void Quant::xDestroyScalingList()
710
0
{
711
0
  if( !m_isScalingListOwner ) return;
712
713
0
  for(uint32_t sizeIdX = 0; sizeIdX < SCALING_LIST_SIZE_NUM; sizeIdX++)
714
0
  {
715
0
    for(uint32_t sizeIdY = 0; sizeIdY < SCALING_LIST_SIZE_NUM; sizeIdY++)
716
0
    {
717
0
      for(uint32_t listId = 0; listId < SCALING_LIST_NUM; listId++)
718
0
      {
719
0
        for(uint32_t qp = 0; qp < SCALING_LIST_REM_NUM; qp++)
720
0
        {
721
0
          if(m_quantCoef[sizeIdX][sizeIdY][listId][qp])
722
0
          {
723
0
            delete [] m_quantCoef[sizeIdX][sizeIdY][listId][qp];
724
0
          }
725
0
          if(m_dequantCoef[sizeIdX][sizeIdY][listId][qp])
726
0
          {
727
0
            delete [] m_dequantCoef[sizeIdX][sizeIdY][listId][qp];
728
0
          }
729
0
        }
730
0
      }
731
0
    }
732
0
  }
733
0
}
734
735
void Quant::quant(TransformUnit& tu, const ComponentID compID, const CCoeffBuf& pSrc, TCoeff &uiAbsSum, const QpParam& cQP, const Ctx& ctx)
736
0
{
737
0
  const SPS &sps            = *tu.cs->sps;
738
0
  const CompArea& rect      = tu.blocks[compID];
739
0
  const uint32_t uiWidth    = rect.width;
740
0
  const uint32_t uiHeight   = rect.height;
741
0
  const int channelBitDepth = sps.bitDepths[toChannelType(compID)];
742
743
0
  const CCoeffBuf&  piCoef  = pSrc;
744
0
        CoeffSigBuf piQCoef = tu.getCoeffs(compID);
745
746
0
  const bool useTransformSkip = tu.mtsIdx[compID] == MTS_SKIP;
747
0
  const int  maxLog2TrDynamicRange = sps.getMaxLog2TrDynamicRange();
748
749
0
  {
750
0
    CoeffCodingContext cctx(tu, compID, tu.cs->slice->signDataHidingEnabled);
751
752
0
    const TCoeff entropyCodingMinimum = -(1 << maxLog2TrDynamicRange);
753
0
    const TCoeff entropyCodingMaximum =  (1 << maxLog2TrDynamicRange) - 1;
754
755
0
    TCoeff deltaU[MAX_TB_SIZEY * MAX_TB_SIZEY];
756
0
    int scalingListType           = getScalingListType(tu.cu->predMode, compID);
757
0
    CHECK(scalingListType >= SCALING_LIST_NUM, "Invalid scaling list");
758
0
    const uint32_t uiLog2TrWidth  = Log2(uiWidth);
759
0
    const uint32_t uiLog2TrHeight = Log2(uiHeight);
760
0
    int *piQuantCoeff             = getQuantCoeff(scalingListType, cQP.rem(useTransformSkip), uiLog2TrWidth, uiLog2TrHeight);
761
762
0
    const bool isLfnstApplied     = tu.cu->lfnstIdx > 0 && (CU::isSepTree(*tu.cu) ? true : isLuma(compID));
763
0
    const bool enableScalingLists = getUseScalingList(uiWidth, uiHeight, useTransformSkip, isLfnstApplied);
764
765
    // for blocks that where width*height != 4^N, the effective scaling applied during transformation cannot be
766
    // compensated by a bit-shift (the quantised result will be sqrt(2) * larger than required).
767
    // The quantScale table and shift is used to compensate for this.
768
0
    const bool needSqrtAdjustment= TU::needsSqrt2Scale( tu, compID );
769
0
    const int defaultQuantisationCoefficient    = g_quantScales[needSqrtAdjustment?1:0][cQP.rem(useTransformSkip)];
770
0
    const int iTransformShift = getTransformShift(channelBitDepth, rect.size(), maxLog2TrDynamicRange) + ( needSqrtAdjustment?-1:0);
771
772
0
    const int iQBits = QUANT_SHIFT + cQP.per(useTransformSkip) + (useTransformSkip ? 0 : iTransformShift);
773
    // QBits will be OK for any internal bit depth as the reduction in transform shift is balanced by an increase in Qp_per due to QpBDOffset
774
775
0
    const int64_t iAdd = int64_t(tu.cs->slice->isIRAP() ? 171 : 85) << int64_t(iQBits - 9);
776
0
    const int qBits8 = iQBits - 8;
777
778
0
    int lastScanPos = -1;
779
780
0
    if (!enableScalingLists)
781
0
      xQuant(tu,compID,piCoef,piQCoef,uiAbsSum,lastScanPos,deltaU,defaultQuantisationCoefficient,iQBits,iAdd,entropyCodingMinimum,entropyCodingMaximum,cctx.signHiding(),m_thrVal);
782
0
    else
783
0
    {
784
0
      const uint32_t lfnstIdx = tu.cu->lfnstIdx;
785
0
      const int maxNumberOfCoeffs = lfnstIdx > 0 ? ( ( ( uiWidth == 4 && uiHeight == 4 ) || ( uiWidth == 8 && uiHeight == 8 ) ) ? 8 : 16 ) : piQCoef.area();
786
787
0
      piQCoef.memset( 0 );
788
0
      for (int uiScanPos = 0; uiScanPos < maxNumberOfCoeffs; uiScanPos++ )
789
0
      {
790
0
        const int uiBlockPos  = cctx.blockPos( uiScanPos );
791
0
        const TCoeff iLevel   = piCoef.buf[uiBlockPos];
792
0
        const TCoeff iSign    = (iLevel < 0 ? -1: 1);
793
794
0
        const int64_t  tmpLevel = (int64_t)abs(iLevel) * (enableScalingLists ? piQuantCoeff[uiBlockPos] : defaultQuantisationCoefficient);
795
0
        const TCoeff quantisedMagnitude = TCoeff((tmpLevel + iAdd ) >> iQBits);
796
0
        deltaU[uiBlockPos] = (TCoeff)((tmpLevel - ((int64_t)quantisedMagnitude<<iQBits) )>> qBits8);
797
798
0
        uiAbsSum += quantisedMagnitude;
799
0
        const TCoeff quantisedCoefficient = quantisedMagnitude * iSign;
800
801
0
        piQCoef.buf[uiBlockPos] = Clip3<TCoeff>( entropyCodingMinimum, entropyCodingMaximum, quantisedCoefficient );
802
0
      } // for n
803
0
    }
804
0
    if (tu.cu->bdpcmM[toChannelType(compID)])
805
0
    {
806
0
      fwdResDPCM( tu, compID );
807
0
    }
808
809
0
    if( uiAbsSum )
810
0
    {
811
0
      for( int scanPos = lastScanPos; scanPos >= 0; scanPos-- )
812
0
      {
813
0
        unsigned blkPos = cctx.blockPos( scanPos );
814
0
        if( piQCoef.buf[blkPos] )
815
0
        {
816
0
          lastScanPos = scanPos;
817
0
          break;
818
0
        }
819
0
      }
820
821
0
      if( cctx.signHiding() )
822
0
      {
823
0
        if( uiAbsSum >= 2 ) //this prevents TUs with only one coefficient of value 1 from being tested
824
0
        {
825
0
          xSignBitHidingHDQ( piQCoef.buf, piCoef.buf, deltaU, cctx, lastScanPos, maxLog2TrDynamicRange );
826
0
        }
827
0
      }
828
0
    }
829
830
0
    tu.lastPos[compID] = lastScanPos;
831
0
  } //if RDOQ
832
  //return;
833
0
}
834
835
bool Quant::xNeedRDOQ(TransformUnit& tu, const ComponentID compID, const CCoeffBuf& pSrc, const QpParam& cQP)
836
0
{
837
0
  const SPS &sps            = *tu.cs->sps;
838
0
  const CompArea& rect      = tu.blocks[compID];
839
0
  const uint32_t uiWidth    = rect.width;
840
0
  const uint32_t uiHeight   = rect.height;
841
0
  const uint32_t efHeight   = std::min<unsigned>( uiHeight, JVET_C0024_ZERO_OUT_TH );
842
0
  const uint32_t efArea     = uiWidth * efHeight;
843
0
  const int channelBitDepth = sps.bitDepths[toChannelType(compID)];
844
0
  const CCoeffBuf piCoef    = pSrc;
845
846
0
  const bool useTransformSkip      = tu.mtsIdx[compID] == MTS_SKIP;
847
0
  const int  maxLog2TrDynamicRange = sps.getMaxLog2TrDynamicRange();
848
849
0
  const int scalingListType     = getScalingListType( tu.cu->predMode, compID );
850
0
  CHECK( scalingListType >= SCALING_LIST_NUM, "Invalid scaling list" );
851
852
0
  const bool        isDq        = tu.cs->slice->depQuantEnabled && !useTransformSkip;
853
0
  const int         qpDQ        = isDq ? cQP.Qp( false ) + 1 : cQP.Qp( useTransformSkip );
854
0
  const int         qpPer       = isDq ? qpDQ / 6 : cQP.per( useTransformSkip );
855
0
  const int         qpRem       = isDq ? qpDQ - 6 * qpPer : cQP.rem( useTransformSkip );
856
857
0
  const uint32_t uiLog2TrWidth  = Log2( uiWidth );
858
0
  const uint32_t uiLog2TrHeight = Log2( uiHeight );
859
0
  int *piQuantCoeff             = getQuantCoeff( scalingListType, qpRem, uiLog2TrWidth, uiLog2TrHeight );
860
861
0
  const bool isLfnstApplied     = tu.cu->lfnstIdx > 0 && ( CU::isSepTree( *tu.cu ) ? true : isLuma( compID ) );
862
0
  const bool enableScalingLists = getUseScalingList( uiWidth, uiHeight, ( useTransformSkip != 0 ), isLfnstApplied );
863
864
0
  const bool needSqrtAdjustment = TU::needsSqrt2Scale( tu, compID );
865
0
  const int defaultQuantisationCoefficient
866
0
                                = g_quantScales[needSqrtAdjustment?1:0][qpRem];
867
0
  const int iTransformShift     = getTransformShift( channelBitDepth, rect.size(), maxLog2TrDynamicRange ) + ( needSqrtAdjustment ? -1 : 0 );
868
869
870
0
  const int iQBits              = QUANT_SHIFT + qpPer + iTransformShift;
871
872
  // QBits will be OK for any internal bit depth as the reduction in transform shift is balanced by an increase in Qp_per due to QpBDOffset
873
  // iAdd is different from the iAdd used in normal quantization
874
0
  const int64_t iAdd = int64_t( compID == COMP_Y ? 171 : 256 ) << ( iQBits - 9 );
875
876
0
  if( !enableScalingLists )
877
0
    return xNeedRdoq( piCoef.buf, efArea, defaultQuantisationCoefficient, iAdd, iQBits );
878
879
0
  for( int uiBlockPos = 0; uiBlockPos < efArea; uiBlockPos++ )
880
0
  {
881
0
    const TCoeff   iLevel           = piCoef.buf[uiBlockPos];
882
0
    const int64_t  tmpLevel         = ( int64_t ) std::abs( iLevel ) * piQuantCoeff[uiBlockPos];
883
0
    const TCoeff quantisedMagnitude = TCoeff( ( tmpLevel + iAdd ) >> iQBits );
884
885
0
    if( quantisedMagnitude != 0 )
886
0
    {
887
0
      return true;
888
0
    }
889
0
  } // for n
890
0
  return false;
891
0
}
892
893
} // namespace vvenc
894
895
//! \}
896