Coverage Report

Created: 2026-08-13 07:04

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/opus/celt/bands.c
Line
Count
Source
1
/* Copyright (c) 2007-2008 CSIRO
2
   Copyright (c) 2007-2009 Xiph.Org Foundation
3
   Copyright (c) 2008-2009 Gregory Maxwell
4
   Written by Jean-Marc Valin and Gregory Maxwell */
5
/*
6
   Redistribution and use in source and binary forms, with or without
7
   modification, are permitted provided that the following conditions
8
   are met:
9
10
   - Redistributions of source code must retain the above copyright
11
   notice, this list of conditions and the following disclaimer.
12
13
   - Redistributions in binary form must reproduce the above copyright
14
   notice, this list of conditions and the following disclaimer in the
15
   documentation and/or other materials provided with the distribution.
16
17
   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18
   ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19
   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20
   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
21
   OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
22
   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
23
   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
24
   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
25
   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
26
   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27
   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
*/
29
30
#ifdef HAVE_CONFIG_H
31
#include "config.h"
32
#endif
33
34
#include <math.h>
35
#include "bands.h"
36
#include "modes.h"
37
#include "vq.h"
38
#include "cwrs.h"
39
#include "stack_alloc.h"
40
#include "os_support.h"
41
#include "mathops.h"
42
#include "rate.h"
43
#include "quant_bands.h"
44
#include "pitch.h"
45
46
int hysteresis_decision(opus_val16 val, const opus_val16 *thresholds, const opus_val16 *hysteresis, int N, int prev)
47
14.5M
{
48
14.5M
   int i;
49
218M
   for (i=0;i<N;i++)
50
218M
   {
51
218M
      if (val < thresholds[i])
52
14.5M
         break;
53
218M
   }
54
14.5M
   if (i>prev && val < thresholds[prev]+hysteresis[prev])
55
30.4k
      i=prev;
56
14.5M
   if (i<prev && val > thresholds[prev-1]-hysteresis[prev-1])
57
1.61k
      i=prev;
58
14.5M
   return i;
59
14.5M
}
60
61
opus_uint32 celt_lcg_rand(opus_uint32 seed)
62
731M
{
63
731M
   return 1664525 * seed + 1013904223;
64
731M
}
65
66
/* This is a cos() approximation designed to be bit-exact on any platform. Bit exactness
67
   with this approximation is important because it has an impact on the bit allocation */
68
opus_int16 bitexact_cos(opus_int16 x)
69
18.6M
{
70
18.6M
   opus_int32 tmp;
71
18.6M
   opus_int16 x2;
72
18.6M
   tmp = (4096+((opus_int32)(x)*(x)))>>13;
73
18.6M
   celt_sig_assert(tmp<=32767);
74
18.6M
   x2 = tmp;
75
18.6M
   x2 = (32767-x2) + FRAC_MUL16(x2, (-7651 + FRAC_MUL16(x2, (8277 + FRAC_MUL16(-626, x2)))));
76
18.6M
   celt_sig_assert(x2<=32766);
77
18.6M
   return 1+x2;
78
18.6M
}
79
80
int bitexact_log2tan(int isin,int icos)
81
9.34M
{
82
9.34M
   int lc;
83
9.34M
   int ls;
84
9.34M
   lc=EC_ILOG(icos);
85
9.34M
   ls=EC_ILOG(isin);
86
9.34M
   icos<<=15-lc;
87
9.34M
   isin<<=15-ls;
88
9.34M
   return (ls-lc)*(1<<11)
89
9.34M
         +FRAC_MUL16(isin, FRAC_MUL16(isin, -2597) + 7932)
90
9.34M
         -FRAC_MUL16(icos, FRAC_MUL16(icos, -2597) + 7932);
91
9.34M
}
92
93
#ifdef FIXED_POINT
94
/* Compute the amplitude (sqrt energy) in each of the bands */
95
void compute_band_energies(const CELTMode *m, const celt_sig *X, celt_ener *bandE, int end, int C, int LM, int arch)
96
{
97
   int i, c, N;
98
   const opus_int16 *eBands = m->eBands;
99
   (void)arch;
100
   N = m->shortMdctSize<<LM;
101
   c=0; do {
102
      for (i=0;i<end;i++)
103
      {
104
         int j;
105
         opus_val32 maxval=0;
106
         opus_val32 sum = 0;
107
108
         maxval = celt_maxabs32(&X[c*N+(eBands[i]<<LM)], (eBands[i+1]-eBands[i])<<LM);
109
         if (maxval > 0)
110
         {
111
            int shift = IMAX(0, 30 - celt_ilog2(maxval+(maxval>>14)+1) - ((((m->logN[i]+7)>>BITRES)+LM+1)>>1));
112
            j=eBands[i]<<LM; do {
113
               opus_val32 x = SHL32(X[j+c*N],shift);
114
               sum = ADD32(sum, MULT32_32_Q31(x, x));
115
            } while (++j<eBands[i+1]<<LM);
116
            bandE[i+c*m->nbEBands] = MAX32(maxval, PSHR32(celt_sqrt32(SHR32(sum,1)), shift));
117
         } else {
118
            bandE[i+c*m->nbEBands] = EPSILON;
119
         }
120
      }
121
   } while (++c<C);
122
}
123
124
/* Normalise each band such that the energy is one. */
125
void normalise_bands(const CELTMode *m, const celt_sig * OPUS_RESTRICT freq, celt_norm * OPUS_RESTRICT X, const celt_ener *bandE, int end, int C, int M)
126
{
127
   int i, c, N;
128
   const opus_int16 *eBands = m->eBands;
129
   N = M*m->shortMdctSize;
130
   c=0; do {
131
      i=0; do {
132
         int j,shift;
133
         opus_val32 E;
134
         opus_val32 g;
135
         E = bandE[i+c*m->nbEBands];
136
         /* For very low energies, we need this to make sure not to prevent energy rounding from
137
            blowing up the normalized signal. */
138
         if (E < 10) E += EPSILON;
139
         shift = 30-celt_zlog2(E);
140
         E = SHL32(E, shift);
141
         g = celt_rcp_norm32(E);
142
         j=M*eBands[i]; do {
143
            X[j+c*N] = PSHR32(MULT32_32_Q31(g, SHL32(freq[j+c*N], shift)), 30-NORM_SHIFT);
144
         } while (++j<M*eBands[i+1]);
145
      } while (++i<end);
146
   } while (++c<C);
147
}
148
149
#else /* FIXED_POINT */
150
/* Compute the amplitude (sqrt energy) in each of the bands */
151
void compute_band_energies(const CELTMode *m, const celt_sig *X, celt_ener *bandE, int end, int C, int LM, int arch)
152
49.0M
{
153
49.0M
   int i, c, N;
154
49.0M
   const opus_int16 *eBands = m->eBands;
155
49.0M
   N = m->shortMdctSize<<LM;
156
63.6M
   c=0; do {
157
1.06G
      for (i=0;i<end;i++)
158
1.00G
      {
159
1.00G
         opus_val32 sum;
160
1.00G
         sum = 1e-27f + celt_inner_prod(&X[c*N+(eBands[i]<<LM)], &X[c*N+(eBands[i]<<LM)], (eBands[i+1]-eBands[i])<<LM, arch);
161
1.00G
         bandE[i+c*m->nbEBands] = celt_sqrt(sum);
162
         /*printf ("%f ", bandE[i+c*m->nbEBands]);*/
163
1.00G
      }
164
63.6M
   } while (++c<C);
165
   /*printf ("\n");*/
166
49.0M
}
167
168
/* Normalise each band such that the energy is one. */
169
void normalise_bands(const CELTMode *m, const celt_sig * OPUS_RESTRICT freq, celt_norm * OPUS_RESTRICT X, const celt_ener *bandE, int end, int C, int M)
170
48.8M
{
171
48.8M
   int i, c, N;
172
48.8M
   const opus_int16 *eBands = m->eBands;
173
48.8M
   N = M*m->shortMdctSize;
174
63.3M
   c=0; do {
175
1.06G
      for (i=0;i<end;i++)
176
1.00G
      {
177
1.00G
         int j;
178
1.00G
         opus_val16 g = 1.f/(1e-27f+bandE[i+c*m->nbEBands]);
179
8.63G
         for (j=M*eBands[i];j<M*eBands[i+1];j++)
180
7.62G
            X[j+c*N] = freq[j+c*N]*g;
181
1.00G
      }
182
63.3M
   } while (++c<C);
183
48.8M
}
184
185
#endif /* FIXED_POINT */
186
187
/* De-normalise the energy to produce the synthesis from the unit-energy bands */
188
void denormalise_bands(const CELTMode *m, const celt_norm * OPUS_RESTRICT X,
189
      celt_sig * OPUS_RESTRICT freq, const celt_glog *bandLogE, int start,
190
      int end, int M, int downsample, int silence)
191
0
{
192
0
   int i, N;
193
0
   int bound;
194
0
   celt_sig * OPUS_RESTRICT f;
195
0
   const celt_norm * OPUS_RESTRICT x;
196
0
   const opus_int16 *eBands = m->eBands;
197
0
   N = M*m->shortMdctSize;
198
0
   bound = M*eBands[end];
199
0
   if (downsample!=1)
200
0
      bound = IMIN(bound, N/downsample);
201
0
   if (silence)
202
0
   {
203
0
      bound = 0;
204
0
      start = end = 0;
205
0
   }
206
0
   f = freq;
207
0
   x = X+M*eBands[start];
208
0
   if (start != 0)
209
0
   {
210
0
      for (i=0;i<M*eBands[start];i++)
211
0
         *f++ = 0;
212
0
   } else {
213
0
      f += M*eBands[start];
214
0
   }
215
0
   for (i=start;i<end;i++)
216
0
   {
217
0
      int j, band_end;
218
0
      opus_val32 g;
219
0
      celt_glog lg;
220
#ifdef FIXED_POINT
221
      int shift;
222
#endif
223
0
      j=M*eBands[i];
224
0
      band_end = M*eBands[i+1];
225
0
      lg = ADD32(bandLogE[i], SHL32((opus_val32)eMeans[i],DB_SHIFT-4));
226
0
#ifndef FIXED_POINT
227
0
      g = celt_exp2_db(MIN32(32.f, lg));
228
#else
229
      /* Handle the integer part of the log energy */
230
      shift = 17-(lg>>DB_SHIFT);
231
      if (shift>=31)
232
      {
233
         shift=0;
234
         g=0;
235
      } else {
236
         /* Handle the fractional part. */
237
         g = SHL32(celt_exp2_db_frac((lg&((1<<DB_SHIFT)-1))), 2);
238
      }
239
      /* Handle extreme gains with negative shift. */
240
      if (shift<0)
241
      {
242
         /* To avoid overflow, we're
243
            capping the gain here, which is equivalent to a cap of 18 on lg.
244
            This shouldn't trigger unless the bitstream is already corrupted. */
245
         g = 2147483647;
246
         shift = 0;
247
      }
248
#endif
249
0
      do {
250
0
         *f++ = PSHR32(MULT32_32_Q31(SHL32(*x, 30-NORM_SHIFT), g), shift);
251
0
         x++;
252
0
      } while (++j<band_end);
253
0
   }
254
0
   celt_assert(start <= end);
255
0
   OPUS_CLEAR(&freq[bound], N-bound);
256
0
}
257
258
/* This prevents energy collapse for transients with multiple short MDCTs */
259
void anti_collapse(const CELTMode *m, celt_norm *X_, unsigned char *collapse_masks, int LM, int C, int size,
260
      int start, int end, const celt_glog *logE, const celt_glog *prev1logE,
261
      const celt_glog *prev2logE, const int *pulses, opus_uint32 seed, int encode, int arch)
262
0
{
263
0
   int c, i, j, k;
264
0
   for (i=start;i<end;i++)
265
0
   {
266
0
      int N0;
267
0
      opus_val16 thresh, sqrt_1;
268
0
      int depth;
269
#ifdef FIXED_POINT
270
      int shift;
271
      opus_val32 thresh32;
272
#endif
273
274
0
      N0 = m->eBands[i+1]-m->eBands[i];
275
      /* depth in 1/8 bits */
276
0
      celt_sig_assert(pulses[i]>=0);
277
0
      depth = celt_udiv(1+pulses[i], (m->eBands[i+1]-m->eBands[i]))>>LM;
278
279
#ifdef FIXED_POINT
280
      thresh32 = SHR32(celt_exp2(-SHL16(depth, 10-BITRES)),1);
281
      thresh = MULT16_32_Q15(QCONST16(0.5f, 15), MIN32(32767,thresh32));
282
      {
283
         opus_val32 t;
284
         t = N0<<LM;
285
         shift = celt_ilog2(t)>>1;
286
         t = SHL32(t, (7-shift)<<1);
287
         sqrt_1 = celt_rsqrt_norm(t);
288
      }
289
#else
290
0
      thresh = .5f*celt_exp2(-.125f*depth);
291
0
      sqrt_1 = celt_rsqrt(N0<<LM);
292
0
#endif
293
294
0
      c=0; do
295
0
      {
296
0
         celt_norm *X;
297
0
         celt_glog prev1;
298
0
         celt_glog prev2;
299
0
         opus_val32 Ediff;
300
0
         celt_norm r;
301
0
         int renormalize=0;
302
0
         prev1 = prev1logE[c*m->nbEBands+i];
303
0
         prev2 = prev2logE[c*m->nbEBands+i];
304
0
         if (!encode && C==1)
305
0
         {
306
0
            prev1 = MAXG(prev1,prev1logE[m->nbEBands+i]);
307
0
            prev2 = MAXG(prev2,prev2logE[m->nbEBands+i]);
308
0
         }
309
0
         Ediff = logE[c*m->nbEBands+i]-MING(prev1,prev2);
310
0
         Ediff = MAX32(0, Ediff);
311
312
#ifdef FIXED_POINT
313
         if (Ediff < GCONST(16.f))
314
         {
315
            opus_val32 r32 = SHR32(celt_exp2_db(-Ediff),1);
316
            r = 2*MIN16(16383,r32);
317
         } else {
318
            r = 0;
319
         }
320
         if (LM==3)
321
            r = MULT16_16_Q14(23170, MIN32(23169, r));
322
         r = SHR16(MIN16(thresh, r),1);
323
         r = VSHR32(MULT16_16_Q15(sqrt_1, r),shift+14-NORM_SHIFT);
324
#else
325
         /* r needs to be multiplied by 2 or 2*sqrt(2) depending on LM because
326
            short blocks don't have the same energy as long */
327
0
         r = 2.f*celt_exp2_db(-Ediff);
328
0
         if (LM==3)
329
0
            r *= 1.41421356f;
330
0
         r = MIN16(thresh, r);
331
0
         r = r*sqrt_1;
332
0
#endif
333
0
         X = X_+c*size+(m->eBands[i]<<LM);
334
0
         for (k=0;k<1<<LM;k++)
335
0
         {
336
            /* Detect collapse */
337
0
            if (!(collapse_masks[i*C+c]&1<<k))
338
0
            {
339
               /* Fill with noise */
340
0
               for (j=0;j<N0;j++)
341
0
               {
342
0
                  seed = celt_lcg_rand(seed);
343
0
                  X[(j<<LM)+k] = (seed&0x8000 ? r : -r);
344
0
               }
345
0
               renormalize = 1;
346
0
            }
347
0
         }
348
         /* We just added some energy, so we need to renormalise */
349
0
         if (renormalize)
350
0
            renormalise_vector(X, N0<<LM, Q31ONE, arch);
351
0
      } while (++c<C);
352
0
   }
353
0
}
354
355
/* Compute the weights to use for optimizing normalized distortion across
356
   channels. We use the amplitude to weight square distortion, which means
357
   that we use the square root of the value we would have been using if we
358
   wanted to minimize the MSE in the non-normalized domain. This roughly
359
   corresponds to some quick-and-dirty perceptual experiments I ran to
360
   measure inter-aural masking (there doesn't seem to be any published data
361
   on the topic). */
362
static void compute_channel_weights(celt_ener Ex, celt_ener Ey, opus_val16 w[2])
363
1.48M
{
364
1.48M
   celt_ener minE;
365
#ifdef FIXED_POINT
366
   int shift;
367
#endif
368
1.48M
   minE = MIN32(Ex, Ey);
369
   /* Adjustment to make the weights a bit more conservative. */
370
1.48M
   Ex = ADD32(Ex, minE/3);
371
1.48M
   Ey = ADD32(Ey, minE/3);
372
#ifdef FIXED_POINT
373
   shift = celt_ilog2(EPSILON+MAX32(Ex, Ey))-14;
374
#endif
375
1.48M
   w[0] = VSHR32(Ex, shift);
376
1.48M
   w[1] = VSHR32(Ey, shift);
377
1.48M
}
378
379
static void intensity_stereo(const CELTMode *m, celt_norm * OPUS_RESTRICT X, const celt_norm * OPUS_RESTRICT Y, const celt_ener *bandE, int bandID, int N)
380
164M
{
381
164M
   int i = bandID;
382
164M
   int j;
383
164M
   opus_val16 a1, a2;
384
164M
   opus_val16 left, right;
385
164M
   opus_val16 norm;
386
#ifdef FIXED_POINT
387
   int shift = celt_zlog2(MAX32(bandE[i], bandE[i+m->nbEBands]))-13;
388
#endif
389
164M
   left = VSHR32(bandE[i],shift);
390
164M
   right = VSHR32(bandE[i+m->nbEBands],shift);
391
164M
   norm = EPSILON + celt_sqrt(EPSILON+MULT16_16(left,left)+MULT16_16(right,right));
392
#ifdef FIXED_POINT
393
   left = MIN32(left, norm-1);
394
   right = MIN32(right, norm-1);
395
#endif
396
164M
   a1 = DIV32_16(SHL32(EXTEND32(left),15),norm);
397
164M
   a2 = DIV32_16(SHL32(EXTEND32(right),15),norm);
398
1.82G
   for (j=0;j<N;j++)
399
1.66G
   {
400
1.66G
      X[j] = ADD32(MULT16_32_Q15(a1, X[j]), MULT16_32_Q15(a2, Y[j]));
401
      /* Side is not encoded, no need to calculate */
402
1.66G
   }
403
164M
}
404
405
static void stereo_split(celt_norm * OPUS_RESTRICT X, celt_norm * OPUS_RESTRICT Y, int N)
406
1.84M
{
407
1.84M
   int j;
408
20.9M
   for (j=0;j<N;j++)
409
19.0M
   {
410
19.0M
      opus_val32 r, l;
411
19.0M
      l = MULT32_32_Q31(QCONST32(.70710678f,31), X[j]);
412
19.0M
      r = MULT32_32_Q31(QCONST32(.70710678f,31), Y[j]);
413
19.0M
      X[j] = ADD32(l, r);
414
19.0M
      Y[j] = SUB32(r, l);
415
19.0M
   }
416
1.84M
}
417
418
static void stereo_merge(celt_norm * OPUS_RESTRICT X, celt_norm * OPUS_RESTRICT Y, opus_val32 mid, int N, int arch)
419
54.1M
{
420
54.1M
   int j;
421
54.1M
   opus_val32 xp=0, side=0;
422
54.1M
   opus_val32 El, Er;
423
#ifdef FIXED_POINT
424
   int kl, kr;
425
#endif
426
54.1M
   opus_val32 t, lgain, rgain;
427
428
   /* Compute the norm of X+Y and X-Y as |X|^2 + |Y|^2 +/- sum(xy) */
429
54.1M
   xp = celt_inner_prod_norm_shift(Y, X, N, arch);
430
54.1M
   side = celt_inner_prod_norm_shift(Y, Y, N, arch);
431
   /* Compensating for the mid normalization */
432
54.1M
   xp = MULT32_32_Q31(mid, xp);
433
   /* mid and side are in Q15, not Q14 like X and Y */
434
54.1M
   El = SHR32(MULT32_32_Q31(mid, mid),3) + side - 2*xp;
435
54.1M
   Er = SHR32(MULT32_32_Q31(mid, mid),3) + side + 2*xp;
436
54.1M
   if (Er < QCONST32(6e-4f, 28) || El < QCONST32(6e-4f, 28))
437
1.01k
   {
438
1.01k
      OPUS_COPY(Y, X, N);
439
1.01k
      return;
440
1.01k
   }
441
442
#ifdef FIXED_POINT
443
   kl = celt_ilog2(El)>>1;
444
   kr = celt_ilog2(Er)>>1;
445
#endif
446
54.1M
   t = VSHR32(El, (kl<<1)-29);
447
54.1M
   lgain = celt_rsqrt_norm32(t);
448
54.1M
   t = VSHR32(Er, (kr<<1)-29);
449
54.1M
   rgain = celt_rsqrt_norm32(t);
450
451
#ifdef FIXED_POINT
452
   if (kl < 7)
453
      kl = 7;
454
   if (kr < 7)
455
      kr = 7;
456
#endif
457
458
773M
   for (j=0;j<N;j++)
459
719M
   {
460
719M
      celt_norm r, l;
461
      /* Apply mid scaling (side is already scaled) */
462
719M
      l = MULT32_32_Q31(mid, X[j]);
463
719M
      r = Y[j];
464
719M
      X[j] = VSHR32(MULT32_32_Q31(lgain, SUB32(l,r)), kl-15);
465
719M
      Y[j] = VSHR32(MULT32_32_Q31(rgain, ADD32(l,r)), kr-15);
466
719M
   }
467
54.1M
}
468
469
/* Decide whether we should spread the pulses in the current frame */
470
int spreading_decision(const CELTMode *m, const celt_norm *X, int *average,
471
      int last_decision, int *hf_average, int *tapset_decision, int update_hf,
472
      int end, int C, int M, const int *spread_weight)
473
918k
{
474
918k
   int i, c, N0;
475
918k
   int sum = 0, nbBands=0;
476
918k
   const opus_int16 * OPUS_RESTRICT eBands = m->eBands;
477
918k
   int decision;
478
918k
   int hf_sum=0;
479
480
918k
   celt_assert(end>0);
481
482
918k
   N0 = M*m->shortMdctSize;
483
484
918k
   if (M*(eBands[end]-eBands[end-1]) <= 8)
485
477k
      return SPREAD_NONE;
486
576k
   c=0; do {
487
10.7M
      for (i=0;i<end;i++)
488
10.1M
      {
489
10.1M
         int j, N, tmp=0;
490
10.1M
         int tcount[3] = {0,0,0};
491
10.1M
         const celt_norm * OPUS_RESTRICT x = X+M*eBands[i]+c*N0;
492
10.1M
         N = M*(eBands[i+1]-eBands[i]);
493
10.1M
         if (N<=8)
494
7.19M
            continue;
495
         /* Compute rough CDF of |x[j]| */
496
91.2M
         for (j=0;j<N;j++)
497
88.2M
         {
498
88.2M
            opus_val32 x2N; /* Q13 */
499
500
88.2M
            x2N = MULT16_16(MULT16_16_Q15(SHR32(x[j], NORM_SHIFT-14), SHR32(x[j], NORM_SHIFT-14)), N);
501
88.2M
            if (x2N < QCONST16(0.25f,13))
502
36.4M
               tcount[0]++;
503
88.2M
            if (x2N < QCONST16(0.0625f,13))
504
20.6M
               tcount[1]++;
505
88.2M
            if (x2N < QCONST16(0.015625f,13))
506
11.4M
               tcount[2]++;
507
88.2M
         }
508
509
         /* Only include four last bands (8 kHz and up) */
510
2.97M
         if (i>m->nbEBands-4)
511
490k
            hf_sum += celt_udiv(32*(tcount[1]+tcount[0]), N);
512
2.97M
         tmp = (2*tcount[2] >= N) + (2*tcount[1] >= N) + (2*tcount[0] >= N);
513
2.97M
         sum += tmp*spread_weight[i];
514
2.97M
         nbBands+=spread_weight[i];
515
2.97M
      }
516
576k
   } while (++c<C);
517
518
441k
   if (update_hf)
519
24.9k
   {
520
24.9k
      if (hf_sum)
521
22.0k
         hf_sum = celt_udiv(hf_sum, C*(4-m->nbEBands+end));
522
24.9k
      *hf_average = (*hf_average+hf_sum)>>1;
523
24.9k
      hf_sum = *hf_average;
524
24.9k
      if (*tapset_decision==2)
525
5.02k
         hf_sum += 4;
526
19.8k
      else if (*tapset_decision==0)
527
18.2k
         hf_sum -= 4;
528
24.9k
      if (hf_sum > 22)
529
5.02k
         *tapset_decision=2;
530
19.8k
      else if (hf_sum > 18)
531
1.66k
         *tapset_decision=1;
532
18.2k
      else
533
18.2k
         *tapset_decision=0;
534
24.9k
   }
535
   /*printf("%d %d %d\n", hf_sum, *hf_average, *tapset_decision);*/
536
441k
   celt_assert(nbBands>0); /* end has to be non-zero */
537
441k
   celt_assert(sum>=0);
538
441k
   sum = celt_udiv((opus_int32)sum<<8, nbBands);
539
   /* Recursive averaging */
540
441k
   sum = (sum+*average)>>1;
541
441k
   *average = sum;
542
   /* Hysteresis */
543
441k
   sum = (3*sum + (((3-last_decision)<<7) + 64) + 2)>>2;
544
441k
   if (sum < 80)
545
261k
   {
546
261k
      decision = SPREAD_AGGRESSIVE;
547
261k
   } else if (sum < 256)
548
142k
   {
549
142k
      decision = SPREAD_NORMAL;
550
142k
   } else if (sum < 384)
551
23.5k
   {
552
23.5k
      decision = SPREAD_LIGHT;
553
23.5k
   } else {
554
13.8k
      decision = SPREAD_NONE;
555
13.8k
   }
556
#ifdef FUZZING
557
   decision = rand()&0x3;
558
   *tapset_decision=rand()%3;
559
#endif
560
441k
   return decision;
561
441k
}
562
563
/* Indexing table for converting from natural Hadamard to ordery Hadamard
564
   This is essentially a bit-reversed Gray, on top of which we've added
565
   an inversion of the order because we want the DC at the end rather than
566
   the beginning. The lines are for N=2, 4, 8, 16 */
567
static const int ordery_table[] = {
568
       1,  0,
569
       3,  0,  2,  1,
570
       7,  0,  4,  3,  6,  1,  5,  2,
571
      15,  0,  8,  7, 12,  3, 11,  4, 14,  1,  9,  6, 13,  2, 10,  5,
572
};
573
574
static void deinterleave_hadamard(celt_norm *X, int N0, int stride, int hadamard)
575
8.27M
{
576
8.27M
   int i,j;
577
8.27M
   VARDECL(celt_norm, tmp);
578
8.27M
   int N;
579
8.27M
   SAVE_STACK;
580
8.27M
   N = N0*stride;
581
8.27M
   ALLOC(tmp, N, celt_norm);
582
8.27M
   celt_assert(stride>0);
583
8.27M
   if (hadamard)
584
1.90M
   {
585
1.90M
      const int *ordery = ordery_table+stride-2;
586
9.40M
      for (i=0;i<stride;i++)
587
7.50M
      {
588
40.8M
         for (j=0;j<N0;j++)
589
33.3M
            tmp[ordery[i]*N0+j] = X[j*stride+i];
590
7.50M
      }
591
6.36M
   } else {
592
49.8M
      for (i=0;i<stride;i++)
593
149M
         for (j=0;j<N0;j++)
594
105M
            tmp[i*N0+j] = X[j*stride+i];
595
6.36M
   }
596
8.27M
   OPUS_COPY(X, tmp, N);
597
8.27M
   RESTORE_STACK;
598
8.27M
}
599
600
static void interleave_hadamard(celt_norm *X, int N0, int stride, int hadamard)
601
1.91M
{
602
1.91M
   int i,j;
603
1.91M
   VARDECL(celt_norm, tmp);
604
1.91M
   int N;
605
1.91M
   SAVE_STACK;
606
1.91M
   N = N0*stride;
607
1.91M
   ALLOC(tmp, N, celt_norm);
608
1.91M
   if (hadamard)
609
709k
   {
610
709k
      const int *ordery = ordery_table+stride-2;
611
3.54M
      for (i=0;i<stride;i++)
612
14.5M
         for (j=0;j<N0;j++)
613
11.6M
            tmp[j*stride+i] = X[ordery[i]*N0+j];
614
1.20M
   } else {
615
9.23M
      for (i=0;i<stride;i++)
616
23.6M
         for (j=0;j<N0;j++)
617
15.6M
            tmp[j*stride+i] = X[i*N0+j];
618
1.20M
   }
619
1.91M
   OPUS_COPY(X, tmp, N);
620
1.91M
   RESTORE_STACK;
621
1.91M
}
622
623
void haar1(celt_norm *X, int N0, int stride)
624
382M
{
625
382M
   int i, j;
626
382M
   N0 >>= 1;
627
1.22G
   for (i=0;i<stride;i++)
628
3.69G
      for (j=0;j<N0;j++)
629
2.84G
      {
630
2.84G
         opus_val32 tmp1, tmp2;
631
2.84G
         tmp1 = MULT32_32_Q31(QCONST32(.70710678f,31), X[stride*2*j+i]);
632
2.84G
         tmp2 = MULT32_32_Q31(QCONST32(.70710678f,31), X[stride*(2*j+1)+i]);
633
2.84G
         X[stride*2*j+i] = ADD32(tmp1, tmp2);
634
2.84G
         X[stride*(2*j+1)+i] = SUB32(tmp1, tmp2);
635
2.84G
      }
636
382M
}
637
638
static int compute_qn(int N, int b, int offset, int pulse_cap, int stereo)
639
193M
{
640
193M
   static const opus_int16 exp2_table8[8] =
641
193M
      {16384, 17866, 19483, 21247, 23170, 25267, 27554, 30048};
642
193M
   int qn, qb;
643
193M
   int N2 = 2*N-1;
644
193M
   if (stereo && N==2)
645
50.1M
      N2--;
646
   /* The upper limit ensures that in a stereo split with itheta==16384, we'll
647
       always have enough bits left over to code at least one pulse in the
648
       side; otherwise it would collapse, since it doesn't get folded. */
649
193M
   qb = celt_sudiv(b+N2*offset, N2);
650
193M
   qb = IMIN(b-pulse_cap-(4<<BITRES), qb);
651
652
193M
   qb = IMIN(8<<BITRES, qb);
653
654
193M
   if (qb<(1<<BITRES>>1)) {
655
162M
      qn = 1;
656
162M
   } else {
657
30.8M
      qn = exp2_table8[qb&0x7]>>(14-(qb>>BITRES));
658
30.8M
      qn = (qn+1)>>1<<1;
659
30.8M
   }
660
193M
   celt_assert(qn <= 256);
661
193M
   return qn;
662
193M
}
663
664
struct band_ctx {
665
   int encode;
666
   int resynth;
667
   const CELTMode *m;
668
   int i;
669
   int intensity;
670
   int spread;
671
   int tf_change;
672
   ec_ctx *ec;
673
   opus_int32 remaining_bits;
674
   const celt_ener *bandE;
675
   opus_uint32 seed;
676
   int arch;
677
   int theta_round;
678
   int disable_inv;
679
   int avoid_split_noise;
680
#ifdef ENABLE_QEXT
681
   ec_ctx *ext_ec;
682
   int extra_bits;
683
   opus_int32 ext_total_bits;
684
   int extra_bands;
685
#endif
686
};
687
688
struct split_ctx {
689
   int inv;
690
   int imid;
691
   int iside;
692
   int delta;
693
   int itheta;
694
#ifdef ENABLE_QEXT
695
   int itheta_q30;
696
#endif
697
   int qalloc;
698
};
699
700
static void compute_theta(struct band_ctx *ctx, struct split_ctx *sctx,
701
      celt_norm *X, celt_norm *Y, int N, int *b, int B, int B0,
702
      int LM,
703
      int stereo, int *fill ARG_QEXT(int *ext_b))
704
193M
{
705
193M
   int qn;
706
193M
   int itheta=0;
707
193M
   int itheta_q30=0;
708
193M
   int delta;
709
193M
   int imid, iside;
710
193M
   int qalloc;
711
193M
   int pulse_cap;
712
193M
   int offset;
713
193M
   opus_int32 tell;
714
193M
   int inv=0;
715
193M
   int encode;
716
193M
   const CELTMode *m;
717
193M
   int i;
718
193M
   int intensity;
719
193M
   ec_ctx *ec;
720
193M
   const celt_ener *bandE;
721
722
193M
   encode = ctx->encode;
723
193M
   m = ctx->m;
724
193M
   i = ctx->i;
725
193M
   intensity = ctx->intensity;
726
193M
   ec = ctx->ec;
727
193M
   bandE = ctx->bandE;
728
729
   /* Decide on the resolution to give to the split parameter theta */
730
193M
   pulse_cap = m->logN[i]+LM*(1<<BITRES);
731
193M
   offset = (pulse_cap>>1) - (stereo&&N==2 ? QTHETA_OFFSET_TWOPHASE : QTHETA_OFFSET);
732
193M
   qn = compute_qn(N, *b, offset, pulse_cap, stereo);
733
193M
   if (stereo && i>=intensity)
734
163M
      qn = 1;
735
193M
   if (encode)
736
193M
   {
737
      /* theta is the atan() of the ratio between the (normalized)
738
         side and mid. With just that parameter, we can re-scale both
739
         mid and side because we know that 1) they have unit norm and
740
         2) they are orthogonal. */
741
193M
      itheta_q30 = stereo_itheta(X, Y, stereo, N, ctx->arch);
742
193M
      itheta = itheta_q30>>16;
743
193M
   }
744
193M
   tell = ec_tell_frac(ec);
745
193M
   if (qn!=1)
746
29.5M
   {
747
29.5M
      if (encode)
748
29.5M
      {
749
29.5M
         if (!stereo || ctx->theta_round == 0)
750
27.7M
         {
751
27.7M
            itheta = (itheta*(opus_int32)qn+8192)>>14;
752
27.7M
            if (!stereo && ctx->avoid_split_noise && itheta > 0 && itheta < qn)
753
246k
            {
754
               /* Check if the selected value of theta will cause the bit allocation
755
                  to inject noise on one side. If so, make sure the energy of that side
756
                  is zero. */
757
246k
               int unquantized = celt_udiv((opus_int32)itheta*16384, qn);
758
246k
               imid = bitexact_cos((opus_int16)unquantized);
759
246k
               iside = bitexact_cos((opus_int16)(16384-unquantized));
760
246k
               delta = FRAC_MUL16((N-1)<<7,bitexact_log2tan(iside,imid));
761
246k
               if (delta > *b)
762
234
                  itheta = qn;
763
246k
               else if (delta < -*b)
764
544
                  itheta = 0;
765
246k
            }
766
27.7M
         } else {
767
1.78M
            int down;
768
            /* Bias quantization towards itheta=0 and itheta=16384. */
769
1.78M
            int bias = itheta > 8192 ? 32767/qn : -32767/qn;
770
1.78M
            down = IMIN(qn-1, IMAX(0, (itheta*(opus_int32)qn + bias)>>14));
771
1.78M
            if (ctx->theta_round < 0)
772
891k
               itheta = down;
773
891k
            else
774
891k
               itheta = down+1;
775
1.78M
         }
776
29.5M
      }
777
      /* Entropy coding of the angle. We use a uniform pdf for the
778
         time split, a step for stereo, and a triangular one for the rest. */
779
29.5M
      if (stereo && N>2)
780
2.10M
      {
781
2.10M
         int p0 = 3;
782
2.10M
         int x = itheta;
783
2.10M
         int x0 = qn/2;
784
2.10M
         int ft = p0*(x0+1) + x0;
785
         /* Use a probability of p0 up to itheta=8192 and then use 1 after */
786
2.10M
         if (encode)
787
2.10M
         {
788
2.10M
            ec_encode(ec,x<=x0?p0*x:(x-1-x0)+(x0+1)*p0,x<=x0?p0*(x+1):(x-x0)+(x0+1)*p0,ft);
789
2.10M
         } else {
790
0
            int fs;
791
0
            fs=ec_decode(ec,ft);
792
0
            if (fs<(x0+1)*p0)
793
0
               x=fs/p0;
794
0
            else
795
0
               x=x0+1+(fs-(x0+1)*p0);
796
0
            ec_dec_update(ec,x<=x0?p0*x:(x-1-x0)+(x0+1)*p0,x<=x0?p0*(x+1):(x-x0)+(x0+1)*p0,ft);
797
0
            itheta = x;
798
0
         }
799
27.4M
      } else if (B0>1 || stereo) {
800
         /* Uniform pdf */
801
4.72M
         if (encode)
802
4.72M
            ec_enc_uint(ec, itheta, qn+1);
803
0
         else
804
0
            itheta = ec_dec_uint(ec, qn+1);
805
22.7M
      } else {
806
22.7M
         int fs=1, ft;
807
22.7M
         ft = ((qn>>1)+1)*((qn>>1)+1);
808
22.7M
         if (encode)
809
22.7M
         {
810
22.7M
            int fl;
811
812
22.7M
            fs = itheta <= (qn>>1) ? itheta + 1 : qn + 1 - itheta;
813
22.7M
            fl = itheta <= (qn>>1) ? itheta*(itheta + 1)>>1 :
814
22.7M
             ft - ((qn + 1 - itheta)*(qn + 2 - itheta)>>1);
815
816
22.7M
            ec_encode(ec, fl, fl+fs, ft);
817
22.7M
         } else {
818
            /* Triangular pdf */
819
0
            int fl=0;
820
0
            int fm;
821
0
            fm = ec_decode(ec, ft);
822
823
0
            if (fm < ((qn>>1)*((qn>>1) + 1)>>1))
824
0
            {
825
0
               itheta = (isqrt32(8*(opus_uint32)fm + 1) - 1)>>1;
826
0
               fs = itheta + 1;
827
0
               fl = itheta*(itheta + 1)>>1;
828
0
            }
829
0
            else
830
0
            {
831
0
               itheta = (2*(qn + 1)
832
0
                - isqrt32(8*(opus_uint32)(ft - fm - 1) + 1))>>1;
833
0
               fs = qn + 1 - itheta;
834
0
               fl = ft - ((qn + 1 - itheta)*(qn + 2 - itheta)>>1);
835
0
            }
836
837
0
            ec_dec_update(ec, fl, fl+fs, ft);
838
0
         }
839
22.7M
      }
840
29.5M
      celt_assert(itheta>=0);
841
29.5M
      itheta = celt_udiv((opus_int32)itheta*16384, qn);
842
#ifdef ENABLE_QEXT
843
      *ext_b = IMIN(*ext_b, ctx->ext_total_bits - (opus_int32)ec_tell_frac(ctx->ext_ec));
844
      if (*ext_b >= 2*N<<BITRES && ctx->ext_total_bits-ec_tell_frac(ctx->ext_ec)-1 > 2<<BITRES) {
845
         int extra_bits;
846
         int ext_tell = ec_tell_frac(ctx->ext_ec);
847
         extra_bits = IMIN(12, IMAX(2, celt_sudiv(*ext_b, (2*N-1)<<BITRES)));
848
         if (encode) {
849
            itheta_q30 = itheta_q30 - (itheta<<16);
850
            itheta_q30 = (itheta_q30*(opus_int64)qn*((1<<extra_bits)-1)+(1<<29))>>30;
851
            itheta_q30 += (1<<(extra_bits-1))-1;
852
            itheta_q30 = IMAX(0, IMIN((1<<extra_bits)-2, itheta_q30));
853
            ec_enc_uint(ctx->ext_ec, itheta_q30, (1<<extra_bits)-1);
854
         } else {
855
            itheta_q30 = ec_dec_uint(ctx->ext_ec, (1<<extra_bits)-1);
856
         }
857
         itheta_q30 -= (1<<(extra_bits-1))-1;
858
         itheta_q30 = (itheta<<16) + itheta_q30*(opus_int64)(1<<30)/(qn*((1<<extra_bits)-1));
859
         /* Hard bounds on itheta (can only trigger on corrupted bitstreams). */
860
         itheta_q30 = IMAX(0, IMIN(itheta_q30, 1073741824));
861
         *ext_b -= ec_tell_frac(ctx->ext_ec) - ext_tell;
862
      } else {
863
         itheta_q30 = (opus_int32)itheta<<16;
864
      }
865
#endif
866
29.5M
      if (encode && stereo)
867
2.75M
      {
868
2.75M
         if (itheta==0)
869
902k
            intensity_stereo(m, X, Y, bandE, i, N);
870
1.84M
         else
871
1.84M
            stereo_split(X, Y, N);
872
2.75M
      }
873
      /* NOTE: Renormalising X and Y *may* help fixed-point a bit at very high rate.
874
               Let's do that at higher complexity */
875
163M
   } else if (stereo) {
876
163M
      if (encode)
877
163M
      {
878
163M
         inv = itheta > 8192 && !ctx->disable_inv;
879
163M
         if (inv)
880
366k
         {
881
366k
            int j;
882
5.67M
            for (j=0;j<N;j++)
883
5.30M
               Y[j] = -Y[j];
884
366k
         }
885
163M
         intensity_stereo(m, X, Y, bandE, i, N);
886
163M
      }
887
163M
      if (*b>2<<BITRES && ctx->remaining_bits > 2<<BITRES)
888
1.95M
      {
889
1.95M
         if (encode)
890
1.95M
            ec_enc_bit_logp(ec, inv, 2);
891
0
         else
892
0
            inv = ec_dec_bit_logp(ec, 2);
893
1.95M
      } else
894
161M
         inv = 0;
895
      /* inv flag override to avoid problems with downmixing. */
896
163M
      if (ctx->disable_inv)
897
72.7M
         inv = 0;
898
163M
      itheta = 0;
899
163M
      itheta_q30 = 0;
900
163M
   }
901
193M
   qalloc = ec_tell_frac(ec) - tell;
902
193M
   *b -= qalloc;
903
904
193M
   if (itheta == 0)
905
183M
   {
906
183M
      imid = 32767;
907
183M
      iside = 0;
908
183M
      *fill &= (1<<B)-1;
909
183M
      delta = -16384;
910
183M
   } else if (itheta == 16384)
911
535k
   {
912
535k
      imid = 0;
913
535k
      iside = 32767;
914
535k
      *fill &= ((1<<B)-1)<<B;
915
535k
      delta = 16384;
916
9.10M
   } else {
917
9.10M
      imid = bitexact_cos((opus_int16)itheta);
918
9.10M
      iside = bitexact_cos((opus_int16)(16384-itheta));
919
      /* This is the mid vs side allocation that minimizes squared error
920
         in that band. */
921
9.10M
      delta = FRAC_MUL16((N-1)<<7,bitexact_log2tan(iside,imid));
922
9.10M
   }
923
924
193M
   sctx->inv = inv;
925
193M
   sctx->imid = imid;
926
193M
   sctx->iside = iside;
927
193M
   sctx->delta = delta;
928
193M
   sctx->itheta = itheta;
929
#ifdef ENABLE_QEXT
930
   sctx->itheta_q30 = itheta_q30;
931
#endif
932
193M
   sctx->qalloc = qalloc;
933
193M
}
934
static unsigned quant_band_n1(struct band_ctx *ctx, celt_norm *X, celt_norm *Y,
935
      celt_norm *lowband_out)
936
212M
{
937
212M
   int c;
938
212M
   int stereo;
939
212M
   celt_norm *x = X;
940
212M
   int encode;
941
212M
   ec_ctx *ec;
942
943
212M
   encode = ctx->encode;
944
212M
   ec = ctx->ec;
945
946
212M
   stereo = Y != NULL;
947
270M
   c=0; do {
948
270M
      int sign=0;
949
270M
      if (ctx->remaining_bits>=1<<BITRES)
950
9.72M
      {
951
9.72M
         if (encode)
952
9.72M
         {
953
9.72M
            sign = x[0]<0;
954
9.72M
            ec_enc_bits(ec, sign, 1);
955
9.72M
         } else {
956
0
            sign = ec_dec_bits(ec, 1);
957
0
         }
958
9.72M
         ctx->remaining_bits -= 1<<BITRES;
959
9.72M
      }
960
270M
      if (ctx->resynth)
961
36.1M
         x[0] = sign ? -NORM_SCALING : NORM_SCALING;
962
270M
      x = Y;
963
270M
   } while (++c<1+stereo);
964
212M
   if (lowband_out)
965
212M
      lowband_out[0] = SHR32(X[0],4);
966
212M
   return 1;
967
212M
}
968
969
/* This function is responsible for encoding and decoding a mono partition.
970
   It can split the band in two and transmit the energy difference with
971
   the two half-bands. It can be called recursively so bands can end up being
972
   split in 8 parts. */
973
static unsigned quant_partition(struct band_ctx *ctx, celt_norm *X,
974
      int N, int b, int B, celt_norm *lowband,
975
      int LM,
976
      opus_val32 gain, int fill
977
      ARG_QEXT(int ext_b))
978
690M
{
979
690M
   const unsigned char *cache;
980
690M
   int q;
981
690M
   int curr_bits;
982
690M
   int imid=0, iside=0;
983
690M
   int B0=B;
984
690M
   opus_val32 mid=0, side=0;
985
690M
   unsigned cm=0;
986
690M
   celt_norm *Y=NULL;
987
690M
   int encode;
988
690M
   const CELTMode *m;
989
690M
   int i;
990
690M
   int spread;
991
690M
   ec_ctx *ec;
992
993
690M
   encode = ctx->encode;
994
690M
   m = ctx->m;
995
690M
   i = ctx->i;
996
690M
   spread = ctx->spread;
997
690M
   ec = ctx->ec;
998
999
   /* If we need 1.5 more bit than we can produce, split the band in two. */
1000
690M
   cache = m->cache.bits + m->cache.index[(LM+1)*m->nbEBands+i];
1001
690M
   if (LM != -1 && b > cache[cache[0]]+12 && N>2)
1002
26.8M
   {
1003
26.8M
      int mbits, sbits, delta;
1004
26.8M
      int itheta;
1005
26.8M
      int qalloc;
1006
26.8M
      struct split_ctx sctx;
1007
26.8M
      celt_norm *next_lowband2=NULL;
1008
26.8M
      opus_int32 rebalance;
1009
1010
26.8M
      N >>= 1;
1011
26.8M
      Y = X+N;
1012
26.8M
      LM -= 1;
1013
26.8M
      if (B==1)
1014
22.7M
         fill = (fill&1)|(fill<<1);
1015
26.8M
      B = (B+1)>>1;
1016
1017
26.8M
      compute_theta(ctx, &sctx, X, Y, N, &b, B, B0, LM, 0, &fill ARG_QEXT(&ext_b));
1018
26.8M
      imid = sctx.imid;
1019
26.8M
      iside = sctx.iside;
1020
26.8M
      delta = sctx.delta;
1021
26.8M
      itheta = sctx.itheta;
1022
26.8M
      qalloc = sctx.qalloc;
1023
#ifdef FIXED_POINT
1024
# ifdef ENABLE_QEXT
1025
      (void)imid;
1026
      (void)iside;
1027
      mid = celt_cos_norm32(sctx.itheta_q30);
1028
      side = celt_cos_norm32((1<<30)-sctx.itheta_q30);
1029
# else
1030
      mid = SHL32(EXTEND32(imid), 16);
1031
      side = SHL32(EXTEND32(iside), 16);
1032
# endif
1033
#else
1034
# ifdef ENABLE_QEXT
1035
      (void)imid;
1036
      (void)iside;
1037
      mid = celt_cos_norm2(sctx.itheta_q30*(1.f/(1<<30)));
1038
      side = celt_cos_norm2(1.f-sctx.itheta_q30*(1.f/(1<<30)));
1039
# else
1040
26.8M
      mid = (1.f/32768)*imid;
1041
26.8M
      side = (1.f/32768)*iside;
1042
26.8M
# endif
1043
26.8M
#endif
1044
1045
      /* Give more bits to low-energy MDCTs than they would otherwise deserve */
1046
26.8M
      if (B0>1 && (itheta&0x3fff))
1047
3.28M
      {
1048
3.28M
         if (itheta > 8192)
1049
            /* Rough approximation for pre-echo masking */
1050
1.31M
            delta -= delta>>(4-LM);
1051
1.96M
         else
1052
            /* Corresponds to a forward-masking slope of 1.5 dB per 10 ms */
1053
1.96M
            delta = IMIN(0, delta + (N<<BITRES>>(5-LM)));
1054
3.28M
      }
1055
26.8M
      mbits = IMAX(0, IMIN(b, (b-delta)/2));
1056
26.8M
      sbits = b-mbits;
1057
26.8M
      ctx->remaining_bits -= qalloc;
1058
1059
26.8M
      if (lowband)
1060
737k
         next_lowband2 = lowband+N; /* >32-bit split case */
1061
1062
26.8M
      rebalance = ctx->remaining_bits;
1063
26.8M
      if (mbits >= sbits)
1064
22.9M
      {
1065
22.9M
         cm = quant_partition(ctx, X, N, mbits, B, lowband, LM,
1066
22.9M
               MULT32_32_Q31(gain,mid), fill ARG_QEXT(ext_b/2));
1067
22.9M
         rebalance = mbits - (rebalance-ctx->remaining_bits);
1068
22.9M
         if (rebalance > 3<<BITRES && itheta!=0)
1069
760k
            sbits += rebalance - (3<<BITRES);
1070
22.9M
         cm |= quant_partition(ctx, Y, N, sbits, B, next_lowband2, LM,
1071
22.9M
               MULT32_32_Q31(gain,side), fill>>B ARG_QEXT(ext_b/2))<<(B0>>1);
1072
22.9M
      } else {
1073
3.82M
         cm = quant_partition(ctx, Y, N, sbits, B, next_lowband2, LM,
1074
3.82M
               MULT32_32_Q31(gain,side), fill>>B ARG_QEXT(ext_b/2))<<(B0>>1);
1075
3.82M
         rebalance = sbits - (rebalance-ctx->remaining_bits);
1076
3.82M
         if (rebalance > 3<<BITRES && itheta!=16384)
1077
482k
            mbits += rebalance - (3<<BITRES);
1078
3.82M
         cm |= quant_partition(ctx, X, N, mbits, B, lowband, LM,
1079
3.82M
               MULT32_32_Q31(gain,mid), fill ARG_QEXT(ext_b/2));
1080
3.82M
      }
1081
663M
   } else {
1082
#ifdef ENABLE_QEXT
1083
      int extra_bits;
1084
      int ext_remaining_bits;
1085
      extra_bits = ext_b/(N-1)>>BITRES;
1086
      ext_remaining_bits = ctx->ext_total_bits-(opus_int32)ec_tell_frac(ctx->ext_ec);
1087
      if (ext_remaining_bits < ((extra_bits+1)*(N-1)+N)<<BITRES) {
1088
         extra_bits = (ext_remaining_bits-(N<<BITRES))/(N-1)>>BITRES;
1089
         extra_bits = IMAX(extra_bits-1, 0);
1090
      }
1091
      extra_bits = IMIN(12, extra_bits);
1092
#endif
1093
      /* This is the basic no-split case */
1094
663M
      q = bits2pulses(m, i, LM, b);
1095
663M
      curr_bits = pulses2bits(m, i, LM, q);
1096
663M
      ctx->remaining_bits -= curr_bits;
1097
1098
      /* Ensures we can never bust the budget */
1099
664M
      while (ctx->remaining_bits < 0 && q > 0)
1100
1.09M
      {
1101
1.09M
         ctx->remaining_bits += curr_bits;
1102
1.09M
         q--;
1103
1.09M
         curr_bits = pulses2bits(m, i, LM, q);
1104
1.09M
         ctx->remaining_bits -= curr_bits;
1105
1.09M
      }
1106
1107
663M
      if (q!=0)
1108
33.2M
      {
1109
33.2M
         int K = get_pulses(q);
1110
1111
         /* Finally do the actual quantization */
1112
33.2M
         if (encode)
1113
33.2M
         {
1114
33.2M
            cm = alg_quant(X, N, K, spread, B, ec, gain, ctx->resynth
1115
33.2M
                           ARG_QEXT(ctx->ext_ec) ARG_QEXT(extra_bits),
1116
33.2M
                           ctx->arch);
1117
33.2M
         } else {
1118
0
            cm = alg_unquant(X, N, K, spread, B, ec, gain
1119
0
                             ARG_QEXT(ctx->ext_ec) ARG_QEXT(extra_bits));
1120
0
         }
1121
#ifdef ENABLE_QEXT
1122
      } else if (ext_b > 2*N<<BITRES)
1123
      {
1124
         extra_bits = ext_b/(N-1)>>BITRES;
1125
         ext_remaining_bits = ctx->ext_total_bits-ec_tell_frac(ctx->ext_ec);
1126
         if (ext_remaining_bits < ((extra_bits+1)*(N-1)+N)<<BITRES) {
1127
            extra_bits = (ext_remaining_bits-(N<<BITRES))/(N-1)>>BITRES;
1128
            extra_bits = IMAX(extra_bits-1, 0);
1129
         }
1130
         extra_bits = IMIN(14, extra_bits);
1131
         if (encode) cm = cubic_quant(X, N, extra_bits, B, ctx->ext_ec, gain, ctx->resynth);
1132
         else cm = cubic_unquant(X, N, extra_bits, B, ctx->ext_ec, gain);
1133
#endif
1134
630M
      } else {
1135
         /* If there's no pulse, fill the band anyway */
1136
630M
         int j;
1137
630M
         if (ctx->resynth)
1138
124M
         {
1139
124M
            unsigned cm_mask;
1140
            /* B can be as large as 16, so this shift might overflow an int on a
1141
               16-bit platform; use a long to get defined behavior.*/
1142
124M
            cm_mask = (unsigned)(1UL<<B)-1;
1143
124M
            fill &= cm_mask;
1144
124M
            if (!fill)
1145
53.5M
            {
1146
53.5M
               OPUS_CLEAR(X, N);
1147
70.5M
            } else {
1148
70.5M
               if (lowband == NULL)
1149
3.36M
               {
1150
                  /* Noise */
1151
21.0M
                  for (j=0;j<N;j++)
1152
17.6M
                  {
1153
17.6M
                     ctx->seed = celt_lcg_rand(ctx->seed);
1154
17.6M
                     X[j] = SHL32((celt_norm)((opus_int32)ctx->seed>>20), NORM_SHIFT-14);
1155
17.6M
                  }
1156
3.36M
                  cm = cm_mask;
1157
67.2M
               } else {
1158
                  /* Folded spectrum */
1159
781M
                  for (j=0;j<N;j++)
1160
713M
                  {
1161
713M
                     opus_val16 tmp;
1162
713M
                     ctx->seed = celt_lcg_rand(ctx->seed);
1163
                     /* About 48 dB below the "normal" folding level */
1164
713M
                     tmp = QCONST16(1.0f/256, NORM_SHIFT-4);
1165
713M
                     tmp = (ctx->seed)&0x8000 ? tmp : -tmp;
1166
713M
                     X[j] = lowband[j]+tmp;
1167
713M
                  }
1168
67.2M
                  cm = fill;
1169
67.2M
               }
1170
70.5M
               renormalise_vector(X, N, gain, ctx->arch);
1171
70.5M
            }
1172
124M
         }
1173
630M
      }
1174
663M
   }
1175
1176
690M
   return cm;
1177
690M
}
1178
1179
#ifdef ENABLE_QEXT
1180
static unsigned cubic_quant_partition(struct band_ctx *ctx, celt_norm *X, int N, int b, int B, ec_ctx *ec, int LM, opus_val32 gain, int resynth, int encode)
1181
{
1182
   celt_assert(LM>=0);
1183
   ctx->remaining_bits = ctx->ec->storage*8*8 - ec_tell_frac(ctx->ec);
1184
   b = IMIN(b, ctx->remaining_bits);
1185
   /* As long as we have at least two bits of depth, split all the way to LM=0 (not -1 like PVQ). */
1186
   if (LM==0 || b<=2*N<<BITRES) {
1187
      int res, ret;
1188
      b = IMIN(b + ((N-1)<<BITRES)/2, ctx->remaining_bits);
1189
      /* Resolution left after taking into account coding the cube face. */
1190
      res = (b-(1<<BITRES)-ctx->m->logN[ctx->i]-(LM<<BITRES)-1)/(N-1)>>BITRES;
1191
      res = IMIN(14, IMAX(0, res));
1192
      if (encode) ret = cubic_quant(X, N, res, B, ec, gain, resynth);
1193
      else ret = cubic_unquant(X, N, res, B, ec, gain);
1194
      ctx->remaining_bits = ctx->ec->storage*8*8 - ec_tell_frac(ctx->ec);
1195
      return ret;
1196
   } else {
1197
      celt_norm *Y;
1198
      opus_int32 itheta_q30;
1199
      opus_val32 g1, g2;
1200
      opus_int32 theta_res;
1201
      opus_int32 qtheta;
1202
      int delta;
1203
      int b1, b2;
1204
      int cm;
1205
      int N0;
1206
      N0 = N;
1207
      N >>= 1;
1208
      Y = X+N;
1209
      LM -= 1;
1210
      B = (B+1)>>1;
1211
      theta_res = IMIN(16, (b>>BITRES)/(N0-1) + 1);
1212
      if (encode) {
1213
         itheta_q30 = stereo_itheta(X, Y, 0, N, ctx->arch);
1214
         qtheta = (itheta_q30+(1<<(29-theta_res)))>>(30-theta_res);
1215
         ec_enc_uint(ec, qtheta, (1<<theta_res)+1);
1216
      } else {
1217
         qtheta = ec_dec_uint(ec, (1<<theta_res)+1);
1218
      }
1219
      itheta_q30 = qtheta<<(30-theta_res);
1220
      b -= theta_res<<BITRES;
1221
      delta = (N0-1) * 23 * ((itheta_q30>>16)-8192) >> (17-BITRES);
1222
1223
#ifdef FIXED_POINT
1224
      g1 = celt_cos_norm32(itheta_q30);
1225
      g2 = celt_cos_norm32((1<<30)-itheta_q30);
1226
#else
1227
      g1 = celt_cos_norm2(itheta_q30*(1.f/(1<<30)));
1228
      g2 = celt_cos_norm2(1.f-itheta_q30*(1.f/(1<<30)));
1229
#endif
1230
      if (itheta_q30 == 0) {
1231
         b1=b;
1232
         b2=0;
1233
      } else if (itheta_q30==1073741824) {
1234
         b1=0;
1235
         b2=b;
1236
      } else {
1237
         b1 = IMIN(b, IMAX(0, (b-delta)/2));
1238
         b2 = b-b1;
1239
      }
1240
      cm  = cubic_quant_partition(ctx, X, N, b1, B, ec, LM, MULT32_32_Q31(gain, g1), resynth, encode);
1241
      cm |= cubic_quant_partition(ctx, Y, N, b2, B, ec, LM, MULT32_32_Q31(gain, g2), resynth, encode);
1242
      return cm;
1243
   }
1244
}
1245
#endif
1246
1247
/* This function is responsible for encoding and decoding a band for the mono case. */
1248
static unsigned quant_band(struct band_ctx *ctx, celt_norm *X,
1249
      int N, int b, int B, celt_norm *lowband,
1250
      int LM, celt_norm *lowband_out,
1251
      opus_val32 gain, celt_norm *lowband_scratch, int fill
1252
      ARG_QEXT(int ext_b))
1253
792M
{
1254
792M
   int N0=N;
1255
792M
   int N_B=N;
1256
792M
   int N_B0;
1257
792M
   int B0=B;
1258
792M
   int time_divide=0;
1259
792M
   int recombine=0;
1260
792M
   int longBlocks;
1261
792M
   unsigned cm=0;
1262
792M
   int k;
1263
792M
   int encode;
1264
792M
   int tf_change;
1265
1266
792M
   encode = ctx->encode;
1267
792M
   tf_change = ctx->tf_change;
1268
1269
792M
   longBlocks = B0==1;
1270
1271
792M
   N_B = celt_udiv(N_B, B);
1272
1273
   /* Special case for one sample */
1274
792M
   if (N==1)
1275
155M
   {
1276
155M
      return quant_band_n1(ctx, X, NULL, lowband_out);
1277
155M
   }
1278
1279
636M
   if (tf_change>0)
1280
1.43M
      recombine = tf_change;
1281
   /* Band recombining to increase frequency resolution */
1282
1283
636M
   if (lowband_scratch && lowband && (recombine || ((N_B&1) == 0 && tf_change<0) || B0>1))
1284
1.12M
   {
1285
1.12M
      OPUS_COPY(lowband_scratch, lowband, N);
1286
1.12M
      lowband = lowband_scratch;
1287
1.12M
   }
1288
1289
639M
   for (k=0;k<recombine;k++)
1290
2.45M
   {
1291
2.45M
      static const unsigned char bit_interleave_table[16]={
1292
2.45M
            0,1,1,1,2,3,3,3,2,3,3,3,2,3,3,3
1293
2.45M
      };
1294
2.45M
      if (encode)
1295
2.45M
         haar1(X, N>>k, 1<<k);
1296
2.45M
      if (lowband)
1297
299k
         haar1(lowband, N>>k, 1<<k);
1298
2.45M
      fill = bit_interleave_table[fill&0xF]|bit_interleave_table[fill>>4]<<2;
1299
2.45M
   }
1300
636M
   B>>=recombine;
1301
636M
   N_B<<=recombine;
1302
1303
   /* Increasing the time resolution */
1304
641M
   while ((N_B&1) == 0 && tf_change<0)
1305
4.86M
   {
1306
4.86M
      if (encode)
1307
4.86M
         haar1(X, N_B, B);
1308
4.86M
      if (lowband)
1309
869k
         haar1(lowband, N_B, B);
1310
4.86M
      fill |= fill<<B;
1311
4.86M
      B <<= 1;
1312
4.86M
      N_B >>= 1;
1313
4.86M
      time_divide++;
1314
4.86M
      tf_change++;
1315
4.86M
   }
1316
636M
   B0=B;
1317
636M
   N_B0 = N_B;
1318
1319
   /* Reorganize the samples in time order instead of frequency order */
1320
636M
   if (B0>1)
1321
7.28M
   {
1322
7.28M
      if (encode)
1323
7.28M
         deinterleave_hadamard(X, N_B>>recombine, B0<<recombine, longBlocks);
1324
7.28M
      if (lowband)
1325
992k
         deinterleave_hadamard(lowband, N_B>>recombine, B0<<recombine, longBlocks);
1326
7.28M
   }
1327
1328
#ifdef ENABLE_QEXT
1329
   if (ctx->extra_bands && b > (3*N<<BITRES)+(ctx->m->logN[ctx->i]+8+8*LM)) {
1330
      cm = cubic_quant_partition(ctx, X, N, b, B, ctx->ec, LM, gain, ctx->resynth, encode);
1331
   } else
1332
#endif
1333
636M
   {
1334
636M
      cm = quant_partition(ctx, X, N, b, B, lowband, LM, gain, fill ARG_QEXT(ext_b));
1335
636M
   }
1336
1337
   /* This code is used by the decoder and by the resynthesis-enabled encoder */
1338
636M
   if (ctx->resynth)
1339
127M
   {
1340
      /* Undo the sample reorganization going from time order to frequency order */
1341
127M
      if (B0>1)
1342
1.91M
         interleave_hadamard(X, N_B>>recombine, B0<<recombine, longBlocks);
1343
1344
      /* Undo time-freq changes that we did earlier */
1345
127M
      N_B = N_B0;
1346
127M
      B = B0;
1347
129M
      for (k=0;k<time_divide;k++)
1348
1.79M
      {
1349
1.79M
         B >>= 1;
1350
1.79M
         N_B <<= 1;
1351
1.79M
         cm |= cm>>B;
1352
1.79M
         haar1(X, N_B, B);
1353
1.79M
      }
1354
1355
127M
      for (k=0;k<recombine;k++)
1356
598k
      {
1357
598k
         static const unsigned char bit_deinterleave_table[16]={
1358
598k
               0x00,0x03,0x0C,0x0F,0x30,0x33,0x3C,0x3F,
1359
598k
               0xC0,0xC3,0xCC,0xCF,0xF0,0xF3,0xFC,0xFF
1360
598k
         };
1361
598k
         cm = bit_deinterleave_table[cm];
1362
598k
         haar1(X, N0>>k, 1<<k);
1363
598k
      }
1364
127M
      B<<=recombine;
1365
1366
      /* Scale output for later folding */
1367
127M
      if (lowband_out)
1368
67.4M
      {
1369
67.4M
         int j;
1370
67.4M
         opus_val16 n;
1371
67.4M
         n = celt_sqrt(SHL32(EXTEND32(N0),22));
1372
677M
         for (j=0;j<N0;j++)
1373
610M
            lowband_out[j] = MULT16_32_Q15(n,X[j]);
1374
67.4M
      }
1375
127M
      cm &= (1<<B)-1;
1376
127M
   }
1377
636M
   return cm;
1378
792M
}
1379
1380
#ifdef FIXED_POINT
1381
#define MIN_STEREO_ENERGY 2
1382
#else
1383
338M
#define MIN_STEREO_ENERGY 1e-10f
1384
#endif
1385
1386
/* This function is responsible for encoding and decoding a band for the stereo case. */
1387
static unsigned quant_band_stereo(struct band_ctx *ctx, celt_norm *X, celt_norm *Y,
1388
      int N, int b, int B, celt_norm *lowband,
1389
      int LM, celt_norm *lowband_out,
1390
      celt_norm *lowband_scratch, int fill
1391
      ARG_QEXT(int ext_b) ARG_QEXT(const int *cap))
1392
223M
{
1393
223M
   int imid=0, iside=0;
1394
223M
   int inv = 0;
1395
223M
   opus_val32 mid=0, side=0;
1396
223M
   unsigned cm=0;
1397
223M
   int mbits, sbits, delta;
1398
223M
   int itheta;
1399
223M
   int qalloc;
1400
223M
   struct split_ctx sctx;
1401
223M
   int orig_fill;
1402
223M
   int encode;
1403
223M
   ec_ctx *ec;
1404
1405
223M
   encode = ctx->encode;
1406
223M
   ec = ctx->ec;
1407
1408
   /* Special case for one sample */
1409
223M
   if (N==1)
1410
57.0M
   {
1411
57.0M
      return quant_band_n1(ctx, X, Y, lowband_out);
1412
57.0M
   }
1413
1414
166M
   orig_fill = fill;
1415
1416
166M
   if (encode) {
1417
166M
      if (ctx->bandE[ctx->i] < MIN_STEREO_ENERGY || ctx->bandE[ctx->m->nbEBands+ctx->i] < MIN_STEREO_ENERGY) {
1418
160M
         if (ctx->bandE[ctx->i] > ctx->bandE[ctx->m->nbEBands+ctx->i]) OPUS_COPY(Y, X, N);
1419
160M
         else OPUS_COPY(X, Y, N);
1420
160M
      }
1421
166M
   }
1422
166M
   compute_theta(ctx, &sctx, X, Y, N, &b, B, B, LM, 1, &fill ARG_QEXT(&ext_b));
1423
166M
   inv = sctx.inv;
1424
166M
   imid = sctx.imid;
1425
166M
   iside = sctx.iside;
1426
166M
   delta = sctx.delta;
1427
166M
   itheta = sctx.itheta;
1428
166M
   qalloc = sctx.qalloc;
1429
#ifdef FIXED_POINT
1430
# ifdef ENABLE_QEXT
1431
   (void)imid;
1432
   (void)iside;
1433
   mid = celt_cos_norm32(sctx.itheta_q30);
1434
   side = celt_cos_norm32((1<<30)-sctx.itheta_q30);
1435
# else
1436
   mid = SHL32(EXTEND32(imid), 16);
1437
   side = SHL32(EXTEND32(iside), 16);
1438
# endif
1439
#else
1440
# ifdef ENABLE_QEXT
1441
   (void)imid;
1442
   (void)iside;
1443
   mid = celt_cos_norm2(sctx.itheta_q30*(1.f/(1<<30)));
1444
   side = celt_cos_norm2(1.f-sctx.itheta_q30*(1.f/(1<<30)));
1445
# else
1446
166M
   mid = (1.f/32768)*imid;
1447
166M
   side = (1.f/32768)*iside;
1448
166M
# endif
1449
166M
#endif
1450
1451
   /* This is a special case for N=2 that only works for stereo and takes
1452
      advantage of the fact that mid and side are orthogonal to encode
1453
      the side with just one bit. */
1454
166M
   if (N==2)
1455
50.1M
   {
1456
50.1M
      int c;
1457
50.1M
      int sign=0;
1458
50.1M
      celt_norm *x2, *y2;
1459
50.1M
      mbits = b;
1460
50.1M
      sbits = 0;
1461
      /* Only need one bit for the side. */
1462
50.1M
      if (itheta != 0 && itheta != 16384)
1463
391k
         sbits = 1<<BITRES;
1464
50.1M
      mbits -= sbits;
1465
50.1M
      c = itheta > 8192;
1466
50.1M
      ctx->remaining_bits -= qalloc+sbits;
1467
1468
50.1M
      x2 = c ? Y : X;
1469
50.1M
      y2 = c ? X : Y;
1470
50.1M
      if (sbits)
1471
391k
      {
1472
391k
         if (encode)
1473
391k
         {
1474
            /* Here we only need to encode a sign for the side. */
1475
            /* FIXME: Need to increase fixed-point precision? */
1476
391k
            sign = MULT32_32_Q31(x2[0],y2[1]) - MULT32_32_Q31(x2[1],y2[0]) < 0;
1477
391k
            ec_enc_bits(ec, sign, 1);
1478
391k
         } else {
1479
0
            sign = ec_dec_bits(ec, 1);
1480
0
         }
1481
391k
      }
1482
50.1M
      sign = 1-2*sign;
1483
      /* We use orig_fill here because we want to fold the side, but if
1484
         itheta==16384, we'll have cleared the low bits of fill. */
1485
50.1M
      cm = quant_band(ctx, x2, N, mbits, B, lowband, LM, lowband_out, Q31ONE,
1486
50.1M
            lowband_scratch, orig_fill ARG_QEXT(ext_b));
1487
      /* We don't split N=2 bands, so cm is either 1 or 0 (for a fold-collapse),
1488
         and there's no need to worry about mixing with the other channel. */
1489
50.1M
      y2[0] = -sign*x2[1];
1490
50.1M
      y2[1] = sign*x2[0];
1491
50.1M
      if (ctx->resynth)
1492
18.8M
      {
1493
18.8M
         celt_norm tmp;
1494
18.8M
         X[0] = MULT32_32_Q31(mid, X[0]);
1495
18.8M
         X[1] = MULT32_32_Q31(mid, X[1]);
1496
18.8M
         Y[0] = MULT32_32_Q31(side, Y[0]);
1497
18.8M
         Y[1] = MULT32_32_Q31(side, Y[1]);
1498
18.8M
         tmp = X[0];
1499
18.8M
         X[0] = SUB32(tmp,Y[0]);
1500
18.8M
         Y[0] = ADD32(tmp,Y[0]);
1501
18.8M
         tmp = X[1];
1502
18.8M
         X[1] = SUB32(tmp,Y[1]);
1503
18.8M
         Y[1] = ADD32(tmp,Y[1]);
1504
18.8M
      }
1505
116M
   } else {
1506
      /* "Normal" split code */
1507
116M
      opus_int32 rebalance;
1508
1509
116M
      mbits = IMAX(0, IMIN(b, (b-delta)/2));
1510
116M
      sbits = b-mbits;
1511
116M
      ctx->remaining_bits -= qalloc;
1512
1513
116M
      rebalance = ctx->remaining_bits;
1514
116M
      if (mbits >= sbits)
1515
115M
      {
1516
#ifdef ENABLE_QEXT
1517
         int qext_extra = 0;
1518
         /* Reallocate any mid bits that cannot be used to extra mid bits. */
1519
         if (cap != NULL && ext_b != 0) qext_extra = IMAX(0, IMIN(ext_b/2, mbits - cap[ctx->i]/2));
1520
#endif
1521
         /* In stereo mode, we do not apply a scaling to the mid because we need the normalized
1522
            mid for folding later. */
1523
115M
         cm = quant_band(ctx, X, N, mbits, B, lowband, LM, lowband_out, Q31ONE,
1524
115M
               lowband_scratch, fill ARG_QEXT(ext_b/2+qext_extra));
1525
115M
         rebalance = mbits - (rebalance-ctx->remaining_bits);
1526
115M
         if (rebalance > 3<<BITRES && itheta!=0)
1527
52.1k
            sbits += rebalance - (3<<BITRES);
1528
#ifdef ENABLE_QEXT
1529
         /* Guard against overflowing the EC with the angle if the cubic quant used too many bits for the mid. */
1530
         if (ctx->extra_bands) sbits = IMIN(sbits, ctx->remaining_bits);
1531
#endif
1532
         /* For a stereo split, the high bits of fill are always zero, so no
1533
            folding will be done to the side. */
1534
115M
         cm |= quant_band(ctx, Y, N, sbits, B, NULL, LM, NULL, side, NULL, fill>>B ARG_QEXT(ext_b/2-qext_extra));
1535
115M
      } else {
1536
#ifdef ENABLE_QEXT
1537
         int qext_extra = 0;
1538
         /* Reallocate any side bits that cannot be used to extra side bits. */
1539
         if (cap != NULL && ext_b != 0) qext_extra = IMAX(0, IMIN(ext_b/2, sbits - cap[ctx->i]/2));
1540
#endif
1541
         /* For a stereo split, the high bits of fill are always zero, so no
1542
            folding will be done to the side. */
1543
397k
         cm = quant_band(ctx, Y, N, sbits, B, NULL, LM, NULL, side, NULL, fill>>B ARG_QEXT(ext_b/2+qext_extra));
1544
397k
         rebalance = sbits - (rebalance-ctx->remaining_bits);
1545
397k
         if (rebalance > 3<<BITRES && itheta!=16384)
1546
12.3k
            mbits += rebalance - (3<<BITRES);
1547
#ifdef ENABLE_QEXT
1548
         /* Guard against overflowing the EC with the angle if the cubic quant used too many bits for the side. */
1549
         if (ctx->extra_bands) mbits = IMIN(mbits, ctx->remaining_bits);
1550
#endif
1551
         /* In stereo mode, we do not apply a scaling to the mid because we need the normalized
1552
            mid for folding later. */
1553
397k
         cm |= quant_band(ctx, X, N, mbits, B, lowband, LM, lowband_out, Q31ONE,
1554
397k
               lowband_scratch, fill ARG_QEXT(ext_b/2-qext_extra));
1555
397k
      }
1556
116M
   }
1557
1558
1559
   /* This code is used by the decoder and by the resynthesis-enabled encoder */
1560
166M
   if (ctx->resynth)
1561
73.0M
   {
1562
73.0M
      if (N!=2)
1563
54.1M
         stereo_merge(X, Y, mid, N, ctx->arch);
1564
73.0M
      if (inv)
1565
138k
      {
1566
138k
         int j;
1567
1.91M
         for (j=0;j<N;j++)
1568
1.77M
            Y[j] = -Y[j];
1569
138k
      }
1570
73.0M
   }
1571
166M
   return cm;
1572
223M
}
1573
1574
#ifndef DISABLE_UPDATE_DRAFT
1575
static void special_hybrid_folding(const CELTMode *m, celt_norm *norm, celt_norm *norm2, int start, int M, int dual_stereo)
1576
48.9M
{
1577
48.9M
   int n1, n2;
1578
48.9M
   const opus_int16 * OPUS_RESTRICT eBands = m->eBands;
1579
48.9M
   n1 = M*(eBands[start+1]-eBands[start]);
1580
48.9M
   n2 = M*(eBands[start+2]-eBands[start+1]);
1581
   /* Duplicate enough of the first band folding data to be able to fold the second band.
1582
      Copies no data for CELT-only mode. */
1583
48.9M
   OPUS_COPY(&norm[n1], &norm[2*n1 - n2], n2-n1);
1584
48.9M
   if (dual_stereo)
1585
50.0k
      OPUS_COPY(&norm2[n1], &norm2[2*n1 - n2], n2-n1);
1586
48.9M
}
1587
#endif
1588
1589
void quant_all_bands(int encode, const CELTMode *m, int start, int end,
1590
      celt_norm *X_, celt_norm *Y_, unsigned char *collapse_masks,
1591
      const celt_ener *bandE, int *pulses, int shortBlocks, int spread,
1592
      int dual_stereo, int intensity, int *tf_res, opus_int32 total_bits,
1593
      opus_int32 balance, ec_ctx *ec, int LM, int codedBands,
1594
      opus_uint32 *seed, int complexity, int arch, int disable_inv
1595
      ARG_QEXT(ec_ctx *ext_ec) ARG_QEXT(int *extra_pulses)
1596
      ARG_QEXT(opus_int32 ext_total_bits) ARG_QEXT(const int *cap))
1597
48.8M
{
1598
48.8M
   int i;
1599
48.8M
   opus_int32 remaining_bits;
1600
48.8M
   const opus_int16 * OPUS_RESTRICT eBands = m->eBands;
1601
48.8M
   celt_norm * OPUS_RESTRICT norm, * OPUS_RESTRICT norm2;
1602
48.8M
   VARDECL(celt_norm, _norm);
1603
48.8M
   VARDECL(celt_norm, _lowband_scratch);
1604
48.8M
   VARDECL(celt_norm, X_save);
1605
48.8M
   VARDECL(celt_norm, Y_save);
1606
48.8M
   VARDECL(celt_norm, X_save2);
1607
48.8M
   VARDECL(celt_norm, Y_save2);
1608
48.8M
   VARDECL(celt_norm, norm_save2);
1609
48.8M
   VARDECL(unsigned char, bytes_save);
1610
48.8M
   int resynth_alloc;
1611
48.8M
   celt_norm *lowband_scratch;
1612
48.8M
   int B;
1613
48.8M
   int M;
1614
48.8M
   int lowband_offset;
1615
48.8M
   int update_lowband = 1;
1616
48.8M
   int C = Y_ != NULL ? 2 : 1;
1617
48.8M
   int norm_offset;
1618
48.8M
   int theta_rdo = encode && Y_!=NULL && !dual_stereo && complexity>=8;
1619
#ifdef RESYNTH
1620
   int resynth = 1;
1621
#else
1622
48.8M
   int resynth = !encode || theta_rdo;
1623
48.8M
#endif
1624
48.8M
   struct band_ctx ctx;
1625
#ifdef ENABLE_QEXT
1626
   int ext_b;
1627
   opus_int32 ext_balance=0;
1628
   opus_int32 ext_tell=0;
1629
   VARDECL(unsigned char, ext_bytes_save);
1630
#endif
1631
48.8M
   SAVE_STACK;
1632
1633
48.8M
   M = 1<<LM;
1634
48.8M
   B = shortBlocks ? M : 1;
1635
48.8M
   norm_offset = M*eBands[start];
1636
   /* No need to allocate norm for the last band because we don't need an
1637
      output in that band. */
1638
48.8M
   ALLOC(_norm, C*(M*eBands[m->nbEBands-1]-norm_offset), celt_norm);
1639
48.8M
   norm = _norm;
1640
48.8M
   norm2 = norm + M*eBands[m->nbEBands-1]-norm_offset;
1641
1642
   /* For decoding, we can use the last band as scratch space because we don't need that
1643
      scratch space for the last band and we don't care about the data there until we're
1644
      decoding the last band. */
1645
48.8M
   if (encode && resynth)
1646
5.63M
      resynth_alloc = M*(eBands[m->nbEBands]-eBands[m->nbEBands-1]);
1647
43.2M
   else
1648
43.2M
      resynth_alloc = ALLOC_NONE;
1649
48.8M
   ALLOC(_lowband_scratch, resynth_alloc, celt_norm);
1650
48.8M
   if (encode && resynth)
1651
5.63M
      lowband_scratch = _lowband_scratch;
1652
43.2M
   else
1653
43.2M
      lowband_scratch = X_+M*eBands[m->effEBands-1];
1654
48.8M
   ALLOC(X_save, resynth_alloc, celt_norm);
1655
48.8M
   ALLOC(Y_save, resynth_alloc, celt_norm);
1656
48.8M
   ALLOC(X_save2, resynth_alloc, celt_norm);
1657
48.8M
   ALLOC(Y_save2, resynth_alloc, celt_norm);
1658
48.8M
   ALLOC(norm_save2, resynth_alloc, celt_norm);
1659
1660
48.8M
   lowband_offset = 0;
1661
48.8M
   ctx.bandE = bandE;
1662
48.8M
   ctx.ec = ec;
1663
48.8M
   ctx.encode = encode;
1664
48.8M
   ctx.intensity = intensity;
1665
48.8M
   ctx.m = m;
1666
48.8M
   ctx.seed = *seed;
1667
48.8M
   ctx.spread = spread;
1668
48.8M
   ctx.arch = arch;
1669
48.8M
   ctx.disable_inv = disable_inv;
1670
48.8M
   ctx.resynth = resynth;
1671
48.8M
   ctx.theta_round = 0;
1672
#ifdef ENABLE_QEXT
1673
   ctx.ext_ec = ext_ec;
1674
   ctx.ext_total_bits = ext_total_bits;
1675
   ctx.extra_bands = end == NB_QEXT_BANDS || end == 2;
1676
   if (ctx.extra_bands) theta_rdo = 0;
1677
   ALLOC(ext_bytes_save, theta_rdo ? QEXT_PACKET_SIZE_CAP : ALLOC_NONE, unsigned char);
1678
#endif
1679
48.8M
   ALLOC(bytes_save, theta_rdo ? 1275 : ALLOC_NONE, unsigned char);
1680
1681
   /* Avoid injecting noise in the first band on transients. */
1682
48.8M
   ctx.avoid_split_noise = B > 1;
1683
779M
   for (i=start;i<end;i++)
1684
731M
   {
1685
731M
      opus_int32 tell;
1686
731M
      int b;
1687
731M
      int N;
1688
731M
      opus_int32 curr_balance;
1689
731M
      int effective_lowband=-1;
1690
731M
      celt_norm * OPUS_RESTRICT X, * OPUS_RESTRICT Y;
1691
731M
      int tf_change=0;
1692
731M
      unsigned x_cm;
1693
731M
      unsigned y_cm;
1694
731M
      int last;
1695
1696
731M
      ctx.i = i;
1697
731M
      last = (i==end-1);
1698
1699
731M
      X = X_+M*eBands[i];
1700
731M
      if (Y_!=NULL)
1701
222M
         Y = Y_+M*eBands[i];
1702
508M
      else
1703
508M
         Y = NULL;
1704
731M
      N = M*eBands[i+1]-M*eBands[i];
1705
731M
      celt_assert(N > 0);
1706
731M
      tell = ec_tell_frac(ec);
1707
1708
      /* Compute how many bits we want to allocate to this band */
1709
731M
      if (i != start)
1710
682M
         balance -= tell;
1711
731M
      remaining_bits = total_bits-tell-1;
1712
731M
      ctx.remaining_bits = remaining_bits;
1713
#ifdef ENABLE_QEXT
1714
      if (i != start) {
1715
         ext_balance += extra_pulses[i-1] + ext_tell;
1716
      }
1717
      ext_tell = ec_tell_frac(ext_ec);
1718
      ctx.extra_bits = extra_pulses[i];
1719
      if (i != start)
1720
         ext_balance -= ext_tell;
1721
      if (i <= codedBands-1)
1722
      {
1723
         opus_int32 ext_curr_balance = celt_sudiv(ext_balance, IMIN(3, codedBands-i));
1724
         ext_b = IMAX(0, IMIN(16383, IMIN(ext_total_bits-ext_tell,extra_pulses[i]+ext_curr_balance)));
1725
      } else {
1726
         ext_b = 0;
1727
      }
1728
#endif
1729
731M
      if (i <= codedBands-1)
1730
75.9M
      {
1731
75.9M
         curr_balance = celt_sudiv(balance, IMIN(3, codedBands-i));
1732
75.9M
         b = IMAX(0, IMIN(16383, IMIN(remaining_bits+1,pulses[i]+curr_balance)));
1733
655M
      } else {
1734
655M
         b = 0;
1735
655M
      }
1736
1737
731M
#ifndef DISABLE_UPDATE_DRAFT
1738
731M
      if (resynth && (M*eBands[i]-N >= M*eBands[start] || i==start+1) && (update_lowband || lowband_offset==0))
1739
7.19M
            lowband_offset = i;
1740
731M
      if (i == start+1)
1741
48.8M
         special_hybrid_folding(m, norm, norm2, start, M, dual_stereo);
1742
#else
1743
      if (resynth && M*eBands[i]-N >= M*eBands[start] && (update_lowband || lowband_offset==0))
1744
            lowband_offset = i;
1745
#endif
1746
1747
731M
      tf_change = tf_res[i];
1748
731M
      ctx.tf_change = tf_change;
1749
731M
      if (i>=m->effEBands)
1750
0
      {
1751
0
         X=norm;
1752
0
         if (Y_!=NULL)
1753
0
            Y = norm;
1754
0
         lowband_scratch = NULL;
1755
0
      }
1756
731M
      if (last && !theta_rdo)
1757
43.2M
         lowband_scratch = NULL;
1758
1759
      /* Get a conservative estimate of the collapse_mask's for the bands we're
1760
         going to be folding from. */
1761
731M
      if (lowband_offset != 0 && (spread!=SPREAD_AGGRESSIVE || B>1 || tf_change<0))
1762
83.6M
      {
1763
83.6M
         int fold_start;
1764
83.6M
         int fold_end;
1765
83.6M
         int fold_i;
1766
         /* This ensures we never repeat spectral content within one band */
1767
83.6M
         effective_lowband = IMAX(0, M*eBands[lowband_offset]-norm_offset-N);
1768
83.6M
         fold_start = lowband_offset;
1769
84.1M
         while(M*eBands[--fold_start] > effective_lowband+norm_offset);
1770
83.6M
         fold_end = lowband_offset-1;
1771
83.6M
#ifndef DISABLE_UPDATE_DRAFT
1772
215M
         while(++fold_end < i && M*eBands[fold_end] < effective_lowband+norm_offset+N);
1773
#else
1774
         while(M*eBands[++fold_end] < effective_lowband+norm_offset+N);
1775
#endif
1776
83.6M
         x_cm = y_cm = 0;
1777
215M
         fold_i = fold_start; do {
1778
215M
           x_cm |= collapse_masks[fold_i*C+0];
1779
215M
           y_cm |= collapse_masks[fold_i*C+C-1];
1780
215M
         } while (++fold_i<fold_end);
1781
83.6M
      }
1782
      /* Otherwise, we'll be using the LCG to fold, so all blocks will (almost
1783
         always) be non-zero. */
1784
647M
      else
1785
647M
         x_cm = y_cm = (1<<B)-1;
1786
1787
731M
      if (dual_stereo && i==intensity)
1788
31.5k
      {
1789
31.5k
         int j;
1790
1791
         /* Switch off dual stereo to do intensity. */
1792
31.5k
         dual_stereo = 0;
1793
31.5k
         if (resynth)
1794
0
            for (j=0;j<M*eBands[i]-norm_offset;j++)
1795
0
               norm[j] = HALF32(norm[j]+norm2[j]);
1796
31.5k
      }
1797
731M
      if (dual_stereo)
1798
652k
      {
1799
652k
         x_cm = quant_band(&ctx, X, N, b/2, B,
1800
652k
               effective_lowband != -1 ? norm+effective_lowband : NULL, LM,
1801
652k
               last?NULL:norm+M*eBands[i]-norm_offset, Q31ONE, lowband_scratch, x_cm ARG_QEXT(ext_b/2));
1802
652k
         y_cm = quant_band(&ctx, Y, N, b/2, B,
1803
652k
               effective_lowband != -1 ? norm2+effective_lowband : NULL, LM,
1804
652k
               last?NULL:norm2+M*eBands[i]-norm_offset, Q31ONE, lowband_scratch, y_cm ARG_QEXT(ext_b/2));
1805
730M
      } else {
1806
730M
         if (Y!=NULL)
1807
222M
         {
1808
222M
            if (theta_rdo && i < intensity)
1809
1.48M
            {
1810
1.48M
               ec_ctx ec_save, ec_save2;
1811
1.48M
               struct band_ctx ctx_save, ctx_save2;
1812
1.48M
               opus_val32 dist0, dist1;
1813
1.48M
               unsigned cm, cm2;
1814
1.48M
               int nstart_bytes, nend_bytes, save_bytes;
1815
1.48M
               unsigned char *bytes_buf;
1816
#ifdef ENABLE_QEXT
1817
               ec_ctx ext_ec_save, ext_ec_save2;
1818
               unsigned char *ext_bytes_buf;
1819
               int ext_nstart_bytes, ext_nend_bytes, ext_save_bytes;
1820
#endif
1821
1.48M
               opus_val16 w[2];
1822
1.48M
               compute_channel_weights(bandE[i], bandE[i+m->nbEBands], w);
1823
               /* Make a copy. */
1824
1.48M
               cm = x_cm|y_cm;
1825
1.48M
               ec_save = *ec;
1826
#ifdef ENABLE_QEXT
1827
               ext_ec_save = *ext_ec;
1828
#endif
1829
1.48M
               ctx_save = ctx;
1830
1.48M
               OPUS_COPY(X_save, X, N);
1831
1.48M
               OPUS_COPY(Y_save, Y, N);
1832
               /* Encode and round down. */
1833
1.48M
               ctx.theta_round = -1;
1834
1.48M
               x_cm = quant_band_stereo(&ctx, X, Y, N, b, B,
1835
1.48M
                     effective_lowband != -1 ? norm+effective_lowband : NULL, LM,
1836
1.48M
                     last?NULL:norm+M*eBands[i]-norm_offset, lowband_scratch, cm ARG_QEXT(ext_b) ARG_QEXT(cap));
1837
1.48M
               dist0 = MULT16_32_Q15(w[0], celt_inner_prod_norm_shift(X_save, X, N, arch)) + MULT16_32_Q15(w[1], celt_inner_prod_norm_shift(Y_save, Y, N, arch));
1838
1839
               /* Save first result. */
1840
1.48M
               cm2 = x_cm;
1841
1.48M
               ec_save2 = *ec;
1842
#ifdef ENABLE_QEXT
1843
               ext_ec_save2 = *ext_ec;
1844
#endif
1845
1.48M
               ctx_save2 = ctx;
1846
1.48M
               OPUS_COPY(X_save2, X, N);
1847
1.48M
               OPUS_COPY(Y_save2, Y, N);
1848
1.48M
               if (!last)
1849
1.46M
                  OPUS_COPY(norm_save2, norm+M*eBands[i]-norm_offset, N);
1850
1.48M
               nstart_bytes = ec_save.offs;
1851
1.48M
               nend_bytes = ec_save.storage;
1852
1.48M
               bytes_buf = ec_save.buf+nstart_bytes;
1853
1.48M
               save_bytes = nend_bytes-nstart_bytes;
1854
1.48M
               OPUS_COPY(bytes_save, bytes_buf, save_bytes);
1855
#ifdef ENABLE_QEXT
1856
               ext_nstart_bytes = ext_ec_save.offs;
1857
               ext_nend_bytes = ext_ec_save.storage;
1858
               ext_bytes_buf = ext_ec_save.buf!=NULL ? ext_ec_save.buf+ext_nstart_bytes : NULL;
1859
               ext_save_bytes = ext_nend_bytes-ext_nstart_bytes;
1860
               if (ext_save_bytes) OPUS_COPY(ext_bytes_save, ext_bytes_buf, ext_save_bytes);
1861
#endif
1862
               /* Restore */
1863
1.48M
               *ec = ec_save;
1864
#ifdef ENABLE_QEXT
1865
               *ext_ec = ext_ec_save;
1866
#endif
1867
1.48M
               ctx = ctx_save;
1868
1.48M
               OPUS_COPY(X, X_save, N);
1869
1.48M
               OPUS_COPY(Y, Y_save, N);
1870
1.48M
#ifndef DISABLE_UPDATE_DRAFT
1871
1.48M
               if (i == start+1)
1872
134k
                  special_hybrid_folding(m, norm, norm2, start, M, dual_stereo);
1873
1.48M
#endif
1874
               /* Encode and round up. */
1875
1.48M
               ctx.theta_round = 1;
1876
1.48M
               x_cm = quant_band_stereo(&ctx, X, Y, N, b, B,
1877
1.48M
                     effective_lowband != -1 ? norm+effective_lowband : NULL, LM,
1878
1.48M
                     last?NULL:norm+M*eBands[i]-norm_offset, lowband_scratch, cm ARG_QEXT(ext_b) ARG_QEXT(cap));
1879
1.48M
               dist1 = MULT16_32_Q15(w[0], celt_inner_prod_norm_shift(X_save, X, N, arch)) + MULT16_32_Q15(w[1], celt_inner_prod_norm_shift(Y_save, Y, N, arch));
1880
1.48M
               if (dist0 >= dist1) {
1881
1.16M
                  x_cm = cm2;
1882
1.16M
                  *ec = ec_save2;
1883
#ifdef ENABLE_QEXT
1884
                  *ext_ec = ext_ec_save2;
1885
#endif
1886
1.16M
                  ctx = ctx_save2;
1887
1.16M
                  OPUS_COPY(X, X_save2, N);
1888
1.16M
                  OPUS_COPY(Y, Y_save2, N);
1889
1.16M
                  if (!last)
1890
1.15M
                     OPUS_COPY(norm+M*eBands[i]-norm_offset, norm_save2, N);
1891
1.16M
                  OPUS_COPY(bytes_buf, bytes_save, save_bytes);
1892
#ifdef ENABLE_QEXT
1893
                  if (ext_save_bytes) OPUS_COPY(ext_bytes_buf, ext_bytes_save, ext_save_bytes);
1894
#endif
1895
1.16M
               }
1896
220M
            } else {
1897
220M
               ctx.theta_round = 0;
1898
220M
               x_cm = quant_band_stereo(&ctx, X, Y, N, b, B,
1899
220M
                     effective_lowband != -1 ? norm+effective_lowband : NULL, LM,
1900
220M
                     last?NULL:norm+M*eBands[i]-norm_offset, lowband_scratch, x_cm|y_cm ARG_QEXT(ext_b) ARG_QEXT(cap));
1901
220M
            }
1902
508M
         } else {
1903
508M
            x_cm = quant_band(&ctx, X, N, b, B,
1904
508M
                  effective_lowband != -1 ? norm+effective_lowband : NULL, LM,
1905
508M
                  last?NULL:norm+M*eBands[i]-norm_offset, Q31ONE, lowband_scratch, x_cm|y_cm ARG_QEXT(ext_b));
1906
508M
         }
1907
730M
         y_cm = x_cm;
1908
730M
      }
1909
731M
      collapse_masks[i*C+0] = (unsigned char)x_cm;
1910
731M
      collapse_masks[i*C+C-1] = (unsigned char)y_cm;
1911
731M
      balance += pulses[i] + tell;
1912
1913
      /* Update the folding position only as long as we have 1 bit/sample depth. */
1914
731M
      update_lowband = b>(N<<BITRES);
1915
      /* We only need to avoid noise on a split for the first band. After that, we
1916
         have folding. */
1917
731M
      ctx.avoid_split_noise = 0;
1918
731M
   }
1919
48.8M
   *seed = ctx.seed;
1920
1921
48.8M
   RESTORE_STACK;
1922
48.8M
}