Coverage Report

Created: 2026-07-25 07:52

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/opus/celt/bands.c
Line
Count
Source
1
/* Copyright (c) 2007-2008 CSIRO
2
   Copyright (c) 2007-2009 Xiph.Org Foundation
3
   Copyright (c) 2008-2009 Gregory Maxwell
4
   Written by Jean-Marc Valin and Gregory Maxwell */
5
/*
6
   Redistribution and use in source and binary forms, with or without
7
   modification, are permitted provided that the following conditions
8
   are met:
9
10
   - Redistributions of source code must retain the above copyright
11
   notice, this list of conditions and the following disclaimer.
12
13
   - Redistributions in binary form must reproduce the above copyright
14
   notice, this list of conditions and the following disclaimer in the
15
   documentation and/or other materials provided with the distribution.
16
17
   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18
   ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19
   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20
   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
21
   OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
22
   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
23
   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
24
   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
25
   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
26
   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27
   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
*/
29
30
#ifdef HAVE_CONFIG_H
31
#include "config.h"
32
#endif
33
34
#include <math.h>
35
#include "bands.h"
36
#include "modes.h"
37
#include "vq.h"
38
#include "cwrs.h"
39
#include "stack_alloc.h"
40
#include "os_support.h"
41
#include "mathops.h"
42
#include "rate.h"
43
#include "quant_bands.h"
44
#include "pitch.h"
45
46
int hysteresis_decision(opus_val16 val, const opus_val16 *thresholds, const opus_val16 *hysteresis, int N, int prev)
47
0
{
48
0
   int i;
49
0
   for (i=0;i<N;i++)
50
0
   {
51
0
      if (val < thresholds[i])
52
0
         break;
53
0
   }
54
0
   if (i>prev && val < thresholds[prev]+hysteresis[prev])
55
0
      i=prev;
56
0
   if (i<prev && val > thresholds[prev-1]-hysteresis[prev-1])
57
0
      i=prev;
58
0
   return i;
59
0
}
60
61
opus_uint32 celt_lcg_rand(opus_uint32 seed)
62
58.3M
{
63
58.3M
   return 1664525 * seed + 1013904223;
64
58.3M
}
65
66
/* This is a cos() approximation designed to be bit-exact on any platform. Bit exactness
67
   with this approximation is important because it has an impact on the bit allocation */
68
opus_int16 bitexact_cos(opus_int16 x)
69
377k
{
70
377k
   opus_int32 tmp;
71
377k
   opus_int16 x2;
72
377k
   tmp = (4096+((opus_int32)(x)*(x)))>>13;
73
377k
   celt_sig_assert(tmp<=32767);
74
377k
   x2 = tmp;
75
377k
   x2 = (32767-x2) + FRAC_MUL16(x2, (-7651 + FRAC_MUL16(x2, (8277 + FRAC_MUL16(-626, x2)))));
76
377k
   celt_sig_assert(x2<=32766);
77
377k
   return 1+x2;
78
377k
}
79
80
int bitexact_log2tan(int isin,int icos)
81
188k
{
82
188k
   int lc;
83
188k
   int ls;
84
188k
   lc=EC_ILOG(icos);
85
188k
   ls=EC_ILOG(isin);
86
188k
   icos<<=15-lc;
87
188k
   isin<<=15-ls;
88
188k
   return (ls-lc)*(1<<11)
89
188k
         +FRAC_MUL16(isin, FRAC_MUL16(isin, -2597) + 7932)
90
188k
         -FRAC_MUL16(icos, FRAC_MUL16(icos, -2597) + 7932);
91
188k
}
92
93
#ifdef FIXED_POINT
94
/* Compute the amplitude (sqrt energy) in each of the bands */
95
void compute_band_energies(const CELTMode *m, const celt_sig *X, celt_ener *bandE, int end, int C, int LM, int arch)
96
{
97
   int i, c, N;
98
   const opus_int16 *eBands = m->eBands;
99
   (void)arch;
100
   N = m->shortMdctSize<<LM;
101
   c=0; do {
102
      for (i=0;i<end;i++)
103
      {
104
         int j;
105
         opus_val32 maxval=0;
106
         opus_val32 sum = 0;
107
108
         maxval = celt_maxabs32(&X[c*N+(eBands[i]<<LM)], (eBands[i+1]-eBands[i])<<LM);
109
         if (maxval > 0)
110
         {
111
            int shift = IMAX(0, 30 - celt_ilog2(maxval+(maxval>>14)+1) - ((((m->logN[i]+7)>>BITRES)+LM+1)>>1));
112
            j=eBands[i]<<LM; do {
113
               opus_val32 x = SHL32(X[j+c*N],shift);
114
               sum = ADD32(sum, MULT32_32_Q31(x, x));
115
            } while (++j<eBands[i+1]<<LM);
116
            bandE[i+c*m->nbEBands] = MAX32(maxval, PSHR32(celt_sqrt32(SHR32(sum,1)), shift));
117
         } else {
118
            bandE[i+c*m->nbEBands] = EPSILON;
119
         }
120
      }
121
   } while (++c<C);
122
}
123
124
/* Normalise each band such that the energy is one. */
125
void normalise_bands(const CELTMode *m, const celt_sig * OPUS_RESTRICT freq, celt_norm * OPUS_RESTRICT X, const celt_ener *bandE, int end, int C, int M)
126
{
127
   int i, c, N;
128
   const opus_int16 *eBands = m->eBands;
129
   N = M*m->shortMdctSize;
130
   c=0; do {
131
      i=0; do {
132
         int j,shift;
133
         opus_val32 E;
134
         opus_val32 g;
135
         E = bandE[i+c*m->nbEBands];
136
         /* For very low energies, we need this to make sure not to prevent energy rounding from
137
            blowing up the normalized signal. */
138
         if (E < 10) E += EPSILON;
139
         shift = 30-celt_zlog2(E);
140
         E = SHL32(E, shift);
141
         g = celt_rcp_norm32(E);
142
         j=M*eBands[i]; do {
143
            X[j+c*N] = PSHR32(MULT32_32_Q31(g, SHL32(freq[j+c*N], shift)), 30-NORM_SHIFT);
144
         } while (++j<M*eBands[i+1]);
145
      } while (++i<end);
146
   } while (++c<C);
147
}
148
149
#else /* FIXED_POINT */
150
/* Compute the amplitude (sqrt energy) in each of the bands */
151
void compute_band_energies(const CELTMode *m, const celt_sig *X, celt_ener *bandE, int end, int C, int LM, int arch)
152
0
{
153
0
   int i, c, N;
154
0
   const opus_int16 *eBands = m->eBands;
155
0
   N = m->shortMdctSize<<LM;
156
0
   c=0; do {
157
0
      for (i=0;i<end;i++)
158
0
      {
159
0
         opus_val32 sum;
160
0
         sum = 1e-27f + celt_inner_prod(&X[c*N+(eBands[i]<<LM)], &X[c*N+(eBands[i]<<LM)], (eBands[i+1]-eBands[i])<<LM, arch);
161
0
         bandE[i+c*m->nbEBands] = celt_sqrt(sum);
162
         /*printf ("%f ", bandE[i+c*m->nbEBands]);*/
163
0
      }
164
0
   } while (++c<C);
165
   /*printf ("\n");*/
166
0
}
167
168
/* Normalise each band such that the energy is one. */
169
void normalise_bands(const CELTMode *m, const celt_sig * OPUS_RESTRICT freq, celt_norm * OPUS_RESTRICT X, const celt_ener *bandE, int end, int C, int M)
170
0
{
171
0
   int i, c, N;
172
0
   const opus_int16 *eBands = m->eBands;
173
0
   N = M*m->shortMdctSize;
174
0
   c=0; do {
175
0
      for (i=0;i<end;i++)
176
0
      {
177
0
         int j;
178
0
         opus_val16 g = 1.f/(1e-27f+bandE[i+c*m->nbEBands]);
179
0
         for (j=M*eBands[i];j<M*eBands[i+1];j++)
180
0
            X[j+c*N] = freq[j+c*N]*g;
181
0
      }
182
0
   } while (++c<C);
183
0
}
184
185
#endif /* FIXED_POINT */
186
187
/* De-normalise the energy to produce the synthesis from the unit-energy bands */
188
void denormalise_bands(const CELTMode *m, const celt_norm * OPUS_RESTRICT X,
189
      celt_sig * OPUS_RESTRICT freq, const celt_glog *bandLogE, int start,
190
      int end, int M, int downsample, int silence)
191
265k
{
192
265k
   int i, N;
193
265k
   int bound;
194
265k
   celt_sig * OPUS_RESTRICT f;
195
265k
   const celt_norm * OPUS_RESTRICT x;
196
265k
   const opus_int16 *eBands = m->eBands;
197
265k
   N = M*m->shortMdctSize;
198
265k
   bound = M*eBands[end];
199
265k
   if (downsample!=1)
200
0
      bound = IMIN(bound, N/downsample);
201
265k
   if (silence)
202
32.5k
   {
203
32.5k
      bound = 0;
204
32.5k
      start = end = 0;
205
32.5k
   }
206
265k
   f = freq;
207
265k
   x = X+M*eBands[start];
208
265k
   if (start != 0)
209
52.6k
   {
210
12.2M
      for (i=0;i<M*eBands[start];i++)
211
12.1M
         *f++ = 0;
212
212k
   } else {
213
212k
      f += M*eBands[start];
214
212k
   }
215
3.69M
   for (i=start;i<end;i++)
216
3.43M
   {
217
3.43M
      int j, band_end;
218
3.43M
      opus_val32 g;
219
3.43M
      celt_glog lg;
220
#ifdef FIXED_POINT
221
      int shift;
222
#endif
223
3.43M
      j=M*eBands[i];
224
3.43M
      band_end = M*eBands[i+1];
225
3.43M
      lg = ADD32(bandLogE[i], SHL32((opus_val32)eMeans[i],DB_SHIFT-4));
226
3.43M
#ifndef FIXED_POINT
227
3.43M
      g = celt_exp2_db(MIN32(32.f, lg));
228
#else
229
      /* Handle the integer part of the log energy */
230
      shift = 17-(lg>>DB_SHIFT);
231
      if (shift>=31)
232
      {
233
         shift=0;
234
         g=0;
235
      } else {
236
         /* Handle the fractional part. */
237
         g = SHL32(celt_exp2_db_frac((lg&((1<<DB_SHIFT)-1))), 2);
238
      }
239
      /* Handle extreme gains with negative shift. */
240
      if (shift<0)
241
      {
242
         /* To avoid overflow, we're
243
            capping the gain here, which is equivalent to a cap of 18 on lg.
244
            This shouldn't trigger unless the bitstream is already corrupted. */
245
         g = 2147483647;
246
         shift = 0;
247
      }
248
#endif
249
60.1M
      do {
250
60.1M
         *f++ = PSHR32(MULT32_32_Q31(SHL32(*x, 30-NORM_SHIFT), g), shift);
251
60.1M
         x++;
252
60.1M
      } while (++j<band_end);
253
3.43M
   }
254
265k
   celt_assert(start <= end);
255
265k
   OPUS_CLEAR(&freq[bound], N-bound);
256
265k
}
257
258
/* This prevents energy collapse for transients with multiple short MDCTs */
259
void anti_collapse(const CELTMode *m, celt_norm *X_, unsigned char *collapse_masks, int LM, int C, int size,
260
      int start, int end, const celt_glog *logE, const celt_glog *prev1logE,
261
      const celt_glog *prev2logE, const int *pulses, opus_uint32 seed, int encode, int arch)
262
2.13k
{
263
2.13k
   int c, i, j, k;
264
34.6k
   for (i=start;i<end;i++)
265
32.4k
   {
266
32.4k
      int N0;
267
32.4k
      opus_val16 thresh, sqrt_1;
268
32.4k
      int depth;
269
#ifdef FIXED_POINT
270
      int shift;
271
      opus_val32 thresh32;
272
#endif
273
274
32.4k
      N0 = m->eBands[i+1]-m->eBands[i];
275
      /* depth in 1/8 bits */
276
32.4k
      celt_sig_assert(pulses[i]>=0);
277
32.4k
      depth = celt_udiv(1+pulses[i], (m->eBands[i+1]-m->eBands[i]))>>LM;
278
279
#ifdef FIXED_POINT
280
      thresh32 = SHR32(celt_exp2(-SHL16(depth, 10-BITRES)),1);
281
      thresh = MULT16_32_Q15(QCONST16(0.5f, 15), MIN32(32767,thresh32));
282
      {
283
         opus_val32 t;
284
         t = N0<<LM;
285
         shift = celt_ilog2(t)>>1;
286
         t = SHL32(t, (7-shift)<<1);
287
         sqrt_1 = celt_rsqrt_norm(t);
288
      }
289
#else
290
32.4k
      thresh = .5f*celt_exp2(-.125f*depth);
291
32.4k
      sqrt_1 = celt_rsqrt(N0<<LM);
292
32.4k
#endif
293
294
32.4k
      c=0; do
295
39.3k
      {
296
39.3k
         celt_norm *X;
297
39.3k
         celt_glog prev1;
298
39.3k
         celt_glog prev2;
299
39.3k
         opus_val32 Ediff;
300
39.3k
         celt_norm r;
301
39.3k
         int renormalize=0;
302
39.3k
         prev1 = prev1logE[c*m->nbEBands+i];
303
39.3k
         prev2 = prev2logE[c*m->nbEBands+i];
304
39.3k
         if (!encode && C==1)
305
25.6k
         {
306
25.6k
            prev1 = MAXG(prev1,prev1logE[m->nbEBands+i]);
307
25.6k
            prev2 = MAXG(prev2,prev2logE[m->nbEBands+i]);
308
25.6k
         }
309
39.3k
         Ediff = logE[c*m->nbEBands+i]-MING(prev1,prev2);
310
39.3k
         Ediff = MAX32(0, Ediff);
311
312
#ifdef FIXED_POINT
313
         if (Ediff < GCONST(16.f))
314
         {
315
            opus_val32 r32 = SHR32(celt_exp2_db(-Ediff),1);
316
            r = 2*MIN16(16383,r32);
317
         } else {
318
            r = 0;
319
         }
320
         if (LM==3)
321
            r = MULT16_16_Q14(23170, MIN32(23169, r));
322
         r = SHR16(MIN16(thresh, r),1);
323
         r = VSHR32(MULT16_16_Q15(sqrt_1, r),shift+14-NORM_SHIFT);
324
#else
325
         /* r needs to be multiplied by 2 or 2*sqrt(2) depending on LM because
326
            short blocks don't have the same energy as long */
327
39.3k
         r = 2.f*celt_exp2_db(-Ediff);
328
39.3k
         if (LM==3)
329
12.8k
            r *= 1.41421356f;
330
39.3k
         r = MIN16(thresh, r);
331
39.3k
         r = r*sqrt_1;
332
39.3k
#endif
333
39.3k
         X = X_+c*size+(m->eBands[i]<<LM);
334
248k
         for (k=0;k<1<<LM;k++)
335
208k
         {
336
            /* Detect collapse */
337
208k
            if (!(collapse_masks[i*C+c]&1<<k))
338
51.3k
            {
339
               /* Fill with noise */
340
187k
               for (j=0;j<N0;j++)
341
136k
               {
342
136k
                  seed = celt_lcg_rand(seed);
343
136k
                  X[(j<<LM)+k] = (seed&0x8000 ? r : -r);
344
136k
               }
345
51.3k
               renormalize = 1;
346
51.3k
            }
347
208k
         }
348
         /* We just added some energy, so we need to renormalise */
349
39.3k
         if (renormalize)
350
15.4k
            renormalise_vector(X, N0<<LM, Q31ONE, arch);
351
39.3k
      } while (++c<C);
352
32.4k
   }
353
2.13k
}
354
355
/* Compute the weights to use for optimizing normalized distortion across
356
   channels. We use the amplitude to weight square distortion, which means
357
   that we use the square root of the value we would have been using if we
358
   wanted to minimize the MSE in the non-normalized domain. This roughly
359
   corresponds to some quick-and-dirty perceptual experiments I ran to
360
   measure inter-aural masking (there doesn't seem to be any published data
361
   on the topic). */
362
static void compute_channel_weights(celt_ener Ex, celt_ener Ey, opus_val16 w[2])
363
0
{
364
0
   celt_ener minE;
365
#ifdef FIXED_POINT
366
   int shift;
367
#endif
368
0
   minE = MIN32(Ex, Ey);
369
   /* Adjustment to make the weights a bit more conservative. */
370
0
   Ex = ADD32(Ex, minE/3);
371
0
   Ey = ADD32(Ey, minE/3);
372
#ifdef FIXED_POINT
373
   shift = celt_ilog2(EPSILON+MAX32(Ex, Ey))-14;
374
#endif
375
0
   w[0] = VSHR32(Ex, shift);
376
0
   w[1] = VSHR32(Ey, shift);
377
0
}
378
379
static void intensity_stereo(const CELTMode *m, celt_norm * OPUS_RESTRICT X, const celt_norm * OPUS_RESTRICT Y, const celt_ener *bandE, int bandID, int N)
380
0
{
381
0
   int i = bandID;
382
0
   int j;
383
0
   opus_val16 a1, a2;
384
0
   opus_val16 left, right;
385
0
   opus_val16 norm;
386
#ifdef FIXED_POINT
387
   int shift = celt_zlog2(MAX32(bandE[i], bandE[i+m->nbEBands]))-13;
388
#endif
389
0
   left = VSHR32(bandE[i],shift);
390
0
   right = VSHR32(bandE[i+m->nbEBands],shift);
391
0
   norm = EPSILON + celt_sqrt(EPSILON+MULT16_16(left,left)+MULT16_16(right,right));
392
#ifdef FIXED_POINT
393
   left = MIN32(left, norm-1);
394
   right = MIN32(right, norm-1);
395
#endif
396
0
   a1 = DIV32_16(SHL32(EXTEND32(left),15),norm);
397
0
   a2 = DIV32_16(SHL32(EXTEND32(right),15),norm);
398
0
   for (j=0;j<N;j++)
399
0
   {
400
0
      X[j] = ADD32(MULT16_32_Q15(a1, X[j]), MULT16_32_Q15(a2, Y[j]));
401
      /* Side is not encoded, no need to calculate */
402
0
   }
403
0
}
404
405
static void stereo_split(celt_norm * OPUS_RESTRICT X, celt_norm * OPUS_RESTRICT Y, int N)
406
0
{
407
0
   int j;
408
0
   for (j=0;j<N;j++)
409
0
   {
410
0
      opus_val32 r, l;
411
0
      l = MULT32_32_Q31(QCONST32(.70710678f,31), X[j]);
412
0
      r = MULT32_32_Q31(QCONST32(.70710678f,31), Y[j]);
413
0
      X[j] = ADD32(l, r);
414
0
      Y[j] = SUB32(r, l);
415
0
   }
416
0
}
417
418
static void stereo_merge(celt_norm * OPUS_RESTRICT X, celt_norm * OPUS_RESTRICT Y, opus_val32 mid, int N, int arch)
419
181k
{
420
181k
   int j;
421
181k
   opus_val32 xp=0, side=0;
422
181k
   opus_val32 El, Er;
423
#ifdef FIXED_POINT
424
   int kl, kr;
425
#endif
426
181k
   opus_val32 t, lgain, rgain;
427
428
   /* Compute the norm of X+Y and X-Y as |X|^2 + |Y|^2 +/- sum(xy) */
429
181k
   xp = celt_inner_prod_norm_shift(Y, X, N, arch);
430
181k
   side = celt_inner_prod_norm_shift(Y, Y, N, arch);
431
   /* Compensating for the mid normalization */
432
181k
   xp = MULT32_32_Q31(mid, xp);
433
   /* mid and side are in Q15, not Q14 like X and Y */
434
181k
   El = SHR32(MULT32_32_Q31(mid, mid),3) + side - 2*xp;
435
181k
   Er = SHR32(MULT32_32_Q31(mid, mid),3) + side + 2*xp;
436
181k
   if (Er < QCONST32(6e-4f, 28) || El < QCONST32(6e-4f, 28))
437
677
   {
438
677
      OPUS_COPY(Y, X, N);
439
677
      return;
440
677
   }
441
442
#ifdef FIXED_POINT
443
   kl = celt_ilog2(El)>>1;
444
   kr = celt_ilog2(Er)>>1;
445
#endif
446
180k
   t = VSHR32(El, (kl<<1)-29);
447
180k
   lgain = celt_rsqrt_norm32(t);
448
180k
   t = VSHR32(Er, (kr<<1)-29);
449
180k
   rgain = celt_rsqrt_norm32(t);
450
451
#ifdef FIXED_POINT
452
   if (kl < 7)
453
      kl = 7;
454
   if (kr < 7)
455
      kr = 7;
456
#endif
457
458
5.17M
   for (j=0;j<N;j++)
459
4.99M
   {
460
4.99M
      celt_norm r, l;
461
      /* Apply mid scaling (side is already scaled) */
462
4.99M
      l = MULT32_32_Q31(mid, X[j]);
463
4.99M
      r = Y[j];
464
4.99M
      X[j] = VSHR32(MULT32_32_Q31(lgain, SUB32(l,r)), kl-15);
465
4.99M
      Y[j] = VSHR32(MULT32_32_Q31(rgain, ADD32(l,r)), kr-15);
466
4.99M
   }
467
180k
}
468
469
/* Decide whether we should spread the pulses in the current frame */
470
int spreading_decision(const CELTMode *m, const celt_norm *X, int *average,
471
      int last_decision, int *hf_average, int *tapset_decision, int update_hf,
472
      int end, int C, int M, const int *spread_weight)
473
0
{
474
0
   int i, c, N0;
475
0
   int sum = 0, nbBands=0;
476
0
   const opus_int16 * OPUS_RESTRICT eBands = m->eBands;
477
0
   int decision;
478
0
   int hf_sum=0;
479
480
0
   celt_assert(end>0);
481
482
0
   N0 = M*m->shortMdctSize;
483
484
0
   if (M*(eBands[end]-eBands[end-1]) <= 8)
485
0
      return SPREAD_NONE;
486
0
   c=0; do {
487
0
      for (i=0;i<end;i++)
488
0
      {
489
0
         int j, N, tmp=0;
490
0
         int tcount[3] = {0,0,0};
491
0
         const celt_norm * OPUS_RESTRICT x = X+M*eBands[i]+c*N0;
492
0
         N = M*(eBands[i+1]-eBands[i]);
493
0
         if (N<=8)
494
0
            continue;
495
         /* Compute rough CDF of |x[j]| */
496
0
         for (j=0;j<N;j++)
497
0
         {
498
0
            opus_val32 x2N; /* Q13 */
499
500
0
            x2N = MULT16_16(MULT16_16_Q15(SHR32(x[j], NORM_SHIFT-14), SHR32(x[j], NORM_SHIFT-14)), N);
501
0
            if (x2N < QCONST16(0.25f,13))
502
0
               tcount[0]++;
503
0
            if (x2N < QCONST16(0.0625f,13))
504
0
               tcount[1]++;
505
0
            if (x2N < QCONST16(0.015625f,13))
506
0
               tcount[2]++;
507
0
         }
508
509
         /* Only include four last bands (8 kHz and up) */
510
0
         if (i>m->nbEBands-4)
511
0
            hf_sum += celt_udiv(32*(tcount[1]+tcount[0]), N);
512
0
         tmp = (2*tcount[2] >= N) + (2*tcount[1] >= N) + (2*tcount[0] >= N);
513
0
         sum += tmp*spread_weight[i];
514
0
         nbBands+=spread_weight[i];
515
0
      }
516
0
   } while (++c<C);
517
518
0
   if (update_hf)
519
0
   {
520
0
      if (hf_sum)
521
0
         hf_sum = celt_udiv(hf_sum, C*(4-m->nbEBands+end));
522
0
      *hf_average = (*hf_average+hf_sum)>>1;
523
0
      hf_sum = *hf_average;
524
0
      if (*tapset_decision==2)
525
0
         hf_sum += 4;
526
0
      else if (*tapset_decision==0)
527
0
         hf_sum -= 4;
528
0
      if (hf_sum > 22)
529
0
         *tapset_decision=2;
530
0
      else if (hf_sum > 18)
531
0
         *tapset_decision=1;
532
0
      else
533
0
         *tapset_decision=0;
534
0
   }
535
   /*printf("%d %d %d\n", hf_sum, *hf_average, *tapset_decision);*/
536
0
   celt_assert(nbBands>0); /* end has to be non-zero */
537
0
   celt_assert(sum>=0);
538
0
   sum = celt_udiv((opus_int32)sum<<8, nbBands);
539
   /* Recursive averaging */
540
0
   sum = (sum+*average)>>1;
541
0
   *average = sum;
542
   /* Hysteresis */
543
0
   sum = (3*sum + (((3-last_decision)<<7) + 64) + 2)>>2;
544
0
   if (sum < 80)
545
0
   {
546
0
      decision = SPREAD_AGGRESSIVE;
547
0
   } else if (sum < 256)
548
0
   {
549
0
      decision = SPREAD_NORMAL;
550
0
   } else if (sum < 384)
551
0
   {
552
0
      decision = SPREAD_LIGHT;
553
0
   } else {
554
0
      decision = SPREAD_NONE;
555
0
   }
556
#ifdef FUZZING
557
   decision = rand()&0x3;
558
   *tapset_decision=rand()%3;
559
#endif
560
0
   return decision;
561
0
}
562
563
/* Indexing table for converting from natural Hadamard to ordery Hadamard
564
   This is essentially a bit-reversed Gray, on top of which we've added
565
   an inversion of the order because we want the DC at the end rather than
566
   the beginning. The lines are for N=2, 4, 8, 16 */
567
static const int ordery_table[] = {
568
       1,  0,
569
       3,  0,  2,  1,
570
       7,  0,  4,  3,  6,  1,  5,  2,
571
      15,  0,  8,  7, 12,  3, 11,  4, 14,  1,  9,  6, 13,  2, 10,  5,
572
};
573
574
static void deinterleave_hadamard(celt_norm *X, int N0, int stride, int hadamard)
575
148k
{
576
148k
   int i,j;
577
148k
   VARDECL(celt_norm, tmp);
578
148k
   int N;
579
148k
   SAVE_STACK;
580
148k
   N = N0*stride;
581
148k
   ALLOC(tmp, N, celt_norm);
582
148k
   celt_assert(stride>0);
583
148k
   if (hadamard)
584
95.0k
   {
585
95.0k
      const int *ordery = ordery_table+stride-2;
586
565k
      for (i=0;i<stride;i++)
587
469k
      {
588
1.83M
         for (j=0;j<N0;j++)
589
1.36M
            tmp[ordery[i]*N0+j] = X[j*stride+i];
590
469k
      }
591
95.0k
   } else {
592
337k
      for (i=0;i<stride;i++)
593
1.03M
         for (j=0;j<N0;j++)
594
750k
            tmp[i*N0+j] = X[j*stride+i];
595
53.1k
   }
596
148k
   OPUS_COPY(X, tmp, N);
597
148k
   RESTORE_STACK;
598
148k
}
599
600
static void interleave_hadamard(celt_norm *X, int N0, int stride, int hadamard)
601
184k
{
602
184k
   int i,j;
603
184k
   VARDECL(celt_norm, tmp);
604
184k
   int N;
605
184k
   SAVE_STACK;
606
184k
   N = N0*stride;
607
184k
   ALLOC(tmp, N, celt_norm);
608
184k
   if (hadamard)
609
122k
   {
610
122k
      const int *ordery = ordery_table+stride-2;
611
727k
      for (i=0;i<stride;i++)
612
2.61M
         for (j=0;j<N0;j++)
613
2.00M
            tmp[j*stride+i] = X[ordery[i]*N0+j];
614
122k
   } else {
615
411k
      for (i=0;i<stride;i++)
616
1.35M
         for (j=0;j<N0;j++)
617
1.00M
            tmp[j*stride+i] = X[i*N0+j];
618
62.6k
   }
619
184k
   OPUS_COPY(X, tmp, N);
620
184k
   RESTORE_STACK;
621
184k
}
622
623
void haar1(celt_norm *X, int N0, int stride)
624
683k
{
625
683k
   int i, j;
626
683k
   N0 >>= 1;
627
1.94M
   for (i=0;i<stride;i++)
628
6.99M
      for (j=0;j<N0;j++)
629
5.73M
      {
630
5.73M
         opus_val32 tmp1, tmp2;
631
5.73M
         tmp1 = MULT32_32_Q31(QCONST32(.70710678f,31), X[stride*2*j+i]);
632
5.73M
         tmp2 = MULT32_32_Q31(QCONST32(.70710678f,31), X[stride*(2*j+1)+i]);
633
5.73M
         X[stride*2*j+i] = ADD32(tmp1, tmp2);
634
5.73M
         X[stride*(2*j+1)+i] = SUB32(tmp1, tmp2);
635
5.73M
      }
636
683k
}
637
638
static int compute_qn(int N, int b, int offset, int pulse_cap, int stereo)
639
419k
{
640
419k
   static const opus_int16 exp2_table8[8] =
641
419k
      {16384, 17866, 19483, 21247, 23170, 25267, 27554, 30048};
642
419k
   int qn, qb;
643
419k
   int N2 = 2*N-1;
644
419k
   if (stereo && N==2)
645
64.6k
      N2--;
646
   /* The upper limit ensures that in a stereo split with itheta==16384, we'll
647
       always have enough bits left over to code at least one pulse in the
648
       side; otherwise it would collapse, since it doesn't get folded. */
649
419k
   qb = celt_sudiv(b+N2*offset, N2);
650
419k
   qb = IMIN(b-pulse_cap-(4<<BITRES), qb);
651
652
419k
   qb = IMIN(8<<BITRES, qb);
653
654
419k
   if (qb<(1<<BITRES>>1)) {
655
171k
      qn = 1;
656
247k
   } else {
657
247k
      qn = exp2_table8[qb&0x7]>>(14-(qb>>BITRES));
658
247k
      qn = (qn+1)>>1<<1;
659
247k
   }
660
419k
   celt_assert(qn <= 256);
661
419k
   return qn;
662
419k
}
663
664
struct band_ctx {
665
   int encode;
666
   int resynth;
667
   const CELTMode *m;
668
   int i;
669
   int intensity;
670
   int spread;
671
   int tf_change;
672
   ec_ctx *ec;
673
   opus_int32 remaining_bits;
674
   const celt_ener *bandE;
675
   opus_uint32 seed;
676
   int arch;
677
   int theta_round;
678
   int disable_inv;
679
   int avoid_split_noise;
680
#ifdef ENABLE_QEXT
681
   ec_ctx *ext_ec;
682
   int extra_bits;
683
   opus_int32 ext_total_bits;
684
   int extra_bands;
685
#endif
686
};
687
688
struct split_ctx {
689
   int inv;
690
   int imid;
691
   int iside;
692
   int delta;
693
   int itheta;
694
#ifdef ENABLE_QEXT
695
   int itheta_q30;
696
#endif
697
   int qalloc;
698
};
699
700
static void compute_theta(struct band_ctx *ctx, struct split_ctx *sctx,
701
      celt_norm *X, celt_norm *Y, int N, int *b, int B, int B0,
702
      int LM,
703
      int stereo, int *fill ARG_QEXT(int *ext_b))
704
419k
{
705
419k
   int qn;
706
419k
   int itheta=0;
707
419k
   int itheta_q30=0;
708
419k
   int delta;
709
419k
   int imid, iside;
710
419k
   int qalloc;
711
419k
   int pulse_cap;
712
419k
   int offset;
713
419k
   opus_int32 tell;
714
419k
   int inv=0;
715
419k
   int encode;
716
419k
   const CELTMode *m;
717
419k
   int i;
718
419k
   int intensity;
719
419k
   ec_ctx *ec;
720
419k
   const celt_ener *bandE;
721
722
419k
   encode = ctx->encode;
723
419k
   m = ctx->m;
724
419k
   i = ctx->i;
725
419k
   intensity = ctx->intensity;
726
419k
   ec = ctx->ec;
727
419k
   bandE = ctx->bandE;
728
729
   /* Decide on the resolution to give to the split parameter theta */
730
419k
   pulse_cap = m->logN[i]+LM*(1<<BITRES);
731
419k
   offset = (pulse_cap>>1) - (stereo&&N==2 ? QTHETA_OFFSET_TWOPHASE : QTHETA_OFFSET);
732
419k
   qn = compute_qn(N, *b, offset, pulse_cap, stereo);
733
419k
   if (stereo && i>=intensity)
734
213k
      qn = 1;
735
419k
   if (encode)
736
0
   {
737
      /* theta is the atan() of the ratio between the (normalized)
738
         side and mid. With just that parameter, we can re-scale both
739
         mid and side because we know that 1) they have unit norm and
740
         2) they are orthogonal. */
741
0
      itheta_q30 = stereo_itheta(X, Y, stereo, N, ctx->arch);
742
0
      itheta = itheta_q30>>16;
743
0
   }
744
419k
   tell = ec_tell_frac(ec);
745
419k
   if (qn!=1)
746
199k
   {
747
199k
      if (encode)
748
0
      {
749
0
         if (!stereo || ctx->theta_round == 0)
750
0
         {
751
0
            itheta = (itheta*(opus_int32)qn+8192)>>14;
752
0
            if (!stereo && ctx->avoid_split_noise && itheta > 0 && itheta < qn)
753
0
            {
754
               /* Check if the selected value of theta will cause the bit allocation
755
                  to inject noise on one side. If so, make sure the energy of that side
756
                  is zero. */
757
0
               int unquantized = celt_udiv((opus_int32)itheta*16384, qn);
758
0
               imid = bitexact_cos((opus_int16)unquantized);
759
0
               iside = bitexact_cos((opus_int16)(16384-unquantized));
760
0
               delta = FRAC_MUL16((N-1)<<7,bitexact_log2tan(iside,imid));
761
0
               if (delta > *b)
762
0
                  itheta = qn;
763
0
               else if (delta < -*b)
764
0
                  itheta = 0;
765
0
            }
766
0
         } else {
767
0
            int down;
768
            /* Bias quantization towards itheta=0 and itheta=16384. */
769
0
            int bias = itheta > 8192 ? 32767/qn : -32767/qn;
770
0
            down = IMIN(qn-1, IMAX(0, (itheta*(opus_int32)qn + bias)>>14));
771
0
            if (ctx->theta_round < 0)
772
0
               itheta = down;
773
0
            else
774
0
               itheta = down+1;
775
0
         }
776
0
      }
777
      /* Entropy coding of the angle. We use a uniform pdf for the
778
         time split, a step for stereo, and a triangular one for the rest. */
779
199k
      if (stereo && N>2)
780
14.1k
      {
781
14.1k
         int p0 = 3;
782
14.1k
         int x = itheta;
783
14.1k
         int x0 = qn/2;
784
14.1k
         int ft = p0*(x0+1) + x0;
785
         /* Use a probability of p0 up to itheta=8192 and then use 1 after */
786
14.1k
         if (encode)
787
0
         {
788
0
            ec_encode(ec,x<=x0?p0*x:(x-1-x0)+(x0+1)*p0,x<=x0?p0*(x+1):(x-x0)+(x0+1)*p0,ft);
789
14.1k
         } else {
790
14.1k
            int fs;
791
14.1k
            fs=ec_decode(ec,ft);
792
14.1k
            if (fs<(x0+1)*p0)
793
10.8k
               x=fs/p0;
794
3.23k
            else
795
3.23k
               x=x0+1+(fs-(x0+1)*p0);
796
14.1k
            ec_dec_update(ec,x<=x0?p0*x:(x-1-x0)+(x0+1)*p0,x<=x0?p0*(x+1):(x-x0)+(x0+1)*p0,ft);
797
14.1k
            itheta = x;
798
14.1k
         }
799
185k
      } else if (B0>1 || stereo) {
800
         /* Uniform pdf */
801
62.8k
         if (encode)
802
0
            ec_enc_uint(ec, itheta, qn+1);
803
62.8k
         else
804
62.8k
            itheta = ec_dec_uint(ec, qn+1);
805
122k
      } else {
806
122k
         int fs=1, ft;
807
122k
         ft = ((qn>>1)+1)*((qn>>1)+1);
808
122k
         if (encode)
809
0
         {
810
0
            int fl;
811
812
0
            fs = itheta <= (qn>>1) ? itheta + 1 : qn + 1 - itheta;
813
0
            fl = itheta <= (qn>>1) ? itheta*(itheta + 1)>>1 :
814
0
             ft - ((qn + 1 - itheta)*(qn + 2 - itheta)>>1);
815
816
0
            ec_encode(ec, fl, fl+fs, ft);
817
122k
         } else {
818
            /* Triangular pdf */
819
122k
            int fl=0;
820
122k
            int fm;
821
122k
            fm = ec_decode(ec, ft);
822
823
122k
            if (fm < ((qn>>1)*((qn>>1) + 1)>>1))
824
60.3k
            {
825
60.3k
               itheta = (isqrt32(8*(opus_uint32)fm + 1) - 1)>>1;
826
60.3k
               fs = itheta + 1;
827
60.3k
               fl = itheta*(itheta + 1)>>1;
828
60.3k
            }
829
62.5k
            else
830
62.5k
            {
831
62.5k
               itheta = (2*(qn + 1)
832
62.5k
                - isqrt32(8*(opus_uint32)(ft - fm - 1) + 1))>>1;
833
62.5k
               fs = qn + 1 - itheta;
834
62.5k
               fl = ft - ((qn + 1 - itheta)*(qn + 2 - itheta)>>1);
835
62.5k
            }
836
837
122k
            ec_dec_update(ec, fl, fl+fs, ft);
838
122k
         }
839
122k
      }
840
199k
      celt_assert(itheta>=0);
841
199k
      itheta = celt_udiv((opus_int32)itheta*16384, qn);
842
#ifdef ENABLE_QEXT
843
      *ext_b = IMIN(*ext_b, ctx->ext_total_bits - (opus_int32)ec_tell_frac(ctx->ext_ec));
844
      if (*ext_b >= 2*N<<BITRES && ctx->ext_total_bits-ec_tell_frac(ctx->ext_ec)-1 > 2<<BITRES) {
845
         int extra_bits;
846
         int ext_tell = ec_tell_frac(ctx->ext_ec);
847
         extra_bits = IMIN(14, IMAX(2, celt_sudiv(*ext_b, (2*N-1)<<BITRES)));
848
         if (encode) {
849
            itheta_q30 = itheta_q30 - (itheta<<16);
850
            itheta_q30 = (itheta_q30*(opus_int64)qn*((1<<extra_bits)-1)+(1<<29))>>30;
851
            itheta_q30 += (1<<(extra_bits-1))-1;
852
            itheta_q30 = IMAX(0, IMIN((1<<extra_bits)-2, itheta_q30));
853
            ec_enc_uint(ctx->ext_ec, itheta_q30, (1<<extra_bits)-1);
854
         } else {
855
            itheta_q30 = ec_dec_uint(ctx->ext_ec, (1<<extra_bits)-1);
856
         }
857
         itheta_q30 -= (1<<(extra_bits-1))-1;
858
         itheta_q30 = (itheta<<16) + itheta_q30*(opus_int64)(1<<30)/(qn*((1<<extra_bits)-1));
859
         /* Hard bounds on itheta (can only trigger on corrupted bitstreams). */
860
         itheta_q30 = IMAX(0, IMIN(itheta_q30, 1073741824));
861
         *ext_b -= ec_tell_frac(ctx->ext_ec) - ext_tell;
862
      } else {
863
         itheta_q30 = (opus_int32)itheta<<16;
864
      }
865
#endif
866
199k
      if (encode && stereo)
867
0
      {
868
0
         if (itheta==0)
869
0
            intensity_stereo(m, X, Y, bandE, i, N);
870
0
         else
871
0
            stereo_split(X, Y, N);
872
0
      }
873
      /* NOTE: Renormalising X and Y *may* help fixed-point a bit at very high rate.
874
               Let's do that at higher complexity */
875
219k
   } else if (stereo) {
876
219k
      if (encode)
877
0
      {
878
0
         inv = itheta > 8192 && !ctx->disable_inv;
879
0
         if (inv)
880
0
         {
881
0
            int j;
882
0
            for (j=0;j<N;j++)
883
0
               Y[j] = -Y[j];
884
0
         }
885
0
         intensity_stereo(m, X, Y, bandE, i, N);
886
0
      }
887
219k
      if (*b>2<<BITRES && ctx->remaining_bits > 2<<BITRES)
888
61.7k
      {
889
61.7k
         if (encode)
890
0
            ec_enc_bit_logp(ec, inv, 2);
891
61.7k
         else
892
61.7k
            inv = ec_dec_bit_logp(ec, 2);
893
61.7k
      } else
894
157k
         inv = 0;
895
      /* inv flag override to avoid problems with downmixing. */
896
219k
      if (ctx->disable_inv)
897
0
         inv = 0;
898
219k
      itheta = 0;
899
219k
      itheta_q30 = 0;
900
219k
   }
901
419k
   qalloc = ec_tell_frac(ec) - tell;
902
419k
   *b -= qalloc;
903
904
419k
   if (itheta == 0)
905
225k
   {
906
225k
      imid = 32767;
907
225k
      iside = 0;
908
225k
      *fill &= (1<<B)-1;
909
225k
      delta = -16384;
910
225k
   } else if (itheta == 16384)
911
4.65k
   {
912
4.65k
      imid = 0;
913
4.65k
      iside = 32767;
914
4.65k
      *fill &= ((1<<B)-1)<<B;
915
4.65k
      delta = 16384;
916
188k
   } else {
917
188k
      imid = bitexact_cos((opus_int16)itheta);
918
188k
      iside = bitexact_cos((opus_int16)(16384-itheta));
919
      /* This is the mid vs side allocation that minimizes squared error
920
         in that band. */
921
188k
      delta = FRAC_MUL16((N-1)<<7,bitexact_log2tan(iside,imid));
922
188k
   }
923
924
419k
   sctx->inv = inv;
925
419k
   sctx->imid = imid;
926
419k
   sctx->iside = iside;
927
419k
   sctx->delta = delta;
928
419k
   sctx->itheta = itheta;
929
#ifdef ENABLE_QEXT
930
   sctx->itheta_q30 = itheta_q30;
931
#endif
932
419k
   sctx->qalloc = qalloc;
933
419k
}
934
static unsigned quant_band_n1(struct band_ctx *ctx, celt_norm *X, celt_norm *Y,
935
      celt_norm *lowband_out)
936
122k
{
937
122k
   int c;
938
122k
   int stereo;
939
122k
   celt_norm *x = X;
940
122k
   int encode;
941
122k
   ec_ctx *ec;
942
943
122k
   encode = ctx->encode;
944
122k
   ec = ctx->ec;
945
946
122k
   stereo = Y != NULL;
947
176k
   c=0; do {
948
176k
      int sign=0;
949
176k
      if (ctx->remaining_bits>=1<<BITRES)
950
61.1k
      {
951
61.1k
         if (encode)
952
0
         {
953
0
            sign = x[0]<0;
954
0
            ec_enc_bits(ec, sign, 1);
955
61.1k
         } else {
956
61.1k
            sign = ec_dec_bits(ec, 1);
957
61.1k
         }
958
61.1k
         ctx->remaining_bits -= 1<<BITRES;
959
61.1k
      }
960
176k
      if (ctx->resynth)
961
176k
         x[0] = sign ? -NORM_SCALING : NORM_SCALING;
962
176k
      x = Y;
963
176k
   } while (++c<1+stereo);
964
122k
   if (lowband_out)
965
122k
      lowband_out[0] = SHR32(X[0],4);
966
122k
   return 1;
967
122k
}
968
969
/* This function is responsible for encoding and decoding a mono partition.
970
   It can split the band in two and transmit the energy difference with
971
   the two half-bands. It can be called recursively so bands can end up being
972
   split in 8 parts. */
973
static unsigned quant_partition(struct band_ctx *ctx, celt_norm *X,
974
      int N, int b, int B, celt_norm *lowband,
975
      int LM,
976
      opus_val32 gain, int fill
977
      ARG_QEXT(int ext_b))
978
1.45M
{
979
1.45M
   const unsigned char *cache;
980
1.45M
   int q;
981
1.45M
   int curr_bits;
982
1.45M
   int imid=0, iside=0;
983
1.45M
   int B0=B;
984
1.45M
   opus_val32 mid=0, side=0;
985
1.45M
   unsigned cm=0;
986
1.45M
   celt_norm *Y=NULL;
987
1.45M
   int encode;
988
1.45M
   const CELTMode *m;
989
1.45M
   int i;
990
1.45M
   int spread;
991
1.45M
   ec_ctx *ec;
992
993
1.45M
   encode = ctx->encode;
994
1.45M
   m = ctx->m;
995
1.45M
   i = ctx->i;
996
1.45M
   spread = ctx->spread;
997
1.45M
   ec = ctx->ec;
998
999
   /* If we need 1.5 more bit than we can produce, split the band in two. */
1000
1.45M
   cache = m->cache.bits + m->cache.index[(LM+1)*m->nbEBands+i];
1001
1.45M
   if (LM != -1 && b > cache[cache[0]]+12 && N>2)
1002
173k
   {
1003
173k
      int mbits, sbits, delta;
1004
173k
      int itheta;
1005
173k
      int qalloc;
1006
173k
      struct split_ctx sctx;
1007
173k
      celt_norm *next_lowband2=NULL;
1008
173k
      opus_int32 rebalance;
1009
1010
173k
      N >>= 1;
1011
173k
      Y = X+N;
1012
173k
      LM -= 1;
1013
173k
      if (B==1)
1014
122k
         fill = (fill&1)|(fill<<1);
1015
173k
      B = (B+1)>>1;
1016
1017
173k
      compute_theta(ctx, &sctx, X, Y, N, &b, B, B0, LM, 0, &fill ARG_QEXT(&ext_b));
1018
173k
      imid = sctx.imid;
1019
173k
      iside = sctx.iside;
1020
173k
      delta = sctx.delta;
1021
173k
      itheta = sctx.itheta;
1022
173k
      qalloc = sctx.qalloc;
1023
#ifdef FIXED_POINT
1024
# ifdef ENABLE_QEXT
1025
      (void)imid;
1026
      (void)iside;
1027
      mid = celt_cos_norm32(sctx.itheta_q30);
1028
      side = celt_cos_norm32((1<<30)-sctx.itheta_q30);
1029
# else
1030
      mid = SHL32(EXTEND32(imid), 16);
1031
      side = SHL32(EXTEND32(iside), 16);
1032
# endif
1033
#else
1034
# ifdef ENABLE_QEXT
1035
      (void)imid;
1036
      (void)iside;
1037
      mid = celt_cos_norm2(sctx.itheta_q30*(1.f/(1<<30)));
1038
      side = celt_cos_norm2(1.f-sctx.itheta_q30*(1.f/(1<<30)));
1039
# else
1040
173k
      mid = (1.f/32768)*imid;
1041
173k
      side = (1.f/32768)*iside;
1042
173k
# endif
1043
173k
#endif
1044
1045
      /* Give more bits to low-energy MDCTs than they would otherwise deserve */
1046
173k
      if (B0>1 && (itheta&0x3fff))
1047
46.4k
      {
1048
46.4k
         if (itheta > 8192)
1049
            /* Rough approximation for pre-echo masking */
1050
20.6k
            delta -= delta>>(4-LM);
1051
25.8k
         else
1052
            /* Corresponds to a forward-masking slope of 1.5 dB per 10 ms */
1053
25.8k
            delta = IMIN(0, delta + (N<<BITRES>>(5-LM)));
1054
46.4k
      }
1055
173k
      mbits = IMAX(0, IMIN(b, (b-delta)/2));
1056
173k
      sbits = b-mbits;
1057
173k
      ctx->remaining_bits -= qalloc;
1058
1059
173k
      if (lowband)
1060
126k
         next_lowband2 = lowband+N; /* >32-bit split case */
1061
1062
173k
      rebalance = ctx->remaining_bits;
1063
173k
      if (mbits >= sbits)
1064
88.1k
      {
1065
88.1k
         cm = quant_partition(ctx, X, N, mbits, B, lowband, LM,
1066
88.1k
               MULT32_32_Q31(gain,mid), fill ARG_QEXT(ext_b/2));
1067
88.1k
         rebalance = mbits - (rebalance-ctx->remaining_bits);
1068
88.1k
         if (rebalance > 3<<BITRES && itheta!=0)
1069
48.8k
            sbits += rebalance - (3<<BITRES);
1070
88.1k
         cm |= quant_partition(ctx, Y, N, sbits, B, next_lowband2, LM,
1071
88.1k
               MULT32_32_Q31(gain,side), fill>>B ARG_QEXT(ext_b/2))<<(B0>>1);
1072
88.1k
      } else {
1073
85.0k
         cm = quant_partition(ctx, Y, N, sbits, B, next_lowband2, LM,
1074
85.0k
               MULT32_32_Q31(gain,side), fill>>B ARG_QEXT(ext_b/2))<<(B0>>1);
1075
85.0k
         rebalance = sbits - (rebalance-ctx->remaining_bits);
1076
85.0k
         if (rebalance > 3<<BITRES && itheta!=16384)
1077
49.7k
            mbits += rebalance - (3<<BITRES);
1078
85.0k
         cm |= quant_partition(ctx, X, N, mbits, B, lowband, LM,
1079
85.0k
               MULT32_32_Q31(gain,mid), fill ARG_QEXT(ext_b/2));
1080
85.0k
      }
1081
1.28M
   } else {
1082
#ifdef ENABLE_QEXT
1083
      int extra_bits;
1084
      int ext_remaining_bits;
1085
      extra_bits = ext_b/(N-1)>>BITRES;
1086
      ext_remaining_bits = ctx->ext_total_bits-(opus_int32)ec_tell_frac(ctx->ext_ec);
1087
      if (ext_remaining_bits < ((extra_bits+1)*(N-1)+N)<<BITRES) {
1088
         extra_bits = (ext_remaining_bits-(N<<BITRES))/(N-1)>>BITRES;
1089
         extra_bits = IMAX(extra_bits-1, 0);
1090
      }
1091
      extra_bits = IMIN(14, extra_bits);
1092
#endif
1093
      /* This is the basic no-split case */
1094
1.28M
      q = bits2pulses(m, i, LM, b);
1095
1.28M
      curr_bits = pulses2bits(m, i, LM, q);
1096
1.28M
      ctx->remaining_bits -= curr_bits;
1097
1098
      /* Ensures we can never bust the budget */
1099
1.29M
      while (ctx->remaining_bits < 0 && q > 0)
1100
15.1k
      {
1101
15.1k
         ctx->remaining_bits += curr_bits;
1102
15.1k
         q--;
1103
15.1k
         curr_bits = pulses2bits(m, i, LM, q);
1104
15.1k
         ctx->remaining_bits -= curr_bits;
1105
15.1k
      }
1106
1107
1.28M
      if (q!=0)
1108
531k
      {
1109
531k
         int K = get_pulses(q);
1110
1111
         /* Finally do the actual quantization */
1112
531k
         if (encode)
1113
0
         {
1114
0
            cm = alg_quant(X, N, K, spread, B, ec, gain, ctx->resynth
1115
0
                           ARG_QEXT(ctx->ext_ec) ARG_QEXT(extra_bits),
1116
0
                           ctx->arch);
1117
531k
         } else {
1118
531k
            cm = alg_unquant(X, N, K, spread, B, ec, gain
1119
531k
                             ARG_QEXT(ctx->ext_ec) ARG_QEXT(extra_bits));
1120
531k
         }
1121
#ifdef ENABLE_QEXT
1122
      } else if (ext_b > 2*N<<BITRES)
1123
      {
1124
         extra_bits = ext_b/(N-1)>>BITRES;
1125
         ext_remaining_bits = ctx->ext_total_bits-ec_tell_frac(ctx->ext_ec);
1126
         if (ext_remaining_bits < ((extra_bits+1)*(N-1)+N)<<BITRES) {
1127
            extra_bits = (ext_remaining_bits-(N<<BITRES))/(N-1)>>BITRES;
1128
            extra_bits = IMAX(extra_bits-1, 0);
1129
         }
1130
         extra_bits = IMIN(14, extra_bits);
1131
         if (encode) cm = cubic_quant(X, N, extra_bits, B, ctx->ext_ec, gain, ctx->resynth);
1132
         else cm = cubic_unquant(X, N, extra_bits, B, ctx->ext_ec, gain);
1133
#endif
1134
751k
      } else {
1135
         /* If there's no pulse, fill the band anyway */
1136
751k
         int j;
1137
751k
         if (ctx->resynth)
1138
751k
         {
1139
751k
            unsigned cm_mask;
1140
            /* B can be as large as 16, so this shift might overflow an int on a
1141
               16-bit platform; use a long to get defined behavior.*/
1142
751k
            cm_mask = (unsigned)(1UL<<B)-1;
1143
751k
            fill &= cm_mask;
1144
751k
            if (!fill)
1145
177k
            {
1146
177k
               OPUS_CLEAR(X, N);
1147
574k
            } else {
1148
574k
               if (lowband == NULL)
1149
37.4k
               {
1150
                  /* Noise */
1151
1.06M
                  for (j=0;j<N;j++)
1152
1.02M
                  {
1153
1.02M
                     ctx->seed = celt_lcg_rand(ctx->seed);
1154
1.02M
                     X[j] = SHL32((celt_norm)((opus_int32)ctx->seed>>20), NORM_SHIFT-14);
1155
1.02M
                  }
1156
37.4k
                  cm = cm_mask;
1157
536k
               } else {
1158
                  /* Folded spectrum */
1159
10.2M
                  for (j=0;j<N;j++)
1160
9.76M
                  {
1161
9.76M
                     opus_val16 tmp;
1162
9.76M
                     ctx->seed = celt_lcg_rand(ctx->seed);
1163
                     /* About 48 dB below the "normal" folding level */
1164
9.76M
                     tmp = QCONST16(1.0f/256, NORM_SHIFT-4);
1165
9.76M
                     tmp = (ctx->seed)&0x8000 ? tmp : -tmp;
1166
9.76M
                     X[j] = lowband[j]+tmp;
1167
9.76M
                  }
1168
536k
                  cm = fill;
1169
536k
               }
1170
574k
               renormalise_vector(X, N, gain, ctx->arch);
1171
574k
            }
1172
751k
         }
1173
751k
      }
1174
1.28M
   }
1175
1176
1.45M
   return cm;
1177
1.45M
}
1178
1179
#ifdef ENABLE_QEXT
1180
static unsigned cubic_quant_partition(struct band_ctx *ctx, celt_norm *X, int N, int b, int B, ec_ctx *ec, int LM, opus_val32 gain, int resynth, int encode)
1181
{
1182
   celt_assert(LM>=0);
1183
   ctx->remaining_bits = ctx->ec->storage*8*8 - ec_tell_frac(ctx->ec);
1184
   b = IMIN(b, ctx->remaining_bits);
1185
   /* As long as we have at least two bits of depth, split all the way to LM=0 (not -1 like PVQ). */
1186
   if (LM==0 || b<=2*N<<BITRES) {
1187
      int res, ret;
1188
      b = IMIN(b + ((N-1)<<BITRES)/2, ctx->remaining_bits);
1189
      /* Resolution left after taking into account coding the cube face. */
1190
      res = (b-(1<<BITRES)-ctx->m->logN[ctx->i]-(LM<<BITRES)-1)/(N-1)>>BITRES;
1191
      res = IMIN(14, IMAX(0, res));
1192
      if (encode) ret = cubic_quant(X, N, res, B, ec, gain, resynth);
1193
      else ret = cubic_unquant(X, N, res, B, ec, gain);
1194
      ctx->remaining_bits = ctx->ec->storage*8*8 - ec_tell_frac(ctx->ec);
1195
      return ret;
1196
   } else {
1197
      celt_norm *Y;
1198
      opus_int32 itheta_q30;
1199
      opus_val32 g1, g2;
1200
      opus_int32 theta_res;
1201
      opus_int32 qtheta;
1202
      int delta;
1203
      int b1, b2;
1204
      int cm;
1205
      int N0;
1206
      N0 = N;
1207
      N >>= 1;
1208
      Y = X+N;
1209
      LM -= 1;
1210
      B = (B+1)>>1;
1211
      theta_res = IMIN(16, (b>>BITRES)/(N0-1) + 1);
1212
      if (encode) {
1213
         itheta_q30 = stereo_itheta(X, Y, 0, N, ctx->arch);
1214
         qtheta = (itheta_q30+(1<<(29-theta_res)))>>(30-theta_res);
1215
         ec_enc_uint(ec, qtheta, (1<<theta_res)+1);
1216
      } else {
1217
         qtheta = ec_dec_uint(ec, (1<<theta_res)+1);
1218
      }
1219
      itheta_q30 = qtheta<<(30-theta_res);
1220
      b -= theta_res<<BITRES;
1221
      delta = (N0-1) * 23 * ((itheta_q30>>16)-8192) >> (17-BITRES);
1222
1223
#ifdef FIXED_POINT
1224
      g1 = celt_cos_norm32(itheta_q30);
1225
      g2 = celt_cos_norm32((1<<30)-itheta_q30);
1226
#else
1227
      g1 = celt_cos_norm2(itheta_q30*(1.f/(1<<30)));
1228
      g2 = celt_cos_norm2(1.f-itheta_q30*(1.f/(1<<30)));
1229
#endif
1230
      if (itheta_q30 == 0) {
1231
         b1=b;
1232
         b2=0;
1233
      } else if (itheta_q30==1073741824) {
1234
         b1=0;
1235
         b2=b;
1236
      } else {
1237
         b1 = IMIN(b, IMAX(0, (b-delta)/2));
1238
         b2 = b-b1;
1239
      }
1240
      cm  = cubic_quant_partition(ctx, X, N, b1, B, ec, LM, MULT32_32_Q31(gain, g1), resynth, encode);
1241
      cm |= cubic_quant_partition(ctx, Y, N, b2, B, ec, LM, MULT32_32_Q31(gain, g2), resynth, encode);
1242
      return cm;
1243
   }
1244
}
1245
#endif
1246
1247
/* This function is responsible for encoding and decoding a band for the mono case. */
1248
static unsigned quant_band(struct band_ctx *ctx, celt_norm *X,
1249
      int N, int b, int B, celt_norm *lowband,
1250
      int LM, celt_norm *lowband_out,
1251
      opus_val32 gain, celt_norm *lowband_scratch, int fill
1252
      ARG_QEXT(int ext_b))
1253
1.17M
{
1254
1.17M
   int N0=N;
1255
1.17M
   int N_B=N;
1256
1.17M
   int N_B0;
1257
1.17M
   int B0=B;
1258
1.17M
   int time_divide=0;
1259
1.17M
   int recombine=0;
1260
1.17M
   int longBlocks;
1261
1.17M
   unsigned cm=0;
1262
1.17M
   int k;
1263
1.17M
   int encode;
1264
1.17M
   int tf_change;
1265
1266
1.17M
   encode = ctx->encode;
1267
1.17M
   tf_change = ctx->tf_change;
1268
1269
1.17M
   longBlocks = B0==1;
1270
1271
1.17M
   N_B = celt_udiv(N_B, B);
1272
1273
   /* Special case for one sample */
1274
1.17M
   if (N==1)
1275
67.7k
   {
1276
67.7k
      return quant_band_n1(ctx, X, NULL, lowband_out);
1277
67.7k
   }
1278
1279
1.10M
   if (tf_change>0)
1280
67.2k
      recombine = tf_change;
1281
   /* Band recombining to increase frequency resolution */
1282
1283
1.10M
   if (lowband_scratch && lowband && (recombine || ((N_B&1) == 0 && tf_change<0) || B0>1))
1284
173k
   {
1285
173k
      OPUS_COPY(lowband_scratch, lowband, N);
1286
173k
      lowband = lowband_scratch;
1287
173k
   }
1288
1289
1.21M
   for (k=0;k<recombine;k++)
1290
110k
   {
1291
110k
      static const unsigned char bit_interleave_table[16]={
1292
110k
            0,1,1,1,2,3,3,3,2,3,3,3,2,3,3,3
1293
110k
      };
1294
110k
      if (encode)
1295
0
         haar1(X, N>>k, 1<<k);
1296
110k
      if (lowband)
1297
89.0k
         haar1(lowband, N>>k, 1<<k);
1298
110k
      fill = bit_interleave_table[fill&0xF]|bit_interleave_table[fill>>4]<<2;
1299
110k
   }
1300
1.10M
   B>>=recombine;
1301
1.10M
   N_B<<=recombine;
1302
1303
   /* Increasing the time resolution */
1304
1.38M
   while ((N_B&1) == 0 && tf_change<0)
1305
271k
   {
1306
271k
      if (encode)
1307
0
         haar1(X, N_B, B);
1308
271k
      if (lowband)
1309
212k
         haar1(lowband, N_B, B);
1310
271k
      fill |= fill<<B;
1311
271k
      B <<= 1;
1312
271k
      N_B >>= 1;
1313
271k
      time_divide++;
1314
271k
      tf_change++;
1315
271k
   }
1316
1.10M
   B0=B;
1317
1.10M
   N_B0 = N_B;
1318
1319
   /* Reorganize the samples in time order instead of frequency order */
1320
1.10M
   if (B0>1)
1321
184k
   {
1322
184k
      if (encode)
1323
0
         deinterleave_hadamard(X, N_B>>recombine, B0<<recombine, longBlocks);
1324
184k
      if (lowband)
1325
148k
         deinterleave_hadamard(lowband, N_B>>recombine, B0<<recombine, longBlocks);
1326
184k
   }
1327
1328
#ifdef ENABLE_QEXT
1329
   if (ctx->extra_bands && b > (3*N<<BITRES)+(ctx->m->logN[ctx->i]+8+8*LM)) {
1330
      cm = cubic_quant_partition(ctx, X, N, b, B, ctx->ec, LM, gain, ctx->resynth, encode);
1331
   } else
1332
#endif
1333
1.10M
   {
1334
1.10M
      cm = quant_partition(ctx, X, N, b, B, lowband, LM, gain, fill ARG_QEXT(ext_b));
1335
1.10M
   }
1336
1337
   /* This code is used by the decoder and by the resynthesis-enabled encoder */
1338
1.10M
   if (ctx->resynth)
1339
1.10M
   {
1340
      /* Undo the sample reorganization going from time order to frequency order */
1341
1.10M
      if (B0>1)
1342
184k
         interleave_hadamard(X, N_B>>recombine, B0<<recombine, longBlocks);
1343
1344
      /* Undo time-freq changes that we did earlier */
1345
1.10M
      N_B = N_B0;
1346
1.10M
      B = B0;
1347
1.38M
      for (k=0;k<time_divide;k++)
1348
271k
      {
1349
271k
         B >>= 1;
1350
271k
         N_B <<= 1;
1351
271k
         cm |= cm>>B;
1352
271k
         haar1(X, N_B, B);
1353
271k
      }
1354
1355
1.21M
      for (k=0;k<recombine;k++)
1356
110k
      {
1357
110k
         static const unsigned char bit_deinterleave_table[16]={
1358
110k
               0x00,0x03,0x0C,0x0F,0x30,0x33,0x3C,0x3F,
1359
110k
               0xC0,0xC3,0xCC,0xCF,0xF0,0xF3,0xFC,0xFF
1360
110k
         };
1361
110k
         cm = bit_deinterleave_table[cm];
1362
110k
         haar1(X, N0>>k, 1<<k);
1363
110k
      }
1364
1.10M
      B<<=recombine;
1365
1366
      /* Scale output for later folding */
1367
1.10M
      if (lowband_out)
1368
842k
      {
1369
842k
         int j;
1370
842k
         opus_val16 n;
1371
842k
         n = celt_sqrt(SHL32(EXTEND32(N0),22));
1372
10.3M
         for (j=0;j<N0;j++)
1373
9.48M
            lowband_out[j] = MULT16_32_Q15(n,X[j]);
1374
842k
      }
1375
1.10M
      cm &= (1<<B)-1;
1376
1.10M
   }
1377
1.10M
   return cm;
1378
1.17M
}
1379
1380
#ifdef FIXED_POINT
1381
#define MIN_STEREO_ENERGY 2
1382
#else
1383
0
#define MIN_STEREO_ENERGY 1e-10f
1384
#endif
1385
1386
/* This function is responsible for encoding and decoding a band for the stereo case. */
1387
static unsigned quant_band_stereo(struct band_ctx *ctx, celt_norm *X, celt_norm *Y,
1388
      int N, int b, int B, celt_norm *lowband,
1389
      int LM, celt_norm *lowband_out,
1390
      celt_norm *lowband_scratch, int fill
1391
      ARG_QEXT(int ext_b) ARG_QEXT(const int *cap))
1392
300k
{
1393
300k
   int imid=0, iside=0;
1394
300k
   int inv = 0;
1395
300k
   opus_val32 mid=0, side=0;
1396
300k
   unsigned cm=0;
1397
300k
   int mbits, sbits, delta;
1398
300k
   int itheta;
1399
300k
   int qalloc;
1400
300k
   struct split_ctx sctx;
1401
300k
   int orig_fill;
1402
300k
   int encode;
1403
300k
   ec_ctx *ec;
1404
1405
300k
   encode = ctx->encode;
1406
300k
   ec = ctx->ec;
1407
1408
   /* Special case for one sample */
1409
300k
   if (N==1)
1410
54.4k
   {
1411
54.4k
      return quant_band_n1(ctx, X, Y, lowband_out);
1412
54.4k
   }
1413
1414
246k
   orig_fill = fill;
1415
1416
246k
   if (encode) {
1417
0
      if (ctx->bandE[ctx->i] < MIN_STEREO_ENERGY || ctx->bandE[ctx->m->nbEBands+ctx->i] < MIN_STEREO_ENERGY) {
1418
0
         if (ctx->bandE[ctx->i] > ctx->bandE[ctx->m->nbEBands+ctx->i]) OPUS_COPY(Y, X, N);
1419
0
         else OPUS_COPY(X, Y, N);
1420
0
      }
1421
0
   }
1422
246k
   compute_theta(ctx, &sctx, X, Y, N, &b, B, B, LM, 1, &fill ARG_QEXT(&ext_b));
1423
246k
   inv = sctx.inv;
1424
246k
   imid = sctx.imid;
1425
246k
   iside = sctx.iside;
1426
246k
   delta = sctx.delta;
1427
246k
   itheta = sctx.itheta;
1428
246k
   qalloc = sctx.qalloc;
1429
#ifdef FIXED_POINT
1430
# ifdef ENABLE_QEXT
1431
   (void)imid;
1432
   (void)iside;
1433
   mid = celt_cos_norm32(sctx.itheta_q30);
1434
   side = celt_cos_norm32((1<<30)-sctx.itheta_q30);
1435
# else
1436
   mid = SHL32(EXTEND32(imid), 16);
1437
   side = SHL32(EXTEND32(iside), 16);
1438
# endif
1439
#else
1440
# ifdef ENABLE_QEXT
1441
   (void)imid;
1442
   (void)iside;
1443
   mid = celt_cos_norm2(sctx.itheta_q30*(1.f/(1<<30)));
1444
   side = celt_cos_norm2(1.f-sctx.itheta_q30*(1.f/(1<<30)));
1445
# else
1446
246k
   mid = (1.f/32768)*imid;
1447
246k
   side = (1.f/32768)*iside;
1448
246k
# endif
1449
246k
#endif
1450
1451
   /* This is a special case for N=2 that only works for stereo and takes
1452
      advantage of the fact that mid and side are orthogonal to encode
1453
      the side with just one bit. */
1454
246k
   if (N==2)
1455
64.6k
   {
1456
64.6k
      int c;
1457
64.6k
      int sign=0;
1458
64.6k
      celt_norm *x2, *y2;
1459
64.6k
      mbits = b;
1460
64.6k
      sbits = 0;
1461
      /* Only need one bit for the side. */
1462
64.6k
      if (itheta != 0 && itheta != 16384)
1463
10.8k
         sbits = 1<<BITRES;
1464
64.6k
      mbits -= sbits;
1465
64.6k
      c = itheta > 8192;
1466
64.6k
      ctx->remaining_bits -= qalloc+sbits;
1467
1468
64.6k
      x2 = c ? Y : X;
1469
64.6k
      y2 = c ? X : Y;
1470
64.6k
      if (sbits)
1471
10.8k
      {
1472
10.8k
         if (encode)
1473
0
         {
1474
            /* Here we only need to encode a sign for the side. */
1475
            /* FIXME: Need to increase fixed-point precision? */
1476
0
            sign = MULT32_32_Q31(x2[0],y2[1]) - MULT32_32_Q31(x2[1],y2[0]) < 0;
1477
0
            ec_enc_bits(ec, sign, 1);
1478
10.8k
         } else {
1479
10.8k
            sign = ec_dec_bits(ec, 1);
1480
10.8k
         }
1481
10.8k
      }
1482
64.6k
      sign = 1-2*sign;
1483
      /* We use orig_fill here because we want to fold the side, but if
1484
         itheta==16384, we'll have cleared the low bits of fill. */
1485
64.6k
      cm = quant_band(ctx, x2, N, mbits, B, lowband, LM, lowband_out, Q31ONE,
1486
64.6k
            lowband_scratch, orig_fill ARG_QEXT(ext_b));
1487
      /* We don't split N=2 bands, so cm is either 1 or 0 (for a fold-collapse),
1488
         and there's no need to worry about mixing with the other channel. */
1489
64.6k
      y2[0] = -sign*x2[1];
1490
64.6k
      y2[1] = sign*x2[0];
1491
64.6k
      if (ctx->resynth)
1492
64.6k
      {
1493
64.6k
         celt_norm tmp;
1494
64.6k
         X[0] = MULT32_32_Q31(mid, X[0]);
1495
64.6k
         X[1] = MULT32_32_Q31(mid, X[1]);
1496
64.6k
         Y[0] = MULT32_32_Q31(side, Y[0]);
1497
64.6k
         Y[1] = MULT32_32_Q31(side, Y[1]);
1498
64.6k
         tmp = X[0];
1499
64.6k
         X[0] = SUB32(tmp,Y[0]);
1500
64.6k
         Y[0] = ADD32(tmp,Y[0]);
1501
64.6k
         tmp = X[1];
1502
64.6k
         X[1] = SUB32(tmp,Y[1]);
1503
64.6k
         Y[1] = ADD32(tmp,Y[1]);
1504
64.6k
      }
1505
181k
   } else {
1506
      /* "Normal" split code */
1507
181k
      opus_int32 rebalance;
1508
1509
181k
      mbits = IMAX(0, IMIN(b, (b-delta)/2));
1510
181k
      sbits = b-mbits;
1511
181k
      ctx->remaining_bits -= qalloc;
1512
1513
181k
      rebalance = ctx->remaining_bits;
1514
181k
      if (mbits >= sbits)
1515
176k
      {
1516
#ifdef ENABLE_QEXT
1517
         int qext_extra = 0;
1518
         /* Reallocate any mid bits that cannot be used to extra mid bits. */
1519
         if (cap != NULL && ext_b != 0) qext_extra = IMAX(0, IMIN(ext_b/2, mbits - cap[ctx->i]/2));
1520
#endif
1521
         /* In stereo mode, we do not apply a scaling to the mid because we need the normalized
1522
            mid for folding later. */
1523
176k
         cm = quant_band(ctx, X, N, mbits, B, lowband, LM, lowband_out, Q31ONE,
1524
176k
               lowband_scratch, fill ARG_QEXT(ext_b/2+qext_extra));
1525
176k
         rebalance = mbits - (rebalance-ctx->remaining_bits);
1526
176k
         if (rebalance > 3<<BITRES && itheta!=0)
1527
998
            sbits += rebalance - (3<<BITRES);
1528
#ifdef ENABLE_QEXT
1529
         /* Guard against overflowing the EC with the angle if the cubic quant used too many bits for the mid. */
1530
         if (ctx->extra_bands) sbits = IMIN(sbits, ctx->remaining_bits);
1531
#endif
1532
         /* For a stereo split, the high bits of fill are always zero, so no
1533
            folding will be done to the side. */
1534
176k
         cm |= quant_band(ctx, Y, N, sbits, B, NULL, LM, NULL, side, NULL, fill>>B ARG_QEXT(ext_b/2-qext_extra));
1535
176k
      } else {
1536
#ifdef ENABLE_QEXT
1537
         int qext_extra = 0;
1538
         /* Reallocate any side bits that cannot be used to extra side bits. */
1539
         if (cap != NULL && ext_b != 0) qext_extra = IMAX(0, IMIN(ext_b/2, sbits - cap[ctx->i]/2));
1540
#endif
1541
         /* For a stereo split, the high bits of fill are always zero, so no
1542
            folding will be done to the side. */
1543
5.22k
         cm = quant_band(ctx, Y, N, sbits, B, NULL, LM, NULL, side, NULL, fill>>B ARG_QEXT(ext_b/2+qext_extra));
1544
5.22k
         rebalance = sbits - (rebalance-ctx->remaining_bits);
1545
5.22k
         if (rebalance > 3<<BITRES && itheta!=16384)
1546
453
            mbits += rebalance - (3<<BITRES);
1547
#ifdef ENABLE_QEXT
1548
         /* Guard against overflowing the EC with the angle if the cubic quant used too many bits for the side. */
1549
         if (ctx->extra_bands) mbits = IMIN(mbits, ctx->remaining_bits);
1550
#endif
1551
         /* In stereo mode, we do not apply a scaling to the mid because we need the normalized
1552
            mid for folding later. */
1553
5.22k
         cm |= quant_band(ctx, X, N, mbits, B, lowband, LM, lowband_out, Q31ONE,
1554
5.22k
               lowband_scratch, fill ARG_QEXT(ext_b/2-qext_extra));
1555
5.22k
      }
1556
181k
   }
1557
1558
1559
   /* This code is used by the decoder and by the resynthesis-enabled encoder */
1560
246k
   if (ctx->resynth)
1561
246k
   {
1562
246k
      if (N!=2)
1563
181k
         stereo_merge(X, Y, mid, N, ctx->arch);
1564
246k
      if (inv)
1565
13.8k
      {
1566
13.8k
         int j;
1567
149k
         for (j=0;j<N;j++)
1568
136k
            Y[j] = -Y[j];
1569
13.8k
      }
1570
246k
   }
1571
246k
   return cm;
1572
300k
}
1573
1574
#ifndef DISABLE_UPDATE_DRAFT
1575
static void special_hybrid_folding(const CELTMode *m, celt_norm *norm, celt_norm *norm2, int start, int M, int dual_stereo)
1576
85.3k
{
1577
85.3k
   int n1, n2;
1578
85.3k
   const opus_int16 * OPUS_RESTRICT eBands = m->eBands;
1579
85.3k
   n1 = M*(eBands[start+1]-eBands[start]);
1580
85.3k
   n2 = M*(eBands[start+2]-eBands[start+1]);
1581
   /* Duplicate enough of the first band folding data to be able to fold the second band.
1582
      Copies no data for CELT-only mode. */
1583
85.3k
   OPUS_COPY(&norm[n1], &norm[2*n1 - n2], n2-n1);
1584
85.3k
   if (dual_stereo)
1585
4.03k
      OPUS_COPY(&norm2[n1], &norm2[2*n1 - n2], n2-n1);
1586
85.3k
}
1587
#endif
1588
1589
void quant_all_bands(int encode, const CELTMode *m, int start, int end,
1590
      celt_norm *X_, celt_norm *Y_, unsigned char *collapse_masks,
1591
      const celt_ener *bandE, int *pulses, int shortBlocks, int spread,
1592
      int dual_stereo, int intensity, int *tf_res, opus_int32 total_bits,
1593
      opus_int32 balance, ec_ctx *ec, int LM, int codedBands,
1594
      opus_uint32 *seed, int complexity, int arch, int disable_inv
1595
      ARG_QEXT(ec_ctx *ext_ec) ARG_QEXT(int *extra_pulses)
1596
      ARG_QEXT(opus_int32 ext_total_bits) ARG_QEXT(const int *cap))
1597
85.3k
{
1598
85.3k
   int i;
1599
85.3k
   opus_int32 remaining_bits;
1600
85.3k
   const opus_int16 * OPUS_RESTRICT eBands = m->eBands;
1601
85.3k
   celt_norm * OPUS_RESTRICT norm, * OPUS_RESTRICT norm2;
1602
85.3k
   VARDECL(celt_norm, _norm);
1603
85.3k
   VARDECL(celt_norm, _lowband_scratch);
1604
85.3k
   VARDECL(celt_norm, X_save);
1605
85.3k
   VARDECL(celt_norm, Y_save);
1606
85.3k
   VARDECL(celt_norm, X_save2);
1607
85.3k
   VARDECL(celt_norm, Y_save2);
1608
85.3k
   VARDECL(celt_norm, norm_save2);
1609
85.3k
   VARDECL(unsigned char, bytes_save);
1610
85.3k
   int resynth_alloc;
1611
85.3k
   celt_norm *lowband_scratch;
1612
85.3k
   int B;
1613
85.3k
   int M;
1614
85.3k
   int lowband_offset;
1615
85.3k
   int update_lowband = 1;
1616
85.3k
   int C = Y_ != NULL ? 2 : 1;
1617
85.3k
   int norm_offset;
1618
85.3k
   int theta_rdo = encode && Y_!=NULL && !dual_stereo && complexity>=8;
1619
#ifdef RESYNTH
1620
   int resynth = 1;
1621
#else
1622
85.3k
   int resynth = !encode || theta_rdo;
1623
85.3k
#endif
1624
85.3k
   struct band_ctx ctx;
1625
#ifdef ENABLE_QEXT
1626
   int ext_b;
1627
   opus_int32 ext_balance=0;
1628
   opus_int32 ext_tell=0;
1629
   VARDECL(unsigned char, ext_bytes_save);
1630
#endif
1631
85.3k
   SAVE_STACK;
1632
1633
85.3k
   M = 1<<LM;
1634
85.3k
   B = shortBlocks ? M : 1;
1635
85.3k
   norm_offset = M*eBands[start];
1636
   /* No need to allocate norm for the last band because we don't need an
1637
      output in that band. */
1638
85.3k
   ALLOC(_norm, C*(M*eBands[m->nbEBands-1]-norm_offset), celt_norm);
1639
85.3k
   norm = _norm;
1640
85.3k
   norm2 = norm + M*eBands[m->nbEBands-1]-norm_offset;
1641
1642
   /* For decoding, we can use the last band as scratch space because we don't need that
1643
      scratch space for the last band and we don't care about the data there until we're
1644
      decoding the last band. */
1645
85.3k
   if (encode && resynth)
1646
0
      resynth_alloc = M*(eBands[m->nbEBands]-eBands[m->nbEBands-1]);
1647
85.3k
   else
1648
85.3k
      resynth_alloc = ALLOC_NONE;
1649
85.3k
   ALLOC(_lowband_scratch, resynth_alloc, celt_norm);
1650
85.3k
   if (encode && resynth)
1651
0
      lowband_scratch = _lowband_scratch;
1652
85.3k
   else
1653
85.3k
      lowband_scratch = X_+M*eBands[m->effEBands-1];
1654
85.3k
   ALLOC(X_save, resynth_alloc, celt_norm);
1655
85.3k
   ALLOC(Y_save, resynth_alloc, celt_norm);
1656
85.3k
   ALLOC(X_save2, resynth_alloc, celt_norm);
1657
85.3k
   ALLOC(Y_save2, resynth_alloc, celt_norm);
1658
85.3k
   ALLOC(norm_save2, resynth_alloc, celt_norm);
1659
1660
85.3k
   lowband_offset = 0;
1661
85.3k
   ctx.bandE = bandE;
1662
85.3k
   ctx.ec = ec;
1663
85.3k
   ctx.encode = encode;
1664
85.3k
   ctx.intensity = intensity;
1665
85.3k
   ctx.m = m;
1666
85.3k
   ctx.seed = *seed;
1667
85.3k
   ctx.spread = spread;
1668
85.3k
   ctx.arch = arch;
1669
85.3k
   ctx.disable_inv = disable_inv;
1670
85.3k
   ctx.resynth = resynth;
1671
85.3k
   ctx.theta_round = 0;
1672
#ifdef ENABLE_QEXT
1673
   ctx.ext_ec = ext_ec;
1674
   ctx.ext_total_bits = ext_total_bits;
1675
   ctx.extra_bands = (cap == NULL);
1676
   if (ctx.extra_bands || ext_total_bits!=0) theta_rdo = 0;
1677
   ALLOC(ext_bytes_save, theta_rdo ? QEXT_PACKET_SIZE_CAP : ALLOC_NONE, unsigned char);
1678
#endif
1679
85.3k
   ALLOC(bytes_save, theta_rdo ? 1275 : ALLOC_NONE, unsigned char);
1680
1681
   /* Avoid injecting noise in the first band on transients. */
1682
85.3k
   ctx.avoid_split_noise = B > 1;
1683
1.10M
   for (i=start;i<end;i++)
1684
1.01M
   {
1685
1.01M
      opus_int32 tell;
1686
1.01M
      int b;
1687
1.01M
      int N;
1688
1.01M
      opus_int32 curr_balance;
1689
1.01M
      int effective_lowband=-1;
1690
1.01M
      celt_norm * OPUS_RESTRICT X, * OPUS_RESTRICT Y;
1691
1.01M
      int tf_change=0;
1692
1.01M
      unsigned x_cm;
1693
1.01M
      unsigned y_cm;
1694
1.01M
      int last;
1695
1696
1.01M
      ctx.i = i;
1697
1.01M
      last = (i==end-1);
1698
1699
1.01M
      X = X_+M*eBands[i];
1700
1.01M
      if (Y_!=NULL)
1701
333k
         Y = Y_+M*eBands[i];
1702
684k
      else
1703
684k
         Y = NULL;
1704
1.01M
      N = M*eBands[i+1]-M*eBands[i];
1705
1.01M
      celt_assert(N > 0);
1706
1.01M
      tell = ec_tell_frac(ec);
1707
1708
      /* Compute how many bits we want to allocate to this band */
1709
1.01M
      if (i != start)
1710
931k
         balance -= tell;
1711
1.01M
      remaining_bits = total_bits-tell-1;
1712
1.01M
      ctx.remaining_bits = remaining_bits;
1713
#ifdef ENABLE_QEXT
1714
      if (i != start) {
1715
         ext_balance += extra_pulses[i-1] + ext_tell;
1716
      }
1717
      ext_tell = ec_tell_frac(ext_ec);
1718
      ctx.extra_bits = extra_pulses[i];
1719
      if (i != start)
1720
         ext_balance -= ext_tell;
1721
      if (i <= codedBands-1)
1722
      {
1723
         opus_int32 ext_curr_balance = celt_sudiv(ext_balance, IMIN(3, codedBands-i));
1724
         ext_b = IMAX(0, IMIN(16383, IMIN(ext_total_bits-ext_tell,extra_pulses[i]+ext_curr_balance)));
1725
      } else {
1726
         ext_b = 0;
1727
      }
1728
#endif
1729
1.01M
      if (i <= codedBands-1)
1730
431k
      {
1731
431k
         curr_balance = celt_sudiv(balance, IMIN(3, codedBands-i));
1732
431k
         b = IMAX(0, IMIN(16383, IMIN(remaining_bits+1,pulses[i]+curr_balance)));
1733
585k
      } else {
1734
585k
         b = 0;
1735
585k
      }
1736
1737
1.01M
#ifndef DISABLE_UPDATE_DRAFT
1738
1.01M
      if (resynth && (M*eBands[i]-N >= M*eBands[start] || i==start+1) && (update_lowband || lowband_offset==0))
1739
337k
            lowband_offset = i;
1740
1.01M
      if (i == start+1)
1741
85.3k
         special_hybrid_folding(m, norm, norm2, start, M, dual_stereo);
1742
#else
1743
      if (resynth && M*eBands[i]-N >= M*eBands[start] && (update_lowband || lowband_offset==0))
1744
            lowband_offset = i;
1745
#endif
1746
1747
1.01M
      tf_change = tf_res[i];
1748
1.01M
      ctx.tf_change = tf_change;
1749
1.01M
      if (i>=m->effEBands)
1750
0
      {
1751
0
         celt_assert(eBands[i+1]-eBands[i] <= eBands[m->nbEBands-1]-eBands[start]);
1752
0
         X=norm;
1753
0
         if (Y_!=NULL)
1754
0
            Y = norm2;
1755
0
         lowband_scratch = NULL;
1756
0
      }
1757
1.01M
      if (last && !theta_rdo)
1758
85.3k
         lowband_scratch = NULL;
1759
1760
      /* Get a conservative estimate of the collapse_mask's for the bands we're
1761
         going to be folding from. */
1762
1.01M
      if (lowband_offset != 0 && (spread!=SPREAD_AGGRESSIVE || B>1 || tf_change<0))
1763
905k
      {
1764
905k
         int fold_start;
1765
905k
         int fold_end;
1766
905k
         int fold_i;
1767
         /* This ensures we never repeat spectral content within one band */
1768
905k
         effective_lowband = IMAX(0, M*eBands[lowband_offset]-norm_offset-N);
1769
905k
         fold_start = lowband_offset;
1770
1.14M
         while(M*eBands[--fold_start] > effective_lowband+norm_offset);
1771
905k
         fold_end = lowband_offset-1;
1772
905k
#ifndef DISABLE_UPDATE_DRAFT
1773
1.68M
         while(++fold_end < i && M*eBands[fold_end] < effective_lowband+norm_offset+N);
1774
#else
1775
         while(M*eBands[++fold_end] < effective_lowband+norm_offset+N);
1776
#endif
1777
905k
         x_cm = y_cm = 0;
1778
1.92M
         fold_i = fold_start; do {
1779
1.92M
           x_cm |= collapse_masks[fold_i*C+0];
1780
1.92M
           y_cm |= collapse_masks[fold_i*C+C-1];
1781
1.92M
         } while (++fold_i<fold_end);
1782
905k
      }
1783
      /* Otherwise, we'll be using the LCG to fold, so all blocks will (almost
1784
         always) be non-zero. */
1785
111k
      else
1786
111k
         x_cm = y_cm = (1<<B)-1;
1787
1788
1.01M
      if (dual_stereo && i==intensity)
1789
3.69k
      {
1790
3.69k
         int j;
1791
1792
         /* Switch off dual stereo to do intensity. */
1793
3.69k
         dual_stereo = 0;
1794
3.69k
         if (resynth)
1795
125k
            for (j=0;j<M*eBands[i]-norm_offset;j++)
1796
121k
               norm[j] = HALF32(norm[j]+norm2[j]);
1797
3.69k
      }
1798
1.01M
      if (dual_stereo)
1799
32.7k
      {
1800
32.7k
         x_cm = quant_band(&ctx, X, N, b/2, B,
1801
32.7k
               effective_lowband != -1 ? norm+effective_lowband : NULL, LM,
1802
32.7k
               last?NULL:norm+M*eBands[i]-norm_offset, Q31ONE, lowband_scratch, x_cm ARG_QEXT(ext_b/2));
1803
32.7k
         y_cm = quant_band(&ctx, Y, N, b/2, B,
1804
32.7k
               effective_lowband != -1 ? norm2+effective_lowband : NULL, LM,
1805
32.7k
               last?NULL:norm2+M*eBands[i]-norm_offset, Q31ONE, lowband_scratch, y_cm ARG_QEXT(ext_b/2));
1806
984k
      } else {
1807
984k
         if (Y!=NULL)
1808
300k
         {
1809
300k
            if (theta_rdo && i < intensity)
1810
0
            {
1811
0
               ec_ctx ec_save, ec_save2;
1812
0
               struct band_ctx ctx_save, ctx_save2;
1813
0
               opus_val32 dist0, dist1;
1814
0
               unsigned cm, cm2;
1815
0
               int nstart_bytes, nend_bytes, save_bytes;
1816
0
               unsigned char *bytes_buf;
1817
#ifdef ENABLE_QEXT
1818
               ec_ctx ext_ec_save, ext_ec_save2;
1819
               unsigned char *ext_bytes_buf;
1820
               int ext_nstart_bytes, ext_nend_bytes, ext_save_bytes;
1821
#endif
1822
0
               opus_val16 w[2];
1823
0
               compute_channel_weights(bandE[i], bandE[i+m->nbEBands], w);
1824
               /* Make a copy. */
1825
0
               cm = x_cm|y_cm;
1826
0
               ec_save = *ec;
1827
#ifdef ENABLE_QEXT
1828
               ext_ec_save = *ext_ec;
1829
#endif
1830
0
               ctx_save = ctx;
1831
0
               OPUS_COPY(X_save, X, N);
1832
0
               OPUS_COPY(Y_save, Y, N);
1833
               /* Encode and round down. */
1834
0
               ctx.theta_round = -1;
1835
0
               x_cm = quant_band_stereo(&ctx, X, Y, N, b, B,
1836
0
                     effective_lowband != -1 ? norm+effective_lowband : NULL, LM,
1837
0
                     last?NULL:norm+M*eBands[i]-norm_offset, lowband_scratch, cm ARG_QEXT(ext_b) ARG_QEXT(cap));
1838
0
               dist0 = MULT16_32_Q15(w[0], celt_inner_prod_norm_shift(X_save, X, N, arch)) + MULT16_32_Q15(w[1], celt_inner_prod_norm_shift(Y_save, Y, N, arch));
1839
1840
               /* Save first result. */
1841
0
               cm2 = x_cm;
1842
0
               ec_save2 = *ec;
1843
#ifdef ENABLE_QEXT
1844
               ext_ec_save2 = *ext_ec;
1845
#endif
1846
0
               ctx_save2 = ctx;
1847
0
               OPUS_COPY(X_save2, X, N);
1848
0
               OPUS_COPY(Y_save2, Y, N);
1849
0
               if (!last)
1850
0
                  OPUS_COPY(norm_save2, norm+M*eBands[i]-norm_offset, N);
1851
0
               nstart_bytes = ec_save.offs;
1852
0
               nend_bytes = ec_save.storage;
1853
0
               bytes_buf = ec_save.buf+nstart_bytes;
1854
0
               save_bytes = nend_bytes-nstart_bytes;
1855
0
               OPUS_COPY(bytes_save, bytes_buf, save_bytes);
1856
#ifdef ENABLE_QEXT
1857
               ext_nstart_bytes = ext_ec_save.offs;
1858
               ext_nend_bytes = ext_ec_save.storage;
1859
               ext_bytes_buf = ext_ec_save.buf!=NULL ? ext_ec_save.buf+ext_nstart_bytes : NULL;
1860
               ext_save_bytes = ext_nend_bytes-ext_nstart_bytes;
1861
               if (ext_save_bytes) OPUS_COPY(ext_bytes_save, ext_bytes_buf, ext_save_bytes);
1862
#endif
1863
               /* Restore */
1864
0
               *ec = ec_save;
1865
#ifdef ENABLE_QEXT
1866
               *ext_ec = ext_ec_save;
1867
#endif
1868
0
               ctx = ctx_save;
1869
0
               OPUS_COPY(X, X_save, N);
1870
0
               OPUS_COPY(Y, Y_save, N);
1871
0
#ifndef DISABLE_UPDATE_DRAFT
1872
0
               if (i == start+1)
1873
0
                  special_hybrid_folding(m, norm, norm2, start, M, dual_stereo);
1874
0
#endif
1875
               /* Encode and round up. */
1876
0
               ctx.theta_round = 1;
1877
0
               x_cm = quant_band_stereo(&ctx, X, Y, N, b, B,
1878
0
                     effective_lowband != -1 ? norm+effective_lowband : NULL, LM,
1879
0
                     last?NULL:norm+M*eBands[i]-norm_offset, lowband_scratch, cm ARG_QEXT(ext_b) ARG_QEXT(cap));
1880
0
               dist1 = MULT16_32_Q15(w[0], celt_inner_prod_norm_shift(X_save, X, N, arch)) + MULT16_32_Q15(w[1], celt_inner_prod_norm_shift(Y_save, Y, N, arch));
1881
0
               if (dist0 >= dist1) {
1882
0
                  x_cm = cm2;
1883
0
                  *ec = ec_save2;
1884
#ifdef ENABLE_QEXT
1885
                  *ext_ec = ext_ec_save2;
1886
#endif
1887
0
                  ctx = ctx_save2;
1888
0
                  OPUS_COPY(X, X_save2, N);
1889
0
                  OPUS_COPY(Y, Y_save2, N);
1890
0
                  if (!last)
1891
0
                     OPUS_COPY(norm+M*eBands[i]-norm_offset, norm_save2, N);
1892
0
                  OPUS_COPY(bytes_buf, bytes_save, save_bytes);
1893
#ifdef ENABLE_QEXT
1894
                  if (ext_save_bytes) OPUS_COPY(ext_bytes_buf, ext_bytes_save, ext_save_bytes);
1895
#endif
1896
0
               }
1897
300k
            } else {
1898
300k
               ctx.theta_round = 0;
1899
300k
               x_cm = quant_band_stereo(&ctx, X, Y, N, b, B,
1900
300k
                     effective_lowband != -1 ? norm+effective_lowband : NULL, LM,
1901
300k
                     last?NULL:norm+M*eBands[i]-norm_offset, lowband_scratch, x_cm|y_cm ARG_QEXT(ext_b) ARG_QEXT(cap));
1902
300k
            }
1903
684k
         } else {
1904
684k
            x_cm = quant_band(&ctx, X, N, b, B,
1905
684k
                  effective_lowband != -1 ? norm+effective_lowband : NULL, LM,
1906
684k
                  last?NULL:norm+M*eBands[i]-norm_offset, Q31ONE, lowband_scratch, x_cm|y_cm ARG_QEXT(ext_b));
1907
684k
         }
1908
984k
         y_cm = x_cm;
1909
984k
      }
1910
1.01M
      collapse_masks[i*C+0] = (unsigned char)x_cm;
1911
1.01M
      collapse_masks[i*C+C-1] = (unsigned char)y_cm;
1912
1.01M
      balance += pulses[i] + tell;
1913
1914
      /* Update the folding position only as long as we have 1 bit/sample depth. */
1915
1.01M
      update_lowband = b>(N<<BITRES);
1916
      /* We only need to avoid noise on a split for the first band. After that, we
1917
         have folding. */
1918
1.01M
      ctx.avoid_split_noise = 0;
1919
1.01M
   }
1920
85.3k
   *seed = ctx.seed;
1921
1922
85.3k
   RESTORE_STACK;
1923
85.3k
}