Coverage Report

Created: 2025-11-16 07:20

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/opus/celt/bands.c
Line
Count
Source
1
/* Copyright (c) 2007-2008 CSIRO
2
   Copyright (c) 2007-2009 Xiph.Org Foundation
3
   Copyright (c) 2008-2009 Gregory Maxwell
4
   Written by Jean-Marc Valin and Gregory Maxwell */
5
/*
6
   Redistribution and use in source and binary forms, with or without
7
   modification, are permitted provided that the following conditions
8
   are met:
9
10
   - Redistributions of source code must retain the above copyright
11
   notice, this list of conditions and the following disclaimer.
12
13
   - Redistributions in binary form must reproduce the above copyright
14
   notice, this list of conditions and the following disclaimer in the
15
   documentation and/or other materials provided with the distribution.
16
17
   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18
   ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19
   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20
   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
21
   OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
22
   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
23
   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
24
   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
25
   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
26
   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27
   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
*/
29
30
#ifdef HAVE_CONFIG_H
31
#include "config.h"
32
#endif
33
34
#include <math.h>
35
#include "bands.h"
36
#include "modes.h"
37
#include "vq.h"
38
#include "cwrs.h"
39
#include "stack_alloc.h"
40
#include "os_support.h"
41
#include "mathops.h"
42
#include "rate.h"
43
#include "quant_bands.h"
44
#include "pitch.h"
45
46
int hysteresis_decision(opus_val16 val, const opus_val16 *thresholds, const opus_val16 *hysteresis, int N, int prev)
47
0
{
48
0
   int i;
49
0
   for (i=0;i<N;i++)
50
0
   {
51
0
      if (val < thresholds[i])
52
0
         break;
53
0
   }
54
0
   if (i>prev && val < thresholds[prev]+hysteresis[prev])
55
0
      i=prev;
56
0
   if (i<prev && val > thresholds[prev-1]-hysteresis[prev-1])
57
0
      i=prev;
58
0
   return i;
59
0
}
60
61
opus_uint32 celt_lcg_rand(opus_uint32 seed)
62
48.4M
{
63
48.4M
   return 1664525 * seed + 1013904223;
64
48.4M
}
65
66
/* This is a cos() approximation designed to be bit-exact on any platform. Bit exactness
67
   with this approximation is important because it has an impact on the bit allocation */
68
opus_int16 bitexact_cos(opus_int16 x)
69
671k
{
70
671k
   opus_int32 tmp;
71
671k
   opus_int16 x2;
72
671k
   tmp = (4096+((opus_int32)(x)*(x)))>>13;
73
671k
   celt_sig_assert(tmp<=32767);
74
671k
   x2 = tmp;
75
671k
   x2 = (32767-x2) + FRAC_MUL16(x2, (-7651 + FRAC_MUL16(x2, (8277 + FRAC_MUL16(-626, x2)))));
76
671k
   celt_sig_assert(x2<=32766);
77
671k
   return 1+x2;
78
671k
}
79
80
int bitexact_log2tan(int isin,int icos)
81
335k
{
82
335k
   int lc;
83
335k
   int ls;
84
335k
   lc=EC_ILOG(icos);
85
335k
   ls=EC_ILOG(isin);
86
335k
   icos<<=15-lc;
87
335k
   isin<<=15-ls;
88
335k
   return (ls-lc)*(1<<11)
89
335k
         +FRAC_MUL16(isin, FRAC_MUL16(isin, -2597) + 7932)
90
335k
         -FRAC_MUL16(icos, FRAC_MUL16(icos, -2597) + 7932);
91
335k
}
92
93
#ifdef FIXED_POINT
94
/* Compute the amplitude (sqrt energy) in each of the bands */
95
void compute_band_energies(const CELTMode *m, const celt_sig *X, celt_ener *bandE, int end, int C, int LM, int arch)
96
{
97
   int i, c, N;
98
   const opus_int16 *eBands = m->eBands;
99
   (void)arch;
100
   N = m->shortMdctSize<<LM;
101
   c=0; do {
102
      for (i=0;i<end;i++)
103
      {
104
         int j;
105
         opus_val32 maxval=0;
106
         opus_val32 sum = 0;
107
108
         maxval = celt_maxabs32(&X[c*N+(eBands[i]<<LM)], (eBands[i+1]-eBands[i])<<LM);
109
         if (maxval > 0)
110
         {
111
            int shift = IMAX(0, 30 - celt_ilog2(maxval+(maxval>>14)+1) - ((((m->logN[i]+7)>>BITRES)+LM+1)>>1));
112
            j=eBands[i]<<LM; do {
113
               opus_val32 x = SHL32(X[j+c*N],shift);
114
               sum = ADD32(sum, MULT32_32_Q31(x, x));
115
            } while (++j<eBands[i+1]<<LM);
116
            bandE[i+c*m->nbEBands] = MAX32(maxval, PSHR32(celt_sqrt32(SHR32(sum,1)), shift));
117
         } else {
118
            bandE[i+c*m->nbEBands] = EPSILON;
119
         }
120
      }
121
   } while (++c<C);
122
}
123
124
/* Normalise each band such that the energy is one. */
125
void normalise_bands(const CELTMode *m, const celt_sig * OPUS_RESTRICT freq, celt_norm * OPUS_RESTRICT X, const celt_ener *bandE, int end, int C, int M)
126
{
127
   int i, c, N;
128
   const opus_int16 *eBands = m->eBands;
129
   N = M*m->shortMdctSize;
130
   c=0; do {
131
      i=0; do {
132
         int j,shift;
133
         opus_val32 E;
134
         opus_val32 g;
135
         E = bandE[i+c*m->nbEBands];
136
         /* For very low energies, we need this to make sure not to prevent energy rounding from
137
            blowing up the normalized signal. */
138
         if (E < 10) E += EPSILON;
139
         shift = 30-celt_zlog2(E);
140
         E = SHL32(E, shift);
141
         g = celt_rcp_norm32(E);
142
         j=M*eBands[i]; do {
143
            X[j+c*N] = PSHR32(MULT32_32_Q31(g, SHL32(freq[j+c*N], shift)), 30-NORM_SHIFT);
144
         } while (++j<M*eBands[i+1]);
145
      } while (++i<end);
146
   } while (++c<C);
147
}
148
149
#else /* FIXED_POINT */
150
/* Compute the amplitude (sqrt energy) in each of the bands */
151
void compute_band_energies(const CELTMode *m, const celt_sig *X, celt_ener *bandE, int end, int C, int LM, int arch)
152
0
{
153
0
   int i, c, N;
154
0
   const opus_int16 *eBands = m->eBands;
155
0
   N = m->shortMdctSize<<LM;
156
0
   c=0; do {
157
0
      for (i=0;i<end;i++)
158
0
      {
159
0
         opus_val32 sum;
160
0
         sum = 1e-27f + celt_inner_prod(&X[c*N+(eBands[i]<<LM)], &X[c*N+(eBands[i]<<LM)], (eBands[i+1]-eBands[i])<<LM, arch);
161
0
         bandE[i+c*m->nbEBands] = celt_sqrt(sum);
162
         /*printf ("%f ", bandE[i+c*m->nbEBands]);*/
163
0
      }
164
0
   } while (++c<C);
165
   /*printf ("\n");*/
166
0
}
167
168
/* Normalise each band such that the energy is one. */
169
void normalise_bands(const CELTMode *m, const celt_sig * OPUS_RESTRICT freq, celt_norm * OPUS_RESTRICT X, const celt_ener *bandE, int end, int C, int M)
170
0
{
171
0
   int i, c, N;
172
0
   const opus_int16 *eBands = m->eBands;
173
0
   N = M*m->shortMdctSize;
174
0
   c=0; do {
175
0
      for (i=0;i<end;i++)
176
0
      {
177
0
         int j;
178
0
         opus_val16 g = 1.f/(1e-27f+bandE[i+c*m->nbEBands]);
179
0
         for (j=M*eBands[i];j<M*eBands[i+1];j++)
180
0
            X[j+c*N] = freq[j+c*N]*g;
181
0
      }
182
0
   } while (++c<C);
183
0
}
184
185
#endif /* FIXED_POINT */
186
187
/* De-normalise the energy to produce the synthesis from the unit-energy bands */
188
void denormalise_bands(const CELTMode *m, const celt_norm * OPUS_RESTRICT X,
189
      celt_sig * OPUS_RESTRICT freq, const celt_glog *bandLogE, int start,
190
      int end, int M, int downsample, int silence)
191
275k
{
192
275k
   int i, N;
193
275k
   int bound;
194
275k
   celt_sig * OPUS_RESTRICT f;
195
275k
   const celt_norm * OPUS_RESTRICT x;
196
275k
   const opus_int16 *eBands = m->eBands;
197
275k
   N = M*m->shortMdctSize;
198
275k
   bound = M*eBands[end];
199
275k
   if (downsample!=1)
200
0
      bound = IMIN(bound, N/downsample);
201
275k
   if (silence)
202
40.4k
   {
203
40.4k
      bound = 0;
204
40.4k
      start = end = 0;
205
40.4k
   }
206
275k
   f = freq;
207
275k
   x = X+M*eBands[start];
208
275k
   if (start != 0)
209
40.4k
   {
210
9.22M
      for (i=0;i<M*eBands[start];i++)
211
9.18M
         *f++ = 0;
212
235k
   } else {
213
235k
      f += M*eBands[start];
214
235k
   }
215
3.71M
   for (i=start;i<end;i++)
216
3.44M
   {
217
3.44M
      int j, band_end;
218
3.44M
      opus_val32 g;
219
3.44M
      celt_glog lg;
220
#ifdef FIXED_POINT
221
      int shift;
222
#endif
223
3.44M
      j=M*eBands[i];
224
3.44M
      band_end = M*eBands[i+1];
225
3.44M
      lg = ADD32(bandLogE[i], SHL32((opus_val32)eMeans[i],DB_SHIFT-4));
226
3.44M
#ifndef FIXED_POINT
227
3.44M
      g = celt_exp2_db(MIN32(32.f, lg));
228
#else
229
      /* Handle the integer part of the log energy */
230
      shift = 17-(lg>>DB_SHIFT);
231
      if (shift>=31)
232
      {
233
         shift=0;
234
         g=0;
235
      } else {
236
         /* Handle the fractional part. */
237
         g = SHL32(celt_exp2_db_frac((lg&((1<<DB_SHIFT)-1))), 2);
238
      }
239
      /* Handle extreme gains with negative shift. */
240
      if (shift<0)
241
      {
242
         /* To avoid overflow, we're
243
            capping the gain here, which is equivalent to a cap of 18 on lg.
244
            This shouldn't trigger unless the bitstream is already corrupted. */
245
         g = 2147483647;
246
         shift = 0;
247
      }
248
#endif
249
51.8M
      do {
250
51.8M
         *f++ = PSHR32(MULT32_32_Q31(SHL32(*x, 30-NORM_SHIFT), g), shift);
251
51.8M
         x++;
252
51.8M
      } while (++j<band_end);
253
3.44M
   }
254
275k
   celt_assert(start <= end);
255
275k
   OPUS_CLEAR(&freq[bound], N-bound);
256
275k
}
257
258
/* This prevents energy collapse for transients with multiple short MDCTs */
259
void anti_collapse(const CELTMode *m, celt_norm *X_, unsigned char *collapse_masks, int LM, int C, int size,
260
      int start, int end, const celt_glog *logE, const celt_glog *prev1logE,
261
      const celt_glog *prev2logE, const int *pulses, opus_uint32 seed, int encode, int arch)
262
2.16k
{
263
2.16k
   int c, i, j, k;
264
37.8k
   for (i=start;i<end;i++)
265
35.6k
   {
266
35.6k
      int N0;
267
35.6k
      opus_val16 thresh, sqrt_1;
268
35.6k
      int depth;
269
#ifdef FIXED_POINT
270
      int shift;
271
      opus_val32 thresh32;
272
#endif
273
274
35.6k
      N0 = m->eBands[i+1]-m->eBands[i];
275
      /* depth in 1/8 bits */
276
35.6k
      celt_sig_assert(pulses[i]>=0);
277
35.6k
      depth = celt_udiv(1+pulses[i], (m->eBands[i+1]-m->eBands[i]))>>LM;
278
279
#ifdef FIXED_POINT
280
      thresh32 = SHR32(celt_exp2(-SHL16(depth, 10-BITRES)),1);
281
      thresh = MULT16_32_Q15(QCONST16(0.5f, 15), MIN32(32767,thresh32));
282
      {
283
         opus_val32 t;
284
         t = N0<<LM;
285
         shift = celt_ilog2(t)>>1;
286
         t = SHL32(t, (7-shift)<<1);
287
         sqrt_1 = celt_rsqrt_norm(t);
288
      }
289
#else
290
35.6k
      thresh = .5f*celt_exp2(-.125f*depth);
291
35.6k
      sqrt_1 = celt_rsqrt(N0<<LM);
292
35.6k
#endif
293
294
35.6k
      c=0; do
295
40.5k
      {
296
40.5k
         celt_norm *X;
297
40.5k
         celt_glog prev1;
298
40.5k
         celt_glog prev2;
299
40.5k
         opus_val32 Ediff;
300
40.5k
         celt_norm r;
301
40.5k
         int renormalize=0;
302
40.5k
         prev1 = prev1logE[c*m->nbEBands+i];
303
40.5k
         prev2 = prev2logE[c*m->nbEBands+i];
304
40.5k
         if (!encode && C==1)
305
30.8k
         {
306
30.8k
            prev1 = MAXG(prev1,prev1logE[m->nbEBands+i]);
307
30.8k
            prev2 = MAXG(prev2,prev2logE[m->nbEBands+i]);
308
30.8k
         }
309
40.5k
         Ediff = logE[c*m->nbEBands+i]-MING(prev1,prev2);
310
40.5k
         Ediff = MAX32(0, Ediff);
311
312
#ifdef FIXED_POINT
313
         if (Ediff < GCONST(16.f))
314
         {
315
            opus_val32 r32 = SHR32(celt_exp2_db(-Ediff),1);
316
            r = 2*MIN16(16383,r32);
317
         } else {
318
            r = 0;
319
         }
320
         if (LM==3)
321
            r = MULT16_16_Q14(23170, MIN32(23169, r));
322
         r = SHR16(MIN16(thresh, r),1);
323
         r = VSHR32(MULT16_16_Q15(sqrt_1, r),shift+14-NORM_SHIFT);
324
#else
325
         /* r needs to be multiplied by 2 or 2*sqrt(2) depending on LM because
326
            short blocks don't have the same energy as long */
327
40.5k
         r = 2.f*celt_exp2_db(-Ediff);
328
40.5k
         if (LM==3)
329
4.54k
            r *= 1.41421356f;
330
40.5k
         r = MIN16(thresh, r);
331
40.5k
         r = r*sqrt_1;
332
40.5k
#endif
333
40.5k
         X = X_+c*size+(m->eBands[i]<<LM);
334
220k
         for (k=0;k<1<<LM;k++)
335
180k
         {
336
            /* Detect collapse */
337
180k
            if (!(collapse_masks[i*C+c]&1<<k))
338
18.9k
            {
339
               /* Fill with noise */
340
67.3k
               for (j=0;j<N0;j++)
341
48.4k
               {
342
48.4k
                  seed = celt_lcg_rand(seed);
343
48.4k
                  X[(j<<LM)+k] = (seed&0x8000 ? r : -r);
344
48.4k
               }
345
18.9k
               renormalize = 1;
346
18.9k
            }
347
180k
         }
348
         /* We just added some energy, so we need to renormalise */
349
40.5k
         if (renormalize)
350
6.75k
            renormalise_vector(X, N0<<LM, Q31ONE, arch);
351
40.5k
      } while (++c<C);
352
35.6k
   }
353
2.16k
}
354
355
/* Compute the weights to use for optimizing normalized distortion across
356
   channels. We use the amplitude to weight square distortion, which means
357
   that we use the square root of the value we would have been using if we
358
   wanted to minimize the MSE in the non-normalized domain. This roughly
359
   corresponds to some quick-and-dirty perceptual experiments I ran to
360
   measure inter-aural masking (there doesn't seem to be any published data
361
   on the topic). */
362
static void compute_channel_weights(celt_ener Ex, celt_ener Ey, opus_val16 w[2])
363
0
{
364
0
   celt_ener minE;
365
#ifdef FIXED_POINT
366
   int shift;
367
#endif
368
0
   minE = MIN32(Ex, Ey);
369
   /* Adjustment to make the weights a bit more conservative. */
370
0
   Ex = ADD32(Ex, minE/3);
371
0
   Ey = ADD32(Ey, minE/3);
372
#ifdef FIXED_POINT
373
   shift = celt_ilog2(EPSILON+MAX32(Ex, Ey))-14;
374
#endif
375
0
   w[0] = VSHR32(Ex, shift);
376
0
   w[1] = VSHR32(Ey, shift);
377
0
}
378
379
static void intensity_stereo(const CELTMode *m, celt_norm * OPUS_RESTRICT X, const celt_norm * OPUS_RESTRICT Y, const celt_ener *bandE, int bandID, int N)
380
0
{
381
0
   int i = bandID;
382
0
   int j;
383
0
   opus_val16 a1, a2;
384
0
   opus_val16 left, right;
385
0
   opus_val16 norm;
386
#ifdef FIXED_POINT
387
   int shift = celt_zlog2(MAX32(bandE[i], bandE[i+m->nbEBands]))-13;
388
#endif
389
0
   left = VSHR32(bandE[i],shift);
390
0
   right = VSHR32(bandE[i+m->nbEBands],shift);
391
0
   norm = EPSILON + celt_sqrt(EPSILON+MULT16_16(left,left)+MULT16_16(right,right));
392
#ifdef FIXED_POINT
393
   left = MIN32(left, norm-1);
394
   right = MIN32(right, norm-1);
395
#endif
396
0
   a1 = DIV32_16(SHL32(EXTEND32(left),15),norm);
397
0
   a2 = DIV32_16(SHL32(EXTEND32(right),15),norm);
398
0
   for (j=0;j<N;j++)
399
0
   {
400
0
      X[j] = ADD32(MULT16_32_Q15(a1, X[j]), MULT16_32_Q15(a2, Y[j]));
401
      /* Side is not encoded, no need to calculate */
402
0
   }
403
0
}
404
405
static void stereo_split(celt_norm * OPUS_RESTRICT X, celt_norm * OPUS_RESTRICT Y, int N)
406
0
{
407
0
   int j;
408
0
   for (j=0;j<N;j++)
409
0
   {
410
0
      opus_val32 r, l;
411
0
      l = MULT32_32_Q31(QCONST32(.70710678f,31), X[j]);
412
0
      r = MULT32_32_Q31(QCONST32(.70710678f,31), Y[j]);
413
0
      X[j] = ADD32(l, r);
414
0
      Y[j] = SUB32(r, l);
415
0
   }
416
0
}
417
418
static void stereo_merge(celt_norm * OPUS_RESTRICT X, celt_norm * OPUS_RESTRICT Y, opus_val32 mid, int N, int arch)
419
182k
{
420
182k
   int j;
421
182k
   opus_val32 xp=0, side=0;
422
182k
   opus_val32 El, Er;
423
#ifdef FIXED_POINT
424
   int kl, kr;
425
#endif
426
182k
   opus_val32 t, lgain, rgain;
427
428
   /* Compute the norm of X+Y and X-Y as |X|^2 + |Y|^2 +/- sum(xy) */
429
182k
   xp = celt_inner_prod_norm_shift(Y, X, N, arch);
430
182k
   side = celt_inner_prod_norm_shift(Y, Y, N, arch);
431
   /* Compensating for the mid normalization */
432
182k
   xp = MULT32_32_Q31(mid, xp);
433
   /* mid and side are in Q15, not Q14 like X and Y */
434
182k
   El = SHR32(MULT32_32_Q31(mid, mid),3) + side - 2*xp;
435
182k
   Er = SHR32(MULT32_32_Q31(mid, mid),3) + side + 2*xp;
436
182k
   if (Er < QCONST32(6e-4f, 28) || El < QCONST32(6e-4f, 28))
437
1.00k
   {
438
1.00k
      OPUS_COPY(Y, X, N);
439
1.00k
      return;
440
1.00k
   }
441
442
#ifdef FIXED_POINT
443
   kl = celt_ilog2(El)>>1;
444
   kr = celt_ilog2(Er)>>1;
445
#endif
446
181k
   t = VSHR32(El, (kl<<1)-29);
447
181k
   lgain = celt_rsqrt_norm32(t);
448
181k
   t = VSHR32(Er, (kr<<1)-29);
449
181k
   rgain = celt_rsqrt_norm32(t);
450
451
#ifdef FIXED_POINT
452
   if (kl < 7)
453
      kl = 7;
454
   if (kr < 7)
455
      kr = 7;
456
#endif
457
458
4.28M
   for (j=0;j<N;j++)
459
4.10M
   {
460
4.10M
      celt_norm r, l;
461
      /* Apply mid scaling (side is already scaled) */
462
4.10M
      l = MULT32_32_Q31(mid, X[j]);
463
4.10M
      r = Y[j];
464
4.10M
      X[j] = VSHR32(MULT32_32_Q31(lgain, SUB32(l,r)), kl-15);
465
4.10M
      Y[j] = VSHR32(MULT32_32_Q31(rgain, ADD32(l,r)), kr-15);
466
4.10M
   }
467
181k
}
468
469
/* Decide whether we should spread the pulses in the current frame */
470
int spreading_decision(const CELTMode *m, const celt_norm *X, int *average,
471
      int last_decision, int *hf_average, int *tapset_decision, int update_hf,
472
      int end, int C, int M, const int *spread_weight)
473
0
{
474
0
   int i, c, N0;
475
0
   int sum = 0, nbBands=0;
476
0
   const opus_int16 * OPUS_RESTRICT eBands = m->eBands;
477
0
   int decision;
478
0
   int hf_sum=0;
479
480
0
   celt_assert(end>0);
481
482
0
   N0 = M*m->shortMdctSize;
483
484
0
   if (M*(eBands[end]-eBands[end-1]) <= 8)
485
0
      return SPREAD_NONE;
486
0
   c=0; do {
487
0
      for (i=0;i<end;i++)
488
0
      {
489
0
         int j, N, tmp=0;
490
0
         int tcount[3] = {0,0,0};
491
0
         const celt_norm * OPUS_RESTRICT x = X+M*eBands[i]+c*N0;
492
0
         N = M*(eBands[i+1]-eBands[i]);
493
0
         if (N<=8)
494
0
            continue;
495
         /* Compute rough CDF of |x[j]| */
496
0
         for (j=0;j<N;j++)
497
0
         {
498
0
            opus_val32 x2N; /* Q13 */
499
500
0
            x2N = MULT16_16(MULT16_16_Q15(SHR32(x[j], NORM_SHIFT-14), SHR32(x[j], NORM_SHIFT-14)), N);
501
0
            if (x2N < QCONST16(0.25f,13))
502
0
               tcount[0]++;
503
0
            if (x2N < QCONST16(0.0625f,13))
504
0
               tcount[1]++;
505
0
            if (x2N < QCONST16(0.015625f,13))
506
0
               tcount[2]++;
507
0
         }
508
509
         /* Only include four last bands (8 kHz and up) */
510
0
         if (i>m->nbEBands-4)
511
0
            hf_sum += celt_udiv(32*(tcount[1]+tcount[0]), N);
512
0
         tmp = (2*tcount[2] >= N) + (2*tcount[1] >= N) + (2*tcount[0] >= N);
513
0
         sum += tmp*spread_weight[i];
514
0
         nbBands+=spread_weight[i];
515
0
      }
516
0
   } while (++c<C);
517
518
0
   if (update_hf)
519
0
   {
520
0
      if (hf_sum)
521
0
         hf_sum = celt_udiv(hf_sum, C*(4-m->nbEBands+end));
522
0
      *hf_average = (*hf_average+hf_sum)>>1;
523
0
      hf_sum = *hf_average;
524
0
      if (*tapset_decision==2)
525
0
         hf_sum += 4;
526
0
      else if (*tapset_decision==0)
527
0
         hf_sum -= 4;
528
0
      if (hf_sum > 22)
529
0
         *tapset_decision=2;
530
0
      else if (hf_sum > 18)
531
0
         *tapset_decision=1;
532
0
      else
533
0
         *tapset_decision=0;
534
0
   }
535
   /*printf("%d %d %d\n", hf_sum, *hf_average, *tapset_decision);*/
536
0
   celt_assert(nbBands>0); /* end has to be non-zero */
537
0
   celt_assert(sum>=0);
538
0
   sum = celt_udiv((opus_int32)sum<<8, nbBands);
539
   /* Recursive averaging */
540
0
   sum = (sum+*average)>>1;
541
0
   *average = sum;
542
   /* Hysteresis */
543
0
   sum = (3*sum + (((3-last_decision)<<7) + 64) + 2)>>2;
544
0
   if (sum < 80)
545
0
   {
546
0
      decision = SPREAD_AGGRESSIVE;
547
0
   } else if (sum < 256)
548
0
   {
549
0
      decision = SPREAD_NORMAL;
550
0
   } else if (sum < 384)
551
0
   {
552
0
      decision = SPREAD_LIGHT;
553
0
   } else {
554
0
      decision = SPREAD_NONE;
555
0
   }
556
#ifdef FUZZING
557
   decision = rand()&0x3;
558
   *tapset_decision=rand()%3;
559
#endif
560
0
   return decision;
561
0
}
562
563
/* Indexing table for converting from natural Hadamard to ordery Hadamard
564
   This is essentially a bit-reversed Gray, on top of which we've added
565
   an inversion of the order because we want the DC at the end rather than
566
   the beginning. The lines are for N=2, 4, 8, 16 */
567
static const int ordery_table[] = {
568
       1,  0,
569
       3,  0,  2,  1,
570
       7,  0,  4,  3,  6,  1,  5,  2,
571
      15,  0,  8,  7, 12,  3, 11,  4, 14,  1,  9,  6, 13,  2, 10,  5,
572
};
573
574
static void deinterleave_hadamard(celt_norm *X, int N0, int stride, int hadamard)
575
205k
{
576
205k
   int i,j;
577
205k
   VARDECL(celt_norm, tmp);
578
205k
   int N;
579
205k
   SAVE_STACK;
580
205k
   N = N0*stride;
581
205k
   ALLOC(tmp, N, celt_norm);
582
205k
   celt_assert(stride>0);
583
205k
   if (hadamard)
584
154k
   {
585
154k
      const int *ordery = ordery_table+stride-2;
586
850k
      for (i=0;i<stride;i++)
587
695k
      {
588
3.09M
         for (j=0;j<N0;j++)
589
2.39M
            tmp[ordery[i]*N0+j] = X[j*stride+i];
590
695k
      }
591
154k
   } else {
592
283k
      for (i=0;i<stride;i++)
593
879k
         for (j=0;j<N0;j++)
594
646k
            tmp[i*N0+j] = X[j*stride+i];
595
50.2k
   }
596
205k
   OPUS_COPY(X, tmp, N);
597
205k
   RESTORE_STACK;
598
205k
}
599
600
static void interleave_hadamard(celt_norm *X, int N0, int stride, int hadamard)
601
254k
{
602
254k
   int i,j;
603
254k
   VARDECL(celt_norm, tmp);
604
254k
   int N;
605
254k
   SAVE_STACK;
606
254k
   N = N0*stride;
607
254k
   ALLOC(tmp, N, celt_norm);
608
254k
   if (hadamard)
609
197k
   {
610
197k
      const int *ordery = ordery_table+stride-2;
611
1.10M
      for (i=0;i<stride;i++)
612
4.32M
         for (j=0;j<N0;j++)
613
3.41M
            tmp[j*stride+i] = X[ordery[i]*N0+j];
614
197k
   } else {
615
334k
      for (i=0;i<stride;i++)
616
1.06M
         for (j=0;j<N0;j++)
617
791k
            tmp[j*stride+i] = X[i*N0+j];
618
57.3k
   }
619
254k
   OPUS_COPY(X, tmp, N);
620
254k
   RESTORE_STACK;
621
254k
}
622
623
void haar1(celt_norm *X, int N0, int stride)
624
1.05M
{
625
1.05M
   int i, j;
626
1.05M
   N0 >>= 1;
627
2.86M
   for (i=0;i<stride;i++)
628
11.0M
      for (j=0;j<N0;j++)
629
9.19M
      {
630
9.19M
         opus_val32 tmp1, tmp2;
631
9.19M
         tmp1 = MULT32_32_Q31(QCONST32(.70710678f,31), X[stride*2*j+i]);
632
9.19M
         tmp2 = MULT32_32_Q31(QCONST32(.70710678f,31), X[stride*(2*j+1)+i]);
633
9.19M
         X[stride*2*j+i] = ADD32(tmp1, tmp2);
634
9.19M
         X[stride*(2*j+1)+i] = SUB32(tmp1, tmp2);
635
9.19M
      }
636
1.05M
}
637
638
static int compute_qn(int N, int b, int offset, int pulse_cap, int stereo)
639
551k
{
640
551k
   static const opus_int16 exp2_table8[8] =
641
551k
      {16384, 17866, 19483, 21247, 23170, 25267, 27554, 30048};
642
551k
   int qn, qb;
643
551k
   int N2 = 2*N-1;
644
551k
   if (stereo && N==2)
645
58.1k
      N2--;
646
   /* The upper limit ensures that in a stereo split with itheta==16384, we'll
647
       always have enough bits left over to code at least one pulse in the
648
       side; otherwise it would collapse, since it doesn't get folded. */
649
551k
   qb = celt_sudiv(b+N2*offset, N2);
650
551k
   qb = IMIN(b-pulse_cap-(4<<BITRES), qb);
651
652
551k
   qb = IMIN(8<<BITRES, qb);
653
654
551k
   if (qb<(1<<BITRES>>1)) {
655
147k
      qn = 1;
656
404k
   } else {
657
404k
      qn = exp2_table8[qb&0x7]>>(14-(qb>>BITRES));
658
404k
      qn = (qn+1)>>1<<1;
659
404k
   }
660
551k
   celt_assert(qn <= 256);
661
551k
   return qn;
662
551k
}
663
664
struct band_ctx {
665
   int encode;
666
   int resynth;
667
   const CELTMode *m;
668
   int i;
669
   int intensity;
670
   int spread;
671
   int tf_change;
672
   ec_ctx *ec;
673
   opus_int32 remaining_bits;
674
   const celt_ener *bandE;
675
   opus_uint32 seed;
676
   int arch;
677
   int theta_round;
678
   int disable_inv;
679
   int avoid_split_noise;
680
#ifdef ENABLE_QEXT
681
   ec_ctx *ext_ec;
682
   int extra_bits;
683
   opus_int32 ext_total_bits;
684
   int extra_bands;
685
#endif
686
};
687
688
struct split_ctx {
689
   int inv;
690
   int imid;
691
   int iside;
692
   int delta;
693
   int itheta;
694
#ifdef ENABLE_QEXT
695
   int itheta_q30;
696
#endif
697
   int qalloc;
698
};
699
700
static void compute_theta(struct band_ctx *ctx, struct split_ctx *sctx,
701
      celt_norm *X, celt_norm *Y, int N, int *b, int B, int B0,
702
      int LM,
703
      int stereo, int *fill ARG_QEXT(int *ext_b))
704
551k
{
705
551k
   int qn;
706
551k
   int itheta=0;
707
551k
   int itheta_q30=0;
708
551k
   int delta;
709
551k
   int imid, iside;
710
551k
   int qalloc;
711
551k
   int pulse_cap;
712
551k
   int offset;
713
551k
   opus_int32 tell;
714
551k
   int inv=0;
715
551k
   int encode;
716
551k
   const CELTMode *m;
717
551k
   int i;
718
551k
   int intensity;
719
551k
   ec_ctx *ec;
720
551k
   const celt_ener *bandE;
721
722
551k
   encode = ctx->encode;
723
551k
   m = ctx->m;
724
551k
   i = ctx->i;
725
551k
   intensity = ctx->intensity;
726
551k
   ec = ctx->ec;
727
551k
   bandE = ctx->bandE;
728
729
   /* Decide on the resolution to give to the split parameter theta */
730
551k
   pulse_cap = m->logN[i]+LM*(1<<BITRES);
731
551k
   offset = (pulse_cap>>1) - (stereo&&N==2 ? QTHETA_OFFSET_TWOPHASE : QTHETA_OFFSET);
732
551k
   qn = compute_qn(N, *b, offset, pulse_cap, stereo);
733
551k
   if (stereo && i>=intensity)
734
196k
      qn = 1;
735
551k
   if (encode)
736
0
   {
737
      /* theta is the atan() of the ratio between the (normalized)
738
         side and mid. With just that parameter, we can re-scale both
739
         mid and side because we know that 1) they have unit norm and
740
         2) they are orthogonal. */
741
0
      itheta_q30 = stereo_itheta(X, Y, stereo, N, ctx->arch);
742
0
      itheta = itheta_q30>>16;
743
0
   }
744
551k
   tell = ec_tell_frac(ec);
745
551k
   if (qn!=1)
746
351k
   {
747
351k
      if (encode)
748
0
      {
749
0
         if (!stereo || ctx->theta_round == 0)
750
0
         {
751
0
            itheta = (itheta*(opus_int32)qn+8192)>>14;
752
0
            if (!stereo && ctx->avoid_split_noise && itheta > 0 && itheta < qn)
753
0
            {
754
               /* Check if the selected value of theta will cause the bit allocation
755
                  to inject noise on one side. If so, make sure the energy of that side
756
                  is zero. */
757
0
               int unquantized = celt_udiv((opus_int32)itheta*16384, qn);
758
0
               imid = bitexact_cos((opus_int16)unquantized);
759
0
               iside = bitexact_cos((opus_int16)(16384-unquantized));
760
0
               delta = FRAC_MUL16((N-1)<<7,bitexact_log2tan(iside,imid));
761
0
               if (delta > *b)
762
0
                  itheta = qn;
763
0
               else if (delta < -*b)
764
0
                  itheta = 0;
765
0
            }
766
0
         } else {
767
0
            int down;
768
            /* Bias quantization towards itheta=0 and itheta=16384. */
769
0
            int bias = itheta > 8192 ? 32767/qn : -32767/qn;
770
0
            down = IMIN(qn-1, IMAX(0, (itheta*(opus_int32)qn + bias)>>14));
771
0
            if (ctx->theta_round < 0)
772
0
               itheta = down;
773
0
            else
774
0
               itheta = down+1;
775
0
         }
776
0
      }
777
      /* Entropy coding of the angle. We use a uniform pdf for the
778
         time split, a step for stereo, and a triangular one for the rest. */
779
351k
      if (stereo && N>2)
780
20.7k
      {
781
20.7k
         int p0 = 3;
782
20.7k
         int x = itheta;
783
20.7k
         int x0 = qn/2;
784
20.7k
         int ft = p0*(x0+1) + x0;
785
         /* Use a probability of p0 up to itheta=8192 and then use 1 after */
786
20.7k
         if (encode)
787
0
         {
788
0
            ec_encode(ec,x<=x0?p0*x:(x-1-x0)+(x0+1)*p0,x<=x0?p0*(x+1):(x-x0)+(x0+1)*p0,ft);
789
20.7k
         } else {
790
20.7k
            int fs;
791
20.7k
            fs=ec_decode(ec,ft);
792
20.7k
            if (fs<(x0+1)*p0)
793
16.5k
               x=fs/p0;
794
4.19k
            else
795
4.19k
               x=x0+1+(fs-(x0+1)*p0);
796
20.7k
            ec_dec_update(ec,x<=x0?p0*x:(x-1-x0)+(x0+1)*p0,x<=x0?p0*(x+1):(x-x0)+(x0+1)*p0,ft);
797
20.7k
            itheta = x;
798
20.7k
         }
799
330k
      } else if (B0>1 || stereo) {
800
         /* Uniform pdf */
801
117k
         if (encode)
802
0
            ec_enc_uint(ec, itheta, qn+1);
803
117k
         else
804
117k
            itheta = ec_dec_uint(ec, qn+1);
805
212k
      } else {
806
212k
         int fs=1, ft;
807
212k
         ft = ((qn>>1)+1)*((qn>>1)+1);
808
212k
         if (encode)
809
0
         {
810
0
            int fl;
811
812
0
            fs = itheta <= (qn>>1) ? itheta + 1 : qn + 1 - itheta;
813
0
            fl = itheta <= (qn>>1) ? itheta*(itheta + 1)>>1 :
814
0
             ft - ((qn + 1 - itheta)*(qn + 2 - itheta)>>1);
815
816
0
            ec_encode(ec, fl, fl+fs, ft);
817
212k
         } else {
818
            /* Triangular pdf */
819
212k
            int fl=0;
820
212k
            int fm;
821
212k
            fm = ec_decode(ec, ft);
822
823
212k
            if (fm < ((qn>>1)*((qn>>1) + 1)>>1))
824
100k
            {
825
100k
               itheta = (isqrt32(8*(opus_uint32)fm + 1) - 1)>>1;
826
100k
               fs = itheta + 1;
827
100k
               fl = itheta*(itheta + 1)>>1;
828
100k
            }
829
112k
            else
830
112k
            {
831
112k
               itheta = (2*(qn + 1)
832
112k
                - isqrt32(8*(opus_uint32)(ft - fm - 1) + 1))>>1;
833
112k
               fs = qn + 1 - itheta;
834
112k
               fl = ft - ((qn + 1 - itheta)*(qn + 2 - itheta)>>1);
835
112k
            }
836
837
212k
            ec_dec_update(ec, fl, fl+fs, ft);
838
212k
         }
839
212k
      }
840
351k
      celt_assert(itheta>=0);
841
351k
      itheta = celt_udiv((opus_int32)itheta*16384, qn);
842
#ifdef ENABLE_QEXT
843
      *ext_b = IMIN(*ext_b, ctx->ext_total_bits - (opus_int32)ec_tell_frac(ctx->ext_ec));
844
      if (*ext_b >= 2*N<<BITRES && ctx->ext_total_bits-ec_tell_frac(ctx->ext_ec)-1 > 2<<BITRES) {
845
         int extra_bits;
846
         int ext_tell = ec_tell_frac(ctx->ext_ec);
847
         extra_bits = IMIN(12, IMAX(2, celt_sudiv(*ext_b, (2*N-1)<<BITRES)));
848
         if (encode) {
849
            itheta_q30 = itheta_q30 - (itheta<<16);
850
            itheta_q30 = (itheta_q30*(opus_int64)qn*((1<<extra_bits)-1)+(1<<29))>>30;
851
            itheta_q30 += (1<<(extra_bits-1))-1;
852
            itheta_q30 = IMAX(0, IMIN((1<<extra_bits)-2, itheta_q30));
853
            ec_enc_uint(ctx->ext_ec, itheta_q30, (1<<extra_bits)-1);
854
         } else {
855
            itheta_q30 = ec_dec_uint(ctx->ext_ec, (1<<extra_bits)-1);
856
         }
857
         itheta_q30 -= (1<<(extra_bits-1))-1;
858
         itheta_q30 = (itheta<<16) + itheta_q30*(opus_int64)(1<<30)/(qn*((1<<extra_bits)-1));
859
         /* Hard bounds on itheta (can only trigger on corrupted bitstreams). */
860
         itheta_q30 = IMAX(0, IMIN(itheta_q30, 1073741824));
861
         *ext_b -= ec_tell_frac(ctx->ext_ec) - ext_tell;
862
      } else {
863
         itheta_q30 = (opus_int32)itheta<<16;
864
      }
865
#endif
866
351k
      if (encode && stereo)
867
0
      {
868
0
         if (itheta==0)
869
0
            intensity_stereo(m, X, Y, bandE, i, N);
870
0
         else
871
0
            stereo_split(X, Y, N);
872
0
      }
873
      /* NOTE: Renormalising X and Y *may* help fixed-point a bit at very high rate.
874
               Let's do that at higher complexity */
875
351k
   } else if (stereo) {
876
200k
      if (encode)
877
0
      {
878
0
         inv = itheta > 8192 && !ctx->disable_inv;
879
0
         if (inv)
880
0
         {
881
0
            int j;
882
0
            for (j=0;j<N;j++)
883
0
               Y[j] = -Y[j];
884
0
         }
885
0
         intensity_stereo(m, X, Y, bandE, i, N);
886
0
      }
887
200k
      if (*b>2<<BITRES && ctx->remaining_bits > 2<<BITRES)
888
66.4k
      {
889
66.4k
         if (encode)
890
0
            ec_enc_bit_logp(ec, inv, 2);
891
66.4k
         else
892
66.4k
            inv = ec_dec_bit_logp(ec, 2);
893
66.4k
      } else
894
133k
         inv = 0;
895
      /* inv flag override to avoid problems with downmixing. */
896
200k
      if (ctx->disable_inv)
897
0
         inv = 0;
898
200k
      itheta = 0;
899
200k
      itheta_q30 = 0;
900
200k
   }
901
551k
   qalloc = ec_tell_frac(ec) - tell;
902
551k
   *b -= qalloc;
903
904
551k
   if (itheta == 0)
905
210k
   {
906
210k
      imid = 32767;
907
210k
      iside = 0;
908
210k
      *fill &= (1<<B)-1;
909
210k
      delta = -16384;
910
341k
   } else if (itheta == 16384)
911
5.79k
   {
912
5.79k
      imid = 0;
913
5.79k
      iside = 32767;
914
5.79k
      *fill &= ((1<<B)-1)<<B;
915
5.79k
      delta = 16384;
916
335k
   } else {
917
335k
      imid = bitexact_cos((opus_int16)itheta);
918
335k
      iside = bitexact_cos((opus_int16)(16384-itheta));
919
      /* This is the mid vs side allocation that minimizes squared error
920
         in that band. */
921
335k
      delta = FRAC_MUL16((N-1)<<7,bitexact_log2tan(iside,imid));
922
335k
   }
923
924
551k
   sctx->inv = inv;
925
551k
   sctx->imid = imid;
926
551k
   sctx->iside = iside;
927
551k
   sctx->delta = delta;
928
551k
   sctx->itheta = itheta;
929
#ifdef ENABLE_QEXT
930
   sctx->itheta_q30 = itheta_q30;
931
#endif
932
551k
   sctx->qalloc = qalloc;
933
551k
}
934
static unsigned quant_band_n1(struct band_ctx *ctx, celt_norm *X, celt_norm *Y,
935
      celt_norm *lowband_out)
936
188k
{
937
188k
   int c;
938
188k
   int stereo;
939
188k
   celt_norm *x = X;
940
188k
   int encode;
941
188k
   ec_ctx *ec;
942
943
188k
   encode = ctx->encode;
944
188k
   ec = ctx->ec;
945
946
188k
   stereo = Y != NULL;
947
221k
   c=0; do {
948
221k
      int sign=0;
949
221k
      if (ctx->remaining_bits>=1<<BITRES)
950
72.4k
      {
951
72.4k
         if (encode)
952
0
         {
953
0
            sign = x[0]<0;
954
0
            ec_enc_bits(ec, sign, 1);
955
72.4k
         } else {
956
72.4k
            sign = ec_dec_bits(ec, 1);
957
72.4k
         }
958
72.4k
         ctx->remaining_bits -= 1<<BITRES;
959
72.4k
      }
960
221k
      if (ctx->resynth)
961
221k
         x[0] = sign ? -NORM_SCALING : NORM_SCALING;
962
221k
      x = Y;
963
221k
   } while (++c<1+stereo);
964
188k
   if (lowband_out)
965
188k
      lowband_out[0] = SHR32(X[0],4);
966
188k
   return 1;
967
188k
}
968
969
/* This function is responsible for encoding and decoding a mono partition.
970
   It can split the band in two and transmit the energy difference with
971
   the two half-bands. It can be called recursively so bands can end up being
972
   split in 8 parts. */
973
static unsigned quant_partition(struct band_ctx *ctx, celt_norm *X,
974
      int N, int b, int B, celt_norm *lowband,
975
      int LM,
976
      opus_val32 gain, int fill
977
      ARG_QEXT(int ext_b))
978
1.96M
{
979
1.96M
   const unsigned char *cache;
980
1.96M
   int q;
981
1.96M
   int curr_bits;
982
1.96M
   int imid=0, iside=0;
983
1.96M
   int B0=B;
984
1.96M
   opus_val32 mid=0, side=0;
985
1.96M
   unsigned cm=0;
986
1.96M
   celt_norm *Y=NULL;
987
1.96M
   int encode;
988
1.96M
   const CELTMode *m;
989
1.96M
   int i;
990
1.96M
   int spread;
991
1.96M
   ec_ctx *ec;
992
993
1.96M
   encode = ctx->encode;
994
1.96M
   m = ctx->m;
995
1.96M
   i = ctx->i;
996
1.96M
   spread = ctx->spread;
997
1.96M
   ec = ctx->ec;
998
999
   /* If we need 1.5 more bit than we can produce, split the band in two. */
1000
1.96M
   cache = m->cache.bits + m->cache.index[(LM+1)*m->nbEBands+i];
1001
1.96M
   if (LM != -1 && b > cache[cache[0]]+12 && N>2)
1002
310k
   {
1003
310k
      int mbits, sbits, delta;
1004
310k
      int itheta;
1005
310k
      int qalloc;
1006
310k
      struct split_ctx sctx;
1007
310k
      celt_norm *next_lowband2=NULL;
1008
310k
      opus_int32 rebalance;
1009
1010
310k
      N >>= 1;
1011
310k
      Y = X+N;
1012
310k
      LM -= 1;
1013
310k
      if (B==1)
1014
212k
         fill = (fill&1)|(fill<<1);
1015
310k
      B = (B+1)>>1;
1016
1017
310k
      compute_theta(ctx, &sctx, X, Y, N, &b, B, B0, LM, 0, &fill ARG_QEXT(&ext_b));
1018
310k
      imid = sctx.imid;
1019
310k
      iside = sctx.iside;
1020
310k
      delta = sctx.delta;
1021
310k
      itheta = sctx.itheta;
1022
310k
      qalloc = sctx.qalloc;
1023
#ifdef FIXED_POINT
1024
# ifdef ENABLE_QEXT
1025
      (void)imid;
1026
      (void)iside;
1027
      mid = celt_cos_norm32(sctx.itheta_q30);
1028
      side = celt_cos_norm32((1<<30)-sctx.itheta_q30);
1029
# else
1030
      mid = SHL32(EXTEND32(imid), 16);
1031
      side = SHL32(EXTEND32(iside), 16);
1032
# endif
1033
#else
1034
# ifdef ENABLE_QEXT
1035
      (void)imid;
1036
      (void)iside;
1037
      mid = celt_cos_norm2(sctx.itheta_q30*(1.f/(1<<30)));
1038
      side = celt_cos_norm2(1.f-sctx.itheta_q30*(1.f/(1<<30)));
1039
# else
1040
310k
      mid = (1.f/32768)*imid;
1041
310k
      side = (1.f/32768)*iside;
1042
310k
# endif
1043
310k
#endif
1044
1045
      /* Give more bits to low-energy MDCTs than they would otherwise deserve */
1046
310k
      if (B0>1 && (itheta&0x3fff))
1047
93.3k
      {
1048
93.3k
         if (itheta > 8192)
1049
            /* Rough approximation for pre-echo masking */
1050
41.6k
            delta -= delta>>(4-LM);
1051
51.6k
         else
1052
            /* Corresponds to a forward-masking slope of 1.5 dB per 10 ms */
1053
51.6k
            delta = IMIN(0, delta + (N<<BITRES>>(5-LM)));
1054
93.3k
      }
1055
310k
      mbits = IMAX(0, IMIN(b, (b-delta)/2));
1056
310k
      sbits = b-mbits;
1057
310k
      ctx->remaining_bits -= qalloc;
1058
1059
310k
      if (lowband)
1060
272k
         next_lowband2 = lowband+N; /* >32-bit split case */
1061
1062
310k
      rebalance = ctx->remaining_bits;
1063
310k
      if (mbits >= sbits)
1064
156k
      {
1065
156k
         cm = quant_partition(ctx, X, N, mbits, B, lowband, LM,
1066
156k
               MULT32_32_Q31(gain,mid), fill ARG_QEXT(ext_b/2));
1067
156k
         rebalance = mbits - (rebalance-ctx->remaining_bits);
1068
156k
         if (rebalance > 3<<BITRES && itheta!=0)
1069
84.6k
            sbits += rebalance - (3<<BITRES);
1070
156k
         cm |= quant_partition(ctx, Y, N, sbits, B, next_lowband2, LM,
1071
156k
               MULT32_32_Q31(gain,side), fill>>B ARG_QEXT(ext_b/2))<<(B0>>1);
1072
156k
      } else {
1073
154k
         cm = quant_partition(ctx, Y, N, sbits, B, next_lowband2, LM,
1074
154k
               MULT32_32_Q31(gain,side), fill>>B ARG_QEXT(ext_b/2))<<(B0>>1);
1075
154k
         rebalance = sbits - (rebalance-ctx->remaining_bits);
1076
154k
         if (rebalance > 3<<BITRES && itheta!=16384)
1077
87.7k
            mbits += rebalance - (3<<BITRES);
1078
154k
         cm |= quant_partition(ctx, X, N, mbits, B, lowband, LM,
1079
154k
               MULT32_32_Q31(gain,mid), fill ARG_QEXT(ext_b/2));
1080
154k
      }
1081
1.65M
   } else {
1082
#ifdef ENABLE_QEXT
1083
      int extra_bits;
1084
      int ext_remaining_bits;
1085
      extra_bits = ext_b/(N-1)>>BITRES;
1086
      ext_remaining_bits = ctx->ext_total_bits-(opus_int32)ec_tell_frac(ctx->ext_ec);
1087
      if (ext_remaining_bits < ((extra_bits+1)*(N-1)+N)<<BITRES) {
1088
         extra_bits = (ext_remaining_bits-(N<<BITRES))/(N-1)>>BITRES;
1089
         extra_bits = IMAX(extra_bits-1, 0);
1090
      }
1091
      extra_bits = IMIN(12, extra_bits);
1092
#endif
1093
      /* This is the basic no-split case */
1094
1.65M
      q = bits2pulses(m, i, LM, b);
1095
1.65M
      curr_bits = pulses2bits(m, i, LM, q);
1096
1.65M
      ctx->remaining_bits -= curr_bits;
1097
1098
      /* Ensures we can never bust the budget */
1099
1.67M
      while (ctx->remaining_bits < 0 && q > 0)
1100
15.3k
      {
1101
15.3k
         ctx->remaining_bits += curr_bits;
1102
15.3k
         q--;
1103
15.3k
         curr_bits = pulses2bits(m, i, LM, q);
1104
15.3k
         ctx->remaining_bits -= curr_bits;
1105
15.3k
      }
1106
1107
1.65M
      if (q!=0)
1108
858k
      {
1109
858k
         int K = get_pulses(q);
1110
1111
         /* Finally do the actual quantization */
1112
858k
         if (encode)
1113
0
         {
1114
0
            cm = alg_quant(X, N, K, spread, B, ec, gain, ctx->resynth
1115
0
                           ARG_QEXT(ctx->ext_ec) ARG_QEXT(extra_bits),
1116
0
                           ctx->arch);
1117
858k
         } else {
1118
858k
            cm = alg_unquant(X, N, K, spread, B, ec, gain
1119
858k
                             ARG_QEXT(ctx->ext_ec) ARG_QEXT(extra_bits));
1120
858k
         }
1121
#ifdef ENABLE_QEXT
1122
      } else if (ext_b > 2*N<<BITRES)
1123
      {
1124
         extra_bits = ext_b/(N-1)>>BITRES;
1125
         ext_remaining_bits = ctx->ext_total_bits-ec_tell_frac(ctx->ext_ec);
1126
         if (ext_remaining_bits < ((extra_bits+1)*(N-1)+N)<<BITRES) {
1127
            extra_bits = (ext_remaining_bits-(N<<BITRES))/(N-1)>>BITRES;
1128
            extra_bits = IMAX(extra_bits-1, 0);
1129
         }
1130
         extra_bits = IMIN(14, extra_bits);
1131
         if (encode) cm = cubic_quant(X, N, extra_bits, B, ctx->ext_ec, gain, ctx->resynth);
1132
         else cm = cubic_unquant(X, N, extra_bits, B, ctx->ext_ec, gain);
1133
#endif
1134
858k
      } else {
1135
         /* If there's no pulse, fill the band anyway */
1136
796k
         int j;
1137
796k
         if (ctx->resynth)
1138
796k
         {
1139
796k
            unsigned cm_mask;
1140
            /* B can be as large as 16, so this shift might overflow an int on a
1141
               16-bit platform; use a long to get defined behavior.*/
1142
796k
            cm_mask = (unsigned)(1UL<<B)-1;
1143
796k
            fill &= cm_mask;
1144
796k
            if (!fill)
1145
174k
            {
1146
174k
               OPUS_CLEAR(X, N);
1147
622k
            } else {
1148
622k
               if (lowband == NULL)
1149
38.9k
               {
1150
                  /* Noise */
1151
1.24M
                  for (j=0;j<N;j++)
1152
1.20M
                  {
1153
1.20M
                     ctx->seed = celt_lcg_rand(ctx->seed);
1154
1.20M
                     X[j] = SHL32((celt_norm)((opus_int32)ctx->seed>>20), NORM_SHIFT-14);
1155
1.20M
                  }
1156
38.9k
                  cm = cm_mask;
1157
583k
               } else {
1158
                  /* Folded spectrum */
1159
9.74M
                  for (j=0;j<N;j++)
1160
9.16M
                  {
1161
9.16M
                     opus_val16 tmp;
1162
9.16M
                     ctx->seed = celt_lcg_rand(ctx->seed);
1163
                     /* About 48 dB below the "normal" folding level */
1164
9.16M
                     tmp = QCONST16(1.0f/256, NORM_SHIFT-4);
1165
9.16M
                     tmp = (ctx->seed)&0x8000 ? tmp : -tmp;
1166
9.16M
                     X[j] = lowband[j]+tmp;
1167
9.16M
                  }
1168
583k
                  cm = fill;
1169
583k
               }
1170
622k
               renormalise_vector(X, N, gain, ctx->arch);
1171
622k
            }
1172
796k
         }
1173
796k
      }
1174
1.65M
   }
1175
1176
1.96M
   return cm;
1177
1.96M
}
1178
1179
#ifdef ENABLE_QEXT
1180
static unsigned cubic_quant_partition(struct band_ctx *ctx, celt_norm *X, int N, int b, int B, ec_ctx *ec, int LM, opus_val32 gain, int resynth, int encode)
1181
{
1182
   celt_assert(LM>=0);
1183
   ctx->remaining_bits = ctx->ec->storage*8*8 - ec_tell_frac(ctx->ec);
1184
   b = IMIN(b, ctx->remaining_bits);
1185
   /* As long as we have at least two bits of depth, split all the way to LM=0 (not -1 like PVQ). */
1186
   if (LM==0 || b<=2*N<<BITRES) {
1187
      int res, ret;
1188
      b = IMIN(b + ((N-1)<<BITRES)/2, ctx->remaining_bits);
1189
      /* Resolution left after taking into account coding the cube face. */
1190
      res = (b-(1<<BITRES)-ctx->m->logN[ctx->i]-(LM<<BITRES)-1)/(N-1)>>BITRES;
1191
      res = IMIN(14, IMAX(0, res));
1192
      if (encode) ret = cubic_quant(X, N, res, B, ec, gain, resynth);
1193
      else ret = cubic_unquant(X, N, res, B, ec, gain);
1194
      ctx->remaining_bits = ctx->ec->storage*8*8 - ec_tell_frac(ctx->ec);
1195
      return ret;
1196
   } else {
1197
      celt_norm *Y;
1198
      opus_int32 itheta_q30;
1199
      opus_val32 g1, g2;
1200
      opus_int32 theta_res;
1201
      opus_int32 qtheta;
1202
      int delta;
1203
      int b1, b2;
1204
      int cm;
1205
      int N0;
1206
      N0 = N;
1207
      N >>= 1;
1208
      Y = X+N;
1209
      LM -= 1;
1210
      B = (B+1)>>1;
1211
      theta_res = IMIN(16, (b>>BITRES)/(N0-1) + 1);
1212
      if (encode) {
1213
         itheta_q30 = stereo_itheta(X, Y, 0, N, ctx->arch);
1214
         qtheta = (itheta_q30+(1<<(29-theta_res)))>>(30-theta_res);
1215
         ec_enc_uint(ec, qtheta, (1<<theta_res)+1);
1216
      } else {
1217
         qtheta = ec_dec_uint(ec, (1<<theta_res)+1);
1218
      }
1219
      itheta_q30 = qtheta<<(30-theta_res);
1220
      b -= theta_res<<BITRES;
1221
      delta = (N0-1) * 23 * ((itheta_q30>>16)-8192) >> (17-BITRES);
1222
1223
#ifdef FIXED_POINT
1224
      g1 = celt_cos_norm32(itheta_q30);
1225
      g2 = celt_cos_norm32((1<<30)-itheta_q30);
1226
#else
1227
      g1 = celt_cos_norm2(itheta_q30*(1.f/(1<<30)));
1228
      g2 = celt_cos_norm2(1.f-itheta_q30*(1.f/(1<<30)));
1229
#endif
1230
      if (itheta_q30 == 0) {
1231
         b1=b;
1232
         b2=0;
1233
      } else if (itheta_q30==1073741824) {
1234
         b1=0;
1235
         b2=b;
1236
      } else {
1237
         b1 = IMIN(b, IMAX(0, (b-delta)/2));
1238
         b2 = b-b1;
1239
      }
1240
      cm  = cubic_quant_partition(ctx, X, N, b1, B, ec, LM, MULT32_32_Q31(gain, g1), resynth, encode);
1241
      cm |= cubic_quant_partition(ctx, Y, N, b2, B, ec, LM, MULT32_32_Q31(gain, g2), resynth, encode);
1242
      return cm;
1243
   }
1244
}
1245
#endif
1246
1247
/* This function is responsible for encoding and decoding a band for the mono case. */
1248
static unsigned quant_band(struct band_ctx *ctx, celt_norm *X,
1249
      int N, int b, int B, celt_norm *lowband,
1250
      int LM, celt_norm *lowband_out,
1251
      opus_val32 gain, celt_norm *lowband_scratch, int fill
1252
      ARG_QEXT(int ext_b))
1253
1.50M
{
1254
1.50M
   int N0=N;
1255
1.50M
   int N_B=N;
1256
1.50M
   int N_B0;
1257
1.50M
   int B0=B;
1258
1.50M
   int time_divide=0;
1259
1.50M
   int recombine=0;
1260
1.50M
   int longBlocks;
1261
1.50M
   unsigned cm=0;
1262
1.50M
   int k;
1263
1.50M
   int encode;
1264
1.50M
   int tf_change;
1265
1266
1.50M
   encode = ctx->encode;
1267
1.50M
   tf_change = ctx->tf_change;
1268
1269
1.50M
   longBlocks = B0==1;
1270
1271
1.50M
   N_B = celt_udiv(N_B, B);
1272
1273
   /* Special case for one sample */
1274
1.50M
   if (N==1)
1275
156k
   {
1276
156k
      return quant_band_n1(ctx, X, NULL, lowband_out);
1277
156k
   }
1278
1279
1.34M
   if (tf_change>0)
1280
120k
      recombine = tf_change;
1281
   /* Band recombining to increase frequency resolution */
1282
1283
1.34M
   if (lowband_scratch && lowband && (recombine || ((N_B&1) == 0 && tf_change<0) || B0>1))
1284
272k
   {
1285
272k
      OPUS_COPY(lowband_scratch, lowband, N);
1286
272k
      lowband = lowband_scratch;
1287
272k
   }
1288
1289
1.52M
   for (k=0;k<recombine;k++)
1290
184k
   {
1291
184k
      static const unsigned char bit_interleave_table[16]={
1292
184k
            0,1,1,1,2,3,3,3,2,3,3,3,2,3,3,3
1293
184k
      };
1294
184k
      if (encode)
1295
0
         haar1(X, N>>k, 1<<k);
1296
184k
      if (lowband)
1297
156k
         haar1(lowband, N>>k, 1<<k);
1298
184k
      fill = bit_interleave_table[fill&0xF]|bit_interleave_table[fill>>4]<<2;
1299
184k
   }
1300
1.34M
   B>>=recombine;
1301
1.34M
   N_B<<=recombine;
1302
1303
   /* Increasing the time resolution */
1304
1.74M
   while ((N_B&1) == 0 && tf_change<0)
1305
402k
   {
1306
402k
      if (encode)
1307
0
         haar1(X, N_B, B);
1308
402k
      if (lowband)
1309
310k
         haar1(lowband, N_B, B);
1310
402k
      fill |= fill<<B;
1311
402k
      B <<= 1;
1312
402k
      N_B >>= 1;
1313
402k
      time_divide++;
1314
402k
      tf_change++;
1315
402k
   }
1316
1.34M
   B0=B;
1317
1.34M
   N_B0 = N_B;
1318
1319
   /* Reorganize the samples in time order instead of frequency order */
1320
1.34M
   if (B0>1)
1321
254k
   {
1322
254k
      if (encode)
1323
0
         deinterleave_hadamard(X, N_B>>recombine, B0<<recombine, longBlocks);
1324
254k
      if (lowband)
1325
205k
         deinterleave_hadamard(lowband, N_B>>recombine, B0<<recombine, longBlocks);
1326
254k
   }
1327
1328
#ifdef ENABLE_QEXT
1329
   if (ctx->extra_bands && b > (3*N<<BITRES)+(ctx->m->logN[ctx->i]+8+8*LM)) {
1330
      cm = cubic_quant_partition(ctx, X, N, b, B, ctx->ec, LM, gain, ctx->resynth, encode);
1331
   } else
1332
#endif
1333
1.34M
   {
1334
1.34M
      cm = quant_partition(ctx, X, N, b, B, lowband, LM, gain, fill ARG_QEXT(ext_b));
1335
1.34M
   }
1336
1337
   /* This code is used by the decoder and by the resynthesis-enabled encoder */
1338
1.34M
   if (ctx->resynth)
1339
1.34M
   {
1340
      /* Undo the sample reorganization going from time order to frequency order */
1341
1.34M
      if (B0>1)
1342
254k
         interleave_hadamard(X, N_B>>recombine, B0<<recombine, longBlocks);
1343
1344
      /* Undo time-freq changes that we did earlier */
1345
1.34M
      N_B = N_B0;
1346
1.34M
      B = B0;
1347
1.74M
      for (k=0;k<time_divide;k++)
1348
402k
      {
1349
402k
         B >>= 1;
1350
402k
         N_B <<= 1;
1351
402k
         cm |= cm>>B;
1352
402k
         haar1(X, N_B, B);
1353
402k
      }
1354
1355
1.52M
      for (k=0;k<recombine;k++)
1356
184k
      {
1357
184k
         static const unsigned char bit_deinterleave_table[16]={
1358
184k
               0x00,0x03,0x0C,0x0F,0x30,0x33,0x3C,0x3F,
1359
184k
               0xC0,0xC3,0xCC,0xCF,0xF0,0xF3,0xFC,0xFF
1360
184k
         };
1361
184k
         cm = bit_deinterleave_table[cm];
1362
184k
         haar1(X, N0>>k, 1<<k);
1363
184k
      }
1364
1.34M
      B<<=recombine;
1365
1366
      /* Scale output for later folding */
1367
1.34M
      if (lowband_out)
1368
1.04M
      {
1369
1.04M
         int j;
1370
1.04M
         opus_val16 n;
1371
1.04M
         n = celt_sqrt(SHL32(EXTEND32(N0),22));
1372
11.2M
         for (j=0;j<N0;j++)
1373
10.1M
            lowband_out[j] = MULT16_32_Q15(n,X[j]);
1374
1.04M
      }
1375
1.34M
      cm &= (1<<B)-1;
1376
1.34M
   }
1377
1.34M
   return cm;
1378
1.50M
}
1379
1380
#ifdef FIXED_POINT
1381
#define MIN_STEREO_ENERGY 2
1382
#else
1383
0
#define MIN_STEREO_ENERGY 1e-10f
1384
#endif
1385
1386
/* This function is responsible for encoding and decoding a band for the stereo case. */
1387
static unsigned quant_band_stereo(struct band_ctx *ctx, celt_norm *X, celt_norm *Y,
1388
      int N, int b, int B, celt_norm *lowband,
1389
      int LM, celt_norm *lowband_out,
1390
      celt_norm *lowband_scratch, int fill
1391
      ARG_QEXT(int ext_b) ARG_QEXT(const int *cap))
1392
273k
{
1393
273k
   int imid=0, iside=0;
1394
273k
   int inv = 0;
1395
273k
   opus_val32 mid=0, side=0;
1396
273k
   unsigned cm=0;
1397
273k
   int mbits, sbits, delta;
1398
273k
   int itheta;
1399
273k
   int qalloc;
1400
273k
   struct split_ctx sctx;
1401
273k
   int orig_fill;
1402
273k
   int encode;
1403
273k
   ec_ctx *ec;
1404
1405
273k
   encode = ctx->encode;
1406
273k
   ec = ctx->ec;
1407
1408
   /* Special case for one sample */
1409
273k
   if (N==1)
1410
32.5k
   {
1411
32.5k
      return quant_band_n1(ctx, X, Y, lowband_out);
1412
32.5k
   }
1413
1414
240k
   orig_fill = fill;
1415
1416
240k
   if (encode) {
1417
0
      if (ctx->bandE[ctx->i] < MIN_STEREO_ENERGY || ctx->bandE[ctx->m->nbEBands+ctx->i] < MIN_STEREO_ENERGY) {
1418
0
         if (ctx->bandE[ctx->i] > ctx->bandE[ctx->m->nbEBands+ctx->i]) OPUS_COPY(Y, X, N);
1419
0
         else OPUS_COPY(X, Y, N);
1420
0
      }
1421
0
   }
1422
240k
   compute_theta(ctx, &sctx, X, Y, N, &b, B, B, LM, 1, &fill ARG_QEXT(&ext_b));
1423
240k
   inv = sctx.inv;
1424
240k
   imid = sctx.imid;
1425
240k
   iside = sctx.iside;
1426
240k
   delta = sctx.delta;
1427
240k
   itheta = sctx.itheta;
1428
240k
   qalloc = sctx.qalloc;
1429
#ifdef FIXED_POINT
1430
# ifdef ENABLE_QEXT
1431
   (void)imid;
1432
   (void)iside;
1433
   mid = celt_cos_norm32(sctx.itheta_q30);
1434
   side = celt_cos_norm32((1<<30)-sctx.itheta_q30);
1435
# else
1436
   mid = SHL32(EXTEND32(imid), 16);
1437
   side = SHL32(EXTEND32(iside), 16);
1438
# endif
1439
#else
1440
# ifdef ENABLE_QEXT
1441
   (void)imid;
1442
   (void)iside;
1443
   mid = celt_cos_norm2(sctx.itheta_q30*(1.f/(1<<30)));
1444
   side = celt_cos_norm2(1.f-sctx.itheta_q30*(1.f/(1<<30)));
1445
# else
1446
240k
   mid = (1.f/32768)*imid;
1447
240k
   side = (1.f/32768)*iside;
1448
240k
# endif
1449
240k
#endif
1450
1451
   /* This is a special case for N=2 that only works for stereo and takes
1452
      advantage of the fact that mid and side are orthogonal to encode
1453
      the side with just one bit. */
1454
240k
   if (N==2)
1455
58.1k
   {
1456
58.1k
      int c;
1457
58.1k
      int sign=0;
1458
58.1k
      celt_norm *x2, *y2;
1459
58.1k
      mbits = b;
1460
58.1k
      sbits = 0;
1461
      /* Only need one bit for the side. */
1462
58.1k
      if (itheta != 0 && itheta != 16384)
1463
16.2k
         sbits = 1<<BITRES;
1464
58.1k
      mbits -= sbits;
1465
58.1k
      c = itheta > 8192;
1466
58.1k
      ctx->remaining_bits -= qalloc+sbits;
1467
1468
58.1k
      x2 = c ? Y : X;
1469
58.1k
      y2 = c ? X : Y;
1470
58.1k
      if (sbits)
1471
16.2k
      {
1472
16.2k
         if (encode)
1473
0
         {
1474
            /* Here we only need to encode a sign for the side. */
1475
            /* FIXME: Need to increase fixed-point precision? */
1476
0
            sign = MULT32_32_Q31(x2[0],y2[1]) - MULT32_32_Q31(x2[1],y2[0]) < 0;
1477
0
            ec_enc_bits(ec, sign, 1);
1478
16.2k
         } else {
1479
16.2k
            sign = ec_dec_bits(ec, 1);
1480
16.2k
         }
1481
16.2k
      }
1482
58.1k
      sign = 1-2*sign;
1483
      /* We use orig_fill here because we want to fold the side, but if
1484
         itheta==16384, we'll have cleared the low bits of fill. */
1485
58.1k
      cm = quant_band(ctx, x2, N, mbits, B, lowband, LM, lowband_out, Q31ONE,
1486
58.1k
            lowband_scratch, orig_fill ARG_QEXT(ext_b));
1487
      /* We don't split N=2 bands, so cm is either 1 or 0 (for a fold-collapse),
1488
         and there's no need to worry about mixing with the other channel. */
1489
58.1k
      y2[0] = -sign*x2[1];
1490
58.1k
      y2[1] = sign*x2[0];
1491
58.1k
      if (ctx->resynth)
1492
58.1k
      {
1493
58.1k
         celt_norm tmp;
1494
58.1k
         X[0] = MULT32_32_Q31(mid, X[0]);
1495
58.1k
         X[1] = MULT32_32_Q31(mid, X[1]);
1496
58.1k
         Y[0] = MULT32_32_Q31(side, Y[0]);
1497
58.1k
         Y[1] = MULT32_32_Q31(side, Y[1]);
1498
58.1k
         tmp = X[0];
1499
58.1k
         X[0] = SUB32(tmp,Y[0]);
1500
58.1k
         Y[0] = ADD32(tmp,Y[0]);
1501
58.1k
         tmp = X[1];
1502
58.1k
         X[1] = SUB32(tmp,Y[1]);
1503
58.1k
         Y[1] = ADD32(tmp,Y[1]);
1504
58.1k
      }
1505
182k
   } else {
1506
      /* "Normal" split code */
1507
182k
      opus_int32 rebalance;
1508
1509
182k
      mbits = IMAX(0, IMIN(b, (b-delta)/2));
1510
182k
      sbits = b-mbits;
1511
182k
      ctx->remaining_bits -= qalloc;
1512
1513
182k
      rebalance = ctx->remaining_bits;
1514
182k
      if (mbits >= sbits)
1515
175k
      {
1516
#ifdef ENABLE_QEXT
1517
         int qext_extra = 0;
1518
         /* Reallocate any mid bits that cannot be used to extra mid bits. */
1519
         if (cap != NULL && ext_b != 0) qext_extra = IMAX(0, IMIN(ext_b/2, mbits - cap[ctx->i]/2));
1520
#endif
1521
         /* In stereo mode, we do not apply a scaling to the mid because we need the normalized
1522
            mid for folding later. */
1523
175k
         cm = quant_band(ctx, X, N, mbits, B, lowband, LM, lowband_out, Q31ONE,
1524
175k
               lowband_scratch, fill ARG_QEXT(ext_b/2+qext_extra));
1525
175k
         rebalance = mbits - (rebalance-ctx->remaining_bits);
1526
175k
         if (rebalance > 3<<BITRES && itheta!=0)
1527
1.16k
            sbits += rebalance - (3<<BITRES);
1528
#ifdef ENABLE_QEXT
1529
         /* Guard against overflowing the EC with the angle if the cubic quant used too many bits for the mid. */
1530
         if (ctx->extra_bands) sbits = IMIN(sbits, ctx->remaining_bits);
1531
#endif
1532
         /* For a stereo split, the high bits of fill are always zero, so no
1533
            folding will be done to the side. */
1534
175k
         cm |= quant_band(ctx, Y, N, sbits, B, NULL, LM, NULL, side, NULL, fill>>B ARG_QEXT(ext_b/2-qext_extra));
1535
175k
      } else {
1536
#ifdef ENABLE_QEXT
1537
         int qext_extra = 0;
1538
         /* Reallocate any side bits that cannot be used to extra side bits. */
1539
         if (cap != NULL && ext_b != 0) qext_extra = IMAX(0, IMIN(ext_b/2, sbits - cap[ctx->i]/2));
1540
#endif
1541
         /* For a stereo split, the high bits of fill are always zero, so no
1542
            folding will be done to the side. */
1543
6.98k
         cm = quant_band(ctx, Y, N, sbits, B, NULL, LM, NULL, side, NULL, fill>>B ARG_QEXT(ext_b/2+qext_extra));
1544
6.98k
         rebalance = sbits - (rebalance-ctx->remaining_bits);
1545
6.98k
         if (rebalance > 3<<BITRES && itheta!=16384)
1546
608
            mbits += rebalance - (3<<BITRES);
1547
#ifdef ENABLE_QEXT
1548
         /* Guard against overflowing the EC with the angle if the cubic quant used too many bits for the side. */
1549
         if (ctx->extra_bands) mbits = IMIN(mbits, ctx->remaining_bits);
1550
#endif
1551
         /* In stereo mode, we do not apply a scaling to the mid because we need the normalized
1552
            mid for folding later. */
1553
6.98k
         cm |= quant_band(ctx, X, N, mbits, B, lowband, LM, lowband_out, Q31ONE,
1554
6.98k
               lowband_scratch, fill ARG_QEXT(ext_b/2-qext_extra));
1555
6.98k
      }
1556
182k
   }
1557
1558
1559
   /* This code is used by the decoder and by the resynthesis-enabled encoder */
1560
240k
   if (ctx->resynth)
1561
240k
   {
1562
240k
      if (N!=2)
1563
182k
         stereo_merge(X, Y, mid, N, ctx->arch);
1564
240k
      if (inv)
1565
15.4k
      {
1566
15.4k
         int j;
1567
229k
         for (j=0;j<N;j++)
1568
213k
            Y[j] = -Y[j];
1569
15.4k
      }
1570
240k
   }
1571
240k
   return cm;
1572
273k
}
1573
1574
#ifndef DISABLE_UPDATE_DRAFT
1575
static void special_hybrid_folding(const CELTMode *m, celt_norm *norm, celt_norm *norm2, int start, int M, int dual_stereo)
1576
112k
{
1577
112k
   int n1, n2;
1578
112k
   const opus_int16 * OPUS_RESTRICT eBands = m->eBands;
1579
112k
   n1 = M*(eBands[start+1]-eBands[start]);
1580
112k
   n2 = M*(eBands[start+2]-eBands[start+1]);
1581
   /* Duplicate enough of the first band folding data to be able to fold the second band.
1582
      Copies no data for CELT-only mode. */
1583
112k
   OPUS_COPY(&norm[n1], &norm[2*n1 - n2], n2-n1);
1584
112k
   if (dual_stereo)
1585
5.21k
      OPUS_COPY(&norm2[n1], &norm2[2*n1 - n2], n2-n1);
1586
112k
}
1587
#endif
1588
1589
void quant_all_bands(int encode, const CELTMode *m, int start, int end,
1590
      celt_norm *X_, celt_norm *Y_, unsigned char *collapse_masks,
1591
      const celt_ener *bandE, int *pulses, int shortBlocks, int spread,
1592
      int dual_stereo, int intensity, int *tf_res, opus_int32 total_bits,
1593
      opus_int32 balance, ec_ctx *ec, int LM, int codedBands,
1594
      opus_uint32 *seed, int complexity, int arch, int disable_inv
1595
      ARG_QEXT(ec_ctx *ext_ec) ARG_QEXT(int *extra_pulses)
1596
      ARG_QEXT(opus_int32 ext_total_bits) ARG_QEXT(const int *cap))
1597
112k
{
1598
112k
   int i;
1599
112k
   opus_int32 remaining_bits;
1600
112k
   const opus_int16 * OPUS_RESTRICT eBands = m->eBands;
1601
112k
   celt_norm * OPUS_RESTRICT norm, * OPUS_RESTRICT norm2;
1602
112k
   VARDECL(celt_norm, _norm);
1603
112k
   VARDECL(celt_norm, _lowband_scratch);
1604
112k
   VARDECL(celt_norm, X_save);
1605
112k
   VARDECL(celt_norm, Y_save);
1606
112k
   VARDECL(celt_norm, X_save2);
1607
112k
   VARDECL(celt_norm, Y_save2);
1608
112k
   VARDECL(celt_norm, norm_save2);
1609
112k
   int resynth_alloc;
1610
112k
   celt_norm *lowband_scratch;
1611
112k
   int B;
1612
112k
   int M;
1613
112k
   int lowband_offset;
1614
112k
   int update_lowband = 1;
1615
112k
   int C = Y_ != NULL ? 2 : 1;
1616
112k
   int norm_offset;
1617
112k
   int theta_rdo = encode && Y_!=NULL && !dual_stereo && complexity>=8;
1618
#ifdef RESYNTH
1619
   int resynth = 1;
1620
#else
1621
112k
   int resynth = !encode || theta_rdo;
1622
112k
#endif
1623
112k
   struct band_ctx ctx;
1624
#ifdef ENABLE_QEXT
1625
   int ext_b;
1626
   opus_int32 ext_balance=0;
1627
   opus_int32 ext_tell=0;
1628
#endif
1629
112k
   SAVE_STACK;
1630
1631
112k
   M = 1<<LM;
1632
112k
   B = shortBlocks ? M : 1;
1633
112k
   norm_offset = M*eBands[start];
1634
   /* No need to allocate norm for the last band because we don't need an
1635
      output in that band. */
1636
112k
   ALLOC(_norm, C*(M*eBands[m->nbEBands-1]-norm_offset), celt_norm);
1637
112k
   norm = _norm;
1638
112k
   norm2 = norm + M*eBands[m->nbEBands-1]-norm_offset;
1639
1640
   /* For decoding, we can use the last band as scratch space because we don't need that
1641
      scratch space for the last band and we don't care about the data there until we're
1642
      decoding the last band. */
1643
112k
   if (encode && resynth)
1644
0
      resynth_alloc = M*(eBands[m->nbEBands]-eBands[m->nbEBands-1]);
1645
112k
   else
1646
112k
      resynth_alloc = ALLOC_NONE;
1647
112k
   ALLOC(_lowband_scratch, resynth_alloc, celt_norm);
1648
112k
   if (encode && resynth)
1649
0
      lowband_scratch = _lowband_scratch;
1650
112k
   else
1651
112k
      lowband_scratch = X_+M*eBands[m->effEBands-1];
1652
112k
   ALLOC(X_save, resynth_alloc, celt_norm);
1653
112k
   ALLOC(Y_save, resynth_alloc, celt_norm);
1654
112k
   ALLOC(X_save2, resynth_alloc, celt_norm);
1655
112k
   ALLOC(Y_save2, resynth_alloc, celt_norm);
1656
112k
   ALLOC(norm_save2, resynth_alloc, celt_norm);
1657
1658
112k
   lowband_offset = 0;
1659
112k
   ctx.bandE = bandE;
1660
112k
   ctx.ec = ec;
1661
112k
   ctx.encode = encode;
1662
112k
   ctx.intensity = intensity;
1663
112k
   ctx.m = m;
1664
112k
   ctx.seed = *seed;
1665
112k
   ctx.spread = spread;
1666
112k
   ctx.arch = arch;
1667
112k
   ctx.disable_inv = disable_inv;
1668
112k
   ctx.resynth = resynth;
1669
112k
   ctx.theta_round = 0;
1670
#ifdef ENABLE_QEXT
1671
   ctx.ext_ec = ext_ec;
1672
   ctx.ext_total_bits = ext_total_bits;
1673
   ctx.extra_bands = end == NB_QEXT_BANDS || end == 2;
1674
   if (ctx.extra_bands) theta_rdo = 0;
1675
#endif
1676
   /* Avoid injecting noise in the first band on transients. */
1677
112k
   ctx.avoid_split_noise = B > 1;
1678
1.42M
   for (i=start;i<end;i++)
1679
1.30M
   {
1680
1.30M
      opus_int32 tell;
1681
1.30M
      int b;
1682
1.30M
      int N;
1683
1.30M
      opus_int32 curr_balance;
1684
1.30M
      int effective_lowband=-1;
1685
1.30M
      celt_norm * OPUS_RESTRICT X, * OPUS_RESTRICT Y;
1686
1.30M
      int tf_change=0;
1687
1.30M
      unsigned x_cm;
1688
1.30M
      unsigned y_cm;
1689
1.30M
      int last;
1690
1691
1.30M
      ctx.i = i;
1692
1.30M
      last = (i==end-1);
1693
1694
1.30M
      X = X_+M*eBands[i];
1695
1.30M
      if (Y_!=NULL)
1696
314k
         Y = Y_+M*eBands[i];
1697
995k
      else
1698
995k
         Y = NULL;
1699
1.30M
      N = M*eBands[i+1]-M*eBands[i];
1700
1.30M
      celt_assert(N > 0);
1701
1.30M
      tell = ec_tell_frac(ec);
1702
1703
      /* Compute how many bits we want to allocate to this band */
1704
1.30M
      if (i != start)
1705
1.19M
         balance -= tell;
1706
1.30M
      remaining_bits = total_bits-tell-1;
1707
1.30M
      ctx.remaining_bits = remaining_bits;
1708
#ifdef ENABLE_QEXT
1709
      if (i != start) {
1710
         ext_balance += extra_pulses[i-1] + ext_tell;
1711
      }
1712
      ext_tell = ec_tell_frac(ext_ec);
1713
      ctx.extra_bits = extra_pulses[i];
1714
      if (i != start)
1715
         ext_balance -= ext_tell;
1716
      if (i <= codedBands-1)
1717
      {
1718
         opus_int32 ext_curr_balance = celt_sudiv(ext_balance, IMIN(3, codedBands-i));
1719
         ext_b = IMAX(0, IMIN(16383, IMIN(ext_total_bits-ext_tell,extra_pulses[i]+ext_curr_balance)));
1720
      } else {
1721
         ext_b = 0;
1722
      }
1723
#endif
1724
1.30M
      if (i <= codedBands-1)
1725
637k
      {
1726
637k
         curr_balance = celt_sudiv(balance, IMIN(3, codedBands-i));
1727
637k
         b = IMAX(0, IMIN(16383, IMIN(remaining_bits+1,pulses[i]+curr_balance)));
1728
672k
      } else {
1729
672k
         b = 0;
1730
672k
      }
1731
1732
1.30M
#ifndef DISABLE_UPDATE_DRAFT
1733
1.30M
      if (resynth && (M*eBands[i]-N >= M*eBands[start] || i==start+1) && (update_lowband || lowband_offset==0))
1734
519k
            lowband_offset = i;
1735
1.30M
      if (i == start+1)
1736
112k
         special_hybrid_folding(m, norm, norm2, start, M, dual_stereo);
1737
#else
1738
      if (resynth && M*eBands[i]-N >= M*eBands[start] && (update_lowband || lowband_offset==0))
1739
            lowband_offset = i;
1740
#endif
1741
1742
1.30M
      tf_change = tf_res[i];
1743
1.30M
      ctx.tf_change = tf_change;
1744
1.30M
      if (i>=m->effEBands)
1745
0
      {
1746
0
         X=norm;
1747
0
         if (Y_!=NULL)
1748
0
            Y = norm;
1749
0
         lowband_scratch = NULL;
1750
0
      }
1751
1.30M
      if (last && !theta_rdo)
1752
112k
         lowband_scratch = NULL;
1753
1754
      /* Get a conservative estimate of the collapse_mask's for the bands we're
1755
         going to be folding from. */
1756
1.30M
      if (lowband_offset != 0 && (spread!=SPREAD_AGGRESSIVE || B>1 || tf_change<0))
1757
1.16M
      {
1758
1.16M
         int fold_start;
1759
1.16M
         int fold_end;
1760
1.16M
         int fold_i;
1761
         /* This ensures we never repeat spectral content within one band */
1762
1.16M
         effective_lowband = IMAX(0, M*eBands[lowband_offset]-norm_offset-N);
1763
1.16M
         fold_start = lowband_offset;
1764
1.48M
         while(M*eBands[--fold_start] > effective_lowband+norm_offset);
1765
1.16M
         fold_end = lowband_offset-1;
1766
1.16M
#ifndef DISABLE_UPDATE_DRAFT
1767
1.93M
         while(++fold_end < i && M*eBands[fold_end] < effective_lowband+norm_offset+N);
1768
#else
1769
         while(M*eBands[++fold_end] < effective_lowband+norm_offset+N);
1770
#endif
1771
1.16M
         x_cm = y_cm = 0;
1772
2.25M
         fold_i = fold_start; do {
1773
2.25M
           x_cm |= collapse_masks[fold_i*C+0];
1774
2.25M
           y_cm |= collapse_masks[fold_i*C+C-1];
1775
2.25M
         } while (++fold_i<fold_end);
1776
1.16M
      }
1777
      /* Otherwise, we'll be using the LCG to fold, so all blocks will (almost
1778
         always) be non-zero. */
1779
141k
      else
1780
141k
         x_cm = y_cm = (1<<B)-1;
1781
1782
1.30M
      if (dual_stereo && i==intensity)
1783
4.60k
      {
1784
4.60k
         int j;
1785
1786
         /* Switch off dual stereo to do intensity. */
1787
4.60k
         dual_stereo = 0;
1788
4.60k
         if (resynth)
1789
183k
            for (j=0;j<M*eBands[i]-norm_offset;j++)
1790
179k
               norm[j] = HALF32(norm[j]+norm2[j]);
1791
4.60k
      }
1792
1.30M
      if (dual_stereo)
1793
40.9k
      {
1794
40.9k
         x_cm = quant_band(&ctx, X, N, b/2, B,
1795
40.9k
               effective_lowband != -1 ? norm+effective_lowband : NULL, LM,
1796
40.9k
               last?NULL:norm+M*eBands[i]-norm_offset, Q31ONE, lowband_scratch, x_cm ARG_QEXT(ext_b/2));
1797
40.9k
         y_cm = quant_band(&ctx, Y, N, b/2, B,
1798
40.9k
               effective_lowband != -1 ? norm2+effective_lowband : NULL, LM,
1799
40.9k
               last?NULL:norm2+M*eBands[i]-norm_offset, Q31ONE, lowband_scratch, y_cm ARG_QEXT(ext_b/2));
1800
1.26M
      } else {
1801
1.26M
         if (Y!=NULL)
1802
273k
         {
1803
273k
            if (theta_rdo && i < intensity)
1804
0
            {
1805
0
               ec_ctx ec_save, ec_save2;
1806
0
               struct band_ctx ctx_save, ctx_save2;
1807
0
               opus_val32 dist0, dist1;
1808
0
               unsigned cm, cm2;
1809
0
               int nstart_bytes, nend_bytes, save_bytes;
1810
0
               unsigned char *bytes_buf;
1811
0
               unsigned char bytes_save[1275];
1812
#ifdef ENABLE_QEXT
1813
               ec_ctx ext_ec_save, ext_ec_save2;
1814
               unsigned char *ext_bytes_buf;
1815
               int ext_nstart_bytes, ext_nend_bytes, ext_save_bytes;
1816
               unsigned char ext_bytes_save[QEXT_PACKET_SIZE_CAP];
1817
#endif
1818
0
               opus_val16 w[2];
1819
0
               compute_channel_weights(bandE[i], bandE[i+m->nbEBands], w);
1820
               /* Make a copy. */
1821
0
               cm = x_cm|y_cm;
1822
0
               ec_save = *ec;
1823
#ifdef ENABLE_QEXT
1824
               ext_ec_save = *ext_ec;
1825
#endif
1826
0
               ctx_save = ctx;
1827
0
               OPUS_COPY(X_save, X, N);
1828
0
               OPUS_COPY(Y_save, Y, N);
1829
               /* Encode and round down. */
1830
0
               ctx.theta_round = -1;
1831
0
               x_cm = quant_band_stereo(&ctx, X, Y, N, b, B,
1832
0
                     effective_lowband != -1 ? norm+effective_lowband : NULL, LM,
1833
0
                     last?NULL:norm+M*eBands[i]-norm_offset, lowband_scratch, cm ARG_QEXT(ext_b) ARG_QEXT(cap));
1834
0
               dist0 = MULT16_32_Q15(w[0], celt_inner_prod_norm_shift(X_save, X, N, arch)) + MULT16_32_Q15(w[1], celt_inner_prod_norm_shift(Y_save, Y, N, arch));
1835
1836
               /* Save first result. */
1837
0
               cm2 = x_cm;
1838
0
               ec_save2 = *ec;
1839
#ifdef ENABLE_QEXT
1840
               ext_ec_save2 = *ext_ec;
1841
#endif
1842
0
               ctx_save2 = ctx;
1843
0
               OPUS_COPY(X_save2, X, N);
1844
0
               OPUS_COPY(Y_save2, Y, N);
1845
0
               if (!last)
1846
0
                  OPUS_COPY(norm_save2, norm+M*eBands[i]-norm_offset, N);
1847
0
               nstart_bytes = ec_save.offs;
1848
0
               nend_bytes = ec_save.storage;
1849
0
               bytes_buf = ec_save.buf+nstart_bytes;
1850
0
               save_bytes = nend_bytes-nstart_bytes;
1851
0
               OPUS_COPY(bytes_save, bytes_buf, save_bytes);
1852
#ifdef ENABLE_QEXT
1853
               ext_nstart_bytes = ext_ec_save.offs;
1854
               ext_nend_bytes = ext_ec_save.storage;
1855
               ext_bytes_buf = ext_ec_save.buf!=NULL ? ext_ec_save.buf+ext_nstart_bytes : NULL;
1856
               ext_save_bytes = ext_nend_bytes-ext_nstart_bytes;
1857
               if (ext_save_bytes) OPUS_COPY(ext_bytes_save, ext_bytes_buf, ext_save_bytes);
1858
#endif
1859
               /* Restore */
1860
0
               *ec = ec_save;
1861
#ifdef ENABLE_QEXT
1862
               *ext_ec = ext_ec_save;
1863
#endif
1864
0
               ctx = ctx_save;
1865
0
               OPUS_COPY(X, X_save, N);
1866
0
               OPUS_COPY(Y, Y_save, N);
1867
0
#ifndef DISABLE_UPDATE_DRAFT
1868
0
               if (i == start+1)
1869
0
                  special_hybrid_folding(m, norm, norm2, start, M, dual_stereo);
1870
0
#endif
1871
               /* Encode and round up. */
1872
0
               ctx.theta_round = 1;
1873
0
               x_cm = quant_band_stereo(&ctx, X, Y, N, b, B,
1874
0
                     effective_lowband != -1 ? norm+effective_lowband : NULL, LM,
1875
0
                     last?NULL:norm+M*eBands[i]-norm_offset, lowband_scratch, cm ARG_QEXT(ext_b) ARG_QEXT(cap));
1876
0
               dist1 = MULT16_32_Q15(w[0], celt_inner_prod_norm_shift(X_save, X, N, arch)) + MULT16_32_Q15(w[1], celt_inner_prod_norm_shift(Y_save, Y, N, arch));
1877
0
               if (dist0 >= dist1) {
1878
0
                  x_cm = cm2;
1879
0
                  *ec = ec_save2;
1880
#ifdef ENABLE_QEXT
1881
                  *ext_ec = ext_ec_save2;
1882
#endif
1883
0
                  ctx = ctx_save2;
1884
0
                  OPUS_COPY(X, X_save2, N);
1885
0
                  OPUS_COPY(Y, Y_save2, N);
1886
0
                  if (!last)
1887
0
                     OPUS_COPY(norm+M*eBands[i]-norm_offset, norm_save2, N);
1888
0
                  OPUS_COPY(bytes_buf, bytes_save, save_bytes);
1889
#ifdef ENABLE_QEXT
1890
                  if (ext_save_bytes) OPUS_COPY(ext_bytes_buf, ext_bytes_save, ext_save_bytes);
1891
#endif
1892
0
               }
1893
273k
            } else {
1894
273k
               ctx.theta_round = 0;
1895
273k
               x_cm = quant_band_stereo(&ctx, X, Y, N, b, B,
1896
273k
                     effective_lowband != -1 ? norm+effective_lowband : NULL, LM,
1897
273k
                     last?NULL:norm+M*eBands[i]-norm_offset, lowband_scratch, x_cm|y_cm ARG_QEXT(ext_b) ARG_QEXT(cap));
1898
273k
            }
1899
995k
         } else {
1900
995k
            x_cm = quant_band(&ctx, X, N, b, B,
1901
995k
                  effective_lowband != -1 ? norm+effective_lowband : NULL, LM,
1902
995k
                  last?NULL:norm+M*eBands[i]-norm_offset, Q31ONE, lowband_scratch, x_cm|y_cm ARG_QEXT(ext_b));
1903
995k
         }
1904
1.26M
         y_cm = x_cm;
1905
1.26M
      }
1906
1.30M
      collapse_masks[i*C+0] = (unsigned char)x_cm;
1907
1.30M
      collapse_masks[i*C+C-1] = (unsigned char)y_cm;
1908
1.30M
      balance += pulses[i] + tell;
1909
1910
      /* Update the folding position only as long as we have 1 bit/sample depth. */
1911
1.30M
      update_lowband = b>(N<<BITRES);
1912
      /* We only need to avoid noise on a split for the first band. After that, we
1913
         have folding. */
1914
1.30M
      ctx.avoid_split_noise = 0;
1915
1.30M
   }
1916
112k
   *seed = ctx.seed;
1917
1918
112k
   RESTORE_STACK;
1919
112k
}