Coverage Report

Created: 2022-08-24 06:11

/src/aom/av1/common/convolve.c
Line
Count
Source (jump to first uncovered line)
1
/*
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
3
 *
4
 * This source code is subject to the terms of the BSD 2 Clause License and
5
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6
 * was not distributed with this source code in the LICENSE file, you can
7
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8
 * Media Patent License 1.0 was not distributed with this source code in the
9
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10
 */
11
12
#include <assert.h>
13
#include <string.h>
14
15
#include "config/aom_dsp_rtcd.h"
16
#include "config/av1_rtcd.h"
17
18
#include "av1/common/av1_common_int.h"
19
#include "av1/common/blockd.h"
20
#include "av1/common/convolve.h"
21
#include "av1/common/filter.h"
22
#include "av1/common/resize.h"
23
#include "aom_dsp/aom_dsp_common.h"
24
#include "aom_ports/mem.h"
25
26
void av1_convolve_horiz_rs_c(const uint8_t *src, int src_stride, uint8_t *dst,
27
                             int dst_stride, int w, int h,
28
                             const int16_t *x_filters, int x0_qn,
29
0
                             int x_step_qn) {
30
0
  src -= UPSCALE_NORMATIVE_TAPS / 2 - 1;
31
0
  for (int y = 0; y < h; ++y) {
32
0
    int x_qn = x0_qn;
33
0
    for (int x = 0; x < w; ++x) {
34
0
      const uint8_t *const src_x = &src[x_qn >> RS_SCALE_SUBPEL_BITS];
35
0
      const int x_filter_idx =
36
0
          (x_qn & RS_SCALE_SUBPEL_MASK) >> RS_SCALE_EXTRA_BITS;
37
0
      assert(x_filter_idx <= RS_SUBPEL_MASK);
38
0
      const int16_t *const x_filter =
39
0
          &x_filters[x_filter_idx * UPSCALE_NORMATIVE_TAPS];
40
0
      int sum = 0;
41
0
      for (int k = 0; k < UPSCALE_NORMATIVE_TAPS; ++k)
42
0
        sum += src_x[k] * x_filter[k];
43
0
      dst[x] = clip_pixel(ROUND_POWER_OF_TWO(sum, FILTER_BITS));
44
0
      x_qn += x_step_qn;
45
0
    }
46
0
    src += src_stride;
47
0
    dst += dst_stride;
48
0
  }
49
0
}
50
51
void av1_highbd_convolve_horiz_rs_c(const uint16_t *src, int src_stride,
52
                                    uint16_t *dst, int dst_stride, int w, int h,
53
                                    const int16_t *x_filters, int x0_qn,
54
0
                                    int x_step_qn, int bd) {
55
0
  src -= UPSCALE_NORMATIVE_TAPS / 2 - 1;
56
0
  for (int y = 0; y < h; ++y) {
57
0
    int x_qn = x0_qn;
58
0
    for (int x = 0; x < w; ++x) {
59
0
      const uint16_t *const src_x = &src[x_qn >> RS_SCALE_SUBPEL_BITS];
60
0
      const int x_filter_idx =
61
0
          (x_qn & RS_SCALE_SUBPEL_MASK) >> RS_SCALE_EXTRA_BITS;
62
0
      assert(x_filter_idx <= RS_SUBPEL_MASK);
63
0
      const int16_t *const x_filter =
64
0
          &x_filters[x_filter_idx * UPSCALE_NORMATIVE_TAPS];
65
0
      int sum = 0;
66
0
      for (int k = 0; k < UPSCALE_NORMATIVE_TAPS; ++k)
67
0
        sum += src_x[k] * x_filter[k];
68
0
      dst[x] = clip_pixel_highbd(ROUND_POWER_OF_TWO(sum, FILTER_BITS), bd);
69
0
      x_qn += x_step_qn;
70
0
    }
71
0
    src += src_stride;
72
0
    dst += dst_stride;
73
0
  }
74
0
}
75
76
void av1_convolve_2d_sobel_y_c(const uint8_t *src, int src_stride, double *dst,
77
                               int dst_stride, int w, int h, int dir,
78
0
                               double norm) {
79
0
  int16_t im_block[(MAX_SB_SIZE + MAX_FILTER_TAP - 1) * MAX_SB_SIZE];
80
0
  DECLARE_ALIGNED(256, static const int16_t, sobel_a[3]) = { 1, 0, -1 };
81
0
  DECLARE_ALIGNED(256, static const int16_t, sobel_b[3]) = { 1, 2, 1 };
82
0
  const int taps = 3;
83
0
  int im_h = h + taps - 1;
84
0
  int im_stride = w;
85
0
  const int fo_vert = 1;
86
0
  const int fo_horiz = 1;
87
88
  // horizontal filter
89
0
  const uint8_t *src_horiz = src - fo_vert * src_stride;
90
0
  const int16_t *x_filter = dir ? sobel_a : sobel_b;
91
0
  for (int y = 0; y < im_h; ++y) {
92
0
    for (int x = 0; x < w; ++x) {
93
0
      int16_t sum = 0;
94
0
      for (int k = 0; k < taps; ++k) {
95
0
        sum += x_filter[k] * src_horiz[y * src_stride + x - fo_horiz + k];
96
0
      }
97
0
      im_block[y * im_stride + x] = sum;
98
0
    }
99
0
  }
100
101
  // vertical filter
102
0
  int16_t *src_vert = im_block + fo_vert * im_stride;
103
0
  const int16_t *y_filter = dir ? sobel_b : sobel_a;
104
0
  for (int y = 0; y < h; ++y) {
105
0
    for (int x = 0; x < w; ++x) {
106
0
      int16_t sum = 0;
107
0
      for (int k = 0; k < taps; ++k) {
108
0
        sum += y_filter[k] * src_vert[(y - fo_vert + k) * im_stride + x];
109
0
      }
110
0
      dst[y * dst_stride + x] = sum * norm;
111
0
    }
112
0
  }
113
0
}
114
115
void av1_convolve_2d_sr_c(const uint8_t *src, int src_stride, uint8_t *dst,
116
                          int dst_stride, int w, int h,
117
                          const InterpFilterParams *filter_params_x,
118
                          const InterpFilterParams *filter_params_y,
119
                          const int subpel_x_qn, const int subpel_y_qn,
120
0
                          ConvolveParams *conv_params) {
121
0
  int16_t im_block[(MAX_SB_SIZE + MAX_FILTER_TAP - 1) * MAX_SB_SIZE];
122
0
  int im_h = h + filter_params_y->taps - 1;
123
0
  int im_stride = w;
124
0
  assert(w <= MAX_SB_SIZE && h <= MAX_SB_SIZE);
125
0
  const int fo_vert = filter_params_y->taps / 2 - 1;
126
0
  const int fo_horiz = filter_params_x->taps / 2 - 1;
127
0
  const int bd = 8;
128
0
  const int bits =
129
0
      FILTER_BITS * 2 - conv_params->round_0 - conv_params->round_1;
130
131
  // horizontal filter
132
0
  const uint8_t *src_horiz = src - fo_vert * src_stride;
133
0
  const int16_t *x_filter = av1_get_interp_filter_subpel_kernel(
134
0
      filter_params_x, subpel_x_qn & SUBPEL_MASK);
135
0
  for (int y = 0; y < im_h; ++y) {
136
0
    for (int x = 0; x < w; ++x) {
137
0
      int32_t sum = (1 << (bd + FILTER_BITS - 1));
138
0
      for (int k = 0; k < filter_params_x->taps; ++k) {
139
0
        sum += x_filter[k] * src_horiz[y * src_stride + x - fo_horiz + k];
140
0
      }
141
0
      assert(0 <= sum && sum < (1 << (bd + FILTER_BITS + 1)));
142
0
      im_block[y * im_stride + x] =
143
0
          (int16_t)ROUND_POWER_OF_TWO(sum, conv_params->round_0);
144
0
    }
145
0
  }
146
147
  // vertical filter
148
0
  int16_t *src_vert = im_block + fo_vert * im_stride;
149
0
  const int16_t *y_filter = av1_get_interp_filter_subpel_kernel(
150
0
      filter_params_y, subpel_y_qn & SUBPEL_MASK);
151
0
  const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0;
152
0
  for (int y = 0; y < h; ++y) {
153
0
    for (int x = 0; x < w; ++x) {
154
0
      int32_t sum = 1 << offset_bits;
155
0
      for (int k = 0; k < filter_params_y->taps; ++k) {
156
0
        sum += y_filter[k] * src_vert[(y - fo_vert + k) * im_stride + x];
157
0
      }
158
0
      assert(0 <= sum && sum < (1 << (offset_bits + 2)));
159
0
      int16_t res = ROUND_POWER_OF_TWO(sum, conv_params->round_1) -
160
0
                    ((1 << (offset_bits - conv_params->round_1)) +
161
0
                     (1 << (offset_bits - conv_params->round_1 - 1)));
162
0
      dst[y * dst_stride + x] = clip_pixel(ROUND_POWER_OF_TWO(res, bits));
163
0
    }
164
0
  }
165
0
}
166
167
void av1_convolve_y_sr_c(const uint8_t *src, int src_stride, uint8_t *dst,
168
                         int dst_stride, int w, int h,
169
                         const InterpFilterParams *filter_params_y,
170
0
                         const int subpel_y_qn) {
171
0
  const int fo_vert = filter_params_y->taps / 2 - 1;
172
173
  // vertical filter
174
0
  const int16_t *y_filter = av1_get_interp_filter_subpel_kernel(
175
0
      filter_params_y, subpel_y_qn & SUBPEL_MASK);
176
0
  for (int y = 0; y < h; ++y) {
177
0
    for (int x = 0; x < w; ++x) {
178
0
      int32_t res = 0;
179
0
      for (int k = 0; k < filter_params_y->taps; ++k) {
180
0
        res += y_filter[k] * src[(y - fo_vert + k) * src_stride + x];
181
0
      }
182
0
      dst[y * dst_stride + x] =
183
0
          clip_pixel(ROUND_POWER_OF_TWO(res, FILTER_BITS));
184
0
    }
185
0
  }
186
0
}
187
188
void av1_convolve_x_sr_c(const uint8_t *src, int src_stride, uint8_t *dst,
189
                         int dst_stride, int w, int h,
190
                         const InterpFilterParams *filter_params_x,
191
0
                         const int subpel_x_qn, ConvolveParams *conv_params) {
192
0
  const int fo_horiz = filter_params_x->taps / 2 - 1;
193
0
  const int bits = FILTER_BITS - conv_params->round_0;
194
195
0
  assert(bits >= 0);
196
0
  assert((FILTER_BITS - conv_params->round_1) >= 0 ||
197
0
         ((conv_params->round_0 + conv_params->round_1) == 2 * FILTER_BITS));
198
199
  // horizontal filter
200
0
  const int16_t *x_filter = av1_get_interp_filter_subpel_kernel(
201
0
      filter_params_x, subpel_x_qn & SUBPEL_MASK);
202
203
0
  for (int y = 0; y < h; ++y) {
204
0
    for (int x = 0; x < w; ++x) {
205
0
      int32_t res = 0;
206
0
      for (int k = 0; k < filter_params_x->taps; ++k) {
207
0
        res += x_filter[k] * src[y * src_stride + x - fo_horiz + k];
208
0
      }
209
0
      res = ROUND_POWER_OF_TWO(res, conv_params->round_0);
210
0
      dst[y * dst_stride + x] = clip_pixel(ROUND_POWER_OF_TWO(res, bits));
211
0
    }
212
0
  }
213
0
}
214
215
void av1_dist_wtd_convolve_2d_c(const uint8_t *src, int src_stride,
216
                                uint8_t *dst, int dst_stride, int w, int h,
217
                                const InterpFilterParams *filter_params_x,
218
                                const InterpFilterParams *filter_params_y,
219
                                const int subpel_x_qn, const int subpel_y_qn,
220
0
                                ConvolveParams *conv_params) {
221
0
  CONV_BUF_TYPE *dst16 = conv_params->dst;
222
0
  int dst16_stride = conv_params->dst_stride;
223
0
  int16_t im_block[(MAX_SB_SIZE + MAX_FILTER_TAP - 1) * MAX_SB_SIZE];
224
0
  int im_h = h + filter_params_y->taps - 1;
225
0
  int im_stride = w;
226
0
  const int fo_vert = filter_params_y->taps / 2 - 1;
227
0
  const int fo_horiz = filter_params_x->taps / 2 - 1;
228
0
  const int bd = 8;
229
0
  const int round_bits =
230
0
      2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1;
231
232
  // horizontal filter
233
0
  const uint8_t *src_horiz = src - fo_vert * src_stride;
234
0
  const int16_t *x_filter = av1_get_interp_filter_subpel_kernel(
235
0
      filter_params_x, subpel_x_qn & SUBPEL_MASK);
236
0
  for (int y = 0; y < im_h; ++y) {
237
0
    for (int x = 0; x < w; ++x) {
238
0
      int32_t sum = (1 << (bd + FILTER_BITS - 1));
239
0
      for (int k = 0; k < filter_params_x->taps; ++k) {
240
0
        sum += x_filter[k] * src_horiz[y * src_stride + x - fo_horiz + k];
241
0
      }
242
0
      assert(0 <= sum && sum < (1 << (bd + FILTER_BITS + 1)));
243
0
      im_block[y * im_stride + x] =
244
0
          (int16_t)ROUND_POWER_OF_TWO(sum, conv_params->round_0);
245
0
    }
246
0
  }
247
248
  // vertical filter
249
0
  int16_t *src_vert = im_block + fo_vert * im_stride;
250
0
  const int16_t *y_filter = av1_get_interp_filter_subpel_kernel(
251
0
      filter_params_y, subpel_y_qn & SUBPEL_MASK);
252
0
  const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0;
253
0
  for (int y = 0; y < h; ++y) {
254
0
    for (int x = 0; x < w; ++x) {
255
0
      int32_t sum = 1 << offset_bits;
256
0
      for (int k = 0; k < filter_params_y->taps; ++k) {
257
0
        sum += y_filter[k] * src_vert[(y - fo_vert + k) * im_stride + x];
258
0
      }
259
0
      assert(0 <= sum && sum < (1 << (offset_bits + 2)));
260
0
      CONV_BUF_TYPE res = ROUND_POWER_OF_TWO(sum, conv_params->round_1);
261
0
      if (conv_params->do_average) {
262
0
        int32_t tmp = dst16[y * dst16_stride + x];
263
0
        if (conv_params->use_dist_wtd_comp_avg) {
264
0
          tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset;
265
0
          tmp = tmp >> DIST_PRECISION_BITS;
266
0
        } else {
267
0
          tmp += res;
268
0
          tmp = tmp >> 1;
269
0
        }
270
0
        tmp -= (1 << (offset_bits - conv_params->round_1)) +
271
0
               (1 << (offset_bits - conv_params->round_1 - 1));
272
0
        dst[y * dst_stride + x] =
273
0
            clip_pixel(ROUND_POWER_OF_TWO(tmp, round_bits));
274
0
      } else {
275
0
        dst16[y * dst16_stride + x] = res;
276
0
      }
277
0
    }
278
0
  }
279
0
}
280
281
void av1_dist_wtd_convolve_y_c(const uint8_t *src, int src_stride, uint8_t *dst,
282
                               int dst_stride, int w, int h,
283
                               const InterpFilterParams *filter_params_y,
284
                               const int subpel_y_qn,
285
0
                               ConvolveParams *conv_params) {
286
0
  CONV_BUF_TYPE *dst16 = conv_params->dst;
287
0
  int dst16_stride = conv_params->dst_stride;
288
0
  const int fo_vert = filter_params_y->taps / 2 - 1;
289
0
  const int bits = FILTER_BITS - conv_params->round_0;
290
0
  const int bd = 8;
291
0
  const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0;
292
0
  const int round_offset = (1 << (offset_bits - conv_params->round_1)) +
293
0
                           (1 << (offset_bits - conv_params->round_1 - 1));
294
0
  const int round_bits =
295
0
      2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1;
296
297
  // vertical filter
298
0
  const int16_t *y_filter = av1_get_interp_filter_subpel_kernel(
299
0
      filter_params_y, subpel_y_qn & SUBPEL_MASK);
300
0
  for (int y = 0; y < h; ++y) {
301
0
    for (int x = 0; x < w; ++x) {
302
0
      int32_t res = 0;
303
0
      for (int k = 0; k < filter_params_y->taps; ++k) {
304
0
        res += y_filter[k] * src[(y - fo_vert + k) * src_stride + x];
305
0
      }
306
0
      res *= (1 << bits);
307
0
      res = ROUND_POWER_OF_TWO(res, conv_params->round_1) + round_offset;
308
309
0
      if (conv_params->do_average) {
310
0
        int32_t tmp = dst16[y * dst16_stride + x];
311
0
        if (conv_params->use_dist_wtd_comp_avg) {
312
0
          tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset;
313
0
          tmp = tmp >> DIST_PRECISION_BITS;
314
0
        } else {
315
0
          tmp += res;
316
0
          tmp = tmp >> 1;
317
0
        }
318
0
        tmp -= round_offset;
319
0
        dst[y * dst_stride + x] =
320
0
            clip_pixel(ROUND_POWER_OF_TWO(tmp, round_bits));
321
0
      } else {
322
0
        dst16[y * dst16_stride + x] = res;
323
0
      }
324
0
    }
325
0
  }
326
0
}
327
328
void av1_dist_wtd_convolve_x_c(const uint8_t *src, int src_stride, uint8_t *dst,
329
                               int dst_stride, int w, int h,
330
                               const InterpFilterParams *filter_params_x,
331
                               const int subpel_x_qn,
332
0
                               ConvolveParams *conv_params) {
333
0
  CONV_BUF_TYPE *dst16 = conv_params->dst;
334
0
  int dst16_stride = conv_params->dst_stride;
335
0
  const int fo_horiz = filter_params_x->taps / 2 - 1;
336
0
  const int bits = FILTER_BITS - conv_params->round_1;
337
0
  const int bd = 8;
338
0
  const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0;
339
0
  const int round_offset = (1 << (offset_bits - conv_params->round_1)) +
340
0
                           (1 << (offset_bits - conv_params->round_1 - 1));
341
0
  const int round_bits =
342
0
      2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1;
343
344
  // horizontal filter
345
0
  const int16_t *x_filter = av1_get_interp_filter_subpel_kernel(
346
0
      filter_params_x, subpel_x_qn & SUBPEL_MASK);
347
0
  for (int y = 0; y < h; ++y) {
348
0
    for (int x = 0; x < w; ++x) {
349
0
      int32_t res = 0;
350
0
      for (int k = 0; k < filter_params_x->taps; ++k) {
351
0
        res += x_filter[k] * src[y * src_stride + x - fo_horiz + k];
352
0
      }
353
0
      res = (1 << bits) * ROUND_POWER_OF_TWO(res, conv_params->round_0);
354
0
      res += round_offset;
355
356
0
      if (conv_params->do_average) {
357
0
        int32_t tmp = dst16[y * dst16_stride + x];
358
0
        if (conv_params->use_dist_wtd_comp_avg) {
359
0
          tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset;
360
0
          tmp = tmp >> DIST_PRECISION_BITS;
361
0
        } else {
362
0
          tmp += res;
363
0
          tmp = tmp >> 1;
364
0
        }
365
0
        tmp -= round_offset;
366
0
        dst[y * dst_stride + x] =
367
0
            clip_pixel(ROUND_POWER_OF_TWO(tmp, round_bits));
368
0
      } else {
369
0
        dst16[y * dst16_stride + x] = res;
370
0
      }
371
0
    }
372
0
  }
373
0
}
374
375
void av1_dist_wtd_convolve_2d_copy_c(const uint8_t *src, int src_stride,
376
                                     uint8_t *dst, int dst_stride, int w, int h,
377
0
                                     ConvolveParams *conv_params) {
378
0
  CONV_BUF_TYPE *dst16 = conv_params->dst;
379
0
  int dst16_stride = conv_params->dst_stride;
380
0
  const int bits =
381
0
      FILTER_BITS * 2 - conv_params->round_1 - conv_params->round_0;
382
0
  const int bd = 8;
383
0
  const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0;
384
0
  const int round_offset = (1 << (offset_bits - conv_params->round_1)) +
385
0
                           (1 << (offset_bits - conv_params->round_1 - 1));
386
387
0
  for (int y = 0; y < h; ++y) {
388
0
    for (int x = 0; x < w; ++x) {
389
0
      CONV_BUF_TYPE res = src[y * src_stride + x] << bits;
390
0
      res += round_offset;
391
392
0
      if (conv_params->do_average) {
393
0
        int32_t tmp = dst16[y * dst16_stride + x];
394
0
        if (conv_params->use_dist_wtd_comp_avg) {
395
0
          tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset;
396
0
          tmp = tmp >> DIST_PRECISION_BITS;
397
0
        } else {
398
0
          tmp += res;
399
0
          tmp = tmp >> 1;
400
0
        }
401
0
        tmp -= round_offset;
402
0
        dst[y * dst_stride + x] = clip_pixel(ROUND_POWER_OF_TWO(tmp, bits));
403
0
      } else {
404
0
        dst16[y * dst16_stride + x] = res;
405
0
      }
406
0
    }
407
0
  }
408
0
}
409
410
void av1_convolve_2d_scale_c(const uint8_t *src, int src_stride, uint8_t *dst,
411
                             int dst_stride, int w, int h,
412
                             const InterpFilterParams *filter_params_x,
413
                             const InterpFilterParams *filter_params_y,
414
                             const int subpel_x_qn, const int x_step_qn,
415
                             const int subpel_y_qn, const int y_step_qn,
416
0
                             ConvolveParams *conv_params) {
417
0
  int16_t im_block[(2 * MAX_SB_SIZE + MAX_FILTER_TAP) * MAX_SB_SIZE];
418
0
  int im_h = (((h - 1) * y_step_qn + subpel_y_qn) >> SCALE_SUBPEL_BITS) +
419
0
             filter_params_y->taps;
420
0
  CONV_BUF_TYPE *dst16 = conv_params->dst;
421
0
  const int dst16_stride = conv_params->dst_stride;
422
0
  const int bits =
423
0
      FILTER_BITS * 2 - conv_params->round_0 - conv_params->round_1;
424
0
  assert(bits >= 0);
425
0
  int im_stride = w;
426
0
  const int fo_vert = filter_params_y->taps / 2 - 1;
427
0
  const int fo_horiz = filter_params_x->taps / 2 - 1;
428
0
  const int bd = 8;
429
430
  // horizontal filter
431
0
  const uint8_t *src_horiz = src - fo_vert * src_stride;
432
0
  for (int y = 0; y < im_h; ++y) {
433
0
    int x_qn = subpel_x_qn;
434
0
    for (int x = 0; x < w; ++x, x_qn += x_step_qn) {
435
0
      const uint8_t *const src_x = &src_horiz[(x_qn >> SCALE_SUBPEL_BITS)];
436
0
      const int x_filter_idx = (x_qn & SCALE_SUBPEL_MASK) >> SCALE_EXTRA_BITS;
437
0
      assert(x_filter_idx < SUBPEL_SHIFTS);
438
0
      const int16_t *x_filter =
439
0
          av1_get_interp_filter_subpel_kernel(filter_params_x, x_filter_idx);
440
0
      int32_t sum = (1 << (bd + FILTER_BITS - 1));
441
0
      for (int k = 0; k < filter_params_x->taps; ++k) {
442
0
        sum += x_filter[k] * src_x[k - fo_horiz];
443
0
      }
444
0
      assert(0 <= sum && sum < (1 << (bd + FILTER_BITS + 1)));
445
0
      im_block[y * im_stride + x] =
446
0
          (int16_t)ROUND_POWER_OF_TWO(sum, conv_params->round_0);
447
0
    }
448
0
    src_horiz += src_stride;
449
0
  }
450
451
  // vertical filter
452
0
  int16_t *src_vert = im_block + fo_vert * im_stride;
453
0
  const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0;
454
0
  for (int x = 0; x < w; ++x) {
455
0
    int y_qn = subpel_y_qn;
456
0
    for (int y = 0; y < h; ++y, y_qn += y_step_qn) {
457
0
      const int16_t *src_y = &src_vert[(y_qn >> SCALE_SUBPEL_BITS) * im_stride];
458
0
      const int y_filter_idx = (y_qn & SCALE_SUBPEL_MASK) >> SCALE_EXTRA_BITS;
459
0
      assert(y_filter_idx < SUBPEL_SHIFTS);
460
0
      const int16_t *y_filter =
461
0
          av1_get_interp_filter_subpel_kernel(filter_params_y, y_filter_idx);
462
0
      int32_t sum = 1 << offset_bits;
463
0
      for (int k = 0; k < filter_params_y->taps; ++k) {
464
0
        sum += y_filter[k] * src_y[(k - fo_vert) * im_stride];
465
0
      }
466
0
      assert(0 <= sum && sum < (1 << (offset_bits + 2)));
467
0
      CONV_BUF_TYPE res = ROUND_POWER_OF_TWO(sum, conv_params->round_1);
468
0
      if (conv_params->is_compound) {
469
0
        if (conv_params->do_average) {
470
0
          int32_t tmp = dst16[y * dst16_stride + x];
471
0
          if (conv_params->use_dist_wtd_comp_avg) {
472
0
            tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset;
473
0
            tmp = tmp >> DIST_PRECISION_BITS;
474
0
          } else {
475
0
            tmp += res;
476
0
            tmp = tmp >> 1;
477
0
          }
478
          /* Subtract round offset and convolve round */
479
0
          tmp = tmp - ((1 << (offset_bits - conv_params->round_1)) +
480
0
                       (1 << (offset_bits - conv_params->round_1 - 1)));
481
0
          dst[y * dst_stride + x] = clip_pixel(ROUND_POWER_OF_TWO(tmp, bits));
482
0
        } else {
483
0
          dst16[y * dst16_stride + x] = res;
484
0
        }
485
0
      } else {
486
        /* Subtract round offset and convolve round */
487
0
        int32_t tmp = res - ((1 << (offset_bits - conv_params->round_1)) +
488
0
                             (1 << (offset_bits - conv_params->round_1 - 1)));
489
0
        dst[y * dst_stride + x] = clip_pixel(ROUND_POWER_OF_TWO(tmp, bits));
490
0
      }
491
0
    }
492
0
    src_vert++;
493
0
  }
494
0
}
495
496
static void convolve_2d_scale_wrapper(
497
    const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w,
498
    int h, const InterpFilterParams *filter_params_x,
499
    const InterpFilterParams *filter_params_y, const int subpel_x_qn,
500
    const int x_step_qn, const int subpel_y_qn, const int y_step_qn,
501
0
    ConvolveParams *conv_params) {
502
0
  if (conv_params->is_compound) {
503
0
    assert(conv_params->dst != NULL);
504
0
  }
505
0
  av1_convolve_2d_scale(src, src_stride, dst, dst_stride, w, h, filter_params_x,
506
0
                        filter_params_y, subpel_x_qn, x_step_qn, subpel_y_qn,
507
0
                        y_step_qn, conv_params);
508
0
}
509
510
static void convolve_2d_facade_compound(
511
    const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w,
512
    int h, const InterpFilterParams *filter_params_x,
513
    const InterpFilterParams *filter_params_y, const int subpel_x_qn,
514
0
    const int subpel_y_qn, ConvolveParams *conv_params) {
515
0
  const bool need_x = subpel_x_qn != 0;
516
0
  const bool need_y = subpel_y_qn != 0;
517
0
  if (!need_x && !need_y) {
518
0
    av1_dist_wtd_convolve_2d_copy(src, src_stride, dst, dst_stride, w, h,
519
0
                                  conv_params);
520
0
  } else if (need_x && !need_y) {
521
0
    av1_dist_wtd_convolve_x(src, src_stride, dst, dst_stride, w, h,
522
0
                            filter_params_x, subpel_x_qn, conv_params);
523
0
  } else if (!need_x && need_y) {
524
0
    av1_dist_wtd_convolve_y(src, src_stride, dst, dst_stride, w, h,
525
0
                            filter_params_y, subpel_y_qn, conv_params);
526
0
  } else {
527
0
    assert(need_y && need_x);
528
0
    av1_dist_wtd_convolve_2d(src, src_stride, dst, dst_stride, w, h,
529
0
                             filter_params_x, filter_params_y, subpel_x_qn,
530
0
                             subpel_y_qn, conv_params);
531
0
  }
532
0
}
533
534
static void convolve_2d_facade_single(
535
    const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w,
536
    int h, const InterpFilterParams *filter_params_x,
537
    const InterpFilterParams *filter_params_y, const int subpel_x_qn,
538
0
    const int subpel_y_qn, ConvolveParams *conv_params) {
539
0
  const bool need_x = subpel_x_qn != 0;
540
0
  const bool need_y = subpel_y_qn != 0;
541
0
  if (!need_x && !need_y) {
542
0
    aom_convolve_copy(src, src_stride, dst, dst_stride, w, h);
543
0
  } else if (need_x && !need_y) {
544
0
    av1_convolve_x_sr(src, src_stride, dst, dst_stride, w, h, filter_params_x,
545
0
                      subpel_x_qn, conv_params);
546
0
  } else if (!need_x && need_y) {
547
0
    av1_convolve_y_sr(src, src_stride, dst, dst_stride, w, h, filter_params_y,
548
0
                      subpel_y_qn);
549
0
  } else {
550
0
    assert(need_x && need_y);
551
0
    av1_convolve_2d_sr(src, src_stride, dst, dst_stride, w, h, filter_params_x,
552
0
                       filter_params_y, subpel_x_qn, subpel_y_qn, conv_params);
553
0
  }
554
0
}
555
556
void av1_convolve_2d_facade(const uint8_t *src, int src_stride, uint8_t *dst,
557
                            int dst_stride, int w, int h,
558
                            const InterpFilterParams *interp_filters[2],
559
                            const int subpel_x_qn, int x_step_q4,
560
                            const int subpel_y_qn, int y_step_q4, int scaled,
561
0
                            ConvolveParams *conv_params) {
562
0
  (void)x_step_q4;
563
0
  (void)y_step_q4;
564
0
  (void)dst;
565
0
  (void)dst_stride;
566
567
0
  const InterpFilterParams *filter_params_x = interp_filters[0];
568
0
  const InterpFilterParams *filter_params_y = interp_filters[1];
569
570
  // TODO(jingning, yunqing): Add SIMD support to 2-tap filter case.
571
  // Do we have SIMD support to 4-tap case?
572
  // 2-tap filter indicates that it is for IntraBC.
573
0
  if (filter_params_x->taps == 2 || filter_params_y->taps == 2) {
574
0
    assert(filter_params_x->taps == 2 && filter_params_y->taps == 2);
575
0
    assert(!scaled);
576
0
    if (subpel_x_qn && subpel_y_qn) {
577
0
      av1_convolve_2d_sr_c(src, src_stride, dst, dst_stride, w, h,
578
0
                           filter_params_x, filter_params_y, subpel_x_qn,
579
0
                           subpel_y_qn, conv_params);
580
0
      return;
581
0
    } else if (subpel_x_qn) {
582
0
      av1_convolve_x_sr_c(src, src_stride, dst, dst_stride, w, h,
583
0
                          filter_params_x, subpel_x_qn, conv_params);
584
0
      return;
585
0
    } else if (subpel_y_qn) {
586
0
      av1_convolve_y_sr_c(src, src_stride, dst, dst_stride, w, h,
587
0
                          filter_params_y, subpel_y_qn);
588
0
      return;
589
0
    }
590
0
  }
591
592
0
  if (scaled) {
593
0
    convolve_2d_scale_wrapper(src, src_stride, dst, dst_stride, w, h,
594
0
                              filter_params_x, filter_params_y, subpel_x_qn,
595
0
                              x_step_q4, subpel_y_qn, y_step_q4, conv_params);
596
0
  } else if (conv_params->is_compound) {
597
0
    convolve_2d_facade_compound(src, src_stride, dst, dst_stride, w, h,
598
0
                                filter_params_x, filter_params_y, subpel_x_qn,
599
0
                                subpel_y_qn, conv_params);
600
0
  } else {
601
0
    convolve_2d_facade_single(src, src_stride, dst, dst_stride, w, h,
602
0
                              filter_params_x, filter_params_y, subpel_x_qn,
603
0
                              subpel_y_qn, conv_params);
604
0
  }
605
0
}
606
607
#if CONFIG_AV1_HIGHBITDEPTH
608
void av1_highbd_convolve_x_sr_c(const uint16_t *src, int src_stride,
609
                                uint16_t *dst, int dst_stride, int w, int h,
610
                                const InterpFilterParams *filter_params_x,
611
                                const int subpel_x_qn,
612
0
                                ConvolveParams *conv_params, int bd) {
613
0
  const int fo_horiz = filter_params_x->taps / 2 - 1;
614
0
  const int bits = FILTER_BITS - conv_params->round_0;
615
616
0
  assert(bits >= 0);
617
0
  assert((FILTER_BITS - conv_params->round_1) >= 0 ||
618
0
         ((conv_params->round_0 + conv_params->round_1) == 2 * FILTER_BITS));
619
620
  // horizontal filter
621
0
  const int16_t *x_filter = av1_get_interp_filter_subpel_kernel(
622
0
      filter_params_x, subpel_x_qn & SUBPEL_MASK);
623
0
  for (int y = 0; y < h; ++y) {
624
0
    for (int x = 0; x < w; ++x) {
625
0
      int32_t res = 0;
626
0
      for (int k = 0; k < filter_params_x->taps; ++k) {
627
0
        res += x_filter[k] * src[y * src_stride + x - fo_horiz + k];
628
0
      }
629
0
      res = ROUND_POWER_OF_TWO(res, conv_params->round_0);
630
0
      dst[y * dst_stride + x] =
631
0
          clip_pixel_highbd(ROUND_POWER_OF_TWO(res, bits), bd);
632
0
    }
633
0
  }
634
0
}
635
636
void av1_highbd_convolve_y_sr_c(const uint16_t *src, int src_stride,
637
                                uint16_t *dst, int dst_stride, int w, int h,
638
                                const InterpFilterParams *filter_params_y,
639
0
                                const int subpel_y_qn, int bd) {
640
0
  const int fo_vert = filter_params_y->taps / 2 - 1;
641
  // vertical filter
642
0
  const int16_t *y_filter = av1_get_interp_filter_subpel_kernel(
643
0
      filter_params_y, subpel_y_qn & SUBPEL_MASK);
644
0
  for (int y = 0; y < h; ++y) {
645
0
    for (int x = 0; x < w; ++x) {
646
0
      int32_t res = 0;
647
0
      for (int k = 0; k < filter_params_y->taps; ++k) {
648
0
        res += y_filter[k] * src[(y - fo_vert + k) * src_stride + x];
649
0
      }
650
0
      dst[y * dst_stride + x] =
651
0
          clip_pixel_highbd(ROUND_POWER_OF_TWO(res, FILTER_BITS), bd);
652
0
    }
653
0
  }
654
0
}
655
656
void av1_highbd_convolve_2d_sr_c(const uint16_t *src, int src_stride,
657
                                 uint16_t *dst, int dst_stride, int w, int h,
658
                                 const InterpFilterParams *filter_params_x,
659
                                 const InterpFilterParams *filter_params_y,
660
                                 const int subpel_x_qn, const int subpel_y_qn,
661
0
                                 ConvolveParams *conv_params, int bd) {
662
0
  int16_t im_block[(MAX_SB_SIZE + MAX_FILTER_TAP - 1) * MAX_SB_SIZE];
663
0
  int im_h = h + filter_params_y->taps - 1;
664
0
  int im_stride = w;
665
0
  assert(w <= MAX_SB_SIZE && h <= MAX_SB_SIZE);
666
0
  const int fo_vert = filter_params_y->taps / 2 - 1;
667
0
  const int fo_horiz = filter_params_x->taps / 2 - 1;
668
0
  const int bits =
669
0
      FILTER_BITS * 2 - conv_params->round_0 - conv_params->round_1;
670
0
  assert(bits >= 0);
671
672
  // horizontal filter
673
0
  const uint16_t *src_horiz = src - fo_vert * src_stride;
674
0
  const int16_t *x_filter = av1_get_interp_filter_subpel_kernel(
675
0
      filter_params_x, subpel_x_qn & SUBPEL_MASK);
676
0
  for (int y = 0; y < im_h; ++y) {
677
0
    for (int x = 0; x < w; ++x) {
678
0
      int32_t sum = (1 << (bd + FILTER_BITS - 1));
679
0
      for (int k = 0; k < filter_params_x->taps; ++k) {
680
0
        sum += x_filter[k] * src_horiz[y * src_stride + x - fo_horiz + k];
681
0
      }
682
0
      assert(0 <= sum && sum < (1 << (bd + FILTER_BITS + 1)));
683
0
      im_block[y * im_stride + x] =
684
0
          ROUND_POWER_OF_TWO(sum, conv_params->round_0);
685
0
    }
686
0
  }
687
688
  // vertical filter
689
0
  int16_t *src_vert = im_block + fo_vert * im_stride;
690
0
  const int16_t *y_filter = av1_get_interp_filter_subpel_kernel(
691
0
      filter_params_y, subpel_y_qn & SUBPEL_MASK);
692
0
  const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0;
693
0
  for (int y = 0; y < h; ++y) {
694
0
    for (int x = 0; x < w; ++x) {
695
0
      int32_t sum = 1 << offset_bits;
696
0
      for (int k = 0; k < filter_params_y->taps; ++k) {
697
0
        sum += y_filter[k] * src_vert[(y - fo_vert + k) * im_stride + x];
698
0
      }
699
0
      assert(0 <= sum && sum < (1 << (offset_bits + 2)));
700
0
      int32_t res = ROUND_POWER_OF_TWO(sum, conv_params->round_1) -
701
0
                    ((1 << (offset_bits - conv_params->round_1)) +
702
0
                     (1 << (offset_bits - conv_params->round_1 - 1)));
703
0
      dst[y * dst_stride + x] =
704
0
          clip_pixel_highbd(ROUND_POWER_OF_TWO(res, bits), bd);
705
0
    }
706
0
  }
707
0
}
708
709
void av1_highbd_dist_wtd_convolve_2d_c(
710
    const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w,
711
    int h, const InterpFilterParams *filter_params_x,
712
    const InterpFilterParams *filter_params_y, const int subpel_x_qn,
713
0
    const int subpel_y_qn, ConvolveParams *conv_params, int bd) {
714
0
  int x, y, k;
715
0
  int16_t im_block[(MAX_SB_SIZE + MAX_FILTER_TAP - 1) * MAX_SB_SIZE];
716
0
  CONV_BUF_TYPE *dst16 = conv_params->dst;
717
0
  int dst16_stride = conv_params->dst_stride;
718
0
  int im_h = h + filter_params_y->taps - 1;
719
0
  int im_stride = w;
720
0
  const int fo_vert = filter_params_y->taps / 2 - 1;
721
0
  const int fo_horiz = filter_params_x->taps / 2 - 1;
722
0
  const int round_bits =
723
0
      2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1;
724
0
  assert(round_bits >= 0);
725
726
  // horizontal filter
727
0
  const uint16_t *src_horiz = src - fo_vert * src_stride;
728
0
  const int16_t *x_filter = av1_get_interp_filter_subpel_kernel(
729
0
      filter_params_x, subpel_x_qn & SUBPEL_MASK);
730
0
  for (y = 0; y < im_h; ++y) {
731
0
    for (x = 0; x < w; ++x) {
732
0
      int32_t sum = (1 << (bd + FILTER_BITS - 1));
733
0
      for (k = 0; k < filter_params_x->taps; ++k) {
734
0
        sum += x_filter[k] * src_horiz[y * src_stride + x - fo_horiz + k];
735
0
      }
736
0
      assert(0 <= sum && sum < (1 << (bd + FILTER_BITS + 1)));
737
0
      (void)bd;
738
0
      im_block[y * im_stride + x] =
739
0
          (int16_t)ROUND_POWER_OF_TWO(sum, conv_params->round_0);
740
0
    }
741
0
  }
742
743
  // vertical filter
744
0
  int16_t *src_vert = im_block + fo_vert * im_stride;
745
0
  const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0;
746
0
  const int16_t *y_filter = av1_get_interp_filter_subpel_kernel(
747
0
      filter_params_y, subpel_y_qn & SUBPEL_MASK);
748
0
  for (y = 0; y < h; ++y) {
749
0
    for (x = 0; x < w; ++x) {
750
0
      int32_t sum = 1 << offset_bits;
751
0
      for (k = 0; k < filter_params_y->taps; ++k) {
752
0
        sum += y_filter[k] * src_vert[(y - fo_vert + k) * im_stride + x];
753
0
      }
754
0
      assert(0 <= sum && sum < (1 << (offset_bits + 2)));
755
0
      CONV_BUF_TYPE res = ROUND_POWER_OF_TWO(sum, conv_params->round_1);
756
0
      if (conv_params->do_average) {
757
0
        int32_t tmp = dst16[y * dst16_stride + x];
758
0
        if (conv_params->use_dist_wtd_comp_avg) {
759
0
          tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset;
760
0
          tmp = tmp >> DIST_PRECISION_BITS;
761
0
        } else {
762
0
          tmp += res;
763
0
          tmp = tmp >> 1;
764
0
        }
765
0
        tmp -= (1 << (offset_bits - conv_params->round_1)) +
766
0
               (1 << (offset_bits - conv_params->round_1 - 1));
767
0
        dst[y * dst_stride + x] =
768
0
            clip_pixel_highbd(ROUND_POWER_OF_TWO(tmp, round_bits), bd);
769
0
      } else {
770
0
        dst16[y * dst16_stride + x] = res;
771
0
      }
772
0
    }
773
0
  }
774
0
}
775
776
void av1_highbd_dist_wtd_convolve_x_c(const uint16_t *src, int src_stride,
777
                                      uint16_t *dst, int dst_stride, int w,
778
                                      int h,
779
                                      const InterpFilterParams *filter_params_x,
780
                                      const int subpel_x_qn,
781
0
                                      ConvolveParams *conv_params, int bd) {
782
0
  CONV_BUF_TYPE *dst16 = conv_params->dst;
783
0
  int dst16_stride = conv_params->dst_stride;
784
0
  const int fo_horiz = filter_params_x->taps / 2 - 1;
785
0
  const int bits = FILTER_BITS - conv_params->round_1;
786
0
  const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0;
787
0
  const int round_offset = (1 << (offset_bits - conv_params->round_1)) +
788
0
                           (1 << (offset_bits - conv_params->round_1 - 1));
789
0
  const int round_bits =
790
0
      2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1;
791
0
  assert(round_bits >= 0);
792
0
  assert(bits >= 0);
793
  // horizontal filter
794
0
  const int16_t *x_filter = av1_get_interp_filter_subpel_kernel(
795
0
      filter_params_x, subpel_x_qn & SUBPEL_MASK);
796
0
  for (int y = 0; y < h; ++y) {
797
0
    for (int x = 0; x < w; ++x) {
798
0
      int32_t res = 0;
799
0
      for (int k = 0; k < filter_params_x->taps; ++k) {
800
0
        res += x_filter[k] * src[y * src_stride + x - fo_horiz + k];
801
0
      }
802
0
      res = (1 << bits) * ROUND_POWER_OF_TWO(res, conv_params->round_0);
803
0
      res += round_offset;
804
805
0
      if (conv_params->do_average) {
806
0
        int32_t tmp = dst16[y * dst16_stride + x];
807
0
        if (conv_params->use_dist_wtd_comp_avg) {
808
0
          tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset;
809
0
          tmp = tmp >> DIST_PRECISION_BITS;
810
0
        } else {
811
0
          tmp += res;
812
0
          tmp = tmp >> 1;
813
0
        }
814
0
        tmp -= round_offset;
815
0
        dst[y * dst_stride + x] =
816
0
            clip_pixel_highbd(ROUND_POWER_OF_TWO(tmp, round_bits), bd);
817
0
      } else {
818
0
        dst16[y * dst16_stride + x] = res;
819
0
      }
820
0
    }
821
0
  }
822
0
}
823
824
void av1_highbd_dist_wtd_convolve_y_c(const uint16_t *src, int src_stride,
825
                                      uint16_t *dst, int dst_stride, int w,
826
                                      int h,
827
                                      const InterpFilterParams *filter_params_y,
828
                                      const int subpel_y_qn,
829
0
                                      ConvolveParams *conv_params, int bd) {
830
0
  CONV_BUF_TYPE *dst16 = conv_params->dst;
831
0
  int dst16_stride = conv_params->dst_stride;
832
0
  const int fo_vert = filter_params_y->taps / 2 - 1;
833
0
  const int bits = FILTER_BITS - conv_params->round_0;
834
0
  const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0;
835
0
  const int round_offset = (1 << (offset_bits - conv_params->round_1)) +
836
0
                           (1 << (offset_bits - conv_params->round_1 - 1));
837
0
  const int round_bits =
838
0
      2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1;
839
0
  assert(round_bits >= 0);
840
0
  assert(bits >= 0);
841
  // vertical filter
842
0
  const int16_t *y_filter = av1_get_interp_filter_subpel_kernel(
843
0
      filter_params_y, subpel_y_qn & SUBPEL_MASK);
844
0
  for (int y = 0; y < h; ++y) {
845
0
    for (int x = 0; x < w; ++x) {
846
0
      int32_t res = 0;
847
0
      for (int k = 0; k < filter_params_y->taps; ++k) {
848
0
        res += y_filter[k] * src[(y - fo_vert + k) * src_stride + x];
849
0
      }
850
0
      res *= (1 << bits);
851
0
      res = ROUND_POWER_OF_TWO(res, conv_params->round_1) + round_offset;
852
853
0
      if (conv_params->do_average) {
854
0
        int32_t tmp = dst16[y * dst16_stride + x];
855
0
        if (conv_params->use_dist_wtd_comp_avg) {
856
0
          tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset;
857
0
          tmp = tmp >> DIST_PRECISION_BITS;
858
0
        } else {
859
0
          tmp += res;
860
0
          tmp = tmp >> 1;
861
0
        }
862
0
        tmp -= round_offset;
863
0
        dst[y * dst_stride + x] =
864
0
            clip_pixel_highbd(ROUND_POWER_OF_TWO(tmp, round_bits), bd);
865
0
      } else {
866
0
        dst16[y * dst16_stride + x] = res;
867
0
      }
868
0
    }
869
0
  }
870
0
}
871
872
void av1_highbd_dist_wtd_convolve_2d_copy_c(const uint16_t *src, int src_stride,
873
                                            uint16_t *dst, int dst_stride,
874
                                            int w, int h,
875
                                            ConvolveParams *conv_params,
876
0
                                            int bd) {
877
0
  CONV_BUF_TYPE *dst16 = conv_params->dst;
878
0
  int dst16_stride = conv_params->dst_stride;
879
0
  const int bits =
880
0
      FILTER_BITS * 2 - conv_params->round_1 - conv_params->round_0;
881
0
  const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0;
882
0
  const int round_offset = (1 << (offset_bits - conv_params->round_1)) +
883
0
                           (1 << (offset_bits - conv_params->round_1 - 1));
884
0
  assert(bits >= 0);
885
886
0
  for (int y = 0; y < h; ++y) {
887
0
    for (int x = 0; x < w; ++x) {
888
0
      CONV_BUF_TYPE res = src[y * src_stride + x] << bits;
889
0
      res += round_offset;
890
0
      if (conv_params->do_average) {
891
0
        int32_t tmp = dst16[y * dst16_stride + x];
892
0
        if (conv_params->use_dist_wtd_comp_avg) {
893
0
          tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset;
894
0
          tmp = tmp >> DIST_PRECISION_BITS;
895
0
        } else {
896
0
          tmp += res;
897
0
          tmp = tmp >> 1;
898
0
        }
899
0
        tmp -= round_offset;
900
0
        dst[y * dst_stride + x] =
901
0
            clip_pixel_highbd(ROUND_POWER_OF_TWO(tmp, bits), bd);
902
0
      } else {
903
0
        dst16[y * dst16_stride + x] = res;
904
0
      }
905
0
    }
906
0
  }
907
0
}
908
909
void av1_highbd_convolve_2d_scale_c(const uint16_t *src, int src_stride,
910
                                    uint16_t *dst, int dst_stride, int w, int h,
911
                                    const InterpFilterParams *filter_params_x,
912
                                    const InterpFilterParams *filter_params_y,
913
                                    const int subpel_x_qn, const int x_step_qn,
914
                                    const int subpel_y_qn, const int y_step_qn,
915
0
                                    ConvolveParams *conv_params, int bd) {
916
0
  int16_t im_block[(2 * MAX_SB_SIZE + MAX_FILTER_TAP) * MAX_SB_SIZE];
917
0
  int im_h = (((h - 1) * y_step_qn + subpel_y_qn) >> SCALE_SUBPEL_BITS) +
918
0
             filter_params_y->taps;
919
0
  int im_stride = w;
920
0
  const int fo_vert = filter_params_y->taps / 2 - 1;
921
0
  const int fo_horiz = filter_params_x->taps / 2 - 1;
922
0
  CONV_BUF_TYPE *dst16 = conv_params->dst;
923
0
  const int dst16_stride = conv_params->dst_stride;
924
0
  const int bits =
925
0
      FILTER_BITS * 2 - conv_params->round_0 - conv_params->round_1;
926
0
  assert(bits >= 0);
927
  // horizontal filter
928
0
  const uint16_t *src_horiz = src - fo_vert * src_stride;
929
0
  for (int y = 0; y < im_h; ++y) {
930
0
    int x_qn = subpel_x_qn;
931
0
    for (int x = 0; x < w; ++x, x_qn += x_step_qn) {
932
0
      const uint16_t *const src_x = &src_horiz[(x_qn >> SCALE_SUBPEL_BITS)];
933
0
      const int x_filter_idx = (x_qn & SCALE_SUBPEL_MASK) >> SCALE_EXTRA_BITS;
934
0
      assert(x_filter_idx < SUBPEL_SHIFTS);
935
0
      const int16_t *x_filter =
936
0
          av1_get_interp_filter_subpel_kernel(filter_params_x, x_filter_idx);
937
0
      int32_t sum = (1 << (bd + FILTER_BITS - 1));
938
0
      for (int k = 0; k < filter_params_x->taps; ++k) {
939
0
        sum += x_filter[k] * src_x[k - fo_horiz];
940
0
      }
941
0
      assert(0 <= sum && sum < (1 << (bd + FILTER_BITS + 1)));
942
0
      im_block[y * im_stride + x] =
943
0
          (int16_t)ROUND_POWER_OF_TWO(sum, conv_params->round_0);
944
0
    }
945
0
    src_horiz += src_stride;
946
0
  }
947
948
  // vertical filter
949
0
  int16_t *src_vert = im_block + fo_vert * im_stride;
950
0
  const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0;
951
0
  for (int x = 0; x < w; ++x) {
952
0
    int y_qn = subpel_y_qn;
953
0
    for (int y = 0; y < h; ++y, y_qn += y_step_qn) {
954
0
      const int16_t *src_y = &src_vert[(y_qn >> SCALE_SUBPEL_BITS) * im_stride];
955
0
      const int y_filter_idx = (y_qn & SCALE_SUBPEL_MASK) >> SCALE_EXTRA_BITS;
956
0
      assert(y_filter_idx < SUBPEL_SHIFTS);
957
0
      const int16_t *y_filter =
958
0
          av1_get_interp_filter_subpel_kernel(filter_params_y, y_filter_idx);
959
0
      int32_t sum = 1 << offset_bits;
960
0
      for (int k = 0; k < filter_params_y->taps; ++k) {
961
0
        sum += y_filter[k] * src_y[(k - fo_vert) * im_stride];
962
0
      }
963
0
      assert(0 <= sum && sum < (1 << (offset_bits + 2)));
964
0
      CONV_BUF_TYPE res = ROUND_POWER_OF_TWO(sum, conv_params->round_1);
965
0
      if (conv_params->is_compound) {
966
0
        if (conv_params->do_average) {
967
0
          int32_t tmp = dst16[y * dst16_stride + x];
968
0
          if (conv_params->use_dist_wtd_comp_avg) {
969
0
            tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset;
970
0
            tmp = tmp >> DIST_PRECISION_BITS;
971
0
          } else {
972
0
            tmp += res;
973
0
            tmp = tmp >> 1;
974
0
          }
975
          /* Subtract round offset and convolve round */
976
0
          tmp = tmp - ((1 << (offset_bits - conv_params->round_1)) +
977
0
                       (1 << (offset_bits - conv_params->round_1 - 1)));
978
0
          dst[y * dst_stride + x] =
979
0
              clip_pixel_highbd(ROUND_POWER_OF_TWO(tmp, bits), bd);
980
0
        } else {
981
0
          dst16[y * dst16_stride + x] = res;
982
0
        }
983
0
      } else {
984
        /* Subtract round offset and convolve round */
985
0
        int32_t tmp = res - ((1 << (offset_bits - conv_params->round_1)) +
986
0
                             (1 << (offset_bits - conv_params->round_1 - 1)));
987
0
        dst[y * dst_stride + x] =
988
0
            clip_pixel_highbd(ROUND_POWER_OF_TWO(tmp, bits), bd);
989
0
      }
990
0
    }
991
0
    src_vert++;
992
0
  }
993
0
}
994
995
static void highbd_convolve_2d_facade_compound(
996
    const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride,
997
    const int w, const int h, const InterpFilterParams *filter_params_x,
998
    const InterpFilterParams *filter_params_y, const int subpel_x_qn,
999
0
    const int subpel_y_qn, ConvolveParams *conv_params, int bd) {
1000
0
  const bool need_x = subpel_x_qn != 0;
1001
0
  const bool need_y = subpel_y_qn != 0;
1002
0
  if (!need_x && !need_y) {
1003
0
    av1_highbd_dist_wtd_convolve_2d_copy(src, src_stride, dst, dst_stride, w, h,
1004
0
                                         conv_params, bd);
1005
0
  } else if (need_x && !need_y) {
1006
0
    av1_highbd_dist_wtd_convolve_x(src, src_stride, dst, dst_stride, w, h,
1007
0
                                   filter_params_x, subpel_x_qn, conv_params,
1008
0
                                   bd);
1009
0
  } else if (!need_x && need_y) {
1010
0
    av1_highbd_dist_wtd_convolve_y(src, src_stride, dst, dst_stride, w, h,
1011
0
                                   filter_params_y, subpel_y_qn, conv_params,
1012
0
                                   bd);
1013
0
  } else {
1014
0
    assert(need_x && need_y);
1015
0
    av1_highbd_dist_wtd_convolve_2d(src, src_stride, dst, dst_stride, w, h,
1016
0
                                    filter_params_x, filter_params_y,
1017
0
                                    subpel_x_qn, subpel_y_qn, conv_params, bd);
1018
0
  }
1019
0
}
1020
1021
static void highbd_convolve_2d_facade_single(
1022
    const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride,
1023
    const int w, const int h, const InterpFilterParams *filter_params_x,
1024
    const InterpFilterParams *filter_params_y, const int subpel_x_qn,
1025
0
    const int subpel_y_qn, ConvolveParams *conv_params, int bd) {
1026
0
  const bool need_x = subpel_x_qn != 0;
1027
0
  const bool need_y = subpel_y_qn != 0;
1028
1029
0
  if (!need_x && !need_y) {
1030
0
    aom_highbd_convolve_copy(src, src_stride, dst, dst_stride, w, h);
1031
0
  } else if (need_x && !need_y) {
1032
0
    av1_highbd_convolve_x_sr(src, src_stride, dst, dst_stride, w, h,
1033
0
                             filter_params_x, subpel_x_qn, conv_params, bd);
1034
0
  } else if (!need_x && need_y) {
1035
0
    av1_highbd_convolve_y_sr(src, src_stride, dst, dst_stride, w, h,
1036
0
                             filter_params_y, subpel_y_qn, bd);
1037
0
  } else {
1038
0
    assert(need_x && need_y);
1039
0
    av1_highbd_convolve_2d_sr(src, src_stride, dst, dst_stride, w, h,
1040
0
                              filter_params_x, filter_params_y, subpel_x_qn,
1041
0
                              subpel_y_qn, conv_params, bd);
1042
0
  }
1043
0
}
1044
1045
void av1_highbd_convolve_2d_facade(const uint8_t *src8, int src_stride,
1046
                                   uint8_t *dst8, int dst_stride, int w, int h,
1047
                                   const InterpFilterParams *interp_filters[2],
1048
                                   const int subpel_x_qn, int x_step_q4,
1049
                                   const int subpel_y_qn, int y_step_q4,
1050
                                   int scaled, ConvolveParams *conv_params,
1051
0
                                   int bd) {
1052
0
  (void)x_step_q4;
1053
0
  (void)y_step_q4;
1054
0
  (void)dst_stride;
1055
0
  const uint16_t *src = CONVERT_TO_SHORTPTR(src8);
1056
1057
0
  const int need_filter_params_x = (subpel_x_qn != 0) | scaled;
1058
0
  const int need_filter_params_y = (subpel_y_qn != 0) | scaled;
1059
0
  const InterpFilterParams *filter_params_x =
1060
0
      need_filter_params_x ? interp_filters[0] : NULL;
1061
0
  const InterpFilterParams *filter_params_y =
1062
0
      need_filter_params_y ? interp_filters[1] : NULL;
1063
1064
0
  uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
1065
0
  if (scaled) {
1066
0
    if (conv_params->is_compound) {
1067
0
      assert(conv_params->dst != NULL);
1068
0
    }
1069
0
    av1_highbd_convolve_2d_scale(src, src_stride, dst, dst_stride, w, h,
1070
0
                                 filter_params_x, filter_params_y, subpel_x_qn,
1071
0
                                 x_step_q4, subpel_y_qn, y_step_q4, conv_params,
1072
0
                                 bd);
1073
0
  } else if (conv_params->is_compound) {
1074
0
    highbd_convolve_2d_facade_compound(
1075
0
        src, src_stride, dst, dst_stride, w, h, filter_params_x,
1076
0
        filter_params_y, subpel_x_qn, subpel_y_qn, conv_params, bd);
1077
0
  } else {
1078
0
    highbd_convolve_2d_facade_single(src, src_stride, dst, dst_stride, w, h,
1079
0
                                     filter_params_x, filter_params_y,
1080
0
                                     subpel_x_qn, subpel_y_qn, conv_params, bd);
1081
0
  }
1082
0
}
1083
#endif  // CONFIG_AV1_HIGHBITDEPTH
1084
1085
// Note: Fixed size intermediate buffers, place limits on parameters
1086
// of some functions. 2d filtering proceeds in 2 steps:
1087
//   (1) Interpolate horizontally into an intermediate buffer, temp.
1088
//   (2) Interpolate temp vertically to derive the sub-pixel result.
1089
// Deriving the maximum number of rows in the temp buffer (135):
1090
// --Smallest scaling factor is x1/2 ==> y_step_q4 = 32 (Normative).
1091
// --Largest block size is 128x128 pixels.
1092
// --128 rows in the downscaled frame span a distance of (128 - 1) * 32 in the
1093
//   original frame (in 1/16th pixel units).
1094
// --Must round-up because block may be located at sub-pixel position.
1095
// --Require an additional SUBPEL_TAPS rows for the 8-tap filter tails.
1096
// --((128 - 1) * 32 + 15) >> 4 + 8 = 263.
1097
#define WIENER_MAX_EXT_SIZE 263
1098
1099
0
static INLINE int horz_scalar_product(const uint8_t *a, const int16_t *b) {
1100
0
  int sum = 0;
1101
0
  for (int k = 0; k < SUBPEL_TAPS; ++k) sum += a[k] * b[k];
1102
0
  return sum;
1103
0
}
1104
1105
#if CONFIG_AV1_HIGHBITDEPTH
1106
static INLINE int highbd_horz_scalar_product(const uint16_t *a,
1107
0
                                             const int16_t *b) {
1108
0
  int sum = 0;
1109
0
  for (int k = 0; k < SUBPEL_TAPS; ++k) sum += a[k] * b[k];
1110
0
  return sum;
1111
0
}
1112
#endif
1113
1114
static INLINE int highbd_vert_scalar_product(const uint16_t *a,
1115
                                             ptrdiff_t a_stride,
1116
0
                                             const int16_t *b) {
1117
0
  int sum = 0;
1118
0
  for (int k = 0; k < SUBPEL_TAPS; ++k) sum += a[k * a_stride] * b[k];
1119
0
  return sum;
1120
0
}
1121
1122
0
static const InterpKernel *get_filter_base(const int16_t *filter) {
1123
  // NOTE: This assumes that the filter table is 256-byte aligned.
1124
  // TODO(agrange) Modify to make independent of table alignment.
1125
0
  return (const InterpKernel *)(((intptr_t)filter) & ~((intptr_t)0xFF));
1126
0
}
1127
1128
0
static int get_filter_offset(const int16_t *f, const InterpKernel *base) {
1129
0
  return (int)((const InterpKernel *)(intptr_t)f - base);
1130
0
}
1131
1132
static void convolve_add_src_horiz_hip(const uint8_t *src, ptrdiff_t src_stride,
1133
                                       uint16_t *dst, ptrdiff_t dst_stride,
1134
                                       const InterpKernel *x_filters, int x0_q4,
1135
                                       int x_step_q4, int w, int h,
1136
0
                                       int round0_bits) {
1137
0
  const int bd = 8;
1138
0
  src -= SUBPEL_TAPS / 2 - 1;
1139
0
  for (int y = 0; y < h; ++y) {
1140
0
    int x_q4 = x0_q4;
1141
0
    for (int x = 0; x < w; ++x) {
1142
0
      const uint8_t *const src_x = &src[x_q4 >> SUBPEL_BITS];
1143
0
      const int16_t *const x_filter = x_filters[x_q4 & SUBPEL_MASK];
1144
0
      const int rounding = ((int)src_x[SUBPEL_TAPS / 2 - 1] << FILTER_BITS) +
1145
0
                           (1 << (bd + FILTER_BITS - 1));
1146
0
      const int sum = horz_scalar_product(src_x, x_filter) + rounding;
1147
0
      dst[x] = (uint16_t)clamp(ROUND_POWER_OF_TWO(sum, round0_bits), 0,
1148
0
                               WIENER_CLAMP_LIMIT(round0_bits, bd) - 1);
1149
0
      x_q4 += x_step_q4;
1150
0
    }
1151
0
    src += src_stride;
1152
0
    dst += dst_stride;
1153
0
  }
1154
0
}
1155
1156
static void convolve_add_src_vert_hip(const uint16_t *src, ptrdiff_t src_stride,
1157
                                      uint8_t *dst, ptrdiff_t dst_stride,
1158
                                      const InterpKernel *y_filters, int y0_q4,
1159
                                      int y_step_q4, int w, int h,
1160
0
                                      int round1_bits) {
1161
0
  const int bd = 8;
1162
0
  src -= src_stride * (SUBPEL_TAPS / 2 - 1);
1163
1164
0
  for (int x = 0; x < w; ++x) {
1165
0
    int y_q4 = y0_q4;
1166
0
    for (int y = 0; y < h; ++y) {
1167
0
      const uint16_t *src_y = &src[(y_q4 >> SUBPEL_BITS) * src_stride];
1168
0
      const int16_t *const y_filter = y_filters[y_q4 & SUBPEL_MASK];
1169
0
      const int rounding =
1170
0
          ((int)src_y[(SUBPEL_TAPS / 2 - 1) * src_stride] << FILTER_BITS) -
1171
0
          (1 << (bd + round1_bits - 1));
1172
0
      const int sum =
1173
0
          highbd_vert_scalar_product(src_y, src_stride, y_filter) + rounding;
1174
0
      dst[y * dst_stride] = clip_pixel(ROUND_POWER_OF_TWO(sum, round1_bits));
1175
0
      y_q4 += y_step_q4;
1176
0
    }
1177
0
    ++src;
1178
0
    ++dst;
1179
0
  }
1180
0
}
1181
1182
void av1_wiener_convolve_add_src_c(const uint8_t *src, ptrdiff_t src_stride,
1183
                                   uint8_t *dst, ptrdiff_t dst_stride,
1184
                                   const int16_t *filter_x, int x_step_q4,
1185
                                   const int16_t *filter_y, int y_step_q4,
1186
                                   int w, int h,
1187
0
                                   const ConvolveParams *conv_params) {
1188
0
  const InterpKernel *const filters_x = get_filter_base(filter_x);
1189
0
  const int x0_q4 = get_filter_offset(filter_x, filters_x);
1190
1191
0
  const InterpKernel *const filters_y = get_filter_base(filter_y);
1192
0
  const int y0_q4 = get_filter_offset(filter_y, filters_y);
1193
1194
0
  uint16_t temp[WIENER_MAX_EXT_SIZE * MAX_SB_SIZE];
1195
0
  const int intermediate_height =
1196
0
      (((h - 1) * y_step_q4 + y0_q4) >> SUBPEL_BITS) + SUBPEL_TAPS - 1;
1197
0
  memset(temp + (intermediate_height * MAX_SB_SIZE), 0, MAX_SB_SIZE);
1198
1199
0
  assert(w <= MAX_SB_SIZE);
1200
0
  assert(h <= MAX_SB_SIZE);
1201
0
  assert(y_step_q4 <= 32);
1202
0
  assert(x_step_q4 <= 32);
1203
1204
0
  convolve_add_src_horiz_hip(src - src_stride * (SUBPEL_TAPS / 2 - 1),
1205
0
                             src_stride, temp, MAX_SB_SIZE, filters_x, x0_q4,
1206
0
                             x_step_q4, w, intermediate_height,
1207
0
                             conv_params->round_0);
1208
0
  convolve_add_src_vert_hip(temp + MAX_SB_SIZE * (SUBPEL_TAPS / 2 - 1),
1209
0
                            MAX_SB_SIZE, dst, dst_stride, filters_y, y0_q4,
1210
0
                            y_step_q4, w, h, conv_params->round_1);
1211
0
}
1212
1213
#if CONFIG_AV1_HIGHBITDEPTH
1214
static void highbd_convolve_add_src_horiz_hip(
1215
    const uint8_t *src8, ptrdiff_t src_stride, uint16_t *dst,
1216
    ptrdiff_t dst_stride, const InterpKernel *x_filters, int x0_q4,
1217
0
    int x_step_q4, int w, int h, int round0_bits, int bd) {
1218
0
  const int extraprec_clamp_limit = WIENER_CLAMP_LIMIT(round0_bits, bd);
1219
0
  uint16_t *src = CONVERT_TO_SHORTPTR(src8);
1220
0
  src -= SUBPEL_TAPS / 2 - 1;
1221
0
  for (int y = 0; y < h; ++y) {
1222
0
    int x_q4 = x0_q4;
1223
0
    for (int x = 0; x < w; ++x) {
1224
0
      const uint16_t *const src_x = &src[x_q4 >> SUBPEL_BITS];
1225
0
      const int16_t *const x_filter = x_filters[x_q4 & SUBPEL_MASK];
1226
0
      const int rounding = ((int)src_x[SUBPEL_TAPS / 2 - 1] << FILTER_BITS) +
1227
0
                           (1 << (bd + FILTER_BITS - 1));
1228
0
      const int sum = highbd_horz_scalar_product(src_x, x_filter) + rounding;
1229
0
      dst[x] = (uint16_t)clamp(ROUND_POWER_OF_TWO(sum, round0_bits), 0,
1230
0
                               extraprec_clamp_limit - 1);
1231
0
      x_q4 += x_step_q4;
1232
0
    }
1233
0
    src += src_stride;
1234
0
    dst += dst_stride;
1235
0
  }
1236
0
}
1237
1238
static void highbd_convolve_add_src_vert_hip(
1239
    const uint16_t *src, ptrdiff_t src_stride, uint8_t *dst8,
1240
    ptrdiff_t dst_stride, const InterpKernel *y_filters, int y0_q4,
1241
0
    int y_step_q4, int w, int h, int round1_bits, int bd) {
1242
0
  uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
1243
0
  src -= src_stride * (SUBPEL_TAPS / 2 - 1);
1244
0
  for (int x = 0; x < w; ++x) {
1245
0
    int y_q4 = y0_q4;
1246
0
    for (int y = 0; y < h; ++y) {
1247
0
      const uint16_t *src_y = &src[(y_q4 >> SUBPEL_BITS) * src_stride];
1248
0
      const int16_t *const y_filter = y_filters[y_q4 & SUBPEL_MASK];
1249
0
      const int rounding =
1250
0
          ((int)src_y[(SUBPEL_TAPS / 2 - 1) * src_stride] << FILTER_BITS) -
1251
0
          (1 << (bd + round1_bits - 1));
1252
0
      const int sum =
1253
0
          highbd_vert_scalar_product(src_y, src_stride, y_filter) + rounding;
1254
0
      dst[y * dst_stride] =
1255
0
          clip_pixel_highbd(ROUND_POWER_OF_TWO(sum, round1_bits), bd);
1256
0
      y_q4 += y_step_q4;
1257
0
    }
1258
0
    ++src;
1259
0
    ++dst;
1260
0
  }
1261
0
}
1262
1263
void av1_highbd_wiener_convolve_add_src_c(
1264
    const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst,
1265
    ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4,
1266
    const int16_t *filter_y, int y_step_q4, int w, int h,
1267
0
    const ConvolveParams *conv_params, int bd) {
1268
0
  const InterpKernel *const filters_x = get_filter_base(filter_x);
1269
0
  const int x0_q4 = get_filter_offset(filter_x, filters_x);
1270
1271
0
  const InterpKernel *const filters_y = get_filter_base(filter_y);
1272
0
  const int y0_q4 = get_filter_offset(filter_y, filters_y);
1273
1274
0
  uint16_t temp[WIENER_MAX_EXT_SIZE * MAX_SB_SIZE];
1275
0
  const int intermediate_height =
1276
0
      (((h - 1) * y_step_q4 + y0_q4) >> SUBPEL_BITS) + SUBPEL_TAPS;
1277
1278
0
  assert(w <= MAX_SB_SIZE);
1279
0
  assert(h <= MAX_SB_SIZE);
1280
0
  assert(y_step_q4 <= 32);
1281
0
  assert(x_step_q4 <= 32);
1282
0
  assert(bd + FILTER_BITS - conv_params->round_0 + 2 <= 16);
1283
1284
0
  highbd_convolve_add_src_horiz_hip(src - src_stride * (SUBPEL_TAPS / 2 - 1),
1285
0
                                    src_stride, temp, MAX_SB_SIZE, filters_x,
1286
0
                                    x0_q4, x_step_q4, w, intermediate_height,
1287
0
                                    conv_params->round_0, bd);
1288
0
  highbd_convolve_add_src_vert_hip(
1289
0
      temp + MAX_SB_SIZE * (SUBPEL_TAPS / 2 - 1), MAX_SB_SIZE, dst, dst_stride,
1290
0
      filters_y, y0_q4, y_step_q4, w, h, conv_params->round_1, bd);
1291
0
}
1292
#endif  // CONFIG_AV1_HIGHBITDEPTH