/src/aom/av1/common/convolve.c
Line | Count | Source (jump to first uncovered line) |
1 | | /* |
2 | | * Copyright (c) 2016, Alliance for Open Media. All rights reserved |
3 | | * |
4 | | * This source code is subject to the terms of the BSD 2 Clause License and |
5 | | * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License |
6 | | * was not distributed with this source code in the LICENSE file, you can |
7 | | * obtain it at www.aomedia.org/license/software. If the Alliance for Open |
8 | | * Media Patent License 1.0 was not distributed with this source code in the |
9 | | * PATENTS file, you can obtain it at www.aomedia.org/license/patent. |
10 | | */ |
11 | | |
12 | | #include <assert.h> |
13 | | #include <string.h> |
14 | | |
15 | | #include "config/aom_dsp_rtcd.h" |
16 | | #include "config/av1_rtcd.h" |
17 | | |
18 | | #include "av1/common/av1_common_int.h" |
19 | | #include "av1/common/blockd.h" |
20 | | #include "av1/common/convolve.h" |
21 | | #include "av1/common/filter.h" |
22 | | #include "av1/common/resize.h" |
23 | | #include "aom_dsp/aom_dsp_common.h" |
24 | | #include "aom_ports/mem.h" |
25 | | |
26 | | void av1_convolve_horiz_rs_c(const uint8_t *src, int src_stride, uint8_t *dst, |
27 | | int dst_stride, int w, int h, |
28 | | const int16_t *x_filters, int x0_qn, |
29 | 0 | int x_step_qn) { |
30 | 0 | src -= UPSCALE_NORMATIVE_TAPS / 2 - 1; |
31 | 0 | for (int y = 0; y < h; ++y) { |
32 | 0 | int x_qn = x0_qn; |
33 | 0 | for (int x = 0; x < w; ++x) { |
34 | 0 | const uint8_t *const src_x = &src[x_qn >> RS_SCALE_SUBPEL_BITS]; |
35 | 0 | const int x_filter_idx = |
36 | 0 | (x_qn & RS_SCALE_SUBPEL_MASK) >> RS_SCALE_EXTRA_BITS; |
37 | 0 | assert(x_filter_idx <= RS_SUBPEL_MASK); |
38 | 0 | const int16_t *const x_filter = |
39 | 0 | &x_filters[x_filter_idx * UPSCALE_NORMATIVE_TAPS]; |
40 | 0 | int sum = 0; |
41 | 0 | for (int k = 0; k < UPSCALE_NORMATIVE_TAPS; ++k) |
42 | 0 | sum += src_x[k] * x_filter[k]; |
43 | 0 | dst[x] = clip_pixel(ROUND_POWER_OF_TWO(sum, FILTER_BITS)); |
44 | 0 | x_qn += x_step_qn; |
45 | 0 | } |
46 | 0 | src += src_stride; |
47 | 0 | dst += dst_stride; |
48 | 0 | } |
49 | 0 | } |
50 | | |
51 | | void av1_highbd_convolve_horiz_rs_c(const uint16_t *src, int src_stride, |
52 | | uint16_t *dst, int dst_stride, int w, int h, |
53 | | const int16_t *x_filters, int x0_qn, |
54 | 0 | int x_step_qn, int bd) { |
55 | 0 | src -= UPSCALE_NORMATIVE_TAPS / 2 - 1; |
56 | 0 | for (int y = 0; y < h; ++y) { |
57 | 0 | int x_qn = x0_qn; |
58 | 0 | for (int x = 0; x < w; ++x) { |
59 | 0 | const uint16_t *const src_x = &src[x_qn >> RS_SCALE_SUBPEL_BITS]; |
60 | 0 | const int x_filter_idx = |
61 | 0 | (x_qn & RS_SCALE_SUBPEL_MASK) >> RS_SCALE_EXTRA_BITS; |
62 | 0 | assert(x_filter_idx <= RS_SUBPEL_MASK); |
63 | 0 | const int16_t *const x_filter = |
64 | 0 | &x_filters[x_filter_idx * UPSCALE_NORMATIVE_TAPS]; |
65 | 0 | int sum = 0; |
66 | 0 | for (int k = 0; k < UPSCALE_NORMATIVE_TAPS; ++k) |
67 | 0 | sum += src_x[k] * x_filter[k]; |
68 | 0 | dst[x] = clip_pixel_highbd(ROUND_POWER_OF_TWO(sum, FILTER_BITS), bd); |
69 | 0 | x_qn += x_step_qn; |
70 | 0 | } |
71 | 0 | src += src_stride; |
72 | 0 | dst += dst_stride; |
73 | 0 | } |
74 | 0 | } |
75 | | |
76 | | void av1_convolve_2d_sobel_y_c(const uint8_t *src, int src_stride, double *dst, |
77 | | int dst_stride, int w, int h, int dir, |
78 | 0 | double norm) { |
79 | 0 | int16_t im_block[(MAX_SB_SIZE + MAX_FILTER_TAP - 1) * MAX_SB_SIZE]; |
80 | 0 | DECLARE_ALIGNED(256, static const int16_t, sobel_a[3]) = { 1, 0, -1 }; |
81 | 0 | DECLARE_ALIGNED(256, static const int16_t, sobel_b[3]) = { 1, 2, 1 }; |
82 | 0 | const int taps = 3; |
83 | 0 | int im_h = h + taps - 1; |
84 | 0 | int im_stride = w; |
85 | 0 | const int fo_vert = 1; |
86 | 0 | const int fo_horiz = 1; |
87 | | |
88 | | // horizontal filter |
89 | 0 | const uint8_t *src_horiz = src - fo_vert * src_stride; |
90 | 0 | const int16_t *x_filter = dir ? sobel_a : sobel_b; |
91 | 0 | for (int y = 0; y < im_h; ++y) { |
92 | 0 | for (int x = 0; x < w; ++x) { |
93 | 0 | int16_t sum = 0; |
94 | 0 | for (int k = 0; k < taps; ++k) { |
95 | 0 | sum += x_filter[k] * src_horiz[y * src_stride + x - fo_horiz + k]; |
96 | 0 | } |
97 | 0 | im_block[y * im_stride + x] = sum; |
98 | 0 | } |
99 | 0 | } |
100 | | |
101 | | // vertical filter |
102 | 0 | int16_t *src_vert = im_block + fo_vert * im_stride; |
103 | 0 | const int16_t *y_filter = dir ? sobel_b : sobel_a; |
104 | 0 | for (int y = 0; y < h; ++y) { |
105 | 0 | for (int x = 0; x < w; ++x) { |
106 | 0 | int16_t sum = 0; |
107 | 0 | for (int k = 0; k < taps; ++k) { |
108 | 0 | sum += y_filter[k] * src_vert[(y - fo_vert + k) * im_stride + x]; |
109 | 0 | } |
110 | 0 | dst[y * dst_stride + x] = sum * norm; |
111 | 0 | } |
112 | 0 | } |
113 | 0 | } |
114 | | |
115 | | void av1_convolve_2d_sr_c(const uint8_t *src, int src_stride, uint8_t *dst, |
116 | | int dst_stride, int w, int h, |
117 | | const InterpFilterParams *filter_params_x, |
118 | | const InterpFilterParams *filter_params_y, |
119 | | const int subpel_x_qn, const int subpel_y_qn, |
120 | 1.18k | ConvolveParams *conv_params) { |
121 | 1.18k | int16_t im_block[(MAX_SB_SIZE + MAX_FILTER_TAP - 1) * MAX_SB_SIZE]; |
122 | 1.18k | int im_h = h + filter_params_y->taps - 1; |
123 | 1.18k | int im_stride = w; |
124 | 1.18k | assert(w <= MAX_SB_SIZE && h <= MAX_SB_SIZE); |
125 | 1.18k | const int fo_vert = filter_params_y->taps / 2 - 1; |
126 | 1.18k | const int fo_horiz = filter_params_x->taps / 2 - 1; |
127 | 1.18k | const int bd = 8; |
128 | 1.18k | const int bits = |
129 | 1.18k | FILTER_BITS * 2 - conv_params->round_0 - conv_params->round_1; |
130 | | |
131 | | // horizontal filter |
132 | 1.18k | const uint8_t *src_horiz = src - fo_vert * src_stride; |
133 | 1.18k | const int16_t *x_filter = av1_get_interp_filter_subpel_kernel( |
134 | 1.18k | filter_params_x, subpel_x_qn & SUBPEL_MASK); |
135 | 10.6k | for (int y = 0; y < im_h; ++y) { |
136 | 118k | for (int x = 0; x < w; ++x) { |
137 | 108k | int32_t sum = (1 << (bd + FILTER_BITS - 1)); |
138 | 325k | for (int k = 0; k < filter_params_x->taps; ++k) { |
139 | 217k | sum += x_filter[k] * src_horiz[y * src_stride + x - fo_horiz + k]; |
140 | 217k | } |
141 | 108k | assert(0 <= sum && sum < (1 << (bd + FILTER_BITS + 1))); |
142 | 108k | im_block[y * im_stride + x] = |
143 | 108k | (int16_t)ROUND_POWER_OF_TWO(sum, conv_params->round_0); |
144 | 108k | } |
145 | 9.43k | } |
146 | | |
147 | | // vertical filter |
148 | 1.18k | int16_t *src_vert = im_block + fo_vert * im_stride; |
149 | 1.18k | const int16_t *y_filter = av1_get_interp_filter_subpel_kernel( |
150 | 1.18k | filter_params_y, subpel_y_qn & SUBPEL_MASK); |
151 | 1.18k | const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0; |
152 | 9.43k | for (int y = 0; y < h; ++y) { |
153 | 106k | for (int x = 0; x < w; ++x) { |
154 | 97.8k | int32_t sum = 1 << offset_bits; |
155 | 293k | for (int k = 0; k < filter_params_y->taps; ++k) { |
156 | 195k | sum += y_filter[k] * src_vert[(y - fo_vert + k) * im_stride + x]; |
157 | 195k | } |
158 | 97.8k | assert(0 <= sum && sum < (1 << (offset_bits + 2))); |
159 | 97.8k | int16_t res = ROUND_POWER_OF_TWO(sum, conv_params->round_1) - |
160 | 97.8k | ((1 << (offset_bits - conv_params->round_1)) + |
161 | 97.8k | (1 << (offset_bits - conv_params->round_1 - 1))); |
162 | 97.8k | dst[y * dst_stride + x] = clip_pixel(ROUND_POWER_OF_TWO(res, bits)); |
163 | 97.8k | } |
164 | 8.25k | } |
165 | 1.18k | } |
166 | | |
167 | | void av1_convolve_y_sr_c(const uint8_t *src, int src_stride, uint8_t *dst, |
168 | | int dst_stride, int w, int h, |
169 | | const InterpFilterParams *filter_params_y, |
170 | 1.21k | const int subpel_y_qn) { |
171 | 1.21k | const int fo_vert = filter_params_y->taps / 2 - 1; |
172 | | |
173 | | // vertical filter |
174 | 1.21k | const int16_t *y_filter = av1_get_interp_filter_subpel_kernel( |
175 | 1.21k | filter_params_y, subpel_y_qn & SUBPEL_MASK); |
176 | 11.5k | for (int y = 0; y < h; ++y) { |
177 | 123k | for (int x = 0; x < w; ++x) { |
178 | 113k | int32_t res = 0; |
179 | 340k | for (int k = 0; k < filter_params_y->taps; ++k) { |
180 | 227k | res += y_filter[k] * src[(y - fo_vert + k) * src_stride + x]; |
181 | 227k | } |
182 | 113k | dst[y * dst_stride + x] = |
183 | 113k | clip_pixel(ROUND_POWER_OF_TWO(res, FILTER_BITS)); |
184 | 113k | } |
185 | 10.3k | } |
186 | 1.21k | } |
187 | | |
188 | | void av1_convolve_x_sr_c(const uint8_t *src, int src_stride, uint8_t *dst, |
189 | | int dst_stride, int w, int h, |
190 | | const InterpFilterParams *filter_params_x, |
191 | 1.49k | const int subpel_x_qn, ConvolveParams *conv_params) { |
192 | 1.49k | const int fo_horiz = filter_params_x->taps / 2 - 1; |
193 | 1.49k | const int bits = FILTER_BITS - conv_params->round_0; |
194 | | |
195 | 1.49k | assert(bits >= 0); |
196 | 1.49k | assert((FILTER_BITS - conv_params->round_1) >= 0 || |
197 | 1.49k | ((conv_params->round_0 + conv_params->round_1) == 2 * FILTER_BITS)); |
198 | | |
199 | | // horizontal filter |
200 | 1.49k | const int16_t *x_filter = av1_get_interp_filter_subpel_kernel( |
201 | 1.49k | filter_params_x, subpel_x_qn & SUBPEL_MASK); |
202 | | |
203 | 14.2k | for (int y = 0; y < h; ++y) { |
204 | 217k | for (int x = 0; x < w; ++x) { |
205 | 204k | int32_t res = 0; |
206 | 614k | for (int k = 0; k < filter_params_x->taps; ++k) { |
207 | 409k | res += x_filter[k] * src[y * src_stride + x - fo_horiz + k]; |
208 | 409k | } |
209 | 204k | res = ROUND_POWER_OF_TWO(res, conv_params->round_0); |
210 | 204k | dst[y * dst_stride + x] = clip_pixel(ROUND_POWER_OF_TWO(res, bits)); |
211 | 204k | } |
212 | 12.8k | } |
213 | 1.49k | } |
214 | | |
215 | | void av1_dist_wtd_convolve_2d_c(const uint8_t *src, int src_stride, |
216 | | uint8_t *dst, int dst_stride, int w, int h, |
217 | | const InterpFilterParams *filter_params_x, |
218 | | const InterpFilterParams *filter_params_y, |
219 | | const int subpel_x_qn, const int subpel_y_qn, |
220 | 0 | ConvolveParams *conv_params) { |
221 | 0 | CONV_BUF_TYPE *dst16 = conv_params->dst; |
222 | 0 | int dst16_stride = conv_params->dst_stride; |
223 | 0 | int16_t im_block[(MAX_SB_SIZE + MAX_FILTER_TAP - 1) * MAX_SB_SIZE]; |
224 | 0 | int im_h = h + filter_params_y->taps - 1; |
225 | 0 | int im_stride = w; |
226 | 0 | const int fo_vert = filter_params_y->taps / 2 - 1; |
227 | 0 | const int fo_horiz = filter_params_x->taps / 2 - 1; |
228 | 0 | const int bd = 8; |
229 | 0 | const int round_bits = |
230 | 0 | 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1; |
231 | | |
232 | | // horizontal filter |
233 | 0 | const uint8_t *src_horiz = src - fo_vert * src_stride; |
234 | 0 | const int16_t *x_filter = av1_get_interp_filter_subpel_kernel( |
235 | 0 | filter_params_x, subpel_x_qn & SUBPEL_MASK); |
236 | 0 | for (int y = 0; y < im_h; ++y) { |
237 | 0 | for (int x = 0; x < w; ++x) { |
238 | 0 | int32_t sum = (1 << (bd + FILTER_BITS - 1)); |
239 | 0 | for (int k = 0; k < filter_params_x->taps; ++k) { |
240 | 0 | sum += x_filter[k] * src_horiz[y * src_stride + x - fo_horiz + k]; |
241 | 0 | } |
242 | 0 | assert(0 <= sum && sum < (1 << (bd + FILTER_BITS + 1))); |
243 | 0 | im_block[y * im_stride + x] = |
244 | 0 | (int16_t)ROUND_POWER_OF_TWO(sum, conv_params->round_0); |
245 | 0 | } |
246 | 0 | } |
247 | | |
248 | | // vertical filter |
249 | 0 | int16_t *src_vert = im_block + fo_vert * im_stride; |
250 | 0 | const int16_t *y_filter = av1_get_interp_filter_subpel_kernel( |
251 | 0 | filter_params_y, subpel_y_qn & SUBPEL_MASK); |
252 | 0 | const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0; |
253 | 0 | for (int y = 0; y < h; ++y) { |
254 | 0 | for (int x = 0; x < w; ++x) { |
255 | 0 | int32_t sum = 1 << offset_bits; |
256 | 0 | for (int k = 0; k < filter_params_y->taps; ++k) { |
257 | 0 | sum += y_filter[k] * src_vert[(y - fo_vert + k) * im_stride + x]; |
258 | 0 | } |
259 | 0 | assert(0 <= sum && sum < (1 << (offset_bits + 2))); |
260 | 0 | CONV_BUF_TYPE res = ROUND_POWER_OF_TWO(sum, conv_params->round_1); |
261 | 0 | if (conv_params->do_average) { |
262 | 0 | int32_t tmp = dst16[y * dst16_stride + x]; |
263 | 0 | if (conv_params->use_dist_wtd_comp_avg) { |
264 | 0 | tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset; |
265 | 0 | tmp = tmp >> DIST_PRECISION_BITS; |
266 | 0 | } else { |
267 | 0 | tmp += res; |
268 | 0 | tmp = tmp >> 1; |
269 | 0 | } |
270 | 0 | tmp -= (1 << (offset_bits - conv_params->round_1)) + |
271 | 0 | (1 << (offset_bits - conv_params->round_1 - 1)); |
272 | 0 | dst[y * dst_stride + x] = |
273 | 0 | clip_pixel(ROUND_POWER_OF_TWO(tmp, round_bits)); |
274 | 0 | } else { |
275 | 0 | dst16[y * dst16_stride + x] = res; |
276 | 0 | } |
277 | 0 | } |
278 | 0 | } |
279 | 0 | } |
280 | | |
281 | | void av1_dist_wtd_convolve_y_c(const uint8_t *src, int src_stride, uint8_t *dst, |
282 | | int dst_stride, int w, int h, |
283 | | const InterpFilterParams *filter_params_y, |
284 | | const int subpel_y_qn, |
285 | 0 | ConvolveParams *conv_params) { |
286 | 0 | CONV_BUF_TYPE *dst16 = conv_params->dst; |
287 | 0 | int dst16_stride = conv_params->dst_stride; |
288 | 0 | const int fo_vert = filter_params_y->taps / 2 - 1; |
289 | 0 | const int bits = FILTER_BITS - conv_params->round_0; |
290 | 0 | const int bd = 8; |
291 | 0 | const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0; |
292 | 0 | const int round_offset = (1 << (offset_bits - conv_params->round_1)) + |
293 | 0 | (1 << (offset_bits - conv_params->round_1 - 1)); |
294 | 0 | const int round_bits = |
295 | 0 | 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1; |
296 | | |
297 | | // vertical filter |
298 | 0 | const int16_t *y_filter = av1_get_interp_filter_subpel_kernel( |
299 | 0 | filter_params_y, subpel_y_qn & SUBPEL_MASK); |
300 | 0 | for (int y = 0; y < h; ++y) { |
301 | 0 | for (int x = 0; x < w; ++x) { |
302 | 0 | int32_t res = 0; |
303 | 0 | for (int k = 0; k < filter_params_y->taps; ++k) { |
304 | 0 | res += y_filter[k] * src[(y - fo_vert + k) * src_stride + x]; |
305 | 0 | } |
306 | 0 | res *= (1 << bits); |
307 | 0 | res = ROUND_POWER_OF_TWO(res, conv_params->round_1) + round_offset; |
308 | |
|
309 | 0 | if (conv_params->do_average) { |
310 | 0 | int32_t tmp = dst16[y * dst16_stride + x]; |
311 | 0 | if (conv_params->use_dist_wtd_comp_avg) { |
312 | 0 | tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset; |
313 | 0 | tmp = tmp >> DIST_PRECISION_BITS; |
314 | 0 | } else { |
315 | 0 | tmp += res; |
316 | 0 | tmp = tmp >> 1; |
317 | 0 | } |
318 | 0 | tmp -= round_offset; |
319 | 0 | dst[y * dst_stride + x] = |
320 | 0 | clip_pixel(ROUND_POWER_OF_TWO(tmp, round_bits)); |
321 | 0 | } else { |
322 | 0 | dst16[y * dst16_stride + x] = res; |
323 | 0 | } |
324 | 0 | } |
325 | 0 | } |
326 | 0 | } |
327 | | |
328 | | void av1_dist_wtd_convolve_x_c(const uint8_t *src, int src_stride, uint8_t *dst, |
329 | | int dst_stride, int w, int h, |
330 | | const InterpFilterParams *filter_params_x, |
331 | | const int subpel_x_qn, |
332 | 0 | ConvolveParams *conv_params) { |
333 | 0 | CONV_BUF_TYPE *dst16 = conv_params->dst; |
334 | 0 | int dst16_stride = conv_params->dst_stride; |
335 | 0 | const int fo_horiz = filter_params_x->taps / 2 - 1; |
336 | 0 | const int bits = FILTER_BITS - conv_params->round_1; |
337 | 0 | const int bd = 8; |
338 | 0 | const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0; |
339 | 0 | const int round_offset = (1 << (offset_bits - conv_params->round_1)) + |
340 | 0 | (1 << (offset_bits - conv_params->round_1 - 1)); |
341 | 0 | const int round_bits = |
342 | 0 | 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1; |
343 | | |
344 | | // horizontal filter |
345 | 0 | const int16_t *x_filter = av1_get_interp_filter_subpel_kernel( |
346 | 0 | filter_params_x, subpel_x_qn & SUBPEL_MASK); |
347 | 0 | for (int y = 0; y < h; ++y) { |
348 | 0 | for (int x = 0; x < w; ++x) { |
349 | 0 | int32_t res = 0; |
350 | 0 | for (int k = 0; k < filter_params_x->taps; ++k) { |
351 | 0 | res += x_filter[k] * src[y * src_stride + x - fo_horiz + k]; |
352 | 0 | } |
353 | 0 | res = (1 << bits) * ROUND_POWER_OF_TWO(res, conv_params->round_0); |
354 | 0 | res += round_offset; |
355 | |
|
356 | 0 | if (conv_params->do_average) { |
357 | 0 | int32_t tmp = dst16[y * dst16_stride + x]; |
358 | 0 | if (conv_params->use_dist_wtd_comp_avg) { |
359 | 0 | tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset; |
360 | 0 | tmp = tmp >> DIST_PRECISION_BITS; |
361 | 0 | } else { |
362 | 0 | tmp += res; |
363 | 0 | tmp = tmp >> 1; |
364 | 0 | } |
365 | 0 | tmp -= round_offset; |
366 | 0 | dst[y * dst_stride + x] = |
367 | 0 | clip_pixel(ROUND_POWER_OF_TWO(tmp, round_bits)); |
368 | 0 | } else { |
369 | 0 | dst16[y * dst16_stride + x] = res; |
370 | 0 | } |
371 | 0 | } |
372 | 0 | } |
373 | 0 | } |
374 | | |
375 | | void av1_dist_wtd_convolve_2d_copy_c(const uint8_t *src, int src_stride, |
376 | | uint8_t *dst, int dst_stride, int w, int h, |
377 | 0 | ConvolveParams *conv_params) { |
378 | 0 | CONV_BUF_TYPE *dst16 = conv_params->dst; |
379 | 0 | int dst16_stride = conv_params->dst_stride; |
380 | 0 | const int bits = |
381 | 0 | FILTER_BITS * 2 - conv_params->round_1 - conv_params->round_0; |
382 | 0 | const int bd = 8; |
383 | 0 | const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0; |
384 | 0 | const int round_offset = (1 << (offset_bits - conv_params->round_1)) + |
385 | 0 | (1 << (offset_bits - conv_params->round_1 - 1)); |
386 | |
|
387 | 0 | for (int y = 0; y < h; ++y) { |
388 | 0 | for (int x = 0; x < w; ++x) { |
389 | 0 | CONV_BUF_TYPE res = src[y * src_stride + x] << bits; |
390 | 0 | res += round_offset; |
391 | |
|
392 | 0 | if (conv_params->do_average) { |
393 | 0 | int32_t tmp = dst16[y * dst16_stride + x]; |
394 | 0 | if (conv_params->use_dist_wtd_comp_avg) { |
395 | 0 | tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset; |
396 | 0 | tmp = tmp >> DIST_PRECISION_BITS; |
397 | 0 | } else { |
398 | 0 | tmp += res; |
399 | 0 | tmp = tmp >> 1; |
400 | 0 | } |
401 | 0 | tmp -= round_offset; |
402 | 0 | dst[y * dst_stride + x] = clip_pixel(ROUND_POWER_OF_TWO(tmp, bits)); |
403 | 0 | } else { |
404 | 0 | dst16[y * dst16_stride + x] = res; |
405 | 0 | } |
406 | 0 | } |
407 | 0 | } |
408 | 0 | } |
409 | | |
410 | | void av1_convolve_2d_scale_c(const uint8_t *src, int src_stride, uint8_t *dst, |
411 | | int dst_stride, int w, int h, |
412 | | const InterpFilterParams *filter_params_x, |
413 | | const InterpFilterParams *filter_params_y, |
414 | | const int subpel_x_qn, const int x_step_qn, |
415 | | const int subpel_y_qn, const int y_step_qn, |
416 | 0 | ConvolveParams *conv_params) { |
417 | 0 | int16_t im_block[(2 * MAX_SB_SIZE + MAX_FILTER_TAP) * MAX_SB_SIZE]; |
418 | 0 | int im_h = (((h - 1) * y_step_qn + subpel_y_qn) >> SCALE_SUBPEL_BITS) + |
419 | 0 | filter_params_y->taps; |
420 | 0 | CONV_BUF_TYPE *dst16 = conv_params->dst; |
421 | 0 | const int dst16_stride = conv_params->dst_stride; |
422 | 0 | const int bits = |
423 | 0 | FILTER_BITS * 2 - conv_params->round_0 - conv_params->round_1; |
424 | 0 | assert(bits >= 0); |
425 | 0 | int im_stride = w; |
426 | 0 | const int fo_vert = filter_params_y->taps / 2 - 1; |
427 | 0 | const int fo_horiz = filter_params_x->taps / 2 - 1; |
428 | 0 | const int bd = 8; |
429 | | |
430 | | // horizontal filter |
431 | 0 | const uint8_t *src_horiz = src - fo_vert * src_stride; |
432 | 0 | for (int y = 0; y < im_h; ++y) { |
433 | 0 | int x_qn = subpel_x_qn; |
434 | 0 | for (int x = 0; x < w; ++x, x_qn += x_step_qn) { |
435 | 0 | const uint8_t *const src_x = &src_horiz[(x_qn >> SCALE_SUBPEL_BITS)]; |
436 | 0 | const int x_filter_idx = (x_qn & SCALE_SUBPEL_MASK) >> SCALE_EXTRA_BITS; |
437 | 0 | assert(x_filter_idx < SUBPEL_SHIFTS); |
438 | 0 | const int16_t *x_filter = |
439 | 0 | av1_get_interp_filter_subpel_kernel(filter_params_x, x_filter_idx); |
440 | 0 | int32_t sum = (1 << (bd + FILTER_BITS - 1)); |
441 | 0 | for (int k = 0; k < filter_params_x->taps; ++k) { |
442 | 0 | sum += x_filter[k] * src_x[k - fo_horiz]; |
443 | 0 | } |
444 | 0 | assert(0 <= sum && sum < (1 << (bd + FILTER_BITS + 1))); |
445 | 0 | im_block[y * im_stride + x] = |
446 | 0 | (int16_t)ROUND_POWER_OF_TWO(sum, conv_params->round_0); |
447 | 0 | } |
448 | 0 | src_horiz += src_stride; |
449 | 0 | } |
450 | | |
451 | | // vertical filter |
452 | 0 | int16_t *src_vert = im_block + fo_vert * im_stride; |
453 | 0 | const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0; |
454 | 0 | for (int x = 0; x < w; ++x) { |
455 | 0 | int y_qn = subpel_y_qn; |
456 | 0 | for (int y = 0; y < h; ++y, y_qn += y_step_qn) { |
457 | 0 | const int16_t *src_y = &src_vert[(y_qn >> SCALE_SUBPEL_BITS) * im_stride]; |
458 | 0 | const int y_filter_idx = (y_qn & SCALE_SUBPEL_MASK) >> SCALE_EXTRA_BITS; |
459 | 0 | assert(y_filter_idx < SUBPEL_SHIFTS); |
460 | 0 | const int16_t *y_filter = |
461 | 0 | av1_get_interp_filter_subpel_kernel(filter_params_y, y_filter_idx); |
462 | 0 | int32_t sum = 1 << offset_bits; |
463 | 0 | for (int k = 0; k < filter_params_y->taps; ++k) { |
464 | 0 | sum += y_filter[k] * src_y[(k - fo_vert) * im_stride]; |
465 | 0 | } |
466 | 0 | assert(0 <= sum && sum < (1 << (offset_bits + 2))); |
467 | 0 | CONV_BUF_TYPE res = ROUND_POWER_OF_TWO(sum, conv_params->round_1); |
468 | 0 | if (conv_params->is_compound) { |
469 | 0 | if (conv_params->do_average) { |
470 | 0 | int32_t tmp = dst16[y * dst16_stride + x]; |
471 | 0 | if (conv_params->use_dist_wtd_comp_avg) { |
472 | 0 | tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset; |
473 | 0 | tmp = tmp >> DIST_PRECISION_BITS; |
474 | 0 | } else { |
475 | 0 | tmp += res; |
476 | 0 | tmp = tmp >> 1; |
477 | 0 | } |
478 | | /* Subtract round offset and convolve round */ |
479 | 0 | tmp = tmp - ((1 << (offset_bits - conv_params->round_1)) + |
480 | 0 | (1 << (offset_bits - conv_params->round_1 - 1))); |
481 | 0 | dst[y * dst_stride + x] = clip_pixel(ROUND_POWER_OF_TWO(tmp, bits)); |
482 | 0 | } else { |
483 | 0 | dst16[y * dst16_stride + x] = res; |
484 | 0 | } |
485 | 0 | } else { |
486 | | /* Subtract round offset and convolve round */ |
487 | 0 | int32_t tmp = res - ((1 << (offset_bits - conv_params->round_1)) + |
488 | 0 | (1 << (offset_bits - conv_params->round_1 - 1))); |
489 | 0 | dst[y * dst_stride + x] = clip_pixel(ROUND_POWER_OF_TWO(tmp, bits)); |
490 | 0 | } |
491 | 0 | } |
492 | 0 | src_vert++; |
493 | 0 | } |
494 | 0 | } |
495 | | |
496 | | static void convolve_2d_scale_wrapper( |
497 | | const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, |
498 | | int h, const InterpFilterParams *filter_params_x, |
499 | | const InterpFilterParams *filter_params_y, const int subpel_x_qn, |
500 | | const int x_step_qn, const int subpel_y_qn, const int y_step_qn, |
501 | 0 | ConvolveParams *conv_params) { |
502 | 0 | if (conv_params->is_compound) { |
503 | 0 | assert(conv_params->dst != NULL); |
504 | 0 | } |
505 | 0 | av1_convolve_2d_scale(src, src_stride, dst, dst_stride, w, h, filter_params_x, |
506 | 0 | filter_params_y, subpel_x_qn, x_step_qn, subpel_y_qn, |
507 | 0 | y_step_qn, conv_params); |
508 | 0 | } |
509 | | |
510 | | static void convolve_2d_facade_compound( |
511 | | const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, |
512 | | int h, const InterpFilterParams *filter_params_x, |
513 | | const InterpFilterParams *filter_params_y, const int subpel_x_qn, |
514 | 0 | const int subpel_y_qn, ConvolveParams *conv_params) { |
515 | 0 | const bool need_x = subpel_x_qn != 0; |
516 | 0 | const bool need_y = subpel_y_qn != 0; |
517 | 0 | if (!need_x && !need_y) { |
518 | 0 | av1_dist_wtd_convolve_2d_copy(src, src_stride, dst, dst_stride, w, h, |
519 | 0 | conv_params); |
520 | 0 | } else if (need_x && !need_y) { |
521 | 0 | av1_dist_wtd_convolve_x(src, src_stride, dst, dst_stride, w, h, |
522 | 0 | filter_params_x, subpel_x_qn, conv_params); |
523 | 0 | } else if (!need_x && need_y) { |
524 | 0 | av1_dist_wtd_convolve_y(src, src_stride, dst, dst_stride, w, h, |
525 | 0 | filter_params_y, subpel_y_qn, conv_params); |
526 | 0 | } else { |
527 | 0 | assert(need_y && need_x); |
528 | 0 | av1_dist_wtd_convolve_2d(src, src_stride, dst, dst_stride, w, h, |
529 | 0 | filter_params_x, filter_params_y, subpel_x_qn, |
530 | 0 | subpel_y_qn, conv_params); |
531 | 0 | } |
532 | 0 | } |
533 | | |
534 | | static void convolve_2d_facade_single( |
535 | | const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, |
536 | | int h, const InterpFilterParams *filter_params_x, |
537 | | const InterpFilterParams *filter_params_y, const int subpel_x_qn, |
538 | 4.33k | const int subpel_y_qn, ConvolveParams *conv_params) { |
539 | 4.33k | const bool need_x = subpel_x_qn != 0; |
540 | 4.33k | const bool need_y = subpel_y_qn != 0; |
541 | 4.33k | if (!need_x && !need_y) { |
542 | 4.33k | aom_convolve_copy(src, src_stride, dst, dst_stride, w, h); |
543 | 4.33k | } else if (need_x && !need_y) { |
544 | 0 | av1_convolve_x_sr(src, src_stride, dst, dst_stride, w, h, filter_params_x, |
545 | 0 | subpel_x_qn, conv_params); |
546 | 0 | } else if (!need_x && need_y) { |
547 | 0 | av1_convolve_y_sr(src, src_stride, dst, dst_stride, w, h, filter_params_y, |
548 | 0 | subpel_y_qn); |
549 | 0 | } else { |
550 | 0 | assert(need_x && need_y); |
551 | 0 | av1_convolve_2d_sr(src, src_stride, dst, dst_stride, w, h, filter_params_x, |
552 | 0 | filter_params_y, subpel_x_qn, subpel_y_qn, conv_params); |
553 | 0 | } |
554 | 4.33k | } |
555 | | |
556 | | void av1_convolve_2d_facade(const uint8_t *src, int src_stride, uint8_t *dst, |
557 | | int dst_stride, int w, int h, |
558 | | const InterpFilterParams *interp_filters[2], |
559 | | const int subpel_x_qn, int x_step_q4, |
560 | | const int subpel_y_qn, int y_step_q4, int scaled, |
561 | 8.22k | ConvolveParams *conv_params) { |
562 | 8.22k | (void)x_step_q4; |
563 | 8.22k | (void)y_step_q4; |
564 | 8.22k | (void)dst; |
565 | 8.22k | (void)dst_stride; |
566 | | |
567 | 8.22k | const InterpFilterParams *filter_params_x = interp_filters[0]; |
568 | 8.22k | const InterpFilterParams *filter_params_y = interp_filters[1]; |
569 | | |
570 | | // TODO(jingning, yunqing): Add SIMD support to 2-tap filter case. |
571 | | // Do we have SIMD support to 4-tap case? |
572 | | // 2-tap filter indicates that it is for IntraBC. |
573 | 8.22k | if (filter_params_x->taps == 2 || filter_params_y->taps == 2) { |
574 | 8.22k | assert(filter_params_x->taps == 2 && filter_params_y->taps == 2); |
575 | 8.22k | assert(!scaled); |
576 | 8.22k | if (subpel_x_qn && subpel_y_qn) { |
577 | 1.18k | av1_convolve_2d_sr_c(src, src_stride, dst, dst_stride, w, h, |
578 | 1.18k | filter_params_x, filter_params_y, subpel_x_qn, |
579 | 1.18k | subpel_y_qn, conv_params); |
580 | 1.18k | return; |
581 | 7.04k | } else if (subpel_x_qn) { |
582 | 1.49k | av1_convolve_x_sr_c(src, src_stride, dst, dst_stride, w, h, |
583 | 1.49k | filter_params_x, subpel_x_qn, conv_params); |
584 | 1.49k | return; |
585 | 5.55k | } else if (subpel_y_qn) { |
586 | 1.21k | av1_convolve_y_sr_c(src, src_stride, dst, dst_stride, w, h, |
587 | 1.21k | filter_params_y, subpel_y_qn); |
588 | 1.21k | return; |
589 | 1.21k | } |
590 | 8.22k | } |
591 | | |
592 | 4.33k | if (scaled) { |
593 | 0 | convolve_2d_scale_wrapper(src, src_stride, dst, dst_stride, w, h, |
594 | 0 | filter_params_x, filter_params_y, subpel_x_qn, |
595 | 0 | x_step_q4, subpel_y_qn, y_step_q4, conv_params); |
596 | 4.33k | } else if (conv_params->is_compound) { |
597 | 0 | convolve_2d_facade_compound(src, src_stride, dst, dst_stride, w, h, |
598 | 0 | filter_params_x, filter_params_y, subpel_x_qn, |
599 | 0 | subpel_y_qn, conv_params); |
600 | 4.33k | } else { |
601 | 4.33k | convolve_2d_facade_single(src, src_stride, dst, dst_stride, w, h, |
602 | 4.33k | filter_params_x, filter_params_y, subpel_x_qn, |
603 | 4.33k | subpel_y_qn, conv_params); |
604 | 4.33k | } |
605 | 4.33k | } |
606 | | |
607 | | #if CONFIG_AV1_HIGHBITDEPTH |
608 | | void av1_highbd_convolve_x_sr_c(const uint16_t *src, int src_stride, |
609 | | uint16_t *dst, int dst_stride, int w, int h, |
610 | | const InterpFilterParams *filter_params_x, |
611 | | const int subpel_x_qn, |
612 | 2.76k | ConvolveParams *conv_params, int bd) { |
613 | 2.76k | const int fo_horiz = filter_params_x->taps / 2 - 1; |
614 | 2.76k | const int bits = FILTER_BITS - conv_params->round_0; |
615 | | |
616 | 2.76k | assert(bits >= 0); |
617 | 2.76k | assert((FILTER_BITS - conv_params->round_1) >= 0 || |
618 | 2.76k | ((conv_params->round_0 + conv_params->round_1) == 2 * FILTER_BITS)); |
619 | | |
620 | | // horizontal filter |
621 | 2.76k | const int16_t *x_filter = av1_get_interp_filter_subpel_kernel( |
622 | 2.76k | filter_params_x, subpel_x_qn & SUBPEL_MASK); |
623 | 29.2k | for (int y = 0; y < h; ++y) { |
624 | 530k | for (int x = 0; x < w; ++x) { |
625 | 504k | int32_t res = 0; |
626 | 1.51M | for (int k = 0; k < filter_params_x->taps; ++k) { |
627 | 1.00M | res += x_filter[k] * src[y * src_stride + x - fo_horiz + k]; |
628 | 1.00M | } |
629 | 504k | res = ROUND_POWER_OF_TWO(res, conv_params->round_0); |
630 | 504k | dst[y * dst_stride + x] = |
631 | 504k | clip_pixel_highbd(ROUND_POWER_OF_TWO(res, bits), bd); |
632 | 504k | } |
633 | 26.4k | } |
634 | 2.76k | } |
635 | | |
636 | | void av1_highbd_convolve_y_sr_c(const uint16_t *src, int src_stride, |
637 | | uint16_t *dst, int dst_stride, int w, int h, |
638 | | const InterpFilterParams *filter_params_y, |
639 | 2.22k | const int subpel_y_qn, int bd) { |
640 | 2.22k | const int fo_vert = filter_params_y->taps / 2 - 1; |
641 | | // vertical filter |
642 | 2.22k | const int16_t *y_filter = av1_get_interp_filter_subpel_kernel( |
643 | 2.22k | filter_params_y, subpel_y_qn & SUBPEL_MASK); |
644 | 20.0k | for (int y = 0; y < h; ++y) { |
645 | 244k | for (int x = 0; x < w; ++x) { |
646 | 226k | int32_t res = 0; |
647 | 680k | for (int k = 0; k < filter_params_y->taps; ++k) { |
648 | 453k | res += y_filter[k] * src[(y - fo_vert + k) * src_stride + x]; |
649 | 453k | } |
650 | 226k | dst[y * dst_stride + x] = |
651 | 226k | clip_pixel_highbd(ROUND_POWER_OF_TWO(res, FILTER_BITS), bd); |
652 | 226k | } |
653 | 17.8k | } |
654 | 2.22k | } |
655 | | |
656 | | void av1_highbd_convolve_2d_sr_c(const uint16_t *src, int src_stride, |
657 | | uint16_t *dst, int dst_stride, int w, int h, |
658 | | const InterpFilterParams *filter_params_x, |
659 | | const InterpFilterParams *filter_params_y, |
660 | | const int subpel_x_qn, const int subpel_y_qn, |
661 | 2.81k | ConvolveParams *conv_params, int bd) { |
662 | 2.81k | int16_t im_block[(MAX_SB_SIZE + MAX_FILTER_TAP - 1) * MAX_SB_SIZE]; |
663 | 2.81k | int im_h = h + filter_params_y->taps - 1; |
664 | 2.81k | int im_stride = w; |
665 | 2.81k | assert(w <= MAX_SB_SIZE && h <= MAX_SB_SIZE); |
666 | 2.81k | const int fo_vert = filter_params_y->taps / 2 - 1; |
667 | 2.81k | const int fo_horiz = filter_params_x->taps / 2 - 1; |
668 | 2.81k | const int bits = |
669 | 2.81k | FILTER_BITS * 2 - conv_params->round_0 - conv_params->round_1; |
670 | 2.81k | assert(bits >= 0); |
671 | | |
672 | | // horizontal filter |
673 | 2.81k | const uint16_t *src_horiz = src - fo_vert * src_stride; |
674 | 2.81k | const int16_t *x_filter = av1_get_interp_filter_subpel_kernel( |
675 | 2.81k | filter_params_x, subpel_x_qn & SUBPEL_MASK); |
676 | 29.4k | for (int y = 0; y < im_h; ++y) { |
677 | 358k | for (int x = 0; x < w; ++x) { |
678 | 331k | int32_t sum = (1 << (bd + FILTER_BITS - 1)); |
679 | 995k | for (int k = 0; k < filter_params_x->taps; ++k) { |
680 | 663k | sum += x_filter[k] * src_horiz[y * src_stride + x - fo_horiz + k]; |
681 | 663k | } |
682 | 331k | assert(0 <= sum && sum < (1 << (bd + FILTER_BITS + 1))); |
683 | 331k | im_block[y * im_stride + x] = |
684 | 331k | ROUND_POWER_OF_TWO(sum, conv_params->round_0); |
685 | 331k | } |
686 | 26.6k | } |
687 | | |
688 | | // vertical filter |
689 | 2.81k | int16_t *src_vert = im_block + fo_vert * im_stride; |
690 | 2.81k | const int16_t *y_filter = av1_get_interp_filter_subpel_kernel( |
691 | 2.81k | filter_params_y, subpel_y_qn & SUBPEL_MASK); |
692 | 2.81k | const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0; |
693 | 26.6k | for (int y = 0; y < h; ++y) { |
694 | 330k | for (int x = 0; x < w; ++x) { |
695 | 306k | int32_t sum = 1 << offset_bits; |
696 | 920k | for (int k = 0; k < filter_params_y->taps; ++k) { |
697 | 613k | sum += y_filter[k] * src_vert[(y - fo_vert + k) * im_stride + x]; |
698 | 613k | } |
699 | 306k | assert(0 <= sum && sum < (1 << (offset_bits + 2))); |
700 | 306k | int32_t res = ROUND_POWER_OF_TWO(sum, conv_params->round_1) - |
701 | 306k | ((1 << (offset_bits - conv_params->round_1)) + |
702 | 306k | (1 << (offset_bits - conv_params->round_1 - 1))); |
703 | 306k | dst[y * dst_stride + x] = |
704 | 306k | clip_pixel_highbd(ROUND_POWER_OF_TWO(res, bits), bd); |
705 | 306k | } |
706 | 23.8k | } |
707 | 2.81k | } |
708 | | |
709 | | void av1_highbd_dist_wtd_convolve_2d_c( |
710 | | const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, |
711 | | int h, const InterpFilterParams *filter_params_x, |
712 | | const InterpFilterParams *filter_params_y, const int subpel_x_qn, |
713 | 0 | const int subpel_y_qn, ConvolveParams *conv_params, int bd) { |
714 | 0 | int x, y, k; |
715 | 0 | int16_t im_block[(MAX_SB_SIZE + MAX_FILTER_TAP - 1) * MAX_SB_SIZE]; |
716 | 0 | CONV_BUF_TYPE *dst16 = conv_params->dst; |
717 | 0 | int dst16_stride = conv_params->dst_stride; |
718 | 0 | int im_h = h + filter_params_y->taps - 1; |
719 | 0 | int im_stride = w; |
720 | 0 | const int fo_vert = filter_params_y->taps / 2 - 1; |
721 | 0 | const int fo_horiz = filter_params_x->taps / 2 - 1; |
722 | 0 | const int round_bits = |
723 | 0 | 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1; |
724 | 0 | assert(round_bits >= 0); |
725 | | |
726 | | // horizontal filter |
727 | 0 | const uint16_t *src_horiz = src - fo_vert * src_stride; |
728 | 0 | const int16_t *x_filter = av1_get_interp_filter_subpel_kernel( |
729 | 0 | filter_params_x, subpel_x_qn & SUBPEL_MASK); |
730 | 0 | for (y = 0; y < im_h; ++y) { |
731 | 0 | for (x = 0; x < w; ++x) { |
732 | 0 | int32_t sum = (1 << (bd + FILTER_BITS - 1)); |
733 | 0 | for (k = 0; k < filter_params_x->taps; ++k) { |
734 | 0 | sum += x_filter[k] * src_horiz[y * src_stride + x - fo_horiz + k]; |
735 | 0 | } |
736 | 0 | assert(0 <= sum && sum < (1 << (bd + FILTER_BITS + 1))); |
737 | 0 | (void)bd; |
738 | 0 | im_block[y * im_stride + x] = |
739 | 0 | (int16_t)ROUND_POWER_OF_TWO(sum, conv_params->round_0); |
740 | 0 | } |
741 | 0 | } |
742 | | |
743 | | // vertical filter |
744 | 0 | int16_t *src_vert = im_block + fo_vert * im_stride; |
745 | 0 | const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0; |
746 | 0 | const int16_t *y_filter = av1_get_interp_filter_subpel_kernel( |
747 | 0 | filter_params_y, subpel_y_qn & SUBPEL_MASK); |
748 | 0 | for (y = 0; y < h; ++y) { |
749 | 0 | for (x = 0; x < w; ++x) { |
750 | 0 | int32_t sum = 1 << offset_bits; |
751 | 0 | for (k = 0; k < filter_params_y->taps; ++k) { |
752 | 0 | sum += y_filter[k] * src_vert[(y - fo_vert + k) * im_stride + x]; |
753 | 0 | } |
754 | 0 | assert(0 <= sum && sum < (1 << (offset_bits + 2))); |
755 | 0 | CONV_BUF_TYPE res = ROUND_POWER_OF_TWO(sum, conv_params->round_1); |
756 | 0 | if (conv_params->do_average) { |
757 | 0 | int32_t tmp = dst16[y * dst16_stride + x]; |
758 | 0 | if (conv_params->use_dist_wtd_comp_avg) { |
759 | 0 | tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset; |
760 | 0 | tmp = tmp >> DIST_PRECISION_BITS; |
761 | 0 | } else { |
762 | 0 | tmp += res; |
763 | 0 | tmp = tmp >> 1; |
764 | 0 | } |
765 | 0 | tmp -= (1 << (offset_bits - conv_params->round_1)) + |
766 | 0 | (1 << (offset_bits - conv_params->round_1 - 1)); |
767 | 0 | dst[y * dst_stride + x] = |
768 | 0 | clip_pixel_highbd(ROUND_POWER_OF_TWO(tmp, round_bits), bd); |
769 | 0 | } else { |
770 | 0 | dst16[y * dst16_stride + x] = res; |
771 | 0 | } |
772 | 0 | } |
773 | 0 | } |
774 | 0 | } |
775 | | |
776 | | void av1_highbd_dist_wtd_convolve_x_c(const uint16_t *src, int src_stride, |
777 | | uint16_t *dst, int dst_stride, int w, |
778 | | int h, |
779 | | const InterpFilterParams *filter_params_x, |
780 | | const int subpel_x_qn, |
781 | 0 | ConvolveParams *conv_params, int bd) { |
782 | 0 | CONV_BUF_TYPE *dst16 = conv_params->dst; |
783 | 0 | int dst16_stride = conv_params->dst_stride; |
784 | 0 | const int fo_horiz = filter_params_x->taps / 2 - 1; |
785 | 0 | const int bits = FILTER_BITS - conv_params->round_1; |
786 | 0 | const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0; |
787 | 0 | const int round_offset = (1 << (offset_bits - conv_params->round_1)) + |
788 | 0 | (1 << (offset_bits - conv_params->round_1 - 1)); |
789 | 0 | const int round_bits = |
790 | 0 | 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1; |
791 | 0 | assert(round_bits >= 0); |
792 | 0 | assert(bits >= 0); |
793 | | // horizontal filter |
794 | 0 | const int16_t *x_filter = av1_get_interp_filter_subpel_kernel( |
795 | 0 | filter_params_x, subpel_x_qn & SUBPEL_MASK); |
796 | 0 | for (int y = 0; y < h; ++y) { |
797 | 0 | for (int x = 0; x < w; ++x) { |
798 | 0 | int32_t res = 0; |
799 | 0 | for (int k = 0; k < filter_params_x->taps; ++k) { |
800 | 0 | res += x_filter[k] * src[y * src_stride + x - fo_horiz + k]; |
801 | 0 | } |
802 | 0 | res = (1 << bits) * ROUND_POWER_OF_TWO(res, conv_params->round_0); |
803 | 0 | res += round_offset; |
804 | |
|
805 | 0 | if (conv_params->do_average) { |
806 | 0 | int32_t tmp = dst16[y * dst16_stride + x]; |
807 | 0 | if (conv_params->use_dist_wtd_comp_avg) { |
808 | 0 | tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset; |
809 | 0 | tmp = tmp >> DIST_PRECISION_BITS; |
810 | 0 | } else { |
811 | 0 | tmp += res; |
812 | 0 | tmp = tmp >> 1; |
813 | 0 | } |
814 | 0 | tmp -= round_offset; |
815 | 0 | dst[y * dst_stride + x] = |
816 | 0 | clip_pixel_highbd(ROUND_POWER_OF_TWO(tmp, round_bits), bd); |
817 | 0 | } else { |
818 | 0 | dst16[y * dst16_stride + x] = res; |
819 | 0 | } |
820 | 0 | } |
821 | 0 | } |
822 | 0 | } |
823 | | |
824 | | void av1_highbd_dist_wtd_convolve_y_c(const uint16_t *src, int src_stride, |
825 | | uint16_t *dst, int dst_stride, int w, |
826 | | int h, |
827 | | const InterpFilterParams *filter_params_y, |
828 | | const int subpel_y_qn, |
829 | 0 | ConvolveParams *conv_params, int bd) { |
830 | 0 | CONV_BUF_TYPE *dst16 = conv_params->dst; |
831 | 0 | int dst16_stride = conv_params->dst_stride; |
832 | 0 | const int fo_vert = filter_params_y->taps / 2 - 1; |
833 | 0 | const int bits = FILTER_BITS - conv_params->round_0; |
834 | 0 | const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0; |
835 | 0 | const int round_offset = (1 << (offset_bits - conv_params->round_1)) + |
836 | 0 | (1 << (offset_bits - conv_params->round_1 - 1)); |
837 | 0 | const int round_bits = |
838 | 0 | 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1; |
839 | 0 | assert(round_bits >= 0); |
840 | 0 | assert(bits >= 0); |
841 | | // vertical filter |
842 | 0 | const int16_t *y_filter = av1_get_interp_filter_subpel_kernel( |
843 | 0 | filter_params_y, subpel_y_qn & SUBPEL_MASK); |
844 | 0 | for (int y = 0; y < h; ++y) { |
845 | 0 | for (int x = 0; x < w; ++x) { |
846 | 0 | int32_t res = 0; |
847 | 0 | for (int k = 0; k < filter_params_y->taps; ++k) { |
848 | 0 | res += y_filter[k] * src[(y - fo_vert + k) * src_stride + x]; |
849 | 0 | } |
850 | 0 | res *= (1 << bits); |
851 | 0 | res = ROUND_POWER_OF_TWO(res, conv_params->round_1) + round_offset; |
852 | |
|
853 | 0 | if (conv_params->do_average) { |
854 | 0 | int32_t tmp = dst16[y * dst16_stride + x]; |
855 | 0 | if (conv_params->use_dist_wtd_comp_avg) { |
856 | 0 | tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset; |
857 | 0 | tmp = tmp >> DIST_PRECISION_BITS; |
858 | 0 | } else { |
859 | 0 | tmp += res; |
860 | 0 | tmp = tmp >> 1; |
861 | 0 | } |
862 | 0 | tmp -= round_offset; |
863 | 0 | dst[y * dst_stride + x] = |
864 | 0 | clip_pixel_highbd(ROUND_POWER_OF_TWO(tmp, round_bits), bd); |
865 | 0 | } else { |
866 | 0 | dst16[y * dst16_stride + x] = res; |
867 | 0 | } |
868 | 0 | } |
869 | 0 | } |
870 | 0 | } |
871 | | |
872 | | void av1_highbd_dist_wtd_convolve_2d_copy_c(const uint16_t *src, int src_stride, |
873 | | uint16_t *dst, int dst_stride, |
874 | | int w, int h, |
875 | | ConvolveParams *conv_params, |
876 | 0 | int bd) { |
877 | 0 | CONV_BUF_TYPE *dst16 = conv_params->dst; |
878 | 0 | int dst16_stride = conv_params->dst_stride; |
879 | 0 | const int bits = |
880 | 0 | FILTER_BITS * 2 - conv_params->round_1 - conv_params->round_0; |
881 | 0 | const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0; |
882 | 0 | const int round_offset = (1 << (offset_bits - conv_params->round_1)) + |
883 | 0 | (1 << (offset_bits - conv_params->round_1 - 1)); |
884 | 0 | assert(bits >= 0); |
885 | |
|
886 | 0 | for (int y = 0; y < h; ++y) { |
887 | 0 | for (int x = 0; x < w; ++x) { |
888 | 0 | CONV_BUF_TYPE res = src[y * src_stride + x] << bits; |
889 | 0 | res += round_offset; |
890 | 0 | if (conv_params->do_average) { |
891 | 0 | int32_t tmp = dst16[y * dst16_stride + x]; |
892 | 0 | if (conv_params->use_dist_wtd_comp_avg) { |
893 | 0 | tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset; |
894 | 0 | tmp = tmp >> DIST_PRECISION_BITS; |
895 | 0 | } else { |
896 | 0 | tmp += res; |
897 | 0 | tmp = tmp >> 1; |
898 | 0 | } |
899 | 0 | tmp -= round_offset; |
900 | 0 | dst[y * dst_stride + x] = |
901 | 0 | clip_pixel_highbd(ROUND_POWER_OF_TWO(tmp, bits), bd); |
902 | 0 | } else { |
903 | 0 | dst16[y * dst16_stride + x] = res; |
904 | 0 | } |
905 | 0 | } |
906 | 0 | } |
907 | 0 | } |
908 | | |
909 | | void av1_highbd_convolve_2d_scale_c(const uint16_t *src, int src_stride, |
910 | | uint16_t *dst, int dst_stride, int w, int h, |
911 | | const InterpFilterParams *filter_params_x, |
912 | | const InterpFilterParams *filter_params_y, |
913 | | const int subpel_x_qn, const int x_step_qn, |
914 | | const int subpel_y_qn, const int y_step_qn, |
915 | 0 | ConvolveParams *conv_params, int bd) { |
916 | 0 | int16_t im_block[(2 * MAX_SB_SIZE + MAX_FILTER_TAP) * MAX_SB_SIZE]; |
917 | 0 | int im_h = (((h - 1) * y_step_qn + subpel_y_qn) >> SCALE_SUBPEL_BITS) + |
918 | 0 | filter_params_y->taps; |
919 | 0 | int im_stride = w; |
920 | 0 | const int fo_vert = filter_params_y->taps / 2 - 1; |
921 | 0 | const int fo_horiz = filter_params_x->taps / 2 - 1; |
922 | 0 | CONV_BUF_TYPE *dst16 = conv_params->dst; |
923 | 0 | const int dst16_stride = conv_params->dst_stride; |
924 | 0 | const int bits = |
925 | 0 | FILTER_BITS * 2 - conv_params->round_0 - conv_params->round_1; |
926 | 0 | assert(bits >= 0); |
927 | | // horizontal filter |
928 | 0 | const uint16_t *src_horiz = src - fo_vert * src_stride; |
929 | 0 | for (int y = 0; y < im_h; ++y) { |
930 | 0 | int x_qn = subpel_x_qn; |
931 | 0 | for (int x = 0; x < w; ++x, x_qn += x_step_qn) { |
932 | 0 | const uint16_t *const src_x = &src_horiz[(x_qn >> SCALE_SUBPEL_BITS)]; |
933 | 0 | const int x_filter_idx = (x_qn & SCALE_SUBPEL_MASK) >> SCALE_EXTRA_BITS; |
934 | 0 | assert(x_filter_idx < SUBPEL_SHIFTS); |
935 | 0 | const int16_t *x_filter = |
936 | 0 | av1_get_interp_filter_subpel_kernel(filter_params_x, x_filter_idx); |
937 | 0 | int32_t sum = (1 << (bd + FILTER_BITS - 1)); |
938 | 0 | for (int k = 0; k < filter_params_x->taps; ++k) { |
939 | 0 | sum += x_filter[k] * src_x[k - fo_horiz]; |
940 | 0 | } |
941 | 0 | assert(0 <= sum && sum < (1 << (bd + FILTER_BITS + 1))); |
942 | 0 | im_block[y * im_stride + x] = |
943 | 0 | (int16_t)ROUND_POWER_OF_TWO(sum, conv_params->round_0); |
944 | 0 | } |
945 | 0 | src_horiz += src_stride; |
946 | 0 | } |
947 | | |
948 | | // vertical filter |
949 | 0 | int16_t *src_vert = im_block + fo_vert * im_stride; |
950 | 0 | const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0; |
951 | 0 | for (int x = 0; x < w; ++x) { |
952 | 0 | int y_qn = subpel_y_qn; |
953 | 0 | for (int y = 0; y < h; ++y, y_qn += y_step_qn) { |
954 | 0 | const int16_t *src_y = &src_vert[(y_qn >> SCALE_SUBPEL_BITS) * im_stride]; |
955 | 0 | const int y_filter_idx = (y_qn & SCALE_SUBPEL_MASK) >> SCALE_EXTRA_BITS; |
956 | 0 | assert(y_filter_idx < SUBPEL_SHIFTS); |
957 | 0 | const int16_t *y_filter = |
958 | 0 | av1_get_interp_filter_subpel_kernel(filter_params_y, y_filter_idx); |
959 | 0 | int32_t sum = 1 << offset_bits; |
960 | 0 | for (int k = 0; k < filter_params_y->taps; ++k) { |
961 | 0 | sum += y_filter[k] * src_y[(k - fo_vert) * im_stride]; |
962 | 0 | } |
963 | 0 | assert(0 <= sum && sum < (1 << (offset_bits + 2))); |
964 | 0 | CONV_BUF_TYPE res = ROUND_POWER_OF_TWO(sum, conv_params->round_1); |
965 | 0 | if (conv_params->is_compound) { |
966 | 0 | if (conv_params->do_average) { |
967 | 0 | int32_t tmp = dst16[y * dst16_stride + x]; |
968 | 0 | if (conv_params->use_dist_wtd_comp_avg) { |
969 | 0 | tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset; |
970 | 0 | tmp = tmp >> DIST_PRECISION_BITS; |
971 | 0 | } else { |
972 | 0 | tmp += res; |
973 | 0 | tmp = tmp >> 1; |
974 | 0 | } |
975 | | /* Subtract round offset and convolve round */ |
976 | 0 | tmp = tmp - ((1 << (offset_bits - conv_params->round_1)) + |
977 | 0 | (1 << (offset_bits - conv_params->round_1 - 1))); |
978 | 0 | dst[y * dst_stride + x] = |
979 | 0 | clip_pixel_highbd(ROUND_POWER_OF_TWO(tmp, bits), bd); |
980 | 0 | } else { |
981 | 0 | dst16[y * dst16_stride + x] = res; |
982 | 0 | } |
983 | 0 | } else { |
984 | | /* Subtract round offset and convolve round */ |
985 | 0 | int32_t tmp = res - ((1 << (offset_bits - conv_params->round_1)) + |
986 | 0 | (1 << (offset_bits - conv_params->round_1 - 1))); |
987 | 0 | dst[y * dst_stride + x] = |
988 | 0 | clip_pixel_highbd(ROUND_POWER_OF_TWO(tmp, bits), bd); |
989 | 0 | } |
990 | 0 | } |
991 | 0 | src_vert++; |
992 | 0 | } |
993 | 0 | } |
994 | | |
995 | | static void highbd_convolve_2d_facade_compound( |
996 | | const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, |
997 | | const int w, const int h, const InterpFilterParams *filter_params_x, |
998 | | const InterpFilterParams *filter_params_y, const int subpel_x_qn, |
999 | 0 | const int subpel_y_qn, ConvolveParams *conv_params, int bd) { |
1000 | 0 | const bool need_x = subpel_x_qn != 0; |
1001 | 0 | const bool need_y = subpel_y_qn != 0; |
1002 | 0 | if (!need_x && !need_y) { |
1003 | 0 | av1_highbd_dist_wtd_convolve_2d_copy(src, src_stride, dst, dst_stride, w, h, |
1004 | 0 | conv_params, bd); |
1005 | 0 | } else if (need_x && !need_y) { |
1006 | 0 | av1_highbd_dist_wtd_convolve_x(src, src_stride, dst, dst_stride, w, h, |
1007 | 0 | filter_params_x, subpel_x_qn, conv_params, |
1008 | 0 | bd); |
1009 | 0 | } else if (!need_x && need_y) { |
1010 | 0 | av1_highbd_dist_wtd_convolve_y(src, src_stride, dst, dst_stride, w, h, |
1011 | 0 | filter_params_y, subpel_y_qn, conv_params, |
1012 | 0 | bd); |
1013 | 0 | } else { |
1014 | 0 | assert(need_x && need_y); |
1015 | 0 | av1_highbd_dist_wtd_convolve_2d(src, src_stride, dst, dst_stride, w, h, |
1016 | 0 | filter_params_x, filter_params_y, |
1017 | 0 | subpel_x_qn, subpel_y_qn, conv_params, bd); |
1018 | 0 | } |
1019 | 0 | } |
1020 | | |
1021 | | static void highbd_convolve_2d_facade_single( |
1022 | | const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, |
1023 | | const int w, const int h, const InterpFilterParams *filter_params_x, |
1024 | | const InterpFilterParams *filter_params_y, const int subpel_x_qn, |
1025 | 16.4k | const int subpel_y_qn, ConvolveParams *conv_params, int bd) { |
1026 | 16.4k | const bool need_x = subpel_x_qn != 0; |
1027 | 16.4k | const bool need_y = subpel_y_qn != 0; |
1028 | | |
1029 | 16.4k | if (!need_x && !need_y) { |
1030 | 8.60k | aom_highbd_convolve_copy(src, src_stride, dst, dst_stride, w, h); |
1031 | 8.60k | } else if (need_x && !need_y) { |
1032 | 2.76k | av1_highbd_convolve_x_sr(src, src_stride, dst, dst_stride, w, h, |
1033 | 2.76k | filter_params_x, subpel_x_qn, conv_params, bd); |
1034 | 5.04k | } else if (!need_x && need_y) { |
1035 | 2.22k | av1_highbd_convolve_y_sr(src, src_stride, dst, dst_stride, w, h, |
1036 | 2.22k | filter_params_y, subpel_y_qn, bd); |
1037 | 2.81k | } else { |
1038 | 2.81k | assert(need_x && need_y); |
1039 | 2.81k | av1_highbd_convolve_2d_sr(src, src_stride, dst, dst_stride, w, h, |
1040 | 2.81k | filter_params_x, filter_params_y, subpel_x_qn, |
1041 | 2.81k | subpel_y_qn, conv_params, bd); |
1042 | 2.81k | } |
1043 | 16.4k | } |
1044 | | |
1045 | | void av1_highbd_convolve_2d_facade(const uint8_t *src8, int src_stride, |
1046 | | uint8_t *dst8, int dst_stride, int w, int h, |
1047 | | const InterpFilterParams *interp_filters[2], |
1048 | | const int subpel_x_qn, int x_step_q4, |
1049 | | const int subpel_y_qn, int y_step_q4, |
1050 | | int scaled, ConvolveParams *conv_params, |
1051 | 16.4k | int bd) { |
1052 | 16.4k | (void)x_step_q4; |
1053 | 16.4k | (void)y_step_q4; |
1054 | 16.4k | (void)dst_stride; |
1055 | 16.4k | const uint16_t *src = CONVERT_TO_SHORTPTR(src8); |
1056 | | |
1057 | 16.4k | const int need_filter_params_x = (subpel_x_qn != 0) | scaled; |
1058 | 16.4k | const int need_filter_params_y = (subpel_y_qn != 0) | scaled; |
1059 | 16.4k | const InterpFilterParams *filter_params_x = |
1060 | 16.4k | need_filter_params_x ? interp_filters[0] : NULL; |
1061 | 16.4k | const InterpFilterParams *filter_params_y = |
1062 | 16.4k | need_filter_params_y ? interp_filters[1] : NULL; |
1063 | | |
1064 | 16.4k | uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); |
1065 | 16.4k | if (scaled) { |
1066 | 0 | if (conv_params->is_compound) { |
1067 | 0 | assert(conv_params->dst != NULL); |
1068 | 0 | } |
1069 | 0 | av1_highbd_convolve_2d_scale(src, src_stride, dst, dst_stride, w, h, |
1070 | 0 | filter_params_x, filter_params_y, subpel_x_qn, |
1071 | 0 | x_step_q4, subpel_y_qn, y_step_q4, conv_params, |
1072 | 0 | bd); |
1073 | 16.4k | } else if (conv_params->is_compound) { |
1074 | 0 | highbd_convolve_2d_facade_compound( |
1075 | 0 | src, src_stride, dst, dst_stride, w, h, filter_params_x, |
1076 | 0 | filter_params_y, subpel_x_qn, subpel_y_qn, conv_params, bd); |
1077 | 16.4k | } else { |
1078 | 16.4k | highbd_convolve_2d_facade_single(src, src_stride, dst, dst_stride, w, h, |
1079 | 16.4k | filter_params_x, filter_params_y, |
1080 | 16.4k | subpel_x_qn, subpel_y_qn, conv_params, bd); |
1081 | 16.4k | } |
1082 | 16.4k | } |
1083 | | #endif // CONFIG_AV1_HIGHBITDEPTH |
1084 | | |
1085 | | // Note: Fixed size intermediate buffers, place limits on parameters |
1086 | | // of some functions. 2d filtering proceeds in 2 steps: |
1087 | | // (1) Interpolate horizontally into an intermediate buffer, temp. |
1088 | | // (2) Interpolate temp vertically to derive the sub-pixel result. |
1089 | | // Deriving the maximum number of rows in the temp buffer (135): |
1090 | | // --Smallest scaling factor is x1/2 ==> y_step_q4 = 32 (Normative). |
1091 | | // --Largest block size is 128x128 pixels. |
1092 | | // --128 rows in the downscaled frame span a distance of (128 - 1) * 32 in the |
1093 | | // original frame (in 1/16th pixel units). |
1094 | | // --Must round-up because block may be located at sub-pixel position. |
1095 | | // --Require an additional SUBPEL_TAPS rows for the 8-tap filter tails. |
1096 | | // --((128 - 1) * 32 + 15) >> 4 + 8 = 263. |
1097 | | #define WIENER_MAX_EXT_SIZE 263 |
1098 | | |
1099 | 0 | static INLINE int horz_scalar_product(const uint8_t *a, const int16_t *b) { |
1100 | 0 | int sum = 0; |
1101 | 0 | for (int k = 0; k < SUBPEL_TAPS; ++k) sum += a[k] * b[k]; |
1102 | 0 | return sum; |
1103 | 0 | } |
1104 | | |
1105 | | #if CONFIG_AV1_HIGHBITDEPTH |
1106 | | static INLINE int highbd_horz_scalar_product(const uint16_t *a, |
1107 | 0 | const int16_t *b) { |
1108 | 0 | int sum = 0; |
1109 | 0 | for (int k = 0; k < SUBPEL_TAPS; ++k) sum += a[k] * b[k]; |
1110 | 0 | return sum; |
1111 | 0 | } |
1112 | | #endif |
1113 | | |
1114 | | static INLINE int highbd_vert_scalar_product(const uint16_t *a, |
1115 | | ptrdiff_t a_stride, |
1116 | 0 | const int16_t *b) { |
1117 | 0 | int sum = 0; |
1118 | 0 | for (int k = 0; k < SUBPEL_TAPS; ++k) sum += a[k * a_stride] * b[k]; |
1119 | 0 | return sum; |
1120 | 0 | } |
1121 | | |
1122 | 0 | static const InterpKernel *get_filter_base(const int16_t *filter) { |
1123 | | // NOTE: This assumes that the filter table is 256-byte aligned. |
1124 | | // TODO(agrange) Modify to make independent of table alignment. |
1125 | 0 | return (const InterpKernel *)(((intptr_t)filter) & ~((intptr_t)0xFF)); |
1126 | 0 | } |
1127 | | |
1128 | 0 | static int get_filter_offset(const int16_t *f, const InterpKernel *base) { |
1129 | 0 | return (int)((const InterpKernel *)(intptr_t)f - base); |
1130 | 0 | } |
1131 | | |
1132 | | static void convolve_add_src_horiz_hip(const uint8_t *src, ptrdiff_t src_stride, |
1133 | | uint16_t *dst, ptrdiff_t dst_stride, |
1134 | | const InterpKernel *x_filters, int x0_q4, |
1135 | | int x_step_q4, int w, int h, |
1136 | 0 | int round0_bits) { |
1137 | 0 | const int bd = 8; |
1138 | 0 | src -= SUBPEL_TAPS / 2 - 1; |
1139 | 0 | for (int y = 0; y < h; ++y) { |
1140 | 0 | int x_q4 = x0_q4; |
1141 | 0 | for (int x = 0; x < w; ++x) { |
1142 | 0 | const uint8_t *const src_x = &src[x_q4 >> SUBPEL_BITS]; |
1143 | 0 | const int16_t *const x_filter = x_filters[x_q4 & SUBPEL_MASK]; |
1144 | 0 | const int rounding = ((int)src_x[SUBPEL_TAPS / 2 - 1] << FILTER_BITS) + |
1145 | 0 | (1 << (bd + FILTER_BITS - 1)); |
1146 | 0 | const int sum = horz_scalar_product(src_x, x_filter) + rounding; |
1147 | 0 | dst[x] = (uint16_t)clamp(ROUND_POWER_OF_TWO(sum, round0_bits), 0, |
1148 | 0 | WIENER_CLAMP_LIMIT(round0_bits, bd) - 1); |
1149 | 0 | x_q4 += x_step_q4; |
1150 | 0 | } |
1151 | 0 | src += src_stride; |
1152 | 0 | dst += dst_stride; |
1153 | 0 | } |
1154 | 0 | } |
1155 | | |
1156 | | static void convolve_add_src_vert_hip(const uint16_t *src, ptrdiff_t src_stride, |
1157 | | uint8_t *dst, ptrdiff_t dst_stride, |
1158 | | const InterpKernel *y_filters, int y0_q4, |
1159 | | int y_step_q4, int w, int h, |
1160 | 0 | int round1_bits) { |
1161 | 0 | const int bd = 8; |
1162 | 0 | src -= src_stride * (SUBPEL_TAPS / 2 - 1); |
1163 | |
|
1164 | 0 | for (int x = 0; x < w; ++x) { |
1165 | 0 | int y_q4 = y0_q4; |
1166 | 0 | for (int y = 0; y < h; ++y) { |
1167 | 0 | const uint16_t *src_y = &src[(y_q4 >> SUBPEL_BITS) * src_stride]; |
1168 | 0 | const int16_t *const y_filter = y_filters[y_q4 & SUBPEL_MASK]; |
1169 | 0 | const int rounding = |
1170 | 0 | ((int)src_y[(SUBPEL_TAPS / 2 - 1) * src_stride] << FILTER_BITS) - |
1171 | 0 | (1 << (bd + round1_bits - 1)); |
1172 | 0 | const int sum = |
1173 | 0 | highbd_vert_scalar_product(src_y, src_stride, y_filter) + rounding; |
1174 | 0 | dst[y * dst_stride] = clip_pixel(ROUND_POWER_OF_TWO(sum, round1_bits)); |
1175 | 0 | y_q4 += y_step_q4; |
1176 | 0 | } |
1177 | 0 | ++src; |
1178 | 0 | ++dst; |
1179 | 0 | } |
1180 | 0 | } |
1181 | | |
1182 | | void av1_wiener_convolve_add_src_c(const uint8_t *src, ptrdiff_t src_stride, |
1183 | | uint8_t *dst, ptrdiff_t dst_stride, |
1184 | | const int16_t *filter_x, int x_step_q4, |
1185 | | const int16_t *filter_y, int y_step_q4, |
1186 | | int w, int h, |
1187 | 0 | const ConvolveParams *conv_params) { |
1188 | 0 | const InterpKernel *const filters_x = get_filter_base(filter_x); |
1189 | 0 | const int x0_q4 = get_filter_offset(filter_x, filters_x); |
1190 | |
|
1191 | 0 | const InterpKernel *const filters_y = get_filter_base(filter_y); |
1192 | 0 | const int y0_q4 = get_filter_offset(filter_y, filters_y); |
1193 | |
|
1194 | 0 | uint16_t temp[WIENER_MAX_EXT_SIZE * MAX_SB_SIZE]; |
1195 | 0 | const int intermediate_height = |
1196 | 0 | (((h - 1) * y_step_q4 + y0_q4) >> SUBPEL_BITS) + SUBPEL_TAPS - 1; |
1197 | 0 | memset(temp + (intermediate_height * MAX_SB_SIZE), 0, MAX_SB_SIZE); |
1198 | |
|
1199 | 0 | assert(w <= MAX_SB_SIZE); |
1200 | 0 | assert(h <= MAX_SB_SIZE); |
1201 | 0 | assert(y_step_q4 <= 32); |
1202 | 0 | assert(x_step_q4 <= 32); |
1203 | |
|
1204 | 0 | convolve_add_src_horiz_hip(src - src_stride * (SUBPEL_TAPS / 2 - 1), |
1205 | 0 | src_stride, temp, MAX_SB_SIZE, filters_x, x0_q4, |
1206 | 0 | x_step_q4, w, intermediate_height, |
1207 | 0 | conv_params->round_0); |
1208 | 0 | convolve_add_src_vert_hip(temp + MAX_SB_SIZE * (SUBPEL_TAPS / 2 - 1), |
1209 | 0 | MAX_SB_SIZE, dst, dst_stride, filters_y, y0_q4, |
1210 | 0 | y_step_q4, w, h, conv_params->round_1); |
1211 | 0 | } |
1212 | | |
1213 | | #if CONFIG_AV1_HIGHBITDEPTH |
1214 | | static void highbd_convolve_add_src_horiz_hip( |
1215 | | const uint8_t *src8, ptrdiff_t src_stride, uint16_t *dst, |
1216 | | ptrdiff_t dst_stride, const InterpKernel *x_filters, int x0_q4, |
1217 | 0 | int x_step_q4, int w, int h, int round0_bits, int bd) { |
1218 | 0 | const int extraprec_clamp_limit = WIENER_CLAMP_LIMIT(round0_bits, bd); |
1219 | 0 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); |
1220 | 0 | src -= SUBPEL_TAPS / 2 - 1; |
1221 | 0 | for (int y = 0; y < h; ++y) { |
1222 | 0 | int x_q4 = x0_q4; |
1223 | 0 | for (int x = 0; x < w; ++x) { |
1224 | 0 | const uint16_t *const src_x = &src[x_q4 >> SUBPEL_BITS]; |
1225 | 0 | const int16_t *const x_filter = x_filters[x_q4 & SUBPEL_MASK]; |
1226 | 0 | const int rounding = ((int)src_x[SUBPEL_TAPS / 2 - 1] << FILTER_BITS) + |
1227 | 0 | (1 << (bd + FILTER_BITS - 1)); |
1228 | 0 | const int sum = highbd_horz_scalar_product(src_x, x_filter) + rounding; |
1229 | 0 | dst[x] = (uint16_t)clamp(ROUND_POWER_OF_TWO(sum, round0_bits), 0, |
1230 | 0 | extraprec_clamp_limit - 1); |
1231 | 0 | x_q4 += x_step_q4; |
1232 | 0 | } |
1233 | 0 | src += src_stride; |
1234 | 0 | dst += dst_stride; |
1235 | 0 | } |
1236 | 0 | } |
1237 | | |
1238 | | static void highbd_convolve_add_src_vert_hip( |
1239 | | const uint16_t *src, ptrdiff_t src_stride, uint8_t *dst8, |
1240 | | ptrdiff_t dst_stride, const InterpKernel *y_filters, int y0_q4, |
1241 | 0 | int y_step_q4, int w, int h, int round1_bits, int bd) { |
1242 | 0 | uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); |
1243 | 0 | src -= src_stride * (SUBPEL_TAPS / 2 - 1); |
1244 | 0 | for (int x = 0; x < w; ++x) { |
1245 | 0 | int y_q4 = y0_q4; |
1246 | 0 | for (int y = 0; y < h; ++y) { |
1247 | 0 | const uint16_t *src_y = &src[(y_q4 >> SUBPEL_BITS) * src_stride]; |
1248 | 0 | const int16_t *const y_filter = y_filters[y_q4 & SUBPEL_MASK]; |
1249 | 0 | const int rounding = |
1250 | 0 | ((int)src_y[(SUBPEL_TAPS / 2 - 1) * src_stride] << FILTER_BITS) - |
1251 | 0 | (1 << (bd + round1_bits - 1)); |
1252 | 0 | const int sum = |
1253 | 0 | highbd_vert_scalar_product(src_y, src_stride, y_filter) + rounding; |
1254 | 0 | dst[y * dst_stride] = |
1255 | 0 | clip_pixel_highbd(ROUND_POWER_OF_TWO(sum, round1_bits), bd); |
1256 | 0 | y_q4 += y_step_q4; |
1257 | 0 | } |
1258 | 0 | ++src; |
1259 | 0 | ++dst; |
1260 | 0 | } |
1261 | 0 | } |
1262 | | |
1263 | | void av1_highbd_wiener_convolve_add_src_c( |
1264 | | const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, |
1265 | | ptrdiff_t dst_stride, const int16_t *filter_x, int x_step_q4, |
1266 | | const int16_t *filter_y, int y_step_q4, int w, int h, |
1267 | 0 | const ConvolveParams *conv_params, int bd) { |
1268 | 0 | const InterpKernel *const filters_x = get_filter_base(filter_x); |
1269 | 0 | const int x0_q4 = get_filter_offset(filter_x, filters_x); |
1270 | |
|
1271 | 0 | const InterpKernel *const filters_y = get_filter_base(filter_y); |
1272 | 0 | const int y0_q4 = get_filter_offset(filter_y, filters_y); |
1273 | |
|
1274 | 0 | uint16_t temp[WIENER_MAX_EXT_SIZE * MAX_SB_SIZE]; |
1275 | 0 | const int intermediate_height = |
1276 | 0 | (((h - 1) * y_step_q4 + y0_q4) >> SUBPEL_BITS) + SUBPEL_TAPS; |
1277 | |
|
1278 | 0 | assert(w <= MAX_SB_SIZE); |
1279 | 0 | assert(h <= MAX_SB_SIZE); |
1280 | 0 | assert(y_step_q4 <= 32); |
1281 | 0 | assert(x_step_q4 <= 32); |
1282 | 0 | assert(bd + FILTER_BITS - conv_params->round_0 + 2 <= 16); |
1283 | |
|
1284 | 0 | highbd_convolve_add_src_horiz_hip(src - src_stride * (SUBPEL_TAPS / 2 - 1), |
1285 | 0 | src_stride, temp, MAX_SB_SIZE, filters_x, |
1286 | 0 | x0_q4, x_step_q4, w, intermediate_height, |
1287 | 0 | conv_params->round_0, bd); |
1288 | 0 | highbd_convolve_add_src_vert_hip( |
1289 | 0 | temp + MAX_SB_SIZE * (SUBPEL_TAPS / 2 - 1), MAX_SB_SIZE, dst, dst_stride, |
1290 | 0 | filters_y, y0_q4, y_step_q4, w, h, conv_params->round_1, bd); |
1291 | 0 | } |
1292 | | #endif // CONFIG_AV1_HIGHBITDEPTH |