/src/libvpx/vpx_dsp/x86/convolve.h
Line | Count | Source |
1 | | /* |
2 | | * Copyright (c) 2015 The WebM project authors. All Rights Reserved. |
3 | | * |
4 | | * Use of this source code is governed by a BSD-style license |
5 | | * that can be found in the LICENSE file in the root of the source |
6 | | * tree. An additional intellectual property rights grant can be found |
7 | | * in the file PATENTS. All contributing project authors may |
8 | | * be found in the AUTHORS file in the root of the source tree. |
9 | | */ |
10 | | #ifndef VPX_VPX_DSP_X86_CONVOLVE_H_ |
11 | | #define VPX_VPX_DSP_X86_CONVOLVE_H_ |
12 | | |
13 | | #include <assert.h> |
14 | | |
15 | | #include "./vpx_config.h" |
16 | | #include "vpx/vpx_integer.h" |
17 | | #include "vpx_ports/compiler_attributes.h" |
18 | | |
19 | | // TODO(chiyotsai@google.com): Refactor the code here. Currently this is pretty |
20 | | // hacky and awful to read. Note that there is a filter_x[3] == 128 check in |
21 | | // HIGHBD_FUN_CONV_2D to avoid seg fault due to the fact that the c function |
22 | | // assumes the filter is always 8 tap. |
23 | | typedef void filter8_1dfunction(const uint8_t *src_ptr, ptrdiff_t src_pitch, |
24 | | uint8_t *output_ptr, ptrdiff_t out_pitch, |
25 | | uint32_t output_height, const int16_t *filter); |
26 | | |
27 | | // TODO(chiyotsai@google.com): Remove the is_avg argument to the MACROS once we |
28 | | // have 4-tap vert avg filter. |
29 | | #define FUN_CONV_1D(name, offset, step_q4, dir, src_start, avg, opt, is_avg) \ |
30 | | void vpx_convolve8_##name##_##opt( \ |
31 | | const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, \ |
32 | | ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, \ |
33 | 529M | int x_step_q4, int y0_q4, int y_step_q4, int w, int h) { \ |
34 | 529M | const int16_t *filter_row = filter[offset]; \ |
35 | 529M | (void)x0_q4; \ |
36 | 529M | (void)x_step_q4; \ |
37 | 529M | (void)y0_q4; \ |
38 | 529M | (void)y_step_q4; \ |
39 | 529M | assert(filter_row[3] != 128); \ |
40 | 529M | assert(step_q4 == 16); \ |
41 | 529M | if (filter_row[0] | filter_row[1] | filter_row[6] | filter_row[7]) { \ |
42 | 463M | const int num_taps = 8; \ |
43 | 542M | while (w >= 16) { \ |
44 | 78.2M | vpx_filter_block1d16_##dir##8_##avg##opt(src_start, src_stride, dst, \ |
45 | 78.2M | dst_stride, h, filter_row); \ |
46 | 78.2M | src += 16; \ |
47 | 78.2M | dst += 16; \ |
48 | 78.2M | w -= 16; \ |
49 | 78.2M | } \ |
50 | 463M | if (w == 8) { \ |
51 | 165M | vpx_filter_block1d8_##dir##8_##avg##opt(src_start, src_stride, dst, \ |
52 | 165M | dst_stride, h, filter_row); \ |
53 | 298M | } else if (w == 4) { \ |
54 | 245M | vpx_filter_block1d4_##dir##8_##avg##opt(src_start, src_stride, dst, \ |
55 | 245M | dst_stride, h, filter_row); \ |
56 | 245M | } \ |
57 | 463M | (void)num_taps; \ |
58 | 463M | } else if (filter_row[2] | filter_row[5]) { \ |
59 | 0 | const int num_taps = is_avg ? 8 : 4; \ |
60 | 0 | while (w >= 16) { \ |
61 | 0 | vpx_filter_block1d16_##dir##4_##avg##opt(src_start, src_stride, dst, \ |
62 | 0 | dst_stride, h, filter_row); \ |
63 | 0 | src += 16; \ |
64 | 0 | dst += 16; \ |
65 | 0 | w -= 16; \ |
66 | 0 | } \ |
67 | 0 | if (w == 8) { \ |
68 | 0 | vpx_filter_block1d8_##dir##4_##avg##opt(src_start, src_stride, dst, \ |
69 | 0 | dst_stride, h, filter_row); \ |
70 | 0 | } else if (w == 4) { \ |
71 | 0 | vpx_filter_block1d4_##dir##4_##avg##opt(src_start, src_stride, dst, \ |
72 | 0 | dst_stride, h, filter_row); \ |
73 | 0 | } \ |
74 | 0 | (void)num_taps; \ |
75 | 66.0M | } else { \ |
76 | 66.0M | const int num_taps = 2; \ |
77 | 83.3M | while (w >= 16) { \ |
78 | 17.3M | vpx_filter_block1d16_##dir##2_##avg##opt(src_start, src_stride, dst, \ |
79 | 17.3M | dst_stride, h, filter_row); \ |
80 | 17.3M | src += 16; \ |
81 | 17.3M | dst += 16; \ |
82 | 17.3M | w -= 16; \ |
83 | 17.3M | } \ |
84 | 66.0M | if (w == 8) { \ |
85 | 21.2M | vpx_filter_block1d8_##dir##2_##avg##opt(src_start, src_stride, dst, \ |
86 | 21.2M | dst_stride, h, filter_row); \ |
87 | 44.7M | } else if (w == 4) { \ |
88 | 31.4M | vpx_filter_block1d4_##dir##2_##avg##opt(src_start, src_stride, dst, \ |
89 | 31.4M | dst_stride, h, filter_row); \ |
90 | 31.4M | } \ |
91 | 66.0M | (void)num_taps; \ |
92 | 66.0M | } \ |
93 | 529M | } |
94 | | |
95 | | #define FUN_CONV_2D(avg, opt, is_avg) \ |
96 | | void vpx_convolve8_##avg##opt( \ |
97 | | const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, \ |
98 | | ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, \ |
99 | 152M | int x_step_q4, int y0_q4, int y_step_q4, int w, int h) { \ |
100 | 152M | const int16_t *filter_x = filter[x0_q4]; \ |
101 | 152M | const int16_t *filter_y = filter[y0_q4]; \ |
102 | 152M | (void)filter_y; \ |
103 | 152M | assert(filter_x[3] != 128); \ |
104 | 152M | assert(filter_y[3] != 128); \ |
105 | 152M | assert(w <= 64); \ |
106 | 152M | assert(h <= 64); \ |
107 | 152M | assert(x_step_q4 == 16); \ |
108 | 152M | assert(y_step_q4 == 16); \ |
109 | 152M | if (filter_x[0] | filter_x[1] | filter_x[6] | filter_x[7]) { \ |
110 | 124M | DECLARE_ALIGNED(16, uint8_t, fdata2[64 * 71] VPX_UNINITIALIZED); \ |
111 | 124M | vpx_convolve8_horiz_##opt(src - 3 * src_stride, src_stride, fdata2, 64, \ |
112 | 124M | filter, x0_q4, x_step_q4, y0_q4, y_step_q4, w, \ |
113 | 124M | h + 7); \ |
114 | 124M | vpx_convolve8_##avg##vert_##opt(fdata2 + 3 * 64, 64, dst, dst_stride, \ |
115 | 124M | filter, x0_q4, x_step_q4, y0_q4, \ |
116 | 124M | y_step_q4, w, h); \ |
117 | 124M | } else if (filter_x[2] | filter_x[5]) { \ |
118 | 0 | const int num_taps = is_avg ? 8 : 4; \ |
119 | 0 | DECLARE_ALIGNED(16, uint8_t, fdata2[64 * 71] VPX_UNINITIALIZED); \ |
120 | 0 | vpx_convolve8_horiz_##opt( \ |
121 | 0 | src - (num_taps / 2 - 1) * src_stride, src_stride, fdata2, 64, \ |
122 | 0 | filter, x0_q4, x_step_q4, y0_q4, y_step_q4, w, h + num_taps - 1); \ |
123 | 0 | vpx_convolve8_##avg##vert_##opt(fdata2 + 64 * (num_taps / 2 - 1), 64, \ |
124 | 0 | dst, dst_stride, filter, x0_q4, \ |
125 | 0 | x_step_q4, y0_q4, y_step_q4, w, h); \ |
126 | 27.9M | } else { \ |
127 | 27.9M | DECLARE_ALIGNED(16, uint8_t, fdata2[64 * 65] VPX_UNINITIALIZED); \ |
128 | 27.9M | vpx_convolve8_horiz_##opt(src, src_stride, fdata2, 64, filter, x0_q4, \ |
129 | 27.9M | x_step_q4, y0_q4, y_step_q4, w, h + 1); \ |
130 | 27.9M | vpx_convolve8_##avg##vert_##opt(fdata2, 64, dst, dst_stride, filter, \ |
131 | 27.9M | x0_q4, x_step_q4, y0_q4, y_step_q4, w, \ |
132 | 27.9M | h); \ |
133 | 27.9M | } \ |
134 | 152M | } |
135 | | |
136 | | #if CONFIG_VP9_HIGHBITDEPTH |
137 | | |
138 | | typedef void highbd_filter8_1dfunction(const uint16_t *src_ptr, |
139 | | const ptrdiff_t src_pitch, |
140 | | uint16_t *output_ptr, |
141 | | ptrdiff_t out_pitch, |
142 | | unsigned int output_height, |
143 | | const int16_t *filter, int bd); |
144 | | |
145 | | #define HIGH_FUN_CONV_1D(name, offset, step_q4, dir, src_start, avg, opt, \ |
146 | | is_avg) \ |
147 | | void vpx_highbd_convolve8_##name##_##opt( \ |
148 | | const uint16_t *src, ptrdiff_t src_stride, uint16_t *dst, \ |
149 | | ptrdiff_t dst_stride, const InterpKernel *filter_kernel, int x0_q4, \ |
150 | 5.40M | int x_step_q4, int y0_q4, int y_step_q4, int w, int h, int bd) { \ |
151 | 5.40M | const int16_t *filter_row = filter_kernel[offset]; \ |
152 | 5.40M | if (step_q4 == 16 && filter_row[3] != 128) { \ |
153 | 5.06M | if (filter_row[0] | filter_row[1] | filter_row[6] | filter_row[7]) { \ |
154 | 4.68M | const int num_taps = 8; \ |
155 | 6.28M | while (w >= 16) { \ |
156 | 1.60M | vpx_highbd_filter_block1d16_##dir##8_##avg##opt( \ |
157 | 1.60M | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ |
158 | 1.60M | src += 16; \ |
159 | 1.60M | dst += 16; \ |
160 | 1.60M | w -= 16; \ |
161 | 1.60M | } \ |
162 | 6.09M | while (w >= 8) { \ |
163 | 1.40M | vpx_highbd_filter_block1d8_##dir##8_##avg##opt( \ |
164 | 1.40M | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ |
165 | 1.40M | src += 8; \ |
166 | 1.40M | dst += 8; \ |
167 | 1.40M | w -= 8; \ |
168 | 1.40M | } \ |
169 | 6.88M | while (w >= 4) { \ |
170 | 2.19M | vpx_highbd_filter_block1d4_##dir##8_##avg##opt( \ |
171 | 2.19M | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ |
172 | 2.19M | src += 4; \ |
173 | 2.19M | dst += 4; \ |
174 | 2.19M | w -= 4; \ |
175 | 2.19M | } \ |
176 | 4.68M | (void)num_taps; \ |
177 | 4.68M | } else if (filter_row[2] | filter_row[5]) { \ |
178 | 0 | const int num_taps = is_avg ? 8 : 4; \ |
179 | 0 | while (w >= 16) { \ |
180 | 0 | vpx_highbd_filter_block1d16_##dir##4_##avg##opt( \ |
181 | 0 | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ |
182 | 0 | src += 16; \ |
183 | 0 | dst += 16; \ |
184 | 0 | w -= 16; \ |
185 | 0 | } \ |
186 | 0 | while (w >= 8) { \ |
187 | 0 | vpx_highbd_filter_block1d8_##dir##4_##avg##opt( \ |
188 | 0 | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ |
189 | 0 | src += 8; \ |
190 | 0 | dst += 8; \ |
191 | 0 | w -= 8; \ |
192 | 0 | } \ |
193 | 0 | while (w >= 4) { \ |
194 | 0 | vpx_highbd_filter_block1d4_##dir##4_##avg##opt( \ |
195 | 0 | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ |
196 | 0 | src += 4; \ |
197 | 0 | dst += 4; \ |
198 | 0 | w -= 4; \ |
199 | 0 | } \ |
200 | 0 | (void)num_taps; \ |
201 | 376k | } else { \ |
202 | 376k | const int num_taps = 2; \ |
203 | 584k | while (w >= 16) { \ |
204 | 208k | vpx_highbd_filter_block1d16_##dir##2_##avg##opt( \ |
205 | 208k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ |
206 | 208k | src += 16; \ |
207 | 208k | dst += 16; \ |
208 | 208k | w -= 16; \ |
209 | 208k | } \ |
210 | 484k | while (w >= 8) { \ |
211 | 108k | vpx_highbd_filter_block1d8_##dir##2_##avg##opt( \ |
212 | 108k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ |
213 | 108k | src += 8; \ |
214 | 108k | dst += 8; \ |
215 | 108k | w -= 8; \ |
216 | 108k | } \ |
217 | 523k | while (w >= 4) { \ |
218 | 147k | vpx_highbd_filter_block1d4_##dir##2_##avg##opt( \ |
219 | 147k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ |
220 | 147k | src += 4; \ |
221 | 147k | dst += 4; \ |
222 | 147k | w -= 4; \ |
223 | 147k | } \ |
224 | 376k | (void)num_taps; \ |
225 | 376k | } \ |
226 | 5.06M | } \ |
227 | 5.40M | if (w) { \ |
228 | 337k | vpx_highbd_convolve8_##name##_c(src, src_stride, dst, dst_stride, \ |
229 | 337k | filter_kernel, x0_q4, x_step_q4, y0_q4, \ |
230 | 337k | y_step_q4, w, h, bd); \ |
231 | 337k | } \ |
232 | 5.40M | } Unexecuted instantiation: vpx_highbd_convolve8_horiz_sse2 Unexecuted instantiation: vpx_highbd_convolve8_vert_sse2 Unexecuted instantiation: vpx_highbd_convolve8_avg_horiz_sse2 Unexecuted instantiation: vpx_highbd_convolve8_avg_vert_sse2 vpx_highbd_convolve8_horiz_avx2 Line | Count | Source | 150 | 2.46M | int x_step_q4, int y0_q4, int y_step_q4, int w, int h, int bd) { \ | 151 | 2.46M | const int16_t *filter_row = filter_kernel[offset]; \ | 152 | 2.46M | if (step_q4 == 16 && filter_row[3] != 128) { \ | 153 | 2.36M | if (filter_row[0] | filter_row[1] | filter_row[6] | filter_row[7]) { \ | 154 | 2.19M | const int num_taps = 8; \ | 155 | 2.91M | while (w >= 16) { \ | 156 | 722k | vpx_highbd_filter_block1d16_##dir##8_##avg##opt( \ | 157 | 722k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 158 | 722k | src += 16; \ | 159 | 722k | dst += 16; \ | 160 | 722k | w -= 16; \ | 161 | 722k | } \ | 162 | 2.84M | while (w >= 8) { \ | 163 | 657k | vpx_highbd_filter_block1d8_##dir##8_##avg##opt( \ | 164 | 657k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 165 | 657k | src += 8; \ | 166 | 657k | dst += 8; \ | 167 | 657k | w -= 8; \ | 168 | 657k | } \ | 169 | 3.22M | while (w >= 4) { \ | 170 | 1.03M | vpx_highbd_filter_block1d4_##dir##8_##avg##opt( \ | 171 | 1.03M | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 172 | 1.03M | src += 4; \ | 173 | 1.03M | dst += 4; \ | 174 | 1.03M | w -= 4; \ | 175 | 1.03M | } \ | 176 | 2.19M | (void)num_taps; \ | 177 | 2.19M | } else if (filter_row[2] | filter_row[5]) { \ | 178 | 0 | const int num_taps = is_avg ? 8 : 4; \ | 179 | 0 | while (w >= 16) { \ | 180 | 0 | vpx_highbd_filter_block1d16_##dir##4_##avg##opt( \ | 181 | 0 | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 182 | 0 | src += 16; \ | 183 | 0 | dst += 16; \ | 184 | 0 | w -= 16; \ | 185 | 0 | } \ | 186 | 0 | while (w >= 8) { \ | 187 | 0 | vpx_highbd_filter_block1d8_##dir##4_##avg##opt( \ | 188 | 0 | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 189 | 0 | src += 8; \ | 190 | 0 | dst += 8; \ | 191 | 0 | w -= 8; \ | 192 | 0 | } \ | 193 | 0 | while (w >= 4) { \ | 194 | 0 | vpx_highbd_filter_block1d4_##dir##4_##avg##opt( \ | 195 | 0 | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 196 | 0 | src += 4; \ | 197 | 0 | dst += 4; \ | 198 | 0 | w -= 4; \ | 199 | 0 | } \ | 200 | 0 | (void)num_taps; \ | 201 | 174k | } else { \ | 202 | 174k | const int num_taps = 2; \ | 203 | 271k | while (w >= 16) { \ | 204 | 96.6k | vpx_highbd_filter_block1d16_##dir##2_##avg##opt( \ | 205 | 96.6k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 206 | 96.6k | src += 16; \ | 207 | 96.6k | dst += 16; \ | 208 | 96.6k | w -= 16; \ | 209 | 96.6k | } \ | 210 | 224k | while (w >= 8) { \ | 211 | 50.1k | vpx_highbd_filter_block1d8_##dir##2_##avg##opt( \ | 212 | 50.1k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 213 | 50.1k | src += 8; \ | 214 | 50.1k | dst += 8; \ | 215 | 50.1k | w -= 8; \ | 216 | 50.1k | } \ | 217 | 242k | while (w >= 4) { \ | 218 | 67.8k | vpx_highbd_filter_block1d4_##dir##2_##avg##opt( \ | 219 | 67.8k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 220 | 67.8k | src += 4; \ | 221 | 67.8k | dst += 4; \ | 222 | 67.8k | w -= 4; \ | 223 | 67.8k | } \ | 224 | 174k | (void)num_taps; \ | 225 | 174k | } \ | 226 | 2.36M | } \ | 227 | 2.46M | if (w) { \ | 228 | 96.5k | vpx_highbd_convolve8_##name##_c(src, src_stride, dst, dst_stride, \ | 229 | 96.5k | filter_kernel, x0_q4, x_step_q4, y0_q4, \ | 230 | 96.5k | y_step_q4, w, h, bd); \ | 231 | 96.5k | } \ | 232 | 2.46M | } |
vpx_highbd_convolve8_vert_avx2 Line | Count | Source | 150 | 2.32M | int x_step_q4, int y0_q4, int y_step_q4, int w, int h, int bd) { \ | 151 | 2.32M | const int16_t *filter_row = filter_kernel[offset]; \ | 152 | 2.32M | if (step_q4 == 16 && filter_row[3] != 128) { \ | 153 | 2.23M | if (filter_row[0] | filter_row[1] | filter_row[6] | filter_row[7]) { \ | 154 | 2.10M | const int num_taps = 8; \ | 155 | 2.83M | while (w >= 16) { \ | 156 | 726k | vpx_highbd_filter_block1d16_##dir##8_##avg##opt( \ | 157 | 726k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 158 | 726k | src += 16; \ | 159 | 726k | dst += 16; \ | 160 | 726k | w -= 16; \ | 161 | 726k | } \ | 162 | 2.73M | while (w >= 8) { \ | 163 | 628k | vpx_highbd_filter_block1d8_##dir##8_##avg##opt( \ | 164 | 628k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 165 | 628k | src += 8; \ | 166 | 628k | dst += 8; \ | 167 | 628k | w -= 8; \ | 168 | 628k | } \ | 169 | 3.08M | while (w >= 4) { \ | 170 | 981k | vpx_highbd_filter_block1d4_##dir##8_##avg##opt( \ | 171 | 981k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 172 | 981k | src += 4; \ | 173 | 981k | dst += 4; \ | 174 | 981k | w -= 4; \ | 175 | 981k | } \ | 176 | 2.10M | (void)num_taps; \ | 177 | 2.10M | } else if (filter_row[2] | filter_row[5]) { \ | 178 | 0 | const int num_taps = is_avg ? 8 : 4; \ | 179 | 0 | while (w >= 16) { \ | 180 | 0 | vpx_highbd_filter_block1d16_##dir##4_##avg##opt( \ | 181 | 0 | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 182 | 0 | src += 16; \ | 183 | 0 | dst += 16; \ | 184 | 0 | w -= 16; \ | 185 | 0 | } \ | 186 | 0 | while (w >= 8) { \ | 187 | 0 | vpx_highbd_filter_block1d8_##dir##4_##avg##opt( \ | 188 | 0 | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 189 | 0 | src += 8; \ | 190 | 0 | dst += 8; \ | 191 | 0 | w -= 8; \ | 192 | 0 | } \ | 193 | 0 | while (w >= 4) { \ | 194 | 0 | vpx_highbd_filter_block1d4_##dir##4_##avg##opt( \ | 195 | 0 | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 196 | 0 | src += 4; \ | 197 | 0 | dst += 4; \ | 198 | 0 | w -= 4; \ | 199 | 0 | } \ | 200 | 0 | (void)num_taps; \ | 201 | 128k | } else { \ | 202 | 128k | const int num_taps = 2; \ | 203 | 194k | while (w >= 16) { \ | 204 | 65.5k | vpx_highbd_filter_block1d16_##dir##2_##avg##opt( \ | 205 | 65.5k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 206 | 65.5k | src += 16; \ | 207 | 65.5k | dst += 16; \ | 208 | 65.5k | w -= 16; \ | 209 | 65.5k | } \ | 210 | 166k | while (w >= 8) { \ | 211 | 37.8k | vpx_highbd_filter_block1d8_##dir##2_##avg##opt( \ | 212 | 37.8k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 213 | 37.8k | src += 8; \ | 214 | 37.8k | dst += 8; \ | 215 | 37.8k | w -= 8; \ | 216 | 37.8k | } \ | 217 | 180k | while (w >= 4) { \ | 218 | 52.0k | vpx_highbd_filter_block1d4_##dir##2_##avg##opt( \ | 219 | 52.0k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 220 | 52.0k | src += 4; \ | 221 | 52.0k | dst += 4; \ | 222 | 52.0k | w -= 4; \ | 223 | 52.0k | } \ | 224 | 128k | (void)num_taps; \ | 225 | 128k | } \ | 226 | 2.23M | } \ | 227 | 2.32M | if (w) { \ | 228 | 91.9k | vpx_highbd_convolve8_##name##_c(src, src_stride, dst, dst_stride, \ | 229 | 91.9k | filter_kernel, x0_q4, x_step_q4, y0_q4, \ | 230 | 91.9k | y_step_q4, w, h, bd); \ | 231 | 91.9k | } \ | 232 | 2.32M | } |
vpx_highbd_convolve8_avg_horiz_avx2 Line | Count | Source | 150 | 190k | int x_step_q4, int y0_q4, int y_step_q4, int w, int h, int bd) { \ | 151 | 190k | const int16_t *filter_row = filter_kernel[offset]; \ | 152 | 190k | if (step_q4 == 16 && filter_row[3] != 128) { \ | 153 | 89.0k | if (filter_row[0] | filter_row[1] | filter_row[6] | filter_row[7]) { \ | 154 | 66.4k | const int num_taps = 8; \ | 155 | 92.8k | while (w >= 16) { \ | 156 | 26.3k | vpx_highbd_filter_block1d16_##dir##8_##avg##opt( \ | 157 | 26.3k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 158 | 26.3k | src += 16; \ | 159 | 26.3k | dst += 16; \ | 160 | 26.3k | w -= 16; \ | 161 | 26.3k | } \ | 162 | 85.1k | while (w >= 8) { \ | 163 | 18.7k | vpx_highbd_filter_block1d8_##dir##8_##avg##opt( \ | 164 | 18.7k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 165 | 18.7k | src += 8; \ | 166 | 18.7k | dst += 8; \ | 167 | 18.7k | w -= 8; \ | 168 | 18.7k | } \ | 169 | 98.2k | while (w >= 4) { \ | 170 | 31.7k | vpx_highbd_filter_block1d4_##dir##8_##avg##opt( \ | 171 | 31.7k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 172 | 31.7k | src += 4; \ | 173 | 31.7k | dst += 4; \ | 174 | 31.7k | w -= 4; \ | 175 | 31.7k | } \ | 176 | 66.4k | (void)num_taps; \ | 177 | 66.4k | } else if (filter_row[2] | filter_row[5]) { \ | 178 | 0 | const int num_taps = is_avg ? 8 : 4; \ | 179 | 0 | while (w >= 16) { \ | 180 | 0 | vpx_highbd_filter_block1d16_##dir##4_##avg##opt( \ | 181 | 0 | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 182 | 0 | src += 16; \ | 183 | 0 | dst += 16; \ | 184 | 0 | w -= 16; \ | 185 | 0 | } \ | 186 | 0 | while (w >= 8) { \ | 187 | 0 | vpx_highbd_filter_block1d8_##dir##4_##avg##opt( \ | 188 | 0 | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 189 | 0 | src += 8; \ | 190 | 0 | dst += 8; \ | 191 | 0 | w -= 8; \ | 192 | 0 | } \ | 193 | 0 | while (w >= 4) { \ | 194 | 0 | vpx_highbd_filter_block1d4_##dir##4_##avg##opt( \ | 195 | 0 | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 196 | 0 | src += 4; \ | 197 | 0 | dst += 4; \ | 198 | 0 | w -= 4; \ | 199 | 0 | } \ | 200 | 0 | (void)num_taps; \ | 201 | 22.6k | } else { \ | 202 | 22.6k | const int num_taps = 2; \ | 203 | 40.8k | while (w >= 16) { \ | 204 | 18.1k | vpx_highbd_filter_block1d16_##dir##2_##avg##opt( \ | 205 | 18.1k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 206 | 18.1k | src += 16; \ | 207 | 18.1k | dst += 16; \ | 208 | 18.1k | w -= 16; \ | 209 | 18.1k | } \ | 210 | 28.5k | while (w >= 8) { \ | 211 | 5.93k | vpx_highbd_filter_block1d8_##dir##2_##avg##opt( \ | 212 | 5.93k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 213 | 5.93k | src += 8; \ | 214 | 5.93k | dst += 8; \ | 215 | 5.93k | w -= 8; \ | 216 | 5.93k | } \ | 217 | 30.4k | while (w >= 4) { \ | 218 | 7.81k | vpx_highbd_filter_block1d4_##dir##2_##avg##opt( \ | 219 | 7.81k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 220 | 7.81k | src += 4; \ | 221 | 7.81k | dst += 4; \ | 222 | 7.81k | w -= 4; \ | 223 | 7.81k | } \ | 224 | 22.6k | (void)num_taps; \ | 225 | 22.6k | } \ | 226 | 89.0k | } \ | 227 | 190k | if (w) { \ | 228 | 101k | vpx_highbd_convolve8_##name##_c(src, src_stride, dst, dst_stride, \ | 229 | 101k | filter_kernel, x0_q4, x_step_q4, y0_q4, \ | 230 | 101k | y_step_q4, w, h, bd); \ | 231 | 101k | } \ | 232 | 190k | } |
vpx_highbd_convolve8_avg_vert_avx2 Line | Count | Source | 150 | 424k | int x_step_q4, int y0_q4, int y_step_q4, int w, int h, int bd) { \ | 151 | 424k | const int16_t *filter_row = filter_kernel[offset]; \ | 152 | 424k | if (step_q4 == 16 && filter_row[3] != 128) { \ | 153 | 376k | if (filter_row[0] | filter_row[1] | filter_row[6] | filter_row[7]) { \ | 154 | 326k | const int num_taps = 8; \ | 155 | 451k | while (w >= 16) { \ | 156 | 125k | vpx_highbd_filter_block1d16_##dir##8_##avg##opt( \ | 157 | 125k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 158 | 125k | src += 16; \ | 159 | 125k | dst += 16; \ | 160 | 125k | w -= 16; \ | 161 | 125k | } \ | 162 | 422k | while (w >= 8) { \ | 163 | 96.1k | vpx_highbd_filter_block1d8_##dir##8_##avg##opt( \ | 164 | 96.1k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 165 | 96.1k | src += 8; \ | 166 | 96.1k | dst += 8; \ | 167 | 96.1k | w -= 8; \ | 168 | 96.1k | } \ | 169 | 476k | while (w >= 4) { \ | 170 | 150k | vpx_highbd_filter_block1d4_##dir##8_##avg##opt( \ | 171 | 150k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 172 | 150k | src += 4; \ | 173 | 150k | dst += 4; \ | 174 | 150k | w -= 4; \ | 175 | 150k | } \ | 176 | 326k | (void)num_taps; \ | 177 | 326k | } else if (filter_row[2] | filter_row[5]) { \ | 178 | 0 | const int num_taps = is_avg ? 8 : 4; \ | 179 | 0 | while (w >= 16) { \ | 180 | 0 | vpx_highbd_filter_block1d16_##dir##4_##avg##opt( \ | 181 | 0 | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 182 | 0 | src += 16; \ | 183 | 0 | dst += 16; \ | 184 | 0 | w -= 16; \ | 185 | 0 | } \ | 186 | 0 | while (w >= 8) { \ | 187 | 0 | vpx_highbd_filter_block1d8_##dir##4_##avg##opt( \ | 188 | 0 | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 189 | 0 | src += 8; \ | 190 | 0 | dst += 8; \ | 191 | 0 | w -= 8; \ | 192 | 0 | } \ | 193 | 0 | while (w >= 4) { \ | 194 | 0 | vpx_highbd_filter_block1d4_##dir##4_##avg##opt( \ | 195 | 0 | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 196 | 0 | src += 4; \ | 197 | 0 | dst += 4; \ | 198 | 0 | w -= 4; \ | 199 | 0 | } \ | 200 | 0 | (void)num_taps; \ | 201 | 50.3k | } else { \ | 202 | 50.3k | const int num_taps = 2; \ | 203 | 78.5k | while (w >= 16) { \ | 204 | 28.1k | vpx_highbd_filter_block1d16_##dir##2_##avg##opt( \ | 205 | 28.1k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 206 | 28.1k | src += 16; \ | 207 | 28.1k | dst += 16; \ | 208 | 28.1k | w -= 16; \ | 209 | 28.1k | } \ | 210 | 64.6k | while (w >= 8) { \ | 211 | 14.2k | vpx_highbd_filter_block1d8_##dir##2_##avg##opt( \ | 212 | 14.2k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 213 | 14.2k | src += 8; \ | 214 | 14.2k | dst += 8; \ | 215 | 14.2k | w -= 8; \ | 216 | 14.2k | } \ | 217 | 69.8k | while (w >= 4) { \ | 218 | 19.4k | vpx_highbd_filter_block1d4_##dir##2_##avg##opt( \ | 219 | 19.4k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 220 | 19.4k | src += 4; \ | 221 | 19.4k | dst += 4; \ | 222 | 19.4k | w -= 4; \ | 223 | 19.4k | } \ | 224 | 50.3k | (void)num_taps; \ | 225 | 50.3k | } \ | 226 | 376k | } \ | 227 | 424k | if (w) { \ | 228 | 48.0k | vpx_highbd_convolve8_##name##_c(src, src_stride, dst, dst_stride, \ | 229 | 48.0k | filter_kernel, x0_q4, x_step_q4, y0_q4, \ | 230 | 48.0k | y_step_q4, w, h, bd); \ | 231 | 48.0k | } \ | 232 | 424k | } |
|
233 | | |
234 | | #define HIGH_FUN_CONV_2D(avg, opt, is_avg) \ |
235 | | void vpx_highbd_convolve8_##avg##opt( \ |
236 | | const uint16_t *src, ptrdiff_t src_stride, uint16_t *dst, \ |
237 | | ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, \ |
238 | 4.34M | int x_step_q4, int y0_q4, int y_step_q4, int w, int h, int bd) { \ |
239 | 4.34M | const int16_t *filter_x = filter[x0_q4]; \ |
240 | 4.34M | assert(w <= 64); \ |
241 | 4.34M | assert(h <= 64); \ |
242 | 4.34M | if (x_step_q4 == 16 && y_step_q4 == 16) { \ |
243 | 1.90M | if ((filter_x[0] | filter_x[1] | filter_x[6] | filter_x[7]) || \ |
244 | 1.90M | filter_x[3] == 128) { \ |
245 | 1.77M | DECLARE_ALIGNED(16, uint16_t, fdata2[64 * 71] VPX_UNINITIALIZED); \ |
246 | 1.77M | vpx_highbd_convolve8_horiz_##opt(src - 3 * src_stride, src_stride, \ |
247 | 1.77M | fdata2, 64, filter, x0_q4, x_step_q4, \ |
248 | 1.77M | y0_q4, y_step_q4, w, h + 7, bd); \ |
249 | 1.77M | vpx_highbd_convolve8_##avg##vert_##opt( \ |
250 | 1.77M | fdata2 + 192, 64, dst, dst_stride, filter, x0_q4, x_step_q4, \ |
251 | 1.77M | y0_q4, y_step_q4, w, h, bd); \ |
252 | 1.77M | } else if (filter_x[2] | filter_x[5]) { \ |
253 | 0 | const int num_taps = is_avg ? 8 : 4; \ |
254 | 0 | DECLARE_ALIGNED(16, uint16_t, fdata2[64 * 71] VPX_UNINITIALIZED); \ |
255 | 0 | vpx_highbd_convolve8_horiz_##opt( \ |
256 | 0 | src - (num_taps / 2 - 1) * src_stride, src_stride, fdata2, 64, \ |
257 | 0 | filter, x0_q4, x_step_q4, y0_q4, y_step_q4, w, h + num_taps - 1, \ |
258 | 0 | bd); \ |
259 | 0 | vpx_highbd_convolve8_##avg##vert_##opt( \ |
260 | 0 | fdata2 + 64 * (num_taps / 2 - 1), 64, dst, dst_stride, filter, \ |
261 | 0 | x0_q4, x_step_q4, y0_q4, y_step_q4, w, h, bd); \ |
262 | 132k | } else { \ |
263 | 132k | DECLARE_ALIGNED(16, uint16_t, fdata2[64 * 65] VPX_UNINITIALIZED); \ |
264 | 132k | vpx_highbd_convolve8_horiz_##opt(src, src_stride, fdata2, 64, filter, \ |
265 | 132k | x0_q4, x_step_q4, y0_q4, y_step_q4, \ |
266 | 132k | w, h + 1, bd); \ |
267 | 132k | vpx_highbd_convolve8_##avg##vert_##opt(fdata2, 64, dst, dst_stride, \ |
268 | 132k | filter, x0_q4, x_step_q4, \ |
269 | 132k | y0_q4, y_step_q4, w, h, bd); \ |
270 | 132k | } \ |
271 | 2.44M | } else { \ |
272 | 2.44M | vpx_highbd_convolve8_##avg##c(src, src_stride, dst, dst_stride, filter, \ |
273 | 2.44M | x0_q4, x_step_q4, y0_q4, y_step_q4, w, h, \ |
274 | 2.44M | bd); \ |
275 | 2.44M | } \ |
276 | 4.34M | } |
277 | | |
278 | | #endif // CONFIG_VP9_HIGHBITDEPTH |
279 | | #endif // VPX_VPX_DSP_X86_CONVOLVE_H_ |