/src/libvpx/vpx_dsp/x86/convolve.h
Line | Count | Source |
1 | | /* |
2 | | * Copyright (c) 2015 The WebM project authors. All Rights Reserved. |
3 | | * |
4 | | * Use of this source code is governed by a BSD-style license |
5 | | * that can be found in the LICENSE file in the root of the source |
6 | | * tree. An additional intellectual property rights grant can be found |
7 | | * in the file PATENTS. All contributing project authors may |
8 | | * be found in the AUTHORS file in the root of the source tree. |
9 | | */ |
10 | | #ifndef VPX_VPX_DSP_X86_CONVOLVE_H_ |
11 | | #define VPX_VPX_DSP_X86_CONVOLVE_H_ |
12 | | |
13 | | #include <assert.h> |
14 | | |
15 | | #include "./vpx_config.h" |
16 | | #include "vpx/vpx_integer.h" |
17 | | #include "vpx_ports/compiler_attributes.h" |
18 | | |
19 | | // TODO(chiyotsai@google.com): Refactor the code here. Currently this is pretty |
20 | | // hacky and awful to read. Note that there is a filter_x[3] == 128 check in |
21 | | // HIGHBD_FUN_CONV_2D to avoid seg fault due to the fact that the c function |
22 | | // assumes the filter is always 8 tap. |
23 | | typedef void filter8_1dfunction(const uint8_t *src_ptr, ptrdiff_t src_pitch, |
24 | | uint8_t *output_ptr, ptrdiff_t out_pitch, |
25 | | uint32_t output_height, const int16_t *filter); |
26 | | |
27 | | // TODO(chiyotsai@google.com): Remove the is_avg argument to the MACROS once we |
28 | | // have 4-tap vert avg filter. |
29 | | #define FUN_CONV_1D(name, offset, step_q4, dir, src_start, avg, opt, is_avg) \ |
30 | | void vpx_convolve8_##name##_##opt( \ |
31 | | const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, \ |
32 | | ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, \ |
33 | 272M | int x_step_q4, int y0_q4, int y_step_q4, int w, int h) { \ |
34 | 272M | const int16_t *filter_row = filter[offset]; \ |
35 | 272M | (void)x0_q4; \ |
36 | 272M | (void)x_step_q4; \ |
37 | 272M | (void)y0_q4; \ |
38 | 272M | (void)y_step_q4; \ |
39 | 272M | assert(filter_row[3] != 128); \ |
40 | 272M | assert(step_q4 == 16); \ |
41 | 272M | if (filter_row[0] | filter_row[1] | filter_row[6] | filter_row[7]) { \ |
42 | 122M | const int num_taps = 8; \ |
43 | 138M | while (w >= 16) { \ |
44 | 15.8M | vpx_filter_block1d16_##dir##8_##avg##opt(src_start, src_stride, dst, \ |
45 | 15.8M | dst_stride, h, filter_row); \ |
46 | 15.8M | src += 16; \ |
47 | 15.8M | dst += 16; \ |
48 | 15.8M | w -= 16; \ |
49 | 15.8M | } \ |
50 | 122M | if (w == 8) { \ |
51 | 34.1M | vpx_filter_block1d8_##dir##8_##avg##opt(src_start, src_stride, dst, \ |
52 | 34.1M | dst_stride, h, filter_row); \ |
53 | 88.6M | } else if (w == 4) { \ |
54 | 75.8M | vpx_filter_block1d4_##dir##8_##avg##opt(src_start, src_stride, dst, \ |
55 | 75.8M | dst_stride, h, filter_row); \ |
56 | 75.8M | } \ |
57 | 122M | (void)num_taps; \ |
58 | 149M | } else if (filter_row[2] | filter_row[5]) { \ |
59 | 143M | const int num_taps = is_avg ? 8 : 4; \ |
60 | 159M | while (w >= 16) { \ |
61 | 15.4M | vpx_filter_block1d16_##dir##4_##avg##opt(src_start, src_stride, dst, \ |
62 | 15.4M | dst_stride, h, filter_row); \ |
63 | 15.4M | src += 16; \ |
64 | 15.4M | dst += 16; \ |
65 | 15.4M | w -= 16; \ |
66 | 15.4M | } \ |
67 | 143M | if (w == 8) { \ |
68 | 35.4M | vpx_filter_block1d8_##dir##4_##avg##opt(src_start, src_stride, dst, \ |
69 | 35.4M | dst_stride, h, filter_row); \ |
70 | 108M | } else if (w == 4) { \ |
71 | 96.4M | vpx_filter_block1d4_##dir##4_##avg##opt(src_start, src_stride, dst, \ |
72 | 96.4M | dst_stride, h, filter_row); \ |
73 | 96.4M | } \ |
74 | 143M | (void)num_taps; \ |
75 | 143M | } else { \ |
76 | 5.93M | const int num_taps = 2; \ |
77 | 8.67M | while (w >= 16) { \ |
78 | 2.73M | vpx_filter_block1d16_##dir##2_##avg##opt(src_start, src_stride, dst, \ |
79 | 2.73M | dst_stride, h, filter_row); \ |
80 | 2.73M | src += 16; \ |
81 | 2.73M | dst += 16; \ |
82 | 2.73M | w -= 16; \ |
83 | 2.73M | } \ |
84 | 5.93M | if (w == 8) { \ |
85 | 1.77M | vpx_filter_block1d8_##dir##2_##avg##opt(src_start, src_stride, dst, \ |
86 | 1.77M | dst_stride, h, filter_row); \ |
87 | 4.16M | } else if (w == 4) { \ |
88 | 2.27M | vpx_filter_block1d4_##dir##2_##avg##opt(src_start, src_stride, dst, \ |
89 | 2.27M | dst_stride, h, filter_row); \ |
90 | 2.27M | } \ |
91 | 5.93M | (void)num_taps; \ |
92 | 5.93M | } \ |
93 | 272M | } |
94 | | |
95 | | #define FUN_CONV_2D(avg, opt, is_avg) \ |
96 | | void vpx_convolve8_##avg##opt( \ |
97 | | const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, \ |
98 | | ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, \ |
99 | 82.0M | int x_step_q4, int y0_q4, int y_step_q4, int w, int h) { \ |
100 | 82.0M | const int16_t *filter_x = filter[x0_q4]; \ |
101 | 82.0M | const int16_t *filter_y = filter[y0_q4]; \ |
102 | 82.0M | (void)filter_y; \ |
103 | 82.0M | assert(filter_x[3] != 128); \ |
104 | 82.0M | assert(filter_y[3] != 128); \ |
105 | 82.0M | assert(w <= 64); \ |
106 | 82.0M | assert(h <= 64); \ |
107 | 82.0M | assert(x_step_q4 == 16); \ |
108 | 82.0M | assert(y_step_q4 == 16); \ |
109 | 82.0M | if (filter_x[0] | filter_x[1] | filter_x[6] | filter_x[7]) { \ |
110 | 38.4M | DECLARE_ALIGNED(16, uint8_t, fdata2[64 * 71] VPX_UNINITIALIZED); \ |
111 | 38.4M | vpx_convolve8_horiz_##opt(src - 3 * src_stride, src_stride, fdata2, 64, \ |
112 | 38.4M | filter, x0_q4, x_step_q4, y0_q4, y_step_q4, w, \ |
113 | 38.4M | h + 7); \ |
114 | 38.4M | vpx_convolve8_##avg##vert_##opt(fdata2 + 3 * 64, 64, dst, dst_stride, \ |
115 | 38.4M | filter, x0_q4, x_step_q4, y0_q4, \ |
116 | 38.4M | y_step_q4, w, h); \ |
117 | 43.6M | } else if (filter_x[2] | filter_x[5]) { \ |
118 | 41.3M | const int num_taps = is_avg ? 8 : 4; \ |
119 | 41.3M | DECLARE_ALIGNED(16, uint8_t, fdata2[64 * 71] VPX_UNINITIALIZED); \ |
120 | 41.3M | vpx_convolve8_horiz_##opt( \ |
121 | 41.3M | src - (num_taps / 2 - 1) * src_stride, src_stride, fdata2, 64, \ |
122 | 41.3M | filter, x0_q4, x_step_q4, y0_q4, y_step_q4, w, h + num_taps - 1); \ |
123 | 41.3M | vpx_convolve8_##avg##vert_##opt(fdata2 + 64 * (num_taps / 2 - 1), 64, \ |
124 | 41.3M | dst, dst_stride, filter, x0_q4, \ |
125 | 41.3M | x_step_q4, y0_q4, y_step_q4, w, h); \ |
126 | 41.3M | } else { \ |
127 | 2.24M | DECLARE_ALIGNED(16, uint8_t, fdata2[64 * 65] VPX_UNINITIALIZED); \ |
128 | 2.24M | vpx_convolve8_horiz_##opt(src, src_stride, fdata2, 64, filter, x0_q4, \ |
129 | 2.24M | x_step_q4, y0_q4, y_step_q4, w, h + 1); \ |
130 | 2.24M | vpx_convolve8_##avg##vert_##opt(fdata2, 64, dst, dst_stride, filter, \ |
131 | 2.24M | x0_q4, x_step_q4, y0_q4, y_step_q4, w, \ |
132 | 2.24M | h); \ |
133 | 2.24M | } \ |
134 | 82.0M | } |
135 | | |
136 | | #if CONFIG_VP9_HIGHBITDEPTH |
137 | | |
138 | | typedef void highbd_filter8_1dfunction(const uint16_t *src_ptr, |
139 | | const ptrdiff_t src_pitch, |
140 | | uint16_t *output_ptr, |
141 | | ptrdiff_t out_pitch, |
142 | | unsigned int output_height, |
143 | | const int16_t *filter, int bd); |
144 | | |
145 | | #define HIGH_FUN_CONV_1D(name, offset, step_q4, dir, src_start, avg, opt, \ |
146 | | is_avg) \ |
147 | | void vpx_highbd_convolve8_##name##_##opt( \ |
148 | | const uint16_t *src, ptrdiff_t src_stride, uint16_t *dst, \ |
149 | | ptrdiff_t dst_stride, const InterpKernel *filter_kernel, int x0_q4, \ |
150 | 9.54M | int x_step_q4, int y0_q4, int y_step_q4, int w, int h, int bd) { \ |
151 | 9.54M | const int16_t *filter_row = filter_kernel[offset]; \ |
152 | 9.54M | if (step_q4 == 16 && filter_row[3] != 128) { \ |
153 | 9.33M | if (filter_row[0] | filter_row[1] | filter_row[6] | filter_row[7]) { \ |
154 | 6.86M | const int num_taps = 8; \ |
155 | 8.60M | while (w >= 16) { \ |
156 | 1.73M | vpx_highbd_filter_block1d16_##dir##8_##avg##opt( \ |
157 | 1.73M | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ |
158 | 1.73M | src += 16; \ |
159 | 1.73M | dst += 16; \ |
160 | 1.73M | w -= 16; \ |
161 | 1.73M | } \ |
162 | 9.19M | while (w >= 8) { \ |
163 | 2.32M | vpx_highbd_filter_block1d8_##dir##8_##avg##opt( \ |
164 | 2.32M | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ |
165 | 2.32M | src += 8; \ |
166 | 2.32M | dst += 8; \ |
167 | 2.32M | w -= 8; \ |
168 | 2.32M | } \ |
169 | 10.2M | while (w >= 4) { \ |
170 | 3.34M | vpx_highbd_filter_block1d4_##dir##8_##avg##opt( \ |
171 | 3.34M | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ |
172 | 3.34M | src += 4; \ |
173 | 3.34M | dst += 4; \ |
174 | 3.34M | w -= 4; \ |
175 | 3.34M | } \ |
176 | 6.86M | (void)num_taps; \ |
177 | 6.86M | } else if (filter_row[2] | filter_row[5]) { \ |
178 | 0 | const int num_taps = is_avg ? 8 : 4; \ |
179 | 0 | while (w >= 16) { \ |
180 | 0 | vpx_highbd_filter_block1d16_##dir##4_##avg##opt( \ |
181 | 0 | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ |
182 | 0 | src += 16; \ |
183 | 0 | dst += 16; \ |
184 | 0 | w -= 16; \ |
185 | 0 | } \ |
186 | 0 | while (w >= 8) { \ |
187 | 0 | vpx_highbd_filter_block1d8_##dir##4_##avg##opt( \ |
188 | 0 | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ |
189 | 0 | src += 8; \ |
190 | 0 | dst += 8; \ |
191 | 0 | w -= 8; \ |
192 | 0 | } \ |
193 | 0 | while (w >= 4) { \ |
194 | 0 | vpx_highbd_filter_block1d4_##dir##4_##avg##opt( \ |
195 | 0 | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ |
196 | 0 | src += 4; \ |
197 | 0 | dst += 4; \ |
198 | 0 | w -= 4; \ |
199 | 0 | } \ |
200 | 0 | (void)num_taps; \ |
201 | 2.46M | } else { \ |
202 | 2.46M | const int num_taps = 2; \ |
203 | 3.45M | while (w >= 16) { \ |
204 | 986k | vpx_highbd_filter_block1d16_##dir##2_##avg##opt( \ |
205 | 986k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ |
206 | 986k | src += 16; \ |
207 | 986k | dst += 16; \ |
208 | 986k | w -= 16; \ |
209 | 986k | } \ |
210 | 3.10M | while (w >= 8) { \ |
211 | 639k | vpx_highbd_filter_block1d8_##dir##2_##avg##opt( \ |
212 | 639k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ |
213 | 639k | src += 8; \ |
214 | 639k | dst += 8; \ |
215 | 639k | w -= 8; \ |
216 | 639k | } \ |
217 | 3.51M | while (w >= 4) { \ |
218 | 1.04M | vpx_highbd_filter_block1d4_##dir##2_##avg##opt( \ |
219 | 1.04M | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ |
220 | 1.04M | src += 4; \ |
221 | 1.04M | dst += 4; \ |
222 | 1.04M | w -= 4; \ |
223 | 1.04M | } \ |
224 | 2.46M | (void)num_taps; \ |
225 | 2.46M | } \ |
226 | 9.33M | } \ |
227 | 9.54M | if (w) { \ |
228 | 208k | vpx_highbd_convolve8_##name##_c(src, src_stride, dst, dst_stride, \ |
229 | 208k | filter_kernel, x0_q4, x_step_q4, y0_q4, \ |
230 | 208k | y_step_q4, w, h, bd); \ |
231 | 208k | } \ |
232 | 9.54M | } Unexecuted instantiation: vpx_highbd_convolve8_horiz_sse2 Unexecuted instantiation: vpx_highbd_convolve8_vert_sse2 Unexecuted instantiation: vpx_highbd_convolve8_avg_horiz_sse2 Unexecuted instantiation: vpx_highbd_convolve8_avg_vert_sse2 vpx_highbd_convolve8_horiz_avx2 Line | Count | Source | 150 | 4.20M | int x_step_q4, int y0_q4, int y_step_q4, int w, int h, int bd) { \ | 151 | 4.20M | const int16_t *filter_row = filter_kernel[offset]; \ | 152 | 4.20M | if (step_q4 == 16 && filter_row[3] != 128) { \ | 153 | 4.19M | if (filter_row[0] | filter_row[1] | filter_row[6] | filter_row[7]) { \ | 154 | 3.05M | const int num_taps = 8; \ | 155 | 3.82M | while (w >= 16) { \ | 156 | 762k | vpx_highbd_filter_block1d16_##dir##8_##avg##opt( \ | 157 | 762k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 158 | 762k | src += 16; \ | 159 | 762k | dst += 16; \ | 160 | 762k | w -= 16; \ | 161 | 762k | } \ | 162 | 4.06M | while (w >= 8) { \ | 163 | 1.01M | vpx_highbd_filter_block1d8_##dir##8_##avg##opt( \ | 164 | 1.01M | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 165 | 1.01M | src += 8; \ | 166 | 1.01M | dst += 8; \ | 167 | 1.01M | w -= 8; \ | 168 | 1.01M | } \ | 169 | 4.58M | while (w >= 4) { \ | 170 | 1.52M | vpx_highbd_filter_block1d4_##dir##8_##avg##opt( \ | 171 | 1.52M | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 172 | 1.52M | src += 4; \ | 173 | 1.52M | dst += 4; \ | 174 | 1.52M | w -= 4; \ | 175 | 1.52M | } \ | 176 | 3.05M | (void)num_taps; \ | 177 | 3.05M | } else if (filter_row[2] | filter_row[5]) { \ | 178 | 0 | const int num_taps = is_avg ? 8 : 4; \ | 179 | 0 | while (w >= 16) { \ | 180 | 0 | vpx_highbd_filter_block1d16_##dir##4_##avg##opt( \ | 181 | 0 | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 182 | 0 | src += 16; \ | 183 | 0 | dst += 16; \ | 184 | 0 | w -= 16; \ | 185 | 0 | } \ | 186 | 0 | while (w >= 8) { \ | 187 | 0 | vpx_highbd_filter_block1d8_##dir##4_##avg##opt( \ | 188 | 0 | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 189 | 0 | src += 8; \ | 190 | 0 | dst += 8; \ | 191 | 0 | w -= 8; \ | 192 | 0 | } \ | 193 | 0 | while (w >= 4) { \ | 194 | 0 | vpx_highbd_filter_block1d4_##dir##4_##avg##opt( \ | 195 | 0 | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 196 | 0 | src += 4; \ | 197 | 0 | dst += 4; \ | 198 | 0 | w -= 4; \ | 199 | 0 | } \ | 200 | 0 | (void)num_taps; \ | 201 | 1.13M | } else { \ | 202 | 1.13M | const int num_taps = 2; \ | 203 | 1.58M | while (w >= 16) { \ | 204 | 445k | vpx_highbd_filter_block1d16_##dir##2_##avg##opt( \ | 205 | 445k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 206 | 445k | src += 16; \ | 207 | 445k | dst += 16; \ | 208 | 445k | w -= 16; \ | 209 | 445k | } \ | 210 | 1.43M | while (w >= 8) { \ | 211 | 295k | vpx_highbd_filter_block1d8_##dir##2_##avg##opt( \ | 212 | 295k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 213 | 295k | src += 8; \ | 214 | 295k | dst += 8; \ | 215 | 295k | w -= 8; \ | 216 | 295k | } \ | 217 | 1.62M | while (w >= 4) { \ | 218 | 488k | vpx_highbd_filter_block1d4_##dir##2_##avg##opt( \ | 219 | 488k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 220 | 488k | src += 4; \ | 221 | 488k | dst += 4; \ | 222 | 488k | w -= 4; \ | 223 | 488k | } \ | 224 | 1.13M | (void)num_taps; \ | 225 | 1.13M | } \ | 226 | 4.19M | } \ | 227 | 4.20M | if (w) { \ | 228 | 10.3k | vpx_highbd_convolve8_##name##_c(src, src_stride, dst, dst_stride, \ | 229 | 10.3k | filter_kernel, x0_q4, x_step_q4, y0_q4, \ | 230 | 10.3k | y_step_q4, w, h, bd); \ | 231 | 10.3k | } \ | 232 | 4.20M | } |
vpx_highbd_convolve8_vert_avx2 Line | Count | Source | 150 | 3.50M | int x_step_q4, int y0_q4, int y_step_q4, int w, int h, int bd) { \ | 151 | 3.50M | const int16_t *filter_row = filter_kernel[offset]; \ | 152 | 3.50M | if (step_q4 == 16 && filter_row[3] != 128) { \ | 153 | 3.37M | if (filter_row[0] | filter_row[1] | filter_row[6] | filter_row[7]) { \ | 154 | 2.51M | const int num_taps = 8; \ | 155 | 3.15M | while (w >= 16) { \ | 156 | 642k | vpx_highbd_filter_block1d16_##dir##8_##avg##opt( \ | 157 | 642k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 158 | 642k | src += 16; \ | 159 | 642k | dst += 16; \ | 160 | 642k | w -= 16; \ | 161 | 642k | } \ | 162 | 3.37M | while (w >= 8) { \ | 163 | 863k | vpx_highbd_filter_block1d8_##dir##8_##avg##opt( \ | 164 | 863k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 165 | 863k | src += 8; \ | 166 | 863k | dst += 8; \ | 167 | 863k | w -= 8; \ | 168 | 863k | } \ | 169 | 3.72M | while (w >= 4) { \ | 170 | 1.21M | vpx_highbd_filter_block1d4_##dir##8_##avg##opt( \ | 171 | 1.21M | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 172 | 1.21M | src += 4; \ | 173 | 1.21M | dst += 4; \ | 174 | 1.21M | w -= 4; \ | 175 | 1.21M | } \ | 176 | 2.51M | (void)num_taps; \ | 177 | 2.51M | } else if (filter_row[2] | filter_row[5]) { \ | 178 | 0 | const int num_taps = is_avg ? 8 : 4; \ | 179 | 0 | while (w >= 16) { \ | 180 | 0 | vpx_highbd_filter_block1d16_##dir##4_##avg##opt( \ | 181 | 0 | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 182 | 0 | src += 16; \ | 183 | 0 | dst += 16; \ | 184 | 0 | w -= 16; \ | 185 | 0 | } \ | 186 | 0 | while (w >= 8) { \ | 187 | 0 | vpx_highbd_filter_block1d8_##dir##4_##avg##opt( \ | 188 | 0 | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 189 | 0 | src += 8; \ | 190 | 0 | dst += 8; \ | 191 | 0 | w -= 8; \ | 192 | 0 | } \ | 193 | 0 | while (w >= 4) { \ | 194 | 0 | vpx_highbd_filter_block1d4_##dir##4_##avg##opt( \ | 195 | 0 | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 196 | 0 | src += 4; \ | 197 | 0 | dst += 4; \ | 198 | 0 | w -= 4; \ | 199 | 0 | } \ | 200 | 0 | (void)num_taps; \ | 201 | 862k | } else { \ | 202 | 862k | const int num_taps = 2; \ | 203 | 1.19M | while (w >= 16) { \ | 204 | 333k | vpx_highbd_filter_block1d16_##dir##2_##avg##opt( \ | 205 | 333k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 206 | 333k | src += 16; \ | 207 | 333k | dst += 16; \ | 208 | 333k | w -= 16; \ | 209 | 333k | } \ | 210 | 1.08M | while (w >= 8) { \ | 211 | 221k | vpx_highbd_filter_block1d8_##dir##2_##avg##opt( \ | 212 | 221k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 213 | 221k | src += 8; \ | 214 | 221k | dst += 8; \ | 215 | 221k | w -= 8; \ | 216 | 221k | } \ | 217 | 1.23M | while (w >= 4) { \ | 218 | 376k | vpx_highbd_filter_block1d4_##dir##2_##avg##opt( \ | 219 | 376k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 220 | 376k | src += 4; \ | 221 | 376k | dst += 4; \ | 222 | 376k | w -= 4; \ | 223 | 376k | } \ | 224 | 862k | (void)num_taps; \ | 225 | 862k | } \ | 226 | 3.37M | } \ | 227 | 3.50M | if (w) { \ | 228 | 129k | vpx_highbd_convolve8_##name##_c(src, src_stride, dst, dst_stride, \ | 229 | 129k | filter_kernel, x0_q4, x_step_q4, y0_q4, \ | 230 | 129k | y_step_q4, w, h, bd); \ | 231 | 129k | } \ | 232 | 3.50M | } |
vpx_highbd_convolve8_avg_horiz_avx2 Line | Count | Source | 150 | 238k | int x_step_q4, int y0_q4, int y_step_q4, int w, int h, int bd) { \ | 151 | 238k | const int16_t *filter_row = filter_kernel[offset]; \ | 152 | 238k | if (step_q4 == 16 && filter_row[3] != 128) { \ | 153 | 228k | if (filter_row[0] | filter_row[1] | filter_row[6] | filter_row[7]) { \ | 154 | 168k | const int num_taps = 8; \ | 155 | 216k | while (w >= 16) { \ | 156 | 47.8k | vpx_highbd_filter_block1d16_##dir##8_##avg##opt( \ | 157 | 47.8k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 158 | 47.8k | src += 16; \ | 159 | 47.8k | dst += 16; \ | 160 | 47.8k | w -= 16; \ | 161 | 47.8k | } \ | 162 | 226k | while (w >= 8) { \ | 163 | 58.4k | vpx_highbd_filter_block1d8_##dir##8_##avg##opt( \ | 164 | 58.4k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 165 | 58.4k | src += 8; \ | 166 | 58.4k | dst += 8; \ | 167 | 58.4k | w -= 8; \ | 168 | 58.4k | } \ | 169 | 247k | while (w >= 4) { \ | 170 | 78.8k | vpx_highbd_filter_block1d4_##dir##8_##avg##opt( \ | 171 | 78.8k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 172 | 78.8k | src += 4; \ | 173 | 78.8k | dst += 4; \ | 174 | 78.8k | w -= 4; \ | 175 | 78.8k | } \ | 176 | 168k | (void)num_taps; \ | 177 | 168k | } else if (filter_row[2] | filter_row[5]) { \ | 178 | 0 | const int num_taps = is_avg ? 8 : 4; \ | 179 | 0 | while (w >= 16) { \ | 180 | 0 | vpx_highbd_filter_block1d16_##dir##4_##avg##opt( \ | 181 | 0 | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 182 | 0 | src += 16; \ | 183 | 0 | dst += 16; \ | 184 | 0 | w -= 16; \ | 185 | 0 | } \ | 186 | 0 | while (w >= 8) { \ | 187 | 0 | vpx_highbd_filter_block1d8_##dir##4_##avg##opt( \ | 188 | 0 | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 189 | 0 | src += 8; \ | 190 | 0 | dst += 8; \ | 191 | 0 | w -= 8; \ | 192 | 0 | } \ | 193 | 0 | while (w >= 4) { \ | 194 | 0 | vpx_highbd_filter_block1d4_##dir##4_##avg##opt( \ | 195 | 0 | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 196 | 0 | src += 4; \ | 197 | 0 | dst += 4; \ | 198 | 0 | w -= 4; \ | 199 | 0 | } \ | 200 | 0 | (void)num_taps; \ | 201 | 60.0k | } else { \ | 202 | 60.0k | const int num_taps = 2; \ | 203 | 91.4k | while (w >= 16) { \ | 204 | 31.4k | vpx_highbd_filter_block1d16_##dir##2_##avg##opt( \ | 205 | 31.4k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 206 | 31.4k | src += 16; \ | 207 | 31.4k | dst += 16; \ | 208 | 31.4k | w -= 16; \ | 209 | 31.4k | } \ | 210 | 74.6k | while (w >= 8) { \ | 211 | 14.5k | vpx_highbd_filter_block1d8_##dir##2_##avg##opt( \ | 212 | 14.5k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 213 | 14.5k | src += 8; \ | 214 | 14.5k | dst += 8; \ | 215 | 14.5k | w -= 8; \ | 216 | 14.5k | } \ | 217 | 82.3k | while (w >= 4) { \ | 218 | 22.2k | vpx_highbd_filter_block1d4_##dir##2_##avg##opt( \ | 219 | 22.2k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 220 | 22.2k | src += 4; \ | 221 | 22.2k | dst += 4; \ | 222 | 22.2k | w -= 4; \ | 223 | 22.2k | } \ | 224 | 60.0k | (void)num_taps; \ | 225 | 60.0k | } \ | 226 | 228k | } \ | 227 | 238k | if (w) { \ | 228 | 10.1k | vpx_highbd_convolve8_##name##_c(src, src_stride, dst, dst_stride, \ | 229 | 10.1k | filter_kernel, x0_q4, x_step_q4, y0_q4, \ | 230 | 10.1k | y_step_q4, w, h, bd); \ | 231 | 10.1k | } \ | 232 | 238k | } |
vpx_highbd_convolve8_avg_vert_avx2 Line | Count | Source | 150 | 1.58M | int x_step_q4, int y0_q4, int y_step_q4, int w, int h, int bd) { \ | 151 | 1.58M | const int16_t *filter_row = filter_kernel[offset]; \ | 152 | 1.58M | if (step_q4 == 16 && filter_row[3] != 128) { \ | 153 | 1.52M | if (filter_row[0] | filter_row[1] | filter_row[6] | filter_row[7]) { \ | 154 | 1.12M | const int num_taps = 8; \ | 155 | 1.40M | while (w >= 16) { \ | 156 | 283k | vpx_highbd_filter_block1d16_##dir##8_##avg##opt( \ | 157 | 283k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 158 | 283k | src += 16; \ | 159 | 283k | dst += 16; \ | 160 | 283k | w -= 16; \ | 161 | 283k | } \ | 162 | 1.51M | while (w >= 8) { \ | 163 | 394k | vpx_highbd_filter_block1d8_##dir##8_##avg##opt( \ | 164 | 394k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 165 | 394k | src += 8; \ | 166 | 394k | dst += 8; \ | 167 | 394k | w -= 8; \ | 168 | 394k | } \ | 169 | 1.65M | while (w >= 4) { \ | 170 | 535k | vpx_highbd_filter_block1d4_##dir##8_##avg##opt( \ | 171 | 535k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 172 | 535k | src += 4; \ | 173 | 535k | dst += 4; \ | 174 | 535k | w -= 4; \ | 175 | 535k | } \ | 176 | 1.12M | (void)num_taps; \ | 177 | 1.12M | } else if (filter_row[2] | filter_row[5]) { \ | 178 | 0 | const int num_taps = is_avg ? 8 : 4; \ | 179 | 0 | while (w >= 16) { \ | 180 | 0 | vpx_highbd_filter_block1d16_##dir##4_##avg##opt( \ | 181 | 0 | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 182 | 0 | src += 16; \ | 183 | 0 | dst += 16; \ | 184 | 0 | w -= 16; \ | 185 | 0 | } \ | 186 | 0 | while (w >= 8) { \ | 187 | 0 | vpx_highbd_filter_block1d8_##dir##4_##avg##opt( \ | 188 | 0 | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 189 | 0 | src += 8; \ | 190 | 0 | dst += 8; \ | 191 | 0 | w -= 8; \ | 192 | 0 | } \ | 193 | 0 | while (w >= 4) { \ | 194 | 0 | vpx_highbd_filter_block1d4_##dir##4_##avg##opt( \ | 195 | 0 | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 196 | 0 | src += 4; \ | 197 | 0 | dst += 4; \ | 198 | 0 | w -= 4; \ | 199 | 0 | } \ | 200 | 0 | (void)num_taps; \ | 201 | 405k | } else { \ | 202 | 405k | const int num_taps = 2; \ | 203 | 580k | while (w >= 16) { \ | 204 | 174k | vpx_highbd_filter_block1d16_##dir##2_##avg##opt( \ | 205 | 174k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 206 | 174k | src += 16; \ | 207 | 174k | dst += 16; \ | 208 | 174k | w -= 16; \ | 209 | 174k | } \ | 210 | 513k | while (w >= 8) { \ | 211 | 107k | vpx_highbd_filter_block1d8_##dir##2_##avg##opt( \ | 212 | 107k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 213 | 107k | src += 8; \ | 214 | 107k | dst += 8; \ | 215 | 107k | w -= 8; \ | 216 | 107k | } \ | 217 | 565k | while (w >= 4) { \ | 218 | 159k | vpx_highbd_filter_block1d4_##dir##2_##avg##opt( \ | 219 | 159k | src_start, src_stride, dst, dst_stride, h, filter_row, bd); \ | 220 | 159k | src += 4; \ | 221 | 159k | dst += 4; \ | 222 | 159k | w -= 4; \ | 223 | 159k | } \ | 224 | 405k | (void)num_taps; \ | 225 | 405k | } \ | 226 | 1.52M | } \ | 227 | 1.58M | if (w) { \ | 228 | 58.3k | vpx_highbd_convolve8_##name##_c(src, src_stride, dst, dst_stride, \ | 229 | 58.3k | filter_kernel, x0_q4, x_step_q4, y0_q4, \ | 230 | 58.3k | y_step_q4, w, h, bd); \ | 231 | 58.3k | } \ | 232 | 1.58M | } |
|
233 | | |
234 | | #define HIGH_FUN_CONV_2D(avg, opt, is_avg) \ |
235 | | void vpx_highbd_convolve8_##avg##opt( \ |
236 | | const uint16_t *src, ptrdiff_t src_stride, uint16_t *dst, \ |
237 | | ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, \ |
238 | 3.71M | int x_step_q4, int y0_q4, int y_step_q4, int w, int h, int bd) { \ |
239 | 3.71M | const int16_t *filter_x = filter[x0_q4]; \ |
240 | 3.71M | assert(w <= 64); \ |
241 | 3.71M | assert(h <= 64); \ |
242 | 3.71M | if (x_step_q4 == 16 && y_step_q4 == 16) { \ |
243 | 3.62M | if ((filter_x[0] | filter_x[1] | filter_x[6] | filter_x[7]) || \ |
244 | 3.62M | filter_x[3] == 128) { \ |
245 | 2.62M | DECLARE_ALIGNED(16, uint16_t, fdata2[64 * 71] VPX_UNINITIALIZED); \ |
246 | 2.62M | vpx_highbd_convolve8_horiz_##opt(src - 3 * src_stride, src_stride, \ |
247 | 2.62M | fdata2, 64, filter, x0_q4, x_step_q4, \ |
248 | 2.62M | y0_q4, y_step_q4, w, h + 7, bd); \ |
249 | 2.62M | vpx_highbd_convolve8_##avg##vert_##opt( \ |
250 | 2.62M | fdata2 + 192, 64, dst, dst_stride, filter, x0_q4, x_step_q4, \ |
251 | 2.62M | y0_q4, y_step_q4, w, h, bd); \ |
252 | 2.62M | } else if (filter_x[2] | filter_x[5]) { \ |
253 | 0 | const int num_taps = is_avg ? 8 : 4; \ |
254 | 0 | DECLARE_ALIGNED(16, uint16_t, fdata2[64 * 71] VPX_UNINITIALIZED); \ |
255 | 0 | vpx_highbd_convolve8_horiz_##opt( \ |
256 | 0 | src - (num_taps / 2 - 1) * src_stride, src_stride, fdata2, 64, \ |
257 | 0 | filter, x0_q4, x_step_q4, y0_q4, y_step_q4, w, h + num_taps - 1, \ |
258 | 0 | bd); \ |
259 | 0 | vpx_highbd_convolve8_##avg##vert_##opt( \ |
260 | 0 | fdata2 + 64 * (num_taps / 2 - 1), 64, dst, dst_stride, filter, \ |
261 | 0 | x0_q4, x_step_q4, y0_q4, y_step_q4, w, h, bd); \ |
262 | 996k | } else { \ |
263 | 996k | DECLARE_ALIGNED(16, uint16_t, fdata2[64 * 65] VPX_UNINITIALIZED); \ |
264 | 996k | vpx_highbd_convolve8_horiz_##opt(src, src_stride, fdata2, 64, filter, \ |
265 | 996k | x0_q4, x_step_q4, y0_q4, y_step_q4, \ |
266 | 996k | w, h + 1, bd); \ |
267 | 996k | vpx_highbd_convolve8_##avg##vert_##opt(fdata2, 64, dst, dst_stride, \ |
268 | 996k | filter, x0_q4, x_step_q4, \ |
269 | 996k | y0_q4, y_step_q4, w, h, bd); \ |
270 | 996k | } \ |
271 | 3.62M | } else { \ |
272 | 94.6k | vpx_highbd_convolve8_##avg##c(src, src_stride, dst, dst_stride, filter, \ |
273 | 94.6k | x0_q4, x_step_q4, y0_q4, y_step_q4, w, h, \ |
274 | 94.6k | bd); \ |
275 | 94.6k | } \ |
276 | 3.71M | } |
277 | | |
278 | | #endif // CONFIG_VP9_HIGHBITDEPTH |
279 | | #endif // VPX_VPX_DSP_X86_CONVOLVE_H_ |