/src/libvpx/vpx_dsp/x86/highbd_variance_sse2.c
Line | Count | Source |
1 | | /* |
2 | | * Copyright (c) 2014 The WebM project authors. All Rights Reserved. |
3 | | * |
4 | | * Use of this source code is governed by a BSD-style license |
5 | | * that can be found in the LICENSE file in the root of the source |
6 | | * tree. An additional intellectual property rights grant can be found |
7 | | * in the file PATENTS. All contributing project authors may |
8 | | * be found in the AUTHORS file in the root of the source tree. |
9 | | */ |
10 | | #include <emmintrin.h> // SSE2 |
11 | | |
12 | | #include "./vpx_config.h" |
13 | | #include "./vpx_dsp_rtcd.h" |
14 | | #include "vpx_ports/mem.h" |
15 | | |
16 | | #if HAVE_X86_ASM |
17 | | typedef uint32_t (*high_variance_fn_t)(const uint16_t *src, int src_stride, |
18 | | const uint16_t *ref, int ref_stride, |
19 | | uint32_t *sse, int *sum); |
20 | | |
21 | | uint32_t vpx_highbd_calc8x8var_sse2(const uint16_t *src, int src_stride, |
22 | | const uint16_t *ref, int ref_stride, |
23 | | uint32_t *sse, int *sum); |
24 | | |
25 | | uint32_t vpx_highbd_calc16x16var_sse2(const uint16_t *src, int src_stride, |
26 | | const uint16_t *ref, int ref_stride, |
27 | | uint32_t *sse, int *sum); |
28 | | |
29 | | static void highbd_8_variance_sse2(const uint16_t *src, int src_stride, |
30 | | const uint16_t *ref, int ref_stride, int w, |
31 | | int h, uint32_t *sse, int *sum, |
32 | 0 | high_variance_fn_t var_fn, int block_size) { |
33 | 0 | int i, j; |
34 | |
|
35 | 0 | *sse = 0; |
36 | 0 | *sum = 0; |
37 | |
|
38 | 0 | for (i = 0; i < h; i += block_size) { |
39 | 0 | for (j = 0; j < w; j += block_size) { |
40 | 0 | unsigned int sse0; |
41 | 0 | int sum0; |
42 | 0 | var_fn(src + src_stride * i + j, src_stride, ref + ref_stride * i + j, |
43 | 0 | ref_stride, &sse0, &sum0); |
44 | 0 | *sse += sse0; |
45 | 0 | *sum += sum0; |
46 | 0 | } |
47 | 0 | } |
48 | 0 | } |
49 | | |
50 | | static void highbd_10_variance_sse2(const uint16_t *src, int src_stride, |
51 | | const uint16_t *ref, int ref_stride, int w, |
52 | | int h, uint32_t *sse, int *sum, |
53 | 168k | high_variance_fn_t var_fn, int block_size) { |
54 | 168k | int i, j; |
55 | 168k | uint64_t sse_long = 0; |
56 | 168k | int32_t sum_long = 0; |
57 | | |
58 | 368k | for (i = 0; i < h; i += block_size) { |
59 | 467k | for (j = 0; j < w; j += block_size) { |
60 | 268k | unsigned int sse0; |
61 | 268k | int sum0; |
62 | 268k | var_fn(src + src_stride * i + j, src_stride, ref + ref_stride * i + j, |
63 | 268k | ref_stride, &sse0, &sum0); |
64 | 268k | sse_long += sse0; |
65 | 268k | sum_long += sum0; |
66 | 268k | } |
67 | 199k | } |
68 | 168k | *sum = ROUND_POWER_OF_TWO(sum_long, 2); |
69 | 168k | *sse = (uint32_t)ROUND_POWER_OF_TWO(sse_long, 4); |
70 | 168k | } |
71 | | |
72 | | static void highbd_12_variance_sse2(const uint16_t *src, int src_stride, |
73 | | const uint16_t *ref, int ref_stride, int w, |
74 | | int h, uint32_t *sse, int *sum, |
75 | 810k | high_variance_fn_t var_fn, int block_size) { |
76 | 810k | int i, j; |
77 | 810k | uint64_t sse_long = 0; |
78 | 810k | int32_t sum_long = 0; |
79 | | |
80 | 1.70M | for (i = 0; i < h; i += block_size) { |
81 | 2.05M | for (j = 0; j < w; j += block_size) { |
82 | 1.16M | unsigned int sse0; |
83 | 1.16M | int sum0; |
84 | 1.16M | var_fn(src + src_stride * i + j, src_stride, ref + ref_stride * i + j, |
85 | 1.16M | ref_stride, &sse0, &sum0); |
86 | 1.16M | sse_long += sse0; |
87 | 1.16M | sum_long += sum0; |
88 | 1.16M | } |
89 | 898k | } |
90 | 810k | *sum = ROUND_POWER_OF_TWO(sum_long, 4); |
91 | 810k | *sse = (uint32_t)ROUND_POWER_OF_TWO(sse_long, 8); |
92 | 810k | } |
93 | | |
94 | | #define HIGH_GET_VAR(S) \ |
95 | | void vpx_highbd_8_get##S##x##S##var_sse2( \ |
96 | | const uint8_t *src8, int src_stride, const uint8_t *ref8, \ |
97 | 0 | int ref_stride, uint32_t *sse, int *sum) { \ |
98 | 0 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ |
99 | 0 | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ |
100 | 0 | vpx_highbd_calc##S##x##S##var_sse2(src, src_stride, ref, ref_stride, sse, \ |
101 | 0 | sum); \ |
102 | 0 | } \ Unexecuted instantiation: vpx_highbd_8_get16x16var_sse2 Unexecuted instantiation: vpx_highbd_8_get8x8var_sse2 |
103 | | \ |
104 | | void vpx_highbd_10_get##S##x##S##var_sse2( \ |
105 | | const uint8_t *src8, int src_stride, const uint8_t *ref8, \ |
106 | 0 | int ref_stride, uint32_t *sse, int *sum) { \ |
107 | 0 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ |
108 | 0 | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ |
109 | 0 | vpx_highbd_calc##S##x##S##var_sse2(src, src_stride, ref, ref_stride, sse, \ |
110 | 0 | sum); \ |
111 | 0 | *sum = ROUND_POWER_OF_TWO(*sum, 2); \ |
112 | 0 | *sse = ROUND_POWER_OF_TWO(*sse, 4); \ |
113 | 0 | } \ Unexecuted instantiation: vpx_highbd_10_get16x16var_sse2 Unexecuted instantiation: vpx_highbd_10_get8x8var_sse2 |
114 | | \ |
115 | | void vpx_highbd_12_get##S##x##S##var_sse2( \ |
116 | | const uint8_t *src8, int src_stride, const uint8_t *ref8, \ |
117 | 0 | int ref_stride, uint32_t *sse, int *sum) { \ |
118 | 0 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ |
119 | 0 | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ |
120 | 0 | vpx_highbd_calc##S##x##S##var_sse2(src, src_stride, ref, ref_stride, sse, \ |
121 | 0 | sum); \ |
122 | 0 | *sum = ROUND_POWER_OF_TWO(*sum, 4); \ |
123 | 0 | *sse = ROUND_POWER_OF_TWO(*sse, 8); \ |
124 | 0 | } Unexecuted instantiation: vpx_highbd_12_get16x16var_sse2 Unexecuted instantiation: vpx_highbd_12_get8x8var_sse2 |
125 | | |
126 | | HIGH_GET_VAR(16) |
127 | | HIGH_GET_VAR(8) |
128 | | |
129 | | #undef HIGH_GET_VAR |
130 | | |
131 | | #define VAR_FN(w, h, block_size, shift) \ |
132 | | uint32_t vpx_highbd_8_variance##w##x##h##_sse2( \ |
133 | | const uint8_t *src8, int src_stride, const uint8_t *ref8, \ |
134 | 0 | int ref_stride, uint32_t *sse) { \ |
135 | 0 | int sum; \ |
136 | 0 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ |
137 | 0 | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ |
138 | 0 | highbd_8_variance_sse2( \ |
139 | 0 | src, src_stride, ref, ref_stride, w, h, sse, &sum, \ |
140 | 0 | vpx_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ |
141 | 0 | return *sse - (uint32_t)(((int64_t)sum * sum) >> (shift)); \ |
142 | 0 | } \ Unexecuted instantiation: vpx_highbd_8_variance64x64_sse2 Unexecuted instantiation: vpx_highbd_8_variance64x32_sse2 Unexecuted instantiation: vpx_highbd_8_variance32x64_sse2 Unexecuted instantiation: vpx_highbd_8_variance32x32_sse2 Unexecuted instantiation: vpx_highbd_8_variance32x16_sse2 Unexecuted instantiation: vpx_highbd_8_variance16x32_sse2 Unexecuted instantiation: vpx_highbd_8_variance16x16_sse2 Unexecuted instantiation: vpx_highbd_8_variance16x8_sse2 Unexecuted instantiation: vpx_highbd_8_variance8x16_sse2 Unexecuted instantiation: vpx_highbd_8_variance8x8_sse2 |
143 | | \ |
144 | | uint32_t vpx_highbd_10_variance##w##x##h##_sse2( \ |
145 | | const uint8_t *src8, int src_stride, const uint8_t *ref8, \ |
146 | 168k | int ref_stride, uint32_t *sse) { \ |
147 | 168k | int sum; \ |
148 | 168k | int64_t var; \ |
149 | 168k | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ |
150 | 168k | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ |
151 | 168k | highbd_10_variance_sse2( \ |
152 | 168k | src, src_stride, ref, ref_stride, w, h, sse, &sum, \ |
153 | 168k | vpx_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ |
154 | 168k | var = (int64_t)(*sse) - (((int64_t)sum * sum) >> (shift)); \ |
155 | 168k | return (var >= 0) ? (uint32_t)var : 0; \ |
156 | 168k | } \ vpx_highbd_10_variance64x64_sse2 Line | Count | Source | 146 | 2.52k | int ref_stride, uint32_t *sse) { \ | 147 | 2.52k | int sum; \ | 148 | 2.52k | int64_t var; \ | 149 | 2.52k | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ | 150 | 2.52k | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ | 151 | 2.52k | highbd_10_variance_sse2( \ | 152 | 2.52k | src, src_stride, ref, ref_stride, w, h, sse, &sum, \ | 153 | 2.52k | vpx_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ | 154 | 2.52k | var = (int64_t)(*sse) - (((int64_t)sum * sum) >> (shift)); \ | 155 | 2.52k | return (var >= 0) ? (uint32_t)var : 0; \ | 156 | 2.52k | } \ |
vpx_highbd_10_variance64x32_sse2 Line | Count | Source | 146 | 726 | int ref_stride, uint32_t *sse) { \ | 147 | 726 | int sum; \ | 148 | 726 | int64_t var; \ | 149 | 726 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ | 150 | 726 | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ | 151 | 726 | highbd_10_variance_sse2( \ | 152 | 726 | src, src_stride, ref, ref_stride, w, h, sse, &sum, \ | 153 | 726 | vpx_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ | 154 | 726 | var = (int64_t)(*sse) - (((int64_t)sum * sum) >> (shift)); \ | 155 | 726 | return (var >= 0) ? (uint32_t)var : 0; \ | 156 | 726 | } \ |
vpx_highbd_10_variance32x64_sse2 Line | Count | Source | 146 | 680 | int ref_stride, uint32_t *sse) { \ | 147 | 680 | int sum; \ | 148 | 680 | int64_t var; \ | 149 | 680 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ | 150 | 680 | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ | 151 | 680 | highbd_10_variance_sse2( \ | 152 | 680 | src, src_stride, ref, ref_stride, w, h, sse, &sum, \ | 153 | 680 | vpx_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ | 154 | 680 | var = (int64_t)(*sse) - (((int64_t)sum * sum) >> (shift)); \ | 155 | 680 | return (var >= 0) ? (uint32_t)var : 0; \ | 156 | 680 | } \ |
vpx_highbd_10_variance32x32_sse2 Line | Count | Source | 146 | 14.3k | int ref_stride, uint32_t *sse) { \ | 147 | 14.3k | int sum; \ | 148 | 14.3k | int64_t var; \ | 149 | 14.3k | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ | 150 | 14.3k | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ | 151 | 14.3k | highbd_10_variance_sse2( \ | 152 | 14.3k | src, src_stride, ref, ref_stride, w, h, sse, &sum, \ | 153 | 14.3k | vpx_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ | 154 | 14.3k | var = (int64_t)(*sse) - (((int64_t)sum * sum) >> (shift)); \ | 155 | 14.3k | return (var >= 0) ? (uint32_t)var : 0; \ | 156 | 14.3k | } \ |
vpx_highbd_10_variance32x16_sse2 Line | Count | Source | 146 | 1.06k | int ref_stride, uint32_t *sse) { \ | 147 | 1.06k | int sum; \ | 148 | 1.06k | int64_t var; \ | 149 | 1.06k | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ | 150 | 1.06k | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ | 151 | 1.06k | highbd_10_variance_sse2( \ | 152 | 1.06k | src, src_stride, ref, ref_stride, w, h, sse, &sum, \ | 153 | 1.06k | vpx_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ | 154 | 1.06k | var = (int64_t)(*sse) - (((int64_t)sum * sum) >> (shift)); \ | 155 | 1.06k | return (var >= 0) ? (uint32_t)var : 0; \ | 156 | 1.06k | } \ |
vpx_highbd_10_variance16x32_sse2 Line | Count | Source | 146 | 1.25k | int ref_stride, uint32_t *sse) { \ | 147 | 1.25k | int sum; \ | 148 | 1.25k | int64_t var; \ | 149 | 1.25k | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ | 150 | 1.25k | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ | 151 | 1.25k | highbd_10_variance_sse2( \ | 152 | 1.25k | src, src_stride, ref, ref_stride, w, h, sse, &sum, \ | 153 | 1.25k | vpx_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ | 154 | 1.25k | var = (int64_t)(*sse) - (((int64_t)sum * sum) >> (shift)); \ | 155 | 1.25k | return (var >= 0) ? (uint32_t)var : 0; \ | 156 | 1.25k | } \ |
vpx_highbd_10_variance16x16_sse2 Line | Count | Source | 146 | 29.5k | int ref_stride, uint32_t *sse) { \ | 147 | 29.5k | int sum; \ | 148 | 29.5k | int64_t var; \ | 149 | 29.5k | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ | 150 | 29.5k | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ | 151 | 29.5k | highbd_10_variance_sse2( \ | 152 | 29.5k | src, src_stride, ref, ref_stride, w, h, sse, &sum, \ | 153 | 29.5k | vpx_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ | 154 | 29.5k | var = (int64_t)(*sse) - (((int64_t)sum * sum) >> (shift)); \ | 155 | 29.5k | return (var >= 0) ? (uint32_t)var : 0; \ | 156 | 29.5k | } \ |
vpx_highbd_10_variance16x8_sse2 Line | Count | Source | 146 | 1.90k | int ref_stride, uint32_t *sse) { \ | 147 | 1.90k | int sum; \ | 148 | 1.90k | int64_t var; \ | 149 | 1.90k | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ | 150 | 1.90k | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ | 151 | 1.90k | highbd_10_variance_sse2( \ | 152 | 1.90k | src, src_stride, ref, ref_stride, w, h, sse, &sum, \ | 153 | 1.90k | vpx_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ | 154 | 1.90k | var = (int64_t)(*sse) - (((int64_t)sum * sum) >> (shift)); \ | 155 | 1.90k | return (var >= 0) ? (uint32_t)var : 0; \ | 156 | 1.90k | } \ |
vpx_highbd_10_variance8x16_sse2 Line | Count | Source | 146 | 4.60k | int ref_stride, uint32_t *sse) { \ | 147 | 4.60k | int sum; \ | 148 | 4.60k | int64_t var; \ | 149 | 4.60k | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ | 150 | 4.60k | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ | 151 | 4.60k | highbd_10_variance_sse2( \ | 152 | 4.60k | src, src_stride, ref, ref_stride, w, h, sse, &sum, \ | 153 | 4.60k | vpx_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ | 154 | 4.60k | var = (int64_t)(*sse) - (((int64_t)sum * sum) >> (shift)); \ | 155 | 4.60k | return (var >= 0) ? (uint32_t)var : 0; \ | 156 | 4.60k | } \ |
vpx_highbd_10_variance8x8_sse2 Line | Count | Source | 146 | 112k | int ref_stride, uint32_t *sse) { \ | 147 | 112k | int sum; \ | 148 | 112k | int64_t var; \ | 149 | 112k | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ | 150 | 112k | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ | 151 | 112k | highbd_10_variance_sse2( \ | 152 | 112k | src, src_stride, ref, ref_stride, w, h, sse, &sum, \ | 153 | 112k | vpx_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ | 154 | 112k | var = (int64_t)(*sse) - (((int64_t)sum * sum) >> (shift)); \ | 155 | 112k | return (var >= 0) ? (uint32_t)var : 0; \ | 156 | 112k | } \ |
|
157 | | \ |
158 | | uint32_t vpx_highbd_12_variance##w##x##h##_sse2( \ |
159 | | const uint8_t *src8, int src_stride, const uint8_t *ref8, \ |
160 | 810k | int ref_stride, uint32_t *sse) { \ |
161 | 810k | int sum; \ |
162 | 810k | int64_t var; \ |
163 | 810k | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ |
164 | 810k | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ |
165 | 810k | highbd_12_variance_sse2( \ |
166 | 810k | src, src_stride, ref, ref_stride, w, h, sse, &sum, \ |
167 | 810k | vpx_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ |
168 | 810k | var = (int64_t)(*sse) - (((int64_t)sum * sum) >> (shift)); \ |
169 | 810k | return (var >= 0) ? (uint32_t)var : 0; \ |
170 | 810k | } vpx_highbd_12_variance64x64_sse2 Line | Count | Source | 160 | 11.2k | int ref_stride, uint32_t *sse) { \ | 161 | 11.2k | int sum; \ | 162 | 11.2k | int64_t var; \ | 163 | 11.2k | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ | 164 | 11.2k | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ | 165 | 11.2k | highbd_12_variance_sse2( \ | 166 | 11.2k | src, src_stride, ref, ref_stride, w, h, sse, &sum, \ | 167 | 11.2k | vpx_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ | 168 | 11.2k | var = (int64_t)(*sse) - (((int64_t)sum * sum) >> (shift)); \ | 169 | 11.2k | return (var >= 0) ? (uint32_t)var : 0; \ | 170 | 11.2k | } |
vpx_highbd_12_variance64x32_sse2 Line | Count | Source | 160 | 2.45k | int ref_stride, uint32_t *sse) { \ | 161 | 2.45k | int sum; \ | 162 | 2.45k | int64_t var; \ | 163 | 2.45k | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ | 164 | 2.45k | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ | 165 | 2.45k | highbd_12_variance_sse2( \ | 166 | 2.45k | src, src_stride, ref, ref_stride, w, h, sse, &sum, \ | 167 | 2.45k | vpx_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ | 168 | 2.45k | var = (int64_t)(*sse) - (((int64_t)sum * sum) >> (shift)); \ | 169 | 2.45k | return (var >= 0) ? (uint32_t)var : 0; \ | 170 | 2.45k | } |
vpx_highbd_12_variance32x64_sse2 Line | Count | Source | 160 | 153 | int ref_stride, uint32_t *sse) { \ | 161 | 153 | int sum; \ | 162 | 153 | int64_t var; \ | 163 | 153 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ | 164 | 153 | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ | 165 | 153 | highbd_12_variance_sse2( \ | 166 | 153 | src, src_stride, ref, ref_stride, w, h, sse, &sum, \ | 167 | 153 | vpx_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ | 168 | 153 | var = (int64_t)(*sse) - (((int64_t)sum * sum) >> (shift)); \ | 169 | 153 | return (var >= 0) ? (uint32_t)var : 0; \ | 170 | 153 | } |
vpx_highbd_12_variance32x32_sse2 Line | Count | Source | 160 | 45.9k | int ref_stride, uint32_t *sse) { \ | 161 | 45.9k | int sum; \ | 162 | 45.9k | int64_t var; \ | 163 | 45.9k | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ | 164 | 45.9k | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ | 165 | 45.9k | highbd_12_variance_sse2( \ | 166 | 45.9k | src, src_stride, ref, ref_stride, w, h, sse, &sum, \ | 167 | 45.9k | vpx_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ | 168 | 45.9k | var = (int64_t)(*sse) - (((int64_t)sum * sum) >> (shift)); \ | 169 | 45.9k | return (var >= 0) ? (uint32_t)var : 0; \ | 170 | 45.9k | } |
vpx_highbd_12_variance32x16_sse2 Line | Count | Source | 160 | 5.85k | int ref_stride, uint32_t *sse) { \ | 161 | 5.85k | int sum; \ | 162 | 5.85k | int64_t var; \ | 163 | 5.85k | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ | 164 | 5.85k | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ | 165 | 5.85k | highbd_12_variance_sse2( \ | 166 | 5.85k | src, src_stride, ref, ref_stride, w, h, sse, &sum, \ | 167 | 5.85k | vpx_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ | 168 | 5.85k | var = (int64_t)(*sse) - (((int64_t)sum * sum) >> (shift)); \ | 169 | 5.85k | return (var >= 0) ? (uint32_t)var : 0; \ | 170 | 5.85k | } |
vpx_highbd_12_variance16x32_sse2 Line | Count | Source | 160 | 367 | int ref_stride, uint32_t *sse) { \ | 161 | 367 | int sum; \ | 162 | 367 | int64_t var; \ | 163 | 367 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ | 164 | 367 | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ | 165 | 367 | highbd_12_variance_sse2( \ | 166 | 367 | src, src_stride, ref, ref_stride, w, h, sse, &sum, \ | 167 | 367 | vpx_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ | 168 | 367 | var = (int64_t)(*sse) - (((int64_t)sum * sum) >> (shift)); \ | 169 | 367 | return (var >= 0) ? (uint32_t)var : 0; \ | 170 | 367 | } |
vpx_highbd_12_variance16x16_sse2 Line | Count | Source | 160 | 116k | int ref_stride, uint32_t *sse) { \ | 161 | 116k | int sum; \ | 162 | 116k | int64_t var; \ | 163 | 116k | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ | 164 | 116k | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ | 165 | 116k | highbd_12_variance_sse2( \ | 166 | 116k | src, src_stride, ref, ref_stride, w, h, sse, &sum, \ | 167 | 116k | vpx_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ | 168 | 116k | var = (int64_t)(*sse) - (((int64_t)sum * sum) >> (shift)); \ | 169 | 116k | return (var >= 0) ? (uint32_t)var : 0; \ | 170 | 116k | } |
vpx_highbd_12_variance16x8_sse2 Line | Count | Source | 160 | 16.0k | int ref_stride, uint32_t *sse) { \ | 161 | 16.0k | int sum; \ | 162 | 16.0k | int64_t var; \ | 163 | 16.0k | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ | 164 | 16.0k | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ | 165 | 16.0k | highbd_12_variance_sse2( \ | 166 | 16.0k | src, src_stride, ref, ref_stride, w, h, sse, &sum, \ | 167 | 16.0k | vpx_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ | 168 | 16.0k | var = (int64_t)(*sse) - (((int64_t)sum * sum) >> (shift)); \ | 169 | 16.0k | return (var >= 0) ? (uint32_t)var : 0; \ | 170 | 16.0k | } |
vpx_highbd_12_variance8x16_sse2 Line | Count | Source | 160 | 4.65k | int ref_stride, uint32_t *sse) { \ | 161 | 4.65k | int sum; \ | 162 | 4.65k | int64_t var; \ | 163 | 4.65k | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ | 164 | 4.65k | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ | 165 | 4.65k | highbd_12_variance_sse2( \ | 166 | 4.65k | src, src_stride, ref, ref_stride, w, h, sse, &sum, \ | 167 | 4.65k | vpx_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ | 168 | 4.65k | var = (int64_t)(*sse) - (((int64_t)sum * sum) >> (shift)); \ | 169 | 4.65k | return (var >= 0) ? (uint32_t)var : 0; \ | 170 | 4.65k | } |
vpx_highbd_12_variance8x8_sse2 Line | Count | Source | 160 | 607k | int ref_stride, uint32_t *sse) { \ | 161 | 607k | int sum; \ | 162 | 607k | int64_t var; \ | 163 | 607k | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ | 164 | 607k | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ | 165 | 607k | highbd_12_variance_sse2( \ | 166 | 607k | src, src_stride, ref, ref_stride, w, h, sse, &sum, \ | 167 | 607k | vpx_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ | 168 | 607k | var = (int64_t)(*sse) - (((int64_t)sum * sum) >> (shift)); \ | 169 | 607k | return (var >= 0) ? (uint32_t)var : 0; \ | 170 | 607k | } |
|
171 | | |
172 | | VAR_FN(64, 64, 16, 12) |
173 | | VAR_FN(64, 32, 16, 11) |
174 | | VAR_FN(32, 64, 16, 11) |
175 | | VAR_FN(32, 32, 16, 10) |
176 | | VAR_FN(32, 16, 16, 9) |
177 | | VAR_FN(16, 32, 16, 9) |
178 | | VAR_FN(16, 16, 16, 8) |
179 | | VAR_FN(16, 8, 8, 7) |
180 | | VAR_FN(8, 16, 8, 7) |
181 | | VAR_FN(8, 8, 8, 6) |
182 | | |
183 | | #undef VAR_FN |
184 | | |
185 | | unsigned int vpx_highbd_8_mse16x16_sse2(const uint8_t *src8, int src_stride, |
186 | | const uint8_t *ref8, int ref_stride, |
187 | 0 | unsigned int *sse) { |
188 | 0 | int sum; |
189 | 0 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); |
190 | 0 | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); |
191 | 0 | highbd_8_variance_sse2(src, src_stride, ref, ref_stride, 16, 16, sse, &sum, |
192 | 0 | vpx_highbd_calc16x16var_sse2, 16); |
193 | 0 | return *sse; |
194 | 0 | } |
195 | | |
196 | | unsigned int vpx_highbd_10_mse16x16_sse2(const uint8_t *src8, int src_stride, |
197 | | const uint8_t *ref8, int ref_stride, |
198 | 0 | unsigned int *sse) { |
199 | 0 | int sum; |
200 | 0 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); |
201 | 0 | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); |
202 | 0 | highbd_10_variance_sse2(src, src_stride, ref, ref_stride, 16, 16, sse, &sum, |
203 | 0 | vpx_highbd_calc16x16var_sse2, 16); |
204 | 0 | return *sse; |
205 | 0 | } |
206 | | |
207 | | unsigned int vpx_highbd_12_mse16x16_sse2(const uint8_t *src8, int src_stride, |
208 | | const uint8_t *ref8, int ref_stride, |
209 | 0 | unsigned int *sse) { |
210 | 0 | int sum; |
211 | 0 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); |
212 | 0 | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); |
213 | 0 | highbd_12_variance_sse2(src, src_stride, ref, ref_stride, 16, 16, sse, &sum, |
214 | 0 | vpx_highbd_calc16x16var_sse2, 16); |
215 | 0 | return *sse; |
216 | 0 | } |
217 | | |
218 | | unsigned int vpx_highbd_8_mse8x8_sse2(const uint8_t *src8, int src_stride, |
219 | | const uint8_t *ref8, int ref_stride, |
220 | 0 | unsigned int *sse) { |
221 | 0 | int sum; |
222 | 0 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); |
223 | 0 | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); |
224 | 0 | highbd_8_variance_sse2(src, src_stride, ref, ref_stride, 8, 8, sse, &sum, |
225 | 0 | vpx_highbd_calc8x8var_sse2, 8); |
226 | 0 | return *sse; |
227 | 0 | } |
228 | | |
229 | | unsigned int vpx_highbd_10_mse8x8_sse2(const uint8_t *src8, int src_stride, |
230 | | const uint8_t *ref8, int ref_stride, |
231 | 0 | unsigned int *sse) { |
232 | 0 | int sum; |
233 | 0 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); |
234 | 0 | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); |
235 | 0 | highbd_10_variance_sse2(src, src_stride, ref, ref_stride, 8, 8, sse, &sum, |
236 | 0 | vpx_highbd_calc8x8var_sse2, 8); |
237 | 0 | return *sse; |
238 | 0 | } |
239 | | |
240 | | unsigned int vpx_highbd_12_mse8x8_sse2(const uint8_t *src8, int src_stride, |
241 | | const uint8_t *ref8, int ref_stride, |
242 | 0 | unsigned int *sse) { |
243 | 0 | int sum; |
244 | 0 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); |
245 | 0 | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); |
246 | 0 | highbd_12_variance_sse2(src, src_stride, ref, ref_stride, 8, 8, sse, &sum, |
247 | 0 | vpx_highbd_calc8x8var_sse2, 8); |
248 | 0 | return *sse; |
249 | 0 | } |
250 | | |
251 | | // The 2 unused parameters are place holders for PIC enabled build. |
252 | | // These definitions are for functions defined in |
253 | | // highbd_subpel_variance_impl_sse2.asm |
254 | | #define DECL(w, opt) \ |
255 | | int vpx_highbd_sub_pixel_variance##w##xh_##opt( \ |
256 | | const uint16_t *src, ptrdiff_t src_stride, int x_offset, int y_offset, \ |
257 | | const uint16_t *ref, ptrdiff_t ref_stride, int height, \ |
258 | | unsigned int *sse, void *unused0, void *unused); |
259 | | #define DECLS(opt) \ |
260 | | DECL(8, opt) \ |
261 | | DECL(16, opt) |
262 | | |
263 | | DECLS(sse2) |
264 | | |
265 | | #undef DECLS |
266 | | #undef DECL |
267 | | |
268 | | #define FN(w, h, wf, wlog2, hlog2, opt, cast) \ |
269 | | uint32_t vpx_highbd_8_sub_pixel_variance##w##x##h##_##opt( \ |
270 | | const uint8_t *src8, int src_stride, int x_offset, int y_offset, \ |
271 | 0 | const uint8_t *ref8, int ref_stride, uint32_t *sse_ptr) { \ |
272 | 0 | uint32_t sse; \ |
273 | 0 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ |
274 | 0 | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ |
275 | 0 | int se = vpx_highbd_sub_pixel_variance##wf##xh_##opt( \ |
276 | 0 | src, src_stride, x_offset, y_offset, ref, ref_stride, h, &sse, NULL, \ |
277 | 0 | NULL); \ |
278 | 0 | if (w > wf) { \ |
279 | 0 | unsigned int sse2; \ |
280 | 0 | int se2 = vpx_highbd_sub_pixel_variance##wf##xh_##opt( \ |
281 | 0 | src + 16, src_stride, x_offset, y_offset, ref + 16, ref_stride, h, \ |
282 | 0 | &sse2, NULL, NULL); \ |
283 | 0 | se += se2; \ |
284 | 0 | sse += sse2; \ |
285 | 0 | if (w > wf * 2) { \ |
286 | 0 | se2 = vpx_highbd_sub_pixel_variance##wf##xh_##opt( \ |
287 | 0 | src + 32, src_stride, x_offset, y_offset, ref + 32, ref_stride, h, \ |
288 | 0 | &sse2, NULL, NULL); \ |
289 | 0 | se += se2; \ |
290 | 0 | sse += sse2; \ |
291 | 0 | se2 = vpx_highbd_sub_pixel_variance##wf##xh_##opt( \ |
292 | 0 | src + 48, src_stride, x_offset, y_offset, ref + 48, ref_stride, h, \ |
293 | 0 | &sse2, NULL, NULL); \ |
294 | 0 | se += se2; \ |
295 | 0 | sse += sse2; \ |
296 | 0 | } \ |
297 | 0 | } \ |
298 | 0 | *sse_ptr = sse; \ |
299 | 0 | return sse - (uint32_t)((cast se * se) >> (wlog2 + hlog2)); \ |
300 | 0 | } \ |
301 | | \ |
302 | | uint32_t vpx_highbd_10_sub_pixel_variance##w##x##h##_##opt( \ |
303 | | const uint8_t *src8, int src_stride, int x_offset, int y_offset, \ |
304 | 42.4k | const uint8_t *ref8, int ref_stride, uint32_t *sse_ptr) { \ |
305 | 42.4k | int64_t var; \ |
306 | 42.4k | uint32_t sse; \ |
307 | 42.4k | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ |
308 | 42.4k | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ |
309 | 42.4k | int se = vpx_highbd_sub_pixel_variance##wf##xh_##opt( \ |
310 | 42.4k | src, src_stride, x_offset, y_offset, ref, ref_stride, h, &sse, NULL, \ |
311 | 42.4k | NULL); \ |
312 | 42.4k | if (w > wf) { \ |
313 | 8.94k | uint32_t sse2; \ |
314 | 8.94k | int se2 = vpx_highbd_sub_pixel_variance##wf##xh_##opt( \ |
315 | 8.94k | src + 16, src_stride, x_offset, y_offset, ref + 16, ref_stride, h, \ |
316 | 8.94k | &sse2, NULL, NULL); \ |
317 | 8.94k | se += se2; \ |
318 | 8.94k | sse += sse2; \ |
319 | 8.94k | if (w > wf * 2) { \ |
320 | 3.19k | se2 = vpx_highbd_sub_pixel_variance##wf##xh_##opt( \ |
321 | 3.19k | src + 32, src_stride, x_offset, y_offset, ref + 32, ref_stride, h, \ |
322 | 3.19k | &sse2, NULL, NULL); \ |
323 | 3.19k | se += se2; \ |
324 | 3.19k | sse += sse2; \ |
325 | 3.19k | se2 = vpx_highbd_sub_pixel_variance##wf##xh_##opt( \ |
326 | 3.19k | src + 48, src_stride, x_offset, y_offset, ref + 48, ref_stride, h, \ |
327 | 3.19k | &sse2, NULL, NULL); \ |
328 | 3.19k | se += se2; \ |
329 | 3.19k | sse += sse2; \ |
330 | 3.19k | } \ |
331 | 8.94k | } \ |
332 | 42.4k | se = ROUND_POWER_OF_TWO(se, 2); \ |
333 | 42.4k | sse = ROUND_POWER_OF_TWO(sse, 4); \ |
334 | 42.4k | *sse_ptr = sse; \ |
335 | 42.4k | var = (int64_t)(sse) - ((cast se * se) >> (wlog2 + hlog2)); \ |
336 | 42.4k | return (var >= 0) ? (uint32_t)var : 0; \ |
337 | 42.4k | } \ |
338 | | \ |
339 | | uint32_t vpx_highbd_12_sub_pixel_variance##w##x##h##_##opt( \ |
340 | | const uint8_t *src8, int src_stride, int x_offset, int y_offset, \ |
341 | 126k | const uint8_t *ref8, int ref_stride, uint32_t *sse_ptr) { \ |
342 | 126k | int start_row; \ |
343 | 126k | uint32_t sse; \ |
344 | 126k | int se = 0; \ |
345 | 126k | int64_t var; \ |
346 | 126k | uint64_t long_sse = 0; \ |
347 | 126k | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ |
348 | 126k | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ |
349 | 269k | for (start_row = 0; start_row < h; start_row += 16) { \ |
350 | 142k | uint32_t sse2; \ |
351 | 142k | int height = h - start_row < 16 ? h - start_row : 16; \ |
352 | 142k | int se2 = vpx_highbd_sub_pixel_variance##wf##xh_##opt( \ |
353 | 142k | src + (start_row * src_stride), src_stride, x_offset, y_offset, \ |
354 | 142k | ref + (start_row * ref_stride), ref_stride, height, &sse2, NULL, \ |
355 | 142k | NULL); \ |
356 | 142k | se += se2; \ |
357 | 142k | long_sse += sse2; \ |
358 | 142k | if (w > wf) { \ |
359 | 27.1k | se2 = vpx_highbd_sub_pixel_variance##wf##xh_##opt( \ |
360 | 27.1k | src + 16 + (start_row * src_stride), src_stride, x_offset, \ |
361 | 27.1k | y_offset, ref + 16 + (start_row * ref_stride), ref_stride, height, \ |
362 | 27.1k | &sse2, NULL, NULL); \ |
363 | 27.1k | se += se2; \ |
364 | 27.1k | long_sse += sse2; \ |
365 | 27.1k | if (w > wf * 2) { \ |
366 | 13.0k | se2 = vpx_highbd_sub_pixel_variance##wf##xh_##opt( \ |
367 | 13.0k | src + 32 + (start_row * src_stride), src_stride, x_offset, \ |
368 | 13.0k | y_offset, ref + 32 + (start_row * ref_stride), ref_stride, \ |
369 | 13.0k | height, &sse2, NULL, NULL); \ |
370 | 13.0k | se += se2; \ |
371 | 13.0k | long_sse += sse2; \ |
372 | 13.0k | se2 = vpx_highbd_sub_pixel_variance##wf##xh_##opt( \ |
373 | 13.0k | src + 48 + (start_row * src_stride), src_stride, x_offset, \ |
374 | 13.0k | y_offset, ref + 48 + (start_row * ref_stride), ref_stride, \ |
375 | 13.0k | height, &sse2, NULL, NULL); \ |
376 | 13.0k | se += se2; \ |
377 | 13.0k | long_sse += sse2; \ |
378 | 13.0k | } \ |
379 | 27.1k | } \ |
380 | 142k | } \ |
381 | 126k | se = ROUND_POWER_OF_TWO(se, 4); \ |
382 | 126k | sse = (uint32_t)ROUND_POWER_OF_TWO(long_sse, 8); \ |
383 | 126k | *sse_ptr = sse; \ |
384 | 126k | var = (int64_t)(sse) - ((cast se * se) >> (wlog2 + hlog2)); \ |
385 | 126k | return (var >= 0) ? (uint32_t)var : 0; \ |
386 | 126k | } |
387 | | |
388 | | #define FNS(opt) \ |
389 | | FN(64, 64, 16, 6, 6, opt, (int64_t)) \ |
390 | | FN(64, 32, 16, 6, 5, opt, (int64_t)) \ |
391 | | FN(32, 64, 16, 5, 6, opt, (int64_t)) \ |
392 | | FN(32, 32, 16, 5, 5, opt, (int64_t)) \ |
393 | | FN(32, 16, 16, 5, 4, opt, (int64_t)) \ |
394 | | FN(16, 32, 16, 4, 5, opt, (int64_t)) \ |
395 | | FN(16, 16, 16, 4, 4, opt, (int64_t)) \ |
396 | | FN(16, 8, 16, 4, 3, opt, (int64_t)) \ |
397 | | FN(8, 16, 8, 3, 4, opt, (int64_t)) \ |
398 | | FN(8, 8, 8, 3, 3, opt, (int64_t)) \ |
399 | | FN(8, 4, 8, 3, 2, opt, (int64_t)) |
400 | | |
401 | 506k | FNS(sse2) Unexecuted instantiation: vpx_highbd_8_sub_pixel_variance64x64_sse2 vpx_highbd_10_sub_pixel_variance64x64_sse2 Line | Count | Source | 401 | | FNS(sse2) |
vpx_highbd_12_sub_pixel_variance64x64_sse2 Line | Count | Source | 401 | | FNS(sse2) |
Unexecuted instantiation: vpx_highbd_8_sub_pixel_variance64x32_sse2 vpx_highbd_10_sub_pixel_variance64x32_sse2 Line | Count | Source | 401 | | FNS(sse2) |
vpx_highbd_12_sub_pixel_variance64x32_sse2 Line | Count | Source | 401 | | FNS(sse2) |
Unexecuted instantiation: vpx_highbd_8_sub_pixel_variance32x64_sse2 vpx_highbd_10_sub_pixel_variance32x64_sse2 Line | Count | Source | 401 | | FNS(sse2) |
vpx_highbd_12_sub_pixel_variance32x64_sse2 Line | Count | Source | 401 | | FNS(sse2) |
Unexecuted instantiation: vpx_highbd_8_sub_pixel_variance32x32_sse2 vpx_highbd_10_sub_pixel_variance32x32_sse2 Line | Count | Source | 401 | | FNS(sse2) |
vpx_highbd_12_sub_pixel_variance32x32_sse2 Line | Count | Source | 401 | | FNS(sse2) |
Unexecuted instantiation: vpx_highbd_8_sub_pixel_variance32x16_sse2 vpx_highbd_10_sub_pixel_variance32x16_sse2 Line | Count | Source | 401 | | FNS(sse2) |
vpx_highbd_12_sub_pixel_variance32x16_sse2 Line | Count | Source | 401 | | FNS(sse2) |
Unexecuted instantiation: vpx_highbd_8_sub_pixel_variance16x32_sse2 vpx_highbd_10_sub_pixel_variance16x32_sse2 Line | Count | Source | 401 | | FNS(sse2) |
vpx_highbd_12_sub_pixel_variance16x32_sse2 Line | Count | Source | 401 | | FNS(sse2) |
Unexecuted instantiation: vpx_highbd_8_sub_pixel_variance16x16_sse2 vpx_highbd_10_sub_pixel_variance16x16_sse2 Line | Count | Source | 401 | | FNS(sse2) |
vpx_highbd_12_sub_pixel_variance16x16_sse2 Line | Count | Source | 401 | | FNS(sse2) |
Unexecuted instantiation: vpx_highbd_8_sub_pixel_variance16x8_sse2 vpx_highbd_10_sub_pixel_variance16x8_sse2 Line | Count | Source | 401 | | FNS(sse2) |
vpx_highbd_12_sub_pixel_variance16x8_sse2 Line | Count | Source | 401 | | FNS(sse2) |
Unexecuted instantiation: vpx_highbd_8_sub_pixel_variance8x16_sse2 vpx_highbd_10_sub_pixel_variance8x16_sse2 Line | Count | Source | 401 | | FNS(sse2) |
vpx_highbd_12_sub_pixel_variance8x16_sse2 Line | Count | Source | 401 | | FNS(sse2) |
Unexecuted instantiation: vpx_highbd_8_sub_pixel_variance8x8_sse2 vpx_highbd_10_sub_pixel_variance8x8_sse2 Line | Count | Source | 401 | | FNS(sse2) |
vpx_highbd_12_sub_pixel_variance8x8_sse2 Line | Count | Source | 401 | | FNS(sse2) |
Unexecuted instantiation: vpx_highbd_8_sub_pixel_variance8x4_sse2 vpx_highbd_10_sub_pixel_variance8x4_sse2 Line | Count | Source | 401 | | FNS(sse2) |
vpx_highbd_12_sub_pixel_variance8x4_sse2 Line | Count | Source | 401 | | FNS(sse2) |
|
402 | 506k | |
403 | 506k | #undef FNS |
404 | 506k | #undef FN |
405 | 506k | |
406 | 506k | // The 2 unused parameters are place holders for PIC enabled build. |
407 | 506k | #define DECL(w, opt) \ |
408 | 506k | int vpx_highbd_sub_pixel_avg_variance##w##xh_##opt( \ |
409 | 506k | const uint16_t *src, ptrdiff_t src_stride, int x_offset, int y_offset, \ |
410 | 506k | const uint16_t *ref, ptrdiff_t ref_stride, const uint16_t *second, \ |
411 | 506k | ptrdiff_t second_stride, int height, unsigned int *sse, void *unused0, \ |
412 | 506k | void *unused); |
413 | 506k | #define DECLS(opt1) \ |
414 | 506k | DECL(16, opt1) \ |
415 | 506k | DECL(8, opt1) |
416 | 506k | |
417 | 506k | DECLS(sse2) |
418 | 506k | #undef DECL |
419 | 506k | #undef DECLS |
420 | 506k | |
421 | 506k | #define FN(w, h, wf, wlog2, hlog2, opt, cast) \ |
422 | 506k | uint32_t vpx_highbd_8_sub_pixel_avg_variance##w##x##h##_##opt( \ |
423 | 506k | const uint8_t *src8, int src_stride, int x_offset, int y_offset, \ |
424 | 506k | const uint8_t *ref8, int ref_stride, uint32_t *sse_ptr, \ |
425 | 506k | const uint8_t *sec8) { \ |
426 | 0 | uint32_t sse; \ |
427 | 0 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ |
428 | 0 | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ |
429 | 0 | uint16_t *sec = CONVERT_TO_SHORTPTR(sec8); \ |
430 | 0 | int se = vpx_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ |
431 | 0 | src, src_stride, x_offset, y_offset, ref, ref_stride, sec, w, h, &sse, \ |
432 | 0 | NULL, NULL); \ |
433 | 0 | if (w > wf) { \ |
434 | 0 | uint32_t sse2; \ |
435 | 0 | int se2 = vpx_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ |
436 | 0 | src + 16, src_stride, x_offset, y_offset, ref + 16, ref_stride, \ |
437 | 0 | sec + 16, w, h, &sse2, NULL, NULL); \ |
438 | 0 | se += se2; \ |
439 | 0 | sse += sse2; \ |
440 | 0 | if (w > wf * 2) { \ |
441 | 0 | se2 = vpx_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ |
442 | 0 | src + 32, src_stride, x_offset, y_offset, ref + 32, ref_stride, \ |
443 | 0 | sec + 32, w, h, &sse2, NULL, NULL); \ |
444 | 0 | se += se2; \ |
445 | 0 | sse += sse2; \ |
446 | 0 | se2 = vpx_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ |
447 | 0 | src + 48, src_stride, x_offset, y_offset, ref + 48, ref_stride, \ |
448 | 0 | sec + 48, w, h, &sse2, NULL, NULL); \ |
449 | 0 | se += se2; \ |
450 | 0 | sse += sse2; \ |
451 | 0 | } \ |
452 | 0 | } \ |
453 | 0 | *sse_ptr = sse; \ |
454 | 0 | return sse - (uint32_t)((cast se * se) >> (wlog2 + hlog2)); \ |
455 | 0 | } \ |
456 | | \ |
457 | | uint32_t vpx_highbd_10_sub_pixel_avg_variance##w##x##h##_##opt( \ |
458 | | const uint8_t *src8, int src_stride, int x_offset, int y_offset, \ |
459 | | const uint8_t *ref8, int ref_stride, uint32_t *sse_ptr, \ |
460 | 0 | const uint8_t *sec8) { \ |
461 | 0 | int64_t var; \ |
462 | 0 | uint32_t sse; \ |
463 | 0 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ |
464 | 0 | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ |
465 | 0 | uint16_t *sec = CONVERT_TO_SHORTPTR(sec8); \ |
466 | 0 | int se = vpx_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ |
467 | 0 | src, src_stride, x_offset, y_offset, ref, ref_stride, sec, w, h, &sse, \ |
468 | 0 | NULL, NULL); \ |
469 | 0 | if (w > wf) { \ |
470 | 0 | uint32_t sse2; \ |
471 | 0 | int se2 = vpx_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ |
472 | 0 | src + 16, src_stride, x_offset, y_offset, ref + 16, ref_stride, \ |
473 | 0 | sec + 16, w, h, &sse2, NULL, NULL); \ |
474 | 0 | se += se2; \ |
475 | 0 | sse += sse2; \ |
476 | 0 | if (w > wf * 2) { \ |
477 | 0 | se2 = vpx_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ |
478 | 0 | src + 32, src_stride, x_offset, y_offset, ref + 32, ref_stride, \ |
479 | 0 | sec + 32, w, h, &sse2, NULL, NULL); \ |
480 | 0 | se += se2; \ |
481 | 0 | sse += sse2; \ |
482 | 0 | se2 = vpx_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ |
483 | 0 | src + 48, src_stride, x_offset, y_offset, ref + 48, ref_stride, \ |
484 | 0 | sec + 48, w, h, &sse2, NULL, NULL); \ |
485 | 0 | se += se2; \ |
486 | 0 | sse += sse2; \ |
487 | 0 | } \ |
488 | 0 | } \ |
489 | 0 | se = ROUND_POWER_OF_TWO(se, 2); \ |
490 | 0 | sse = ROUND_POWER_OF_TWO(sse, 4); \ |
491 | 0 | *sse_ptr = sse; \ |
492 | 0 | var = (int64_t)(sse) - ((cast se * se) >> (wlog2 + hlog2)); \ |
493 | 0 | return (var >= 0) ? (uint32_t)var : 0; \ |
494 | 0 | } \ |
495 | | \ |
496 | | uint32_t vpx_highbd_12_sub_pixel_avg_variance##w##x##h##_##opt( \ |
497 | | const uint8_t *src8, int src_stride, int x_offset, int y_offset, \ |
498 | | const uint8_t *ref8, int ref_stride, uint32_t *sse_ptr, \ |
499 | 0 | const uint8_t *sec8) { \ |
500 | 0 | int start_row; \ |
501 | 0 | int64_t var; \ |
502 | 0 | uint32_t sse; \ |
503 | 0 | int se = 0; \ |
504 | 0 | uint64_t long_sse = 0; \ |
505 | 0 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ |
506 | 0 | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ |
507 | 0 | uint16_t *sec = CONVERT_TO_SHORTPTR(sec8); \ |
508 | 0 | for (start_row = 0; start_row < h; start_row += 16) { \ |
509 | 0 | uint32_t sse2; \ |
510 | 0 | int height = h - start_row < 16 ? h - start_row : 16; \ |
511 | 0 | int se2 = vpx_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ |
512 | 0 | src + (start_row * src_stride), src_stride, x_offset, y_offset, \ |
513 | 0 | ref + (start_row * ref_stride), ref_stride, sec + (start_row * w), \ |
514 | 0 | w, height, &sse2, NULL, NULL); \ |
515 | 0 | se += se2; \ |
516 | 0 | long_sse += sse2; \ |
517 | 0 | if (w > wf) { \ |
518 | 0 | se2 = vpx_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ |
519 | 0 | src + 16 + (start_row * src_stride), src_stride, x_offset, \ |
520 | 0 | y_offset, ref + 16 + (start_row * ref_stride), ref_stride, \ |
521 | 0 | sec + 16 + (start_row * w), w, height, &sse2, NULL, NULL); \ |
522 | 0 | se += se2; \ |
523 | 0 | long_sse += sse2; \ |
524 | 0 | if (w > wf * 2) { \ |
525 | 0 | se2 = vpx_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ |
526 | 0 | src + 32 + (start_row * src_stride), src_stride, x_offset, \ |
527 | 0 | y_offset, ref + 32 + (start_row * ref_stride), ref_stride, \ |
528 | 0 | sec + 32 + (start_row * w), w, height, &sse2, NULL, NULL); \ |
529 | 0 | se += se2; \ |
530 | 0 | long_sse += sse2; \ |
531 | 0 | se2 = vpx_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ |
532 | 0 | src + 48 + (start_row * src_stride), src_stride, x_offset, \ |
533 | 0 | y_offset, ref + 48 + (start_row * ref_stride), ref_stride, \ |
534 | 0 | sec + 48 + (start_row * w), w, height, &sse2, NULL, NULL); \ |
535 | 0 | se += se2; \ |
536 | 0 | long_sse += sse2; \ |
537 | 0 | } \ |
538 | 0 | } \ |
539 | 0 | } \ |
540 | 0 | se = ROUND_POWER_OF_TWO(se, 4); \ |
541 | 0 | sse = (uint32_t)ROUND_POWER_OF_TWO(long_sse, 8); \ |
542 | 0 | *sse_ptr = sse; \ |
543 | 0 | var = (int64_t)(sse) - ((cast se * se) >> (wlog2 + hlog2)); \ |
544 | 0 | return (var >= 0) ? (uint32_t)var : 0; \ |
545 | 0 | } |
546 | | |
547 | | #define FNS(opt1) \ |
548 | | FN(64, 64, 16, 6, 6, opt1, (int64_t)) \ |
549 | | FN(64, 32, 16, 6, 5, opt1, (int64_t)) \ |
550 | | FN(32, 64, 16, 5, 6, opt1, (int64_t)) \ |
551 | | FN(32, 32, 16, 5, 5, opt1, (int64_t)) \ |
552 | | FN(32, 16, 16, 5, 4, opt1, (int64_t)) \ |
553 | | FN(16, 32, 16, 4, 5, opt1, (int64_t)) \ |
554 | | FN(16, 16, 16, 4, 4, opt1, (int64_t)) \ |
555 | | FN(16, 8, 16, 4, 3, opt1, (int64_t)) \ |
556 | | FN(8, 16, 8, 4, 3, opt1, (int64_t)) \ |
557 | | FN(8, 8, 8, 3, 3, opt1, (int64_t)) \ |
558 | | FN(8, 4, 8, 3, 2, opt1, (int64_t)) |
559 | | |
560 | 0 | FNS(sse2) Unexecuted instantiation: vpx_highbd_8_sub_pixel_avg_variance64x64_sse2 Unexecuted instantiation: vpx_highbd_10_sub_pixel_avg_variance64x64_sse2 Unexecuted instantiation: vpx_highbd_12_sub_pixel_avg_variance64x64_sse2 Unexecuted instantiation: vpx_highbd_8_sub_pixel_avg_variance64x32_sse2 Unexecuted instantiation: vpx_highbd_10_sub_pixel_avg_variance64x32_sse2 Unexecuted instantiation: vpx_highbd_12_sub_pixel_avg_variance64x32_sse2 Unexecuted instantiation: vpx_highbd_8_sub_pixel_avg_variance32x64_sse2 Unexecuted instantiation: vpx_highbd_10_sub_pixel_avg_variance32x64_sse2 Unexecuted instantiation: vpx_highbd_12_sub_pixel_avg_variance32x64_sse2 Unexecuted instantiation: vpx_highbd_8_sub_pixel_avg_variance32x32_sse2 Unexecuted instantiation: vpx_highbd_10_sub_pixel_avg_variance32x32_sse2 Unexecuted instantiation: vpx_highbd_12_sub_pixel_avg_variance32x32_sse2 Unexecuted instantiation: vpx_highbd_8_sub_pixel_avg_variance32x16_sse2 Unexecuted instantiation: vpx_highbd_10_sub_pixel_avg_variance32x16_sse2 Unexecuted instantiation: vpx_highbd_12_sub_pixel_avg_variance32x16_sse2 Unexecuted instantiation: vpx_highbd_8_sub_pixel_avg_variance16x32_sse2 Unexecuted instantiation: vpx_highbd_10_sub_pixel_avg_variance16x32_sse2 Unexecuted instantiation: vpx_highbd_12_sub_pixel_avg_variance16x32_sse2 Unexecuted instantiation: vpx_highbd_8_sub_pixel_avg_variance16x16_sse2 Unexecuted instantiation: vpx_highbd_10_sub_pixel_avg_variance16x16_sse2 Unexecuted instantiation: vpx_highbd_12_sub_pixel_avg_variance16x16_sse2 Unexecuted instantiation: vpx_highbd_8_sub_pixel_avg_variance16x8_sse2 Unexecuted instantiation: vpx_highbd_10_sub_pixel_avg_variance16x8_sse2 Unexecuted instantiation: vpx_highbd_12_sub_pixel_avg_variance16x8_sse2 Unexecuted instantiation: vpx_highbd_8_sub_pixel_avg_variance8x16_sse2 Unexecuted instantiation: vpx_highbd_10_sub_pixel_avg_variance8x16_sse2 Unexecuted instantiation: vpx_highbd_12_sub_pixel_avg_variance8x16_sse2 Unexecuted instantiation: vpx_highbd_8_sub_pixel_avg_variance8x8_sse2 Unexecuted instantiation: vpx_highbd_10_sub_pixel_avg_variance8x8_sse2 Unexecuted instantiation: vpx_highbd_12_sub_pixel_avg_variance8x8_sse2 Unexecuted instantiation: vpx_highbd_8_sub_pixel_avg_variance8x4_sse2 Unexecuted instantiation: vpx_highbd_10_sub_pixel_avg_variance8x4_sse2 Unexecuted instantiation: vpx_highbd_12_sub_pixel_avg_variance8x4_sse2 |
561 | 0 |
|
562 | 0 | #undef FNS |
563 | 0 | #undef FN |
564 | 0 | #endif // HAVE_X86_ASM |
565 | 0 |
|
566 | 0 | void vpx_highbd_comp_avg_pred_sse2(uint16_t *comp_pred, const uint16_t *pred, |
567 | 0 | int width, int height, const uint16_t *ref, |
568 | 0 | int ref_stride) { |
569 | 0 | int i, j; |
570 | 0 | if (width > 8) { |
571 | 0 | for (i = 0; i < height; ++i) { |
572 | 0 | for (j = 0; j < width; j += 16) { |
573 | 0 | const __m128i p0 = _mm_loadu_si128((const __m128i *)&pred[j]); |
574 | 0 | const __m128i p1 = _mm_loadu_si128((const __m128i *)&pred[j + 8]); |
575 | 0 | const __m128i r0 = _mm_loadu_si128((const __m128i *)&ref[j]); |
576 | 0 | const __m128i r1 = _mm_loadu_si128((const __m128i *)&ref[j + 8]); |
577 | 0 | _mm_storeu_si128((__m128i *)&comp_pred[j], _mm_avg_epu16(p0, r0)); |
578 | 0 | _mm_storeu_si128((__m128i *)&comp_pred[j + 8], _mm_avg_epu16(p1, r1)); |
579 | 0 | } |
580 | 0 | comp_pred += width; |
581 | 0 | pred += width; |
582 | 0 | ref += ref_stride; |
583 | 0 | } |
584 | 0 | } else if (width == 8) { |
585 | 0 | for (i = 0; i < height; i += 2) { |
586 | 0 | const __m128i p0 = _mm_loadu_si128((const __m128i *)&pred[0]); |
587 | 0 | const __m128i p1 = _mm_loadu_si128((const __m128i *)&pred[8]); |
588 | 0 | const __m128i r0 = _mm_loadu_si128((const __m128i *)&ref[0]); |
589 | 0 | const __m128i r1 = _mm_loadu_si128((const __m128i *)&ref[ref_stride]); |
590 | 0 | _mm_storeu_si128((__m128i *)&comp_pred[0], _mm_avg_epu16(p0, r0)); |
591 | 0 | _mm_storeu_si128((__m128i *)&comp_pred[8], _mm_avg_epu16(p1, r1)); |
592 | 0 | comp_pred += 8 << 1; |
593 | 0 | pred += 8 << 1; |
594 | 0 | ref += ref_stride << 1; |
595 | 0 | } |
596 | 0 | } else { |
597 | 0 | assert(width == 4); |
598 | 0 | for (i = 0; i < height; i += 2) { |
599 | 0 | const __m128i p0 = _mm_loadl_epi64((const __m128i *)&pred[0]); |
600 | 0 | const __m128i p1 = _mm_loadl_epi64((const __m128i *)&pred[4]); |
601 | 0 | const __m128i r0 = _mm_loadl_epi64((const __m128i *)&ref[0]); |
602 | 0 | const __m128i r1 = _mm_loadl_epi64((const __m128i *)&ref[ref_stride]); |
603 | 0 | _mm_storel_epi64((__m128i *)&comp_pred[0], _mm_avg_epu16(p0, r0)); |
604 | 0 | _mm_storel_epi64((__m128i *)&comp_pred[4], _mm_avg_epu16(p1, r1)); |
605 | 0 | comp_pred += 4 << 1; |
606 | 0 | pred += 4 << 1; |
607 | 0 | ref += ref_stride << 1; |
608 | 0 | } |
609 | 0 | } |
610 | 0 | } |