/src/aom/aom_dsp/x86/blend_sse4.h
Line | Count | Source |
1 | | /* |
2 | | * Copyright (c) 2016, Alliance for Open Media. All rights reserved. |
3 | | * |
4 | | * This source code is subject to the terms of the BSD 2 Clause License and |
5 | | * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License |
6 | | * was not distributed with this source code in the LICENSE file, you can |
7 | | * obtain it at www.aomedia.org/license/software. If the Alliance for Open |
8 | | * Media Patent License 1.0 was not distributed with this source code in the |
9 | | * PATENTS file, you can obtain it at www.aomedia.org/license/patent. |
10 | | */ |
11 | | |
12 | | #ifndef AOM_AOM_DSP_X86_BLEND_SSE4_H_ |
13 | | #define AOM_AOM_DSP_X86_BLEND_SSE4_H_ |
14 | | |
15 | | #include "aom_dsp/blend.h" |
16 | | #include "aom_dsp/x86/synonyms.h" |
17 | | static const uint8_t g_blend_a64_mask_shuffle[32] = { |
18 | | 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, |
19 | | 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15, |
20 | | }; |
21 | | |
22 | | ////////////////////////////////////////////////////////////////////////////// |
23 | | // Common kernels |
24 | | ////////////////////////////////////////////////////////////////////////////// |
25 | | |
26 | | static inline __m128i blend_4(const uint8_t *src0, const uint8_t *src1, |
27 | 163k | const __m128i *v_m0_w, const __m128i *v_m1_w) { |
28 | 163k | const __m128i v_s0_b = xx_loadl_32(src0); |
29 | 163k | const __m128i v_s1_b = xx_loadl_32(src1); |
30 | 163k | const __m128i v_s0_w = _mm_cvtepu8_epi16(v_s0_b); |
31 | 163k | const __m128i v_s1_w = _mm_cvtepu8_epi16(v_s1_b); |
32 | | |
33 | 163k | const __m128i v_p0_w = _mm_mullo_epi16(v_s0_w, *v_m0_w); |
34 | 163k | const __m128i v_p1_w = _mm_mullo_epi16(v_s1_w, *v_m1_w); |
35 | 163k | const __m128i v_sum_w = _mm_add_epi16(v_p0_w, v_p1_w); |
36 | 163k | const __m128i v_res_w = xx_roundn_epu16(v_sum_w, AOM_BLEND_A64_ROUND_BITS); |
37 | | |
38 | 163k | return v_res_w; |
39 | 163k | } Unexecuted instantiation: blend_a64_mask_sse4.c:blend_4 blend_a64_vmask_sse4.c:blend_4 Line | Count | Source | 27 | 163k | const __m128i *v_m0_w, const __m128i *v_m1_w) { | 28 | 163k | const __m128i v_s0_b = xx_loadl_32(src0); | 29 | 163k | const __m128i v_s1_b = xx_loadl_32(src1); | 30 | 163k | const __m128i v_s0_w = _mm_cvtepu8_epi16(v_s0_b); | 31 | 163k | const __m128i v_s1_w = _mm_cvtepu8_epi16(v_s1_b); | 32 | | | 33 | 163k | const __m128i v_p0_w = _mm_mullo_epi16(v_s0_w, *v_m0_w); | 34 | 163k | const __m128i v_p1_w = _mm_mullo_epi16(v_s1_w, *v_m1_w); | 35 | 163k | const __m128i v_sum_w = _mm_add_epi16(v_p0_w, v_p1_w); | 36 | 163k | const __m128i v_res_w = xx_roundn_epu16(v_sum_w, AOM_BLEND_A64_ROUND_BITS); | 37 | | | 38 | 163k | return v_res_w; | 39 | 163k | } |
Unexecuted instantiation: blend_a64_mask_avx2.c:blend_4 |
40 | | |
41 | | static inline __m128i blend_8(const uint8_t *src0, const uint8_t *src1, |
42 | 2.92M | const __m128i *v_m0_w, const __m128i *v_m1_w) { |
43 | 2.92M | const __m128i v_s0_b = xx_loadl_64(src0); |
44 | 2.92M | const __m128i v_s1_b = xx_loadl_64(src1); |
45 | 2.92M | const __m128i v_s0_w = _mm_cvtepu8_epi16(v_s0_b); |
46 | 2.92M | const __m128i v_s1_w = _mm_cvtepu8_epi16(v_s1_b); |
47 | | |
48 | 2.92M | const __m128i v_p0_w = _mm_mullo_epi16(v_s0_w, *v_m0_w); |
49 | 2.92M | const __m128i v_p1_w = _mm_mullo_epi16(v_s1_w, *v_m1_w); |
50 | | |
51 | 2.92M | const __m128i v_sum_w = _mm_add_epi16(v_p0_w, v_p1_w); |
52 | | |
53 | 2.92M | const __m128i v_res_w = xx_roundn_epu16(v_sum_w, AOM_BLEND_A64_ROUND_BITS); |
54 | | |
55 | 2.92M | return v_res_w; |
56 | 2.92M | } Unexecuted instantiation: blend_a64_mask_sse4.c:blend_8 blend_a64_vmask_sse4.c:blend_8 Line | Count | Source | 42 | 2.92M | const __m128i *v_m0_w, const __m128i *v_m1_w) { | 43 | 2.92M | const __m128i v_s0_b = xx_loadl_64(src0); | 44 | 2.92M | const __m128i v_s1_b = xx_loadl_64(src1); | 45 | 2.92M | const __m128i v_s0_w = _mm_cvtepu8_epi16(v_s0_b); | 46 | 2.92M | const __m128i v_s1_w = _mm_cvtepu8_epi16(v_s1_b); | 47 | | | 48 | 2.92M | const __m128i v_p0_w = _mm_mullo_epi16(v_s0_w, *v_m0_w); | 49 | 2.92M | const __m128i v_p1_w = _mm_mullo_epi16(v_s1_w, *v_m1_w); | 50 | | | 51 | 2.92M | const __m128i v_sum_w = _mm_add_epi16(v_p0_w, v_p1_w); | 52 | | | 53 | 2.92M | const __m128i v_res_w = xx_roundn_epu16(v_sum_w, AOM_BLEND_A64_ROUND_BITS); | 54 | | | 55 | 2.92M | return v_res_w; | 56 | 2.92M | } |
Unexecuted instantiation: blend_a64_mask_avx2.c:blend_8 |
57 | | |
58 | | static inline __m128i blend_4_u8(const uint8_t *src0, const uint8_t *src1, |
59 | | const __m128i *v_m0_b, const __m128i *v_m1_b, |
60 | 2.10M | const __m128i *rounding) { |
61 | 2.10M | const __m128i v_s0_b = xx_loadl_32(src0); |
62 | 2.10M | const __m128i v_s1_b = xx_loadl_32(src1); |
63 | | |
64 | 2.10M | const __m128i v_p0_w = _mm_maddubs_epi16(_mm_unpacklo_epi8(v_s0_b, v_s1_b), |
65 | 2.10M | _mm_unpacklo_epi8(*v_m0_b, *v_m1_b)); |
66 | | |
67 | 2.10M | const __m128i v_res_w = _mm_mulhrs_epi16(v_p0_w, *rounding); |
68 | 2.10M | const __m128i v_res = _mm_packus_epi16(v_res_w, v_res_w); |
69 | 2.10M | return v_res; |
70 | 2.10M | } blend_a64_mask_sse4.c:blend_4_u8 Line | Count | Source | 60 | 1.84M | const __m128i *rounding) { | 61 | 1.84M | const __m128i v_s0_b = xx_loadl_32(src0); | 62 | 1.84M | const __m128i v_s1_b = xx_loadl_32(src1); | 63 | | | 64 | 1.84M | const __m128i v_p0_w = _mm_maddubs_epi16(_mm_unpacklo_epi8(v_s0_b, v_s1_b), | 65 | 1.84M | _mm_unpacklo_epi8(*v_m0_b, *v_m1_b)); | 66 | | | 67 | 1.84M | const __m128i v_res_w = _mm_mulhrs_epi16(v_p0_w, *rounding); | 68 | 1.84M | const __m128i v_res = _mm_packus_epi16(v_res_w, v_res_w); | 69 | 1.84M | return v_res; | 70 | 1.84M | } |
Unexecuted instantiation: blend_a64_vmask_sse4.c:blend_4_u8 blend_a64_mask_avx2.c:blend_4_u8 Line | Count | Source | 60 | 267k | const __m128i *rounding) { | 61 | 267k | const __m128i v_s0_b = xx_loadl_32(src0); | 62 | 267k | const __m128i v_s1_b = xx_loadl_32(src1); | 63 | | | 64 | 267k | const __m128i v_p0_w = _mm_maddubs_epi16(_mm_unpacklo_epi8(v_s0_b, v_s1_b), | 65 | 267k | _mm_unpacklo_epi8(*v_m0_b, *v_m1_b)); | 66 | | | 67 | 267k | const __m128i v_res_w = _mm_mulhrs_epi16(v_p0_w, *rounding); | 68 | 267k | const __m128i v_res = _mm_packus_epi16(v_res_w, v_res_w); | 69 | 267k | return v_res; | 70 | 267k | } |
|
71 | | |
72 | | static inline __m128i blend_8_u8(const uint8_t *src0, const uint8_t *src1, |
73 | | const __m128i *v_m0_b, const __m128i *v_m1_b, |
74 | 2.50M | const __m128i *rounding) { |
75 | 2.50M | const __m128i v_s0_b = xx_loadl_64(src0); |
76 | 2.50M | const __m128i v_s1_b = xx_loadl_64(src1); |
77 | | |
78 | 2.50M | const __m128i v_p0_w = _mm_maddubs_epi16(_mm_unpacklo_epi8(v_s0_b, v_s1_b), |
79 | 2.50M | _mm_unpacklo_epi8(*v_m0_b, *v_m1_b)); |
80 | | |
81 | 2.50M | const __m128i v_res_w = _mm_mulhrs_epi16(v_p0_w, *rounding); |
82 | 2.50M | const __m128i v_res = _mm_packus_epi16(v_res_w, v_res_w); |
83 | 2.50M | return v_res; |
84 | 2.50M | } blend_a64_mask_sse4.c:blend_8_u8 Line | Count | Source | 74 | 1.22M | const __m128i *rounding) { | 75 | 1.22M | const __m128i v_s0_b = xx_loadl_64(src0); | 76 | 1.22M | const __m128i v_s1_b = xx_loadl_64(src1); | 77 | | | 78 | 1.22M | const __m128i v_p0_w = _mm_maddubs_epi16(_mm_unpacklo_epi8(v_s0_b, v_s1_b), | 79 | 1.22M | _mm_unpacklo_epi8(*v_m0_b, *v_m1_b)); | 80 | | | 81 | 1.22M | const __m128i v_res_w = _mm_mulhrs_epi16(v_p0_w, *rounding); | 82 | 1.22M | const __m128i v_res = _mm_packus_epi16(v_res_w, v_res_w); | 83 | 1.22M | return v_res; | 84 | 1.22M | } |
Unexecuted instantiation: blend_a64_vmask_sse4.c:blend_8_u8 blend_a64_mask_avx2.c:blend_8_u8 Line | Count | Source | 74 | 1.27M | const __m128i *rounding) { | 75 | 1.27M | const __m128i v_s0_b = xx_loadl_64(src0); | 76 | 1.27M | const __m128i v_s1_b = xx_loadl_64(src1); | 77 | | | 78 | 1.27M | const __m128i v_p0_w = _mm_maddubs_epi16(_mm_unpacklo_epi8(v_s0_b, v_s1_b), | 79 | 1.27M | _mm_unpacklo_epi8(*v_m0_b, *v_m1_b)); | 80 | | | 81 | 1.27M | const __m128i v_res_w = _mm_mulhrs_epi16(v_p0_w, *rounding); | 82 | 1.27M | const __m128i v_res = _mm_packus_epi16(v_res_w, v_res_w); | 83 | 1.27M | return v_res; | 84 | 1.27M | } |
|
85 | | |
86 | | static inline __m128i blend_16_u8(const uint8_t *src0, const uint8_t *src1, |
87 | | const __m128i *v_m0_b, const __m128i *v_m1_b, |
88 | 1.87M | const __m128i *rounding) { |
89 | 1.87M | const __m128i v_s0_b = xx_loadu_128(src0); |
90 | 1.87M | const __m128i v_s1_b = xx_loadu_128(src1); |
91 | | |
92 | 1.87M | const __m128i v_p0_w = _mm_maddubs_epi16(_mm_unpacklo_epi8(v_s0_b, v_s1_b), |
93 | 1.87M | _mm_unpacklo_epi8(*v_m0_b, *v_m1_b)); |
94 | 1.87M | const __m128i v_p1_w = _mm_maddubs_epi16(_mm_unpackhi_epi8(v_s0_b, v_s1_b), |
95 | 1.87M | _mm_unpackhi_epi8(*v_m0_b, *v_m1_b)); |
96 | | |
97 | 1.87M | const __m128i v_res0_w = _mm_mulhrs_epi16(v_p0_w, *rounding); |
98 | 1.87M | const __m128i v_res1_w = _mm_mulhrs_epi16(v_p1_w, *rounding); |
99 | 1.87M | const __m128i v_res = _mm_packus_epi16(v_res0_w, v_res1_w); |
100 | 1.87M | return v_res; |
101 | 1.87M | } blend_a64_mask_sse4.c:blend_16_u8 Line | Count | Source | 88 | 611k | const __m128i *rounding) { | 89 | 611k | const __m128i v_s0_b = xx_loadu_128(src0); | 90 | 611k | const __m128i v_s1_b = xx_loadu_128(src1); | 91 | | | 92 | 611k | const __m128i v_p0_w = _mm_maddubs_epi16(_mm_unpacklo_epi8(v_s0_b, v_s1_b), | 93 | 611k | _mm_unpacklo_epi8(*v_m0_b, *v_m1_b)); | 94 | 611k | const __m128i v_p1_w = _mm_maddubs_epi16(_mm_unpackhi_epi8(v_s0_b, v_s1_b), | 95 | 611k | _mm_unpackhi_epi8(*v_m0_b, *v_m1_b)); | 96 | | | 97 | 611k | const __m128i v_res0_w = _mm_mulhrs_epi16(v_p0_w, *rounding); | 98 | 611k | const __m128i v_res1_w = _mm_mulhrs_epi16(v_p1_w, *rounding); | 99 | 611k | const __m128i v_res = _mm_packus_epi16(v_res0_w, v_res1_w); | 100 | 611k | return v_res; | 101 | 611k | } |
Unexecuted instantiation: blend_a64_vmask_sse4.c:blend_16_u8 blend_a64_mask_avx2.c:blend_16_u8 Line | Count | Source | 88 | 1.25M | const __m128i *rounding) { | 89 | 1.25M | const __m128i v_s0_b = xx_loadu_128(src0); | 90 | 1.25M | const __m128i v_s1_b = xx_loadu_128(src1); | 91 | | | 92 | 1.25M | const __m128i v_p0_w = _mm_maddubs_epi16(_mm_unpacklo_epi8(v_s0_b, v_s1_b), | 93 | 1.25M | _mm_unpacklo_epi8(*v_m0_b, *v_m1_b)); | 94 | 1.25M | const __m128i v_p1_w = _mm_maddubs_epi16(_mm_unpackhi_epi8(v_s0_b, v_s1_b), | 95 | 1.25M | _mm_unpackhi_epi8(*v_m0_b, *v_m1_b)); | 96 | | | 97 | 1.25M | const __m128i v_res0_w = _mm_mulhrs_epi16(v_p0_w, *rounding); | 98 | 1.25M | const __m128i v_res1_w = _mm_mulhrs_epi16(v_p1_w, *rounding); | 99 | 1.25M | const __m128i v_res = _mm_packus_epi16(v_res0_w, v_res1_w); | 100 | 1.25M | return v_res; | 101 | 1.25M | } |
|
102 | | |
103 | | typedef __m128i (*blend_unit_fn)(const uint16_t *src0, const uint16_t *src1, |
104 | | const __m128i v_m0_w, const __m128i v_m1_w); |
105 | | |
106 | | static inline __m128i blend_4_b10(const uint16_t *src0, const uint16_t *src1, |
107 | 2.53M | const __m128i v_m0_w, const __m128i v_m1_w) { |
108 | 2.53M | const __m128i v_s0_w = xx_loadl_64(src0); |
109 | 2.53M | const __m128i v_s1_w = xx_loadl_64(src1); |
110 | | |
111 | 2.53M | const __m128i v_p0_w = _mm_mullo_epi16(v_s0_w, v_m0_w); |
112 | 2.53M | const __m128i v_p1_w = _mm_mullo_epi16(v_s1_w, v_m1_w); |
113 | | |
114 | 2.53M | const __m128i v_sum_w = _mm_add_epi16(v_p0_w, v_p1_w); |
115 | | |
116 | 2.53M | const __m128i v_res_w = xx_roundn_epu16(v_sum_w, AOM_BLEND_A64_ROUND_BITS); |
117 | | |
118 | 2.53M | return v_res_w; |
119 | 2.53M | } blend_a64_mask_sse4.c:blend_4_b10 Line | Count | Source | 107 | 2.34M | const __m128i v_m0_w, const __m128i v_m1_w) { | 108 | 2.34M | const __m128i v_s0_w = xx_loadl_64(src0); | 109 | 2.34M | const __m128i v_s1_w = xx_loadl_64(src1); | 110 | | | 111 | 2.34M | const __m128i v_p0_w = _mm_mullo_epi16(v_s0_w, v_m0_w); | 112 | 2.34M | const __m128i v_p1_w = _mm_mullo_epi16(v_s1_w, v_m1_w); | 113 | | | 114 | 2.34M | const __m128i v_sum_w = _mm_add_epi16(v_p0_w, v_p1_w); | 115 | | | 116 | 2.34M | const __m128i v_res_w = xx_roundn_epu16(v_sum_w, AOM_BLEND_A64_ROUND_BITS); | 117 | | | 118 | 2.34M | return v_res_w; | 119 | 2.34M | } |
blend_a64_vmask_sse4.c:blend_4_b10 Line | Count | Source | 107 | 190k | const __m128i v_m0_w, const __m128i v_m1_w) { | 108 | 190k | const __m128i v_s0_w = xx_loadl_64(src0); | 109 | 190k | const __m128i v_s1_w = xx_loadl_64(src1); | 110 | | | 111 | 190k | const __m128i v_p0_w = _mm_mullo_epi16(v_s0_w, v_m0_w); | 112 | 190k | const __m128i v_p1_w = _mm_mullo_epi16(v_s1_w, v_m1_w); | 113 | | | 114 | 190k | const __m128i v_sum_w = _mm_add_epi16(v_p0_w, v_p1_w); | 115 | | | 116 | 190k | const __m128i v_res_w = xx_roundn_epu16(v_sum_w, AOM_BLEND_A64_ROUND_BITS); | 117 | | | 118 | 190k | return v_res_w; | 119 | 190k | } |
Unexecuted instantiation: blend_a64_mask_avx2.c:blend_4_b10 |
120 | | |
121 | | static inline __m128i blend_8_b10(const uint16_t *src0, const uint16_t *src1, |
122 | 9.37M | const __m128i v_m0_w, const __m128i v_m1_w) { |
123 | 9.37M | const __m128i v_s0_w = xx_loadu_128(src0); |
124 | 9.37M | const __m128i v_s1_w = xx_loadu_128(src1); |
125 | | |
126 | 9.37M | const __m128i v_p0_w = _mm_mullo_epi16(v_s0_w, v_m0_w); |
127 | 9.37M | const __m128i v_p1_w = _mm_mullo_epi16(v_s1_w, v_m1_w); |
128 | | |
129 | 9.37M | const __m128i v_sum_w = _mm_add_epi16(v_p0_w, v_p1_w); |
130 | | |
131 | 9.37M | const __m128i v_res_w = xx_roundn_epu16(v_sum_w, AOM_BLEND_A64_ROUND_BITS); |
132 | | |
133 | 9.37M | return v_res_w; |
134 | 9.37M | } blend_a64_mask_sse4.c:blend_8_b10 Line | Count | Source | 122 | 6.55M | const __m128i v_m0_w, const __m128i v_m1_w) { | 123 | 6.55M | const __m128i v_s0_w = xx_loadu_128(src0); | 124 | 6.55M | const __m128i v_s1_w = xx_loadu_128(src1); | 125 | | | 126 | 6.55M | const __m128i v_p0_w = _mm_mullo_epi16(v_s0_w, v_m0_w); | 127 | 6.55M | const __m128i v_p1_w = _mm_mullo_epi16(v_s1_w, v_m1_w); | 128 | | | 129 | 6.55M | const __m128i v_sum_w = _mm_add_epi16(v_p0_w, v_p1_w); | 130 | | | 131 | 6.55M | const __m128i v_res_w = xx_roundn_epu16(v_sum_w, AOM_BLEND_A64_ROUND_BITS); | 132 | | | 133 | 6.55M | return v_res_w; | 134 | 6.55M | } |
blend_a64_vmask_sse4.c:blend_8_b10 Line | Count | Source | 122 | 2.82M | const __m128i v_m0_w, const __m128i v_m1_w) { | 123 | 2.82M | const __m128i v_s0_w = xx_loadu_128(src0); | 124 | 2.82M | const __m128i v_s1_w = xx_loadu_128(src1); | 125 | | | 126 | 2.82M | const __m128i v_p0_w = _mm_mullo_epi16(v_s0_w, v_m0_w); | 127 | 2.82M | const __m128i v_p1_w = _mm_mullo_epi16(v_s1_w, v_m1_w); | 128 | | | 129 | 2.82M | const __m128i v_sum_w = _mm_add_epi16(v_p0_w, v_p1_w); | 130 | | | 131 | 2.82M | const __m128i v_res_w = xx_roundn_epu16(v_sum_w, AOM_BLEND_A64_ROUND_BITS); | 132 | | | 133 | 2.82M | return v_res_w; | 134 | 2.82M | } |
Unexecuted instantiation: blend_a64_mask_avx2.c:blend_8_b10 |
135 | | |
136 | | static inline __m128i blend_4_b12(const uint16_t *src0, const uint16_t *src1, |
137 | 163k | const __m128i v_m0_w, const __m128i v_m1_w) { |
138 | 163k | const __m128i v_s0_w = xx_loadl_64(src0); |
139 | 163k | const __m128i v_s1_w = xx_loadl_64(src1); |
140 | | |
141 | | // Interleave |
142 | 163k | const __m128i v_m01_w = _mm_unpacklo_epi16(v_m0_w, v_m1_w); |
143 | 163k | const __m128i v_s01_w = _mm_unpacklo_epi16(v_s0_w, v_s1_w); |
144 | | |
145 | | // Multiply-Add |
146 | 163k | const __m128i v_sum_d = _mm_madd_epi16(v_s01_w, v_m01_w); |
147 | | |
148 | | // Scale |
149 | 163k | const __m128i v_ssum_d = |
150 | 163k | _mm_srli_epi32(v_sum_d, AOM_BLEND_A64_ROUND_BITS - 1); |
151 | | |
152 | | // Pack |
153 | 163k | const __m128i v_pssum_d = _mm_packs_epi32(v_ssum_d, v_ssum_d); |
154 | | |
155 | | // Round |
156 | 163k | const __m128i v_res_w = xx_round_epu16(v_pssum_d); |
157 | | |
158 | 163k | return v_res_w; |
159 | 163k | } blend_a64_mask_sse4.c:blend_4_b12 Line | Count | Source | 137 | 155k | const __m128i v_m0_w, const __m128i v_m1_w) { | 138 | 155k | const __m128i v_s0_w = xx_loadl_64(src0); | 139 | 155k | const __m128i v_s1_w = xx_loadl_64(src1); | 140 | | | 141 | | // Interleave | 142 | 155k | const __m128i v_m01_w = _mm_unpacklo_epi16(v_m0_w, v_m1_w); | 143 | 155k | const __m128i v_s01_w = _mm_unpacklo_epi16(v_s0_w, v_s1_w); | 144 | | | 145 | | // Multiply-Add | 146 | 155k | const __m128i v_sum_d = _mm_madd_epi16(v_s01_w, v_m01_w); | 147 | | | 148 | | // Scale | 149 | 155k | const __m128i v_ssum_d = | 150 | 155k | _mm_srli_epi32(v_sum_d, AOM_BLEND_A64_ROUND_BITS - 1); | 151 | | | 152 | | // Pack | 153 | 155k | const __m128i v_pssum_d = _mm_packs_epi32(v_ssum_d, v_ssum_d); | 154 | | | 155 | | // Round | 156 | 155k | const __m128i v_res_w = xx_round_epu16(v_pssum_d); | 157 | | | 158 | 155k | return v_res_w; | 159 | 155k | } |
blend_a64_vmask_sse4.c:blend_4_b12 Line | Count | Source | 137 | 8.86k | const __m128i v_m0_w, const __m128i v_m1_w) { | 138 | 8.86k | const __m128i v_s0_w = xx_loadl_64(src0); | 139 | 8.86k | const __m128i v_s1_w = xx_loadl_64(src1); | 140 | | | 141 | | // Interleave | 142 | 8.86k | const __m128i v_m01_w = _mm_unpacklo_epi16(v_m0_w, v_m1_w); | 143 | 8.86k | const __m128i v_s01_w = _mm_unpacklo_epi16(v_s0_w, v_s1_w); | 144 | | | 145 | | // Multiply-Add | 146 | 8.86k | const __m128i v_sum_d = _mm_madd_epi16(v_s01_w, v_m01_w); | 147 | | | 148 | | // Scale | 149 | 8.86k | const __m128i v_ssum_d = | 150 | 8.86k | _mm_srli_epi32(v_sum_d, AOM_BLEND_A64_ROUND_BITS - 1); | 151 | | | 152 | | // Pack | 153 | 8.86k | const __m128i v_pssum_d = _mm_packs_epi32(v_ssum_d, v_ssum_d); | 154 | | | 155 | | // Round | 156 | 8.86k | const __m128i v_res_w = xx_round_epu16(v_pssum_d); | 157 | | | 158 | 8.86k | return v_res_w; | 159 | 8.86k | } |
Unexecuted instantiation: blend_a64_mask_avx2.c:blend_4_b12 |
160 | | |
161 | | static inline __m128i blend_8_b12(const uint16_t *src0, const uint16_t *src1, |
162 | 934k | const __m128i v_m0_w, const __m128i v_m1_w) { |
163 | 934k | const __m128i v_s0_w = xx_loadu_128(src0); |
164 | 934k | const __m128i v_s1_w = xx_loadu_128(src1); |
165 | | |
166 | | // Interleave |
167 | 934k | const __m128i v_m01l_w = _mm_unpacklo_epi16(v_m0_w, v_m1_w); |
168 | 934k | const __m128i v_m01h_w = _mm_unpackhi_epi16(v_m0_w, v_m1_w); |
169 | 934k | const __m128i v_s01l_w = _mm_unpacklo_epi16(v_s0_w, v_s1_w); |
170 | 934k | const __m128i v_s01h_w = _mm_unpackhi_epi16(v_s0_w, v_s1_w); |
171 | | |
172 | | // Multiply-Add |
173 | 934k | const __m128i v_suml_d = _mm_madd_epi16(v_s01l_w, v_m01l_w); |
174 | 934k | const __m128i v_sumh_d = _mm_madd_epi16(v_s01h_w, v_m01h_w); |
175 | | |
176 | | // Scale |
177 | 934k | const __m128i v_ssuml_d = |
178 | 934k | _mm_srli_epi32(v_suml_d, AOM_BLEND_A64_ROUND_BITS - 1); |
179 | 934k | const __m128i v_ssumh_d = |
180 | 934k | _mm_srli_epi32(v_sumh_d, AOM_BLEND_A64_ROUND_BITS - 1); |
181 | | |
182 | | // Pack |
183 | 934k | const __m128i v_pssum_d = _mm_packs_epi32(v_ssuml_d, v_ssumh_d); |
184 | | |
185 | | // Round |
186 | 934k | const __m128i v_res_w = xx_round_epu16(v_pssum_d); |
187 | | |
188 | 934k | return v_res_w; |
189 | 934k | } blend_a64_mask_sse4.c:blend_8_b12 Line | Count | Source | 162 | 648k | const __m128i v_m0_w, const __m128i v_m1_w) { | 163 | 648k | const __m128i v_s0_w = xx_loadu_128(src0); | 164 | 648k | const __m128i v_s1_w = xx_loadu_128(src1); | 165 | | | 166 | | // Interleave | 167 | 648k | const __m128i v_m01l_w = _mm_unpacklo_epi16(v_m0_w, v_m1_w); | 168 | 648k | const __m128i v_m01h_w = _mm_unpackhi_epi16(v_m0_w, v_m1_w); | 169 | 648k | const __m128i v_s01l_w = _mm_unpacklo_epi16(v_s0_w, v_s1_w); | 170 | 648k | const __m128i v_s01h_w = _mm_unpackhi_epi16(v_s0_w, v_s1_w); | 171 | | | 172 | | // Multiply-Add | 173 | 648k | const __m128i v_suml_d = _mm_madd_epi16(v_s01l_w, v_m01l_w); | 174 | 648k | const __m128i v_sumh_d = _mm_madd_epi16(v_s01h_w, v_m01h_w); | 175 | | | 176 | | // Scale | 177 | 648k | const __m128i v_ssuml_d = | 178 | 648k | _mm_srli_epi32(v_suml_d, AOM_BLEND_A64_ROUND_BITS - 1); | 179 | 648k | const __m128i v_ssumh_d = | 180 | 648k | _mm_srli_epi32(v_sumh_d, AOM_BLEND_A64_ROUND_BITS - 1); | 181 | | | 182 | | // Pack | 183 | 648k | const __m128i v_pssum_d = _mm_packs_epi32(v_ssuml_d, v_ssumh_d); | 184 | | | 185 | | // Round | 186 | 648k | const __m128i v_res_w = xx_round_epu16(v_pssum_d); | 187 | | | 188 | 648k | return v_res_w; | 189 | 648k | } |
blend_a64_vmask_sse4.c:blend_8_b12 Line | Count | Source | 162 | 286k | const __m128i v_m0_w, const __m128i v_m1_w) { | 163 | 286k | const __m128i v_s0_w = xx_loadu_128(src0); | 164 | 286k | const __m128i v_s1_w = xx_loadu_128(src1); | 165 | | | 166 | | // Interleave | 167 | 286k | const __m128i v_m01l_w = _mm_unpacklo_epi16(v_m0_w, v_m1_w); | 168 | 286k | const __m128i v_m01h_w = _mm_unpackhi_epi16(v_m0_w, v_m1_w); | 169 | 286k | const __m128i v_s01l_w = _mm_unpacklo_epi16(v_s0_w, v_s1_w); | 170 | 286k | const __m128i v_s01h_w = _mm_unpackhi_epi16(v_s0_w, v_s1_w); | 171 | | | 172 | | // Multiply-Add | 173 | 286k | const __m128i v_suml_d = _mm_madd_epi16(v_s01l_w, v_m01l_w); | 174 | 286k | const __m128i v_sumh_d = _mm_madd_epi16(v_s01h_w, v_m01h_w); | 175 | | | 176 | | // Scale | 177 | 286k | const __m128i v_ssuml_d = | 178 | 286k | _mm_srli_epi32(v_suml_d, AOM_BLEND_A64_ROUND_BITS - 1); | 179 | 286k | const __m128i v_ssumh_d = | 180 | 286k | _mm_srli_epi32(v_sumh_d, AOM_BLEND_A64_ROUND_BITS - 1); | 181 | | | 182 | | // Pack | 183 | 286k | const __m128i v_pssum_d = _mm_packs_epi32(v_ssuml_d, v_ssumh_d); | 184 | | | 185 | | // Round | 186 | 286k | const __m128i v_res_w = xx_round_epu16(v_pssum_d); | 187 | | | 188 | 286k | return v_res_w; | 189 | 286k | } |
Unexecuted instantiation: blend_a64_mask_avx2.c:blend_8_b12 |
190 | | |
191 | | #endif // AOM_AOM_DSP_X86_BLEND_SSE4_H_ |