Coverage Report

Created: 2025-06-13 07:07

/src/aom/aom_dsp/x86/txfm_common_avx2.h
Line
Count
Source (jump to first uncovered line)
1
/*
2
 * Copyright (c) 2018, Alliance for Open Media. All rights reserved.
3
 *
4
 * This source code is subject to the terms of the BSD 2 Clause License and
5
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6
 * was not distributed with this source code in the LICENSE file, you can
7
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8
 * Media Patent License 1.0 was not distributed with this source code in the
9
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10
 */
11
12
#ifndef AOM_AOM_DSP_X86_TXFM_COMMON_AVX2_H_
13
#define AOM_AOM_DSP_X86_TXFM_COMMON_AVX2_H_
14
15
#include <emmintrin.h>
16
#include "aom/aom_integer.h"
17
#include "aom_dsp/x86/synonyms.h"
18
19
#ifdef __cplusplus
20
extern "C" {
21
#endif
22
23
58.7M
static inline __m256i pair_set_w16_epi16(int16_t a, int16_t b) {
24
58.7M
  return _mm256_set1_epi32(
25
58.7M
      (int32_t)(((uint16_t)(a)) | (((uint32_t)(uint16_t)(b)) << 16)));
26
58.7M
}
av1_inv_txfm_avx2.c:pair_set_w16_epi16
Line
Count
Source
23
58.7M
static inline __m256i pair_set_w16_epi16(int16_t a, int16_t b) {
24
58.7M
  return _mm256_set1_epi32(
25
58.7M
      (int32_t)(((uint16_t)(a)) | (((uint32_t)(uint16_t)(b)) << 16)));
26
58.7M
}
Unexecuted instantiation: highbd_inv_txfm_avx2.c:pair_set_w16_epi16
27
28
static inline void btf_16_w16_avx2(const __m256i w0, const __m256i w1,
29
                                   __m256i *in0, __m256i *in1, const __m256i _r,
30
54.2M
                                   const int32_t cos_bit) {
31
54.2M
  __m256i t0 = _mm256_unpacklo_epi16(*in0, *in1);
32
54.2M
  __m256i t1 = _mm256_unpackhi_epi16(*in0, *in1);
33
54.2M
  __m256i u0 = _mm256_madd_epi16(t0, w0);
34
54.2M
  __m256i u1 = _mm256_madd_epi16(t1, w0);
35
54.2M
  __m256i v0 = _mm256_madd_epi16(t0, w1);
36
54.2M
  __m256i v1 = _mm256_madd_epi16(t1, w1);
37
38
54.2M
  __m256i a0 = _mm256_add_epi32(u0, _r);
39
54.2M
  __m256i a1 = _mm256_add_epi32(u1, _r);
40
54.2M
  __m256i b0 = _mm256_add_epi32(v0, _r);
41
54.2M
  __m256i b1 = _mm256_add_epi32(v1, _r);
42
43
54.2M
  __m256i c0 = _mm256_srai_epi32(a0, cos_bit);
44
54.2M
  __m256i c1 = _mm256_srai_epi32(a1, cos_bit);
45
54.2M
  __m256i d0 = _mm256_srai_epi32(b0, cos_bit);
46
54.2M
  __m256i d1 = _mm256_srai_epi32(b1, cos_bit);
47
48
54.2M
  *in0 = _mm256_packs_epi32(c0, c1);
49
54.2M
  *in1 = _mm256_packs_epi32(d0, d1);
50
54.2M
}
av1_inv_txfm_avx2.c:btf_16_w16_avx2
Line
Count
Source
30
54.2M
                                   const int32_t cos_bit) {
31
54.2M
  __m256i t0 = _mm256_unpacklo_epi16(*in0, *in1);
32
54.2M
  __m256i t1 = _mm256_unpackhi_epi16(*in0, *in1);
33
54.2M
  __m256i u0 = _mm256_madd_epi16(t0, w0);
34
54.2M
  __m256i u1 = _mm256_madd_epi16(t1, w0);
35
54.2M
  __m256i v0 = _mm256_madd_epi16(t0, w1);
36
54.2M
  __m256i v1 = _mm256_madd_epi16(t1, w1);
37
38
54.2M
  __m256i a0 = _mm256_add_epi32(u0, _r);
39
54.2M
  __m256i a1 = _mm256_add_epi32(u1, _r);
40
54.2M
  __m256i b0 = _mm256_add_epi32(v0, _r);
41
54.2M
  __m256i b1 = _mm256_add_epi32(v1, _r);
42
43
54.2M
  __m256i c0 = _mm256_srai_epi32(a0, cos_bit);
44
54.2M
  __m256i c1 = _mm256_srai_epi32(a1, cos_bit);
45
54.2M
  __m256i d0 = _mm256_srai_epi32(b0, cos_bit);
46
54.2M
  __m256i d1 = _mm256_srai_epi32(b1, cos_bit);
47
48
54.2M
  *in0 = _mm256_packs_epi32(c0, c1);
49
54.2M
  *in1 = _mm256_packs_epi32(d0, d1);
50
54.2M
}
Unexecuted instantiation: highbd_inv_txfm_avx2.c:btf_16_w16_avx2
51
52
104M
static inline void btf_16_adds_subs_avx2(__m256i *in0, __m256i *in1) {
53
104M
  const __m256i _in0 = *in0;
54
104M
  const __m256i _in1 = *in1;
55
104M
  *in0 = _mm256_adds_epi16(_in0, _in1);
56
104M
  *in1 = _mm256_subs_epi16(_in0, _in1);
57
104M
}
av1_inv_txfm_avx2.c:btf_16_adds_subs_avx2
Line
Count
Source
52
104M
static inline void btf_16_adds_subs_avx2(__m256i *in0, __m256i *in1) {
53
104M
  const __m256i _in0 = *in0;
54
104M
  const __m256i _in1 = *in1;
55
104M
  *in0 = _mm256_adds_epi16(_in0, _in1);
56
104M
  *in1 = _mm256_subs_epi16(_in0, _in1);
57
104M
}
Unexecuted instantiation: highbd_inv_txfm_avx2.c:btf_16_adds_subs_avx2
58
59
0
static inline void btf_32_add_sub_avx2(__m256i *in0, __m256i *in1) {
60
0
  const __m256i _in0 = *in0;
61
0
  const __m256i _in1 = *in1;
62
0
  *in0 = _mm256_add_epi32(_in0, _in1);
63
0
  *in1 = _mm256_sub_epi32(_in0, _in1);
64
0
}
Unexecuted instantiation: av1_inv_txfm_avx2.c:btf_32_add_sub_avx2
Unexecuted instantiation: highbd_inv_txfm_avx2.c:btf_32_add_sub_avx2
65
66
static inline void btf_16_adds_subs_out_avx2(__m256i *out0, __m256i *out1,
67
39.6M
                                             __m256i in0, __m256i in1) {
68
39.6M
  const __m256i _in0 = in0;
69
39.6M
  const __m256i _in1 = in1;
70
39.6M
  *out0 = _mm256_adds_epi16(_in0, _in1);
71
39.6M
  *out1 = _mm256_subs_epi16(_in0, _in1);
72
39.6M
}
av1_inv_txfm_avx2.c:btf_16_adds_subs_out_avx2
Line
Count
Source
67
39.6M
                                             __m256i in0, __m256i in1) {
68
39.6M
  const __m256i _in0 = in0;
69
39.6M
  const __m256i _in1 = in1;
70
39.6M
  *out0 = _mm256_adds_epi16(_in0, _in1);
71
39.6M
  *out1 = _mm256_subs_epi16(_in0, _in1);
72
39.6M
}
Unexecuted instantiation: highbd_inv_txfm_avx2.c:btf_16_adds_subs_out_avx2
73
74
static inline void btf_32_add_sub_out_avx2(__m256i *out0, __m256i *out1,
75
0
                                           __m256i in0, __m256i in1) {
76
0
  const __m256i _in0 = in0;
77
0
  const __m256i _in1 = in1;
78
0
  *out0 = _mm256_add_epi32(_in0, _in1);
79
0
  *out1 = _mm256_sub_epi32(_in0, _in1);
80
0
}
Unexecuted instantiation: av1_inv_txfm_avx2.c:btf_32_add_sub_out_avx2
Unexecuted instantiation: highbd_inv_txfm_avx2.c:btf_32_add_sub_out_avx2
81
82
0
static inline __m256i load_16bit_to_16bit_avx2(const int16_t *a) {
83
0
  return _mm256_load_si256((const __m256i *)a);
84
0
}
Unexecuted instantiation: av1_inv_txfm_avx2.c:load_16bit_to_16bit_avx2
Unexecuted instantiation: highbd_inv_txfm_avx2.c:load_16bit_to_16bit_avx2
85
86
static inline void load_buffer_16bit_to_16bit_avx2(const int16_t *in,
87
                                                   int stride, __m256i *out,
88
0
                                                   int out_size) {
89
0
  for (int i = 0; i < out_size; ++i) {
90
0
    out[i] = load_16bit_to_16bit_avx2(in + i * stride);
91
0
  }
92
0
}
Unexecuted instantiation: av1_inv_txfm_avx2.c:load_buffer_16bit_to_16bit_avx2
Unexecuted instantiation: highbd_inv_txfm_avx2.c:load_buffer_16bit_to_16bit_avx2
93
94
static inline void load_buffer_16bit_to_16bit_flip_avx2(const int16_t *in,
95
                                                        int stride,
96
                                                        __m256i *out,
97
0
                                                        int out_size) {
98
0
  for (int i = 0; i < out_size; ++i) {
99
0
    out[out_size - i - 1] = load_16bit_to_16bit_avx2(in + i * stride);
100
0
  }
101
0
}
Unexecuted instantiation: av1_inv_txfm_avx2.c:load_buffer_16bit_to_16bit_flip_avx2
Unexecuted instantiation: highbd_inv_txfm_avx2.c:load_buffer_16bit_to_16bit_flip_avx2
102
103
27.8M
static inline __m256i load_32bit_to_16bit_w16_avx2(const int32_t *a) {
104
27.8M
  const __m256i a_low = _mm256_lddqu_si256((const __m256i *)a);
105
27.8M
  const __m256i b = _mm256_packs_epi32(a_low, *(const __m256i *)(a + 8));
106
27.8M
  return _mm256_permute4x64_epi64(b, 0xD8);
107
27.8M
}
av1_inv_txfm_avx2.c:load_32bit_to_16bit_w16_avx2
Line
Count
Source
103
27.8M
static inline __m256i load_32bit_to_16bit_w16_avx2(const int32_t *a) {
104
27.8M
  const __m256i a_low = _mm256_lddqu_si256((const __m256i *)a);
105
27.8M
  const __m256i b = _mm256_packs_epi32(a_low, *(const __m256i *)(a + 8));
106
27.8M
  return _mm256_permute4x64_epi64(b, 0xD8);
107
27.8M
}
Unexecuted instantiation: highbd_inv_txfm_avx2.c:load_32bit_to_16bit_w16_avx2
108
109
static inline void load_buffer_32bit_to_16bit_w16_avx2(const int32_t *in,
110
                                                       int stride, __m256i *out,
111
1.58M
                                                       int out_size) {
112
28.5M
  for (int i = 0; i < out_size; ++i) {
113
26.9M
    out[i] = load_32bit_to_16bit_w16_avx2(in + i * stride);
114
26.9M
  }
115
1.58M
}
av1_inv_txfm_avx2.c:load_buffer_32bit_to_16bit_w16_avx2
Line
Count
Source
111
1.58M
                                                       int out_size) {
112
28.5M
  for (int i = 0; i < out_size; ++i) {
113
26.9M
    out[i] = load_32bit_to_16bit_w16_avx2(in + i * stride);
114
26.9M
  }
115
1.58M
}
Unexecuted instantiation: highbd_inv_txfm_avx2.c:load_buffer_32bit_to_16bit_w16_avx2
116
117
static inline void transpose2_8x8_avx2(const __m256i *const in,
118
5.91M
                                       __m256i *const out) {
119
5.91M
  __m256i t[16], u[16];
120
  // (1st, 2nd) ==> (lo, hi)
121
  //   (0, 1)   ==>  (0, 1)
122
  //   (2, 3)   ==>  (2, 3)
123
  //   (4, 5)   ==>  (4, 5)
124
  //   (6, 7)   ==>  (6, 7)
125
29.5M
  for (int i = 0; i < 4; i++) {
126
23.6M
    t[2 * i] = _mm256_unpacklo_epi16(in[2 * i], in[2 * i + 1]);
127
23.6M
    t[2 * i + 1] = _mm256_unpackhi_epi16(in[2 * i], in[2 * i + 1]);
128
23.6M
  }
129
130
  // (1st, 2nd) ==> (lo, hi)
131
  //   (0, 2)   ==>  (0, 2)
132
  //   (1, 3)   ==>  (1, 3)
133
  //   (4, 6)   ==>  (4, 6)
134
  //   (5, 7)   ==>  (5, 7)
135
17.7M
  for (int i = 0; i < 2; i++) {
136
11.8M
    u[i] = _mm256_unpacklo_epi32(t[i], t[i + 2]);
137
11.8M
    u[i + 2] = _mm256_unpackhi_epi32(t[i], t[i + 2]);
138
139
11.8M
    u[i + 4] = _mm256_unpacklo_epi32(t[i + 4], t[i + 6]);
140
11.8M
    u[i + 6] = _mm256_unpackhi_epi32(t[i + 4], t[i + 6]);
141
11.8M
  }
142
143
  // (1st, 2nd) ==> (lo, hi)
144
  //   (0, 4)   ==>  (0, 1)
145
  //   (1, 5)   ==>  (4, 5)
146
  //   (2, 6)   ==>  (2, 3)
147
  //   (3, 7)   ==>  (6, 7)
148
17.7M
  for (int i = 0; i < 2; i++) {
149
11.8M
    out[2 * i] = _mm256_unpacklo_epi64(u[2 * i], u[2 * i + 4]);
150
11.8M
    out[2 * i + 1] = _mm256_unpackhi_epi64(u[2 * i], u[2 * i + 4]);
151
152
11.8M
    out[2 * i + 4] = _mm256_unpacklo_epi64(u[2 * i + 1], u[2 * i + 5]);
153
11.8M
    out[2 * i + 5] = _mm256_unpackhi_epi64(u[2 * i + 1], u[2 * i + 5]);
154
11.8M
  }
155
5.91M
}
av1_inv_txfm_avx2.c:transpose2_8x8_avx2
Line
Count
Source
118
5.91M
                                       __m256i *const out) {
119
5.91M
  __m256i t[16], u[16];
120
  // (1st, 2nd) ==> (lo, hi)
121
  //   (0, 1)   ==>  (0, 1)
122
  //   (2, 3)   ==>  (2, 3)
123
  //   (4, 5)   ==>  (4, 5)
124
  //   (6, 7)   ==>  (6, 7)
125
29.5M
  for (int i = 0; i < 4; i++) {
126
23.6M
    t[2 * i] = _mm256_unpacklo_epi16(in[2 * i], in[2 * i + 1]);
127
23.6M
    t[2 * i + 1] = _mm256_unpackhi_epi16(in[2 * i], in[2 * i + 1]);
128
23.6M
  }
129
130
  // (1st, 2nd) ==> (lo, hi)
131
  //   (0, 2)   ==>  (0, 2)
132
  //   (1, 3)   ==>  (1, 3)
133
  //   (4, 6)   ==>  (4, 6)
134
  //   (5, 7)   ==>  (5, 7)
135
17.7M
  for (int i = 0; i < 2; i++) {
136
11.8M
    u[i] = _mm256_unpacklo_epi32(t[i], t[i + 2]);
137
11.8M
    u[i + 2] = _mm256_unpackhi_epi32(t[i], t[i + 2]);
138
139
11.8M
    u[i + 4] = _mm256_unpacklo_epi32(t[i + 4], t[i + 6]);
140
11.8M
    u[i + 6] = _mm256_unpackhi_epi32(t[i + 4], t[i + 6]);
141
11.8M
  }
142
143
  // (1st, 2nd) ==> (lo, hi)
144
  //   (0, 4)   ==>  (0, 1)
145
  //   (1, 5)   ==>  (4, 5)
146
  //   (2, 6)   ==>  (2, 3)
147
  //   (3, 7)   ==>  (6, 7)
148
17.7M
  for (int i = 0; i < 2; i++) {
149
11.8M
    out[2 * i] = _mm256_unpacklo_epi64(u[2 * i], u[2 * i + 4]);
150
11.8M
    out[2 * i + 1] = _mm256_unpackhi_epi64(u[2 * i], u[2 * i + 4]);
151
152
11.8M
    out[2 * i + 4] = _mm256_unpacklo_epi64(u[2 * i + 1], u[2 * i + 5]);
153
11.8M
    out[2 * i + 5] = _mm256_unpackhi_epi64(u[2 * i + 1], u[2 * i + 5]);
154
11.8M
  }
155
5.91M
}
Unexecuted instantiation: highbd_inv_txfm_avx2.c:transpose2_8x8_avx2
156
157
static inline void transpose_16bit_16x16_avx2(const __m256i *const in,
158
2.95M
                                              __m256i *const out) {
159
2.95M
  __m256i t[16];
160
161
2.95M
#define LOADL(idx)                                                            \
162
23.6M
  t[idx] = _mm256_castsi128_si256(_mm_load_si128((__m128i const *)&in[idx])); \
163
23.6M
  t[idx] = _mm256_inserti128_si256(                                           \
164
23.6M
      t[idx], _mm_load_si128((__m128i const *)&in[idx + 8]), 1);
165
166
2.95M
#define LOADR(idx)                                                           \
167
23.6M
  t[8 + idx] =                                                               \
168
23.6M
      _mm256_castsi128_si256(_mm_load_si128((__m128i const *)&in[idx] + 1)); \
169
23.6M
  t[8 + idx] = _mm256_inserti128_si256(                                      \
170
23.6M
      t[8 + idx], _mm_load_si128((__m128i const *)&in[idx + 8] + 1), 1);
171
172
  // load left 8x16
173
2.95M
  LOADL(0)
174
2.95M
  LOADL(1)
175
2.95M
  LOADL(2)
176
2.95M
  LOADL(3)
177
2.95M
  LOADL(4)
178
2.95M
  LOADL(5)
179
2.95M
  LOADL(6)
180
2.95M
  LOADL(7)
181
182
  // load right 8x16
183
2.95M
  LOADR(0)
184
2.95M
  LOADR(1)
185
2.95M
  LOADR(2)
186
2.95M
  LOADR(3)
187
2.95M
  LOADR(4)
188
2.95M
  LOADR(5)
189
2.95M
  LOADR(6)
190
2.95M
  LOADR(7)
191
192
  // get the top 16x8 result
193
2.95M
  transpose2_8x8_avx2(t, out);
194
  // get the bottom 16x8 result
195
2.95M
  transpose2_8x8_avx2(&t[8], &out[8]);
196
2.95M
}
av1_inv_txfm_avx2.c:transpose_16bit_16x16_avx2
Line
Count
Source
158
2.95M
                                              __m256i *const out) {
159
2.95M
  __m256i t[16];
160
161
2.95M
#define LOADL(idx)                                                            \
162
2.95M
  t[idx] = _mm256_castsi128_si256(_mm_load_si128((__m128i const *)&in[idx])); \
163
2.95M
  t[idx] = _mm256_inserti128_si256(                                           \
164
2.95M
      t[idx], _mm_load_si128((__m128i const *)&in[idx + 8]), 1);
165
166
2.95M
#define LOADR(idx)                                                           \
167
2.95M
  t[8 + idx] =                                                               \
168
2.95M
      _mm256_castsi128_si256(_mm_load_si128((__m128i const *)&in[idx] + 1)); \
169
2.95M
  t[8 + idx] = _mm256_inserti128_si256(                                      \
170
2.95M
      t[8 + idx], _mm_load_si128((__m128i const *)&in[idx + 8] + 1), 1);
171
172
  // load left 8x16
173
2.95M
  LOADL(0)
174
2.95M
  LOADL(1)
175
2.95M
  LOADL(2)
176
2.95M
  LOADL(3)
177
2.95M
  LOADL(4)
178
2.95M
  LOADL(5)
179
2.95M
  LOADL(6)
180
2.95M
  LOADL(7)
181
182
  // load right 8x16
183
2.95M
  LOADR(0)
184
2.95M
  LOADR(1)
185
2.95M
  LOADR(2)
186
2.95M
  LOADR(3)
187
2.95M
  LOADR(4)
188
2.95M
  LOADR(5)
189
2.95M
  LOADR(6)
190
2.95M
  LOADR(7)
191
192
  // get the top 16x8 result
193
2.95M
  transpose2_8x8_avx2(t, out);
194
  // get the bottom 16x8 result
195
2.95M
  transpose2_8x8_avx2(&t[8], &out[8]);
196
2.95M
}
Unexecuted instantiation: highbd_inv_txfm_avx2.c:transpose_16bit_16x16_avx2
197
198
static inline void transpose_16bit_16x8_avx2(const __m256i *const in,
199
0
                                             __m256i *const out) {
200
0
  const __m256i a0 = _mm256_unpacklo_epi16(in[0], in[1]);
201
0
  const __m256i a1 = _mm256_unpacklo_epi16(in[2], in[3]);
202
0
  const __m256i a2 = _mm256_unpacklo_epi16(in[4], in[5]);
203
0
  const __m256i a3 = _mm256_unpacklo_epi16(in[6], in[7]);
204
0
  const __m256i a4 = _mm256_unpackhi_epi16(in[0], in[1]);
205
0
  const __m256i a5 = _mm256_unpackhi_epi16(in[2], in[3]);
206
0
  const __m256i a6 = _mm256_unpackhi_epi16(in[4], in[5]);
207
0
  const __m256i a7 = _mm256_unpackhi_epi16(in[6], in[7]);
208
0
209
0
  const __m256i b0 = _mm256_unpacklo_epi32(a0, a1);
210
0
  const __m256i b1 = _mm256_unpacklo_epi32(a2, a3);
211
0
  const __m256i b2 = _mm256_unpacklo_epi32(a4, a5);
212
0
  const __m256i b3 = _mm256_unpacklo_epi32(a6, a7);
213
0
  const __m256i b4 = _mm256_unpackhi_epi32(a0, a1);
214
0
  const __m256i b5 = _mm256_unpackhi_epi32(a2, a3);
215
0
  const __m256i b6 = _mm256_unpackhi_epi32(a4, a5);
216
0
  const __m256i b7 = _mm256_unpackhi_epi32(a6, a7);
217
0
218
0
  out[0] = _mm256_unpacklo_epi64(b0, b1);
219
0
  out[1] = _mm256_unpackhi_epi64(b0, b1);
220
0
  out[2] = _mm256_unpacklo_epi64(b4, b5);
221
0
  out[3] = _mm256_unpackhi_epi64(b4, b5);
222
0
  out[4] = _mm256_unpacklo_epi64(b2, b3);
223
0
  out[5] = _mm256_unpackhi_epi64(b2, b3);
224
0
  out[6] = _mm256_unpacklo_epi64(b6, b7);
225
0
  out[7] = _mm256_unpackhi_epi64(b6, b7);
226
0
}
Unexecuted instantiation: av1_inv_txfm_avx2.c:transpose_16bit_16x8_avx2
Unexecuted instantiation: highbd_inv_txfm_avx2.c:transpose_16bit_16x8_avx2
227
228
16.9k
static inline void flip_buf_avx2(__m256i *in, __m256i *out, int size) {
229
288k
  for (int i = 0; i < size; ++i) {
230
271k
    out[size - i - 1] = in[i];
231
271k
  }
232
16.9k
}
av1_inv_txfm_avx2.c:flip_buf_avx2
Line
Count
Source
228
16.9k
static inline void flip_buf_avx2(__m256i *in, __m256i *out, int size) {
229
288k
  for (int i = 0; i < size; ++i) {
230
271k
    out[size - i - 1] = in[i];
231
271k
  }
232
16.9k
}
Unexecuted instantiation: highbd_inv_txfm_avx2.c:flip_buf_avx2
233
234
18.3k
static inline void round_shift_16bit_w16_avx2(__m256i *in, int size, int bit) {
235
18.3k
  if (bit < 0) {
236
18.3k
    bit = -bit;
237
18.3k
    __m256i round = _mm256_set1_epi16(1 << (bit - 1));
238
312k
    for (int i = 0; i < size; ++i) {
239
294k
      in[i] = _mm256_adds_epi16(in[i], round);
240
294k
      in[i] = _mm256_srai_epi16(in[i], bit);
241
294k
    }
242
18.3k
  } else if (bit > 0) {
243
0
    for (int i = 0; i < size; ++i) {
244
0
      in[i] = _mm256_slli_epi16(in[i], bit);
245
0
    }
246
0
  }
247
18.3k
}
av1_inv_txfm_avx2.c:round_shift_16bit_w16_avx2
Line
Count
Source
234
18.3k
static inline void round_shift_16bit_w16_avx2(__m256i *in, int size, int bit) {
235
18.3k
  if (bit < 0) {
236
18.3k
    bit = -bit;
237
18.3k
    __m256i round = _mm256_set1_epi16(1 << (bit - 1));
238
312k
    for (int i = 0; i < size; ++i) {
239
294k
      in[i] = _mm256_adds_epi16(in[i], round);
240
294k
      in[i] = _mm256_srai_epi16(in[i], bit);
241
294k
    }
242
18.3k
  } else if (bit > 0) {
243
0
    for (int i = 0; i < size; ++i) {
244
0
      in[i] = _mm256_slli_epi16(in[i], bit);
245
0
    }
246
0
  }
247
18.3k
}
Unexecuted instantiation: highbd_inv_txfm_avx2.c:round_shift_16bit_w16_avx2
248
249
235M
static inline __m256i round_shift_32_avx2(__m256i vec, int bit) {
250
235M
  __m256i tmp, round;
251
235M
  round = _mm256_set1_epi32(1 << (bit - 1));
252
235M
  tmp = _mm256_add_epi32(vec, round);
253
235M
  return _mm256_srai_epi32(tmp, bit);
254
235M
}
Unexecuted instantiation: av1_inv_txfm_avx2.c:round_shift_32_avx2
highbd_inv_txfm_avx2.c:round_shift_32_avx2
Line
Count
Source
249
235M
static inline __m256i round_shift_32_avx2(__m256i vec, int bit) {
250
235M
  __m256i tmp, round;
251
235M
  round = _mm256_set1_epi32(1 << (bit - 1));
252
235M
  tmp = _mm256_add_epi32(vec, round);
253
235M
  return _mm256_srai_epi32(tmp, bit);
254
235M
}
255
256
static inline void round_shift_array_32_avx2(__m256i *input, __m256i *output,
257
11.4M
                                             const int size, const int bit) {
258
11.4M
  if (bit > 0) {
259
11.4M
    int i;
260
234M
    for (i = 0; i < size; i++) {
261
223M
      output[i] = round_shift_32_avx2(input[i], bit);
262
223M
    }
263
18.4E
  } else {
264
18.4E
    int i;
265
18.4E
    for (i = 0; i < size; i++) {
266
0
      output[i] = _mm256_slli_epi32(input[i], -bit);
267
0
    }
268
18.4E
  }
269
11.4M
}
Unexecuted instantiation: av1_inv_txfm_avx2.c:round_shift_array_32_avx2
highbd_inv_txfm_avx2.c:round_shift_array_32_avx2
Line
Count
Source
257
11.4M
                                             const int size, const int bit) {
258
11.4M
  if (bit > 0) {
259
11.4M
    int i;
260
234M
    for (i = 0; i < size; i++) {
261
223M
      output[i] = round_shift_32_avx2(input[i], bit);
262
223M
    }
263
18.4E
  } else {
264
18.4E
    int i;
265
18.4E
    for (i = 0; i < size; i++) {
266
0
      output[i] = _mm256_slli_epi32(input[i], -bit);
267
0
    }
268
18.4E
  }
269
11.4M
}
270
271
static inline void round_shift_rect_array_32_avx2(__m256i *input,
272
                                                  __m256i *output,
273
                                                  const int size, const int bit,
274
1.22M
                                                  const int val) {
275
1.22M
  const __m256i sqrt2 = _mm256_set1_epi32(val);
276
1.22M
  if (bit > 0) {
277
0
    int i;
278
0
    for (i = 0; i < size; i++) {
279
0
      const __m256i r0 = round_shift_32_avx2(input[i], bit);
280
0
      const __m256i r1 = _mm256_mullo_epi32(sqrt2, r0);
281
0
      output[i] = round_shift_32_avx2(r1, NewSqrt2Bits);
282
0
    }
283
1.22M
  } else {
284
1.22M
    int i;
285
13.7M
    for (i = 0; i < size; i++) {
286
12.5M
      const __m256i r0 = _mm256_slli_epi32(input[i], -bit);
287
12.5M
      const __m256i r1 = _mm256_mullo_epi32(sqrt2, r0);
288
12.5M
      output[i] = round_shift_32_avx2(r1, NewSqrt2Bits);
289
12.5M
    }
290
1.22M
  }
291
1.22M
}
Unexecuted instantiation: av1_inv_txfm_avx2.c:round_shift_rect_array_32_avx2
highbd_inv_txfm_avx2.c:round_shift_rect_array_32_avx2
Line
Count
Source
274
1.22M
                                                  const int val) {
275
1.22M
  const __m256i sqrt2 = _mm256_set1_epi32(val);
276
1.22M
  if (bit > 0) {
277
0
    int i;
278
0
    for (i = 0; i < size; i++) {
279
0
      const __m256i r0 = round_shift_32_avx2(input[i], bit);
280
0
      const __m256i r1 = _mm256_mullo_epi32(sqrt2, r0);
281
0
      output[i] = round_shift_32_avx2(r1, NewSqrt2Bits);
282
0
    }
283
1.22M
  } else {
284
1.22M
    int i;
285
13.7M
    for (i = 0; i < size; i++) {
286
12.5M
      const __m256i r0 = _mm256_slli_epi32(input[i], -bit);
287
12.5M
      const __m256i r1 = _mm256_mullo_epi32(sqrt2, r0);
288
12.5M
      output[i] = round_shift_32_avx2(r1, NewSqrt2Bits);
289
12.5M
    }
290
1.22M
  }
291
1.22M
}
292
293
0
static inline __m256i scale_round_avx2(const __m256i a, const int scale) {
294
0
  const __m256i scale_rounding =
295
0
      pair_set_w16_epi16(scale, 1 << (NewSqrt2Bits - 1));
296
0
  const __m256i b = _mm256_madd_epi16(a, scale_rounding);
297
0
  return _mm256_srai_epi32(b, NewSqrt2Bits);
298
0
}
Unexecuted instantiation: av1_inv_txfm_avx2.c:scale_round_avx2
Unexecuted instantiation: highbd_inv_txfm_avx2.c:scale_round_avx2
299
300
static inline void store_rect_16bit_to_32bit_w8_avx2(const __m256i a,
301
0
                                                     int32_t *const b) {
302
0
  const __m256i one = _mm256_set1_epi16(1);
303
0
  const __m256i a_lo = _mm256_unpacklo_epi16(a, one);
304
0
  const __m256i a_hi = _mm256_unpackhi_epi16(a, one);
305
0
  const __m256i b_lo = scale_round_avx2(a_lo, NewSqrt2);
306
0
  const __m256i b_hi = scale_round_avx2(a_hi, NewSqrt2);
307
0
  const __m256i temp = _mm256_permute2f128_si256(b_lo, b_hi, 0x31);
308
0
  _mm_store_si128((__m128i *)b, _mm256_castsi256_si128(b_lo));
309
0
  _mm_store_si128((__m128i *)(b + 4), _mm256_castsi256_si128(b_hi));
310
0
  _mm256_store_si256((__m256i *)(b + 64), temp);
311
0
}
Unexecuted instantiation: av1_inv_txfm_avx2.c:store_rect_16bit_to_32bit_w8_avx2
Unexecuted instantiation: highbd_inv_txfm_avx2.c:store_rect_16bit_to_32bit_w8_avx2
312
313
static inline void store_rect_buffer_16bit_to_32bit_w8_avx2(
314
    const __m256i *const in, int32_t *const out, const int stride,
315
0
    const int out_size) {
316
0
  for (int i = 0; i < out_size; ++i) {
317
0
    store_rect_16bit_to_32bit_w8_avx2(in[i], out + i * stride);
318
0
  }
319
0
}
Unexecuted instantiation: av1_inv_txfm_avx2.c:store_rect_buffer_16bit_to_32bit_w8_avx2
Unexecuted instantiation: highbd_inv_txfm_avx2.c:store_rect_buffer_16bit_to_32bit_w8_avx2
320
321
static inline void pack_reg(const __m128i *in1, const __m128i *in2,
322
0
                            __m256i *out) {
323
0
  out[0] = _mm256_insertf128_si256(_mm256_castsi128_si256(in1[0]), in2[0], 0x1);
324
0
  out[1] = _mm256_insertf128_si256(_mm256_castsi128_si256(in1[1]), in2[1], 0x1);
325
0
  out[2] = _mm256_insertf128_si256(_mm256_castsi128_si256(in1[2]), in2[2], 0x1);
326
0
  out[3] = _mm256_insertf128_si256(_mm256_castsi128_si256(in1[3]), in2[3], 0x1);
327
0
  out[4] = _mm256_insertf128_si256(_mm256_castsi128_si256(in1[4]), in2[4], 0x1);
328
0
  out[5] = _mm256_insertf128_si256(_mm256_castsi128_si256(in1[5]), in2[5], 0x1);
329
0
  out[6] = _mm256_insertf128_si256(_mm256_castsi128_si256(in1[6]), in2[6], 0x1);
330
0
  out[7] = _mm256_insertf128_si256(_mm256_castsi128_si256(in1[7]), in2[7], 0x1);
331
0
}
Unexecuted instantiation: av1_inv_txfm_avx2.c:pack_reg
Unexecuted instantiation: highbd_inv_txfm_avx2.c:pack_reg
332
333
0
static inline void extract_reg(const __m256i *in, __m128i *out1) {
334
0
  out1[0] = _mm256_castsi256_si128(in[0]);
335
0
  out1[1] = _mm256_castsi256_si128(in[1]);
336
0
  out1[2] = _mm256_castsi256_si128(in[2]);
337
0
  out1[3] = _mm256_castsi256_si128(in[3]);
338
0
  out1[4] = _mm256_castsi256_si128(in[4]);
339
0
  out1[5] = _mm256_castsi256_si128(in[5]);
340
0
  out1[6] = _mm256_castsi256_si128(in[6]);
341
0
  out1[7] = _mm256_castsi256_si128(in[7]);
342
0
343
0
  out1[8] = _mm256_extracti128_si256(in[0], 0x01);
344
0
  out1[9] = _mm256_extracti128_si256(in[1], 0x01);
345
0
  out1[10] = _mm256_extracti128_si256(in[2], 0x01);
346
0
  out1[11] = _mm256_extracti128_si256(in[3], 0x01);
347
0
  out1[12] = _mm256_extracti128_si256(in[4], 0x01);
348
0
  out1[13] = _mm256_extracti128_si256(in[5], 0x01);
349
0
  out1[14] = _mm256_extracti128_si256(in[6], 0x01);
350
0
  out1[15] = _mm256_extracti128_si256(in[7], 0x01);
351
0
}
Unexecuted instantiation: av1_inv_txfm_avx2.c:extract_reg
Unexecuted instantiation: highbd_inv_txfm_avx2.c:extract_reg
352
353
#ifdef __cplusplus
354
}
355
#endif
356
357
#endif  // AOM_AOM_DSP_X86_TXFM_COMMON_AVX2_H_