Coverage Report

Created: 2026-09-14 06:49

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/avm/av2/common/cdef_block_simd.h
Line
Count
Source
1
/*
2
 * Copyright (c) 2021, Alliance for Open Media. All rights reserved
3
 *
4
 * This source code is subject to the terms of the BSD 3-Clause Clear License
5
 * and the Alliance for Open Media Patent License 1.0. If the BSD 3-Clause Clear
6
 * License was not distributed with this source code in the LICENSE file, you
7
 * can obtain it at aomedia.org/license/software-license/bsd-3-c-c/.  If the
8
 * Alliance for Open Media Patent License 1.0 was not distributed with this
9
 * source code in the PATENTS file, you can obtain it at
10
 * aomedia.org/license/patent-license/.
11
 */
12
13
#ifndef AVM_AV2_COMMON_CDEF_BLOCK_SIMD_H_
14
#define AVM_AV2_COMMON_CDEF_BLOCK_SIMD_H_
15
16
#include "config/av2_rtcd.h"
17
18
#include "av2/common/cdef_block.h"
19
20
/* partial A is a 16-bit vector of the form:
21
   [x8 x7 x6 x5 x4 x3 x2 x1] and partial B has the form:
22
   [0  y1 y2 y3 y4 y5 y6 y7].
23
   This function computes (x1^2+y1^2)*C1 + (x2^2+y2^2)*C2 + ...
24
   (x7^2+y2^7)*C7 + (x8^2+0^2)*C8 where the C1..C8 constants are in const1
25
   and const2. */
26
static INLINE v128 fold_mul_and_sum(v128 partiala, v128 partialb, v128 const1,
27
18
                                    v128 const2) {
28
18
  v128 tmp;
29
  /* Reverse partial B. */
30
18
  partialb = v128_shuffle_8(
31
18
      partialb, v128_from_32(0x0f0e0100, 0x03020504, 0x07060908, 0x0b0a0d0c));
32
  /* Interleave the x and y values of identical indices and pair x8 with 0. */
33
18
  tmp = partiala;
34
18
  partiala = v128_ziplo_16(partialb, partiala);
35
18
  partialb = v128_ziphi_16(partialb, tmp);
36
  /* Square and add the corresponding x and y values. */
37
18
  partiala = v128_madd_s16(partiala, partiala);
38
18
  partialb = v128_madd_s16(partialb, partialb);
39
  /* Multiply by constant. */
40
18
  partiala = v128_mullo_s32(partiala, const1);
41
18
  partialb = v128_mullo_s32(partialb, const2);
42
  /* Sum all results. */
43
18
  partiala = v128_add_32(partiala, partialb);
44
18
  return partiala;
45
18
}
Unexecuted instantiation: cdef_block_sse2.c:fold_mul_and_sum
Unexecuted instantiation: cdef_block_ssse3.c:fold_mul_and_sum
Unexecuted instantiation: cdef_block_sse4.c:fold_mul_and_sum
cdef_block_avx2.c:fold_mul_and_sum
Line
Count
Source
27
18
                                    v128 const2) {
28
18
  v128 tmp;
29
  /* Reverse partial B. */
30
18
  partialb = v128_shuffle_8(
31
18
      partialb, v128_from_32(0x0f0e0100, 0x03020504, 0x07060908, 0x0b0a0d0c));
32
  /* Interleave the x and y values of identical indices and pair x8 with 0. */
33
18
  tmp = partiala;
34
18
  partiala = v128_ziplo_16(partialb, partiala);
35
18
  partialb = v128_ziphi_16(partialb, tmp);
36
  /* Square and add the corresponding x and y values. */
37
18
  partiala = v128_madd_s16(partiala, partiala);
38
18
  partialb = v128_madd_s16(partialb, partialb);
39
  /* Multiply by constant. */
40
18
  partiala = v128_mullo_s32(partiala, const1);
41
18
  partialb = v128_mullo_s32(partialb, const2);
42
  /* Sum all results. */
43
18
  partiala = v128_add_32(partiala, partialb);
44
18
  return partiala;
45
18
}
46
47
6
static INLINE v128 hsum4(v128 x0, v128 x1, v128 x2, v128 x3) {
48
6
  v128 t0, t1, t2, t3;
49
6
  t0 = v128_ziplo_32(x1, x0);
50
6
  t1 = v128_ziplo_32(x3, x2);
51
6
  t2 = v128_ziphi_32(x1, x0);
52
6
  t3 = v128_ziphi_32(x3, x2);
53
6
  x0 = v128_ziplo_64(t1, t0);
54
6
  x1 = v128_ziphi_64(t1, t0);
55
6
  x2 = v128_ziplo_64(t3, t2);
56
6
  x3 = v128_ziphi_64(t3, t2);
57
6
  return v128_add_32(v128_add_32(x0, x1), v128_add_32(x2, x3));
58
6
}
Unexecuted instantiation: cdef_block_sse2.c:hsum4
Unexecuted instantiation: cdef_block_ssse3.c:hsum4
Unexecuted instantiation: cdef_block_sse4.c:hsum4
cdef_block_avx2.c:hsum4
Line
Count
Source
47
6
static INLINE v128 hsum4(v128 x0, v128 x1, v128 x2, v128 x3) {
48
6
  v128 t0, t1, t2, t3;
49
6
  t0 = v128_ziplo_32(x1, x0);
50
6
  t1 = v128_ziplo_32(x3, x2);
51
6
  t2 = v128_ziphi_32(x1, x0);
52
6
  t3 = v128_ziphi_32(x3, x2);
53
6
  x0 = v128_ziplo_64(t1, t0);
54
6
  x1 = v128_ziphi_64(t1, t0);
55
6
  x2 = v128_ziplo_64(t3, t2);
56
6
  x3 = v128_ziphi_64(t3, t2);
57
6
  return v128_add_32(v128_add_32(x0, x1), v128_add_32(x2, x3));
58
6
}
59
60
/* Computes cost for directions 0, 5, 6 and 7. We can call this function again
61
   to compute the remaining directions. */
62
6
static INLINE v128 compute_directions(v128 lines[8], int32_t tmp_cost1[4]) {
63
6
  v128 partial4a, partial4b, partial5a, partial5b, partial7a, partial7b;
64
6
  v128 partial6;
65
6
  v128 tmp;
66
  /* Partial sums for lines 0 and 1. */
67
6
  partial4a = v128_shl_n_byte(lines[0], 14);
68
6
  partial4b = v128_shr_n_byte(lines[0], 2);
69
6
  partial4a = v128_add_16(partial4a, v128_shl_n_byte(lines[1], 12));
70
6
  partial4b = v128_add_16(partial4b, v128_shr_n_byte(lines[1], 4));
71
6
  tmp = v128_add_16(lines[0], lines[1]);
72
6
  partial5a = v128_shl_n_byte(tmp, 10);
73
6
  partial5b = v128_shr_n_byte(tmp, 6);
74
6
  partial7a = v128_shl_n_byte(tmp, 4);
75
6
  partial7b = v128_shr_n_byte(tmp, 12);
76
6
  partial6 = tmp;
77
78
  /* Partial sums for lines 2 and 3. */
79
6
  partial4a = v128_add_16(partial4a, v128_shl_n_byte(lines[2], 10));
80
6
  partial4b = v128_add_16(partial4b, v128_shr_n_byte(lines[2], 6));
81
6
  partial4a = v128_add_16(partial4a, v128_shl_n_byte(lines[3], 8));
82
6
  partial4b = v128_add_16(partial4b, v128_shr_n_byte(lines[3], 8));
83
6
  tmp = v128_add_16(lines[2], lines[3]);
84
6
  partial5a = v128_add_16(partial5a, v128_shl_n_byte(tmp, 8));
85
6
  partial5b = v128_add_16(partial5b, v128_shr_n_byte(tmp, 8));
86
6
  partial7a = v128_add_16(partial7a, v128_shl_n_byte(tmp, 6));
87
6
  partial7b = v128_add_16(partial7b, v128_shr_n_byte(tmp, 10));
88
6
  partial6 = v128_add_16(partial6, tmp);
89
90
  /* Partial sums for lines 4 and 5. */
91
6
  partial4a = v128_add_16(partial4a, v128_shl_n_byte(lines[4], 6));
92
6
  partial4b = v128_add_16(partial4b, v128_shr_n_byte(lines[4], 10));
93
6
  partial4a = v128_add_16(partial4a, v128_shl_n_byte(lines[5], 4));
94
6
  partial4b = v128_add_16(partial4b, v128_shr_n_byte(lines[5], 12));
95
6
  tmp = v128_add_16(lines[4], lines[5]);
96
6
  partial5a = v128_add_16(partial5a, v128_shl_n_byte(tmp, 6));
97
6
  partial5b = v128_add_16(partial5b, v128_shr_n_byte(tmp, 10));
98
6
  partial7a = v128_add_16(partial7a, v128_shl_n_byte(tmp, 8));
99
6
  partial7b = v128_add_16(partial7b, v128_shr_n_byte(tmp, 8));
100
6
  partial6 = v128_add_16(partial6, tmp);
101
102
  /* Partial sums for lines 6 and 7. */
103
6
  partial4a = v128_add_16(partial4a, v128_shl_n_byte(lines[6], 2));
104
6
  partial4b = v128_add_16(partial4b, v128_shr_n_byte(lines[6], 14));
105
6
  partial4a = v128_add_16(partial4a, lines[7]);
106
6
  tmp = v128_add_16(lines[6], lines[7]);
107
6
  partial5a = v128_add_16(partial5a, v128_shl_n_byte(tmp, 4));
108
6
  partial5b = v128_add_16(partial5b, v128_shr_n_byte(tmp, 12));
109
6
  partial7a = v128_add_16(partial7a, v128_shl_n_byte(tmp, 10));
110
6
  partial7b = v128_add_16(partial7b, v128_shr_n_byte(tmp, 6));
111
6
  partial6 = v128_add_16(partial6, tmp);
112
113
  /* Compute costs in terms of partial sums. */
114
6
  partial4a =
115
6
      fold_mul_and_sum(partial4a, partial4b, v128_from_32(210, 280, 420, 840),
116
6
                       v128_from_32(105, 120, 140, 168));
117
6
  partial7a =
118
6
      fold_mul_and_sum(partial7a, partial7b, v128_from_32(210, 420, 0, 0),
119
6
                       v128_from_32(105, 105, 105, 140));
120
6
  partial5a =
121
6
      fold_mul_and_sum(partial5a, partial5b, v128_from_32(210, 420, 0, 0),
122
6
                       v128_from_32(105, 105, 105, 140));
123
6
  partial6 = v128_madd_s16(partial6, partial6);
124
6
  partial6 = v128_mullo_s32(partial6, v128_dup_32(105));
125
126
6
  partial4a = hsum4(partial4a, partial5a, partial6, partial7a);
127
6
  v128_store_unaligned(tmp_cost1, partial4a);
128
6
  return partial4a;
129
6
}
Unexecuted instantiation: cdef_block_sse2.c:compute_directions
Unexecuted instantiation: cdef_block_ssse3.c:compute_directions
Unexecuted instantiation: cdef_block_sse4.c:compute_directions
cdef_block_avx2.c:compute_directions
Line
Count
Source
62
6
static INLINE v128 compute_directions(v128 lines[8], int32_t tmp_cost1[4]) {
63
6
  v128 partial4a, partial4b, partial5a, partial5b, partial7a, partial7b;
64
6
  v128 partial6;
65
6
  v128 tmp;
66
  /* Partial sums for lines 0 and 1. */
67
6
  partial4a = v128_shl_n_byte(lines[0], 14);
68
6
  partial4b = v128_shr_n_byte(lines[0], 2);
69
6
  partial4a = v128_add_16(partial4a, v128_shl_n_byte(lines[1], 12));
70
6
  partial4b = v128_add_16(partial4b, v128_shr_n_byte(lines[1], 4));
71
6
  tmp = v128_add_16(lines[0], lines[1]);
72
6
  partial5a = v128_shl_n_byte(tmp, 10);
73
6
  partial5b = v128_shr_n_byte(tmp, 6);
74
6
  partial7a = v128_shl_n_byte(tmp, 4);
75
6
  partial7b = v128_shr_n_byte(tmp, 12);
76
6
  partial6 = tmp;
77
78
  /* Partial sums for lines 2 and 3. */
79
6
  partial4a = v128_add_16(partial4a, v128_shl_n_byte(lines[2], 10));
80
6
  partial4b = v128_add_16(partial4b, v128_shr_n_byte(lines[2], 6));
81
6
  partial4a = v128_add_16(partial4a, v128_shl_n_byte(lines[3], 8));
82
6
  partial4b = v128_add_16(partial4b, v128_shr_n_byte(lines[3], 8));
83
6
  tmp = v128_add_16(lines[2], lines[3]);
84
6
  partial5a = v128_add_16(partial5a, v128_shl_n_byte(tmp, 8));
85
6
  partial5b = v128_add_16(partial5b, v128_shr_n_byte(tmp, 8));
86
6
  partial7a = v128_add_16(partial7a, v128_shl_n_byte(tmp, 6));
87
6
  partial7b = v128_add_16(partial7b, v128_shr_n_byte(tmp, 10));
88
6
  partial6 = v128_add_16(partial6, tmp);
89
90
  /* Partial sums for lines 4 and 5. */
91
6
  partial4a = v128_add_16(partial4a, v128_shl_n_byte(lines[4], 6));
92
6
  partial4b = v128_add_16(partial4b, v128_shr_n_byte(lines[4], 10));
93
6
  partial4a = v128_add_16(partial4a, v128_shl_n_byte(lines[5], 4));
94
6
  partial4b = v128_add_16(partial4b, v128_shr_n_byte(lines[5], 12));
95
6
  tmp = v128_add_16(lines[4], lines[5]);
96
6
  partial5a = v128_add_16(partial5a, v128_shl_n_byte(tmp, 6));
97
6
  partial5b = v128_add_16(partial5b, v128_shr_n_byte(tmp, 10));
98
6
  partial7a = v128_add_16(partial7a, v128_shl_n_byte(tmp, 8));
99
6
  partial7b = v128_add_16(partial7b, v128_shr_n_byte(tmp, 8));
100
6
  partial6 = v128_add_16(partial6, tmp);
101
102
  /* Partial sums for lines 6 and 7. */
103
6
  partial4a = v128_add_16(partial4a, v128_shl_n_byte(lines[6], 2));
104
6
  partial4b = v128_add_16(partial4b, v128_shr_n_byte(lines[6], 14));
105
6
  partial4a = v128_add_16(partial4a, lines[7]);
106
6
  tmp = v128_add_16(lines[6], lines[7]);
107
6
  partial5a = v128_add_16(partial5a, v128_shl_n_byte(tmp, 4));
108
6
  partial5b = v128_add_16(partial5b, v128_shr_n_byte(tmp, 12));
109
6
  partial7a = v128_add_16(partial7a, v128_shl_n_byte(tmp, 10));
110
6
  partial7b = v128_add_16(partial7b, v128_shr_n_byte(tmp, 6));
111
6
  partial6 = v128_add_16(partial6, tmp);
112
113
  /* Compute costs in terms of partial sums. */
114
6
  partial4a =
115
6
      fold_mul_and_sum(partial4a, partial4b, v128_from_32(210, 280, 420, 840),
116
6
                       v128_from_32(105, 120, 140, 168));
117
6
  partial7a =
118
6
      fold_mul_and_sum(partial7a, partial7b, v128_from_32(210, 420, 0, 0),
119
6
                       v128_from_32(105, 105, 105, 140));
120
6
  partial5a =
121
6
      fold_mul_and_sum(partial5a, partial5b, v128_from_32(210, 420, 0, 0),
122
6
                       v128_from_32(105, 105, 105, 140));
123
6
  partial6 = v128_madd_s16(partial6, partial6);
124
6
  partial6 = v128_mullo_s32(partial6, v128_dup_32(105));
125
126
6
  partial4a = hsum4(partial4a, partial5a, partial6, partial7a);
127
6
  v128_store_unaligned(tmp_cost1, partial4a);
128
6
  return partial4a;
129
6
}
130
131
/* transpose and reverse the order of the lines -- equivalent to a 90-degree
132
   counter-clockwise rotation of the pixels. */
133
3
static INLINE void array_reverse_transpose_8x8(v128 *in, v128 *res) {
134
3
  const v128 tr0_0 = v128_ziplo_16(in[1], in[0]);
135
3
  const v128 tr0_1 = v128_ziplo_16(in[3], in[2]);
136
3
  const v128 tr0_2 = v128_ziphi_16(in[1], in[0]);
137
3
  const v128 tr0_3 = v128_ziphi_16(in[3], in[2]);
138
3
  const v128 tr0_4 = v128_ziplo_16(in[5], in[4]);
139
3
  const v128 tr0_5 = v128_ziplo_16(in[7], in[6]);
140
3
  const v128 tr0_6 = v128_ziphi_16(in[5], in[4]);
141
3
  const v128 tr0_7 = v128_ziphi_16(in[7], in[6]);
142
143
3
  const v128 tr1_0 = v128_ziplo_32(tr0_1, tr0_0);
144
3
  const v128 tr1_1 = v128_ziplo_32(tr0_5, tr0_4);
145
3
  const v128 tr1_2 = v128_ziphi_32(tr0_1, tr0_0);
146
3
  const v128 tr1_3 = v128_ziphi_32(tr0_5, tr0_4);
147
3
  const v128 tr1_4 = v128_ziplo_32(tr0_3, tr0_2);
148
3
  const v128 tr1_5 = v128_ziplo_32(tr0_7, tr0_6);
149
3
  const v128 tr1_6 = v128_ziphi_32(tr0_3, tr0_2);
150
3
  const v128 tr1_7 = v128_ziphi_32(tr0_7, tr0_6);
151
152
3
  res[7] = v128_ziplo_64(tr1_1, tr1_0);
153
3
  res[6] = v128_ziphi_64(tr1_1, tr1_0);
154
3
  res[5] = v128_ziplo_64(tr1_3, tr1_2);
155
3
  res[4] = v128_ziphi_64(tr1_3, tr1_2);
156
3
  res[3] = v128_ziplo_64(tr1_5, tr1_4);
157
3
  res[2] = v128_ziphi_64(tr1_5, tr1_4);
158
3
  res[1] = v128_ziplo_64(tr1_7, tr1_6);
159
3
  res[0] = v128_ziphi_64(tr1_7, tr1_6);
160
3
}
Unexecuted instantiation: cdef_block_sse2.c:array_reverse_transpose_8x8
Unexecuted instantiation: cdef_block_ssse3.c:array_reverse_transpose_8x8
Unexecuted instantiation: cdef_block_sse4.c:array_reverse_transpose_8x8
cdef_block_avx2.c:array_reverse_transpose_8x8
Line
Count
Source
133
3
static INLINE void array_reverse_transpose_8x8(v128 *in, v128 *res) {
134
3
  const v128 tr0_0 = v128_ziplo_16(in[1], in[0]);
135
3
  const v128 tr0_1 = v128_ziplo_16(in[3], in[2]);
136
3
  const v128 tr0_2 = v128_ziphi_16(in[1], in[0]);
137
3
  const v128 tr0_3 = v128_ziphi_16(in[3], in[2]);
138
3
  const v128 tr0_4 = v128_ziplo_16(in[5], in[4]);
139
3
  const v128 tr0_5 = v128_ziplo_16(in[7], in[6]);
140
3
  const v128 tr0_6 = v128_ziphi_16(in[5], in[4]);
141
3
  const v128 tr0_7 = v128_ziphi_16(in[7], in[6]);
142
143
3
  const v128 tr1_0 = v128_ziplo_32(tr0_1, tr0_0);
144
3
  const v128 tr1_1 = v128_ziplo_32(tr0_5, tr0_4);
145
3
  const v128 tr1_2 = v128_ziphi_32(tr0_1, tr0_0);
146
3
  const v128 tr1_3 = v128_ziphi_32(tr0_5, tr0_4);
147
3
  const v128 tr1_4 = v128_ziplo_32(tr0_3, tr0_2);
148
3
  const v128 tr1_5 = v128_ziplo_32(tr0_7, tr0_6);
149
3
  const v128 tr1_6 = v128_ziphi_32(tr0_3, tr0_2);
150
3
  const v128 tr1_7 = v128_ziphi_32(tr0_7, tr0_6);
151
152
3
  res[7] = v128_ziplo_64(tr1_1, tr1_0);
153
3
  res[6] = v128_ziphi_64(tr1_1, tr1_0);
154
3
  res[5] = v128_ziplo_64(tr1_3, tr1_2);
155
3
  res[4] = v128_ziphi_64(tr1_3, tr1_2);
156
3
  res[3] = v128_ziplo_64(tr1_5, tr1_4);
157
3
  res[2] = v128_ziphi_64(tr1_5, tr1_4);
158
3
  res[1] = v128_ziplo_64(tr1_7, tr1_6);
159
3
  res[0] = v128_ziphi_64(tr1_7, tr1_6);
160
3
}
161
162
int SIMD_FUNC(av2_cdef_find_dir)(const uint16_t *img, int stride, int32_t *var,
163
3
                                 int coeff_shift) {
164
3
  int i;
165
3
  int32_t cost[8];
166
3
  int32_t best_cost = 0;
167
3
  int best_dir = 0;
168
3
  v128 lines[8];
169
27
  for (i = 0; i < 8; i++) {
170
24
    lines[i] = v128_load_unaligned(&img[i * stride]);
171
24
    lines[i] =
172
24
        v128_sub_16(v128_shr_s16(lines[i], coeff_shift), v128_dup_16(128));
173
24
  }
174
175
  /* Compute "mostly vertical" directions. */
176
3
  v128 dir47 = compute_directions(lines, cost + 4);
177
178
3
  array_reverse_transpose_8x8(lines, lines);
179
180
  /* Compute "mostly horizontal" directions. */
181
3
  v128 dir03 = compute_directions(lines, cost);
182
183
3
  v128 max = v128_max_s32(dir03, dir47);
184
3
  max = v128_max_s32(max, v128_align(max, max, 8));
185
3
  max = v128_max_s32(max, v128_align(max, max, 4));
186
3
  best_cost = v128_low_u32(max);
187
3
  v128 t =
188
3
      v128_pack_s32_s16(v128_cmpeq_32(max, dir47), v128_cmpeq_32(max, dir03));
189
3
  best_dir = v128_movemask_8(v128_pack_s16_s8(t, t));
190
3
  best_dir = get_msb(best_dir ^ (best_dir - 1));  // Count trailing zeros
191
192
  /* Difference between the optimal variance and the variance along the
193
     orthogonal direction. Again, the sum(x^2) terms cancel out. */
194
3
  *var = best_cost - cost[(best_dir + 4) & 7];
195
  /* We'd normally divide by 840, but dividing by 1024 is close enough
196
     for what we're going to do with this. */
197
3
  *var >>= 10;
198
3
  return best_dir;
199
3
}
Unexecuted instantiation: av2_cdef_find_dir_sse2
Unexecuted instantiation: av2_cdef_find_dir_ssse3
Unexecuted instantiation: av2_cdef_find_dir_sse4_1
av2_cdef_find_dir_avx2
Line
Count
Source
163
3
                                 int coeff_shift) {
164
3
  int i;
165
3
  int32_t cost[8];
166
3
  int32_t best_cost = 0;
167
3
  int best_dir = 0;
168
3
  v128 lines[8];
169
27
  for (i = 0; i < 8; i++) {
170
24
    lines[i] = v128_load_unaligned(&img[i * stride]);
171
24
    lines[i] =
172
24
        v128_sub_16(v128_shr_s16(lines[i], coeff_shift), v128_dup_16(128));
173
24
  }
174
175
  /* Compute "mostly vertical" directions. */
176
3
  v128 dir47 = compute_directions(lines, cost + 4);
177
178
3
  array_reverse_transpose_8x8(lines, lines);
179
180
  /* Compute "mostly horizontal" directions. */
181
3
  v128 dir03 = compute_directions(lines, cost);
182
183
3
  v128 max = v128_max_s32(dir03, dir47);
184
3
  max = v128_max_s32(max, v128_align(max, max, 8));
185
3
  max = v128_max_s32(max, v128_align(max, max, 4));
186
3
  best_cost = v128_low_u32(max);
187
3
  v128 t =
188
3
      v128_pack_s32_s16(v128_cmpeq_32(max, dir47), v128_cmpeq_32(max, dir03));
189
3
  best_dir = v128_movemask_8(v128_pack_s16_s8(t, t));
190
3
  best_dir = get_msb(best_dir ^ (best_dir - 1));  // Count trailing zeros
191
192
  /* Difference between the optimal variance and the variance along the
193
     orthogonal direction. Again, the sum(x^2) terms cancel out. */
194
3
  *var = best_cost - cost[(best_dir + 4) & 7];
195
  /* We'd normally divide by 840, but dividing by 1024 is close enough
196
     for what we're going to do with this. */
197
3
  *var >>= 10;
198
3
  return best_dir;
199
3
}
200
201
// sign(a-b) * min(abs(a-b), max(0, threshold - (abs(a-b) >> adjdamp)))
202
SIMD_INLINE v256 constrain16(v256 a, v256 b, unsigned int threshold,
203
100k
                             unsigned int adjdamp) {
204
100k
  v256 diff = v256_sub_16(a, b);
205
100k
  const v256 sign = v256_shr_n_s16(diff, 15);
206
100k
  diff = v256_abs_s16(diff);
207
100k
  const v256 s =
208
100k
      v256_ssub_u16(v256_dup_16(threshold), v256_shr_u16(diff, adjdamp));
209
100k
  return v256_xor(v256_add_16(sign, v256_min_s16(diff, s)), sign);
210
100k
}
Unexecuted instantiation: cdef_block_sse2.c:constrain16
Unexecuted instantiation: cdef_block_ssse3.c:constrain16
Unexecuted instantiation: cdef_block_sse4.c:constrain16
cdef_block_avx2.c:constrain16
Line
Count
Source
203
100k
                             unsigned int adjdamp) {
204
100k
  v256 diff = v256_sub_16(a, b);
205
100k
  const v256 sign = v256_shr_n_s16(diff, 15);
206
100k
  diff = v256_abs_s16(diff);
207
100k
  const v256 s =
208
100k
      v256_ssub_u16(v256_dup_16(threshold), v256_shr_u16(diff, adjdamp));
209
100k
  return v256_xor(v256_add_16(sign, v256_min_s16(diff, s)), sign);
210
100k
}
211
212
/* Computes the maximum pixel value used during primary CDEF filtering. */
213
SIMD_INLINE v256 get_max_primary(const v256 *const tap, v256 max,
214
2.78k
                                 v256 cdef_large_value_mask) {
215
  /* Convert CDEF_VERY_LARGE to 0 before calculating max. */
216
2.78k
  max = v256_max_s16(max, v256_and(tap[0], cdef_large_value_mask));
217
2.78k
  max = v256_max_s16(max, v256_and(tap[1], cdef_large_value_mask));
218
2.78k
  max = v256_max_s16(max, v256_and(tap[2], cdef_large_value_mask));
219
2.78k
  max = v256_max_s16(max, v256_and(tap[3], cdef_large_value_mask));
220
2.78k
  return max;
221
2.78k
}
Unexecuted instantiation: cdef_block_sse2.c:get_max_primary
Unexecuted instantiation: cdef_block_ssse3.c:get_max_primary
Unexecuted instantiation: cdef_block_sse4.c:get_max_primary
cdef_block_avx2.c:get_max_primary
Line
Count
Source
214
2.78k
                                 v256 cdef_large_value_mask) {
215
  /* Convert CDEF_VERY_LARGE to 0 before calculating max. */
216
2.78k
  max = v256_max_s16(max, v256_and(tap[0], cdef_large_value_mask));
217
2.78k
  max = v256_max_s16(max, v256_and(tap[1], cdef_large_value_mask));
218
2.78k
  max = v256_max_s16(max, v256_and(tap[2], cdef_large_value_mask));
219
2.78k
  max = v256_max_s16(max, v256_and(tap[3], cdef_large_value_mask));
220
2.78k
  return max;
221
2.78k
}
222
223
/* Computes the maximum pixel value used during secondary CDEF filtering. */
224
SIMD_INLINE v256 get_max_secondary(const v256 *const tap, v256 max,
225
2.78k
                                   v256 cdef_large_value_mask) {
226
  /* Convert CDEF_VERY_LARGE to 0 before calculating max. */
227
2.78k
  max = v256_max_s16(max, v256_and(tap[0], cdef_large_value_mask));
228
2.78k
  max = v256_max_s16(max, v256_and(tap[1], cdef_large_value_mask));
229
2.78k
  max = v256_max_s16(max, v256_and(tap[2], cdef_large_value_mask));
230
2.78k
  max = v256_max_s16(max, v256_and(tap[3], cdef_large_value_mask));
231
2.78k
  max = v256_max_s16(max, v256_and(tap[4], cdef_large_value_mask));
232
2.78k
  max = v256_max_s16(max, v256_and(tap[5], cdef_large_value_mask));
233
2.78k
  max = v256_max_s16(max, v256_and(tap[6], cdef_large_value_mask));
234
2.78k
  max = v256_max_s16(max, v256_and(tap[7], cdef_large_value_mask));
235
2.78k
  return max;
236
2.78k
}
Unexecuted instantiation: cdef_block_sse2.c:get_max_secondary
Unexecuted instantiation: cdef_block_ssse3.c:get_max_secondary
Unexecuted instantiation: cdef_block_sse4.c:get_max_secondary
cdef_block_avx2.c:get_max_secondary
Line
Count
Source
225
2.78k
                                   v256 cdef_large_value_mask) {
226
  /* Convert CDEF_VERY_LARGE to 0 before calculating max. */
227
2.78k
  max = v256_max_s16(max, v256_and(tap[0], cdef_large_value_mask));
228
2.78k
  max = v256_max_s16(max, v256_and(tap[1], cdef_large_value_mask));
229
2.78k
  max = v256_max_s16(max, v256_and(tap[2], cdef_large_value_mask));
230
2.78k
  max = v256_max_s16(max, v256_and(tap[3], cdef_large_value_mask));
231
2.78k
  max = v256_max_s16(max, v256_and(tap[4], cdef_large_value_mask));
232
2.78k
  max = v256_max_s16(max, v256_and(tap[5], cdef_large_value_mask));
233
2.78k
  max = v256_max_s16(max, v256_and(tap[6], cdef_large_value_mask));
234
2.78k
  max = v256_max_s16(max, v256_and(tap[7], cdef_large_value_mask));
235
2.78k
  return max;
236
2.78k
}
237
238
/* Applies CDEF filtering for block width equals 4. */
239
SIMD_INLINE void filter_block_4x4(uint16_t *const dest, int dstride,
240
                                  const uint16_t *in, int pri_strength,
241
                                  int sec_strength, int dir, int pri_damping,
242
                                  int sec_damping, int coeff_shift, int height,
243
4.86k
                                  int enable_primary, int enable_secondary) {
244
4.86k
  const int clipping_required = enable_primary && enable_secondary;
245
4.86k
  v256 p0, p1, p2, p3;
246
4.86k
  v256 sum, row, res;
247
4.86k
  const v256 cdef_large_value_mask = v256_dup_16((uint16_t)~CDEF_VERY_LARGE);
248
4.86k
  v256 max, min;
249
4.86k
  const int po1 = av2_cdef_directions[dir][0];
250
4.86k
  const int po2 = av2_cdef_directions[dir][1];
251
4.86k
  const int s1o1 = av2_cdef_directions[dir + 2][0];
252
4.86k
  const int s1o2 = av2_cdef_directions[dir + 2][1];
253
4.86k
  const int s2o1 = av2_cdef_directions[dir - 2][0];
254
4.86k
  const int s2o2 = av2_cdef_directions[dir - 2][1];
255
4.86k
  const int *pri_taps = av2_cdef_pri_taps[(pri_strength >> coeff_shift) & 1];
256
4.86k
  const int *sec_taps = av2_cdef_sec_taps;
257
4.86k
  int i;
258
259
4.86k
  if (enable_primary && pri_strength)
260
3.31k
    pri_damping = AVMMAX(0, pri_damping - get_msb(pri_strength));
261
4.86k
  if (enable_secondary && sec_strength)
262
1.74k
    sec_damping = AVMMAX(0, sec_damping - get_msb(sec_strength));
263
264
9.72k
  for (i = 0; i < height; i += 4) {
265
4.86k
    sum = v256_zero();
266
4.86k
    row = v256_from_v64(v64_load_aligned(&in[(i + 0) * CDEF_BSTRIDE]),
267
4.86k
                        v64_load_aligned(&in[(i + 1) * CDEF_BSTRIDE]),
268
4.86k
                        v64_load_aligned(&in[(i + 2) * CDEF_BSTRIDE]),
269
4.86k
                        v64_load_aligned(&in[(i + 3) * CDEF_BSTRIDE]));
270
4.86k
    max = min = row;
271
272
4.86k
    if (enable_primary) {
273
3.31k
      v256 tap[4];
274
      // Primary near taps
275
3.31k
      tap[0] =
276
3.31k
          v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE + po1]),
277
3.31k
                        v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + po1]),
278
3.31k
                        v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE + po1]),
279
3.31k
                        v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE + po1]));
280
3.31k
      p0 = constrain16(tap[0], row, pri_strength, pri_damping);
281
3.31k
      tap[1] =
282
3.31k
          v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE - po1]),
283
3.31k
                        v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - po1]),
284
3.31k
                        v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE - po1]),
285
3.31k
                        v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE - po1]));
286
3.31k
      p1 = constrain16(tap[1], row, pri_strength, pri_damping);
287
288
      // sum += pri_taps[0] * (p0 + p1)
289
3.31k
      sum = v256_add_16(
290
3.31k
          sum, v256_mullo_s16(v256_dup_16(pri_taps[0]), v256_add_16(p0, p1)));
291
292
      // Primary far taps
293
3.31k
      tap[2] =
294
3.31k
          v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE + po2]),
295
3.31k
                        v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + po2]),
296
3.31k
                        v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE + po2]),
297
3.31k
                        v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE + po2]));
298
3.31k
      p0 = constrain16(tap[2], row, pri_strength, pri_damping);
299
3.31k
      tap[3] =
300
3.31k
          v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE - po2]),
301
3.31k
                        v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - po2]),
302
3.31k
                        v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE - po2]),
303
3.31k
                        v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE - po2]));
304
3.31k
      p1 = constrain16(tap[3], row, pri_strength, pri_damping);
305
306
      // sum += pri_taps[1] * (p0 + p1)
307
3.31k
      sum = v256_add_16(
308
3.31k
          sum, v256_mullo_s16(v256_dup_16(pri_taps[1]), v256_add_16(p0, p1)));
309
3.31k
      if (clipping_required) {
310
192
        max = get_max_primary(tap, max, cdef_large_value_mask);
311
312
192
        min = v256_min_s16(min, tap[0]);
313
192
        min = v256_min_s16(min, tap[1]);
314
192
        min = v256_min_s16(min, tap[2]);
315
192
        min = v256_min_s16(min, tap[3]);
316
192
      }
317
3.31k
    }
318
319
4.86k
    if (enable_secondary) {
320
1.74k
      v256 tap[8];
321
      // Secondary near taps
322
1.74k
      tap[0] =
323
1.74k
          v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE + s1o1]),
324
1.74k
                        v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + s1o1]),
325
1.74k
                        v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE + s1o1]),
326
1.74k
                        v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE + s1o1]));
327
1.74k
      p0 = constrain16(tap[0], row, sec_strength, sec_damping);
328
1.74k
      tap[1] =
329
1.74k
          v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE - s1o1]),
330
1.74k
                        v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - s1o1]),
331
1.74k
                        v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE - s1o1]),
332
1.74k
                        v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE - s1o1]));
333
1.74k
      p1 = constrain16(tap[1], row, sec_strength, sec_damping);
334
1.74k
      tap[2] =
335
1.74k
          v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE + s2o1]),
336
1.74k
                        v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + s2o1]),
337
1.74k
                        v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE + s2o1]),
338
1.74k
                        v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE + s2o1]));
339
1.74k
      p2 = constrain16(tap[2], row, sec_strength, sec_damping);
340
1.74k
      tap[3] =
341
1.74k
          v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE - s2o1]),
342
1.74k
                        v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - s2o1]),
343
1.74k
                        v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE - s2o1]),
344
1.74k
                        v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE - s2o1]));
345
1.74k
      p3 = constrain16(tap[3], row, sec_strength, sec_damping);
346
347
      // sum += sec_taps[0] * (p0 + p1 + p2 + p3)
348
1.74k
      sum = v256_add_16(sum, v256_mullo_s16(v256_dup_16(sec_taps[0]),
349
1.74k
                                            v256_add_16(v256_add_16(p0, p1),
350
1.74k
                                                        v256_add_16(p2, p3))));
351
352
      // Secondary far taps
353
1.74k
      tap[4] =
354
1.74k
          v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE + s1o2]),
355
1.74k
                        v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + s1o2]),
356
1.74k
                        v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE + s1o2]),
357
1.74k
                        v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE + s1o2]));
358
1.74k
      p0 = constrain16(tap[4], row, sec_strength, sec_damping);
359
1.74k
      tap[5] =
360
1.74k
          v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE - s1o2]),
361
1.74k
                        v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - s1o2]),
362
1.74k
                        v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE - s1o2]),
363
1.74k
                        v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE - s1o2]));
364
1.74k
      p1 = constrain16(tap[5], row, sec_strength, sec_damping);
365
1.74k
      tap[6] =
366
1.74k
          v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE + s2o2]),
367
1.74k
                        v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + s2o2]),
368
1.74k
                        v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE + s2o2]),
369
1.74k
                        v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE + s2o2]));
370
1.74k
      p2 = constrain16(tap[6], row, sec_strength, sec_damping);
371
1.74k
      tap[7] =
372
1.74k
          v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE - s2o2]),
373
1.74k
                        v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - s2o2]),
374
1.74k
                        v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE - s2o2]),
375
1.74k
                        v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE - s2o2]));
376
1.74k
      p3 = constrain16(tap[7], row, sec_strength, sec_damping);
377
378
      // sum += sec_taps[1] * (p0 + p1 + p2 + p3)
379
1.74k
      sum = v256_add_16(sum, v256_mullo_s16(v256_dup_16(sec_taps[1]),
380
1.74k
                                            v256_add_16(v256_add_16(p0, p1),
381
1.74k
                                                        v256_add_16(p2, p3))));
382
383
1.74k
      if (clipping_required) {
384
192
        max = get_max_secondary(tap, max, cdef_large_value_mask);
385
192
        min = v256_min_s16(min, tap[0]);
386
192
        min = v256_min_s16(min, tap[1]);
387
192
        min = v256_min_s16(min, tap[2]);
388
192
        min = v256_min_s16(min, tap[3]);
389
192
        min = v256_min_s16(min, tap[4]);
390
192
        min = v256_min_s16(min, tap[5]);
391
192
        min = v256_min_s16(min, tap[6]);
392
192
        min = v256_min_s16(min, tap[7]);
393
192
      }
394
1.74k
    }
395
396
    // res = row + ((sum - (sum < 0) + 8) >> 4)
397
4.86k
    sum = v256_add_16(sum, v256_cmplt_s16(sum, v256_zero()));
398
4.86k
    res = v256_add_16(sum, v256_dup_16(8));
399
4.86k
    res = v256_shr_n_s16(res, 4);
400
4.86k
    res = v256_add_16(row, res);
401
4.86k
    if (clipping_required) {
402
192
      res = v256_min_s16(v256_max_s16(res, min), max);
403
192
    }
404
405
4.86k
    v64_store_aligned(&dest[(i + 0) * dstride],
406
4.86k
                      v128_high_v64(v256_high_v128(res)));
407
4.86k
    v64_store_aligned(&dest[(i + 1) * dstride],
408
4.86k
                      v128_low_v64(v256_high_v128(res)));
409
4.86k
    v64_store_aligned(&dest[(i + 2) * dstride],
410
4.86k
                      v128_high_v64(v256_low_v128(res)));
411
4.86k
    v64_store_aligned(&dest[(i + 3) * dstride],
412
4.86k
                      v128_low_v64(v256_low_v128(res)));
413
4.86k
  }
414
4.86k
}
Unexecuted instantiation: cdef_block_sse2.c:filter_block_4x4
Unexecuted instantiation: cdef_block_ssse3.c:filter_block_4x4
Unexecuted instantiation: cdef_block_sse4.c:filter_block_4x4
cdef_block_avx2.c:filter_block_4x4
Line
Count
Source
243
4.86k
                                  int enable_primary, int enable_secondary) {
244
4.86k
  const int clipping_required = enable_primary && enable_secondary;
245
4.86k
  v256 p0, p1, p2, p3;
246
4.86k
  v256 sum, row, res;
247
4.86k
  const v256 cdef_large_value_mask = v256_dup_16((uint16_t)~CDEF_VERY_LARGE);
248
4.86k
  v256 max, min;
249
4.86k
  const int po1 = av2_cdef_directions[dir][0];
250
4.86k
  const int po2 = av2_cdef_directions[dir][1];
251
4.86k
  const int s1o1 = av2_cdef_directions[dir + 2][0];
252
4.86k
  const int s1o2 = av2_cdef_directions[dir + 2][1];
253
4.86k
  const int s2o1 = av2_cdef_directions[dir - 2][0];
254
4.86k
  const int s2o2 = av2_cdef_directions[dir - 2][1];
255
4.86k
  const int *pri_taps = av2_cdef_pri_taps[(pri_strength >> coeff_shift) & 1];
256
4.86k
  const int *sec_taps = av2_cdef_sec_taps;
257
4.86k
  int i;
258
259
4.86k
  if (enable_primary && pri_strength)
260
3.31k
    pri_damping = AVMMAX(0, pri_damping - get_msb(pri_strength));
261
4.86k
  if (enable_secondary && sec_strength)
262
1.74k
    sec_damping = AVMMAX(0, sec_damping - get_msb(sec_strength));
263
264
9.72k
  for (i = 0; i < height; i += 4) {
265
4.86k
    sum = v256_zero();
266
4.86k
    row = v256_from_v64(v64_load_aligned(&in[(i + 0) * CDEF_BSTRIDE]),
267
4.86k
                        v64_load_aligned(&in[(i + 1) * CDEF_BSTRIDE]),
268
4.86k
                        v64_load_aligned(&in[(i + 2) * CDEF_BSTRIDE]),
269
4.86k
                        v64_load_aligned(&in[(i + 3) * CDEF_BSTRIDE]));
270
4.86k
    max = min = row;
271
272
4.86k
    if (enable_primary) {
273
3.31k
      v256 tap[4];
274
      // Primary near taps
275
3.31k
      tap[0] =
276
3.31k
          v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE + po1]),
277
3.31k
                        v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + po1]),
278
3.31k
                        v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE + po1]),
279
3.31k
                        v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE + po1]));
280
3.31k
      p0 = constrain16(tap[0], row, pri_strength, pri_damping);
281
3.31k
      tap[1] =
282
3.31k
          v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE - po1]),
283
3.31k
                        v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - po1]),
284
3.31k
                        v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE - po1]),
285
3.31k
                        v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE - po1]));
286
3.31k
      p1 = constrain16(tap[1], row, pri_strength, pri_damping);
287
288
      // sum += pri_taps[0] * (p0 + p1)
289
3.31k
      sum = v256_add_16(
290
3.31k
          sum, v256_mullo_s16(v256_dup_16(pri_taps[0]), v256_add_16(p0, p1)));
291
292
      // Primary far taps
293
3.31k
      tap[2] =
294
3.31k
          v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE + po2]),
295
3.31k
                        v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + po2]),
296
3.31k
                        v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE + po2]),
297
3.31k
                        v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE + po2]));
298
3.31k
      p0 = constrain16(tap[2], row, pri_strength, pri_damping);
299
3.31k
      tap[3] =
300
3.31k
          v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE - po2]),
301
3.31k
                        v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - po2]),
302
3.31k
                        v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE - po2]),
303
3.31k
                        v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE - po2]));
304
3.31k
      p1 = constrain16(tap[3], row, pri_strength, pri_damping);
305
306
      // sum += pri_taps[1] * (p0 + p1)
307
3.31k
      sum = v256_add_16(
308
3.31k
          sum, v256_mullo_s16(v256_dup_16(pri_taps[1]), v256_add_16(p0, p1)));
309
3.31k
      if (clipping_required) {
310
192
        max = get_max_primary(tap, max, cdef_large_value_mask);
311
312
192
        min = v256_min_s16(min, tap[0]);
313
192
        min = v256_min_s16(min, tap[1]);
314
192
        min = v256_min_s16(min, tap[2]);
315
192
        min = v256_min_s16(min, tap[3]);
316
192
      }
317
3.31k
    }
318
319
4.86k
    if (enable_secondary) {
320
1.74k
      v256 tap[8];
321
      // Secondary near taps
322
1.74k
      tap[0] =
323
1.74k
          v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE + s1o1]),
324
1.74k
                        v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + s1o1]),
325
1.74k
                        v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE + s1o1]),
326
1.74k
                        v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE + s1o1]));
327
1.74k
      p0 = constrain16(tap[0], row, sec_strength, sec_damping);
328
1.74k
      tap[1] =
329
1.74k
          v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE - s1o1]),
330
1.74k
                        v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - s1o1]),
331
1.74k
                        v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE - s1o1]),
332
1.74k
                        v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE - s1o1]));
333
1.74k
      p1 = constrain16(tap[1], row, sec_strength, sec_damping);
334
1.74k
      tap[2] =
335
1.74k
          v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE + s2o1]),
336
1.74k
                        v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + s2o1]),
337
1.74k
                        v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE + s2o1]),
338
1.74k
                        v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE + s2o1]));
339
1.74k
      p2 = constrain16(tap[2], row, sec_strength, sec_damping);
340
1.74k
      tap[3] =
341
1.74k
          v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE - s2o1]),
342
1.74k
                        v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - s2o1]),
343
1.74k
                        v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE - s2o1]),
344
1.74k
                        v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE - s2o1]));
345
1.74k
      p3 = constrain16(tap[3], row, sec_strength, sec_damping);
346
347
      // sum += sec_taps[0] * (p0 + p1 + p2 + p3)
348
1.74k
      sum = v256_add_16(sum, v256_mullo_s16(v256_dup_16(sec_taps[0]),
349
1.74k
                                            v256_add_16(v256_add_16(p0, p1),
350
1.74k
                                                        v256_add_16(p2, p3))));
351
352
      // Secondary far taps
353
1.74k
      tap[4] =
354
1.74k
          v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE + s1o2]),
355
1.74k
                        v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + s1o2]),
356
1.74k
                        v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE + s1o2]),
357
1.74k
                        v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE + s1o2]));
358
1.74k
      p0 = constrain16(tap[4], row, sec_strength, sec_damping);
359
1.74k
      tap[5] =
360
1.74k
          v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE - s1o2]),
361
1.74k
                        v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - s1o2]),
362
1.74k
                        v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE - s1o2]),
363
1.74k
                        v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE - s1o2]));
364
1.74k
      p1 = constrain16(tap[5], row, sec_strength, sec_damping);
365
1.74k
      tap[6] =
366
1.74k
          v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE + s2o2]),
367
1.74k
                        v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + s2o2]),
368
1.74k
                        v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE + s2o2]),
369
1.74k
                        v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE + s2o2]));
370
1.74k
      p2 = constrain16(tap[6], row, sec_strength, sec_damping);
371
1.74k
      tap[7] =
372
1.74k
          v256_from_v64(v64_load_unaligned(&in[(i + 0) * CDEF_BSTRIDE - s2o2]),
373
1.74k
                        v64_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - s2o2]),
374
1.74k
                        v64_load_unaligned(&in[(i + 2) * CDEF_BSTRIDE - s2o2]),
375
1.74k
                        v64_load_unaligned(&in[(i + 3) * CDEF_BSTRIDE - s2o2]));
376
1.74k
      p3 = constrain16(tap[7], row, sec_strength, sec_damping);
377
378
      // sum += sec_taps[1] * (p0 + p1 + p2 + p3)
379
1.74k
      sum = v256_add_16(sum, v256_mullo_s16(v256_dup_16(sec_taps[1]),
380
1.74k
                                            v256_add_16(v256_add_16(p0, p1),
381
1.74k
                                                        v256_add_16(p2, p3))));
382
383
1.74k
      if (clipping_required) {
384
192
        max = get_max_secondary(tap, max, cdef_large_value_mask);
385
192
        min = v256_min_s16(min, tap[0]);
386
192
        min = v256_min_s16(min, tap[1]);
387
192
        min = v256_min_s16(min, tap[2]);
388
192
        min = v256_min_s16(min, tap[3]);
389
192
        min = v256_min_s16(min, tap[4]);
390
192
        min = v256_min_s16(min, tap[5]);
391
192
        min = v256_min_s16(min, tap[6]);
392
192
        min = v256_min_s16(min, tap[7]);
393
192
      }
394
1.74k
    }
395
396
    // res = row + ((sum - (sum < 0) + 8) >> 4)
397
4.86k
    sum = v256_add_16(sum, v256_cmplt_s16(sum, v256_zero()));
398
4.86k
    res = v256_add_16(sum, v256_dup_16(8));
399
4.86k
    res = v256_shr_n_s16(res, 4);
400
4.86k
    res = v256_add_16(row, res);
401
4.86k
    if (clipping_required) {
402
192
      res = v256_min_s16(v256_max_s16(res, min), max);
403
192
    }
404
405
4.86k
    v64_store_aligned(&dest[(i + 0) * dstride],
406
4.86k
                      v128_high_v64(v256_high_v128(res)));
407
4.86k
    v64_store_aligned(&dest[(i + 1) * dstride],
408
4.86k
                      v128_low_v64(v256_high_v128(res)));
409
4.86k
    v64_store_aligned(&dest[(i + 2) * dstride],
410
4.86k
                      v128_high_v64(v256_low_v128(res)));
411
4.86k
    v64_store_aligned(&dest[(i + 3) * dstride],
412
4.86k
                      v128_low_v64(v256_low_v128(res)));
413
4.86k
  }
414
4.86k
}
415
416
/* Applies CDEF filtering for block width equals 8. */
417
SIMD_INLINE void filter_block_8x8(uint16_t *const dest, int dstride,
418
                                  const uint16_t *in, int pri_strength,
419
                                  int sec_strength, int dir, int pri_damping,
420
                                  int sec_damping, int coeff_shift, int height,
421
2.68k
                                  int enable_primary, int enable_secondary) {
422
2.68k
  const int clipping_required = enable_primary && enable_secondary;
423
2.68k
  int i;
424
2.68k
  v256 sum, p0, p1, p2, p3, row, res;
425
2.68k
  const v256 cdef_large_value_mask = v256_dup_16((uint16_t)~CDEF_VERY_LARGE);
426
2.68k
  v256 max, min;
427
2.68k
  const int po1 = av2_cdef_directions[dir][0];
428
2.68k
  const int po2 = av2_cdef_directions[dir][1];
429
2.68k
  const int s1o1 = av2_cdef_directions[dir + 2][0];
430
2.68k
  const int s1o2 = av2_cdef_directions[dir + 2][1];
431
2.68k
  const int s2o1 = av2_cdef_directions[dir - 2][0];
432
2.68k
  const int s2o2 = av2_cdef_directions[dir - 2][1];
433
2.68k
  const int *pri_taps = av2_cdef_pri_taps[(pri_strength >> coeff_shift) & 1];
434
2.68k
  const int *sec_taps = av2_cdef_sec_taps;
435
436
2.68k
  if (enable_primary && pri_strength)
437
2.10k
    pri_damping = AVMMAX(0, pri_damping - get_msb(pri_strength));
438
2.68k
  if (enable_secondary && sec_strength)
439
1.23k
    sec_damping = AVMMAX(0, sec_damping - get_msb(sec_strength));
440
441
13.4k
  for (i = 0; i < height; i += 2) {
442
10.7k
    v256 tap[8];
443
10.7k
    sum = v256_zero();
444
10.7k
    row = v256_from_v128(v128_load_aligned(&in[i * CDEF_BSTRIDE]),
445
10.7k
                         v128_load_aligned(&in[(i + 1) * CDEF_BSTRIDE]));
446
447
10.7k
    min = max = row;
448
10.7k
    if (enable_primary) {
449
      // Primary near taps
450
8.40k
      tap[0] = v256_from_v128(
451
8.40k
          v128_load_unaligned(&in[i * CDEF_BSTRIDE + po1]),
452
8.40k
          v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + po1]));
453
8.40k
      tap[1] = v256_from_v128(
454
8.40k
          v128_load_unaligned(&in[i * CDEF_BSTRIDE - po1]),
455
8.40k
          v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - po1]));
456
8.40k
      p0 = constrain16(tap[0], row, pri_strength, pri_damping);
457
8.40k
      p1 = constrain16(tap[1], row, pri_strength, pri_damping);
458
459
      // sum += pri_taps[0] * (p0 + p1)
460
8.40k
      sum = v256_add_16(
461
8.40k
          sum, v256_mullo_s16(v256_dup_16(pri_taps[0]), v256_add_16(p0, p1)));
462
463
      // Primary far taps
464
8.40k
      tap[2] = v256_from_v128(
465
8.40k
          v128_load_unaligned(&in[i * CDEF_BSTRIDE + po2]),
466
8.40k
          v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + po2]));
467
8.40k
      tap[3] = v256_from_v128(
468
8.40k
          v128_load_unaligned(&in[i * CDEF_BSTRIDE - po2]),
469
8.40k
          v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - po2]));
470
8.40k
      p0 = constrain16(tap[2], row, pri_strength, pri_damping);
471
8.40k
      p1 = constrain16(tap[3], row, pri_strength, pri_damping);
472
473
      // sum += pri_taps[1] * (p0 + p1)
474
8.40k
      sum = v256_add_16(
475
8.40k
          sum, v256_mullo_s16(v256_dup_16(pri_taps[1]), v256_add_16(p0, p1)));
476
477
8.40k
      if (clipping_required) {
478
2.59k
        max = get_max_primary(tap, max, cdef_large_value_mask);
479
2.59k
        min = v256_min_s16(min, tap[0]);
480
2.59k
        min = v256_min_s16(min, tap[1]);
481
2.59k
        min = v256_min_s16(min, tap[2]);
482
2.59k
        min = v256_min_s16(min, tap[3]);
483
2.59k
      }
484
      // End primary
485
8.40k
    }
486
487
10.7k
    if (enable_secondary) {
488
      // Secondary near taps
489
4.94k
      tap[0] = v256_from_v128(
490
4.94k
          v128_load_unaligned(&in[i * CDEF_BSTRIDE + s1o1]),
491
4.94k
          v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + s1o1]));
492
4.94k
      tap[1] = v256_from_v128(
493
4.94k
          v128_load_unaligned(&in[i * CDEF_BSTRIDE - s1o1]),
494
4.94k
          v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - s1o1]));
495
4.94k
      tap[2] = v256_from_v128(
496
4.94k
          v128_load_unaligned(&in[i * CDEF_BSTRIDE + s2o1]),
497
4.94k
          v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + s2o1]));
498
4.94k
      tap[3] = v256_from_v128(
499
4.94k
          v128_load_unaligned(&in[i * CDEF_BSTRIDE - s2o1]),
500
4.94k
          v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - s2o1]));
501
4.94k
      p0 = constrain16(tap[0], row, sec_strength, sec_damping);
502
4.94k
      p1 = constrain16(tap[1], row, sec_strength, sec_damping);
503
4.94k
      p2 = constrain16(tap[2], row, sec_strength, sec_damping);
504
4.94k
      p3 = constrain16(tap[3], row, sec_strength, sec_damping);
505
506
      // sum += sec_taps[0] * (p0 + p1 + p2 + p3)
507
4.94k
      sum = v256_add_16(sum, v256_mullo_s16(v256_dup_16(sec_taps[0]),
508
4.94k
                                            v256_add_16(v256_add_16(p0, p1),
509
4.94k
                                                        v256_add_16(p2, p3))));
510
511
      // Secondary far taps
512
4.94k
      tap[4] = v256_from_v128(
513
4.94k
          v128_load_unaligned(&in[i * CDEF_BSTRIDE + s1o2]),
514
4.94k
          v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + s1o2]));
515
4.94k
      tap[5] = v256_from_v128(
516
4.94k
          v128_load_unaligned(&in[i * CDEF_BSTRIDE - s1o2]),
517
4.94k
          v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - s1o2]));
518
4.94k
      tap[6] = v256_from_v128(
519
4.94k
          v128_load_unaligned(&in[i * CDEF_BSTRIDE + s2o2]),
520
4.94k
          v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + s2o2]));
521
4.94k
      tap[7] = v256_from_v128(
522
4.94k
          v128_load_unaligned(&in[i * CDEF_BSTRIDE - s2o2]),
523
4.94k
          v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - s2o2]));
524
4.94k
      p0 = constrain16(tap[4], row, sec_strength, sec_damping);
525
4.94k
      p1 = constrain16(tap[5], row, sec_strength, sec_damping);
526
4.94k
      p2 = constrain16(tap[6], row, sec_strength, sec_damping);
527
4.94k
      p3 = constrain16(tap[7], row, sec_strength, sec_damping);
528
529
      // sum += sec_taps[1] * (p0 + p1 + p2 + p3)
530
4.94k
      sum = v256_add_16(sum, v256_mullo_s16(v256_dup_16(sec_taps[1]),
531
4.94k
                                            v256_add_16(v256_add_16(p0, p1),
532
4.94k
                                                        v256_add_16(p2, p3))));
533
534
4.94k
      if (clipping_required) {
535
2.59k
        max = get_max_secondary(tap, max, cdef_large_value_mask);
536
2.59k
        min = v256_min_s16(min, tap[0]);
537
2.59k
        min = v256_min_s16(min, tap[1]);
538
2.59k
        min = v256_min_s16(min, tap[2]);
539
2.59k
        min = v256_min_s16(min, tap[3]);
540
2.59k
        min = v256_min_s16(min, tap[4]);
541
2.59k
        min = v256_min_s16(min, tap[5]);
542
2.59k
        min = v256_min_s16(min, tap[6]);
543
2.59k
        min = v256_min_s16(min, tap[7]);
544
2.59k
      }
545
      // End secondary
546
4.94k
    }
547
548
    // res = row + ((sum - (sum < 0) + 8) >> 4)
549
10.7k
    sum = v256_add_16(sum, v256_cmplt_s16(sum, v256_zero()));
550
10.7k
    res = v256_add_16(sum, v256_dup_16(8));
551
10.7k
    res = v256_shr_n_s16(res, 4);
552
10.7k
    res = v256_add_16(row, res);
553
10.7k
    if (clipping_required) {
554
2.59k
      res = v256_min_s16(v256_max_s16(res, min), max);
555
2.59k
    }
556
557
10.7k
    v128_store_unaligned(&dest[i * dstride], v256_high_v128(res));
558
10.7k
    v128_store_unaligned(&dest[(i + 1) * dstride], v256_low_v128(res));
559
10.7k
  }
560
2.68k
}
Unexecuted instantiation: cdef_block_sse2.c:filter_block_8x8
Unexecuted instantiation: cdef_block_ssse3.c:filter_block_8x8
Unexecuted instantiation: cdef_block_sse4.c:filter_block_8x8
cdef_block_avx2.c:filter_block_8x8
Line
Count
Source
421
2.68k
                                  int enable_primary, int enable_secondary) {
422
2.68k
  const int clipping_required = enable_primary && enable_secondary;
423
2.68k
  int i;
424
2.68k
  v256 sum, p0, p1, p2, p3, row, res;
425
2.68k
  const v256 cdef_large_value_mask = v256_dup_16((uint16_t)~CDEF_VERY_LARGE);
426
2.68k
  v256 max, min;
427
2.68k
  const int po1 = av2_cdef_directions[dir][0];
428
2.68k
  const int po2 = av2_cdef_directions[dir][1];
429
2.68k
  const int s1o1 = av2_cdef_directions[dir + 2][0];
430
2.68k
  const int s1o2 = av2_cdef_directions[dir + 2][1];
431
2.68k
  const int s2o1 = av2_cdef_directions[dir - 2][0];
432
2.68k
  const int s2o2 = av2_cdef_directions[dir - 2][1];
433
2.68k
  const int *pri_taps = av2_cdef_pri_taps[(pri_strength >> coeff_shift) & 1];
434
2.68k
  const int *sec_taps = av2_cdef_sec_taps;
435
436
2.68k
  if (enable_primary && pri_strength)
437
2.10k
    pri_damping = AVMMAX(0, pri_damping - get_msb(pri_strength));
438
2.68k
  if (enable_secondary && sec_strength)
439
1.23k
    sec_damping = AVMMAX(0, sec_damping - get_msb(sec_strength));
440
441
13.4k
  for (i = 0; i < height; i += 2) {
442
10.7k
    v256 tap[8];
443
10.7k
    sum = v256_zero();
444
10.7k
    row = v256_from_v128(v128_load_aligned(&in[i * CDEF_BSTRIDE]),
445
10.7k
                         v128_load_aligned(&in[(i + 1) * CDEF_BSTRIDE]));
446
447
10.7k
    min = max = row;
448
10.7k
    if (enable_primary) {
449
      // Primary near taps
450
8.40k
      tap[0] = v256_from_v128(
451
8.40k
          v128_load_unaligned(&in[i * CDEF_BSTRIDE + po1]),
452
8.40k
          v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + po1]));
453
8.40k
      tap[1] = v256_from_v128(
454
8.40k
          v128_load_unaligned(&in[i * CDEF_BSTRIDE - po1]),
455
8.40k
          v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - po1]));
456
8.40k
      p0 = constrain16(tap[0], row, pri_strength, pri_damping);
457
8.40k
      p1 = constrain16(tap[1], row, pri_strength, pri_damping);
458
459
      // sum += pri_taps[0] * (p0 + p1)
460
8.40k
      sum = v256_add_16(
461
8.40k
          sum, v256_mullo_s16(v256_dup_16(pri_taps[0]), v256_add_16(p0, p1)));
462
463
      // Primary far taps
464
8.40k
      tap[2] = v256_from_v128(
465
8.40k
          v128_load_unaligned(&in[i * CDEF_BSTRIDE + po2]),
466
8.40k
          v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + po2]));
467
8.40k
      tap[3] = v256_from_v128(
468
8.40k
          v128_load_unaligned(&in[i * CDEF_BSTRIDE - po2]),
469
8.40k
          v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - po2]));
470
8.40k
      p0 = constrain16(tap[2], row, pri_strength, pri_damping);
471
8.40k
      p1 = constrain16(tap[3], row, pri_strength, pri_damping);
472
473
      // sum += pri_taps[1] * (p0 + p1)
474
8.40k
      sum = v256_add_16(
475
8.40k
          sum, v256_mullo_s16(v256_dup_16(pri_taps[1]), v256_add_16(p0, p1)));
476
477
8.40k
      if (clipping_required) {
478
2.59k
        max = get_max_primary(tap, max, cdef_large_value_mask);
479
2.59k
        min = v256_min_s16(min, tap[0]);
480
2.59k
        min = v256_min_s16(min, tap[1]);
481
2.59k
        min = v256_min_s16(min, tap[2]);
482
2.59k
        min = v256_min_s16(min, tap[3]);
483
2.59k
      }
484
      // End primary
485
8.40k
    }
486
487
10.7k
    if (enable_secondary) {
488
      // Secondary near taps
489
4.94k
      tap[0] = v256_from_v128(
490
4.94k
          v128_load_unaligned(&in[i * CDEF_BSTRIDE + s1o1]),
491
4.94k
          v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + s1o1]));
492
4.94k
      tap[1] = v256_from_v128(
493
4.94k
          v128_load_unaligned(&in[i * CDEF_BSTRIDE - s1o1]),
494
4.94k
          v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - s1o1]));
495
4.94k
      tap[2] = v256_from_v128(
496
4.94k
          v128_load_unaligned(&in[i * CDEF_BSTRIDE + s2o1]),
497
4.94k
          v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + s2o1]));
498
4.94k
      tap[3] = v256_from_v128(
499
4.94k
          v128_load_unaligned(&in[i * CDEF_BSTRIDE - s2o1]),
500
4.94k
          v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - s2o1]));
501
4.94k
      p0 = constrain16(tap[0], row, sec_strength, sec_damping);
502
4.94k
      p1 = constrain16(tap[1], row, sec_strength, sec_damping);
503
4.94k
      p2 = constrain16(tap[2], row, sec_strength, sec_damping);
504
4.94k
      p3 = constrain16(tap[3], row, sec_strength, sec_damping);
505
506
      // sum += sec_taps[0] * (p0 + p1 + p2 + p3)
507
4.94k
      sum = v256_add_16(sum, v256_mullo_s16(v256_dup_16(sec_taps[0]),
508
4.94k
                                            v256_add_16(v256_add_16(p0, p1),
509
4.94k
                                                        v256_add_16(p2, p3))));
510
511
      // Secondary far taps
512
4.94k
      tap[4] = v256_from_v128(
513
4.94k
          v128_load_unaligned(&in[i * CDEF_BSTRIDE + s1o2]),
514
4.94k
          v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + s1o2]));
515
4.94k
      tap[5] = v256_from_v128(
516
4.94k
          v128_load_unaligned(&in[i * CDEF_BSTRIDE - s1o2]),
517
4.94k
          v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - s1o2]));
518
4.94k
      tap[6] = v256_from_v128(
519
4.94k
          v128_load_unaligned(&in[i * CDEF_BSTRIDE + s2o2]),
520
4.94k
          v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE + s2o2]));
521
4.94k
      tap[7] = v256_from_v128(
522
4.94k
          v128_load_unaligned(&in[i * CDEF_BSTRIDE - s2o2]),
523
4.94k
          v128_load_unaligned(&in[(i + 1) * CDEF_BSTRIDE - s2o2]));
524
4.94k
      p0 = constrain16(tap[4], row, sec_strength, sec_damping);
525
4.94k
      p1 = constrain16(tap[5], row, sec_strength, sec_damping);
526
4.94k
      p2 = constrain16(tap[6], row, sec_strength, sec_damping);
527
4.94k
      p3 = constrain16(tap[7], row, sec_strength, sec_damping);
528
529
      // sum += sec_taps[1] * (p0 + p1 + p2 + p3)
530
4.94k
      sum = v256_add_16(sum, v256_mullo_s16(v256_dup_16(sec_taps[1]),
531
4.94k
                                            v256_add_16(v256_add_16(p0, p1),
532
4.94k
                                                        v256_add_16(p2, p3))));
533
534
4.94k
      if (clipping_required) {
535
2.59k
        max = get_max_secondary(tap, max, cdef_large_value_mask);
536
2.59k
        min = v256_min_s16(min, tap[0]);
537
2.59k
        min = v256_min_s16(min, tap[1]);
538
2.59k
        min = v256_min_s16(min, tap[2]);
539
2.59k
        min = v256_min_s16(min, tap[3]);
540
2.59k
        min = v256_min_s16(min, tap[4]);
541
2.59k
        min = v256_min_s16(min, tap[5]);
542
2.59k
        min = v256_min_s16(min, tap[6]);
543
2.59k
        min = v256_min_s16(min, tap[7]);
544
2.59k
      }
545
      // End secondary
546
4.94k
    }
547
548
    // res = row + ((sum - (sum < 0) + 8) >> 4)
549
10.7k
    sum = v256_add_16(sum, v256_cmplt_s16(sum, v256_zero()));
550
10.7k
    res = v256_add_16(sum, v256_dup_16(8));
551
10.7k
    res = v256_shr_n_s16(res, 4);
552
10.7k
    res = v256_add_16(row, res);
553
10.7k
    if (clipping_required) {
554
2.59k
      res = v256_min_s16(v256_max_s16(res, min), max);
555
2.59k
    }
556
557
10.7k
    v128_store_unaligned(&dest[i * dstride], v256_high_v128(res));
558
10.7k
    v128_store_unaligned(&dest[(i + 1) * dstride], v256_low_v128(res));
559
10.7k
  }
560
2.68k
}
561
562
/* Copy function specialized for block width equals 4, invoked when both primary
563
 * and secondary strengths are zero. */
564
SIMD_INLINE void copy_block_4xh(uint16_t *const dest, int dstride,
565
0
                                const uint16_t *in, int height) {
566
0
  int i;
567
0
  for (i = 0; i < height; i += 4) {
568
0
    const v128 row0 =
569
0
        v128_from_v64(v64_load_aligned(&in[(i + 0) * CDEF_BSTRIDE]),
570
0
                      v64_load_aligned(&in[(i + 1) * CDEF_BSTRIDE]));
571
0
    const v128 row1 =
572
0
        v128_from_v64(v64_load_aligned(&in[(i + 2) * CDEF_BSTRIDE]),
573
0
                      v64_load_aligned(&in[(i + 3) * CDEF_BSTRIDE]));
574
0
    v64_store_aligned(&dest[(i + 0) * dstride], v128_high_v64(row0));
575
0
    v64_store_aligned(&dest[(i + 1) * dstride], v128_low_v64(row0));
576
0
    v64_store_aligned(&dest[(i + 2) * dstride], v128_high_v64(row1));
577
0
    v64_store_aligned(&dest[(i + 3) * dstride], v128_low_v64(row1));
578
0
  }
579
0
}
Unexecuted instantiation: cdef_block_sse2.c:copy_block_4xh
Unexecuted instantiation: cdef_block_ssse3.c:copy_block_4xh
Unexecuted instantiation: cdef_block_sse4.c:copy_block_4xh
Unexecuted instantiation: cdef_block_avx2.c:copy_block_4xh
580
581
/* Copy function specialized for block width equals 8, invoked when both primary
582
 * and secondary strengths are zero. */
583
SIMD_INLINE void copy_block_8xh(uint16_t *const dest, int dstride,
584
1.10k
                                const uint16_t *in, int height) {
585
1.10k
  int i;
586
5.50k
  for (i = 0; i < height; i += 2) {
587
4.40k
    const v128 row0 = v128_load_aligned(&in[i * CDEF_BSTRIDE]);
588
4.40k
    const v128 row1 = v128_load_aligned(&in[(i + 1) * CDEF_BSTRIDE]);
589
4.40k
    v128_store_unaligned(&dest[i * dstride], row0);
590
4.40k
    v128_store_unaligned(&dest[(i + 1) * dstride], row1);
591
4.40k
  }
592
1.10k
}
Unexecuted instantiation: cdef_block_sse2.c:copy_block_8xh
Unexecuted instantiation: cdef_block_ssse3.c:copy_block_8xh
Unexecuted instantiation: cdef_block_sse4.c:copy_block_8xh
cdef_block_avx2.c:copy_block_8xh
Line
Count
Source
584
1.10k
                                const uint16_t *in, int height) {
585
1.10k
  int i;
586
5.50k
  for (i = 0; i < height; i += 2) {
587
4.40k
    const v128 row0 = v128_load_aligned(&in[i * CDEF_BSTRIDE]);
588
4.40k
    const v128 row1 = v128_load_aligned(&in[(i + 1) * CDEF_BSTRIDE]);
589
4.40k
    v128_store_unaligned(&dest[i * dstride], row0);
590
4.40k
    v128_store_unaligned(&dest[(i + 1) * dstride], row1);
591
4.40k
  }
592
1.10k
}
593
594
/* Wrapper function which invokes block width specific CDEF SIMD functions when
595
 * primary and secondary strengths are non-zero. */
596
void SIMD_FUNC(av2_cdef_filter_16_0)(uint16_t *const dest, int dstride,
597
                                     const uint16_t *in, int pri_strength,
598
                                     int sec_strength, int dir, int pri_damping,
599
                                     int sec_damping, int coeff_shift,
600
841
                                     int block_width, int block_height) {
601
841
  if (block_width == 8) {
602
649
    filter_block_8x8(dest, dstride, in, pri_strength, sec_strength, dir,
603
649
                     pri_damping, sec_damping, coeff_shift, block_height,
604
649
                     /*enable_primary=*/1,
605
649
                     /*enable_secondary=*/1);
606
649
  } else {
607
192
    filter_block_4x4(dest, dstride, in, pri_strength, sec_strength, dir,
608
192
                     pri_damping, sec_damping, coeff_shift, block_height,
609
192
                     /*enable_primary=*/1,
610
192
                     /*enable_secondary=*/1);
611
192
  }
612
841
}
Unexecuted instantiation: av2_cdef_filter_16_0_sse2
Unexecuted instantiation: av2_cdef_filter_16_0_ssse3
Unexecuted instantiation: av2_cdef_filter_16_0_sse4_1
av2_cdef_filter_16_0_avx2
Line
Count
Source
600
841
                                     int block_width, int block_height) {
601
841
  if (block_width == 8) {
602
649
    filter_block_8x8(dest, dstride, in, pri_strength, sec_strength, dir,
603
649
                     pri_damping, sec_damping, coeff_shift, block_height,
604
649
                     /*enable_primary=*/1,
605
649
                     /*enable_secondary=*/1);
606
649
  } else {
607
192
    filter_block_4x4(dest, dstride, in, pri_strength, sec_strength, dir,
608
192
                     pri_damping, sec_damping, coeff_shift, block_height,
609
192
                     /*enable_primary=*/1,
610
192
                     /*enable_secondary=*/1);
611
192
  }
612
841
}
613
614
/* Wrapper function which invokes block width specific CDEF SIMD functions when
615
 * primary strength is non-zero and secondary strength is zero. */
616
void SIMD_FUNC(av2_cdef_filter_16_1)(uint16_t *const dest, int dstride,
617
                                     const uint16_t *in, int pri_strength,
618
                                     int sec_strength, int dir, int pri_damping,
619
                                     int sec_damping, int coeff_shift,
620
4.57k
                                     int block_width, int block_height) {
621
4.57k
  if (block_width == 8) {
622
1.45k
    filter_block_8x8(dest, dstride, in, pri_strength, sec_strength, dir,
623
1.45k
                     pri_damping, sec_damping, coeff_shift, block_height,
624
1.45k
                     /*enable_primary=*/1,
625
1.45k
                     /*enable_secondary=*/0);
626
3.12k
  } else {
627
3.12k
    filter_block_4x4(dest, dstride, in, pri_strength, sec_strength, dir,
628
3.12k
                     pri_damping, sec_damping, coeff_shift, block_height,
629
3.12k
                     /*enable_primary=*/1,
630
3.12k
                     /*enable_secondary=*/0);
631
3.12k
  }
632
4.57k
}
Unexecuted instantiation: av2_cdef_filter_16_1_sse2
Unexecuted instantiation: av2_cdef_filter_16_1_ssse3
Unexecuted instantiation: av2_cdef_filter_16_1_sse4_1
av2_cdef_filter_16_1_avx2
Line
Count
Source
620
4.57k
                                     int block_width, int block_height) {
621
4.57k
  if (block_width == 8) {
622
1.45k
    filter_block_8x8(dest, dstride, in, pri_strength, sec_strength, dir,
623
1.45k
                     pri_damping, sec_damping, coeff_shift, block_height,
624
1.45k
                     /*enable_primary=*/1,
625
1.45k
                     /*enable_secondary=*/0);
626
3.12k
  } else {
627
3.12k
    filter_block_4x4(dest, dstride, in, pri_strength, sec_strength, dir,
628
3.12k
                     pri_damping, sec_damping, coeff_shift, block_height,
629
3.12k
                     /*enable_primary=*/1,
630
3.12k
                     /*enable_secondary=*/0);
631
3.12k
  }
632
4.57k
}
633
634
/* Wrapper function which invokes block width specific CDEF SIMD functions when
635
 * primary strength is zero and secondary strength is non-zero. */
636
void SIMD_FUNC(av2_cdef_filter_16_2)(uint16_t *const dest, int dstride,
637
                                     const uint16_t *in, int pri_strength,
638
                                     int sec_strength, int dir, int pri_damping,
639
                                     int sec_damping, int coeff_shift,
640
2.13k
                                     int block_width, int block_height) {
641
2.13k
  if (block_width == 8) {
642
586
    filter_block_8x8(dest, dstride, in, pri_strength, sec_strength, dir,
643
586
                     pri_damping, sec_damping, coeff_shift, block_height,
644
586
                     /*enable_primary=*/0,
645
586
                     /*enable_secondary=*/1);
646
1.54k
  } else {
647
1.54k
    filter_block_4x4(dest, dstride, in, pri_strength, sec_strength, dir,
648
1.54k
                     pri_damping, sec_damping, coeff_shift, block_height,
649
1.54k
                     /*enable_primary=*/0,
650
1.54k
                     /*enable_secondary=*/1);
651
1.54k
  }
652
2.13k
}
Unexecuted instantiation: av2_cdef_filter_16_2_sse2
Unexecuted instantiation: av2_cdef_filter_16_2_ssse3
Unexecuted instantiation: av2_cdef_filter_16_2_sse4_1
av2_cdef_filter_16_2_avx2
Line
Count
Source
640
2.13k
                                     int block_width, int block_height) {
641
2.13k
  if (block_width == 8) {
642
586
    filter_block_8x8(dest, dstride, in, pri_strength, sec_strength, dir,
643
586
                     pri_damping, sec_damping, coeff_shift, block_height,
644
586
                     /*enable_primary=*/0,
645
586
                     /*enable_secondary=*/1);
646
1.54k
  } else {
647
1.54k
    filter_block_4x4(dest, dstride, in, pri_strength, sec_strength, dir,
648
1.54k
                     pri_damping, sec_damping, coeff_shift, block_height,
649
1.54k
                     /*enable_primary=*/0,
650
1.54k
                     /*enable_secondary=*/1);
651
1.54k
  }
652
2.13k
}
653
654
/* Wrapper function which invokes block width specific CDEF SIMD functions when
655
 * both primary and secondary strengths are zero. */
656
void SIMD_FUNC(av2_cdef_filter_16_3)(uint16_t *const dest, int dstride,
657
                                     const uint16_t *in, int pri_strength,
658
                                     int sec_strength, int dir, int pri_damping,
659
                                     int sec_damping, int coeff_shift,
660
1.10k
                                     int block_width, int block_height) {
661
1.10k
  (void)pri_strength;
662
1.10k
  (void)sec_strength;
663
1.10k
  (void)dir;
664
1.10k
  (void)pri_damping;
665
1.10k
  (void)sec_damping;
666
1.10k
  (void)coeff_shift;
667
1.10k
  (void)block_width;
668
1.10k
  if (block_width == 8) {
669
1.10k
    copy_block_8xh(dest, dstride, in, block_height);
670
1.10k
  } else {
671
0
    copy_block_4xh(dest, dstride, in, block_height);
672
0
  }
673
1.10k
}
Unexecuted instantiation: av2_cdef_filter_16_3_sse2
Unexecuted instantiation: av2_cdef_filter_16_3_ssse3
Unexecuted instantiation: av2_cdef_filter_16_3_sse4_1
av2_cdef_filter_16_3_avx2
Line
Count
Source
660
1.10k
                                     int block_width, int block_height) {
661
1.10k
  (void)pri_strength;
662
1.10k
  (void)sec_strength;
663
1.10k
  (void)dir;
664
1.10k
  (void)pri_damping;
665
1.10k
  (void)sec_damping;
666
1.10k
  (void)coeff_shift;
667
1.10k
  (void)block_width;
668
1.10k
  if (block_width == 8) {
669
1.10k
    copy_block_8xh(dest, dstride, in, block_height);
670
1.10k
  } else {
671
0
    copy_block_4xh(dest, dstride, in, block_height);
672
0
  }
673
1.10k
}
674
675
void SIMD_FUNC(av2_cdef_copy_rect8_16bit_to_16bit)(uint16_t *const dst,
676
                                                   int dstride,
677
                                                   const uint16_t *src,
678
1.90k
                                                   int sstride, int v, int h) {
679
1.90k
  int i, j;
680
17.5k
  for (i = 0; i < v; i++) {
681
64.6k
    for (j = 0; j < (h & ~0x7); j += 8) {
682
48.9k
      v128 row = v128_load_unaligned(&src[i * sstride + j]);
683
48.9k
      v128_store_unaligned(&dst[i * dstride + j], row);
684
48.9k
    }
685
15.7k
    for (; j < h; j++) {
686
64
      dst[i * dstride + j] = src[i * sstride + j];
687
64
    }
688
15.6k
  }
689
1.90k
}
Unexecuted instantiation: av2_cdef_copy_rect8_16bit_to_16bit_sse2
Unexecuted instantiation: av2_cdef_copy_rect8_16bit_to_16bit_ssse3
Unexecuted instantiation: av2_cdef_copy_rect8_16bit_to_16bit_sse4_1
av2_cdef_copy_rect8_16bit_to_16bit_avx2
Line
Count
Source
678
1.90k
                                                   int sstride, int v, int h) {
679
1.90k
  int i, j;
680
17.5k
  for (i = 0; i < v; i++) {
681
64.6k
    for (j = 0; j < (h & ~0x7); j += 8) {
682
48.9k
      v128 row = v128_load_unaligned(&src[i * sstride + j]);
683
48.9k
      v128_store_unaligned(&dst[i * dstride + j], row);
684
48.9k
    }
685
15.7k
    for (; j < h; j++) {
686
64
      dst[i * dstride + j] = src[i * sstride + j];
687
64
    }
688
15.6k
  }
689
1.90k
}
690
691
#endif  // AVM_AV2_COMMON_CDEF_BLOCK_SIMD_H_