/src/aom/aom_dsp/x86/convolve_common_intrin.h

Source (jump to first uncovered line)
/*
 * Copyright (c) 2018, Alliance for Open Media. All rights reserved.
 *
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
 */

#ifndef AOM_AOM_DSP_X86_CONVOLVE_COMMON_INTRIN_H_
#define AOM_AOM_DSP_X86_CONVOLVE_COMMON_INTRIN_H_

// Note:
//  This header file should be put below any x86 intrinsics head file

static inline void add_store(CONV_BUF_TYPE *const dst, const __m128i *const res,
                             const int do_average) {
  __m128i d;
  if (do_average) {
    d = _mm_load_si128((__m128i *)dst);
    d = _mm_add_epi32(d, *res);
    d = _mm_srai_epi32(d, 1);
  } else {
    d = *res;
  }
  _mm_store_si128((__m128i *)dst, d);
}

static inline void prepare_coeffs_12tap(const InterpFilterParams *filter_params,
                                        int subpel_q4,
                                        __m128i *coeffs /* [6] */) {
  const int16_t *const y_filter = av1_get_interp_filter_subpel_kernel(
      filter_params, subpel_q4 & SUBPEL_MASK);

  __m128i coeffs_y = _mm_loadu_si128((__m128i *)y_filter);

  coeffs[0] = _mm_shuffle_epi32(coeffs_y, 0);    // coeffs 0 1 0 1 0 1 0 1
  coeffs[1] = _mm_shuffle_epi32(coeffs_y, 85);   // coeffs 2 3 2 3 2 3 2 3
  coeffs[2] = _mm_shuffle_epi32(coeffs_y, 170);  // coeffs 4 5 4 5 4 5 4 5
  coeffs[3] = _mm_shuffle_epi32(coeffs_y, 255);  // coeffs 6 7 6 7 6 7 6 7

  coeffs_y = _mm_loadl_epi64((__m128i *)(y_filter + 8));

  coeffs[4] = _mm_shuffle_epi32(coeffs_y, 0);  // coeffs 8 9 8 9 8 9 8 9
  coeffs[5] =
      _mm_shuffle_epi32(coeffs_y, 85);  // coeffs 10 11 10 11 10 11 10 11
}

static inline __m128i convolve_12tap(const __m128i *s, const __m128i *coeffs) {
  const __m128i d0 = _mm_madd_epi16(s[0], coeffs[0]);
  const __m128i d1 = _mm_madd_epi16(s[1], coeffs[1]);
  const __m128i d2 = _mm_madd_epi16(s[2], coeffs[2]);
  const __m128i d3 = _mm_madd_epi16(s[3], coeffs[3]);
  const __m128i d4 = _mm_madd_epi16(s[4], coeffs[4]);
  const __m128i d5 = _mm_madd_epi16(s[5], coeffs[5]);
  const __m128i d_0123 =
      _mm_add_epi32(_mm_add_epi32(d0, d1), _mm_add_epi32(d2, d3));
  const __m128i d = _mm_add_epi32(_mm_add_epi32(d4, d5), d_0123);
  return d;
}

static inline __m128i convolve_lo_x_12tap(const __m128i *s,
                                          const __m128i *coeffs,
                                          const __m128i zero) {
  __m128i ss[6];
  ss[0] = _mm_unpacklo_epi8(s[0], zero);  //  0  1  1  2  2  3  3  4
  ss[1] = _mm_unpacklo_epi8(s[1], zero);  //  2  3  3  4  4  5  5  6
  ss[2] = _mm_unpacklo_epi8(s[2], zero);  //  4  5  5  6  6  7  7  8
  ss[3] = _mm_unpacklo_epi8(s[3], zero);  //  6  7  7  8  8  9  9 10
  ss[4] = _mm_unpackhi_epi8(s[2], zero);  //  8  9  9 10 10 11 11 12
  ss[5] = _mm_unpackhi_epi8(s[3], zero);  // 10 11 11 12 12 13 13 14
  return convolve_12tap(ss, coeffs);
}

static inline __m128i convolve_lo_y_12tap(const __m128i *s,
                                          const __m128i *coeffs) {
  __m128i ss[6];
  const __m128i zero = _mm_setzero_si128();
  ss[0] = _mm_unpacklo_epi8(s[0], zero);
  ss[1] = _mm_unpacklo_epi8(s[2], zero);
  ss[2] = _mm_unpacklo_epi8(s[4], zero);
  ss[3] = _mm_unpacklo_epi8(s[6], zero);
  ss[4] = _mm_unpacklo_epi8(s[8], zero);
  ss[5] = _mm_unpacklo_epi8(s[10], zero);
  return convolve_12tap(ss, coeffs);
}

static inline __m128i convolve_hi_y_12tap(const __m128i *s,
                                          const __m128i *coeffs) {
  __m128i ss[6];
  const __m128i zero = _mm_setzero_si128();
  ss[0] = _mm_unpackhi_epi8(s[0], zero);
  ss[1] = _mm_unpackhi_epi8(s[2], zero);
  ss[2] = _mm_unpackhi_epi8(s[4], zero);
  ss[3] = _mm_unpackhi_epi8(s[6], zero);
  ss[4] = _mm_unpackhi_epi8(s[8], zero);
  ss[5] = _mm_unpackhi_epi8(s[10], zero);
  return convolve_12tap(ss, coeffs);
}
#endif  // AOM_AOM_DSP_X86_CONVOLVE_COMMON_INTRIN_H_

Coverage Report

Created: 2025-06-13 07:07

Line	Count	Source (jump to first uncovered line)
1		/*
2		* Copyright (c) 2018, Alliance for Open Media. All rights reserved.
3		*
4		* This source code is subject to the terms of the BSD 2 Clause License and
5		* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6		* was not distributed with this source code in the LICENSE file, you can
7		* obtain it at www.aomedia.org/license/software. If the Alliance for Open
8		* Media Patent License 1.0 was not distributed with this source code in the
9		* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10		*/
11
12		#ifndef AOM_AOM_DSP_X86_CONVOLVE_COMMON_INTRIN_H_
13		#define AOM_AOM_DSP_X86_CONVOLVE_COMMON_INTRIN_H_
14
15		// Note:
16		// This header file should be put below any x86 intrinsics head file
17
18		static inline void add_store(CONV_BUF_TYPE const dst, const __m128i const res,
19	0	const int do_average) {
20	0	__m128i d;
21	0	if (do_average) {
22	0	d = _mm_load_si128((__m128i *)dst);
23	0	d = _mm_add_epi32(d, *res);
24	0	d = _mm_srai_epi32(d, 1);
25	0	} else {
26	0	d = *res;
27	0	}
28	0	_mm_store_si128((__m128i *)dst, d);
29	0	} Unexecuted instantiation: highbd_convolve_ssse3.c:add_store Unexecuted instantiation: convolve_2d_sse2.c:add_store Unexecuted instantiation: convolve_sse2.c:add_store Unexecuted instantiation: highbd_convolve_2d_ssse3.c:add_store Unexecuted instantiation: convolve_avx2.c:add_store Unexecuted instantiation: jnt_convolve_avx2.c:add_store Unexecuted instantiation: highbd_jnt_convolve_avx2.c:add_store
30
31		static inline void prepare_coeffs_12tap(const InterpFilterParams *filter_params,
32		int subpel_q4,
33	0	__m128i coeffs / [6] */) {
34	0	const int16_t *const y_filter = av1_get_interp_filter_subpel_kernel(
35	0	filter_params, subpel_q4 & SUBPEL_MASK);
36
37	0	__m128i coeffs_y = _mm_loadu_si128((__m128i *)y_filter);
38
39	0	coeffs[0] = _mm_shuffle_epi32(coeffs_y, 0); // coeffs 0 1 0 1 0 1 0 1
40	0	coeffs[1] = _mm_shuffle_epi32(coeffs_y, 85); // coeffs 2 3 2 3 2 3 2 3
41	0	coeffs[2] = _mm_shuffle_epi32(coeffs_y, 170); // coeffs 4 5 4 5 4 5 4 5
42	0	coeffs[3] = _mm_shuffle_epi32(coeffs_y, 255); // coeffs 6 7 6 7 6 7 6 7
43
44	0	coeffs_y = _mm_loadl_epi64((__m128i *)(y_filter + 8));
45
46	0	coeffs[4] = _mm_shuffle_epi32(coeffs_y, 0); // coeffs 8 9 8 9 8 9 8 9
47	0	coeffs[5] =
48	0	_mm_shuffle_epi32(coeffs_y, 85); // coeffs 10 11 10 11 10 11 10 11
49	0	} Unexecuted instantiation: highbd_convolve_ssse3.c:prepare_coeffs_12tap Unexecuted instantiation: convolve_2d_sse2.c:prepare_coeffs_12tap Unexecuted instantiation: convolve_sse2.c:prepare_coeffs_12tap Unexecuted instantiation: highbd_convolve_2d_ssse3.c:prepare_coeffs_12tap Unexecuted instantiation: convolve_avx2.c:prepare_coeffs_12tap Unexecuted instantiation: jnt_convolve_avx2.c:prepare_coeffs_12tap Unexecuted instantiation: highbd_jnt_convolve_avx2.c:prepare_coeffs_12tap
50
51	0	static inline __m128i convolve_12tap(const __m128i s, const __m128i coeffs) {
52	0	const __m128i d0 = _mm_madd_epi16(s[0], coeffs[0]);
53	0	const __m128i d1 = _mm_madd_epi16(s[1], coeffs[1]);
54	0	const __m128i d2 = _mm_madd_epi16(s[2], coeffs[2]);
55	0	const __m128i d3 = _mm_madd_epi16(s[3], coeffs[3]);
56	0	const __m128i d4 = _mm_madd_epi16(s[4], coeffs[4]);
57	0	const __m128i d5 = _mm_madd_epi16(s[5], coeffs[5]);
58	0	const __m128i d_0123 =
59	0	_mm_add_epi32(_mm_add_epi32(d0, d1), _mm_add_epi32(d2, d3));
60	0	const __m128i d = _mm_add_epi32(_mm_add_epi32(d4, d5), d_0123);
61	0	return d;
62	0	} Unexecuted instantiation: highbd_convolve_ssse3.c:convolve_12tap Unexecuted instantiation: convolve_2d_sse2.c:convolve_12tap Unexecuted instantiation: convolve_sse2.c:convolve_12tap Unexecuted instantiation: highbd_convolve_2d_ssse3.c:convolve_12tap Unexecuted instantiation: convolve_avx2.c:convolve_12tap Unexecuted instantiation: jnt_convolve_avx2.c:convolve_12tap Unexecuted instantiation: highbd_jnt_convolve_avx2.c:convolve_12tap
63
64		static inline __m128i convolve_lo_x_12tap(const __m128i *s,
65		const __m128i *coeffs,
66	0	const __m128i zero) {
67	0	__m128i ss[6];
68	0	ss[0] = _mm_unpacklo_epi8(s[0], zero); // 0 1 1 2 2 3 3 4
69	0	ss[1] = _mm_unpacklo_epi8(s[1], zero); // 2 3 3 4 4 5 5 6
70	0	ss[2] = _mm_unpacklo_epi8(s[2], zero); // 4 5 5 6 6 7 7 8
71	0	ss[3] = _mm_unpacklo_epi8(s[3], zero); // 6 7 7 8 8 9 9 10
72	0	ss[4] = _mm_unpackhi_epi8(s[2], zero); // 8 9 9 10 10 11 11 12
73	0	ss[5] = _mm_unpackhi_epi8(s[3], zero); // 10 11 11 12 12 13 13 14
74	0	return convolve_12tap(ss, coeffs);
75	0	} Unexecuted instantiation: highbd_convolve_ssse3.c:convolve_lo_x_12tap Unexecuted instantiation: convolve_2d_sse2.c:convolve_lo_x_12tap Unexecuted instantiation: convolve_sse2.c:convolve_lo_x_12tap Unexecuted instantiation: highbd_convolve_2d_ssse3.c:convolve_lo_x_12tap Unexecuted instantiation: convolve_avx2.c:convolve_lo_x_12tap Unexecuted instantiation: jnt_convolve_avx2.c:convolve_lo_x_12tap Unexecuted instantiation: highbd_jnt_convolve_avx2.c:convolve_lo_x_12tap
76
77		static inline __m128i convolve_lo_y_12tap(const __m128i *s,
78	0	const __m128i *coeffs) {
79	0	__m128i ss[6];
80	0	const __m128i zero = _mm_setzero_si128();
81	0	ss[0] = _mm_unpacklo_epi8(s[0], zero);
82	0	ss[1] = _mm_unpacklo_epi8(s[2], zero);
83	0	ss[2] = _mm_unpacklo_epi8(s[4], zero);
84	0	ss[3] = _mm_unpacklo_epi8(s[6], zero);
85	0	ss[4] = _mm_unpacklo_epi8(s[8], zero);
86	0	ss[5] = _mm_unpacklo_epi8(s[10], zero);
87	0	return convolve_12tap(ss, coeffs);
88	0	} Unexecuted instantiation: highbd_convolve_ssse3.c:convolve_lo_y_12tap Unexecuted instantiation: convolve_2d_sse2.c:convolve_lo_y_12tap Unexecuted instantiation: convolve_sse2.c:convolve_lo_y_12tap Unexecuted instantiation: highbd_convolve_2d_ssse3.c:convolve_lo_y_12tap Unexecuted instantiation: convolve_avx2.c:convolve_lo_y_12tap Unexecuted instantiation: jnt_convolve_avx2.c:convolve_lo_y_12tap Unexecuted instantiation: highbd_jnt_convolve_avx2.c:convolve_lo_y_12tap
89
90		static inline __m128i convolve_hi_y_12tap(const __m128i *s,
91	0	const __m128i *coeffs) {
92	0	__m128i ss[6];
93	0	const __m128i zero = _mm_setzero_si128();
94	0	ss[0] = _mm_unpackhi_epi8(s[0], zero);
95	0	ss[1] = _mm_unpackhi_epi8(s[2], zero);
96	0	ss[2] = _mm_unpackhi_epi8(s[4], zero);
97	0	ss[3] = _mm_unpackhi_epi8(s[6], zero);
98	0	ss[4] = _mm_unpackhi_epi8(s[8], zero);
99	0	ss[5] = _mm_unpackhi_epi8(s[10], zero);
100	0	return convolve_12tap(ss, coeffs);
101	0	} Unexecuted instantiation: highbd_convolve_ssse3.c:convolve_hi_y_12tap Unexecuted instantiation: convolve_2d_sse2.c:convolve_hi_y_12tap Unexecuted instantiation: convolve_sse2.c:convolve_hi_y_12tap Unexecuted instantiation: highbd_convolve_2d_ssse3.c:convolve_hi_y_12tap Unexecuted instantiation: convolve_avx2.c:convolve_hi_y_12tap Unexecuted instantiation: jnt_convolve_avx2.c:convolve_hi_y_12tap Unexecuted instantiation: highbd_jnt_convolve_avx2.c:convolve_hi_y_12tap
102		#endif // AOM_AOM_DSP_X86_CONVOLVE_COMMON_INTRIN_H_