/*

 * Copyright (c) 2017, Alliance for Open Media. All rights reserved.

 *

 * This source code is subject to the terms of the BSD 2 Clause License and

 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License

 * was not distributed with this source code in the LICENSE file, you can

 * obtain it at www.aomedia.org/license/software. If the Alliance for Open

 * Media Patent License 1.0 was not distributed with this source code in the

 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.

 */

#include <immintrin.h>

#include <stdint.h>

#include "config/av1_rtcd.h"

#include "aom_dsp/x86/convolve_avx2.h"

#include "aom_dsp/aom_filter.h"

#include "aom_dsp/x86/synonyms.h"

#include "av1/common/convolve.h"

static void convolve_2d_sr_w4_avx2(

    const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride,

    int32_t w, int32_t h, const InterpFilterParams *filter_params_x,

    const InterpFilterParams *filter_params_y, const int32_t subpel_x_qn,

    const int32_t subpel_y_qn, ConvolveParams *conv_params) {

  int i;

  DECLARE_ALIGNED(32, int16_t, im_block[(MAX_SB_SIZE + MAX_FILTER_TAP) * 4]);

  uint8_t *dst_ptr = dst;

  assert(conv_params->round_0 == 3);

  assert(conv_params->round_1 == 11);

  const __m128i round_const_h = _mm_set1_epi16(1 << (conv_params->round_0 - 2));

  const __m256i round_const_v =

      _mm256_set1_epi32(1 << (conv_params->round_1 - 1));

  __m128i filt[2], coeffs_h[2] = { 0 };

  __m256i coeffs_v[4] = { 0 };

  const int horiz_tap = get_filter_tap(filter_params_x, subpel_x_qn);

  const int vert_tap = get_filter_tap(filter_params_y, subpel_y_qn);

  assert(horiz_tap == 2 || horiz_tap == 4);

  assert(vert_tap == 2 || vert_tap == 4 || vert_tap == 6 || vert_tap == 8);

  if (horiz_tap == 2)

    prepare_coeffs_2t_ssse3(filter_params_x, subpel_x_qn, coeffs_h);

  else

    prepare_coeffs_4t_ssse3(filter_params_x, subpel_x_qn, coeffs_h);

  if (vert_tap == 2)

    prepare_coeffs_2t(filter_params_y, subpel_y_qn, coeffs_v);

  else if (vert_tap == 4)

    prepare_coeffs_4t(filter_params_y, subpel_y_qn, coeffs_v);

  else if (vert_tap == 6)

    prepare_coeffs_6t(filter_params_y, subpel_y_qn, coeffs_v);

  else

    prepare_coeffs(filter_params_y, subpel_y_qn, coeffs_v);

  int im_h = h + vert_tap - 1;

  const int fo_vert = vert_tap / 2 - 1;

  const int fo_horiz = horiz_tap / 2 - 1;

  const uint8_t *const src_ptr = src - fo_vert * src_stride - fo_horiz;

  filt[0] = _mm_load_si128((__m128i const *)filt1_global_sse2);

  filt[1] = _mm_load_si128((__m128i const *)filt2_global_sse2);

  if (horiz_tap == 2) {

    CONVOLVE_SR_HOR_FILTER_2TAP_W4

  } else {

    CONVOLVE_SR_HOR_FILTER_4TAP_W4

  }

  if (vert_tap == 2) {

    CONVOLVE_SR_VER_FILTER_2TAP_W4

  } else if (vert_tap == 4) {

    CONVOLVE_SR_VER_FILTER_4TAP_W4

  } else if (vert_tap == 6) {

    CONVOLVE_SR_VER_FILTER_6TAP_W4

  } else {

    CONVOLVE_SR_VER_FILTER_8TAP_W4

  }

}

static void convolve_2d_sr_avx2(const uint8_t *src, int src_stride,

                                uint8_t *dst, int dst_stride, int w, int h,

                                const InterpFilterParams *filter_params_x,

                                const InterpFilterParams *filter_params_y,

                                const int subpel_x_qn, const int subpel_y_qn,

                                ConvolveParams *conv_params) {

  if (filter_params_x->taps > 8) {

    const int bd = 8;

    int im_stride = 8, i;

    const int strip_stride = (MAX_SB_SIZE + MAX_FILTER_TAP) * 8;

    DECLARE_ALIGNED(

        32, int16_t,

        im_block_buf[(MAX_SB_SIZE / 8) * (MAX_SB_SIZE + MAX_FILTER_TAP) * 8]);

    const int bits =

        FILTER_BITS * 2 - conv_params->round_0 - conv_params->round_1;

    const int offset_bits = bd + 2 * FILTER_BITS - conv_params->round_0;

    assert(conv_params->round_0 > 0);

    const __m256i round_const_h12 = _mm256_set1_epi32(

        ((1 << (conv_params->round_0)) >> 1) + (1 << (bd + FILTER_BITS - 1)));

    const __m128i round_shift_h12 = _mm_cvtsi32_si128(conv_params->round_0);

    const __m256i sum_round_v = _mm256_set1_epi32(

        (1 << offset_bits) + ((1 << conv_params->round_1) >> 1));

    const __m128i sum_shift_v = _mm_cvtsi32_si128(conv_params->round_1);

    const __m256i round_const_v = _mm256_set1_epi32(

        ((1 << bits) >> 1) - (1 << (offset_bits - conv_params->round_1)) -

        ((1 << (offset_bits - conv_params->round_1)) >> 1));

    const __m128i round_shift_v = _mm_cvtsi32_si128(bits);

    __m256i coeffs_h[6] = { 0 }, coeffs_v[6] = { 0 };

    int horiz_tap = 12;

    int vert_tap = 12;

    prepare_coeffs_12taps(filter_params_x, subpel_x_qn, coeffs_h);

    prepare_coeffs_12taps(filter_params_y, subpel_y_qn, coeffs_v);

    int im_h = h + vert_tap - 1;

    const int fo_vert = vert_tap / 2 - 1;

    const int fo_horiz = horiz_tap / 2 - 1;

    const uint8_t *const src_ptr = src - fo_vert * src_stride - fo_horiz;

    const __m256i v_zero = _mm256_setzero_si256();

    __m256i s[12];

    if (w <= 4) {

      for (i = 0; i < im_h; i += 2) {

        for (int j = 0; j < w; j += 8) {

          int16_t *strip_im_block = &im_block_buf[(j / 8) * strip_stride];

          const __m256i data = _mm256_permute2x128_si256(

              _mm256_castsi128_si256(

                  _mm_loadu_si128((__m128i *)(&src_ptr[i * src_stride + j]))),

              _mm256_castsi128_si256(_mm_loadu_si128(

                  (__m128i *)(&src_ptr[i * src_stride + src_stride + j]))),

              0x20);

          const __m256i s_16lo = _mm256_unpacklo_epi8(data, v_zero);

          const __m256i s_16hi = _mm256_unpackhi_epi8(data, v_zero);

          const __m256i s_lolo = _mm256_unpacklo_epi16(s_16lo, s_16lo);

          const __m256i s_lohi = _mm256_unpackhi_epi16(s_16lo, s_16lo);

          const __m256i s_hilo = _mm256_unpacklo_epi16(s_16hi, s_16hi);

          const __m256i s_hihi = _mm256_unpackhi_epi16(s_16hi, s_16hi);

          s[0] = _mm256_alignr_epi8(s_lohi, s_lolo, 2);

          s[1] = _mm256_alignr_epi8(s_lohi, s_lolo, 10);

          s[2] = _mm256_alignr_epi8(s_hilo, s_lohi, 2);

          s[3] = _mm256_alignr_epi8(s_hilo, s_lohi, 10);

          s[4] = _mm256_alignr_epi8(s_hihi, s_hilo, 2);

          s[5] = _mm256_alignr_epi8(s_hihi, s_hilo, 10);

          const __m256i res_lo = convolve_12taps(s, coeffs_h);

          __m256i res_32b_lo = _mm256_sra_epi32(

              _mm256_add_epi32(res_lo, round_const_h12), round_shift_h12);

          __m256i res_16b_lo = _mm256_packs_epi32(res_32b_lo, res_32b_lo);

          const __m128i res_0 = _mm256_extracti128_si256(res_16b_lo, 0);

          const __m128i res_1 = _mm256_extracti128_si256(res_16b_lo, 1);

          if (w > 2) {

            _mm_storel_epi64((__m128i *)&strip_im_block[i * im_stride], res_0);

            _mm_storel_epi64(

                (__m128i *)&strip_im_block[i * im_stride + im_stride], res_1);

          } else {

            uint32_t horiz_2;

            horiz_2 = (uint32_t)_mm_cvtsi128_si32(res_0);

            strip_im_block[i * im_stride] = (uint16_t)horiz_2;

            strip_im_block[i * im_stride + 1] = (uint16_t)(horiz_2 >> 16);

            horiz_2 = (uint32_t)_mm_cvtsi128_si32(res_1);

            strip_im_block[i * im_stride + im_stride] = (uint16_t)horiz_2;

            strip_im_block[i * im_stride + im_stride + 1] =

                (uint16_t)(horiz_2 >> 16);

          }

        }

      }

    } else {

      for (i = 0; i < im_h; i++) {

        for (int j = 0; j < w; j += 8) {

          int16_t *strip_im_block = &im_block_buf[(j / 8) * strip_stride];

          const __m256i data = _mm256_permute2x128_si256(

              _mm256_castsi128_si256(

                  _mm_loadu_si128((__m128i *)(&src_ptr[i * src_stride + j]))),

              _mm256_castsi128_si256(_mm_loadu_si128(

                  (__m128i *)(&src_ptr[i * src_stride + j + 4]))),

              0x20);

          const __m256i s_16lo = _mm256_unpacklo_epi8(data, v_zero);

          const __m256i s_16hi = _mm256_unpackhi_epi8(data, v_zero);

          const __m256i s_lolo = _mm256_unpacklo_epi16(s_16lo, s_16lo);

          const __m256i s_lohi = _mm256_unpackhi_epi16(s_16lo, s_16lo);

          const __m256i s_hilo = _mm256_unpacklo_epi16(s_16hi, s_16hi);

          const __m256i s_hihi = _mm256_unpackhi_epi16(s_16hi, s_16hi);

          s[0] = _mm256_alignr_epi8(s_lohi, s_lolo, 2);

          s[1] = _mm256_alignr_epi8(s_lohi, s_lolo, 10);

          s[2] = _mm256_alignr_epi8(s_hilo, s_lohi, 2);

          s[3] = _mm256_alignr_epi8(s_hilo, s_lohi, 10);

          s[4] = _mm256_alignr_epi8(s_hihi, s_hilo, 2);

          s[5] = _mm256_alignr_epi8(s_hihi, s_hilo, 10);

          const __m256i res_lo = convolve_12taps(s, coeffs_h);

          __m256i res_32b_lo = _mm256_sra_epi32(

              _mm256_add_epi32(res_lo, round_const_h12), round_shift_h12);

          __m256i res_16b_lo = _mm256_packs_epi32(res_32b_lo, res_32b_lo);

          _mm_store_si128((__m128i *)&strip_im_block[i * im_stride],

                          _mm256_extracti128_si256(

                              _mm256_permute4x64_epi64(res_16b_lo, 0x88), 0));

        }

      }

    }

    for (int j = 0; j < w; j += 8) {

      const int16_t *im_block = &im_block_buf[(j / 8) * strip_stride];

      CONVOLVE_SR_VERTICAL_FILTER_12TAP

    }

  } else {

    int im_stride = 8, i;

    const int strip_stride = (MAX_SB_SIZE + MAX_FILTER_TAP) * 8;

    DECLARE_ALIGNED(

        32, int16_t,

        im_block_buf[(MAX_SB_SIZE / 8) * (MAX_SB_SIZE + MAX_FILTER_TAP) * 8]);

    assert(conv_params->round_0 == 3);

    assert(conv_params->round_1 == 11);

    const __m256i round_const_h =

        _mm256_set1_epi16(1 << (conv_params->round_0 - 2));

    const __m256i round_const_v =

        _mm256_set1_epi32(1 << (conv_params->round_1 - 1));

    __m256i filt[4], coeffs_h[4] = { 0 }, coeffs_v[4] = { 0 };

    int horiz_tap = get_filter_tap(filter_params_x, subpel_x_qn);

    int vert_tap = get_filter_tap(filter_params_y, subpel_y_qn);

    assert(horiz_tap == 2 || horiz_tap == 4 || horiz_tap == 6 ||

           horiz_tap == 8);

    assert(vert_tap == 2 || vert_tap == 4 || vert_tap == 6 || vert_tap == 8);

    if (horiz_tap == 2)

      prepare_coeffs_2t_lowbd(filter_params_x, subpel_x_qn, coeffs_h);

    else if (horiz_tap == 4)

      prepare_coeffs_4t_lowbd(filter_params_x, subpel_x_qn, coeffs_h);

    else if (horiz_tap == 6)

      prepare_coeffs_6t_lowbd(filter_params_x, subpel_x_qn, coeffs_h);

    else

      prepare_coeffs_lowbd(filter_params_x, subpel_x_qn, coeffs_h);

    if (vert_tap == 2)

      prepare_coeffs_2t(filter_params_y, subpel_y_qn, coeffs_v);

    else if (vert_tap == 4)

      prepare_coeffs_4t(filter_params_y, subpel_y_qn, coeffs_v);

    else if (vert_tap == 6)

      prepare_coeffs_6t(filter_params_y, subpel_y_qn, coeffs_v);

    else

      prepare_coeffs(filter_params_y, subpel_y_qn, coeffs_v);

    int im_h = h + vert_tap - 1;

    const int fo_vert = vert_tap / 2 - 1;

    const int fo_horiz = horiz_tap / 2 - 1;

    const uint8_t *const src_ptr = src - fo_vert * src_stride - fo_horiz;

    filt[0] = _mm256_load_si256((__m256i const *)filt1_global_avx2);

    filt[1] = _mm256_load_si256((__m256i const *)filt2_global_avx2);

    filt[2] = _mm256_load_si256((__m256i const *)filt3_global_avx2);

    filt[3] = _mm256_load_si256((__m256i const *)filt4_global_avx2);

    for (i = 0; i < (im_h - 1); i += 2) {

      const uint8_t *src_row0 = &src_ptr[i * src_stride];

      const uint8_t *src_row1 = &src_ptr[(i + 1) * src_stride];

      for (int j = 0; j < w; j += 8) {

        int16_t *strip_im_block = &im_block_buf[(j / 8) * strip_stride];

        __m256i data =

            _mm256_castsi128_si256(_mm_loadu_si128((__m128i *)&src_row0[j]));

        data = _mm256_inserti128_si256(

            data, _mm_loadu_si128((__m128i *)&src_row1[j]), 1);

        __m256i res;

        if (horiz_tap == 2)

          res = convolve_lowbd_x_2tap(data, coeffs_h, filt);

        else if (horiz_tap == 4)

          res = convolve_lowbd_x_4tap(data, coeffs_h, filt);

        else if (horiz_tap == 6)

          res = convolve_lowbd_x_6tap(data, coeffs_h, filt);

        else

          res = convolve_lowbd_x(data, coeffs_h, filt);

        res = _mm256_srai_epi16(_mm256_add_epi16(res, round_const_h), 2);

        _mm256_store_si256((__m256i *)&strip_im_block[i * 8], res);

      }

    }

    {

      const uint8_t *src_row0 = &src_ptr[i * src_stride];

      for (int j = 0; j < w; j += 8) {

        int16_t *strip_im_block = &im_block_buf[(j / 8) * strip_stride];

        __m256i data_1 =

            _mm256_castsi128_si256(_mm_loadu_si128((__m128i *)&src_row0[j]));

        __m256i res;

        if (horiz_tap == 2)

          res = convolve_lowbd_x_2tap(data_1, coeffs_h, filt);

        else if (horiz_tap == 4)

          res = convolve_lowbd_x_4tap(data_1, coeffs_h, filt);

        else if (horiz_tap == 6)

          res = convolve_lowbd_x_6tap(data_1, coeffs_h, filt);

        else

          res = convolve_lowbd_x(data_1, coeffs_h, filt);

        res = _mm256_srai_epi16(_mm256_add_epi16(res, round_const_h), 2);

        _mm_store_si128((__m128i *)&strip_im_block[i * 8],

                        _mm256_castsi256_si128(res));

      }

    }

    for (int j = 0; j < w; j += 8) {

      const int16_t *im_block = &im_block_buf[(j / 8) * strip_stride];

      uint8_t *dst_ptr = dst + j;

      if (vert_tap == 2) {

        CONVOLVE_SR_VERTICAL_FILTER_2TAP

      } else if (vert_tap == 4) {

        CONVOLVE_SR_VERTICAL_FILTER_4TAP

      } else if (vert_tap == 6) {

        CONVOLVE_SR_VERTICAL_FILTER_6TAP

      } else {

        CONVOLVE_SR_VERTICAL_FILTER_8TAP

      }

    }

  }

}

void av1_convolve_2d_sr_avx2(

    const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride,

    int32_t w, int32_t h, const InterpFilterParams *filter_params_x,

    const InterpFilterParams *filter_params_y, const int32_t subpel_x_qn,

    const int32_t subpel_y_qn, ConvolveParams *conv_params) {

  const int32_t tap_x = get_filter_tap(filter_params_x, subpel_x_qn);

  const int32_t tap_y = get_filter_tap(filter_params_y, subpel_y_qn);

  const bool use_12tap = (tap_x == 12 || tap_y == 12);

  if (w <= 4 && !use_12tap) {

    convolve_2d_sr_w4_avx2(src, src_stride, dst, dst_stride, w, h,

                           filter_params_x, filter_params_y, subpel_x_qn,

                           subpel_y_qn, conv_params);

  } else {

    convolve_2d_sr_avx2(src, src_stride, dst, dst_stride, w, h, filter_params_x,

                        filter_params_y, subpel_x_qn, subpel_y_qn, conv_params);

  }

}