/* -----------------------------------------------------------------------------

The copyright in this software is being made available under the Clear BSD

License, included below. No patent rights, trademark rights and/or 

other Intellectual Property Rights other than the copyrights concerning 

the Software are granted under this license.

The Clear BSD License

Copyright (c) 2019-2026, Fraunhofer-Gesellschaft zur Förderung der angewandten Forschung e.V. & The VVenC Authors.

All rights reserved.

Redistribution and use in source and binary forms, with or without modification,

are permitted (subject to the limitations in the disclaimer below) provided that

the following conditions are met:

     * Redistributions of source code must retain the above copyright notice,

     this list of conditions and the following disclaimer.

     * Redistributions in binary form must reproduce the above copyright

     notice, this list of conditions and the following disclaimer in the

     documentation and/or other materials provided with the distribution.

     * Neither the name of the copyright holder nor the names of its

     contributors may be used to endorse or promote products derived from this

     software without specific prior written permission.

NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY

THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND

CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT

LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A

PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR

CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,

EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,

PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR

BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER

IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)

ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

POSSIBILITY OF SUCH DAMAGE.

------------------------------------------------------------------------------------------- */

/** \file     YuvX86.cpp

    \brief    SIMD averaging.

*/

#pragma once

#define DONT_UNDEF_SIZE_AWARE_PER_EL_OP 1

#include "CommonDefX86.h"

#include "Unit.h"

#include "InterpolationFilter.h"

#ifdef TARGET_SIMD_X86

#if ENABLE_SIMD_OPT_BUFFER

//! \ingroup CommonLib

//! \{

namespace vvenc {

#if USE_AVX2

template<bool isAligned> static inline __m256i load_aligned_avx2       ( const void* addr );

template<>                      inline __m256i load_aligned_avx2<true> ( const void* addr ) { return _mm256_load_si256 ( (const __m256i *) addr );}

template<>                      inline __m256i load_aligned_avx2<false>( const void* addr ) { return _mm256_loadu_si256( (const __m256i *) addr );}

#endif

template<bool isAligned> static inline __m128i load_aligned       ( const void* addr );

template<>                      inline __m128i load_aligned<true> ( const void* addr ) { return _mm_load_si128 ( (const __m128i *) addr );}

Unexecuted instantiation: Buffer_sse41.cpp:long long __vector(2) vvenc::load_aligned<true>(void const*)

Unexecuted instantiation: Buffer_avx2.cpp:long long __vector(2) vvenc::load_aligned<true>(void const*)

template<>                      inline __m128i load_aligned<false>( const void* addr ) { return _mm_loadu_si128( (const __m128i *) addr );}

Unexecuted instantiation: Buffer_sse41.cpp:long long __vector(2) vvenc::load_aligned<false>(void const*)

Unexecuted instantiation: Buffer_avx2.cpp:long long __vector(2) vvenc::load_aligned<false>(void const*)

template< X86_VEXT vext >

void weightCiip_SSE( Pel* res, const Pel* src, const int numSamples, int numIntra )

{

#if USE_AVX2

  int n = 16;

  if( numIntra == 1 )

  {

    __m256i vres;

    __m256i vpred = _mm256_load_si256((const __m256i*)&res[0]);

    __m256i vsrc  = _mm256_load_si256((const __m256i*)&src[0]);

    for( ; n < numSamples; n+=16)

    {

      vres = _mm256_avg_epu16( vpred, vsrc );

      vpred = _mm256_load_si256((const __m256i*)&res[n]);

      vsrc  = _mm256_load_si256((const __m256i*)&src[n]);

      _mm256_storeu_si256( ( __m256i * )&res[n-16], vres );

    }

    vres = _mm256_avg_epu16( vpred, vsrc );

    _mm256_storeu_si256( ( __m256i * )&res[n-16], vres );

  }

  else

  {

    const Pel* scale   = ( numIntra == 0 ) ? res : src;

    const Pel* unscale = ( numIntra == 0 ) ? src : res;

    __m256i vres;

    __m256i voffset = _mm256_set1_epi16(2);

    __m256i vscl = _mm256_load_si256((const __m256i*)&scale[0]);

    __m256i vuns = _mm256_load_si256((const __m256i*)&unscale[0]);

    for( ; n < numSamples; n+=16)

    {

      vres = _mm256_srai_epi16( _mm256_adds_epi16( _mm256_adds_epi16(_mm256_adds_epi16( vscl, vscl),_mm256_adds_epi16( vscl, vuns)), voffset), 2 );

      vscl = _mm256_load_si256((const __m256i*)&scale[n]);

      vuns = _mm256_load_si256((const __m256i*)&unscale[n]);

      _mm256_storeu_si256( ( __m256i * )&res[n-16], vres );

    }

    vres = _mm256_srai_epi16( _mm256_adds_epi16( _mm256_adds_epi16(_mm256_adds_epi16( vscl, vscl),_mm256_adds_epi16( vscl, vuns)), voffset), 2 );

    _mm256_storeu_si256( ( __m256i * )&res[n-16], vres );

  }

#else

  int n = 8;

  if( numIntra == 1 )

  {

    __m128i vres;

    __m128i vpred = _mm_load_si128((const __m128i*)&res[0]);

    __m128i vsrc  = _mm_load_si128((const __m128i*)&src[0]);

    for( ; n < numSamples; n+=8)

    {

      vres = _mm_avg_epu16( vpred, vsrc );

      vpred = _mm_load_si128((const __m128i*)&res[n]);

      vsrc  = _mm_load_si128((const __m128i*)&src[n]);

      _mm_storeu_si128( ( __m128i * )&res[n-8], vres );

    }

    vres = _mm_avg_epu16( vpred, vsrc );

    _mm_storeu_si128( ( __m128i * )&res[n-8], vres );

  }

  else

  {

    const Pel* scale   = ( numIntra == 0 ) ? res : src;

    const Pel* unscale = ( numIntra == 0 ) ? src : res;

    __m128i vres;

    __m128i voffset = _mm_set1_epi16(2);

    __m128i vscl = _mm_load_si128((const __m128i*)&scale[0]);

    __m128i vuns = _mm_load_si128((const __m128i*)&unscale[0]);

    for( ; n < numSamples; n+=8)

    {

      vres = _mm_srai_epi16( _mm_adds_epi16( _mm_adds_epi16(_mm_adds_epi16( vscl, vscl),_mm_adds_epi16( vscl, vuns)), voffset), 2 );

      vscl = _mm_load_si128((const __m128i*)&scale[n]);

      vuns = _mm_load_si128((const __m128i*)&unscale[n]);

      _mm_storeu_si128( ( __m128i * )&res[n-8], vres );

    }

    vres = _mm_srai_epi16( _mm_adds_epi16( _mm_adds_epi16(_mm_adds_epi16( vscl, vscl),_mm_adds_epi16( vscl, vuns)), voffset), 2 );

    _mm_storeu_si128( ( __m128i * )&res[n-8], vres );

  }

#endif

}

Unexecuted instantiation: void vvenc::weightCiip_SSE<(vvenc::x86_simd::X86_VEXT)1>(short*, short const*, int, int)

Unexecuted instantiation: void vvenc::weightCiip_SSE<(vvenc::x86_simd::X86_VEXT)4>(short*, short const*, int, int)

template< X86_VEXT vext, unsigned inputSize, unsigned outputSize >

void mipMatrixMul_SSE( Pel* res, const Pel* input, const uint8_t* weight, const int maxVal, const int inputOffset, bool transpose )

{

  int sum = 0;

  for( int i = 0; i < inputSize; i++ ) { sum += input[i]; }

  const int offset = (1 << (MIP_SHIFT_MATRIX - 1)) - MIP_OFFSET_MATRIX * sum + (inputOffset << MIP_SHIFT_MATRIX);

  CHECK( inputSize != 4 * (inputSize >> 2), "Error, input size not divisible by four" );

#if USE_AVX2

#if !ENABLE_VALGRIND_CODE

  static

#endif

  const __m256i perm = _mm256_setr_epi32(0,4,1,5,2,6,3,7);

  __m256i vibdimin  = _mm256_set1_epi16( 0 );

  __m256i vibdimax  = _mm256_set1_epi16( maxVal );

  if( inputSize == 4 && outputSize == 4)

  {

    __m256i voffset   = _mm256_set1_epi32( offset );

    __m256i vin = _mm256_set1_epi64x( *((const int64_t*)input) );

    __m256i vw = _mm256_load_si256((const __m256i*)(weight));

    __m256i w0 = _mm256_cvtepi8_epi16( _mm256_castsi256_si128( vw ) );   //w0 - w16

    __m256i w1 = _mm256_cvtepi8_epi16( _mm256_extracti128_si256( vw, 1 ) ); //w16 -w 32

    __m256i r0 = _mm256_madd_epi16( vin, w0 );

    __m256i r1 = _mm256_madd_epi16( vin, w1 );

    __m256i r2 = _mm256_hadd_epi32( r0 , r1);

            r2 = _mm256_add_epi32( r2, voffset );

    __m256i r3 = _mm256_srai_epi32( r2, MIP_SHIFT_MATRIX );

            vw = _mm256_load_si256((const __m256i*)(weight+32));

            w0 = _mm256_cvtepi8_epi16( _mm256_castsi256_si128( vw ) );   //w0 - w16

            w1 = _mm256_cvtepi8_epi16( _mm256_extracti128_si256( vw, 1 ) ); //w16 -w 32

            r0 = _mm256_madd_epi16( vin, w0 );

            r1 = _mm256_madd_epi16( vin, w1 );

            r2 = _mm256_hadd_epi32( r0 , r1);

            r2 = _mm256_add_epi32( r2, voffset );

            r2 = _mm256_srai_epi32( r2, MIP_SHIFT_MATRIX );

            r2 = _mm256_packs_epi32( r3, r2 );

            r2 = _mm256_permutevar8x32_epi32 ( r2, perm );

            r2 = _mm256_min_epi16( vibdimax, _mm256_max_epi16( vibdimin, r2 ) );

      if( transpose )

      {

        __m256i vshuf0 = _mm256_set_epi8( 0xf, 0xe, 0xb, 0xa, 0x7, 0x6, 0x3, 0x2, 0xd, 0xc, 0x9, 0x8, 0x5, 0x4, 0x1, 0x0,

                                      0xf, 0xe, 0xb, 0xa, 0x7, 0x6, 0x3, 0x2, 0xd, 0xc, 0x9, 0x8, 0x5, 0x4, 0x1, 0x0);

         r2 = _mm256_permutevar8x32_epi32( r2, _mm256_set_epi32(7,5,3,1,6,4,2,0) );

         r2 = _mm256_shuffle_epi8 ( r2, vshuf0);

      }

      _mm256_store_si256( ( __m256i * )&res[0], r2 );

  }

  else if( inputSize == 8 )

  {

    __m256i voffset   = _mm256_set1_epi32( offset );

    __m128i inv =_mm_load_si128( ( __m128i* )input );

    __m256i vin = _mm256_permute2f128_si256(_mm256_castsi128_si256(inv), _mm256_castsi128_si256(inv), 2); 

    __m256i r2;

    for( int i = 0; i < outputSize*outputSize; i+=16)

    {

      __m256i vw = _mm256_load_si256((const __m256i*)(weight));

      __m256i w0 = _mm256_cvtepi8_epi16( _mm256_castsi256_si128( vw ) );   //w0 - w16

      __m256i w1 = _mm256_cvtepi8_epi16( _mm256_extracti128_si256( vw, 1 ) ); //w16 -w 32

      __m256i r0 = _mm256_madd_epi16( vin, w0 );

      __m256i r1 = _mm256_madd_epi16( vin, w1 );

              r2 = _mm256_hadd_epi32( r0 , r1);

              vw = _mm256_load_si256((const __m256i*)(weight+32));

              w0 = _mm256_cvtepi8_epi16( _mm256_castsi256_si128( vw ) );   //w0 - w16

              w1 = _mm256_cvtepi8_epi16( _mm256_extracti128_si256( vw, 1 ) ); //w16 -w 32

              r0 = _mm256_madd_epi16( vin, w0 );

              r1 = _mm256_madd_epi16( vin, w1 );

      __m256i r4 = _mm256_hadd_epi32( r0 , r1);

              r2 = _mm256_hadd_epi32( r2 , r4);

              r2 = _mm256_add_epi32( r2, voffset );

      __m256i r3 = _mm256_srai_epi32( r2, MIP_SHIFT_MATRIX );

              vw = _mm256_load_si256((const __m256i*)(weight+64));

              w0 = _mm256_cvtepi8_epi16( _mm256_castsi256_si128( vw ) );   //w0 - w16

              w1 = _mm256_cvtepi8_epi16( _mm256_extracti128_si256( vw, 1 ) ); //w16 -w 32

              r0 = _mm256_madd_epi16( vin, w0 );

              r1 = _mm256_madd_epi16( vin, w1 );

              r2 = _mm256_hadd_epi32( r0 , r1);

              vw = _mm256_load_si256((const __m256i*)(weight+96));

              w0 = _mm256_cvtepi8_epi16( _mm256_castsi256_si128( vw ) );   //w0 - w16

              w1 = _mm256_cvtepi8_epi16( _mm256_extracti128_si256( vw, 1 ) ); //w16 -w 32

              r0 = _mm256_madd_epi16( vin, w0 );

              r1 = _mm256_madd_epi16( vin, w1 );

              r2 = _mm256_hadd_epi32( r2 , _mm256_hadd_epi32( r0 , r1));

              r2 = _mm256_add_epi32( r2, voffset );

              r2 = _mm256_srai_epi32( r2, MIP_SHIFT_MATRIX );

              r2 = _mm256_permutevar8x32_epi32 ( r2, perm );

              r3 = _mm256_permutevar8x32_epi32 ( r3, perm );

              r3 = _mm256_packs_epi32( r3, r2 );

              r2 = _mm256_permute4x64_epi64( r3, 0xd8 );

              r2 = _mm256_min_epi16( vibdimax, _mm256_max_epi16( vibdimin, r2 ) );

        _mm256_store_si256( ( __m256i * )&res[0], r2 );

        res+=16;

        weight+=128;

    }

    if( transpose )

    {

      if( outputSize == 4 )

      {

        __m256i vshuf0 = _mm256_set_epi8( 0xf, 0xe, 0xb, 0xa, 0x7, 0x6, 0x3, 0x2, 0xd, 0xc, 0x9, 0x8, 0x5, 0x4, 0x1, 0x0,

                                      0xf, 0xe, 0xb, 0xa, 0x7, 0x6, 0x3, 0x2, 0xd, 0xc, 0x9, 0x8, 0x5, 0x4, 0x1, 0x0);

         r2 = _mm256_permutevar8x32_epi32( r2, _mm256_set_epi32(7,5,3,1,6,4,2,0) );

         r2 = _mm256_shuffle_epi8 ( r2, vshuf0);

        _mm256_store_si256( ( __m256i * )(res-16), r2 );

      }

      else

      {

        res -= 64;

        __m256i va, vb, vc, vd, wa, wb, wc, wd;

        va = _mm256_load_si256( ( const __m256i* ) res ); 

        vb = _mm256_load_si256( ( const __m256i* ) (res+16) ); 

        vc = _mm256_load_si256( ( const __m256i* ) (res+32) ); 

        va =_mm256_permute4x64_epi64(va, 0xd8); 

        vb =_mm256_permute4x64_epi64(vb, 0xd8);

        vc =_mm256_permute4x64_epi64(vc, 0xd8);

        vd =_mm256_permute4x64_epi64(r2, 0xd8);

        wa = _mm256_unpacklo_epi16( va, vb );

        wb = _mm256_unpackhi_epi16( va, vb );

        wc = _mm256_unpacklo_epi16( vc, vd );

        wd = _mm256_unpackhi_epi16( vc, vd );

        va = _mm256_unpacklo_epi16( wa, wb );

        vb = _mm256_unpackhi_epi16( wa, wb );

        vc = _mm256_unpacklo_epi16( wc, wd );

        vd = _mm256_unpackhi_epi16( wc, wd );

        va =_mm256_permute4x64_epi64(va, 0xd8); 

        vb =_mm256_permute4x64_epi64(vb, 0xd8);

        vc =_mm256_permute4x64_epi64(vc, 0xd8);

        vd =_mm256_permute4x64_epi64(vd, 0xd8);

        wa = _mm256_unpacklo_epi64( va, vc );

        wb = _mm256_unpacklo_epi64( vb, vd );

        wc = _mm256_unpackhi_epi64( va, vc );

        wd = _mm256_unpackhi_epi64( vb, vd );

        _mm256_store_si256( ( __m256i* ) res, wa ); 

        _mm256_store_si256( ( __m256i* ) (res+16), wb );

        _mm256_store_si256( ( __m256i* ) (res+32), wc );

        _mm256_store_si256( ( __m256i* ) (res+48), wd );

      }

    }

  }

#else

  __m128i zero  = _mm_set1_epi16( 0 );

  __m128i vibdimax  = _mm_set1_epi16( maxVal );

  if( inputSize == 4 && outputSize == 4)

  {

    __m128i vin = _mm_set1_epi64x( *((const int64_t*)input) );

    __m128i voffset = _mm_set1_epi32( offset );

    __m128i r2 = vin;

    __m128i r;

    for( int i = 0; i < 2; i++)

    {

             r = r2; // save the result from the first interation

    __m128i vw = _mm_load_si128((const __m128i*)weight);

    __m128i w0 = _mm_unpacklo_epi8( vw, zero );

    __m128i w1 = _mm_unpackhi_epi8( vw, zero );

    __m128i r0 = _mm_madd_epi16( vin, w0 );

    __m128i r1 = _mm_madd_epi16( vin, w1 );

            r2 = _mm_hadd_epi32( r0 , r1);

            r2 = _mm_add_epi32( r2, voffset );

    __m128i r3 = _mm_srai_epi32( r2, MIP_SHIFT_MATRIX );

            vw = _mm_load_si128((const __m128i*)(weight+16));

            w0 = _mm_unpacklo_epi8( vw, zero );

            w1 = _mm_unpackhi_epi8( vw, zero );

            r0 = _mm_madd_epi16( vin, w0 );

            r1 = _mm_madd_epi16( vin, w1 );

            r2 = _mm_hadd_epi32( r0 , r1);

            r2 = _mm_add_epi32( r2, voffset );

            r2 = _mm_srai_epi32( r2, MIP_SHIFT_MATRIX );

            r2 = _mm_packs_epi32( r3, r2 );

            r2 = _mm_min_epi16( vibdimax, _mm_max_epi16( zero, r2 ) );

      _mm_store_si128( ( __m128i * )&res[0], r2 );

      res +=8;

      weight += 32;

    }

    if( transpose)

    {

      __m128i vc, vd, va, vb;

      vc = _mm_unpacklo_epi16( r, r2 );

      vd = _mm_unpackhi_epi16( r, r2 );

      va = _mm_unpacklo_epi16( vc, vd );

      vb = _mm_unpackhi_epi16( vc, vd );

      _mm_store_si128( ( __m128i* ) (res-16), va ); 

      _mm_store_si128( ( __m128i* ) (res-8), vb ); 

    }

  }

  else

  {

    __m128i vin = _mm_load_si128( (const __m128i*)input);

    __m128i voffset = _mm_set1_epi32( offset );

    for( int i = 0; i < outputSize*outputSize; i+=4)

    {

    __m128i vw = _mm_load_si128((const __m128i*)(weight));

    __m128i w0 = _mm_unpacklo_epi8( vw, zero );

    __m128i w1 = _mm_unpackhi_epi8( vw, zero );

    __m128i r0 = _mm_madd_epi16( vin, w0 );

    __m128i r1 = _mm_madd_epi16( vin, w1 );

    __m128i r2 = _mm_hadd_epi32( r0 , r1);

            vw = _mm_load_si128((const __m128i*)(weight+16));

            w0 = _mm_unpacklo_epi8( vw, zero );

            w1 = _mm_unpackhi_epi8( vw, zero );

            r0 = _mm_madd_epi16( vin, w0 );

            r1 = _mm_madd_epi16( vin, w1 );

            r2 = _mm_hadd_epi32( r2 , _mm_hadd_epi32( r0 , r1));

            r2 = _mm_add_epi32( r2, voffset );

            r2 = _mm_srai_epi32( r2, MIP_SHIFT_MATRIX );

            r2 = _mm_packs_epi32( r2, r2 );

            r2 = _mm_min_epi16( vibdimax, _mm_max_epi16( zero, r2 ) );

      _vv_storel_epi64( ( __m128i * )&res[0], r2 );

      res +=4;

      weight += 32;

    }

    if( transpose )

    {

      if( outputSize == 4)

      {

        res -= 16;

        __m128i vc, vd, va, vb;

        va = _mm_load_si128( ( const __m128i* ) (res) );

        vb = _mm_load_si128( ( const __m128i* ) (res+8) );

        vc = _mm_unpacklo_epi16( va, vb );

        vd = _mm_unpackhi_epi16( va, vb );

        va = _mm_unpacklo_epi16( vc, vd );

        vb = _mm_unpackhi_epi16( vc, vd );

        _mm_store_si128( ( __m128i* ) (res), va ); 

        _mm_store_si128( ( __m128i* ) (res+8), vb ); 

      }

      else

      {

        res -= 64;

        __m128i va, vb, vc, vd, ve, vf, vg, vh;

        va = _mm_load_si128( ( const __m128i* ) (res) );

        vb = _mm_load_si128( ( const __m128i* ) (res+8) );

        vc = _mm_load_si128( ( const __m128i* ) (res+16) );

        vd = _mm_load_si128( ( const __m128i* ) (res+24) );

        ve = _mm_load_si128( ( const __m128i* ) (res+32) );

        vf = _mm_load_si128( ( const __m128i* ) (res+40) );

        vg = _mm_load_si128( ( const __m128i* ) (res+48) );

        vh = _mm_load_si128( ( const __m128i* ) (res+56) );

        __m128i va01b01 = _mm_unpacklo_epi16( va, vb );

        __m128i va23b23 = _mm_unpackhi_epi16( va, vb );

        __m128i vc01d01 = _mm_unpacklo_epi16( vc, vd );

        __m128i vc23d23 = _mm_unpackhi_epi16( vc, vd );

        __m128i ve01f01 = _mm_unpacklo_epi16( ve, vf );

        __m128i ve23f23 = _mm_unpackhi_epi16( ve, vf );

        __m128i vg01h01 = _mm_unpacklo_epi16( vg, vh );

        __m128i vg23h23 = _mm_unpackhi_epi16( vg, vh );

        va = _mm_unpacklo_epi32( va01b01, vc01d01 );

        vb = _mm_unpackhi_epi32( va01b01, vc01d01 );

        vc = _mm_unpacklo_epi32( va23b23, vc23d23 );

        vd = _mm_unpackhi_epi32( va23b23, vc23d23 );

        ve = _mm_unpacklo_epi32( ve01f01, vg01h01 );

        vf = _mm_unpackhi_epi32( ve01f01, vg01h01 );

        vg = _mm_unpacklo_epi32( ve23f23, vg23h23 );

        vh = _mm_unpackhi_epi32( ve23f23, vg23h23 );

        va01b01 = _mm_unpacklo_epi64( va, ve );

        va23b23 = _mm_unpackhi_epi64( va, ve );

        vc01d01 = _mm_unpacklo_epi64( vb, vf );

        vc23d23 = _mm_unpackhi_epi64( vb, vf );

        ve01f01 = _mm_unpacklo_epi64( vc, vg );

        ve23f23 = _mm_unpackhi_epi64( vc, vg );

        vg01h01 = _mm_unpacklo_epi64( vd, vh );

        vg23h23 = _mm_unpackhi_epi64( vd, vh );

        _mm_store_si128( ( __m128i* ) (res),    va01b01 );

        _mm_store_si128( ( __m128i* ) (res+8) , va23b23 );

        _mm_store_si128( ( __m128i* ) (res+16), vc01d01 );

        _mm_store_si128( ( __m128i* ) (res+24), vc23d23 );

        _mm_store_si128( ( __m128i* ) (res+32), ve01f01 );

        _mm_store_si128( ( __m128i* ) (res+40), ve23f23 );

        _mm_store_si128( ( __m128i* ) (res+48), vg01h01 );

        _mm_store_si128( ( __m128i* ) (res+56), vg23h23 );

      }

    }

  }

#endif

}

Unexecuted instantiation: void vvenc::mipMatrixMul_SSE<(vvenc::x86_simd::X86_VEXT)1, 4u, 4u>(short*, short const*, unsigned char const*, int, int, bool)

Unexecuted instantiation: void vvenc::mipMatrixMul_SSE<(vvenc::x86_simd::X86_VEXT)1, 8u, 4u>(short*, short const*, unsigned char const*, int, int, bool)

Unexecuted instantiation: void vvenc::mipMatrixMul_SSE<(vvenc::x86_simd::X86_VEXT)1, 8u, 8u>(short*, short const*, unsigned char const*, int, int, bool)

Unexecuted instantiation: void vvenc::mipMatrixMul_SSE<(vvenc::x86_simd::X86_VEXT)4, 4u, 4u>(short*, short const*, unsigned char const*, int, int, bool)

Unexecuted instantiation: void vvenc::mipMatrixMul_SSE<(vvenc::x86_simd::X86_VEXT)4, 8u, 4u>(short*, short const*, unsigned char const*, int, int, bool)

Unexecuted instantiation: void vvenc::mipMatrixMul_SSE<(vvenc::x86_simd::X86_VEXT)4, 8u, 8u>(short*, short const*, unsigned char const*, int, int, bool)

template< X86_VEXT vext>

void addAvg_SSE( const Pel* src0, const Pel* src1, Pel* dst, int numSamples, unsigned shift, int offset, const ClpRng& clpRng )

{

#if USE_AVX2

  if( numSamples >= 16 )

  {

    const __m256i voffset   = _mm256_set1_epi32( offset );

    const __m256i vibdimin  = _mm256_set1_epi16( clpRng.min() );

    const __m256i vibdimax  = _mm256_set1_epi16( clpRng.max() );

    const __m256i vone      = _mm256_set1_epi16( 1 );

    for( int col = 0; col < numSamples; col += 16 )

    {

      __m256i vsrc0 = _mm256_load_si256( ( const __m256i* )&src0[col] );

      __m256i vsrc1 = _mm256_load_si256( ( const __m256i* )&src1[col] );

      __m256i vsum, vdst;

      vsum = _mm256_unpacklo_epi16    ( vsrc0, vsrc1 );

      vsum = _mm256_madd_epi16        ( vsum, vone );

      vsum = _mm256_add_epi32         ( vsum, voffset );

      vdst = _mm256_srai_epi32        ( vsum, shift );

      vsum = _mm256_unpackhi_epi16    ( vsrc0, vsrc1 );

      vsum = _mm256_madd_epi16        ( vsum, vone );

      vsum = _mm256_add_epi32         ( vsum, voffset );

      vsum = _mm256_srai_epi32        ( vsum, shift );

      vdst = _mm256_packs_epi32       ( vdst, vsum );

      vdst = _mm256_min_epi16( vibdimax, _mm256_max_epi16( vibdimin, vdst ) );

      _mm256_store_si256( ( __m256i * )&dst[col], vdst );

    }

  }

  else

#endif

  if( numSamples >= 8 )

  {

    const __m128i vone     = _mm_set1_epi16( 1 );

    const __m128i voffset  = _mm_set1_epi32( offset );

    const __m128i vibdimin = _mm_set1_epi16( clpRng.min() );

    const __m128i vibdimax = _mm_set1_epi16( clpRng.max() );

    for( int col = 0; col < numSamples; col += 8 )

    {

      __m128i vsrc0 = _mm_load_si128 ( (const __m128i *)&src0[col] );

      __m128i vsrc1 = _mm_load_si128 ( (const __m128i *)&src1[col] );

      __m128i vsum, vdst;

      vsum = _mm_unpacklo_epi16    ( vsrc0, vsrc1 );

      vsum = _mm_madd_epi16        ( vsum, vone );

      vsum = _mm_add_epi32         ( vsum, voffset );

      vdst = _mm_srai_epi32        ( vsum, shift );

      vsum = _mm_unpackhi_epi16    ( vsrc0, vsrc1 );

      vsum = _mm_madd_epi16        ( vsum, vone );

      vsum = _mm_add_epi32         ( vsum, voffset );

      vsum = _mm_srai_epi32        ( vsum, shift );

      vdst = _mm_packs_epi32       ( vdst, vsum );

      vdst = _mm_min_epi16( vibdimax, _mm_max_epi16( vibdimin, vdst ) );

      _mm_store_si128( ( __m128i * )&dst[col], vdst );

    }

  }

  else if( numSamples == 4 )

  {

    const __m128i vone      = _mm_set1_epi16( 1 );

    const __m128i vzero     = _mm_setzero_si128();

    const __m128i voffset   = _mm_set1_epi32( offset );

    const __m128i vibdimin  = _mm_set1_epi16( clpRng.min() );

    const __m128i vibdimax  = _mm_set1_epi16( clpRng.max() );

    __m128i vsum = _vv_loadl_epi64  ( ( const __m128i * )&src0[0] );

    __m128i vdst = _vv_loadl_epi64  ( ( const __m128i * )&src1[0] );

    vsum = _mm_unpacklo_epi16    ( vsum, vdst );

    vsum = _mm_madd_epi16        ( vsum, vone );

    vsum = _mm_add_epi32         ( vsum, voffset );

    vsum = _mm_srai_epi32        ( vsum, shift );

    vdst = _mm_packs_epi32       ( vsum, vzero );

    vdst = _mm_min_epi16( vibdimax, _mm_max_epi16( vibdimin, vdst ) );

    _vv_storel_epi64( ( __m128i * )&dst[0], vdst );

  }

  else

  {

    THROW( "Unsupported size" );

  }

#if USE_AVX2

  _mm256_zeroupper();

#endif

}

Unexecuted instantiation: void vvenc::addAvg_SSE<(vvenc::x86_simd::X86_VEXT)1>(short const*, short const*, short*, int, unsigned int, int, vvenc::ClpRng const&)

Unexecuted instantiation: void vvenc::addAvg_SSE<(vvenc::x86_simd::X86_VEXT)4>(short const*, short const*, short*, int, unsigned int, int, vvenc::ClpRng const&)

template< X86_VEXT vext>

void roundGeo_SSE( const Pel* src, Pel* dst, const int numSamples, unsigned shift, int offset, const ClpRng &clpRng)

{

#if USE_AVX2

  if( numSamples >= 16 )

  {

    __m256i voffset   = _mm256_set1_epi16( offset );

    __m256i vibdimin  = _mm256_set1_epi16( clpRng.min() );

    __m256i vibdimax  = _mm256_set1_epi16( clpRng.max() );

    for( int col = 0; col < numSamples; col += 16 )

    {

      __m256i val = _mm256_load_si256( ( const __m256i* )&src[col] );

      val = _mm256_adds_epi16        ( val, voffset );

      val = _mm256_srai_epi16        ( val, shift );

      val = _mm256_min_epi16( vibdimax, _mm256_max_epi16( vibdimin, val ) );

      _mm256_store_si256( ( __m256i * )&dst[col], val );

    }

  }

  else

#endif

  {

    __m128i voffset   = _mm_set1_epi16( offset );

    __m128i vibdimin  = _mm_set1_epi16( clpRng.min() );

    __m128i vibdimax  = _mm_set1_epi16( clpRng.max() );

    if( numSamples >= 8 )

    {

      for( int col = 0; col < numSamples; col += 8 )

      {

        __m128i val = _mm_load_si128 ( (const __m128i *)&src[col] );

        val  = _mm_adds_epi16        ( val, voffset );

        val  = _mm_srai_epi16        ( val, shift );

        val  = _mm_min_epi16( vibdimax, _mm_max_epi16( vibdimin, val ) );

        _mm_store_si128( ( __m128i * )&dst[col], val );

      }

    }

    else //if( numSamples == 4 )

    {

      __m128i val = _vv_loadl_epi64  ( ( const __m128i * )&src[0] );

      val = _mm_adds_epi16           ( val, voffset );

      val = _mm_srai_epi16           ( val, shift );

      val = _mm_min_epi16( vibdimax, _mm_max_epi16( vibdimin, val ) );

      _vv_storel_epi64( ( __m128i * )&dst[0], val );

    }

  }

#if USE_AVX2

  _mm256_zeroupper();

#endif

}

Unexecuted instantiation: void vvenc::roundGeo_SSE<(vvenc::x86_simd::X86_VEXT)1>(short const*, short*, int, unsigned int, int, vvenc::ClpRng const&)

Unexecuted instantiation: void vvenc::roundGeo_SSE<(vvenc::x86_simd::X86_VEXT)4>(short const*, short*, int, unsigned int, int, vvenc::ClpRng const&)

template< X86_VEXT vext >

void recoCore_SSE( const Pel* src0, const Pel* src1, Pel* dst, int numSamples, const ClpRng& clpRng )

{

#if USE_AVX2

  if( vext >= AVX2 && numSamples >= 16 )

  {

    __m256i vbdmin = _mm256_set1_epi16( clpRng.min() );

    __m256i vbdmax = _mm256_set1_epi16( clpRng.max() );

    for( int n = 0; n < numSamples; n += 16 )

    {

      __m256i vdest = _mm256_load_si256 ( ( const __m256i * )&src0[n] );

      __m256i vsrc1 = _mm256_load_si256( ( const __m256i * )&src1[n] );

      vdest = _mm256_adds_epi16( vdest, vsrc1 );

      vdest = _mm256_min_epi16 ( vbdmax, _mm256_max_epi16( vbdmin, vdest ) );

      _mm256_store_si256( ( __m256i * )&dst[n], vdest );

    }

  }

  else

#endif

  if( numSamples >= 8 )

  {

    __m128i vbdmin = _mm_set1_epi16( clpRng.min() );

    __m128i vbdmax = _mm_set1_epi16( clpRng.max() );

    for( int n = 0; n < numSamples; n += 8 )

    {

      __m128i vdest = _mm_load_si128 ( ( const __m128i * )&src0[n] );

      __m128i vsrc1 = _mm_load_si128( ( const __m128i * )&src1[n] );

      vdest = _mm_adds_epi16( vdest, vsrc1 );

      vdest = _mm_min_epi16 ( vbdmax, _mm_max_epi16( vbdmin, vdest ) );

      _mm_store_si128( ( __m128i * )&dst[n], vdest );

    }

  }

  else

  {

    __m128i vbdmin = _mm_set1_epi16( clpRng.min() );

    __m128i vbdmax = _mm_set1_epi16( clpRng.max() );

    __m128i vsrc = _vv_loadl_epi64( ( const __m128i * )&src0[0] );

    __m128i vdst = _vv_loadl_epi64( ( const __m128i * )&src1[0] );

    vdst = _mm_adds_epi16( vdst, vsrc );

    vdst = _mm_min_epi16 ( vbdmax, _mm_max_epi16( vbdmin, vdst ) );

    _vv_storel_epi64( ( __m128i * )&dst[0], vdst );

  }

#if USE_AVX2

  _mm256_zeroupper();

#endif

}

Unexecuted instantiation: void vvenc::recoCore_SSE<(vvenc::x86_simd::X86_VEXT)1>(short const*, short const*, short*, int, vvenc::ClpRng const&)

Unexecuted instantiation: void vvenc::recoCore_SSE<(vvenc::x86_simd::X86_VEXT)4>(short const*, short const*, short*, int, vvenc::ClpRng const&)

template<X86_VEXT vext>

void copyClip_SSE( const Pel* src, Pel* dst, int numSamples, const ClpRng& clpRng )

{

  if( vext >= AVX2 && numSamples >= 16 )

  {

#if USE_AVX2

    __m256i vbdmin   = _mm256_set1_epi16( clpRng.min() );

    __m256i vbdmax   = _mm256_set1_epi16( clpRng.max() );

    for( int col = 0; col < numSamples; col += 16 )

    {

      __m256i val = _mm256_loadu_si256  ( ( const __m256i * ) &src[col] );

      val = _mm256_min_epi16            ( vbdmax, _mm256_max_epi16( vbdmin, val ) );

      _mm256_storeu_si256               ( ( __m256i * )&dst[col], val );

    }

#endif

  }

  else if(numSamples >= 8 )

  {

    __m128i vbdmin = _mm_set1_epi16( clpRng.min() );

    __m128i vbdmax = _mm_set1_epi16( clpRng.max() );

    for( int col = 0; col < numSamples; col += 8 )

    {

      __m128i val = _mm_loadu_si128 ( ( const __m128i * ) &src[col] );

      val = _mm_min_epi16           ( vbdmax, _mm_max_epi16( vbdmin, val ) );

      _mm_storeu_si128              ( ( __m128i * )&dst[col], val );

    }

  }

  else

  {

    __m128i vbdmin  = _mm_set1_epi16( clpRng.min() );

    __m128i vbdmax  = _mm_set1_epi16( clpRng.max() );

    __m128i val;

    val = _vv_loadl_epi64   ( ( const __m128i * )&src[0] );

    val = _mm_min_epi16     ( vbdmax, _mm_max_epi16( vbdmin, val ) );

    _vv_storel_epi64        ( ( __m128i * )&dst[0], val );

  }

#if USE_AVX2

  _mm256_zeroupper();

#endif

}

Unexecuted instantiation: void vvenc::copyClip_SSE<(vvenc::x86_simd::X86_VEXT)1>(short const*, short*, int, vvenc::ClpRng const&)

Unexecuted instantiation: void vvenc::copyClip_SSE<(vvenc::x86_simd::X86_VEXT)4>(short const*, short*, int, vvenc::ClpRng const&)

template< X86_VEXT vext, int W, bool srcAligned >

void addAvg_SSE_algn( const int16_t* src0, int src0Stride, const int16_t* src1, int src1Stride, int16_t *dst, ptrdiff_t dstStride, int width, int height, unsigned shift, int offset, const ClpRng& clpRng )

{

#if USE_AVX2

  if( W == 16 )

  {

    const __m256i voffset   = _mm256_set1_epi32( offset );

    const __m256i vibdimin  = _mm256_set1_epi16( clpRng.min() );

    const __m256i vibdimax  = _mm256_set1_epi16( clpRng.max() );

    const __m256i vone      = _mm256_set1_epi16( 1 );

    for( int row = 0; row < height; row++ )

    {

      for( int col = 0; col < width; col += 16 )

      {

        __m256i vsrc0 = load_aligned_avx2<srcAligned>( ( const void* )&src0[col] );

        __m256i vsrc1 = load_aligned_avx2<srcAligned>( ( const void* )&src1[col] );

        __m256i vsum, vdst;

        vsum = _mm256_unpacklo_epi16    ( vsrc0, vsrc1 );

        vsum = _mm256_madd_epi16        ( vsum, vone );

        vsum = _mm256_add_epi32         ( vsum, voffset );

        vdst = _mm256_srai_epi32        ( vsum, shift );

        vsum = _mm256_unpackhi_epi16    ( vsrc0, vsrc1 );

        vsum = _mm256_madd_epi16        ( vsum, vone );

        vsum = _mm256_add_epi32         ( vsum, voffset );

        vsum = _mm256_srai_epi32        ( vsum, shift );

        vdst = _mm256_packs_epi32       ( vdst, vsum );

        vdst = _mm256_min_epi16( vibdimax, _mm256_max_epi16( vibdimin, vdst ) );

        _mm256_storeu_si256( ( __m256i * )&dst[col], vdst );

      }

      src0 += src0Stride;

      src1 += src1Stride;

      dst  +=  dstStride;

    }

  }

  else

#endif

#if USE_AVX2

  if( W >= 8 )

  {

    __m256i voffset  = _mm256_set1_epi32( offset );

    __m128i vibdimin = _mm_set1_epi16   ( clpRng.min() );

    __m128i vibdimax = _mm_set1_epi16   ( clpRng.max() );

    for( int row = 0; row < height; row++ )

    {

      for( int col = 0; col < width; col += 8 )

      {

        __m256i vsrc0 = _mm256_cvtepi16_epi32( load_aligned<srcAligned>( ( const void* )&src0[col] ) );

        __m256i vsrc1 = _mm256_cvtepi16_epi32( load_aligned<srcAligned>( ( const void* )&src1[col] ) );

        __m256i

        vsum = _mm256_add_epi32        ( vsrc0, vsrc1 );

        vsum = _mm256_add_epi32        ( vsum, voffset );

        vsum = _mm256_srai_epi32       ( vsum, shift );

        vsum = _mm256_packs_epi32      ( vsum, vsum );

        vsum = _mm256_permute4x64_epi64( vsum, 0 + ( 2 << 2 ) + ( 2 << 4 ) + ( 3 << 6 ) );

        __m128i

        xsum = _mm_min_epi16( vibdimax, _mm_max_epi16( vibdimin, _mm256_castsi256_si128( vsum ) ) );

        _mm_storeu_si128( ( __m128i * )&dst[col], xsum );

      }

      src0 += src0Stride;

      src1 += src1Stride;

      dst  +=  dstStride;

    }

  }

#else

  if( W >= 8 )

  {

    const __m128i voffset  = _mm_set1_epi32( offset );

    const __m128i vibdimin = _mm_set1_epi16( clpRng.min() );

    const __m128i vibdimax = _mm_set1_epi16( clpRng.max() );

    const __m128i vone     = _mm_set1_epi16( 1 );

    for( int row = 0; row < height; row++ )

    {

      for( int col = 0; col < width; col += 8 )

      {

        __m128i vsrc0 = load_aligned<srcAligned>( ( const void* )&src0[col] );

        __m128i vsrc1 = load_aligned<srcAligned>( ( const void* )&src1[col] );

        __m128i vsum, vdst;

        vsum = _mm_unpacklo_epi16    ( vsrc0, vsrc1 );

        vsum = _mm_madd_epi16        ( vsum, vone );

        vsum = _mm_add_epi32         ( vsum, voffset );

        vdst = _mm_srai_epi32        ( vsum, shift );

        vsum = _mm_unpackhi_epi16    ( vsrc0, vsrc1 );

        vsum = _mm_madd_epi16        ( vsum, vone );

        vsum = _mm_add_epi32         ( vsum, voffset );

        vsum = _mm_srai_epi32        ( vsum, shift );

        vdst = _mm_packs_epi32       ( vdst, vsum );

        vdst = _mm_min_epi16( vibdimax, _mm_max_epi16( vibdimin, vdst ) );

        _mm_storeu_si128( ( __m128i * )&dst[col], vdst );

      }

      src0 += src0Stride;

      src1 += src1Stride;

      dst  +=  dstStride;

    }

  }

#endif

  else if( W == 4 )

  {

    __m128i vzero     = _mm_setzero_si128();

    __m128i voffset   = _mm_set1_epi32( offset );

    __m128i vibdimin  = _mm_set1_epi16( clpRng.min() );

    __m128i vibdimax  = _mm_set1_epi16( clpRng.max() );

    for( int row = 0; row < height; row++ )

    {

      for( int col = 0; col < width; col += 4 )

      {

        __m128i vsum = _vv_loadl_epi64  ( ( const __m128i * )&src0[col] );

        __m128i vdst = _vv_loadl_epi64  ( ( const __m128i * )&src1[col] );

        vsum = _mm_cvtepi16_epi32       ( vsum );

        vdst = _mm_cvtepi16_epi32       ( vdst );

        vsum = _mm_add_epi32            ( vsum, vdst );

        vsum = _mm_add_epi32            ( vsum, voffset );

        vsum = _mm_srai_epi32           ( vsum, shift );

        vsum = _mm_packs_epi32          ( vsum, vzero );

        vsum = _mm_min_epi16( vibdimax, _mm_max_epi16( vibdimin, vsum ) );

        _vv_storel_epi64( ( __m128i * )&dst[col], vsum );

      }

      src0 += src0Stride;

      src1 += src1Stride;

      dst  +=  dstStride;

    }

  }

  else

  {

    THROW( "Unsupported size" );

  }

#if USE_AVX2

  _mm256_zeroupper();

#endif

}

Unexecuted instantiation: void vvenc::addAvg_SSE_algn<(vvenc::x86_simd::X86_VEXT)1, 4, false>(short const*, int, short const*, int, short*, long, int, int, unsigned int, int, vvenc::ClpRng const&)

Unexecuted instantiation: void vvenc::addAvg_SSE_algn<(vvenc::x86_simd::X86_VEXT)1, 8, false>(short const*, int, short const*, int, short*, long, int, int, unsigned int, int, vvenc::ClpRng const&)

Unexecuted instantiation: void vvenc::addAvg_SSE_algn<(vvenc::x86_simd::X86_VEXT)1, 16, false>(short const*, int, short const*, int, short*, long, int, int, unsigned int, int, vvenc::ClpRng const&)

Unexecuted instantiation: void vvenc::addAvg_SSE_algn<(vvenc::x86_simd::X86_VEXT)4, 4, false>(short const*, int, short const*, int, short*, long, int, int, unsigned int, int, vvenc::ClpRng const&)

Unexecuted instantiation: void vvenc::addAvg_SSE_algn<(vvenc::x86_simd::X86_VEXT)4, 8, false>(short const*, int, short const*, int, short*, long, int, int, unsigned int, int, vvenc::ClpRng const&)

Unexecuted instantiation: void vvenc::addAvg_SSE_algn<(vvenc::x86_simd::X86_VEXT)4, 16, false>(short const*, int, short const*, int, short*, long, int, int, unsigned int, int, vvenc::ClpRng const&)

template< X86_VEXT vext, int W >

void addAvg_SSE( const int16_t* src0, int src0Stride, const int16_t* src1, int src1Stride, int16_t *dst, int dstStride, int width, int height, unsigned shift, int offset, const ClpRng& clpRng/*, bool srcAligned*/ )

{

/*

  if( srcAligned )

  {

    addAvg_SSE_algn<vext, W, true>( src0, src0Stride, src1, src1Stride, dst, dstStride, width, height, shift, offset, clpRng );

  }

  else

*/

  {

    addAvg_SSE_algn<vext, W, false>( src0, src0Stride, src1, src1Stride, dst, dstStride, width, height, shift, offset, clpRng );

  }

}

Unexecuted instantiation: void vvenc::addAvg_SSE<(vvenc::x86_simd::X86_VEXT)1, 4>(short const*, int, short const*, int, short*, int, int, int, unsigned int, int, vvenc::ClpRng const&)

Unexecuted instantiation: void vvenc::addAvg_SSE<(vvenc::x86_simd::X86_VEXT)1, 8>(short const*, int, short const*, int, short*, int, int, int, unsigned int, int, vvenc::ClpRng const&)

Unexecuted instantiation: void vvenc::addAvg_SSE<(vvenc::x86_simd::X86_VEXT)1, 16>(short const*, int, short const*, int, short*, int, int, int, unsigned int, int, vvenc::ClpRng const&)

Unexecuted instantiation: void vvenc::addAvg_SSE<(vvenc::x86_simd::X86_VEXT)4, 4>(short const*, int, short const*, int, short*, int, int, int, unsigned int, int, vvenc::ClpRng const&)

Unexecuted instantiation: void vvenc::addAvg_SSE<(vvenc::x86_simd::X86_VEXT)4, 8>(short const*, int, short const*, int, short*, int, int, int, unsigned int, int, vvenc::ClpRng const&)

Unexecuted instantiation: void vvenc::addAvg_SSE<(vvenc::x86_simd::X86_VEXT)4, 16>(short const*, int, short const*, int, short*, int, int, int, unsigned int, int, vvenc::ClpRng const&)