/src/libtheora/lib/x86/sse2idct.c

Source
/********************************************************************
 *                                                                  *
 * THIS FILE IS PART OF THE OggTheora SOFTWARE CODEC SOURCE CODE.   *
 * USE, DISTRIBUTION AND REPRODUCTION OF THIS LIBRARY SOURCE IS     *
 * GOVERNED BY A BSD-STYLE SOURCE LICENSE INCLUDED WITH THIS SOURCE *
 * IN 'COPYING'. PLEASE READ THESE TERMS BEFORE DISTRIBUTING.       *
 *                                                                  *
 * THE Theora SOURCE CODE IS COPYRIGHT (C) 2002-2009,2025           *
 * by the Xiph.Org Foundation and contributors                      *
 * https://www.xiph.org/                                            *
 *                                                                  *
 ********************************************************************

  function:

 ********************************************************************/

/*SSE2 acceleration of Theora's iDCT.*/
#include "x86int.h"
#include "sse2trans.h"
#include "../dct.h"

#if defined(OC_X86_ASM)

/*A table of constants used by the MMX routines.*/
const unsigned short __attribute__((aligned(16),used)) OC_IDCT_CONSTS[64]={
        8,      8,      8,      8,      8,      8,      8,      8,
  OC_C1S7,OC_C1S7,OC_C1S7,OC_C1S7,OC_C1S7,OC_C1S7,OC_C1S7,OC_C1S7,
  OC_C2S6,OC_C2S6,OC_C2S6,OC_C2S6,OC_C2S6,OC_C2S6,OC_C2S6,OC_C2S6,
  OC_C3S5,OC_C3S5,OC_C3S5,OC_C3S5,OC_C3S5,OC_C3S5,OC_C3S5,OC_C3S5,
  OC_C4S4,OC_C4S4,OC_C4S4,OC_C4S4,OC_C4S4,OC_C4S4,OC_C4S4,OC_C4S4,
  OC_C5S3,OC_C5S3,OC_C5S3,OC_C5S3,OC_C5S3,OC_C5S3,OC_C5S3,OC_C5S3,
  OC_C6S2,OC_C6S2,OC_C6S2,OC_C6S2,OC_C6S2,OC_C6S2,OC_C6S2,OC_C6S2,
  OC_C7S1,OC_C7S1,OC_C7S1,OC_C7S1,OC_C7S1,OC_C7S1,OC_C7S1,OC_C7S1
};


/*Performs the first three stages of the iDCT.
  xmm2, xmm6, xmm3, and xmm5 must contain the corresponding rows of the input
   (accessed in that order).
  The remaining rows must be in _x at their corresponding locations.
  On output, xmm7 down to xmm4 contain rows 0 through 3, and xmm0 up to xmm3
   contain rows 4 through 7.*/
#define OC_IDCT_8x8_ABC(_x) \
  "#OC_IDCT_8x8_ABC\n\t" \
  /*Stage 1:*/ \
  /*2-3 rotation by 6pi/16. \
    xmm4=xmm7=C6, xmm0=xmm1=C2, xmm2=X2, xmm6=X6.*/ \
  "movdqa "OC_MEM_OFFS(0x20,c)",%%xmm1\n\t" \
  "movdqa "OC_MEM_OFFS(0x60,c)",%%xmm4\n\t" \
  "movdqa %%xmm1,%%xmm0\n\t" \
  "pmulhw %%xmm2,%%xmm1\n\t" \
  "movdqa %%xmm4,%%xmm7\n\t" \
  "pmulhw %%xmm6,%%xmm0\n\t" \
  "pmulhw %%xmm2,%%xmm7\n\t" \
  "pmulhw %%xmm6,%%xmm4\n\t" \
  "paddw %%xmm6,%%xmm0\n\t" \
  "movdqa "OC_MEM_OFFS(0x30,c)",%%xmm6\n\t" \
  "paddw %%xmm1,%%xmm2\n\t" \
  "psubw %%xmm0,%%xmm7\n\t" \
  "movdqa %%xmm7,"OC_MEM_OFFS(0x00,buf)"\n\t" \
  "paddw %%xmm4,%%xmm2\n\t" \
  "movdqa "OC_MEM_OFFS(0x50,c)",%%xmm4\n\t" \
  "movdqa %%xmm2,"OC_MEM_OFFS(0x10,buf)"\n\t" \
  /*5-6 rotation by 3pi/16. \
    xmm4=xmm2=C5, xmm1=xmm6=C3, xmm3=X3, xmm5=X5.*/ \
  "movdqa %%xmm4,%%xmm2\n\t" \
  "movdqa %%xmm6,%%xmm1\n\t" \
  "pmulhw %%xmm3,%%xmm4\n\t" \
  "pmulhw %%xmm5,%%xmm1\n\t" \
  "pmulhw %%xmm3,%%xmm6\n\t" \
  "pmulhw %%xmm5,%%xmm2\n\t" \
  "paddw %%xmm3,%%xmm4\n\t" \
  "paddw %%xmm5,%%xmm3\n\t" \
  "paddw %%xmm6,%%xmm3\n\t" \
  "movdqa "OC_MEM_OFFS(0x70,_x)",%%xmm6\n\t" \
  "paddw %%xmm5,%%xmm1\n\t" \
  "movdqa "OC_MEM_OFFS(0x10,_x)",%%xmm5\n\t" \
  "paddw %%xmm3,%%xmm2\n\t" \
  "movdqa "OC_MEM_OFFS(0x70,c)",%%xmm3\n\t" \
  "psubw %%xmm4,%%xmm1\n\t" \
  "movdqa "OC_MEM_OFFS(0x10,c)",%%xmm4\n\t" \
  /*4-7 rotation by 7pi/16. \
    xmm4=xmm7=C1, xmm3=xmm0=C7, xmm5=X1, xmm6=X7.*/ \
  "movdqa %%xmm3,%%xmm0\n\t" \
  "movdqa %%xmm4,%%xmm7\n\t" \
  "pmulhw %%xmm5,%%xmm3\n\t" \
  "pmulhw %%xmm5,%%xmm7\n\t" \
  "pmulhw %%xmm6,%%xmm4\n\t" \
  "pmulhw %%xmm6,%%xmm0\n\t" \
  "paddw %%xmm6,%%xmm4\n\t" \
  "movdqa "OC_MEM_OFFS(0x40,_x)",%%xmm6\n\t" \
  "paddw %%xmm5,%%xmm7\n\t" \
  "psubw %%xmm4,%%xmm3\n\t" \
  "movdqa "OC_MEM_OFFS(0x40,c)",%%xmm4\n\t" \
  "paddw %%xmm7,%%xmm0\n\t" \
  "movdqa "OC_MEM_OFFS(0x00,_x)",%%xmm7\n\t" \
  /*0-1 butterfly. \
    xmm4=xmm5=C4, xmm7=X0, xmm6=X4.*/ \
  "paddw %%xmm7,%%xmm6\n\t" \
  "movdqa %%xmm4,%%xmm5\n\t" \
  "pmulhw %%xmm6,%%xmm4\n\t" \
  "paddw %%xmm7,%%xmm7\n\t" \
  "psubw %%xmm6,%%xmm7\n\t" \
  "paddw %%xmm6,%%xmm4\n\t" \
  /*Stage 2:*/ \
  /*4-5 butterfly: xmm3=t[4], xmm1=t[5] \
    7-6 butterfly: xmm2=t[6], xmm0=t[7]*/ \
  "movdqa %%xmm3,%%xmm6\n\t" \
  "paddw %%xmm1,%%xmm3\n\t" \
  "psubw %%xmm1,%%xmm6\n\t" \
  "movdqa %%xmm5,%%xmm1\n\t" \
  "pmulhw %%xmm7,%%xmm5\n\t" \
  "paddw %%xmm7,%%xmm5\n\t" \
  "movdqa %%xmm0,%%xmm7\n\t" \
  "paddw %%xmm2,%%xmm0\n\t" \
  "psubw %%xmm2,%%xmm7\n\t" \
  "movdqa %%xmm1,%%xmm2\n\t" \
  "pmulhw %%xmm6,%%xmm1\n\t" \
  "pmulhw %%xmm7,%%xmm2\n\t" \
  "paddw %%xmm6,%%xmm1\n\t" \
  "movdqa "OC_MEM_OFFS(0x00,buf)",%%xmm6\n\t" \
  "paddw %%xmm7,%%xmm2\n\t" \
  "movdqa "OC_MEM_OFFS(0x10,buf)",%%xmm7\n\t" \
  /*Stage 3: \
    6-5 butterfly: xmm1=t[5], xmm2=t[6] -> xmm1=t[6]+t[5], xmm2=t[6]-t[5] \
    0-3 butterfly: xmm4=t[0], xmm7=t[3] -> xmm7=t[0]+t[3], xmm4=t[0]-t[3] \
    1-2 butterfly: xmm5=t[1], xmm6=t[2] -> xmm6=t[1]+t[2], xmm5=t[1]-t[2]*/ \
  "paddw %%xmm2,%%xmm1\n\t" \
  "paddw %%xmm5,%%xmm6\n\t" \
  "paddw %%xmm4,%%xmm7\n\t" \
  "paddw %%xmm2,%%xmm2\n\t" \
  "paddw %%xmm4,%%xmm4\n\t" \
  "paddw %%xmm5,%%xmm5\n\t" \
  "psubw %%xmm1,%%xmm2\n\t" \
  "psubw %%xmm7,%%xmm4\n\t" \
  "psubw %%xmm6,%%xmm5\n\t" \

/*Performs the last stage of the iDCT.
  On input, xmm7 down to xmm4 contain rows 0 through 3, and xmm0 up to xmm3
   contain rows 4 through 7.
  On output, xmm0 through xmm7 contain the corresponding rows.*/
#define OC_IDCT_8x8_D \
  "#OC_IDCT_8x8_D\n\t" \
  /*Stage 4: \
    0-7 butterfly: xmm7=t[0], xmm0=t[7] -> xmm0=t[0]+t[7], xmm7=t[0]-t[7] \
    1-6 butterfly: xmm6=t[1], xmm1=t[6] -> xmm1=t[1]+t[6], xmm6=t[1]-t[6] \
    2-5 butterfly: xmm5=t[2], xmm2=t[5] -> xmm2=t[2]+t[5], xmm5=t[2]-t[5] \
    3-4 butterfly: xmm4=t[3], xmm3=t[4] -> xmm3=t[3]+t[4], xmm4=t[3]-t[4]*/ \
  "psubw %%xmm0,%%xmm7\n\t" \
  "psubw %%xmm1,%%xmm6\n\t" \
  "psubw %%xmm2,%%xmm5\n\t" \
  "psubw %%xmm3,%%xmm4\n\t" \
  "paddw %%xmm0,%%xmm0\n\t" \
  "paddw %%xmm1,%%xmm1\n\t" \
  "paddw %%xmm2,%%xmm2\n\t" \
  "paddw %%xmm3,%%xmm3\n\t" \
  "paddw %%xmm7,%%xmm0\n\t" \
  "paddw %%xmm6,%%xmm1\n\t" \
  "paddw %%xmm5,%%xmm2\n\t" \
  "paddw %%xmm4,%%xmm3\n\t" \

/*Performs the last stage of the iDCT.
  On input, xmm7 down to xmm4 contain rows 0 through 3, and xmm0 up to xmm3
   contain rows 4 through 7.
  On output, xmm0 through xmm7 contain the corresponding rows.*/
#define OC_IDCT_8x8_D_STORE \
  "#OC_IDCT_8x8_D_STORE\n\t" \
  /*Stage 4: \
    0-7 butterfly: xmm7=t[0], xmm0=t[7] -> xmm0=t[0]+t[7], xmm7=t[0]-t[7] \
    1-6 butterfly: xmm6=t[1], xmm1=t[6] -> xmm1=t[1]+t[6], xmm6=t[1]-t[6] \
    2-5 butterfly: xmm5=t[2], xmm2=t[5] -> xmm2=t[2]+t[5], xmm5=t[2]-t[5] \
    3-4 butterfly: xmm4=t[3], xmm3=t[4] -> xmm3=t[3]+t[4], xmm4=t[3]-t[4]*/ \
  "psubw %%xmm3,%%xmm4\n\t" \
  "movdqa %%xmm4,"OC_MEM_OFFS(0x40,y)"\n\t" \
  "movdqa "OC_MEM_OFFS(0x00,c)",%%xmm4\n\t" \
  "psubw %%xmm0,%%xmm7\n\t" \
  "psubw %%xmm1,%%xmm6\n\t" \
  "psubw %%xmm2,%%xmm5\n\t" \
  "paddw %%xmm4,%%xmm7\n\t" \
  "paddw %%xmm4,%%xmm6\n\t" \
  "paddw %%xmm4,%%xmm5\n\t" \
  "paddw "OC_MEM_OFFS(0x40,y)",%%xmm4\n\t" \
  "paddw %%xmm0,%%xmm0\n\t" \
  "paddw %%xmm1,%%xmm1\n\t" \
  "paddw %%xmm2,%%xmm2\n\t" \
  "paddw %%xmm3,%%xmm3\n\t" \
  "paddw %%xmm7,%%xmm0\n\t" \
  "paddw %%xmm6,%%xmm1\n\t" \
  "psraw $4,%%xmm0\n\t" \
  "paddw %%xmm5,%%xmm2\n\t" \
  "movdqa %%xmm0,"OC_MEM_OFFS(0x00,y)"\n\t" \
  "psraw $4,%%xmm1\n\t" \
  "paddw %%xmm4,%%xmm3\n\t" \
  "movdqa %%xmm1,"OC_MEM_OFFS(0x10,y)"\n\t" \
  "psraw $4,%%xmm2\n\t" \
  "movdqa %%xmm2,"OC_MEM_OFFS(0x20,y)"\n\t" \
  "psraw $4,%%xmm3\n\t" \
  "movdqa %%xmm3,"OC_MEM_OFFS(0x30,y)"\n\t" \
  "psraw $4,%%xmm4\n\t" \
  "movdqa %%xmm4,"OC_MEM_OFFS(0x40,y)"\n\t" \
  "psraw $4,%%xmm5\n\t" \
  "movdqa %%xmm5,"OC_MEM_OFFS(0x50,y)"\n\t" \
  "psraw $4,%%xmm6\n\t" \
  "movdqa %%xmm6,"OC_MEM_OFFS(0x60,y)"\n\t" \
  "psraw $4,%%xmm7\n\t" \
  "movdqa %%xmm7,"OC_MEM_OFFS(0x70,y)"\n\t" \

static void __attribute__((target("sse2"))) oc_idct8x8_slow_sse2(ogg_int16_t _y[64],ogg_int16_t _x[64]){
  OC_ALIGN16(ogg_int16_t buf[16]);
  int i;
  /*This routine accepts an 8x8 matrix pre-transposed.*/
  __asm__ __volatile__(
    /*Load rows 2, 3, 5, and 6 for the first stage of the iDCT.*/
    "movdqa "OC_MEM_OFFS(0x20,x)",%%xmm2\n\t"
    "movdqa "OC_MEM_OFFS(0x60,x)",%%xmm6\n\t"
    "movdqa "OC_MEM_OFFS(0x30,x)",%%xmm3\n\t"
    "movdqa "OC_MEM_OFFS(0x50,x)",%%xmm5\n\t"
    OC_IDCT_8x8_ABC(x)
    OC_IDCT_8x8_D
    OC_TRANSPOSE_8x8
    /*Clear out rows 0, 1, 4, and 7 for the first stage of the iDCT.*/
    "movdqa %%xmm7,"OC_MEM_OFFS(0x70,y)"\n\t"
    "movdqa %%xmm4,"OC_MEM_OFFS(0x40,y)"\n\t"
    "movdqa %%xmm1,"OC_MEM_OFFS(0x10,y)"\n\t"
    "movdqa %%xmm0,"OC_MEM_OFFS(0x00,y)"\n\t"
    OC_IDCT_8x8_ABC(y)
    OC_IDCT_8x8_D_STORE
    :[buf]"=m"(OC_ARRAY_OPERAND(ogg_int16_t,buf,16)),
     [y]"=m"(OC_ARRAY_OPERAND(ogg_int16_t,_y,64))
    :[x]"m"(OC_CONST_ARRAY_OPERAND(ogg_int16_t,_x,64)),
     [c]"m"(OC_CONST_ARRAY_OPERAND(ogg_int16_t,OC_IDCT_CONSTS,64))
    :"%xmm0", "%xmm1", "%xmm2", "%xmm3",
     "%xmm4", "%xmm5", "%xmm6", "%xmm7"
  );
  /*Store 0 in zr and use it in the loop. This translates to nothing with -O1.*/
  register sse2_reg zr;
  __asm__ __volatile__("pxor %[zr],%[zr]\n\t":[zr]"=x"(zr));
  /*Clear input data for next block (decoder only).*/
  for(i=0;i<2;i++){
    __asm__ __volatile__(
      "movdqa %[zr],"OC_MEM_OFFS(0x00,x)"\n\t"
      "movdqa %[zr],"OC_MEM_OFFS(0x10,x)"\n\t"
      "movdqa %[zr],"OC_MEM_OFFS(0x20,x)"\n\t"
      "movdqa %[zr],"OC_MEM_OFFS(0x30,x)"\n\t"
      :[x]"=m"(OC_ARRAY_OPERAND(ogg_int16_t,_x+i*32,32))
      :[zr]"x"(zr)
    );
  }
}

/*For the first step of the 10-coefficient version of the 8x8 iDCT, we only
   need to work with four columns at a time.
  Doing this in MMX is faster on processors with a 64-bit data path.*/
#define OC_IDCT_8x8_10_MMX \
  "#OC_IDCT_8x8_10_MMX\n\t" \
  /*Stage 1:*/ \
  /*2-3 rotation by 6pi/16. \
    mm7=C6, mm6=C2, mm2=X2, X6=0.*/ \
  "movq "OC_MEM_OFFS(0x60,c)",%%mm7\n\t" \
  "movq "OC_MEM_OFFS(0x20,c)",%%mm6\n\t" \
  "pmulhw %%mm2,%%mm6\n\t" \
  "pmulhw %%mm2,%%mm7\n\t" \
  "movq "OC_MEM_OFFS(0x50,c)",%%mm5\n\t" \
  "paddw %%mm6,%%mm2\n\t" \
  "movq %%mm2,"OC_MEM_OFFS(0x10,buf)"\n\t" \
  "movq "OC_MEM_OFFS(0x30,c)",%%mm2\n\t" \
  "movq %%mm7,"OC_MEM_OFFS(0x00,buf)"\n\t" \
  /*5-6 rotation by 3pi/16. \
    mm5=C5, mm2=C3, mm3=X3, X5=0.*/ \
  "pmulhw %%mm3,%%mm5\n\t" \
  "pmulhw %%mm3,%%mm2\n\t" \
  "movq "OC_MEM_OFFS(0x10,c)",%%mm7\n\t" \
  "paddw %%mm3,%%mm5\n\t" \
  "paddw %%mm3,%%mm2\n\t" \
  "movq "OC_MEM_OFFS(0x70,c)",%%mm3\n\t" \
  /*4-7 rotation by 7pi/16. \
    mm7=C1, mm3=C7, mm1=X1, X7=0.*/ \
  "pmulhw %%mm1,%%mm3\n\t" \
  "pmulhw %%mm1,%%mm7\n\t" \
  "movq "OC_MEM_OFFS(0x40,c)",%%mm4\n\t" \
  "movq %%mm3,%%mm6\n\t" \
  "paddw %%mm1,%%mm7\n\t" \
  /*0-1 butterfly. \
    mm4=C4, mm0=X0, X4=0.*/ \
  /*Stage 2:*/ \
  /*4-5 butterfly: mm3=t[4], mm5=t[5] \
    7-6 butterfly: mm2=t[6], mm7=t[7]*/ \
  "psubw %%mm5,%%mm3\n\t" \
  "paddw %%mm5,%%mm6\n\t" \
  "movq %%mm4,%%mm1\n\t" \
  "pmulhw %%mm0,%%mm4\n\t" \
  "paddw %%mm0,%%mm4\n\t" \
  "movq %%mm7,%%mm0\n\t" \
  "movq %%mm4,%%mm5\n\t" \
  "paddw %%mm2,%%mm0\n\t" \
  "psubw %%mm2,%%mm7\n\t" \
  "movq %%mm1,%%mm2\n\t" \
  "pmulhw %%mm6,%%mm1\n\t" \
  "pmulhw %%mm7,%%mm2\n\t" \
  "paddw %%mm6,%%mm1\n\t" \
  "movq "OC_MEM_OFFS(0x00,buf)",%%mm6\n\t" \
  "paddw %%mm7,%%mm2\n\t" \
  "movq "OC_MEM_OFFS(0x10,buf)",%%mm7\n\t" \
  /*Stage 3: \
    6-5 butterfly: mm1=t[5], mm2=t[6] -> mm1=t[6]+t[5], mm2=t[6]-t[5] \
    0-3 butterfly: mm4=t[0], mm7=t[3] -> mm7=t[0]+t[3], mm4=t[0]-t[3] \
    1-2 butterfly: mm5=t[1], mm6=t[2] -> mm6=t[1]+t[2], mm5=t[1]-t[2]*/ \
  "paddw %%mm2,%%mm1\n\t" \
  "paddw %%mm5,%%mm6\n\t" \
  "paddw %%mm4,%%mm7\n\t" \
  "paddw %%mm2,%%mm2\n\t" \
  "paddw %%mm4,%%mm4\n\t" \
  "paddw %%mm5,%%mm5\n\t" \
  "psubw %%mm1,%%mm2\n\t" \
  "psubw %%mm7,%%mm4\n\t" \
  "psubw %%mm6,%%mm5\n\t" \
  /*Stage 4: \
    0-7 butterfly: mm7=t[0], mm0=t[7] -> mm0=t[0]+t[7], mm7=t[0]-t[7] \
    1-6 butterfly: mm6=t[1], mm1=t[6] -> mm1=t[1]+t[6], mm6=t[1]-t[6] \
    2-5 butterfly: mm5=t[2], mm2=t[5] -> mm2=t[2]+t[5], mm5=t[2]-t[5] \
    3-4 butterfly: mm4=t[3], mm3=t[4] -> mm3=t[3]+t[4], mm4=t[3]-t[4]*/ \
  "psubw %%mm0,%%mm7\n\t" \
  "psubw %%mm1,%%mm6\n\t" \
  "psubw %%mm2,%%mm5\n\t" \
  "psubw %%mm3,%%mm4\n\t" \
  "paddw %%mm0,%%mm0\n\t" \
  "paddw %%mm1,%%mm1\n\t" \
  "paddw %%mm2,%%mm2\n\t" \
  "paddw %%mm3,%%mm3\n\t" \
  "paddw %%mm7,%%mm0\n\t" \
  "paddw %%mm6,%%mm1\n\t" \
  "paddw %%mm5,%%mm2\n\t" \
  "paddw %%mm4,%%mm3\n\t" \

#define OC_IDCT_8x8_10_ABC \
  "#OC_IDCT_8x8_10_ABC\n\t" \
  /*Stage 1:*/ \
  /*2-3 rotation by 6pi/16. \
    xmm7=C6, xmm6=C2, xmm2=X2, X6=0.*/ \
  "movdqa "OC_MEM_OFFS(0x60,c)",%%xmm7\n\t" \
  "movdqa "OC_MEM_OFFS(0x20,c)",%%xmm6\n\t" \
  "pmulhw %%xmm2,%%xmm6\n\t" \
  "pmulhw %%xmm2,%%xmm7\n\t" \
  "movdqa "OC_MEM_OFFS(0x50,c)",%%xmm5\n\t" \
  "paddw %%xmm6,%%xmm2\n\t" \
  "movdqa %%xmm2,"OC_MEM_OFFS(0x10,buf)"\n\t" \
  "movdqa "OC_MEM_OFFS(0x30,c)",%%xmm2\n\t" \
  "movdqa %%xmm7,"OC_MEM_OFFS(0x00,buf)"\n\t" \
  /*5-6 rotation by 3pi/16. \
    xmm5=C5, xmm2=C3, xmm3=X3, X5=0.*/ \
  "pmulhw %%xmm3,%%xmm5\n\t" \
  "pmulhw %%xmm3,%%xmm2\n\t" \
  "movdqa "OC_MEM_OFFS(0x10,c)",%%xmm7\n\t" \
  "paddw %%xmm3,%%xmm5\n\t" \
  "paddw %%xmm3,%%xmm2\n\t" \
  "movdqa "OC_MEM_OFFS(0x70,c)",%%xmm3\n\t" \
  /*4-7 rotation by 7pi/16. \
    xmm7=C1, xmm3=C7, xmm1=X1, X7=0.*/ \
  "pmulhw %%xmm1,%%xmm3\n\t" \
  "pmulhw %%xmm1,%%xmm7\n\t" \
  "movdqa "OC_MEM_OFFS(0x40,c)",%%xmm4\n\t" \
  "movdqa %%xmm3,%%xmm6\n\t" \
  "paddw %%xmm1,%%xmm7\n\t" \
  /*0-1 butterfly. \
    xmm4=C4, xmm0=X0, X4=0.*/ \
  /*Stage 2:*/ \
  /*4-5 butterfly: xmm3=t[4], xmm5=t[5] \
    7-6 butterfly: xmm2=t[6], xmm7=t[7]*/ \
  "psubw %%xmm5,%%xmm3\n\t" \
  "paddw %%xmm5,%%xmm6\n\t" \
  "movdqa %%xmm4,%%xmm1\n\t" \
  "pmulhw %%xmm0,%%xmm4\n\t" \
  "paddw %%xmm0,%%xmm4\n\t" \
  "movdqa %%xmm7,%%xmm0\n\t" \
  "movdqa %%xmm4,%%xmm5\n\t" \
  "paddw %%xmm2,%%xmm0\n\t" \
  "psubw %%xmm2,%%xmm7\n\t" \
  "movdqa %%xmm1,%%xmm2\n\t" \
  "pmulhw %%xmm6,%%xmm1\n\t" \
  "pmulhw %%xmm7,%%xmm2\n\t" \
  "paddw %%xmm6,%%xmm1\n\t" \
  "movdqa "OC_MEM_OFFS(0x00,buf)",%%xmm6\n\t" \
  "paddw %%xmm7,%%xmm2\n\t" \
  "movdqa "OC_MEM_OFFS(0x10,buf)",%%xmm7\n\t" \
  /*Stage 3: \
    6-5 butterfly: xmm1=t[5], xmm2=t[6] -> xmm1=t[6]+t[5], xmm2=t[6]-t[5] \
    0-3 butterfly: xmm4=t[0], xmm7=t[3] -> xmm7=t[0]+t[3], xmm4=t[0]-t[3] \
    1-2 butterfly: xmm5=t[1], xmm6=t[2] -> xmm6=t[1]+t[2], xmm5=t[1]-t[2]*/ \
  "paddw %%xmm2,%%xmm1\n\t" \
  "paddw %%xmm5,%%xmm6\n\t" \
  "paddw %%xmm4,%%xmm7\n\t" \
  "paddw %%xmm2,%%xmm2\n\t" \
  "paddw %%xmm4,%%xmm4\n\t" \
  "paddw %%xmm5,%%xmm5\n\t" \
  "psubw %%xmm1,%%xmm2\n\t" \
  "psubw %%xmm7,%%xmm4\n\t" \
  "psubw %%xmm6,%%xmm5\n\t" \

static __attribute__((target("sse2"))) void oc_idct8x8_10_sse2(ogg_int16_t _y[64],ogg_int16_t _x[64]){
  OC_ALIGN16(ogg_int16_t buf[16]);
  /*This routine accepts an 8x8 matrix pre-transposed.*/
  __asm__ __volatile__(
    "movq "OC_MEM_OFFS(0x20,x)",%%mm2\n\t"
    "movq "OC_MEM_OFFS(0x30,x)",%%mm3\n\t"
    "movq "OC_MEM_OFFS(0x10,x)",%%mm1\n\t"
    "movq "OC_MEM_OFFS(0x00,x)",%%mm0\n\t"
    OC_IDCT_8x8_10_MMX
    OC_TRANSPOSE_8x4_MMX2SSE
    OC_IDCT_8x8_10_ABC
    OC_IDCT_8x8_D_STORE
    :[buf]"=m"(OC_ARRAY_OPERAND(short,buf,16)),
     [y]"=m"(OC_ARRAY_OPERAND(ogg_int16_t,_y,64))
    :[x]"m"OC_CONST_ARRAY_OPERAND(ogg_int16_t,_x,64),
     [c]"m"(OC_CONST_ARRAY_OPERAND(ogg_int16_t,OC_IDCT_CONSTS,64))
    :"%xmm0", "%xmm1", "%xmm2", "%xmm3",
     "%xmm4", "%xmm5", "%xmm6", "%xmm7"
  );
  /*Clear input data for next block (decoder only).*/
  __asm__ __volatile__(
    "pxor %%mm0,%%mm0\n\t"
    "movq %%mm0,"OC_MEM_OFFS(0x00,x)"\n\t"
    "movq %%mm0,"OC_MEM_OFFS(0x10,x)"\n\t"
    "movq %%mm0,"OC_MEM_OFFS(0x20,x)"\n\t"
    "movq %%mm0,"OC_MEM_OFFS(0x30,x)"\n\t"
    :[x]"+m"(OC_ARRAY_OPERAND(ogg_int16_t,_x,28))
  );
}

/*Performs an inverse 8x8 Type-II DCT transform.
  The input is assumed to be scaled by a factor of 4 relative to orthonormal
   version of the transform.*/
void __attribute__((target("sse2"))) oc_idct8x8_sse2(ogg_int16_t _y[64],ogg_int16_t _x[64],int _last_zzi){
  /*_last_zzi is subtly different from an actual count of the number of
     coefficients we decoded for this block.
    It contains the value of zzi BEFORE the final token in the block was
     decoded.
    In most cases this is an EOB token (the continuation of an EOB run from a
     previous block counts), and so this is the same as the coefficient count.
    However, in the case that the last token was NOT an EOB token, but filled
     the block up with exactly 64 coefficients, _last_zzi will be less than 64.
    Provided the last token was not a pure zero run, the minimum value it can
     be is 46, and so that doesn't affect any of the cases in this routine.
    However, if the last token WAS a pure zero run of length 63, then _last_zzi
     will be 1 while the number of coefficients decoded is 64.
    Thus, we will trigger the following special case, where the real
     coefficient count would not.
    Note also that a zero run of length 64 will give _last_zzi a value of 0,
     but we still process the DC coefficient, which might have a non-zero value
     due to DC prediction.
    Although convoluted, this is arguably the correct behavior: it allows us to
     use a smaller transform when the block ends with a long zero run instead
     of a normal EOB token.
    It could be smarter... multiple separate zero runs at the end of a block
     will fool it, but an encoder that generates these really deserves what it
     gets.
    Needless to say we inherited this approach from VP3.*/
  /*Then perform the iDCT.*/
  if(_last_zzi<=10)oc_idct8x8_10_sse2(_y,_x);
  else oc_idct8x8_slow_sse2(_y,_x);
}

#endif

Coverage Report

Created: 2026-05-24 06:16

Line	Count	Source
1		/********************************************************************
2		* *
3		* THIS FILE IS PART OF THE OggTheora SOFTWARE CODEC SOURCE CODE. *
4		* USE, DISTRIBUTION AND REPRODUCTION OF THIS LIBRARY SOURCE IS *
5		* GOVERNED BY A BSD-STYLE SOURCE LICENSE INCLUDED WITH THIS SOURCE *
6		* IN 'COPYING'. PLEASE READ THESE TERMS BEFORE DISTRIBUTING. *
7		* *
8		* THE Theora SOURCE CODE IS COPYRIGHT (C) 2002-2009,2025 *
9		* by the Xiph.Org Foundation and contributors *
10		* https://www.xiph.org/ *
11		* *
12		********************************************************************
13
14		function:
15
16		********************************************************************/
17
18		/SSE2 acceleration of Theora's iDCT./
19		#include "x86int.h"
20		#include "sse2trans.h"
21		#include "../dct.h"
22
23		#if defined(OC_X86_ASM)
24
25		/A table of constants used by the MMX routines./
26		const unsigned short __attribute__((aligned(16),used)) OC_IDCT_CONSTS[64]={
27		8, 8, 8, 8, 8, 8, 8, 8,
28		OC_C1S7,OC_C1S7,OC_C1S7,OC_C1S7,OC_C1S7,OC_C1S7,OC_C1S7,OC_C1S7,
29		OC_C2S6,OC_C2S6,OC_C2S6,OC_C2S6,OC_C2S6,OC_C2S6,OC_C2S6,OC_C2S6,
30		OC_C3S5,OC_C3S5,OC_C3S5,OC_C3S5,OC_C3S5,OC_C3S5,OC_C3S5,OC_C3S5,
31		OC_C4S4,OC_C4S4,OC_C4S4,OC_C4S4,OC_C4S4,OC_C4S4,OC_C4S4,OC_C4S4,
32		OC_C5S3,OC_C5S3,OC_C5S3,OC_C5S3,OC_C5S3,OC_C5S3,OC_C5S3,OC_C5S3,
33		OC_C6S2,OC_C6S2,OC_C6S2,OC_C6S2,OC_C6S2,OC_C6S2,OC_C6S2,OC_C6S2,
34		OC_C7S1,OC_C7S1,OC_C7S1,OC_C7S1,OC_C7S1,OC_C7S1,OC_C7S1,OC_C7S1
35		};
36
37
38		/*Performs the first three stages of the iDCT.
39		xmm2, xmm6, xmm3, and xmm5 must contain the corresponding rows of the input
40		(accessed in that order).
41		The remaining rows must be in _x at their corresponding locations.
42		On output, xmm7 down to xmm4 contain rows 0 through 3, and xmm0 up to xmm3
43		contain rows 4 through 7.*/
44		#define OC_IDCT_8x8_ABC(_x) \
45		"#OC_IDCT_8x8_ABC\n\t" \
46		/Stage 1:/ \
47		/*2-3 rotation by 6pi/16. \
48		xmm4=xmm7=C6, xmm0=xmm1=C2, xmm2=X2, xmm6=X6.*/ \
49		"movdqa "OC_MEM_OFFS(0x20,c)",%%xmm1\n\t" \
50		"movdqa "OC_MEM_OFFS(0x60,c)",%%xmm4\n\t" \
51		"movdqa %%xmm1,%%xmm0\n\t" \
52		"pmulhw %%xmm2,%%xmm1\n\t" \
53		"movdqa %%xmm4,%%xmm7\n\t" \
54		"pmulhw %%xmm6,%%xmm0\n\t" \
55		"pmulhw %%xmm2,%%xmm7\n\t" \
56		"pmulhw %%xmm6,%%xmm4\n\t" \
57		"paddw %%xmm6,%%xmm0\n\t" \
58		"movdqa "OC_MEM_OFFS(0x30,c)",%%xmm6\n\t" \
59		"paddw %%xmm1,%%xmm2\n\t" \
60		"psubw %%xmm0,%%xmm7\n\t" \
61		"movdqa %%xmm7,"OC_MEM_OFFS(0x00,buf)"\n\t" \
62		"paddw %%xmm4,%%xmm2\n\t" \
63		"movdqa "OC_MEM_OFFS(0x50,c)",%%xmm4\n\t" \
64		"movdqa %%xmm2,"OC_MEM_OFFS(0x10,buf)"\n\t" \
65		/*5-6 rotation by 3pi/16. \
66		xmm4=xmm2=C5, xmm1=xmm6=C3, xmm3=X3, xmm5=X5.*/ \
67		"movdqa %%xmm4,%%xmm2\n\t" \
68		"movdqa %%xmm6,%%xmm1\n\t" \
69		"pmulhw %%xmm3,%%xmm4\n\t" \
70		"pmulhw %%xmm5,%%xmm1\n\t" \
71		"pmulhw %%xmm3,%%xmm6\n\t" \
72		"pmulhw %%xmm5,%%xmm2\n\t" \
73		"paddw %%xmm3,%%xmm4\n\t" \
74		"paddw %%xmm5,%%xmm3\n\t" \
75		"paddw %%xmm6,%%xmm3\n\t" \
76		"movdqa "OC_MEM_OFFS(0x70,_x)",%%xmm6\n\t" \
77		"paddw %%xmm5,%%xmm1\n\t" \
78		"movdqa "OC_MEM_OFFS(0x10,_x)",%%xmm5\n\t" \
79		"paddw %%xmm3,%%xmm2\n\t" \
80		"movdqa "OC_MEM_OFFS(0x70,c)",%%xmm3\n\t" \
81		"psubw %%xmm4,%%xmm1\n\t" \
82		"movdqa "OC_MEM_OFFS(0x10,c)",%%xmm4\n\t" \
83		/*4-7 rotation by 7pi/16. \
84		xmm4=xmm7=C1, xmm3=xmm0=C7, xmm5=X1, xmm6=X7.*/ \
85		"movdqa %%xmm3,%%xmm0\n\t" \
86		"movdqa %%xmm4,%%xmm7\n\t" \
87		"pmulhw %%xmm5,%%xmm3\n\t" \
88		"pmulhw %%xmm5,%%xmm7\n\t" \
89		"pmulhw %%xmm6,%%xmm4\n\t" \
90		"pmulhw %%xmm6,%%xmm0\n\t" \
91		"paddw %%xmm6,%%xmm4\n\t" \
92		"movdqa "OC_MEM_OFFS(0x40,_x)",%%xmm6\n\t" \
93		"paddw %%xmm5,%%xmm7\n\t" \
94		"psubw %%xmm4,%%xmm3\n\t" \
95		"movdqa "OC_MEM_OFFS(0x40,c)",%%xmm4\n\t" \
96		"paddw %%xmm7,%%xmm0\n\t" \
97		"movdqa "OC_MEM_OFFS(0x00,_x)",%%xmm7\n\t" \
98		/*0-1 butterfly. \
99		xmm4=xmm5=C4, xmm7=X0, xmm6=X4.*/ \
100		"paddw %%xmm7,%%xmm6\n\t" \
101		"movdqa %%xmm4,%%xmm5\n\t" \
102		"pmulhw %%xmm6,%%xmm4\n\t" \
103		"paddw %%xmm7,%%xmm7\n\t" \
104		"psubw %%xmm6,%%xmm7\n\t" \
105		"paddw %%xmm6,%%xmm4\n\t" \
106		/Stage 2:/ \
107		/*4-5 butterfly: xmm3=t[4], xmm1=t[5] \
108		7-6 butterfly: xmm2=t[6], xmm0=t[7]*/ \
109		"movdqa %%xmm3,%%xmm6\n\t" \
110		"paddw %%xmm1,%%xmm3\n\t" \
111		"psubw %%xmm1,%%xmm6\n\t" \
112		"movdqa %%xmm5,%%xmm1\n\t" \
113		"pmulhw %%xmm7,%%xmm5\n\t" \
114		"paddw %%xmm7,%%xmm5\n\t" \
115		"movdqa %%xmm0,%%xmm7\n\t" \
116		"paddw %%xmm2,%%xmm0\n\t" \
117		"psubw %%xmm2,%%xmm7\n\t" \
118		"movdqa %%xmm1,%%xmm2\n\t" \
119		"pmulhw %%xmm6,%%xmm1\n\t" \
120		"pmulhw %%xmm7,%%xmm2\n\t" \
121		"paddw %%xmm6,%%xmm1\n\t" \
122		"movdqa "OC_MEM_OFFS(0x00,buf)",%%xmm6\n\t" \
123		"paddw %%xmm7,%%xmm2\n\t" \
124		"movdqa "OC_MEM_OFFS(0x10,buf)",%%xmm7\n\t" \
125		/*Stage 3: \
126		6-5 butterfly: xmm1=t[5], xmm2=t[6] -> xmm1=t[6]+t[5], xmm2=t[6]-t[5] \
127		0-3 butterfly: xmm4=t[0], xmm7=t[3] -> xmm7=t[0]+t[3], xmm4=t[0]-t[3] \
128		1-2 butterfly: xmm5=t[1], xmm6=t[2] -> xmm6=t[1]+t[2], xmm5=t[1]-t[2]*/ \
129		"paddw %%xmm2,%%xmm1\n\t" \
130		"paddw %%xmm5,%%xmm6\n\t" \
131		"paddw %%xmm4,%%xmm7\n\t" \
132		"paddw %%xmm2,%%xmm2\n\t" \
133		"paddw %%xmm4,%%xmm4\n\t" \
134		"paddw %%xmm5,%%xmm5\n\t" \
135		"psubw %%xmm1,%%xmm2\n\t" \
136		"psubw %%xmm7,%%xmm4\n\t" \
137		"psubw %%xmm6,%%xmm5\n\t" \
138
139		/*Performs the last stage of the iDCT.
140		On input, xmm7 down to xmm4 contain rows 0 through 3, and xmm0 up to xmm3
141		contain rows 4 through 7.
142		On output, xmm0 through xmm7 contain the corresponding rows.*/
143		#define OC_IDCT_8x8_D \
144		"#OC_IDCT_8x8_D\n\t" \
145		/*Stage 4: \
146		0-7 butterfly: xmm7=t[0], xmm0=t[7] -> xmm0=t[0]+t[7], xmm7=t[0]-t[7] \
147		1-6 butterfly: xmm6=t[1], xmm1=t[6] -> xmm1=t[1]+t[6], xmm6=t[1]-t[6] \
148		2-5 butterfly: xmm5=t[2], xmm2=t[5] -> xmm2=t[2]+t[5], xmm5=t[2]-t[5] \
149		3-4 butterfly: xmm4=t[3], xmm3=t[4] -> xmm3=t[3]+t[4], xmm4=t[3]-t[4]*/ \
150		"psubw %%xmm0,%%xmm7\n\t" \
151		"psubw %%xmm1,%%xmm6\n\t" \
152		"psubw %%xmm2,%%xmm5\n\t" \
153		"psubw %%xmm3,%%xmm4\n\t" \
154		"paddw %%xmm0,%%xmm0\n\t" \
155		"paddw %%xmm1,%%xmm1\n\t" \
156		"paddw %%xmm2,%%xmm2\n\t" \
157		"paddw %%xmm3,%%xmm3\n\t" \
158		"paddw %%xmm7,%%xmm0\n\t" \
159		"paddw %%xmm6,%%xmm1\n\t" \
160		"paddw %%xmm5,%%xmm2\n\t" \
161		"paddw %%xmm4,%%xmm3\n\t" \
162
163		/*Performs the last stage of the iDCT.
164		On input, xmm7 down to xmm4 contain rows 0 through 3, and xmm0 up to xmm3
165		contain rows 4 through 7.
166		On output, xmm0 through xmm7 contain the corresponding rows.*/
167		#define OC_IDCT_8x8_D_STORE \
168		"#OC_IDCT_8x8_D_STORE\n\t" \
169		/*Stage 4: \
170		0-7 butterfly: xmm7=t[0], xmm0=t[7] -> xmm0=t[0]+t[7], xmm7=t[0]-t[7] \
171		1-6 butterfly: xmm6=t[1], xmm1=t[6] -> xmm1=t[1]+t[6], xmm6=t[1]-t[6] \
172		2-5 butterfly: xmm5=t[2], xmm2=t[5] -> xmm2=t[2]+t[5], xmm5=t[2]-t[5] \
173		3-4 butterfly: xmm4=t[3], xmm3=t[4] -> xmm3=t[3]+t[4], xmm4=t[3]-t[4]*/ \
174		"psubw %%xmm3,%%xmm4\n\t" \
175		"movdqa %%xmm4,"OC_MEM_OFFS(0x40,y)"\n\t" \
176		"movdqa "OC_MEM_OFFS(0x00,c)",%%xmm4\n\t" \
177		"psubw %%xmm0,%%xmm7\n\t" \
178		"psubw %%xmm1,%%xmm6\n\t" \
179		"psubw %%xmm2,%%xmm5\n\t" \
180		"paddw %%xmm4,%%xmm7\n\t" \
181		"paddw %%xmm4,%%xmm6\n\t" \
182		"paddw %%xmm4,%%xmm5\n\t" \
183		"paddw "OC_MEM_OFFS(0x40,y)",%%xmm4\n\t" \
184		"paddw %%xmm0,%%xmm0\n\t" \
185		"paddw %%xmm1,%%xmm1\n\t" \
186		"paddw %%xmm2,%%xmm2\n\t" \
187		"paddw %%xmm3,%%xmm3\n\t" \
188		"paddw %%xmm7,%%xmm0\n\t" \
189		"paddw %%xmm6,%%xmm1\n\t" \
190		"psraw $4,%%xmm0\n\t" \
191		"paddw %%xmm5,%%xmm2\n\t" \
192		"movdqa %%xmm0,"OC_MEM_OFFS(0x00,y)"\n\t" \
193		"psraw $4,%%xmm1\n\t" \
194		"paddw %%xmm4,%%xmm3\n\t" \
195		"movdqa %%xmm1,"OC_MEM_OFFS(0x10,y)"\n\t" \
196		"psraw $4,%%xmm2\n\t" \
197		"movdqa %%xmm2,"OC_MEM_OFFS(0x20,y)"\n\t" \
198		"psraw $4,%%xmm3\n\t" \
199		"movdqa %%xmm3,"OC_MEM_OFFS(0x30,y)"\n\t" \
200		"psraw $4,%%xmm4\n\t" \
201		"movdqa %%xmm4,"OC_MEM_OFFS(0x40,y)"\n\t" \
202		"psraw $4,%%xmm5\n\t" \
203		"movdqa %%xmm5,"OC_MEM_OFFS(0x50,y)"\n\t" \
204		"psraw $4,%%xmm6\n\t" \
205		"movdqa %%xmm6,"OC_MEM_OFFS(0x60,y)"\n\t" \
206		"psraw $4,%%xmm7\n\t" \
207		"movdqa %%xmm7,"OC_MEM_OFFS(0x70,y)"\n\t" \
208
209	2.05M	static void __attribute__((target("sse2"))) oc_idct8x8_slow_sse2(ogg_int16_t _y[64],ogg_int16_t _x[64]){
210	2.05M	OC_ALIGN16(ogg_int16_t buf[16]);
211	2.05M	int i;
212		/This routine accepts an 8x8 matrix pre-transposed./
213	2.05M	__asm__ __volatile__(
214		/Load rows 2, 3, 5, and 6 for the first stage of the iDCT./
215	2.05M	"movdqa "OC_MEM_OFFS(0x20,x)",%%xmm2\n\t"
216	2.05M	"movdqa "OC_MEM_OFFS(0x60,x)",%%xmm6\n\t"
217	2.05M	"movdqa "OC_MEM_OFFS(0x30,x)",%%xmm3\n\t"
218	2.05M	"movdqa "OC_MEM_OFFS(0x50,x)",%%xmm5\n\t"
219	2.05M	OC_IDCT_8x8_ABC(x)
220	2.05M	OC_IDCT_8x8_D
221	2.05M	OC_TRANSPOSE_8x8
222		/Clear out rows 0, 1, 4, and 7 for the first stage of the iDCT./
223	2.05M	"movdqa %%xmm7,"OC_MEM_OFFS(0x70,y)"\n\t"
224	2.05M	"movdqa %%xmm4,"OC_MEM_OFFS(0x40,y)"\n\t"
225	2.05M	"movdqa %%xmm1,"OC_MEM_OFFS(0x10,y)"\n\t"
226	2.05M	"movdqa %%xmm0,"OC_MEM_OFFS(0x00,y)"\n\t"
227	2.05M	OC_IDCT_8x8_ABC(y)
228	2.05M	OC_IDCT_8x8_D_STORE
229	2.05M	:[buf]"=m"(OC_ARRAY_OPERAND(ogg_int16_t,buf,16)),
230	2.05M	[y]"=m"(OC_ARRAY_OPERAND(ogg_int16_t,_y,64))
231	2.05M	:[x]"m"(OC_CONST_ARRAY_OPERAND(ogg_int16_t,_x,64)),
232	2.05M	[c]"m"(OC_CONST_ARRAY_OPERAND(ogg_int16_t,OC_IDCT_CONSTS,64))
233	2.05M	:"%xmm0", "%xmm1", "%xmm2", "%xmm3",
234	2.05M	"%xmm4", "%xmm5", "%xmm6", "%xmm7"
235	2.05M	);
236		/Store 0 in zr and use it in the loop. This translates to nothing with -O1./
237	2.05M	register sse2_reg zr;
238	2.05M	__asm__ __volatile__("pxor %[zr],%[zr]\n\t":[zr]"=x"(zr));
239		/Clear input data for next block (decoder only)./
240	6.17M	for(i=0;i<2;i++){
241	4.11M	__asm__ __volatile__(
242	4.11M	"movdqa %[zr],"OC_MEM_OFFS(0x00,x)"\n\t"
243	4.11M	"movdqa %[zr],"OC_MEM_OFFS(0x10,x)"\n\t"
244	4.11M	"movdqa %[zr],"OC_MEM_OFFS(0x20,x)"\n\t"
245	4.11M	"movdqa %[zr],"OC_MEM_OFFS(0x30,x)"\n\t"
246	4.11M	:[x]"=m"(OC_ARRAY_OPERAND(ogg_int16_t,_x+i*32,32))
247	4.11M	:[zr]"x"(zr)
248	4.11M	);
249	4.11M	}
250	2.05M	}
251
252		/*For the first step of the 10-coefficient version of the 8x8 iDCT, we only
253		need to work with four columns at a time.
254		Doing this in MMX is faster on processors with a 64-bit data path.*/
255		#define OC_IDCT_8x8_10_MMX \
256		"#OC_IDCT_8x8_10_MMX\n\t" \
257		/Stage 1:/ \
258		/*2-3 rotation by 6pi/16. \
259		mm7=C6, mm6=C2, mm2=X2, X6=0.*/ \
260		"movq "OC_MEM_OFFS(0x60,c)",%%mm7\n\t" \
261		"movq "OC_MEM_OFFS(0x20,c)",%%mm6\n\t" \
262		"pmulhw %%mm2,%%mm6\n\t" \
263		"pmulhw %%mm2,%%mm7\n\t" \
264		"movq "OC_MEM_OFFS(0x50,c)",%%mm5\n\t" \
265		"paddw %%mm6,%%mm2\n\t" \
266		"movq %%mm2,"OC_MEM_OFFS(0x10,buf)"\n\t" \
267		"movq "OC_MEM_OFFS(0x30,c)",%%mm2\n\t" \
268		"movq %%mm7,"OC_MEM_OFFS(0x00,buf)"\n\t" \
269		/*5-6 rotation by 3pi/16. \
270		mm5=C5, mm2=C3, mm3=X3, X5=0.*/ \
271		"pmulhw %%mm3,%%mm5\n\t" \
272		"pmulhw %%mm3,%%mm2\n\t" \
273		"movq "OC_MEM_OFFS(0x10,c)",%%mm7\n\t" \
274		"paddw %%mm3,%%mm5\n\t" \
275		"paddw %%mm3,%%mm2\n\t" \
276		"movq "OC_MEM_OFFS(0x70,c)",%%mm3\n\t" \
277		/*4-7 rotation by 7pi/16. \
278		mm7=C1, mm3=C7, mm1=X1, X7=0.*/ \
279		"pmulhw %%mm1,%%mm3\n\t" \
280		"pmulhw %%mm1,%%mm7\n\t" \
281		"movq "OC_MEM_OFFS(0x40,c)",%%mm4\n\t" \
282		"movq %%mm3,%%mm6\n\t" \
283		"paddw %%mm1,%%mm7\n\t" \
284		/*0-1 butterfly. \
285		mm4=C4, mm0=X0, X4=0.*/ \
286		/Stage 2:/ \
287		/*4-5 butterfly: mm3=t[4], mm5=t[5] \
288		7-6 butterfly: mm2=t[6], mm7=t[7]*/ \
289		"psubw %%mm5,%%mm3\n\t" \
290		"paddw %%mm5,%%mm6\n\t" \
291		"movq %%mm4,%%mm1\n\t" \
292		"pmulhw %%mm0,%%mm4\n\t" \
293		"paddw %%mm0,%%mm4\n\t" \
294		"movq %%mm7,%%mm0\n\t" \
295		"movq %%mm4,%%mm5\n\t" \
296		"paddw %%mm2,%%mm0\n\t" \
297		"psubw %%mm2,%%mm7\n\t" \
298		"movq %%mm1,%%mm2\n\t" \
299		"pmulhw %%mm6,%%mm1\n\t" \
300		"pmulhw %%mm7,%%mm2\n\t" \
301		"paddw %%mm6,%%mm1\n\t" \
302		"movq "OC_MEM_OFFS(0x00,buf)",%%mm6\n\t" \
303		"paddw %%mm7,%%mm2\n\t" \
304		"movq "OC_MEM_OFFS(0x10,buf)",%%mm7\n\t" \
305		/*Stage 3: \
306		6-5 butterfly: mm1=t[5], mm2=t[6] -> mm1=t[6]+t[5], mm2=t[6]-t[5] \
307		0-3 butterfly: mm4=t[0], mm7=t[3] -> mm7=t[0]+t[3], mm4=t[0]-t[3] \
308		1-2 butterfly: mm5=t[1], mm6=t[2] -> mm6=t[1]+t[2], mm5=t[1]-t[2]*/ \
309		"paddw %%mm2,%%mm1\n\t" \
310		"paddw %%mm5,%%mm6\n\t" \
311		"paddw %%mm4,%%mm7\n\t" \
312		"paddw %%mm2,%%mm2\n\t" \
313		"paddw %%mm4,%%mm4\n\t" \
314		"paddw %%mm5,%%mm5\n\t" \
315		"psubw %%mm1,%%mm2\n\t" \
316		"psubw %%mm7,%%mm4\n\t" \
317		"psubw %%mm6,%%mm5\n\t" \
318		/*Stage 4: \
319		0-7 butterfly: mm7=t[0], mm0=t[7] -> mm0=t[0]+t[7], mm7=t[0]-t[7] \
320		1-6 butterfly: mm6=t[1], mm1=t[6] -> mm1=t[1]+t[6], mm6=t[1]-t[6] \
321		2-5 butterfly: mm5=t[2], mm2=t[5] -> mm2=t[2]+t[5], mm5=t[2]-t[5] \
322		3-4 butterfly: mm4=t[3], mm3=t[4] -> mm3=t[3]+t[4], mm4=t[3]-t[4]*/ \
323		"psubw %%mm0,%%mm7\n\t" \
324		"psubw %%mm1,%%mm6\n\t" \
325		"psubw %%mm2,%%mm5\n\t" \
326		"psubw %%mm3,%%mm4\n\t" \
327		"paddw %%mm0,%%mm0\n\t" \
328		"paddw %%mm1,%%mm1\n\t" \
329		"paddw %%mm2,%%mm2\n\t" \
330		"paddw %%mm3,%%mm3\n\t" \
331		"paddw %%mm7,%%mm0\n\t" \
332		"paddw %%mm6,%%mm1\n\t" \
333		"paddw %%mm5,%%mm2\n\t" \
334		"paddw %%mm4,%%mm3\n\t" \
335
336		#define OC_IDCT_8x8_10_ABC \
337		"#OC_IDCT_8x8_10_ABC\n\t" \
338		/Stage 1:/ \
339		/*2-3 rotation by 6pi/16. \
340		xmm7=C6, xmm6=C2, xmm2=X2, X6=0.*/ \
341		"movdqa "OC_MEM_OFFS(0x60,c)",%%xmm7\n\t" \
342		"movdqa "OC_MEM_OFFS(0x20,c)",%%xmm6\n\t" \
343		"pmulhw %%xmm2,%%xmm6\n\t" \
344		"pmulhw %%xmm2,%%xmm7\n\t" \
345		"movdqa "OC_MEM_OFFS(0x50,c)",%%xmm5\n\t" \
346		"paddw %%xmm6,%%xmm2\n\t" \
347		"movdqa %%xmm2,"OC_MEM_OFFS(0x10,buf)"\n\t" \
348		"movdqa "OC_MEM_OFFS(0x30,c)",%%xmm2\n\t" \
349		"movdqa %%xmm7,"OC_MEM_OFFS(0x00,buf)"\n\t" \
350		/*5-6 rotation by 3pi/16. \
351		xmm5=C5, xmm2=C3, xmm3=X3, X5=0.*/ \
352		"pmulhw %%xmm3,%%xmm5\n\t" \
353		"pmulhw %%xmm3,%%xmm2\n\t" \
354		"movdqa "OC_MEM_OFFS(0x10,c)",%%xmm7\n\t" \
355		"paddw %%xmm3,%%xmm5\n\t" \
356		"paddw %%xmm3,%%xmm2\n\t" \
357		"movdqa "OC_MEM_OFFS(0x70,c)",%%xmm3\n\t" \
358		/*4-7 rotation by 7pi/16. \
359		xmm7=C1, xmm3=C7, xmm1=X1, X7=0.*/ \
360		"pmulhw %%xmm1,%%xmm3\n\t" \
361		"pmulhw %%xmm1,%%xmm7\n\t" \
362		"movdqa "OC_MEM_OFFS(0x40,c)",%%xmm4\n\t" \
363		"movdqa %%xmm3,%%xmm6\n\t" \
364		"paddw %%xmm1,%%xmm7\n\t" \
365		/*0-1 butterfly. \
366		xmm4=C4, xmm0=X0, X4=0.*/ \
367		/Stage 2:/ \
368		/*4-5 butterfly: xmm3=t[4], xmm5=t[5] \
369		7-6 butterfly: xmm2=t[6], xmm7=t[7]*/ \
370		"psubw %%xmm5,%%xmm3\n\t" \
371		"paddw %%xmm5,%%xmm6\n\t" \
372		"movdqa %%xmm4,%%xmm1\n\t" \
373		"pmulhw %%xmm0,%%xmm4\n\t" \
374		"paddw %%xmm0,%%xmm4\n\t" \
375		"movdqa %%xmm7,%%xmm0\n\t" \
376		"movdqa %%xmm4,%%xmm5\n\t" \
377		"paddw %%xmm2,%%xmm0\n\t" \
378		"psubw %%xmm2,%%xmm7\n\t" \
379		"movdqa %%xmm1,%%xmm2\n\t" \
380		"pmulhw %%xmm6,%%xmm1\n\t" \
381		"pmulhw %%xmm7,%%xmm2\n\t" \
382		"paddw %%xmm6,%%xmm1\n\t" \
383		"movdqa "OC_MEM_OFFS(0x00,buf)",%%xmm6\n\t" \
384		"paddw %%xmm7,%%xmm2\n\t" \
385		"movdqa "OC_MEM_OFFS(0x10,buf)",%%xmm7\n\t" \
386		/*Stage 3: \
387		6-5 butterfly: xmm1=t[5], xmm2=t[6] -> xmm1=t[6]+t[5], xmm2=t[6]-t[5] \
388		0-3 butterfly: xmm4=t[0], xmm7=t[3] -> xmm7=t[0]+t[3], xmm4=t[0]-t[3] \
389		1-2 butterfly: xmm5=t[1], xmm6=t[2] -> xmm6=t[1]+t[2], xmm5=t[1]-t[2]*/ \
390		"paddw %%xmm2,%%xmm1\n\t" \
391		"paddw %%xmm5,%%xmm6\n\t" \
392		"paddw %%xmm4,%%xmm7\n\t" \
393		"paddw %%xmm2,%%xmm2\n\t" \
394		"paddw %%xmm4,%%xmm4\n\t" \
395		"paddw %%xmm5,%%xmm5\n\t" \
396		"psubw %%xmm1,%%xmm2\n\t" \
397		"psubw %%xmm7,%%xmm4\n\t" \
398		"psubw %%xmm6,%%xmm5\n\t" \
399
400	2.50M	static __attribute__((target("sse2"))) void oc_idct8x8_10_sse2(ogg_int16_t _y[64],ogg_int16_t _x[64]){
401	2.50M	OC_ALIGN16(ogg_int16_t buf[16]);
402		/This routine accepts an 8x8 matrix pre-transposed./
403	2.50M	__asm__ __volatile__(
404	2.50M	"movq "OC_MEM_OFFS(0x20,x)",%%mm2\n\t"
405	2.50M	"movq "OC_MEM_OFFS(0x30,x)",%%mm3\n\t"
406	2.50M	"movq "OC_MEM_OFFS(0x10,x)",%%mm1\n\t"
407	2.50M	"movq "OC_MEM_OFFS(0x00,x)",%%mm0\n\t"
408	2.50M	OC_IDCT_8x8_10_MMX
409	2.50M	OC_TRANSPOSE_8x4_MMX2SSE
410	2.50M	OC_IDCT_8x8_10_ABC
411	2.50M	OC_IDCT_8x8_D_STORE
412	2.50M	:[buf]"=m"(OC_ARRAY_OPERAND(short,buf,16)),
413	2.50M	[y]"=m"(OC_ARRAY_OPERAND(ogg_int16_t,_y,64))
414	2.50M	:[x]"m"OC_CONST_ARRAY_OPERAND(ogg_int16_t,_x,64),
415	2.50M	[c]"m"(OC_CONST_ARRAY_OPERAND(ogg_int16_t,OC_IDCT_CONSTS,64))
416	2.50M	:"%xmm0", "%xmm1", "%xmm2", "%xmm3",
417	2.50M	"%xmm4", "%xmm5", "%xmm6", "%xmm7"
418	2.50M	);
419		/Clear input data for next block (decoder only)./
420	2.50M	__asm__ __volatile__(
421	2.50M	"pxor %%mm0,%%mm0\n\t"
422	2.50M	"movq %%mm0,"OC_MEM_OFFS(0x00,x)"\n\t"
423	2.50M	"movq %%mm0,"OC_MEM_OFFS(0x10,x)"\n\t"
424	2.50M	"movq %%mm0,"OC_MEM_OFFS(0x20,x)"\n\t"
425	2.50M	"movq %%mm0,"OC_MEM_OFFS(0x30,x)"\n\t"
426	2.50M	:[x]"+m"(OC_ARRAY_OPERAND(ogg_int16_t,_x,28))
427	2.50M	);
428	2.50M	}
429
430		/*Performs an inverse 8x8 Type-II DCT transform.
431		The input is assumed to be scaled by a factor of 4 relative to orthonormal
432		version of the transform.*/
433	4.55M	void __attribute__((target("sse2"))) oc_idct8x8_sse2(ogg_int16_t _y[64],ogg_int16_t _x[64],int _last_zzi){
434		/*_last_zzi is subtly different from an actual count of the number of
435		coefficients we decoded for this block.
436		It contains the value of zzi BEFORE the final token in the block was
437		decoded.
438		In most cases this is an EOB token (the continuation of an EOB run from a
439		previous block counts), and so this is the same as the coefficient count.
440		However, in the case that the last token was NOT an EOB token, but filled
441		the block up with exactly 64 coefficients, _last_zzi will be less than 64.
442		Provided the last token was not a pure zero run, the minimum value it can
443		be is 46, and so that doesn't affect any of the cases in this routine.
444		However, if the last token WAS a pure zero run of length 63, then _last_zzi
445		will be 1 while the number of coefficients decoded is 64.
446		Thus, we will trigger the following special case, where the real
447		coefficient count would not.
448		Note also that a zero run of length 64 will give _last_zzi a value of 0,
449		but we still process the DC coefficient, which might have a non-zero value
450		due to DC prediction.
451		Although convoluted, this is arguably the correct behavior: it allows us to
452		use a smaller transform when the block ends with a long zero run instead
453		of a normal EOB token.
454		It could be smarter... multiple separate zero runs at the end of a block
455		will fool it, but an encoder that generates these really deserves what it
456		gets.
457		Needless to say we inherited this approach from VP3.*/
458		/Then perform the iDCT./
459	4.55M	if(_last_zzi<=10)oc_idct8x8_10_sse2(_y,_x);
460	2.05M	else oc_idct8x8_slow_sse2(_y,_x);
461	4.55M	}
462
463		#endif