/rust/registry/src/index.crates.io-1949cf8c6b5b557f/jpeg-encoder-0.7.1/src/avx2/fdct.rs

Source
/*
 * Ported from mozjpeg / jfdctint-avx2.asm to rust
 * Copyright 2009 Pierre Ossman <ossman@cendio.se> for Cendio AB
 * Copyright (C) 2009, 2016, 2018, 2020, D. R. Commander.
 *
 * Based on the x86 SIMD extension for IJG JPEG library
 * Copyright (C) 1999-2006, MIYASAKA Masaru.
 */

#[cfg(target_arch = "x86")]
use core::arch::x86::{
    __m256i, _mm256_add_epi16, _mm256_add_epi32, _mm256_loadu_si256, _mm256_madd_epi16,
    _mm256_packs_epi32, _mm256_permute2x128_si256, _mm256_permute4x64_epi64, _mm256_set_epi16,
    _mm256_set_epi32, _mm256_sign_epi16, _mm256_slli_epi16, _mm256_srai_epi16, _mm256_srai_epi32,
    _mm256_storeu_si256, _mm256_sub_epi16, _mm256_unpackhi_epi16, _mm256_unpackhi_epi32,
    _mm256_unpacklo_epi16, _mm256_unpacklo_epi32,
};

#[cfg(target_arch = "x86_64")]
use core::arch::x86_64::{
    __m256i, _mm256_add_epi16, _mm256_add_epi32, _mm256_loadu_si256, _mm256_madd_epi16,
    _mm256_packs_epi32, _mm256_permute2x128_si256, _mm256_permute4x64_epi64, _mm256_set_epi16,
    _mm256_set_epi32, _mm256_sign_epi16, _mm256_slli_epi16, _mm256_srai_epi16, _mm256_srai_epi32,
    _mm256_storeu_si256, _mm256_sub_epi16, _mm256_unpackhi_epi16, _mm256_unpackhi_epi32,
    _mm256_unpacklo_epi16, _mm256_unpacklo_epi32,
};

use crate::encoder::AlignedBlock;

const CONST_BITS: i32 = 13;
const PASS1_BITS: i32 = 2;

// FIX(0.298631336)
const F_0_298: i16 = 2446;
// FIX(0.390180644)
const F_0_390: i16 = 3196;
// FIX(0.541196100)
const F_0_541: i16 = 4433;
// FIX(0.765366865)
const F_0_765: i16 = 6270;
//FIX(0.899976223)
const F_0_899: i16 = 7373;
//FIX(1.175875602)
const F_1_175: i16 = 9633;
//FIX(1.501321110)
const F_1_501: i16 = 12299;
//FIX(1.847759065)
const F_1_847: i16 = 15137;
//FIX(1.961570560)
const F_1_961: i16 = 16069;
//FIX(2.053119869)
const F_2_053: i16 = 16819;
//FIX(2.562915447)
const F_2_562: i16 = 20995;
//FIX(3.072711026)
const F_3_072: i16 = 25172;

const DESCALE_P1: i32 = CONST_BITS - PASS1_BITS;
const DESCALE_P2: i32 = CONST_BITS + PASS1_BITS;

#[inline(always)]
pub fn fdct_avx2(data: &mut AlignedBlock) {
    unsafe {
        fdct_avx2_internal(data);
    }
}

#[target_feature(enable = "avx2")]
fn fdct_avx2_internal(data: &mut AlignedBlock) {
    #[target_feature(enable = "avx2")]
    #[allow(non_snake_case)]
    #[inline]
    fn PW_F130_F054_MF130_F054() -> __m256i {
        _mm256_set_epi16(
            F_0_541,
            F_0_541 - F_1_847,
            F_0_541,
            F_0_541 - F_1_847,
            F_0_541,
            F_0_541 - F_1_847,
            F_0_541,
            F_0_541 - F_1_847,
            F_0_541,
            F_0_541 + F_0_765,
            F_0_541,
            F_0_541 + F_0_765,
            F_0_541,
            F_0_541 + F_0_765,
            F_0_541,
            F_0_541 + F_0_765,
        )
    }

    #[target_feature(enable = "avx2")]
    #[allow(non_snake_case)]
    #[inline]
    fn PW_MF078_F117_F078_F117() -> __m256i {
        _mm256_set_epi16(
            F_1_175,
            F_1_175 - F_0_390,
            F_1_175,
            F_1_175 - F_0_390,
            F_1_175,
            F_1_175 - F_0_390,
            F_1_175,
            F_1_175 - F_0_390,
            F_1_175,
            F_1_175 - F_1_961,
            F_1_175,
            F_1_175 - F_1_961,
            F_1_175,
            F_1_175 - F_1_961,
            F_1_175,
            F_1_175 - F_1_961,
        )
    }

    #[target_feature(enable = "avx2")]
    #[allow(non_snake_case)]
    #[inline]
    fn PW_MF060_MF089_MF050_MF256() -> __m256i {
        _mm256_set_epi16(
            -F_2_562,
            F_2_053 - F_2_562,
            -F_2_562,
            F_2_053 - F_2_562,
            -F_2_562,
            F_2_053 - F_2_562,
            -F_2_562,
            F_2_053 - F_2_562,
            -F_0_899,
            F_0_298 - F_0_899,
            -F_0_899,
            F_0_298 - F_0_899,
            -F_0_899,
            F_0_298 - F_0_899,
            -F_0_899,
            F_0_298 - F_0_899,
        )
    }

    #[target_feature(enable = "avx2")]
    #[allow(non_snake_case)]
    #[inline]
    fn PW_F050_MF256_F060_MF089() -> __m256i {
        _mm256_set_epi16(
            -F_0_899,
            F_1_501 - F_0_899,
            -F_0_899,
            F_1_501 - F_0_899,
            -F_0_899,
            F_1_501 - F_0_899,
            -F_0_899,
            F_1_501 - F_0_899,
            -F_2_562,
            F_3_072 - F_2_562,
            -F_2_562,
            F_3_072 - F_2_562,
            -F_2_562,
            F_3_072 - F_2_562,
            -F_2_562,
            F_3_072 - F_2_562,
        )
    }

    #[target_feature(enable = "avx2")]
    #[allow(non_snake_case)]
    #[inline]
    fn PD_DESCALE_P(first_pass: bool) -> __m256i {
        if first_pass {
            _mm256_set_epi32(
                1 << (DESCALE_P1 - 1),
                1 << (DESCALE_P1 - 1),
                1 << (DESCALE_P1 - 1),
                1 << (DESCALE_P1 - 1),
                1 << (DESCALE_P1 - 1),
                1 << (DESCALE_P1 - 1),
                1 << (DESCALE_P1 - 1),
                1 << (DESCALE_P1 - 1),
            )
        } else {
            _mm256_set_epi32(
                1 << (DESCALE_P2 - 1),
                1 << (DESCALE_P2 - 1),
                1 << (DESCALE_P2 - 1),
                1 << (DESCALE_P2 - 1),
                1 << (DESCALE_P2 - 1),
                1 << (DESCALE_P2 - 1),
                1 << (DESCALE_P2 - 1),
                1 << (DESCALE_P2 - 1),
            )
        }
    }

    #[target_feature(enable = "avx2")]
    #[allow(non_snake_case)]
    #[inline]
    fn PW_DESCALE_P2X() -> __m256i {
        _mm256_set_epi32(
            1 << (PASS1_BITS - 1),
            1 << (PASS1_BITS - 1),
            1 << (PASS1_BITS - 1),
            1 << (PASS1_BITS - 1),
            1 << (PASS1_BITS - 1),
            1 << (PASS1_BITS - 1),
            1 << (PASS1_BITS - 1),
            1 << (PASS1_BITS - 1),
        )
    }

    // In-place 8x8x16-bit matrix transpose using AVX2 instructions
    #[target_feature(enable = "avx2")]
    #[inline]
    fn do_transpose(
        i1: __m256i,
        i2: __m256i,
        i3: __m256i,
        i4: __m256i,
    ) -> (__m256i, __m256i, __m256i, __m256i) {
        //i1=(00 01 02 03 04 05 06 07  40 41 42 43 44 45 46 47)
        //i2=(10 11 12 13 14 15 16 17  50 51 52 53 54 55 56 57)
        //i3=(20 21 22 23 24 25 26 27  60 61 62 63 64 65 66 67)
        //i4=(30 31 32 33 34 35 36 37  70 71 72 73 74 75 76 77)

        let t5 = _mm256_unpacklo_epi16(i1, i2);
        let t6 = _mm256_unpackhi_epi16(i1, i2);
        let t7 = _mm256_unpacklo_epi16(i3, i4);
        let t8 = _mm256_unpackhi_epi16(i3, i4);

        // transpose coefficients(phase 1)
        // t1=(00 10 01 11 02 12 03 13  40 50 41 51 42 52 43 53)
        // t2=(04 14 05 15 06 16 07 17  44 54 45 55 46 56 47 57)
        // t3=(20 30 21 31 22 32 23 33  60 70 61 71 62 72 63 73)
        // t4=(24 34 25 35 26 36 27 37  64 74 65 75 66 76 67 77)

        let t1 = _mm256_unpacklo_epi32(t5, t7);
        let t2 = _mm256_unpackhi_epi32(t5, t7);
        let t3 = _mm256_unpacklo_epi32(t6, t8);
        let t4 = _mm256_unpackhi_epi32(t6, t8);

        // transpose coefficients(phase 2)
        // t5=(00 10 20 30 01 11 21 31  40 50 60 70 41 51 61 71)
        // t6=(02 12 22 32 03 13 23 33  42 52 62 72 43 53 63 73)
        // t7=(04 14 24 34 05 15 25 35  44 54 64 74 45 55 65 75)
        // t8=(06 16 26 36 07 17 27 37  46 56 66 76 47 57 67 77)

        (
            _mm256_permute4x64_epi64(t1, 0x8D),
            _mm256_permute4x64_epi64(t2, 0x8D),
            _mm256_permute4x64_epi64(t3, 0xD8),
            _mm256_permute4x64_epi64(t4, 0xD8),
        )
    }

    // In-place 8x8x16-bit accurate integer forward DCT using AVX2 instructions
    #[target_feature(enable = "avx2")]
    #[inline]
    fn do_dct(
        first_pass: bool,
        i1: __m256i,
        i2: __m256i,
        i3: __m256i,
        i4: __m256i,
    ) -> (__m256i, __m256i, __m256i, __m256i) {
        let t5 = _mm256_sub_epi16(i1, i4); // data1_0 - data6_7 = tmp6_7
        let t6 = _mm256_add_epi16(i1, i4); // data1_0 + data6_7 = tmp1_0
        let t7 = _mm256_add_epi16(i2, i3); // data3_2 + data4_5 = tmp3_2
        let t8 = _mm256_sub_epi16(i2, i3); // data3_2 - data4_5 = tmp4_5

        // Even part

        let t6 = _mm256_permute2x128_si256(t6, t6, 0x01); // t6=tmp0_1
        let t1 = _mm256_add_epi16(t6, t7); // t1 = tmp0_1 + tmp3_2 = tmp10_11
        let t6 = _mm256_sub_epi16(t6, t7); // t6 = tmp0_1 - tmp3_2 = tmp13_12

        let t7 = _mm256_permute2x128_si256(t1, t1, 0x01); // t7 = tmp11_10
        let t1 = _mm256_sign_epi16(
            t1,
            _mm256_set_epi16(-1, -1, -1, -1, -1, -1, -1, -1, 1, 1, 1, 1, 1, 1, 1, 1),
        ); // tmp10_neg11

        let t7 = _mm256_add_epi16(t7, t1); // t7 = (tmp10 + tmp11)_(tmp10 - tmp11)

        let t1 = if first_pass {
            _mm256_slli_epi16(t7, PASS1_BITS)
        } else {
            let t7 = _mm256_add_epi16(t7, PW_DESCALE_P2X());
            _mm256_srai_epi16(t7, PASS1_BITS)
        };

        // (Original)
        // z1 = (tmp12 + tmp13) * 0.541196100;
        // data2 = z1 + tmp13 * 0.765366865;
        // data6 = z1 + tmp12 * -1.847759065;
        //
        // (This implementation)
        // data2 = tmp13 * (0.541196100 + 0.765366865) + tmp12 * 0.541196100;
        // data6 = tmp13 * 0.541196100 + tmp12 * (0.541196100 - 1.847759065);

        let t7 = _mm256_permute2x128_si256(t6, t6, 0x01); // t7 = tmp12_13
        let t2 = _mm256_unpacklo_epi16(t6, t7);
        let t6 = _mm256_unpackhi_epi16(t6, t7);

        let t2 = _mm256_madd_epi16(t2, PW_F130_F054_MF130_F054()); // t2 = data2_6L
        let t6 = _mm256_madd_epi16(t6, PW_F130_F054_MF130_F054()); // t6 = data2_6H

        let t2 = _mm256_add_epi32(t2, PD_DESCALE_P(first_pass));
        let t6 = _mm256_add_epi32(t6, PD_DESCALE_P(first_pass));

        let t2 = if first_pass {
            _mm256_srai_epi32(t2, DESCALE_P1)
        } else {
            _mm256_srai_epi32(t2, DESCALE_P2)
        };
        let t6 = if first_pass {
            _mm256_srai_epi32(t6, DESCALE_P1)
        } else {
            _mm256_srai_epi32(t6, DESCALE_P2)
        };

        let t3 = _mm256_packs_epi32(t2, t6); // t6 = data2_6

        // Odd part

        let t7 = _mm256_add_epi16(t8, t5); // t7 = tmp4_5 + tmp6_7 = z3_4

        // (Original)
        // z5 = (z3 + z4) * 1.175875602;
        // z3 = z3 * -1.961570560;
        // z4 = z4 * -0.390180644;
        // z3 += z5;
        // z4 += z5;
        //
        // (This implementation)
        // z3 = z3 * (1.175875602 - 1.961570560) + z4 * 1.175875602;
        // z4 = z3 * 1.175875602 + z4 * (1.175875602 - 0.390180644);

        let t2 = _mm256_permute2x128_si256(t7, t7, 0x01); // t2 = z4_3
        let t6 = _mm256_unpacklo_epi16(t7, t2);
        let t7 = _mm256_unpackhi_epi16(t7, t2);

        let t6 = _mm256_madd_epi16(t6, PW_MF078_F117_F078_F117()); // t6 = z3_4L
        let t7 = _mm256_madd_epi16(t7, PW_MF078_F117_F078_F117()); // t7 = z3_4H

        // (Original)
        // z1 = tmp4 + tmp7;
        // z2 = tmp5 + tmp6;
        // tmp4 = tmp4 * 0.298631336;
        // tmp5 = tmp5 * 2.053119869;
        // tmp6 = tmp6 * 3.072711026;
        // tmp7 = tmp7 * 1.501321110;
        // z1 = z1 * -0.899976223;
        // z2 = z2 * -2.562915447;
        // data7 = tmp4 + z1 + z3;
        // data5 = tmp5 + z2 + z4;
        // data3 = tmp6 + z2 + z3;
        // data1 = tmp7 + z1 + z4;
        //
        // (This implementation)
        // tmp4 = tmp4 * (0.298631336 - 0.899976223) + tmp7 * -0.899976223;
        // tmp5 = tmp5 * (2.053119869 - 2.562915447) + tmp6 * -2.562915447;
        // tmp6 = tmp5 * -2.562915447 + tmp6 * (3.072711026 - 2.562915447);
        // tmp7 = tmp4 * -0.899976223 + tmp7 * (1.501321110 - 0.899976223);
        // data7 = tmp4 + z3;
        // data5 = tmp5 + z4;
        // data3 = tmp6 + z3;
        // data1 = tmp7 + z4;

        let t4 = _mm256_permute2x128_si256(t5, t5, 0x01); // t4 = tmp7_6
        let t2 = _mm256_unpacklo_epi16(t8, t4);
        let t4 = _mm256_unpackhi_epi16(t8, t4);

        let t2 = _mm256_madd_epi16(t2, PW_MF060_MF089_MF050_MF256()); //t2 = tmp4_5L
        let t4 = _mm256_madd_epi16(t4, PW_MF060_MF089_MF050_MF256()); // t4 = tmp4_5H

        let t2 = _mm256_add_epi32(t2, t6); // t2 = data7_5L
        let t4 = _mm256_add_epi32(t4, t7); // t4 = data7_5H

        let t2 = _mm256_add_epi32(t2, PD_DESCALE_P(first_pass));
        let t4 = _mm256_add_epi32(t4, PD_DESCALE_P(first_pass));

        let t2 = if first_pass {
            _mm256_srai_epi32(t2, DESCALE_P1)
        } else {
            _mm256_srai_epi32(t2, DESCALE_P2)
        };
        let t4 = if first_pass {
            _mm256_srai_epi32(t4, DESCALE_P1)
        } else {
            _mm256_srai_epi32(t4, DESCALE_P2)
        };

        let t4 = _mm256_packs_epi32(t2, t4); // t4 = data7_5

        let t2 = _mm256_permute2x128_si256(t8, t8, 0x01); // t2 = tmp5_4

        let t8 = _mm256_unpacklo_epi16(t5, t2);
        let t5 = _mm256_unpackhi_epi16(t5, t2);

        let t8 = _mm256_madd_epi16(t8, PW_F050_MF256_F060_MF089()); // t8 = tmp6_7L
        let t5 = _mm256_madd_epi16(t5, PW_F050_MF256_F060_MF089()); // t5 = tmp6_7H

        let t8 = _mm256_add_epi32(t8, t6); // t8 = data3_1L
        let t5 = _mm256_add_epi32(t5, t7); // t5 = data3_1H

        let t8 = _mm256_add_epi32(t8, PD_DESCALE_P(first_pass));
        let t5 = _mm256_add_epi32(t5, PD_DESCALE_P(first_pass));

        let t8 = if first_pass {
            _mm256_srai_epi32(t8, DESCALE_P1)
        } else {
            _mm256_srai_epi32(t8, DESCALE_P2)
        };
        let t5 = if first_pass {
            _mm256_srai_epi32(t5, DESCALE_P1)
        } else {
            _mm256_srai_epi32(t5, DESCALE_P2)
        };

        let t2 = _mm256_packs_epi32(t8, t5); // t2 = data3_1

        (t1, t2, t3, t4)
    }

    let data = &mut data.data;

    let ymm4 = avx_load(&data[0..16]);
    let ymm5 = avx_load(&data[16..32]);
    let ymm6 = avx_load(&data[32..48]);
    let ymm7 = avx_load(&data[48..64]);

    // ---- Pass 1: process rows.
    // ymm4=(00 01 02 03 04 05 06 07  10 11 12 13 14 15 16 17)
    // ymm5=(20 21 22 23 24 25 26 27  30 31 32 33 34 35 36 37)
    // ymm6=(40 41 42 43 44 45 46 47  50 51 52 53 54 55 56 57)
    // ymm7=(60 61 62 63 64 65 66 67  70 71 72 73 74 75 76 77)

    let ymm0 = _mm256_permute2x128_si256(ymm4, ymm6, 0x20);
    let ymm1 = _mm256_permute2x128_si256(ymm4, ymm6, 0x31);
    let ymm2 = _mm256_permute2x128_si256(ymm5, ymm7, 0x20);
    let ymm3 = _mm256_permute2x128_si256(ymm5, ymm7, 0x31);

    // ymm0=(00 01 02 03 04 05 06 07  40 41 42 43 44 45 46 47)
    // ymm1=(10 11 12 13 14 15 16 17  50 51 52 53 54 55 56 57)
    // ymm2=(20 21 22 23 24 25 26 27  60 61 62 63 64 65 66 67)
    // ymm3=(30 31 32 33 34 35 36 37  70 71 72 73 74 75 76 77)

    let (ymm0, ymm1, ymm2, ymm3) = do_transpose(ymm0, ymm1, ymm2, ymm3);
    let (ymm0, ymm1, ymm2, ymm3) = do_dct(true, ymm0, ymm1, ymm2, ymm3);

    // ---- Pass 2: process columns.

    let ymm4 = _mm256_permute2x128_si256(ymm1, ymm3, 0x20); // ymm4=data3_7
    let ymm1 = _mm256_permute2x128_si256(ymm1, ymm3, 0x31); // ymm1=data1_5

    let (ymm0, ymm1, ymm2, ymm4) = do_transpose(ymm0, ymm1, ymm2, ymm4);
    let (ymm0, ymm1, ymm2, ymm4) = do_dct(false, ymm0, ymm1, ymm2, ymm4);

    let ymm3 = _mm256_permute2x128_si256(ymm0, ymm1, 0x30); // ymm3=data0_1
    let ymm5 = _mm256_permute2x128_si256(ymm2, ymm1, 0x20); // ymm5=data2_3
    let ymm6 = _mm256_permute2x128_si256(ymm0, ymm4, 0x31); // ymm6=data4_5
    let ymm7 = _mm256_permute2x128_si256(ymm2, ymm4, 0x21); // ymm7=data6_7

    avx_store(ymm3, &mut data[0..16]);
    avx_store(ymm5, &mut data[16..32]);
    avx_store(ymm6, &mut data[32..48]);
    avx_store(ymm7, &mut data[48..64]);
}

/// Safe wrapper for an unaligned AVX load
#[target_feature(enable = "avx2")]
#[inline]
fn avx_load(input: &[i16]) -> __m256i {
    assert!(input.len() == 16);
    assert!(core::mem::size_of::<[i16; 16]>() == core::mem::size_of::<__m256i>());
    // SAFETY: we've checked sizes above. The load is unaligned, so no alignment requirements.
    unsafe { _mm256_loadu_si256(input.as_ptr() as *const __m256i) }
}

/// Safe wrapper for an unaligned AVX store
#[target_feature(enable = "avx2")]
#[inline]
fn avx_store(input: __m256i, output: &mut [i16]) {
    assert!(output.len() == 16);
    assert!(core::mem::size_of::<[i16; 16]>() == core::mem::size_of::<__m256i>());
    // SAFETY: we've checked sizes above. The load is unaligned, so no alignment requirements.
    unsafe { _mm256_storeu_si256(output.as_mut_ptr() as *mut __m256i, input) }
}

Coverage Report

Created: 2026-08-13 08:17

Line	Count	Source
1		/*
2		* Ported from mozjpeg / jfdctint-avx2.asm to rust
3		* Copyright 2009 Pierre Ossman <ossman@cendio.se> for Cendio AB
4		* Copyright (C) 2009, 2016, 2018, 2020, D. R. Commander.
5		*
6		* Based on the x86 SIMD extension for IJG JPEG library
7		* Copyright (C) 1999-2006, MIYASAKA Masaru.
8		*/
9
10		#[cfg(target_arch = "x86")]
11		use core::arch::x86::{
12		__m256i, _mm256_add_epi16, _mm256_add_epi32, _mm256_loadu_si256, _mm256_madd_epi16,
13		_mm256_packs_epi32, _mm256_permute2x128_si256, _mm256_permute4x64_epi64, _mm256_set_epi16,
14		_mm256_set_epi32, _mm256_sign_epi16, _mm256_slli_epi16, _mm256_srai_epi16, _mm256_srai_epi32,
15		_mm256_storeu_si256, _mm256_sub_epi16, _mm256_unpackhi_epi16, _mm256_unpackhi_epi32,
16		_mm256_unpacklo_epi16, _mm256_unpacklo_epi32,
17		};
18
19		#[cfg(target_arch = "x86_64")]
20		use core::arch::x86_64::{
21		__m256i, _mm256_add_epi16, _mm256_add_epi32, _mm256_loadu_si256, _mm256_madd_epi16,
22		_mm256_packs_epi32, _mm256_permute2x128_si256, _mm256_permute4x64_epi64, _mm256_set_epi16,
23		_mm256_set_epi32, _mm256_sign_epi16, _mm256_slli_epi16, _mm256_srai_epi16, _mm256_srai_epi32,
24		_mm256_storeu_si256, _mm256_sub_epi16, _mm256_unpackhi_epi16, _mm256_unpackhi_epi32,
25		_mm256_unpacklo_epi16, _mm256_unpacklo_epi32,
26		};
27
28		use crate::encoder::AlignedBlock;
29
30		const CONST_BITS: i32 = 13;
31		const PASS1_BITS: i32 = 2;
32
33		// FIX(0.298631336)
34		const F_0_298: i16 = 2446;
35		// FIX(0.390180644)
36		const F_0_390: i16 = 3196;
37		// FIX(0.541196100)
38		const F_0_541: i16 = 4433;
39		// FIX(0.765366865)
40		const F_0_765: i16 = 6270;
41		//FIX(0.899976223)
42		const F_0_899: i16 = 7373;
43		//FIX(1.175875602)
44		const F_1_175: i16 = 9633;
45		//FIX(1.501321110)
46		const F_1_501: i16 = 12299;
47		//FIX(1.847759065)
48		const F_1_847: i16 = 15137;
49		//FIX(1.961570560)
50		const F_1_961: i16 = 16069;
51		//FIX(2.053119869)
52		const F_2_053: i16 = 16819;
53		//FIX(2.562915447)
54		const F_2_562: i16 = 20995;
55		//FIX(3.072711026)
56		const F_3_072: i16 = 25172;
57
58		const DESCALE_P1: i32 = CONST_BITS - PASS1_BITS;
59		const DESCALE_P2: i32 = CONST_BITS + PASS1_BITS;
60
61		#[inline(always)]
62	0	pub fn fdct_avx2(data: &mut AlignedBlock) {
63	0	unsafe {
64	0	fdct_avx2_internal(data);
65	0	}
66	0	}
67
68		#[target_feature(enable = "avx2")]
69	0	fn fdct_avx2_internal(data: &mut AlignedBlock) {
70		#[target_feature(enable = "avx2")]
71		#[allow(non_snake_case)]
72		#[inline]
73	0	fn PW_F130_F054_MF130_F054() -> __m256i {
74	0	_mm256_set_epi16(
75		F_0_541,
76	0	F_0_541 - F_1_847,
77		F_0_541,
78	0	F_0_541 - F_1_847,
79		F_0_541,
80	0	F_0_541 - F_1_847,
81		F_0_541,
82	0	F_0_541 - F_1_847,
83		F_0_541,
84	0	F_0_541 + F_0_765,
85		F_0_541,
86	0	F_0_541 + F_0_765,
87		F_0_541,
88	0	F_0_541 + F_0_765,
89		F_0_541,
90	0	F_0_541 + F_0_765,
91		)
92	0	}
93
94		#[target_feature(enable = "avx2")]
95		#[allow(non_snake_case)]
96		#[inline]
97	0	fn PW_MF078_F117_F078_F117() -> __m256i {
98	0	_mm256_set_epi16(
99		F_1_175,
100	0	F_1_175 - F_0_390,
101		F_1_175,
102	0	F_1_175 - F_0_390,
103		F_1_175,
104	0	F_1_175 - F_0_390,
105		F_1_175,
106	0	F_1_175 - F_0_390,
107		F_1_175,
108	0	F_1_175 - F_1_961,
109		F_1_175,
110	0	F_1_175 - F_1_961,
111		F_1_175,
112	0	F_1_175 - F_1_961,
113		F_1_175,
114	0	F_1_175 - F_1_961,
115		)
116	0	}
117
118		#[target_feature(enable = "avx2")]
119		#[allow(non_snake_case)]
120		#[inline]
121	0	fn PW_MF060_MF089_MF050_MF256() -> __m256i {
122	0	_mm256_set_epi16(
123	0	-F_2_562,
124	0	F_2_053 - F_2_562,
125	0	-F_2_562,
126	0	F_2_053 - F_2_562,
127	0	-F_2_562,
128	0	F_2_053 - F_2_562,
129	0	-F_2_562,
130	0	F_2_053 - F_2_562,
131	0	-F_0_899,
132	0	F_0_298 - F_0_899,
133	0	-F_0_899,
134	0	F_0_298 - F_0_899,
135	0	-F_0_899,
136	0	F_0_298 - F_0_899,
137	0	-F_0_899,
138	0	F_0_298 - F_0_899,
139		)
140	0	}
141
142		#[target_feature(enable = "avx2")]
143		#[allow(non_snake_case)]
144		#[inline]
145	0	fn PW_F050_MF256_F060_MF089() -> __m256i {
146	0	_mm256_set_epi16(
147	0	-F_0_899,
148	0	F_1_501 - F_0_899,
149	0	-F_0_899,
150	0	F_1_501 - F_0_899,
151	0	-F_0_899,
152	0	F_1_501 - F_0_899,
153	0	-F_0_899,
154	0	F_1_501 - F_0_899,
155	0	-F_2_562,
156	0	F_3_072 - F_2_562,
157	0	-F_2_562,
158	0	F_3_072 - F_2_562,
159	0	-F_2_562,
160	0	F_3_072 - F_2_562,
161	0	-F_2_562,
162	0	F_3_072 - F_2_562,
163		)
164	0	}
165
166		#[target_feature(enable = "avx2")]
167		#[allow(non_snake_case)]
168		#[inline]
169	0	fn PD_DESCALE_P(first_pass: bool) -> __m256i {
170	0	if first_pass {
171	0	_mm256_set_epi32(
172	0	1 << (DESCALE_P1 - 1),
173	0	1 << (DESCALE_P1 - 1),
174	0	1 << (DESCALE_P1 - 1),
175	0	1 << (DESCALE_P1 - 1),
176	0	1 << (DESCALE_P1 - 1),
177	0	1 << (DESCALE_P1 - 1),
178	0	1 << (DESCALE_P1 - 1),
179	0	1 << (DESCALE_P1 - 1),
180		)
181		} else {
182	0	_mm256_set_epi32(
183	0	1 << (DESCALE_P2 - 1),
184	0	1 << (DESCALE_P2 - 1),
185	0	1 << (DESCALE_P2 - 1),
186	0	1 << (DESCALE_P2 - 1),
187	0	1 << (DESCALE_P2 - 1),
188	0	1 << (DESCALE_P2 - 1),
189	0	1 << (DESCALE_P2 - 1),
190	0	1 << (DESCALE_P2 - 1),
191		)
192		}
193	0	}
194
195		#[target_feature(enable = "avx2")]
196		#[allow(non_snake_case)]
197		#[inline]
198	0	fn PW_DESCALE_P2X() -> __m256i {
199	0	_mm256_set_epi32(
200	0	1 << (PASS1_BITS - 1),
201	0	1 << (PASS1_BITS - 1),
202	0	1 << (PASS1_BITS - 1),
203	0	1 << (PASS1_BITS - 1),
204	0	1 << (PASS1_BITS - 1),
205	0	1 << (PASS1_BITS - 1),
206	0	1 << (PASS1_BITS - 1),
207	0	1 << (PASS1_BITS - 1),
208		)
209	0	}
210
211		// In-place 8x8x16-bit matrix transpose using AVX2 instructions
212		#[target_feature(enable = "avx2")]
213		#[inline]
214	0	fn do_transpose(
215	0	i1: __m256i,
216	0	i2: __m256i,
217	0	i3: __m256i,
218	0	i4: __m256i,
219	0	) -> (__m256i, __m256i, __m256i, __m256i) {
220		//i1=(00 01 02 03 04 05 06 07 40 41 42 43 44 45 46 47)
221		//i2=(10 11 12 13 14 15 16 17 50 51 52 53 54 55 56 57)
222		//i3=(20 21 22 23 24 25 26 27 60 61 62 63 64 65 66 67)
223		//i4=(30 31 32 33 34 35 36 37 70 71 72 73 74 75 76 77)
224
225	0	let t5 = _mm256_unpacklo_epi16(i1, i2);
226	0	let t6 = _mm256_unpackhi_epi16(i1, i2);
227	0	let t7 = _mm256_unpacklo_epi16(i3, i4);
228	0	let t8 = _mm256_unpackhi_epi16(i3, i4);
229
230		// transpose coefficients(phase 1)
231		// t1=(00 10 01 11 02 12 03 13 40 50 41 51 42 52 43 53)
232		// t2=(04 14 05 15 06 16 07 17 44 54 45 55 46 56 47 57)
233		// t3=(20 30 21 31 22 32 23 33 60 70 61 71 62 72 63 73)
234		// t4=(24 34 25 35 26 36 27 37 64 74 65 75 66 76 67 77)
235
236	0	let t1 = _mm256_unpacklo_epi32(t5, t7);
237	0	let t2 = _mm256_unpackhi_epi32(t5, t7);
238	0	let t3 = _mm256_unpacklo_epi32(t6, t8);
239	0	let t4 = _mm256_unpackhi_epi32(t6, t8);
240
241		// transpose coefficients(phase 2)
242		// t5=(00 10 20 30 01 11 21 31 40 50 60 70 41 51 61 71)
243		// t6=(02 12 22 32 03 13 23 33 42 52 62 72 43 53 63 73)
244		// t7=(04 14 24 34 05 15 25 35 44 54 64 74 45 55 65 75)
245		// t8=(06 16 26 36 07 17 27 37 46 56 66 76 47 57 67 77)
246
247	0	(
248	0	_mm256_permute4x64_epi64(t1, 0x8D),
249	0	_mm256_permute4x64_epi64(t2, 0x8D),
250	0	_mm256_permute4x64_epi64(t3, 0xD8),
251	0	_mm256_permute4x64_epi64(t4, 0xD8),
252	0	)
253	0	}
254
255		// In-place 8x8x16-bit accurate integer forward DCT using AVX2 instructions
256		#[target_feature(enable = "avx2")]
257		#[inline]
258	0	fn do_dct(
259	0	first_pass: bool,
260	0	i1: __m256i,
261	0	i2: __m256i,
262	0	i3: __m256i,
263	0	i4: __m256i,
264	0	) -> (__m256i, __m256i, __m256i, __m256i) {
265	0	let t5 = _mm256_sub_epi16(i1, i4); // data1_0 - data6_7 = tmp6_7
266	0	let t6 = _mm256_add_epi16(i1, i4); // data1_0 + data6_7 = tmp1_0
267	0	let t7 = _mm256_add_epi16(i2, i3); // data3_2 + data4_5 = tmp3_2
268	0	let t8 = _mm256_sub_epi16(i2, i3); // data3_2 - data4_5 = tmp4_5
269
270		// Even part
271
272	0	let t6 = _mm256_permute2x128_si256(t6, t6, 0x01); // t6=tmp0_1
273	0	let t1 = _mm256_add_epi16(t6, t7); // t1 = tmp0_1 + tmp3_2 = tmp10_11
274	0	let t6 = _mm256_sub_epi16(t6, t7); // t6 = tmp0_1 - tmp3_2 = tmp13_12
275
276	0	let t7 = _mm256_permute2x128_si256(t1, t1, 0x01); // t7 = tmp11_10
277	0	let t1 = _mm256_sign_epi16(
278	0	t1,
279	0	_mm256_set_epi16(-1, -1, -1, -1, -1, -1, -1, -1, 1, 1, 1, 1, 1, 1, 1, 1),
280		); // tmp10_neg11
281
282	0	let t7 = _mm256_add_epi16(t7, t1); // t7 = (tmp10 + tmp11)_(tmp10 - tmp11)
283
284	0	let t1 = if first_pass {
285	0	_mm256_slli_epi16(t7, PASS1_BITS)
286		} else {
287	0	let t7 = _mm256_add_epi16(t7, PW_DESCALE_P2X());
288	0	_mm256_srai_epi16(t7, PASS1_BITS)
289		};
290
291		// (Original)
292		// z1 = (tmp12 + tmp13) * 0.541196100;
293		// data2 = z1 + tmp13 * 0.765366865;
294		// data6 = z1 + tmp12 * -1.847759065;
295		//
296		// (This implementation)
297		// data2 = tmp13 * (0.541196100 + 0.765366865) + tmp12 * 0.541196100;
298		// data6 = tmp13 * 0.541196100 + tmp12 * (0.541196100 - 1.847759065);
299
300	0	let t7 = _mm256_permute2x128_si256(t6, t6, 0x01); // t7 = tmp12_13
301	0	let t2 = _mm256_unpacklo_epi16(t6, t7);
302	0	let t6 = _mm256_unpackhi_epi16(t6, t7);
303
304	0	let t2 = _mm256_madd_epi16(t2, PW_F130_F054_MF130_F054()); // t2 = data2_6L
305	0	let t6 = _mm256_madd_epi16(t6, PW_F130_F054_MF130_F054()); // t6 = data2_6H
306
307	0	let t2 = _mm256_add_epi32(t2, PD_DESCALE_P(first_pass));
308	0	let t6 = _mm256_add_epi32(t6, PD_DESCALE_P(first_pass));
309
310	0	let t2 = if first_pass {
311	0	_mm256_srai_epi32(t2, DESCALE_P1)
312		} else {
313	0	_mm256_srai_epi32(t2, DESCALE_P2)
314		};
315	0	let t6 = if first_pass {
316	0	_mm256_srai_epi32(t6, DESCALE_P1)
317		} else {
318	0	_mm256_srai_epi32(t6, DESCALE_P2)
319		};
320
321	0	let t3 = _mm256_packs_epi32(t2, t6); // t6 = data2_6
322
323		// Odd part
324
325	0	let t7 = _mm256_add_epi16(t8, t5); // t7 = tmp4_5 + tmp6_7 = z3_4
326
327		// (Original)
328		// z5 = (z3 + z4) * 1.175875602;
329		// z3 = z3 * -1.961570560;
330		// z4 = z4 * -0.390180644;
331		// z3 += z5;
332		// z4 += z5;
333		//
334		// (This implementation)
335		// z3 = z3 * (1.175875602 - 1.961570560) + z4 * 1.175875602;
336		// z4 = z3 * 1.175875602 + z4 * (1.175875602 - 0.390180644);
337
338	0	let t2 = _mm256_permute2x128_si256(t7, t7, 0x01); // t2 = z4_3
339	0	let t6 = _mm256_unpacklo_epi16(t7, t2);
340	0	let t7 = _mm256_unpackhi_epi16(t7, t2);
341
342	0	let t6 = _mm256_madd_epi16(t6, PW_MF078_F117_F078_F117()); // t6 = z3_4L
343	0	let t7 = _mm256_madd_epi16(t7, PW_MF078_F117_F078_F117()); // t7 = z3_4H
344
345		// (Original)
346		// z1 = tmp4 + tmp7;
347		// z2 = tmp5 + tmp6;
348		// tmp4 = tmp4 * 0.298631336;
349		// tmp5 = tmp5 * 2.053119869;
350		// tmp6 = tmp6 * 3.072711026;
351		// tmp7 = tmp7 * 1.501321110;
352		// z1 = z1 * -0.899976223;
353		// z2 = z2 * -2.562915447;
354		// data7 = tmp4 + z1 + z3;
355		// data5 = tmp5 + z2 + z4;
356		// data3 = tmp6 + z2 + z3;
357		// data1 = tmp7 + z1 + z4;
358		//
359		// (This implementation)
360		// tmp4 = tmp4 * (0.298631336 - 0.899976223) + tmp7 * -0.899976223;
361		// tmp5 = tmp5 * (2.053119869 - 2.562915447) + tmp6 * -2.562915447;
362		// tmp6 = tmp5 * -2.562915447 + tmp6 * (3.072711026 - 2.562915447);
363		// tmp7 = tmp4 * -0.899976223 + tmp7 * (1.501321110 - 0.899976223);
364		// data7 = tmp4 + z3;
365		// data5 = tmp5 + z4;
366		// data3 = tmp6 + z3;
367		// data1 = tmp7 + z4;
368
369	0	let t4 = _mm256_permute2x128_si256(t5, t5, 0x01); // t4 = tmp7_6
370	0	let t2 = _mm256_unpacklo_epi16(t8, t4);
371	0	let t4 = _mm256_unpackhi_epi16(t8, t4);
372
373	0	let t2 = _mm256_madd_epi16(t2, PW_MF060_MF089_MF050_MF256()); //t2 = tmp4_5L
374	0	let t4 = _mm256_madd_epi16(t4, PW_MF060_MF089_MF050_MF256()); // t4 = tmp4_5H
375
376	0	let t2 = _mm256_add_epi32(t2, t6); // t2 = data7_5L
377	0	let t4 = _mm256_add_epi32(t4, t7); // t4 = data7_5H
378
379	0	let t2 = _mm256_add_epi32(t2, PD_DESCALE_P(first_pass));
380	0	let t4 = _mm256_add_epi32(t4, PD_DESCALE_P(first_pass));
381
382	0	let t2 = if first_pass {
383	0	_mm256_srai_epi32(t2, DESCALE_P1)
384		} else {
385	0	_mm256_srai_epi32(t2, DESCALE_P2)
386		};
387	0	let t4 = if first_pass {
388	0	_mm256_srai_epi32(t4, DESCALE_P1)
389		} else {
390	0	_mm256_srai_epi32(t4, DESCALE_P2)
391		};
392
393	0	let t4 = _mm256_packs_epi32(t2, t4); // t4 = data7_5
394
395	0	let t2 = _mm256_permute2x128_si256(t8, t8, 0x01); // t2 = tmp5_4
396
397	0	let t8 = _mm256_unpacklo_epi16(t5, t2);
398	0	let t5 = _mm256_unpackhi_epi16(t5, t2);
399
400	0	let t8 = _mm256_madd_epi16(t8, PW_F050_MF256_F060_MF089()); // t8 = tmp6_7L
401	0	let t5 = _mm256_madd_epi16(t5, PW_F050_MF256_F060_MF089()); // t5 = tmp6_7H
402
403	0	let t8 = _mm256_add_epi32(t8, t6); // t8 = data3_1L
404	0	let t5 = _mm256_add_epi32(t5, t7); // t5 = data3_1H
405
406	0	let t8 = _mm256_add_epi32(t8, PD_DESCALE_P(first_pass));
407	0	let t5 = _mm256_add_epi32(t5, PD_DESCALE_P(first_pass));
408
409	0	let t8 = if first_pass {
410	0	_mm256_srai_epi32(t8, DESCALE_P1)
411		} else {
412	0	_mm256_srai_epi32(t8, DESCALE_P2)
413		};
414	0	let t5 = if first_pass {
415	0	_mm256_srai_epi32(t5, DESCALE_P1)
416		} else {
417	0	_mm256_srai_epi32(t5, DESCALE_P2)
418		};
419
420	0	let t2 = _mm256_packs_epi32(t8, t5); // t2 = data3_1
421
422	0	(t1, t2, t3, t4)
423	0	}
424
425	0	let data = &mut data.data;
426
427	0	let ymm4 = avx_load(&data[0..16]);
428	0	let ymm5 = avx_load(&data[16..32]);
429	0	let ymm6 = avx_load(&data[32..48]);
430	0	let ymm7 = avx_load(&data[48..64]);
431
432		// ---- Pass 1: process rows.
433		// ymm4=(00 01 02 03 04 05 06 07 10 11 12 13 14 15 16 17)
434		// ymm5=(20 21 22 23 24 25 26 27 30 31 32 33 34 35 36 37)
435		// ymm6=(40 41 42 43 44 45 46 47 50 51 52 53 54 55 56 57)
436		// ymm7=(60 61 62 63 64 65 66 67 70 71 72 73 74 75 76 77)
437
438	0	let ymm0 = _mm256_permute2x128_si256(ymm4, ymm6, 0x20);
439	0	let ymm1 = _mm256_permute2x128_si256(ymm4, ymm6, 0x31);
440	0	let ymm2 = _mm256_permute2x128_si256(ymm5, ymm7, 0x20);
441	0	let ymm3 = _mm256_permute2x128_si256(ymm5, ymm7, 0x31);
442
443		// ymm0=(00 01 02 03 04 05 06 07 40 41 42 43 44 45 46 47)
444		// ymm1=(10 11 12 13 14 15 16 17 50 51 52 53 54 55 56 57)
445		// ymm2=(20 21 22 23 24 25 26 27 60 61 62 63 64 65 66 67)
446		// ymm3=(30 31 32 33 34 35 36 37 70 71 72 73 74 75 76 77)
447
448	0	let (ymm0, ymm1, ymm2, ymm3) = do_transpose(ymm0, ymm1, ymm2, ymm3);
449	0	let (ymm0, ymm1, ymm2, ymm3) = do_dct(true, ymm0, ymm1, ymm2, ymm3);
450
451		// ---- Pass 2: process columns.
452
453	0	let ymm4 = _mm256_permute2x128_si256(ymm1, ymm3, 0x20); // ymm4=data3_7
454	0	let ymm1 = _mm256_permute2x128_si256(ymm1, ymm3, 0x31); // ymm1=data1_5
455
456	0	let (ymm0, ymm1, ymm2, ymm4) = do_transpose(ymm0, ymm1, ymm2, ymm4);
457	0	let (ymm0, ymm1, ymm2, ymm4) = do_dct(false, ymm0, ymm1, ymm2, ymm4);
458
459	0	let ymm3 = _mm256_permute2x128_si256(ymm0, ymm1, 0x30); // ymm3=data0_1
460	0	let ymm5 = _mm256_permute2x128_si256(ymm2, ymm1, 0x20); // ymm5=data2_3
461	0	let ymm6 = _mm256_permute2x128_si256(ymm0, ymm4, 0x31); // ymm6=data4_5
462	0	let ymm7 = _mm256_permute2x128_si256(ymm2, ymm4, 0x21); // ymm7=data6_7
463
464	0	avx_store(ymm3, &mut data[0..16]);
465	0	avx_store(ymm5, &mut data[16..32]);
466	0	avx_store(ymm6, &mut data[32..48]);
467	0	avx_store(ymm7, &mut data[48..64]);
468	0	}
469
470		/// Safe wrapper for an unaligned AVX load
471		#[target_feature(enable = "avx2")]
472		#[inline]
473	0	fn avx_load(input: &[i16]) -> __m256i {
474	0	assert!(input.len() == 16);
475	0	assert!(core::mem::size_of::<[i16; 16]>() == core::mem::size_of::<__m256i>());
476		// SAFETY: we've checked sizes above. The load is unaligned, so no alignment requirements.
477	0	unsafe { _mm256_loadu_si256(input.as_ptr() as *const __m256i) }
478	0	}
479
480		/// Safe wrapper for an unaligned AVX store
481		#[target_feature(enable = "avx2")]
482		#[inline]
483	0	fn avx_store(input: __m256i, output: &mut [i16]) {
484	0	assert!(output.len() == 16);
485	0	assert!(core::mem::size_of::<[i16; 16]>() == core::mem::size_of::<__m256i>());
486		// SAFETY: we've checked sizes above. The load is unaligned, so no alignment requirements.
487	0	unsafe { _mm256_storeu_si256(output.as_mut_ptr() as *mut __m256i, input) }
488	0	}