/*

* Copyright(c) 2019 Intel Corporation

* Copyright (c) 2016, Alliance for Open Media. All rights reserved

*

* This source code is subject to the terms of the BSD 3-Clause Clear License and

* the Alliance for Open Media Patent License 1.0. If the BSD 3-Clause Clear License

* was not distributed with this source code in the LICENSE file, you can

* obtain it at https://www.aomedia.org/license. If the Alliance for Open

* Media Patent License 1.0 was not distributed with this source code in the

* PATENTS file, you can obtain it at https://www.aomedia.org/license/patent-license.

*/

#include "definitions.h"

#include "full_loop.h"

#include "pcs.h"

#include "rd_cost.h"

#include "aom_dsp_rtcd.h"

#include "sequence_control_set.h"

#include "utility.h"

#include "ac_bias.h"

const int av1_get_tx_scale_tab[TX_SIZES_ALL] = {0, 0, 0, 1, 2, 0, 0, 0, 0, 1, 1, 2, 2, 0, 0, 0, 0, 1, 1};

void     svt_aom_residual_kernel(uint8_t* input, uint32_t input_offset, uint32_t input_stride, uint8_t* pred,

                                 uint32_t pred_offset, uint32_t pred_stride, int16_t* residual, uint32_t residual_offset,

                                 uint32_t residual_stride, bool hbd, uint32_t area_width, uint32_t area_height);

uint64_t svt_spatial_full_distortion_ssim_kernel(uint8_t* input, uint32_t input_offset, uint32_t input_stride,

                                                 uint8_t* recon, int32_t recon_offset, uint32_t recon_stride,

                                                 uint32_t area_width, uint32_t area_height, bool hbd, double ac_bias);

void svt_aom_quantize_b_c(const TranLow* coeff_ptr, intptr_t n_coeffs, const int16_t* zbin_ptr,

                          const int16_t* round_ptr, const int16_t* quant_ptr, const int16_t* quant_shift_ptr,

                          TranLow* qcoeff_ptr, TranLow* dqcoeff_ptr, const int16_t* dequant_ptr, uint16_t* eob_ptr,

                          const int16_t* scan, const int16_t* iscan, const QmVal* qm_ptr, const QmVal* iqm_ptr,

                          const int32_t log_scale) {

    const int32_t zbins[2]  = {ROUND_POWER_OF_TWO(zbin_ptr[0], log_scale), ROUND_POWER_OF_TWO(zbin_ptr[1], log_scale)};

    const int32_t nzbins[2] = {zbins[0] * -1, zbins[1] * -1};

    intptr_t      non_zero_count = n_coeffs, eob = -1;

    (void)iscan;

    memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));

    memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));

    // Pre-scan pass

    for (intptr_t i = n_coeffs - 1; i >= 0; i--) {

        const int32_t rc    = scan[i];

        const QmVal   wt    = qm_ptr != NULL ? qm_ptr[rc] : (1 << AOM_QM_BITS);

        const int32_t coeff = coeff_ptr[rc] * wt;

        if (coeff < (zbins[rc != 0] * (1 << AOM_QM_BITS)) && coeff > (nzbins[rc != 0] * (1 << AOM_QM_BITS))) {

            non_zero_count--;

        } else {

            break;

        }

    }

    // Quantization pass: All coefficients with index >= zero_flag are

    // skippable. Note: zero_flag can be zero.

    for (intptr_t i = 0; i < non_zero_count; i++) {

        const int32_t rc         = scan[i];

        const int32_t coeff      = coeff_ptr[rc];

        const int     coeff_sign = coeff < 0 ? -1 : 0;

        const int32_t abs_coeff  = (coeff ^ coeff_sign) - coeff_sign;

        const QmVal wt = qm_ptr != NULL ? qm_ptr[rc] : (1 << AOM_QM_BITS);

        if (abs_coeff * wt >= (zbins[rc != 0] << AOM_QM_BITS)) {

            int64_t tmp = clamp(abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], log_scale), INT16_MIN, INT16_MAX);

            tmp *= wt;

            int32_t tmp32         = (int32_t)(((((tmp * quant_ptr[rc != 0]) >> 16) + tmp) * quant_shift_ptr[rc != 0]) >>

                                      (16 - log_scale + AOM_QM_BITS)); // quantization

            qcoeff_ptr[rc]        = (tmp32 ^ coeff_sign) - coeff_sign;

            const int32_t iwt     = iqm_ptr != NULL ? iqm_ptr[rc] : (1 << AOM_QM_BITS);

            const int32_t dequant = (dequant_ptr[rc != 0] * iwt + (1 << (AOM_QM_BITS - 1))) >> AOM_QM_BITS;

            const TranLow abs_dqcoeff = (tmp32 * dequant) >> log_scale;

            dqcoeff_ptr[rc]           = (TranLow)((abs_dqcoeff ^ coeff_sign) - coeff_sign);

            if (tmp32) {

                eob = i;

            }

        }

    }

    *eob_ptr = (uint16_t)(eob + 1);

}

void svt_aom_highbd_quantize_b_c(const TranLow* coeff_ptr, intptr_t n_coeffs, const int16_t* zbin_ptr,

                                 const int16_t* round_ptr, const int16_t* quant_ptr, const int16_t* quant_shift_ptr,

                                 TranLow* qcoeff_ptr, TranLow* dqcoeff_ptr, const int16_t* dequant_ptr,

                                 uint16_t* eob_ptr, const int16_t* scan, const int16_t* iscan, const QmVal* qm_ptr,

                                 const QmVal* iqm_ptr, const int32_t log_scale) {

    intptr_t eob = -1;

    (void)iscan;

    memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));

    memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));

    const int32_t zbins[2]  = {ROUND_POWER_OF_TWO(zbin_ptr[0], log_scale), ROUND_POWER_OF_TWO(zbin_ptr[1], log_scale)};

    const int32_t nzbins[2] = {zbins[0] * -1, zbins[1] * -1};

    intptr_t      idx_arr[4096];

    int           idx = 0;

    // Pre-scan pass

    for (intptr_t i = 0; i < n_coeffs; i++) {

        const int32_t rc    = scan[i];

        const QmVal   wt    = qm_ptr != NULL ? qm_ptr[rc] : (1 << AOM_QM_BITS);

        const int32_t coeff = coeff_ptr[rc] * wt;

        // If the coefficient is out of the base ZBIN range, keep it for

        // quantization.

        if (coeff >= (zbins[rc != 0] * (1 << AOM_QM_BITS)) || coeff <= (nzbins[rc != 0] * (1 << AOM_QM_BITS))) {

            idx_arr[idx++] = i;

        }

    }

    // Quantization pass: only process the coefficients selected in

    // pre-scan pass. Note: idx can be zero.

    for (int i = 0; i < idx; i++) {

        const int32_t rc          = scan[idx_arr[i]];

        const int32_t coeff       = coeff_ptr[rc];

        const int     coeff_sign  = coeff < 0 ? -1 : 0;

        const QmVal   wt          = qm_ptr != NULL ? qm_ptr[rc] : (1 << AOM_QM_BITS);

        const QmVal   iwt         = iqm_ptr != NULL ? iqm_ptr[rc] : (1 << AOM_QM_BITS);

        const int32_t abs_coeff   = (coeff ^ coeff_sign) - coeff_sign;

        const int64_t tmp1        = abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], log_scale);

        const int64_t tmpw        = tmp1 * wt;

        const int64_t tmp2        = ((tmpw * quant_ptr[rc != 0]) >> 16) + tmpw;

        const int32_t abs_qcoeff  = (int32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> (16 - log_scale + AOM_QM_BITS));

        qcoeff_ptr[rc]            = (TranLow)((abs_qcoeff ^ coeff_sign) - coeff_sign);

        int32_t       dequant     = (dequant_ptr[rc != 0] * iwt + (1 << (AOM_QM_BITS - 1))) >> AOM_QM_BITS;

        const TranLow abs_dqcoeff = (abs_qcoeff * dequant) >> log_scale;

        dqcoeff_ptr[rc]           = (TranLow)((abs_dqcoeff ^ coeff_sign) - coeff_sign);

        if (abs_qcoeff) {

            eob = idx_arr[i];

        }

    }

    *eob_ptr = (uint16_t)(eob + 1);

}

#if CONFIG_ENABLE_HIGH_BIT_DEPTH

void svt_av1_highbd_quantize_b_facade(const TranLow* coeff_ptr, intptr_t n_coeffs, const MacroblockPlane* p,

                                      TranLow* qcoeff_ptr, TranLow* dqcoeff_ptr, uint16_t* eob_ptr, const ScanOrder* sc,

                                      const QuantParam* qparam) {

    const QmVal* qm_ptr  = qparam->qmatrix;

    const QmVal* iqm_ptr = qparam->iqmatrix;

    if (qm_ptr || iqm_ptr) {

        svt_av1_highbd_quantize_b_qm(coeff_ptr,

                                     n_coeffs,

                                     p->zbin_qtx,

                                     p->round_qtx,

                                     p->quant_qtx,

                                     p->quant_shift_qtx,

                                     qcoeff_ptr,

                                     dqcoeff_ptr,

                                     p->dequant_qtx,

                                     eob_ptr,

                                     sc->scan,

                                     sc->iscan,

                                     qm_ptr,

                                     iqm_ptr,

                                     qparam->log_scale);

    } else {

        svt_aom_highbd_quantize_b(coeff_ptr,

                                  n_coeffs,

                                  p->zbin_qtx,

                                  p->round_qtx,

                                  p->quant_qtx,

                                  p->quant_shift_qtx,

                                  qcoeff_ptr,

                                  dqcoeff_ptr,

                                  p->dequant_qtx,

                                  eob_ptr,

                                  sc->scan,

                                  sc->iscan,

                                  NULL,

                                  NULL,

                                  qparam->log_scale);

    }

    assert(qparam->log_scale <= 2);

}

#endif

static void av1_quantize_b_facade_ii(const TranLow* coeff_ptr, intptr_t n_coeffs, const MacroblockPlane* p,

                                     TranLow* qcoeff_ptr, TranLow* dqcoeff_ptr, uint16_t* eob_ptr, const ScanOrder* sc,

                                     const QuantParam* qparam) {

    const QmVal* qm_ptr  = qparam->qmatrix;

    const QmVal* iqm_ptr = qparam->iqmatrix;

    if (qm_ptr || iqm_ptr) {

        svt_av1_quantize_b_qm(coeff_ptr,

                              n_coeffs,

                              p->zbin_qtx,

                              p->round_qtx,

                              p->quant_qtx,

                              p->quant_shift_qtx,

                              qcoeff_ptr,

                              dqcoeff_ptr,

                              p->dequant_qtx,

                              eob_ptr,

                              sc->scan,

                              sc->iscan,

                              qm_ptr,

                              iqm_ptr,

                              qparam->log_scale);

    } else {

        svt_aom_quantize_b(coeff_ptr,

                           n_coeffs,

                           p->zbin_qtx,

                           p->round_qtx,

                           p->quant_qtx,

                           p->quant_shift_qtx,

                           qcoeff_ptr,

                           dqcoeff_ptr,

                           p->dequant_qtx,

                           eob_ptr,

                           sc->scan,

                           sc->iscan,

                           NULL,

                           NULL,

                           qparam->log_scale);

    }

    assert(qparam->log_scale <= 2);

}

static void quantize_fp_helper_c(const TranLow* coeff_ptr, intptr_t n_coeffs, const int16_t* zbin_ptr,

                                 const int16_t* round_ptr, const int16_t* quant_ptr, const int16_t* quant_shift_ptr,

                                 TranLow* qcoeff_ptr, TranLow* dqcoeff_ptr, const int16_t* dequant_ptr,

                                 uint16_t* eob_ptr, const int16_t* scan, const int16_t* iscan, const QmVal* qm_ptr,

                                 const QmVal* iqm_ptr, int log_scale) {

    int       i, eob = -1;

    const int rounding[2] = {ROUND_POWER_OF_TWO(round_ptr[0], log_scale), ROUND_POWER_OF_TWO(round_ptr[1], log_scale)};

    (void)zbin_ptr;

    (void)quant_shift_ptr;

    (void)iscan;

    memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));

    memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));

    if (qm_ptr == NULL && iqm_ptr == NULL) {

        for (i = 0; i < n_coeffs; i++) {

            const int     rc         = scan[i];

            const int32_t thresh     = (int32_t)(dequant_ptr[rc != 0]);

            const int     coeff      = coeff_ptr[rc];

            const int     coeff_sign = coeff < 0 ? -1 : 0;

            int64_t       abs_coeff  = (coeff ^ coeff_sign) - coeff_sign;

            int           tmp32      = 0;

            if ((abs_coeff << (1 + log_scale)) >= thresh) {

                abs_coeff = clamp64(abs_coeff + rounding[rc != 0], INT16_MIN, INT16_MAX);

                tmp32     = (int)((abs_coeff * quant_ptr[rc != 0]) >> (16 - log_scale));

                if (tmp32) {

                    qcoeff_ptr[rc]            = (tmp32 ^ coeff_sign) - coeff_sign;

                    const TranLow abs_dqcoeff = (tmp32 * dequant_ptr[rc != 0]) >> log_scale;

                    dqcoeff_ptr[rc]           = (abs_dqcoeff ^ coeff_sign) - coeff_sign;

                }

            }

            if (tmp32) {

                eob = i;

            }

        }

    } else {

        // Quantization pass: All coefficients with index >= zero_flag are

        // skippable. Note: zero_flag can be zero.

        for (i = 0; i < n_coeffs; i++) {

            const int   rc         = scan[i];

            const int   coeff      = coeff_ptr[rc];

            const QmVal wt         = qm_ptr ? qm_ptr[rc] : (1 << AOM_QM_BITS);

            const QmVal iwt        = iqm_ptr ? iqm_ptr[rc] : (1 << AOM_QM_BITS);

            const int   dequant    = (dequant_ptr[rc != 0] * iwt + (1 << (AOM_QM_BITS - 1))) >> AOM_QM_BITS;

            const int   coeff_sign = coeff < 0 ? -1 : 0;

            int64_t     abs_coeff  = (coeff ^ coeff_sign) - coeff_sign;

            int         tmp32      = 0;

            if (abs_coeff * wt >= (dequant_ptr[rc != 0] << (AOM_QM_BITS - (1 + log_scale)))) {

                abs_coeff += rounding[rc != 0];

                abs_coeff      = clamp64(abs_coeff, INT16_MIN, INT16_MAX);

                tmp32          = (int)((abs_coeff * wt * quant_ptr[rc != 0]) >> (16 - log_scale + AOM_QM_BITS));

                qcoeff_ptr[rc] = (tmp32 ^ coeff_sign) - coeff_sign;

                const TranLow abs_dqcoeff = (tmp32 * dequant) >> log_scale;

                dqcoeff_ptr[rc]           = (abs_dqcoeff ^ coeff_sign) - coeff_sign;

            }

            if (tmp32) {

                eob = i;

            }

        }

    }

    *eob_ptr = eob + 1;

}

void svt_av1_quantize_fp_c(const TranLow* coeff_ptr, intptr_t n_coeffs, const int16_t* zbin_ptr,

                           const int16_t* round_ptr, const int16_t* quant_ptr, const int16_t* quant_shift_ptr,

                           TranLow* qcoeff_ptr, TranLow* dqcoeff_ptr, const int16_t* dequant_ptr, uint16_t* eob_ptr,

                           const int16_t* scan, const int16_t* iscan) {

    quantize_fp_helper_c(coeff_ptr,

                         n_coeffs,

                         zbin_ptr,

                         round_ptr,

                         quant_ptr,

                         quant_shift_ptr,

                         qcoeff_ptr,

                         dqcoeff_ptr,

                         dequant_ptr,

                         eob_ptr,

                         scan,

                         iscan,

                         NULL,

                         NULL,

                         0);

}

void svt_av1_quantize_fp_qm_c(const TranLow* coeff_ptr, intptr_t n_coeffs, const int16_t* zbin_ptr,

                              const int16_t* round_ptr, const int16_t* quant_ptr, const int16_t* quant_shift_ptr,

                              TranLow* qcoeff_ptr, TranLow* dqcoeff_ptr, const int16_t* dequant_ptr, uint16_t* eob_ptr,

                              const int16_t* scan, const int16_t* iscan, const QmVal* qm_ptr, const QmVal* iqm_ptr,

                              int16_t log_scale) {

    quantize_fp_helper_c(coeff_ptr,

                         n_coeffs,

                         zbin_ptr,

                         round_ptr,

                         quant_ptr,

                         quant_shift_ptr,

                         qcoeff_ptr,

                         dqcoeff_ptr,

                         dequant_ptr,

                         eob_ptr,

                         scan,

                         iscan,

                         qm_ptr,

                         iqm_ptr,

                         log_scale);

}

static void highbd_quantize_fp_helper_c(const TranLow* coeff_ptr, intptr_t count, const int16_t* zbin_ptr,

                                        const int16_t* round_ptr, const int16_t* quant_ptr,

                                        const int16_t* quant_shift_ptr, TranLow* qcoeff_ptr, TranLow* dqcoeff_ptr,

                                        const int16_t* dequant_ptr, uint16_t* eob_ptr, const int16_t* scan,

                                        const int16_t* iscan, const QmVal* qm_ptr, const QmVal* iqm_ptr,

                                        int16_t log_scale) {

    int       i;

    int       eob   = -1;

    const int shift = 16 - log_scale;

    (void)zbin_ptr;

    (void)quant_shift_ptr;

    (void)iscan;

    if (qm_ptr || iqm_ptr) {

        // Quantization pass: All coefficients with index >= zero_flag are

        // skippable. Note: zero_flag can be zero.

        for (i = 0; i < count; i++) {

            const int     rc         = scan[i];

            const int     coeff      = coeff_ptr[rc];

            const QmVal   wt         = qm_ptr != NULL ? qm_ptr[rc] : (1 << AOM_QM_BITS);

            const QmVal   iwt        = iqm_ptr != NULL ? iqm_ptr[rc] : (1 << AOM_QM_BITS);

            const int     dequant    = (dequant_ptr[rc != 0] * iwt + (1 << (AOM_QM_BITS - 1))) >> AOM_QM_BITS;

            const int     coeff_sign = coeff < 0 ? -1 : 0;

            const int64_t abs_coeff  = (coeff ^ coeff_sign) - coeff_sign;

            if (abs_coeff * wt >= (dequant_ptr[rc != 0] << (AOM_QM_BITS - (1 + log_scale)))) {

                const int64_t tmp         = abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], log_scale);

                const int     abs_qcoeff  = (int)((tmp * quant_ptr[rc != 0] * wt) >> (shift + AOM_QM_BITS));

                qcoeff_ptr[rc]            = (TranLow)((abs_qcoeff ^ coeff_sign) - coeff_sign);

                const TranLow abs_dqcoeff = (abs_qcoeff * dequant) >> log_scale;

                dqcoeff_ptr[rc]           = (TranLow)((abs_dqcoeff ^ coeff_sign) - coeff_sign);

                if (abs_qcoeff) {

                    eob = i;

                }

            } else {

                qcoeff_ptr[rc]  = 0;

                dqcoeff_ptr[rc] = 0;

            }

        }

    } else {

        const int log_scaled_round_arr[2] = {

            ROUND_POWER_OF_TWO(round_ptr[0], log_scale),

            ROUND_POWER_OF_TWO(round_ptr[1], log_scale),

        };

        for (i = 0; i < count; i++) {

            const int rc               = scan[i];

            const int coeff            = coeff_ptr[rc];

            const int rc01             = (rc != 0);

            const int coeff_sign       = coeff < 0 ? -1 : 0;

            const int abs_coeff        = (coeff ^ coeff_sign) - coeff_sign;

            const int log_scaled_round = log_scaled_round_arr[rc01];

            if ((abs_coeff << (1 + log_scale)) >= dequant_ptr[rc01]) {

                const int     quant       = quant_ptr[rc01];

                const int     dequant     = dequant_ptr[rc01];

                const int64_t tmp         = (int64_t)abs_coeff + log_scaled_round;

                const int     abs_qcoeff  = (int)((tmp * quant) >> shift);

                qcoeff_ptr[rc]            = (TranLow)((abs_qcoeff ^ coeff_sign) - coeff_sign);

                const TranLow abs_dqcoeff = (abs_qcoeff * dequant) >> log_scale;

                if (abs_qcoeff) {

                    eob = i;

                }

                dqcoeff_ptr[rc] = (TranLow)((abs_dqcoeff ^ coeff_sign) - coeff_sign);

            } else {

                qcoeff_ptr[rc]  = 0;

                dqcoeff_ptr[rc] = 0;

            }

        }

    }

    *eob_ptr = eob + 1;

}

void svt_av1_highbd_quantize_fp_c(const TranLow* coeff_ptr, intptr_t count, const int16_t* zbin_ptr,

                                  const int16_t* round_ptr, const int16_t* quant_ptr, const int16_t* quant_shift_ptr,

                                  TranLow* qcoeff_ptr, TranLow* dqcoeff_ptr, const int16_t* dequant_ptr,

                                  uint16_t* eob_ptr, const int16_t* scan, const int16_t* iscan, int16_t log_scale) {

    highbd_quantize_fp_helper_c(coeff_ptr,

                                count,

                                zbin_ptr,

                                round_ptr,

                                quant_ptr,

                                quant_shift_ptr,

                                qcoeff_ptr,

                                dqcoeff_ptr,

                                dequant_ptr,

                                eob_ptr,

                                scan,

                                iscan,

                                NULL,

                                NULL,

                                log_scale);

}

void svt_av1_quantize_fp_32x32_c(const TranLow* coeff_ptr, intptr_t n_coeffs, const int16_t* zbin_ptr,

                                 const int16_t* round_ptr, const int16_t* quant_ptr, const int16_t* quant_shift_ptr,

                                 TranLow* qcoeff_ptr, TranLow* dqcoeff_ptr, const int16_t* dequant_ptr,

                                 uint16_t* eob_ptr, const int16_t* scan, const int16_t* iscan) {

    quantize_fp_helper_c(coeff_ptr,

                         n_coeffs,

                         zbin_ptr,

                         round_ptr,

                         quant_ptr,

                         quant_shift_ptr,

                         qcoeff_ptr,

                         dqcoeff_ptr,

                         dequant_ptr,

                         eob_ptr,

                         scan,

                         iscan,

                         NULL,

                         NULL,

                         1);

}

void svt_av1_quantize_fp_64x64_c(const TranLow* coeff_ptr, intptr_t n_coeffs, const int16_t* zbin_ptr,

                                 const int16_t* round_ptr, const int16_t* quant_ptr, const int16_t* quant_shift_ptr,

                                 TranLow* qcoeff_ptr, TranLow* dqcoeff_ptr, const int16_t* dequant_ptr,

                                 uint16_t* eob_ptr, const int16_t* scan, const int16_t* iscan) {

    quantize_fp_helper_c(coeff_ptr,

                         n_coeffs,

                         zbin_ptr,

                         round_ptr,

                         quant_ptr,

                         quant_shift_ptr,

                         qcoeff_ptr,

                         dqcoeff_ptr,

                         dequant_ptr,

                         eob_ptr,

                         scan,

                         iscan,

                         NULL,

                         NULL,

                         2);

}

void svt_av1_quantize_fp_facade(const TranLow* coeff_ptr, intptr_t n_coeffs, const MacroblockPlane* p,

                                TranLow* qcoeff_ptr, TranLow* dqcoeff_ptr, uint16_t* eob_ptr, const ScanOrder* sc,

                                const QuantParam* qparam) {

    const QmVal* qm_ptr  = qparam->qmatrix;

    const QmVal* iqm_ptr = qparam->iqmatrix;

    if (qm_ptr || iqm_ptr) {

        svt_av1_quantize_fp_qm(coeff_ptr,

                               n_coeffs,

                               p->zbin_qtx,

                               p->round_fp_qtx,

                               p->quant_fp_qtx,

                               p->quant_shift_qtx,

                               qcoeff_ptr,

                               dqcoeff_ptr,

                               p->dequant_qtx,

                               eob_ptr,

                               sc->scan,

                               sc->iscan,

                               qm_ptr,

                               iqm_ptr,

                               qparam->log_scale);

    } else {

        switch (qparam->log_scale) {

        case 0:

            svt_av1_quantize_fp(coeff_ptr,

                                n_coeffs,

                                p->zbin_qtx,

                                p->round_fp_qtx,

                                p->quant_fp_qtx,

                                p->quant_shift_qtx,

                                qcoeff_ptr,

                                dqcoeff_ptr,

                                p->dequant_qtx,

                                eob_ptr,

                                sc->scan,

                                sc->iscan);

            break;

        case 1:

            svt_av1_quantize_fp_32x32(coeff_ptr,

                                      n_coeffs,

                                      p->zbin_qtx,

                                      p->round_fp_qtx,

                                      p->quant_fp_qtx,

                                      p->quant_shift_qtx,

                                      qcoeff_ptr,

                                      dqcoeff_ptr,

                                      p->dequant_qtx,

                                      eob_ptr,

                                      sc->scan,

                                      sc->iscan);

            break;

        case 2:

            svt_av1_quantize_fp_64x64(coeff_ptr,

                                      n_coeffs,

                                      p->zbin_qtx,

                                      p->round_fp_qtx,

                                      p->quant_fp_qtx,

                                      p->quant_shift_qtx,

                                      qcoeff_ptr,

                                      dqcoeff_ptr,

                                      p->dequant_qtx,

                                      eob_ptr,

                                      sc->scan,

                                      sc->iscan);

            break;

        default:

            assert(0);

        }

    }

}

#if CONFIG_ENABLE_HIGH_BIT_DEPTH

void svt_av1_highbd_quantize_fp_facade(const TranLow* coeff_ptr, intptr_t n_coeffs, const MacroblockPlane* p,

                                       TranLow* qcoeff_ptr, TranLow* dqcoeff_ptr, uint16_t* eob_ptr,

                                       const ScanOrder* sc, const QuantParam* qparam) {

    const QmVal* qm_ptr  = qparam->qmatrix;

    const QmVal* iqm_ptr = qparam->iqmatrix;

    if (qm_ptr != NULL && iqm_ptr != NULL) {

        svt_av1_highbd_quantize_fp_qm(coeff_ptr,

                                      n_coeffs,

                                      p->zbin_qtx,

                                      p->round_fp_qtx,

                                      p->quant_fp_qtx,

                                      p->quant_shift_qtx,

                                      qcoeff_ptr,

                                      dqcoeff_ptr,

                                      p->dequant_qtx,

                                      eob_ptr,

                                      sc->scan,

                                      sc->iscan,

                                      qm_ptr,

                                      iqm_ptr,

                                      qparam->log_scale);

    } else {

        svt_av1_highbd_quantize_fp(coeff_ptr,

                                   n_coeffs,

                                   p->zbin_qtx,

                                   p->round_fp_qtx,

                                   p->quant_fp_qtx,

                                   p->quant_shift_qtx,

                                   qcoeff_ptr,

                                   dqcoeff_ptr,

                                   p->dequant_qtx,

                                   eob_ptr,

                                   sc->scan,

                                   sc->iscan,

                                   qparam->log_scale);

    }

}

#endif

void svt_av1_highbd_quantize_fp_qm_c(const TranLow* coeff_ptr, intptr_t count, const int16_t* zbin_ptr,

                                     const int16_t* round_ptr, const int16_t* quant_ptr, const int16_t* quant_shift_ptr,

                                     TranLow* qcoeff_ptr, TranLow* dqcoeff_ptr, const int16_t* dequant_ptr,

                                     uint16_t* eob_ptr, const int16_t* scan, const int16_t* iscan, const QmVal* qm_ptr,

                                     const QmVal* iqm_ptr, int16_t log_scale) {

    highbd_quantize_fp_helper_c(coeff_ptr,

                                count,

                                zbin_ptr,

                                round_ptr,

                                quant_ptr,

                                quant_shift_ptr,

                                qcoeff_ptr,

                                dqcoeff_ptr,

                                dequant_ptr,

                                eob_ptr,

                                scan,

                                iscan,

                                qm_ptr,

                                iqm_ptr,

                                log_scale);

}

static INLINE int get_lower_levels_ctx_general(int is_last, int scan_idx, int bwl, int height, const uint8_t* levels,

                                               int coeff_idx, TxSize tx_size, TxClass tx_class) {

    if (is_last) {

        if (scan_idx == 0) {

            return 0;

        }

        if (scan_idx <= (height << bwl) >> 3) {

            return 1;

        }

        if (scan_idx <= (height << bwl) >> 2) {

            return 2;

        }

        return 3;

    }

    return get_lower_levels_ctx(levels, coeff_idx, bwl, tx_size, tx_class);

}

static INLINE int32_t get_golomb_cost(int32_t abs_qc) {

    if (abs_qc >= 1 + NUM_BASE_LEVELS + COEFF_BASE_RANGE) {

        const int32_t r      = abs_qc - COEFF_BASE_RANGE - NUM_BASE_LEVELS;

        const int32_t length = get_msb(r) + 1;

        return av1_cost_literal(2 * length - 1);

    }

    return 0;

}

static INLINE int get_br_cost(TranLow level, const int* coeff_lps) {

    const int base_range = AOMMIN(level - 1 - NUM_BASE_LEVELS, COEFF_BASE_RANGE);

    return coeff_lps[base_range] + get_golomb_cost(level);

}

static INLINE int get_coeff_cost_general(int is_last, int ci, TranLow abs_qc, int sign, int coeff_ctx, int dc_sign_ctx,

                                         const LvMapCoeffCost* txb_costs, int bwl, TxClass tx_class,

                                         const uint8_t* levels) {

    int cost = 0;

    if (is_last) {

        cost += txb_costs->base_eob_cost[coeff_ctx][AOMMIN(abs_qc, 3) - 1];

    } else {

        cost += txb_costs->base_cost[coeff_ctx][AOMMIN(abs_qc, 3)];

    }

    // All callers invoke this only on the non-zero coeff path, so abs_qc != 0 always.

    if (ci == 0) {

        cost += txb_costs->dc_sign_cost[dc_sign_ctx][sign];

    } else {

        cost += av1_cost_literal(1);

    }

    if (abs_qc > NUM_BASE_LEVELS) {

        int br_ctx;

        if (is_last) {

            br_ctx = get_br_ctx_eob(ci, bwl, tx_class);

        } else {

            br_ctx = get_br_ctx(levels, ci, bwl, tx_class);

        }

        cost += get_br_cost(abs_qc, txb_costs->lps_cost[br_ctx]);

    }

    return cost;

}

static INLINE int64_t get_coeff_dist(TranLow tcoeff, TranLow dqcoeff, int shift) {

    return SQR(((int64_t)tcoeff - dqcoeff) * (int64_t)(1lu << shift));

}

static INLINE void get_qc_dqc_low(TranLow abs_qc, int sign, int dqv, int shift, TranLow* qc_low, TranLow* dqc_low) {

    TranLow abs_qc_low = abs_qc - 1;

    *qc_low            = (-sign ^ abs_qc_low) + sign;

    assert((sign ? -abs_qc_low : abs_qc_low) == *qc_low);

    TranLow abs_dqc_low = (abs_qc_low * dqv) >> shift;

    *dqc_low            = (-sign ^ abs_dqc_low) + sign;

    assert((sign ? -abs_dqc_low : abs_dqc_low) == *dqc_low);

}

static const int golomb_bits_cost[32] = {0,       512,     512 * 3, 512 * 3, 512 * 5, 512 * 5, 512 * 5, 512 * 5,

                                         512 * 7, 512 * 7, 512 * 7, 512 * 7, 512 * 7, 512 * 7, 512 * 7, 512 * 7,

                                         512 * 9, 512 * 9, 512 * 9, 512 * 9, 512 * 9, 512 * 9, 512 * 9, 512 * 9,

                                         512 * 9, 512 * 9, 512 * 9, 512 * 9, 512 * 9, 512 * 9, 512 * 9, 512 * 9};

static const int golomb_cost_diff[32] = {0,       512, 512 * 2, 0, 512 * 2, 0, 0, 0, 512 * 2, 0, 0, 0, 0, 0, 0, 0,

                                         512 * 2, 0,   0,       0, 0,       0, 0, 0, 0,       0, 0, 0, 0, 0, 0, 0};

static INLINE int get_br_cost_with_diff(TranLow level, const int* coeff_lps, int* diff) {

    const int base_range  = AOMMIN(level - 1 - NUM_BASE_LEVELS, COEFF_BASE_RANGE);

    int       golomb_bits = 0;

    if (level <= COEFF_BASE_RANGE + 1 + NUM_BASE_LEVELS) {

        *diff += coeff_lps[base_range + COEFF_BASE_RANGE + 1];

    }

    if (level >= COEFF_BASE_RANGE + 1 + NUM_BASE_LEVELS) {

        int r = level - COEFF_BASE_RANGE - NUM_BASE_LEVELS;

        if (r < 32) {

            golomb_bits = golomb_bits_cost[r];

            *diff += golomb_cost_diff[r];

        } else {

            golomb_bits = get_golomb_cost(level);

            *diff += (r & (r - 1)) == 0 ? 1024 : 0;

        }

    }

    return coeff_lps[base_range] + golomb_bits;

}

static AOM_FORCE_INLINE int get_two_coeff_cost_simple(int ci, TranLow abs_qc, int coeff_ctx,

                                                      const LvMapCoeffCost* txb_costs, int bwl, TxClass tx_class,

                                                      const uint8_t* levels, int* cost_low) {

    // this simple version assumes the coeff's scan_idx is not DC (scan_idx != 0)

    // and not the last (scan_idx != eob - 1)

    assert(ci > 0);

    //assert(abs_qc + 4 < 4);

    int cost = txb_costs->base_cost[coeff_ctx][AOMMIN(abs_qc, 3)];

    int diff = 0;

    if (abs_qc <= 3) {

        diff = txb_costs->base_cost[coeff_ctx][abs_qc + 4];

    }

    // Caller (update_coeff_simple) only invokes this with abs_qc != 0.

    cost += av1_cost_literal(1);

    if (abs_qc > NUM_BASE_LEVELS) {

        const int br_ctx      = get_br_ctx(levels, ci, bwl, tx_class);

        int       brcost_diff = 0;

        cost += get_br_cost_with_diff(abs_qc, txb_costs->lps_cost[br_ctx], &brcost_diff);

        diff += brcost_diff;

    }

    *cost_low = cost - diff;

    return cost;

}

static INLINE int get_coeff_cost_eob(int ci, TranLow abs_qc, int sign, int coeff_ctx, int dc_sign_ctx,

                                     const LvMapCoeffCost* txb_costs, int bwl, TxClass tx_class) {

    int cost = 0;

    cost += txb_costs->base_eob_cost[coeff_ctx][AOMMIN(abs_qc, 3) - 1];

    // Callers only invoke this on the non-zero coeff path, so abs_qc != 0 always.

    if (ci == 0) {

        cost += txb_costs->dc_sign_cost[dc_sign_ctx][sign];

    } else {

        cost += av1_cost_literal(1);

    }

    if (abs_qc > NUM_BASE_LEVELS) {

        const int br_ctx = get_br_ctx_eob(ci, bwl, tx_class);

        cost += get_br_cost(abs_qc, txb_costs->lps_cost[br_ctx]);

    }

    return cost;

}

static INLINE int get_dqv(const int16_t* dequant, int coeff_idx, const QmVal* iqm_ptr) {

    int dqv = dequant[!!coeff_idx];

    if (iqm_ptr != NULL) {

        dqv = ((iqm_ptr[coeff_idx] * dqv) + (1 << (AOM_QM_BITS - 1))) >> AOM_QM_BITS;

    }

    return dqv;

}

static AOM_FORCE_INLINE void update_coeff_eob(int* accu_rate, int64_t* accu_dist, uint16_t* eob, int* nz_num,

                                              int* nz_ci, int si, TxSize tx_size, TxClass tx_class, int bwl, int height,

                                              int dc_sign_ctx, int64_t rdmult, int shift, const int16_t* dequant,

                                              const int16_t* scan, const LvMapEobCost* txb_eob_costs,

                                              const LvMapCoeffCost* txb_costs, const TranLow* tcoeff, TranLow* qcoeff,

                                              TranLow* dqcoeff, uint8_t* levels, int sharpness, const QmVal* iqm_ptr) {

    assert(si != *eob - 1);

    const int     ci        = scan[si];

    const TranLow qc        = qcoeff[ci];

    const int     coeff_ctx = get_lower_levels_ctx(levels, ci, bwl, tx_size, tx_class);

    if (qc == 0) {

        *accu_rate += txb_costs->base_cost[coeff_ctx][0];

    } else {

        int           lower_level = 0;

        const TranLow abs_qc      = abs(qc);

        const TranLow tqc         = tcoeff[ci];

        const TranLow dqc         = dqcoeff[ci];

        const int     sign        = (qc < 0) ? 1 : 0;

        const int64_t dist0       = get_coeff_dist(tqc, 0, shift);

        int64_t       dist        = get_coeff_dist(tqc, dqc, shift) - dist0;

        int           rate        = get_coeff_cost_general(

            0, ci, abs_qc, sign, coeff_ctx, dc_sign_ctx, txb_costs, bwl, tx_class, levels);

        int64_t rd = RDCOST(rdmult, *accu_rate + rate, *accu_dist + dist);

        TranLow qc_low, dqc_low;

        TranLow abs_qc_low;

        int64_t dist_low, rd_low;

        int     rate_low;

        if (abs_qc == 1) {

            abs_qc_low = 0;

            dqc_low = qc_low = 0;

            dist_low         = 0;

            rate_low         = txb_costs->base_cost[coeff_ctx][0];

            rd_low           = RDCOST(rdmult, *accu_rate + rate_low, *accu_dist);

        } else {

            const int dqv = get_dqv(dequant, ci, iqm_ptr);

            get_qc_dqc_low(abs_qc, sign, dqv, shift, &qc_low, &dqc_low);

            abs_qc_low = abs_qc - 1;

            dist_low   = get_coeff_dist(tqc, dqc_low, shift) - dist0;

            rate_low   = get_coeff_cost_general(

                0, ci, abs_qc_low, sign, coeff_ctx, dc_sign_ctx, txb_costs, bwl, tx_class, levels);

            rd_low = RDCOST(rdmult, *accu_rate + rate_low, *accu_dist + dist_low);

        }

        int       lower_level_new_eob = 0;

        const int new_eob             = si + 1;

        const int coeff_ctx_new_eob   = get_lower_levels_ctx_eob(bwl, height, si);

        const int new_eob_cost        = get_eob_cost(new_eob, txb_eob_costs, txb_costs, tx_class);

        int       rate_coeff_eob      = new_eob_cost +

            get_coeff_cost_eob(ci, abs_qc, sign, coeff_ctx_new_eob, dc_sign_ctx, txb_costs, bwl, tx_class);

        int64_t dist_new_eob = dist;

        int64_t rd_new_eob   = RDCOST(rdmult, rate_coeff_eob, dist_new_eob);

        if (abs_qc_low > 0) {

            const int rate_coeff_eob_low = new_eob_cost +

                get_coeff_cost_eob(ci, abs_qc_low, sign, coeff_ctx_new_eob, dc_sign_ctx, txb_costs, bwl, tx_class);

            const int64_t dist_new_eob_low = dist_low;

            const int64_t rd_new_eob_low   = RDCOST(rdmult, rate_coeff_eob_low, dist_new_eob_low);

            if (rd_new_eob_low < rd_new_eob) {

                lower_level_new_eob = 1;

                rd_new_eob          = rd_new_eob_low;

                rate_coeff_eob      = rate_coeff_eob_low;

                dist_new_eob        = dist_new_eob_low;

            }

        }

        if (rd_low < rd) {

            lower_level = 1;

            rd          = rd_low;

            rate        = rate_low;

            dist        = dist_low;

        }

        if (sharpness == 0 && rd_new_eob < rd) {

            for (int ni = 0; ni < *nz_num; ++ni) {

                int last_ci                          = nz_ci[ni];

                levels[get_padded_idx(last_ci, bwl)] = 0;

                qcoeff[last_ci]                      = 0;

                dqcoeff[last_ci]                     = 0;

            }

            *eob        = new_eob;

            *nz_num     = 0;

            *accu_rate  = rate_coeff_eob;

            *accu_dist  = dist_new_eob;

            lower_level = lower_level_new_eob;

        } else {

            *accu_rate += rate;

            *accu_dist += dist;

        }

        if (lower_level) {

            qcoeff[ci]                      = qc_low;

            dqcoeff[ci]                     = dqc_low;

            levels[get_padded_idx(ci, bwl)] = AOMMIN(abs_qc_low, INT8_MAX);

        }

        if (qcoeff[ci]) {

            nz_ci[*nz_num] = ci;

            ++*nz_num;

        }

    }

}

static INLINE void update_coeff_general(int* accu_rate, int64_t* accu_dist, int si, int eob, TxSize tx_size,

                                        TxClass tx_class, int bwl, int height, int64_t rdmult, int shift,

                                        int dc_sign_ctx, const int16_t* dequant, const int16_t* scan,

                                        const LvMapCoeffCost* txb_costs, const TranLow* tcoeff, TranLow* qcoeff,

                                        TranLow* dqcoeff, uint8_t* levels, const QmVal* iqm_ptr) {

    const int     ci        = scan[si];

    const TranLow qc        = qcoeff[ci];

    const int     is_last   = si == (eob - 1);

    const int     coeff_ctx = get_lower_levels_ctx_general(is_last, si, bwl, height, levels, ci, tx_size, tx_class);

    if (qc == 0) {

        *accu_rate += txb_costs->base_cost[coeff_ctx][0];

    } else {

        const int     sign   = (qc < 0) ? 1 : 0;

        const TranLow abs_qc = abs(qc);

        const TranLow tqc    = tcoeff[ci];

        const TranLow dqc    = dqcoeff[ci];

        const int64_t dist   = get_coeff_dist(tqc, dqc, shift);

        const int64_t dist0  = get_coeff_dist(tqc, 0, shift);

        const int     rate   = get_coeff_cost_general(

            is_last, ci, abs_qc, sign, coeff_ctx, dc_sign_ctx, txb_costs, bwl, tx_class, levels);

        const int64_t rd = RDCOST(rdmult, rate, dist);

        TranLow qc_low, dqc_low;

        TranLow abs_qc_low;

        int64_t dist_low, rd_low;

        int     rate_low;

        if (abs_qc == 1) {

            abs_qc_low = qc_low = dqc_low = 0;

            dist_low                      = dist0;

            rate_low                      = txb_costs->base_cost[coeff_ctx][0];

        } else {

            const int dqv = get_dqv(dequant, ci, iqm_ptr);

            get_qc_dqc_low(abs_qc, sign, dqv, shift, &qc_low, &dqc_low);

            abs_qc_low = abs_qc - 1;

            dist_low   = get_coeff_dist(tqc, dqc_low, shift);

            rate_low   = get_coeff_cost_general(

                is_last, ci, abs_qc_low, sign, coeff_ctx, dc_sign_ctx, txb_costs, bwl, tx_class, levels);

        }

        rd_low = RDCOST(rdmult, rate_low, dist_low);

        if (rd_low < rd) {

            qcoeff[ci]                      = qc_low;

            dqcoeff[ci]                     = dqc_low;

            levels[get_padded_idx(ci, bwl)] = AOMMIN(abs_qc_low, INT8_MAX);

            *accu_rate += rate_low;

            *accu_dist += dist_low - dist0;

        } else {

            *accu_rate += rate;

            *accu_dist += dist - dist0;

        }

    }

}

static AOM_FORCE_INLINE void update_coeff_simple(int* accu_rate, int si, int eob, TxSize tx_size, TxClass tx_class,

                                                 int bwl, int64_t rdmult, int shift, const int16_t* dequant,

                                                 const int16_t* scan, const LvMapCoeffCost* txb_costs,

                                                 const TranLow* tcoeff, TranLow* qcoeff, TranLow* dqcoeff,

                                                 uint8_t* levels, const QmVal* iqm_ptr) {

    (void)eob;

    // this simple version assumes the coeff's scan_idx is not DC (scan_idx != 0)

    // and not the last (scan_idx != eob - 1)

    assert(si != eob - 1);

    assert(si > 0);

    const int     ci        = scan[si];

    const TranLow qc        = qcoeff[ci];

    const int     coeff_ctx = get_lower_levels_ctx(levels, ci, bwl, tx_size, tx_class);

    if (qc == 0) {

        *accu_rate += txb_costs->base_cost[coeff_ctx][0];

    } else {

        const TranLow abs_qc   = abs(qc);

        const TranLow abs_tqc  = abs(tcoeff[ci]);

        const TranLow abs_dqc  = abs(dqcoeff[ci]);

        int           rate_low = 0;

        const int rate = get_two_coeff_cost_simple(ci, abs_qc, coeff_ctx, txb_costs, bwl, tx_class, levels, &rate_low);

        if (abs_dqc < abs_tqc) {