// Copyright (c) the JPEG XL Project Authors. All rights reserved.

//

// Use of this source code is governed by a BSD-style

// license that can be found in the LICENSE file.

#include <cstddef>

#include <cstdint>

#if defined(LIB_JXL_QUANTIZER_INL_H_) == defined(HWY_TARGET_TOGGLE)

#ifdef LIB_JXL_QUANTIZER_INL_H_

#undef LIB_JXL_QUANTIZER_INL_H_

#else

#define LIB_JXL_QUANTIZER_INL_H_

#endif

#include <hwy/highway.h>

HWY_BEFORE_NAMESPACE();

namespace jxl {

namespace HWY_NAMESPACE {

namespace {

// These templates are not found via ADL.

using hwy::HWY_NAMESPACE::And;

using hwy::HWY_NAMESPACE::AndNot;

using hwy::HWY_NAMESPACE::ApproximateReciprocal;

using hwy::HWY_NAMESPACE::Gt;

using hwy::HWY_NAMESPACE::IfThenElse;

using hwy::HWY_NAMESPACE::IfThenElseZero;

using hwy::HWY_NAMESPACE::Lt;

using hwy::HWY_NAMESPACE::Rebind;

using hwy::HWY_NAMESPACE::Vec;

using hwy::HWY_NAMESPACE::Xor;

template <class DI>

HWY_INLINE HWY_MAYBE_UNUSED Vec<Rebind<float, DI>> AdjustQuantBias(

    DI di, const size_t c, const Vec<DI> quant_i,

    const float* HWY_RESTRICT biases) {

  const Rebind<float, DI> df;

  const auto quant = ConvertTo(df, quant_i);

  // Compare |quant|, keep sign bit for negating result.

  const auto kSign = BitCast(df, Set(di, INT32_MIN));

  const auto sign = And(quant, kSign);  // TODO(janwas): = abs ^ orig

  const auto abs_quant = AndNot(kSign, quant);

  // If |x| is 1, kZeroBias creates a different bias for each channel.

  // We're implementing the following:

  // if (quant == 0) return 0;

  // if (quant == 1) return biases[c];

  // if (quant == -1) return -biases[c];

  // return quant - biases[3] / quant;

  // Integer comparison is not helpful because Clang incurs bypass penalties

  // from unnecessarily mixing integer and float.

  const auto is_01 = Lt(abs_quant, Set(df, 1.125f));

  const auto not_0 = Gt(abs_quant, Zero(df));

  // Bitwise logic is faster than quant * biases[c].

  const auto one_bias = IfThenElseZero(not_0, Xor(Set(df, biases[c]), sign));

  // About 2E-5 worse than ReciprocalNR or division.

  const auto bias =

      NegMulAdd(Set(df, biases[3]), ApproximateReciprocal(quant), quant);

  return IfThenElse(is_01, one_bias, bias);

}

Unexecuted instantiation: enc_group.cc:decltype (Zero((hwy::N_SSE4::Simd<int, 4ul, 0>::Rebind<float>)())) jxl::N_SSE4::(anonymous namespace)::AdjustQuantBias<hwy::N_SSE4::Simd<int, 4ul, 0> >(hwy::N_SSE4::Simd<int, 4ul, 0>, unsigned long, decltype (Zero((hwy::N_SSE4::Simd<int, 4ul, 0>)())), float const*)

enc_group.cc:decltype (Zero((hwy::N_AVX2::Simd<int, 8ul, 0>::Rebind<float>)())) jxl::N_AVX2::(anonymous namespace)::AdjustQuantBias<hwy::N_AVX2::Simd<int, 8ul, 0> >(hwy::N_AVX2::Simd<int, 8ul, 0>, unsigned long, decltype (Zero((hwy::N_AVX2::Simd<int, 8ul, 0>)())), float const*)

Line

Count

Source

37

43.4M

    const float* HWY_RESTRICT biases) {

38

43.4M

  const Rebind<float, DI> df;

39

40

43.4M

  const auto quant = ConvertTo(df, quant_i);

41

42

  // Compare |quant|, keep sign bit for negating result.

43

43.4M

  const auto kSign = BitCast(df, Set(di, INT32_MIN));

44

43.4M

  const auto sign = And(quant, kSign);  // TODO(janwas): = abs ^ orig

45

43.4M

  const auto abs_quant = AndNot(kSign, quant);

46

47

  // If |x| is 1, kZeroBias creates a different bias for each channel.

48

  // We're implementing the following:

49

  // if (quant == 0) return 0;

50

  // if (quant == 1) return biases[c];

51

  // if (quant == -1) return -biases[c];

52

  // return quant - biases[3] / quant;

53

54

  // Integer comparison is not helpful because Clang incurs bypass penalties

55

  // from unnecessarily mixing integer and float.

56

43.4M

  const auto is_01 = Lt(abs_quant, Set(df, 1.125f));

57

43.4M

  const auto not_0 = Gt(abs_quant, Zero(df));

58

59

  // Bitwise logic is faster than quant * biases[c].

60

43.4M

  const auto one_bias = IfThenElseZero(not_0, Xor(Set(df, biases[c]), sign));

61

62

  // About 2E-5 worse than ReciprocalNR or division.

63

43.4M

  const auto bias =

64

43.4M

      NegMulAdd(Set(df, biases[3]), ApproximateReciprocal(quant), quant);

65

66

43.4M

  return IfThenElse(is_01, one_bias, bias);

67

43.4M

}

Unexecuted instantiation: enc_group.cc:decltype (Zero((hwy::N_SSE2::Simd<int, 4ul, 0>::Rebind<float>)())) jxl::N_SSE2::(anonymous namespace)::AdjustQuantBias<hwy::N_SSE2::Simd<int, 4ul, 0> >(hwy::N_SSE2::Simd<int, 4ul, 0>, unsigned long, decltype (Zero((hwy::N_SSE2::Simd<int, 4ul, 0>)())), float const*)

Unexecuted instantiation: dec_group.cc:decltype (Zero((hwy::N_SSE4::Simd<int, 4ul, 0>::Rebind<float>)())) jxl::N_SSE4::(anonymous namespace)::AdjustQuantBias<hwy::N_SSE4::Simd<int, 4ul, 0> >(hwy::N_SSE4::Simd<int, 4ul, 0>, unsigned long, decltype (Zero((hwy::N_SSE4::Simd<int, 4ul, 0>)())), float const*)

dec_group.cc:decltype (Zero((hwy::N_AVX2::Simd<int, 8ul, 0>::Rebind<float>)())) jxl::N_AVX2::(anonymous namespace)::AdjustQuantBias<hwy::N_AVX2::Simd<int, 8ul, 0> >(hwy::N_AVX2::Simd<int, 8ul, 0>, unsigned long, decltype (Zero((hwy::N_AVX2::Simd<int, 8ul, 0>)())), float const*)

Line

Count

Source

37

404M

    const float* HWY_RESTRICT biases) {

38

404M

  const Rebind<float, DI> df;

39

40

404M

  const auto quant = ConvertTo(df, quant_i);

41

42

  // Compare |quant|, keep sign bit for negating result.

43

404M

  const auto kSign = BitCast(df, Set(di, INT32_MIN));

44

404M

  const auto sign = And(quant, kSign);  // TODO(janwas): = abs ^ orig

45

404M

  const auto abs_quant = AndNot(kSign, quant);

46

47

  // If |x| is 1, kZeroBias creates a different bias for each channel.

48

  // We're implementing the following:

49

  // if (quant == 0) return 0;

50

  // if (quant == 1) return biases[c];

51

  // if (quant == -1) return -biases[c];

52

  // return quant - biases[3] / quant;

53

54

  // Integer comparison is not helpful because Clang incurs bypass penalties

55

  // from unnecessarily mixing integer and float.

56

404M

  const auto is_01 = Lt(abs_quant, Set(df, 1.125f));

57

404M

  const auto not_0 = Gt(abs_quant, Zero(df));

58

59

  // Bitwise logic is faster than quant * biases[c].

60

404M

  const auto one_bias = IfThenElseZero(not_0, Xor(Set(df, biases[c]), sign));

61

62

  // About 2E-5 worse than ReciprocalNR or division.

63

404M

  const auto bias =

64

404M

      NegMulAdd(Set(df, biases[3]), ApproximateReciprocal(quant), quant);

65

66

404M

  return IfThenElse(is_01, one_bias, bias);

67

404M

}

Unexecuted instantiation: dec_group.cc:decltype (Zero((hwy::N_SSE2::Simd<int, 4ul, 0>::Rebind<float>)())) jxl::N_SSE2::(anonymous namespace)::AdjustQuantBias<hwy::N_SSE2::Simd<int, 4ul, 0> >(hwy::N_SSE2::Simd<int, 4ul, 0>, unsigned long, decltype (Zero((hwy::N_SSE2::Simd<int, 4ul, 0>)())), float const*)

}  // namespace

// NOLINTNEXTLINE(google-readability-namespace-comments)

}  // namespace HWY_NAMESPACE

}  // namespace jxl

HWY_AFTER_NAMESPACE();

#endif  // LIB_JXL_QUANTIZER_INL_H_