Coverage Report

Created: 2025-08-12 07:37

/src/libjxl/lib/jxl/base/fast_math-inl.h
Line
Count
Source (jump to first uncovered line)
1
// Copyright (c) the JPEG XL Project Authors. All rights reserved.
2
//
3
// Use of this source code is governed by a BSD-style
4
// license that can be found in the LICENSE file.
5
6
// Fast SIMD math ops (log2, encoder only, cos, erf for splines)
7
8
#include <cstdint>
9
10
#if defined(LIB_JXL_BASE_FAST_MATH_INL_H_) == defined(HWY_TARGET_TOGGLE)
11
#ifdef LIB_JXL_BASE_FAST_MATH_INL_H_
12
#undef LIB_JXL_BASE_FAST_MATH_INL_H_
13
#else
14
#define LIB_JXL_BASE_FAST_MATH_INL_H_
15
#endif
16
17
#include <hwy/highway.h>
18
19
#include "lib/jxl/base/common.h"
20
#include "lib/jxl/base/rational_polynomial-inl.h"
21
HWY_BEFORE_NAMESPACE();
22
namespace jxl {
23
namespace HWY_NAMESPACE {
24
25
// These templates are not found via ADL.
26
using hwy::HWY_NAMESPACE::Abs;
27
using hwy::HWY_NAMESPACE::Add;
28
using hwy::HWY_NAMESPACE::Eq;
29
using hwy::HWY_NAMESPACE::Floor;
30
using hwy::HWY_NAMESPACE::Ge;
31
using hwy::HWY_NAMESPACE::GetLane;
32
using hwy::HWY_NAMESPACE::IfThenElse;
33
using hwy::HWY_NAMESPACE::IfThenZeroElse;
34
using hwy::HWY_NAMESPACE::Le;
35
using hwy::HWY_NAMESPACE::Min;
36
using hwy::HWY_NAMESPACE::Mul;
37
using hwy::HWY_NAMESPACE::MulAdd;
38
using hwy::HWY_NAMESPACE::NegMulAdd;
39
using hwy::HWY_NAMESPACE::Rebind;
40
using hwy::HWY_NAMESPACE::ShiftLeft;
41
using hwy::HWY_NAMESPACE::ShiftRight;
42
using hwy::HWY_NAMESPACE::Sub;
43
using hwy::HWY_NAMESPACE::Xor;
44
45
// Computes base-2 logarithm like std::log2. Undefined if negative / NaN.
46
// L1 error ~3.9E-6
47
template <class DF, class V>
48
76.6M
V FastLog2f(const DF df, V x) {
49
  // 2,2 rational polynomial approximation of std::log1p(x) / std::log(2).
50
76.6M
  HWY_ALIGN const float p[4 * (2 + 1)] = {HWY_REP4(-1.8503833400518310E-06f),
51
76.6M
                                          HWY_REP4(1.4287160470083755E+00f),
52
76.6M
                                          HWY_REP4(7.4245873327820566E-01f)};
53
76.6M
  HWY_ALIGN const float q[4 * (2 + 1)] = {HWY_REP4(9.9032814277590719E-01f),
54
76.6M
                                          HWY_REP4(1.0096718572241148E+00f),
55
76.6M
                                          HWY_REP4(1.7409343003366853E-01f)};
56
57
76.6M
  const Rebind<int32_t, DF> di;
58
76.6M
  const auto x_bits = BitCast(di, x);
59
60
  // Range reduction to [-1/3, 1/3] - 3 integer, 2 float ops
61
76.6M
  const auto exp_bits = Sub(x_bits, Set(di, 0x3f2aaaab));  // = 2/3
62
  // Shifted exponent = log2; also used to clear mantissa.
63
76.6M
  const auto exp_shifted = ShiftRight<23>(exp_bits);
64
76.6M
  const auto mantissa = BitCast(df, Sub(x_bits, ShiftLeft<23>(exp_shifted)));
65
76.6M
  const auto exp_val = ConvertTo(df, exp_shifted);
66
76.6M
  return Add(EvalRationalPolynomial(df, Sub(mantissa, Set(df, 1.0f)), p, q),
67
76.6M
             exp_val);
68
76.6M
}
Unexecuted instantiation: hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::FastLog2f<hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul> >(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>)
hwy::N_AVX2::Vec256<float> jxl::N_AVX2::FastLog2f<hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float> >(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>)
Line
Count
Source
48
74.9M
V FastLog2f(const DF df, V x) {
49
  // 2,2 rational polynomial approximation of std::log1p(x) / std::log(2).
50
74.9M
  HWY_ALIGN const float p[4 * (2 + 1)] = {HWY_REP4(-1.8503833400518310E-06f),
51
74.9M
                                          HWY_REP4(1.4287160470083755E+00f),
52
74.9M
                                          HWY_REP4(7.4245873327820566E-01f)};
53
74.9M
  HWY_ALIGN const float q[4 * (2 + 1)] = {HWY_REP4(9.9032814277590719E-01f),
54
74.9M
                                          HWY_REP4(1.0096718572241148E+00f),
55
74.9M
                                          HWY_REP4(1.7409343003366853E-01f)};
56
57
74.9M
  const Rebind<int32_t, DF> di;
58
74.9M
  const auto x_bits = BitCast(di, x);
59
60
  // Range reduction to [-1/3, 1/3] - 3 integer, 2 float ops
61
74.9M
  const auto exp_bits = Sub(x_bits, Set(di, 0x3f2aaaab));  // = 2/3
62
  // Shifted exponent = log2; also used to clear mantissa.
63
74.9M
  const auto exp_shifted = ShiftRight<23>(exp_bits);
64
74.9M
  const auto mantissa = BitCast(df, Sub(x_bits, ShiftLeft<23>(exp_shifted)));
65
74.9M
  const auto exp_val = ConvertTo(df, exp_shifted);
66
74.9M
  return Add(EvalRationalPolynomial(df, Sub(mantissa, Set(df, 1.0f)), p, q),
67
74.9M
             exp_val);
68
74.9M
}
Unexecuted instantiation: hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::FastLog2f<hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul> >(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>)
Unexecuted instantiation: hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::FastLog2f<hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul> >(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>)
hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::FastLog2f<hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul> >(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>)
Line
Count
Source
48
344k
V FastLog2f(const DF df, V x) {
49
  // 2,2 rational polynomial approximation of std::log1p(x) / std::log(2).
50
344k
  HWY_ALIGN const float p[4 * (2 + 1)] = {HWY_REP4(-1.8503833400518310E-06f),
51
344k
                                          HWY_REP4(1.4287160470083755E+00f),
52
344k
                                          HWY_REP4(7.4245873327820566E-01f)};
53
344k
  HWY_ALIGN const float q[4 * (2 + 1)] = {HWY_REP4(9.9032814277590719E-01f),
54
344k
                                          HWY_REP4(1.0096718572241148E+00f),
55
344k
                                          HWY_REP4(1.7409343003366853E-01f)};
56
57
344k
  const Rebind<int32_t, DF> di;
58
344k
  const auto x_bits = BitCast(di, x);
59
60
  // Range reduction to [-1/3, 1/3] - 3 integer, 2 float ops
61
344k
  const auto exp_bits = Sub(x_bits, Set(di, 0x3f2aaaab));  // = 2/3
62
  // Shifted exponent = log2; also used to clear mantissa.
63
344k
  const auto exp_shifted = ShiftRight<23>(exp_bits);
64
344k
  const auto mantissa = BitCast(df, Sub(x_bits, ShiftLeft<23>(exp_shifted)));
65
344k
  const auto exp_val = ConvertTo(df, exp_shifted);
66
344k
  return Add(EvalRationalPolynomial(df, Sub(mantissa, Set(df, 1.0f)), p, q),
67
344k
             exp_val);
68
344k
}
hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::FastLog2f<hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul> >(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>)
Line
Count
Source
48
79.4k
V FastLog2f(const DF df, V x) {
49
  // 2,2 rational polynomial approximation of std::log1p(x) / std::log(2).
50
79.4k
  HWY_ALIGN const float p[4 * (2 + 1)] = {HWY_REP4(-1.8503833400518310E-06f),
51
79.4k
                                          HWY_REP4(1.4287160470083755E+00f),
52
79.4k
                                          HWY_REP4(7.4245873327820566E-01f)};
53
79.4k
  HWY_ALIGN const float q[4 * (2 + 1)] = {HWY_REP4(9.9032814277590719E-01f),
54
79.4k
                                          HWY_REP4(1.0096718572241148E+00f),
55
79.4k
                                          HWY_REP4(1.7409343003366853E-01f)};
56
57
79.4k
  const Rebind<int32_t, DF> di;
58
79.4k
  const auto x_bits = BitCast(di, x);
59
60
  // Range reduction to [-1/3, 1/3] - 3 integer, 2 float ops
61
79.4k
  const auto exp_bits = Sub(x_bits, Set(di, 0x3f2aaaab));  // = 2/3
62
  // Shifted exponent = log2; also used to clear mantissa.
63
79.4k
  const auto exp_shifted = ShiftRight<23>(exp_bits);
64
79.4k
  const auto mantissa = BitCast(df, Sub(x_bits, ShiftLeft<23>(exp_shifted)));
65
79.4k
  const auto exp_val = ConvertTo(df, exp_shifted);
66
79.4k
  return Add(EvalRationalPolynomial(df, Sub(mantissa, Set(df, 1.0f)), p, q),
67
79.4k
             exp_val);
68
79.4k
}
hwy::N_AVX2::Vec128<float, 4ul> jxl::N_AVX2::FastLog2f<hwy::N_AVX2::Simd<float, 4ul, 0>, hwy::N_AVX2::Vec128<float, 4ul> >(hwy::N_AVX2::Simd<float, 4ul, 0>, hwy::N_AVX2::Vec128<float, 4ul>)
Line
Count
Source
48
1.30M
V FastLog2f(const DF df, V x) {
49
  // 2,2 rational polynomial approximation of std::log1p(x) / std::log(2).
50
1.30M
  HWY_ALIGN const float p[4 * (2 + 1)] = {HWY_REP4(-1.8503833400518310E-06f),
51
1.30M
                                          HWY_REP4(1.4287160470083755E+00f),
52
1.30M
                                          HWY_REP4(7.4245873327820566E-01f)};
53
1.30M
  HWY_ALIGN const float q[4 * (2 + 1)] = {HWY_REP4(9.9032814277590719E-01f),
54
1.30M
                                          HWY_REP4(1.0096718572241148E+00f),
55
1.30M
                                          HWY_REP4(1.7409343003366853E-01f)};
56
57
1.30M
  const Rebind<int32_t, DF> di;
58
1.30M
  const auto x_bits = BitCast(di, x);
59
60
  // Range reduction to [-1/3, 1/3] - 3 integer, 2 float ops
61
1.30M
  const auto exp_bits = Sub(x_bits, Set(di, 0x3f2aaaab));  // = 2/3
62
  // Shifted exponent = log2; also used to clear mantissa.
63
1.30M
  const auto exp_shifted = ShiftRight<23>(exp_bits);
64
1.30M
  const auto mantissa = BitCast(df, Sub(x_bits, ShiftLeft<23>(exp_shifted)));
65
1.30M
  const auto exp_val = ConvertTo(df, exp_shifted);
66
1.30M
  return Add(EvalRationalPolynomial(df, Sub(mantissa, Set(df, 1.0f)), p, q),
67
1.30M
             exp_val);
68
1.30M
}
69
70
// max relative error ~3e-7
71
template <class DF, class V>
72
2.48M
V FastPow2f(const DF df, V x) {
73
2.48M
  const Rebind<int32_t, DF> di;
74
2.48M
  auto floorx = Floor(x);
75
2.48M
  auto exp =
76
2.48M
      BitCast(df, ShiftLeft<23>(Add(ConvertTo(di, floorx), Set(di, 127))));
77
2.48M
  auto frac = Sub(x, floorx);
78
2.48M
  auto num = Add(frac, Set(df, 1.01749063e+01));
79
2.48M
  num = MulAdd(num, frac, Set(df, 4.88687798e+01));
80
2.48M
  num = MulAdd(num, frac, Set(df, 9.85506591e+01));
81
2.48M
  num = Mul(num, exp);
82
2.48M
  auto den = MulAdd(frac, Set(df, 2.10242958e-01), Set(df, -2.22328856e-02));
83
2.48M
  den = MulAdd(den, frac, Set(df, -1.94414990e+01));
84
2.48M
  den = MulAdd(den, frac, Set(df, 9.85506633e+01));
85
2.48M
  return Div(num, den);
86
2.48M
}
Unexecuted instantiation: hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::FastPow2f<hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul> >(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>)
hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::FastPow2f<hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul> >(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>)
Line
Count
Source
72
697k
V FastPow2f(const DF df, V x) {
73
697k
  const Rebind<int32_t, DF> di;
74
697k
  auto floorx = Floor(x);
75
697k
  auto exp =
76
697k
      BitCast(df, ShiftLeft<23>(Add(ConvertTo(di, floorx), Set(di, 127))));
77
697k
  auto frac = Sub(x, floorx);
78
697k
  auto num = Add(frac, Set(df, 1.01749063e+01));
79
697k
  num = MulAdd(num, frac, Set(df, 4.88687798e+01));
80
697k
  num = MulAdd(num, frac, Set(df, 9.85506591e+01));
81
697k
  num = Mul(num, exp);
82
697k
  auto den = MulAdd(frac, Set(df, 2.10242958e-01), Set(df, -2.22328856e-02));
83
697k
  den = MulAdd(den, frac, Set(df, -1.94414990e+01));
84
697k
  den = MulAdd(den, frac, Set(df, 9.85506633e+01));
85
697k
  return Div(num, den);
86
697k
}
Unexecuted instantiation: hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::FastPow2f<hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul> >(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>)
Unexecuted instantiation: hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::FastPow2f<hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul> >(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>)
hwy::N_AVX2::Vec128<float, 4ul> jxl::N_AVX2::FastPow2f<hwy::N_AVX2::Simd<float, 4ul, 0>, hwy::N_AVX2::Vec128<float, 4ul> >(hwy::N_AVX2::Simd<float, 4ul, 0>, hwy::N_AVX2::Vec128<float, 4ul>)
Line
Count
Source
72
1.30M
V FastPow2f(const DF df, V x) {
73
1.30M
  const Rebind<int32_t, DF> di;
74
1.30M
  auto floorx = Floor(x);
75
1.30M
  auto exp =
76
1.30M
      BitCast(df, ShiftLeft<23>(Add(ConvertTo(di, floorx), Set(di, 127))));
77
1.30M
  auto frac = Sub(x, floorx);
78
1.30M
  auto num = Add(frac, Set(df, 1.01749063e+01));
79
1.30M
  num = MulAdd(num, frac, Set(df, 4.88687798e+01));
80
1.30M
  num = MulAdd(num, frac, Set(df, 9.85506591e+01));
81
1.30M
  num = Mul(num, exp);
82
1.30M
  auto den = MulAdd(frac, Set(df, 2.10242958e-01), Set(df, -2.22328856e-02));
83
1.30M
  den = MulAdd(den, frac, Set(df, -1.94414990e+01));
84
1.30M
  den = MulAdd(den, frac, Set(df, 9.85506633e+01));
85
1.30M
  return Div(num, den);
86
1.30M
}
Unexecuted instantiation: hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::FastPow2f<hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul> >(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>)
hwy::N_AVX2::Vec256<float> jxl::N_AVX2::FastPow2f<hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float> >(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>)
Line
Count
Source
72
478k
V FastPow2f(const DF df, V x) {
73
478k
  const Rebind<int32_t, DF> di;
74
478k
  auto floorx = Floor(x);
75
478k
  auto exp =
76
478k
      BitCast(df, ShiftLeft<23>(Add(ConvertTo(di, floorx), Set(di, 127))));
77
478k
  auto frac = Sub(x, floorx);
78
478k
  auto num = Add(frac, Set(df, 1.01749063e+01));
79
478k
  num = MulAdd(num, frac, Set(df, 4.88687798e+01));
80
478k
  num = MulAdd(num, frac, Set(df, 9.85506591e+01));
81
478k
  num = Mul(num, exp);
82
478k
  auto den = MulAdd(frac, Set(df, 2.10242958e-01), Set(df, -2.22328856e-02));
83
478k
  den = MulAdd(den, frac, Set(df, -1.94414990e+01));
84
478k
  den = MulAdd(den, frac, Set(df, 9.85506633e+01));
85
478k
  return Div(num, den);
86
478k
}
87
88
// max relative error ~3e-5
89
template <class DF, class V>
90
2.12M
V FastPowf(const DF df, V base, V exponent) {
91
2.12M
  return FastPow2f(df, Mul(FastLog2f(df, base), exponent));
92
2.12M
}
Unexecuted instantiation: hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::FastPowf<hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul> >(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>)
hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::FastPowf<hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul> >(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>)
Line
Count
Source
90
344k
V FastPowf(const DF df, V base, V exponent) {
91
344k
  return FastPow2f(df, Mul(FastLog2f(df, base), exponent));
92
344k
}
Unexecuted instantiation: hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::FastPowf<hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul> >(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>)
Unexecuted instantiation: hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::FastPowf<hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul> >(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>)
hwy::N_AVX2::Vec128<float, 4ul> jxl::N_AVX2::FastPowf<hwy::N_AVX2::Simd<float, 4ul, 0>, hwy::N_AVX2::Vec128<float, 4ul> >(hwy::N_AVX2::Simd<float, 4ul, 0>, hwy::N_AVX2::Vec128<float, 4ul>, hwy::N_AVX2::Vec128<float, 4ul>)
Line
Count
Source
90
1.30M
V FastPowf(const DF df, V base, V exponent) {
91
1.30M
  return FastPow2f(df, Mul(FastLog2f(df, base), exponent));
92
1.30M
}
Unexecuted instantiation: hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::FastPowf<hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul> >(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>)
hwy::N_AVX2::Vec256<float> jxl::N_AVX2::FastPowf<hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float> >(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>)
Line
Count
Source
90
478k
V FastPowf(const DF df, V base, V exponent) {
91
478k
  return FastPow2f(df, Mul(FastLog2f(df, base), exponent));
92
478k
}
93
94
// Computes cosine like std::cos.
95
// L1 error 7e-5.
96
template <class DF, class V>
97
494M
V FastCosf(const DF df, V x) {
98
  // Step 1: range reduction to [0, 2pi)
99
494M
  const auto pi2 = Set(df, kPi * 2.0f);
100
494M
  const auto pi2_inv = Set(df, 0.5f / kPi);
101
494M
  const auto npi2 = Mul(Floor(Mul(x, pi2_inv)), pi2);
102
494M
  const auto xmodpi2 = Sub(x, npi2);
103
  // Step 2: range reduction to [0, pi]
104
494M
  const auto x_pi = Min(xmodpi2, Sub(pi2, xmodpi2));
105
  // Step 3: range reduction to [0, pi/2]
106
494M
  const auto above_pihalf = Ge(x_pi, Set(df, kPi / 2.0f));
107
494M
  const auto x_pihalf = IfThenElse(above_pihalf, Sub(Set(df, kPi), x_pi), x_pi);
108
  // Step 4: Taylor-like approximation, scaled by 2**0.75 to make angle
109
  // duplication steps faster, on x/4.
110
494M
  const auto xs = Mul(x_pihalf, Set(df, 0.25f));
111
494M
  const auto x2 = Mul(xs, xs);
112
494M
  const auto x4 = Mul(x2, x2);
113
494M
  const auto cosx_prescaling =
114
494M
      MulAdd(x4, Set(df, 0.06960438),
115
494M
             MulAdd(x2, Set(df, -0.84087373), Set(df, 1.68179268)));
116
  // Step 5: angle duplication.
117
494M
  const auto cosx_scale1 =
118
494M
      MulAdd(cosx_prescaling, cosx_prescaling, Set(df, -1.414213562));
119
494M
  const auto cosx_scale2 = MulAdd(cosx_scale1, cosx_scale1, Set(df, -1));
120
  // Step 6: change sign if needed.
121
494M
  const Rebind<uint32_t, DF> du;
122
494M
  auto signbit = ShiftLeft<31>(BitCast(du, VecFromMask(df, above_pihalf)));
123
494M
  return BitCast(df, Xor(signbit, BitCast(du, cosx_scale2)));
124
494M
}
Unexecuted instantiation: hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::FastCosf<hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul> >(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>)
Unexecuted instantiation: hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::FastCosf<hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul> >(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>)
Unexecuted instantiation: hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::FastCosf<hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul> >(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>)
hwy::N_AVX2::Vec256<float> jxl::N_AVX2::FastCosf<hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float> >(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>)
Line
Count
Source
97
494M
V FastCosf(const DF df, V x) {
98
  // Step 1: range reduction to [0, 2pi)
99
494M
  const auto pi2 = Set(df, kPi * 2.0f);
100
494M
  const auto pi2_inv = Set(df, 0.5f / kPi);
101
494M
  const auto npi2 = Mul(Floor(Mul(x, pi2_inv)), pi2);
102
494M
  const auto xmodpi2 = Sub(x, npi2);
103
  // Step 2: range reduction to [0, pi]
104
494M
  const auto x_pi = Min(xmodpi2, Sub(pi2, xmodpi2));
105
  // Step 3: range reduction to [0, pi/2]
106
494M
  const auto above_pihalf = Ge(x_pi, Set(df, kPi / 2.0f));
107
494M
  const auto x_pihalf = IfThenElse(above_pihalf, Sub(Set(df, kPi), x_pi), x_pi);
108
  // Step 4: Taylor-like approximation, scaled by 2**0.75 to make angle
109
  // duplication steps faster, on x/4.
110
494M
  const auto xs = Mul(x_pihalf, Set(df, 0.25f));
111
494M
  const auto x2 = Mul(xs, xs);
112
494M
  const auto x4 = Mul(x2, x2);
113
494M
  const auto cosx_prescaling =
114
494M
      MulAdd(x4, Set(df, 0.06960438),
115
494M
             MulAdd(x2, Set(df, -0.84087373), Set(df, 1.68179268)));
116
  // Step 5: angle duplication.
117
494M
  const auto cosx_scale1 =
118
494M
      MulAdd(cosx_prescaling, cosx_prescaling, Set(df, -1.414213562));
119
494M
  const auto cosx_scale2 = MulAdd(cosx_scale1, cosx_scale1, Set(df, -1));
120
  // Step 6: change sign if needed.
121
494M
  const Rebind<uint32_t, DF> du;
122
494M
  auto signbit = ShiftLeft<31>(BitCast(du, VecFromMask(df, above_pihalf)));
123
494M
  return BitCast(df, Xor(signbit, BitCast(du, cosx_scale2)));
124
494M
}
Unexecuted instantiation: hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::FastCosf<hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul> >(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>)
Unexecuted instantiation: hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::FastCosf<hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul> >(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>)
125
126
// Computes the error function like std::erf.
127
// L1 error 7e-4.
128
template <class DF, class V>
129
2.06M
V FastErff(const DF df, V x) {
130
  // Formula from
131
  // https://en.wikipedia.org/wiki/Error_function#Numerical_approximations
132
  // but constants have been recomputed.
133
2.06M
  const auto xle0 = Le(x, Zero(df));
134
2.06M
  const auto absx = Abs(x);
135
  // Compute 1 - 1 / ((((x * a + b) * x + c) * x + d) * x + 1)**4
136
2.06M
  const auto denom1 =
137
2.06M
      MulAdd(absx, Set(df, 7.77394369e-02), Set(df, 2.05260015e-04));
138
2.06M
  const auto denom2 = MulAdd(denom1, absx, Set(df, 2.32120216e-01));
139
2.06M
  const auto denom3 = MulAdd(denom2, absx, Set(df, 2.77820801e-01));
140
2.06M
  const auto denom4 = MulAdd(denom3, absx, Set(df, 1.0f));
141
2.06M
  const auto denom5 = Mul(denom4, denom4);
142
2.06M
  const auto inv_denom5 = Div(Set(df, 1.0f), denom5);
143
2.06M
  const auto result = NegMulAdd(inv_denom5, inv_denom5, Set(df, 1.0f));
144
  // Change sign if needed.
145
2.06M
  const Rebind<uint32_t, DF> du;
146
2.06M
  auto signbit = ShiftLeft<31>(BitCast(du, VecFromMask(df, xle0)));
147
2.06M
  return BitCast(df, Xor(signbit, BitCast(du, result)));
148
2.06M
}
Unexecuted instantiation: hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::FastErff<hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul> >(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>)
hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::FastErff<hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul> >(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>)
Line
Count
Source
129
1.06M
V FastErff(const DF df, V x) {
130
  // Formula from
131
  // https://en.wikipedia.org/wiki/Error_function#Numerical_approximations
132
  // but constants have been recomputed.
133
1.06M
  const auto xle0 = Le(x, Zero(df));
134
1.06M
  const auto absx = Abs(x);
135
  // Compute 1 - 1 / ((((x * a + b) * x + c) * x + d) * x + 1)**4
136
1.06M
  const auto denom1 =
137
1.06M
      MulAdd(absx, Set(df, 7.77394369e-02), Set(df, 2.05260015e-04));
138
1.06M
  const auto denom2 = MulAdd(denom1, absx, Set(df, 2.32120216e-01));
139
1.06M
  const auto denom3 = MulAdd(denom2, absx, Set(df, 2.77820801e-01));
140
1.06M
  const auto denom4 = MulAdd(denom3, absx, Set(df, 1.0f));
141
1.06M
  const auto denom5 = Mul(denom4, denom4);
142
1.06M
  const auto inv_denom5 = Div(Set(df, 1.0f), denom5);
143
1.06M
  const auto result = NegMulAdd(inv_denom5, inv_denom5, Set(df, 1.0f));
144
  // Change sign if needed.
145
1.06M
  const Rebind<uint32_t, DF> du;
146
1.06M
  auto signbit = ShiftLeft<31>(BitCast(du, VecFromMask(df, xle0)));
147
1.06M
  return BitCast(df, Xor(signbit, BitCast(du, result)));
148
1.06M
}
Unexecuted instantiation: hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::FastErff<hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul> >(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>)
hwy::N_AVX2::Vec256<float> jxl::N_AVX2::FastErff<hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float> >(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>)
Line
Count
Source
129
995k
V FastErff(const DF df, V x) {
130
  // Formula from
131
  // https://en.wikipedia.org/wiki/Error_function#Numerical_approximations
132
  // but constants have been recomputed.
133
995k
  const auto xle0 = Le(x, Zero(df));
134
995k
  const auto absx = Abs(x);
135
  // Compute 1 - 1 / ((((x * a + b) * x + c) * x + d) * x + 1)**4
136
995k
  const auto denom1 =
137
995k
      MulAdd(absx, Set(df, 7.77394369e-02), Set(df, 2.05260015e-04));
138
995k
  const auto denom2 = MulAdd(denom1, absx, Set(df, 2.32120216e-01));
139
995k
  const auto denom3 = MulAdd(denom2, absx, Set(df, 2.77820801e-01));
140
995k
  const auto denom4 = MulAdd(denom3, absx, Set(df, 1.0f));
141
995k
  const auto denom5 = Mul(denom4, denom4);
142
995k
  const auto inv_denom5 = Div(Set(df, 1.0f), denom5);
143
995k
  const auto result = NegMulAdd(inv_denom5, inv_denom5, Set(df, 1.0f));
144
  // Change sign if needed.
145
995k
  const Rebind<uint32_t, DF> du;
146
995k
  auto signbit = ShiftLeft<31>(BitCast(du, VecFromMask(df, xle0)));
147
995k
  return BitCast(df, Xor(signbit, BitCast(du, result)));
148
995k
}
Unexecuted instantiation: hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::FastErff<hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul> >(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>)
Unexecuted instantiation: hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::FastErff<hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul> >(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>)
149
150
79.4k
inline float FastLog2f(float f) {
151
79.4k
  HWY_CAPPED(float, 1) D;
152
79.4k
  return GetLane(FastLog2f(D, Set(D, f)));
153
79.4k
}
Unexecuted instantiation: jxl::N_SSE4::FastLog2f(float)
Unexecuted instantiation: jxl::N_AVX2::FastLog2f(float)
jxl::N_SSE2::FastLog2f(float)
Line
Count
Source
150
79.4k
inline float FastLog2f(float f) {
151
79.4k
  HWY_CAPPED(float, 1) D;
152
79.4k
  return GetLane(FastLog2f(D, Set(D, f)));
153
79.4k
}
154
155
353k
inline float FastPow2f(float f) {
156
353k
  HWY_CAPPED(float, 1) D;
157
353k
  return GetLane(FastPow2f(D, Set(D, f)));
158
353k
}
Unexecuted instantiation: jxl::N_SSE4::FastPow2f(float)
jxl::N_AVX2::FastPow2f(float)
Line
Count
Source
155
353k
inline float FastPow2f(float f) {
156
353k
  HWY_CAPPED(float, 1) D;
157
353k
  return GetLane(FastPow2f(D, Set(D, f)));
158
353k
}
Unexecuted instantiation: jxl::N_SSE2::FastPow2f(float)
159
160
344k
inline float FastPowf(float b, float e) {
161
344k
  HWY_CAPPED(float, 1) D;
162
344k
  return GetLane(FastPowf(D, Set(D, b), Set(D, e)));
163
344k
}
Unexecuted instantiation: jxl::N_SSE4::FastPowf(float, float)
jxl::N_AVX2::FastPowf(float, float)
Line
Count
Source
160
344k
inline float FastPowf(float b, float e) {
161
344k
  HWY_CAPPED(float, 1) D;
162
344k
  return GetLane(FastPowf(D, Set(D, b), Set(D, e)));
163
344k
}
Unexecuted instantiation: jxl::N_SSE2::FastPowf(float, float)
164
165
0
inline float FastCosf(float f) {
166
0
  HWY_CAPPED(float, 1) D;
167
0
  return GetLane(FastCosf(D, Set(D, f)));
168
0
}
Unexecuted instantiation: jxl::N_SSE4::FastCosf(float)
Unexecuted instantiation: jxl::N_AVX2::FastCosf(float)
Unexecuted instantiation: jxl::N_SSE2::FastCosf(float)
169
170
0
inline float FastErff(float f) {
171
0
  HWY_CAPPED(float, 1) D;
172
0
  return GetLane(FastErff(D, Set(D, f)));
173
0
}
Unexecuted instantiation: jxl::N_SSE4::FastErff(float)
Unexecuted instantiation: jxl::N_AVX2::FastErff(float)
Unexecuted instantiation: jxl::N_SSE2::FastErff(float)
174
175
// Returns cbrt(x) + add with 6 ulp max error.
176
// Modified from vectormath_exp.h, Apache 2 license.
177
// https://www.agner.org/optimize/vectorclass.zip
178
template <class V>
179
8.43M
V CubeRootAndAdd(const V x, const V add) {
180
8.43M
  const HWY_FULL(float) df;
181
8.43M
  const HWY_FULL(int32_t) di;
182
183
8.43M
  const auto kExpBias = Set(di, 0x54800000);  // cast(1.) + cast(1.) / 3
184
8.43M
  const auto kExpMul = Set(di, 0x002AAAAA);   // shifted 1/3
185
8.43M
  const auto k1_3 = Set(df, 1.0f / 3);
186
8.43M
  const auto k4_3 = Set(df, 4.0f / 3);
187
188
8.43M
  const auto xa = x;  // assume inputs never negative
189
8.43M
  const auto xa_3 = Mul(k1_3, xa);
190
191
  // Multiply exponent by -1/3
192
8.43M
  const auto m1 = BitCast(di, xa);
193
  // Special case for 0. 0 is represented with an exponent of 0, so the
194
  // "kExpBias - 1/3 * exp" below gives the wrong result. The IfThenZeroElse()
195
  // sets those values as 0, which prevents having NaNs in the computations
196
  // below.
197
  // TODO(eustas): use fused op
198
8.43M
  const auto m2 = IfThenZeroElse(
199
8.43M
      Eq(m1, Zero(di)), Sub(kExpBias, Mul((ShiftRight<23>(m1)), kExpMul)));
200
8.43M
  auto r = BitCast(df, m2);
201
202
  // Newton-Raphson iterations
203
33.7M
  for (int i = 0; i < 3; i++) {
204
25.2M
    const auto r2 = Mul(r, r);
205
25.2M
    r = NegMulAdd(xa_3, Mul(r2, r2), Mul(k4_3, r));
206
25.2M
  }
207
  // Final iteration
208
8.43M
  auto r2 = Mul(r, r);
209
8.43M
  r = MulAdd(k1_3, NegMulAdd(xa, Mul(r2, r2), r), r);
210
8.43M
  r2 = Mul(r, r);
211
8.43M
  r = MulAdd(r2, x, add);
212
213
8.43M
  return r;
214
8.43M
}
Unexecuted instantiation: hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::CubeRootAndAdd<hwy::N_SSE4::Vec128<float, 4ul> >(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>)
hwy::N_AVX2::Vec256<float> jxl::N_AVX2::CubeRootAndAdd<hwy::N_AVX2::Vec256<float> >(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>)
Line
Count
Source
179
8.43M
V CubeRootAndAdd(const V x, const V add) {
180
8.43M
  const HWY_FULL(float) df;
181
8.43M
  const HWY_FULL(int32_t) di;
182
183
8.43M
  const auto kExpBias = Set(di, 0x54800000);  // cast(1.) + cast(1.) / 3
184
8.43M
  const auto kExpMul = Set(di, 0x002AAAAA);   // shifted 1/3
185
8.43M
  const auto k1_3 = Set(df, 1.0f / 3);
186
8.43M
  const auto k4_3 = Set(df, 4.0f / 3);
187
188
8.43M
  const auto xa = x;  // assume inputs never negative
189
8.43M
  const auto xa_3 = Mul(k1_3, xa);
190
191
  // Multiply exponent by -1/3
192
8.43M
  const auto m1 = BitCast(di, xa);
193
  // Special case for 0. 0 is represented with an exponent of 0, so the
194
  // "kExpBias - 1/3 * exp" below gives the wrong result. The IfThenZeroElse()
195
  // sets those values as 0, which prevents having NaNs in the computations
196
  // below.
197
  // TODO(eustas): use fused op
198
8.43M
  const auto m2 = IfThenZeroElse(
199
8.43M
      Eq(m1, Zero(di)), Sub(kExpBias, Mul((ShiftRight<23>(m1)), kExpMul)));
200
8.43M
  auto r = BitCast(df, m2);
201
202
  // Newton-Raphson iterations
203
33.7M
  for (int i = 0; i < 3; i++) {
204
25.2M
    const auto r2 = Mul(r, r);
205
25.2M
    r = NegMulAdd(xa_3, Mul(r2, r2), Mul(k4_3, r));
206
25.2M
  }
207
  // Final iteration
208
8.43M
  auto r2 = Mul(r, r);
209
8.43M
  r = MulAdd(k1_3, NegMulAdd(xa, Mul(r2, r2), r), r);
210
8.43M
  r2 = Mul(r, r);
211
8.43M
  r = MulAdd(r2, x, add);
212
213
8.43M
  return r;
214
8.43M
}
Unexecuted instantiation: hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::CubeRootAndAdd<hwy::N_SSE2::Vec128<float, 4ul> >(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>)
215
216
// NOLINTNEXTLINE(google-readability-namespace-comments)
217
}  // namespace HWY_NAMESPACE
218
}  // namespace jxl
219
HWY_AFTER_NAMESPACE();
220
221
#endif  // LIB_JXL_BASE_FAST_MATH_INL_H_
222
223
#if HWY_ONCE
224
#ifndef LIB_JXL_BASE_FAST_MATH_ONCE
225
#define LIB_JXL_BASE_FAST_MATH_ONCE
226
227
namespace jxl {
228
79.4k
inline float FastLog2f(float f) { return HWY_STATIC_DISPATCH(FastLog2f)(f); }
229
0
inline float FastPow2f(float f) { return HWY_STATIC_DISPATCH(FastPow2f)(f); }
230
0
inline float FastPowf(float b, float e) {
231
0
  return HWY_STATIC_DISPATCH(FastPowf)(b, e);
232
0
}
233
0
inline float FastCosf(float f) { return HWY_STATIC_DISPATCH(FastCosf)(f); }
234
0
inline float FastErff(float f) { return HWY_STATIC_DISPATCH(FastErff)(f); }
235
}  // namespace jxl
236
237
#endif  // LIB_JXL_BASE_FAST_MATH_ONCE
238
#endif  // HWY_ONCE