Coverage Report

Created: 2026-05-16 07:22

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/libjxl/lib/jxl/base/rational_polynomial-inl.h
Line
Count
Source
1
// Copyright (c) the JPEG XL Project Authors. All rights reserved.
2
//
3
// Use of this source code is governed by a BSD-style
4
// license that can be found in the LICENSE file.
5
6
// Fast SIMD evaluation of rational polynomials for approximating functions.
7
8
#if defined(LIB_JXL_BASE_RATIONAL_POLYNOMIAL_INL_H_) == \
9
    defined(HWY_TARGET_TOGGLE)
10
#ifdef LIB_JXL_BASE_RATIONAL_POLYNOMIAL_INL_H_
11
#undef LIB_JXL_BASE_RATIONAL_POLYNOMIAL_INL_H_
12
#else
13
#define LIB_JXL_BASE_RATIONAL_POLYNOMIAL_INL_H_
14
#endif
15
16
#include <jxl/types.h>
17
#include <stddef.h>
18
19
#include <hwy/highway.h>
20
HWY_BEFORE_NAMESPACE();
21
namespace jxl {
22
namespace HWY_NAMESPACE {
23
namespace {
24
25
// These templates are not found via ADL.
26
using hwy::HWY_NAMESPACE::ApproximateReciprocal;
27
using hwy::HWY_NAMESPACE::Div;
28
using hwy::HWY_NAMESPACE::MulAdd;
29
30
// Primary template: default to actual division.
31
template <typename T, class V>
32
struct FastDivision {
33
  HWY_INLINE V operator()(const V n, const V d) const { return n / d; }
34
};
35
// Partial specialization for float vectors.
36
template <class V>
37
struct FastDivision<float, V> {
38
  // One Newton-Raphson iteration.
39
  static HWY_INLINE V ReciprocalNR(const V x) {
40
    const auto rcp = ApproximateReciprocal(x);
41
    const auto sum = Add(rcp, rcp);
42
    const auto x_rcp = Mul(x, rcp);
43
    return NegMulAdd(x_rcp, rcp, sum);
44
  }
45
46
1.14G
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
1.14G
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
1.14G
  }
Unexecuted instantiation: enc_cluster.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
enc_cluster.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Line
Count
Source
46
657M
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
657M
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
657M
  }
Unexecuted instantiation: enc_cluster.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: enc_cluster.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_cluster.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_cluster.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
enc_lz77.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Line
Count
Source
46
1.07M
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
1.07M
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
1.07M
  }
Unexecuted instantiation: enc_xyb.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
enc_xyb.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Line
Count
Source
46
104M
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
104M
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
104M
  }
Unexecuted instantiation: enc_xyb.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: enc_xyb.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_xyb.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_xyb.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: butteraugli.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
Unexecuted instantiation: butteraugli.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Unexecuted instantiation: butteraugli.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: butteraugli.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: butteraugli.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: butteraugli.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
enc_adaptive_quantization.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Line
Count
Source
46
4.40M
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
4.40M
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
4.40M
  }
Unexecuted instantiation: enc_adaptive_quantization.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
Unexecuted instantiation: enc_adaptive_quantization.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: enc_adaptive_quantization.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_adaptive_quantization.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_adaptive_quantization.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_ac_strategy.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
enc_ac_strategy.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Line
Count
Source
46
4.01M
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
4.01M
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
4.01M
  }
Unexecuted instantiation: enc_ac_strategy.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: jxl_cms.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
Unexecuted instantiation: jxl_cms.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Unexecuted instantiation: jxl_cms.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: jxl_cms.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: jxl_cms.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: jxl_cms.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_ma.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
enc_ma.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Line
Count
Source
46
176M
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
176M
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
176M
  }
Unexecuted instantiation: enc_ma.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: enc_ma.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_ma.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_ma.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: quant_weights.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: quant_weights.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: quant_weights.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: quant_weights.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
quant_weights.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 4ul> >::operator()(hwy::N_AVX2::Vec128<float, 4ul>, hwy::N_AVX2::Vec128<float, 4ul>) const
Line
Count
Source
46
17.6M
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
17.6M
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
17.6M
  }
Unexecuted instantiation: quant_weights.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
stage_from_linear.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Line
Count
Source
46
183M
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
183M
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
183M
  }
Unexecuted instantiation: stage_from_linear.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
Unexecuted instantiation: stage_from_linear.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: stage_from_linear.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: stage_from_linear.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: stage_from_linear.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: stage_to_linear.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Unexecuted instantiation: stage_to_linear.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
Unexecuted instantiation: stage_to_linear.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: stage_to_linear.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: stage_to_linear.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: stage_to_linear.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: stage_tone_mapping.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
Unexecuted instantiation: stage_tone_mapping.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Unexecuted instantiation: stage_tone_mapping.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: stage_tone_mapping.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: stage_tone_mapping.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: stage_tone_mapping.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: splines.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: splines.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: splines.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
53
};
54
55
// Approximates smooth functions via rational polynomials (i.e. dividing two
56
// polynomials). Evaluates polynomials via Horner's scheme, which is faster than
57
// Clenshaw recurrence for Chebyshev polynomials. LoadDup128 allows us to
58
// specify constants (replicated 4x) independently of the lane count.
59
template <size_t NP, size_t NQ, class D, class V, typename T>
60
HWY_INLINE HWY_MAYBE_UNUSED V EvalRationalPolynomial(const D d, const V x,
61
                                                     const T (&p)[NP],
62
1.14G
                                                     const T (&q)[NQ]) {
63
1.14G
  constexpr size_t kDegP = NP / 4 - 1;
64
1.14G
  constexpr size_t kDegQ = NQ / 4 - 1;
65
1.14G
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
1.14G
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
1.14G
  HWY_FENCE;
72
1.14G
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
1.14G
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
1.14G
  HWY_FENCE;
75
1.14G
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
1.14G
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
1.14G
  HWY_FENCE;
78
1.14G
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
1.14G
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
1.14G
  HWY_FENCE;
81
1.14G
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
1.14G
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
1.14G
  HWY_FENCE;
84
1.14G
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
1.14G
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
1.14G
  HWY_FENCE;
87
1.14G
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
1.14G
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
1.14G
  HWY_FENCE;
90
1.14G
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
1.14G
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
1.14G
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
1.14G
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
1.14G
  return FastDivision<T, V>()(yp, yq);
97
1.14G
}
Unexecuted instantiation: enc_cluster.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
enc_cluster.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [12ul], float const (&) [12ul])
Line
Count
Source
62
657M
                                                     const T (&q)[NQ]) {
63
657M
  constexpr size_t kDegP = NP / 4 - 1;
64
657M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
657M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
657M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
657M
  HWY_FENCE;
72
657M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
657M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
657M
  HWY_FENCE;
75
657M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
657M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
657M
  HWY_FENCE;
78
657M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
657M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
657M
  HWY_FENCE;
81
657M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
657M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
657M
  HWY_FENCE;
84
657M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
657M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
657M
  HWY_FENCE;
87
657M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
657M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
657M
  HWY_FENCE;
90
657M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
657M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
657M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
657M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
657M
  return FastDivision<T, V>()(yp, yq);
97
657M
}
Unexecuted instantiation: enc_cluster.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_cluster.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_cluster.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_cluster.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
enc_lz77.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Line
Count
Source
62
1.07M
                                                     const T (&q)[NQ]) {
63
1.07M
  constexpr size_t kDegP = NP / 4 - 1;
64
1.07M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
1.07M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
1.07M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
1.07M
  HWY_FENCE;
72
1.07M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
1.07M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
1.07M
  HWY_FENCE;
75
1.07M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
1.07M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
1.07M
  HWY_FENCE;
78
1.07M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
1.07M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
1.07M
  HWY_FENCE;
81
1.07M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
1.07M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
1.07M
  HWY_FENCE;
84
1.07M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
1.07M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
1.07M
  HWY_FENCE;
87
1.07M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
1.07M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
1.07M
  HWY_FENCE;
90
1.07M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
1.07M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
1.07M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
1.07M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
1.07M
  return FastDivision<T, V>()(yp, yq);
97
1.07M
}
Unexecuted instantiation: enc_xyb.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
enc_xyb.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [20ul], float const (&) [20ul])
Line
Count
Source
62
104M
                                                     const T (&q)[NQ]) {
63
104M
  constexpr size_t kDegP = NP / 4 - 1;
64
104M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
104M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
104M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
104M
  HWY_FENCE;
72
104M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
104M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
104M
  HWY_FENCE;
75
104M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
104M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
104M
  HWY_FENCE;
78
104M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
104M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
104M
  HWY_FENCE;
81
104M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
104M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
104M
  HWY_FENCE;
84
104M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
104M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
104M
  HWY_FENCE;
87
104M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
104M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
104M
  HWY_FENCE;
90
104M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
104M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
104M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
104M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
104M
  return FastDivision<T, V>()(yp, yq);
97
104M
}
Unexecuted instantiation: enc_xyb.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: enc_xyb.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_xyb.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_xyb.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: butteraugli.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: butteraugli.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: butteraugli.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: butteraugli.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: butteraugli.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: butteraugli.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
enc_adaptive_quantization.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [12ul], float const (&) [12ul])
Line
Count
Source
62
4.40M
                                                     const T (&q)[NQ]) {
63
4.40M
  constexpr size_t kDegP = NP / 4 - 1;
64
4.40M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
4.40M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
4.40M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
4.40M
  HWY_FENCE;
72
4.40M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
4.40M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
4.40M
  HWY_FENCE;
75
4.40M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
4.40M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
4.40M
  HWY_FENCE;
78
4.40M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
4.40M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
4.40M
  HWY_FENCE;
81
4.40M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
4.40M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
4.40M
  HWY_FENCE;
84
4.40M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
4.40M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
4.40M
  HWY_FENCE;
87
4.40M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
4.40M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
4.40M
  HWY_FENCE;
90
4.40M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
4.40M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
4.40M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
4.40M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
4.40M
  return FastDivision<T, V>()(yp, yq);
97
4.40M
}
Unexecuted instantiation: enc_adaptive_quantization.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_adaptive_quantization.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_adaptive_quantization.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_adaptive_quantization.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_adaptive_quantization.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_ac_strategy.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
enc_ac_strategy.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Line
Count
Source
62
4.01M
                                                     const T (&q)[NQ]) {
63
4.01M
  constexpr size_t kDegP = NP / 4 - 1;
64
4.01M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
4.01M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
4.01M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
4.01M
  HWY_FENCE;
72
4.01M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
4.01M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
4.01M
  HWY_FENCE;
75
4.01M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
4.01M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
4.01M
  HWY_FENCE;
78
4.01M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
4.01M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
4.01M
  HWY_FENCE;
81
4.01M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
4.01M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
4.01M
  HWY_FENCE;
84
4.01M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
4.01M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
4.01M
  HWY_FENCE;
87
4.01M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
4.01M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
4.01M
  HWY_FENCE;
90
4.01M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
4.01M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
4.01M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
4.01M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
4.01M
  return FastDivision<T, V>()(yp, yq);
97
4.01M
}
Unexecuted instantiation: enc_ac_strategy.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: jxl_cms.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: jxl_cms.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: jxl_cms.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: jxl_cms.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: jxl_cms.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: jxl_cms.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_ma.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
enc_ma.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [12ul], float const (&) [12ul])
Line
Count
Source
62
176M
                                                     const T (&q)[NQ]) {
63
176M
  constexpr size_t kDegP = NP / 4 - 1;
64
176M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
176M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
176M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
176M
  HWY_FENCE;
72
176M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
176M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
176M
  HWY_FENCE;
75
176M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
176M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
176M
  HWY_FENCE;
78
176M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
176M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
176M
  HWY_FENCE;
81
176M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
176M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
176M
  HWY_FENCE;
84
176M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
176M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
176M
  HWY_FENCE;
87
176M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
176M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
176M
  HWY_FENCE;
90
176M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
176M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
176M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
176M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
176M
  return FastDivision<T, V>()(yp, yq);
97
176M
}
Unexecuted instantiation: enc_ma.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_ma.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_ma.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_ma.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: quant_weights.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: quant_weights.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: quant_weights.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: quant_weights.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
quant_weights.cc:hwy::N_AVX2::Vec128<float, 4ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 4ul, 0>, hwy::N_AVX2::Vec128<float, 4ul>, float>(hwy::N_AVX2::Simd<float, 4ul, 0>, hwy::N_AVX2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Line
Count
Source
62
17.6M
                                                     const T (&q)[NQ]) {
63
17.6M
  constexpr size_t kDegP = NP / 4 - 1;
64
17.6M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
17.6M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
17.6M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
17.6M
  HWY_FENCE;
72
17.6M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
17.6M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
17.6M
  HWY_FENCE;
75
17.6M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
17.6M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
17.6M
  HWY_FENCE;
78
17.6M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
17.6M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
17.6M
  HWY_FENCE;
81
17.6M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
17.6M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
17.6M
  HWY_FENCE;
84
17.6M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
17.6M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
17.6M
  HWY_FENCE;
87
17.6M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
17.6M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
17.6M
  HWY_FENCE;
90
17.6M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
17.6M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
17.6M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
17.6M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
17.6M
  return FastDivision<T, V>()(yp, yq);
97
17.6M
}
Unexecuted instantiation: quant_weights.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
stage_from_linear.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [20ul], float const (&) [20ul])
Line
Count
Source
62
178M
                                                     const T (&q)[NQ]) {
63
178M
  constexpr size_t kDegP = NP / 4 - 1;
64
178M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
178M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
178M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
178M
  HWY_FENCE;
72
178M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
178M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
178M
  HWY_FENCE;
75
178M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
178M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
178M
  HWY_FENCE;
78
178M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
178M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
178M
  HWY_FENCE;
81
178M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
178M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
178M
  HWY_FENCE;
84
178M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
178M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
178M
  HWY_FENCE;
87
178M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
178M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
178M
  HWY_FENCE;
90
178M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
178M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
178M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
178M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
178M
  return FastDivision<T, V>()(yp, yq);
97
178M
}
stage_from_linear.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [12ul], float const (&) [12ul])
Line
Count
Source
62
4.69M
                                                     const T (&q)[NQ]) {
63
4.69M
  constexpr size_t kDegP = NP / 4 - 1;
64
4.69M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
4.69M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
4.69M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
4.69M
  HWY_FENCE;
72
4.69M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
4.69M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
4.69M
  HWY_FENCE;
75
4.69M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
4.69M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
4.69M
  HWY_FENCE;
78
4.69M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
4.69M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
4.69M
  HWY_FENCE;
81
4.69M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
4.69M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
4.69M
  HWY_FENCE;
84
4.69M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
4.69M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
4.69M
  HWY_FENCE;
87
4.69M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
4.69M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
4.69M
  HWY_FENCE;
90
4.69M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
4.69M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
4.69M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
4.69M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
4.69M
  return FastDivision<T, V>()(yp, yq);
97
4.69M
}
Unexecuted instantiation: stage_from_linear.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: stage_from_linear.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_from_linear.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: stage_from_linear.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_from_linear.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_from_linear.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_from_linear.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: splines.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: splines.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: splines.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
98
99
}  // namespace
100
// NOLINTNEXTLINE(google-readability-namespace-comments)
101
}  // namespace HWY_NAMESPACE
102
}  // namespace jxl
103
HWY_AFTER_NAMESPACE();
104
#endif  // LIB_JXL_BASE_RATIONAL_POLYNOMIAL_INL_H_