Coverage Report

Created: 2026-09-28 06:47

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/libjxl/lib/jxl/base/rational_polynomial-inl.h
Line
Count
Source
1
// Copyright (c) the JPEG XL Project Authors. All rights reserved.
2
//
3
// Use of this source code is governed by a BSD-style
4
// license that can be found in the LICENSE file.
5
6
// Fast SIMD evaluation of rational polynomials for approximating functions.
7
8
#if defined(LIB_JXL_BASE_RATIONAL_POLYNOMIAL_INL_H_) == \
9
    defined(HWY_TARGET_TOGGLE)
10
#ifdef LIB_JXL_BASE_RATIONAL_POLYNOMIAL_INL_H_
11
#undef LIB_JXL_BASE_RATIONAL_POLYNOMIAL_INL_H_
12
#else
13
#define LIB_JXL_BASE_RATIONAL_POLYNOMIAL_INL_H_
14
#endif
15
16
#include <jxl/types.h>
17
#include <stddef.h>
18
19
#include <hwy/highway.h>
20
HWY_BEFORE_NAMESPACE();
21
namespace jxl {
22
namespace HWY_NAMESPACE {
23
namespace {
24
25
// These templates are not found via ADL.
26
using hwy::HWY_NAMESPACE::ApproximateReciprocal;
27
using hwy::HWY_NAMESPACE::Div;
28
using hwy::HWY_NAMESPACE::MulAdd;
29
30
// Primary template: default to actual division.
31
template <typename T, class V>
32
struct FastDivision {
33
  HWY_INLINE V operator()(const V n, const V d) const { return n / d; }
34
};
35
// Partial specialization for float vectors.
36
template <class V>
37
struct FastDivision<float, V> {
38
  // One Newton-Raphson iteration.
39
  static HWY_INLINE V ReciprocalNR(const V x) {
40
    const auto rcp = ApproximateReciprocal(x);
41
    const auto sum = Add(rcp, rcp);
42
    const auto x_rcp = Mul(x, rcp);
43
    return NegMulAdd(x_rcp, rcp, sum);
44
  }
45
46
1.33G
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
1.33G
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
1.33G
  }
Unexecuted instantiation: enc_cluster.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
enc_cluster.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Line
Count
Source
46
773M
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
773M
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
773M
  }
Unexecuted instantiation: enc_cluster.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: enc_cluster.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_cluster.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_cluster.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
enc_lz77.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Line
Count
Source
46
1.27M
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
1.27M
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
1.27M
  }
Unexecuted instantiation: enc_xyb.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
enc_xyb.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Line
Count
Source
46
129M
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
129M
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
129M
  }
Unexecuted instantiation: enc_xyb.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: enc_xyb.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_xyb.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_xyb.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: butteraugli.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
Unexecuted instantiation: butteraugli.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Unexecuted instantiation: butteraugli.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: butteraugli.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: butteraugli.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: butteraugli.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
enc_adaptive_quantization.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Line
Count
Source
46
5.43M
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
5.43M
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
5.43M
  }
Unexecuted instantiation: enc_adaptive_quantization.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
Unexecuted instantiation: enc_adaptive_quantization.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: enc_adaptive_quantization.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_adaptive_quantization.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_adaptive_quantization.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_ac_strategy.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
enc_ac_strategy.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Line
Count
Source
46
4.96M
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
4.96M
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
4.96M
  }
Unexecuted instantiation: enc_ac_strategy.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: jxl_cms.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
Unexecuted instantiation: jxl_cms.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Unexecuted instantiation: jxl_cms.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: jxl_cms.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: jxl_cms.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: jxl_cms.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_ma.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
enc_ma.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Line
Count
Source
46
217M
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
217M
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
217M
  }
Unexecuted instantiation: enc_ma.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: enc_ma.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_ma.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_ma.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: quant_weights.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: quant_weights.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: quant_weights.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: quant_weights.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
quant_weights.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 4ul> >::operator()(hwy::N_AVX2::Vec128<float, 4ul>, hwy::N_AVX2::Vec128<float, 4ul>) const
Line
Count
Source
46
18.9M
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
18.9M
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
18.9M
  }
Unexecuted instantiation: quant_weights.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
stage_from_linear.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Line
Count
Source
46
181M
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
181M
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
181M
  }
Unexecuted instantiation: stage_from_linear.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
Unexecuted instantiation: stage_from_linear.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: stage_from_linear.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: stage_from_linear.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: stage_from_linear.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: stage_to_linear.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Unexecuted instantiation: stage_to_linear.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
Unexecuted instantiation: stage_to_linear.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: stage_to_linear.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: stage_to_linear.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: stage_to_linear.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: stage_tone_mapping.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
Unexecuted instantiation: stage_tone_mapping.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Unexecuted instantiation: stage_tone_mapping.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: stage_tone_mapping.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: stage_tone_mapping.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: stage_tone_mapping.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: splines.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: splines.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: splines.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
53
};
54
55
// Approximates smooth functions via rational polynomials (i.e. dividing two
56
// polynomials). Evaluates polynomials via Horner's scheme, which is faster than
57
// Clenshaw recurrence for Chebyshev polynomials. LoadDup128 allows us to
58
// specify constants (replicated 4x) independently of the lane count.
59
template <size_t NP, size_t NQ, class D, class V, typename T>
60
HWY_INLINE HWY_MAYBE_UNUSED V EvalRationalPolynomial(const D d, const V x,
61
                                                     const T (&p)[NP],
62
1.33G
                                                     const T (&q)[NQ]) {
63
1.33G
  constexpr size_t kDegP = NP / 4 - 1;
64
1.33G
  constexpr size_t kDegQ = NQ / 4 - 1;
65
1.33G
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
1.33G
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
1.33G
  HWY_FENCE;
72
1.33G
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
1.33G
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
1.33G
  HWY_FENCE;
75
1.33G
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
1.33G
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
1.33G
  HWY_FENCE;
78
1.33G
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
1.33G
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
1.33G
  HWY_FENCE;
81
1.33G
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
1.33G
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
1.33G
  HWY_FENCE;
84
1.33G
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
1.33G
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
1.33G
  HWY_FENCE;
87
1.33G
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
1.33G
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
1.33G
  HWY_FENCE;
90
1.33G
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
1.33G
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
1.33G
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
1.33G
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
1.33G
  return FastDivision<T, V>()(yp, yq);
97
1.33G
}
Unexecuted instantiation: enc_cluster.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
enc_cluster.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [12ul], float const (&) [12ul])
Line
Count
Source
62
773M
                                                     const T (&q)[NQ]) {
63
773M
  constexpr size_t kDegP = NP / 4 - 1;
64
773M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
773M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
773M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
773M
  HWY_FENCE;
72
773M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
773M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
773M
  HWY_FENCE;
75
773M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
773M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
773M
  HWY_FENCE;
78
773M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
773M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
773M
  HWY_FENCE;
81
773M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
773M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
773M
  HWY_FENCE;
84
773M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
773M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
773M
  HWY_FENCE;
87
773M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
773M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
773M
  HWY_FENCE;
90
773M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
773M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
773M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
773M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
773M
  return FastDivision<T, V>()(yp, yq);
97
773M
}
Unexecuted instantiation: enc_cluster.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_cluster.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_cluster.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_cluster.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
enc_lz77.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Line
Count
Source
62
1.27M
                                                     const T (&q)[NQ]) {
63
1.27M
  constexpr size_t kDegP = NP / 4 - 1;
64
1.27M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
1.27M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
1.27M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
1.27M
  HWY_FENCE;
72
1.27M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
1.27M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
1.27M
  HWY_FENCE;
75
1.27M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
1.27M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
1.27M
  HWY_FENCE;
78
1.27M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
1.27M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
1.27M
  HWY_FENCE;
81
1.27M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
1.27M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
1.27M
  HWY_FENCE;
84
1.27M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
1.27M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
1.27M
  HWY_FENCE;
87
1.27M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
1.27M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
1.27M
  HWY_FENCE;
90
1.27M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
1.27M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
1.27M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
1.27M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
1.27M
  return FastDivision<T, V>()(yp, yq);
97
1.27M
}
Unexecuted instantiation: enc_xyb.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
enc_xyb.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [20ul], float const (&) [20ul])
Line
Count
Source
62
129M
                                                     const T (&q)[NQ]) {
63
129M
  constexpr size_t kDegP = NP / 4 - 1;
64
129M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
129M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
129M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
129M
  HWY_FENCE;
72
129M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
129M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
129M
  HWY_FENCE;
75
129M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
129M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
129M
  HWY_FENCE;
78
129M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
129M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
129M
  HWY_FENCE;
81
129M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
129M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
129M
  HWY_FENCE;
84
129M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
129M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
129M
  HWY_FENCE;
87
129M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
129M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
129M
  HWY_FENCE;
90
129M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
129M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
129M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
129M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
129M
  return FastDivision<T, V>()(yp, yq);
97
129M
}
Unexecuted instantiation: enc_xyb.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: enc_xyb.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_xyb.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_xyb.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: butteraugli.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: butteraugli.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: butteraugli.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: butteraugli.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: butteraugli.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: butteraugli.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
enc_adaptive_quantization.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [12ul], float const (&) [12ul])
Line
Count
Source
62
5.43M
                                                     const T (&q)[NQ]) {
63
5.43M
  constexpr size_t kDegP = NP / 4 - 1;
64
5.43M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
5.43M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
5.43M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
5.43M
  HWY_FENCE;
72
5.43M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
5.43M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
5.43M
  HWY_FENCE;
75
5.43M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
5.43M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
5.43M
  HWY_FENCE;
78
5.43M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
5.43M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
5.43M
  HWY_FENCE;
81
5.43M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
5.43M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
5.43M
  HWY_FENCE;
84
5.43M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
5.43M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
5.43M
  HWY_FENCE;
87
5.43M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
5.43M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
5.43M
  HWY_FENCE;
90
5.43M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
5.43M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
5.43M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
5.43M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
5.43M
  return FastDivision<T, V>()(yp, yq);
97
5.43M
}
Unexecuted instantiation: enc_adaptive_quantization.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_adaptive_quantization.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_adaptive_quantization.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_adaptive_quantization.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_adaptive_quantization.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_ac_strategy.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
enc_ac_strategy.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Line
Count
Source
62
4.96M
                                                     const T (&q)[NQ]) {
63
4.96M
  constexpr size_t kDegP = NP / 4 - 1;
64
4.96M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
4.96M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
4.96M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
4.96M
  HWY_FENCE;
72
4.96M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
4.96M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
4.96M
  HWY_FENCE;
75
4.96M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
4.96M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
4.96M
  HWY_FENCE;
78
4.96M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
4.96M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
4.96M
  HWY_FENCE;
81
4.96M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
4.96M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
4.96M
  HWY_FENCE;
84
4.96M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
4.96M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
4.96M
  HWY_FENCE;
87
4.96M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
4.96M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
4.96M
  HWY_FENCE;
90
4.96M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
4.96M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
4.96M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
4.96M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
4.96M
  return FastDivision<T, V>()(yp, yq);
97
4.96M
}
Unexecuted instantiation: enc_ac_strategy.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: jxl_cms.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: jxl_cms.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: jxl_cms.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: jxl_cms.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: jxl_cms.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: jxl_cms.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_ma.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
enc_ma.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [12ul], float const (&) [12ul])
Line
Count
Source
62
217M
                                                     const T (&q)[NQ]) {
63
217M
  constexpr size_t kDegP = NP / 4 - 1;
64
217M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
217M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
217M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
217M
  HWY_FENCE;
72
217M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
217M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
217M
  HWY_FENCE;
75
217M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
217M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
217M
  HWY_FENCE;
78
217M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
217M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
217M
  HWY_FENCE;
81
217M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
217M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
217M
  HWY_FENCE;
84
217M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
217M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
217M
  HWY_FENCE;
87
217M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
217M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
217M
  HWY_FENCE;
90
217M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
217M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
217M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
217M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
217M
  return FastDivision<T, V>()(yp, yq);
97
217M
}
Unexecuted instantiation: enc_ma.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_ma.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_ma.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_ma.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: quant_weights.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: quant_weights.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: quant_weights.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: quant_weights.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
quant_weights.cc:hwy::N_AVX2::Vec128<float, 4ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 4ul, 0>, hwy::N_AVX2::Vec128<float, 4ul>, float>(hwy::N_AVX2::Simd<float, 4ul, 0>, hwy::N_AVX2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Line
Count
Source
62
18.9M
                                                     const T (&q)[NQ]) {
63
18.9M
  constexpr size_t kDegP = NP / 4 - 1;
64
18.9M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
18.9M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
18.9M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
18.9M
  HWY_FENCE;
72
18.9M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
18.9M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
18.9M
  HWY_FENCE;
75
18.9M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
18.9M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
18.9M
  HWY_FENCE;
78
18.9M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
18.9M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
18.9M
  HWY_FENCE;
81
18.9M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
18.9M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
18.9M
  HWY_FENCE;
84
18.9M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
18.9M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
18.9M
  HWY_FENCE;
87
18.9M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
18.9M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
18.9M
  HWY_FENCE;
90
18.9M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
18.9M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
18.9M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
18.9M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
18.9M
  return FastDivision<T, V>()(yp, yq);
97
18.9M
}
Unexecuted instantiation: quant_weights.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
stage_from_linear.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [20ul], float const (&) [20ul])
Line
Count
Source
62
176M
                                                     const T (&q)[NQ]) {
63
176M
  constexpr size_t kDegP = NP / 4 - 1;
64
176M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
176M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
176M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
176M
  HWY_FENCE;
72
176M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
176M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
176M
  HWY_FENCE;
75
176M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
176M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
176M
  HWY_FENCE;
78
176M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
176M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
176M
  HWY_FENCE;
81
176M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
176M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
176M
  HWY_FENCE;
84
176M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
176M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
176M
  HWY_FENCE;
87
176M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
176M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
176M
  HWY_FENCE;
90
176M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
176M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
176M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
176M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
176M
  return FastDivision<T, V>()(yp, yq);
97
176M
}
stage_from_linear.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [12ul], float const (&) [12ul])
Line
Count
Source
62
5.18M
                                                     const T (&q)[NQ]) {
63
5.18M
  constexpr size_t kDegP = NP / 4 - 1;
64
5.18M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
5.18M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
5.18M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
5.18M
  HWY_FENCE;
72
5.18M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
5.18M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
5.18M
  HWY_FENCE;
75
5.18M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
5.18M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
5.18M
  HWY_FENCE;
78
5.18M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
5.18M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
5.18M
  HWY_FENCE;
81
5.18M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
5.18M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
5.18M
  HWY_FENCE;
84
5.18M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
5.18M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
5.18M
  HWY_FENCE;
87
5.18M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
5.18M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
5.18M
  HWY_FENCE;
90
5.18M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
5.18M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
5.18M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
5.18M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
5.18M
  return FastDivision<T, V>()(yp, yq);
97
5.18M
}
Unexecuted instantiation: stage_from_linear.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: stage_from_linear.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_from_linear.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: stage_from_linear.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_from_linear.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_from_linear.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_from_linear.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: splines.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: splines.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: splines.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
98
99
}  // namespace
100
// NOLINTNEXTLINE(google-readability-namespace-comments)
101
}  // namespace HWY_NAMESPACE
102
}  // namespace jxl
103
HWY_AFTER_NAMESPACE();
104
#endif  // LIB_JXL_BASE_RATIONAL_POLYNOMIAL_INL_H_