Coverage Report

Created: 2026-03-31 06:56

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/libjxl/lib/jxl/base/rational_polynomial-inl.h
Line
Count
Source
1
// Copyright (c) the JPEG XL Project Authors. All rights reserved.
2
//
3
// Use of this source code is governed by a BSD-style
4
// license that can be found in the LICENSE file.
5
6
// Fast SIMD evaluation of rational polynomials for approximating functions.
7
8
#if defined(LIB_JXL_BASE_RATIONAL_POLYNOMIAL_INL_H_) == \
9
    defined(HWY_TARGET_TOGGLE)
10
#ifdef LIB_JXL_BASE_RATIONAL_POLYNOMIAL_INL_H_
11
#undef LIB_JXL_BASE_RATIONAL_POLYNOMIAL_INL_H_
12
#else
13
#define LIB_JXL_BASE_RATIONAL_POLYNOMIAL_INL_H_
14
#endif
15
16
#include <jxl/types.h>
17
#include <stddef.h>
18
19
#include <hwy/highway.h>
20
HWY_BEFORE_NAMESPACE();
21
namespace jxl {
22
namespace HWY_NAMESPACE {
23
namespace {
24
25
// These templates are not found via ADL.
26
using hwy::HWY_NAMESPACE::ApproximateReciprocal;
27
using hwy::HWY_NAMESPACE::Div;
28
using hwy::HWY_NAMESPACE::MulAdd;
29
30
// Primary template: default to actual division.
31
template <typename T, class V>
32
struct FastDivision {
33
  HWY_INLINE V operator()(const V n, const V d) const { return n / d; }
34
};
35
// Partial specialization for float vectors.
36
template <class V>
37
struct FastDivision<float, V> {
38
  // One Newton-Raphson iteration.
39
  static HWY_INLINE V ReciprocalNR(const V x) {
40
    const auto rcp = ApproximateReciprocal(x);
41
    const auto sum = Add(rcp, rcp);
42
    const auto x_rcp = Mul(x, rcp);
43
    return NegMulAdd(x_rcp, rcp, sum);
44
  }
45
46
1.13G
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
1.13G
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
1.13G
  }
Unexecuted instantiation: enc_cluster.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
enc_cluster.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Line
Count
Source
46
672M
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
672M
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
672M
  }
Unexecuted instantiation: enc_cluster.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: enc_cluster.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_cluster.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_cluster.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
enc_lz77.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Line
Count
Source
46
1.03M
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
1.03M
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
1.03M
  }
Unexecuted instantiation: enc_xyb.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
enc_xyb.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Line
Count
Source
46
98.4M
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
98.4M
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
98.4M
  }
Unexecuted instantiation: enc_xyb.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: enc_xyb.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_xyb.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_xyb.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: butteraugli.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
Unexecuted instantiation: butteraugli.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Unexecuted instantiation: butteraugli.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: butteraugli.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: butteraugli.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: butteraugli.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
enc_adaptive_quantization.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Line
Count
Source
46
4.12M
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
4.12M
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
4.12M
  }
Unexecuted instantiation: enc_adaptive_quantization.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
Unexecuted instantiation: enc_adaptive_quantization.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: enc_adaptive_quantization.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_adaptive_quantization.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_adaptive_quantization.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_ac_strategy.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
enc_ac_strategy.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Line
Count
Source
46
3.81M
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
3.81M
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
3.81M
  }
Unexecuted instantiation: enc_ac_strategy.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: jxl_cms.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
Unexecuted instantiation: jxl_cms.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Unexecuted instantiation: jxl_cms.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: jxl_cms.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: jxl_cms.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: jxl_cms.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_ma.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
enc_ma.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Line
Count
Source
46
167M
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
167M
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
167M
  }
Unexecuted instantiation: enc_ma.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: enc_ma.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_ma.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_ma.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: quant_weights.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: quant_weights.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: quant_weights.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: quant_weights.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
quant_weights.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 4ul> >::operator()(hwy::N_AVX2::Vec128<float, 4ul>, hwy::N_AVX2::Vec128<float, 4ul>) const
Line
Count
Source
46
17.4M
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
17.4M
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
17.4M
  }
Unexecuted instantiation: quant_weights.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
stage_from_linear.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Line
Count
Source
46
170M
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
170M
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
170M
  }
Unexecuted instantiation: stage_from_linear.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
Unexecuted instantiation: stage_from_linear.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: stage_from_linear.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: stage_from_linear.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: stage_from_linear.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: stage_to_linear.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Unexecuted instantiation: stage_to_linear.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
Unexecuted instantiation: stage_to_linear.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: stage_to_linear.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: stage_to_linear.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: stage_to_linear.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: stage_tone_mapping.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
Unexecuted instantiation: stage_tone_mapping.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Unexecuted instantiation: stage_tone_mapping.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: stage_tone_mapping.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: stage_tone_mapping.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: stage_tone_mapping.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: splines.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: splines.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: splines.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
53
};
54
55
// Approximates smooth functions via rational polynomials (i.e. dividing two
56
// polynomials). Evaluates polynomials via Horner's scheme, which is faster than
57
// Clenshaw recurrence for Chebyshev polynomials. LoadDup128 allows us to
58
// specify constants (replicated 4x) independently of the lane count.
59
template <size_t NP, size_t NQ, class D, class V, typename T>
60
HWY_INLINE HWY_MAYBE_UNUSED V EvalRationalPolynomial(const D d, const V x,
61
                                                     const T (&p)[NP],
62
1.13G
                                                     const T (&q)[NQ]) {
63
1.13G
  constexpr size_t kDegP = NP / 4 - 1;
64
1.13G
  constexpr size_t kDegQ = NQ / 4 - 1;
65
1.13G
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
1.13G
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
1.13G
  HWY_FENCE;
72
1.13G
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
1.13G
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
1.13G
  HWY_FENCE;
75
1.13G
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
1.13G
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
1.13G
  HWY_FENCE;
78
1.13G
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
1.13G
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
1.13G
  HWY_FENCE;
81
1.13G
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
1.13G
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
1.13G
  HWY_FENCE;
84
1.13G
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
1.13G
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
1.13G
  HWY_FENCE;
87
1.13G
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
1.13G
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
1.13G
  HWY_FENCE;
90
1.13G
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
1.13G
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
1.13G
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
1.13G
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
1.13G
  return FastDivision<T, V>()(yp, yq);
97
1.13G
}
Unexecuted instantiation: enc_cluster.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
enc_cluster.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [12ul], float const (&) [12ul])
Line
Count
Source
62
672M
                                                     const T (&q)[NQ]) {
63
672M
  constexpr size_t kDegP = NP / 4 - 1;
64
672M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
672M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
672M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
672M
  HWY_FENCE;
72
672M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
672M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
672M
  HWY_FENCE;
75
672M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
672M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
672M
  HWY_FENCE;
78
672M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
672M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
672M
  HWY_FENCE;
81
672M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
672M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
672M
  HWY_FENCE;
84
672M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
672M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
672M
  HWY_FENCE;
87
672M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
672M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
672M
  HWY_FENCE;
90
672M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
672M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
672M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
672M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
672M
  return FastDivision<T, V>()(yp, yq);
97
672M
}
Unexecuted instantiation: enc_cluster.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_cluster.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_cluster.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_cluster.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
enc_lz77.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Line
Count
Source
62
1.03M
                                                     const T (&q)[NQ]) {
63
1.03M
  constexpr size_t kDegP = NP / 4 - 1;
64
1.03M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
1.03M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
1.03M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
1.03M
  HWY_FENCE;
72
1.03M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
1.03M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
1.03M
  HWY_FENCE;
75
1.03M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
1.03M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
1.03M
  HWY_FENCE;
78
1.03M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
1.03M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
1.03M
  HWY_FENCE;
81
1.03M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
1.03M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
1.03M
  HWY_FENCE;
84
1.03M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
1.03M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
1.03M
  HWY_FENCE;
87
1.03M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
1.03M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
1.03M
  HWY_FENCE;
90
1.03M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
1.03M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
1.03M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
1.03M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
1.03M
  return FastDivision<T, V>()(yp, yq);
97
1.03M
}
Unexecuted instantiation: enc_xyb.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
enc_xyb.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [20ul], float const (&) [20ul])
Line
Count
Source
62
98.4M
                                                     const T (&q)[NQ]) {
63
98.4M
  constexpr size_t kDegP = NP / 4 - 1;
64
98.4M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
98.4M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
98.4M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
98.4M
  HWY_FENCE;
72
98.4M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
98.4M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
98.4M
  HWY_FENCE;
75
98.4M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
98.4M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
98.4M
  HWY_FENCE;
78
98.4M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
98.4M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
98.4M
  HWY_FENCE;
81
98.4M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
98.4M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
98.4M
  HWY_FENCE;
84
98.4M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
98.4M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
98.4M
  HWY_FENCE;
87
98.4M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
98.4M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
98.4M
  HWY_FENCE;
90
98.4M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
98.4M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
98.4M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
98.4M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
98.4M
  return FastDivision<T, V>()(yp, yq);
97
98.4M
}
Unexecuted instantiation: enc_xyb.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: enc_xyb.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_xyb.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_xyb.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: butteraugli.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: butteraugli.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: butteraugli.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: butteraugli.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: butteraugli.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: butteraugli.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
enc_adaptive_quantization.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [12ul], float const (&) [12ul])
Line
Count
Source
62
4.12M
                                                     const T (&q)[NQ]) {
63
4.12M
  constexpr size_t kDegP = NP / 4 - 1;
64
4.12M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
4.12M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
4.12M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
4.12M
  HWY_FENCE;
72
4.12M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
4.12M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
4.12M
  HWY_FENCE;
75
4.12M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
4.12M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
4.12M
  HWY_FENCE;
78
4.12M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
4.12M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
4.12M
  HWY_FENCE;
81
4.12M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
4.12M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
4.12M
  HWY_FENCE;
84
4.12M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
4.12M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
4.12M
  HWY_FENCE;
87
4.12M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
4.12M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
4.12M
  HWY_FENCE;
90
4.12M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
4.12M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
4.12M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
4.12M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
4.12M
  return FastDivision<T, V>()(yp, yq);
97
4.12M
}
Unexecuted instantiation: enc_adaptive_quantization.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_adaptive_quantization.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_adaptive_quantization.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_adaptive_quantization.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_adaptive_quantization.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_ac_strategy.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
enc_ac_strategy.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Line
Count
Source
62
3.81M
                                                     const T (&q)[NQ]) {
63
3.81M
  constexpr size_t kDegP = NP / 4 - 1;
64
3.81M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
3.81M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
3.81M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
3.81M
  HWY_FENCE;
72
3.81M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
3.81M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
3.81M
  HWY_FENCE;
75
3.81M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
3.81M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
3.81M
  HWY_FENCE;
78
3.81M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
3.81M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
3.81M
  HWY_FENCE;
81
3.81M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
3.81M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
3.81M
  HWY_FENCE;
84
3.81M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
3.81M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
3.81M
  HWY_FENCE;
87
3.81M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
3.81M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
3.81M
  HWY_FENCE;
90
3.81M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
3.81M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
3.81M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
3.81M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
3.81M
  return FastDivision<T, V>()(yp, yq);
97
3.81M
}
Unexecuted instantiation: enc_ac_strategy.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: jxl_cms.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: jxl_cms.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: jxl_cms.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: jxl_cms.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: jxl_cms.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: jxl_cms.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_ma.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
enc_ma.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [12ul], float const (&) [12ul])
Line
Count
Source
62
167M
                                                     const T (&q)[NQ]) {
63
167M
  constexpr size_t kDegP = NP / 4 - 1;
64
167M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
167M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
167M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
167M
  HWY_FENCE;
72
167M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
167M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
167M
  HWY_FENCE;
75
167M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
167M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
167M
  HWY_FENCE;
78
167M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
167M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
167M
  HWY_FENCE;
81
167M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
167M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
167M
  HWY_FENCE;
84
167M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
167M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
167M
  HWY_FENCE;
87
167M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
167M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
167M
  HWY_FENCE;
90
167M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
167M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
167M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
167M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
167M
  return FastDivision<T, V>()(yp, yq);
97
167M
}
Unexecuted instantiation: enc_ma.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_ma.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_ma.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_ma.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: quant_weights.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: quant_weights.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: quant_weights.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: quant_weights.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
quant_weights.cc:hwy::N_AVX2::Vec128<float, 4ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 4ul, 0>, hwy::N_AVX2::Vec128<float, 4ul>, float>(hwy::N_AVX2::Simd<float, 4ul, 0>, hwy::N_AVX2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Line
Count
Source
62
17.4M
                                                     const T (&q)[NQ]) {
63
17.4M
  constexpr size_t kDegP = NP / 4 - 1;
64
17.4M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
17.4M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
17.4M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
17.4M
  HWY_FENCE;
72
17.4M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
17.4M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
17.4M
  HWY_FENCE;
75
17.4M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
17.4M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
17.4M
  HWY_FENCE;
78
17.4M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
17.4M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
17.4M
  HWY_FENCE;
81
17.4M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
17.4M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
17.4M
  HWY_FENCE;
84
17.4M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
17.4M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
17.4M
  HWY_FENCE;
87
17.4M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
17.4M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
17.4M
  HWY_FENCE;
90
17.4M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
17.4M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
17.4M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
17.4M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
17.4M
  return FastDivision<T, V>()(yp, yq);
97
17.4M
}
Unexecuted instantiation: quant_weights.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
stage_from_linear.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [20ul], float const (&) [20ul])
Line
Count
Source
62
166M
                                                     const T (&q)[NQ]) {
63
166M
  constexpr size_t kDegP = NP / 4 - 1;
64
166M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
166M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
166M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
166M
  HWY_FENCE;
72
166M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
166M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
166M
  HWY_FENCE;
75
166M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
166M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
166M
  HWY_FENCE;
78
166M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
166M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
166M
  HWY_FENCE;
81
166M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
166M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
166M
  HWY_FENCE;
84
166M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
166M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
166M
  HWY_FENCE;
87
166M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
166M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
166M
  HWY_FENCE;
90
166M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
166M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
166M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
166M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
166M
  return FastDivision<T, V>()(yp, yq);
97
166M
}
stage_from_linear.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [12ul], float const (&) [12ul])
Line
Count
Source
62
3.63M
                                                     const T (&q)[NQ]) {
63
3.63M
  constexpr size_t kDegP = NP / 4 - 1;
64
3.63M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
3.63M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
3.63M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
3.63M
  HWY_FENCE;
72
3.63M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
3.63M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
3.63M
  HWY_FENCE;
75
3.63M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
3.63M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
3.63M
  HWY_FENCE;
78
3.63M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
3.63M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
3.63M
  HWY_FENCE;
81
3.63M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
3.63M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
3.63M
  HWY_FENCE;
84
3.63M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
3.63M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
3.63M
  HWY_FENCE;
87
3.63M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
3.63M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
3.63M
  HWY_FENCE;
90
3.63M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
3.63M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
3.63M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
3.63M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
3.63M
  return FastDivision<T, V>()(yp, yq);
97
3.63M
}
Unexecuted instantiation: stage_from_linear.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: stage_from_linear.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_from_linear.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: stage_from_linear.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_from_linear.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_from_linear.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_from_linear.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: splines.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: splines.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: splines.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
98
99
}  // namespace
100
// NOLINTNEXTLINE(google-readability-namespace-comments)
101
}  // namespace HWY_NAMESPACE
102
}  // namespace jxl
103
HWY_AFTER_NAMESPACE();
104
#endif  // LIB_JXL_BASE_RATIONAL_POLYNOMIAL_INL_H_