Coverage Report

Created: 2026-02-14 07:11

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/libjxl/lib/jxl/base/rational_polynomial-inl.h
Line
Count
Source
1
// Copyright (c) the JPEG XL Project Authors. All rights reserved.
2
//
3
// Use of this source code is governed by a BSD-style
4
// license that can be found in the LICENSE file.
5
6
// Fast SIMD evaluation of rational polynomials for approximating functions.
7
8
#if defined(LIB_JXL_BASE_RATIONAL_POLYNOMIAL_INL_H_) == \
9
    defined(HWY_TARGET_TOGGLE)
10
#ifdef LIB_JXL_BASE_RATIONAL_POLYNOMIAL_INL_H_
11
#undef LIB_JXL_BASE_RATIONAL_POLYNOMIAL_INL_H_
12
#else
13
#define LIB_JXL_BASE_RATIONAL_POLYNOMIAL_INL_H_
14
#endif
15
16
#include <jxl/types.h>
17
#include <stddef.h>
18
19
#include <hwy/highway.h>
20
HWY_BEFORE_NAMESPACE();
21
namespace jxl {
22
namespace HWY_NAMESPACE {
23
namespace {
24
25
// These templates are not found via ADL.
26
using hwy::HWY_NAMESPACE::ApproximateReciprocal;
27
using hwy::HWY_NAMESPACE::Div;
28
using hwy::HWY_NAMESPACE::MulAdd;
29
30
// Primary template: default to actual division.
31
template <typename T, class V>
32
struct FastDivision {
33
  HWY_INLINE V operator()(const V n, const V d) const { return n / d; }
34
};
35
// Partial specialization for float vectors.
36
template <class V>
37
struct FastDivision<float, V> {
38
  // One Newton-Raphson iteration.
39
  static HWY_INLINE V ReciprocalNR(const V x) {
40
    const auto rcp = ApproximateReciprocal(x);
41
    const auto sum = Add(rcp, rcp);
42
    const auto x_rcp = Mul(x, rcp);
43
    return NegMulAdd(x_rcp, rcp, sum);
44
  }
45
46
1.08G
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
1.08G
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
1.08G
  }
Unexecuted instantiation: enc_cluster.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
enc_cluster.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Line
Count
Source
46
672M
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
672M
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
672M
  }
Unexecuted instantiation: enc_cluster.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: enc_cluster.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_cluster.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_cluster.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
enc_lz77.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Line
Count
Source
46
1.01M
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
1.01M
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
1.01M
  }
Unexecuted instantiation: enc_xyb.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
enc_xyb.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Line
Count
Source
46
91.3M
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
91.3M
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
91.3M
  }
Unexecuted instantiation: enc_xyb.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: enc_xyb.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_xyb.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_xyb.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: butteraugli.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
Unexecuted instantiation: butteraugli.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Unexecuted instantiation: butteraugli.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: butteraugli.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: butteraugli.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: butteraugli.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
enc_adaptive_quantization.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Line
Count
Source
46
3.83M
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
3.83M
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
3.83M
  }
Unexecuted instantiation: enc_adaptive_quantization.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
Unexecuted instantiation: enc_adaptive_quantization.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: enc_adaptive_quantization.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_adaptive_quantization.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_adaptive_quantization.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_ac_strategy.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
enc_ac_strategy.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Line
Count
Source
46
3.51M
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
3.51M
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
3.51M
  }
Unexecuted instantiation: enc_ac_strategy.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: jxl_cms.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
Unexecuted instantiation: jxl_cms.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Unexecuted instantiation: jxl_cms.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: jxl_cms.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: jxl_cms.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: jxl_cms.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_ma.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
enc_ma.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Line
Count
Source
46
146M
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
146M
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
146M
  }
Unexecuted instantiation: enc_ma.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: enc_ma.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_ma.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: enc_ma.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: quant_weights.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: quant_weights.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: quant_weights.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: quant_weights.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
quant_weights.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 4ul> >::operator()(hwy::N_AVX2::Vec128<float, 4ul>, hwy::N_AVX2::Vec128<float, 4ul>) const
Line
Count
Source
46
16.5M
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
16.5M
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
16.5M
  }
Unexecuted instantiation: quant_weights.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
stage_from_linear.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Line
Count
Source
46
147M
  V operator()(const V n, const V d) const {
47
#if JXL_TRUE  // Faster on SKX
48
147M
    return Div(n, d);
49
#else
50
    return n * ReciprocalNR(d);
51
#endif
52
147M
  }
Unexecuted instantiation: stage_from_linear.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
Unexecuted instantiation: stage_from_linear.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: stage_from_linear.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: stage_from_linear.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: stage_from_linear.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: stage_to_linear.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Unexecuted instantiation: stage_to_linear.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
Unexecuted instantiation: stage_to_linear.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: stage_to_linear.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: stage_to_linear.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: stage_to_linear.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: stage_tone_mapping.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 4ul> >::operator()(hwy::N_SSE4::Vec128<float, 4ul>, hwy::N_SSE4::Vec128<float, 4ul>) const
Unexecuted instantiation: stage_tone_mapping.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec256<float> >::operator()(hwy::N_AVX2::Vec256<float>, hwy::N_AVX2::Vec256<float>) const
Unexecuted instantiation: stage_tone_mapping.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 4ul> >::operator()(hwy::N_SSE2::Vec128<float, 4ul>, hwy::N_SSE2::Vec128<float, 4ul>) const
Unexecuted instantiation: stage_tone_mapping.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: stage_tone_mapping.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: stage_tone_mapping.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
Unexecuted instantiation: splines.cc:jxl::N_SSE4::(anonymous namespace)::FastDivision<float, hwy::N_SSE4::Vec128<float, 1ul> >::operator()(hwy::N_SSE4::Vec128<float, 1ul>, hwy::N_SSE4::Vec128<float, 1ul>) const
Unexecuted instantiation: splines.cc:jxl::N_AVX2::(anonymous namespace)::FastDivision<float, hwy::N_AVX2::Vec128<float, 1ul> >::operator()(hwy::N_AVX2::Vec128<float, 1ul>, hwy::N_AVX2::Vec128<float, 1ul>) const
Unexecuted instantiation: splines.cc:jxl::N_SSE2::(anonymous namespace)::FastDivision<float, hwy::N_SSE2::Vec128<float, 1ul> >::operator()(hwy::N_SSE2::Vec128<float, 1ul>, hwy::N_SSE2::Vec128<float, 1ul>) const
53
};
54
55
// Approximates smooth functions via rational polynomials (i.e. dividing two
56
// polynomials). Evaluates polynomials via Horner's scheme, which is faster than
57
// Clenshaw recurrence for Chebyshev polynomials. LoadDup128 allows us to
58
// specify constants (replicated 4x) independently of the lane count.
59
template <size_t NP, size_t NQ, class D, class V, typename T>
60
HWY_INLINE HWY_MAYBE_UNUSED V EvalRationalPolynomial(const D d, const V x,
61
                                                     const T (&p)[NP],
62
1.08G
                                                     const T (&q)[NQ]) {
63
1.08G
  constexpr size_t kDegP = NP / 4 - 1;
64
1.08G
  constexpr size_t kDegQ = NQ / 4 - 1;
65
1.08G
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
1.08G
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
1.08G
  HWY_FENCE;
72
1.08G
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
1.08G
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
1.08G
  HWY_FENCE;
75
1.08G
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
1.08G
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
1.08G
  HWY_FENCE;
78
1.08G
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
1.08G
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
1.08G
  HWY_FENCE;
81
1.08G
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
1.08G
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
1.08G
  HWY_FENCE;
84
1.08G
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
1.08G
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
1.08G
  HWY_FENCE;
87
1.08G
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
1.08G
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
1.08G
  HWY_FENCE;
90
1.08G
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
1.08G
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
1.08G
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
1.08G
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
1.08G
  return FastDivision<T, V>()(yp, yq);
97
1.08G
}
Unexecuted instantiation: enc_cluster.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
enc_cluster.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [12ul], float const (&) [12ul])
Line
Count
Source
62
672M
                                                     const T (&q)[NQ]) {
63
672M
  constexpr size_t kDegP = NP / 4 - 1;
64
672M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
672M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
672M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
672M
  HWY_FENCE;
72
672M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
672M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
672M
  HWY_FENCE;
75
672M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
672M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
672M
  HWY_FENCE;
78
672M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
672M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
672M
  HWY_FENCE;
81
672M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
672M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
672M
  HWY_FENCE;
84
672M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
672M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
672M
  HWY_FENCE;
87
672M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
672M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
672M
  HWY_FENCE;
90
672M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
672M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
672M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
672M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
672M
  return FastDivision<T, V>()(yp, yq);
97
672M
}
Unexecuted instantiation: enc_cluster.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_cluster.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_cluster.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_cluster.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
enc_lz77.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Line
Count
Source
62
1.01M
                                                     const T (&q)[NQ]) {
63
1.01M
  constexpr size_t kDegP = NP / 4 - 1;
64
1.01M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
1.01M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
1.01M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
1.01M
  HWY_FENCE;
72
1.01M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
1.01M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
1.01M
  HWY_FENCE;
75
1.01M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
1.01M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
1.01M
  HWY_FENCE;
78
1.01M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
1.01M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
1.01M
  HWY_FENCE;
81
1.01M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
1.01M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
1.01M
  HWY_FENCE;
84
1.01M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
1.01M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
1.01M
  HWY_FENCE;
87
1.01M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
1.01M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
1.01M
  HWY_FENCE;
90
1.01M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
1.01M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
1.01M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
1.01M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
1.01M
  return FastDivision<T, V>()(yp, yq);
97
1.01M
}
Unexecuted instantiation: enc_xyb.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
enc_xyb.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [20ul], float const (&) [20ul])
Line
Count
Source
62
91.3M
                                                     const T (&q)[NQ]) {
63
91.3M
  constexpr size_t kDegP = NP / 4 - 1;
64
91.3M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
91.3M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
91.3M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
91.3M
  HWY_FENCE;
72
91.3M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
91.3M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
91.3M
  HWY_FENCE;
75
91.3M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
91.3M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
91.3M
  HWY_FENCE;
78
91.3M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
91.3M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
91.3M
  HWY_FENCE;
81
91.3M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
91.3M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
91.3M
  HWY_FENCE;
84
91.3M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
91.3M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
91.3M
  HWY_FENCE;
87
91.3M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
91.3M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
91.3M
  HWY_FENCE;
90
91.3M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
91.3M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
91.3M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
91.3M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
91.3M
  return FastDivision<T, V>()(yp, yq);
97
91.3M
}
Unexecuted instantiation: enc_xyb.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: enc_xyb.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_xyb.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_xyb.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: butteraugli.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: butteraugli.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: butteraugli.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: butteraugli.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: butteraugli.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: butteraugli.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
enc_adaptive_quantization.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [12ul], float const (&) [12ul])
Line
Count
Source
62
3.83M
                                                     const T (&q)[NQ]) {
63
3.83M
  constexpr size_t kDegP = NP / 4 - 1;
64
3.83M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
3.83M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
3.83M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
3.83M
  HWY_FENCE;
72
3.83M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
3.83M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
3.83M
  HWY_FENCE;
75
3.83M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
3.83M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
3.83M
  HWY_FENCE;
78
3.83M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
3.83M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
3.83M
  HWY_FENCE;
81
3.83M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
3.83M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
3.83M
  HWY_FENCE;
84
3.83M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
3.83M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
3.83M
  HWY_FENCE;
87
3.83M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
3.83M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
3.83M
  HWY_FENCE;
90
3.83M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
3.83M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
3.83M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
3.83M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
3.83M
  return FastDivision<T, V>()(yp, yq);
97
3.83M
}
Unexecuted instantiation: enc_adaptive_quantization.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_adaptive_quantization.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_adaptive_quantization.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_adaptive_quantization.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_adaptive_quantization.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_ac_strategy.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
enc_ac_strategy.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Line
Count
Source
62
3.51M
                                                     const T (&q)[NQ]) {
63
3.51M
  constexpr size_t kDegP = NP / 4 - 1;
64
3.51M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
3.51M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
3.51M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
3.51M
  HWY_FENCE;
72
3.51M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
3.51M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
3.51M
  HWY_FENCE;
75
3.51M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
3.51M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
3.51M
  HWY_FENCE;
78
3.51M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
3.51M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
3.51M
  HWY_FENCE;
81
3.51M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
3.51M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
3.51M
  HWY_FENCE;
84
3.51M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
3.51M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
3.51M
  HWY_FENCE;
87
3.51M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
3.51M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
3.51M
  HWY_FENCE;
90
3.51M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
3.51M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
3.51M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
3.51M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
3.51M
  return FastDivision<T, V>()(yp, yq);
97
3.51M
}
Unexecuted instantiation: enc_ac_strategy.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: jxl_cms.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: jxl_cms.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: jxl_cms.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: jxl_cms.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: jxl_cms.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: jxl_cms.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_ma.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
enc_ma.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [12ul], float const (&) [12ul])
Line
Count
Source
62
146M
                                                     const T (&q)[NQ]) {
63
146M
  constexpr size_t kDegP = NP / 4 - 1;
64
146M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
146M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
146M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
146M
  HWY_FENCE;
72
146M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
146M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
146M
  HWY_FENCE;
75
146M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
146M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
146M
  HWY_FENCE;
78
146M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
146M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
146M
  HWY_FENCE;
81
146M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
146M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
146M
  HWY_FENCE;
84
146M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
146M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
146M
  HWY_FENCE;
87
146M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
146M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
146M
  HWY_FENCE;
90
146M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
146M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
146M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
146M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
146M
  return FastDivision<T, V>()(yp, yq);
97
146M
}
Unexecuted instantiation: enc_ma.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_ma.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_ma.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: enc_ma.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: quant_weights.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: quant_weights.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: quant_weights.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: quant_weights.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
quant_weights.cc:hwy::N_AVX2::Vec128<float, 4ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 4ul, 0>, hwy::N_AVX2::Vec128<float, 4ul>, float>(hwy::N_AVX2::Simd<float, 4ul, 0>, hwy::N_AVX2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Line
Count
Source
62
16.5M
                                                     const T (&q)[NQ]) {
63
16.5M
  constexpr size_t kDegP = NP / 4 - 1;
64
16.5M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
16.5M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
16.5M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
16.5M
  HWY_FENCE;
72
16.5M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
16.5M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
16.5M
  HWY_FENCE;
75
16.5M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
16.5M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
16.5M
  HWY_FENCE;
78
16.5M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
16.5M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
16.5M
  HWY_FENCE;
81
16.5M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
16.5M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
16.5M
  HWY_FENCE;
84
16.5M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
16.5M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
16.5M
  HWY_FENCE;
87
16.5M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
16.5M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
16.5M
  HWY_FENCE;
90
16.5M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
16.5M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
16.5M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
16.5M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
16.5M
  return FastDivision<T, V>()(yp, yq);
97
16.5M
}
Unexecuted instantiation: quant_weights.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
stage_from_linear.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [20ul], float const (&) [20ul])
Line
Count
Source
62
144M
                                                     const T (&q)[NQ]) {
63
144M
  constexpr size_t kDegP = NP / 4 - 1;
64
144M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
144M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
144M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
144M
  HWY_FENCE;
72
144M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
144M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
144M
  HWY_FENCE;
75
144M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
144M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
144M
  HWY_FENCE;
78
144M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
144M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
144M
  HWY_FENCE;
81
144M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
144M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
144M
  HWY_FENCE;
84
144M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
144M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
144M
  HWY_FENCE;
87
144M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
144M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
144M
  HWY_FENCE;
90
144M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
144M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
144M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
144M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
144M
  return FastDivision<T, V>()(yp, yq);
97
144M
}
stage_from_linear.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [12ul], float const (&) [12ul])
Line
Count
Source
62
2.74M
                                                     const T (&q)[NQ]) {
63
2.74M
  constexpr size_t kDegP = NP / 4 - 1;
64
2.74M
  constexpr size_t kDegQ = NQ / 4 - 1;
65
2.74M
  auto yp = LoadDup128(d, &p[kDegP * 4]);
66
2.74M
  auto yq = LoadDup128(d, &q[kDegQ * 4]);
67
  // We use pointer arithmetic to refer to &p[(kDegP - n) * 4] to avoid a
68
  // compiler warning that the index is out of bounds since we are already
69
  // checking that it is not out of bounds with (kDegP >= n) and the access
70
  // will be optimized away. Similarly with q and kDegQ.
71
2.74M
  HWY_FENCE;
72
2.74M
  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 1) * 4)));
73
2.74M
  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 1) * 4)));
74
2.74M
  HWY_FENCE;
75
2.74M
  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 2) * 4)));
76
2.74M
  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 2) * 4)));
77
2.74M
  HWY_FENCE;
78
2.74M
  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 3) * 4)));
79
2.74M
  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 3) * 4)));
80
2.74M
  HWY_FENCE;
81
2.74M
  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 4) * 4)));
82
2.74M
  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 4) * 4)));
83
2.74M
  HWY_FENCE;
84
2.74M
  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 5) * 4)));
85
2.74M
  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 5) * 4)));
86
2.74M
  HWY_FENCE;
87
2.74M
  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 6) * 4)));
88
2.74M
  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 6) * 4)));
89
2.74M
  HWY_FENCE;
90
2.74M
  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, p + ((kDegP - 7) * 4)));
91
2.74M
  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, q + ((kDegQ - 7) * 4)));
92
93
2.74M
  static_assert(kDegP < 8, "Polynomial degree is too high");
94
2.74M
  static_assert(kDegQ < 8, "Polynomial degree is too high");
95
96
2.74M
  return FastDivision<T, V>()(yp, yq);
97
2.74M
}
Unexecuted instantiation: stage_from_linear.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: stage_from_linear.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_from_linear.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: stage_from_linear.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_from_linear.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_from_linear.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_from_linear.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_to_linear.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_SSE4::Vec128<float, 4ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float>(hwy::N_SSE4::Simd<float, 4ul, 0>, hwy::N_SSE4::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_AVX2::Vec256<float> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float>(hwy::N_AVX2::Simd<float, 8ul, 0>, hwy::N_AVX2::Vec256<float>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<20ul, 20ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [20ul], float const (&) [20ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_SSE2::Vec128<float, 4ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float>(hwy::N_SSE2::Simd<float, 4ul, 0>, hwy::N_SSE2::Vec128<float, 4ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: stage_tone_mapping.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: splines.cc:hwy::N_SSE4::Vec128<float, 1ul> jxl::N_SSE4::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float>(hwy::N_SSE4::Simd<float, 1ul, 0>, hwy::N_SSE4::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: splines.cc:hwy::N_AVX2::Vec128<float, 1ul> jxl::N_AVX2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float>(hwy::N_AVX2::Simd<float, 1ul, 0>, hwy::N_AVX2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
Unexecuted instantiation: splines.cc:hwy::N_SSE2::Vec128<float, 1ul> jxl::N_SSE2::(anonymous namespace)::EvalRationalPolynomial<12ul, 12ul, hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float>(hwy::N_SSE2::Simd<float, 1ul, 0>, hwy::N_SSE2::Vec128<float, 1ul>, float const (&) [12ul], float const (&) [12ul])
98
99
}  // namespace
100
// NOLINTNEXTLINE(google-readability-namespace-comments)
101
}  // namespace HWY_NAMESPACE
102
}  // namespace jxl
103
HWY_AFTER_NAMESPACE();
104
#endif  // LIB_JXL_BASE_RATIONAL_POLYNOMIAL_INL_H_