Coverage Report

Created: 2026-09-13 07:02

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/libjxl/lib/jxl/enc_modular_simd.cc
Line
Count
Source
1
// Copyright (c) the JPEG XL Project Authors. All rights reserved.
2
//
3
// Use of this source code is governed by a BSD-style
4
// license that can be found in the LICENSE file.
5
6
#include "lib/jxl/enc_modular_simd.h"
7
8
#include <jxl/memory_manager.h>
9
10
#include <algorithm>
11
#include <array>
12
#include <cstddef>
13
#include <cstdint>
14
15
#include "lib/jxl/base/common.h"
16
#include "lib/jxl/base/compiler_specific.h"
17
#include "lib/jxl/base/status.h"
18
#include "lib/jxl/dec_ans.h"
19
#include "lib/jxl/enc_ans_params.h"
20
#include "lib/jxl/memory_manager_internal.h"
21
#include "lib/jxl/modular/modular_image.h"
22
23
#undef HWY_TARGET_INCLUDE
24
#define HWY_TARGET_INCLUDE "lib/jxl/enc_modular_simd.cc"
25
#include <hwy/foreach_target.h>
26
#include <hwy/highway.h>
27
28
#if HWY_TARGET == HWY_SCALAR
29
#include "lib/jxl/modular/encoding/context_predict.h"
30
#include "lib/jxl/pack_signed.h"
31
#endif
32
33
HWY_BEFORE_NAMESPACE();
34
namespace jxl {
35
namespace HWY_NAMESPACE {
36
37
// These templates are not found via ADL.
38
using hwy::HWY_NAMESPACE::Add;
39
using hwy::HWY_NAMESPACE::And;
40
using hwy::HWY_NAMESPACE::Ge;
41
using hwy::HWY_NAMESPACE::GetLane;
42
using hwy::HWY_NAMESPACE::Gt;
43
using hwy::HWY_NAMESPACE::IfThenElse;
44
using hwy::HWY_NAMESPACE::IfThenElseZero;
45
using hwy::HWY_NAMESPACE::Iota;
46
using hwy::HWY_NAMESPACE::Load;
47
using hwy::HWY_NAMESPACE::LoadU;
48
using hwy::HWY_NAMESPACE::Lt;
49
using hwy::HWY_NAMESPACE::Max;
50
using hwy::HWY_NAMESPACE::Min;
51
using hwy::HWY_NAMESPACE::Mul;
52
using hwy::HWY_NAMESPACE::Not;
53
using hwy::HWY_NAMESPACE::Set;
54
using hwy::HWY_NAMESPACE::ShiftLeft;
55
using hwy::HWY_NAMESPACE::ShiftRight;
56
using hwy::HWY_NAMESPACE::Store;
57
using hwy::HWY_NAMESPACE::StoreU;
58
using hwy::HWY_NAMESPACE::Sub;
59
using hwy::HWY_NAMESPACE::Xor;
60
using hwy::HWY_NAMESPACE::Zero;
61
62
3.53k
StatusOr<float> EstimateCost(const Image& img) {
63
3.53k
  size_t histo_cost = 0;
64
3.53k
  float histo_cost_frac = 0.0f;
65
3.53k
  size_t extra_bits = 0;
66
67
#if HWY_TARGET == HWY_SCALAR
68
  HybridUintConfig config;
69
  uint32_t cutoffs[] = {0,  1,  3,  5,   7,   11,  15,  23, 31,
70
                        47, 63, 95, 127, 191, 255, 392, 500};
71
  constexpr size_t nc = sizeof(cutoffs) / sizeof(*cutoffs) + 1;
72
  Histogram histo[nc] = {};
73
  for (const Channel& ch : img.channel) {
74
    const ptrdiff_t onerow = ch.plane.PixelsPerRow();
75
    for (size_t y = 0; y < ch.h; y++) {
76
      const pixel_type* JXL_RESTRICT r = ch.Row(y);
77
      for (size_t x = 0; x < ch.w; x++) {
78
        pixel_type_w left = (x ? r[x - 1] : y ? *(r + x - onerow) : 0);
79
        pixel_type_w top = (y ? *(r + x - onerow) : left);
80
        pixel_type_w topleft = (x && y ? *(r + x - 1 - onerow) : left);
81
        size_t max_diff =
82
            std::max({left, top, topleft}) - std::min({left, top, topleft});
83
        size_t ctx = 0;
84
        for (uint32_t c : cutoffs) {
85
          ctx += (max_diff < c) ? 1 : 0;
86
        }
87
        pixel_type res = r[x] - ClampedGradient(top, left, topleft);
88
        uint32_t token;
89
        uint32_t nbits;
90
        uint32_t bits;
91
        config.Encode(PackSigned(res), &token, &nbits, &bits);
92
        histo[ctx].Add(token);
93
        extra_bits += nbits;
94
      }
95
    }
96
    for (auto& h : histo) {
97
      float f_cost = h.ShannonEntropy();
98
      size_t i_cost = f_cost;
99
      histo_cost += i_cost;
100
      histo_cost_frac += f_cost - i_cost;
101
      h.Clear();
102
    }
103
  }
104
#else
105
3.53k
  JxlMemoryManager* memory_manager = img.memory_manager();
106
3.53k
  const auto& ctx_map = estimate_cost_detail::ContextMap();
107
3.53k
  const HWY_FULL(int32_t) di;
108
3.53k
  const HWY_FULL(uint32_t) du;
109
3.53k
  const HWY_FULL(float) df;
110
3.53k
  const auto kOne = Set(du, 1);
111
3.53k
  const auto kSplit = Set(du, 16);
112
3.53k
  const auto kExpOffset2 = Set(du, 129);  // 127 + 2
113
3.53k
  const auto kTokenBias = Set(du, 8);
114
3.53k
  const auto kTokenMul = Set(du, 4);
115
3.53k
  const auto kMsbMask = Set(du, 3);
116
3.53k
  const auto kMaxDiffCap = Set(du, estimate_cost_detail::kLastThreshold - 1);
117
3.53k
  const auto kLanes = Set(du, Lanes(du));
118
3.53k
  const auto kIota = Iota(du, 0);
119
3.53k
  const auto kLargeThreshold = Set(du, (1 << 22) - 1);
120
3.53k
  constexpr size_t kLargeShiftVal = 10;
121
3.53k
  const auto kLargeShift = Set(du, kLargeShiftVal);
122
123
3.53k
  size_t max_w = 0;
124
3.53k
  for (const Channel& ch : img.channel) {
125
3.53k
    if (ch.h == 0) continue;
126
3.53k
    max_w = std::max(max_w, ch.w);
127
3.53k
  }
128
3.53k
  max_w = RoundUpTo(max_w, Lanes(du));
129
3.53k
  max_w = std::max(max_w, 2 * Lanes(du));
130
131
3.53k
  JXL_ASSIGN_OR_RETURN(
132
3.53k
      AlignedMemory buffer,
133
3.53k
      AlignedMemory::Create(memory_manager, max_w * 2 * sizeof(uint32_t)));
134
3.53k
  uint32_t* max_diff_row = buffer.address<uint32_t>();
135
3.53k
  uint32_t* token_row = max_diff_row + max_w;
136
3.53k
  int32_t* primer = buffer.address<int32_t>();
137
3.53k
  int32_t* top_primer = primer + max_w;
138
139
3.53k
  HybridUintConfig config;
140
141
3.53k
  Histogram histo[estimate_cost_detail::kLastCtx + 1] = {};
142
3.53k
  auto extra_bits_lanes = Zero(du);
143
3.53k
  for (const Channel& ch : img.channel) {
144
3.53k
    if (ch.h == 0 || ch.w == 0) continue;
145
60.0k
    for (auto& h : histo) {
146
60.0k
      h.EnsureCapacity(32 * 4);
147
60.0k
    }
148
3.53k
    const pixel_type* JXL_RESTRICT r = ch.Row(0);
149
3.53k
    const pixel_type* JXL_RESTRICT last = primer;
150
3.53k
    primer[0] = 0;
151
3.53k
    StoreU(Load(di, r), di, primer + 1);
152
3.53k
    auto pos = kIota;
153
3.53k
    const auto last_pos = Set(du, ch.w);
154
96.0k
    for (size_t x = 0; x < ch.w; x += Lanes(di)) {
155
92.5k
      const auto left = LoadU(di, last);
156
92.5k
      const auto central = Load(di, r + x);
157
92.5k
      const auto ures = BitCast(du, Sub(central, left));
158
92.5k
      const auto packed =
159
92.5k
          Xor(ShiftLeft<1>(ures), Sub(ShiftRight<31>(Not(ures)), kOne));
160
92.5k
      const auto is_large = Gt(packed, kLargeThreshold);
161
92.5k
      const auto packed_shifted = ShiftRight<kLargeShiftVal>(packed);
162
92.5k
      const auto not_literal = Ge(packed, kSplit);
163
92.5k
      const auto packed_fixed = IfThenElse(is_large, packed_shifted, packed);
164
92.5k
      const auto v = BitCast(du, ConvertTo(df, packed_fixed));
165
92.5k
      const auto eb_raw = Sub(ShiftRight<23>(v), kExpOffset2);
166
92.5k
      const auto eb = IfThenElse(is_large, Add(eb_raw, kLargeShift), eb_raw);
167
92.5k
      const auto token = Add(Add(kTokenBias, Mul(eb, kTokenMul)),
168
92.5k
                             And(ShiftRight<21>(v), kMsbMask));
169
92.5k
      const auto tail_mask = Lt(pos, last_pos);
170
92.5k
      const auto eb_fixed = IfThenElseZero(not_literal, eb);
171
92.5k
      const auto token_fixed = IfThenElse(not_literal, token, packed);
172
92.5k
      extra_bits_lanes =
173
92.5k
          Add(extra_bits_lanes, IfThenElseZero(tail_mask, eb_fixed));
174
92.5k
      Store(token_fixed, du, token_row + x);
175
92.5k
      pos = Add(pos, kLanes);
176
92.5k
      last = r + x + Lanes(di) - 1;
177
92.5k
    }
178
738k
    for (size_t x = 0; x < ch.w; x++) {
179
734k
      histo[0].FastAdd(token_row[x]);
180
734k
    }
181
753k
    for (size_t y = 1; y < ch.h; y++) {
182
749k
      r = ch.Row(y);
183
749k
      const pixel_type* JXL_RESTRICT t = ch.Row(y - 1);
184
749k
      last = primer;
185
749k
      primer[0] = t[0];
186
749k
      StoreU(Load(di, r), di, primer + 1);
187
749k
      top_primer[0] = t[0];
188
749k
      StoreU(Load(di, t), di, top_primer + 1);
189
749k
      const pixel_type* JXL_RESTRICT top_last = top_primer;
190
749k
      pos = kIota;
191
23.9M
      for (size_t x = 0; x < ch.w; x += Lanes(di)) {
192
23.2M
        const auto left = LoadU(di, last);
193
23.2M
        const auto central = Load(di, r + x);
194
23.2M
        const auto topleft = LoadU(di, top_last);
195
23.2M
        const auto top = Load(di, t + x);
196
23.2M
        const auto l_ge_t = Ge(left, top);
197
23.2M
        const auto m = IfThenElse(l_ge_t, top, left);
198
23.2M
        const auto M = IfThenElse(l_ge_t, left, top);
199
23.2M
        const auto maxx = Max(topleft, M);
200
23.2M
        const auto minn = Min(topleft, m);
201
23.2M
        const auto max_diff = BitCast(du, Sub(maxx, minn));
202
23.2M
        Store(Min(max_diff, kMaxDiffCap), du, max_diff_row + x);
203
23.2M
        const auto overshoot = Lt(topleft, m);
204
23.2M
        const auto undershoot = Gt(topleft, M);
205
23.2M
        const auto grad =
206
23.2M
            BitCast(di, Sub(Add(BitCast(du, top), BitCast(du, left)),
207
23.2M
                            BitCast(du, topleft)));
208
23.2M
        const auto prediction =
209
23.2M
            IfThenElse(undershoot, m, IfThenElse(overshoot, M, grad));
210
23.2M
        const auto ures = BitCast(du, Sub(central, prediction));
211
23.2M
        const auto packed =
212
23.2M
            Xor(ShiftLeft<1>(ures), Sub(ShiftRight<31>(Not(ures)), kOne));
213
23.2M
        const auto is_large = Gt(packed, kLargeThreshold);
214
23.2M
        const auto packed_shifted = ShiftRight<kLargeShiftVal>(packed);
215
23.2M
        const auto not_literal = Ge(packed, kSplit);
216
23.2M
        const auto packed_fixed = IfThenElse(is_large, packed_shifted, packed);
217
23.2M
        const auto v = BitCast(du, ConvertTo(df, packed_fixed));
218
23.2M
        const auto eb_raw = Sub(ShiftRight<23>(v), kExpOffset2);
219
23.2M
        const auto eb = IfThenElse(is_large, Add(eb_raw, kLargeShift), eb_raw);
220
23.2M
        const auto token = Add(Add(kTokenBias, Mul(eb, kTokenMul)),
221
23.2M
                               And(ShiftRight<21>(v), kMsbMask));
222
23.2M
        const auto tail_mask = Lt(pos, last_pos);
223
23.2M
        const auto eb_fixed = IfThenElseZero(not_literal, eb);
224
23.2M
        const auto token_fixed = IfThenElse(not_literal, token, packed);
225
23.2M
        extra_bits_lanes =
226
23.2M
            Add(extra_bits_lanes, IfThenElseZero(tail_mask, eb_fixed));
227
23.2M
        Store(token_fixed, du, token_row + x);
228
23.2M
        pos = Add(pos, kLanes);
229
23.2M
        last = r + x + Lanes(di) - 1;
230
23.2M
        top_last = t + x + Lanes(di) - 1;
231
23.2M
      }
232
185M
      for (size_t x = 0; x < ch.w; x++) {
233
184M
        size_t ctx = ctx_map[max_diff_row[x]];
234
184M
        histo[ctx].FastAdd(token_row[x]);
235
184M
      }
236
749k
    }
237
60.0k
    for (auto& h : histo) {
238
60.0k
      h.Condition();
239
60.0k
      float f_cost = h.ShannonEntropy();
240
60.0k
      size_t i_cost = f_cost;
241
60.0k
      histo_cost += i_cost;
242
60.0k
      histo_cost_frac += f_cost - i_cost;
243
60.0k
      h.Clear();
244
60.0k
    }
245
3.53k
  }
246
3.53k
  extra_bits = GetLane(SumOfLanes(du, extra_bits_lanes));
247
3.53k
#endif
248
3.53k
  size_t total_cost =
249
3.53k
      extra_bits + histo_cost + static_cast<size_t>(histo_cost_frac);
250
3.53k
  return total_cost;
251
3.53k
}
Unexecuted instantiation: jxl::N_SSE4::EstimateCost(jxl::Image const&)
jxl::N_AVX2::EstimateCost(jxl::Image const&)
Line
Count
Source
62
3.53k
StatusOr<float> EstimateCost(const Image& img) {
63
3.53k
  size_t histo_cost = 0;
64
3.53k
  float histo_cost_frac = 0.0f;
65
3.53k
  size_t extra_bits = 0;
66
67
#if HWY_TARGET == HWY_SCALAR
68
  HybridUintConfig config;
69
  uint32_t cutoffs[] = {0,  1,  3,  5,   7,   11,  15,  23, 31,
70
                        47, 63, 95, 127, 191, 255, 392, 500};
71
  constexpr size_t nc = sizeof(cutoffs) / sizeof(*cutoffs) + 1;
72
  Histogram histo[nc] = {};
73
  for (const Channel& ch : img.channel) {
74
    const ptrdiff_t onerow = ch.plane.PixelsPerRow();
75
    for (size_t y = 0; y < ch.h; y++) {
76
      const pixel_type* JXL_RESTRICT r = ch.Row(y);
77
      for (size_t x = 0; x < ch.w; x++) {
78
        pixel_type_w left = (x ? r[x - 1] : y ? *(r + x - onerow) : 0);
79
        pixel_type_w top = (y ? *(r + x - onerow) : left);
80
        pixel_type_w topleft = (x && y ? *(r + x - 1 - onerow) : left);
81
        size_t max_diff =
82
            std::max({left, top, topleft}) - std::min({left, top, topleft});
83
        size_t ctx = 0;
84
        for (uint32_t c : cutoffs) {
85
          ctx += (max_diff < c) ? 1 : 0;
86
        }
87
        pixel_type res = r[x] - ClampedGradient(top, left, topleft);
88
        uint32_t token;
89
        uint32_t nbits;
90
        uint32_t bits;
91
        config.Encode(PackSigned(res), &token, &nbits, &bits);
92
        histo[ctx].Add(token);
93
        extra_bits += nbits;
94
      }
95
    }
96
    for (auto& h : histo) {
97
      float f_cost = h.ShannonEntropy();
98
      size_t i_cost = f_cost;
99
      histo_cost += i_cost;
100
      histo_cost_frac += f_cost - i_cost;
101
      h.Clear();
102
    }
103
  }
104
#else
105
3.53k
  JxlMemoryManager* memory_manager = img.memory_manager();
106
3.53k
  const auto& ctx_map = estimate_cost_detail::ContextMap();
107
3.53k
  const HWY_FULL(int32_t) di;
108
3.53k
  const HWY_FULL(uint32_t) du;
109
3.53k
  const HWY_FULL(float) df;
110
3.53k
  const auto kOne = Set(du, 1);
111
3.53k
  const auto kSplit = Set(du, 16);
112
3.53k
  const auto kExpOffset2 = Set(du, 129);  // 127 + 2
113
3.53k
  const auto kTokenBias = Set(du, 8);
114
3.53k
  const auto kTokenMul = Set(du, 4);
115
3.53k
  const auto kMsbMask = Set(du, 3);
116
3.53k
  const auto kMaxDiffCap = Set(du, estimate_cost_detail::kLastThreshold - 1);
117
3.53k
  const auto kLanes = Set(du, Lanes(du));
118
3.53k
  const auto kIota = Iota(du, 0);
119
3.53k
  const auto kLargeThreshold = Set(du, (1 << 22) - 1);
120
3.53k
  constexpr size_t kLargeShiftVal = 10;
121
3.53k
  const auto kLargeShift = Set(du, kLargeShiftVal);
122
123
3.53k
  size_t max_w = 0;
124
3.53k
  for (const Channel& ch : img.channel) {
125
3.53k
    if (ch.h == 0) continue;
126
3.53k
    max_w = std::max(max_w, ch.w);
127
3.53k
  }
128
3.53k
  max_w = RoundUpTo(max_w, Lanes(du));
129
3.53k
  max_w = std::max(max_w, 2 * Lanes(du));
130
131
3.53k
  JXL_ASSIGN_OR_RETURN(
132
3.53k
      AlignedMemory buffer,
133
3.53k
      AlignedMemory::Create(memory_manager, max_w * 2 * sizeof(uint32_t)));
134
3.53k
  uint32_t* max_diff_row = buffer.address<uint32_t>();
135
3.53k
  uint32_t* token_row = max_diff_row + max_w;
136
3.53k
  int32_t* primer = buffer.address<int32_t>();
137
3.53k
  int32_t* top_primer = primer + max_w;
138
139
3.53k
  HybridUintConfig config;
140
141
3.53k
  Histogram histo[estimate_cost_detail::kLastCtx + 1] = {};
142
3.53k
  auto extra_bits_lanes = Zero(du);
143
3.53k
  for (const Channel& ch : img.channel) {
144
3.53k
    if (ch.h == 0 || ch.w == 0) continue;
145
60.0k
    for (auto& h : histo) {
146
60.0k
      h.EnsureCapacity(32 * 4);
147
60.0k
    }
148
3.53k
    const pixel_type* JXL_RESTRICT r = ch.Row(0);
149
3.53k
    const pixel_type* JXL_RESTRICT last = primer;
150
3.53k
    primer[0] = 0;
151
3.53k
    StoreU(Load(di, r), di, primer + 1);
152
3.53k
    auto pos = kIota;
153
3.53k
    const auto last_pos = Set(du, ch.w);
154
96.0k
    for (size_t x = 0; x < ch.w; x += Lanes(di)) {
155
92.5k
      const auto left = LoadU(di, last);
156
92.5k
      const auto central = Load(di, r + x);
157
92.5k
      const auto ures = BitCast(du, Sub(central, left));
158
92.5k
      const auto packed =
159
92.5k
          Xor(ShiftLeft<1>(ures), Sub(ShiftRight<31>(Not(ures)), kOne));
160
92.5k
      const auto is_large = Gt(packed, kLargeThreshold);
161
92.5k
      const auto packed_shifted = ShiftRight<kLargeShiftVal>(packed);
162
92.5k
      const auto not_literal = Ge(packed, kSplit);
163
92.5k
      const auto packed_fixed = IfThenElse(is_large, packed_shifted, packed);
164
92.5k
      const auto v = BitCast(du, ConvertTo(df, packed_fixed));
165
92.5k
      const auto eb_raw = Sub(ShiftRight<23>(v), kExpOffset2);
166
92.5k
      const auto eb = IfThenElse(is_large, Add(eb_raw, kLargeShift), eb_raw);
167
92.5k
      const auto token = Add(Add(kTokenBias, Mul(eb, kTokenMul)),
168
92.5k
                             And(ShiftRight<21>(v), kMsbMask));
169
92.5k
      const auto tail_mask = Lt(pos, last_pos);
170
92.5k
      const auto eb_fixed = IfThenElseZero(not_literal, eb);
171
92.5k
      const auto token_fixed = IfThenElse(not_literal, token, packed);
172
92.5k
      extra_bits_lanes =
173
92.5k
          Add(extra_bits_lanes, IfThenElseZero(tail_mask, eb_fixed));
174
92.5k
      Store(token_fixed, du, token_row + x);
175
92.5k
      pos = Add(pos, kLanes);
176
92.5k
      last = r + x + Lanes(di) - 1;
177
92.5k
    }
178
738k
    for (size_t x = 0; x < ch.w; x++) {
179
734k
      histo[0].FastAdd(token_row[x]);
180
734k
    }
181
753k
    for (size_t y = 1; y < ch.h; y++) {
182
749k
      r = ch.Row(y);
183
749k
      const pixel_type* JXL_RESTRICT t = ch.Row(y - 1);
184
749k
      last = primer;
185
749k
      primer[0] = t[0];
186
749k
      StoreU(Load(di, r), di, primer + 1);
187
749k
      top_primer[0] = t[0];
188
749k
      StoreU(Load(di, t), di, top_primer + 1);
189
749k
      const pixel_type* JXL_RESTRICT top_last = top_primer;
190
749k
      pos = kIota;
191
23.9M
      for (size_t x = 0; x < ch.w; x += Lanes(di)) {
192
23.2M
        const auto left = LoadU(di, last);
193
23.2M
        const auto central = Load(di, r + x);
194
23.2M
        const auto topleft = LoadU(di, top_last);
195
23.2M
        const auto top = Load(di, t + x);
196
23.2M
        const auto l_ge_t = Ge(left, top);
197
23.2M
        const auto m = IfThenElse(l_ge_t, top, left);
198
23.2M
        const auto M = IfThenElse(l_ge_t, left, top);
199
23.2M
        const auto maxx = Max(topleft, M);
200
23.2M
        const auto minn = Min(topleft, m);
201
23.2M
        const auto max_diff = BitCast(du, Sub(maxx, minn));
202
23.2M
        Store(Min(max_diff, kMaxDiffCap), du, max_diff_row + x);
203
23.2M
        const auto overshoot = Lt(topleft, m);
204
23.2M
        const auto undershoot = Gt(topleft, M);
205
23.2M
        const auto grad =
206
23.2M
            BitCast(di, Sub(Add(BitCast(du, top), BitCast(du, left)),
207
23.2M
                            BitCast(du, topleft)));
208
23.2M
        const auto prediction =
209
23.2M
            IfThenElse(undershoot, m, IfThenElse(overshoot, M, grad));
210
23.2M
        const auto ures = BitCast(du, Sub(central, prediction));
211
23.2M
        const auto packed =
212
23.2M
            Xor(ShiftLeft<1>(ures), Sub(ShiftRight<31>(Not(ures)), kOne));
213
23.2M
        const auto is_large = Gt(packed, kLargeThreshold);
214
23.2M
        const auto packed_shifted = ShiftRight<kLargeShiftVal>(packed);
215
23.2M
        const auto not_literal = Ge(packed, kSplit);
216
23.2M
        const auto packed_fixed = IfThenElse(is_large, packed_shifted, packed);
217
23.2M
        const auto v = BitCast(du, ConvertTo(df, packed_fixed));
218
23.2M
        const auto eb_raw = Sub(ShiftRight<23>(v), kExpOffset2);
219
23.2M
        const auto eb = IfThenElse(is_large, Add(eb_raw, kLargeShift), eb_raw);
220
23.2M
        const auto token = Add(Add(kTokenBias, Mul(eb, kTokenMul)),
221
23.2M
                               And(ShiftRight<21>(v), kMsbMask));
222
23.2M
        const auto tail_mask = Lt(pos, last_pos);
223
23.2M
        const auto eb_fixed = IfThenElseZero(not_literal, eb);
224
23.2M
        const auto token_fixed = IfThenElse(not_literal, token, packed);
225
23.2M
        extra_bits_lanes =
226
23.2M
            Add(extra_bits_lanes, IfThenElseZero(tail_mask, eb_fixed));
227
23.2M
        Store(token_fixed, du, token_row + x);
228
23.2M
        pos = Add(pos, kLanes);
229
23.2M
        last = r + x + Lanes(di) - 1;
230
23.2M
        top_last = t + x + Lanes(di) - 1;
231
23.2M
      }
232
185M
      for (size_t x = 0; x < ch.w; x++) {
233
184M
        size_t ctx = ctx_map[max_diff_row[x]];
234
184M
        histo[ctx].FastAdd(token_row[x]);
235
184M
      }
236
749k
    }
237
60.0k
    for (auto& h : histo) {
238
60.0k
      h.Condition();
239
60.0k
      float f_cost = h.ShannonEntropy();
240
60.0k
      size_t i_cost = f_cost;
241
60.0k
      histo_cost += i_cost;
242
60.0k
      histo_cost_frac += f_cost - i_cost;
243
60.0k
      h.Clear();
244
60.0k
    }
245
3.53k
  }
246
3.53k
  extra_bits = GetLane(SumOfLanes(du, extra_bits_lanes));
247
3.53k
#endif
248
3.53k
  size_t total_cost =
249
3.53k
      extra_bits + histo_cost + static_cast<size_t>(histo_cost_frac);
250
3.53k
  return total_cost;
251
3.53k
}
Unexecuted instantiation: jxl::N_SSE2::EstimateCost(jxl::Image const&)
252
253
// NOLINTNEXTLINE(google-readability-namespace-comments)
254
}  // namespace HWY_NAMESPACE
255
}  // namespace jxl
256
HWY_AFTER_NAMESPACE();
257
258
#if HWY_ONCE
259
namespace jxl {
260
261
HWY_EXPORT(EstimateCost);
262
263
3.53k
StatusOr<float> EstimateCost(const Image& img) {
264
3.53k
  return HWY_DYNAMIC_DISPATCH(EstimateCost)(img);
265
3.53k
}
266
267
namespace estimate_cost_detail {
268
/*
269
cutoffs = [0, 1, 3, 5, 7, 11, 15, 23, 31, 47, 63, 95, 127, 191, 255, 392, 500]
270
ctx_map = [[c for c,v in enumerate(cutoffs) if v <= i][0] for i in range(501)]
271
*/
272
3.53k
const std::array<uint8_t, kLastThreshold>& ContextMap() {
273
3.53k
  static const std::array<uint8_t, kLastThreshold> kCtxMap = {
274
3.53k
      0,  1,  1,  2,  2,  3,  3,  4,  4,  4,  4,  5,  5,  5,  5,  6,  6,  6,
275
3.53k
      6,  6,  6,  6,  6,  7,  7,  7,  7,  7,  7,  7,  7,  8,  8,  8,  8,  8,
276
3.53k
      8,  8,  8,  8,  8,  8,  8,  8,  8,  8,  8,  9,  9,  9,  9,  9,  9,  9,
277
3.53k
      9,  9,  9,  9,  9,  9,  9,  9,  9,  10, 10, 10, 10, 10, 10, 10, 10, 10,
278
3.53k
      10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,
279
3.53k
      10, 10, 10, 10, 10, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
280
3.53k
      11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
281
3.53k
      11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
282
3.53k
      12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
283
3.53k
      12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
284
3.53k
      12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 13, 13, 13, 13, 13, 13, 13,
285
3.53k
      13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
286
3.53k
      13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
287
3.53k
      13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
288
3.53k
      13, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
289
3.53k
      14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
290
3.53k
      14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
291
3.53k
      14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
292
3.53k
      14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
293
3.53k
      14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
294
3.53k
      14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
295
3.53k
      14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 15, 15, 15, 15,
296
3.53k
      15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
297
3.53k
      15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
298
3.53k
      15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
299
3.53k
      15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
300
3.53k
      15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
301
3.53k
      15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 16};
302
3.53k
  return kCtxMap;
303
3.53k
}
304
}  // namespace estimate_cost_detail
305
306
}  // namespace jxl
307
#endif