Coverage Report

Created: 2026-11-12 07:35

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/libjxl/lib/jxl/enc_modular_simd.cc
Line
Count
Source
1
// Copyright (c) the JPEG XL Project Authors. All rights reserved.
2
//
3
// Use of this source code is governed by a BSD-style
4
// license that can be found in the LICENSE file.
5
6
#include "lib/jxl/enc_modular_simd.h"
7
8
#include <jxl/memory_manager.h>
9
10
#include <algorithm>
11
#include <array>
12
#include <cstddef>
13
#include <cstdint>
14
15
#include "lib/jxl/base/common.h"
16
#include "lib/jxl/base/compiler_specific.h"
17
#include "lib/jxl/base/status.h"
18
#include "lib/jxl/dec_ans.h"
19
#include "lib/jxl/enc_ans_params.h"
20
#include "lib/jxl/memory_manager_internal.h"
21
#include "lib/jxl/modular/modular_image.h"
22
23
#undef HWY_TARGET_INCLUDE
24
#define HWY_TARGET_INCLUDE "lib/jxl/enc_modular_simd.cc"
25
#include <hwy/foreach_target.h>
26
#include <hwy/highway.h>
27
28
#if HWY_TARGET == HWY_SCALAR
29
#include "lib/jxl/modular/encoding/context_predict.h"
30
#include "lib/jxl/pack_signed.h"
31
#endif
32
33
HWY_BEFORE_NAMESPACE();
34
namespace jxl {
35
namespace HWY_NAMESPACE {
36
37
// These templates are not found via ADL.
38
using hwy::HWY_NAMESPACE::Add;
39
using hwy::HWY_NAMESPACE::And;
40
using hwy::HWY_NAMESPACE::Ge;
41
using hwy::HWY_NAMESPACE::GetLane;
42
using hwy::HWY_NAMESPACE::Gt;
43
using hwy::HWY_NAMESPACE::IfThenElse;
44
using hwy::HWY_NAMESPACE::IfThenElseZero;
45
using hwy::HWY_NAMESPACE::Iota;
46
using hwy::HWY_NAMESPACE::Load;
47
using hwy::HWY_NAMESPACE::LoadU;
48
using hwy::HWY_NAMESPACE::Lt;
49
using hwy::HWY_NAMESPACE::Max;
50
using hwy::HWY_NAMESPACE::Min;
51
using hwy::HWY_NAMESPACE::Mul;
52
using hwy::HWY_NAMESPACE::Not;
53
using hwy::HWY_NAMESPACE::Set;
54
using hwy::HWY_NAMESPACE::ShiftLeft;
55
using hwy::HWY_NAMESPACE::ShiftRight;
56
using hwy::HWY_NAMESPACE::Store;
57
using hwy::HWY_NAMESPACE::StoreU;
58
using hwy::HWY_NAMESPACE::Sub;
59
using hwy::HWY_NAMESPACE::Xor;
60
using hwy::HWY_NAMESPACE::Zero;
61
62
3.40k
StatusOr<float> EstimateCost(const Image& img) {
63
3.40k
  size_t histo_cost = 0;
64
3.40k
  float histo_cost_frac = 0.0f;
65
3.40k
  size_t extra_bits = 0;
66
67
#if HWY_TARGET == HWY_SCALAR
68
  HybridUintConfig config;
69
  uint32_t cutoffs[] = {0,  1,  3,  5,   7,   11,  15,  23, 31,
70
                        47, 63, 95, 127, 191, 255, 392, 500};
71
  constexpr size_t nc = sizeof(cutoffs) / sizeof(*cutoffs) + 1;
72
  Histogram histo[nc] = {};
73
  for (const Channel& ch : img.channel) {
74
    const ptrdiff_t onerow = ch.plane.PixelsPerRow();
75
    for (size_t y = 0; y < ch.h; y++) {
76
      const pixel_type* JXL_RESTRICT r = ch.Row(y);
77
      for (size_t x = 0; x < ch.w; x++) {
78
        pixel_type_w left = (x ? r[x - 1] : y ? *(r + x - onerow) : 0);
79
        pixel_type_w top = (y ? *(r + x - onerow) : left);
80
        pixel_type_w topleft = (x && y ? *(r + x - 1 - onerow) : left);
81
        size_t max_diff =
82
            std::max({left, top, topleft}) - std::min({left, top, topleft});
83
        size_t ctx = 0;
84
        for (uint32_t c : cutoffs) {
85
          ctx += (max_diff < c) ? 1 : 0;
86
        }
87
        pixel_type res = r[x] - ClampedGradient(top, left, topleft);
88
        uint32_t token;
89
        uint32_t nbits;
90
        uint32_t bits;
91
        config.Encode(PackSigned(res), &token, &nbits, &bits);
92
        histo[ctx].Add(token);
93
        extra_bits += nbits;
94
      }
95
    }
96
    for (auto& h : histo) {
97
      float f_cost = h.ShannonEntropy();
98
      size_t i_cost = f_cost;
99
      histo_cost += i_cost;
100
      histo_cost_frac += f_cost - i_cost;
101
      h.Clear();
102
    }
103
  }
104
#else
105
3.40k
  JxlMemoryManager* memory_manager = img.memory_manager();
106
3.40k
  const auto& ctx_map = estimate_cost_detail::ContextMap();
107
3.40k
  const HWY_FULL(int32_t) di;
108
3.40k
  const HWY_FULL(uint32_t) du;
109
3.40k
  const HWY_FULL(float) df;
110
3.40k
  const auto kOne = Set(du, 1);
111
3.40k
  const auto kSplit = Set(du, 16);
112
3.40k
  const auto kExpOffset2 = Set(du, 129);  // 127 + 2
113
3.40k
  const auto kTokenBias = Set(du, 8);
114
3.40k
  const auto kTokenMul = Set(du, 4);
115
3.40k
  const auto kMsbMask = Set(du, 3);
116
3.40k
  const auto kMaxDiffCap = Set(du, estimate_cost_detail::kLastThreshold - 1);
117
3.40k
  const auto kLanes = Set(du, Lanes(du));
118
3.40k
  const auto kIota = Iota(du, 0);
119
3.40k
  const auto kLargeThreshold = Set(du, (1 << 22) - 1);
120
3.40k
  constexpr size_t kLargeShiftVal = 10;
121
3.40k
  const auto kLargeShift = Set(du, kLargeShiftVal);
122
123
3.40k
  size_t max_w = 0;
124
3.40k
  for (const Channel& ch : img.channel) {
125
3.40k
    if (ch.h == 0) continue;
126
3.40k
    max_w = std::max(max_w, ch.w);
127
3.40k
  }
128
3.40k
  max_w = RoundUpTo(max_w, Lanes(du));
129
3.40k
  max_w = std::max(max_w, 2 * Lanes(du));
130
131
3.40k
  JXL_ASSIGN_OR_RETURN(
132
3.40k
      AlignedMemory buffer,
133
3.40k
      AlignedMemory::Create(memory_manager, max_w * 2 * sizeof(uint32_t)));
134
3.40k
  uint32_t* max_diff_row = buffer.address<uint32_t>();
135
3.40k
  uint32_t* token_row = max_diff_row + max_w;
136
3.40k
  int32_t* primer = buffer.address<int32_t>();
137
3.40k
  int32_t* top_primer = primer + max_w;
138
139
3.40k
  HybridUintConfig config;
140
141
3.40k
  Histogram histo[estimate_cost_detail::kLastCtx + 1] = {};
142
3.40k
  auto extra_bits_lanes = Zero(du);
143
3.40k
  for (const Channel& ch : img.channel) {
144
3.40k
    if (ch.h == 0 || ch.w == 0) continue;
145
57.9k
    for (auto& h : histo) {
146
57.9k
      h.EnsureCapacity(32 * 4);
147
57.9k
    }
148
3.40k
    const pixel_type* JXL_RESTRICT r = ch.Row(0);
149
3.40k
    const pixel_type* JXL_RESTRICT last = primer;
150
3.40k
    primer[0] = 0;
151
3.40k
    StoreU(Load(di, r), di, primer + 1);
152
3.40k
    auto pos = kIota;
153
3.40k
    const auto last_pos = Set(du, ch.w);
154
92.4k
    for (size_t x = 0; x < ch.w; x += Lanes(di)) {
155
89.0k
      const auto left = LoadU(di, last);
156
89.0k
      const auto central = Load(di, r + x);
157
89.0k
      const auto ures = BitCast(du, Sub(central, left));
158
89.0k
      const auto packed =
159
89.0k
          Xor(ShiftLeft<1>(ures), Sub(ShiftRight<31>(Not(ures)), kOne));
160
89.0k
      const auto is_large = Gt(packed, kLargeThreshold);
161
89.0k
      const auto packed_shifted = ShiftRight<kLargeShiftVal>(packed);
162
89.0k
      const auto not_literal = Ge(packed, kSplit);
163
89.0k
      const auto packed_fixed = IfThenElse(is_large, packed_shifted, packed);
164
89.0k
      const auto v = BitCast(du, ConvertTo(df, packed_fixed));
165
89.0k
      const auto eb_raw = Sub(ShiftRight<23>(v), kExpOffset2);
166
89.0k
      const auto eb = IfThenElse(is_large, Add(eb_raw, kLargeShift), eb_raw);
167
89.0k
      const auto token = Add(Add(kTokenBias, Mul(eb, kTokenMul)),
168
89.0k
                             And(ShiftRight<21>(v), kMsbMask));
169
89.0k
      const auto tail_mask = Lt(pos, last_pos);
170
89.0k
      const auto eb_fixed = IfThenElseZero(not_literal, eb);
171
89.0k
      const auto token_fixed = IfThenElse(not_literal, token, packed);
172
89.0k
      extra_bits_lanes =
173
89.0k
          Add(extra_bits_lanes, IfThenElseZero(tail_mask, eb_fixed));
174
89.0k
      Store(token_fixed, du, token_row + x);
175
89.0k
      pos = Add(pos, kLanes);
176
89.0k
      last = r + x + Lanes(di) - 1;
177
89.0k
    }
178
709k
    for (size_t x = 0; x < ch.w; x++) {
179
706k
      histo[0].FastAdd(token_row[x]);
180
706k
    }
181
723k
    for (size_t y = 1; y < ch.h; y++) {
182
719k
      r = ch.Row(y);
183
719k
      const pixel_type* JXL_RESTRICT t = ch.Row(y - 1);
184
719k
      last = primer;
185
719k
      primer[0] = t[0];
186
719k
      StoreU(Load(di, r), di, primer + 1);
187
719k
      top_primer[0] = t[0];
188
719k
      StoreU(Load(di, t), di, top_primer + 1);
189
719k
      const pixel_type* JXL_RESTRICT top_last = top_primer;
190
719k
      pos = kIota;
191
23.0M
      for (size_t x = 0; x < ch.w; x += Lanes(di)) {
192
22.3M
        const auto left = LoadU(di, last);
193
22.3M
        const auto central = Load(di, r + x);
194
22.3M
        const auto topleft = LoadU(di, top_last);
195
22.3M
        const auto top = Load(di, t + x);
196
22.3M
        const auto l_ge_t = Ge(left, top);
197
22.3M
        const auto m = IfThenElse(l_ge_t, top, left);
198
22.3M
        const auto M = IfThenElse(l_ge_t, left, top);
199
22.3M
        const auto maxx = Max(topleft, M);
200
22.3M
        const auto minn = Min(topleft, m);
201
22.3M
        const auto max_diff = BitCast(du, Sub(maxx, minn));
202
22.3M
        Store(Min(max_diff, kMaxDiffCap), du, max_diff_row + x);
203
22.3M
        const auto overshoot = Lt(topleft, m);
204
22.3M
        const auto undershoot = Gt(topleft, M);
205
22.3M
        const auto grad =
206
22.3M
            BitCast(di, Sub(Add(BitCast(du, top), BitCast(du, left)),
207
22.3M
                            BitCast(du, topleft)));
208
22.3M
        const auto prediction =
209
22.3M
            IfThenElse(undershoot, m, IfThenElse(overshoot, M, grad));
210
22.3M
        const auto ures = BitCast(du, Sub(central, prediction));
211
22.3M
        const auto packed =
212
22.3M
            Xor(ShiftLeft<1>(ures), Sub(ShiftRight<31>(Not(ures)), kOne));
213
22.3M
        const auto is_large = Gt(packed, kLargeThreshold);
214
22.3M
        const auto packed_shifted = ShiftRight<kLargeShiftVal>(packed);
215
22.3M
        const auto not_literal = Ge(packed, kSplit);
216
22.3M
        const auto packed_fixed = IfThenElse(is_large, packed_shifted, packed);
217
22.3M
        const auto v = BitCast(du, ConvertTo(df, packed_fixed));
218
22.3M
        const auto eb_raw = Sub(ShiftRight<23>(v), kExpOffset2);
219
22.3M
        const auto eb = IfThenElse(is_large, Add(eb_raw, kLargeShift), eb_raw);
220
22.3M
        const auto token = Add(Add(kTokenBias, Mul(eb, kTokenMul)),
221
22.3M
                               And(ShiftRight<21>(v), kMsbMask));
222
22.3M
        const auto tail_mask = Lt(pos, last_pos);
223
22.3M
        const auto eb_fixed = IfThenElseZero(not_literal, eb);
224
22.3M
        const auto token_fixed = IfThenElse(not_literal, token, packed);
225
22.3M
        extra_bits_lanes =
226
22.3M
            Add(extra_bits_lanes, IfThenElseZero(tail_mask, eb_fixed));
227
22.3M
        Store(token_fixed, du, token_row + x);
228
22.3M
        pos = Add(pos, kLanes);
229
22.3M
        last = r + x + Lanes(di) - 1;
230
22.3M
        top_last = t + x + Lanes(di) - 1;
231
22.3M
      }
232
177M
      for (size_t x = 0; x < ch.w; x++) {
233
177M
        size_t ctx = ctx_map[max_diff_row[x]];
234
177M
        histo[ctx].FastAdd(token_row[x]);
235
177M
      }
236
719k
    }
237
57.9k
    for (auto& h : histo) {
238
57.9k
      h.Condition();
239
57.9k
      float f_cost = h.ShannonEntropy();
240
57.9k
      size_t i_cost = f_cost;
241
57.9k
      histo_cost += i_cost;
242
57.9k
      histo_cost_frac += f_cost - i_cost;
243
57.9k
      h.Clear();
244
57.9k
    }
245
3.40k
  }
246
3.40k
  extra_bits = GetLane(SumOfLanes(du, extra_bits_lanes));
247
3.40k
#endif
248
3.40k
  size_t total_cost =
249
3.40k
      extra_bits + histo_cost + static_cast<size_t>(histo_cost_frac);
250
3.40k
  return total_cost;
251
3.40k
}
Unexecuted instantiation: jxl::N_SSE4::EstimateCost(jxl::Image const&)
jxl::N_AVX2::EstimateCost(jxl::Image const&)
Line
Count
Source
62
3.40k
StatusOr<float> EstimateCost(const Image& img) {
63
3.40k
  size_t histo_cost = 0;
64
3.40k
  float histo_cost_frac = 0.0f;
65
3.40k
  size_t extra_bits = 0;
66
67
#if HWY_TARGET == HWY_SCALAR
68
  HybridUintConfig config;
69
  uint32_t cutoffs[] = {0,  1,  3,  5,   7,   11,  15,  23, 31,
70
                        47, 63, 95, 127, 191, 255, 392, 500};
71
  constexpr size_t nc = sizeof(cutoffs) / sizeof(*cutoffs) + 1;
72
  Histogram histo[nc] = {};
73
  for (const Channel& ch : img.channel) {
74
    const ptrdiff_t onerow = ch.plane.PixelsPerRow();
75
    for (size_t y = 0; y < ch.h; y++) {
76
      const pixel_type* JXL_RESTRICT r = ch.Row(y);
77
      for (size_t x = 0; x < ch.w; x++) {
78
        pixel_type_w left = (x ? r[x - 1] : y ? *(r + x - onerow) : 0);
79
        pixel_type_w top = (y ? *(r + x - onerow) : left);
80
        pixel_type_w topleft = (x && y ? *(r + x - 1 - onerow) : left);
81
        size_t max_diff =
82
            std::max({left, top, topleft}) - std::min({left, top, topleft});
83
        size_t ctx = 0;
84
        for (uint32_t c : cutoffs) {
85
          ctx += (max_diff < c) ? 1 : 0;
86
        }
87
        pixel_type res = r[x] - ClampedGradient(top, left, topleft);
88
        uint32_t token;
89
        uint32_t nbits;
90
        uint32_t bits;
91
        config.Encode(PackSigned(res), &token, &nbits, &bits);
92
        histo[ctx].Add(token);
93
        extra_bits += nbits;
94
      }
95
    }
96
    for (auto& h : histo) {
97
      float f_cost = h.ShannonEntropy();
98
      size_t i_cost = f_cost;
99
      histo_cost += i_cost;
100
      histo_cost_frac += f_cost - i_cost;
101
      h.Clear();
102
    }
103
  }
104
#else
105
3.40k
  JxlMemoryManager* memory_manager = img.memory_manager();
106
3.40k
  const auto& ctx_map = estimate_cost_detail::ContextMap();
107
3.40k
  const HWY_FULL(int32_t) di;
108
3.40k
  const HWY_FULL(uint32_t) du;
109
3.40k
  const HWY_FULL(float) df;
110
3.40k
  const auto kOne = Set(du, 1);
111
3.40k
  const auto kSplit = Set(du, 16);
112
3.40k
  const auto kExpOffset2 = Set(du, 129);  // 127 + 2
113
3.40k
  const auto kTokenBias = Set(du, 8);
114
3.40k
  const auto kTokenMul = Set(du, 4);
115
3.40k
  const auto kMsbMask = Set(du, 3);
116
3.40k
  const auto kMaxDiffCap = Set(du, estimate_cost_detail::kLastThreshold - 1);
117
3.40k
  const auto kLanes = Set(du, Lanes(du));
118
3.40k
  const auto kIota = Iota(du, 0);
119
3.40k
  const auto kLargeThreshold = Set(du, (1 << 22) - 1);
120
3.40k
  constexpr size_t kLargeShiftVal = 10;
121
3.40k
  const auto kLargeShift = Set(du, kLargeShiftVal);
122
123
3.40k
  size_t max_w = 0;
124
3.40k
  for (const Channel& ch : img.channel) {
125
3.40k
    if (ch.h == 0) continue;
126
3.40k
    max_w = std::max(max_w, ch.w);
127
3.40k
  }
128
3.40k
  max_w = RoundUpTo(max_w, Lanes(du));
129
3.40k
  max_w = std::max(max_w, 2 * Lanes(du));
130
131
3.40k
  JXL_ASSIGN_OR_RETURN(
132
3.40k
      AlignedMemory buffer,
133
3.40k
      AlignedMemory::Create(memory_manager, max_w * 2 * sizeof(uint32_t)));
134
3.40k
  uint32_t* max_diff_row = buffer.address<uint32_t>();
135
3.40k
  uint32_t* token_row = max_diff_row + max_w;
136
3.40k
  int32_t* primer = buffer.address<int32_t>();
137
3.40k
  int32_t* top_primer = primer + max_w;
138
139
3.40k
  HybridUintConfig config;
140
141
3.40k
  Histogram histo[estimate_cost_detail::kLastCtx + 1] = {};
142
3.40k
  auto extra_bits_lanes = Zero(du);
143
3.40k
  for (const Channel& ch : img.channel) {
144
3.40k
    if (ch.h == 0 || ch.w == 0) continue;
145
57.9k
    for (auto& h : histo) {
146
57.9k
      h.EnsureCapacity(32 * 4);
147
57.9k
    }
148
3.40k
    const pixel_type* JXL_RESTRICT r = ch.Row(0);
149
3.40k
    const pixel_type* JXL_RESTRICT last = primer;
150
3.40k
    primer[0] = 0;
151
3.40k
    StoreU(Load(di, r), di, primer + 1);
152
3.40k
    auto pos = kIota;
153
3.40k
    const auto last_pos = Set(du, ch.w);
154
92.4k
    for (size_t x = 0; x < ch.w; x += Lanes(di)) {
155
89.0k
      const auto left = LoadU(di, last);
156
89.0k
      const auto central = Load(di, r + x);
157
89.0k
      const auto ures = BitCast(du, Sub(central, left));
158
89.0k
      const auto packed =
159
89.0k
          Xor(ShiftLeft<1>(ures), Sub(ShiftRight<31>(Not(ures)), kOne));
160
89.0k
      const auto is_large = Gt(packed, kLargeThreshold);
161
89.0k
      const auto packed_shifted = ShiftRight<kLargeShiftVal>(packed);
162
89.0k
      const auto not_literal = Ge(packed, kSplit);
163
89.0k
      const auto packed_fixed = IfThenElse(is_large, packed_shifted, packed);
164
89.0k
      const auto v = BitCast(du, ConvertTo(df, packed_fixed));
165
89.0k
      const auto eb_raw = Sub(ShiftRight<23>(v), kExpOffset2);
166
89.0k
      const auto eb = IfThenElse(is_large, Add(eb_raw, kLargeShift), eb_raw);
167
89.0k
      const auto token = Add(Add(kTokenBias, Mul(eb, kTokenMul)),
168
89.0k
                             And(ShiftRight<21>(v), kMsbMask));
169
89.0k
      const auto tail_mask = Lt(pos, last_pos);
170
89.0k
      const auto eb_fixed = IfThenElseZero(not_literal, eb);
171
89.0k
      const auto token_fixed = IfThenElse(not_literal, token, packed);
172
89.0k
      extra_bits_lanes =
173
89.0k
          Add(extra_bits_lanes, IfThenElseZero(tail_mask, eb_fixed));
174
89.0k
      Store(token_fixed, du, token_row + x);
175
89.0k
      pos = Add(pos, kLanes);
176
89.0k
      last = r + x + Lanes(di) - 1;
177
89.0k
    }
178
709k
    for (size_t x = 0; x < ch.w; x++) {
179
706k
      histo[0].FastAdd(token_row[x]);
180
706k
    }
181
723k
    for (size_t y = 1; y < ch.h; y++) {
182
719k
      r = ch.Row(y);
183
719k
      const pixel_type* JXL_RESTRICT t = ch.Row(y - 1);
184
719k
      last = primer;
185
719k
      primer[0] = t[0];
186
719k
      StoreU(Load(di, r), di, primer + 1);
187
719k
      top_primer[0] = t[0];
188
719k
      StoreU(Load(di, t), di, top_primer + 1);
189
719k
      const pixel_type* JXL_RESTRICT top_last = top_primer;
190
719k
      pos = kIota;
191
23.0M
      for (size_t x = 0; x < ch.w; x += Lanes(di)) {
192
22.3M
        const auto left = LoadU(di, last);
193
22.3M
        const auto central = Load(di, r + x);
194
22.3M
        const auto topleft = LoadU(di, top_last);
195
22.3M
        const auto top = Load(di, t + x);
196
22.3M
        const auto l_ge_t = Ge(left, top);
197
22.3M
        const auto m = IfThenElse(l_ge_t, top, left);
198
22.3M
        const auto M = IfThenElse(l_ge_t, left, top);
199
22.3M
        const auto maxx = Max(topleft, M);
200
22.3M
        const auto minn = Min(topleft, m);
201
22.3M
        const auto max_diff = BitCast(du, Sub(maxx, minn));
202
22.3M
        Store(Min(max_diff, kMaxDiffCap), du, max_diff_row + x);
203
22.3M
        const auto overshoot = Lt(topleft, m);
204
22.3M
        const auto undershoot = Gt(topleft, M);
205
22.3M
        const auto grad =
206
22.3M
            BitCast(di, Sub(Add(BitCast(du, top), BitCast(du, left)),
207
22.3M
                            BitCast(du, topleft)));
208
22.3M
        const auto prediction =
209
22.3M
            IfThenElse(undershoot, m, IfThenElse(overshoot, M, grad));
210
22.3M
        const auto ures = BitCast(du, Sub(central, prediction));
211
22.3M
        const auto packed =
212
22.3M
            Xor(ShiftLeft<1>(ures), Sub(ShiftRight<31>(Not(ures)), kOne));
213
22.3M
        const auto is_large = Gt(packed, kLargeThreshold);
214
22.3M
        const auto packed_shifted = ShiftRight<kLargeShiftVal>(packed);
215
22.3M
        const auto not_literal = Ge(packed, kSplit);
216
22.3M
        const auto packed_fixed = IfThenElse(is_large, packed_shifted, packed);
217
22.3M
        const auto v = BitCast(du, ConvertTo(df, packed_fixed));
218
22.3M
        const auto eb_raw = Sub(ShiftRight<23>(v), kExpOffset2);
219
22.3M
        const auto eb = IfThenElse(is_large, Add(eb_raw, kLargeShift), eb_raw);
220
22.3M
        const auto token = Add(Add(kTokenBias, Mul(eb, kTokenMul)),
221
22.3M
                               And(ShiftRight<21>(v), kMsbMask));
222
22.3M
        const auto tail_mask = Lt(pos, last_pos);
223
22.3M
        const auto eb_fixed = IfThenElseZero(not_literal, eb);
224
22.3M
        const auto token_fixed = IfThenElse(not_literal, token, packed);
225
22.3M
        extra_bits_lanes =
226
22.3M
            Add(extra_bits_lanes, IfThenElseZero(tail_mask, eb_fixed));
227
22.3M
        Store(token_fixed, du, token_row + x);
228
22.3M
        pos = Add(pos, kLanes);
229
22.3M
        last = r + x + Lanes(di) - 1;
230
22.3M
        top_last = t + x + Lanes(di) - 1;
231
22.3M
      }
232
177M
      for (size_t x = 0; x < ch.w; x++) {
233
177M
        size_t ctx = ctx_map[max_diff_row[x]];
234
177M
        histo[ctx].FastAdd(token_row[x]);
235
177M
      }
236
719k
    }
237
57.9k
    for (auto& h : histo) {
238
57.9k
      h.Condition();
239
57.9k
      float f_cost = h.ShannonEntropy();
240
57.9k
      size_t i_cost = f_cost;
241
57.9k
      histo_cost += i_cost;
242
57.9k
      histo_cost_frac += f_cost - i_cost;
243
57.9k
      h.Clear();
244
57.9k
    }
245
3.40k
  }
246
3.40k
  extra_bits = GetLane(SumOfLanes(du, extra_bits_lanes));
247
3.40k
#endif
248
3.40k
  size_t total_cost =
249
3.40k
      extra_bits + histo_cost + static_cast<size_t>(histo_cost_frac);
250
3.40k
  return total_cost;
251
3.40k
}
Unexecuted instantiation: jxl::N_SSE2::EstimateCost(jxl::Image const&)
252
253
// NOLINTNEXTLINE(google-readability-namespace-comments)
254
}  // namespace HWY_NAMESPACE
255
}  // namespace jxl
256
HWY_AFTER_NAMESPACE();
257
258
#if HWY_ONCE
259
namespace jxl {
260
261
HWY_EXPORT(EstimateCost);
262
263
3.40k
StatusOr<float> EstimateCost(const Image& img) {
264
3.40k
  return HWY_DYNAMIC_DISPATCH(EstimateCost)(img);
265
3.40k
}
266
267
namespace estimate_cost_detail {
268
/*
269
cutoffs = [0, 1, 3, 5, 7, 11, 15, 23, 31, 47, 63, 95, 127, 191, 255, 392, 500]
270
ctx_map = [[c for c,v in enumerate(cutoffs) if v <= i][0] for i in range(501)]
271
*/
272
3.40k
const std::array<uint8_t, kLastThreshold>& ContextMap() {
273
3.40k
  static const std::array<uint8_t, kLastThreshold> kCtxMap = {
274
3.40k
      0,  1,  1,  2,  2,  3,  3,  4,  4,  4,  4,  5,  5,  5,  5,  6,  6,  6,
275
3.40k
      6,  6,  6,  6,  6,  7,  7,  7,  7,  7,  7,  7,  7,  8,  8,  8,  8,  8,
276
3.40k
      8,  8,  8,  8,  8,  8,  8,  8,  8,  8,  8,  9,  9,  9,  9,  9,  9,  9,
277
3.40k
      9,  9,  9,  9,  9,  9,  9,  9,  9,  10, 10, 10, 10, 10, 10, 10, 10, 10,
278
3.40k
      10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,
279
3.40k
      10, 10, 10, 10, 10, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
280
3.40k
      11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
281
3.40k
      11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
282
3.40k
      12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
283
3.40k
      12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
284
3.40k
      12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 13, 13, 13, 13, 13, 13, 13,
285
3.40k
      13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
286
3.40k
      13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
287
3.40k
      13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
288
3.40k
      13, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
289
3.40k
      14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
290
3.40k
      14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
291
3.40k
      14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
292
3.40k
      14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
293
3.40k
      14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
294
3.40k
      14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
295
3.40k
      14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 15, 15, 15, 15,
296
3.40k
      15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
297
3.40k
      15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
298
3.40k
      15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
299
3.40k
      15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
300
3.40k
      15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
301
3.40k
      15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 16};
302
3.40k
  return kCtxMap;
303
3.40k
}
304
}  // namespace estimate_cost_detail
305
306
}  // namespace jxl
307
#endif