/src/libjxl/lib/jxl/compressed_dc.cc
Line | Count | Source |
1 | | // Copyright (c) the JPEG XL Project Authors. All rights reserved. |
2 | | // |
3 | | // Use of this source code is governed by a BSD-style |
4 | | // license that can be found in the LICENSE file. |
5 | | |
6 | | #include "lib/jxl/compressed_dc.h" |
7 | | |
8 | | #include <jxl/memory_manager.h> |
9 | | |
10 | | #include <algorithm> |
11 | | #include <cstdint> |
12 | | #include <cstdlib> |
13 | | #include <cstring> |
14 | | #include <vector> |
15 | | |
16 | | #include "lib/jxl/ac_context.h" |
17 | | #include "lib/jxl/frame_header.h" |
18 | | #include "lib/jxl/modular/modular_image.h" |
19 | | |
20 | | #undef HWY_TARGET_INCLUDE |
21 | | #define HWY_TARGET_INCLUDE "lib/jxl/compressed_dc.cc" |
22 | | #include <hwy/foreach_target.h> |
23 | | #include <hwy/highway.h> |
24 | | |
25 | | #include "lib/jxl/base/compiler_specific.h" |
26 | | #include "lib/jxl/base/data_parallel.h" |
27 | | #include "lib/jxl/base/rect.h" |
28 | | #include "lib/jxl/base/status.h" |
29 | | #include "lib/jxl/image.h" |
30 | | HWY_BEFORE_NAMESPACE(); |
31 | | namespace jxl { |
32 | | namespace HWY_NAMESPACE { |
33 | | |
34 | | using D = HWY_FULL(float); |
35 | | using DScalar = HWY_CAPPED(float, 1); |
36 | | |
37 | | // These templates are not found via ADL. |
38 | | using hwy::HWY_NAMESPACE::Abs; |
39 | | using hwy::HWY_NAMESPACE::Add; |
40 | | using hwy::HWY_NAMESPACE::Div; |
41 | | using hwy::HWY_NAMESPACE::Max; |
42 | | using hwy::HWY_NAMESPACE::Mul; |
43 | | using hwy::HWY_NAMESPACE::MulAdd; |
44 | | using hwy::HWY_NAMESPACE::Rebind; |
45 | | using hwy::HWY_NAMESPACE::Sub; |
46 | | using hwy::HWY_NAMESPACE::Vec; |
47 | | using hwy::HWY_NAMESPACE::ZeroIfNegative; |
48 | | |
49 | | // TODO(veluca): optimize constants. |
50 | | const float w1 = 0.20345139757231578f; |
51 | | const float w2 = 0.0334829185968739f; |
52 | | const float w0 = 1.0f - 4.0f * (w1 + w2); |
53 | | |
54 | | template <class V> |
55 | 9.16k | V MaxWorkaround(V a, V b) { |
56 | | #if (HWY_TARGET == HWY_AVX3) && HWY_COMPILER_CLANG <= 800 |
57 | | // Prevents "Do not know how to split the result of this operator" error |
58 | | return IfThenElse(a > b, a, b); |
59 | | #else |
60 | 9.16k | return Max(a, b); |
61 | 9.16k | #endif |
62 | 9.16k | } |
63 | | |
64 | | template <typename D> |
65 | | JXL_INLINE void ComputePixelChannel(const D d, const float dc_factor, |
66 | | const float* JXL_RESTRICT row_top, |
67 | | const float* JXL_RESTRICT row, |
68 | | const float* JXL_RESTRICT row_bottom, |
69 | | Vec<D>* JXL_RESTRICT mc, |
70 | | Vec<D>* JXL_RESTRICT sm, |
71 | 10.5k | Vec<D>* JXL_RESTRICT gap, size_t x) { |
72 | 10.5k | const auto tl = LoadU(d, row_top + x - 1); |
73 | 10.5k | const auto tc = Load(d, row_top + x); |
74 | 10.5k | const auto tr = LoadU(d, row_top + x + 1); |
75 | | |
76 | 10.5k | const auto ml = LoadU(d, row + x - 1); |
77 | 10.5k | *mc = Load(d, row + x); |
78 | 10.5k | const auto mr = LoadU(d, row + x + 1); |
79 | | |
80 | 10.5k | const auto bl = LoadU(d, row_bottom + x - 1); |
81 | 10.5k | const auto bc = Load(d, row_bottom + x); |
82 | 10.5k | const auto br = LoadU(d, row_bottom + x + 1); |
83 | | |
84 | 10.5k | const auto w_center = Set(d, w0); |
85 | 10.5k | const auto w_side = Set(d, w1); |
86 | 10.5k | const auto w_corner = Set(d, w2); |
87 | | |
88 | 10.5k | const auto corner = Add(Add(tl, tr), Add(bl, br)); |
89 | 10.5k | const auto side = Add(Add(ml, mr), Add(tc, bc)); |
90 | 10.5k | *sm = MulAdd(corner, w_corner, MulAdd(side, w_side, Mul(*mc, w_center))); |
91 | | |
92 | 10.5k | const auto dc_quant = Set(d, dc_factor); |
93 | 10.5k | *gap = MaxWorkaround(*gap, Abs(Div(Sub(*mc, *sm), dc_quant))); |
94 | 10.5k | } |
95 | | |
96 | | template <typename D> |
97 | | JXL_INLINE void ComputePixel( |
98 | | const float* JXL_RESTRICT dc_factors, |
99 | | const float* JXL_RESTRICT* JXL_RESTRICT rows_top, |
100 | | const float* JXL_RESTRICT* JXL_RESTRICT rows, |
101 | | const float* JXL_RESTRICT* JXL_RESTRICT rows_bottom, |
102 | 5.26k | float* JXL_RESTRICT* JXL_RESTRICT out_rows, size_t x) { |
103 | 5.26k | const D d; |
104 | 5.26k | auto mc_x = Undefined(d); |
105 | 5.26k | auto mc_y = Undefined(d); |
106 | 5.26k | auto mc_b = Undefined(d); |
107 | 5.26k | auto sm_x = Undefined(d); |
108 | 5.26k | auto sm_y = Undefined(d); |
109 | 5.26k | auto sm_b = Undefined(d); |
110 | 5.26k | auto gap = Set(d, 0.5f); |
111 | 5.26k | ComputePixelChannel(d, dc_factors[0], rows_top[0], rows[0], rows_bottom[0], |
112 | 5.26k | &mc_x, &sm_x, &gap, x); |
113 | 5.26k | ComputePixelChannel(d, dc_factors[1], rows_top[1], rows[1], rows_bottom[1], |
114 | 5.26k | &mc_y, &sm_y, &gap, x); |
115 | 5.26k | ComputePixelChannel(d, dc_factors[2], rows_top[2], rows[2], rows_bottom[2], |
116 | 5.26k | &mc_b, &sm_b, &gap, x); |
117 | 5.26k | auto factor = MulAdd(Set(d, -4.0f), gap, Set(d, 3.0f)); |
118 | 5.26k | factor = ZeroIfNegative(factor); |
119 | | |
120 | 5.26k | auto out = MulAdd(Sub(sm_x, mc_x), factor, mc_x); |
121 | 5.26k | Store(out, d, out_rows[0] + x); |
122 | 5.26k | out = MulAdd(Sub(sm_y, mc_y), factor, mc_y); |
123 | 5.26k | Store(out, d, out_rows[1] + x); |
124 | 5.26k | out = MulAdd(Sub(sm_b, mc_b), factor, mc_b); |
125 | 5.26k | Store(out, d, out_rows[2] + x); |
126 | 5.26k | } |
127 | | |
128 | | Status AdaptiveDCSmoothing(JxlMemoryManager* memory_manager, |
129 | | const float* dc_factors, Image3F* dc, |
130 | 4.74k | ThreadPool* pool) { |
131 | 4.74k | const size_t xsize = dc->xsize(); |
132 | 4.74k | const size_t ysize = dc->ysize(); |
133 | 4.74k | if (ysize <= 2 || xsize <= 2) return true; |
134 | | |
135 | | // TODO(veluca): use tile-based processing? |
136 | | // TODO(veluca): decide if changes to the y channel should be propagated to |
137 | | // the x and b channels through color correlation. |
138 | 7 | JXL_ENSURE(w1 + w2 < 0.25f); |
139 | | |
140 | 14 | JXL_ASSIGN_OR_RETURN(Image3F smoothed, |
141 | 14 | Image3F::Create(memory_manager, xsize, ysize)); |
142 | | // Fill in borders that the loop below will not. First and last are unused. |
143 | 28 | for (size_t c = 0; c < 3; c++) { |
144 | 42 | for (size_t y : {static_cast<size_t>(0), ysize - 1}) { |
145 | 42 | memcpy(smoothed.PlaneRow(c, y), dc->PlaneRow(c, y), |
146 | 42 | xsize * sizeof(float)); |
147 | 42 | } |
148 | 21 | } |
149 | 209 | auto process_row = [&](const uint32_t y, size_t /*thread*/) -> Status { |
150 | 209 | const float* JXL_RESTRICT rows_top[3]{ |
151 | 209 | dc->ConstPlaneRow(0, y - 1), |
152 | 209 | dc->ConstPlaneRow(1, y - 1), |
153 | 209 | dc->ConstPlaneRow(2, y - 1), |
154 | 209 | }; |
155 | 209 | const float* JXL_RESTRICT rows[3] = { |
156 | 209 | dc->ConstPlaneRow(0, y), |
157 | 209 | dc->ConstPlaneRow(1, y), |
158 | 209 | dc->ConstPlaneRow(2, y), |
159 | 209 | }; |
160 | 209 | const float* JXL_RESTRICT rows_bottom[3] = { |
161 | 209 | dc->ConstPlaneRow(0, y + 1), |
162 | 209 | dc->ConstPlaneRow(1, y + 1), |
163 | 209 | dc->ConstPlaneRow(2, y + 1), |
164 | 209 | }; |
165 | 209 | float* JXL_RESTRICT rows_out[3] = { |
166 | 209 | smoothed.PlaneRow(0, y), |
167 | 209 | smoothed.PlaneRow(1, y), |
168 | 209 | smoothed.PlaneRow(2, y), |
169 | 209 | }; |
170 | 416 | for (size_t x : {static_cast<size_t>(0), xsize - 1}) { |
171 | 1.66k | for (size_t c = 0; c < 3; c++) { |
172 | 1.24k | rows_out[c][x] = rows[c][x]; |
173 | 1.24k | } |
174 | 416 | } |
175 | | |
176 | 209 | size_t x = 1; |
177 | | // First pixels |
178 | 209 | const size_t N = Lanes(D()); |
179 | 209 | for (; x < std::min(N, xsize - 1); x++) { |
180 | 0 | ComputePixel<DScalar>(dc_factors, rows_top, rows, rows_bottom, rows_out, |
181 | 0 | x); |
182 | 0 | } |
183 | | // Full vectors. |
184 | 5.45k | for (; x + N <= xsize - 1; x += N) { |
185 | 5.25k | ComputePixel<D>(dc_factors, rows_top, rows, rows_bottom, rows_out, x); |
186 | 5.25k | } |
187 | | // Last pixels. |
188 | 209 | for (; x < xsize - 1; x++) { |
189 | 0 | ComputePixel<DScalar>(dc_factors, rows_top, rows, rows_bottom, rows_out, |
190 | 0 | x); |
191 | 0 | } |
192 | 209 | return true; |
193 | 209 | }; |
194 | 14 | JXL_RETURN_IF_ERROR(RunOnPool(pool, 1, ysize - 1, ThreadPool::NoInit, |
195 | 14 | process_row, "DCSmoothingRow")); |
196 | 7 | dc->Swap(smoothed); |
197 | 7 | return true; |
198 | 14 | } |
199 | | |
200 | | // DC dequantization. |
201 | | void DequantDC(const Rect& r, Image3F* dc, ImageB* quant_dc, const Image& in, |
202 | | const float* dc_factors, float mul, const float* cfl_factors, |
203 | | const YCbCrChromaSubsampling& chroma_subsampling, |
204 | 4.79k | const BlockCtxMap& bctx) { |
205 | 4.79k | const HWY_FULL(float) df; |
206 | 4.79k | const Rebind<pixel_type, HWY_FULL(float)> di; // assumes pixel_type <= float |
207 | 4.79k | if (chroma_subsampling.Is444()) { |
208 | 4.79k | const auto fac_x = Set(df, dc_factors[0] * mul); |
209 | 4.79k | const auto fac_y = Set(df, dc_factors[1] * mul); |
210 | 4.79k | const auto fac_b = Set(df, dc_factors[2] * mul); |
211 | 4.79k | const auto cfl_fac_x = Set(df, cfl_factors[0]); |
212 | 4.79k | const auto cfl_fac_b = Set(df, cfl_factors[2]); |
213 | 14.6k | for (size_t y = 0; y < r.ysize(); y++) { |
214 | 9.90k | float* dec_row_x = r.PlaneRow(dc, 0, y); |
215 | 9.90k | float* dec_row_y = r.PlaneRow(dc, 1, y); |
216 | 9.90k | float* dec_row_b = r.PlaneRow(dc, 2, y); |
217 | 9.90k | const int32_t* quant_row_x = in.channel[1].plane.Row(y); |
218 | 9.90k | const int32_t* quant_row_y = in.channel[0].plane.Row(y); |
219 | 9.90k | const int32_t* quant_row_b = in.channel[2].plane.Row(y); |
220 | 72.8k | for (size_t x = 0; x < r.xsize(); x += Lanes(di)) { |
221 | 62.9k | const auto in_q_x = Load(di, quant_row_x + x); |
222 | 62.9k | const auto in_q_y = Load(di, quant_row_y + x); |
223 | 62.9k | const auto in_q_b = Load(di, quant_row_b + x); |
224 | 62.9k | const auto in_x = Mul(ConvertTo(df, in_q_x), fac_x); |
225 | 62.9k | const auto in_y = Mul(ConvertTo(df, in_q_y), fac_y); |
226 | 62.9k | const auto in_b = Mul(ConvertTo(df, in_q_b), fac_b); |
227 | 62.9k | Store(in_y, df, dec_row_y + x); |
228 | 62.9k | Store(MulAdd(in_y, cfl_fac_x, in_x), df, dec_row_x + x); |
229 | 62.9k | Store(MulAdd(in_y, cfl_fac_b, in_b), df, dec_row_b + x); |
230 | 62.9k | } |
231 | 9.90k | } |
232 | 4.79k | } else { |
233 | 0 | for (size_t c : {1, 0, 2}) { |
234 | 0 | Rect rect(r.x0() >> chroma_subsampling.HShift(c), |
235 | 0 | r.y0() >> chroma_subsampling.VShift(c), |
236 | 0 | r.xsize() >> chroma_subsampling.HShift(c), |
237 | 0 | r.ysize() >> chroma_subsampling.VShift(c)); |
238 | 0 | const auto fac = Set(df, dc_factors[c] * mul); |
239 | 0 | const Channel& ch = in.channel[c < 2 ? c ^ 1 : c]; |
240 | 0 | for (size_t y = 0; y < rect.ysize(); y++) { |
241 | 0 | const int32_t* quant_row = ch.plane.Row(y); |
242 | 0 | float* row = rect.PlaneRow(dc, c, y); |
243 | 0 | for (size_t x = 0; x < rect.xsize(); x += Lanes(di)) { |
244 | 0 | const auto in_q = Load(di, quant_row + x); |
245 | 0 | const auto out = Mul(ConvertTo(df, in_q), fac); |
246 | 0 | Store(out, df, row + x); |
247 | 0 | } |
248 | 0 | } |
249 | 0 | } |
250 | 0 | } |
251 | 4.79k | if (bctx.num_dc_ctxs <= 1) { |
252 | 14.6k | for (size_t y = 0; y < r.ysize(); y++) { |
253 | 9.90k | uint8_t* qdc_row = r.Row(quant_dc, y); |
254 | 9.90k | memset(qdc_row, 0, sizeof(*qdc_row) * r.xsize()); |
255 | 9.90k | } |
256 | 4.79k | } else { |
257 | 0 | for (size_t y = 0; y < r.ysize(); y++) { |
258 | 0 | uint8_t* qdc_row_val = r.Row(quant_dc, y); |
259 | 0 | const int32_t* quant_row_x = |
260 | 0 | in.channel[1].plane.Row(y >> chroma_subsampling.VShift(0)); |
261 | 0 | const int32_t* quant_row_y = |
262 | 0 | in.channel[0].plane.Row(y >> chroma_subsampling.VShift(1)); |
263 | 0 | const int32_t* quant_row_b = |
264 | 0 | in.channel[2].plane.Row(y >> chroma_subsampling.VShift(2)); |
265 | 0 | for (size_t x = 0; x < r.xsize(); x++) { |
266 | 0 | int bucket_x = 0; |
267 | 0 | int bucket_y = 0; |
268 | 0 | int bucket_b = 0; |
269 | 0 | for (int t : bctx.dc_thresholds[0]) { |
270 | 0 | if (quant_row_x[x >> chroma_subsampling.HShift(0)] > t) bucket_x++; |
271 | 0 | } |
272 | 0 | for (int t : bctx.dc_thresholds[1]) { |
273 | 0 | if (quant_row_y[x >> chroma_subsampling.HShift(1)] > t) bucket_y++; |
274 | 0 | } |
275 | 0 | for (int t : bctx.dc_thresholds[2]) { |
276 | 0 | if (quant_row_b[x >> chroma_subsampling.HShift(2)] > t) bucket_b++; |
277 | 0 | } |
278 | 0 | int bucket = bucket_x; |
279 | 0 | bucket *= bctx.dc_thresholds[2].size() + 1; |
280 | 0 | bucket += bucket_b; |
281 | 0 | bucket *= bctx.dc_thresholds[1].size() + 1; |
282 | 0 | bucket += bucket_y; |
283 | 0 | qdc_row_val[x] = bucket; |
284 | 0 | } |
285 | 0 | } |
286 | 0 | } |
287 | 4.79k | } |
288 | | |
289 | | // NOLINTNEXTLINE(google-readability-namespace-comments) |
290 | | } // namespace HWY_NAMESPACE |
291 | | } // namespace jxl |
292 | | HWY_AFTER_NAMESPACE(); |
293 | | |
294 | | #if HWY_ONCE |
295 | | namespace jxl { |
296 | | |
297 | | HWY_EXPORT(DequantDC); |
298 | | HWY_EXPORT(AdaptiveDCSmoothing); |
299 | | Status AdaptiveDCSmoothing(JxlMemoryManager* memory_manager, |
300 | | const float* dc_factors, Image3F* dc, |
301 | 4.74k | ThreadPool* pool) { |
302 | 4.74k | return HWY_DYNAMIC_DISPATCH(AdaptiveDCSmoothing)(memory_manager, dc_factors, |
303 | 4.74k | dc, pool); |
304 | 4.74k | } |
305 | | |
306 | | void DequantDC(const Rect& r, Image3F* dc, ImageB* quant_dc, const Image& in, |
307 | | const float* dc_factors, float mul, const float* cfl_factors, |
308 | | const YCbCrChromaSubsampling& chroma_subsampling, |
309 | 4.79k | const BlockCtxMap& bctx) { |
310 | 4.79k | HWY_DYNAMIC_DISPATCH(DequantDC) |
311 | 4.79k | (r, dc, quant_dc, in, dc_factors, mul, cfl_factors, chroma_subsampling, bctx); |
312 | 4.79k | } |
313 | | |
314 | | } // namespace jxl |
315 | | #endif // HWY_ONCE |