/src/libjxl/lib/jxl/dec_patch_dictionary.cc
Line | Count | Source |
1 | | // Copyright (c) the JPEG XL Project Authors. All rights reserved. |
2 | | // |
3 | | // Use of this source code is governed by a BSD-style |
4 | | // license that can be found in the LICENSE file. |
5 | | |
6 | | #include "lib/jxl/dec_patch_dictionary.h" |
7 | | |
8 | | #include <jxl/memory_manager.h> |
9 | | |
10 | | #include <algorithm> |
11 | | #include <cstddef> |
12 | | #include <cstdint> |
13 | | #include <cstdlib> |
14 | | #include <utility> |
15 | | #include <vector> |
16 | | |
17 | | #include "lib/jxl/base/printf_macros.h" |
18 | | #include "lib/jxl/base/status.h" |
19 | | #include "lib/jxl/blending.h" |
20 | | #include "lib/jxl/common.h" // kMaxNumReferenceFrames |
21 | | #include "lib/jxl/dec_ans.h" |
22 | | #include "lib/jxl/dec_bit_reader.h" |
23 | | #include "lib/jxl/image.h" |
24 | | #include "lib/jxl/image_bundle.h" |
25 | | #include "lib/jxl/image_metadata.h" |
26 | | #include "lib/jxl/pack_signed.h" |
27 | | #include "lib/jxl/patch_dictionary_internal.h" |
28 | | |
29 | | namespace jxl { |
30 | | |
31 | | Status PatchDictionary::Decode(JxlMemoryManager* memory_manager, BitReader* br, |
32 | | size_t xsize, size_t ysize, |
33 | | size_t num_extra_channels, |
34 | 1.37k | bool* uses_extra_channels) { |
35 | | // The dictionary is reused across frames: blendings_ is indexed by position |
36 | | // index, so stale entries from a previous frame must not remain. |
37 | 1.37k | positions_.clear(); |
38 | 1.37k | ref_positions_.clear(); |
39 | 1.37k | blendings_.clear(); |
40 | 1.37k | blendings_stride_ = num_extra_channels + 1; |
41 | 1.37k | std::vector<uint8_t> context_map; |
42 | 1.37k | ANSCode code; |
43 | 1.37k | JXL_RETURN_IF_ERROR(DecodeHistograms( |
44 | 1.37k | memory_manager, br, kNumPatchDictionaryContexts, &code, &context_map)); |
45 | 2.73k | JXL_ASSIGN_OR_RETURN(ANSSymbolReader decoder, |
46 | 2.73k | ANSSymbolReader::Create(&code, br)); |
47 | | |
48 | 3.19M | auto read_num = [&](size_t context) -> size_t { |
49 | 3.19M | size_t r = decoder.ReadHybridUint(context, br, context_map); |
50 | 3.19M | return r; |
51 | 3.19M | }; |
52 | | |
53 | 2.73k | size_t num_ref_patch = read_num(kNumRefPatchContext); |
54 | | // Limit max memory usage of patches to about 66 bytes per pixel (assuming 8 |
55 | | // bytes per size_t) |
56 | 2.73k | const size_t num_pixels = xsize * ysize; |
57 | 2.73k | const size_t max_ref_patches = 1024 + num_pixels / 4; |
58 | 2.73k | const size_t max_patches = max_ref_patches * 4; |
59 | 2.73k | const size_t max_blending_infos = max_patches * 4; |
60 | 2.73k | if (num_ref_patch > max_ref_patches) { |
61 | 0 | return JXL_FAILURE("Too many patches in dictionary"); |
62 | 0 | } |
63 | | |
64 | 1.36k | size_t total_patches = 0; |
65 | 1.36k | size_t next_size = 1; |
66 | | |
67 | 28.2k | for (size_t id = 0; id < num_ref_patch; id++) { |
68 | 26.9k | PatchReferencePosition ref_pos; |
69 | 26.9k | ref_pos.ref = read_num(kReferenceFrameContext); |
70 | 26.9k | if (ref_pos.ref >= kMaxNumReferenceFrames || |
71 | 26.9k | reference_frames_->at(ref_pos.ref).frame->xsize() == 0) { |
72 | 0 | return JXL_FAILURE("Invalid reference frame ID"); |
73 | 0 | } |
74 | 26.9k | if (!reference_frames_->at(ref_pos.ref).ib_is_in_xyb) { |
75 | 0 | return JXL_FAILURE( |
76 | 0 | "Patches cannot use frames saved post color transforms"); |
77 | 0 | } |
78 | 26.9k | const ImageBundle& ib = *reference_frames_->at(ref_pos.ref).frame; |
79 | 26.9k | ref_pos.x0 = read_num(kPatchReferencePositionContext); |
80 | 26.9k | ref_pos.y0 = read_num(kPatchReferencePositionContext); |
81 | 26.9k | ref_pos.xsize = read_num(kPatchSizeContext) + 1; |
82 | 26.9k | ref_pos.ysize = read_num(kPatchSizeContext) + 1; |
83 | 26.9k | if (ref_pos.x0 + ref_pos.xsize > ib.xsize()) { |
84 | 0 | return JXL_FAILURE("Invalid position specified in reference frame"); |
85 | 0 | } |
86 | 26.9k | if (ref_pos.y0 + ref_pos.ysize > ib.ysize()) { |
87 | 0 | return JXL_FAILURE("Invalid position specified in reference frame"); |
88 | 0 | } |
89 | 26.9k | size_t id_count = read_num(kPatchCountContext); |
90 | 26.9k | if (id_count > max_patches) { |
91 | 0 | return JXL_FAILURE("Too many patches in dictionary"); |
92 | 0 | } |
93 | 26.9k | id_count++; |
94 | 26.9k | total_patches += id_count; |
95 | 26.9k | if (total_patches > max_patches) { |
96 | 0 | return JXL_FAILURE("Too many patches in dictionary"); |
97 | 0 | } |
98 | 26.9k | if (next_size < total_patches) { |
99 | 6.83k | next_size *= 2; |
100 | 6.83k | next_size = std::min<size_t>(next_size, max_patches); |
101 | 6.83k | } |
102 | 26.9k | if (next_size * blendings_stride_ > max_blending_infos) { |
103 | 0 | return JXL_FAILURE("Too many patches in dictionary"); |
104 | 0 | } |
105 | 26.9k | positions_.reserve(next_size); |
106 | 26.9k | blendings_.reserve(next_size * blendings_stride_); |
107 | 26.9k | bool choose_alpha = (num_extra_channels > 1); |
108 | 1.03M | for (size_t i = 0; i < id_count; i++) { |
109 | 1.01M | PatchPosition pos; |
110 | 1.01M | pos.ref_pos_idx = ref_positions_.size(); |
111 | 1.01M | if (i == 0) { |
112 | 26.9k | pos.x = read_num(kPatchPositionContext); |
113 | 26.9k | pos.y = read_num(kPatchPositionContext); |
114 | 983k | } else { |
115 | 983k | ptrdiff_t deltax = UnpackSigned(read_num(kPatchOffsetContext)); |
116 | 983k | if (deltax < 0 && static_cast<size_t>(-deltax) > positions_.back().x) { |
117 | 0 | return JXL_FAILURE("Invalid patch: negative x coordinate (%" PRIuS |
118 | 0 | " base x %" PRIdS " delta x)", |
119 | 0 | positions_.back().x, deltax); |
120 | 0 | } |
121 | 983k | pos.x = positions_.back().x + deltax; |
122 | 983k | ptrdiff_t deltay = UnpackSigned(read_num(kPatchOffsetContext)); |
123 | 983k | if (deltay < 0 && static_cast<size_t>(-deltay) > positions_.back().y) { |
124 | 0 | return JXL_FAILURE("Invalid patch: negative y coordinate (%" PRIuS |
125 | 0 | " base y %" PRIdS " delta y)", |
126 | 0 | positions_.back().y, deltay); |
127 | 0 | } |
128 | 983k | pos.y = positions_.back().y + deltay; |
129 | 983k | } |
130 | 1.01M | if (pos.x + ref_pos.xsize > xsize) { |
131 | 1 | return JXL_FAILURE("Invalid patch x: at %" PRIuS " + %" PRIuS |
132 | 1 | " > %" PRIuS, |
133 | 1 | pos.x, ref_pos.xsize, xsize); |
134 | 1 | } |
135 | 1.01M | if (pos.y + ref_pos.ysize > ysize) { |
136 | 1 | return JXL_FAILURE("Invalid patch y: at %" PRIuS " + %" PRIuS |
137 | 1 | " > %" PRIuS, |
138 | 1 | pos.y, ref_pos.ysize, ysize); |
139 | 1 | } |
140 | 2.02M | for (size_t j = 0; j < blendings_stride_; j++) { |
141 | 1.01M | uint32_t blend_mode = read_num(kPatchBlendModeContext); |
142 | 1.01M | if (blend_mode >= kNumPatchBlendModes) { |
143 | 0 | return JXL_FAILURE("Invalid patch blend mode: %u", blend_mode); |
144 | 0 | } |
145 | 1.01M | PatchBlending info; |
146 | 1.01M | info.mode = static_cast<PatchBlendMode>(blend_mode); |
147 | 1.01M | if (UsesAlpha(info.mode)) { |
148 | 3 | *uses_extra_channels = true; |
149 | 3 | } |
150 | 1.01M | if (info.mode != PatchBlendMode::kNone && j > 0) { |
151 | 631 | *uses_extra_channels = true; |
152 | 631 | } |
153 | 1.01M | if (UsesAlpha(info.mode) && choose_alpha) { |
154 | 0 | info.alpha_channel = read_num(kPatchAlphaChannelContext); |
155 | 0 | if (info.alpha_channel >= num_extra_channels) { |
156 | 0 | return JXL_FAILURE( |
157 | 0 | "Invalid alpha channel for blending: %u out of %u\n", |
158 | 0 | info.alpha_channel, static_cast<uint32_t>(num_extra_channels)); |
159 | 0 | } |
160 | 1.01M | } else { |
161 | 1.01M | info.alpha_channel = 0; |
162 | 1.01M | } |
163 | 1.01M | if (UsesClamp(info.mode)) { |
164 | 55 | info.clamp = static_cast<bool>(read_num(kPatchClampContext)); |
165 | 1.01M | } else { |
166 | 1.01M | info.clamp = false; |
167 | 1.01M | } |
168 | 1.01M | blendings_.push_back(info); |
169 | 1.01M | } |
170 | 1.01M | positions_.emplace_back(pos); |
171 | 1.01M | } |
172 | 26.9k | ref_positions_.emplace_back(ref_pos); |
173 | 26.9k | } |
174 | 1.36k | positions_.shrink_to_fit(); |
175 | | |
176 | 1.36k | if (!decoder.CheckANSFinalState()) { |
177 | 0 | return JXL_FAILURE("ANS checksum failure."); |
178 | 0 | } |
179 | | |
180 | 1.36k | ComputePatchTree(); |
181 | 1.36k | return true; |
182 | 1.36k | } |
183 | | |
184 | 1.33k | int PatchDictionary::GetReferences() const { |
185 | 1.33k | int result = 0; |
186 | 25.4k | for (const auto& ref_pos : ref_positions_) { |
187 | 25.4k | result |= (1 << static_cast<int>(ref_pos.ref)); |
188 | 25.4k | } |
189 | 1.33k | return result; |
190 | 1.33k | } |
191 | | |
192 | | namespace { |
193 | | struct PatchInterval { |
194 | | size_t idx; |
195 | | size_t y0, y1; |
196 | | }; |
197 | | } // namespace |
198 | | |
199 | 17.3k | void PatchDictionary::ComputePatchTree() { |
200 | 17.3k | patch_tree_.clear(); |
201 | 17.3k | num_patches_.clear(); |
202 | 17.3k | sorted_patches_y0_.clear(); |
203 | 17.3k | sorted_patches_y1_.clear(); |
204 | 17.3k | if (positions_.empty()) { |
205 | 16.4k | return; |
206 | 16.4k | } |
207 | | // Create a y-interval for each patch. |
208 | 931 | std::vector<PatchInterval> intervals(positions_.size()); |
209 | 1.01M | for (size_t i = 0; i < positions_.size(); ++i) { |
210 | 1.01M | const auto& pos = positions_[i]; |
211 | 1.01M | intervals[i].idx = i; |
212 | 1.01M | intervals[i].y0 = pos.y; |
213 | 1.01M | intervals[i].y1 = pos.y + ref_positions_[pos.ref_pos_idx].ysize; |
214 | 1.01M | } |
215 | 76.7k | auto sort_by_y0 = [&intervals](size_t start, size_t end) { |
216 | 76.7k | std::sort(intervals.data() + start, intervals.data() + end, |
217 | 25.2M | [](const PatchInterval& i0, const PatchInterval& i1) { |
218 | 25.2M | return i0.y0 < i1.y0; |
219 | 25.2M | }); |
220 | 76.7k | }; |
221 | 39.3k | auto sort_by_y1 = [&intervals](size_t start, size_t end) { |
222 | 39.3k | std::sort(intervals.data() + start, intervals.data() + end, |
223 | 21.3M | [](const PatchInterval& i0, const PatchInterval& i1) { |
224 | 21.3M | return i0.y1 < i1.y1; |
225 | 21.3M | }); |
226 | 39.3k | }; |
227 | | // Count the number of patches for each row. |
228 | 931 | sort_by_y1(0, intervals.size()); |
229 | 931 | num_patches_.resize(intervals.back().y1); |
230 | 1.01M | for (auto iv : intervals) { |
231 | 27.8M | for (size_t y = iv.y0; y < iv.y1; ++y) num_patches_[y]++; |
232 | 1.01M | } |
233 | 931 | PatchTreeNode root; |
234 | 931 | root.start = 0; |
235 | 931 | root.num = intervals.size(); |
236 | 931 | patch_tree_.push_back(root); |
237 | 931 | size_t next = 0; |
238 | 39.3k | while (next < patch_tree_.size()) { |
239 | 38.3k | auto& node = patch_tree_[next]; |
240 | 38.3k | size_t start = node.start; |
241 | 38.3k | size_t end = node.start + node.num; |
242 | | // Choose the y_center for this node to be the median of interval starts. |
243 | 38.3k | sort_by_y0(start, end); |
244 | 38.3k | size_t middle_idx = start + node.num / 2; |
245 | 38.3k | node.y_center = intervals[middle_idx].y0; |
246 | | // Divide the intervals in [start, end) into three groups: |
247 | | // * those completely to the right of y_center: [right_start, end) |
248 | | // * those overlapping y_center: [left_end, right_start) |
249 | | // * those completely to the left of y_center: [start, left_end) |
250 | 38.3k | size_t right_start = middle_idx; |
251 | 187k | while (right_start < end && intervals[right_start].y0 == node.y_center) { |
252 | 148k | ++right_start; |
253 | 148k | } |
254 | 38.3k | sort_by_y1(start, right_start); |
255 | 38.3k | size_t left_end = right_start; |
256 | 1.04M | while (left_end > start && intervals[left_end - 1].y1 > node.y_center) { |
257 | 1.01M | --left_end; |
258 | 1.01M | } |
259 | | // Fill in sorted_patches_y0_ and sorted_patches_y1_ for the current node. |
260 | 38.3k | node.num = right_start - left_end; |
261 | 38.3k | node.start = sorted_patches_y0_.size(); |
262 | 38.3k | for (ptrdiff_t i = static_cast<ptrdiff_t>(right_start) - 1; |
263 | 1.04M | i >= static_cast<ptrdiff_t>(left_end); --i) { |
264 | 1.01M | sorted_patches_y1_.emplace_back(intervals[i].y1, intervals[i].idx); |
265 | 1.01M | } |
266 | 38.3k | sort_by_y0(left_end, right_start); |
267 | 1.04M | for (size_t i = left_end; i < right_start; ++i) { |
268 | 1.01M | sorted_patches_y0_.emplace_back(intervals[i].y0, intervals[i].idx); |
269 | 1.01M | } |
270 | | // Create the left and right nodes (if not empty). |
271 | 38.3k | node.left_child = node.right_child = -1; |
272 | 38.3k | if (left_end > start) { |
273 | 17.8k | PatchTreeNode left; |
274 | 17.8k | left.start = start; |
275 | 17.8k | left.num = left_end - left.start; |
276 | 17.8k | patch_tree_[next].left_child = patch_tree_.size(); |
277 | 17.8k | patch_tree_.push_back(left); |
278 | 17.8k | } |
279 | 38.3k | if (right_start < end) { |
280 | 19.6k | PatchTreeNode right; |
281 | 19.6k | right.start = right_start; |
282 | 19.6k | right.num = end - right.start; |
283 | 19.6k | patch_tree_[next].right_child = patch_tree_.size(); |
284 | 19.6k | patch_tree_.push_back(right); |
285 | 19.6k | } |
286 | 38.3k | ++next; |
287 | 38.3k | } |
288 | 931 | } |
289 | | |
290 | 105k | std::vector<size_t> PatchDictionary::GetPatchesForRow(size_t y) const { |
291 | 105k | std::vector<size_t> result; |
292 | 105k | if (y < num_patches_.size() && num_patches_[y] > 0) { |
293 | 67.5k | result.reserve(num_patches_[y]); |
294 | 523k | for (ptrdiff_t tree_idx = 0; tree_idx != -1;) { |
295 | 456k | JXL_DASSERT(tree_idx < static_cast<ptrdiff_t>(patch_tree_.size())); |
296 | 456k | const auto& node = patch_tree_[tree_idx]; |
297 | 456k | if (y <= node.y_center) { |
298 | 8.07M | for (size_t i = 0; i < node.num; ++i) { |
299 | 8.04M | const auto& p = sorted_patches_y0_[node.start + i]; |
300 | 8.04M | if (y < p.first) break; |
301 | 7.93M | result.push_back(p.second); |
302 | 7.93M | } |
303 | 144k | tree_idx = y < node.y_center ? node.left_child : -1; |
304 | 311k | } else { |
305 | 17.2M | for (size_t i = 0; i < node.num; ++i) { |
306 | 17.2M | const auto& p = sorted_patches_y1_[node.start + i]; |
307 | 17.2M | if (y >= p.first) break; |
308 | 16.9M | result.push_back(p.second); |
309 | 16.9M | } |
310 | 311k | tree_idx = node.right_child; |
311 | 311k | } |
312 | 456k | } |
313 | | // Ensure that he relative order of patches that affect the same pixels is |
314 | | // preserved. This is important for patches that have a blend mode |
315 | | // different from kAdd. |
316 | 67.5k | std::sort(result.begin(), result.end()); |
317 | 67.5k | } |
318 | 105k | return result; |
319 | 105k | } |
320 | | |
321 | | // Adds patches to a segment of `xsize` pixels, starting at `inout`, assumed |
322 | | // to be located at position (x0, y) in the frame. |
323 | | Status PatchDictionary::AddOneRow( |
324 | | float* const* inout, size_t y, size_t x0, size_t xsize, |
325 | 105k | const std::vector<ExtraChannelInfo>& extra_channel_info) const { |
326 | 105k | size_t num_ec = extra_channel_info.size(); |
327 | 105k | JXL_ENSURE(num_ec + 1 <= blendings_stride_); |
328 | 105k | std::vector<const float*> fg_ptrs(3 + num_ec); |
329 | 24.9M | for (size_t pos_idx : GetPatchesForRow(y)) { |
330 | 24.9M | const size_t blending_idx = pos_idx * blendings_stride_; |
331 | 24.9M | const PatchPosition& pos = positions_[pos_idx]; |
332 | 24.9M | const PatchReferencePosition& ref_pos = ref_positions_[pos.ref_pos_idx]; |
333 | 24.9M | size_t by = pos.y; |
334 | 24.9M | size_t bx = pos.x; |
335 | 24.9M | size_t patch_xsize = ref_pos.xsize; |
336 | 24.9M | JXL_ENSURE(y >= by); |
337 | 24.9M | JXL_ENSURE(y < by + ref_pos.ysize); |
338 | 24.9M | size_t iy = y - by; |
339 | 24.9M | size_t ref = ref_pos.ref; |
340 | 24.9M | if (bx >= x0 + xsize) continue; |
341 | 24.9M | if (bx + patch_xsize < x0) continue; |
342 | 24.9M | size_t patch_x0 = std::max(bx, x0); |
343 | 24.9M | size_t patch_x1 = std::min(bx + patch_xsize, x0 + xsize); |
344 | 99.6M | for (size_t c = 0; c < 3; c++) { |
345 | 74.7M | fg_ptrs[c] = reference_frames_->at(ref).frame->color()->ConstPlaneRow( |
346 | 74.7M | c, ref_pos.y0 + iy) + |
347 | 74.7M | ref_pos.x0 + x0 - bx; |
348 | 74.7M | } |
349 | 24.9M | for (size_t i = 0; i < num_ec; i++) { |
350 | 33.8k | fg_ptrs[3 + i] = |
351 | 33.8k | reference_frames_->at(ref).frame->extra_channels()[i].ConstRow( |
352 | 33.8k | ref_pos.y0 + iy) + |
353 | 33.8k | ref_pos.x0 + x0 - bx; |
354 | 33.8k | } |
355 | 24.9M | JXL_RETURN_IF_ERROR(PerformBlending( |
356 | 24.9M | memory_manager_, inout, fg_ptrs.data(), inout, patch_x0 - x0, |
357 | 24.9M | patch_x1 - patch_x0, blendings_[blending_idx], |
358 | 24.9M | blendings_.data() + blending_idx + 1, extra_channel_info)); |
359 | 24.9M | } |
360 | 105k | return true; |
361 | 105k | } |
362 | | } // namespace jxl |