/src/libjxl/lib/jxl/enc_detect_dots.cc
Line | Count | Source (jump to first uncovered line) |
1 | | // Copyright (c) the JPEG XL Project Authors. All rights reserved. |
2 | | // |
3 | | // Use of this source code is governed by a BSD-style |
4 | | // license that can be found in the LICENSE file. |
5 | | |
6 | | #include "lib/jxl/enc_detect_dots.h" |
7 | | |
8 | | #include <jxl/memory_manager.h> |
9 | | |
10 | | #include <algorithm> |
11 | | #include <array> |
12 | | #include <cmath> |
13 | | #include <cstdint> |
14 | | #include <cstdio> |
15 | | #include <utility> |
16 | | #include <vector> |
17 | | |
18 | | #undef HWY_TARGET_INCLUDE |
19 | | #define HWY_TARGET_INCLUDE "lib/jxl/enc_detect_dots.cc" |
20 | | #include <hwy/foreach_target.h> |
21 | | #include <hwy/highway.h> |
22 | | |
23 | | #include "lib/jxl/base/common.h" |
24 | | #include "lib/jxl/base/compiler_specific.h" |
25 | | #include "lib/jxl/base/data_parallel.h" |
26 | | #include "lib/jxl/base/printf_macros.h" |
27 | | #include "lib/jxl/base/rect.h" |
28 | | #include "lib/jxl/base/status.h" |
29 | | #include "lib/jxl/convolve.h" |
30 | | #include "lib/jxl/enc_linalg.h" |
31 | | #include "lib/jxl/image.h" |
32 | | #include "lib/jxl/image_ops.h" |
33 | | |
34 | | // Set JXL_DEBUG_DOT_DETECT to 1 to enable debugging. |
35 | | #ifndef JXL_DEBUG_DOT_DETECT |
36 | | #define JXL_DEBUG_DOT_DETECT 0 |
37 | | #endif |
38 | | |
39 | | HWY_BEFORE_NAMESPACE(); |
40 | | namespace jxl { |
41 | | namespace HWY_NAMESPACE { |
42 | | |
43 | | // These templates are not found via ADL. |
44 | | using hwy::HWY_NAMESPACE::Add; |
45 | | using hwy::HWY_NAMESPACE::Mul; |
46 | | using hwy::HWY_NAMESPACE::Sub; |
47 | | |
48 | | StatusOr<ImageF> SumOfSquareDifferences(const Image3F& forig, |
49 | | const Image3F& smooth, |
50 | 0 | ThreadPool* pool) { |
51 | 0 | const HWY_FULL(float) d; |
52 | 0 | const auto color_coef0 = Set(d, 0.0f); |
53 | 0 | const auto color_coef1 = Set(d, 10.0f); |
54 | 0 | const auto color_coef2 = Set(d, 0.0f); |
55 | 0 | JxlMemoryManager* memory_manager = forig.memory_manager(); |
56 | |
|
57 | 0 | JXL_ASSIGN_OR_RETURN( |
58 | 0 | ImageF sum_of_squares, |
59 | 0 | ImageF::Create(memory_manager, forig.xsize(), forig.ysize())); |
60 | 0 | const auto process_row = [&](const uint32_t task, size_t thread) -> Status { |
61 | 0 | const size_t y = static_cast<size_t>(task); |
62 | 0 | const float* JXL_RESTRICT orig_row0 = forig.Plane(0).ConstRow(y); |
63 | 0 | const float* JXL_RESTRICT orig_row1 = forig.Plane(1).ConstRow(y); |
64 | 0 | const float* JXL_RESTRICT orig_row2 = forig.Plane(2).ConstRow(y); |
65 | 0 | const float* JXL_RESTRICT smooth_row0 = smooth.Plane(0).ConstRow(y); |
66 | 0 | const float* JXL_RESTRICT smooth_row1 = smooth.Plane(1).ConstRow(y); |
67 | 0 | const float* JXL_RESTRICT smooth_row2 = smooth.Plane(2).ConstRow(y); |
68 | 0 | float* JXL_RESTRICT sos_row = sum_of_squares.Row(y); |
69 | |
|
70 | 0 | for (size_t x = 0; x < forig.xsize(); x += Lanes(d)) { |
71 | 0 | auto v0 = Sub(Load(d, orig_row0 + x), Load(d, smooth_row0 + x)); |
72 | 0 | auto v1 = Sub(Load(d, orig_row1 + x), Load(d, smooth_row1 + x)); |
73 | 0 | auto v2 = Sub(Load(d, orig_row2 + x), Load(d, smooth_row2 + x)); |
74 | 0 | v0 = Mul(Mul(v0, v0), color_coef0); |
75 | 0 | v1 = Mul(Mul(v1, v1), color_coef1); |
76 | 0 | v2 = Mul(Mul(v2, v2), color_coef2); |
77 | 0 | const auto sos = Add(v0, Add(v1, v2)); // weighted sum of square diffs |
78 | 0 | Store(sos, d, sos_row + x); |
79 | 0 | } |
80 | 0 | return true; |
81 | 0 | }; |
82 | 0 | JXL_RETURN_IF_ERROR(RunOnPool(pool, 0, forig.ysize(), ThreadPool::NoInit, |
83 | 0 | process_row, "ComputeEnergyImage")); |
84 | 0 | return sum_of_squares; |
85 | 0 | } |
86 | | |
87 | | // NOLINTNEXTLINE(google-readability-namespace-comments) |
88 | | } // namespace HWY_NAMESPACE |
89 | | } // namespace jxl |
90 | | HWY_AFTER_NAMESPACE(); |
91 | | |
92 | | #if HWY_ONCE |
93 | | namespace jxl { |
94 | | HWY_EXPORT(SumOfSquareDifferences); // Local function |
95 | | |
96 | | const int kEllipseWindowSize = 5; |
97 | | |
98 | | namespace { |
99 | | struct GaussianEllipse { |
100 | | double x; // position in x |
101 | | double y; // position in y |
102 | | double sigma_x; // scale in x |
103 | | double sigma_y; // scale in y |
104 | | double angle; // ellipse rotation in radians |
105 | | std::array<double, 3> intensity; // intensity in each channel |
106 | | |
107 | | // The following variables do not need to be encoded |
108 | | double l2_loss; // error after the Gaussian was fit |
109 | | double l1_loss; |
110 | | double ridge_loss; // the l2_loss plus regularization term |
111 | | double custom_loss; // experimental custom loss |
112 | | std::array<double, 3> bgColor; // best background color |
113 | | size_t neg_pixels; // number of negative pixels when subtracting dot |
114 | | std::array<double, 3> neg_value; // debt due to channel truncation |
115 | | }; |
116 | | double DotGaussianModel(double dx, double dy, double ct, double st, |
117 | 0 | double sigma_x, double sigma_y, double intensity) { |
118 | 0 | double rx = ct * dx + st * dy; |
119 | 0 | double ry = -st * dx + ct * dy; |
120 | 0 | double md = (rx * rx / sigma_x) + (ry * ry / sigma_y); |
121 | 0 | double value = intensity * exp(-0.5 * md); |
122 | 0 | return value; |
123 | 0 | } |
124 | | |
125 | | constexpr bool kOptimizeBackground = true; |
126 | | |
127 | | // Gaussian that smooths noise but preserves dots |
128 | 0 | const WeightsSeparable5& WeightsSeparable5Gaussian0_65() { |
129 | 0 | constexpr float w0 = 0.558311f; |
130 | 0 | constexpr float w1 = 0.210395f; |
131 | 0 | constexpr float w2 = 0.010449f; |
132 | 0 | static constexpr WeightsSeparable5 weights = { |
133 | 0 | {HWY_REP4(w0), HWY_REP4(w1), HWY_REP4(w2)}, |
134 | 0 | {HWY_REP4(w0), HWY_REP4(w1), HWY_REP4(w2)}}; |
135 | 0 | return weights; |
136 | 0 | } |
137 | | |
138 | | // (Iterated) Gaussian that removes dots. |
139 | 0 | const WeightsSeparable5& WeightsSeparable5Gaussian3() { |
140 | 0 | constexpr float w0 = 0.222338f; |
141 | 0 | constexpr float w1 = 0.210431f; |
142 | 0 | constexpr float w2 = 0.1784f; |
143 | 0 | static constexpr WeightsSeparable5 weights = { |
144 | 0 | {HWY_REP4(w0), HWY_REP4(w1), HWY_REP4(w2)}, |
145 | 0 | {HWY_REP4(w0), HWY_REP4(w1), HWY_REP4(w2)}}; |
146 | 0 | return weights; |
147 | 0 | } |
148 | | |
149 | | StatusOr<ImageF> ComputeEnergyImage(const Image3F& orig, Image3F* smooth, |
150 | 0 | ThreadPool* pool) { |
151 | 0 | JxlMemoryManager* memory_manager = orig.memory_manager(); |
152 | | // Prepare guidance images for dot selection. |
153 | 0 | JXL_ASSIGN_OR_RETURN( |
154 | 0 | Image3F forig, |
155 | 0 | Image3F::Create(memory_manager, orig.xsize(), orig.ysize())); |
156 | 0 | JXL_ASSIGN_OR_RETURN( |
157 | 0 | *smooth, Image3F::Create(memory_manager, orig.xsize(), orig.ysize())); |
158 | 0 | Rect rect(orig); |
159 | |
|
160 | 0 | const auto& weights1 = WeightsSeparable5Gaussian0_65(); |
161 | 0 | const auto& weights3 = WeightsSeparable5Gaussian3(); |
162 | |
|
163 | 0 | for (size_t c = 0; c < 3; ++c) { |
164 | | // Use forig as temporary storage to reduce memory and keep it warmer. |
165 | 0 | JXL_RETURN_IF_ERROR( |
166 | 0 | Separable5(orig.Plane(c), rect, weights3, pool, &forig.Plane(c))); |
167 | 0 | JXL_RETURN_IF_ERROR( |
168 | 0 | Separable5(forig.Plane(c), rect, weights3, pool, &smooth->Plane(c))); |
169 | 0 | JXL_RETURN_IF_ERROR( |
170 | 0 | Separable5(orig.Plane(c), rect, weights1, pool, &forig.Plane(c))); |
171 | 0 | } |
172 | | |
173 | 0 | return HWY_DYNAMIC_DISPATCH(SumOfSquareDifferences)(forig, *smooth, pool); |
174 | 0 | } |
175 | | |
176 | | struct Pixel { |
177 | | int x; |
178 | | int y; |
179 | | }; |
180 | | |
181 | 0 | Pixel operator+(const Pixel& a, const Pixel& b) { |
182 | 0 | return Pixel{a.x + b.x, a.y + b.y}; |
183 | 0 | } |
184 | | |
185 | | // Maximum area in pixels of a ellipse |
186 | | const size_t kMaxCCSize = 1000; |
187 | | |
188 | | // Extracts a connected component from a Binary image where seed is part |
189 | | // of the component |
190 | | bool ExtractComponent(const Rect& rect, ImageF* img, std::vector<Pixel>* pixels, |
191 | 0 | const Pixel& seed, double threshold) { |
192 | 0 | static const std::vector<Pixel> neighbors{{1, -1}, {1, 0}, {1, 1}, {0, -1}, |
193 | 0 | {0, 1}, {-1, -1}, {-1, 1}, {1, 0}}; |
194 | 0 | std::vector<Pixel> q{seed}; |
195 | 0 | while (!q.empty()) { |
196 | 0 | Pixel current = q.back(); |
197 | 0 | q.pop_back(); |
198 | 0 | pixels->push_back(current); |
199 | 0 | if (pixels->size() > kMaxCCSize) return false; |
200 | 0 | for (const Pixel& delta : neighbors) { |
201 | 0 | Pixel child = current + delta; |
202 | 0 | if (child.x >= 0 && static_cast<size_t>(child.x) < rect.xsize() && |
203 | 0 | child.y >= 0 && static_cast<size_t>(child.y) < rect.ysize()) { |
204 | 0 | float* value = &rect.Row(img, child.y)[child.x]; |
205 | 0 | if (*value > threshold) { |
206 | 0 | *value = 0.0; |
207 | 0 | q.push_back(child); |
208 | 0 | } |
209 | 0 | } |
210 | 0 | } |
211 | 0 | } |
212 | 0 | return true; |
213 | 0 | } |
214 | | |
215 | 0 | inline bool PointInRect(const Rect& r, const Pixel& p) { |
216 | 0 | return (static_cast<size_t>(p.x) >= r.x0() && |
217 | 0 | static_cast<size_t>(p.x) < (r.x0() + r.xsize()) && |
218 | 0 | static_cast<size_t>(p.y) >= r.y0() && |
219 | 0 | static_cast<size_t>(p.y) < (r.y0() + r.ysize())); |
220 | 0 | } |
221 | | |
222 | | struct ConnectedComponent { |
223 | | ConnectedComponent(const Rect& bounds, const std::vector<Pixel>&& pixels) |
224 | 0 | : bounds(bounds), pixels(pixels) {} |
225 | | Rect bounds; |
226 | | std::vector<Pixel> pixels; |
227 | | float maxEnergy; |
228 | | float meanEnergy; |
229 | | float varEnergy; |
230 | | float meanBg; |
231 | | float varBg; |
232 | | float score; |
233 | | Pixel mode; |
234 | | |
235 | 0 | void CompStats(const ImageF& energy, const Rect& rect, int extra) { |
236 | 0 | maxEnergy = 0.0; |
237 | 0 | meanEnergy = 0.0; |
238 | 0 | varEnergy = 0.0; |
239 | 0 | meanBg = 0.0; |
240 | 0 | varBg = 0.0; |
241 | 0 | int nIn = 0; |
242 | 0 | int nOut = 0; |
243 | 0 | mode.x = 0; |
244 | 0 | mode.y = 0; |
245 | 0 | for (int sy = -extra; sy < (static_cast<int>(bounds.ysize()) + extra); |
246 | 0 | sy++) { |
247 | 0 | int y = sy + static_cast<int>(bounds.y0()); |
248 | 0 | if (y < 0 || static_cast<size_t>(y) >= rect.ysize()) continue; |
249 | 0 | const float* JXL_RESTRICT erow = rect.ConstRow(energy, y); |
250 | 0 | for (int sx = -extra; sx < (static_cast<int>(bounds.xsize()) + extra); |
251 | 0 | sx++) { |
252 | 0 | int x = sx + static_cast<int>(bounds.x0()); |
253 | 0 | if (x < 0 || static_cast<size_t>(x) >= rect.xsize()) continue; |
254 | 0 | if (erow[x] > maxEnergy) { |
255 | 0 | maxEnergy = erow[x]; |
256 | 0 | mode.x = x; |
257 | 0 | mode.y = y; |
258 | 0 | } |
259 | 0 | if (PointInRect(bounds, Pixel{x, y})) { |
260 | 0 | meanEnergy += erow[x]; |
261 | 0 | varEnergy += erow[x] * erow[x]; |
262 | 0 | nIn++; |
263 | 0 | } else { |
264 | 0 | meanBg += erow[x]; |
265 | 0 | varBg += erow[x] * erow[x]; |
266 | 0 | nOut++; |
267 | 0 | } |
268 | 0 | } |
269 | 0 | } |
270 | 0 | meanEnergy = meanEnergy / nIn; |
271 | 0 | meanBg = meanBg / nOut; |
272 | 0 | varEnergy = (varEnergy / nIn) - meanEnergy * meanEnergy; |
273 | 0 | varBg = (varBg / nOut) - meanBg * meanBg; |
274 | 0 | score = (meanEnergy - meanBg) / std::sqrt(varBg); |
275 | 0 | } |
276 | | }; |
277 | | |
278 | 0 | Rect BoundingRectangle(const std::vector<Pixel>& pixels) { |
279 | 0 | JXL_DASSERT(!pixels.empty()); |
280 | 0 | int low_x; |
281 | 0 | int high_x; |
282 | 0 | int low_y; |
283 | 0 | int high_y; |
284 | 0 | low_x = high_x = pixels[0].x; |
285 | 0 | low_y = high_y = pixels[0].y; |
286 | 0 | for (const Pixel& p : pixels) { |
287 | 0 | low_x = std::min(low_x, p.x); |
288 | 0 | high_x = std::max(high_x, p.x); |
289 | 0 | low_y = std::min(low_y, p.y); |
290 | 0 | high_y = std::max(high_y, p.y); |
291 | 0 | } |
292 | 0 | return Rect(low_x, low_y, high_x - low_x + 1, high_y - low_y + 1); |
293 | 0 | } |
294 | | |
295 | | StatusOr<std::vector<ConnectedComponent>> FindCC(const ImageF& energy, |
296 | | const Rect& rect, double t_low, |
297 | | double t_high, |
298 | | uint32_t maxWindow, |
299 | 0 | double minScore) { |
300 | 0 | const int kExtraRect = 4; |
301 | 0 | JxlMemoryManager* memory_manager = energy.memory_manager(); |
302 | 0 | JXL_ASSIGN_OR_RETURN( |
303 | 0 | ImageF img, |
304 | 0 | ImageF::Create(memory_manager, energy.xsize(), energy.ysize())); |
305 | 0 | JXL_RETURN_IF_ERROR(CopyImageTo(energy, &img)); |
306 | 0 | std::vector<ConnectedComponent> ans; |
307 | 0 | for (size_t y = 0; y < rect.ysize(); y++) { |
308 | 0 | float* JXL_RESTRICT row = rect.Row(&img, y); |
309 | 0 | for (size_t x = 0; x < rect.xsize(); x++) { |
310 | 0 | if (row[x] > t_high) { |
311 | 0 | std::vector<Pixel> pixels; |
312 | 0 | row[x] = 0.0; |
313 | 0 | Pixel seed = Pixel{static_cast<int>(x), static_cast<int>(y)}; |
314 | 0 | bool success = ExtractComponent(rect, &img, &pixels, seed, t_low); |
315 | 0 | if (!success) continue; |
316 | | #if JXL_DEBUG_DOT_DETECT |
317 | | for (size_t i = 0; i < pixels.size(); i++) { |
318 | | fprintf(stderr, "(%d,%d) ", pixels[i].x, pixels[i].y); |
319 | | } |
320 | | fprintf(stderr, "\n"); |
321 | | #endif // JXL_DEBUG_DOT_DETECT |
322 | 0 | Rect bounds = BoundingRectangle(pixels); |
323 | 0 | if (bounds.xsize() < maxWindow && bounds.ysize() < maxWindow) { |
324 | 0 | ConnectedComponent cc{bounds, std::move(pixels)}; |
325 | 0 | cc.CompStats(energy, rect, kExtraRect); |
326 | 0 | if (cc.score < minScore) continue; |
327 | 0 | JXL_DEBUG(JXL_DEBUG_DOT_DETECT, |
328 | 0 | "cc mode: (%d,%d), max: %f, bgMean: %f bgVar: " |
329 | 0 | "%f bound:(%" PRIuS ",%" PRIuS ",%" PRIuS ",%" PRIuS ")\n", |
330 | 0 | cc.mode.x, cc.mode.y, cc.maxEnergy, cc.meanEnergy, |
331 | 0 | cc.varEnergy, cc.bounds.x0(), cc.bounds.y0(), |
332 | 0 | cc.bounds.xsize(), cc.bounds.ysize()); |
333 | 0 | ans.push_back(cc); |
334 | 0 | } |
335 | 0 | } |
336 | 0 | } |
337 | 0 | } |
338 | 0 | return ans; |
339 | 0 | } |
340 | | |
341 | | // TODO(sggonzalez): Adapt this function for the different color spaces or |
342 | | // remove it if the color space with the best performance does not need it |
343 | | void ComputeDotLosses(GaussianEllipse* ellipse, const ConnectedComponent& cc, |
344 | | const Rect& rect, const Image3F& img, |
345 | 0 | const Image3F& background) { |
346 | 0 | const int rectBounds = 2; |
347 | 0 | const double kIntensityR = 0.0; // 0.015; |
348 | 0 | const double kSigmaR = 0.0; // 0.01; |
349 | 0 | const double kZeroEpsilon = 0.1; // Tolerance to consider a value negative |
350 | 0 | double ct = cos(ellipse->angle); |
351 | 0 | double st = sin(ellipse->angle); |
352 | 0 | const std::array<double, 3> channelGains{{1.0, 1.0, 1.0}}; |
353 | 0 | int N = 0; |
354 | 0 | ellipse->l1_loss = 0.0; |
355 | 0 | ellipse->l2_loss = 0.0; |
356 | 0 | ellipse->neg_pixels = 0; |
357 | 0 | ellipse->neg_value.fill(0.0); |
358 | 0 | double distMeanModeSq = (cc.mode.x - ellipse->x) * (cc.mode.x - ellipse->x) + |
359 | 0 | (cc.mode.y - ellipse->y) * (cc.mode.y - ellipse->y); |
360 | 0 | ellipse->custom_loss = 0.0; |
361 | 0 | for (int c = 0; c < 3; c++) { |
362 | 0 | for (int sy = -rectBounds; |
363 | 0 | sy < (static_cast<int>(cc.bounds.ysize()) + rectBounds); sy++) { |
364 | 0 | int y = sy + cc.bounds.y0(); |
365 | 0 | if (y < 0 || static_cast<size_t>(y) >= rect.ysize()) continue; |
366 | 0 | const float* JXL_RESTRICT row = rect.ConstPlaneRow(img, c, y); |
367 | | // bgrow is only used if kOptimizeBackground is false. |
368 | | // NOLINTNEXTLINE(clang-analyzer-deadcode.DeadStores) |
369 | 0 | const float* JXL_RESTRICT bgrow = rect.ConstPlaneRow(background, c, y); |
370 | 0 | for (int sx = -rectBounds; |
371 | 0 | sx < (static_cast<int>(cc.bounds.xsize()) + rectBounds); sx++) { |
372 | 0 | int x = sx + cc.bounds.x0(); |
373 | 0 | if (x < 0 || static_cast<size_t>(x) >= rect.xsize()) continue; |
374 | 0 | double target = row[x]; |
375 | 0 | double dotDelta = DotGaussianModel( |
376 | 0 | x - ellipse->x, y - ellipse->y, ct, st, ellipse->sigma_x, |
377 | 0 | ellipse->sigma_y, ellipse->intensity[c]); |
378 | 0 | if (dotDelta > target + kZeroEpsilon) { |
379 | 0 | ellipse->neg_pixels++; |
380 | 0 | ellipse->neg_value[c] += dotDelta - target; |
381 | 0 | } |
382 | 0 | double bkg = kOptimizeBackground ? ellipse->bgColor[c] : bgrow[x]; |
383 | 0 | double pred = bkg + dotDelta; |
384 | 0 | double diff = target - pred; |
385 | 0 | double l2 = channelGains[c] * diff * diff; |
386 | 0 | double l1 = channelGains[c] * std::fabs(diff); |
387 | 0 | ellipse->l2_loss += l2; |
388 | 0 | ellipse->l1_loss += l1; |
389 | 0 | double w = DotGaussianModel(x - cc.mode.x, y - cc.mode.y, 1.0, 0.0, |
390 | 0 | 1.0 + ellipse->sigma_x, |
391 | 0 | 1.0 + ellipse->sigma_y, 1.0); |
392 | 0 | ellipse->custom_loss += w * l2; |
393 | 0 | N++; |
394 | 0 | } |
395 | 0 | } |
396 | 0 | } |
397 | 0 | ellipse->l2_loss /= N; |
398 | 0 | ellipse->custom_loss /= N; |
399 | 0 | ellipse->custom_loss += 20.0 * distMeanModeSq + ellipse->neg_value[1]; |
400 | 0 | ellipse->l1_loss /= N; |
401 | 0 | double ridgeTerm = kSigmaR * ellipse->sigma_x + kSigmaR * ellipse->sigma_y; |
402 | 0 | for (int c = 0; c < 3; c++) { |
403 | 0 | ridgeTerm += kIntensityR * ellipse->intensity[c] * ellipse->intensity[c]; |
404 | 0 | } |
405 | 0 | ellipse->ridge_loss = ellipse->l2_loss + ridgeTerm; |
406 | 0 | } |
407 | | |
408 | | StatusOr<GaussianEllipse> FitGaussianFast(const ConnectedComponent& cc, |
409 | | const Rect& rect, const Image3F& img, |
410 | 0 | const Image3F& background) { |
411 | 0 | constexpr bool leastSqIntensity = true; |
412 | 0 | constexpr double kEpsilon = 1e-6; |
413 | 0 | GaussianEllipse ans; |
414 | 0 | constexpr int kRectBounds = (kEllipseWindowSize >> 1); |
415 | | |
416 | | // Compute the 1st and 2nd moments of the CC |
417 | 0 | double sum = 0.0; |
418 | 0 | int N = 0; |
419 | 0 | std::array<double, 3> m1{{0.0, 0.0, 0.0}}; |
420 | 0 | std::array<double, 3> m2{{0.0, 0.0, 0.0}}; |
421 | 0 | std::array<double, 3> color{{0.0, 0.0, 0.0}}; |
422 | 0 | std::array<double, 3> bgColor{{0.0, 0.0, 0.0}}; |
423 | |
|
424 | 0 | JXL_DEBUG(JXL_DEBUG_DOT_DETECT, |
425 | 0 | "%" PRIuS " %" PRIuS " %" PRIuS " %" PRIuS "\n", cc.bounds.x0(), |
426 | 0 | cc.bounds.y0(), cc.bounds.xsize(), cc.bounds.ysize()); |
427 | 0 | for (int c = 0; c < 3; c++) { |
428 | 0 | color[c] = rect.ConstPlaneRow(img, c, cc.mode.y)[cc.mode.x] - |
429 | 0 | rect.ConstPlaneRow(background, c, cc.mode.y)[cc.mode.x]; |
430 | 0 | } |
431 | 0 | double sign = (color[1] > 0) ? 1 : -1; |
432 | 0 | for (int sy = -kRectBounds; sy <= kRectBounds; sy++) { |
433 | 0 | int y = sy + cc.mode.y; |
434 | 0 | if (y < 0 || static_cast<size_t>(y) >= rect.ysize()) continue; |
435 | 0 | const float* JXL_RESTRICT row = rect.ConstPlaneRow(img, 1, y); |
436 | 0 | const float* JXL_RESTRICT bgrow = rect.ConstPlaneRow(background, 1, y); |
437 | 0 | for (int sx = -kRectBounds; sx <= kRectBounds; sx++) { |
438 | 0 | int x = sx + cc.mode.x; |
439 | 0 | if (x < 0 || static_cast<size_t>(x) >= rect.xsize()) continue; |
440 | 0 | double w = std::max(kEpsilon, sign * (row[x] - bgrow[x])); |
441 | 0 | sum += w; |
442 | |
|
443 | 0 | m1[0] += w * x; |
444 | 0 | m1[1] += w * y; |
445 | 0 | m2[0] += w * x * x; |
446 | 0 | m2[1] += w * x * y; |
447 | 0 | m2[2] += w * y * y; |
448 | 0 | for (int c = 0; c < 3; c++) { |
449 | 0 | bgColor[c] += rect.ConstPlaneRow(background, c, y)[x]; |
450 | 0 | } |
451 | 0 | N++; |
452 | 0 | } |
453 | 0 | } |
454 | 0 | JXL_ENSURE(N > 0); |
455 | | |
456 | 0 | for (int i = 0; i < 3; i++) { |
457 | 0 | m1[i] /= sum; |
458 | 0 | m2[i] /= sum; |
459 | 0 | bgColor[i] /= N; |
460 | 0 | } |
461 | | |
462 | | // Some magic constants |
463 | 0 | constexpr double kSigmaMult = 1.0; |
464 | 0 | constexpr std::array<double, 3> kScaleMult{{1.1, 1.1, 1.1}}; |
465 | | |
466 | | // Now set the parameters of the Gaussian |
467 | 0 | ans.x = m1[0]; |
468 | 0 | ans.y = m1[1]; |
469 | 0 | for (int j = 0; j < 3; j++) { |
470 | 0 | ans.intensity[j] = kScaleMult[j] * color[j]; |
471 | 0 | } |
472 | |
|
473 | 0 | Matrix2x2 Sigma; |
474 | 0 | Vector2 d; |
475 | 0 | Matrix2x2 U; |
476 | 0 | Sigma[0][0] = m2[0] - m1[0] * m1[0]; |
477 | 0 | Sigma[1][1] = m2[2] - m1[1] * m1[1]; |
478 | 0 | Sigma[0][1] = Sigma[1][0] = m2[1] - m1[0] * m1[1]; |
479 | 0 | ConvertToDiagonal(Sigma, d, U); |
480 | 0 | Vector2& u = U[1]; |
481 | 0 | int p1 = 0; |
482 | 0 | int p2 = 1; |
483 | 0 | if (d[0] < d[1]) std::swap(p1, p2); |
484 | 0 | ans.sigma_x = kSigmaMult * d[p1]; |
485 | 0 | ans.sigma_y = kSigmaMult * d[p2]; |
486 | 0 | ans.angle = std::atan2(u[p1], u[p2]); |
487 | 0 | ans.l2_loss = 0.0; |
488 | 0 | ans.bgColor = bgColor; |
489 | 0 | if (leastSqIntensity) { |
490 | 0 | GaussianEllipse* ellipse = &ans; |
491 | 0 | double ct = cos(ans.angle); |
492 | 0 | double st = sin(ans.angle); |
493 | | // Estimate intensity with least squares (fixed background) |
494 | 0 | for (int c = 0; c < 3; c++) { |
495 | 0 | double gg = 0.0; |
496 | 0 | double gd = 0.0; |
497 | 0 | int yc = static_cast<int>(cc.mode.y); |
498 | 0 | int xc = static_cast<int>(cc.mode.x); |
499 | 0 | for (int y = yc - kRectBounds; y <= yc + kRectBounds; y++) { |
500 | 0 | if (y < 0 || static_cast<size_t>(y) >= rect.ysize()) continue; |
501 | 0 | const float* JXL_RESTRICT row = rect.ConstPlaneRow(img, c, y); |
502 | 0 | const float* JXL_RESTRICT bgrow = rect.ConstPlaneRow(background, c, y); |
503 | 0 | for (int x = xc - kRectBounds; x <= xc + kRectBounds; x++) { |
504 | 0 | if (x < 0 || static_cast<size_t>(x) >= rect.xsize()) continue; |
505 | 0 | double target = row[x] - bgrow[x]; |
506 | 0 | double gaussian = |
507 | 0 | DotGaussianModel(x - ellipse->x, y - ellipse->y, ct, st, |
508 | 0 | ellipse->sigma_x, ellipse->sigma_y, 1.0); |
509 | 0 | gg += gaussian * gaussian; |
510 | 0 | gd += gaussian * target; |
511 | 0 | } |
512 | 0 | } |
513 | 0 | ans.intensity[c] = gd / (gg + 1e-6); // Regularized least squares |
514 | 0 | } |
515 | 0 | } |
516 | 0 | ComputeDotLosses(&ans, cc, rect, img, background); |
517 | 0 | return ans; |
518 | 0 | } |
519 | | |
520 | | StatusOr<GaussianEllipse> FitGaussian(const ConnectedComponent& cc, |
521 | | const Rect& rect, const Image3F& img, |
522 | 0 | const Image3F& background) { |
523 | 0 | JXL_ASSIGN_OR_RETURN(GaussianEllipse ellipse, |
524 | 0 | FitGaussianFast(cc, rect, img, background)); |
525 | 0 | if (ellipse.sigma_x < ellipse.sigma_y) { |
526 | 0 | std::swap(ellipse.sigma_x, ellipse.sigma_y); |
527 | 0 | ellipse.angle += kPi / 2.0; |
528 | 0 | } |
529 | 0 | ellipse.angle -= kPi * std::floor(ellipse.angle / kPi); |
530 | 0 | if (fabs(ellipse.angle - kPi) < 1e-6 || fabs(ellipse.angle) < 1e-6) { |
531 | 0 | ellipse.angle = 0.0; |
532 | 0 | } |
533 | 0 | JXL_ENSURE(ellipse.angle >= 0 && ellipse.angle <= kPi && |
534 | 0 | ellipse.sigma_x >= ellipse.sigma_y); |
535 | 0 | JXL_DEBUG(JXL_DEBUG_DOT_DETECT, |
536 | 0 | "Ellipse mu=(%lf,%lf) sigma=(%lf,%lf) angle=%lf " |
537 | 0 | "intensity=(%lf,%lf,%lf) bg=(%lf,%lf,%lf) l2_loss=%lf " |
538 | 0 | "custom_loss=%lf, neg_pix=%" PRIuS ", neg_v=(%lf,%lf,%lf)\n", |
539 | 0 | ellipse.x, ellipse.y, ellipse.sigma_x, ellipse.sigma_y, |
540 | 0 | ellipse.angle, ellipse.intensity[0], ellipse.intensity[1], |
541 | 0 | ellipse.intensity[2], ellipse.bgColor[0], ellipse.bgColor[1], |
542 | 0 | ellipse.bgColor[2], ellipse.l2_loss, ellipse.custom_loss, |
543 | 0 | ellipse.neg_pixels, ellipse.neg_value[0], ellipse.neg_value[1], |
544 | 0 | ellipse.neg_value[2]); |
545 | 0 | return ellipse; |
546 | 0 | } |
547 | | |
548 | | } // namespace |
549 | | |
550 | | StatusOr<std::vector<PatchInfo>> DetectGaussianEllipses( |
551 | | const Image3F& opsin, const Rect& rect, const GaussianDetectParams& params, |
552 | 0 | const EllipseQuantParams& qParams, ThreadPool* pool) { |
553 | 0 | JxlMemoryManager* memory_manager = opsin.memory_manager(); |
554 | 0 | std::vector<PatchInfo> dots; |
555 | 0 | JXL_ASSIGN_OR_RETURN( |
556 | 0 | Image3F smooth, |
557 | 0 | Image3F::Create(memory_manager, opsin.xsize(), opsin.ysize())); |
558 | 0 | JXL_ASSIGN_OR_RETURN(ImageF energy, ComputeEnergyImage(opsin, &smooth, pool)); |
559 | 0 | JXL_ASSIGN_OR_RETURN(std::vector<ConnectedComponent> components, |
560 | 0 | FindCC(energy, rect, params.t_low, params.t_high, |
561 | 0 | params.maxWinSize, params.minScore)); |
562 | 0 | size_t numCC = |
563 | 0 | std::min(params.maxCC, (components.size() * params.percCC) / 100); |
564 | 0 | if (components.size() > numCC) { |
565 | 0 | std::sort( |
566 | 0 | components.begin(), components.end(), |
567 | 0 | [](const ConnectedComponent& a, const ConnectedComponent& b) -> bool { |
568 | 0 | return a.score > b.score; |
569 | 0 | }); |
570 | 0 | components.erase(components.begin() + numCC, components.end()); |
571 | 0 | } |
572 | 0 | for (const auto& cc : components) { |
573 | 0 | JXL_ASSIGN_OR_RETURN(GaussianEllipse ellipse, |
574 | 0 | FitGaussian(cc, rect, opsin, smooth)); |
575 | 0 | if (ellipse.x < 0.0 || |
576 | 0 | std::ceil(ellipse.x) >= static_cast<double>(rect.xsize()) || |
577 | 0 | ellipse.y < 0.0 || |
578 | 0 | std::ceil(ellipse.y) >= static_cast<double>(rect.ysize())) { |
579 | 0 | continue; |
580 | 0 | } |
581 | 0 | if (ellipse.neg_pixels > params.maxNegPixels) continue; |
582 | 0 | double intensity = 0.21 * ellipse.intensity[0] + |
583 | 0 | 0.72 * ellipse.intensity[1] + |
584 | 0 | 0.07 * ellipse.intensity[2]; |
585 | 0 | double intensitySq = intensity * intensity; |
586 | | // for (int c = 0; c < 3; c++) { |
587 | | // intensitySq += ellipse.intensity[c] * ellipse.intensity[c]; |
588 | | //} |
589 | 0 | double sqDistMeanMode = (ellipse.x - cc.mode.x) * (ellipse.x - cc.mode.x) + |
590 | 0 | (ellipse.y - cc.mode.y) * (ellipse.y - cc.mode.y); |
591 | 0 | if (ellipse.l2_loss < params.maxL2Loss && |
592 | 0 | ellipse.custom_loss < params.maxCustomLoss && |
593 | 0 | intensitySq > (params.minIntensity * params.minIntensity) && |
594 | 0 | sqDistMeanMode < params.maxDistMeanMode * params.maxDistMeanMode) { |
595 | 0 | size_t x0 = cc.bounds.x0(); |
596 | 0 | size_t y0 = cc.bounds.y0(); |
597 | 0 | dots.emplace_back(); |
598 | 0 | dots.back().second.emplace_back(x0, y0); |
599 | 0 | QuantizedPatch& patch = dots.back().first; |
600 | 0 | patch.xsize = cc.bounds.xsize(); |
601 | 0 | patch.ysize = cc.bounds.ysize(); |
602 | 0 | for (size_t y = 0; y < patch.ysize; y++) { |
603 | 0 | for (size_t x = 0; x < patch.xsize; x++) { |
604 | 0 | for (size_t c = 0; c < 3; c++) { |
605 | 0 | patch.fpixels[c][y * patch.xsize + x] = |
606 | 0 | rect.ConstPlaneRow(opsin, c, y0 + y)[x0 + x] - |
607 | 0 | rect.ConstPlaneRow(smooth, c, y0 + y)[x0 + x]; |
608 | 0 | } |
609 | 0 | } |
610 | 0 | } |
611 | 0 | } |
612 | 0 | } |
613 | 0 | return dots; |
614 | 0 | } |
615 | | |
616 | | } // namespace jxl |
617 | | #endif // HWY_ONCE |