/src/libjxl/lib/jxl/enc_detect_dots.cc
Line | Count | Source (jump to first uncovered line) |
1 | | // Copyright (c) the JPEG XL Project Authors. All rights reserved. |
2 | | // |
3 | | // Use of this source code is governed by a BSD-style |
4 | | // license that can be found in the LICENSE file. |
5 | | |
6 | | #include "lib/jxl/enc_detect_dots.h" |
7 | | |
8 | | #include <jxl/memory_manager.h> |
9 | | |
10 | | #include <algorithm> |
11 | | #include <array> |
12 | | #include <cmath> |
13 | | #include <cstdint> |
14 | | #include <cstdio> |
15 | | #include <utility> |
16 | | #include <vector> |
17 | | |
18 | | #include "lib/jxl/enc_patch_dictionary.h" |
19 | | |
20 | | #undef HWY_TARGET_INCLUDE |
21 | | #define HWY_TARGET_INCLUDE "lib/jxl/enc_detect_dots.cc" |
22 | | #include <hwy/foreach_target.h> |
23 | | #include <hwy/highway.h> |
24 | | |
25 | | #include "lib/jxl/base/common.h" |
26 | | #include "lib/jxl/base/compiler_specific.h" |
27 | | #include "lib/jxl/base/data_parallel.h" |
28 | | #include "lib/jxl/base/printf_macros.h" |
29 | | #include "lib/jxl/base/rect.h" |
30 | | #include "lib/jxl/base/status.h" |
31 | | #include "lib/jxl/convolve.h" |
32 | | #include "lib/jxl/enc_linalg.h" |
33 | | #include "lib/jxl/image.h" |
34 | | #include "lib/jxl/image_ops.h" |
35 | | |
36 | | // Set JXL_DEBUG_DOT_DETECT to 1 to enable debugging. |
37 | | #ifndef JXL_DEBUG_DOT_DETECT |
38 | | #define JXL_DEBUG_DOT_DETECT 0 |
39 | | #endif |
40 | | |
41 | | HWY_BEFORE_NAMESPACE(); |
42 | | namespace jxl { |
43 | | namespace HWY_NAMESPACE { |
44 | | |
45 | | // These templates are not found via ADL. |
46 | | using hwy::HWY_NAMESPACE::Add; |
47 | | using hwy::HWY_NAMESPACE::Mul; |
48 | | using hwy::HWY_NAMESPACE::Sub; |
49 | | |
50 | | StatusOr<ImageF> SumOfSquareDifferences(const Image3F& forig, |
51 | | const Image3F& smooth, |
52 | 0 | ThreadPool* pool) { |
53 | 0 | const HWY_FULL(float) d; |
54 | 0 | const auto color_coef0 = Set(d, 0.0f); |
55 | 0 | const auto color_coef1 = Set(d, 10.0f); |
56 | 0 | const auto color_coef2 = Set(d, 0.0f); |
57 | 0 | JxlMemoryManager* memory_manager = forig.memory_manager(); |
58 | |
|
59 | 0 | JXL_ASSIGN_OR_RETURN( |
60 | 0 | ImageF sum_of_squares, |
61 | 0 | ImageF::Create(memory_manager, forig.xsize(), forig.ysize())); |
62 | 0 | const auto process_row = [&](const uint32_t task, size_t thread) -> Status { |
63 | 0 | const size_t y = static_cast<size_t>(task); |
64 | 0 | const float* JXL_RESTRICT orig_row0 = forig.Plane(0).ConstRow(y); |
65 | 0 | const float* JXL_RESTRICT orig_row1 = forig.Plane(1).ConstRow(y); |
66 | 0 | const float* JXL_RESTRICT orig_row2 = forig.Plane(2).ConstRow(y); |
67 | 0 | const float* JXL_RESTRICT smooth_row0 = smooth.Plane(0).ConstRow(y); |
68 | 0 | const float* JXL_RESTRICT smooth_row1 = smooth.Plane(1).ConstRow(y); |
69 | 0 | const float* JXL_RESTRICT smooth_row2 = smooth.Plane(2).ConstRow(y); |
70 | 0 | float* JXL_RESTRICT sos_row = sum_of_squares.Row(y); |
71 | |
|
72 | 0 | for (size_t x = 0; x < forig.xsize(); x += Lanes(d)) { |
73 | 0 | auto v0 = Sub(Load(d, orig_row0 + x), Load(d, smooth_row0 + x)); |
74 | 0 | auto v1 = Sub(Load(d, orig_row1 + x), Load(d, smooth_row1 + x)); |
75 | 0 | auto v2 = Sub(Load(d, orig_row2 + x), Load(d, smooth_row2 + x)); |
76 | 0 | v0 = Mul(Mul(v0, v0), color_coef0); |
77 | 0 | v1 = Mul(Mul(v1, v1), color_coef1); |
78 | 0 | v2 = Mul(Mul(v2, v2), color_coef2); |
79 | 0 | const auto sos = Add(v0, Add(v1, v2)); // weighted sum of square diffs |
80 | 0 | Store(sos, d, sos_row + x); |
81 | 0 | } |
82 | 0 | return true; |
83 | 0 | }; Unexecuted instantiation: enc_detect_dots.cc:jxl::N_SSE4::SumOfSquareDifferences(jxl::Image3<float> const&, jxl::Image3<float> const&, jxl::ThreadPool*)::$_0::operator()(unsigned int, unsigned long) const Unexecuted instantiation: enc_detect_dots.cc:jxl::N_AVX2::SumOfSquareDifferences(jxl::Image3<float> const&, jxl::Image3<float> const&, jxl::ThreadPool*)::$_0::operator()(unsigned int, unsigned long) const Unexecuted instantiation: enc_detect_dots.cc:jxl::N_SSE2::SumOfSquareDifferences(jxl::Image3<float> const&, jxl::Image3<float> const&, jxl::ThreadPool*)::$_0::operator()(unsigned int, unsigned long) const |
84 | 0 | JXL_RETURN_IF_ERROR(RunOnPool(pool, 0, forig.ysize(), ThreadPool::NoInit, |
85 | 0 | process_row, "ComputeEnergyImage")); |
86 | 0 | return sum_of_squares; |
87 | 0 | } Unexecuted instantiation: jxl::N_SSE4::SumOfSquareDifferences(jxl::Image3<float> const&, jxl::Image3<float> const&, jxl::ThreadPool*) Unexecuted instantiation: jxl::N_AVX2::SumOfSquareDifferences(jxl::Image3<float> const&, jxl::Image3<float> const&, jxl::ThreadPool*) Unexecuted instantiation: jxl::N_SSE2::SumOfSquareDifferences(jxl::Image3<float> const&, jxl::Image3<float> const&, jxl::ThreadPool*) |
88 | | |
89 | | // NOLINTNEXTLINE(google-readability-namespace-comments) |
90 | | } // namespace HWY_NAMESPACE |
91 | | } // namespace jxl |
92 | | HWY_AFTER_NAMESPACE(); |
93 | | |
94 | | #if HWY_ONCE |
95 | | namespace jxl { |
96 | | HWY_EXPORT(SumOfSquareDifferences); // Local function |
97 | | |
98 | | const int kEllipseWindowSize = 5; |
99 | | |
100 | | namespace { |
101 | | struct GaussianEllipse { |
102 | | double x; // position in x |
103 | | double y; // position in y |
104 | | double sigma_x; // scale in x |
105 | | double sigma_y; // scale in y |
106 | | double angle; // ellipse rotation in radians |
107 | | std::array<double, 3> intensity; // intensity in each channel |
108 | | |
109 | | // The following variables do not need to be encoded |
110 | | double l2_loss; // error after the Gaussian was fit |
111 | | double l1_loss; |
112 | | double ridge_loss; // the l2_loss plus regularization term |
113 | | double custom_loss; // experimental custom loss |
114 | | std::array<double, 3> bgColor; // best background color |
115 | | size_t neg_pixels; // number of negative pixels when subtracting dot |
116 | | std::array<double, 3> neg_value; // debt due to channel truncation |
117 | | }; |
118 | | double DotGaussianModel(double dx, double dy, double ct, double st, |
119 | 0 | double sigma_x, double sigma_y, double intensity) { |
120 | 0 | double rx = ct * dx + st * dy; |
121 | 0 | double ry = -st * dx + ct * dy; |
122 | 0 | double md = (rx * rx / sigma_x) + (ry * ry / sigma_y); |
123 | 0 | double value = intensity * exp(-0.5 * md); |
124 | 0 | return value; |
125 | 0 | } |
126 | | |
127 | | constexpr bool kOptimizeBackground = true; |
128 | | |
129 | | // Gaussian that smooths noise but preserves dots |
130 | 0 | const WeightsSeparable5& WeightsSeparable5Gaussian0_65() { |
131 | 0 | constexpr float w0 = 0.558311f; |
132 | 0 | constexpr float w1 = 0.210395f; |
133 | 0 | constexpr float w2 = 0.010449f; |
134 | 0 | static constexpr WeightsSeparable5 weights = { |
135 | 0 | {HWY_REP4(w0), HWY_REP4(w1), HWY_REP4(w2)}, |
136 | 0 | {HWY_REP4(w0), HWY_REP4(w1), HWY_REP4(w2)}}; |
137 | 0 | return weights; |
138 | 0 | } |
139 | | |
140 | | // (Iterated) Gaussian that removes dots. |
141 | 0 | const WeightsSeparable5& WeightsSeparable5Gaussian3() { |
142 | 0 | constexpr float w0 = 0.222338f; |
143 | 0 | constexpr float w1 = 0.210431f; |
144 | 0 | constexpr float w2 = 0.1784f; |
145 | 0 | static constexpr WeightsSeparable5 weights = { |
146 | 0 | {HWY_REP4(w0), HWY_REP4(w1), HWY_REP4(w2)}, |
147 | 0 | {HWY_REP4(w0), HWY_REP4(w1), HWY_REP4(w2)}}; |
148 | 0 | return weights; |
149 | 0 | } |
150 | | |
151 | | StatusOr<ImageF> ComputeEnergyImage(const Image3F& orig, Image3F* smooth, |
152 | 0 | ThreadPool* pool) { |
153 | 0 | JxlMemoryManager* memory_manager = orig.memory_manager(); |
154 | | // Prepare guidance images for dot selection. |
155 | 0 | JXL_ASSIGN_OR_RETURN( |
156 | 0 | Image3F forig, |
157 | 0 | Image3F::Create(memory_manager, orig.xsize(), orig.ysize())); |
158 | 0 | JXL_ASSIGN_OR_RETURN( |
159 | 0 | *smooth, Image3F::Create(memory_manager, orig.xsize(), orig.ysize())); |
160 | 0 | Rect rect(orig); |
161 | |
|
162 | 0 | const auto& weights1 = WeightsSeparable5Gaussian0_65(); |
163 | 0 | const auto& weights3 = WeightsSeparable5Gaussian3(); |
164 | |
|
165 | 0 | for (size_t c = 0; c < 3; ++c) { |
166 | | // Use forig as temporary storage to reduce memory and keep it warmer. |
167 | 0 | JXL_RETURN_IF_ERROR( |
168 | 0 | Separable5(orig.Plane(c), rect, weights3, pool, &forig.Plane(c))); |
169 | 0 | JXL_RETURN_IF_ERROR( |
170 | 0 | Separable5(forig.Plane(c), rect, weights3, pool, &smooth->Plane(c))); |
171 | 0 | JXL_RETURN_IF_ERROR( |
172 | 0 | Separable5(orig.Plane(c), rect, weights1, pool, &forig.Plane(c))); |
173 | 0 | } |
174 | | |
175 | 0 | return HWY_DYNAMIC_DISPATCH(SumOfSquareDifferences)(forig, *smooth, pool); |
176 | 0 | } |
177 | | |
178 | | struct Pixel { |
179 | | int x; |
180 | | int y; |
181 | | }; |
182 | | |
183 | 0 | Pixel operator+(const Pixel& a, const Pixel& b) { |
184 | 0 | return Pixel{a.x + b.x, a.y + b.y}; |
185 | 0 | } |
186 | | |
187 | | // Maximum area in pixels of a ellipse |
188 | | const size_t kMaxCCSize = 1000; |
189 | | |
190 | | // Extracts a connected component from a Binary image where seed is part |
191 | | // of the component |
192 | | bool ExtractComponent(const Rect& rect, ImageF* img, std::vector<Pixel>* pixels, |
193 | 0 | const Pixel& seed, double threshold) { |
194 | 0 | static const std::vector<Pixel> neighbors{{1, -1}, {1, 0}, {1, 1}, {0, -1}, |
195 | 0 | {0, 1}, {-1, -1}, {-1, 1}, {1, 0}}; |
196 | 0 | std::vector<Pixel> q{seed}; |
197 | 0 | while (!q.empty()) { |
198 | 0 | Pixel current = q.back(); |
199 | 0 | q.pop_back(); |
200 | 0 | pixels->push_back(current); |
201 | 0 | if (pixels->size() > kMaxCCSize) return false; |
202 | 0 | for (const Pixel& delta : neighbors) { |
203 | 0 | Pixel child = current + delta; |
204 | 0 | if (child.x >= 0 && static_cast<size_t>(child.x) < rect.xsize() && |
205 | 0 | child.y >= 0 && static_cast<size_t>(child.y) < rect.ysize()) { |
206 | 0 | float* value = &rect.Row(img, child.y)[child.x]; |
207 | 0 | if (*value > threshold) { |
208 | 0 | *value = 0.0; |
209 | 0 | q.push_back(child); |
210 | 0 | } |
211 | 0 | } |
212 | 0 | } |
213 | 0 | } |
214 | 0 | return true; |
215 | 0 | } |
216 | | |
217 | 0 | inline bool PointInRect(const Rect& r, const Pixel& p) { |
218 | 0 | return (static_cast<size_t>(p.x) >= r.x0() && |
219 | 0 | static_cast<size_t>(p.x) < (r.x0() + r.xsize()) && |
220 | 0 | static_cast<size_t>(p.y) >= r.y0() && |
221 | 0 | static_cast<size_t>(p.y) < (r.y0() + r.ysize())); |
222 | 0 | } |
223 | | |
224 | | struct ConnectedComponent { |
225 | | ConnectedComponent(const Rect& bounds, const std::vector<Pixel>&& pixels) |
226 | 0 | : bounds(bounds), pixels(pixels) {} |
227 | | Rect bounds; |
228 | | std::vector<Pixel> pixels; |
229 | | float maxEnergy; |
230 | | float meanEnergy; |
231 | | float varEnergy; |
232 | | float meanBg; |
233 | | float varBg; |
234 | | float score; |
235 | | Pixel mode; |
236 | | |
237 | 0 | void CompStats(const ImageF& energy, const Rect& rect, int extra) { |
238 | 0 | maxEnergy = 0.0; |
239 | 0 | meanEnergy = 0.0; |
240 | 0 | varEnergy = 0.0; |
241 | 0 | meanBg = 0.0; |
242 | 0 | varBg = 0.0; |
243 | 0 | int nIn = 0; |
244 | 0 | int nOut = 0; |
245 | 0 | mode.x = 0; |
246 | 0 | mode.y = 0; |
247 | 0 | for (int sy = -extra; sy < (static_cast<int>(bounds.ysize()) + extra); |
248 | 0 | sy++) { |
249 | 0 | int y = sy + static_cast<int>(bounds.y0()); |
250 | 0 | if (y < 0 || static_cast<size_t>(y) >= rect.ysize()) continue; |
251 | 0 | const float* JXL_RESTRICT erow = rect.ConstRow(energy, y); |
252 | 0 | for (int sx = -extra; sx < (static_cast<int>(bounds.xsize()) + extra); |
253 | 0 | sx++) { |
254 | 0 | int x = sx + static_cast<int>(bounds.x0()); |
255 | 0 | if (x < 0 || static_cast<size_t>(x) >= rect.xsize()) continue; |
256 | 0 | if (erow[x] > maxEnergy) { |
257 | 0 | maxEnergy = erow[x]; |
258 | 0 | mode.x = x; |
259 | 0 | mode.y = y; |
260 | 0 | } |
261 | 0 | if (PointInRect(bounds, Pixel{x, y})) { |
262 | 0 | meanEnergy += erow[x]; |
263 | 0 | varEnergy += erow[x] * erow[x]; |
264 | 0 | nIn++; |
265 | 0 | } else { |
266 | 0 | meanBg += erow[x]; |
267 | 0 | varBg += erow[x] * erow[x]; |
268 | 0 | nOut++; |
269 | 0 | } |
270 | 0 | } |
271 | 0 | } |
272 | 0 | meanEnergy = meanEnergy / nIn; |
273 | 0 | meanBg = meanBg / nOut; |
274 | 0 | varEnergy = (varEnergy / nIn) - meanEnergy * meanEnergy; |
275 | 0 | varBg = (varBg / nOut) - meanBg * meanBg; |
276 | 0 | score = (meanEnergy - meanBg) / std::sqrt(varBg); |
277 | 0 | } |
278 | | }; |
279 | | |
280 | 0 | Rect BoundingRectangle(const std::vector<Pixel>& pixels) { |
281 | 0 | JXL_DASSERT(!pixels.empty()); |
282 | 0 | int low_x; |
283 | 0 | int high_x; |
284 | 0 | int low_y; |
285 | 0 | int high_y; |
286 | 0 | low_x = high_x = pixels[0].x; |
287 | 0 | low_y = high_y = pixels[0].y; |
288 | 0 | for (const Pixel& p : pixels) { |
289 | 0 | low_x = std::min(low_x, p.x); |
290 | 0 | high_x = std::max(high_x, p.x); |
291 | 0 | low_y = std::min(low_y, p.y); |
292 | 0 | high_y = std::max(high_y, p.y); |
293 | 0 | } |
294 | 0 | return Rect(low_x, low_y, high_x - low_x + 1, high_y - low_y + 1); |
295 | 0 | } |
296 | | |
297 | | StatusOr<std::vector<ConnectedComponent>> FindCC(const ImageF& energy, |
298 | | const Rect& rect, double t_low, |
299 | | double t_high, |
300 | | uint32_t maxWindow, |
301 | 0 | double minScore) { |
302 | 0 | const int kExtraRect = 4; |
303 | 0 | JxlMemoryManager* memory_manager = energy.memory_manager(); |
304 | 0 | JXL_ASSIGN_OR_RETURN( |
305 | 0 | ImageF img, |
306 | 0 | ImageF::Create(memory_manager, energy.xsize(), energy.ysize())); |
307 | 0 | JXL_RETURN_IF_ERROR(CopyImageTo(energy, &img)); |
308 | 0 | std::vector<ConnectedComponent> ans; |
309 | 0 | for (size_t y = 0; y < rect.ysize(); y++) { |
310 | 0 | float* JXL_RESTRICT row = rect.Row(&img, y); |
311 | 0 | for (size_t x = 0; x < rect.xsize(); x++) { |
312 | 0 | if (row[x] > t_high) { |
313 | 0 | std::vector<Pixel> pixels; |
314 | 0 | row[x] = 0.0; |
315 | 0 | Pixel seed = Pixel{static_cast<int>(x), static_cast<int>(y)}; |
316 | 0 | bool success = ExtractComponent(rect, &img, &pixels, seed, t_low); |
317 | 0 | if (!success) continue; |
318 | | #if JXL_DEBUG_DOT_DETECT |
319 | | for (size_t i = 0; i < pixels.size(); i++) { |
320 | | fprintf(stderr, "(%d,%d) ", pixels[i].x, pixels[i].y); |
321 | | } |
322 | | fprintf(stderr, "\n"); |
323 | | #endif // JXL_DEBUG_DOT_DETECT |
324 | 0 | Rect bounds = BoundingRectangle(pixels); |
325 | 0 | if (bounds.xsize() < maxWindow && bounds.ysize() < maxWindow) { |
326 | 0 | ConnectedComponent cc{bounds, std::move(pixels)}; |
327 | 0 | cc.CompStats(energy, rect, kExtraRect); |
328 | 0 | if (cc.score < minScore) continue; |
329 | 0 | JXL_DEBUG(JXL_DEBUG_DOT_DETECT, |
330 | 0 | "cc mode: (%d,%d), max: %f, bgMean: %f bgVar: " |
331 | 0 | "%f bound:(%" PRIuS ",%" PRIuS ",%" PRIuS ",%" PRIuS ")\n", |
332 | 0 | cc.mode.x, cc.mode.y, cc.maxEnergy, cc.meanEnergy, |
333 | 0 | cc.varEnergy, cc.bounds.x0(), cc.bounds.y0(), |
334 | 0 | cc.bounds.xsize(), cc.bounds.ysize()); |
335 | 0 | ans.push_back(cc); |
336 | 0 | } |
337 | 0 | } |
338 | 0 | } |
339 | 0 | } |
340 | 0 | return ans; |
341 | 0 | } |
342 | | |
343 | | // TODO(sggonzalez): Adapt this function for the different color spaces or |
344 | | // remove it if the color space with the best performance does not need it |
345 | | void ComputeDotLosses(GaussianEllipse* ellipse, const ConnectedComponent& cc, |
346 | | const Rect& rect, const Image3F& img, |
347 | 0 | const Image3F& background) { |
348 | 0 | const int rectBounds = 2; |
349 | 0 | const double kIntensityR = 0.0; // 0.015; |
350 | 0 | const double kSigmaR = 0.0; // 0.01; |
351 | 0 | const double kZeroEpsilon = 0.1; // Tolerance to consider a value negative |
352 | 0 | double ct = cos(ellipse->angle); |
353 | 0 | double st = sin(ellipse->angle); |
354 | 0 | const std::array<double, 3> channelGains{{1.0, 1.0, 1.0}}; |
355 | 0 | int N = 0; |
356 | 0 | ellipse->l1_loss = 0.0; |
357 | 0 | ellipse->l2_loss = 0.0; |
358 | 0 | ellipse->neg_pixels = 0; |
359 | 0 | ellipse->neg_value.fill(0.0); |
360 | 0 | double distMeanModeSq = (cc.mode.x - ellipse->x) * (cc.mode.x - ellipse->x) + |
361 | 0 | (cc.mode.y - ellipse->y) * (cc.mode.y - ellipse->y); |
362 | 0 | ellipse->custom_loss = 0.0; |
363 | 0 | for (int c = 0; c < 3; c++) { |
364 | 0 | for (int sy = -rectBounds; |
365 | 0 | sy < (static_cast<int>(cc.bounds.ysize()) + rectBounds); sy++) { |
366 | 0 | int y = sy + cc.bounds.y0(); |
367 | 0 | if (y < 0 || static_cast<size_t>(y) >= rect.ysize()) continue; |
368 | 0 | const float* JXL_RESTRICT row = rect.ConstPlaneRow(img, c, y); |
369 | | // bgrow is only used if kOptimizeBackground is false. |
370 | | // NOLINTNEXTLINE(clang-analyzer-deadcode.DeadStores) |
371 | 0 | const float* JXL_RESTRICT bgrow = rect.ConstPlaneRow(background, c, y); |
372 | 0 | for (int sx = -rectBounds; |
373 | 0 | sx < (static_cast<int>(cc.bounds.xsize()) + rectBounds); sx++) { |
374 | 0 | int x = sx + cc.bounds.x0(); |
375 | 0 | if (x < 0 || static_cast<size_t>(x) >= rect.xsize()) continue; |
376 | 0 | double target = row[x]; |
377 | 0 | double dotDelta = DotGaussianModel( |
378 | 0 | x - ellipse->x, y - ellipse->y, ct, st, ellipse->sigma_x, |
379 | 0 | ellipse->sigma_y, ellipse->intensity[c]); |
380 | 0 | if (dotDelta > target + kZeroEpsilon) { |
381 | 0 | ellipse->neg_pixels++; |
382 | 0 | ellipse->neg_value[c] += dotDelta - target; |
383 | 0 | } |
384 | 0 | double bkg = kOptimizeBackground ? ellipse->bgColor[c] : bgrow[x]; |
385 | 0 | double pred = bkg + dotDelta; |
386 | 0 | double diff = target - pred; |
387 | 0 | double l2 = channelGains[c] * diff * diff; |
388 | 0 | double l1 = channelGains[c] * std::fabs(diff); |
389 | 0 | ellipse->l2_loss += l2; |
390 | 0 | ellipse->l1_loss += l1; |
391 | 0 | double w = DotGaussianModel(x - cc.mode.x, y - cc.mode.y, 1.0, 0.0, |
392 | 0 | 1.0 + ellipse->sigma_x, |
393 | 0 | 1.0 + ellipse->sigma_y, 1.0); |
394 | 0 | ellipse->custom_loss += w * l2; |
395 | 0 | N++; |
396 | 0 | } |
397 | 0 | } |
398 | 0 | } |
399 | 0 | ellipse->l2_loss /= N; |
400 | 0 | ellipse->custom_loss /= N; |
401 | 0 | ellipse->custom_loss += 20.0 * distMeanModeSq + ellipse->neg_value[1]; |
402 | 0 | ellipse->l1_loss /= N; |
403 | 0 | double ridgeTerm = kSigmaR * ellipse->sigma_x + kSigmaR * ellipse->sigma_y; |
404 | 0 | for (int c = 0; c < 3; c++) { |
405 | 0 | ridgeTerm += kIntensityR * ellipse->intensity[c] * ellipse->intensity[c]; |
406 | 0 | } |
407 | 0 | ellipse->ridge_loss = ellipse->l2_loss + ridgeTerm; |
408 | 0 | } |
409 | | |
410 | | StatusOr<GaussianEllipse> FitGaussianFast(const ConnectedComponent& cc, |
411 | | const Rect& rect, const Image3F& img, |
412 | 0 | const Image3F& background) { |
413 | 0 | constexpr bool leastSqIntensity = true; |
414 | 0 | constexpr double kEpsilon = 1e-6; |
415 | 0 | GaussianEllipse ans; |
416 | 0 | constexpr int kRectBounds = (kEllipseWindowSize >> 1); |
417 | | |
418 | | // Compute the 1st and 2nd moments of the CC |
419 | 0 | double sum = 0.0; |
420 | 0 | int N = 0; |
421 | 0 | std::array<double, 3> m1{{0.0, 0.0, 0.0}}; |
422 | 0 | std::array<double, 3> m2{{0.0, 0.0, 0.0}}; |
423 | 0 | std::array<double, 3> color{{0.0, 0.0, 0.0}}; |
424 | 0 | std::array<double, 3> bgColor{{0.0, 0.0, 0.0}}; |
425 | |
|
426 | 0 | JXL_DEBUG(JXL_DEBUG_DOT_DETECT, |
427 | 0 | "%" PRIuS " %" PRIuS " %" PRIuS " %" PRIuS "\n", cc.bounds.x0(), |
428 | 0 | cc.bounds.y0(), cc.bounds.xsize(), cc.bounds.ysize()); |
429 | 0 | for (int c = 0; c < 3; c++) { |
430 | 0 | color[c] = rect.ConstPlaneRow(img, c, cc.mode.y)[cc.mode.x] - |
431 | 0 | rect.ConstPlaneRow(background, c, cc.mode.y)[cc.mode.x]; |
432 | 0 | } |
433 | 0 | double sign = (color[1] > 0) ? 1 : -1; |
434 | 0 | for (int sy = -kRectBounds; sy <= kRectBounds; sy++) { |
435 | 0 | int y = sy + cc.mode.y; |
436 | 0 | if (y < 0 || static_cast<size_t>(y) >= rect.ysize()) continue; |
437 | 0 | const float* JXL_RESTRICT row = rect.ConstPlaneRow(img, 1, y); |
438 | 0 | const float* JXL_RESTRICT bgrow = rect.ConstPlaneRow(background, 1, y); |
439 | 0 | for (int sx = -kRectBounds; sx <= kRectBounds; sx++) { |
440 | 0 | int x = sx + cc.mode.x; |
441 | 0 | if (x < 0 || static_cast<size_t>(x) >= rect.xsize()) continue; |
442 | 0 | double w = std::max(kEpsilon, sign * (row[x] - bgrow[x])); |
443 | 0 | sum += w; |
444 | |
|
445 | 0 | m1[0] += w * x; |
446 | 0 | m1[1] += w * y; |
447 | 0 | m2[0] += w * x * x; |
448 | 0 | m2[1] += w * x * y; |
449 | 0 | m2[2] += w * y * y; |
450 | 0 | for (int c = 0; c < 3; c++) { |
451 | 0 | bgColor[c] += rect.ConstPlaneRow(background, c, y)[x]; |
452 | 0 | } |
453 | 0 | N++; |
454 | 0 | } |
455 | 0 | } |
456 | 0 | JXL_ENSURE(N > 0); |
457 | | |
458 | 0 | for (int i = 0; i < 3; i++) { |
459 | 0 | m1[i] /= sum; |
460 | 0 | m2[i] /= sum; |
461 | 0 | bgColor[i] /= N; |
462 | 0 | } |
463 | | |
464 | | // Some magic constants |
465 | 0 | constexpr double kSigmaMult = 1.0; |
466 | 0 | constexpr std::array<double, 3> kScaleMult{{1.1, 1.1, 1.1}}; |
467 | | |
468 | | // Now set the parameters of the Gaussian |
469 | 0 | ans.x = m1[0]; |
470 | 0 | ans.y = m1[1]; |
471 | 0 | for (int j = 0; j < 3; j++) { |
472 | 0 | ans.intensity[j] = kScaleMult[j] * color[j]; |
473 | 0 | } |
474 | |
|
475 | 0 | Matrix2x2 Sigma; |
476 | 0 | Vector2 d; |
477 | 0 | Matrix2x2 U; |
478 | 0 | Sigma[0][0] = m2[0] - m1[0] * m1[0]; |
479 | 0 | Sigma[1][1] = m2[2] - m1[1] * m1[1]; |
480 | 0 | Sigma[0][1] = Sigma[1][0] = m2[1] - m1[0] * m1[1]; |
481 | 0 | ConvertToDiagonal(Sigma, d, U); |
482 | 0 | Vector2& u = U[1]; |
483 | 0 | int p1 = 0; |
484 | 0 | int p2 = 1; |
485 | 0 | if (d[0] < d[1]) std::swap(p1, p2); |
486 | 0 | ans.sigma_x = kSigmaMult * d[p1]; |
487 | 0 | ans.sigma_y = kSigmaMult * d[p2]; |
488 | 0 | ans.angle = std::atan2(u[p1], u[p2]); |
489 | 0 | ans.l2_loss = 0.0; |
490 | 0 | ans.bgColor = bgColor; |
491 | 0 | if (leastSqIntensity) { |
492 | 0 | GaussianEllipse* ellipse = &ans; |
493 | 0 | double ct = cos(ans.angle); |
494 | 0 | double st = sin(ans.angle); |
495 | | // Estimate intensity with least squares (fixed background) |
496 | 0 | for (int c = 0; c < 3; c++) { |
497 | 0 | double gg = 0.0; |
498 | 0 | double gd = 0.0; |
499 | 0 | int yc = static_cast<int>(cc.mode.y); |
500 | 0 | int xc = static_cast<int>(cc.mode.x); |
501 | 0 | for (int y = yc - kRectBounds; y <= yc + kRectBounds; y++) { |
502 | 0 | if (y < 0 || static_cast<size_t>(y) >= rect.ysize()) continue; |
503 | 0 | const float* JXL_RESTRICT row = rect.ConstPlaneRow(img, c, y); |
504 | 0 | const float* JXL_RESTRICT bgrow = rect.ConstPlaneRow(background, c, y); |
505 | 0 | for (int x = xc - kRectBounds; x <= xc + kRectBounds; x++) { |
506 | 0 | if (x < 0 || static_cast<size_t>(x) >= rect.xsize()) continue; |
507 | 0 | double target = row[x] - bgrow[x]; |
508 | 0 | double gaussian = |
509 | 0 | DotGaussianModel(x - ellipse->x, y - ellipse->y, ct, st, |
510 | 0 | ellipse->sigma_x, ellipse->sigma_y, 1.0); |
511 | 0 | gg += gaussian * gaussian; |
512 | 0 | gd += gaussian * target; |
513 | 0 | } |
514 | 0 | } |
515 | 0 | ans.intensity[c] = gd / (gg + 1e-6); // Regularized least squares |
516 | 0 | } |
517 | 0 | } |
518 | 0 | ComputeDotLosses(&ans, cc, rect, img, background); |
519 | 0 | return ans; |
520 | 0 | } |
521 | | |
522 | | StatusOr<GaussianEllipse> FitGaussian(const ConnectedComponent& cc, |
523 | | const Rect& rect, const Image3F& img, |
524 | 0 | const Image3F& background) { |
525 | 0 | JXL_ASSIGN_OR_RETURN(GaussianEllipse ellipse, |
526 | 0 | FitGaussianFast(cc, rect, img, background)); |
527 | 0 | if (ellipse.sigma_x < ellipse.sigma_y) { |
528 | 0 | std::swap(ellipse.sigma_x, ellipse.sigma_y); |
529 | 0 | ellipse.angle += kPi / 2.0; |
530 | 0 | } |
531 | 0 | ellipse.angle -= kPi * std::floor(ellipse.angle / kPi); |
532 | 0 | if (std::fabs(ellipse.angle - kPi) < 1e-6 || |
533 | 0 | std::fabs(ellipse.angle) < 1e-6) { |
534 | 0 | ellipse.angle = 0.0; |
535 | 0 | } |
536 | 0 | JXL_ENSURE(ellipse.angle >= 0 && ellipse.angle <= kPi && |
537 | 0 | ellipse.sigma_x >= ellipse.sigma_y); |
538 | 0 | JXL_DEBUG(JXL_DEBUG_DOT_DETECT, |
539 | 0 | "Ellipse mu=(%lf,%lf) sigma=(%lf,%lf) angle=%lf " |
540 | 0 | "intensity=(%lf,%lf,%lf) bg=(%lf,%lf,%lf) l2_loss=%lf " |
541 | 0 | "custom_loss=%lf, neg_pix=%" PRIuS ", neg_v=(%lf,%lf,%lf)\n", |
542 | 0 | ellipse.x, ellipse.y, ellipse.sigma_x, ellipse.sigma_y, |
543 | 0 | ellipse.angle, ellipse.intensity[0], ellipse.intensity[1], |
544 | 0 | ellipse.intensity[2], ellipse.bgColor[0], ellipse.bgColor[1], |
545 | 0 | ellipse.bgColor[2], ellipse.l2_loss, ellipse.custom_loss, |
546 | 0 | ellipse.neg_pixels, ellipse.neg_value[0], ellipse.neg_value[1], |
547 | 0 | ellipse.neg_value[2]); |
548 | 0 | return ellipse; |
549 | 0 | } |
550 | | |
551 | | } // namespace |
552 | | |
553 | | StatusOr<std::vector<PatchInfo>> DetectGaussianEllipses( |
554 | | const Image3F& opsin, const Rect& rect, const GaussianDetectParams& params, |
555 | 0 | const EllipseQuantParams& qParams, ThreadPool* pool) { |
556 | 0 | JxlMemoryManager* memory_manager = opsin.memory_manager(); |
557 | 0 | std::vector<PatchInfo> dots; |
558 | 0 | JXL_ASSIGN_OR_RETURN( |
559 | 0 | Image3F smooth, |
560 | 0 | Image3F::Create(memory_manager, opsin.xsize(), opsin.ysize())); |
561 | 0 | JXL_ASSIGN_OR_RETURN(ImageF energy, ComputeEnergyImage(opsin, &smooth, pool)); |
562 | 0 | JXL_ASSIGN_OR_RETURN(std::vector<ConnectedComponent> components, |
563 | 0 | FindCC(energy, rect, params.t_low, params.t_high, |
564 | 0 | params.maxWinSize, params.minScore)); |
565 | 0 | size_t numCC = |
566 | 0 | std::min(params.maxCC, (components.size() * params.percCC) / 100); |
567 | 0 | if (components.size() > numCC) { |
568 | 0 | std::sort( |
569 | 0 | components.begin(), components.end(), |
570 | 0 | [](const ConnectedComponent& a, const ConnectedComponent& b) -> bool { |
571 | 0 | return a.score > b.score; |
572 | 0 | }); |
573 | 0 | components.erase(components.begin() + numCC, components.end()); |
574 | 0 | } |
575 | 0 | for (const auto& cc : components) { |
576 | 0 | JXL_ASSIGN_OR_RETURN(GaussianEllipse ellipse, |
577 | 0 | FitGaussian(cc, rect, opsin, smooth)); |
578 | 0 | if (ellipse.x < 0.0 || |
579 | 0 | std::ceil(ellipse.x) >= static_cast<double>(rect.xsize()) || |
580 | 0 | ellipse.y < 0.0 || |
581 | 0 | std::ceil(ellipse.y) >= static_cast<double>(rect.ysize())) { |
582 | 0 | continue; |
583 | 0 | } |
584 | 0 | if (ellipse.neg_pixels > params.maxNegPixels) continue; |
585 | 0 | double intensity = 0.21 * ellipse.intensity[0] + |
586 | 0 | 0.72 * ellipse.intensity[1] + |
587 | 0 | 0.07 * ellipse.intensity[2]; |
588 | 0 | double intensitySq = intensity * intensity; |
589 | | // for (int c = 0; c < 3; c++) { |
590 | | // intensitySq += ellipse.intensity[c] * ellipse.intensity[c]; |
591 | | //} |
592 | 0 | double sqDistMeanMode = (ellipse.x - cc.mode.x) * (ellipse.x - cc.mode.x) + |
593 | 0 | (ellipse.y - cc.mode.y) * (ellipse.y - cc.mode.y); |
594 | 0 | if (ellipse.l2_loss < params.maxL2Loss && |
595 | 0 | ellipse.custom_loss < params.maxCustomLoss && |
596 | 0 | intensitySq > (params.minIntensity * params.minIntensity) && |
597 | 0 | sqDistMeanMode < params.maxDistMeanMode * params.maxDistMeanMode) { |
598 | 0 | size_t x0 = cc.bounds.x0(); |
599 | 0 | size_t y0 = cc.bounds.y0(); |
600 | 0 | dots.emplace_back(); |
601 | 0 | dots.back().second.emplace_back(x0, y0); |
602 | 0 | QuantizedPatch& patch = dots.back().first; |
603 | 0 | patch.xsize = cc.bounds.xsize(); |
604 | 0 | patch.ysize = cc.bounds.ysize(); |
605 | 0 | for (size_t y = 0; y < patch.ysize; y++) { |
606 | 0 | for (size_t x = 0; x < patch.xsize; x++) { |
607 | 0 | for (size_t c = 0; c < 3; c++) { |
608 | 0 | patch.fpixels[c][y * patch.xsize + x] = |
609 | 0 | rect.ConstPlaneRow(opsin, c, y0 + y)[x0 + x] - |
610 | 0 | rect.ConstPlaneRow(smooth, c, y0 + y)[x0 + x]; |
611 | 0 | } |
612 | 0 | } |
613 | 0 | } |
614 | 0 | } |
615 | 0 | } |
616 | 0 | return dots; |
617 | 0 | } |
618 | | |
619 | | } // namespace jxl |
620 | | #endif // HWY_ONCE |