/src/libvpx/vp8/encoder/denoising.c
Line | Count | Source (jump to first uncovered line) |
1 | | /* |
2 | | * Copyright (c) 2012 The WebM project authors. All Rights Reserved. |
3 | | * |
4 | | * Use of this source code is governed by a BSD-style license |
5 | | * that can be found in the LICENSE file in the root of the source |
6 | | * tree. An additional intellectual property rights grant can be found |
7 | | * in the file PATENTS. All contributing project authors may |
8 | | * be found in the AUTHORS file in the root of the source tree. |
9 | | */ |
10 | | |
11 | | #include <limits.h> |
12 | | |
13 | | #include "denoising.h" |
14 | | |
15 | | #include "vp8/common/reconinter.h" |
16 | | #include "vpx/vpx_integer.h" |
17 | | #include "vpx_mem/vpx_mem.h" |
18 | | #include "vp8_rtcd.h" |
19 | | |
20 | | static const unsigned int NOISE_MOTION_THRESHOLD = 25 * 25; |
21 | | /* SSE_DIFF_THRESHOLD is selected as ~95% confidence assuming |
22 | | * var(noise) ~= 100. |
23 | | */ |
24 | | static const unsigned int SSE_DIFF_THRESHOLD = 16 * 16 * 20; |
25 | | static const unsigned int SSE_THRESHOLD = 16 * 16 * 40; |
26 | | static const unsigned int SSE_THRESHOLD_HIGH = 16 * 16 * 80; |
27 | | |
28 | | /* |
29 | | * The filter function was modified to reduce the computational complexity. |
30 | | * Step 1: |
31 | | * Instead of applying tap coefficients for each pixel, we calculated the |
32 | | * pixel adjustments vs. pixel diff value ahead of time. |
33 | | * adjustment = filtered_value - current_raw |
34 | | * = (filter_coefficient * diff + 128) >> 8 |
35 | | * where |
36 | | * filter_coefficient = (255 << 8) / (256 + ((absdiff * 330) >> 3)); |
37 | | * filter_coefficient += filter_coefficient / |
38 | | * (3 + motion_magnitude_adjustment); |
39 | | * filter_coefficient is clamped to 0 ~ 255. |
40 | | * |
41 | | * Step 2: |
42 | | * The adjustment vs. diff curve becomes flat very quick when diff increases. |
43 | | * This allowed us to use only several levels to approximate the curve without |
44 | | * changing the filtering algorithm too much. |
45 | | * The adjustments were further corrected by checking the motion magnitude. |
46 | | * The levels used are: |
47 | | * diff adjustment w/o motion correction adjustment w/ motion correction |
48 | | * [-255, -16] -6 -7 |
49 | | * [-15, -8] -4 -5 |
50 | | * [-7, -4] -3 -4 |
51 | | * [-3, 3] diff diff |
52 | | * [4, 7] 3 4 |
53 | | * [8, 15] 4 5 |
54 | | * [16, 255] 6 7 |
55 | | */ |
56 | | |
57 | | int vp8_denoiser_filter_c(unsigned char *mc_running_avg_y, int mc_avg_y_stride, |
58 | | unsigned char *running_avg_y, int avg_y_stride, |
59 | | unsigned char *sig, int sig_stride, |
60 | | unsigned int motion_magnitude, |
61 | 0 | int increase_denoising) { |
62 | 0 | unsigned char *running_avg_y_start = running_avg_y; |
63 | 0 | unsigned char *sig_start = sig; |
64 | 0 | int sum_diff_thresh; |
65 | 0 | int r, c; |
66 | 0 | int sum_diff = 0; |
67 | 0 | int adj_val[3] = { 3, 4, 6 }; |
68 | 0 | int shift_inc1 = 0; |
69 | 0 | int shift_inc2 = 1; |
70 | 0 | int col_sum[16] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; |
71 | | /* If motion_magnitude is small, making the denoiser more aggressive by |
72 | | * increasing the adjustment for each level. Add another increment for |
73 | | * blocks that are labeled for increase denoising. */ |
74 | 0 | if (motion_magnitude <= MOTION_MAGNITUDE_THRESHOLD) { |
75 | 0 | if (increase_denoising) { |
76 | 0 | shift_inc1 = 1; |
77 | 0 | shift_inc2 = 2; |
78 | 0 | } |
79 | 0 | adj_val[0] += shift_inc2; |
80 | 0 | adj_val[1] += shift_inc2; |
81 | 0 | adj_val[2] += shift_inc2; |
82 | 0 | } |
83 | |
|
84 | 0 | for (r = 0; r < 16; ++r) { |
85 | 0 | for (c = 0; c < 16; ++c) { |
86 | 0 | int diff = 0; |
87 | 0 | int adjustment = 0; |
88 | 0 | int absdiff = 0; |
89 | |
|
90 | 0 | diff = mc_running_avg_y[c] - sig[c]; |
91 | 0 | absdiff = abs(diff); |
92 | | |
93 | | // When |diff| <= |3 + shift_inc1|, use pixel value from |
94 | | // last denoised raw. |
95 | 0 | if (absdiff <= 3 + shift_inc1) { |
96 | 0 | running_avg_y[c] = mc_running_avg_y[c]; |
97 | 0 | col_sum[c] += diff; |
98 | 0 | } else { |
99 | 0 | if (absdiff >= 4 + shift_inc1 && absdiff <= 7) { |
100 | 0 | adjustment = adj_val[0]; |
101 | 0 | } else if (absdiff >= 8 && absdiff <= 15) { |
102 | 0 | adjustment = adj_val[1]; |
103 | 0 | } else { |
104 | 0 | adjustment = adj_val[2]; |
105 | 0 | } |
106 | |
|
107 | 0 | if (diff > 0) { |
108 | 0 | if ((sig[c] + adjustment) > 255) { |
109 | 0 | running_avg_y[c] = 255; |
110 | 0 | } else { |
111 | 0 | running_avg_y[c] = sig[c] + adjustment; |
112 | 0 | } |
113 | |
|
114 | 0 | col_sum[c] += adjustment; |
115 | 0 | } else { |
116 | 0 | if ((sig[c] - adjustment) < 0) { |
117 | 0 | running_avg_y[c] = 0; |
118 | 0 | } else { |
119 | 0 | running_avg_y[c] = sig[c] - adjustment; |
120 | 0 | } |
121 | |
|
122 | 0 | col_sum[c] -= adjustment; |
123 | 0 | } |
124 | 0 | } |
125 | 0 | } |
126 | | |
127 | | /* Update pointers for next iteration. */ |
128 | 0 | sig += sig_stride; |
129 | 0 | mc_running_avg_y += mc_avg_y_stride; |
130 | 0 | running_avg_y += avg_y_stride; |
131 | 0 | } |
132 | |
|
133 | 0 | for (c = 0; c < 16; ++c) { |
134 | | // Below we clip the value in the same way which SSE code use. |
135 | | // When adopting aggressive denoiser, the adj_val for each pixel |
136 | | // could be at most 8 (this is current max adjustment of the map). |
137 | | // In SSE code, we calculate the sum of adj_val for |
138 | | // the columns, so the sum could be up to 128(16 rows). However, |
139 | | // the range of the value is -128 ~ 127 in SSE code, that's why |
140 | | // we do this change in C code. |
141 | | // We don't do this for UV denoiser, since there are only 8 rows, |
142 | | // and max adjustments <= 8, so the sum of the columns will not |
143 | | // exceed 64. |
144 | 0 | if (col_sum[c] >= 128) { |
145 | 0 | col_sum[c] = 127; |
146 | 0 | } |
147 | 0 | sum_diff += col_sum[c]; |
148 | 0 | } |
149 | |
|
150 | 0 | sum_diff_thresh = SUM_DIFF_THRESHOLD; |
151 | 0 | if (increase_denoising) sum_diff_thresh = SUM_DIFF_THRESHOLD_HIGH; |
152 | 0 | if (abs(sum_diff) > sum_diff_thresh) { |
153 | | // Before returning to copy the block (i.e., apply no denoising), check |
154 | | // if we can still apply some (weaker) temporal filtering to this block, |
155 | | // that would otherwise not be denoised at all. Simplest is to apply |
156 | | // an additional adjustment to running_avg_y to bring it closer to sig. |
157 | | // The adjustment is capped by a maximum delta, and chosen such that |
158 | | // in most cases the resulting sum_diff will be within the |
159 | | // accceptable range given by sum_diff_thresh. |
160 | | |
161 | | // The delta is set by the excess of absolute pixel diff over threshold. |
162 | 0 | int delta = ((abs(sum_diff) - sum_diff_thresh) >> 8) + 1; |
163 | | // Only apply the adjustment for max delta up to 3. |
164 | 0 | if (delta < 4) { |
165 | 0 | sig -= sig_stride * 16; |
166 | 0 | mc_running_avg_y -= mc_avg_y_stride * 16; |
167 | 0 | running_avg_y -= avg_y_stride * 16; |
168 | 0 | for (r = 0; r < 16; ++r) { |
169 | 0 | for (c = 0; c < 16; ++c) { |
170 | 0 | int diff = mc_running_avg_y[c] - sig[c]; |
171 | 0 | int adjustment = abs(diff); |
172 | 0 | if (adjustment > delta) adjustment = delta; |
173 | 0 | if (diff > 0) { |
174 | | // Bring denoised signal down. |
175 | 0 | if (running_avg_y[c] - adjustment < 0) { |
176 | 0 | running_avg_y[c] = 0; |
177 | 0 | } else { |
178 | 0 | running_avg_y[c] = running_avg_y[c] - adjustment; |
179 | 0 | } |
180 | 0 | col_sum[c] -= adjustment; |
181 | 0 | } else if (diff < 0) { |
182 | | // Bring denoised signal up. |
183 | 0 | if (running_avg_y[c] + adjustment > 255) { |
184 | 0 | running_avg_y[c] = 255; |
185 | 0 | } else { |
186 | 0 | running_avg_y[c] = running_avg_y[c] + adjustment; |
187 | 0 | } |
188 | 0 | col_sum[c] += adjustment; |
189 | 0 | } |
190 | 0 | } |
191 | | // TODO(marpan): Check here if abs(sum_diff) has gone below the |
192 | | // threshold sum_diff_thresh, and if so, we can exit the row loop. |
193 | 0 | sig += sig_stride; |
194 | 0 | mc_running_avg_y += mc_avg_y_stride; |
195 | 0 | running_avg_y += avg_y_stride; |
196 | 0 | } |
197 | |
|
198 | 0 | sum_diff = 0; |
199 | 0 | for (c = 0; c < 16; ++c) { |
200 | 0 | if (col_sum[c] >= 128) { |
201 | 0 | col_sum[c] = 127; |
202 | 0 | } |
203 | 0 | sum_diff += col_sum[c]; |
204 | 0 | } |
205 | |
|
206 | 0 | if (abs(sum_diff) > sum_diff_thresh) return COPY_BLOCK; |
207 | 0 | } else { |
208 | 0 | return COPY_BLOCK; |
209 | 0 | } |
210 | 0 | } |
211 | | |
212 | 0 | vp8_copy_mem16x16(running_avg_y_start, avg_y_stride, sig_start, sig_stride); |
213 | 0 | return FILTER_BLOCK; |
214 | 0 | } |
215 | | |
216 | | int vp8_denoiser_filter_uv_c(unsigned char *mc_running_avg, int mc_avg_stride, |
217 | | unsigned char *running_avg, int avg_stride, |
218 | | unsigned char *sig, int sig_stride, |
219 | | unsigned int motion_magnitude, |
220 | 0 | int increase_denoising) { |
221 | 0 | unsigned char *running_avg_start = running_avg; |
222 | 0 | unsigned char *sig_start = sig; |
223 | 0 | int sum_diff_thresh; |
224 | 0 | int r, c; |
225 | 0 | int sum_diff = 0; |
226 | 0 | int sum_block = 0; |
227 | 0 | int adj_val[3] = { 3, 4, 6 }; |
228 | 0 | int shift_inc1 = 0; |
229 | 0 | int shift_inc2 = 1; |
230 | | /* If motion_magnitude is small, making the denoiser more aggressive by |
231 | | * increasing the adjustment for each level. Add another increment for |
232 | | * blocks that are labeled for increase denoising. */ |
233 | 0 | if (motion_magnitude <= MOTION_MAGNITUDE_THRESHOLD_UV) { |
234 | 0 | if (increase_denoising) { |
235 | 0 | shift_inc1 = 1; |
236 | 0 | shift_inc2 = 2; |
237 | 0 | } |
238 | 0 | adj_val[0] += shift_inc2; |
239 | 0 | adj_val[1] += shift_inc2; |
240 | 0 | adj_val[2] += shift_inc2; |
241 | 0 | } |
242 | | |
243 | | // Avoid denoising color signal if its close to average level. |
244 | 0 | for (r = 0; r < 8; ++r) { |
245 | 0 | for (c = 0; c < 8; ++c) { |
246 | 0 | sum_block += sig[c]; |
247 | 0 | } |
248 | 0 | sig += sig_stride; |
249 | 0 | } |
250 | 0 | if (abs(sum_block - (128 * 8 * 8)) < SUM_DIFF_FROM_AVG_THRESH_UV) { |
251 | 0 | return COPY_BLOCK; |
252 | 0 | } |
253 | | |
254 | 0 | sig -= sig_stride * 8; |
255 | 0 | for (r = 0; r < 8; ++r) { |
256 | 0 | for (c = 0; c < 8; ++c) { |
257 | 0 | int diff = 0; |
258 | 0 | int adjustment = 0; |
259 | 0 | int absdiff = 0; |
260 | |
|
261 | 0 | diff = mc_running_avg[c] - sig[c]; |
262 | 0 | absdiff = abs(diff); |
263 | | |
264 | | // When |diff| <= |3 + shift_inc1|, use pixel value from |
265 | | // last denoised raw. |
266 | 0 | if (absdiff <= 3 + shift_inc1) { |
267 | 0 | running_avg[c] = mc_running_avg[c]; |
268 | 0 | sum_diff += diff; |
269 | 0 | } else { |
270 | 0 | if (absdiff >= 4 && absdiff <= 7) { |
271 | 0 | adjustment = adj_val[0]; |
272 | 0 | } else if (absdiff >= 8 && absdiff <= 15) { |
273 | 0 | adjustment = adj_val[1]; |
274 | 0 | } else { |
275 | 0 | adjustment = adj_val[2]; |
276 | 0 | } |
277 | 0 | if (diff > 0) { |
278 | 0 | if ((sig[c] + adjustment) > 255) { |
279 | 0 | running_avg[c] = 255; |
280 | 0 | } else { |
281 | 0 | running_avg[c] = sig[c] + adjustment; |
282 | 0 | } |
283 | 0 | sum_diff += adjustment; |
284 | 0 | } else { |
285 | 0 | if ((sig[c] - adjustment) < 0) { |
286 | 0 | running_avg[c] = 0; |
287 | 0 | } else { |
288 | 0 | running_avg[c] = sig[c] - adjustment; |
289 | 0 | } |
290 | 0 | sum_diff -= adjustment; |
291 | 0 | } |
292 | 0 | } |
293 | 0 | } |
294 | | /* Update pointers for next iteration. */ |
295 | 0 | sig += sig_stride; |
296 | 0 | mc_running_avg += mc_avg_stride; |
297 | 0 | running_avg += avg_stride; |
298 | 0 | } |
299 | |
|
300 | 0 | sum_diff_thresh = SUM_DIFF_THRESHOLD_UV; |
301 | 0 | if (increase_denoising) sum_diff_thresh = SUM_DIFF_THRESHOLD_HIGH_UV; |
302 | 0 | if (abs(sum_diff) > sum_diff_thresh) { |
303 | | // Before returning to copy the block (i.e., apply no denoising), check |
304 | | // if we can still apply some (weaker) temporal filtering to this block, |
305 | | // that would otherwise not be denoised at all. Simplest is to apply |
306 | | // an additional adjustment to running_avg_y to bring it closer to sig. |
307 | | // The adjustment is capped by a maximum delta, and chosen such that |
308 | | // in most cases the resulting sum_diff will be within the |
309 | | // accceptable range given by sum_diff_thresh. |
310 | | |
311 | | // The delta is set by the excess of absolute pixel diff over threshold. |
312 | 0 | int delta = ((abs(sum_diff) - sum_diff_thresh) >> 8) + 1; |
313 | | // Only apply the adjustment for max delta up to 3. |
314 | 0 | if (delta < 4) { |
315 | 0 | sig -= sig_stride * 8; |
316 | 0 | mc_running_avg -= mc_avg_stride * 8; |
317 | 0 | running_avg -= avg_stride * 8; |
318 | 0 | for (r = 0; r < 8; ++r) { |
319 | 0 | for (c = 0; c < 8; ++c) { |
320 | 0 | int diff = mc_running_avg[c] - sig[c]; |
321 | 0 | int adjustment = abs(diff); |
322 | 0 | if (adjustment > delta) adjustment = delta; |
323 | 0 | if (diff > 0) { |
324 | | // Bring denoised signal down. |
325 | 0 | if (running_avg[c] - adjustment < 0) { |
326 | 0 | running_avg[c] = 0; |
327 | 0 | } else { |
328 | 0 | running_avg[c] = running_avg[c] - adjustment; |
329 | 0 | } |
330 | 0 | sum_diff -= adjustment; |
331 | 0 | } else if (diff < 0) { |
332 | | // Bring denoised signal up. |
333 | 0 | if (running_avg[c] + adjustment > 255) { |
334 | 0 | running_avg[c] = 255; |
335 | 0 | } else { |
336 | 0 | running_avg[c] = running_avg[c] + adjustment; |
337 | 0 | } |
338 | 0 | sum_diff += adjustment; |
339 | 0 | } |
340 | 0 | } |
341 | | // TODO(marpan): Check here if abs(sum_diff) has gone below the |
342 | | // threshold sum_diff_thresh, and if so, we can exit the row loop. |
343 | 0 | sig += sig_stride; |
344 | 0 | mc_running_avg += mc_avg_stride; |
345 | 0 | running_avg += avg_stride; |
346 | 0 | } |
347 | 0 | if (abs(sum_diff) > sum_diff_thresh) return COPY_BLOCK; |
348 | 0 | } else { |
349 | 0 | return COPY_BLOCK; |
350 | 0 | } |
351 | 0 | } |
352 | | |
353 | 0 | vp8_copy_mem8x8(running_avg_start, avg_stride, sig_start, sig_stride); |
354 | 0 | return FILTER_BLOCK; |
355 | 0 | } |
356 | | |
357 | 0 | void vp8_denoiser_set_parameters(VP8_DENOISER *denoiser, int mode) { |
358 | 0 | assert(mode > 0); // Denoiser is allocated only if mode > 0. |
359 | 0 | if (mode == 1) { |
360 | 0 | denoiser->denoiser_mode = kDenoiserOnYOnly; |
361 | 0 | } else if (mode == 2) { |
362 | 0 | denoiser->denoiser_mode = kDenoiserOnYUV; |
363 | 0 | } else if (mode == 3) { |
364 | 0 | denoiser->denoiser_mode = kDenoiserOnYUVAggressive; |
365 | 0 | } else { |
366 | 0 | denoiser->denoiser_mode = kDenoiserOnYUV; |
367 | 0 | } |
368 | 0 | if (denoiser->denoiser_mode != kDenoiserOnYUVAggressive) { |
369 | 0 | denoiser->denoise_pars.scale_sse_thresh = 1; |
370 | 0 | denoiser->denoise_pars.scale_motion_thresh = 8; |
371 | 0 | denoiser->denoise_pars.scale_increase_filter = 0; |
372 | 0 | denoiser->denoise_pars.denoise_mv_bias = 95; |
373 | 0 | denoiser->denoise_pars.pickmode_mv_bias = 100; |
374 | 0 | denoiser->denoise_pars.qp_thresh = 0; |
375 | 0 | denoiser->denoise_pars.consec_zerolast = UINT_MAX; |
376 | 0 | denoiser->denoise_pars.spatial_blur = 0; |
377 | 0 | } else { |
378 | 0 | denoiser->denoise_pars.scale_sse_thresh = 2; |
379 | 0 | denoiser->denoise_pars.scale_motion_thresh = 16; |
380 | 0 | denoiser->denoise_pars.scale_increase_filter = 1; |
381 | 0 | denoiser->denoise_pars.denoise_mv_bias = 60; |
382 | 0 | denoiser->denoise_pars.pickmode_mv_bias = 75; |
383 | 0 | denoiser->denoise_pars.qp_thresh = 80; |
384 | 0 | denoiser->denoise_pars.consec_zerolast = 15; |
385 | 0 | denoiser->denoise_pars.spatial_blur = 0; |
386 | 0 | } |
387 | 0 | } |
388 | | |
389 | | int vp8_denoiser_allocate(VP8_DENOISER *denoiser, int width, int height, |
390 | 0 | int num_mb_rows, int num_mb_cols, int mode) { |
391 | 0 | int i; |
392 | 0 | assert(denoiser); |
393 | 0 | denoiser->num_mb_cols = num_mb_cols; |
394 | |
|
395 | 0 | for (i = 0; i < MAX_REF_FRAMES; ++i) { |
396 | 0 | denoiser->yv12_running_avg[i].flags = 0; |
397 | |
|
398 | 0 | if (vp8_yv12_alloc_frame_buffer(&(denoiser->yv12_running_avg[i]), width, |
399 | 0 | height, VP8BORDERINPIXELS) < 0) { |
400 | 0 | vp8_denoiser_free(denoiser); |
401 | 0 | return 1; |
402 | 0 | } |
403 | 0 | memset(denoiser->yv12_running_avg[i].buffer_alloc, 0, |
404 | 0 | denoiser->yv12_running_avg[i].frame_size); |
405 | 0 | } |
406 | 0 | denoiser->yv12_mc_running_avg.flags = 0; |
407 | |
|
408 | 0 | if (vp8_yv12_alloc_frame_buffer(&(denoiser->yv12_mc_running_avg), width, |
409 | 0 | height, VP8BORDERINPIXELS) < 0) { |
410 | 0 | vp8_denoiser_free(denoiser); |
411 | 0 | return 1; |
412 | 0 | } |
413 | | |
414 | 0 | memset(denoiser->yv12_mc_running_avg.buffer_alloc, 0, |
415 | 0 | denoiser->yv12_mc_running_avg.frame_size); |
416 | |
|
417 | 0 | if (vp8_yv12_alloc_frame_buffer(&denoiser->yv12_last_source, width, height, |
418 | 0 | VP8BORDERINPIXELS) < 0) { |
419 | 0 | vp8_denoiser_free(denoiser); |
420 | 0 | return 1; |
421 | 0 | } |
422 | 0 | memset(denoiser->yv12_last_source.buffer_alloc, 0, |
423 | 0 | denoiser->yv12_last_source.frame_size); |
424 | |
|
425 | 0 | denoiser->denoise_state = vpx_calloc((num_mb_rows * num_mb_cols), 1); |
426 | 0 | if (!denoiser->denoise_state) { |
427 | 0 | vp8_denoiser_free(denoiser); |
428 | 0 | return 1; |
429 | 0 | } |
430 | 0 | memset(denoiser->denoise_state, 0, (num_mb_rows * num_mb_cols)); |
431 | 0 | vp8_denoiser_set_parameters(denoiser, mode); |
432 | 0 | denoiser->nmse_source_diff = 0; |
433 | 0 | denoiser->nmse_source_diff_count = 0; |
434 | 0 | denoiser->qp_avg = 0; |
435 | | // QP threshold below which we can go up to aggressive mode. |
436 | 0 | denoiser->qp_threshold_up = 80; |
437 | | // QP threshold above which we can go back down to normal mode. |
438 | | // For now keep this second threshold high, so not used currently. |
439 | 0 | denoiser->qp_threshold_down = 128; |
440 | | // Bitrate thresholds and noise metric (nmse) thresholds for switching to |
441 | | // aggressive mode. |
442 | | // TODO(marpan): Adjust thresholds, including effect on resolution. |
443 | 0 | denoiser->bitrate_threshold = 400000; // (bits/sec). |
444 | 0 | denoiser->threshold_aggressive_mode = 80; |
445 | 0 | if (width * height > 1280 * 720) { |
446 | 0 | denoiser->bitrate_threshold = 3000000; |
447 | 0 | denoiser->threshold_aggressive_mode = 200; |
448 | 0 | } else if (width * height > 960 * 540) { |
449 | 0 | denoiser->bitrate_threshold = 1200000; |
450 | 0 | denoiser->threshold_aggressive_mode = 120; |
451 | 0 | } else if (width * height > 640 * 480) { |
452 | 0 | denoiser->bitrate_threshold = 600000; |
453 | 0 | denoiser->threshold_aggressive_mode = 100; |
454 | 0 | } |
455 | 0 | return 0; |
456 | 0 | } |
457 | | |
458 | 4.91k | void vp8_denoiser_free(VP8_DENOISER *denoiser) { |
459 | 4.91k | int i; |
460 | 4.91k | assert(denoiser); |
461 | | |
462 | 24.5k | for (i = 0; i < MAX_REF_FRAMES; ++i) { |
463 | 19.6k | vp8_yv12_de_alloc_frame_buffer(&denoiser->yv12_running_avg[i]); |
464 | 19.6k | } |
465 | 4.91k | vp8_yv12_de_alloc_frame_buffer(&denoiser->yv12_mc_running_avg); |
466 | 4.91k | vp8_yv12_de_alloc_frame_buffer(&denoiser->yv12_last_source); |
467 | 4.91k | vpx_free(denoiser->denoise_state); |
468 | 4.91k | } |
469 | | |
470 | | void vp8_denoiser_denoise_mb(VP8_DENOISER *denoiser, MACROBLOCK *x, |
471 | | unsigned int best_sse, unsigned int zero_mv_sse, |
472 | | int recon_yoffset, int recon_uvoffset, |
473 | | loop_filter_info_n *lfi_n, int mb_row, int mb_col, |
474 | | int block_index, int consec_zero_last) |
475 | | |
476 | 0 | { |
477 | 0 | int mv_row; |
478 | 0 | int mv_col; |
479 | 0 | unsigned int motion_threshold; |
480 | 0 | unsigned int motion_magnitude2; |
481 | 0 | unsigned int sse_thresh; |
482 | 0 | int sse_diff_thresh = 0; |
483 | | // Spatial loop filter: only applied selectively based on |
484 | | // temporal filter state of block relative to top/left neighbors. |
485 | 0 | int apply_spatial_loop_filter = 1; |
486 | 0 | MV_REFERENCE_FRAME frame = x->best_reference_frame; |
487 | 0 | MV_REFERENCE_FRAME zero_frame = x->best_zeromv_reference_frame; |
488 | |
|
489 | 0 | enum vp8_denoiser_decision decision = FILTER_BLOCK; |
490 | 0 | enum vp8_denoiser_decision decision_u = COPY_BLOCK; |
491 | 0 | enum vp8_denoiser_decision decision_v = COPY_BLOCK; |
492 | |
|
493 | 0 | if (zero_frame) { |
494 | 0 | YV12_BUFFER_CONFIG *src = &denoiser->yv12_running_avg[frame]; |
495 | 0 | YV12_BUFFER_CONFIG *dst = &denoiser->yv12_mc_running_avg; |
496 | 0 | YV12_BUFFER_CONFIG saved_pre, saved_dst; |
497 | 0 | MB_MODE_INFO saved_mbmi; |
498 | 0 | MACROBLOCKD *filter_xd = &x->e_mbd; |
499 | 0 | MB_MODE_INFO *mbmi = &filter_xd->mode_info_context->mbmi; |
500 | 0 | int sse_diff = 0; |
501 | | // Bias on zero motion vector sse. |
502 | 0 | const int zero_bias = denoiser->denoise_pars.denoise_mv_bias; |
503 | 0 | zero_mv_sse = (unsigned int)((int64_t)zero_mv_sse * zero_bias / 100); |
504 | 0 | sse_diff = (int)zero_mv_sse - (int)best_sse; |
505 | |
|
506 | 0 | saved_mbmi = *mbmi; |
507 | | |
508 | | /* Use the best MV for the compensation. */ |
509 | 0 | mbmi->ref_frame = x->best_reference_frame; |
510 | 0 | mbmi->mode = x->best_sse_inter_mode; |
511 | 0 | mbmi->mv = x->best_sse_mv; |
512 | 0 | mbmi->need_to_clamp_mvs = x->need_to_clamp_best_mvs; |
513 | 0 | mv_col = x->best_sse_mv.as_mv.col; |
514 | 0 | mv_row = x->best_sse_mv.as_mv.row; |
515 | | // Bias to zero_mv if small amount of motion. |
516 | | // Note sse_diff_thresh is intialized to zero, so this ensures |
517 | | // we will always choose zero_mv for denoising if |
518 | | // zero_mv_see <= best_sse (i.e., sse_diff <= 0). |
519 | 0 | if ((unsigned int)(mv_row * mv_row + mv_col * mv_col) <= |
520 | 0 | NOISE_MOTION_THRESHOLD) { |
521 | 0 | sse_diff_thresh = (int)SSE_DIFF_THRESHOLD; |
522 | 0 | } |
523 | |
|
524 | 0 | if (frame == INTRA_FRAME || sse_diff <= sse_diff_thresh) { |
525 | | /* |
526 | | * Handle intra blocks as referring to last frame with zero motion |
527 | | * and let the absolute pixel difference affect the filter factor. |
528 | | * Also consider small amount of motion as being random walk due |
529 | | * to noise, if it doesn't mean that we get a much bigger error. |
530 | | * Note that any changes to the mode info only affects the |
531 | | * denoising. |
532 | | */ |
533 | 0 | x->denoise_zeromv = 1; |
534 | 0 | mbmi->ref_frame = x->best_zeromv_reference_frame; |
535 | |
|
536 | 0 | src = &denoiser->yv12_running_avg[zero_frame]; |
537 | |
|
538 | 0 | mbmi->mode = ZEROMV; |
539 | 0 | mbmi->mv.as_int = 0; |
540 | 0 | x->best_sse_inter_mode = ZEROMV; |
541 | 0 | x->best_sse_mv.as_int = 0; |
542 | 0 | best_sse = zero_mv_sse; |
543 | 0 | } |
544 | |
|
545 | 0 | mv_row = x->best_sse_mv.as_mv.row; |
546 | 0 | mv_col = x->best_sse_mv.as_mv.col; |
547 | 0 | motion_magnitude2 = mv_row * mv_row + mv_col * mv_col; |
548 | 0 | motion_threshold = |
549 | 0 | denoiser->denoise_pars.scale_motion_thresh * NOISE_MOTION_THRESHOLD; |
550 | |
|
551 | 0 | if (motion_magnitude2 < |
552 | 0 | denoiser->denoise_pars.scale_increase_filter * NOISE_MOTION_THRESHOLD) { |
553 | 0 | x->increase_denoising = 1; |
554 | 0 | } |
555 | |
|
556 | 0 | sse_thresh = denoiser->denoise_pars.scale_sse_thresh * SSE_THRESHOLD; |
557 | 0 | if (x->increase_denoising) { |
558 | 0 | sse_thresh = denoiser->denoise_pars.scale_sse_thresh * SSE_THRESHOLD_HIGH; |
559 | 0 | } |
560 | |
|
561 | 0 | if (best_sse > sse_thresh || motion_magnitude2 > motion_threshold) { |
562 | 0 | decision = COPY_BLOCK; |
563 | 0 | } |
564 | | |
565 | | // If block is considered skin, don't denoise if the block |
566 | | // (1) is selected as non-zero motion for current frame, or |
567 | | // (2) has not been selected as ZERO_LAST mode at least x past frames |
568 | | // in a row. |
569 | | // TODO(marpan): Parameter "x" should be varied with framerate. |
570 | | // In particualar, should be reduced for layers (base layer/LAST). |
571 | 0 | if (x->is_skin && (consec_zero_last < 2 || motion_magnitude2 > 0)) { |
572 | 0 | decision = COPY_BLOCK; |
573 | 0 | } |
574 | |
|
575 | 0 | if (decision == FILTER_BLOCK) { |
576 | 0 | saved_pre = filter_xd->pre; |
577 | 0 | saved_dst = filter_xd->dst; |
578 | | |
579 | | /* Compensate the running average. */ |
580 | 0 | filter_xd->pre.y_buffer = src->y_buffer + recon_yoffset; |
581 | 0 | filter_xd->pre.u_buffer = src->u_buffer + recon_uvoffset; |
582 | 0 | filter_xd->pre.v_buffer = src->v_buffer + recon_uvoffset; |
583 | | /* Write the compensated running average to the destination buffer. */ |
584 | 0 | filter_xd->dst.y_buffer = dst->y_buffer + recon_yoffset; |
585 | 0 | filter_xd->dst.u_buffer = dst->u_buffer + recon_uvoffset; |
586 | 0 | filter_xd->dst.v_buffer = dst->v_buffer + recon_uvoffset; |
587 | |
|
588 | 0 | if (!x->skip) { |
589 | 0 | vp8_build_inter_predictors_mb(filter_xd); |
590 | 0 | } else { |
591 | 0 | vp8_build_inter16x16_predictors_mb( |
592 | 0 | filter_xd, filter_xd->dst.y_buffer, filter_xd->dst.u_buffer, |
593 | 0 | filter_xd->dst.v_buffer, filter_xd->dst.y_stride, |
594 | 0 | filter_xd->dst.uv_stride); |
595 | 0 | } |
596 | 0 | filter_xd->pre = saved_pre; |
597 | 0 | filter_xd->dst = saved_dst; |
598 | 0 | *mbmi = saved_mbmi; |
599 | 0 | } |
600 | 0 | } else { |
601 | | // zero_frame should always be 1 for real-time mode, as the |
602 | | // ZEROMV mode is always checked, so we should never go into this branch. |
603 | | // If case ZEROMV is not checked, then we will force no denoise (COPY). |
604 | 0 | decision = COPY_BLOCK; |
605 | 0 | } |
606 | |
|
607 | 0 | if (decision == FILTER_BLOCK) { |
608 | 0 | unsigned char *mc_running_avg_y = |
609 | 0 | denoiser->yv12_mc_running_avg.y_buffer + recon_yoffset; |
610 | 0 | int mc_avg_y_stride = denoiser->yv12_mc_running_avg.y_stride; |
611 | 0 | unsigned char *running_avg_y = |
612 | 0 | denoiser->yv12_running_avg[INTRA_FRAME].y_buffer + recon_yoffset; |
613 | 0 | int avg_y_stride = denoiser->yv12_running_avg[INTRA_FRAME].y_stride; |
614 | | |
615 | | /* Filter. */ |
616 | 0 | decision = vp8_denoiser_filter(mc_running_avg_y, mc_avg_y_stride, |
617 | 0 | running_avg_y, avg_y_stride, x->thismb, 16, |
618 | 0 | motion_magnitude2, x->increase_denoising); |
619 | 0 | denoiser->denoise_state[block_index] = |
620 | 0 | motion_magnitude2 > 0 ? kFilterNonZeroMV : kFilterZeroMV; |
621 | | // Only denoise UV for zero motion, and if y channel was denoised. |
622 | 0 | if (denoiser->denoiser_mode != kDenoiserOnYOnly && motion_magnitude2 == 0 && |
623 | 0 | decision == FILTER_BLOCK) { |
624 | 0 | unsigned char *mc_running_avg_u = |
625 | 0 | denoiser->yv12_mc_running_avg.u_buffer + recon_uvoffset; |
626 | 0 | unsigned char *running_avg_u = |
627 | 0 | denoiser->yv12_running_avg[INTRA_FRAME].u_buffer + recon_uvoffset; |
628 | 0 | unsigned char *mc_running_avg_v = |
629 | 0 | denoiser->yv12_mc_running_avg.v_buffer + recon_uvoffset; |
630 | 0 | unsigned char *running_avg_v = |
631 | 0 | denoiser->yv12_running_avg[INTRA_FRAME].v_buffer + recon_uvoffset; |
632 | 0 | int mc_avg_uv_stride = denoiser->yv12_mc_running_avg.uv_stride; |
633 | 0 | int avg_uv_stride = denoiser->yv12_running_avg[INTRA_FRAME].uv_stride; |
634 | 0 | int signal_stride = x->block[16].src_stride; |
635 | 0 | decision_u = vp8_denoiser_filter_uv( |
636 | 0 | mc_running_avg_u, mc_avg_uv_stride, running_avg_u, avg_uv_stride, |
637 | 0 | x->block[16].src + *x->block[16].base_src, signal_stride, |
638 | 0 | motion_magnitude2, 0); |
639 | 0 | decision_v = vp8_denoiser_filter_uv( |
640 | 0 | mc_running_avg_v, mc_avg_uv_stride, running_avg_v, avg_uv_stride, |
641 | 0 | x->block[20].src + *x->block[20].base_src, signal_stride, |
642 | 0 | motion_magnitude2, 0); |
643 | 0 | } |
644 | 0 | } |
645 | 0 | if (decision == COPY_BLOCK) { |
646 | | /* No filtering of this block; it differs too much from the predictor, |
647 | | * or the motion vector magnitude is considered too big. |
648 | | */ |
649 | 0 | x->denoise_zeromv = 0; |
650 | 0 | vp8_copy_mem16x16( |
651 | 0 | x->thismb, 16, |
652 | 0 | denoiser->yv12_running_avg[INTRA_FRAME].y_buffer + recon_yoffset, |
653 | 0 | denoiser->yv12_running_avg[INTRA_FRAME].y_stride); |
654 | 0 | denoiser->denoise_state[block_index] = kNoFilter; |
655 | 0 | } |
656 | 0 | if (denoiser->denoiser_mode != kDenoiserOnYOnly) { |
657 | 0 | if (decision_u == COPY_BLOCK) { |
658 | 0 | vp8_copy_mem8x8( |
659 | 0 | x->block[16].src + *x->block[16].base_src, x->block[16].src_stride, |
660 | 0 | denoiser->yv12_running_avg[INTRA_FRAME].u_buffer + recon_uvoffset, |
661 | 0 | denoiser->yv12_running_avg[INTRA_FRAME].uv_stride); |
662 | 0 | } |
663 | 0 | if (decision_v == COPY_BLOCK) { |
664 | 0 | vp8_copy_mem8x8( |
665 | 0 | x->block[20].src + *x->block[20].base_src, x->block[16].src_stride, |
666 | 0 | denoiser->yv12_running_avg[INTRA_FRAME].v_buffer + recon_uvoffset, |
667 | 0 | denoiser->yv12_running_avg[INTRA_FRAME].uv_stride); |
668 | 0 | } |
669 | 0 | } |
670 | | // Option to selectively deblock the denoised signal, for y channel only. |
671 | 0 | if (apply_spatial_loop_filter) { |
672 | 0 | loop_filter_info lfi; |
673 | 0 | int apply_filter_col = 0; |
674 | 0 | int apply_filter_row = 0; |
675 | 0 | int apply_filter = 0; |
676 | 0 | int y_stride = denoiser->yv12_running_avg[INTRA_FRAME].y_stride; |
677 | 0 | int uv_stride = denoiser->yv12_running_avg[INTRA_FRAME].uv_stride; |
678 | | |
679 | | // Fix filter level to some nominal value for now. |
680 | 0 | int filter_level = 48; |
681 | |
|
682 | 0 | int hev_index = lfi_n->hev_thr_lut[INTER_FRAME][filter_level]; |
683 | 0 | lfi.mblim = lfi_n->mblim[filter_level]; |
684 | 0 | lfi.blim = lfi_n->blim[filter_level]; |
685 | 0 | lfi.lim = lfi_n->lim[filter_level]; |
686 | 0 | lfi.hev_thr = lfi_n->hev_thr[hev_index]; |
687 | | |
688 | | // Apply filter if there is a difference in the denoiser filter state |
689 | | // between the current and left/top block, or if non-zero motion vector |
690 | | // is used for the motion-compensated filtering. |
691 | 0 | if (mb_col > 0) { |
692 | 0 | apply_filter_col = |
693 | 0 | !((denoiser->denoise_state[block_index] == |
694 | 0 | denoiser->denoise_state[block_index - 1]) && |
695 | 0 | denoiser->denoise_state[block_index] != kFilterNonZeroMV); |
696 | 0 | if (apply_filter_col) { |
697 | | // Filter left vertical edge. |
698 | 0 | apply_filter = 1; |
699 | 0 | vp8_loop_filter_mbv( |
700 | 0 | denoiser->yv12_running_avg[INTRA_FRAME].y_buffer + recon_yoffset, |
701 | 0 | NULL, NULL, y_stride, uv_stride, &lfi); |
702 | 0 | } |
703 | 0 | } |
704 | 0 | if (mb_row > 0) { |
705 | 0 | apply_filter_row = |
706 | 0 | !((denoiser->denoise_state[block_index] == |
707 | 0 | denoiser->denoise_state[block_index - denoiser->num_mb_cols]) && |
708 | 0 | denoiser->denoise_state[block_index] != kFilterNonZeroMV); |
709 | 0 | if (apply_filter_row) { |
710 | | // Filter top horizontal edge. |
711 | 0 | apply_filter = 1; |
712 | 0 | vp8_loop_filter_mbh( |
713 | 0 | denoiser->yv12_running_avg[INTRA_FRAME].y_buffer + recon_yoffset, |
714 | 0 | NULL, NULL, y_stride, uv_stride, &lfi); |
715 | 0 | } |
716 | 0 | } |
717 | 0 | if (apply_filter) { |
718 | | // Update the signal block |x|. Pixel changes are only to top and/or |
719 | | // left boundary pixels: can we avoid full block copy here. |
720 | 0 | vp8_copy_mem16x16( |
721 | 0 | denoiser->yv12_running_avg[INTRA_FRAME].y_buffer + recon_yoffset, |
722 | 0 | y_stride, x->thismb, 16); |
723 | 0 | } |
724 | 0 | } |
725 | 0 | } |