/src/aom/av1/encoder/av1_noise_estimate.c
Line | Count | Source (jump to first uncovered line) |
1 | | /* |
2 | | * Copyright (c) 2020, Alliance for Open Media. All rights reserved |
3 | | * |
4 | | * This source code is subject to the terms of the BSD 2 Clause License and |
5 | | * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License |
6 | | * was not distributed with this source code in the LICENSE file, you can |
7 | | * obtain it at www.aomedia.org/license/software. If the Alliance for Open |
8 | | * Media Patent License 1.0 was not distributed with this source code in the |
9 | | * PATENTS file, you can obtain it at www.aomedia.org/license/patent. |
10 | | */ |
11 | | |
12 | | #include <assert.h> |
13 | | #include <limits.h> |
14 | | #include <math.h> |
15 | | |
16 | | #include "config/aom_dsp_rtcd.h" |
17 | | #include "aom_dsp/aom_dsp_common.h" |
18 | | #include "aom_scale/yv12config.h" |
19 | | #include "aom/aom_integer.h" |
20 | | #include "av1/encoder/context_tree.h" |
21 | | #include "av1/encoder/av1_noise_estimate.h" |
22 | | #include "av1/encoder/encoder.h" |
23 | | #if CONFIG_AV1_TEMPORAL_DENOISING |
24 | | #include "av1/encoder/av1_temporal_denoiser.h" |
25 | | #endif |
26 | | |
27 | | #if CONFIG_AV1_TEMPORAL_DENOISING |
28 | | // For SVC: only do noise estimation on top spatial layer. |
29 | | static INLINE int noise_est_svc(const struct AV1_COMP *const cpi) { |
30 | | return (!cpi->ppi->use_svc || |
31 | | (cpi->ppi->use_svc && |
32 | | cpi->svc.spatial_layer_id == cpi->svc.number_spatial_layers - 1)); |
33 | | } |
34 | | #endif |
35 | | |
36 | 0 | void av1_noise_estimate_init(NOISE_ESTIMATE *const ne, int width, int height) { |
37 | 0 | ne->enabled = 0; |
38 | 0 | ne->level = (width * height < 1280 * 720) ? kLowLow : kLow; |
39 | 0 | ne->value = 0; |
40 | 0 | ne->count = 0; |
41 | 0 | ne->thresh = 90; |
42 | 0 | ne->last_w = 0; |
43 | 0 | ne->last_h = 0; |
44 | 0 | if (width * height >= 1920 * 1080) { |
45 | 0 | ne->thresh = 200; |
46 | 0 | } else if (width * height >= 1280 * 720) { |
47 | 0 | ne->thresh = 140; |
48 | 0 | } else if (width * height >= 640 * 360) { |
49 | 0 | ne->thresh = 115; |
50 | 0 | } |
51 | 0 | ne->num_frames_estimate = 15; |
52 | 0 | ne->adapt_thresh = (3 * ne->thresh) >> 1; |
53 | 0 | } |
54 | | |
55 | 0 | static int enable_noise_estimation(AV1_COMP *const cpi) { |
56 | 0 | const int resize_pending = is_frame_resize_pending(cpi); |
57 | |
|
58 | 0 | #if CONFIG_AV1_HIGHBITDEPTH |
59 | 0 | if (cpi->common.seq_params->use_highbitdepth) return 0; |
60 | 0 | #endif |
61 | | // Enable noise estimation if denoising is on. |
62 | | #if CONFIG_AV1_TEMPORAL_DENOISING |
63 | | if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi) && |
64 | | cpi->common.width >= 320 && cpi->common.height >= 180) |
65 | | return 1; |
66 | | #endif |
67 | | // Only allow noise estimate under certain encoding mode. |
68 | | // Enabled for 1 pass CBR, speed >=5, and if resolution is same as original. |
69 | | // Not enabled for SVC mode and screen_content_mode. |
70 | | // Not enabled for low resolutions. |
71 | 0 | if (cpi->oxcf.pass == AOM_RC_ONE_PASS && cpi->oxcf.rc_cfg.mode == AOM_CBR && |
72 | 0 | cpi->oxcf.q_cfg.aq_mode == CYCLIC_REFRESH_AQ && cpi->oxcf.speed >= 5 && |
73 | 0 | resize_pending == 0 && !cpi->ppi->use_svc && |
74 | 0 | cpi->oxcf.tune_cfg.content != AOM_CONTENT_SCREEN && |
75 | 0 | cpi->common.width * cpi->common.height >= 640 * 360) |
76 | 0 | return 1; |
77 | 0 | else |
78 | 0 | return 0; |
79 | 0 | } |
80 | | |
81 | | #if CONFIG_AV1_TEMPORAL_DENOISING |
82 | | static void copy_frame(YV12_BUFFER_CONFIG *const dest, |
83 | | const YV12_BUFFER_CONFIG *const src) { |
84 | | const uint8_t *srcbuf = src->y_buffer; |
85 | | uint8_t *destbuf = dest->y_buffer; |
86 | | |
87 | | assert(dest->y_width == src->y_width); |
88 | | assert(dest->y_height == src->y_height); |
89 | | |
90 | | for (int r = 0; r < dest->y_height; ++r) { |
91 | | memcpy(destbuf, srcbuf, dest->y_width); |
92 | | destbuf += dest->y_stride; |
93 | | srcbuf += src->y_stride; |
94 | | } |
95 | | } |
96 | | #endif // CONFIG_AV1_TEMPORAL_DENOISING |
97 | | |
98 | 0 | NOISE_LEVEL av1_noise_estimate_extract_level(NOISE_ESTIMATE *const ne) { |
99 | 0 | int noise_level = kLowLow; |
100 | 0 | if (ne->value > (ne->thresh << 1)) { |
101 | 0 | noise_level = kHigh; |
102 | 0 | } else { |
103 | 0 | if (ne->value > ne->thresh) |
104 | 0 | noise_level = kMedium; |
105 | 0 | else if (ne->value > (ne->thresh >> 1)) |
106 | 0 | noise_level = kLow; |
107 | 0 | else |
108 | 0 | noise_level = kLowLow; |
109 | 0 | } |
110 | 0 | return noise_level; |
111 | 0 | } |
112 | | |
113 | 0 | void av1_update_noise_estimate(AV1_COMP *const cpi) { |
114 | 0 | const AV1_COMMON *const cm = &cpi->common; |
115 | 0 | const CommonModeInfoParams *const mi_params = &cm->mi_params; |
116 | |
|
117 | 0 | NOISE_ESTIMATE *const ne = &cpi->noise_estimate; |
118 | 0 | const int low_res = (cm->width <= 352 && cm->height <= 288); |
119 | | // Estimate of noise level every frame_period frames. |
120 | 0 | int frame_period = 8; |
121 | 0 | int thresh_consec_zeromv = 2; |
122 | 0 | int frame_counter = cm->current_frame.frame_number; |
123 | | // Estimate is between current source and last source. |
124 | 0 | YV12_BUFFER_CONFIG *last_source = cpi->last_source; |
125 | | #if CONFIG_AV1_TEMPORAL_DENOISING |
126 | | if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi)) { |
127 | | last_source = &cpi->denoiser.last_source; |
128 | | // Tune these thresholds for different resolutions when denoising is |
129 | | // enabled. |
130 | | if (cm->width > 640 && cm->width <= 1920) { |
131 | | thresh_consec_zeromv = 2; |
132 | | } |
133 | | } |
134 | | #endif |
135 | 0 | ne->enabled = enable_noise_estimation(cpi); |
136 | 0 | if (cpi->svc.number_spatial_layers > 1) |
137 | 0 | frame_counter = cpi->svc.current_superframe; |
138 | 0 | if (!ne->enabled || frame_counter % frame_period != 0 || |
139 | 0 | last_source == NULL || |
140 | 0 | (cpi->svc.number_spatial_layers == 1 && |
141 | 0 | (ne->last_w != cm->width || ne->last_h != cm->height))) { |
142 | | #if CONFIG_AV1_TEMPORAL_DENOISING |
143 | | if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi)) |
144 | | copy_frame(&cpi->denoiser.last_source, cpi->source); |
145 | | #endif |
146 | 0 | if (last_source != NULL) { |
147 | 0 | ne->last_w = cm->width; |
148 | 0 | ne->last_h = cm->height; |
149 | 0 | } |
150 | 0 | return; |
151 | 0 | } else if (frame_counter > 60 && cpi->svc.num_encoded_top_layer > 1 && |
152 | 0 | cpi->rc.frames_since_key > cpi->svc.number_spatial_layers && |
153 | 0 | cpi->svc.spatial_layer_id == cpi->svc.number_spatial_layers - 1 && |
154 | 0 | cpi->rc.avg_frame_low_motion < (low_res ? 60 : 40)) { |
155 | | // Force noise estimation to 0 and denoiser off if content has high motion. |
156 | 0 | ne->level = kLowLow; |
157 | 0 | ne->count = 0; |
158 | 0 | ne->num_frames_estimate = 10; |
159 | | #if CONFIG_AV1_TEMPORAL_DENOISING |
160 | | if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi) && |
161 | | cpi->svc.current_superframe > 1) { |
162 | | av1_denoiser_set_noise_level(cpi, ne->level); |
163 | | copy_frame(&cpi->denoiser.last_source, cpi->source); |
164 | | } |
165 | | #endif |
166 | 0 | return; |
167 | 0 | } else { |
168 | 0 | unsigned int bin_size = 100; |
169 | 0 | unsigned int hist[MAX_VAR_HIST_BINS] = { 0 }; |
170 | 0 | unsigned int hist_avg[MAX_VAR_HIST_BINS]; |
171 | 0 | unsigned int max_bin = 0; |
172 | 0 | unsigned int max_bin_count = 0; |
173 | 0 | unsigned int bin_cnt; |
174 | 0 | int bsize = BLOCK_16X16; |
175 | | // Loop over sub-sample of 16x16 blocks of frame, and for blocks that have |
176 | | // been encoded as zero/small mv at least x consecutive frames, compute |
177 | | // the variance to update estimate of noise in the source. |
178 | 0 | const uint8_t *src_y = cpi->source->y_buffer; |
179 | 0 | const int src_ystride = cpi->source->y_stride; |
180 | 0 | const uint8_t *last_src_y = last_source->y_buffer; |
181 | 0 | const int last_src_ystride = last_source->y_stride; |
182 | 0 | int mi_row, mi_col; |
183 | 0 | int num_low_motion = 0; |
184 | 0 | int frame_low_motion = 1; |
185 | 0 | for (mi_row = 0; mi_row < mi_params->mi_rows; mi_row += 2) { |
186 | 0 | for (mi_col = 0; mi_col < mi_params->mi_cols; mi_col += 2) { |
187 | 0 | int bl_index = |
188 | 0 | (mi_row >> 1) * (mi_params->mi_cols >> 1) + (mi_col >> 1); |
189 | 0 | if (cpi->consec_zero_mv[bl_index] > thresh_consec_zeromv) |
190 | 0 | num_low_motion++; |
191 | 0 | } |
192 | 0 | } |
193 | 0 | if (num_low_motion < |
194 | 0 | (((3 * (mi_params->mi_rows * mi_params->mi_cols) >> 2)) >> 3)) |
195 | 0 | frame_low_motion = 0; |
196 | 0 | for (mi_row = 0; mi_row < mi_params->mi_rows; mi_row++) { |
197 | 0 | for (mi_col = 0; mi_col < mi_params->mi_cols; mi_col++) { |
198 | | // 16x16 blocks, 1/4 sample of frame. |
199 | 0 | if (mi_row % 8 == 0 && mi_col % 8 == 0 && |
200 | 0 | mi_row < mi_params->mi_rows - 3 && |
201 | 0 | mi_col < mi_params->mi_cols - 3) { |
202 | 0 | int bl_index = |
203 | 0 | (mi_row >> 1) * (mi_params->mi_cols >> 1) + (mi_col >> 1); |
204 | 0 | int bl_index1 = bl_index + 1; |
205 | 0 | int bl_index2 = bl_index + (mi_params->mi_cols >> 1); |
206 | 0 | int bl_index3 = bl_index2 + 1; |
207 | 0 | int consec_zeromv = |
208 | 0 | AOMMIN(cpi->consec_zero_mv[bl_index], |
209 | 0 | AOMMIN(cpi->consec_zero_mv[bl_index1], |
210 | 0 | AOMMIN(cpi->consec_zero_mv[bl_index2], |
211 | 0 | cpi->consec_zero_mv[bl_index3]))); |
212 | | // Only consider blocks that are likely steady background. i.e, have |
213 | | // been encoded as zero/low motion x (= thresh_consec_zeromv) frames |
214 | | // in a row. consec_zero_mv[] defined for 8x8 blocks, so consider all |
215 | | // 4 sub-blocks for 16x16 block. And exclude this frame if |
216 | | // high_source_sad is true (i.e., scene/content change). |
217 | 0 | if (frame_low_motion && consec_zeromv > thresh_consec_zeromv && |
218 | 0 | !cpi->rc.high_source_sad) { |
219 | 0 | unsigned int sse; |
220 | | // Compute variance between co-located blocks from current and |
221 | | // last input frames. |
222 | 0 | unsigned int variance = cpi->ppi->fn_ptr[bsize].vf( |
223 | 0 | src_y, src_ystride, last_src_y, last_src_ystride, &sse); |
224 | 0 | unsigned int hist_index = variance / bin_size; |
225 | 0 | if (hist_index < MAX_VAR_HIST_BINS) |
226 | 0 | hist[hist_index]++; |
227 | 0 | else if (hist_index < 3 * (MAX_VAR_HIST_BINS >> 1)) |
228 | 0 | hist[MAX_VAR_HIST_BINS - 1]++; // Account for the tail |
229 | 0 | } |
230 | 0 | } |
231 | 0 | src_y += 4; |
232 | 0 | last_src_y += 4; |
233 | 0 | } |
234 | 0 | src_y += (src_ystride << 2) - (mi_params->mi_cols << 2); |
235 | 0 | last_src_y += (last_src_ystride << 2) - (mi_params->mi_cols << 2); |
236 | 0 | } |
237 | 0 | ne->last_w = cm->width; |
238 | 0 | ne->last_h = cm->height; |
239 | | // Adjust histogram to account for effect that histogram flattens |
240 | | // and shifts to zero as scene darkens. |
241 | 0 | if (hist[0] > 10 && (hist[MAX_VAR_HIST_BINS - 1] > hist[0] >> 2)) { |
242 | 0 | hist[0] = 0; |
243 | 0 | hist[1] >>= 2; |
244 | 0 | hist[2] >>= 2; |
245 | 0 | hist[3] >>= 2; |
246 | 0 | hist[4] >>= 1; |
247 | 0 | hist[5] >>= 1; |
248 | 0 | hist[6] = 3 * hist[6] >> 1; |
249 | 0 | hist[MAX_VAR_HIST_BINS - 1] >>= 1; |
250 | 0 | } |
251 | | |
252 | | // Average hist[] and find largest bin |
253 | 0 | for (bin_cnt = 0; bin_cnt < MAX_VAR_HIST_BINS; bin_cnt++) { |
254 | 0 | if (bin_cnt == 0) |
255 | 0 | hist_avg[bin_cnt] = (hist[0] + hist[1] + hist[2]) / 3; |
256 | 0 | else if (bin_cnt == MAX_VAR_HIST_BINS - 1) |
257 | 0 | hist_avg[bin_cnt] = hist[MAX_VAR_HIST_BINS - 1] >> 2; |
258 | 0 | else if (bin_cnt == MAX_VAR_HIST_BINS - 2) |
259 | 0 | hist_avg[bin_cnt] = (hist[bin_cnt - 1] + 2 * hist[bin_cnt] + |
260 | 0 | (hist[bin_cnt + 1] >> 1) + 2) >> |
261 | 0 | 2; |
262 | 0 | else |
263 | 0 | hist_avg[bin_cnt] = |
264 | 0 | (hist[bin_cnt - 1] + 2 * hist[bin_cnt] + hist[bin_cnt + 1] + 2) >> |
265 | 0 | 2; |
266 | |
|
267 | 0 | if (hist_avg[bin_cnt] > max_bin_count) { |
268 | 0 | max_bin_count = hist_avg[bin_cnt]; |
269 | 0 | max_bin = bin_cnt; |
270 | 0 | } |
271 | 0 | } |
272 | | // Scale by 40 to work with existing thresholds |
273 | 0 | ne->value = (int)((3 * ne->value + max_bin * 40) >> 2); |
274 | | // Quickly increase VNR strength when the noise level increases suddenly. |
275 | 0 | if (ne->level < kMedium && ne->value > ne->adapt_thresh) { |
276 | 0 | ne->count = ne->num_frames_estimate; |
277 | 0 | } else { |
278 | 0 | ne->count++; |
279 | 0 | } |
280 | 0 | if (ne->count == ne->num_frames_estimate) { |
281 | | // Reset counter and check noise level condition. |
282 | 0 | ne->num_frames_estimate = 30; |
283 | 0 | ne->count = 0; |
284 | 0 | ne->level = av1_noise_estimate_extract_level(ne); |
285 | | #if CONFIG_AV1_TEMPORAL_DENOISING |
286 | | if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi)) |
287 | | av1_denoiser_set_noise_level(cpi, ne->level); |
288 | | #endif |
289 | 0 | } |
290 | 0 | } |
291 | | #if CONFIG_AV1_TEMPORAL_DENOISING |
292 | | if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi)) |
293 | | copy_frame(&cpi->denoiser.last_source, cpi->source); |
294 | | #endif |
295 | 0 | } |