Coverage Report

Created: 2022-08-24 06:11

/src/aom/av1/encoder/av1_noise_estimate.c
Line
Count
Source (jump to first uncovered line)
1
/*
2
 * Copyright (c) 2020, Alliance for Open Media. All rights reserved
3
 *
4
 * This source code is subject to the terms of the BSD 2 Clause License and
5
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6
 * was not distributed with this source code in the LICENSE file, you can
7
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8
 * Media Patent License 1.0 was not distributed with this source code in the
9
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10
 */
11
12
#include <assert.h>
13
#include <limits.h>
14
#include <math.h>
15
16
#include "config/aom_dsp_rtcd.h"
17
#include "aom_dsp/aom_dsp_common.h"
18
#include "aom_scale/yv12config.h"
19
#include "aom/aom_integer.h"
20
#include "av1/encoder/context_tree.h"
21
#include "av1/encoder/av1_noise_estimate.h"
22
#include "av1/encoder/encoder.h"
23
#if CONFIG_AV1_TEMPORAL_DENOISING
24
#include "av1/encoder/av1_temporal_denoiser.h"
25
#endif
26
27
#if CONFIG_AV1_TEMPORAL_DENOISING
28
// For SVC: only do noise estimation on top spatial layer.
29
static INLINE int noise_est_svc(const struct AV1_COMP *const cpi) {
30
  return (!cpi->ppi->use_svc ||
31
          (cpi->ppi->use_svc &&
32
           cpi->svc.spatial_layer_id == cpi->svc.number_spatial_layers - 1));
33
}
34
#endif
35
36
0
void av1_noise_estimate_init(NOISE_ESTIMATE *const ne, int width, int height) {
37
0
  ne->enabled = 0;
38
0
  ne->level = (width * height < 1280 * 720) ? kLowLow : kLow;
39
0
  ne->value = 0;
40
0
  ne->count = 0;
41
0
  ne->thresh = 90;
42
0
  ne->last_w = 0;
43
0
  ne->last_h = 0;
44
0
  if (width * height >= 1920 * 1080) {
45
0
    ne->thresh = 200;
46
0
  } else if (width * height >= 1280 * 720) {
47
0
    ne->thresh = 140;
48
0
  } else if (width * height >= 640 * 360) {
49
0
    ne->thresh = 115;
50
0
  }
51
0
  ne->num_frames_estimate = 15;
52
0
  ne->adapt_thresh = (3 * ne->thresh) >> 1;
53
0
}
54
55
0
static int enable_noise_estimation(AV1_COMP *const cpi) {
56
0
  const int resize_pending = is_frame_resize_pending(cpi);
57
58
0
#if CONFIG_AV1_HIGHBITDEPTH
59
0
  if (cpi->common.seq_params->use_highbitdepth) return 0;
60
0
#endif
61
// Enable noise estimation if denoising is on.
62
#if CONFIG_AV1_TEMPORAL_DENOISING
63
  if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi) &&
64
      cpi->common.width >= 320 && cpi->common.height >= 180)
65
    return 1;
66
#endif
67
  // Only allow noise estimate under certain encoding mode.
68
  // Enabled for 1 pass CBR, speed >=5, and if resolution is same as original.
69
  // Not enabled for SVC mode and screen_content_mode.
70
  // Not enabled for low resolutions.
71
0
  if (cpi->oxcf.pass == AOM_RC_ONE_PASS && cpi->oxcf.rc_cfg.mode == AOM_CBR &&
72
0
      cpi->oxcf.q_cfg.aq_mode == CYCLIC_REFRESH_AQ && cpi->oxcf.speed >= 5 &&
73
0
      resize_pending == 0 && !cpi->ppi->use_svc &&
74
0
      cpi->oxcf.tune_cfg.content != AOM_CONTENT_SCREEN &&
75
0
      cpi->common.width * cpi->common.height >= 640 * 360)
76
0
    return 1;
77
0
  else
78
0
    return 0;
79
0
}
80
81
#if CONFIG_AV1_TEMPORAL_DENOISING
82
static void copy_frame(YV12_BUFFER_CONFIG *const dest,
83
                       const YV12_BUFFER_CONFIG *const src) {
84
  const uint8_t *srcbuf = src->y_buffer;
85
  uint8_t *destbuf = dest->y_buffer;
86
87
  assert(dest->y_width == src->y_width);
88
  assert(dest->y_height == src->y_height);
89
90
  for (int r = 0; r < dest->y_height; ++r) {
91
    memcpy(destbuf, srcbuf, dest->y_width);
92
    destbuf += dest->y_stride;
93
    srcbuf += src->y_stride;
94
  }
95
}
96
#endif  // CONFIG_AV1_TEMPORAL_DENOISING
97
98
0
NOISE_LEVEL av1_noise_estimate_extract_level(NOISE_ESTIMATE *const ne) {
99
0
  int noise_level = kLowLow;
100
0
  if (ne->value > (ne->thresh << 1)) {
101
0
    noise_level = kHigh;
102
0
  } else {
103
0
    if (ne->value > ne->thresh)
104
0
      noise_level = kMedium;
105
0
    else if (ne->value > (ne->thresh >> 1))
106
0
      noise_level = kLow;
107
0
    else
108
0
      noise_level = kLowLow;
109
0
  }
110
0
  return noise_level;
111
0
}
112
113
0
void av1_update_noise_estimate(AV1_COMP *const cpi) {
114
0
  const AV1_COMMON *const cm = &cpi->common;
115
0
  const CommonModeInfoParams *const mi_params = &cm->mi_params;
116
117
0
  NOISE_ESTIMATE *const ne = &cpi->noise_estimate;
118
0
  const int low_res = (cm->width <= 352 && cm->height <= 288);
119
  // Estimate of noise level every frame_period frames.
120
0
  int frame_period = 8;
121
0
  int thresh_consec_zeromv = 2;
122
0
  int frame_counter = cm->current_frame.frame_number;
123
  // Estimate is between current source and last source.
124
0
  YV12_BUFFER_CONFIG *last_source = cpi->last_source;
125
#if CONFIG_AV1_TEMPORAL_DENOISING
126
  if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi)) {
127
    last_source = &cpi->denoiser.last_source;
128
    // Tune these thresholds for different resolutions when denoising is
129
    // enabled.
130
    if (cm->width > 640 && cm->width <= 1920) {
131
      thresh_consec_zeromv = 2;
132
    }
133
  }
134
#endif
135
0
  ne->enabled = enable_noise_estimation(cpi);
136
0
  if (cpi->svc.number_spatial_layers > 1)
137
0
    frame_counter = cpi->svc.current_superframe;
138
0
  if (!ne->enabled || frame_counter % frame_period != 0 ||
139
0
      last_source == NULL ||
140
0
      (cpi->svc.number_spatial_layers == 1 &&
141
0
       (ne->last_w != cm->width || ne->last_h != cm->height))) {
142
#if CONFIG_AV1_TEMPORAL_DENOISING
143
    if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi))
144
      copy_frame(&cpi->denoiser.last_source, cpi->source);
145
#endif
146
0
    if (last_source != NULL) {
147
0
      ne->last_w = cm->width;
148
0
      ne->last_h = cm->height;
149
0
    }
150
0
    return;
151
0
  } else if (frame_counter > 60 && cpi->svc.num_encoded_top_layer > 1 &&
152
0
             cpi->rc.frames_since_key > cpi->svc.number_spatial_layers &&
153
0
             cpi->svc.spatial_layer_id == cpi->svc.number_spatial_layers - 1 &&
154
0
             cpi->rc.avg_frame_low_motion < (low_res ? 60 : 40)) {
155
    // Force noise estimation to 0 and denoiser off if content has high motion.
156
0
    ne->level = kLowLow;
157
0
    ne->count = 0;
158
0
    ne->num_frames_estimate = 10;
159
#if CONFIG_AV1_TEMPORAL_DENOISING
160
    if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi) &&
161
        cpi->svc.current_superframe > 1) {
162
      av1_denoiser_set_noise_level(cpi, ne->level);
163
      copy_frame(&cpi->denoiser.last_source, cpi->source);
164
    }
165
#endif
166
0
    return;
167
0
  } else {
168
0
    unsigned int bin_size = 100;
169
0
    unsigned int hist[MAX_VAR_HIST_BINS] = { 0 };
170
0
    unsigned int hist_avg[MAX_VAR_HIST_BINS];
171
0
    unsigned int max_bin = 0;
172
0
    unsigned int max_bin_count = 0;
173
0
    unsigned int bin_cnt;
174
0
    int bsize = BLOCK_16X16;
175
    // Loop over sub-sample of 16x16 blocks of frame, and for blocks that have
176
    // been encoded as zero/small mv at least x consecutive frames, compute
177
    // the variance to update estimate of noise in the source.
178
0
    const uint8_t *src_y = cpi->source->y_buffer;
179
0
    const int src_ystride = cpi->source->y_stride;
180
0
    const uint8_t *last_src_y = last_source->y_buffer;
181
0
    const int last_src_ystride = last_source->y_stride;
182
0
    int mi_row, mi_col;
183
0
    int num_low_motion = 0;
184
0
    int frame_low_motion = 1;
185
0
    for (mi_row = 0; mi_row < mi_params->mi_rows; mi_row += 2) {
186
0
      for (mi_col = 0; mi_col < mi_params->mi_cols; mi_col += 2) {
187
0
        int bl_index =
188
0
            (mi_row >> 1) * (mi_params->mi_cols >> 1) + (mi_col >> 1);
189
0
        if (cpi->consec_zero_mv[bl_index] > thresh_consec_zeromv)
190
0
          num_low_motion++;
191
0
      }
192
0
    }
193
0
    if (num_low_motion <
194
0
        (((3 * (mi_params->mi_rows * mi_params->mi_cols) >> 2)) >> 3))
195
0
      frame_low_motion = 0;
196
0
    for (mi_row = 0; mi_row < mi_params->mi_rows; mi_row++) {
197
0
      for (mi_col = 0; mi_col < mi_params->mi_cols; mi_col++) {
198
        // 16x16 blocks, 1/4 sample of frame.
199
0
        if (mi_row % 8 == 0 && mi_col % 8 == 0 &&
200
0
            mi_row < mi_params->mi_rows - 3 &&
201
0
            mi_col < mi_params->mi_cols - 3) {
202
0
          int bl_index =
203
0
              (mi_row >> 1) * (mi_params->mi_cols >> 1) + (mi_col >> 1);
204
0
          int bl_index1 = bl_index + 1;
205
0
          int bl_index2 = bl_index + (mi_params->mi_cols >> 1);
206
0
          int bl_index3 = bl_index2 + 1;
207
0
          int consec_zeromv =
208
0
              AOMMIN(cpi->consec_zero_mv[bl_index],
209
0
                     AOMMIN(cpi->consec_zero_mv[bl_index1],
210
0
                            AOMMIN(cpi->consec_zero_mv[bl_index2],
211
0
                                   cpi->consec_zero_mv[bl_index3])));
212
          // Only consider blocks that are likely steady background. i.e, have
213
          // been encoded as zero/low motion x (= thresh_consec_zeromv) frames
214
          // in a row. consec_zero_mv[] defined for 8x8 blocks, so consider all
215
          // 4 sub-blocks for 16x16 block. And exclude this frame if
216
          // high_source_sad is true (i.e., scene/content change).
217
0
          if (frame_low_motion && consec_zeromv > thresh_consec_zeromv &&
218
0
              !cpi->rc.high_source_sad) {
219
0
            unsigned int sse;
220
            // Compute variance between co-located blocks from current and
221
            // last input frames.
222
0
            unsigned int variance = cpi->ppi->fn_ptr[bsize].vf(
223
0
                src_y, src_ystride, last_src_y, last_src_ystride, &sse);
224
0
            unsigned int hist_index = variance / bin_size;
225
0
            if (hist_index < MAX_VAR_HIST_BINS)
226
0
              hist[hist_index]++;
227
0
            else if (hist_index < 3 * (MAX_VAR_HIST_BINS >> 1))
228
0
              hist[MAX_VAR_HIST_BINS - 1]++;  // Account for the tail
229
0
          }
230
0
        }
231
0
        src_y += 4;
232
0
        last_src_y += 4;
233
0
      }
234
0
      src_y += (src_ystride << 2) - (mi_params->mi_cols << 2);
235
0
      last_src_y += (last_src_ystride << 2) - (mi_params->mi_cols << 2);
236
0
    }
237
0
    ne->last_w = cm->width;
238
0
    ne->last_h = cm->height;
239
    // Adjust histogram to account for effect that histogram flattens
240
    // and shifts to zero as scene darkens.
241
0
    if (hist[0] > 10 && (hist[MAX_VAR_HIST_BINS - 1] > hist[0] >> 2)) {
242
0
      hist[0] = 0;
243
0
      hist[1] >>= 2;
244
0
      hist[2] >>= 2;
245
0
      hist[3] >>= 2;
246
0
      hist[4] >>= 1;
247
0
      hist[5] >>= 1;
248
0
      hist[6] = 3 * hist[6] >> 1;
249
0
      hist[MAX_VAR_HIST_BINS - 1] >>= 1;
250
0
    }
251
252
    // Average hist[] and find largest bin
253
0
    for (bin_cnt = 0; bin_cnt < MAX_VAR_HIST_BINS; bin_cnt++) {
254
0
      if (bin_cnt == 0)
255
0
        hist_avg[bin_cnt] = (hist[0] + hist[1] + hist[2]) / 3;
256
0
      else if (bin_cnt == MAX_VAR_HIST_BINS - 1)
257
0
        hist_avg[bin_cnt] = hist[MAX_VAR_HIST_BINS - 1] >> 2;
258
0
      else if (bin_cnt == MAX_VAR_HIST_BINS - 2)
259
0
        hist_avg[bin_cnt] = (hist[bin_cnt - 1] + 2 * hist[bin_cnt] +
260
0
                             (hist[bin_cnt + 1] >> 1) + 2) >>
261
0
                            2;
262
0
      else
263
0
        hist_avg[bin_cnt] =
264
0
            (hist[bin_cnt - 1] + 2 * hist[bin_cnt] + hist[bin_cnt + 1] + 2) >>
265
0
            2;
266
267
0
      if (hist_avg[bin_cnt] > max_bin_count) {
268
0
        max_bin_count = hist_avg[bin_cnt];
269
0
        max_bin = bin_cnt;
270
0
      }
271
0
    }
272
    // Scale by 40 to work with existing thresholds
273
0
    ne->value = (int)((3 * ne->value + max_bin * 40) >> 2);
274
    // Quickly increase VNR strength when the noise level increases suddenly.
275
0
    if (ne->level < kMedium && ne->value > ne->adapt_thresh) {
276
0
      ne->count = ne->num_frames_estimate;
277
0
    } else {
278
0
      ne->count++;
279
0
    }
280
0
    if (ne->count == ne->num_frames_estimate) {
281
      // Reset counter and check noise level condition.
282
0
      ne->num_frames_estimate = 30;
283
0
      ne->count = 0;
284
0
      ne->level = av1_noise_estimate_extract_level(ne);
285
#if CONFIG_AV1_TEMPORAL_DENOISING
286
      if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi))
287
        av1_denoiser_set_noise_level(cpi, ne->level);
288
#endif
289
0
    }
290
0
  }
291
#if CONFIG_AV1_TEMPORAL_DENOISING
292
  if (cpi->oxcf.noise_sensitivity > 0 && noise_est_svc(cpi))
293
    copy_frame(&cpi->denoiser.last_source, cpi->source);
294
#endif
295
0
}