Coverage Report

Created: 2025-06-22 08:04

/src/aom/av1/encoder/global_motion_facade.c
Line
Count
Source (jump to first uncovered line)
1
/*
2
 * Copyright (c) 2020, Alliance for Open Media. All rights reserved.
3
 *
4
 * This source code is subject to the terms of the BSD 2 Clause License and
5
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6
 * was not distributed with this source code in the LICENSE file, you can
7
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8
 * Media Patent License 1.0 was not distributed with this source code in the
9
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10
 */
11
12
#include "aom_dsp/binary_codes_writer.h"
13
14
#include "aom_dsp/flow_estimation/corner_detect.h"
15
#include "aom_dsp/flow_estimation/flow_estimation.h"
16
#include "aom_dsp/pyramid.h"
17
#include "av1/common/warped_motion.h"
18
#include "av1/encoder/encoder.h"
19
#include "av1/encoder/ethread.h"
20
#include "av1/encoder/rdopt.h"
21
#include "av1/encoder/global_motion_facade.h"
22
23
// Range of model types to search
24
0
#define FIRST_GLOBAL_TRANS_TYPE ROTZOOM
25
0
#define LAST_GLOBAL_TRANS_TYPE ROTZOOM
26
27
// Computes the cost for the warp parameters.
28
static int gm_get_params_cost(const WarpedMotionParams *gm,
29
0
                              const WarpedMotionParams *ref_gm, int allow_hp) {
30
0
  int params_cost = 0;
31
0
  int trans_bits, trans_prec_diff;
32
0
  switch (gm->wmtype) {
33
0
    case AFFINE:
34
0
    case ROTZOOM:
35
0
      params_cost += aom_count_signed_primitive_refsubexpfin(
36
0
          GM_ALPHA_MAX + 1, SUBEXPFIN_K,
37
0
          (ref_gm->wmmat[2] >> GM_ALPHA_PREC_DIFF) - (1 << GM_ALPHA_PREC_BITS),
38
0
          (gm->wmmat[2] >> GM_ALPHA_PREC_DIFF) - (1 << GM_ALPHA_PREC_BITS));
39
0
      params_cost += aom_count_signed_primitive_refsubexpfin(
40
0
          GM_ALPHA_MAX + 1, SUBEXPFIN_K,
41
0
          (ref_gm->wmmat[3] >> GM_ALPHA_PREC_DIFF),
42
0
          (gm->wmmat[3] >> GM_ALPHA_PREC_DIFF));
43
0
      if (gm->wmtype >= AFFINE) {
44
0
        params_cost += aom_count_signed_primitive_refsubexpfin(
45
0
            GM_ALPHA_MAX + 1, SUBEXPFIN_K,
46
0
            (ref_gm->wmmat[4] >> GM_ALPHA_PREC_DIFF),
47
0
            (gm->wmmat[4] >> GM_ALPHA_PREC_DIFF));
48
0
        params_cost += aom_count_signed_primitive_refsubexpfin(
49
0
            GM_ALPHA_MAX + 1, SUBEXPFIN_K,
50
0
            (ref_gm->wmmat[5] >> GM_ALPHA_PREC_DIFF) -
51
0
                (1 << GM_ALPHA_PREC_BITS),
52
0
            (gm->wmmat[5] >> GM_ALPHA_PREC_DIFF) - (1 << GM_ALPHA_PREC_BITS));
53
0
      }
54
0
      AOM_FALLTHROUGH_INTENDED;
55
0
    case TRANSLATION:
56
0
      trans_bits = (gm->wmtype == TRANSLATION)
57
0
                       ? GM_ABS_TRANS_ONLY_BITS - !allow_hp
58
0
                       : GM_ABS_TRANS_BITS;
59
0
      trans_prec_diff = (gm->wmtype == TRANSLATION)
60
0
                            ? GM_TRANS_ONLY_PREC_DIFF + !allow_hp
61
0
                            : GM_TRANS_PREC_DIFF;
62
0
      params_cost += aom_count_signed_primitive_refsubexpfin(
63
0
          (1 << trans_bits) + 1, SUBEXPFIN_K,
64
0
          (ref_gm->wmmat[0] >> trans_prec_diff),
65
0
          (gm->wmmat[0] >> trans_prec_diff));
66
0
      params_cost += aom_count_signed_primitive_refsubexpfin(
67
0
          (1 << trans_bits) + 1, SUBEXPFIN_K,
68
0
          (ref_gm->wmmat[1] >> trans_prec_diff),
69
0
          (gm->wmmat[1] >> trans_prec_diff));
70
0
      AOM_FALLTHROUGH_INTENDED;
71
0
    case IDENTITY: break;
72
0
    default: assert(0);
73
0
  }
74
0
  return (params_cost << AV1_PROB_COST_SHIFT);
75
0
}
76
77
// For the given reference frame, computes the global motion parameters for
78
// different motion models and finds the best.
79
static inline void compute_global_motion_for_ref_frame(
80
    AV1_COMP *cpi, struct aom_internal_error_info *error_info,
81
    YV12_BUFFER_CONFIG *ref_buf[REF_FRAMES], int frame,
82
    MotionModel *motion_models, uint8_t *segment_map, const int segment_map_w,
83
0
    const int segment_map_h, const WarpedMotionParams *ref_params) {
84
0
  AV1_COMMON *const cm = &cpi->common;
85
0
  MACROBLOCKD *const xd = &cpi->td.mb.e_mbd;
86
0
  int src_width = cpi->source->y_crop_width;
87
0
  int src_height = cpi->source->y_crop_height;
88
0
  int src_stride = cpi->source->y_stride;
89
0
  assert(ref_buf[frame] != NULL);
90
0
  int bit_depth = cpi->common.seq_params->bit_depth;
91
0
  GlobalMotionMethod global_motion_method = default_global_motion_method;
92
0
  int downsample_level = cpi->sf.gm_sf.downsample_level;
93
0
  int num_refinements = cpi->sf.gm_sf.num_refinement_steps;
94
0
  bool mem_alloc_failed = false;
95
96
  // Select the best model based on fractional error reduction.
97
  // By initializing this to erroradv_tr, the same logic which is used to
98
  // select the best model will automatically filter out any model which
99
  // doesn't meet the required quality threshold
100
0
  double best_erroradv = erroradv_tr;
101
0
  for (TransformationType model = FIRST_GLOBAL_TRANS_TYPE;
102
0
       model <= LAST_GLOBAL_TRANS_TYPE; ++model) {
103
0
    if (!aom_compute_global_motion(model, cpi->source, ref_buf[frame],
104
0
                                   bit_depth, global_motion_method,
105
0
                                   downsample_level, motion_models,
106
0
                                   RANSAC_NUM_MOTIONS, &mem_alloc_failed)) {
107
0
      if (mem_alloc_failed) {
108
0
        aom_internal_error(error_info, AOM_CODEC_MEM_ERROR,
109
0
                           "Failed to allocate global motion buffers");
110
0
      }
111
0
      continue;
112
0
    }
113
114
0
    for (int i = 0; i < RANSAC_NUM_MOTIONS; ++i) {
115
0
      if (motion_models[i].num_inliers == 0) continue;
116
117
0
      WarpedMotionParams tmp_wm_params;
118
0
      av1_convert_model_to_params(motion_models[i].params, &tmp_wm_params);
119
120
      // Check that the generated model is warp-able
121
0
      if (!av1_get_shear_params(&tmp_wm_params)) continue;
122
123
      // Skip models that we won't use (IDENTITY or TRANSLATION)
124
      //
125
      // For IDENTITY type models, we don't need to evaluate anything because
126
      // all the following logic is effectively comparing the estimated model
127
      // to an identity model.
128
      //
129
      // For TRANSLATION type global motion models, gm_get_motion_vector() gives
130
      // the wrong motion vector (see comments in that function for details).
131
      // As translation-type models do not give much gain, we can avoid this bug
132
      // by never choosing a TRANSLATION type model
133
0
      if (tmp_wm_params.wmtype <= TRANSLATION) continue;
134
135
0
      av1_compute_feature_segmentation_map(
136
0
          segment_map, segment_map_w, segment_map_h, motion_models[i].inliers,
137
0
          motion_models[i].num_inliers);
138
139
0
      int64_t ref_frame_error = av1_segmented_frame_error(
140
0
          is_cur_buf_hbd(xd), xd->bd, ref_buf[frame]->y_buffer,
141
0
          ref_buf[frame]->y_stride, cpi->source->y_buffer, src_stride,
142
0
          src_width, src_height, segment_map, segment_map_w);
143
144
0
      if (ref_frame_error == 0) continue;
145
146
0
      const int64_t warp_error = av1_refine_integerized_param(
147
0
          &tmp_wm_params, tmp_wm_params.wmtype, is_cur_buf_hbd(xd), xd->bd,
148
0
          ref_buf[frame]->y_buffer, ref_buf[frame]->y_crop_width,
149
0
          ref_buf[frame]->y_crop_height, ref_buf[frame]->y_stride,
150
0
          cpi->source->y_buffer, src_width, src_height, src_stride,
151
0
          num_refinements, ref_frame_error, segment_map, segment_map_w);
152
153
      // av1_refine_integerized_param() can return a simpler model type than
154
      // its input, so re-check model type here
155
0
      if (tmp_wm_params.wmtype <= TRANSLATION) continue;
156
157
0
      double erroradvantage = (double)warp_error / ref_frame_error;
158
159
      // Check that the model signaling cost is not too high
160
0
      if (!av1_is_enough_erroradvantage(
161
0
              erroradvantage,
162
0
              gm_get_params_cost(&tmp_wm_params, ref_params,
163
0
                                 cm->features.allow_high_precision_mv))) {
164
0
        continue;
165
0
      }
166
167
0
      if (erroradvantage < best_erroradv) {
168
0
        best_erroradv = erroradvantage;
169
        // Save the wm_params modified by
170
        // av1_refine_integerized_param() rather than motion index to
171
        // avoid rerunning refine() below.
172
0
        memcpy(&(cm->global_motion[frame]), &tmp_wm_params,
173
0
               sizeof(WarpedMotionParams));
174
0
      }
175
0
    }
176
0
  }
177
0
}
178
179
// Computes global motion for the given reference frame.
180
void av1_compute_gm_for_valid_ref_frames(
181
    AV1_COMP *cpi, struct aom_internal_error_info *error_info,
182
    YV12_BUFFER_CONFIG *ref_buf[REF_FRAMES], int frame,
183
    MotionModel *motion_models, uint8_t *segment_map, int segment_map_w,
184
0
    int segment_map_h) {
185
0
  AV1_COMMON *const cm = &cpi->common;
186
0
  const WarpedMotionParams *ref_params =
187
0
      cm->prev_frame ? &cm->prev_frame->global_motion[frame]
188
0
                     : &default_warp_params;
189
190
0
  compute_global_motion_for_ref_frame(cpi, error_info, ref_buf, frame,
191
0
                                      motion_models, segment_map, segment_map_w,
192
0
                                      segment_map_h, ref_params);
193
0
}
194
195
// Loops over valid reference frames and computes global motion estimation.
196
static inline void compute_global_motion_for_references(
197
    AV1_COMP *cpi, YV12_BUFFER_CONFIG *ref_buf[REF_FRAMES],
198
    FrameDistPair reference_frame[REF_FRAMES - 1], int num_ref_frames,
199
    MotionModel *motion_models, uint8_t *segment_map, const int segment_map_w,
200
0
    const int segment_map_h) {
201
0
  AV1_COMMON *const cm = &cpi->common;
202
0
  struct aom_internal_error_info *const error_info =
203
0
      cpi->td.mb.e_mbd.error_info;
204
  // Compute global motion w.r.t. reference frames starting from the nearest ref
205
  // frame in a given direction.
206
0
  for (int frame = 0; frame < num_ref_frames; frame++) {
207
0
    int ref_frame = reference_frame[frame].frame;
208
0
    av1_compute_gm_for_valid_ref_frames(cpi, error_info, ref_buf, ref_frame,
209
0
                                        motion_models, segment_map,
210
0
                                        segment_map_w, segment_map_h);
211
    // If global motion w.r.t. current ref frame is
212
    // INVALID/TRANSLATION/IDENTITY, skip the evaluation of global motion w.r.t
213
    // the remaining ref frames in that direction.
214
0
    if (cpi->sf.gm_sf.prune_ref_frame_for_gm_search &&
215
0
        cm->global_motion[ref_frame].wmtype <= TRANSLATION)
216
0
      break;
217
0
  }
218
0
}
219
220
// Compares the distance in 'a' and 'b'. Returns 1 if the frame corresponding to
221
// 'a' is farther, -1 if the frame corresponding to 'b' is farther, 0 otherwise.
222
0
static int compare_distance(const void *a, const void *b) {
223
0
  const int diff =
224
0
      ((FrameDistPair *)a)->distance - ((FrameDistPair *)b)->distance;
225
0
  if (diff > 0)
226
0
    return 1;
227
0
  else if (diff < 0)
228
0
    return -1;
229
0
  return 0;
230
0
}
231
232
0
static int disable_gm_search_based_on_stats(const AV1_COMP *const cpi) {
233
0
  int is_gm_present = 1;
234
235
  // Check number of GM models only in GF groups with ARF frames. GM param
236
  // estimation is always done in the case of GF groups with no ARF frames (flat
237
  // gops)
238
0
  if (cpi->ppi->gf_group.arf_index > -1) {
239
    // valid_gm_model_found is initialized to INT32_MAX in the beginning of
240
    // every GF group.
241
    // Therefore, GM param estimation is always done for all frames until
242
    // at least 1 frame each of ARF_UPDATE, INTNL_ARF_UPDATE and LF_UPDATE are
243
    // encoded in a GF group For subsequent frames, GM param estimation is
244
    // disabled, if no valid models have been found in all the three update
245
    // types.
246
0
    is_gm_present = (cpi->ppi->valid_gm_model_found[ARF_UPDATE] != 0) ||
247
0
                    (cpi->ppi->valid_gm_model_found[INTNL_ARF_UPDATE] != 0) ||
248
0
                    (cpi->ppi->valid_gm_model_found[LF_UPDATE] != 0);
249
0
  }
250
0
  return !is_gm_present;
251
0
}
252
253
// Prunes reference frames for global motion estimation based on the speed
254
// feature 'gm_search_type'.
255
0
static int do_gm_search_logic(SPEED_FEATURES *const sf, int frame) {
256
0
  (void)frame;
257
0
  switch (sf->gm_sf.gm_search_type) {
258
0
    case GM_FULL_SEARCH: return 1;
259
0
    case GM_REDUCED_REF_SEARCH_SKIP_L2_L3:
260
0
      return !(frame == LAST2_FRAME || frame == LAST3_FRAME);
261
0
    case GM_REDUCED_REF_SEARCH_SKIP_L2_L3_ARF2:
262
0
      return !(frame == LAST2_FRAME || frame == LAST3_FRAME ||
263
0
               (frame == ALTREF2_FRAME));
264
0
    case GM_SEARCH_CLOSEST_REFS_ONLY: return 1;
265
0
    case GM_DISABLE_SEARCH: return 0;
266
0
    default: assert(0);
267
0
  }
268
0
  return 1;
269
0
}
270
271
// Populates valid reference frames in past/future directions in
272
// 'reference_frames' and their count in 'num_ref_frames'.
273
static inline void update_valid_ref_frames_for_gm(
274
    AV1_COMP *cpi, YV12_BUFFER_CONFIG *ref_buf[REF_FRAMES],
275
    FrameDistPair reference_frames[MAX_DIRECTIONS][REF_FRAMES - 1],
276
0
    int *num_ref_frames) {
277
0
  AV1_COMMON *const cm = &cpi->common;
278
0
  int *num_past_ref_frames = &num_ref_frames[0];
279
0
  int *num_future_ref_frames = &num_ref_frames[1];
280
0
  const GF_GROUP *gf_group = &cpi->ppi->gf_group;
281
0
  int ref_pruning_enabled = is_frame_eligible_for_ref_pruning(
282
0
      gf_group, cpi->sf.inter_sf.selective_ref_frame, 1, cpi->gf_frame_index);
283
0
  int cur_frame_gm_disabled = 0;
284
0
  int pyr_lvl = cm->cur_frame->pyramid_level;
285
286
0
  if (cpi->sf.gm_sf.disable_gm_search_based_on_stats) {
287
0
    cur_frame_gm_disabled = disable_gm_search_based_on_stats(cpi);
288
0
  }
289
290
0
  for (int frame = ALTREF_FRAME; frame >= LAST_FRAME; --frame) {
291
0
    const MV_REFERENCE_FRAME ref_frame[2] = { frame, NONE_FRAME };
292
0
    RefCntBuffer *buf = get_ref_frame_buf(cm, frame);
293
0
    const int ref_disabled =
294
0
        !(cpi->ref_frame_flags & av1_ref_frame_flag_list[frame]);
295
0
    ref_buf[frame] = NULL;
296
0
    cm->global_motion[frame] = default_warp_params;
297
    // Skip global motion estimation for invalid ref frames
298
0
    if (buf == NULL ||
299
0
        (ref_disabled && cpi->sf.hl_sf.recode_loop != DISALLOW_RECODE)) {
300
0
      continue;
301
0
    } else {
302
0
      ref_buf[frame] = &buf->buf;
303
0
    }
304
305
0
    int prune_ref_frames =
306
0
        ref_pruning_enabled &&
307
0
        prune_ref_by_selective_ref_frame(cpi, NULL, ref_frame,
308
0
                                         cm->cur_frame->ref_display_order_hint);
309
0
    int ref_pyr_lvl = buf->pyramid_level;
310
311
0
    if (ref_buf[frame]->y_crop_width == cpi->source->y_crop_width &&
312
0
        ref_buf[frame]->y_crop_height == cpi->source->y_crop_height &&
313
0
        do_gm_search_logic(&cpi->sf, frame) && !prune_ref_frames &&
314
0
        ref_pyr_lvl <= pyr_lvl && !cur_frame_gm_disabled) {
315
0
      assert(ref_buf[frame] != NULL);
316
0
      const int relative_frame_dist = av1_encoder_get_relative_dist(
317
0
          buf->display_order_hint, cm->cur_frame->display_order_hint);
318
      // Populate past and future ref frames.
319
      // reference_frames[0][] indicates past direction and
320
      // reference_frames[1][] indicates future direction.
321
0
      if (relative_frame_dist == 0) {
322
        // Skip global motion estimation for frames at the same nominal instant.
323
        // This will generally be either a "real" frame coded against a
324
        // temporal filtered version, or a higher spatial layer coded against
325
        // a lower spatial layer. In either case, the optimal motion model will
326
        // be IDENTITY, so we don't need to search explicitly.
327
0
      } else if (relative_frame_dist < 0) {
328
0
        reference_frames[0][*num_past_ref_frames].distance =
329
0
            abs(relative_frame_dist);
330
0
        reference_frames[0][*num_past_ref_frames].frame = frame;
331
0
        (*num_past_ref_frames)++;
332
0
      } else {
333
0
        reference_frames[1][*num_future_ref_frames].distance =
334
0
            abs(relative_frame_dist);
335
0
        reference_frames[1][*num_future_ref_frames].frame = frame;
336
0
        (*num_future_ref_frames)++;
337
0
      }
338
0
    }
339
0
  }
340
0
}
341
342
// Initializes parameters used for computing global motion.
343
0
static inline void setup_global_motion_info_params(AV1_COMP *cpi) {
344
0
  GlobalMotionInfo *const gm_info = &cpi->gm_info;
345
0
  YV12_BUFFER_CONFIG *source = cpi->source;
346
347
0
  gm_info->segment_map_w =
348
0
      (source->y_crop_width + WARP_ERROR_BLOCK - 1) >> WARP_ERROR_BLOCK_LOG;
349
0
  gm_info->segment_map_h =
350
0
      (source->y_crop_height + WARP_ERROR_BLOCK - 1) >> WARP_ERROR_BLOCK_LOG;
351
352
0
  memset(gm_info->reference_frames, -1,
353
0
         sizeof(gm_info->reference_frames[0][0]) * MAX_DIRECTIONS *
354
0
             (REF_FRAMES - 1));
355
0
  av1_zero(gm_info->num_ref_frames);
356
357
  // Populate ref_buf for valid ref frames in global motion
358
0
  update_valid_ref_frames_for_gm(cpi, gm_info->ref_buf,
359
0
                                 gm_info->reference_frames,
360
0
                                 gm_info->num_ref_frames);
361
362
  // Sort the past and future ref frames in the ascending order of their
363
  // distance from the current frame. reference_frames[0] => past direction
364
  // and reference_frames[1] => future direction.
365
0
  qsort(gm_info->reference_frames[0], gm_info->num_ref_frames[0],
366
0
        sizeof(gm_info->reference_frames[0][0]), compare_distance);
367
0
  qsort(gm_info->reference_frames[1], gm_info->num_ref_frames[1],
368
0
        sizeof(gm_info->reference_frames[1][0]), compare_distance);
369
370
0
  if (cpi->sf.gm_sf.gm_search_type == GM_SEARCH_CLOSEST_REFS_ONLY) {
371
    // Filter down to the nearest two ref frames.
372
    // Prefer one past and one future ref over two past refs, even if
373
    // the second past ref is closer
374
0
    if (gm_info->num_ref_frames[1] > 0) {
375
0
      gm_info->num_ref_frames[0] = AOMMIN(gm_info->num_ref_frames[0], 1);
376
0
      gm_info->num_ref_frames[1] = AOMMIN(gm_info->num_ref_frames[1], 1);
377
0
    } else {
378
0
      gm_info->num_ref_frames[0] = AOMMIN(gm_info->num_ref_frames[0], 2);
379
0
    }
380
0
  }
381
0
}
382
383
// Computes global motion w.r.t. valid reference frames.
384
0
static inline void global_motion_estimation(AV1_COMP *cpi) {
385
0
  GlobalMotionInfo *const gm_info = &cpi->gm_info;
386
0
  GlobalMotionData *gm_data = &cpi->td.gm_data;
387
388
  // Compute global motion w.r.t. past reference frames and future reference
389
  // frames
390
0
  for (int dir = 0; dir < MAX_DIRECTIONS; dir++) {
391
0
    if (gm_info->num_ref_frames[dir] > 0)
392
0
      compute_global_motion_for_references(
393
0
          cpi, gm_info->ref_buf, gm_info->reference_frames[dir],
394
0
          gm_info->num_ref_frames[dir], gm_data->motion_models,
395
0
          gm_data->segment_map, gm_info->segment_map_w, gm_info->segment_map_h);
396
0
  }
397
0
}
398
399
// Global motion estimation for the current frame is computed.This computation
400
// happens once per frame and the winner motion model parameters are stored in
401
// cm->cur_frame->global_motion.
402
0
void av1_compute_global_motion_facade(AV1_COMP *cpi) {
403
0
  AV1_COMMON *const cm = &cpi->common;
404
0
  GlobalMotionInfo *const gm_info = &cpi->gm_info;
405
406
0
  if (cpi->oxcf.tool_cfg.enable_global_motion) {
407
0
    if (cpi->gf_frame_index == 0) {
408
0
      for (int i = 0; i < FRAME_UPDATE_TYPES; i++) {
409
0
        cpi->ppi->valid_gm_model_found[i] = INT32_MAX;
410
#if CONFIG_FPMT_TEST
411
        if (cpi->ppi->fpmt_unit_test_cfg == PARALLEL_SIMULATION_ENCODE)
412
          cpi->ppi->temp_valid_gm_model_found[i] = INT32_MAX;
413
#endif
414
0
      }
415
0
    }
416
0
  }
417
418
0
  if (cpi->common.current_frame.frame_type == INTER_FRAME && cpi->source &&
419
0
      cpi->oxcf.tool_cfg.enable_global_motion && !gm_info->search_done &&
420
0
      cpi->sf.gm_sf.gm_search_type != GM_DISABLE_SEARCH) {
421
0
    setup_global_motion_info_params(cpi);
422
    // Terminate early if the total number of reference frames is zero.
423
0
    if (cpi->gm_info.num_ref_frames[0] || cpi->gm_info.num_ref_frames[1]) {
424
0
      gm_alloc_data(cpi, &cpi->td.gm_data);
425
0
      if (cpi->mt_info.num_workers > 1)
426
0
        av1_global_motion_estimation_mt(cpi);
427
0
      else
428
0
        global_motion_estimation(cpi);
429
0
      gm_dealloc_data(&cpi->td.gm_data);
430
0
      gm_info->search_done = 1;
431
0
    }
432
0
  }
433
0
  memcpy(cm->cur_frame->global_motion, cm->global_motion,
434
0
         sizeof(cm->cur_frame->global_motion));
435
0
}