/src/libvpx/vp9/common/vp9_reconintra.c

Source
/*
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "./vpx_config.h"
#include "./vpx_dsp_rtcd.h"

#if CONFIG_VP9_HIGHBITDEPTH
#include "vpx_dsp/vpx_dsp_common.h"
#endif  // CONFIG_VP9_HIGHBITDEPTH
#include "vpx_mem/vpx_mem.h"
#include "vpx_ports/mem.h"
#include "vpx_ports/vpx_once.h"

#include "vp9/common/vp9_reconintra.h"
#include "vp9/common/vp9_onyxc_int.h"

const TX_TYPE intra_mode_to_tx_type_lookup[INTRA_MODES] = {
  DCT_DCT,    // DC
  ADST_DCT,   // V
  DCT_ADST,   // H
  DCT_DCT,    // D45
  ADST_ADST,  // D135
  ADST_DCT,   // D117
  DCT_ADST,   // D153
  DCT_ADST,   // D207
  ADST_DCT,   // D63
  ADST_ADST,  // TM
};

enum {
  NEED_LEFT = 1 << 1,
  NEED_ABOVE = 1 << 2,
  NEED_ABOVERIGHT = 1 << 3,
};

static const uint8_t extend_modes[INTRA_MODES] = {
  NEED_ABOVE | NEED_LEFT,  // DC
  NEED_ABOVE,              // V
  NEED_LEFT,               // H
  NEED_ABOVERIGHT,         // D45
  NEED_LEFT | NEED_ABOVE,  // D135
  NEED_LEFT | NEED_ABOVE,  // D117
  NEED_LEFT | NEED_ABOVE,  // D153
  NEED_LEFT,               // D207
  NEED_ABOVERIGHT,         // D63
  NEED_LEFT | NEED_ABOVE,  // TM
};

typedef void (*intra_pred_fn)(uint8_t *dst, ptrdiff_t stride,
                              const uint8_t *above, const uint8_t *left);

static intra_pred_fn pred[INTRA_MODES][TX_SIZES];
static intra_pred_fn dc_pred[2][2][TX_SIZES];

#if CONFIG_VP9_HIGHBITDEPTH
typedef void (*intra_high_pred_fn)(uint16_t *dst, ptrdiff_t stride,
                                   const uint16_t *above, const uint16_t *left,
                                   int bd);
static intra_high_pred_fn pred_high[INTRA_MODES][4];
static intra_high_pred_fn dc_pred_high[2][2][4];
#endif  // CONFIG_VP9_HIGHBITDEPTH

static void vp9_init_intra_predictors_internal(void) {
#define INIT_ALL_SIZES(p, type)               \
  p[TX_4X4] = vpx_##type##_predictor_4x4;     \
  p[TX_8X8] = vpx_##type##_predictor_8x8;     \
  p[TX_16X16] = vpx_##type##_predictor_16x16; \
  p[TX_32X32] = vpx_##type##_predictor_32x32

  INIT_ALL_SIZES(pred[V_PRED], v);
  INIT_ALL_SIZES(pred[H_PRED], h);
  INIT_ALL_SIZES(pred[D207_PRED], d207);
  INIT_ALL_SIZES(pred[D45_PRED], d45);
  INIT_ALL_SIZES(pred[D63_PRED], d63);
  INIT_ALL_SIZES(pred[D117_PRED], d117);
  INIT_ALL_SIZES(pred[D135_PRED], d135);
  INIT_ALL_SIZES(pred[D153_PRED], d153);
  INIT_ALL_SIZES(pred[TM_PRED], tm);

  INIT_ALL_SIZES(dc_pred[0][0], dc_128);
  INIT_ALL_SIZES(dc_pred[0][1], dc_top);
  INIT_ALL_SIZES(dc_pred[1][0], dc_left);
  INIT_ALL_SIZES(dc_pred[1][1], dc);

#if CONFIG_VP9_HIGHBITDEPTH
  INIT_ALL_SIZES(pred_high[V_PRED], highbd_v);
  INIT_ALL_SIZES(pred_high[H_PRED], highbd_h);
  INIT_ALL_SIZES(pred_high[D207_PRED], highbd_d207);
  INIT_ALL_SIZES(pred_high[D45_PRED], highbd_d45);
  INIT_ALL_SIZES(pred_high[D63_PRED], highbd_d63);
  INIT_ALL_SIZES(pred_high[D117_PRED], highbd_d117);
  INIT_ALL_SIZES(pred_high[D135_PRED], highbd_d135);
  INIT_ALL_SIZES(pred_high[D153_PRED], highbd_d153);
  INIT_ALL_SIZES(pred_high[TM_PRED], highbd_tm);

  INIT_ALL_SIZES(dc_pred_high[0][0], highbd_dc_128);
  INIT_ALL_SIZES(dc_pred_high[0][1], highbd_dc_top);
  INIT_ALL_SIZES(dc_pred_high[1][0], highbd_dc_left);
  INIT_ALL_SIZES(dc_pred_high[1][1], highbd_dc);
#endif  // CONFIG_VP9_HIGHBITDEPTH

#undef intra_pred_allsizes
}

#if CONFIG_VP9_HIGHBITDEPTH
static void build_intra_predictors_high(
    const MACROBLOCKD *xd, const uint8_t *ref8, int ref_stride, uint8_t *dst8,
    int dst_stride, PREDICTION_MODE mode, TX_SIZE tx_size, int up_available,
    int left_available, int right_available, int x, int y, int plane, int bd) {
  int i;
  uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
  uint16_t *ref = CONVERT_TO_SHORTPTR(ref8);
  DECLARE_ALIGNED(16, uint16_t, left_col[32]);
  DECLARE_ALIGNED(16, uint16_t, above_data[64 + 16]);
  uint16_t *above_row = above_data + 16;
  const uint16_t *const_above_row = above_row;
  const int bs = 4 << tx_size;
  int frame_width, frame_height;
  int x0, y0;
  const struct macroblockd_plane *const pd = &xd->plane[plane];
  const int need_left = extend_modes[mode] & NEED_LEFT;
  const int need_above = extend_modes[mode] & NEED_ABOVE;
  const int need_aboveright = extend_modes[mode] & NEED_ABOVERIGHT;
  int base = 128 << (bd - 8);
  // 127 127 127 .. 127 127 127 127 127 127
  // 129  A   B  ..  Y   Z
  // 129  C   D  ..  W   X
  // 129  E   F  ..  U   V
  // 129  G   H  ..  S   T   T   T   T   T
  // For 10 bit and 12 bit, 127 and 129 are replaced by base -1 and base + 1.

  // Get current frame pointer, width and height.
  if (plane == 0) {
    frame_width = xd->cur_buf->y_width;
    frame_height = xd->cur_buf->y_height;
  } else {
    frame_width = xd->cur_buf->uv_width;
    frame_height = xd->cur_buf->uv_height;
  }

  // Get block position in current frame.
  x0 = (-xd->mb_to_left_edge >> (3 + pd->subsampling_x)) + x;
  y0 = (-xd->mb_to_top_edge >> (3 + pd->subsampling_y)) + y;

  // NEED_LEFT
  if (need_left) {
    if (left_available) {
      if (xd->mb_to_bottom_edge < 0) {
        /* slower path if the block needs border extension */
        if (y0 + bs <= frame_height) {
          for (i = 0; i < bs; ++i) left_col[i] = ref[i * ref_stride - 1];
        } else {
          const int extend_bottom = frame_height - y0;
          for (i = 0; i < extend_bottom; ++i)
            left_col[i] = ref[i * ref_stride - 1];
          for (; i < bs; ++i)
            left_col[i] = ref[(extend_bottom - 1) * ref_stride - 1];
        }
      } else {
        /* faster path if the block does not need extension */
        for (i = 0; i < bs; ++i) left_col[i] = ref[i * ref_stride - 1];
      }
    } else {
      vpx_memset16(left_col, base + 1, bs);
    }
  }

  // NEED_ABOVE
  if (need_above) {
    if (up_available) {
      const uint16_t *above_ref = ref - ref_stride;
      if (xd->mb_to_right_edge < 0) {
        /* slower path if the block needs border extension */
        if (x0 + bs <= frame_width) {
          memcpy(above_row, above_ref, bs * sizeof(above_row[0]));
        } else if (x0 <= frame_width) {
          const int r = frame_width - x0;
          memcpy(above_row, above_ref, r * sizeof(above_row[0]));
          vpx_memset16(above_row + r, above_row[r - 1], x0 + bs - frame_width);
        }
      } else {
        /* faster path if the block does not need extension */
        if (bs == 4 && right_available && left_available) {
          const_above_row = above_ref;
        } else {
          memcpy(above_row, above_ref, bs * sizeof(above_row[0]));
        }
      }
      above_row[-1] = left_available ? above_ref[-1] : (base + 1);
    } else {
      vpx_memset16(above_row, base - 1, bs);
      above_row[-1] = base - 1;
    }
  }

  // NEED_ABOVERIGHT
  if (need_aboveright) {
    if (up_available) {
      const uint16_t *above_ref = ref - ref_stride;
      if (xd->mb_to_right_edge < 0) {
        /* slower path if the block needs border extension */
        if (x0 + 2 * bs <= frame_width) {
          if (right_available && bs == 4) {
            memcpy(above_row, above_ref, 2 * bs * sizeof(above_row[0]));
          } else {
            memcpy(above_row, above_ref, bs * sizeof(above_row[0]));
            vpx_memset16(above_row + bs, above_row[bs - 1], bs);
          }
        } else if (x0 + bs <= frame_width) {
          const int r = frame_width - x0;
          if (right_available && bs == 4) {
            memcpy(above_row, above_ref, r * sizeof(above_row[0]));
            vpx_memset16(above_row + r, above_row[r - 1],
                         x0 + 2 * bs - frame_width);
          } else {
            memcpy(above_row, above_ref, bs * sizeof(above_row[0]));
            vpx_memset16(above_row + bs, above_row[bs - 1], bs);
          }
        } else if (x0 <= frame_width) {
          const int r = frame_width - x0;
          memcpy(above_row, above_ref, r * sizeof(above_row[0]));
          vpx_memset16(above_row + r, above_row[r - 1],
                       x0 + 2 * bs - frame_width);
        }
        above_row[-1] = left_available ? above_ref[-1] : (base + 1);
      } else {
        /* faster path if the block does not need extension */
        if (bs == 4 && right_available && left_available) {
          const_above_row = above_ref;
        } else {
          memcpy(above_row, above_ref, bs * sizeof(above_row[0]));
          if (bs == 4 && right_available)
            memcpy(above_row + bs, above_ref + bs, bs * sizeof(above_row[0]));
          else
            vpx_memset16(above_row + bs, above_row[bs - 1], bs);
          above_row[-1] = left_available ? above_ref[-1] : (base + 1);
        }
      }
    } else {
      vpx_memset16(above_row, base - 1, bs * 2);
      above_row[-1] = base - 1;
    }
  }

  // predict
  if (mode == DC_PRED) {
    dc_pred_high[left_available][up_available][tx_size](
        dst, dst_stride, const_above_row, left_col, xd->bd);
  } else {
    pred_high[mode][tx_size](dst, dst_stride, const_above_row, left_col,
                             xd->bd);
  }
}
#endif  // CONFIG_VP9_HIGHBITDEPTH

static void build_intra_predictors(const MACROBLOCKD *xd, const uint8_t *ref,
                                   int ref_stride, uint8_t *dst, int dst_stride,
                                   PREDICTION_MODE mode, TX_SIZE tx_size,
                                   int up_available, int left_available,
                                   int right_available, int x, int y,
                                   int plane) {
  int i;
  DECLARE_ALIGNED(16, uint8_t, left_col[32]);
  DECLARE_ALIGNED(16, uint8_t, above_data[64 + 16]);
  uint8_t *above_row = above_data + 16;
  const uint8_t *const_above_row = above_row;
  const int bs = 4 << tx_size;
  int frame_width, frame_height;
  int x0, y0;
  const struct macroblockd_plane *const pd = &xd->plane[plane];

  // 127 127 127 .. 127 127 127 127 127 127
  // 129  A   B  ..  Y   Z
  // 129  C   D  ..  W   X
  // 129  E   F  ..  U   V
  // 129  G   H  ..  S   T   T   T   T   T
  // ..

  // Get current frame pointer, width and height.
  if (plane == 0) {
    frame_width = xd->cur_buf->y_width;
    frame_height = xd->cur_buf->y_height;
  } else {
    frame_width = xd->cur_buf->uv_width;
    frame_height = xd->cur_buf->uv_height;
  }

  // Get block position in current frame.
  x0 = (-xd->mb_to_left_edge >> (3 + pd->subsampling_x)) + x;
  y0 = (-xd->mb_to_top_edge >> (3 + pd->subsampling_y)) + y;

  // NEED_LEFT
  if (extend_modes[mode] & NEED_LEFT) {
    if (left_available) {
      if (xd->mb_to_bottom_edge < 0) {
        /* slower path if the block needs border extension */
        if (y0 + bs <= frame_height) {
          for (i = 0; i < bs; ++i) left_col[i] = ref[i * ref_stride - 1];
        } else {
          const int extend_bottom = frame_height - y0;
          for (i = 0; i < extend_bottom; ++i)
            left_col[i] = ref[i * ref_stride - 1];
          for (; i < bs; ++i)
            left_col[i] = ref[(extend_bottom - 1) * ref_stride - 1];
        }
      } else {
        /* faster path if the block does not need extension */
        for (i = 0; i < bs; ++i) left_col[i] = ref[i * ref_stride - 1];
      }
    } else {
      memset(left_col, 129, bs);
    }
  }

  // NEED_ABOVE
  if (extend_modes[mode] & NEED_ABOVE) {
    if (up_available) {
      const uint8_t *above_ref = ref - ref_stride;
      if (xd->mb_to_right_edge < 0) {
        /* slower path if the block needs border extension */
        if (x0 + bs <= frame_width) {
          memcpy(above_row, above_ref, bs);
        } else if (x0 <= frame_width) {
          const int r = frame_width - x0;
          memcpy(above_row, above_ref, r);
          memset(above_row + r, above_row[r - 1], x0 + bs - frame_width);
        }
      } else {
        /* faster path if the block does not need extension */
        if (bs == 4 && right_available && left_available) {
          const_above_row = above_ref;
        } else {
          memcpy(above_row, above_ref, bs);
        }
      }
      above_row[-1] = left_available ? above_ref[-1] : 129;
    } else {
      memset(above_row, 127, bs);
      above_row[-1] = 127;
    }
  }

  // NEED_ABOVERIGHT
  if (extend_modes[mode] & NEED_ABOVERIGHT) {
    if (up_available) {
      const uint8_t *above_ref = ref - ref_stride;
      if (xd->mb_to_right_edge < 0) {
        /* slower path if the block needs border extension */
        if (x0 + 2 * bs <= frame_width) {
          if (right_available && bs == 4) {
            memcpy(above_row, above_ref, 2 * bs);
          } else {
            memcpy(above_row, above_ref, bs);
            memset(above_row + bs, above_row[bs - 1], bs);
          }
        } else if (x0 + bs <= frame_width) {
          const int r = frame_width - x0;
          if (right_available && bs == 4) {
            memcpy(above_row, above_ref, r);
            memset(above_row + r, above_row[r - 1], x0 + 2 * bs - frame_width);
          } else {
            memcpy(above_row, above_ref, bs);
            memset(above_row + bs, above_row[bs - 1], bs);
          }
        } else if (x0 <= frame_width) {
          const int r = frame_width - x0;
          memcpy(above_row, above_ref, r);
          memset(above_row + r, above_row[r - 1], x0 + 2 * bs - frame_width);
        }
      } else {
        /* faster path if the block does not need extension */
        if (bs == 4 && right_available && left_available) {
          const_above_row = above_ref;
        } else {
          memcpy(above_row, above_ref, bs);
          if (bs == 4 && right_available)
            memcpy(above_row + bs, above_ref + bs, bs);
          else
            memset(above_row + bs, above_row[bs - 1], bs);
        }
      }
      above_row[-1] = left_available ? above_ref[-1] : 129;
    } else {
      memset(above_row, 127, bs * 2);
      above_row[-1] = 127;
    }
  }

  // predict
  if (mode == DC_PRED) {
    dc_pred[left_available][up_available][tx_size](dst, dst_stride,
                                                   const_above_row, left_col);
  } else {
    pred[mode][tx_size](dst, dst_stride, const_above_row, left_col);
  }
}

void vp9_predict_intra_block(const MACROBLOCKD *xd, int bwl_in, TX_SIZE tx_size,
                             PREDICTION_MODE mode, const uint8_t *ref,
                             int ref_stride, uint8_t *dst, int dst_stride,
                             int aoff, int loff, int plane) {
  const int bw = (1 << bwl_in);
  const int txw = (1 << tx_size);
  const int have_top = loff || (xd->above_mi != NULL);
  const int have_left = aoff || (xd->left_mi != NULL);
  const int have_right = (aoff + txw) < bw;
  const int x = aoff * 4;
  const int y = loff * 4;

#if CONFIG_VP9_HIGHBITDEPTH
  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
    build_intra_predictors_high(xd, ref, ref_stride, dst, dst_stride, mode,
                                tx_size, have_top, have_left, have_right, x, y,
                                plane, xd->bd);
    return;
  }
#endif
  build_intra_predictors(xd, ref, ref_stride, dst, dst_stride, mode, tx_size,
                         have_top, have_left, have_right, x, y, plane);
}

void vp9_init_intra_predictors(void) {
  once(vp9_init_intra_predictors_internal);
}

Coverage Report

Created: 2025-12-12 07:23

Line	Count	Source
1		/*
2		* Copyright (c) 2010 The WebM project authors. All Rights Reserved.
3		*
4		* Use of this source code is governed by a BSD-style license
5		* that can be found in the LICENSE file in the root of the source
6		* tree. An additional intellectual property rights grant can be found
7		* in the file PATENTS. All contributing project authors may
8		* be found in the AUTHORS file in the root of the source tree.
9		*/
10
11		#include "./vpx_config.h"
12		#include "./vpx_dsp_rtcd.h"
13
14		#if CONFIG_VP9_HIGHBITDEPTH
15		#include "vpx_dsp/vpx_dsp_common.h"
16		#endif // CONFIG_VP9_HIGHBITDEPTH
17		#include "vpx_mem/vpx_mem.h"
18		#include "vpx_ports/mem.h"
19		#include "vpx_ports/vpx_once.h"
20
21		#include "vp9/common/vp9_reconintra.h"
22		#include "vp9/common/vp9_onyxc_int.h"
23
24		const TX_TYPE intra_mode_to_tx_type_lookup[INTRA_MODES] = {
25		DCT_DCT, // DC
26		ADST_DCT, // V
27		DCT_ADST, // H
28		DCT_DCT, // D45
29		ADST_ADST, // D135
30		ADST_DCT, // D117
31		DCT_ADST, // D153
32		DCT_ADST, // D207
33		ADST_DCT, // D63
34		ADST_ADST, // TM
35		};
36
37		enum {
38		NEED_LEFT = 1 << 1,
39		NEED_ABOVE = 1 << 2,
40		NEED_ABOVERIGHT = 1 << 3,
41		};
42
43		static const uint8_t extend_modes[INTRA_MODES] = {
44		NEED_ABOVE \| NEED_LEFT, // DC
45		NEED_ABOVE, // V
46		NEED_LEFT, // H
47		NEED_ABOVERIGHT, // D45
48		NEED_LEFT \| NEED_ABOVE, // D135
49		NEED_LEFT \| NEED_ABOVE, // D117
50		NEED_LEFT \| NEED_ABOVE, // D153
51		NEED_LEFT, // D207
52		NEED_ABOVERIGHT, // D63
53		NEED_LEFT \| NEED_ABOVE, // TM
54		};
55
56		typedef void (intra_pred_fn)(uint8_t dst, ptrdiff_t stride,
57		const uint8_t above, const uint8_t left);
58
59		static intra_pred_fn pred[INTRA_MODES][TX_SIZES];
60		static intra_pred_fn dc_pred[2][2][TX_SIZES];
61
62		#if CONFIG_VP9_HIGHBITDEPTH
63		typedef void (intra_high_pred_fn)(uint16_t dst, ptrdiff_t stride,
64		const uint16_t above, const uint16_t left,
65		int bd);
66		static intra_high_pred_fn pred_high[INTRA_MODES][4];
67		static intra_high_pred_fn dc_pred_high[2][2][4];
68		#endif // CONFIG_VP9_HIGHBITDEPTH
69
70	1	static void vp9_init_intra_predictors_internal(void) {
71	1	#define INIT_ALL_SIZES(p, type) \
72	26	p[TX_4X4] = vpx_##type##_predictor_4x4; \
73	26	p[TX_8X8] = vpx_##type##_predictor_8x8; \
74	26	p[TX_16X16] = vpx_##type##_predictor_16x16; \
75	26	p[TX_32X32] = vpx_##type##_predictor_32x32
76
77	1	INIT_ALL_SIZES(pred[V_PRED], v);
78	1	INIT_ALL_SIZES(pred[H_PRED], h);
79	1	INIT_ALL_SIZES(pred[D207_PRED], d207);
80	1	INIT_ALL_SIZES(pred[D45_PRED], d45);
81	1	INIT_ALL_SIZES(pred[D63_PRED], d63);
82	1	INIT_ALL_SIZES(pred[D117_PRED], d117);
83	1	INIT_ALL_SIZES(pred[D135_PRED], d135);
84	1	INIT_ALL_SIZES(pred[D153_PRED], d153);
85	1	INIT_ALL_SIZES(pred[TM_PRED], tm);
86
87	1	INIT_ALL_SIZES(dc_pred[0][0], dc_128);
88	1	INIT_ALL_SIZES(dc_pred[0][1], dc_top);
89	1	INIT_ALL_SIZES(dc_pred[1][0], dc_left);
90	1	INIT_ALL_SIZES(dc_pred[1][1], dc);
91
92	1	#if CONFIG_VP9_HIGHBITDEPTH
93	1	INIT_ALL_SIZES(pred_high[V_PRED], highbd_v);
94	1	INIT_ALL_SIZES(pred_high[H_PRED], highbd_h);
95	1	INIT_ALL_SIZES(pred_high[D207_PRED], highbd_d207);
96	1	INIT_ALL_SIZES(pred_high[D45_PRED], highbd_d45);
97	1	INIT_ALL_SIZES(pred_high[D63_PRED], highbd_d63);
98	1	INIT_ALL_SIZES(pred_high[D117_PRED], highbd_d117);
99	1	INIT_ALL_SIZES(pred_high[D135_PRED], highbd_d135);
100	1	INIT_ALL_SIZES(pred_high[D153_PRED], highbd_d153);
101	1	INIT_ALL_SIZES(pred_high[TM_PRED], highbd_tm);
102
103	1	INIT_ALL_SIZES(dc_pred_high[0][0], highbd_dc_128);
104	1	INIT_ALL_SIZES(dc_pred_high[0][1], highbd_dc_top);
105	1	INIT_ALL_SIZES(dc_pred_high[1][0], highbd_dc_left);
106	1	INIT_ALL_SIZES(dc_pred_high[1][1], highbd_dc);
107	1	#endif // CONFIG_VP9_HIGHBITDEPTH
108
109	1	#undef intra_pred_allsizes
110	1	}
111
112		#if CONFIG_VP9_HIGHBITDEPTH
113		static void build_intra_predictors_high(
114		const MACROBLOCKD xd, const uint8_t ref8, int ref_stride, uint8_t *dst8,
115		int dst_stride, PREDICTION_MODE mode, TX_SIZE tx_size, int up_available,
116	73.8M	int left_available, int right_available, int x, int y, int plane, int bd) {
117	73.8M	int i;
118	73.8M	uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
119	73.8M	uint16_t *ref = CONVERT_TO_SHORTPTR(ref8);
120	73.8M	DECLARE_ALIGNED(16, uint16_t, left_col[32]);
121	73.8M	DECLARE_ALIGNED(16, uint16_t, above_data[64 + 16]);
122	73.8M	uint16_t *above_row = above_data + 16;
123	73.8M	const uint16_t *const_above_row = above_row;
124	73.8M	const int bs = 4 << tx_size;
125	73.8M	int frame_width, frame_height;
126	73.8M	int x0, y0;
127	73.8M	const struct macroblockd_plane *const pd = &xd->plane[plane];
128	73.8M	const int need_left = extend_modes[mode] & NEED_LEFT;
129	73.8M	const int need_above = extend_modes[mode] & NEED_ABOVE;
130	73.8M	const int need_aboveright = extend_modes[mode] & NEED_ABOVERIGHT;
131	73.8M	int base = 128 << (bd - 8);
132		// 127 127 127 .. 127 127 127 127 127 127
133		// 129 A B .. Y Z
134		// 129 C D .. W X
135		// 129 E F .. U V
136		// 129 G H .. S T T T T T
137		// For 10 bit and 12 bit, 127 and 129 are replaced by base -1 and base + 1.
138
139		// Get current frame pointer, width and height.
140	73.8M	if (plane == 0) {
141	43.0M	frame_width = xd->cur_buf->y_width;
142	43.0M	frame_height = xd->cur_buf->y_height;
143	43.0M	} else {
144	30.8M	frame_width = xd->cur_buf->uv_width;
145	30.8M	frame_height = xd->cur_buf->uv_height;
146	30.8M	}
147
148		// Get block position in current frame.
149	73.8M	x0 = (-xd->mb_to_left_edge >> (3 + pd->subsampling_x)) + x;
150	73.8M	y0 = (-xd->mb_to_top_edge >> (3 + pd->subsampling_y)) + y;
151
152		// NEED_LEFT
153	73.8M	if (need_left) {
154	69.3M	if (left_available) {
155	67.4M	if (xd->mb_to_bottom_edge < 0) {
156		/* slower path if the block needs border extension */
157	9.25M	if (y0 + bs <= frame_height) {
158	50.5M	for (i = 0; i < bs; ++i) left_col[i] = ref[i * ref_stride - 1];
159	8.22M	} else {
160	1.02M	const int extend_bottom = frame_height - y0;
161	11.6M	for (i = 0; i < extend_bottom; ++i)
162	10.5M	left_col[i] = ref[i * ref_stride - 1];
163	11.7M	for (; i < bs; ++i)
164	10.6M	left_col[i] = ref[(extend_bottom - 1) * ref_stride - 1];
165	1.02M	}
166	58.1M	} else {
167		/* faster path if the block does not need extension */
168	366M	for (i = 0; i < bs; ++i) left_col[i] = ref[i * ref_stride - 1];
169	58.1M	}
170	67.4M	} else {
171	1.94M	vpx_memset16(left_col, base + 1, bs);
172	1.94M	}
173	69.3M	}
174
175		// NEED_ABOVE
176	73.8M	if (need_above) {
177	65.5M	if (up_available) {
178	62.2M	const uint16_t *above_ref = ref - ref_stride;
179	62.2M	if (xd->mb_to_right_edge < 0) {
180		/* slower path if the block needs border extension */
181	9.10M	if (x0 + bs <= frame_width) {
182	8.89M	memcpy(above_row, above_ref, bs * sizeof(above_row[0]));
183	8.89M	} else if (x0 <= frame_width) {
184	204k	const int r = frame_width - x0;
185	204k	memcpy(above_row, above_ref, r * sizeof(above_row[0]));
186	204k	vpx_memset16(above_row + r, above_row[r - 1], x0 + bs - frame_width);
187	204k	}
188	53.1M	} else {
189		/* faster path if the block does not need extension */
190	53.1M	if (bs == 4 && right_available && left_available) {
191	39.7M	const_above_row = above_ref;
192	39.7M	} else {
193	13.3M	memcpy(above_row, above_ref, bs * sizeof(above_row[0]));
194	13.3M	}
195	53.1M	}
196	62.2M	above_row[-1] = left_available ? above_ref[-1] : (base + 1);
197	62.2M	} else {
198	3.31M	vpx_memset16(above_row, base - 1, bs);
199	3.31M	above_row[-1] = base - 1;
200	3.31M	}
201	65.5M	}
202
203		// NEED_ABOVERIGHT
204	73.8M	if (need_aboveright) {
205	1.85M	if (up_available) {
206	1.81M	const uint16_t *above_ref = ref - ref_stride;
207	1.81M	if (xd->mb_to_right_edge < 0) {
208		/* slower path if the block needs border extension */
209	129k	if (x0 + 2 * bs <= frame_width) {
210	96.9k	if (right_available && bs == 4) {
211	64.3k	memcpy(above_row, above_ref, 2 * bs * sizeof(above_row[0]));
212	64.3k	} else {
213	32.5k	memcpy(above_row, above_ref, bs * sizeof(above_row[0]));
214	32.5k	vpx_memset16(above_row + bs, above_row[bs - 1], bs);
215	32.5k	}
216	96.9k	} else if (x0 + bs <= frame_width) {
217	26.5k	const int r = frame_width - x0;
218	26.5k	if (right_available && bs == 4) {
219	8.78k	memcpy(above_row, above_ref, r * sizeof(above_row[0]));
220	8.78k	vpx_memset16(above_row + r, above_row[r - 1],
221	8.78k	x0 + 2 * bs - frame_width);
222	17.7k	} else {
223	17.7k	memcpy(above_row, above_ref, bs * sizeof(above_row[0]));
224	17.7k	vpx_memset16(above_row + bs, above_row[bs - 1], bs);
225	17.7k	}
226	26.5k	} else if (x0 <= frame_width) {
227	6.13k	const int r = frame_width - x0;
228	6.13k	memcpy(above_row, above_ref, r * sizeof(above_row[0]));
229	6.13k	vpx_memset16(above_row + r, above_row[r - 1],
230	6.13k	x0 + 2 * bs - frame_width);
231	6.13k	}
232	129k	above_row[-1] = left_available ? above_ref[-1] : (base + 1);
233	1.68M	} else {
234		/* faster path if the block does not need extension */
235	1.68M	if (bs == 4 && right_available && left_available) {
236	1.08M	const_above_row = above_ref;
237	1.08M	} else {
238	600k	memcpy(above_row, above_ref, bs * sizeof(above_row[0]));
239	600k	if (bs == 4 && right_available)
240	10.7k	memcpy(above_row + bs, above_ref + bs, bs * sizeof(above_row[0]));
241	589k	else
242	589k	vpx_memset16(above_row + bs, above_row[bs - 1], bs);
243	600k	above_row[-1] = left_available ? above_ref[-1] : (base + 1);
244	600k	}
245	1.68M	}
246	1.81M	} else {
247	40.6k	vpx_memset16(above_row, base - 1, bs * 2);
248	40.6k	above_row[-1] = base - 1;
249	40.6k	}
250	1.85M	}
251
252		// predict
253	73.8M	if (mode == DC_PRED) {
254	56.6M	dc_pred_high[left_available][up_available][tx_size](
255	56.6M	dst, dst_stride, const_above_row, left_col, xd->bd);
256	56.6M	} else {
257	17.1M	pred_high[mode][tx_size](dst, dst_stride, const_above_row, left_col,
258	17.1M	xd->bd);
259	17.1M	}
260	73.8M	}
261		#endif // CONFIG_VP9_HIGHBITDEPTH
262
263		static void build_intra_predictors(const MACROBLOCKD xd, const uint8_t ref,
264		int ref_stride, uint8_t *dst, int dst_stride,
265		PREDICTION_MODE mode, TX_SIZE tx_size,
266		int up_available, int left_available,
267		int right_available, int x, int y,
268	798M	int plane) {
269	798M	int i;
270	798M	DECLARE_ALIGNED(16, uint8_t, left_col[32]);
271	798M	DECLARE_ALIGNED(16, uint8_t, above_data[64 + 16]);
272	798M	uint8_t *above_row = above_data + 16;
273	798M	const uint8_t *const_above_row = above_row;
274	798M	const int bs = 4 << tx_size;
275	798M	int frame_width, frame_height;
276	798M	int x0, y0;
277	798M	const struct macroblockd_plane *const pd = &xd->plane[plane];
278
279		// 127 127 127 .. 127 127 127 127 127 127
280		// 129 A B .. Y Z
281		// 129 C D .. W X
282		// 129 E F .. U V
283		// 129 G H .. S T T T T T
284		// ..
285
286		// Get current frame pointer, width and height.
287	798M	if (plane == 0) {
288	538M	frame_width = xd->cur_buf->y_width;
289	538M	frame_height = xd->cur_buf->y_height;
290	538M	} else {
291	260M	frame_width = xd->cur_buf->uv_width;
292	260M	frame_height = xd->cur_buf->uv_height;
293	260M	}
294
295		// Get block position in current frame.
296	798M	x0 = (-xd->mb_to_left_edge >> (3 + pd->subsampling_x)) + x;
297	798M	y0 = (-xd->mb_to_top_edge >> (3 + pd->subsampling_y)) + y;
298
299		// NEED_LEFT
300	798M	if (extend_modes[mode] & NEED_LEFT) {
301	787M	if (left_available) {
302	783M	if (xd->mb_to_bottom_edge < 0) {
303		/* slower path if the block needs border extension */
304	16.4M	if (y0 + bs <= frame_height) {
305	86.4M	for (i = 0; i < bs; ++i) left_col[i] = ref[i * ref_stride - 1];
306	15.9M	} else {
307	542k	const int extend_bottom = frame_height - y0;
308	7.97M	for (i = 0; i < extend_bottom; ++i)
309	7.42M	left_col[i] = ref[i * ref_stride - 1];
310	6.60M	for (; i < bs; ++i)
311	6.06M	left_col[i] = ref[(extend_bottom - 1) * ref_stride - 1];
312	542k	}
313	767M	} else {
314		/* faster path if the block does not need extension */
315	3.98G	for (i = 0; i < bs; ++i) left_col[i] = ref[i * ref_stride - 1];
316	767M	}
317	783M	} else {
318	3.65M	memset(left_col, 129, bs);
319	3.65M	}
320	787M	}
321
322		// NEED_ABOVE
323	798M	if (extend_modes[mode] & NEED_ABOVE) {
324	771M	if (up_available) {
325	766M	const uint8_t *above_ref = ref - ref_stride;
326	766M	if (xd->mb_to_right_edge < 0) {
327		/* slower path if the block needs border extension */
328	12.9M	if (x0 + bs <= frame_width) {
329	12.6M	memcpy(above_row, above_ref, bs);
330	12.6M	} else if (x0 <= frame_width) {
331	251k	const int r = frame_width - x0;
332	251k	memcpy(above_row, above_ref, r);
333	251k	memset(above_row + r, above_row[r - 1], x0 + bs - frame_width);
334	251k	}
335	753M	} else {
336		/* faster path if the block does not need extension */
337	753M	if (bs == 4 && right_available && left_available) {
338	670M	const_above_row = above_ref;
339	670M	} else {
340	83.0M	memcpy(above_row, above_ref, bs);
341	83.0M	}
342	753M	}
343	766M	above_row[-1] = left_available ? above_ref[-1] : 129;
344	766M	} else {
345	4.66M	memset(above_row, 127, bs);
346	4.66M	above_row[-1] = 127;
347	4.66M	}
348	771M	}
349
350		// NEED_ABOVERIGHT
351	798M	if (extend_modes[mode] & NEED_ABOVERIGHT) {
352	5.42M	if (up_available) {
353	5.37M	const uint8_t *above_ref = ref - ref_stride;
354	5.37M	if (xd->mb_to_right_edge < 0) {
355		/* slower path if the block needs border extension */
356	46.7k	if (x0 + 2 * bs <= frame_width) {
357	36.5k	if (right_available && bs == 4) {
358	34.9k	memcpy(above_row, above_ref, 2 * bs);
359	34.9k	} else {
360	1.57k	memcpy(above_row, above_ref, bs);
361	1.57k	memset(above_row + bs, above_row[bs - 1], bs);
362	1.57k	}
363	36.5k	} else if (x0 + bs <= frame_width) {
364	8.44k	const int r = frame_width - x0;
365	8.44k	if (right_available && bs == 4) {
366	7.36k	memcpy(above_row, above_ref, r);
367	7.36k	memset(above_row + r, above_row[r - 1], x0 + 2 * bs - frame_width);
368	7.36k	} else {
369	1.08k	memcpy(above_row, above_ref, bs);
370	1.08k	memset(above_row + bs, above_row[bs - 1], bs);
371	1.08k	}
372	8.44k	} else if (x0 <= frame_width) {
373	1.76k	const int r = frame_width - x0;
374	1.76k	memcpy(above_row, above_ref, r);
375	1.76k	memset(above_row + r, above_row[r - 1], x0 + 2 * bs - frame_width);
376	1.76k	}
377	5.32M	} else {
378		/* faster path if the block does not need extension */
379	5.32M	if (bs == 4 && right_available && left_available) {
380	3.98M	const_above_row = above_ref;
381	3.98M	} else {
382	1.33M	memcpy(above_row, above_ref, bs);
383	1.33M	if (bs == 4 && right_available)
384	13.2k	memcpy(above_row + bs, above_ref + bs, bs);
385	1.32M	else
386	1.32M	memset(above_row + bs, above_row[bs - 1], bs);
387	1.33M	}
388	5.32M	}
389	5.37M	above_row[-1] = left_available ? above_ref[-1] : 129;
390	5.37M	} else {
391	50.2k	memset(above_row, 127, bs * 2);
392	50.2k	above_row[-1] = 127;
393	50.2k	}
394	5.42M	}
395
396		// predict
397	798M	if (mode == DC_PRED) {
398	740M	dc_pred[left_available][up_available][tx_size](dst, dst_stride,
399	740M	const_above_row, left_col);
400	740M	} else {
401	57.8M	pred[mode][tx_size](dst, dst_stride, const_above_row, left_col);
402	57.8M	}
403	798M	}
404
405		void vp9_predict_intra_block(const MACROBLOCKD *xd, int bwl_in, TX_SIZE tx_size,
406		PREDICTION_MODE mode, const uint8_t *ref,
407		int ref_stride, uint8_t *dst, int dst_stride,
408	877M	int aoff, int loff, int plane) {
409	877M	const int bw = (1 << bwl_in);
410	877M	const int txw = (1 << tx_size);
411	877M	const int have_top = loff \|\| (xd->above_mi != NULL);
412	877M	const int have_left = aoff \|\| (xd->left_mi != NULL);
413	877M	const int have_right = (aoff + txw) < bw;
414	877M	const int x = aoff * 4;
415	877M	const int y = loff * 4;
416
417	877M	#if CONFIG_VP9_HIGHBITDEPTH
418	877M	if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
419	74.1M	build_intra_predictors_high(xd, ref, ref_stride, dst, dst_stride, mode,
420	74.1M	tx_size, have_top, have_left, have_right, x, y,
421	74.1M	plane, xd->bd);
422	74.1M	return;
423	74.1M	}
424	803M	#endif
425	803M	build_intra_predictors(xd, ref, ref_stride, dst, dst_stride, mode, tx_size,
426	803M	have_top, have_left, have_right, x, y, plane);
427	803M	}
428
429	1	void vp9_init_intra_predictors(void) {
430	1	once(vp9_init_intra_predictors_internal);
431	1	}