/src/aom/av1/common/reconintra.c
Line | Count | Source (jump to first uncovered line) |
1 | | /* |
2 | | * Copyright (c) 2016, Alliance for Open Media. All rights reserved. |
3 | | * |
4 | | * This source code is subject to the terms of the BSD 2 Clause License and |
5 | | * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License |
6 | | * was not distributed with this source code in the LICENSE file, you can |
7 | | * obtain it at www.aomedia.org/license/software. If the Alliance for Open |
8 | | * Media Patent License 1.0 was not distributed with this source code in the |
9 | | * PATENTS file, you can obtain it at www.aomedia.org/license/patent. |
10 | | */ |
11 | | |
12 | | #include <assert.h> |
13 | | #include <math.h> |
14 | | |
15 | | #include "config/aom_config.h" |
16 | | #include "config/aom_dsp_rtcd.h" |
17 | | #include "config/av1_rtcd.h" |
18 | | |
19 | | #include "aom_dsp/aom_dsp_common.h" |
20 | | #include "aom_mem/aom_mem.h" |
21 | | #include "aom_ports/aom_once.h" |
22 | | #include "aom_ports/mem.h" |
23 | | #include "av1/common/av1_common_int.h" |
24 | | #include "av1/common/cfl.h" |
25 | | #include "av1/common/reconintra.h" |
26 | | |
27 | | enum { |
28 | | NEED_LEFT = 1 << 1, |
29 | | NEED_ABOVE = 1 << 2, |
30 | | NEED_ABOVERIGHT = 1 << 3, |
31 | | NEED_ABOVELEFT = 1 << 4, |
32 | | NEED_BOTTOMLEFT = 1 << 5, |
33 | | }; |
34 | | |
35 | | #define INTRA_EDGE_FILT 3 |
36 | 0 | #define INTRA_EDGE_TAPS 5 |
37 | | #define MAX_UPSAMPLE_SZ 16 |
38 | 226M | #define NUM_INTRA_NEIGHBOUR_PIXELS (MAX_TX_SIZE * 2 + 32) |
39 | | |
40 | | static const uint8_t extend_modes[INTRA_MODES] = { |
41 | | NEED_ABOVE | NEED_LEFT, // DC |
42 | | NEED_ABOVE, // V |
43 | | NEED_LEFT, // H |
44 | | NEED_ABOVE | NEED_ABOVERIGHT, // D45 |
45 | | NEED_LEFT | NEED_ABOVE | NEED_ABOVELEFT, // D135 |
46 | | NEED_LEFT | NEED_ABOVE | NEED_ABOVELEFT, // D113 |
47 | | NEED_LEFT | NEED_ABOVE | NEED_ABOVELEFT, // D157 |
48 | | NEED_LEFT | NEED_BOTTOMLEFT, // D203 |
49 | | NEED_ABOVE | NEED_ABOVERIGHT, // D67 |
50 | | NEED_LEFT | NEED_ABOVE, // SMOOTH |
51 | | NEED_LEFT | NEED_ABOVE, // SMOOTH_V |
52 | | NEED_LEFT | NEED_ABOVE, // SMOOTH_H |
53 | | NEED_LEFT | NEED_ABOVE | NEED_ABOVELEFT, // PAETH |
54 | | }; |
55 | | |
56 | | // Tables to store if the top-right reference pixels are available. The flags |
57 | | // are represented with bits, packed into 8-bit integers. E.g., for the 32x32 |
58 | | // blocks in a 128x128 superblock, the index of the "o" block is 10 (in raster |
59 | | // order), so its flag is stored at the 3rd bit of the 2nd entry in the table, |
60 | | // i.e. (table[10 / 8] >> (10 % 8)) & 1. |
61 | | // . . . . |
62 | | // . . . . |
63 | | // . . o . |
64 | | // . . . . |
65 | | static uint8_t has_tr_4x4[128] = { |
66 | | 255, 255, 255, 255, 85, 85, 85, 85, 119, 119, 119, 119, 85, 85, 85, 85, |
67 | | 127, 127, 127, 127, 85, 85, 85, 85, 119, 119, 119, 119, 85, 85, 85, 85, |
68 | | 255, 127, 255, 127, 85, 85, 85, 85, 119, 119, 119, 119, 85, 85, 85, 85, |
69 | | 127, 127, 127, 127, 85, 85, 85, 85, 119, 119, 119, 119, 85, 85, 85, 85, |
70 | | 255, 255, 255, 127, 85, 85, 85, 85, 119, 119, 119, 119, 85, 85, 85, 85, |
71 | | 127, 127, 127, 127, 85, 85, 85, 85, 119, 119, 119, 119, 85, 85, 85, 85, |
72 | | 255, 127, 255, 127, 85, 85, 85, 85, 119, 119, 119, 119, 85, 85, 85, 85, |
73 | | 127, 127, 127, 127, 85, 85, 85, 85, 119, 119, 119, 119, 85, 85, 85, 85, |
74 | | }; |
75 | | static uint8_t has_tr_4x8[64] = { |
76 | | 255, 255, 255, 255, 119, 119, 119, 119, 127, 127, 127, 127, 119, |
77 | | 119, 119, 119, 255, 127, 255, 127, 119, 119, 119, 119, 127, 127, |
78 | | 127, 127, 119, 119, 119, 119, 255, 255, 255, 127, 119, 119, 119, |
79 | | 119, 127, 127, 127, 127, 119, 119, 119, 119, 255, 127, 255, 127, |
80 | | 119, 119, 119, 119, 127, 127, 127, 127, 119, 119, 119, 119, |
81 | | }; |
82 | | static uint8_t has_tr_8x4[64] = { |
83 | | 255, 255, 0, 0, 85, 85, 0, 0, 119, 119, 0, 0, 85, 85, 0, 0, |
84 | | 127, 127, 0, 0, 85, 85, 0, 0, 119, 119, 0, 0, 85, 85, 0, 0, |
85 | | 255, 127, 0, 0, 85, 85, 0, 0, 119, 119, 0, 0, 85, 85, 0, 0, |
86 | | 127, 127, 0, 0, 85, 85, 0, 0, 119, 119, 0, 0, 85, 85, 0, 0, |
87 | | }; |
88 | | static uint8_t has_tr_8x8[32] = { |
89 | | 255, 255, 85, 85, 119, 119, 85, 85, 127, 127, 85, 85, 119, 119, 85, 85, |
90 | | 255, 127, 85, 85, 119, 119, 85, 85, 127, 127, 85, 85, 119, 119, 85, 85, |
91 | | }; |
92 | | static uint8_t has_tr_8x16[16] = { |
93 | | 255, 255, 119, 119, 127, 127, 119, 119, |
94 | | 255, 127, 119, 119, 127, 127, 119, 119, |
95 | | }; |
96 | | static uint8_t has_tr_16x8[16] = { |
97 | | 255, 0, 85, 0, 119, 0, 85, 0, 127, 0, 85, 0, 119, 0, 85, 0, |
98 | | }; |
99 | | static uint8_t has_tr_16x16[8] = { |
100 | | 255, 85, 119, 85, 127, 85, 119, 85, |
101 | | }; |
102 | | static uint8_t has_tr_16x32[4] = { 255, 119, 127, 119 }; |
103 | | static uint8_t has_tr_32x16[4] = { 15, 5, 7, 5 }; |
104 | | static uint8_t has_tr_32x32[2] = { 95, 87 }; |
105 | | static uint8_t has_tr_32x64[1] = { 127 }; |
106 | | static uint8_t has_tr_64x32[1] = { 19 }; |
107 | | static uint8_t has_tr_64x64[1] = { 7 }; |
108 | | static uint8_t has_tr_64x128[1] = { 3 }; |
109 | | static uint8_t has_tr_128x64[1] = { 1 }; |
110 | | static uint8_t has_tr_128x128[1] = { 1 }; |
111 | | static uint8_t has_tr_4x16[32] = { |
112 | | 255, 255, 255, 255, 127, 127, 127, 127, 255, 127, 255, |
113 | | 127, 127, 127, 127, 127, 255, 255, 255, 127, 127, 127, |
114 | | 127, 127, 255, 127, 255, 127, 127, 127, 127, 127, |
115 | | }; |
116 | | static uint8_t has_tr_16x4[32] = { |
117 | | 255, 0, 0, 0, 85, 0, 0, 0, 119, 0, 0, 0, 85, 0, 0, 0, |
118 | | 127, 0, 0, 0, 85, 0, 0, 0, 119, 0, 0, 0, 85, 0, 0, 0, |
119 | | }; |
120 | | static uint8_t has_tr_8x32[8] = { |
121 | | 255, 255, 127, 127, 255, 127, 127, 127, |
122 | | }; |
123 | | static uint8_t has_tr_32x8[8] = { |
124 | | 15, 0, 5, 0, 7, 0, 5, 0, |
125 | | }; |
126 | | static uint8_t has_tr_16x64[2] = { 255, 127 }; |
127 | | static uint8_t has_tr_64x16[2] = { 3, 1 }; |
128 | | |
129 | | static const uint8_t *const has_tr_tables[BLOCK_SIZES_ALL] = { |
130 | | // 4X4 |
131 | | has_tr_4x4, |
132 | | // 4X8, 8X4, 8X8 |
133 | | has_tr_4x8, has_tr_8x4, has_tr_8x8, |
134 | | // 8X16, 16X8, 16X16 |
135 | | has_tr_8x16, has_tr_16x8, has_tr_16x16, |
136 | | // 16X32, 32X16, 32X32 |
137 | | has_tr_16x32, has_tr_32x16, has_tr_32x32, |
138 | | // 32X64, 64X32, 64X64 |
139 | | has_tr_32x64, has_tr_64x32, has_tr_64x64, |
140 | | // 64x128, 128x64, 128x128 |
141 | | has_tr_64x128, has_tr_128x64, has_tr_128x128, |
142 | | // 4x16, 16x4, 8x32 |
143 | | has_tr_4x16, has_tr_16x4, has_tr_8x32, |
144 | | // 32x8, 16x64, 64x16 |
145 | | has_tr_32x8, has_tr_16x64, has_tr_64x16 |
146 | | }; |
147 | | |
148 | | static uint8_t has_tr_vert_8x8[32] = { |
149 | | 255, 255, 0, 0, 119, 119, 0, 0, 127, 127, 0, 0, 119, 119, 0, 0, |
150 | | 255, 127, 0, 0, 119, 119, 0, 0, 127, 127, 0, 0, 119, 119, 0, 0, |
151 | | }; |
152 | | static uint8_t has_tr_vert_16x16[8] = { |
153 | | 255, 0, 119, 0, 127, 0, 119, 0, |
154 | | }; |
155 | | static uint8_t has_tr_vert_32x32[2] = { 15, 7 }; |
156 | | static uint8_t has_tr_vert_64x64[1] = { 3 }; |
157 | | |
158 | | // The _vert_* tables are like the ordinary tables above, but describe the |
159 | | // order we visit square blocks when doing a PARTITION_VERT_A or |
160 | | // PARTITION_VERT_B. This is the same order as normal except for on the last |
161 | | // split where we go vertically (TL, BL, TR, BR). We treat the rectangular block |
162 | | // as a pair of squares, which means that these tables work correctly for both |
163 | | // mixed vertical partition types. |
164 | | // |
165 | | // There are tables for each of the square sizes. Vertical rectangles (like |
166 | | // BLOCK_16X32) use their respective "non-vert" table |
167 | | static const uint8_t *const has_tr_vert_tables[BLOCK_SIZES] = { |
168 | | // 4X4 |
169 | | NULL, |
170 | | // 4X8, 8X4, 8X8 |
171 | | has_tr_4x8, NULL, has_tr_vert_8x8, |
172 | | // 8X16, 16X8, 16X16 |
173 | | has_tr_8x16, NULL, has_tr_vert_16x16, |
174 | | // 16X32, 32X16, 32X32 |
175 | | has_tr_16x32, NULL, has_tr_vert_32x32, |
176 | | // 32X64, 64X32, 64X64 |
177 | | has_tr_32x64, NULL, has_tr_vert_64x64, |
178 | | // 64x128, 128x64, 128x128 |
179 | | has_tr_64x128, NULL, has_tr_128x128 |
180 | | }; |
181 | | |
182 | | static const uint8_t *get_has_tr_table(PARTITION_TYPE partition, |
183 | 681k | BLOCK_SIZE bsize) { |
184 | 681k | const uint8_t *ret = NULL; |
185 | | // If this is a mixed vertical partition, look up bsize in orders_vert. |
186 | 681k | if (partition == PARTITION_VERT_A || partition == PARTITION_VERT_B) { |
187 | 64.3k | assert(bsize < BLOCK_SIZES); |
188 | 64.3k | ret = has_tr_vert_tables[bsize]; |
189 | 617k | } else { |
190 | 617k | ret = has_tr_tables[bsize]; |
191 | 617k | } |
192 | 681k | assert(ret); |
193 | 681k | return ret; |
194 | 681k | } |
195 | | |
196 | | static int has_top_right(BLOCK_SIZE sb_size, BLOCK_SIZE bsize, int mi_row, |
197 | | int mi_col, int top_available, int right_available, |
198 | | PARTITION_TYPE partition, TX_SIZE txsz, int row_off, |
199 | 1.29M | int col_off, int ss_x, int ss_y) { |
200 | 1.29M | if (!top_available || !right_available) return 0; |
201 | | |
202 | 1.21M | const int bw_unit = mi_size_wide[bsize]; |
203 | 1.21M | const int plane_bw_unit = AOMMAX(bw_unit >> ss_x, 1); |
204 | 1.21M | const int top_right_count_unit = tx_size_wide_unit[txsz]; |
205 | | |
206 | 1.21M | if (row_off > 0) { // Just need to check if enough pixels on the right. |
207 | 221k | if (block_size_wide[bsize] > block_size_wide[BLOCK_64X64]) { |
208 | | // Special case: For 128x128 blocks, the transform unit whose |
209 | | // top-right corner is at the center of the block does in fact have |
210 | | // pixels available at its top-right corner. |
211 | 113k | if (row_off == mi_size_high[BLOCK_64X64] >> ss_y && |
212 | 113k | col_off + top_right_count_unit == mi_size_wide[BLOCK_64X64] >> ss_x) { |
213 | 16.4k | return 1; |
214 | 16.4k | } |
215 | 97.2k | const int plane_bw_unit_64 = mi_size_wide[BLOCK_64X64] >> ss_x; |
216 | 97.2k | const int col_off_64 = col_off % plane_bw_unit_64; |
217 | 97.2k | return col_off_64 + top_right_count_unit < plane_bw_unit_64; |
218 | 113k | } |
219 | 107k | return col_off + top_right_count_unit < plane_bw_unit; |
220 | 990k | } else { |
221 | | // All top-right pixels are in the block above, which is already available. |
222 | 990k | if (col_off + top_right_count_unit < plane_bw_unit) return 1; |
223 | | |
224 | 938k | const int bw_in_mi_log2 = mi_size_wide_log2[bsize]; |
225 | 938k | const int bh_in_mi_log2 = mi_size_high_log2[bsize]; |
226 | 938k | const int sb_mi_size = mi_size_high[sb_size]; |
227 | 938k | const int blk_row_in_sb = (mi_row & (sb_mi_size - 1)) >> bh_in_mi_log2; |
228 | 938k | const int blk_col_in_sb = (mi_col & (sb_mi_size - 1)) >> bw_in_mi_log2; |
229 | | |
230 | | // Top row of superblock: so top-right pixels are in the top and/or |
231 | | // top-right superblocks, both of which are already available. |
232 | 938k | if (blk_row_in_sb == 0) return 1; |
233 | | |
234 | | // Rightmost column of superblock (and not the top row): so top-right pixels |
235 | | // fall in the right superblock, which is not available yet. |
236 | 823k | if (((blk_col_in_sb + 1) << bw_in_mi_log2) >= sb_mi_size) { |
237 | 142k | return 0; |
238 | 142k | } |
239 | | |
240 | | // General case (neither top row nor rightmost column): check if the |
241 | | // top-right block is coded before the current block. |
242 | 681k | const int this_blk_index = |
243 | 681k | ((blk_row_in_sb + 0) << (MAX_MIB_SIZE_LOG2 - bw_in_mi_log2)) + |
244 | 681k | blk_col_in_sb + 0; |
245 | 681k | const int idx1 = this_blk_index / 8; |
246 | 681k | const int idx2 = this_blk_index % 8; |
247 | 681k | const uint8_t *has_tr_table = get_has_tr_table(partition, bsize); |
248 | 681k | return (has_tr_table[idx1] >> idx2) & 1; |
249 | 823k | } |
250 | 1.21M | } |
251 | | |
252 | | // Similar to the has_tr_* tables, but store if the bottom-left reference |
253 | | // pixels are available. |
254 | | static uint8_t has_bl_4x4[128] = { |
255 | | 84, 85, 85, 85, 16, 17, 17, 17, 84, 85, 85, 85, 0, 1, 1, 1, 84, 85, 85, |
256 | | 85, 16, 17, 17, 17, 84, 85, 85, 85, 0, 0, 1, 0, 84, 85, 85, 85, 16, 17, |
257 | | 17, 17, 84, 85, 85, 85, 0, 1, 1, 1, 84, 85, 85, 85, 16, 17, 17, 17, 84, |
258 | | 85, 85, 85, 0, 0, 0, 0, 84, 85, 85, 85, 16, 17, 17, 17, 84, 85, 85, 85, |
259 | | 0, 1, 1, 1, 84, 85, 85, 85, 16, 17, 17, 17, 84, 85, 85, 85, 0, 0, 1, |
260 | | 0, 84, 85, 85, 85, 16, 17, 17, 17, 84, 85, 85, 85, 0, 1, 1, 1, 84, 85, |
261 | | 85, 85, 16, 17, 17, 17, 84, 85, 85, 85, 0, 0, 0, 0, |
262 | | }; |
263 | | static uint8_t has_bl_4x8[64] = { |
264 | | 16, 17, 17, 17, 0, 1, 1, 1, 16, 17, 17, 17, 0, 0, 1, 0, |
265 | | 16, 17, 17, 17, 0, 1, 1, 1, 16, 17, 17, 17, 0, 0, 0, 0, |
266 | | 16, 17, 17, 17, 0, 1, 1, 1, 16, 17, 17, 17, 0, 0, 1, 0, |
267 | | 16, 17, 17, 17, 0, 1, 1, 1, 16, 17, 17, 17, 0, 0, 0, 0, |
268 | | }; |
269 | | static uint8_t has_bl_8x4[64] = { |
270 | | 254, 255, 84, 85, 254, 255, 16, 17, 254, 255, 84, 85, 254, 255, 0, 1, |
271 | | 254, 255, 84, 85, 254, 255, 16, 17, 254, 255, 84, 85, 254, 255, 0, 0, |
272 | | 254, 255, 84, 85, 254, 255, 16, 17, 254, 255, 84, 85, 254, 255, 0, 1, |
273 | | 254, 255, 84, 85, 254, 255, 16, 17, 254, 255, 84, 85, 254, 255, 0, 0, |
274 | | }; |
275 | | static uint8_t has_bl_8x8[32] = { |
276 | | 84, 85, 16, 17, 84, 85, 0, 1, 84, 85, 16, 17, 84, 85, 0, 0, |
277 | | 84, 85, 16, 17, 84, 85, 0, 1, 84, 85, 16, 17, 84, 85, 0, 0, |
278 | | }; |
279 | | static uint8_t has_bl_8x16[16] = { |
280 | | 16, 17, 0, 1, 16, 17, 0, 0, 16, 17, 0, 1, 16, 17, 0, 0, |
281 | | }; |
282 | | static uint8_t has_bl_16x8[16] = { |
283 | | 254, 84, 254, 16, 254, 84, 254, 0, 254, 84, 254, 16, 254, 84, 254, 0, |
284 | | }; |
285 | | static uint8_t has_bl_16x16[8] = { |
286 | | 84, 16, 84, 0, 84, 16, 84, 0, |
287 | | }; |
288 | | static uint8_t has_bl_16x32[4] = { 16, 0, 16, 0 }; |
289 | | static uint8_t has_bl_32x16[4] = { 78, 14, 78, 14 }; |
290 | | static uint8_t has_bl_32x32[2] = { 4, 4 }; |
291 | | static uint8_t has_bl_32x64[1] = { 0 }; |
292 | | static uint8_t has_bl_64x32[1] = { 34 }; |
293 | | static uint8_t has_bl_64x64[1] = { 0 }; |
294 | | static uint8_t has_bl_64x128[1] = { 0 }; |
295 | | static uint8_t has_bl_128x64[1] = { 0 }; |
296 | | static uint8_t has_bl_128x128[1] = { 0 }; |
297 | | static uint8_t has_bl_4x16[32] = { |
298 | | 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, |
299 | | 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, |
300 | | }; |
301 | | static uint8_t has_bl_16x4[32] = { |
302 | | 254, 254, 254, 84, 254, 254, 254, 16, 254, 254, 254, 84, 254, 254, 254, 0, |
303 | | 254, 254, 254, 84, 254, 254, 254, 16, 254, 254, 254, 84, 254, 254, 254, 0, |
304 | | }; |
305 | | static uint8_t has_bl_8x32[8] = { |
306 | | 0, 1, 0, 0, 0, 1, 0, 0, |
307 | | }; |
308 | | static uint8_t has_bl_32x8[8] = { |
309 | | 238, 78, 238, 14, 238, 78, 238, 14, |
310 | | }; |
311 | | static uint8_t has_bl_16x64[2] = { 0, 0 }; |
312 | | static uint8_t has_bl_64x16[2] = { 42, 42 }; |
313 | | |
314 | | static const uint8_t *const has_bl_tables[BLOCK_SIZES_ALL] = { |
315 | | // 4X4 |
316 | | has_bl_4x4, |
317 | | // 4X8, 8X4, 8X8 |
318 | | has_bl_4x8, has_bl_8x4, has_bl_8x8, |
319 | | // 8X16, 16X8, 16X16 |
320 | | has_bl_8x16, has_bl_16x8, has_bl_16x16, |
321 | | // 16X32, 32X16, 32X32 |
322 | | has_bl_16x32, has_bl_32x16, has_bl_32x32, |
323 | | // 32X64, 64X32, 64X64 |
324 | | has_bl_32x64, has_bl_64x32, has_bl_64x64, |
325 | | // 64x128, 128x64, 128x128 |
326 | | has_bl_64x128, has_bl_128x64, has_bl_128x128, |
327 | | // 4x16, 16x4, 8x32 |
328 | | has_bl_4x16, has_bl_16x4, has_bl_8x32, |
329 | | // 32x8, 16x64, 64x16 |
330 | | has_bl_32x8, has_bl_16x64, has_bl_64x16 |
331 | | }; |
332 | | |
333 | | static uint8_t has_bl_vert_8x8[32] = { |
334 | | 254, 255, 16, 17, 254, 255, 0, 1, 254, 255, 16, 17, 254, 255, 0, 0, |
335 | | 254, 255, 16, 17, 254, 255, 0, 1, 254, 255, 16, 17, 254, 255, 0, 0, |
336 | | }; |
337 | | static uint8_t has_bl_vert_16x16[8] = { |
338 | | 254, 16, 254, 0, 254, 16, 254, 0, |
339 | | }; |
340 | | static uint8_t has_bl_vert_32x32[2] = { 14, 14 }; |
341 | | static uint8_t has_bl_vert_64x64[1] = { 2 }; |
342 | | |
343 | | // The _vert_* tables are like the ordinary tables above, but describe the |
344 | | // order we visit square blocks when doing a PARTITION_VERT_A or |
345 | | // PARTITION_VERT_B. This is the same order as normal except for on the last |
346 | | // split where we go vertically (TL, BL, TR, BR). We treat the rectangular block |
347 | | // as a pair of squares, which means that these tables work correctly for both |
348 | | // mixed vertical partition types. |
349 | | // |
350 | | // There are tables for each of the square sizes. Vertical rectangles (like |
351 | | // BLOCK_16X32) use their respective "non-vert" table |
352 | | static const uint8_t *const has_bl_vert_tables[BLOCK_SIZES] = { |
353 | | // 4X4 |
354 | | NULL, |
355 | | // 4X8, 8X4, 8X8 |
356 | | has_bl_4x8, NULL, has_bl_vert_8x8, |
357 | | // 8X16, 16X8, 16X16 |
358 | | has_bl_8x16, NULL, has_bl_vert_16x16, |
359 | | // 16X32, 32X16, 32X32 |
360 | | has_bl_16x32, NULL, has_bl_vert_32x32, |
361 | | // 32X64, 64X32, 64X64 |
362 | | has_bl_32x64, NULL, has_bl_vert_64x64, |
363 | | // 64x128, 128x64, 128x128 |
364 | | has_bl_64x128, NULL, has_bl_128x128 |
365 | | }; |
366 | | |
367 | | static const uint8_t *get_has_bl_table(PARTITION_TYPE partition, |
368 | 987k | BLOCK_SIZE bsize) { |
369 | 987k | const uint8_t *ret = NULL; |
370 | | // If this is a mixed vertical partition, look up bsize in orders_vert. |
371 | 987k | if (partition == PARTITION_VERT_A || partition == PARTITION_VERT_B) { |
372 | 78.1k | assert(bsize < BLOCK_SIZES); |
373 | 78.1k | ret = has_bl_vert_tables[bsize]; |
374 | 909k | } else { |
375 | 909k | ret = has_bl_tables[bsize]; |
376 | 909k | } |
377 | 987k | assert(ret); |
378 | 987k | return ret; |
379 | 987k | } |
380 | | |
381 | | static int has_bottom_left(BLOCK_SIZE sb_size, BLOCK_SIZE bsize, int mi_row, |
382 | | int mi_col, int bottom_available, int left_available, |
383 | | PARTITION_TYPE partition, TX_SIZE txsz, int row_off, |
384 | 2.08M | int col_off, int ss_x, int ss_y) { |
385 | 2.08M | if (!bottom_available || !left_available) return 0; |
386 | | |
387 | | // Special case for 128x* blocks, when col_off is half the block width. |
388 | | // This is needed because 128x* superblocks are divided into 64x* blocks in |
389 | | // raster order |
390 | 1.86M | if (block_size_wide[bsize] > block_size_wide[BLOCK_64X64] && col_off > 0) { |
391 | 170k | const int plane_bw_unit_64 = mi_size_wide[BLOCK_64X64] >> ss_x; |
392 | 170k | const int col_off_64 = col_off % plane_bw_unit_64; |
393 | 170k | if (col_off_64 == 0) { |
394 | | // We are at the left edge of top-right or bottom-right 64x* block. |
395 | 70.3k | const int plane_bh_unit_64 = mi_size_high[BLOCK_64X64] >> ss_y; |
396 | 70.3k | const int row_off_64 = row_off % plane_bh_unit_64; |
397 | 70.3k | const int plane_bh_unit = |
398 | 70.3k | AOMMIN(mi_size_high[bsize] >> ss_y, plane_bh_unit_64); |
399 | | // Check if all bottom-left pixels are in the left 64x* block (which is |
400 | | // already coded). |
401 | 70.3k | return row_off_64 + tx_size_high_unit[txsz] < plane_bh_unit; |
402 | 70.3k | } |
403 | 170k | } |
404 | | |
405 | 1.79M | if (col_off > 0) { |
406 | | // Bottom-left pixels are in the bottom-left block, which is not available. |
407 | 339k | return 0; |
408 | 1.45M | } else { |
409 | 1.45M | const int bh_unit = mi_size_high[bsize]; |
410 | 1.45M | const int plane_bh_unit = AOMMAX(bh_unit >> ss_y, 1); |
411 | 1.45M | const int bottom_left_count_unit = tx_size_high_unit[txsz]; |
412 | | |
413 | | // All bottom-left pixels are in the left block, which is already available. |
414 | 1.45M | if (row_off + bottom_left_count_unit < plane_bh_unit) return 1; |
415 | | |
416 | 1.37M | const int bw_in_mi_log2 = mi_size_wide_log2[bsize]; |
417 | 1.37M | const int bh_in_mi_log2 = mi_size_high_log2[bsize]; |
418 | 1.37M | const int sb_mi_size = mi_size_high[sb_size]; |
419 | 1.37M | const int blk_row_in_sb = (mi_row & (sb_mi_size - 1)) >> bh_in_mi_log2; |
420 | 1.37M | const int blk_col_in_sb = (mi_col & (sb_mi_size - 1)) >> bw_in_mi_log2; |
421 | | |
422 | | // Leftmost column of superblock: so bottom-left pixels maybe in the left |
423 | | // and/or bottom-left superblocks. But only the left superblock is |
424 | | // available, so check if all required pixels fall in that superblock. |
425 | 1.37M | if (blk_col_in_sb == 0) { |
426 | 227k | const int blk_start_row_off = |
427 | 227k | blk_row_in_sb << (bh_in_mi_log2 + MI_SIZE_LOG2 - MI_SIZE_LOG2) >> |
428 | 227k | ss_y; |
429 | 227k | const int row_off_in_sb = blk_start_row_off + row_off; |
430 | 227k | const int sb_height_unit = sb_mi_size >> ss_y; |
431 | 227k | return row_off_in_sb + bottom_left_count_unit < sb_height_unit; |
432 | 227k | } |
433 | | |
434 | | // Bottom row of superblock (and not the leftmost column): so bottom-left |
435 | | // pixels fall in the bottom superblock, which is not available yet. |
436 | 1.15M | if (((blk_row_in_sb + 1) << bh_in_mi_log2) >= sb_mi_size) return 0; |
437 | | |
438 | | // General case (neither leftmost column nor bottom row): check if the |
439 | | // bottom-left block is coded before the current block. |
440 | 987k | const int this_blk_index = |
441 | 987k | ((blk_row_in_sb + 0) << (MAX_MIB_SIZE_LOG2 - bw_in_mi_log2)) + |
442 | 987k | blk_col_in_sb + 0; |
443 | 987k | const int idx1 = this_blk_index / 8; |
444 | 987k | const int idx2 = this_blk_index % 8; |
445 | 987k | const uint8_t *has_bl_table = get_has_bl_table(partition, bsize); |
446 | 987k | return (has_bl_table[idx1] >> idx2) & 1; |
447 | 1.15M | } |
448 | 1.79M | } |
449 | | |
450 | | typedef void (*intra_pred_fn)(uint8_t *dst, ptrdiff_t stride, |
451 | | const uint8_t *above, const uint8_t *left); |
452 | | |
453 | | static intra_pred_fn pred[INTRA_MODES][TX_SIZES_ALL]; |
454 | | static intra_pred_fn dc_pred[2][2][TX_SIZES_ALL]; |
455 | | |
456 | | #if CONFIG_AV1_HIGHBITDEPTH |
457 | | typedef void (*intra_high_pred_fn)(uint16_t *dst, ptrdiff_t stride, |
458 | | const uint16_t *above, const uint16_t *left, |
459 | | int bd); |
460 | | static intra_high_pred_fn pred_high[INTRA_MODES][TX_SIZES_ALL]; |
461 | | static intra_high_pred_fn dc_pred_high[2][2][TX_SIZES_ALL]; |
462 | | #endif |
463 | | |
464 | 1 | static void init_intra_predictors_internal(void) { |
465 | 1 | assert(NELEMENTS(mode_to_angle_map) == INTRA_MODES); |
466 | | |
467 | | #if CONFIG_REALTIME_ONLY && !CONFIG_AV1_DECODER |
468 | | #define INIT_RECTANGULAR(p, type) \ |
469 | | p[TX_4X8] = aom_##type##_predictor_4x8; \ |
470 | | p[TX_8X4] = aom_##type##_predictor_8x4; \ |
471 | | p[TX_8X16] = aom_##type##_predictor_8x16; \ |
472 | | p[TX_16X8] = aom_##type##_predictor_16x8; \ |
473 | | p[TX_16X32] = aom_##type##_predictor_16x32; \ |
474 | | p[TX_32X16] = aom_##type##_predictor_32x16; \ |
475 | | p[TX_32X64] = aom_##type##_predictor_32x64; \ |
476 | | p[TX_64X32] = aom_##type##_predictor_64x32; |
477 | | #else |
478 | 1 | #define INIT_RECTANGULAR(p, type) \ |
479 | 20 | p[TX_4X8] = aom_##type##_predictor_4x8; \ |
480 | 20 | p[TX_8X4] = aom_##type##_predictor_8x4; \ |
481 | 20 | p[TX_8X16] = aom_##type##_predictor_8x16; \ |
482 | 20 | p[TX_16X8] = aom_##type##_predictor_16x8; \ |
483 | 20 | p[TX_16X32] = aom_##type##_predictor_16x32; \ |
484 | 20 | p[TX_32X16] = aom_##type##_predictor_32x16; \ |
485 | 20 | p[TX_32X64] = aom_##type##_predictor_32x64; \ |
486 | 20 | p[TX_64X32] = aom_##type##_predictor_64x32; \ |
487 | 20 | p[TX_4X16] = aom_##type##_predictor_4x16; \ |
488 | 20 | p[TX_16X4] = aom_##type##_predictor_16x4; \ |
489 | 20 | p[TX_8X32] = aom_##type##_predictor_8x32; \ |
490 | 20 | p[TX_32X8] = aom_##type##_predictor_32x8; \ |
491 | 20 | p[TX_16X64] = aom_##type##_predictor_16x64; \ |
492 | 20 | p[TX_64X16] = aom_##type##_predictor_64x16; |
493 | 1 | #endif // CONFIG_REALTIME_ONLY && !CONFIG_AV1_DECODER |
494 | | |
495 | 1 | #define INIT_NO_4X4(p, type) \ |
496 | 20 | p[TX_8X8] = aom_##type##_predictor_8x8; \ |
497 | 20 | p[TX_16X16] = aom_##type##_predictor_16x16; \ |
498 | 20 | p[TX_32X32] = aom_##type##_predictor_32x32; \ |
499 | 20 | p[TX_64X64] = aom_##type##_predictor_64x64; \ |
500 | 20 | INIT_RECTANGULAR(p, type) |
501 | | |
502 | 1 | #define INIT_ALL_SIZES(p, type) \ |
503 | 20 | p[TX_4X4] = aom_##type##_predictor_4x4; \ |
504 | 20 | INIT_NO_4X4(p, type) |
505 | | |
506 | 1 | INIT_ALL_SIZES(pred[V_PRED], v) |
507 | 1 | INIT_ALL_SIZES(pred[H_PRED], h) |
508 | 1 | INIT_ALL_SIZES(pred[PAETH_PRED], paeth) |
509 | 1 | INIT_ALL_SIZES(pred[SMOOTH_PRED], smooth) |
510 | 1 | INIT_ALL_SIZES(pred[SMOOTH_V_PRED], smooth_v) |
511 | 1 | INIT_ALL_SIZES(pred[SMOOTH_H_PRED], smooth_h) |
512 | 1 | INIT_ALL_SIZES(dc_pred[0][0], dc_128) |
513 | 1 | INIT_ALL_SIZES(dc_pred[0][1], dc_top) |
514 | 1 | INIT_ALL_SIZES(dc_pred[1][0], dc_left) |
515 | 1 | INIT_ALL_SIZES(dc_pred[1][1], dc) |
516 | 1 | #if CONFIG_AV1_HIGHBITDEPTH |
517 | 1 | INIT_ALL_SIZES(pred_high[V_PRED], highbd_v) |
518 | 1 | INIT_ALL_SIZES(pred_high[H_PRED], highbd_h) |
519 | 1 | INIT_ALL_SIZES(pred_high[PAETH_PRED], highbd_paeth) |
520 | 1 | INIT_ALL_SIZES(pred_high[SMOOTH_PRED], highbd_smooth) |
521 | 1 | INIT_ALL_SIZES(pred_high[SMOOTH_V_PRED], highbd_smooth_v) |
522 | 1 | INIT_ALL_SIZES(pred_high[SMOOTH_H_PRED], highbd_smooth_h) |
523 | 1 | INIT_ALL_SIZES(dc_pred_high[0][0], highbd_dc_128) |
524 | 1 | INIT_ALL_SIZES(dc_pred_high[0][1], highbd_dc_top) |
525 | 1 | INIT_ALL_SIZES(dc_pred_high[1][0], highbd_dc_left) |
526 | 1 | INIT_ALL_SIZES(dc_pred_high[1][1], highbd_dc) |
527 | 1 | #endif |
528 | 1 | #undef intra_pred_allsizes |
529 | 1 | } |
530 | | |
531 | | // Directional prediction, zone 1: 0 < angle < 90 |
532 | | void av1_dr_prediction_z1_c(uint8_t *dst, ptrdiff_t stride, int bw, int bh, |
533 | | const uint8_t *above, const uint8_t *left, |
534 | 0 | int upsample_above, int dx, int dy) { |
535 | 0 | int r, c, x, base, shift, val; |
536 | |
|
537 | 0 | (void)left; |
538 | 0 | (void)dy; |
539 | 0 | assert(dy == 1); |
540 | 0 | assert(dx > 0); |
541 | | |
542 | 0 | const int max_base_x = ((bw + bh) - 1) << upsample_above; |
543 | 0 | const int frac_bits = 6 - upsample_above; |
544 | 0 | const int base_inc = 1 << upsample_above; |
545 | 0 | x = dx; |
546 | 0 | for (r = 0; r < bh; ++r, dst += stride, x += dx) { |
547 | 0 | base = x >> frac_bits; |
548 | 0 | shift = ((x << upsample_above) & 0x3F) >> 1; |
549 | |
|
550 | 0 | if (base >= max_base_x) { |
551 | 0 | for (int i = r; i < bh; ++i) { |
552 | 0 | memset(dst, above[max_base_x], bw * sizeof(dst[0])); |
553 | 0 | dst += stride; |
554 | 0 | } |
555 | 0 | return; |
556 | 0 | } |
557 | | |
558 | 0 | for (c = 0; c < bw; ++c, base += base_inc) { |
559 | 0 | if (base < max_base_x) { |
560 | 0 | val = above[base] * (32 - shift) + above[base + 1] * shift; |
561 | 0 | dst[c] = ROUND_POWER_OF_TWO(val, 5); |
562 | 0 | } else { |
563 | 0 | dst[c] = above[max_base_x]; |
564 | 0 | } |
565 | 0 | } |
566 | 0 | } |
567 | 0 | } |
568 | | |
569 | | // Directional prediction, zone 2: 90 < angle < 180 |
570 | | void av1_dr_prediction_z2_c(uint8_t *dst, ptrdiff_t stride, int bw, int bh, |
571 | | const uint8_t *above, const uint8_t *left, |
572 | | int upsample_above, int upsample_left, int dx, |
573 | 0 | int dy) { |
574 | 0 | assert(dx > 0); |
575 | 0 | assert(dy > 0); |
576 | | |
577 | 0 | const int min_base_x = -(1 << upsample_above); |
578 | 0 | const int min_base_y = -(1 << upsample_left); |
579 | 0 | (void)min_base_y; |
580 | 0 | const int frac_bits_x = 6 - upsample_above; |
581 | 0 | const int frac_bits_y = 6 - upsample_left; |
582 | |
|
583 | 0 | for (int r = 0; r < bh; ++r) { |
584 | 0 | for (int c = 0; c < bw; ++c) { |
585 | 0 | int val; |
586 | 0 | int y = r + 1; |
587 | 0 | int x = (c << 6) - y * dx; |
588 | 0 | const int base_x = x >> frac_bits_x; |
589 | 0 | if (base_x >= min_base_x) { |
590 | 0 | const int shift = ((x * (1 << upsample_above)) & 0x3F) >> 1; |
591 | 0 | val = above[base_x] * (32 - shift) + above[base_x + 1] * shift; |
592 | 0 | val = ROUND_POWER_OF_TWO(val, 5); |
593 | 0 | } else { |
594 | 0 | x = c + 1; |
595 | 0 | y = (r << 6) - x * dy; |
596 | 0 | const int base_y = y >> frac_bits_y; |
597 | 0 | assert(base_y >= min_base_y); |
598 | 0 | const int shift = ((y * (1 << upsample_left)) & 0x3F) >> 1; |
599 | 0 | val = left[base_y] * (32 - shift) + left[base_y + 1] * shift; |
600 | 0 | val = ROUND_POWER_OF_TWO(val, 5); |
601 | 0 | } |
602 | 0 | dst[c] = val; |
603 | 0 | } |
604 | 0 | dst += stride; |
605 | 0 | } |
606 | 0 | } |
607 | | |
608 | | // Directional prediction, zone 3: 180 < angle < 270 |
609 | | void av1_dr_prediction_z3_c(uint8_t *dst, ptrdiff_t stride, int bw, int bh, |
610 | | const uint8_t *above, const uint8_t *left, |
611 | 0 | int upsample_left, int dx, int dy) { |
612 | 0 | int r, c, y, base, shift, val; |
613 | |
|
614 | 0 | (void)above; |
615 | 0 | (void)dx; |
616 | |
|
617 | 0 | assert(dx == 1); |
618 | 0 | assert(dy > 0); |
619 | | |
620 | 0 | const int max_base_y = (bw + bh - 1) << upsample_left; |
621 | 0 | const int frac_bits = 6 - upsample_left; |
622 | 0 | const int base_inc = 1 << upsample_left; |
623 | 0 | y = dy; |
624 | 0 | for (c = 0; c < bw; ++c, y += dy) { |
625 | 0 | base = y >> frac_bits; |
626 | 0 | shift = ((y << upsample_left) & 0x3F) >> 1; |
627 | |
|
628 | 0 | for (r = 0; r < bh; ++r, base += base_inc) { |
629 | 0 | if (base < max_base_y) { |
630 | 0 | val = left[base] * (32 - shift) + left[base + 1] * shift; |
631 | 0 | dst[r * stride + c] = ROUND_POWER_OF_TWO(val, 5); |
632 | 0 | } else { |
633 | 0 | for (; r < bh; ++r) dst[r * stride + c] = left[max_base_y]; |
634 | 0 | break; |
635 | 0 | } |
636 | 0 | } |
637 | 0 | } |
638 | 0 | } |
639 | | |
640 | | static void dr_predictor(uint8_t *dst, ptrdiff_t stride, TX_SIZE tx_size, |
641 | | const uint8_t *above, const uint8_t *left, |
642 | 4.48M | int upsample_above, int upsample_left, int angle) { |
643 | 4.48M | const int dx = av1_get_dx(angle); |
644 | 4.48M | const int dy = av1_get_dy(angle); |
645 | 4.48M | const int bw = tx_size_wide[tx_size]; |
646 | 4.48M | const int bh = tx_size_high[tx_size]; |
647 | 4.48M | assert(angle > 0 && angle < 270); |
648 | | |
649 | 4.48M | if (angle > 0 && angle < 90) { |
650 | 469k | av1_dr_prediction_z1(dst, stride, bw, bh, above, left, upsample_above, dx, |
651 | 469k | dy); |
652 | 4.01M | } else if (angle > 90 && angle < 180) { |
653 | 1.28M | av1_dr_prediction_z2(dst, stride, bw, bh, above, left, upsample_above, |
654 | 1.28M | upsample_left, dx, dy); |
655 | 2.72M | } else if (angle > 180 && angle < 270) { |
656 | 741k | av1_dr_prediction_z3(dst, stride, bw, bh, above, left, upsample_left, dx, |
657 | 741k | dy); |
658 | 1.98M | } else if (angle == 90) { |
659 | 455k | pred[V_PRED][tx_size](dst, stride, above, left); |
660 | 1.53M | } else if (angle == 180) { |
661 | 1.53M | pred[H_PRED][tx_size](dst, stride, above, left); |
662 | 1.53M | } |
663 | 4.48M | } |
664 | | |
665 | | #if CONFIG_AV1_HIGHBITDEPTH |
666 | | // Directional prediction, zone 1: 0 < angle < 90 |
667 | | void av1_highbd_dr_prediction_z1_c(uint16_t *dst, ptrdiff_t stride, int bw, |
668 | | int bh, const uint16_t *above, |
669 | | const uint16_t *left, int upsample_above, |
670 | 0 | int dx, int dy, int bd) { |
671 | 0 | int r, c, x, base, shift, val; |
672 | |
|
673 | 0 | (void)left; |
674 | 0 | (void)dy; |
675 | 0 | (void)bd; |
676 | 0 | assert(dy == 1); |
677 | 0 | assert(dx > 0); |
678 | | |
679 | 0 | const int max_base_x = ((bw + bh) - 1) << upsample_above; |
680 | 0 | const int frac_bits = 6 - upsample_above; |
681 | 0 | const int base_inc = 1 << upsample_above; |
682 | 0 | x = dx; |
683 | 0 | for (r = 0; r < bh; ++r, dst += stride, x += dx) { |
684 | 0 | base = x >> frac_bits; |
685 | 0 | shift = ((x << upsample_above) & 0x3F) >> 1; |
686 | |
|
687 | 0 | if (base >= max_base_x) { |
688 | 0 | for (int i = r; i < bh; ++i) { |
689 | 0 | aom_memset16(dst, above[max_base_x], bw); |
690 | 0 | dst += stride; |
691 | 0 | } |
692 | 0 | return; |
693 | 0 | } |
694 | | |
695 | 0 | for (c = 0; c < bw; ++c, base += base_inc) { |
696 | 0 | if (base < max_base_x) { |
697 | 0 | val = above[base] * (32 - shift) + above[base + 1] * shift; |
698 | 0 | dst[c] = ROUND_POWER_OF_TWO(val, 5); |
699 | 0 | } else { |
700 | 0 | dst[c] = above[max_base_x]; |
701 | 0 | } |
702 | 0 | } |
703 | 0 | } |
704 | 0 | } |
705 | | |
706 | | // Directional prediction, zone 2: 90 < angle < 180 |
707 | | void av1_highbd_dr_prediction_z2_c(uint16_t *dst, ptrdiff_t stride, int bw, |
708 | | int bh, const uint16_t *above, |
709 | | const uint16_t *left, int upsample_above, |
710 | 0 | int upsample_left, int dx, int dy, int bd) { |
711 | 0 | (void)bd; |
712 | 0 | assert(dx > 0); |
713 | 0 | assert(dy > 0); |
714 | | |
715 | 0 | const int min_base_x = -(1 << upsample_above); |
716 | 0 | const int min_base_y = -(1 << upsample_left); |
717 | 0 | (void)min_base_y; |
718 | 0 | const int frac_bits_x = 6 - upsample_above; |
719 | 0 | const int frac_bits_y = 6 - upsample_left; |
720 | |
|
721 | 0 | for (int r = 0; r < bh; ++r) { |
722 | 0 | for (int c = 0; c < bw; ++c) { |
723 | 0 | int val; |
724 | 0 | int y = r + 1; |
725 | 0 | int x = (c << 6) - y * dx; |
726 | 0 | const int base_x = x >> frac_bits_x; |
727 | 0 | if (base_x >= min_base_x) { |
728 | 0 | const int shift = ((x * (1 << upsample_above)) & 0x3F) >> 1; |
729 | 0 | val = above[base_x] * (32 - shift) + above[base_x + 1] * shift; |
730 | 0 | val = ROUND_POWER_OF_TWO(val, 5); |
731 | 0 | } else { |
732 | 0 | x = c + 1; |
733 | 0 | y = (r << 6) - x * dy; |
734 | 0 | const int base_y = y >> frac_bits_y; |
735 | 0 | assert(base_y >= min_base_y); |
736 | 0 | const int shift = ((y * (1 << upsample_left)) & 0x3F) >> 1; |
737 | 0 | val = left[base_y] * (32 - shift) + left[base_y + 1] * shift; |
738 | 0 | val = ROUND_POWER_OF_TWO(val, 5); |
739 | 0 | } |
740 | 0 | dst[c] = val; |
741 | 0 | } |
742 | 0 | dst += stride; |
743 | 0 | } |
744 | 0 | } |
745 | | |
746 | | // Directional prediction, zone 3: 180 < angle < 270 |
747 | | void av1_highbd_dr_prediction_z3_c(uint16_t *dst, ptrdiff_t stride, int bw, |
748 | | int bh, const uint16_t *above, |
749 | | const uint16_t *left, int upsample_left, |
750 | 0 | int dx, int dy, int bd) { |
751 | 0 | int r, c, y, base, shift, val; |
752 | |
|
753 | 0 | (void)above; |
754 | 0 | (void)dx; |
755 | 0 | (void)bd; |
756 | 0 | assert(dx == 1); |
757 | 0 | assert(dy > 0); |
758 | | |
759 | 0 | const int max_base_y = (bw + bh - 1) << upsample_left; |
760 | 0 | const int frac_bits = 6 - upsample_left; |
761 | 0 | const int base_inc = 1 << upsample_left; |
762 | 0 | y = dy; |
763 | 0 | for (c = 0; c < bw; ++c, y += dy) { |
764 | 0 | base = y >> frac_bits; |
765 | 0 | shift = ((y << upsample_left) & 0x3F) >> 1; |
766 | |
|
767 | 0 | for (r = 0; r < bh; ++r, base += base_inc) { |
768 | 0 | if (base < max_base_y) { |
769 | 0 | val = left[base] * (32 - shift) + left[base + 1] * shift; |
770 | 0 | dst[r * stride + c] = ROUND_POWER_OF_TWO(val, 5); |
771 | 0 | } else { |
772 | 0 | for (; r < bh; ++r) dst[r * stride + c] = left[max_base_y]; |
773 | 0 | break; |
774 | 0 | } |
775 | 0 | } |
776 | 0 | } |
777 | 0 | } |
778 | | |
779 | | static void highbd_dr_predictor(uint16_t *dst, ptrdiff_t stride, |
780 | | TX_SIZE tx_size, const uint16_t *above, |
781 | | const uint16_t *left, int upsample_above, |
782 | 6.14M | int upsample_left, int angle, int bd) { |
783 | 6.14M | const int dx = av1_get_dx(angle); |
784 | 6.14M | const int dy = av1_get_dy(angle); |
785 | 6.14M | const int bw = tx_size_wide[tx_size]; |
786 | 6.14M | const int bh = tx_size_high[tx_size]; |
787 | 6.14M | assert(angle > 0 && angle < 270); |
788 | | |
789 | 6.14M | if (angle > 0 && angle < 90) { |
790 | 765k | av1_highbd_dr_prediction_z1(dst, stride, bw, bh, above, left, |
791 | 765k | upsample_above, dx, dy, bd); |
792 | 5.38M | } else if (angle > 90 && angle < 180) { |
793 | 1.63M | av1_highbd_dr_prediction_z2(dst, stride, bw, bh, above, left, |
794 | 1.63M | upsample_above, upsample_left, dx, dy, bd); |
795 | 3.74M | } else if (angle > 180 && angle < 270) { |
796 | 1.14M | av1_highbd_dr_prediction_z3(dst, stride, bw, bh, above, left, upsample_left, |
797 | 1.14M | dx, dy, bd); |
798 | 2.59M | } else if (angle == 90) { |
799 | 680k | pred_high[V_PRED][tx_size](dst, stride, above, left, bd); |
800 | 1.91M | } else if (angle == 180) { |
801 | 1.91M | pred_high[H_PRED][tx_size](dst, stride, above, left, bd); |
802 | 1.91M | } |
803 | 6.14M | } |
804 | | #endif // CONFIG_AV1_HIGHBITDEPTH |
805 | | |
806 | | DECLARE_ALIGNED(16, const int8_t, |
807 | | av1_filter_intra_taps[FILTER_INTRA_MODES][8][8]) = { |
808 | | { |
809 | | { -6, 10, 0, 0, 0, 12, 0, 0 }, |
810 | | { -5, 2, 10, 0, 0, 9, 0, 0 }, |
811 | | { -3, 1, 1, 10, 0, 7, 0, 0 }, |
812 | | { -3, 1, 1, 2, 10, 5, 0, 0 }, |
813 | | { -4, 6, 0, 0, 0, 2, 12, 0 }, |
814 | | { -3, 2, 6, 0, 0, 2, 9, 0 }, |
815 | | { -3, 2, 2, 6, 0, 2, 7, 0 }, |
816 | | { -3, 1, 2, 2, 6, 3, 5, 0 }, |
817 | | }, |
818 | | { |
819 | | { -10, 16, 0, 0, 0, 10, 0, 0 }, |
820 | | { -6, 0, 16, 0, 0, 6, 0, 0 }, |
821 | | { -4, 0, 0, 16, 0, 4, 0, 0 }, |
822 | | { -2, 0, 0, 0, 16, 2, 0, 0 }, |
823 | | { -10, 16, 0, 0, 0, 0, 10, 0 }, |
824 | | { -6, 0, 16, 0, 0, 0, 6, 0 }, |
825 | | { -4, 0, 0, 16, 0, 0, 4, 0 }, |
826 | | { -2, 0, 0, 0, 16, 0, 2, 0 }, |
827 | | }, |
828 | | { |
829 | | { -8, 8, 0, 0, 0, 16, 0, 0 }, |
830 | | { -8, 0, 8, 0, 0, 16, 0, 0 }, |
831 | | { -8, 0, 0, 8, 0, 16, 0, 0 }, |
832 | | { -8, 0, 0, 0, 8, 16, 0, 0 }, |
833 | | { -4, 4, 0, 0, 0, 0, 16, 0 }, |
834 | | { -4, 0, 4, 0, 0, 0, 16, 0 }, |
835 | | { -4, 0, 0, 4, 0, 0, 16, 0 }, |
836 | | { -4, 0, 0, 0, 4, 0, 16, 0 }, |
837 | | }, |
838 | | { |
839 | | { -2, 8, 0, 0, 0, 10, 0, 0 }, |
840 | | { -1, 3, 8, 0, 0, 6, 0, 0 }, |
841 | | { -1, 2, 3, 8, 0, 4, 0, 0 }, |
842 | | { 0, 1, 2, 3, 8, 2, 0, 0 }, |
843 | | { -1, 4, 0, 0, 0, 3, 10, 0 }, |
844 | | { -1, 3, 4, 0, 0, 4, 6, 0 }, |
845 | | { -1, 2, 3, 4, 0, 4, 4, 0 }, |
846 | | { -1, 2, 2, 3, 4, 3, 3, 0 }, |
847 | | }, |
848 | | { |
849 | | { -12, 14, 0, 0, 0, 14, 0, 0 }, |
850 | | { -10, 0, 14, 0, 0, 12, 0, 0 }, |
851 | | { -9, 0, 0, 14, 0, 11, 0, 0 }, |
852 | | { -8, 0, 0, 0, 14, 10, 0, 0 }, |
853 | | { -10, 12, 0, 0, 0, 0, 14, 0 }, |
854 | | { -9, 1, 12, 0, 0, 0, 12, 0 }, |
855 | | { -8, 0, 0, 12, 0, 1, 11, 0 }, |
856 | | { -7, 0, 0, 1, 12, 1, 9, 0 }, |
857 | | }, |
858 | | }; |
859 | | |
860 | | void av1_filter_intra_predictor_c(uint8_t *dst, ptrdiff_t stride, |
861 | | TX_SIZE tx_size, const uint8_t *above, |
862 | 0 | const uint8_t *left, int mode) { |
863 | 0 | int r, c; |
864 | 0 | uint8_t buffer[33][33]; |
865 | 0 | const int bw = tx_size_wide[tx_size]; |
866 | 0 | const int bh = tx_size_high[tx_size]; |
867 | |
|
868 | 0 | assert(bw <= 32 && bh <= 32); |
869 | | |
870 | 0 | for (r = 0; r < bh; ++r) buffer[r + 1][0] = left[r]; |
871 | 0 | memcpy(buffer[0], &above[-1], (bw + 1) * sizeof(uint8_t)); |
872 | |
|
873 | 0 | for (r = 1; r < bh + 1; r += 2) |
874 | 0 | for (c = 1; c < bw + 1; c += 4) { |
875 | 0 | const uint8_t p0 = buffer[r - 1][c - 1]; |
876 | 0 | const uint8_t p1 = buffer[r - 1][c]; |
877 | 0 | const uint8_t p2 = buffer[r - 1][c + 1]; |
878 | 0 | const uint8_t p3 = buffer[r - 1][c + 2]; |
879 | 0 | const uint8_t p4 = buffer[r - 1][c + 3]; |
880 | 0 | const uint8_t p5 = buffer[r][c - 1]; |
881 | 0 | const uint8_t p6 = buffer[r + 1][c - 1]; |
882 | 0 | for (int k = 0; k < 8; ++k) { |
883 | 0 | int r_offset = k >> 2; |
884 | 0 | int c_offset = k & 0x03; |
885 | 0 | int pr = av1_filter_intra_taps[mode][k][0] * p0 + |
886 | 0 | av1_filter_intra_taps[mode][k][1] * p1 + |
887 | 0 | av1_filter_intra_taps[mode][k][2] * p2 + |
888 | 0 | av1_filter_intra_taps[mode][k][3] * p3 + |
889 | 0 | av1_filter_intra_taps[mode][k][4] * p4 + |
890 | 0 | av1_filter_intra_taps[mode][k][5] * p5 + |
891 | 0 | av1_filter_intra_taps[mode][k][6] * p6; |
892 | | // Section 7.11.2.3 specifies the right-hand side of the assignment as |
893 | | // Clip1( Round2Signed( pr, INTRA_FILTER_SCALE_BITS ) ). |
894 | | // Since Clip1() clips a negative value to 0, it is safe to replace |
895 | | // Round2Signed() with Round2(). |
896 | 0 | buffer[r + r_offset][c + c_offset] = |
897 | 0 | clip_pixel(ROUND_POWER_OF_TWO(pr, FILTER_INTRA_SCALE_BITS)); |
898 | 0 | } |
899 | 0 | } |
900 | |
|
901 | 0 | for (r = 0; r < bh; ++r) { |
902 | 0 | memcpy(dst, &buffer[r + 1][1], bw * sizeof(uint8_t)); |
903 | 0 | dst += stride; |
904 | 0 | } |
905 | 0 | } |
906 | | |
907 | | #if CONFIG_AV1_HIGHBITDEPTH |
908 | | static void highbd_filter_intra_predictor(uint16_t *dst, ptrdiff_t stride, |
909 | | TX_SIZE tx_size, |
910 | | const uint16_t *above, |
911 | | const uint16_t *left, int mode, |
912 | 744k | int bd) { |
913 | 744k | int r, c; |
914 | 744k | uint16_t buffer[33][33]; |
915 | 744k | const int bw = tx_size_wide[tx_size]; |
916 | 744k | const int bh = tx_size_high[tx_size]; |
917 | | |
918 | 744k | assert(bw <= 32 && bh <= 32); |
919 | | |
920 | 7.84M | for (r = 0; r < bh; ++r) buffer[r + 1][0] = left[r]; |
921 | 744k | memcpy(buffer[0], &above[-1], (bw + 1) * sizeof(buffer[0][0])); |
922 | | |
923 | 4.28M | for (r = 1; r < bh + 1; r += 2) |
924 | 15.6M | for (c = 1; c < bw + 1; c += 4) { |
925 | 12.1M | const uint16_t p0 = buffer[r - 1][c - 1]; |
926 | 12.1M | const uint16_t p1 = buffer[r - 1][c]; |
927 | 12.1M | const uint16_t p2 = buffer[r - 1][c + 1]; |
928 | 12.1M | const uint16_t p3 = buffer[r - 1][c + 2]; |
929 | 12.1M | const uint16_t p4 = buffer[r - 1][c + 3]; |
930 | 12.1M | const uint16_t p5 = buffer[r][c - 1]; |
931 | 12.1M | const uint16_t p6 = buffer[r + 1][c - 1]; |
932 | 109M | for (int k = 0; k < 8; ++k) { |
933 | 96.9M | int r_offset = k >> 2; |
934 | 96.9M | int c_offset = k & 0x03; |
935 | 96.9M | int pr = av1_filter_intra_taps[mode][k][0] * p0 + |
936 | 96.9M | av1_filter_intra_taps[mode][k][1] * p1 + |
937 | 96.9M | av1_filter_intra_taps[mode][k][2] * p2 + |
938 | 96.9M | av1_filter_intra_taps[mode][k][3] * p3 + |
939 | 96.9M | av1_filter_intra_taps[mode][k][4] * p4 + |
940 | 96.9M | av1_filter_intra_taps[mode][k][5] * p5 + |
941 | 96.9M | av1_filter_intra_taps[mode][k][6] * p6; |
942 | | // Section 7.11.2.3 specifies the right-hand side of the assignment as |
943 | | // Clip1( Round2Signed( pr, INTRA_FILTER_SCALE_BITS ) ). |
944 | | // Since Clip1() clips a negative value to 0, it is safe to replace |
945 | | // Round2Signed() with Round2(). |
946 | 96.9M | buffer[r + r_offset][c + c_offset] = clip_pixel_highbd( |
947 | 96.9M | ROUND_POWER_OF_TWO(pr, FILTER_INTRA_SCALE_BITS), bd); |
948 | 96.9M | } |
949 | 12.1M | } |
950 | | |
951 | 7.83M | for (r = 0; r < bh; ++r) { |
952 | 7.09M | memcpy(dst, &buffer[r + 1][1], bw * sizeof(dst[0])); |
953 | 7.09M | dst += stride; |
954 | 7.09M | } |
955 | 744k | } |
956 | | #endif // CONFIG_AV1_HIGHBITDEPTH |
957 | | |
958 | 20.9M | static int is_smooth(const MB_MODE_INFO *mbmi, int plane) { |
959 | 20.9M | if (plane == 0) { |
960 | 10.2M | const PREDICTION_MODE mode = mbmi->mode; |
961 | 10.2M | return (mode == SMOOTH_PRED || mode == SMOOTH_V_PRED || |
962 | 10.2M | mode == SMOOTH_H_PRED); |
963 | 10.6M | } else { |
964 | | // uv_mode is not set for inter blocks, so need to explicitly |
965 | | // detect that case. |
966 | 10.6M | if (is_inter_block(mbmi)) return 0; |
967 | | |
968 | 9.50M | const UV_PREDICTION_MODE uv_mode = mbmi->uv_mode; |
969 | 9.50M | return (uv_mode == UV_SMOOTH_PRED || uv_mode == UV_SMOOTH_V_PRED || |
970 | 9.50M | uv_mode == UV_SMOOTH_H_PRED); |
971 | 10.6M | } |
972 | 20.9M | } |
973 | | |
974 | 12.3M | static int get_intra_edge_filter_type(const MACROBLOCKD *xd, int plane) { |
975 | 12.3M | const MB_MODE_INFO *above; |
976 | 12.3M | const MB_MODE_INFO *left; |
977 | | |
978 | 12.3M | if (plane == 0) { |
979 | 6.26M | above = xd->above_mbmi; |
980 | 6.26M | left = xd->left_mbmi; |
981 | 6.26M | } else { |
982 | 6.11M | above = xd->chroma_above_mbmi; |
983 | 6.11M | left = xd->chroma_left_mbmi; |
984 | 6.11M | } |
985 | | |
986 | 12.3M | return (above && is_smooth(above, plane)) || (left && is_smooth(left, plane)); |
987 | 12.3M | } |
988 | | |
989 | 7.15M | static int intra_edge_filter_strength(int bs0, int bs1, int delta, int type) { |
990 | 7.15M | const int d = abs(delta); |
991 | 7.15M | int strength = 0; |
992 | | |
993 | 7.15M | const int blk_wh = bs0 + bs1; |
994 | 7.15M | if (type == 0) { |
995 | 5.63M | if (blk_wh <= 8) { |
996 | 1.53M | if (d >= 56) strength = 1; |
997 | 4.09M | } else if (blk_wh <= 12) { |
998 | 516k | if (d >= 40) strength = 1; |
999 | 3.57M | } else if (blk_wh <= 16) { |
1000 | 788k | if (d >= 40) strength = 1; |
1001 | 2.78M | } else if (blk_wh <= 24) { |
1002 | 1.03M | if (d >= 8) strength = 1; |
1003 | 1.03M | if (d >= 16) strength = 2; |
1004 | 1.03M | if (d >= 32) strength = 3; |
1005 | 1.75M | } else if (blk_wh <= 32) { |
1006 | 530k | if (d >= 1) strength = 1; |
1007 | 530k | if (d >= 4) strength = 2; |
1008 | 530k | if (d >= 32) strength = 3; |
1009 | 1.22M | } else { |
1010 | 1.22M | if (d >= 1) strength = 3; |
1011 | 1.22M | } |
1012 | 5.63M | } else { |
1013 | 1.52M | if (blk_wh <= 8) { |
1014 | 249k | if (d >= 40) strength = 1; |
1015 | 249k | if (d >= 64) strength = 2; |
1016 | 1.27M | } else if (blk_wh <= 16) { |
1017 | 411k | if (d >= 20) strength = 1; |
1018 | 411k | if (d >= 48) strength = 2; |
1019 | 860k | } else if (blk_wh <= 24) { |
1020 | 334k | if (d >= 4) strength = 3; |
1021 | 526k | } else { |
1022 | 526k | if (d >= 1) strength = 3; |
1023 | 526k | } |
1024 | 1.52M | } |
1025 | 7.15M | return strength; |
1026 | 7.15M | } |
1027 | | |
1028 | 0 | void av1_filter_intra_edge_c(uint8_t *p, int sz, int strength) { |
1029 | 0 | if (!strength) return; |
1030 | | |
1031 | 0 | const int kernel[INTRA_EDGE_FILT][INTRA_EDGE_TAPS] = { { 0, 4, 8, 4, 0 }, |
1032 | 0 | { 0, 5, 6, 5, 0 }, |
1033 | 0 | { 2, 4, 4, 4, 2 } }; |
1034 | 0 | const int filt = strength - 1; |
1035 | 0 | uint8_t edge[129]; |
1036 | |
|
1037 | 0 | memcpy(edge, p, sz * sizeof(*p)); |
1038 | 0 | for (int i = 1; i < sz; i++) { |
1039 | 0 | int s = 0; |
1040 | 0 | for (int j = 0; j < INTRA_EDGE_TAPS; j++) { |
1041 | 0 | int k = i - 2 + j; |
1042 | 0 | k = (k < 0) ? 0 : k; |
1043 | 0 | k = (k > sz - 1) ? sz - 1 : k; |
1044 | 0 | s += edge[k] * kernel[filt][j]; |
1045 | 0 | } |
1046 | 0 | s = (s + 8) >> 4; |
1047 | 0 | p[i] = s; |
1048 | 0 | } |
1049 | 0 | } |
1050 | | |
1051 | 416k | static void filter_intra_edge_corner(uint8_t *p_above, uint8_t *p_left) { |
1052 | 416k | const int kernel[3] = { 5, 6, 5 }; |
1053 | | |
1054 | 416k | int s = (p_left[0] * kernel[0]) + (p_above[-1] * kernel[1]) + |
1055 | 416k | (p_above[0] * kernel[2]); |
1056 | 416k | s = (s + 8) >> 4; |
1057 | 416k | p_above[-1] = s; |
1058 | 416k | p_left[-1] = s; |
1059 | 416k | } |
1060 | | |
1061 | 0 | void av1_upsample_intra_edge_c(uint8_t *p, int sz) { |
1062 | | // interpolate half-sample positions |
1063 | 0 | assert(sz <= MAX_UPSAMPLE_SZ); |
1064 | | |
1065 | 0 | uint8_t in[MAX_UPSAMPLE_SZ + 3]; |
1066 | | // copy p[-1..(sz-1)] and extend first and last samples |
1067 | 0 | in[0] = p[-1]; |
1068 | 0 | in[1] = p[-1]; |
1069 | 0 | for (int i = 0; i < sz; i++) { |
1070 | 0 | in[i + 2] = p[i]; |
1071 | 0 | } |
1072 | 0 | in[sz + 2] = p[sz - 1]; |
1073 | | |
1074 | | // interpolate half-sample edge positions |
1075 | 0 | p[-2] = in[0]; |
1076 | 0 | for (int i = 0; i < sz; i++) { |
1077 | 0 | int s = -in[i] + (9 * in[i + 1]) + (9 * in[i + 2]) - in[i + 3]; |
1078 | 0 | s = clip_pixel((s + 8) >> 4); |
1079 | 0 | p[2 * i - 1] = s; |
1080 | 0 | p[2 * i] = in[i + 2]; |
1081 | 0 | } |
1082 | 0 | } |
1083 | | |
1084 | | static void build_directional_and_filter_intra_predictors( |
1085 | | const uint8_t *ref, int ref_stride, uint8_t *dst, int dst_stride, |
1086 | | PREDICTION_MODE mode, int p_angle, FILTER_INTRA_MODE filter_intra_mode, |
1087 | | TX_SIZE tx_size, int disable_edge_filter, int n_top_px, int n_topright_px, |
1088 | 5.21M | int n_left_px, int n_bottomleft_px, int intra_edge_filter_type) { |
1089 | 5.21M | int i; |
1090 | 5.21M | const uint8_t *above_ref = ref - ref_stride; |
1091 | 5.21M | const uint8_t *left_ref = ref - 1; |
1092 | 5.21M | DECLARE_ALIGNED(16, uint8_t, left_data[NUM_INTRA_NEIGHBOUR_PIXELS]); |
1093 | 5.21M | DECLARE_ALIGNED(16, uint8_t, above_data[NUM_INTRA_NEIGHBOUR_PIXELS]); |
1094 | 5.21M | uint8_t *const above_row = above_data + 16; |
1095 | 5.21M | uint8_t *const left_col = left_data + 16; |
1096 | 5.21M | const int txwpx = tx_size_wide[tx_size]; |
1097 | 5.21M | const int txhpx = tx_size_high[tx_size]; |
1098 | 5.21M | int need_left = extend_modes[mode] & NEED_LEFT; |
1099 | 5.21M | int need_above = extend_modes[mode] & NEED_ABOVE; |
1100 | 5.21M | int need_above_left = extend_modes[mode] & NEED_ABOVELEFT; |
1101 | 5.21M | const int is_dr_mode = av1_is_directional_mode(mode); |
1102 | 5.21M | const int use_filter_intra = filter_intra_mode != FILTER_INTRA_MODES; |
1103 | 5.21M | assert(use_filter_intra || is_dr_mode); |
1104 | | // The left_data, above_data buffers must be zeroed to fix some intermittent |
1105 | | // valgrind errors. Uninitialized reads in intra pred modules (e.g. width = 4 |
1106 | | // path in av1_dr_prediction_z1_avx2()) from left_data, above_data are seen to |
1107 | | // be the potential reason for this issue. |
1108 | 5.21M | memset(left_data, 129, NUM_INTRA_NEIGHBOUR_PIXELS); |
1109 | 5.21M | memset(above_data, 127, NUM_INTRA_NEIGHBOUR_PIXELS); |
1110 | | |
1111 | | // The default values if ref pixels are not available: |
1112 | | // 128 127 127 .. 127 127 127 127 127 127 |
1113 | | // 129 A B .. Y Z |
1114 | | // 129 C D .. W X |
1115 | | // 129 E F .. U V |
1116 | | // 129 G H .. S T T T T T |
1117 | | // .. |
1118 | | |
1119 | 5.21M | if (is_dr_mode) { |
1120 | 4.54M | if (p_angle <= 90) |
1121 | 953k | need_above = 1, need_left = 0, need_above_left = 1; |
1122 | 3.59M | else if (p_angle < 180) |
1123 | 1.28M | need_above = 1, need_left = 1, need_above_left = 1; |
1124 | 2.30M | else |
1125 | 2.30M | need_above = 0, need_left = 1, need_above_left = 1; |
1126 | 4.54M | } |
1127 | 5.21M | if (use_filter_intra) need_left = need_above = need_above_left = 1; |
1128 | | |
1129 | 5.21M | assert(n_top_px >= 0); |
1130 | 5.21M | assert(n_topright_px >= -1); |
1131 | 5.21M | assert(n_left_px >= 0); |
1132 | 5.21M | assert(n_bottomleft_px >= -1); |
1133 | | |
1134 | 5.21M | if ((!need_above && n_left_px == 0) || (!need_left && n_top_px == 0)) { |
1135 | 62.1k | int val; |
1136 | 62.1k | if (need_left) { |
1137 | 33.8k | val = (n_top_px > 0) ? above_ref[0] : 129; |
1138 | 33.8k | } else { |
1139 | 28.3k | val = (n_left_px > 0) ? left_ref[0] : 127; |
1140 | 28.3k | } |
1141 | 1.96M | for (i = 0; i < txhpx; ++i) { |
1142 | 1.90M | memset(dst, val, txwpx); |
1143 | 1.90M | dst += dst_stride; |
1144 | 1.90M | } |
1145 | 62.1k | return; |
1146 | 62.1k | } |
1147 | | |
1148 | | // NEED_LEFT |
1149 | 5.15M | if (need_left) { |
1150 | 4.23M | const int num_left_pixels_needed = |
1151 | 4.23M | txhpx + (n_bottomleft_px >= 0 ? txwpx : 0); |
1152 | 4.23M | i = 0; |
1153 | 4.23M | if (n_left_px > 0) { |
1154 | 55.1M | for (; i < n_left_px; i++) left_col[i] = left_ref[i * ref_stride]; |
1155 | 4.17M | if (n_bottomleft_px > 0) { |
1156 | 296k | assert(i == txhpx); |
1157 | 3.67M | for (; i < txhpx + n_bottomleft_px; i++) |
1158 | 3.38M | left_col[i] = left_ref[i * ref_stride]; |
1159 | 296k | } |
1160 | 4.17M | if (i < num_left_pixels_needed) |
1161 | 623k | memset(&left_col[i], left_col[i - 1], num_left_pixels_needed - i); |
1162 | 4.17M | } else if (n_top_px > 0) { |
1163 | 36.4k | memset(left_col, above_ref[0], num_left_pixels_needed); |
1164 | 36.4k | } |
1165 | 4.23M | } |
1166 | | |
1167 | | // NEED_ABOVE |
1168 | 5.15M | if (need_above) { |
1169 | 2.88M | const int num_top_pixels_needed = txwpx + (n_topright_px >= 0 ? txhpx : 0); |
1170 | 2.88M | if (n_top_px > 0) { |
1171 | 2.82M | memcpy(above_row, above_ref, n_top_px); |
1172 | 2.82M | i = n_top_px; |
1173 | 2.82M | if (n_topright_px > 0) { |
1174 | 288k | assert(n_top_px == txwpx); |
1175 | 288k | memcpy(above_row + txwpx, above_ref + txwpx, n_topright_px); |
1176 | 288k | i += n_topright_px; |
1177 | 288k | } |
1178 | 2.82M | if (i < num_top_pixels_needed) |
1179 | 254k | memset(&above_row[i], above_row[i - 1], num_top_pixels_needed - i); |
1180 | 2.82M | } else if (n_left_px > 0) { |
1181 | 42.6k | memset(above_row, left_ref[0], num_top_pixels_needed); |
1182 | 42.6k | } |
1183 | 2.88M | } |
1184 | | |
1185 | 5.15M | if (need_above_left) { |
1186 | 5.15M | if (n_top_px > 0 && n_left_px > 0) { |
1187 | 4.98M | above_row[-1] = above_ref[-1]; |
1188 | 4.98M | } else if (n_top_px > 0) { |
1189 | 59.2k | above_row[-1] = above_ref[0]; |
1190 | 114k | } else if (n_left_px > 0) { |
1191 | 98.2k | above_row[-1] = left_ref[0]; |
1192 | 98.2k | } else { |
1193 | 16.6k | above_row[-1] = 128; |
1194 | 16.6k | } |
1195 | 5.15M | left_col[-1] = above_row[-1]; |
1196 | 5.15M | } |
1197 | | |
1198 | 5.15M | if (use_filter_intra) { |
1199 | 670k | av1_filter_intra_predictor(dst, dst_stride, tx_size, above_row, left_col, |
1200 | 670k | filter_intra_mode); |
1201 | 670k | return; |
1202 | 670k | } |
1203 | | |
1204 | 4.48M | assert(is_dr_mode); |
1205 | 4.48M | int upsample_above = 0; |
1206 | 4.48M | int upsample_left = 0; |
1207 | 4.48M | if (!disable_edge_filter) { |
1208 | 4.31M | const int need_right = p_angle < 90; |
1209 | 4.31M | const int need_bottom = p_angle > 180; |
1210 | 4.31M | if (p_angle != 90 && p_angle != 180) { |
1211 | 2.38M | assert(need_above_left); |
1212 | 2.38M | const int ab_le = 1; |
1213 | 2.38M | if (need_above && need_left && (txwpx + txhpx >= 24)) { |
1214 | 416k | filter_intra_edge_corner(above_row, left_col); |
1215 | 416k | } |
1216 | 2.38M | if (need_above && n_top_px > 0) { |
1217 | 1.64M | const int strength = intra_edge_filter_strength( |
1218 | 1.64M | txwpx, txhpx, p_angle - 90, intra_edge_filter_type); |
1219 | 1.64M | const int n_px = n_top_px + ab_le + (need_right ? txhpx : 0); |
1220 | 1.64M | av1_filter_intra_edge(above_row - ab_le, n_px, strength); |
1221 | 1.64M | } |
1222 | 2.38M | if (need_left && n_left_px > 0) { |
1223 | 1.92M | const int strength = intra_edge_filter_strength( |
1224 | 1.92M | txhpx, txwpx, p_angle - 180, intra_edge_filter_type); |
1225 | 1.92M | const int n_px = n_left_px + ab_le + (need_bottom ? txwpx : 0); |
1226 | 1.92M | av1_filter_intra_edge(left_col - ab_le, n_px, strength); |
1227 | 1.92M | } |
1228 | 2.38M | } |
1229 | 4.31M | upsample_above = av1_use_intra_edge_upsample(txwpx, txhpx, p_angle - 90, |
1230 | 4.31M | intra_edge_filter_type); |
1231 | 4.31M | if (need_above && upsample_above) { |
1232 | 241k | const int n_px = txwpx + (need_right ? txhpx : 0); |
1233 | 241k | av1_upsample_intra_edge(above_row, n_px); |
1234 | 241k | } |
1235 | 4.31M | upsample_left = av1_use_intra_edge_upsample(txhpx, txwpx, p_angle - 180, |
1236 | 4.31M | intra_edge_filter_type); |
1237 | 4.31M | if (need_left && upsample_left) { |
1238 | 735k | const int n_px = txhpx + (need_bottom ? txwpx : 0); |
1239 | 735k | av1_upsample_intra_edge(left_col, n_px); |
1240 | 735k | } |
1241 | 4.31M | } |
1242 | 4.48M | dr_predictor(dst, dst_stride, tx_size, above_row, left_col, upsample_above, |
1243 | 4.48M | upsample_left, p_angle); |
1244 | 4.48M | } |
1245 | | |
1246 | | // This function generates the pred data of a given block for non-directional |
1247 | | // intra prediction modes (i.e., DC, SMOOTH, SMOOTH_H, SMOOTH_V and PAETH). |
1248 | | static void build_non_directional_intra_predictors( |
1249 | | const uint8_t *ref, int ref_stride, uint8_t *dst, int dst_stride, |
1250 | 12.5M | PREDICTION_MODE mode, TX_SIZE tx_size, int n_top_px, int n_left_px) { |
1251 | 12.5M | const uint8_t *above_ref = ref - ref_stride; |
1252 | 12.5M | const uint8_t *left_ref = ref - 1; |
1253 | 12.5M | const int txwpx = tx_size_wide[tx_size]; |
1254 | 12.5M | const int txhpx = tx_size_high[tx_size]; |
1255 | 12.5M | const int need_left = extend_modes[mode] & NEED_LEFT; |
1256 | 12.5M | const int need_above = extend_modes[mode] & NEED_ABOVE; |
1257 | 12.5M | const int need_above_left = extend_modes[mode] & NEED_ABOVELEFT; |
1258 | 12.5M | int i = 0; |
1259 | 12.5M | assert(n_top_px >= 0); |
1260 | 12.5M | assert(n_left_px >= 0); |
1261 | 12.5M | assert(mode == DC_PRED || mode == SMOOTH_PRED || mode == SMOOTH_V_PRED || |
1262 | 12.5M | mode == SMOOTH_H_PRED || mode == PAETH_PRED); |
1263 | | |
1264 | 12.5M | if ((!need_above && n_left_px == 0) || (!need_left && n_top_px == 0)) { |
1265 | 0 | int val = 0; |
1266 | 0 | if (need_left) { |
1267 | 0 | val = (n_top_px > 0) ? above_ref[0] : 129; |
1268 | 0 | } else { |
1269 | 0 | val = (n_left_px > 0) ? left_ref[0] : 127; |
1270 | 0 | } |
1271 | 0 | for (i = 0; i < txhpx; ++i) { |
1272 | 0 | memset(dst, val, txwpx); |
1273 | 0 | dst += dst_stride; |
1274 | 0 | } |
1275 | 0 | return; |
1276 | 0 | } |
1277 | | |
1278 | 12.5M | DECLARE_ALIGNED(16, uint8_t, left_data[NUM_INTRA_NEIGHBOUR_PIXELS]); |
1279 | 12.5M | DECLARE_ALIGNED(16, uint8_t, above_data[NUM_INTRA_NEIGHBOUR_PIXELS]); |
1280 | 12.5M | uint8_t *const above_row = above_data + 16; |
1281 | 12.5M | uint8_t *const left_col = left_data + 16; |
1282 | | |
1283 | 12.5M | if (need_left) { |
1284 | 12.5M | memset(left_data, 129, NUM_INTRA_NEIGHBOUR_PIXELS); |
1285 | 12.5M | if (n_left_px > 0) { |
1286 | 200M | for (i = 0; i < n_left_px; i++) left_col[i] = left_ref[i * ref_stride]; |
1287 | 12.1M | if (i < txhpx) memset(&left_col[i], left_col[i - 1], txhpx - i); |
1288 | 12.1M | } else if (n_top_px > 0) { |
1289 | 274k | memset(left_col, above_ref[0], txhpx); |
1290 | 274k | } |
1291 | 12.5M | } |
1292 | | |
1293 | 12.5M | if (need_above) { |
1294 | 12.5M | memset(above_data, 127, NUM_INTRA_NEIGHBOUR_PIXELS); |
1295 | 12.5M | if (n_top_px > 0) { |
1296 | 12.1M | memcpy(above_row, above_ref, n_top_px); |
1297 | 12.1M | i = n_top_px; |
1298 | 12.1M | if (i < txwpx) memset(&above_row[i], above_row[i - 1], txwpx - i); |
1299 | 12.1M | } else if (n_left_px > 0) { |
1300 | 333k | memset(above_row, left_ref[0], txwpx); |
1301 | 333k | } |
1302 | 12.5M | } |
1303 | | |
1304 | 12.5M | if (need_above_left) { |
1305 | 2.59M | if (n_top_px > 0 && n_left_px > 0) { |
1306 | 2.46M | above_row[-1] = above_ref[-1]; |
1307 | 2.46M | } else if (n_top_px > 0) { |
1308 | 68.6k | above_row[-1] = above_ref[0]; |
1309 | 68.6k | } else if (n_left_px > 0) { |
1310 | 52.3k | above_row[-1] = left_ref[0]; |
1311 | 52.3k | } else { |
1312 | 4.01k | above_row[-1] = 128; |
1313 | 4.01k | } |
1314 | 2.59M | left_col[-1] = above_row[-1]; |
1315 | 2.59M | } |
1316 | | |
1317 | 12.5M | if (mode == DC_PRED) { |
1318 | 8.08M | dc_pred[n_left_px > 0][n_top_px > 0][tx_size](dst, dst_stride, above_row, |
1319 | 8.08M | left_col); |
1320 | 8.08M | } else { |
1321 | 4.45M | pred[mode][tx_size](dst, dst_stride, above_row, left_col); |
1322 | 4.45M | } |
1323 | 12.5M | } |
1324 | | |
1325 | | #if CONFIG_AV1_HIGHBITDEPTH |
1326 | 0 | void av1_highbd_filter_intra_edge_c(uint16_t *p, int sz, int strength) { |
1327 | 0 | if (!strength) return; |
1328 | | |
1329 | 0 | const int kernel[INTRA_EDGE_FILT][INTRA_EDGE_TAPS] = { { 0, 4, 8, 4, 0 }, |
1330 | 0 | { 0, 5, 6, 5, 0 }, |
1331 | 0 | { 2, 4, 4, 4, 2 } }; |
1332 | 0 | const int filt = strength - 1; |
1333 | 0 | uint16_t edge[129]; |
1334 | |
|
1335 | 0 | memcpy(edge, p, sz * sizeof(*p)); |
1336 | 0 | for (int i = 1; i < sz; i++) { |
1337 | 0 | int s = 0; |
1338 | 0 | for (int j = 0; j < INTRA_EDGE_TAPS; j++) { |
1339 | 0 | int k = i - 2 + j; |
1340 | 0 | k = (k < 0) ? 0 : k; |
1341 | 0 | k = (k > sz - 1) ? sz - 1 : k; |
1342 | 0 | s += edge[k] * kernel[filt][j]; |
1343 | 0 | } |
1344 | 0 | s = (s + 8) >> 4; |
1345 | 0 | p[i] = s; |
1346 | 0 | } |
1347 | 0 | } |
1348 | | |
1349 | | static void highbd_filter_intra_edge_corner(uint16_t *p_above, |
1350 | 475k | uint16_t *p_left) { |
1351 | 475k | const int kernel[3] = { 5, 6, 5 }; |
1352 | | |
1353 | 475k | int s = (p_left[0] * kernel[0]) + (p_above[-1] * kernel[1]) + |
1354 | 475k | (p_above[0] * kernel[2]); |
1355 | 475k | s = (s + 8) >> 4; |
1356 | 475k | p_above[-1] = s; |
1357 | 475k | p_left[-1] = s; |
1358 | 475k | } |
1359 | | |
1360 | 0 | void av1_highbd_upsample_intra_edge_c(uint16_t *p, int sz, int bd) { |
1361 | | // interpolate half-sample positions |
1362 | 0 | assert(sz <= MAX_UPSAMPLE_SZ); |
1363 | | |
1364 | 0 | uint16_t in[MAX_UPSAMPLE_SZ + 3]; |
1365 | | // copy p[-1..(sz-1)] and extend first and last samples |
1366 | 0 | in[0] = p[-1]; |
1367 | 0 | in[1] = p[-1]; |
1368 | 0 | for (int i = 0; i < sz; i++) { |
1369 | 0 | in[i + 2] = p[i]; |
1370 | 0 | } |
1371 | 0 | in[sz + 2] = p[sz - 1]; |
1372 | | |
1373 | | // interpolate half-sample edge positions |
1374 | 0 | p[-2] = in[0]; |
1375 | 0 | for (int i = 0; i < sz; i++) { |
1376 | 0 | int s = -in[i] + (9 * in[i + 1]) + (9 * in[i + 2]) - in[i + 3]; |
1377 | 0 | s = (s + 8) >> 4; |
1378 | 0 | s = clip_pixel_highbd(s, bd); |
1379 | 0 | p[2 * i - 1] = s; |
1380 | 0 | p[2 * i] = in[i + 2]; |
1381 | 0 | } |
1382 | 0 | } |
1383 | | |
1384 | | static void highbd_build_directional_and_filter_intra_predictors( |
1385 | | const uint8_t *ref8, int ref_stride, uint8_t *dst8, int dst_stride, |
1386 | | PREDICTION_MODE mode, int p_angle, FILTER_INTRA_MODE filter_intra_mode, |
1387 | | TX_SIZE tx_size, int disable_edge_filter, int n_top_px, int n_topright_px, |
1388 | | int n_left_px, int n_bottomleft_px, int intra_edge_filter_type, |
1389 | 7.16M | int bit_depth) { |
1390 | 7.16M | int i; |
1391 | 7.16M | uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); |
1392 | 7.16M | const uint16_t *const ref = CONVERT_TO_SHORTPTR(ref8); |
1393 | 7.16M | DECLARE_ALIGNED(16, uint16_t, left_data[NUM_INTRA_NEIGHBOUR_PIXELS]); |
1394 | 7.16M | DECLARE_ALIGNED(16, uint16_t, above_data[NUM_INTRA_NEIGHBOUR_PIXELS]); |
1395 | 7.16M | uint16_t *const above_row = above_data + 16; |
1396 | 7.16M | uint16_t *const left_col = left_data + 16; |
1397 | 7.16M | const int txwpx = tx_size_wide[tx_size]; |
1398 | 7.16M | const int txhpx = tx_size_high[tx_size]; |
1399 | 7.16M | int need_left = extend_modes[mode] & NEED_LEFT; |
1400 | 7.16M | int need_above = extend_modes[mode] & NEED_ABOVE; |
1401 | 7.16M | int need_above_left = extend_modes[mode] & NEED_ABOVELEFT; |
1402 | 7.16M | const uint16_t *above_ref = ref - ref_stride; |
1403 | 7.16M | const uint16_t *left_ref = ref - 1; |
1404 | 7.16M | const int is_dr_mode = av1_is_directional_mode(mode); |
1405 | 7.16M | const int use_filter_intra = filter_intra_mode != FILTER_INTRA_MODES; |
1406 | 7.16M | assert(use_filter_intra || is_dr_mode); |
1407 | 7.16M | const int base = 128 << (bit_depth - 8); |
1408 | | // The left_data, above_data buffers must be zeroed to fix some intermittent |
1409 | | // valgrind errors. Uninitialized reads in intra pred modules (e.g. width = 4 |
1410 | | // path in av1_highbd_dr_prediction_z2_avx2()) from left_data, above_data are |
1411 | | // seen to be the potential reason for this issue. |
1412 | 7.16M | aom_memset16(left_data, base + 1, NUM_INTRA_NEIGHBOUR_PIXELS); |
1413 | 7.16M | aom_memset16(above_data, base - 1, NUM_INTRA_NEIGHBOUR_PIXELS); |
1414 | | |
1415 | | // The default values if ref pixels are not available: |
1416 | | // base base-1 base-1 .. base-1 base-1 base-1 base-1 base-1 base-1 |
1417 | | // base+1 A B .. Y Z |
1418 | | // base+1 C D .. W X |
1419 | | // base+1 E F .. U V |
1420 | | // base+1 G H .. S T T T T T |
1421 | | |
1422 | 7.16M | if (is_dr_mode) { |
1423 | 6.41M | if (p_angle <= 90) |
1424 | 1.51M | need_above = 1, need_left = 0, need_above_left = 1; |
1425 | 4.90M | else if (p_angle < 180) |
1426 | 1.63M | need_above = 1, need_left = 1, need_above_left = 1; |
1427 | 3.26M | else |
1428 | 3.26M | need_above = 0, need_left = 1, need_above_left = 1; |
1429 | 6.41M | } |
1430 | 7.16M | if (use_filter_intra) need_left = need_above = need_above_left = 1; |
1431 | | |
1432 | 7.16M | assert(n_top_px >= 0); |
1433 | 7.15M | assert(n_topright_px >= -1); |
1434 | 7.15M | assert(n_left_px >= 0); |
1435 | 7.15M | assert(n_bottomleft_px >= -1); |
1436 | | |
1437 | 7.15M | if ((!need_above && n_left_px == 0) || (!need_left && n_top_px == 0)) { |
1438 | 268k | int val; |
1439 | 268k | if (need_left) { |
1440 | 204k | val = (n_top_px > 0) ? above_ref[0] : base + 1; |
1441 | 204k | } else { |
1442 | 64.3k | val = (n_left_px > 0) ? left_ref[0] : base - 1; |
1443 | 64.3k | } |
1444 | 5.15M | for (i = 0; i < txhpx; ++i) { |
1445 | 4.88M | aom_memset16(dst, val, txwpx); |
1446 | 4.88M | dst += dst_stride; |
1447 | 4.88M | } |
1448 | 268k | return; |
1449 | 268k | } |
1450 | | |
1451 | | // NEED_LEFT |
1452 | 6.89M | if (need_left) { |
1453 | 5.44M | const int num_left_pixels_needed = |
1454 | 5.44M | txhpx + (n_bottomleft_px >= 0 ? txwpx : 0); |
1455 | 5.44M | i = 0; |
1456 | 5.44M | if (n_left_px > 0) { |
1457 | 68.2M | for (; i < n_left_px; i++) left_col[i] = left_ref[i * ref_stride]; |
1458 | 5.37M | if (n_bottomleft_px > 0) { |
1459 | 403k | assert(i == txhpx); |
1460 | 4.65M | for (; i < txhpx + n_bottomleft_px; i++) |
1461 | 4.25M | left_col[i] = left_ref[i * ref_stride]; |
1462 | 403k | } |
1463 | 5.37M | if (i < num_left_pixels_needed) |
1464 | 980k | aom_memset16(&left_col[i], left_col[i - 1], num_left_pixels_needed - i); |
1465 | 5.37M | } else if (n_top_px > 0) { |
1466 | 62.0k | aom_memset16(left_col, above_ref[0], num_left_pixels_needed); |
1467 | 62.0k | } |
1468 | 5.44M | } |
1469 | | |
1470 | | // NEED_ABOVE |
1471 | 6.89M | if (need_above) { |
1472 | 3.82M | const int num_top_pixels_needed = txwpx + (n_topright_px >= 0 ? txhpx : 0); |
1473 | 3.82M | if (n_top_px > 0) { |
1474 | 3.76M | memcpy(above_row, above_ref, n_top_px * sizeof(above_ref[0])); |
1475 | 3.76M | i = n_top_px; |
1476 | 3.76M | if (n_topright_px > 0) { |
1477 | 410k | assert(n_top_px == txwpx); |
1478 | 410k | memcpy(above_row + txwpx, above_ref + txwpx, |
1479 | 410k | n_topright_px * sizeof(above_ref[0])); |
1480 | 410k | i += n_topright_px; |
1481 | 410k | } |
1482 | 3.76M | if (i < num_top_pixels_needed) |
1483 | 470k | aom_memset16(&above_row[i], above_row[i - 1], |
1484 | 470k | num_top_pixels_needed - i); |
1485 | 3.76M | } else if (n_left_px > 0) { |
1486 | 58.4k | aom_memset16(above_row, left_ref[0], num_top_pixels_needed); |
1487 | 58.4k | } |
1488 | 3.82M | } |
1489 | | |
1490 | 6.89M | if (need_above_left) { |
1491 | 6.89M | if (n_top_px > 0 && n_left_px > 0) { |
1492 | 6.61M | above_row[-1] = above_ref[-1]; |
1493 | 6.61M | } else if (n_top_px > 0) { |
1494 | 104k | above_row[-1] = above_ref[0]; |
1495 | 166k | } else if (n_left_px > 0) { |
1496 | 164k | above_row[-1] = left_ref[0]; |
1497 | 164k | } else { |
1498 | 2.88k | above_row[-1] = base; |
1499 | 2.88k | } |
1500 | 6.89M | left_col[-1] = above_row[-1]; |
1501 | 6.89M | } |
1502 | | |
1503 | 6.89M | if (use_filter_intra) { |
1504 | 744k | highbd_filter_intra_predictor(dst, dst_stride, tx_size, above_row, left_col, |
1505 | 744k | filter_intra_mode, bit_depth); |
1506 | 744k | return; |
1507 | 744k | } |
1508 | | |
1509 | 6.14M | assert(is_dr_mode); |
1510 | 6.14M | int upsample_above = 0; |
1511 | 6.14M | int upsample_left = 0; |
1512 | 6.14M | if (!disable_edge_filter) { |
1513 | 4.85M | const int need_right = p_angle < 90; |
1514 | 4.85M | const int need_bottom = p_angle > 180; |
1515 | 4.85M | if (p_angle != 90 && p_angle != 180) { |
1516 | 2.54M | assert(need_above_left); |
1517 | 2.54M | const int ab_le = 1; |
1518 | 2.54M | if (need_above && need_left && (txwpx + txhpx >= 24)) { |
1519 | 475k | highbd_filter_intra_edge_corner(above_row, left_col); |
1520 | 475k | } |
1521 | 2.54M | if (need_above && n_top_px > 0) { |
1522 | 1.60M | const int strength = intra_edge_filter_strength( |
1523 | 1.60M | txwpx, txhpx, p_angle - 90, intra_edge_filter_type); |
1524 | 1.60M | const int n_px = n_top_px + ab_le + (need_right ? txhpx : 0); |
1525 | 1.60M | av1_highbd_filter_intra_edge(above_row - ab_le, n_px, strength); |
1526 | 1.60M | } |
1527 | 2.54M | if (need_left && n_left_px > 0) { |
1528 | 1.97M | const int strength = intra_edge_filter_strength( |
1529 | 1.97M | txhpx, txwpx, p_angle - 180, intra_edge_filter_type); |
1530 | 1.97M | const int n_px = n_left_px + ab_le + (need_bottom ? txwpx : 0); |
1531 | 1.97M | av1_highbd_filter_intra_edge(left_col - ab_le, n_px, strength); |
1532 | 1.97M | } |
1533 | 2.54M | } |
1534 | 4.85M | upsample_above = av1_use_intra_edge_upsample(txwpx, txhpx, p_angle - 90, |
1535 | 4.85M | intra_edge_filter_type); |
1536 | 4.85M | if (need_above && upsample_above) { |
1537 | 301k | const int n_px = txwpx + (need_right ? txhpx : 0); |
1538 | 301k | av1_highbd_upsample_intra_edge(above_row, n_px, bit_depth); |
1539 | 301k | } |
1540 | 4.85M | upsample_left = av1_use_intra_edge_upsample(txhpx, txwpx, p_angle - 180, |
1541 | 4.85M | intra_edge_filter_type); |
1542 | 4.85M | if (need_left && upsample_left) { |
1543 | 545k | const int n_px = txhpx + (need_bottom ? txwpx : 0); |
1544 | 545k | av1_highbd_upsample_intra_edge(left_col, n_px, bit_depth); |
1545 | 545k | } |
1546 | 4.85M | } |
1547 | 6.14M | highbd_dr_predictor(dst, dst_stride, tx_size, above_row, left_col, |
1548 | 6.14M | upsample_above, upsample_left, p_angle, bit_depth); |
1549 | 6.14M | } |
1550 | | |
1551 | | // For HBD encode/decode, this function generates the pred data of a given |
1552 | | // block for non-directional intra prediction modes (i.e., DC, SMOOTH, SMOOTH_H, |
1553 | | // SMOOTH_V and PAETH). |
1554 | | static void highbd_build_non_directional_intra_predictors( |
1555 | | const uint8_t *ref8, int ref_stride, uint8_t *dst8, int dst_stride, |
1556 | | PREDICTION_MODE mode, TX_SIZE tx_size, int n_top_px, int n_left_px, |
1557 | 88.3M | int bit_depth) { |
1558 | 88.3M | int i = 0; |
1559 | 88.3M | uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); |
1560 | 88.3M | const uint16_t *const ref = CONVERT_TO_SHORTPTR(ref8); |
1561 | 88.3M | const int txwpx = tx_size_wide[tx_size]; |
1562 | 88.3M | const int txhpx = tx_size_high[tx_size]; |
1563 | 88.3M | int need_left = extend_modes[mode] & NEED_LEFT; |
1564 | 88.3M | int need_above = extend_modes[mode] & NEED_ABOVE; |
1565 | 88.3M | int need_above_left = extend_modes[mode] & NEED_ABOVELEFT; |
1566 | 88.3M | const uint16_t *above_ref = ref - ref_stride; |
1567 | 88.3M | const uint16_t *left_ref = ref - 1; |
1568 | 88.3M | const int base = 128 << (bit_depth - 8); |
1569 | | |
1570 | 88.3M | assert(n_top_px >= 0); |
1571 | 88.3M | assert(n_left_px >= 0); |
1572 | 88.3M | assert(mode == DC_PRED || mode == SMOOTH_PRED || mode == SMOOTH_V_PRED || |
1573 | 88.3M | mode == SMOOTH_H_PRED || mode == PAETH_PRED); |
1574 | | |
1575 | 88.3M | if ((!need_above && n_left_px == 0) || (!need_left && n_top_px == 0)) { |
1576 | 0 | int val = 0; |
1577 | 0 | if (need_left) { |
1578 | 0 | val = (n_top_px > 0) ? above_ref[0] : base + 1; |
1579 | 0 | } else { |
1580 | 0 | val = (n_left_px > 0) ? left_ref[0] : base - 1; |
1581 | 0 | } |
1582 | 0 | for (i = 0; i < txhpx; ++i) { |
1583 | 0 | aom_memset16(dst, val, txwpx); |
1584 | 0 | dst += dst_stride; |
1585 | 0 | } |
1586 | 0 | return; |
1587 | 0 | } |
1588 | | |
1589 | 88.3M | DECLARE_ALIGNED(16, uint16_t, left_data[NUM_INTRA_NEIGHBOUR_PIXELS]); |
1590 | 88.3M | DECLARE_ALIGNED(16, uint16_t, above_data[NUM_INTRA_NEIGHBOUR_PIXELS]); |
1591 | 88.3M | uint16_t *const above_row = above_data + 16; |
1592 | 88.3M | uint16_t *const left_col = left_data + 16; |
1593 | | |
1594 | 88.3M | if (need_left) { |
1595 | 88.3M | aom_memset16(left_data, base + 1, NUM_INTRA_NEIGHBOUR_PIXELS); |
1596 | 88.3M | if (n_left_px > 0) { |
1597 | 608M | for (i = 0; i < n_left_px; i++) left_col[i] = left_ref[i * ref_stride]; |
1598 | 86.5M | if (i < txhpx) aom_memset16(&left_col[i], left_col[i - 1], txhpx - i); |
1599 | 86.5M | } else if (n_top_px > 0) { |
1600 | 1.75M | aom_memset16(left_col, above_ref[0], txhpx); |
1601 | 1.75M | } |
1602 | 88.3M | } |
1603 | | |
1604 | 88.3M | if (need_above) { |
1605 | 88.3M | aom_memset16(above_data, base - 1, NUM_INTRA_NEIGHBOUR_PIXELS); |
1606 | 88.3M | if (n_top_px > 0) { |
1607 | 87.6M | memcpy(above_row, above_ref, n_top_px * sizeof(above_ref[0])); |
1608 | 87.6M | i = n_top_px; |
1609 | 87.6M | if (i < txwpx) aom_memset16(&above_row[i], above_row[i - 1], (txwpx - i)); |
1610 | 87.6M | } else if (n_left_px > 0) { |
1611 | 617k | aom_memset16(above_row, left_ref[0], txwpx); |
1612 | 617k | } |
1613 | 88.3M | } |
1614 | | |
1615 | 88.3M | if (need_above_left) { |
1616 | 3.04M | if (n_top_px > 0 && n_left_px > 0) { |
1617 | 2.87M | above_row[-1] = above_ref[-1]; |
1618 | 2.87M | } else if (n_top_px > 0) { |
1619 | 71.0k | above_row[-1] = above_ref[0]; |
1620 | 94.5k | } else if (n_left_px > 0) { |
1621 | 92.3k | above_row[-1] = left_ref[0]; |
1622 | 92.3k | } else { |
1623 | 2.25k | above_row[-1] = base; |
1624 | 2.25k | } |
1625 | 3.04M | left_col[-1] = above_row[-1]; |
1626 | 3.04M | } |
1627 | | |
1628 | 88.3M | if (mode == DC_PRED) { |
1629 | 82.3M | dc_pred_high[n_left_px > 0][n_top_px > 0][tx_size]( |
1630 | 82.3M | dst, dst_stride, above_row, left_col, bit_depth); |
1631 | 82.3M | } else { |
1632 | 5.98M | pred_high[mode][tx_size](dst, dst_stride, above_row, left_col, bit_depth); |
1633 | 5.98M | } |
1634 | 88.3M | } |
1635 | | #endif // CONFIG_AV1_HIGHBITDEPTH |
1636 | | |
1637 | | static inline BLOCK_SIZE scale_chroma_bsize(BLOCK_SIZE bsize, int subsampling_x, |
1638 | 3.35M | int subsampling_y) { |
1639 | 3.35M | assert(subsampling_x >= 0 && subsampling_x < 2); |
1640 | 3.35M | assert(subsampling_y >= 0 && subsampling_y < 2); |
1641 | 3.35M | BLOCK_SIZE bs = bsize; |
1642 | 3.35M | switch (bsize) { |
1643 | 16.8k | case BLOCK_4X4: |
1644 | 16.8k | if (subsampling_x == 1 && subsampling_y == 1) |
1645 | 16.7k | bs = BLOCK_8X8; |
1646 | 82 | else if (subsampling_x == 1) |
1647 | 82 | bs = BLOCK_8X4; |
1648 | 0 | else if (subsampling_y == 1) |
1649 | 0 | bs = BLOCK_4X8; |
1650 | 16.8k | break; |
1651 | 30.9k | case BLOCK_4X8: |
1652 | 30.9k | if (subsampling_x == 1 && subsampling_y == 1) |
1653 | 30.9k | bs = BLOCK_8X8; |
1654 | 0 | else if (subsampling_x == 1) |
1655 | 0 | bs = BLOCK_8X8; |
1656 | 0 | else if (subsampling_y == 1) |
1657 | 0 | bs = BLOCK_4X8; |
1658 | 30.9k | break; |
1659 | 48.1k | case BLOCK_8X4: |
1660 | 48.1k | if (subsampling_x == 1 && subsampling_y == 1) |
1661 | 47.0k | bs = BLOCK_8X8; |
1662 | 1.14k | else if (subsampling_x == 1) |
1663 | 1.14k | bs = BLOCK_8X4; |
1664 | 0 | else if (subsampling_y == 1) |
1665 | 0 | bs = BLOCK_8X8; |
1666 | 48.1k | break; |
1667 | 36.6k | case BLOCK_4X16: |
1668 | 36.6k | if (subsampling_x == 1 && subsampling_y == 1) |
1669 | 36.6k | bs = BLOCK_8X16; |
1670 | 0 | else if (subsampling_x == 1) |
1671 | 0 | bs = BLOCK_8X16; |
1672 | 0 | else if (subsampling_y == 1) |
1673 | 0 | bs = BLOCK_4X16; |
1674 | 36.6k | break; |
1675 | 65.8k | case BLOCK_16X4: |
1676 | 65.8k | if (subsampling_x == 1 && subsampling_y == 1) |
1677 | 65.1k | bs = BLOCK_16X8; |
1678 | 700 | else if (subsampling_x == 1) |
1679 | 700 | bs = BLOCK_16X4; |
1680 | 0 | else if (subsampling_y == 1) |
1681 | 0 | bs = BLOCK_16X8; |
1682 | 65.8k | break; |
1683 | 3.16M | default: break; |
1684 | 3.35M | } |
1685 | 3.35M | return bs; |
1686 | 3.35M | } |
1687 | | |
1688 | | void av1_predict_intra_block(const MACROBLOCKD *xd, BLOCK_SIZE sb_size, |
1689 | | int enable_intra_edge_filter, int wpx, int hpx, |
1690 | | TX_SIZE tx_size, PREDICTION_MODE mode, |
1691 | | int angle_delta, int use_palette, |
1692 | | FILTER_INTRA_MODE filter_intra_mode, |
1693 | | const uint8_t *ref, int ref_stride, uint8_t *dst, |
1694 | | int dst_stride, int col_off, int row_off, |
1695 | 113M | int plane) { |
1696 | 113M | const MB_MODE_INFO *const mbmi = xd->mi[0]; |
1697 | 113M | const int txwpx = tx_size_wide[tx_size]; |
1698 | 113M | const int txhpx = tx_size_high[tx_size]; |
1699 | 113M | const int x = col_off << MI_SIZE_LOG2; |
1700 | 113M | const int y = row_off << MI_SIZE_LOG2; |
1701 | 113M | const int is_hbd = is_cur_buf_hbd(xd); |
1702 | | |
1703 | 113M | assert(mode < INTRA_MODES); |
1704 | | |
1705 | 113M | if (use_palette) { |
1706 | 252k | int r, c; |
1707 | 252k | const uint8_t *const map = xd->plane[plane != 0].color_index_map + |
1708 | 252k | xd->color_index_map_offset[plane != 0]; |
1709 | 252k | const uint16_t *const palette = |
1710 | 252k | mbmi->palette_mode_info.palette_colors + plane * PALETTE_MAX_SIZE; |
1711 | 252k | if (is_hbd) { |
1712 | 148k | uint16_t *dst16 = CONVERT_TO_SHORTPTR(dst); |
1713 | 1.51M | for (r = 0; r < txhpx; ++r) { |
1714 | 17.9M | for (c = 0; c < txwpx; ++c) { |
1715 | 16.5M | dst16[r * dst_stride + c] = palette[map[(r + y) * wpx + c + x]]; |
1716 | 16.5M | } |
1717 | 1.36M | } |
1718 | 148k | } else { |
1719 | 991k | for (r = 0; r < txhpx; ++r) { |
1720 | 11.2M | for (c = 0; c < txwpx; ++c) { |
1721 | 10.3M | dst[r * dst_stride + c] = |
1722 | 10.3M | (uint8_t)palette[map[(r + y) * wpx + c + x]]; |
1723 | 10.3M | } |
1724 | 887k | } |
1725 | 103k | } |
1726 | 252k | return; |
1727 | 252k | } |
1728 | | |
1729 | 113M | const struct macroblockd_plane *const pd = &xd->plane[plane]; |
1730 | 113M | const int ss_x = pd->subsampling_x; |
1731 | 113M | const int ss_y = pd->subsampling_y; |
1732 | 113M | const int have_top = |
1733 | 113M | row_off || (ss_y ? xd->chroma_up_available : xd->up_available); |
1734 | 113M | const int have_left = |
1735 | 113M | col_off || (ss_x ? xd->chroma_left_available : xd->left_available); |
1736 | | |
1737 | | // Distance between the right edge of this prediction block to |
1738 | | // the frame right edge |
1739 | 113M | const int xr = (xd->mb_to_right_edge >> (3 + ss_x)) + wpx - x - txwpx; |
1740 | | // Distance between the bottom edge of this prediction block to |
1741 | | // the frame bottom edge |
1742 | 113M | const int yd = (xd->mb_to_bottom_edge >> (3 + ss_y)) + hpx - y - txhpx; |
1743 | 113M | const int use_filter_intra = filter_intra_mode != FILTER_INTRA_MODES; |
1744 | 113M | const int is_dr_mode = av1_is_directional_mode(mode); |
1745 | | |
1746 | | // The computations in this function, as well as in build_intra_predictors(), |
1747 | | // are generalized for all intra modes. Some of these operations are not |
1748 | | // required since non-directional intra modes (i.e., DC, SMOOTH, SMOOTH_H, |
1749 | | // SMOOTH_V, and PAETH) specifically require left and top neighbors. Hence, a |
1750 | | // separate function build_non_directional_intra_predictors() is introduced |
1751 | | // for these modes to avoid redundant computations while generating pred data. |
1752 | | |
1753 | 113M | const int n_top_px = have_top ? AOMMIN(txwpx, xr + txwpx) : 0; |
1754 | 113M | const int n_left_px = have_left ? AOMMIN(txhpx, yd + txhpx) : 0; |
1755 | 113M | if (!use_filter_intra && !is_dr_mode) { |
1756 | 100M | #if CONFIG_AV1_HIGHBITDEPTH |
1757 | 100M | if (is_hbd) { |
1758 | 88.3M | highbd_build_non_directional_intra_predictors( |
1759 | 88.3M | ref, ref_stride, dst, dst_stride, mode, tx_size, n_top_px, n_left_px, |
1760 | 88.3M | xd->bd); |
1761 | 88.3M | return; |
1762 | 88.3M | } |
1763 | 12.5M | #endif // CONFIG_AV1_HIGHBITDEPTH |
1764 | 12.5M | build_non_directional_intra_predictors(ref, ref_stride, dst, dst_stride, |
1765 | 12.5M | mode, tx_size, n_top_px, n_left_px); |
1766 | 12.5M | return; |
1767 | 100M | } |
1768 | | |
1769 | 12.3M | const int txw = tx_size_wide_unit[tx_size]; |
1770 | 12.3M | const int txh = tx_size_high_unit[tx_size]; |
1771 | 12.3M | const int mi_row = -xd->mb_to_top_edge >> (3 + MI_SIZE_LOG2); |
1772 | 12.3M | const int mi_col = -xd->mb_to_left_edge >> (3 + MI_SIZE_LOG2); |
1773 | 12.3M | const int right_available = |
1774 | 12.3M | mi_col + ((col_off + txw) << ss_x) < xd->tile.mi_col_end; |
1775 | 12.3M | const int bottom_available = |
1776 | 12.3M | (yd > 0) && (mi_row + ((row_off + txh) << ss_y) < xd->tile.mi_row_end); |
1777 | | |
1778 | 12.3M | const PARTITION_TYPE partition = mbmi->partition; |
1779 | | |
1780 | 12.3M | BLOCK_SIZE bsize = mbmi->bsize; |
1781 | | // force 4x4 chroma component block size. |
1782 | 12.3M | if (ss_x || ss_y) { |
1783 | 3.35M | bsize = scale_chroma_bsize(bsize, ss_x, ss_y); |
1784 | 3.35M | } |
1785 | | |
1786 | 12.3M | int p_angle = 0; |
1787 | 12.3M | int need_top_right = extend_modes[mode] & NEED_ABOVERIGHT; |
1788 | 12.3M | int need_bottom_left = extend_modes[mode] & NEED_BOTTOMLEFT; |
1789 | | |
1790 | 12.3M | if (use_filter_intra) { |
1791 | 1.41M | need_top_right = 0; |
1792 | 1.41M | need_bottom_left = 0; |
1793 | 1.41M | } |
1794 | 12.3M | if (is_dr_mode) { |
1795 | 10.9M | p_angle = mode_to_angle_map[mode] + angle_delta; |
1796 | 10.9M | need_top_right = p_angle < 90; |
1797 | 10.9M | need_bottom_left = p_angle > 180; |
1798 | 10.9M | } |
1799 | | |
1800 | | // Possible states for have_top_right(TR) and have_bottom_left(BL) |
1801 | | // -1 : TR and BL are not needed |
1802 | | // 0 : TR and BL are needed but not available |
1803 | | // > 0 : TR and BL are needed and pixels are available |
1804 | 12.3M | const int have_top_right = |
1805 | 12.3M | need_top_right ? has_top_right(sb_size, bsize, mi_row, mi_col, have_top, |
1806 | 1.29M | right_available, partition, tx_size, |
1807 | 1.29M | row_off, col_off, ss_x, ss_y) |
1808 | 12.3M | : -1; |
1809 | 12.3M | const int have_bottom_left = |
1810 | 12.3M | need_bottom_left ? has_bottom_left(sb_size, bsize, mi_row, mi_col, |
1811 | 2.08M | bottom_available, have_left, partition, |
1812 | 2.08M | tx_size, row_off, col_off, ss_x, ss_y) |
1813 | 12.3M | : -1; |
1814 | | |
1815 | 12.3M | const int disable_edge_filter = !enable_intra_edge_filter; |
1816 | 12.3M | const int intra_edge_filter_type = get_intra_edge_filter_type(xd, plane); |
1817 | 12.3M | const int n_topright_px = |
1818 | 12.3M | have_top_right > 0 ? AOMMIN(txwpx, xr) : have_top_right; |
1819 | 12.3M | const int n_bottomleft_px = |
1820 | 12.3M | have_bottom_left > 0 ? AOMMIN(txhpx, yd) : have_bottom_left; |
1821 | 12.3M | #if CONFIG_AV1_HIGHBITDEPTH |
1822 | 12.3M | if (is_hbd) { |
1823 | 7.16M | highbd_build_directional_and_filter_intra_predictors( |
1824 | 7.16M | ref, ref_stride, dst, dst_stride, mode, p_angle, filter_intra_mode, |
1825 | 7.16M | tx_size, disable_edge_filter, n_top_px, n_topright_px, n_left_px, |
1826 | 7.16M | n_bottomleft_px, intra_edge_filter_type, xd->bd); |
1827 | 7.16M | return; |
1828 | 7.16M | } |
1829 | 5.19M | #endif |
1830 | 5.19M | build_directional_and_filter_intra_predictors( |
1831 | 5.19M | ref, ref_stride, dst, dst_stride, mode, p_angle, filter_intra_mode, |
1832 | 5.19M | tx_size, disable_edge_filter, n_top_px, n_topright_px, n_left_px, |
1833 | 5.19M | n_bottomleft_px, intra_edge_filter_type); |
1834 | 5.19M | } |
1835 | | |
1836 | | void av1_predict_intra_block_facade(const AV1_COMMON *cm, MACROBLOCKD *xd, |
1837 | | int plane, int blk_col, int blk_row, |
1838 | 112M | TX_SIZE tx_size) { |
1839 | 112M | const MB_MODE_INFO *const mbmi = xd->mi[0]; |
1840 | 112M | struct macroblockd_plane *const pd = &xd->plane[plane]; |
1841 | 112M | const int dst_stride = pd->dst.stride; |
1842 | 112M | uint8_t *dst = &pd->dst.buf[(blk_row * dst_stride + blk_col) << MI_SIZE_LOG2]; |
1843 | 112M | const PREDICTION_MODE mode = |
1844 | 112M | (plane == AOM_PLANE_Y) ? mbmi->mode : get_uv_mode(mbmi->uv_mode); |
1845 | 112M | const int use_palette = mbmi->palette_mode_info.palette_size[plane != 0] > 0; |
1846 | 112M | const FILTER_INTRA_MODE filter_intra_mode = |
1847 | 112M | (plane == AOM_PLANE_Y && mbmi->filter_intra_mode_info.use_filter_intra) |
1848 | 112M | ? mbmi->filter_intra_mode_info.filter_intra_mode |
1849 | 112M | : FILTER_INTRA_MODES; |
1850 | 112M | const int angle_delta = mbmi->angle_delta[plane != AOM_PLANE_Y] * ANGLE_STEP; |
1851 | 112M | const SequenceHeader *seq_params = cm->seq_params; |
1852 | | |
1853 | 112M | #if !CONFIG_REALTIME_ONLY || CONFIG_AV1_DECODER |
1854 | 112M | if (plane != AOM_PLANE_Y && mbmi->uv_mode == UV_CFL_PRED) { |
1855 | | #if CONFIG_DEBUG |
1856 | | assert(is_cfl_allowed(xd)); |
1857 | | const BLOCK_SIZE plane_bsize = |
1858 | | get_plane_block_size(mbmi->bsize, pd->subsampling_x, pd->subsampling_y); |
1859 | | (void)plane_bsize; |
1860 | | assert(plane_bsize < BLOCK_SIZES_ALL); |
1861 | | if (!xd->lossless[mbmi->segment_id]) { |
1862 | | assert(blk_col == 0); |
1863 | | assert(blk_row == 0); |
1864 | | assert(block_size_wide[plane_bsize] == tx_size_wide[tx_size]); |
1865 | | assert(block_size_high[plane_bsize] == tx_size_high[tx_size]); |
1866 | | } |
1867 | | #endif |
1868 | 2.97M | CFL_CTX *const cfl = &xd->cfl; |
1869 | 2.97M | CFL_PRED_TYPE pred_plane = get_cfl_pred_type(plane); |
1870 | 2.97M | if (!cfl->dc_pred_is_cached[pred_plane]) { |
1871 | 2.97M | av1_predict_intra_block(xd, seq_params->sb_size, |
1872 | 2.97M | seq_params->enable_intra_edge_filter, pd->width, |
1873 | 2.97M | pd->height, tx_size, mode, angle_delta, |
1874 | 2.97M | use_palette, filter_intra_mode, dst, dst_stride, |
1875 | 2.97M | dst, dst_stride, blk_col, blk_row, plane); |
1876 | 2.97M | if (cfl->use_dc_pred_cache) { |
1877 | 0 | cfl_store_dc_pred(xd, dst, pred_plane, tx_size_wide[tx_size]); |
1878 | 0 | cfl->dc_pred_is_cached[pred_plane] = true; |
1879 | 0 | } |
1880 | 18.4E | } else { |
1881 | 18.4E | cfl_load_dc_pred(xd, dst, dst_stride, tx_size, pred_plane); |
1882 | 18.4E | } |
1883 | 2.97M | av1_cfl_predict_block(xd, dst, dst_stride, tx_size, plane); |
1884 | 2.97M | return; |
1885 | 2.97M | } |
1886 | 109M | #endif // !CONFIG_REALTIME_ONLY || CONFIG_AV1_DECODER |
1887 | 109M | av1_predict_intra_block( |
1888 | 109M | xd, seq_params->sb_size, seq_params->enable_intra_edge_filter, pd->width, |
1889 | 109M | pd->height, tx_size, mode, angle_delta, use_palette, filter_intra_mode, |
1890 | 109M | dst, dst_stride, dst, dst_stride, blk_col, blk_row, plane); |
1891 | 109M | } |
1892 | | |
1893 | 16.1k | void av1_init_intra_predictors(void) { |
1894 | 16.1k | aom_once(init_intra_predictors_internal); |
1895 | 16.1k | } |