/src/aom/av1/encoder/txb_rdopt.c
Line | Count | Source (jump to first uncovered line) |
1 | | /* |
2 | | * Copyright (c) 2021, Alliance for Open Media. All rights reserved |
3 | | * |
4 | | * This source code is subject to the terms of the BSD 2 Clause License and |
5 | | * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License |
6 | | * was not distributed with this source code in the LICENSE file, you can |
7 | | * obtain it at www.aomedia.org/license/software. If the Alliance for Open |
8 | | * Media Patent License 1.0 was not distributed with this source code in the |
9 | | * PATENTS file, you can obtain it at www.aomedia.org/license/patent. |
10 | | */ |
11 | | |
12 | | #include "av1/encoder/txb_rdopt.h" |
13 | | #include "av1/encoder/txb_rdopt_utils.h" |
14 | | |
15 | | #include "av1/common/idct.h" |
16 | | |
17 | | static INLINE void update_coeff_general( |
18 | | int *accu_rate, int64_t *accu_dist, int si, int eob, TX_SIZE tx_size, |
19 | | TX_CLASS tx_class, int bwl, int height, int64_t rdmult, int shift, |
20 | | int dc_sign_ctx, const int16_t *dequant, const int16_t *scan, |
21 | | const LV_MAP_COEFF_COST *txb_costs, const tran_low_t *tcoeff, |
22 | | tran_low_t *qcoeff, tran_low_t *dqcoeff, uint8_t *levels, |
23 | 102k | const qm_val_t *iqmatrix) { |
24 | 102k | const int dqv = get_dqv(dequant, scan[si], iqmatrix); |
25 | 102k | const int ci = scan[si]; |
26 | 102k | const tran_low_t qc = qcoeff[ci]; |
27 | 102k | const int is_last = si == (eob - 1); |
28 | 102k | const int coeff_ctx = get_lower_levels_ctx_general( |
29 | 102k | is_last, si, bwl, height, levels, ci, tx_size, tx_class); |
30 | 102k | if (qc == 0) { |
31 | 0 | *accu_rate += txb_costs->base_cost[coeff_ctx][0]; |
32 | 102k | } else { |
33 | 102k | const int sign = (qc < 0) ? 1 : 0; |
34 | 102k | const tran_low_t abs_qc = abs(qc); |
35 | 102k | const tran_low_t tqc = tcoeff[ci]; |
36 | 102k | const tran_low_t dqc = dqcoeff[ci]; |
37 | 102k | const int64_t dist = get_coeff_dist(tqc, dqc, shift); |
38 | 102k | const int64_t dist0 = get_coeff_dist(tqc, 0, shift); |
39 | 102k | const int rate = |
40 | 102k | get_coeff_cost_general(is_last, ci, abs_qc, sign, coeff_ctx, |
41 | 102k | dc_sign_ctx, txb_costs, bwl, tx_class, levels); |
42 | 102k | const int64_t rd = RDCOST(rdmult, rate, dist); |
43 | | |
44 | 102k | tran_low_t qc_low, dqc_low; |
45 | 102k | tran_low_t abs_qc_low; |
46 | 102k | int64_t dist_low, rd_low; |
47 | 102k | int rate_low; |
48 | 102k | if (abs_qc == 1) { |
49 | 0 | abs_qc_low = qc_low = dqc_low = 0; |
50 | 0 | dist_low = dist0; |
51 | 0 | rate_low = txb_costs->base_cost[coeff_ctx][0]; |
52 | 102k | } else { |
53 | 102k | get_qc_dqc_low(abs_qc, sign, dqv, shift, &qc_low, &dqc_low); |
54 | 102k | abs_qc_low = abs_qc - 1; |
55 | 102k | dist_low = get_coeff_dist(tqc, dqc_low, shift); |
56 | 102k | rate_low = |
57 | 102k | get_coeff_cost_general(is_last, ci, abs_qc_low, sign, coeff_ctx, |
58 | 102k | dc_sign_ctx, txb_costs, bwl, tx_class, levels); |
59 | 102k | } |
60 | | |
61 | 102k | rd_low = RDCOST(rdmult, rate_low, dist_low); |
62 | 102k | if (rd_low < rd) { |
63 | 2.06k | qcoeff[ci] = qc_low; |
64 | 2.06k | dqcoeff[ci] = dqc_low; |
65 | 2.06k | levels[get_padded_idx(ci, bwl)] = AOMMIN(abs_qc_low, INT8_MAX); |
66 | 2.06k | *accu_rate += rate_low; |
67 | 2.06k | *accu_dist += dist_low - dist0; |
68 | 100k | } else { |
69 | 100k | *accu_rate += rate; |
70 | 100k | *accu_dist += dist - dist0; |
71 | 100k | } |
72 | 102k | } |
73 | 102k | } |
74 | | |
75 | | static AOM_FORCE_INLINE void update_coeff_simple( |
76 | | int *accu_rate, int si, int eob, TX_SIZE tx_size, TX_CLASS tx_class, |
77 | | int bwl, int64_t rdmult, int shift, const int16_t *dequant, |
78 | | const int16_t *scan, const LV_MAP_COEFF_COST *txb_costs, |
79 | | const tran_low_t *tcoeff, tran_low_t *qcoeff, tran_low_t *dqcoeff, |
80 | 1.06M | uint8_t *levels, const qm_val_t *iqmatrix) { |
81 | 1.06M | const int dqv = get_dqv(dequant, scan[si], iqmatrix); |
82 | 1.06M | (void)eob; |
83 | | // this simple version assumes the coeff's scan_idx is not DC (scan_idx != 0) |
84 | | // and not the last (scan_idx != eob - 1) |
85 | 1.06M | assert(si != eob - 1); |
86 | 1.06M | assert(si > 0); |
87 | 1.06M | const int ci = scan[si]; |
88 | 1.06M | const tran_low_t qc = qcoeff[ci]; |
89 | 1.06M | const int coeff_ctx = |
90 | 1.06M | get_lower_levels_ctx(levels, ci, bwl, tx_size, tx_class); |
91 | 1.06M | if (qc == 0) { |
92 | 894k | *accu_rate += txb_costs->base_cost[coeff_ctx][0]; |
93 | 894k | } else { |
94 | 170k | const tran_low_t abs_qc = abs(qc); |
95 | 170k | const tran_low_t abs_tqc = abs(tcoeff[ci]); |
96 | 170k | const tran_low_t abs_dqc = abs(dqcoeff[ci]); |
97 | 170k | int rate_low = 0; |
98 | 170k | const int rate = get_two_coeff_cost_simple( |
99 | 170k | ci, abs_qc, coeff_ctx, txb_costs, bwl, tx_class, levels, &rate_low); |
100 | 170k | if (abs_dqc < abs_tqc) { |
101 | 85.3k | *accu_rate += rate; |
102 | 85.3k | return; |
103 | 85.3k | } |
104 | | |
105 | 84.7k | const int64_t dist = get_coeff_dist(abs_tqc, abs_dqc, shift); |
106 | 84.7k | const int64_t rd = RDCOST(rdmult, rate, dist); |
107 | | |
108 | 84.7k | const tran_low_t abs_qc_low = abs_qc - 1; |
109 | 84.7k | const tran_low_t abs_dqc_low = (abs_qc_low * dqv) >> shift; |
110 | 84.7k | const int64_t dist_low = get_coeff_dist(abs_tqc, abs_dqc_low, shift); |
111 | 84.7k | const int64_t rd_low = RDCOST(rdmult, rate_low, dist_low); |
112 | | |
113 | 84.7k | if (rd_low < rd) { |
114 | 8.50k | const int sign = (qc < 0) ? 1 : 0; |
115 | 8.50k | qcoeff[ci] = (-sign ^ abs_qc_low) + sign; |
116 | 8.50k | dqcoeff[ci] = (-sign ^ abs_dqc_low) + sign; |
117 | 8.50k | levels[get_padded_idx(ci, bwl)] = AOMMIN(abs_qc_low, INT8_MAX); |
118 | 8.50k | *accu_rate += rate_low; |
119 | 76.2k | } else { |
120 | 76.2k | *accu_rate += rate; |
121 | 76.2k | } |
122 | 84.7k | } |
123 | 1.06M | } |
124 | | |
125 | | static AOM_FORCE_INLINE void update_coeff_eob( |
126 | | int *accu_rate, int64_t *accu_dist, int *eob, int *nz_num, int *nz_ci, |
127 | | int si, TX_SIZE tx_size, TX_CLASS tx_class, int bwl, int height, |
128 | | int dc_sign_ctx, int64_t rdmult, int shift, const int16_t *dequant, |
129 | | const int16_t *scan, const LV_MAP_EOB_COST *txb_eob_costs, |
130 | | const LV_MAP_COEFF_COST *txb_costs, const tran_low_t *tcoeff, |
131 | | tran_low_t *qcoeff, tran_low_t *dqcoeff, uint8_t *levels, int sharpness, |
132 | 514k | const qm_val_t *iqmatrix) { |
133 | 514k | const int dqv = get_dqv(dequant, scan[si], iqmatrix); |
134 | 514k | assert(si != *eob - 1); |
135 | 514k | const int ci = scan[si]; |
136 | 514k | const tran_low_t qc = qcoeff[ci]; |
137 | 514k | const int coeff_ctx = |
138 | 514k | get_lower_levels_ctx(levels, ci, bwl, tx_size, tx_class); |
139 | 514k | if (qc == 0) { |
140 | 449k | *accu_rate += txb_costs->base_cost[coeff_ctx][0]; |
141 | 449k | } else { |
142 | 65.8k | int lower_level = 0; |
143 | 65.8k | const tran_low_t abs_qc = abs(qc); |
144 | 65.8k | const tran_low_t tqc = tcoeff[ci]; |
145 | 65.8k | const tran_low_t dqc = dqcoeff[ci]; |
146 | 65.8k | const int sign = (qc < 0) ? 1 : 0; |
147 | 65.8k | const int64_t dist0 = get_coeff_dist(tqc, 0, shift); |
148 | 65.8k | int64_t dist = get_coeff_dist(tqc, dqc, shift) - dist0; |
149 | 65.8k | int rate = |
150 | 65.8k | get_coeff_cost_general(0, ci, abs_qc, sign, coeff_ctx, dc_sign_ctx, |
151 | 65.8k | txb_costs, bwl, tx_class, levels); |
152 | 65.8k | int64_t rd = RDCOST(rdmult, *accu_rate + rate, *accu_dist + dist); |
153 | | |
154 | 65.8k | tran_low_t qc_low, dqc_low; |
155 | 65.8k | tran_low_t abs_qc_low; |
156 | 65.8k | int64_t dist_low, rd_low; |
157 | 65.8k | int rate_low; |
158 | | |
159 | 65.8k | if (abs_qc == 1) { |
160 | 26.0k | abs_qc_low = 0; |
161 | 26.0k | dqc_low = qc_low = 0; |
162 | 26.0k | dist_low = 0; |
163 | 26.0k | rate_low = txb_costs->base_cost[coeff_ctx][0]; |
164 | 26.0k | rd_low = RDCOST(rdmult, *accu_rate + rate_low, *accu_dist); |
165 | 39.8k | } else { |
166 | 39.8k | get_qc_dqc_low(abs_qc, sign, dqv, shift, &qc_low, &dqc_low); |
167 | 39.8k | abs_qc_low = abs_qc - 1; |
168 | 39.8k | dist_low = get_coeff_dist(tqc, dqc_low, shift) - dist0; |
169 | 39.8k | rate_low = |
170 | 39.8k | get_coeff_cost_general(0, ci, abs_qc_low, sign, coeff_ctx, |
171 | 39.8k | dc_sign_ctx, txb_costs, bwl, tx_class, levels); |
172 | 39.8k | rd_low = RDCOST(rdmult, *accu_rate + rate_low, *accu_dist + dist_low); |
173 | 39.8k | } |
174 | | |
175 | 65.8k | int lower_level_new_eob = 0; |
176 | 65.8k | const int new_eob = si + 1; |
177 | 65.8k | const int coeff_ctx_new_eob = get_lower_levels_ctx_eob(bwl, height, si); |
178 | 65.8k | const int new_eob_cost = |
179 | 65.8k | get_eob_cost(new_eob, txb_eob_costs, txb_costs, tx_class); |
180 | 65.8k | int rate_coeff_eob = |
181 | 65.8k | new_eob_cost + get_coeff_cost_eob(ci, abs_qc, sign, coeff_ctx_new_eob, |
182 | 65.8k | dc_sign_ctx, txb_costs, bwl, |
183 | 65.8k | tx_class); |
184 | 65.8k | int64_t dist_new_eob = dist; |
185 | 65.8k | int64_t rd_new_eob = RDCOST(rdmult, rate_coeff_eob, dist_new_eob); |
186 | | |
187 | 65.8k | if (abs_qc_low > 0) { |
188 | 39.8k | const int rate_coeff_eob_low = |
189 | 39.8k | new_eob_cost + get_coeff_cost_eob(ci, abs_qc_low, sign, |
190 | 39.8k | coeff_ctx_new_eob, dc_sign_ctx, |
191 | 39.8k | txb_costs, bwl, tx_class); |
192 | 39.8k | const int64_t dist_new_eob_low = dist_low; |
193 | 39.8k | const int64_t rd_new_eob_low = |
194 | 39.8k | RDCOST(rdmult, rate_coeff_eob_low, dist_new_eob_low); |
195 | 39.8k | if (rd_new_eob_low < rd_new_eob) { |
196 | 2.45k | lower_level_new_eob = 1; |
197 | 2.45k | rd_new_eob = rd_new_eob_low; |
198 | 2.45k | rate_coeff_eob = rate_coeff_eob_low; |
199 | 2.45k | dist_new_eob = dist_new_eob_low; |
200 | 2.45k | } |
201 | 39.8k | } |
202 | | |
203 | 65.8k | if (sharpness == 0 || abs_qc > 1) { |
204 | 65.8k | if (rd_low < rd) { |
205 | 2.84k | lower_level = 1; |
206 | 2.84k | rd = rd_low; |
207 | 2.84k | rate = rate_low; |
208 | 2.84k | dist = dist_low; |
209 | 2.84k | } |
210 | 65.8k | } |
211 | | |
212 | 65.8k | if (sharpness == 0 && rd_new_eob < rd) { |
213 | 11.8k | for (int ni = 0; ni < *nz_num; ++ni) { |
214 | 6.01k | int last_ci = nz_ci[ni]; |
215 | 6.01k | levels[get_padded_idx(last_ci, bwl)] = 0; |
216 | 6.01k | qcoeff[last_ci] = 0; |
217 | 6.01k | dqcoeff[last_ci] = 0; |
218 | 6.01k | } |
219 | 5.83k | *eob = new_eob; |
220 | 5.83k | *nz_num = 0; |
221 | 5.83k | *accu_rate = rate_coeff_eob; |
222 | 5.83k | *accu_dist = dist_new_eob; |
223 | 5.83k | lower_level = lower_level_new_eob; |
224 | 60.0k | } else { |
225 | 60.0k | *accu_rate += rate; |
226 | 60.0k | *accu_dist += dist; |
227 | 60.0k | } |
228 | | |
229 | 65.8k | if (lower_level) { |
230 | 1.55k | qcoeff[ci] = qc_low; |
231 | 1.55k | dqcoeff[ci] = dqc_low; |
232 | 1.55k | levels[get_padded_idx(ci, bwl)] = AOMMIN(abs_qc_low, INT8_MAX); |
233 | 1.55k | } |
234 | 65.8k | if (qcoeff[ci]) { |
235 | 65.5k | nz_ci[*nz_num] = ci; |
236 | 65.5k | ++*nz_num; |
237 | 65.5k | } |
238 | 65.8k | } |
239 | 514k | } |
240 | | |
241 | | static INLINE void update_skip(int *accu_rate, int64_t accu_dist, int *eob, |
242 | | int nz_num, int *nz_ci, int64_t rdmult, |
243 | | int skip_cost, int non_skip_cost, |
244 | 172k | tran_low_t *qcoeff, tran_low_t *dqcoeff) { |
245 | 172k | const int64_t rd = RDCOST(rdmult, *accu_rate + non_skip_cost, accu_dist); |
246 | 172k | const int64_t rd_new_eob = RDCOST(rdmult, skip_cost, 0); |
247 | 172k | if (rd_new_eob < rd) { |
248 | 15.7k | for (int i = 0; i < nz_num; ++i) { |
249 | 7.88k | const int ci = nz_ci[i]; |
250 | 7.88k | qcoeff[ci] = 0; |
251 | 7.88k | dqcoeff[ci] = 0; |
252 | | // no need to set up levels because this is the last step |
253 | | // levels[get_padded_idx(ci, bwl)] = 0; |
254 | 7.88k | } |
255 | 7.88k | *accu_rate = 0; |
256 | 7.88k | *eob = 0; |
257 | 7.88k | } |
258 | 172k | } |
259 | | |
260 | | // TODO(angiebird): use this function whenever it's possible |
261 | | static int get_tx_type_cost(const MACROBLOCK *x, const MACROBLOCKD *xd, |
262 | | int plane, TX_SIZE tx_size, TX_TYPE tx_type, |
263 | 444k | int reduced_tx_set_used) { |
264 | 444k | if (plane > 0) return 0; |
265 | | |
266 | 345k | const TX_SIZE square_tx_size = txsize_sqr_map[tx_size]; |
267 | | |
268 | 345k | const MB_MODE_INFO *mbmi = xd->mi[0]; |
269 | 345k | const int is_inter = is_inter_block(mbmi); |
270 | 345k | if (get_ext_tx_types(tx_size, is_inter, reduced_tx_set_used) > 1 && |
271 | 345k | !xd->lossless[xd->mi[0]->segment_id]) { |
272 | 178k | const int ext_tx_set = |
273 | 178k | get_ext_tx_set(tx_size, is_inter, reduced_tx_set_used); |
274 | 178k | if (is_inter) { |
275 | 0 | if (ext_tx_set > 0) |
276 | 0 | return x->mode_costs |
277 | 0 | .inter_tx_type_costs[ext_tx_set][square_tx_size][tx_type]; |
278 | 178k | } else { |
279 | 178k | if (ext_tx_set > 0) { |
280 | 178k | PREDICTION_MODE intra_dir; |
281 | 178k | if (mbmi->filter_intra_mode_info.use_filter_intra) |
282 | 38.9k | intra_dir = fimode_to_intradir[mbmi->filter_intra_mode_info |
283 | 38.9k | .filter_intra_mode]; |
284 | 139k | else |
285 | 139k | intra_dir = mbmi->mode; |
286 | 178k | return x->mode_costs.intra_tx_type_costs[ext_tx_set][square_tx_size] |
287 | 178k | [intra_dir][tx_type]; |
288 | 178k | } |
289 | 178k | } |
290 | 178k | } |
291 | 167k | return 0; |
292 | 345k | } |
293 | | |
294 | | int av1_optimize_txb(const struct AV1_COMP *cpi, MACROBLOCK *x, int plane, |
295 | | int block, TX_SIZE tx_size, TX_TYPE tx_type, |
296 | | const TXB_CTX *const txb_ctx, int *rate_cost, |
297 | 200k | int sharpness) { |
298 | 200k | MACROBLOCKD *xd = &x->e_mbd; |
299 | 200k | const struct macroblock_plane *p = &x->plane[plane]; |
300 | 200k | const SCAN_ORDER *scan_order = get_scan(tx_size, tx_type); |
301 | 200k | const int16_t *scan = scan_order->scan; |
302 | 200k | const int shift = av1_get_tx_scale(tx_size); |
303 | 200k | int eob = p->eobs[block]; |
304 | 200k | const int16_t *dequant = p->dequant_QTX; |
305 | 200k | const qm_val_t *iqmatrix = |
306 | 200k | av1_get_iqmatrix(&cpi->common.quant_params, xd, plane, tx_size, tx_type); |
307 | 200k | const int block_offset = BLOCK_OFFSET(block); |
308 | 200k | tran_low_t *qcoeff = p->qcoeff + block_offset; |
309 | 200k | tran_low_t *dqcoeff = p->dqcoeff + block_offset; |
310 | 200k | const tran_low_t *tcoeff = p->coeff + block_offset; |
311 | 200k | const CoeffCosts *coeff_costs = &x->coeff_costs; |
312 | | |
313 | | // This function is not called if eob = 0. |
314 | 200k | assert(eob > 0); |
315 | | |
316 | 200k | const AV1_COMMON *cm = &cpi->common; |
317 | 200k | const PLANE_TYPE plane_type = get_plane_type(plane); |
318 | 200k | const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size); |
319 | 200k | const TX_CLASS tx_class = tx_type_to_class[tx_type]; |
320 | 200k | const MB_MODE_INFO *mbmi = xd->mi[0]; |
321 | 200k | const int bwl = get_txb_bwl(tx_size); |
322 | 200k | const int width = get_txb_wide(tx_size); |
323 | 200k | const int height = get_txb_high(tx_size); |
324 | 200k | assert(width == (1 << bwl)); |
325 | 200k | const int is_inter = is_inter_block(mbmi); |
326 | 200k | const LV_MAP_COEFF_COST *txb_costs = |
327 | 200k | &coeff_costs->coeff_costs[txs_ctx][plane_type]; |
328 | 200k | const int eob_multi_size = txsize_log2_minus4[tx_size]; |
329 | 200k | const LV_MAP_EOB_COST *txb_eob_costs = |
330 | 200k | &coeff_costs->eob_costs[eob_multi_size][plane_type]; |
331 | | |
332 | 200k | const int rshift = 2; |
333 | | |
334 | 200k | const int64_t rdmult = |
335 | 200k | (((int64_t)x->rdmult * |
336 | 200k | (plane_rd_mult[is_inter][plane_type] << (2 * (xd->bd - 8)))) + |
337 | 200k | 2) >> |
338 | 200k | rshift; |
339 | | |
340 | 200k | uint8_t levels_buf[TX_PAD_2D]; |
341 | 200k | uint8_t *const levels = set_levels(levels_buf, width); |
342 | | |
343 | 200k | if (eob > 1) av1_txb_init_levels(qcoeff, width, height, levels); |
344 | | |
345 | | // TODO(angirbird): check iqmatrix |
346 | | |
347 | 200k | const int non_skip_cost = txb_costs->txb_skip_cost[txb_ctx->txb_skip_ctx][0]; |
348 | 200k | const int skip_cost = txb_costs->txb_skip_cost[txb_ctx->txb_skip_ctx][1]; |
349 | 200k | const int eob_cost = get_eob_cost(eob, txb_eob_costs, txb_costs, tx_class); |
350 | 200k | int accu_rate = eob_cost; |
351 | 200k | int64_t accu_dist = 0; |
352 | 200k | int si = eob - 1; |
353 | 200k | const int ci = scan[si]; |
354 | 200k | const tran_low_t qc = qcoeff[ci]; |
355 | 200k | const tran_low_t abs_qc = abs(qc); |
356 | 200k | const int sign = qc < 0; |
357 | 200k | const int max_nz_num = 2; |
358 | 200k | int nz_num = 1; |
359 | 200k | int nz_ci[3] = { ci, 0, 0 }; |
360 | 200k | if (abs_qc >= 2) { |
361 | 76.8k | update_coeff_general(&accu_rate, &accu_dist, si, eob, tx_size, tx_class, |
362 | 76.8k | bwl, height, rdmult, shift, txb_ctx->dc_sign_ctx, |
363 | 76.8k | dequant, scan, txb_costs, tcoeff, qcoeff, dqcoeff, |
364 | 76.8k | levels, iqmatrix); |
365 | 76.8k | --si; |
366 | 123k | } else { |
367 | 123k | assert(abs_qc == 1); |
368 | 123k | const int coeff_ctx = get_lower_levels_ctx_eob(bwl, height, si); |
369 | 123k | accu_rate += |
370 | 123k | get_coeff_cost_eob(ci, abs_qc, sign, coeff_ctx, txb_ctx->dc_sign_ctx, |
371 | 123k | txb_costs, bwl, tx_class); |
372 | 123k | const tran_low_t tqc = tcoeff[ci]; |
373 | 123k | const tran_low_t dqc = dqcoeff[ci]; |
374 | 123k | const int64_t dist = get_coeff_dist(tqc, dqc, shift); |
375 | 123k | const int64_t dist0 = get_coeff_dist(tqc, 0, shift); |
376 | 123k | accu_dist += dist - dist0; |
377 | 123k | --si; |
378 | 123k | } |
379 | | |
380 | 200k | #define UPDATE_COEFF_EOB_CASE(tx_class_literal) \ |
381 | 200k | case tx_class_literal: \ |
382 | 715k | for (; si >= 0 && nz_num <= max_nz_num; --si) { \ |
383 | 514k | update_coeff_eob(&accu_rate, &accu_dist, &eob, &nz_num, nz_ci, si, \ |
384 | 514k | tx_size, tx_class_literal, bwl, height, \ |
385 | 514k | txb_ctx->dc_sign_ctx, rdmult, shift, dequant, scan, \ |
386 | 514k | txb_eob_costs, txb_costs, tcoeff, qcoeff, dqcoeff, \ |
387 | 514k | levels, sharpness, iqmatrix); \ |
388 | 514k | } \ |
389 | 200k | break; |
390 | 200k | switch (tx_class) { |
391 | 199k | UPDATE_COEFF_EOB_CASE(TX_CLASS_2D); |
392 | 1.45k | UPDATE_COEFF_EOB_CASE(TX_CLASS_HORIZ); |
393 | 29 | UPDATE_COEFF_EOB_CASE(TX_CLASS_VERT); |
394 | 0 | #undef UPDATE_COEFF_EOB_CASE |
395 | 0 | default: assert(false); |
396 | 200k | } |
397 | | |
398 | 200k | if (si == -1 && nz_num <= max_nz_num && sharpness == 0) { |
399 | 172k | update_skip(&accu_rate, accu_dist, &eob, nz_num, nz_ci, rdmult, skip_cost, |
400 | 172k | non_skip_cost, qcoeff, dqcoeff); |
401 | 172k | } |
402 | | |
403 | 200k | #define UPDATE_COEFF_SIMPLE_CASE(tx_class_literal) \ |
404 | 200k | case tx_class_literal: \ |
405 | 1.26M | for (; si >= 1; --si) { \ |
406 | 1.06M | update_coeff_simple(&accu_rate, si, eob, tx_size, tx_class_literal, bwl, \ |
407 | 1.06M | rdmult, shift, dequant, scan, txb_costs, tcoeff, \ |
408 | 1.06M | qcoeff, dqcoeff, levels, iqmatrix); \ |
409 | 1.06M | } \ |
410 | 200k | break; |
411 | 200k | switch (tx_class) { |
412 | 199k | UPDATE_COEFF_SIMPLE_CASE(TX_CLASS_2D); |
413 | 1.45k | UPDATE_COEFF_SIMPLE_CASE(TX_CLASS_HORIZ); |
414 | 29 | UPDATE_COEFF_SIMPLE_CASE(TX_CLASS_VERT); |
415 | 0 | #undef UPDATE_COEFF_SIMPLE_CASE |
416 | 0 | default: assert(false); |
417 | 200k | } |
418 | | |
419 | | // DC position |
420 | 200k | if (si == 0) { |
421 | | // no need to update accu_dist because it's not used after this point |
422 | 25.3k | int64_t dummy_dist = 0; |
423 | 25.3k | update_coeff_general(&accu_rate, &dummy_dist, si, eob, tx_size, tx_class, |
424 | 25.3k | bwl, height, rdmult, shift, txb_ctx->dc_sign_ctx, |
425 | 25.3k | dequant, scan, txb_costs, tcoeff, qcoeff, dqcoeff, |
426 | 25.3k | levels, iqmatrix); |
427 | 25.3k | } |
428 | | |
429 | 200k | const int tx_type_cost = get_tx_type_cost(x, xd, plane, tx_size, tx_type, |
430 | 200k | cm->features.reduced_tx_set_used); |
431 | 200k | if (eob == 0) |
432 | 7.88k | accu_rate += skip_cost; |
433 | 192k | else |
434 | 192k | accu_rate += non_skip_cost + tx_type_cost; |
435 | | |
436 | 200k | p->eobs[block] = eob; |
437 | 200k | p->txb_entropy_ctx[block] = |
438 | 200k | av1_get_txb_entropy_context(qcoeff, scan_order, p->eobs[block]); |
439 | | |
440 | 200k | *rate_cost = accu_rate; |
441 | 200k | return eob; |
442 | 200k | } |
443 | | |
444 | | static AOM_FORCE_INLINE int warehouse_efficients_txb( |
445 | | const MACROBLOCK *x, const int plane, const int block, |
446 | | const TX_SIZE tx_size, const TXB_CTX *const txb_ctx, |
447 | | const struct macroblock_plane *p, const int eob, |
448 | | const PLANE_TYPE plane_type, const LV_MAP_COEFF_COST *const coeff_costs, |
449 | | const MACROBLOCKD *const xd, const TX_TYPE tx_type, const TX_CLASS tx_class, |
450 | 153k | int reduced_tx_set_used) { |
451 | 153k | const tran_low_t *const qcoeff = p->qcoeff + BLOCK_OFFSET(block); |
452 | 153k | const int txb_skip_ctx = txb_ctx->txb_skip_ctx; |
453 | 153k | const int bwl = get_txb_bwl(tx_size); |
454 | 153k | const int width = get_txb_wide(tx_size); |
455 | 153k | const int height = get_txb_high(tx_size); |
456 | 153k | const SCAN_ORDER *const scan_order = get_scan(tx_size, tx_type); |
457 | 153k | const int16_t *const scan = scan_order->scan; |
458 | 153k | uint8_t levels_buf[TX_PAD_2D]; |
459 | 153k | uint8_t *const levels = set_levels(levels_buf, width); |
460 | 153k | DECLARE_ALIGNED(16, int8_t, coeff_contexts[MAX_TX_SQUARE]); |
461 | 153k | const int eob_multi_size = txsize_log2_minus4[tx_size]; |
462 | 153k | const LV_MAP_EOB_COST *const eob_costs = |
463 | 153k | &x->coeff_costs.eob_costs[eob_multi_size][plane_type]; |
464 | 153k | int cost = coeff_costs->txb_skip_cost[txb_skip_ctx][0]; |
465 | | |
466 | 153k | av1_txb_init_levels(qcoeff, width, height, levels); |
467 | | |
468 | 153k | cost += get_tx_type_cost(x, xd, plane, tx_size, tx_type, reduced_tx_set_used); |
469 | | |
470 | 153k | cost += get_eob_cost(eob, eob_costs, coeff_costs, tx_class); |
471 | | |
472 | 153k | av1_get_nz_map_contexts(levels, scan, eob, tx_size, tx_class, coeff_contexts); |
473 | | |
474 | 153k | const int(*lps_cost)[COEFF_BASE_RANGE + 1 + COEFF_BASE_RANGE + 1] = |
475 | 153k | coeff_costs->lps_cost; |
476 | 153k | int c = eob - 1; |
477 | 153k | { |
478 | 153k | const int pos = scan[c]; |
479 | 153k | const tran_low_t v = qcoeff[pos]; |
480 | 153k | const int sign = AOMSIGN(v); |
481 | 153k | const int level = (v ^ sign) - sign; |
482 | 153k | const int coeff_ctx = coeff_contexts[pos]; |
483 | 153k | cost += coeff_costs->base_eob_cost[coeff_ctx][AOMMIN(level, 3) - 1]; |
484 | | |
485 | 153k | if (v) { |
486 | | // sign bit cost |
487 | 153k | if (level > NUM_BASE_LEVELS) { |
488 | 137k | const int ctx = get_br_ctx_eob(pos, bwl, tx_class); |
489 | 137k | cost += get_br_cost(level, lps_cost[ctx]); |
490 | 137k | } |
491 | 153k | if (c) { |
492 | 22.0k | cost += av1_cost_literal(1); |
493 | 131k | } else { |
494 | 131k | const int sign01 = (sign ^ sign) - sign; |
495 | 131k | const int dc_sign_ctx = txb_ctx->dc_sign_ctx; |
496 | 131k | cost += coeff_costs->dc_sign_cost[dc_sign_ctx][sign01]; |
497 | 131k | return cost; |
498 | 131k | } |
499 | 153k | } |
500 | 153k | } |
501 | 22.0k | const int(*base_cost)[8] = coeff_costs->base_cost; |
502 | 1.17M | for (c = eob - 2; c >= 1; --c) { |
503 | 1.15M | const int pos = scan[c]; |
504 | 1.15M | const int coeff_ctx = coeff_contexts[pos]; |
505 | 1.15M | const tran_low_t v = qcoeff[pos]; |
506 | 1.15M | const int level = abs(v); |
507 | 1.15M | cost += base_cost[coeff_ctx][AOMMIN(level, 3)]; |
508 | 1.15M | if (v) { |
509 | | // sign bit cost |
510 | 350k | cost += av1_cost_literal(1); |
511 | 350k | if (level > NUM_BASE_LEVELS) { |
512 | 184k | const int ctx = get_br_ctx(levels, pos, bwl, tx_class); |
513 | 184k | cost += get_br_cost(level, lps_cost[ctx]); |
514 | 184k | } |
515 | 350k | } |
516 | 1.15M | } |
517 | | // c == 0 after previous loop |
518 | 22.0k | { |
519 | 22.0k | const int pos = scan[c]; |
520 | 22.0k | const tran_low_t v = qcoeff[pos]; |
521 | 22.0k | const int coeff_ctx = coeff_contexts[pos]; |
522 | 22.0k | const int sign = AOMSIGN(v); |
523 | 22.0k | const int level = (v ^ sign) - sign; |
524 | 22.0k | cost += base_cost[coeff_ctx][AOMMIN(level, 3)]; |
525 | | |
526 | 22.0k | if (v) { |
527 | | // sign bit cost |
528 | 22.0k | const int sign01 = (sign ^ sign) - sign; |
529 | 22.0k | const int dc_sign_ctx = txb_ctx->dc_sign_ctx; |
530 | 22.0k | cost += coeff_costs->dc_sign_cost[dc_sign_ctx][sign01]; |
531 | 22.0k | if (level > NUM_BASE_LEVELS) { |
532 | 22.0k | const int ctx = get_br_ctx(levels, pos, bwl, tx_class); |
533 | 22.0k | cost += get_br_cost(level, lps_cost[ctx]); |
534 | 22.0k | } |
535 | 22.0k | } |
536 | 22.0k | } |
537 | 22.0k | return cost; |
538 | 153k | } |
539 | | |
540 | | int av1_cost_coeffs_txb_estimate(const MACROBLOCK *x, const int plane, |
541 | | const int block, const TX_SIZE tx_size, |
542 | 89.8k | const TX_TYPE tx_type) { |
543 | 89.8k | assert(plane == 0); |
544 | | |
545 | 89.8k | int cost = 0; |
546 | 89.8k | const struct macroblock_plane *p = &x->plane[plane]; |
547 | 89.8k | const SCAN_ORDER *scan_order = get_scan(tx_size, tx_type); |
548 | 89.8k | const int16_t *scan = scan_order->scan; |
549 | 89.8k | tran_low_t *qcoeff = p->qcoeff + BLOCK_OFFSET(block); |
550 | | |
551 | 89.8k | int eob = p->eobs[block]; |
552 | | |
553 | | // coeffs |
554 | 89.8k | int c = eob - 1; |
555 | | // eob |
556 | 89.8k | { |
557 | 89.8k | const int pos = scan[c]; |
558 | 89.8k | const tran_low_t v = abs(qcoeff[pos]) - 1; |
559 | 89.8k | cost += (v << (AV1_PROB_COST_SHIFT + 2)); |
560 | 89.8k | } |
561 | | // other coeffs |
562 | 4.15M | for (c = eob - 2; c >= 0; c--) { |
563 | 4.06M | const int pos = scan[c]; |
564 | 4.06M | const tran_low_t v = abs(qcoeff[pos]); |
565 | 4.06M | const int idx = AOMMIN(v, 14); |
566 | | |
567 | 4.06M | cost += costLUT[idx]; |
568 | 4.06M | } |
569 | | |
570 | | // const_term does not contain DC, and log(e) does not contain eob, so both |
571 | | // (eob-1) |
572 | 89.8k | cost += (const_term + loge_par) * (eob - 1); |
573 | | |
574 | 89.8k | return cost; |
575 | 89.8k | } |
576 | | |
577 | | static AOM_FORCE_INLINE int warehouse_efficients_txb_laplacian( |
578 | | const MACROBLOCK *x, const int plane, const int block, |
579 | | const TX_SIZE tx_size, const TXB_CTX *const txb_ctx, const int eob, |
580 | | const PLANE_TYPE plane_type, const LV_MAP_COEFF_COST *const coeff_costs, |
581 | | const MACROBLOCKD *const xd, const TX_TYPE tx_type, const TX_CLASS tx_class, |
582 | 89.8k | int reduced_tx_set_used) { |
583 | 89.8k | const int txb_skip_ctx = txb_ctx->txb_skip_ctx; |
584 | | |
585 | 89.8k | const int eob_multi_size = txsize_log2_minus4[tx_size]; |
586 | 89.8k | const LV_MAP_EOB_COST *const eob_costs = |
587 | 89.8k | &x->coeff_costs.eob_costs[eob_multi_size][plane_type]; |
588 | 89.8k | int cost = coeff_costs->txb_skip_cost[txb_skip_ctx][0]; |
589 | | |
590 | 89.8k | cost += get_tx_type_cost(x, xd, plane, tx_size, tx_type, reduced_tx_set_used); |
591 | | |
592 | 89.8k | cost += get_eob_cost(eob, eob_costs, coeff_costs, tx_class); |
593 | | |
594 | 89.8k | cost += av1_cost_coeffs_txb_estimate(x, plane, block, tx_size, tx_type); |
595 | 89.8k | return cost; |
596 | 89.8k | } |
597 | | |
598 | | int av1_cost_coeffs_txb(const MACROBLOCK *x, const int plane, const int block, |
599 | | const TX_SIZE tx_size, const TX_TYPE tx_type, |
600 | 41.2M | const TXB_CTX *const txb_ctx, int reduced_tx_set_used) { |
601 | 41.2M | const struct macroblock_plane *p = &x->plane[plane]; |
602 | 41.2M | const int eob = p->eobs[block]; |
603 | 41.2M | const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size); |
604 | 41.2M | const PLANE_TYPE plane_type = get_plane_type(plane); |
605 | 41.2M | const LV_MAP_COEFF_COST *const coeff_costs = |
606 | 41.2M | &x->coeff_costs.coeff_costs[txs_ctx][plane_type]; |
607 | 41.2M | if (eob == 0) { |
608 | 41.0M | return coeff_costs->txb_skip_cost[txb_ctx->txb_skip_ctx][1]; |
609 | 41.0M | } |
610 | | |
611 | 161k | const MACROBLOCKD *const xd = &x->e_mbd; |
612 | 161k | const TX_CLASS tx_class = tx_type_to_class[tx_type]; |
613 | | |
614 | 161k | return warehouse_efficients_txb(x, plane, block, tx_size, txb_ctx, p, eob, |
615 | 161k | plane_type, coeff_costs, xd, tx_type, |
616 | 161k | tx_class, reduced_tx_set_used); |
617 | 41.2M | } |
618 | | |
619 | | int av1_cost_coeffs_txb_laplacian(const MACROBLOCK *x, const int plane, |
620 | | const int block, const TX_SIZE tx_size, |
621 | | const TX_TYPE tx_type, |
622 | | const TXB_CTX *const txb_ctx, |
623 | | const int reduced_tx_set_used, |
624 | 1.40M | const int adjust_eob) { |
625 | 1.40M | const struct macroblock_plane *p = &x->plane[plane]; |
626 | 1.40M | int eob = p->eobs[block]; |
627 | | |
628 | 1.40M | if (adjust_eob) { |
629 | 0 | const SCAN_ORDER *scan_order = get_scan(tx_size, tx_type); |
630 | 0 | const int16_t *scan = scan_order->scan; |
631 | 0 | tran_low_t *tcoeff = p->coeff + BLOCK_OFFSET(block); |
632 | 0 | tran_low_t *qcoeff = p->qcoeff + BLOCK_OFFSET(block); |
633 | 0 | tran_low_t *dqcoeff = p->dqcoeff + BLOCK_OFFSET(block); |
634 | 0 | update_coeff_eob_fast(&eob, av1_get_tx_scale(tx_size), p->dequant_QTX, scan, |
635 | 0 | tcoeff, qcoeff, dqcoeff); |
636 | 0 | p->eobs[block] = eob; |
637 | 0 | } |
638 | | |
639 | 1.40M | const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size); |
640 | 1.40M | const PLANE_TYPE plane_type = get_plane_type(plane); |
641 | 1.40M | const LV_MAP_COEFF_COST *const coeff_costs = |
642 | 1.40M | &x->coeff_costs.coeff_costs[txs_ctx][plane_type]; |
643 | 1.40M | if (eob == 0) { |
644 | 1.31M | return coeff_costs->txb_skip_cost[txb_ctx->txb_skip_ctx][1]; |
645 | 1.31M | } |
646 | | |
647 | 89.9k | const MACROBLOCKD *const xd = &x->e_mbd; |
648 | 89.9k | const TX_CLASS tx_class = tx_type_to_class[tx_type]; |
649 | | |
650 | 89.9k | return warehouse_efficients_txb_laplacian( |
651 | 89.9k | x, plane, block, tx_size, txb_ctx, eob, plane_type, coeff_costs, xd, |
652 | 89.9k | tx_type, tx_class, reduced_tx_set_used); |
653 | 1.40M | } |