/src/aom/av1/encoder/txb_rdopt.c
Line | Count | Source (jump to first uncovered line) |
1 | | /* |
2 | | * Copyright (c) 2021, Alliance for Open Media. All rights reserved |
3 | | * |
4 | | * This source code is subject to the terms of the BSD 2 Clause License and |
5 | | * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License |
6 | | * was not distributed with this source code in the LICENSE file, you can |
7 | | * obtain it at www.aomedia.org/license/software. If the Alliance for Open |
8 | | * Media Patent License 1.0 was not distributed with this source code in the |
9 | | * PATENTS file, you can obtain it at www.aomedia.org/license/patent. |
10 | | */ |
11 | | |
12 | | #include "av1/encoder/txb_rdopt.h" |
13 | | #include "av1/encoder/txb_rdopt_utils.h" |
14 | | |
15 | | #include "av1/common/idct.h" |
16 | | |
17 | | static INLINE void update_coeff_general( |
18 | | int *accu_rate, int64_t *accu_dist, int si, int eob, TX_SIZE tx_size, |
19 | | TX_CLASS tx_class, int bwl, int height, int64_t rdmult, int shift, |
20 | | int dc_sign_ctx, const int16_t *dequant, const int16_t *scan, |
21 | | const LV_MAP_COEFF_COST *txb_costs, const tran_low_t *tcoeff, |
22 | | tran_low_t *qcoeff, tran_low_t *dqcoeff, uint8_t *levels, |
23 | 0 | const qm_val_t *iqmatrix) { |
24 | 0 | const int dqv = get_dqv(dequant, scan[si], iqmatrix); |
25 | 0 | const int ci = scan[si]; |
26 | 0 | const tran_low_t qc = qcoeff[ci]; |
27 | 0 | const int is_last = si == (eob - 1); |
28 | 0 | const int coeff_ctx = get_lower_levels_ctx_general( |
29 | 0 | is_last, si, bwl, height, levels, ci, tx_size, tx_class); |
30 | 0 | if (qc == 0) { |
31 | 0 | *accu_rate += txb_costs->base_cost[coeff_ctx][0]; |
32 | 0 | } else { |
33 | 0 | const int sign = (qc < 0) ? 1 : 0; |
34 | 0 | const tran_low_t abs_qc = abs(qc); |
35 | 0 | const tran_low_t tqc = tcoeff[ci]; |
36 | 0 | const tran_low_t dqc = dqcoeff[ci]; |
37 | 0 | const int64_t dist = get_coeff_dist(tqc, dqc, shift); |
38 | 0 | const int64_t dist0 = get_coeff_dist(tqc, 0, shift); |
39 | 0 | const int rate = |
40 | 0 | get_coeff_cost_general(is_last, ci, abs_qc, sign, coeff_ctx, |
41 | 0 | dc_sign_ctx, txb_costs, bwl, tx_class, levels); |
42 | 0 | const int64_t rd = RDCOST(rdmult, rate, dist); |
43 | |
|
44 | 0 | tran_low_t qc_low, dqc_low; |
45 | 0 | tran_low_t abs_qc_low; |
46 | 0 | int64_t dist_low, rd_low; |
47 | 0 | int rate_low; |
48 | 0 | if (abs_qc == 1) { |
49 | 0 | abs_qc_low = qc_low = dqc_low = 0; |
50 | 0 | dist_low = dist0; |
51 | 0 | rate_low = txb_costs->base_cost[coeff_ctx][0]; |
52 | 0 | } else { |
53 | 0 | get_qc_dqc_low(abs_qc, sign, dqv, shift, &qc_low, &dqc_low); |
54 | 0 | abs_qc_low = abs_qc - 1; |
55 | 0 | dist_low = get_coeff_dist(tqc, dqc_low, shift); |
56 | 0 | rate_low = |
57 | 0 | get_coeff_cost_general(is_last, ci, abs_qc_low, sign, coeff_ctx, |
58 | 0 | dc_sign_ctx, txb_costs, bwl, tx_class, levels); |
59 | 0 | } |
60 | |
|
61 | 0 | rd_low = RDCOST(rdmult, rate_low, dist_low); |
62 | 0 | if (rd_low < rd) { |
63 | 0 | qcoeff[ci] = qc_low; |
64 | 0 | dqcoeff[ci] = dqc_low; |
65 | 0 | levels[get_padded_idx(ci, bwl)] = AOMMIN(abs_qc_low, INT8_MAX); |
66 | 0 | *accu_rate += rate_low; |
67 | 0 | *accu_dist += dist_low - dist0; |
68 | 0 | } else { |
69 | 0 | *accu_rate += rate; |
70 | 0 | *accu_dist += dist - dist0; |
71 | 0 | } |
72 | 0 | } |
73 | 0 | } |
74 | | |
75 | | static AOM_FORCE_INLINE void update_coeff_simple( |
76 | | int *accu_rate, int si, int eob, TX_SIZE tx_size, TX_CLASS tx_class, |
77 | | int bwl, int64_t rdmult, int shift, const int16_t *dequant, |
78 | | const int16_t *scan, const LV_MAP_COEFF_COST *txb_costs, |
79 | | const tran_low_t *tcoeff, tran_low_t *qcoeff, tran_low_t *dqcoeff, |
80 | 0 | uint8_t *levels, const qm_val_t *iqmatrix) { |
81 | 0 | const int dqv = get_dqv(dequant, scan[si], iqmatrix); |
82 | 0 | (void)eob; |
83 | | // this simple version assumes the coeff's scan_idx is not DC (scan_idx != 0) |
84 | | // and not the last (scan_idx != eob - 1) |
85 | 0 | assert(si != eob - 1); |
86 | 0 | assert(si > 0); |
87 | 0 | const int ci = scan[si]; |
88 | 0 | const tran_low_t qc = qcoeff[ci]; |
89 | 0 | const int coeff_ctx = |
90 | 0 | get_lower_levels_ctx(levels, ci, bwl, tx_size, tx_class); |
91 | 0 | if (qc == 0) { |
92 | 0 | *accu_rate += txb_costs->base_cost[coeff_ctx][0]; |
93 | 0 | } else { |
94 | 0 | const tran_low_t abs_qc = abs(qc); |
95 | 0 | const tran_low_t abs_tqc = abs(tcoeff[ci]); |
96 | 0 | const tran_low_t abs_dqc = abs(dqcoeff[ci]); |
97 | 0 | int rate_low = 0; |
98 | 0 | const int rate = get_two_coeff_cost_simple( |
99 | 0 | ci, abs_qc, coeff_ctx, txb_costs, bwl, tx_class, levels, &rate_low); |
100 | 0 | if (abs_dqc < abs_tqc) { |
101 | 0 | *accu_rate += rate; |
102 | 0 | return; |
103 | 0 | } |
104 | | |
105 | 0 | const int64_t dist = get_coeff_dist(abs_tqc, abs_dqc, shift); |
106 | 0 | const int64_t rd = RDCOST(rdmult, rate, dist); |
107 | |
|
108 | 0 | const tran_low_t abs_qc_low = abs_qc - 1; |
109 | 0 | const tran_low_t abs_dqc_low = (abs_qc_low * dqv) >> shift; |
110 | 0 | const int64_t dist_low = get_coeff_dist(abs_tqc, abs_dqc_low, shift); |
111 | 0 | const int64_t rd_low = RDCOST(rdmult, rate_low, dist_low); |
112 | |
|
113 | 0 | if (rd_low < rd) { |
114 | 0 | const int sign = (qc < 0) ? 1 : 0; |
115 | 0 | qcoeff[ci] = (-sign ^ abs_qc_low) + sign; |
116 | 0 | dqcoeff[ci] = (-sign ^ abs_dqc_low) + sign; |
117 | 0 | levels[get_padded_idx(ci, bwl)] = AOMMIN(abs_qc_low, INT8_MAX); |
118 | 0 | *accu_rate += rate_low; |
119 | 0 | } else { |
120 | 0 | *accu_rate += rate; |
121 | 0 | } |
122 | 0 | } |
123 | 0 | } |
124 | | |
125 | | static AOM_FORCE_INLINE void update_coeff_eob( |
126 | | int *accu_rate, int64_t *accu_dist, int *eob, int *nz_num, int *nz_ci, |
127 | | int si, TX_SIZE tx_size, TX_CLASS tx_class, int bwl, int height, |
128 | | int dc_sign_ctx, int64_t rdmult, int shift, const int16_t *dequant, |
129 | | const int16_t *scan, const LV_MAP_EOB_COST *txb_eob_costs, |
130 | | const LV_MAP_COEFF_COST *txb_costs, const tran_low_t *tcoeff, |
131 | | tran_low_t *qcoeff, tran_low_t *dqcoeff, uint8_t *levels, int sharpness, |
132 | 0 | const qm_val_t *iqmatrix) { |
133 | 0 | const int dqv = get_dqv(dequant, scan[si], iqmatrix); |
134 | 0 | assert(si != *eob - 1); |
135 | 0 | const int ci = scan[si]; |
136 | 0 | const tran_low_t qc = qcoeff[ci]; |
137 | 0 | const int coeff_ctx = |
138 | 0 | get_lower_levels_ctx(levels, ci, bwl, tx_size, tx_class); |
139 | 0 | if (qc == 0) { |
140 | 0 | *accu_rate += txb_costs->base_cost[coeff_ctx][0]; |
141 | 0 | } else { |
142 | 0 | int lower_level = 0; |
143 | 0 | const tran_low_t abs_qc = abs(qc); |
144 | 0 | const tran_low_t tqc = tcoeff[ci]; |
145 | 0 | const tran_low_t dqc = dqcoeff[ci]; |
146 | 0 | const int sign = (qc < 0) ? 1 : 0; |
147 | 0 | const int64_t dist0 = get_coeff_dist(tqc, 0, shift); |
148 | 0 | int64_t dist = get_coeff_dist(tqc, dqc, shift) - dist0; |
149 | 0 | int rate = |
150 | 0 | get_coeff_cost_general(0, ci, abs_qc, sign, coeff_ctx, dc_sign_ctx, |
151 | 0 | txb_costs, bwl, tx_class, levels); |
152 | 0 | int64_t rd = RDCOST(rdmult, *accu_rate + rate, *accu_dist + dist); |
153 | |
|
154 | 0 | tran_low_t qc_low, dqc_low; |
155 | 0 | tran_low_t abs_qc_low; |
156 | 0 | int64_t dist_low, rd_low; |
157 | 0 | int rate_low; |
158 | |
|
159 | 0 | if (abs_qc == 1) { |
160 | 0 | abs_qc_low = 0; |
161 | 0 | dqc_low = qc_low = 0; |
162 | 0 | dist_low = 0; |
163 | 0 | rate_low = txb_costs->base_cost[coeff_ctx][0]; |
164 | 0 | rd_low = RDCOST(rdmult, *accu_rate + rate_low, *accu_dist); |
165 | 0 | } else { |
166 | 0 | get_qc_dqc_low(abs_qc, sign, dqv, shift, &qc_low, &dqc_low); |
167 | 0 | abs_qc_low = abs_qc - 1; |
168 | 0 | dist_low = get_coeff_dist(tqc, dqc_low, shift) - dist0; |
169 | 0 | rate_low = |
170 | 0 | get_coeff_cost_general(0, ci, abs_qc_low, sign, coeff_ctx, |
171 | 0 | dc_sign_ctx, txb_costs, bwl, tx_class, levels); |
172 | 0 | rd_low = RDCOST(rdmult, *accu_rate + rate_low, *accu_dist + dist_low); |
173 | 0 | } |
174 | |
|
175 | 0 | int lower_level_new_eob = 0; |
176 | 0 | const int new_eob = si + 1; |
177 | 0 | const int coeff_ctx_new_eob = get_lower_levels_ctx_eob(bwl, height, si); |
178 | 0 | const int new_eob_cost = |
179 | 0 | get_eob_cost(new_eob, txb_eob_costs, txb_costs, tx_class); |
180 | 0 | int rate_coeff_eob = |
181 | 0 | new_eob_cost + get_coeff_cost_eob(ci, abs_qc, sign, coeff_ctx_new_eob, |
182 | 0 | dc_sign_ctx, txb_costs, bwl, |
183 | 0 | tx_class); |
184 | 0 | int64_t dist_new_eob = dist; |
185 | 0 | int64_t rd_new_eob = RDCOST(rdmult, rate_coeff_eob, dist_new_eob); |
186 | |
|
187 | 0 | if (abs_qc_low > 0) { |
188 | 0 | const int rate_coeff_eob_low = |
189 | 0 | new_eob_cost + get_coeff_cost_eob(ci, abs_qc_low, sign, |
190 | 0 | coeff_ctx_new_eob, dc_sign_ctx, |
191 | 0 | txb_costs, bwl, tx_class); |
192 | 0 | const int64_t dist_new_eob_low = dist_low; |
193 | 0 | const int64_t rd_new_eob_low = |
194 | 0 | RDCOST(rdmult, rate_coeff_eob_low, dist_new_eob_low); |
195 | 0 | if (rd_new_eob_low < rd_new_eob) { |
196 | 0 | lower_level_new_eob = 1; |
197 | 0 | rd_new_eob = rd_new_eob_low; |
198 | 0 | rate_coeff_eob = rate_coeff_eob_low; |
199 | 0 | dist_new_eob = dist_new_eob_low; |
200 | 0 | } |
201 | 0 | } |
202 | |
|
203 | 0 | if (sharpness == 0 || abs_qc > 1) { |
204 | 0 | if (rd_low < rd) { |
205 | 0 | lower_level = 1; |
206 | 0 | rd = rd_low; |
207 | 0 | rate = rate_low; |
208 | 0 | dist = dist_low; |
209 | 0 | } |
210 | 0 | } |
211 | |
|
212 | 0 | if (sharpness == 0 && rd_new_eob < rd) { |
213 | 0 | for (int ni = 0; ni < *nz_num; ++ni) { |
214 | 0 | int last_ci = nz_ci[ni]; |
215 | 0 | levels[get_padded_idx(last_ci, bwl)] = 0; |
216 | 0 | qcoeff[last_ci] = 0; |
217 | 0 | dqcoeff[last_ci] = 0; |
218 | 0 | } |
219 | 0 | *eob = new_eob; |
220 | 0 | *nz_num = 0; |
221 | 0 | *accu_rate = rate_coeff_eob; |
222 | 0 | *accu_dist = dist_new_eob; |
223 | 0 | lower_level = lower_level_new_eob; |
224 | 0 | } else { |
225 | 0 | *accu_rate += rate; |
226 | 0 | *accu_dist += dist; |
227 | 0 | } |
228 | |
|
229 | 0 | if (lower_level) { |
230 | 0 | qcoeff[ci] = qc_low; |
231 | 0 | dqcoeff[ci] = dqc_low; |
232 | 0 | levels[get_padded_idx(ci, bwl)] = AOMMIN(abs_qc_low, INT8_MAX); |
233 | 0 | } |
234 | 0 | if (qcoeff[ci]) { |
235 | 0 | nz_ci[*nz_num] = ci; |
236 | 0 | ++*nz_num; |
237 | 0 | } |
238 | 0 | } |
239 | 0 | } |
240 | | |
241 | | static INLINE void update_skip(int *accu_rate, int64_t accu_dist, int *eob, |
242 | | int nz_num, int *nz_ci, int64_t rdmult, |
243 | | int skip_cost, int non_skip_cost, |
244 | 0 | tran_low_t *qcoeff, tran_low_t *dqcoeff) { |
245 | 0 | const int64_t rd = RDCOST(rdmult, *accu_rate + non_skip_cost, accu_dist); |
246 | 0 | const int64_t rd_new_eob = RDCOST(rdmult, skip_cost, 0); |
247 | 0 | if (rd_new_eob < rd) { |
248 | 0 | for (int i = 0; i < nz_num; ++i) { |
249 | 0 | const int ci = nz_ci[i]; |
250 | 0 | qcoeff[ci] = 0; |
251 | 0 | dqcoeff[ci] = 0; |
252 | | // no need to set up levels because this is the last step |
253 | | // levels[get_padded_idx(ci, bwl)] = 0; |
254 | 0 | } |
255 | 0 | *accu_rate = 0; |
256 | 0 | *eob = 0; |
257 | 0 | } |
258 | 0 | } |
259 | | |
260 | | // TODO(angiebird): use this function whenever it's possible |
261 | | static int get_tx_type_cost(const MACROBLOCK *x, const MACROBLOCKD *xd, |
262 | | int plane, TX_SIZE tx_size, TX_TYPE tx_type, |
263 | 0 | int reduced_tx_set_used) { |
264 | 0 | if (plane > 0) return 0; |
265 | | |
266 | 0 | const TX_SIZE square_tx_size = txsize_sqr_map[tx_size]; |
267 | |
|
268 | 0 | const MB_MODE_INFO *mbmi = xd->mi[0]; |
269 | 0 | const int is_inter = is_inter_block(mbmi); |
270 | 0 | if (get_ext_tx_types(tx_size, is_inter, reduced_tx_set_used) > 1 && |
271 | 0 | !xd->lossless[xd->mi[0]->segment_id]) { |
272 | 0 | const int ext_tx_set = |
273 | 0 | get_ext_tx_set(tx_size, is_inter, reduced_tx_set_used); |
274 | 0 | if (is_inter) { |
275 | 0 | if (ext_tx_set > 0) |
276 | 0 | return x->mode_costs |
277 | 0 | .inter_tx_type_costs[ext_tx_set][square_tx_size][tx_type]; |
278 | 0 | } else { |
279 | 0 | if (ext_tx_set > 0) { |
280 | 0 | PREDICTION_MODE intra_dir; |
281 | 0 | if (mbmi->filter_intra_mode_info.use_filter_intra) |
282 | 0 | intra_dir = fimode_to_intradir[mbmi->filter_intra_mode_info |
283 | 0 | .filter_intra_mode]; |
284 | 0 | else |
285 | 0 | intra_dir = mbmi->mode; |
286 | 0 | return x->mode_costs.intra_tx_type_costs[ext_tx_set][square_tx_size] |
287 | 0 | [intra_dir][tx_type]; |
288 | 0 | } |
289 | 0 | } |
290 | 0 | } |
291 | 0 | return 0; |
292 | 0 | } |
293 | | |
294 | | int av1_optimize_txb(const struct AV1_COMP *cpi, MACROBLOCK *x, int plane, |
295 | | int block, TX_SIZE tx_size, TX_TYPE tx_type, |
296 | | const TXB_CTX *const txb_ctx, int *rate_cost, |
297 | 0 | int sharpness) { |
298 | 0 | MACROBLOCKD *xd = &x->e_mbd; |
299 | 0 | const struct macroblock_plane *p = &x->plane[plane]; |
300 | 0 | const SCAN_ORDER *scan_order = get_scan(tx_size, tx_type); |
301 | 0 | const int16_t *scan = scan_order->scan; |
302 | 0 | const int shift = av1_get_tx_scale(tx_size); |
303 | 0 | int eob = p->eobs[block]; |
304 | 0 | const int16_t *dequant = p->dequant_QTX; |
305 | 0 | const qm_val_t *iqmatrix = |
306 | 0 | av1_get_iqmatrix(&cpi->common.quant_params, xd, plane, tx_size, tx_type); |
307 | 0 | const int block_offset = BLOCK_OFFSET(block); |
308 | 0 | tran_low_t *qcoeff = p->qcoeff + block_offset; |
309 | 0 | tran_low_t *dqcoeff = p->dqcoeff + block_offset; |
310 | 0 | const tran_low_t *tcoeff = p->coeff + block_offset; |
311 | 0 | const CoeffCosts *coeff_costs = &x->coeff_costs; |
312 | | |
313 | | // This function is not called if eob = 0. |
314 | 0 | assert(eob > 0); |
315 | |
|
316 | 0 | const AV1_COMMON *cm = &cpi->common; |
317 | 0 | const PLANE_TYPE plane_type = get_plane_type(plane); |
318 | 0 | const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size); |
319 | 0 | const TX_CLASS tx_class = tx_type_to_class[tx_type]; |
320 | 0 | const MB_MODE_INFO *mbmi = xd->mi[0]; |
321 | 0 | const int bwl = get_txb_bwl(tx_size); |
322 | 0 | const int width = get_txb_wide(tx_size); |
323 | 0 | const int height = get_txb_high(tx_size); |
324 | 0 | assert(width == (1 << bwl)); |
325 | 0 | const int is_inter = is_inter_block(mbmi); |
326 | 0 | const LV_MAP_COEFF_COST *txb_costs = |
327 | 0 | &coeff_costs->coeff_costs[txs_ctx][plane_type]; |
328 | 0 | const int eob_multi_size = txsize_log2_minus4[tx_size]; |
329 | 0 | const LV_MAP_EOB_COST *txb_eob_costs = |
330 | 0 | &coeff_costs->eob_costs[eob_multi_size][plane_type]; |
331 | |
|
332 | 0 | const int rshift = 2; |
333 | |
|
334 | 0 | const int64_t rdmult = |
335 | 0 | (((int64_t)x->rdmult * |
336 | 0 | (plane_rd_mult[is_inter][plane_type] << (2 * (xd->bd - 8)))) + |
337 | 0 | 2) >> |
338 | 0 | rshift; |
339 | |
|
340 | 0 | uint8_t levels_buf[TX_PAD_2D]; |
341 | 0 | uint8_t *const levels = set_levels(levels_buf, width); |
342 | |
|
343 | 0 | if (eob > 1) av1_txb_init_levels(qcoeff, width, height, levels); |
344 | | |
345 | | // TODO(angirbird): check iqmatrix |
346 | |
|
347 | 0 | const int non_skip_cost = txb_costs->txb_skip_cost[txb_ctx->txb_skip_ctx][0]; |
348 | 0 | const int skip_cost = txb_costs->txb_skip_cost[txb_ctx->txb_skip_ctx][1]; |
349 | 0 | const int eob_cost = get_eob_cost(eob, txb_eob_costs, txb_costs, tx_class); |
350 | 0 | int accu_rate = eob_cost; |
351 | 0 | int64_t accu_dist = 0; |
352 | 0 | int si = eob - 1; |
353 | 0 | const int ci = scan[si]; |
354 | 0 | const tran_low_t qc = qcoeff[ci]; |
355 | 0 | const tran_low_t abs_qc = abs(qc); |
356 | 0 | const int sign = qc < 0; |
357 | 0 | const int max_nz_num = 2; |
358 | 0 | int nz_num = 1; |
359 | 0 | int nz_ci[3] = { ci, 0, 0 }; |
360 | 0 | if (abs_qc >= 2) { |
361 | 0 | update_coeff_general(&accu_rate, &accu_dist, si, eob, tx_size, tx_class, |
362 | 0 | bwl, height, rdmult, shift, txb_ctx->dc_sign_ctx, |
363 | 0 | dequant, scan, txb_costs, tcoeff, qcoeff, dqcoeff, |
364 | 0 | levels, iqmatrix); |
365 | 0 | --si; |
366 | 0 | } else { |
367 | 0 | assert(abs_qc == 1); |
368 | 0 | const int coeff_ctx = get_lower_levels_ctx_eob(bwl, height, si); |
369 | 0 | accu_rate += |
370 | 0 | get_coeff_cost_eob(ci, abs_qc, sign, coeff_ctx, txb_ctx->dc_sign_ctx, |
371 | 0 | txb_costs, bwl, tx_class); |
372 | 0 | const tran_low_t tqc = tcoeff[ci]; |
373 | 0 | const tran_low_t dqc = dqcoeff[ci]; |
374 | 0 | const int64_t dist = get_coeff_dist(tqc, dqc, shift); |
375 | 0 | const int64_t dist0 = get_coeff_dist(tqc, 0, shift); |
376 | 0 | accu_dist += dist - dist0; |
377 | 0 | --si; |
378 | 0 | } |
379 | |
|
380 | 0 | #define UPDATE_COEFF_EOB_CASE(tx_class_literal) \ |
381 | 0 | case tx_class_literal: \ |
382 | 0 | for (; si >= 0 && nz_num <= max_nz_num; --si) { \ |
383 | 0 | update_coeff_eob(&accu_rate, &accu_dist, &eob, &nz_num, nz_ci, si, \ |
384 | 0 | tx_size, tx_class_literal, bwl, height, \ |
385 | 0 | txb_ctx->dc_sign_ctx, rdmult, shift, dequant, scan, \ |
386 | 0 | txb_eob_costs, txb_costs, tcoeff, qcoeff, dqcoeff, \ |
387 | 0 | levels, sharpness, iqmatrix); \ |
388 | 0 | } \ |
389 | 0 | break; |
390 | 0 | switch (tx_class) { |
391 | 0 | UPDATE_COEFF_EOB_CASE(TX_CLASS_2D); |
392 | 0 | UPDATE_COEFF_EOB_CASE(TX_CLASS_HORIZ); |
393 | 0 | UPDATE_COEFF_EOB_CASE(TX_CLASS_VERT); |
394 | 0 | #undef UPDATE_COEFF_EOB_CASE |
395 | 0 | default: assert(false); |
396 | 0 | } |
397 | | |
398 | 0 | if (si == -1 && nz_num <= max_nz_num && sharpness == 0) { |
399 | 0 | update_skip(&accu_rate, accu_dist, &eob, nz_num, nz_ci, rdmult, skip_cost, |
400 | 0 | non_skip_cost, qcoeff, dqcoeff); |
401 | 0 | } |
402 | |
|
403 | 0 | #define UPDATE_COEFF_SIMPLE_CASE(tx_class_literal) \ |
404 | 0 | case tx_class_literal: \ |
405 | 0 | for (; si >= 1; --si) { \ |
406 | 0 | update_coeff_simple(&accu_rate, si, eob, tx_size, tx_class_literal, bwl, \ |
407 | 0 | rdmult, shift, dequant, scan, txb_costs, tcoeff, \ |
408 | 0 | qcoeff, dqcoeff, levels, iqmatrix); \ |
409 | 0 | } \ |
410 | 0 | break; |
411 | 0 | switch (tx_class) { |
412 | 0 | UPDATE_COEFF_SIMPLE_CASE(TX_CLASS_2D); |
413 | 0 | UPDATE_COEFF_SIMPLE_CASE(TX_CLASS_HORIZ); |
414 | 0 | UPDATE_COEFF_SIMPLE_CASE(TX_CLASS_VERT); |
415 | 0 | #undef UPDATE_COEFF_SIMPLE_CASE |
416 | 0 | default: assert(false); |
417 | 0 | } |
418 | | |
419 | | // DC position |
420 | 0 | if (si == 0) { |
421 | | // no need to update accu_dist because it's not used after this point |
422 | 0 | int64_t dummy_dist = 0; |
423 | 0 | update_coeff_general(&accu_rate, &dummy_dist, si, eob, tx_size, tx_class, |
424 | 0 | bwl, height, rdmult, shift, txb_ctx->dc_sign_ctx, |
425 | 0 | dequant, scan, txb_costs, tcoeff, qcoeff, dqcoeff, |
426 | 0 | levels, iqmatrix); |
427 | 0 | } |
428 | |
|
429 | 0 | const int tx_type_cost = get_tx_type_cost(x, xd, plane, tx_size, tx_type, |
430 | 0 | cm->features.reduced_tx_set_used); |
431 | 0 | if (eob == 0) |
432 | 0 | accu_rate += skip_cost; |
433 | 0 | else |
434 | 0 | accu_rate += non_skip_cost + tx_type_cost; |
435 | |
|
436 | 0 | p->eobs[block] = eob; |
437 | 0 | p->txb_entropy_ctx[block] = |
438 | 0 | av1_get_txb_entropy_context(qcoeff, scan_order, p->eobs[block]); |
439 | |
|
440 | 0 | *rate_cost = accu_rate; |
441 | 0 | return eob; |
442 | 0 | } |
443 | | |
444 | | static AOM_FORCE_INLINE int warehouse_efficients_txb( |
445 | | const MACROBLOCK *x, const int plane, const int block, |
446 | | const TX_SIZE tx_size, const TXB_CTX *const txb_ctx, |
447 | | const struct macroblock_plane *p, const int eob, |
448 | | const PLANE_TYPE plane_type, const LV_MAP_COEFF_COST *const coeff_costs, |
449 | | const MACROBLOCKD *const xd, const TX_TYPE tx_type, const TX_CLASS tx_class, |
450 | 0 | int reduced_tx_set_used) { |
451 | 0 | const tran_low_t *const qcoeff = p->qcoeff + BLOCK_OFFSET(block); |
452 | 0 | const int txb_skip_ctx = txb_ctx->txb_skip_ctx; |
453 | 0 | const int bwl = get_txb_bwl(tx_size); |
454 | 0 | const int width = get_txb_wide(tx_size); |
455 | 0 | const int height = get_txb_high(tx_size); |
456 | 0 | const SCAN_ORDER *const scan_order = get_scan(tx_size, tx_type); |
457 | 0 | const int16_t *const scan = scan_order->scan; |
458 | 0 | uint8_t levels_buf[TX_PAD_2D]; |
459 | 0 | uint8_t *const levels = set_levels(levels_buf, width); |
460 | 0 | DECLARE_ALIGNED(16, int8_t, coeff_contexts[MAX_TX_SQUARE]); |
461 | 0 | const int eob_multi_size = txsize_log2_minus4[tx_size]; |
462 | 0 | const LV_MAP_EOB_COST *const eob_costs = |
463 | 0 | &x->coeff_costs.eob_costs[eob_multi_size][plane_type]; |
464 | 0 | int cost = coeff_costs->txb_skip_cost[txb_skip_ctx][0]; |
465 | |
|
466 | 0 | av1_txb_init_levels(qcoeff, width, height, levels); |
467 | |
|
468 | 0 | cost += get_tx_type_cost(x, xd, plane, tx_size, tx_type, reduced_tx_set_used); |
469 | |
|
470 | 0 | cost += get_eob_cost(eob, eob_costs, coeff_costs, tx_class); |
471 | |
|
472 | 0 | av1_get_nz_map_contexts(levels, scan, eob, tx_size, tx_class, coeff_contexts); |
473 | |
|
474 | 0 | const int(*lps_cost)[COEFF_BASE_RANGE + 1 + COEFF_BASE_RANGE + 1] = |
475 | 0 | coeff_costs->lps_cost; |
476 | 0 | int c = eob - 1; |
477 | 0 | { |
478 | 0 | const int pos = scan[c]; |
479 | 0 | const tran_low_t v = qcoeff[pos]; |
480 | 0 | const int sign = AOMSIGN(v); |
481 | 0 | const int level = (v ^ sign) - sign; |
482 | 0 | const int coeff_ctx = coeff_contexts[pos]; |
483 | 0 | cost += coeff_costs->base_eob_cost[coeff_ctx][AOMMIN(level, 3) - 1]; |
484 | |
|
485 | 0 | if (v) { |
486 | | // sign bit cost |
487 | 0 | if (level > NUM_BASE_LEVELS) { |
488 | 0 | const int ctx = get_br_ctx_eob(pos, bwl, tx_class); |
489 | 0 | cost += get_br_cost(level, lps_cost[ctx]); |
490 | 0 | } |
491 | 0 | if (c) { |
492 | 0 | cost += av1_cost_literal(1); |
493 | 0 | } else { |
494 | 0 | const int sign01 = (sign ^ sign) - sign; |
495 | 0 | const int dc_sign_ctx = txb_ctx->dc_sign_ctx; |
496 | 0 | cost += coeff_costs->dc_sign_cost[dc_sign_ctx][sign01]; |
497 | 0 | return cost; |
498 | 0 | } |
499 | 0 | } |
500 | 0 | } |
501 | 0 | const int(*base_cost)[8] = coeff_costs->base_cost; |
502 | 0 | for (c = eob - 2; c >= 1; --c) { |
503 | 0 | const int pos = scan[c]; |
504 | 0 | const int coeff_ctx = coeff_contexts[pos]; |
505 | 0 | const tran_low_t v = qcoeff[pos]; |
506 | 0 | const int level = abs(v); |
507 | 0 | cost += base_cost[coeff_ctx][AOMMIN(level, 3)]; |
508 | 0 | if (v) { |
509 | | // sign bit cost |
510 | 0 | cost += av1_cost_literal(1); |
511 | 0 | if (level > NUM_BASE_LEVELS) { |
512 | 0 | const int ctx = get_br_ctx(levels, pos, bwl, tx_class); |
513 | 0 | cost += get_br_cost(level, lps_cost[ctx]); |
514 | 0 | } |
515 | 0 | } |
516 | 0 | } |
517 | | // c == 0 after previous loop |
518 | 0 | { |
519 | 0 | const int pos = scan[c]; |
520 | 0 | const tran_low_t v = qcoeff[pos]; |
521 | 0 | const int coeff_ctx = coeff_contexts[pos]; |
522 | 0 | const int sign = AOMSIGN(v); |
523 | 0 | const int level = (v ^ sign) - sign; |
524 | 0 | cost += base_cost[coeff_ctx][AOMMIN(level, 3)]; |
525 | |
|
526 | 0 | if (v) { |
527 | | // sign bit cost |
528 | 0 | const int sign01 = (sign ^ sign) - sign; |
529 | 0 | const int dc_sign_ctx = txb_ctx->dc_sign_ctx; |
530 | 0 | cost += coeff_costs->dc_sign_cost[dc_sign_ctx][sign01]; |
531 | 0 | if (level > NUM_BASE_LEVELS) { |
532 | 0 | const int ctx = get_br_ctx(levels, pos, bwl, tx_class); |
533 | 0 | cost += get_br_cost(level, lps_cost[ctx]); |
534 | 0 | } |
535 | 0 | } |
536 | 0 | } |
537 | 0 | return cost; |
538 | 0 | } |
539 | | |
540 | | int av1_cost_coeffs_txb_estimate(const MACROBLOCK *x, const int plane, |
541 | | const int block, const TX_SIZE tx_size, |
542 | 0 | const TX_TYPE tx_type) { |
543 | 0 | assert(plane == 0); |
544 | |
|
545 | 0 | int cost = 0; |
546 | 0 | const struct macroblock_plane *p = &x->plane[plane]; |
547 | 0 | const SCAN_ORDER *scan_order = get_scan(tx_size, tx_type); |
548 | 0 | const int16_t *scan = scan_order->scan; |
549 | 0 | tran_low_t *qcoeff = p->qcoeff + BLOCK_OFFSET(block); |
550 | |
|
551 | 0 | int eob = p->eobs[block]; |
552 | | |
553 | | // coeffs |
554 | 0 | int c = eob - 1; |
555 | | // eob |
556 | 0 | { |
557 | 0 | const int pos = scan[c]; |
558 | 0 | const tran_low_t v = abs(qcoeff[pos]) - 1; |
559 | 0 | cost += (v << (AV1_PROB_COST_SHIFT + 2)); |
560 | 0 | } |
561 | | // other coeffs |
562 | 0 | for (c = eob - 2; c >= 0; c--) { |
563 | 0 | const int pos = scan[c]; |
564 | 0 | const tran_low_t v = abs(qcoeff[pos]); |
565 | 0 | const int idx = AOMMIN(v, 14); |
566 | |
|
567 | 0 | cost += costLUT[idx]; |
568 | 0 | } |
569 | | |
570 | | // const_term does not contain DC, and log(e) does not contain eob, so both |
571 | | // (eob-1) |
572 | 0 | cost += (const_term + loge_par) * (eob - 1); |
573 | |
|
574 | 0 | return cost; |
575 | 0 | } |
576 | | |
577 | | static AOM_FORCE_INLINE int warehouse_efficients_txb_laplacian( |
578 | | const MACROBLOCK *x, const int plane, const int block, |
579 | | const TX_SIZE tx_size, const TXB_CTX *const txb_ctx, const int eob, |
580 | | const PLANE_TYPE plane_type, const LV_MAP_COEFF_COST *const coeff_costs, |
581 | | const MACROBLOCKD *const xd, const TX_TYPE tx_type, const TX_CLASS tx_class, |
582 | 0 | int reduced_tx_set_used) { |
583 | 0 | const int txb_skip_ctx = txb_ctx->txb_skip_ctx; |
584 | |
|
585 | 0 | const int eob_multi_size = txsize_log2_minus4[tx_size]; |
586 | 0 | const LV_MAP_EOB_COST *const eob_costs = |
587 | 0 | &x->coeff_costs.eob_costs[eob_multi_size][plane_type]; |
588 | 0 | int cost = coeff_costs->txb_skip_cost[txb_skip_ctx][0]; |
589 | |
|
590 | 0 | cost += get_tx_type_cost(x, xd, plane, tx_size, tx_type, reduced_tx_set_used); |
591 | |
|
592 | 0 | cost += get_eob_cost(eob, eob_costs, coeff_costs, tx_class); |
593 | |
|
594 | 0 | cost += av1_cost_coeffs_txb_estimate(x, plane, block, tx_size, tx_type); |
595 | 0 | return cost; |
596 | 0 | } |
597 | | |
598 | | int av1_cost_coeffs_txb(const MACROBLOCK *x, const int plane, const int block, |
599 | | const TX_SIZE tx_size, const TX_TYPE tx_type, |
600 | 0 | const TXB_CTX *const txb_ctx, int reduced_tx_set_used) { |
601 | 0 | const struct macroblock_plane *p = &x->plane[plane]; |
602 | 0 | const int eob = p->eobs[block]; |
603 | 0 | const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size); |
604 | 0 | const PLANE_TYPE plane_type = get_plane_type(plane); |
605 | 0 | const LV_MAP_COEFF_COST *const coeff_costs = |
606 | 0 | &x->coeff_costs.coeff_costs[txs_ctx][plane_type]; |
607 | 0 | if (eob == 0) { |
608 | 0 | return coeff_costs->txb_skip_cost[txb_ctx->txb_skip_ctx][1]; |
609 | 0 | } |
610 | | |
611 | 0 | const MACROBLOCKD *const xd = &x->e_mbd; |
612 | 0 | const TX_CLASS tx_class = tx_type_to_class[tx_type]; |
613 | |
|
614 | 0 | return warehouse_efficients_txb(x, plane, block, tx_size, txb_ctx, p, eob, |
615 | 0 | plane_type, coeff_costs, xd, tx_type, |
616 | 0 | tx_class, reduced_tx_set_used); |
617 | 0 | } |
618 | | |
619 | | int av1_cost_coeffs_txb_laplacian(const MACROBLOCK *x, const int plane, |
620 | | const int block, const TX_SIZE tx_size, |
621 | | const TX_TYPE tx_type, |
622 | | const TXB_CTX *const txb_ctx, |
623 | | const int reduced_tx_set_used, |
624 | 0 | const int adjust_eob) { |
625 | 0 | const struct macroblock_plane *p = &x->plane[plane]; |
626 | 0 | int eob = p->eobs[block]; |
627 | |
|
628 | 0 | if (adjust_eob) { |
629 | 0 | const SCAN_ORDER *scan_order = get_scan(tx_size, tx_type); |
630 | 0 | const int16_t *scan = scan_order->scan; |
631 | 0 | tran_low_t *tcoeff = p->coeff + BLOCK_OFFSET(block); |
632 | 0 | tran_low_t *qcoeff = p->qcoeff + BLOCK_OFFSET(block); |
633 | 0 | tran_low_t *dqcoeff = p->dqcoeff + BLOCK_OFFSET(block); |
634 | 0 | update_coeff_eob_fast(&eob, av1_get_tx_scale(tx_size), p->dequant_QTX, scan, |
635 | 0 | tcoeff, qcoeff, dqcoeff); |
636 | 0 | p->eobs[block] = eob; |
637 | 0 | } |
638 | |
|
639 | 0 | const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size); |
640 | 0 | const PLANE_TYPE plane_type = get_plane_type(plane); |
641 | 0 | const LV_MAP_COEFF_COST *const coeff_costs = |
642 | 0 | &x->coeff_costs.coeff_costs[txs_ctx][plane_type]; |
643 | 0 | if (eob == 0) { |
644 | 0 | return coeff_costs->txb_skip_cost[txb_ctx->txb_skip_ctx][1]; |
645 | 0 | } |
646 | | |
647 | 0 | const MACROBLOCKD *const xd = &x->e_mbd; |
648 | 0 | const TX_CLASS tx_class = tx_type_to_class[tx_type]; |
649 | |
|
650 | 0 | return warehouse_efficients_txb_laplacian( |
651 | 0 | x, plane, block, tx_size, txb_ctx, eob, plane_type, coeff_costs, xd, |
652 | 0 | tx_type, tx_class, reduced_tx_set_used); |
653 | 0 | } |