Line | Count | Source |
1 | | /***************************************************************************** |
2 | | * set.c: quantization init |
3 | | ***************************************************************************** |
4 | | * Copyright (C) 2005-2025 x264 project |
5 | | * |
6 | | * Authors: Loren Merritt <lorenm@u.washington.edu> |
7 | | * |
8 | | * This program is free software; you can redistribute it and/or modify |
9 | | * it under the terms of the GNU General Public License as published by |
10 | | * the Free Software Foundation; either version 2 of the License, or |
11 | | * (at your option) any later version. |
12 | | * |
13 | | * This program is distributed in the hope that it will be useful, |
14 | | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
15 | | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
16 | | * GNU General Public License for more details. |
17 | | * |
18 | | * You should have received a copy of the GNU General Public License |
19 | | * along with this program; if not, write to the Free Software |
20 | | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111, USA. |
21 | | * |
22 | | * This program is also available under a commercial proprietary license. |
23 | | * For more information, contact us at licensing@x264.com. |
24 | | *****************************************************************************/ |
25 | | |
26 | | #include "common.h" |
27 | | |
28 | 0 | #define SHIFT(x,s) ((s)<=0 ? (x)<<-(s) : ((x)+(1<<((s)-1)))>>(s)) |
29 | 0 | #define DIV(n,d) (((n) + ((d)>>1)) / (d)) |
30 | | |
31 | | static const uint8_t dequant4_scale[6][3] = |
32 | | { |
33 | | { 10, 13, 16 }, |
34 | | { 11, 14, 18 }, |
35 | | { 13, 16, 20 }, |
36 | | { 14, 18, 23 }, |
37 | | { 16, 20, 25 }, |
38 | | { 18, 23, 29 } |
39 | | }; |
40 | | static const uint16_t quant4_scale[6][3] = |
41 | | { |
42 | | { 13107, 8066, 5243 }, |
43 | | { 11916, 7490, 4660 }, |
44 | | { 10082, 6554, 4194 }, |
45 | | { 9362, 5825, 3647 }, |
46 | | { 8192, 5243, 3355 }, |
47 | | { 7282, 4559, 2893 }, |
48 | | }; |
49 | | |
50 | | static const uint8_t quant8_scan[16] = |
51 | | { |
52 | | 0,3,4,3, 3,1,5,1, 4,5,2,5, 3,1,5,1 |
53 | | }; |
54 | | static const uint8_t dequant8_scale[6][6] = |
55 | | { |
56 | | { 20, 18, 32, 19, 25, 24 }, |
57 | | { 22, 19, 35, 21, 28, 26 }, |
58 | | { 26, 23, 42, 24, 33, 31 }, |
59 | | { 28, 25, 45, 26, 35, 33 }, |
60 | | { 32, 28, 51, 30, 40, 38 }, |
61 | | { 36, 32, 58, 34, 46, 43 }, |
62 | | }; |
63 | | static const uint16_t quant8_scale[6][6] = |
64 | | { |
65 | | { 13107, 11428, 20972, 12222, 16777, 15481 }, |
66 | | { 11916, 10826, 19174, 11058, 14980, 14290 }, |
67 | | { 10082, 8943, 15978, 9675, 12710, 11985 }, |
68 | | { 9362, 8228, 14913, 8931, 11984, 11259 }, |
69 | | { 8192, 7346, 13159, 7740, 10486, 9777 }, |
70 | | { 7282, 6428, 11570, 6830, 9118, 8640 } |
71 | | }; |
72 | | |
73 | | int x264_cqm_init( x264_t *h ) |
74 | 0 | { |
75 | 0 | int def_quant4[6][16]; |
76 | 0 | int def_quant8[6][64]; |
77 | 0 | int def_dequant4[6][16]; |
78 | 0 | int def_dequant8[6][64]; |
79 | 0 | int quant4_mf[4][6][16]; |
80 | 0 | int quant8_mf[4][6][64]; |
81 | 0 | int deadzone[4] = { 32 - h->param.analyse.i_luma_deadzone[1], |
82 | 0 | 32 - h->param.analyse.i_luma_deadzone[0], |
83 | 0 | 32 - 11, 32 - 21 }; |
84 | 0 | int max_qp_err = -1; |
85 | 0 | int max_chroma_qp_err = -1; |
86 | 0 | int min_qp_err = QP_MAX+1; |
87 | 0 | int num_8x8_lists = h->sps->i_chroma_format_idc == CHROMA_444 ? 4 |
88 | 0 | : h->param.analyse.b_transform_8x8 ? 2 : 0; /* Checkasm may segfault if optimized out by --chroma-format */ |
89 | |
|
90 | 0 | #define CQM_ALLOC( w, count )\ |
91 | 0 | for( int i = 0; i < count; i++ )\ |
92 | 0 | {\ |
93 | 0 | int size = w*w;\ |
94 | 0 | int start = w == 8 ? 4 : 0;\ |
95 | 0 | int j;\ |
96 | 0 | for( j = 0; j < i; j++ )\ |
97 | 0 | if( !memcmp( h->sps->scaling_list[i+start], h->sps->scaling_list[j+start], size*sizeof(uint8_t) ) )\ |
98 | 0 | break;\ |
99 | 0 | if( j < i )\ |
100 | 0 | {\ |
101 | 0 | h-> quant##w##_mf[i] = h-> quant##w##_mf[j];\ |
102 | 0 | h->dequant##w##_mf[i] = h->dequant##w##_mf[j];\ |
103 | 0 | h->unquant##w##_mf[i] = h->unquant##w##_mf[j];\ |
104 | 0 | }\ |
105 | 0 | else\ |
106 | 0 | {\ |
107 | 0 | CHECKED_MALLOC( h-> quant##w##_mf[i], (QP_MAX_SPEC+1)*size*sizeof(udctcoef) );\ |
108 | 0 | CHECKED_MALLOC( h->dequant##w##_mf[i], 6*size*sizeof(int) );\ |
109 | 0 | CHECKED_MALLOC( h->unquant##w##_mf[i], (QP_MAX_SPEC+1)*size*sizeof(int) );\ |
110 | 0 | }\ |
111 | 0 | for( j = 0; j < i; j++ )\ |
112 | 0 | if( deadzone[j] == deadzone[i] &&\ |
113 | 0 | !memcmp( h->sps->scaling_list[i+start], h->sps->scaling_list[j+start], size*sizeof(uint8_t) ) )\ |
114 | 0 | break;\ |
115 | 0 | if( j < i )\ |
116 | 0 | {\ |
117 | 0 | h->quant##w##_bias[i] = h->quant##w##_bias[j];\ |
118 | 0 | h->quant##w##_bias0[i] = h->quant##w##_bias0[j];\ |
119 | 0 | }\ |
120 | 0 | else\ |
121 | 0 | {\ |
122 | 0 | CHECKED_MALLOC( h->quant##w##_bias[i], (QP_MAX_SPEC+1)*size*sizeof(udctcoef) );\ |
123 | 0 | CHECKED_MALLOC( h->quant##w##_bias0[i], (QP_MAX_SPEC+1)*size*sizeof(udctcoef) );\ |
124 | 0 | }\ |
125 | 0 | } |
126 | |
|
127 | 0 | CQM_ALLOC( 4, 4 ) |
128 | 0 | CQM_ALLOC( 8, num_8x8_lists ) |
129 | | |
130 | 0 | for( int q = 0; q < 6; q++ ) |
131 | 0 | { |
132 | 0 | for( int i = 0; i < 16; i++ ) |
133 | 0 | { |
134 | 0 | int j = (i&1) + ((i>>2)&1); |
135 | 0 | def_dequant4[q][i] = dequant4_scale[q][j]; |
136 | 0 | def_quant4[q][i] = quant4_scale[q][j]; |
137 | 0 | } |
138 | 0 | for( int i = 0; i < 64; i++ ) |
139 | 0 | { |
140 | 0 | int j = quant8_scan[((i>>1)&12) | (i&3)]; |
141 | 0 | def_dequant8[q][i] = dequant8_scale[q][j]; |
142 | 0 | def_quant8[q][i] = quant8_scale[q][j]; |
143 | 0 | } |
144 | 0 | } |
145 | |
|
146 | 0 | for( int q = 0; q < 6; q++ ) |
147 | 0 | { |
148 | 0 | for( int i_list = 0; i_list < 4; i_list++ ) |
149 | 0 | for( int i = 0; i < 16; i++ ) |
150 | 0 | { |
151 | 0 | h->dequant4_mf[i_list][q][i] = def_dequant4[q][i] * h->sps->scaling_list[i_list][i]; |
152 | 0 | quant4_mf[i_list][q][i] = DIV(def_quant4[q][i] * 16, h->sps->scaling_list[i_list][i]); |
153 | 0 | } |
154 | 0 | for( int i_list = 0; i_list < num_8x8_lists; i_list++ ) |
155 | 0 | for( int i = 0; i < 64; i++ ) |
156 | 0 | { |
157 | 0 | h->dequant8_mf[i_list][q][i] = def_dequant8[q][i] * h->sps->scaling_list[4+i_list][i]; |
158 | 0 | quant8_mf[i_list][q][i] = DIV(def_quant8[q][i] * 16, h->sps->scaling_list[4+i_list][i]); |
159 | 0 | } |
160 | 0 | } |
161 | |
|
162 | 0 | #define MAX_MF X264_MIN( 0xffff, (1 << (25 - BIT_DEPTH)) - 1 ) |
163 | |
|
164 | 0 | for( int q = 0; q <= QP_MAX_SPEC; q++ ) |
165 | 0 | { |
166 | 0 | int j; |
167 | 0 | for( int i_list = 0; i_list < 4; i_list++ ) |
168 | 0 | for( int i = 0; i < 16; i++ ) |
169 | 0 | { |
170 | 0 | h->unquant4_mf[i_list][q][i] = (1ULL << (q/6 + 15 + 8)) / quant4_mf[i_list][q%6][i]; |
171 | 0 | j = SHIFT(quant4_mf[i_list][q%6][i], q/6 - 1); |
172 | 0 | h->quant4_mf[i_list][q][i] = (uint16_t)j; |
173 | 0 | if( !j ) |
174 | 0 | { |
175 | 0 | min_qp_err = X264_MIN( min_qp_err, q ); |
176 | 0 | continue; |
177 | 0 | } |
178 | | // round to nearest, unless that would cause the deadzone to be negative |
179 | 0 | h->quant4_bias[i_list][q][i] = X264_MIN( DIV(deadzone[i_list]<<10, j), (1<<15)/j ); |
180 | 0 | h->quant4_bias0[i_list][q][i] = (1<<15)/j; |
181 | 0 | if( j > MAX_MF && q > max_qp_err && (i_list == CQM_4IY || i_list == CQM_4PY) ) |
182 | 0 | max_qp_err = q; |
183 | 0 | if( j > MAX_MF && q > max_chroma_qp_err && (i_list == CQM_4IC || i_list == CQM_4PC) ) |
184 | 0 | max_chroma_qp_err = q; |
185 | 0 | } |
186 | 0 | if( h->param.analyse.b_transform_8x8 ) |
187 | 0 | for( int i_list = 0; i_list < num_8x8_lists; i_list++ ) |
188 | 0 | for( int i = 0; i < 64; i++ ) |
189 | 0 | { |
190 | 0 | h->unquant8_mf[i_list][q][i] = (1ULL << (q/6 + 16 + 8)) / quant8_mf[i_list][q%6][i]; |
191 | 0 | j = SHIFT(quant8_mf[i_list][q%6][i], q/6); |
192 | 0 | h->quant8_mf[i_list][q][i] = (uint16_t)j; |
193 | |
|
194 | 0 | if( !j ) |
195 | 0 | { |
196 | 0 | min_qp_err = X264_MIN( min_qp_err, q ); |
197 | 0 | continue; |
198 | 0 | } |
199 | 0 | h->quant8_bias[i_list][q][i] = X264_MIN( DIV(deadzone[i_list]<<10, j), (1<<15)/j ); |
200 | 0 | h->quant8_bias0[i_list][q][i] = (1<<15)/j; |
201 | 0 | if( j > MAX_MF && q > max_qp_err && (i_list == CQM_8IY || i_list == CQM_8PY) ) |
202 | 0 | max_qp_err = q; |
203 | 0 | if( j > MAX_MF && q > max_chroma_qp_err && (i_list == CQM_8IC || i_list == CQM_8PC) ) |
204 | 0 | max_chroma_qp_err = q; |
205 | 0 | } |
206 | 0 | } |
207 | | |
208 | | /* Emergency mode denoising. */ |
209 | 0 | x264_emms(); |
210 | 0 | CHECKED_MALLOC( h->nr_offset_emergency, sizeof(*h->nr_offset_emergency)*(QP_MAX-QP_MAX_SPEC) ); |
211 | 0 | for( int q = 0; q < QP_MAX - QP_MAX_SPEC; q++ ) |
212 | 0 | for( int cat = 0; cat < 3 + CHROMA444; cat++ ) |
213 | 0 | { |
214 | 0 | int dct8x8 = cat&1; |
215 | 0 | if( !h->param.analyse.b_transform_8x8 && dct8x8 ) |
216 | 0 | continue; |
217 | | |
218 | 0 | int size = dct8x8 ? 64 : 16; |
219 | 0 | udctcoef *nr_offset = h->nr_offset_emergency[q][cat]; |
220 | | /* Denoise chroma first (due to h264's chroma QP offset), then luma, then DC. */ |
221 | 0 | int dc_threshold = (QP_MAX-QP_MAX_SPEC)*2/3; |
222 | 0 | int luma_threshold = (QP_MAX-QP_MAX_SPEC)*2/3; |
223 | 0 | int chroma_threshold = 0; |
224 | |
|
225 | 0 | for( int i = 0; i < size; i++ ) |
226 | 0 | { |
227 | 0 | int max = (1 << (7 + BIT_DEPTH)) - 1; |
228 | | /* True "emergency mode": remove all DCT coefficients */ |
229 | 0 | if( q == QP_MAX - QP_MAX_SPEC - 1 ) |
230 | 0 | { |
231 | 0 | nr_offset[i] = max; |
232 | 0 | continue; |
233 | 0 | } |
234 | | |
235 | 0 | int thresh = i == 0 ? dc_threshold : cat >= 2 ? chroma_threshold : luma_threshold; |
236 | 0 | if( q < thresh ) |
237 | 0 | { |
238 | 0 | nr_offset[i] = 0; |
239 | 0 | continue; |
240 | 0 | } |
241 | 0 | double pos = (double)(q-thresh+1) / (QP_MAX - QP_MAX_SPEC - thresh); |
242 | | |
243 | | /* XXX: this math is largely tuned for /dev/random input. */ |
244 | 0 | double start = dct8x8 ? h->unquant8_mf[CQM_8PY][QP_MAX_SPEC][i] |
245 | 0 | : h->unquant4_mf[CQM_4PY][QP_MAX_SPEC][i]; |
246 | | /* Formula chosen as an exponential scale to vaguely mimic the effects |
247 | | * of a higher quantizer. */ |
248 | 0 | double bias = (pow( 2, pos*(QP_MAX - QP_MAX_SPEC)/10. )*0.003-0.003) * start; |
249 | 0 | nr_offset[i] = X264_MIN( bias + 0.5, max ); |
250 | 0 | } |
251 | 0 | } |
252 | |
|
253 | 0 | if( !h->mb.b_lossless ) |
254 | 0 | { |
255 | 0 | while( h->chroma_qp_table[SPEC_QP(h->param.rc.i_qp_min)] <= max_chroma_qp_err ) |
256 | 0 | h->param.rc.i_qp_min++; |
257 | 0 | if( min_qp_err <= h->param.rc.i_qp_max ) |
258 | 0 | h->param.rc.i_qp_max = min_qp_err-1; |
259 | 0 | if( max_qp_err >= h->param.rc.i_qp_min ) |
260 | 0 | h->param.rc.i_qp_min = max_qp_err+1; |
261 | | /* If long level-codes aren't allowed, we need to allow QP high enough to avoid them. */ |
262 | 0 | if( !h->param.b_cabac && h->sps->i_profile_idc < PROFILE_HIGH ) |
263 | 0 | while( h->chroma_qp_table[SPEC_QP(h->param.rc.i_qp_max)] <= 12 || h->param.rc.i_qp_max <= 12 ) |
264 | 0 | h->param.rc.i_qp_max++; |
265 | 0 | if( h->param.rc.i_qp_min > h->param.rc.i_qp_max ) |
266 | 0 | { |
267 | 0 | x264_log( h, X264_LOG_ERROR, "Impossible QP constraints for CQM (min=%d, max=%d)\n", h->param.rc.i_qp_min, h->param.rc.i_qp_max ); |
268 | 0 | return -1; |
269 | 0 | } |
270 | 0 | } |
271 | 0 | return 0; |
272 | 0 | fail: |
273 | 0 | x264_cqm_delete( h ); |
274 | 0 | return -1; |
275 | 0 | } Unexecuted instantiation: x264_8_cqm_init Unexecuted instantiation: x264_10_cqm_init |
276 | | |
277 | | #define CQM_DELETE( n, max )\ |
278 | 0 | for( int i = 0; i < (max); i++ )\ |
279 | 0 | {\ |
280 | 0 | int j;\ |
281 | 0 | for( j = 0; j < i; j++ )\ |
282 | 0 | if( h->quant##n##_mf[i] == h->quant##n##_mf[j] )\ |
283 | 0 | break;\ |
284 | 0 | if( j == i )\ |
285 | 0 | {\ |
286 | 0 | x264_free( h-> quant##n##_mf[i] );\ |
287 | 0 | x264_free( h->dequant##n##_mf[i] );\ |
288 | 0 | x264_free( h->unquant##n##_mf[i] );\ |
289 | 0 | }\ |
290 | 0 | for( j = 0; j < i; j++ )\ |
291 | 0 | if( h->quant##n##_bias[i] == h->quant##n##_bias[j] )\ |
292 | 0 | break;\ |
293 | 0 | if( j == i )\ |
294 | 0 | {\ |
295 | 0 | x264_free( h->quant##n##_bias[i] );\ |
296 | 0 | x264_free( h->quant##n##_bias0[i] );\ |
297 | 0 | }\ |
298 | 0 | } |
299 | | |
300 | | void x264_cqm_delete( x264_t *h ) |
301 | 0 | { |
302 | 0 | CQM_DELETE( 4, 4 ); |
303 | 0 | CQM_DELETE( 8, CHROMA444 ? 4 : 2 ); |
304 | 0 | x264_free( h->nr_offset_emergency ); |
305 | 0 | } Unexecuted instantiation: x264_8_cqm_delete Unexecuted instantiation: x264_10_cqm_delete |
306 | | |
307 | | static int cqm_parse_jmlist( x264_t *h, const char *buf, const char *name, |
308 | | uint8_t *cqm, const uint8_t *jvt, int length ) |
309 | 0 | { |
310 | 0 | int i; |
311 | |
|
312 | 0 | char *p = strstr( buf, name ); |
313 | 0 | if( !p ) |
314 | 0 | { |
315 | 0 | memset( cqm, 16, length ); |
316 | 0 | return 0; |
317 | 0 | } |
318 | | |
319 | 0 | p += strlen( name ); |
320 | 0 | if( *p == 'U' || *p == 'V' ) |
321 | 0 | p++; |
322 | |
|
323 | 0 | char *nextvar = strstr( p, "INT" ); |
324 | |
|
325 | 0 | for( i = 0; i < length && (p = strpbrk( p, " \t\n," )) && (p = strpbrk( p, "0123456789" )); i++ ) |
326 | 0 | { |
327 | 0 | int coef = -1; |
328 | 0 | sscanf( p, "%d", &coef ); |
329 | 0 | if( i == 0 && coef == 0 ) |
330 | 0 | { |
331 | 0 | memcpy( cqm, jvt, length ); |
332 | 0 | return 0; |
333 | 0 | } |
334 | 0 | if( coef < 1 || coef > 255 ) |
335 | 0 | { |
336 | 0 | x264_log( h, X264_LOG_ERROR, "bad coefficient in list '%s'\n", name ); |
337 | 0 | return -1; |
338 | 0 | } |
339 | 0 | cqm[i] = coef; |
340 | 0 | } |
341 | | |
342 | 0 | if( (nextvar && p > nextvar) || i != length ) |
343 | 0 | { |
344 | 0 | x264_log( h, X264_LOG_ERROR, "not enough coefficients in list '%s'\n", name ); |
345 | 0 | return -1; |
346 | 0 | } |
347 | | |
348 | 0 | return 0; |
349 | 0 | } |
350 | | |
351 | | int x264_cqm_parse_file( x264_t *h, const char *filename ) |
352 | 0 | { |
353 | 0 | char *p; |
354 | 0 | int b_error = 0; |
355 | |
|
356 | 0 | h->param.i_cqm_preset = X264_CQM_CUSTOM; |
357 | |
|
358 | 0 | char *buf = x264_slurp_file( filename ); |
359 | 0 | if( !buf ) |
360 | 0 | { |
361 | 0 | x264_log( h, X264_LOG_ERROR, "can't open file '%s'\n", filename ); |
362 | 0 | return -1; |
363 | 0 | } |
364 | | |
365 | 0 | while( (p = strchr( buf, '#' )) != NULL ) |
366 | 0 | memset( p, ' ', strcspn( p, "\n" ) ); |
367 | |
|
368 | 0 | b_error |= cqm_parse_jmlist( h, buf, "INTRA4X4_LUMA", h->param.cqm_4iy, x264_cqm_jvt4i, 16 ); |
369 | 0 | b_error |= cqm_parse_jmlist( h, buf, "INTER4X4_LUMA", h->param.cqm_4py, x264_cqm_jvt4p, 16 ); |
370 | 0 | b_error |= cqm_parse_jmlist( h, buf, "INTRA4X4_CHROMA", h->param.cqm_4ic, x264_cqm_jvt4i, 16 ); |
371 | 0 | b_error |= cqm_parse_jmlist( h, buf, "INTER4X4_CHROMA", h->param.cqm_4pc, x264_cqm_jvt4p, 16 ); |
372 | 0 | b_error |= cqm_parse_jmlist( h, buf, "INTRA8X8_LUMA", h->param.cqm_8iy, x264_cqm_jvt8i, 64 ); |
373 | 0 | b_error |= cqm_parse_jmlist( h, buf, "INTER8X8_LUMA", h->param.cqm_8py, x264_cqm_jvt8p, 64 ); |
374 | 0 | if( CHROMA444 ) |
375 | 0 | { |
376 | 0 | b_error |= cqm_parse_jmlist( h, buf, "INTRA8X8_CHROMA", h->param.cqm_8ic, x264_cqm_jvt8i, 64 ); |
377 | 0 | b_error |= cqm_parse_jmlist( h, buf, "INTER8X8_CHROMA", h->param.cqm_8pc, x264_cqm_jvt8p, 64 ); |
378 | 0 | } |
379 | |
|
380 | 0 | x264_free( buf ); |
381 | 0 | return b_error; |
382 | 0 | } Unexecuted instantiation: x264_8_cqm_parse_file Unexecuted instantiation: x264_10_cqm_parse_file |
383 | | |