1 /*****************************************************************************
2 * set.c: quantization init
3 *****************************************************************************
4 * Copyright (C) 2005-2014 x264 project
5 *
6 * Authors: Loren Merritt <lorenm@u.washington.edu>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with this program; if not, write to the Free Software
20 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111, USA.
21 *
22 * This program is also available under a commercial proprietary license.
23 * For more information, contact us at licensing@x264.com.
24 *****************************************************************************/
25
26 #define _ISOC99_SOURCE
27 #include "common.h"
28
29 #define SHIFT(x,s) ((s)<=0 ? (x)<<-(s) : ((x)+(1<<((s)-1)))>>(s))
30 #define DIV(n,d) (((n) + ((d)>>1)) / (d))
31
32 static const uint8_t dequant4_scale[6][3] =
33 {
34 { 10, 13, 16 },
35 { 11, 14, 18 },
36 { 13, 16, 20 },
37 { 14, 18, 23 },
38 { 16, 20, 25 },
39 { 18, 23, 29 }
40 };
41 static const uint16_t quant4_scale[6][3] =
42 {
43 { 13107, 8066, 5243 },
44 { 11916, 7490, 4660 },
45 { 10082, 6554, 4194 },
46 { 9362, 5825, 3647 },
47 { 8192, 5243, 3355 },
48 { 7282, 4559, 2893 },
49 };
50
51 static const uint8_t quant8_scan[16] =
52 {
53 0,3,4,3, 3,1,5,1, 4,5,2,5, 3,1,5,1
54 };
55 static const uint8_t dequant8_scale[6][6] =
56 {
57 { 20, 18, 32, 19, 25, 24 },
58 { 22, 19, 35, 21, 28, 26 },
59 { 26, 23, 42, 24, 33, 31 },
60 { 28, 25, 45, 26, 35, 33 },
61 { 32, 28, 51, 30, 40, 38 },
62 { 36, 32, 58, 34, 46, 43 },
63 };
64 static const uint16_t quant8_scale[6][6] =
65 {
66 { 13107, 11428, 20972, 12222, 16777, 15481 },
67 { 11916, 10826, 19174, 11058, 14980, 14290 },
68 { 10082, 8943, 15978, 9675, 12710, 11985 },
69 { 9362, 8228, 14913, 8931, 11984, 11259 },
70 { 8192, 7346, 13159, 7740, 10486, 9777 },
71 { 7282, 6428, 11570, 6830, 9118, 8640 }
72 };
73
x264_cqm_init(x264_t * h)74 int x264_cqm_init( x264_t *h )
75 {
76 int def_quant4[6][16];
77 int def_quant8[6][64];
78 int def_dequant4[6][16];
79 int def_dequant8[6][64];
80 int quant4_mf[4][6][16];
81 int quant8_mf[4][6][64];
82 int deadzone[4] = { 32 - h->param.analyse.i_luma_deadzone[1],
83 32 - h->param.analyse.i_luma_deadzone[0],
84 32 - 11, 32 - 21 };
85 int max_qp_err = -1;
86 int max_chroma_qp_err = -1;
87 int min_qp_err = QP_MAX+1;
88 int num_8x8_lists = h->sps->i_chroma_format_idc == CHROMA_444 ? 4
89 : h->param.analyse.b_transform_8x8 ? 2 : 0; /* Checkasm may segfault if optimized out by --chroma-format */
90
91 int i;
92 int q;
93
94 #define CQM_ALLOC( w, count )\
95 for( i = 0; i < count; i++ )\
96 {\
97 int size = w*w;\
98 int start = w == 8 ? 4 : 0;\
99 int j;\
100 for( j = 0; j < i; j++ )\
101 if( !memcmp( h->pps->scaling_list[i+start], h->pps->scaling_list[j+start], size*sizeof(uint8_t) ) )\
102 break;\
103 if( j < i )\
104 {\
105 h-> quant##w##_mf[i] = h-> quant##w##_mf[j];\
106 h->dequant##w##_mf[i] = h->dequant##w##_mf[j];\
107 h->unquant##w##_mf[i] = h->unquant##w##_mf[j];\
108 }\
109 else\
110 {\
111 CHECKED_MALLOC( h-> quant##w##_mf[i], (QP_MAX_SPEC+1)*size*sizeof(udctcoef) );\
112 CHECKED_MALLOC( h->dequant##w##_mf[i], 6*size*sizeof(int) );\
113 CHECKED_MALLOC( h->unquant##w##_mf[i], (QP_MAX_SPEC+1)*size*sizeof(int) );\
114 }\
115 for( j = 0; j < i; j++ )\
116 if( deadzone[j] == deadzone[i] &&\
117 !memcmp( h->pps->scaling_list[i+start], h->pps->scaling_list[j+start], size*sizeof(uint8_t) ) )\
118 break;\
119 if( j < i )\
120 {\
121 h->quant##w##_bias[i] = h->quant##w##_bias[j];\
122 h->quant##w##_bias0[i] = h->quant##w##_bias0[j];\
123 }\
124 else\
125 {\
126 CHECKED_MALLOC( h->quant##w##_bias[i], (QP_MAX_SPEC+1)*size*sizeof(udctcoef) );\
127 CHECKED_MALLOC( h->quant##w##_bias0[i], (QP_MAX_SPEC+1)*size*sizeof(udctcoef) );\
128 }\
129 }
130
131 CQM_ALLOC( 4, 4 )
132 CQM_ALLOC( 8, num_8x8_lists )
133
134 for( q = 0; q < 6; q++ )
135 {
136 for( i = 0; i < 16; i++ )
137 {
138 int j = (i&1) + ((i>>2)&1);
139 def_dequant4[q][i] = dequant4_scale[q][j];
140 def_quant4[q][i] = quant4_scale[q][j];
141 }
142 for( i = 0; i < 64; i++ )
143 {
144 int j = quant8_scan[((i>>1)&12) | (i&3)];
145 def_dequant8[q][i] = dequant8_scale[q][j];
146 def_quant8[q][i] = quant8_scale[q][j];
147 }
148 }
149
150 for( q = 0; q < 6; q++ )
151 {
152 int i_list;
153
154 for( i_list = 0; i_list < 4; i_list++ )
155 for( i = 0; i < 16; i++ )
156 {
157 h->dequant4_mf[i_list][q][i] = def_dequant4[q][i] * h->pps->scaling_list[i_list][i];
158 quant4_mf[i_list][q][i] = DIV(def_quant4[q][i] * 16, h->pps->scaling_list[i_list][i]);
159 }
160 for( i_list = 0; i_list < num_8x8_lists; i_list++ )
161 for( i = 0; i < 64; i++ )
162 {
163 h->dequant8_mf[i_list][q][i] = def_dequant8[q][i] * h->pps->scaling_list[4+i_list][i];
164 quant8_mf[i_list][q][i] = DIV(def_quant8[q][i] * 16, h->pps->scaling_list[4+i_list][i]);
165 }
166 }
167 for( q = 0; q <= QP_MAX_SPEC; q++ )
168 {
169 int j;
170 int i_list;
171
172 for( i_list = 0; i_list < 4; i_list++ )
173 for( i = 0; i < 16; i++ )
174 {
175 h->unquant4_mf[i_list][q][i] = (ULLN(1) << (q/6 + 15 + 8)) / quant4_mf[i_list][q%6][i];
176 h->quant4_mf[i_list][q][i] = j = SHIFT(quant4_mf[i_list][q%6][i], q/6 - 1);
177 if( !j )
178 {
179 min_qp_err = X264_MIN( min_qp_err, q );
180 continue;
181 }
182 // round to nearest, unless that would cause the deadzone to be negative
183 h->quant4_bias[i_list][q][i] = X264_MIN( DIV(deadzone[i_list]<<10, j), (1<<15)/j );
184 h->quant4_bias0[i_list][q][i] = (1<<15)/j;
185 if( j > 0xffff && q > max_qp_err && (i_list == CQM_4IY || i_list == CQM_4PY) )
186 max_qp_err = q;
187 if( j > 0xffff && q > max_chroma_qp_err && (i_list == CQM_4IC || i_list == CQM_4PC) )
188 max_chroma_qp_err = q;
189 }
190 if( h->param.analyse.b_transform_8x8 )
191 for( i_list = 0; i_list < num_8x8_lists; i_list++ )
192 for( i = 0; i < 64; i++ )
193 {
194 h->unquant8_mf[i_list][q][i] = (ULLN(1) << (q/6 + 16 + 8)) / quant8_mf[i_list][q%6][i];
195 j = SHIFT(quant8_mf[i_list][q%6][i], q/6);
196 h->quant8_mf[i_list][q][i] = (uint16_t)j;
197
198 if( !j )
199 {
200 min_qp_err = X264_MIN( min_qp_err, q );
201 continue;
202 }
203 h->quant8_bias[i_list][q][i] = X264_MIN( DIV(deadzone[i_list]<<10, j), (1<<15)/j );
204 h->quant8_bias0[i_list][q][i] = (1<<15)/j;
205 if( j > 0xffff && q > max_qp_err && (i_list == CQM_8IY || i_list == CQM_8PY) )
206 max_qp_err = q;
207 if( j > 0xffff && q > max_chroma_qp_err && (i_list == CQM_8IC || i_list == CQM_8PC) )
208 max_chroma_qp_err = q;
209 }
210 }
211
212 /* Emergency mode denoising. */
213 x264_emms();
214 CHECKED_MALLOC( h->nr_offset_emergency, sizeof(*h->nr_offset_emergency)*(QP_MAX-QP_MAX_SPEC) );
215 for( q = 0; q < QP_MAX - QP_MAX_SPEC; q++ ) {
216 int cat;
217 for( cat = 0; cat < 3 + CHROMA444; cat++ )
218 {
219 int dct8x8 = cat&1;
220 int size;
221 udctcoef *nr_offset;
222 /* Denoise chroma first (due to h264's chroma QP offset), then luma, then DC. */
223 int dc_threshold;
224 int luma_threshold;
225 int chroma_threshold;
226
227 if( !h->param.analyse.b_transform_8x8 && dct8x8 )
228 continue;
229
230 size = dct8x8 ? 64 : 16;
231 nr_offset = h->nr_offset_emergency[q][cat];
232 /* Denoise chroma first (due to h264's chroma QP offset), then luma, then DC. */
233 dc_threshold = (QP_MAX-QP_MAX_SPEC)*2/3;
234 luma_threshold = (QP_MAX-QP_MAX_SPEC)*2/3;
235 chroma_threshold = 0;
236
237 for( i = 0; i < size; i++ )
238 {
239 int thresh;
240 int max = (1 << (7 + BIT_DEPTH)) - 1;
241 double pos;
242 double start;
243 double bias;
244
245 /* True "emergency mode": remove all DCT coefficients */
246 if( q == QP_MAX - QP_MAX_SPEC - 1 )
247 {
248 nr_offset[i] = max;
249 continue;
250 }
251
252 thresh = i == 0 ? dc_threshold : cat >= 2 ? chroma_threshold : luma_threshold;
253 if( q < thresh )
254 {
255 nr_offset[i] = 0;
256 continue;
257 }
258 pos = (double)(q-thresh+1) / (QP_MAX - QP_MAX_SPEC - thresh);
259
260 /* XXX: this math is largely tuned for /dev/random input. */
261 start = dct8x8 ? h->unquant8_mf[CQM_8PY][QP_MAX_SPEC][i]
262 : h->unquant4_mf[CQM_4PY][QP_MAX_SPEC][i];
263 /* Formula chosen as an exponential scale to vaguely mimic the effects
264 * of a higher quantizer. */
265 bias = (pow( 2, pos*(QP_MAX - QP_MAX_SPEC)/10. )*0.003-0.003) * start;
266 nr_offset[i] = X264_MIN( bias + 0.5, max );
267 }
268 }
269 }
270
271 if( !h->mb.b_lossless )
272 {
273 while( h->chroma_qp_table[SPEC_QP(h->param.rc.i_qp_min)] <= max_chroma_qp_err )
274 h->param.rc.i_qp_min++;
275 if( min_qp_err <= h->param.rc.i_qp_max )
276 h->param.rc.i_qp_max = min_qp_err-1;
277 if( max_qp_err >= h->param.rc.i_qp_min )
278 h->param.rc.i_qp_min = max_qp_err+1;
279 /* If long level-codes aren't allowed, we need to allow QP high enough to avoid them. */
280 if( !h->param.b_cabac && h->sps->i_profile_idc < PROFILE_HIGH )
281 while( h->chroma_qp_table[SPEC_QP(h->param.rc.i_qp_max)] <= 12 || h->param.rc.i_qp_max <= 12 )
282 h->param.rc.i_qp_max++;
283 if( h->param.rc.i_qp_min > h->param.rc.i_qp_max )
284 {
285 x264_log( h, X264_LOG_ERROR, "Impossible QP constraints for CQM (min=%d, max=%d)\n", h->param.rc.i_qp_min, h->param.rc.i_qp_max );
286 return -1;
287 }
288 }
289 return 0;
290 fail:
291 x264_cqm_delete( h );
292 return -1;
293 }
294
295 #define CQM_DELETE( n, max )\
296 for( i = 0; i < (max); i++ )\
297 {\
298 int j;\
299 for( j = 0; j < i; j++ )\
300 if( h->quant##n##_mf[i] == h->quant##n##_mf[j] )\
301 break;\
302 if( j == i )\
303 {\
304 x264_free( h-> quant##n##_mf[i] );\
305 x264_free( h->dequant##n##_mf[i] );\
306 x264_free( h->unquant##n##_mf[i] );\
307 }\
308 for( j = 0; j < i; j++ )\
309 if( h->quant##n##_bias[i] == h->quant##n##_bias[j] )\
310 break;\
311 if( j == i )\
312 {\
313 x264_free( h->quant##n##_bias[i] );\
314 x264_free( h->quant##n##_bias0[i] );\
315 }\
316 }
317
x264_cqm_delete(x264_t * h)318 void x264_cqm_delete( x264_t *h )
319 {
320 int i;
321 CQM_DELETE( 4, 4 );
322 CQM_DELETE( 8, CHROMA444 ? 4 : 2 );
323 x264_free( h->nr_offset_emergency );
324 }
325
x264_cqm_parse_jmlist(x264_t * h,const char * buf,const char * name,uint8_t * cqm,const uint8_t * jvt,int length)326 static int x264_cqm_parse_jmlist( x264_t *h, const char *buf, const char *name,
327 uint8_t *cqm, const uint8_t *jvt, int length )
328 {
329 int i;
330 char *nextvar;
331 char *p = strstr( buf, name );
332
333 if( !p )
334 {
335 memset( cqm, 16, length );
336 return 0;
337 }
338
339 p += strlen( name );
340 if( *p == 'U' || *p == 'V' )
341 p++;
342
343 nextvar = strstr( p, "INT" );
344
345 for( i = 0; i < length && (p = strpbrk( p, " \t\n," )) && (p = strpbrk( p, "0123456789" )); i++ )
346 {
347 int coef = -1;
348 sscanf( p, "%d", &coef );
349 if( i == 0 && coef == 0 )
350 {
351 memcpy( cqm, jvt, length );
352 return 0;
353 }
354 if( coef < 1 || coef > 255 )
355 {
356 x264_log( h, X264_LOG_ERROR, "bad coefficient in list '%s'\n", name );
357 return -1;
358 }
359 cqm[i] = coef;
360 }
361
362 if( (nextvar && p > nextvar) || i != length )
363 {
364 x264_log( h, X264_LOG_ERROR, "not enough coefficients in list '%s'\n", name );
365 return -1;
366 }
367
368 return 0;
369 }
370
x264_cqm_parse_file(x264_t * h,const char * filename)371 int x264_cqm_parse_file( x264_t *h, const char *filename )
372 {
373 char *p;
374 int b_error = 0;
375 char *buf;
376
377 h->param.i_cqm_preset = X264_CQM_CUSTOM;
378
379 buf = x264_slurp_file( filename );
380 if( !buf )
381 {
382 x264_log( h, X264_LOG_ERROR, "can't open file '%s'\n", filename );
383 return -1;
384 }
385
386 while( (p = strchr( buf, '#' )) != NULL )
387 memset( p, ' ', strcspn( p, "\n" ) );
388
389 b_error |= x264_cqm_parse_jmlist( h, buf, "INTRA4X4_LUMA", h->param.cqm_4iy, x264_cqm_jvt4i, 16 );
390 b_error |= x264_cqm_parse_jmlist( h, buf, "INTER4X4_LUMA", h->param.cqm_4py, x264_cqm_jvt4p, 16 );
391 b_error |= x264_cqm_parse_jmlist( h, buf, "INTRA4X4_CHROMA", h->param.cqm_4ic, x264_cqm_jvt4i, 16 );
392 b_error |= x264_cqm_parse_jmlist( h, buf, "INTER4X4_CHROMA", h->param.cqm_4pc, x264_cqm_jvt4p, 16 );
393 b_error |= x264_cqm_parse_jmlist( h, buf, "INTRA8X8_LUMA", h->param.cqm_8iy, x264_cqm_jvt8i, 64 );
394 b_error |= x264_cqm_parse_jmlist( h, buf, "INTER8X8_LUMA", h->param.cqm_8py, x264_cqm_jvt8p, 64 );
395 if( CHROMA444 )
396 {
397 b_error |= x264_cqm_parse_jmlist( h, buf, "INTRA8X8_CHROMA", h->param.cqm_8ic, x264_cqm_jvt8i, 64 );
398 b_error |= x264_cqm_parse_jmlist( h, buf, "INTER8X8_CHROMA", h->param.cqm_8pc, x264_cqm_jvt8p, 64 );
399 }
400
401 x264_free( buf );
402 return b_error;
403 }
404
405