1 /*
2 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
3 *
4 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10 */
11
12 #ifndef AOM_DSP_TXFM_COMMON_H_
13 #define AOM_DSP_TXFM_COMMON_H_
14
15 #include "aom_dsp/aom_dsp_common.h"
16 #include "av1/common/enums.h"
17
18 // Constants and Macros used by all idct/dct functions
19 #define DCT_CONST_BITS 14
20 #define DCT_CONST_ROUNDING (1 << (DCT_CONST_BITS - 1))
21
22 #define UNIT_QUANT_SHIFT 2
23 #define UNIT_QUANT_FACTOR (1 << UNIT_QUANT_SHIFT)
24
25 typedef struct txfm_param {
26 // for both forward and inverse transforms
27 TX_TYPE tx_type;
28 TX_SIZE tx_size;
29 int lossless;
30 int bd;
31 #if CONFIG_MRC_TX || CONFIG_LGT
32 int is_inter;
33 #endif // CONFIG_MRC_TX || CONFIG_LGT
34 #if CONFIG_MRC_TX || CONFIG_LGT_FROM_PRED
35 int stride;
36 uint8_t *dst;
37 #if CONFIG_MRC_TX
38 int *valid_mask;
39 uint8_t *mask;
40 #endif // CONFIG_MRC_TX
41 #if CONFIG_LGT_FROM_PRED
42 int mode;
43 int use_lgt;
44 #endif // CONFIG_LGT_FROM_PRED
45 #endif // CONFIG_MRC_TX || CONFIG_LGT_FROM_PRED
46 // for inverse transforms only
47 #if CONFIG_ADAPT_SCAN
48 const int16_t *eob_threshold;
49 #endif
50 int eob;
51 } TxfmParam;
52
53 // Constants:
54 // for (int i = 1; i< 32; ++i)
55 // printf("static const int cospi_%d_64 = %.0f;\n", i,
56 // round(16384 * cos(i*M_PI/64)));
57 // Note: sin(k*Pi/64) = cos((32-k)*Pi/64)
58 static const tran_high_t cospi_1_64 = 16364;
59 static const tran_high_t cospi_2_64 = 16305;
60 static const tran_high_t cospi_3_64 = 16207;
61 static const tran_high_t cospi_4_64 = 16069;
62 static const tran_high_t cospi_5_64 = 15893;
63 static const tran_high_t cospi_6_64 = 15679;
64 static const tran_high_t cospi_7_64 = 15426;
65 static const tran_high_t cospi_8_64 = 15137;
66 static const tran_high_t cospi_9_64 = 14811;
67 static const tran_high_t cospi_10_64 = 14449;
68 static const tran_high_t cospi_11_64 = 14053;
69 static const tran_high_t cospi_12_64 = 13623;
70 static const tran_high_t cospi_13_64 = 13160;
71 static const tran_high_t cospi_14_64 = 12665;
72 static const tran_high_t cospi_15_64 = 12140;
73 static const tran_high_t cospi_16_64 = 11585;
74 static const tran_high_t cospi_17_64 = 11003;
75 static const tran_high_t cospi_18_64 = 10394;
76 static const tran_high_t cospi_19_64 = 9760;
77 static const tran_high_t cospi_20_64 = 9102;
78 static const tran_high_t cospi_21_64 = 8423;
79 static const tran_high_t cospi_22_64 = 7723;
80 static const tran_high_t cospi_23_64 = 7005;
81 static const tran_high_t cospi_24_64 = 6270;
82 static const tran_high_t cospi_25_64 = 5520;
83 static const tran_high_t cospi_26_64 = 4756;
84 static const tran_high_t cospi_27_64 = 3981;
85 static const tran_high_t cospi_28_64 = 3196;
86 static const tran_high_t cospi_29_64 = 2404;
87 static const tran_high_t cospi_30_64 = 1606;
88 static const tran_high_t cospi_31_64 = 804;
89
90 // 16384 * sqrt(2) * sin(kPi/9) * 2 / 3
91 static const tran_high_t sinpi_1_9 = 5283;
92 static const tran_high_t sinpi_2_9 = 9929;
93 static const tran_high_t sinpi_3_9 = 13377;
94 static const tran_high_t sinpi_4_9 = 15212;
95
96 // 16384 * sqrt(2)
97 static const tran_high_t Sqrt2 = 23170;
98 static const tran_high_t InvSqrt2 = 11585;
99
fdct_round_shift(tran_high_t input)100 static INLINE tran_high_t fdct_round_shift(tran_high_t input) {
101 tran_high_t rv = ROUND_POWER_OF_TWO(input, DCT_CONST_BITS);
102 return rv;
103 }
104
105 #if CONFIG_LGT_FROM_PRED
106 // Use negative numbers so they do not coincide with lgt*[0][0], which are
107 // always nonnegative.
108 typedef enum {
109 DCT4 = -1,
110 ADST4 = -2,
111 DCT8 = -3,
112 ADST8 = -4,
113 DCT16 = -5,
114 ADST16 = -6,
115 DCT32 = -7,
116 ADST32 = -8,
117 } ButterflyLgt;
118
119 /* These are some LGTs already implementated in the codec. When any of them
120 * is chosen, the flgt or ilgt function will call the existing fast
121 * transform instead of the matrix product implementation. Thus, we
122 * do not need the actual basis functions here */
123 static const tran_high_t lgt4_000[1][1] = { { (tran_high_t)DCT4 } };
124 static const tran_high_t lgt4_100[1][1] = { { (tran_high_t)ADST4 } };
125 static const tran_high_t lgt8_000[1][1] = { { (tran_high_t)DCT8 } };
126 static const tran_high_t lgt8_200[1][1] = { { (tran_high_t)ADST8 } };
127 static const tran_high_t lgt16_000[1][1] = { { (tran_high_t)DCT16 } };
128 static const tran_high_t lgt16_200[1][1] = { { (tran_high_t)ADST16 } };
129 static const tran_high_t lgt32_000[1][1] = { { (tran_high_t)DCT32 } };
130 static const tran_high_t lgt32_200[1][1] = { { (tran_high_t)ADST32 } };
131
132 /* The Line Graph Transforms (LGTs) matrices are written as follows.
133 Each 2D array is sqrt(2)*16384 times an LGT matrix, which is the
134 matrix of eigenvectors of the graph Laplacian matrix of the associated
135 line graph. Some of those transforms have fast algorithms but not
136 implemented yet for now. */
137
138 // LGT4 name: lgt4_150_000w3
139 // Self loops: 1.500, 0.000, 0.000, 0.000
140 // Edges: 1.000, 1.000, 0.000
141 static const tran_high_t lgt4_150_000w3[4][4] = {
142 { 0, 0, 0, 23170 },
143 { 5991, 13537, 17825, 0 },
144 { 15515, 10788, -13408, 0 },
145 { 16133, -15403, 6275, 0 },
146 };
147
148 // LGT4 name: lgt4_100_000w3
149 // Self loops: 1.000, 0.000, 0.000, 0.000
150 // Edges: 1.000, 1.000, 0.000
151 static const tran_high_t lgt4_100_000w3[4][4] = {
152 { 0, 0, 0, 23170 },
153 { 7600, 13694, 17076, 0 },
154 { 17076, 7600, -13694, 0 },
155 { 13694, -17076, 7600, 0 },
156 };
157
158 // LGT4 name: lgt4_060_000w3
159 // Self loops: 0.600, 0.000, 0.000, 0.000
160 // Edges: 1.000, 1.000, 0.000
161 static const tran_high_t lgt4_060_000w3[4][4] = {
162 { 0, 0, 0, 23170 },
163 { 9449, 13755, 16075, 0 },
164 { 17547, 4740, -14370, 0 },
165 { 11819, -18034, 8483, 0 },
166 };
167
168 // LGT4 name: lgt4_000w3
169 // Self loops: 0.000, 0.000, 0.000, 0.000
170 // Edges: 1.000, 1.000, 0.000
171 static const tran_high_t lgt4_000w3[4][4] = {
172 { 0, 0, 0, 23170 },
173 { 13377, 13377, 13377, 0 },
174 { 16384, 0, -16384, 0 },
175 { 9459, -18919, 9459, 0 },
176 };
177
178 // LGT4 name: lgt4_150_000w2
179 // Self loops: 1.500, 0.000, 0.000, 0.000
180 // Edges: 1.000, 0.000, 1.000
181 static const tran_high_t lgt4_150_000w2[4][4] = {
182 { 10362, 20724, 0, 0 },
183 { 20724, -10362, 0, 0 },
184 { 0, 0, 16384, 16384 },
185 { 0, 0, 16384, -16384 },
186 };
187
188 // LGT4 name: lgt4_100_000w2
189 // Self loops: 1.000, 0.000, 0.000, 0.000
190 // Edges: 1.000, 0.000, 1.000
191 static const tran_high_t lgt4_100_000w2[4][4] = {
192 { 12181, 19710, 0, 0 },
193 { 19710, -12181, 0, 0 },
194 { 0, 0, 16384, 16384 },
195 { 0, 0, 16384, -16384 },
196 };
197
198 // LGT4 name: lgt4_060_000w2
199 // Self loops: 0.600, 0.000, 0.000, 0.000
200 // Edges: 1.000, 0.000, 1.000
201 static const tran_high_t lgt4_060_000w2[4][4] = {
202 { 13831, 18590, 0, 0 },
203 { 18590, -13831, 0, 0 },
204 { 0, 0, 16384, 16384 },
205 { 0, 0, 16384, -16384 },
206 };
207
208 // LGT4 name: lgt4_000w2
209 // Self loops: 0.000, 0.000, 0.000, 0.000
210 // Edges: 1.000, 0.000, 1.000
211 static const tran_high_t lgt4_000w2[4][4] = {
212 { 16384, 16384, 0, 0 },
213 { 16384, -16384, 0, 0 },
214 { 0, 0, 16384, 16384 },
215 { 0, 0, 16384, -16384 },
216 };
217
218 // LGT4 name: lgt4_150_000w1
219 // Self loops: 1.500, 0.000, 0.000, 0.000
220 // Edges: 0.000, 1.000, 1.000
221 static const tran_high_t lgt4_150_000w1[4][4] = {
222 { 23170, 0, 0, 0 },
223 { 0, 13377, 13377, 13377 },
224 { 0, 16384, 0, -16384 },
225 { 0, 9459, -18919, 9459 },
226 };
227
228 // LGT4 name: lgt4_100_000w1
229 // Self loops: 1.000, 0.000, 0.000, 0.000
230 // Edges: 0.000, 1.000, 1.000
231 static const tran_high_t lgt4_100_000w1[4][4] = {
232 { 23170, 0, 0, 0 },
233 { 0, 13377, 13377, 13377 },
234 { 0, 16384, 0, -16384 },
235 { 0, 9459, -18919, 9459 },
236 };
237
238 // LGT4 name: lgt4_060_000w1
239 // Self loops: 0.600, 0.000, 0.000, 0.000
240 // Edges: 0.000, 1.000, 1.000
241 static const tran_high_t lgt4_060_000w1[4][4] = {
242 { 23170, 0, 0, 0 },
243 { 0, 13377, 13377, 13377 },
244 { 0, 16384, 0, -16384 },
245 { 0, 9459, -18919, 9459 },
246 };
247
248 // LGT4 name: lgt4_000w1
249 // Self loops: 0.000, 0.000, 0.000, 0.000
250 // Edges: 0.000, 1.000, 1.000
251 static const tran_high_t lgt4_000w1[4][4] = {
252 { 23170, 0, 0, 0 },
253 { 0, 13377, 13377, 13377 },
254 { 0, 16384, 0, -16384 },
255 { 0, 9459, -18919, 9459 },
256 };
257
258 // LGT4 name: lgt4_060
259 // Self loops: 0.600, 0.000, 0.000, 0.000
260 // Edges: 1.000, 1.000, 1.000
261 static const tran_high_t lgt4_060[4][4] = {
262 { 6971, 10504, 13060, 14400 },
263 { 14939, 11211, -2040, -13559 },
264 { 14096, -8258, -12561, 10593 },
265 { 8150, -15253, 14295, -5784 },
266 };
267
268 // LGT4 name: lgt4_150
269 // Self loops: 1.500, 0.000, 0.000, 0.000
270 // Edges: 1.000, 1.000, 1.000
271 static const tran_high_t lgt4_150[4][4] = {
272 { 3998, 9435, 13547, 15759 },
273 { 11106, 15105, 1886, -13483 },
274 { 15260, -1032, -14674, 9361 },
275 { 12833, -14786, 11596, -4372 },
276 };
277
278 // LGT8 name: lgt8_150_000w7
279 // Self loops: 1.500, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
280 // Edges: 1.000, 1.000, 1.000, 1.000, 1.000, 1.000, 0.000
281 static const tran_high_t lgt8_150_000w7[8][8] = {
282 { 0, 0, 0, 0, 0, 0, 0, 32768 },
283 { 2522, 6185, 9551, 12461, 14775, 16381, 17204, 0 },
284 { 7390, 15399, 16995, 11515, 1240, -9551, -16365, 0 },
285 { 11716, 16625, 3560, -13353, -15831, -1194, 14733, 0 },
286 { 15073, 8866, -14291, -10126, 13398, 11308, -12401, 0 },
287 { 16848, -4177, -13724, 14441, 2923, -16628, 9513, 0 },
288 { 15942, -14888, 5405, 7137, -15640, 15288, -6281, 0 },
289 { 10501, -14293, 16099, -15670, 13063, -8642, 3021, 0 },
290 };
291
292 // LGT8 name: lgt8_100_000w7
293 // Self loops: 1.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
294 // Edges: 1.000, 1.000, 1.000, 1.000, 1.000, 1.000, 0.000
295 static const tran_high_t lgt8_100_000w7[8][8] = {
296 { 0, 0, 0, 0, 0, 0, 0, 32768 },
297 { 3518, 6883, 9946, 12575, 14654, 16093, 16829, 0 },
298 { 9946, 16093, 16093, 9946, 0, -9946, -16093, 0 },
299 { 14654, 14654, 0, -14654, -14654, 0, 14654, 0 },
300 { 16829, 3518, -16093, -6883, 14654, 9946, -12575, 0 },
301 { 16093, -9946, -9946, 16093, 0, -16093, 9946, 0 },
302 { 12575, -16829, 9946, 3518, -14654, 16093, -6883, 0 },
303 { 6883, -12575, 16093, -16829, 14654, -9946, 3518, 0 },
304 };
305
306 // LGT8 name: lgt8_060_000w7
307 // Self loops: 0.600, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
308 // Edges: 1.000, 1.000, 1.000, 1.000, 1.000, 1.000, 0.000
309 static const tran_high_t lgt8_060_000w7[8][8] = {
310 { 0, 0, 0, 0, 0, 0, 0, 32768 },
311 { 5087, 7951, 10521, 12701, 14411, 15587, 16186, 0 },
312 { 13015, 16486, 14464, 7621, -1762, -10557, -15834, 0 },
313 { 16581, 11475, -4050, -15898, -13311, 1362, 14798, 0 },
314 { 16536, -1414, -16981, -3927, 15746, 8879, -12953, 0 },
315 { 14104, -13151, -7102, 16932, -1912, -15914, 10385, 0 },
316 { 10156, -17168, 11996, 1688, -14174, 16602, -7249, 0 },
317 { 5295, -11721, 15961, -17224, 15274, -10476, 3723, 0 },
318 };
319
320 // LGT8 name: lgt8_000w7
321 // Self loops: 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
322 // Edges: 1.000, 1.000, 1.000, 1.000, 1.000, 1.000, 0.000
323 static const tran_high_t lgt8_000w7[8][8] = {
324 { 0, 0, 0, 0, 0, 0, 0, 32768 },
325 { 12385, 12385, 12385, 12385, 12385, 12385, 12385, 0 },
326 { 17076, 13694, 7600, 0, -7600, -13694, -17076, 0 },
327 { 15781, 3898, -10921, -17515, -10921, 3898, 15781, 0 },
328 { 13694, -7600, -17076, 0, 17076, 7600, -13694, 0 },
329 { 10921, -15781, -3898, 17515, -3898, -15781, 10921, 0 },
330 { 7600, -17076, 13694, 0, -13694, 17076, -7600, 0 },
331 { 3898, -10921, 15781, -17515, 15781, -10921, 3898, 0 },
332 };
333
334 // LGT8 name: lgt8_150_000w6
335 // Self loops: 1.500, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
336 // Edges: 1.000, 1.000, 1.000, 1.000, 1.000, 0.000, 1.000
337 static const tran_high_t lgt8_150_000w6[8][8] = {
338 { 0, 0, 0, 0, 0, 0, 23170, 23170 },
339 { 0, 0, 0, 0, 0, 0, 23170, -23170 },
340 { 3157, 7688, 11723, 15002, 17312, 18506, 0, 0 },
341 { 9167, 17832, 16604, 6164, -7696, -17286, 0, 0 },
342 { 14236, 15584, -4969, -18539, -6055, 14938, 0, 0 },
343 { 17558, 1891, -18300, 5288, 16225, -11653, 0, 0 },
344 { 17776, -13562, -647, 14380, -17514, 7739, 0, 0 },
345 { 12362, -16318, 17339, -15240, 10399, -3688, 0, 0 },
346 };
347
348 // LGT8 name: lgt8_100_000w6
349 // Self loops: 1.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
350 // Edges: 1.000, 1.000, 1.000, 1.000, 1.000, 0.000, 1.000
351 static const tran_high_t lgt8_100_000w6[8][8] = {
352 { 0, 0, 0, 0, 0, 0, 23170, 23170 },
353 { 0, 0, 0, 0, 0, 0, 23170, -23170 },
354 { 4350, 8447, 12053, 14959, 16995, 18044, 0, 0 },
355 { 12053, 18044, 14959, 4350, -8447, -16995, 0, 0 },
356 { 16995, 12053, -8447, -18044, -4350, 14959, 0, 0 },
357 { 18044, -4350, -16995, 8447, 14959, -12053, 0, 0 },
358 { 14959, -16995, 4350, 12053, -18044, 8447, 0, 0 },
359 { 8447, -14959, 18044, -16995, 12053, -4350, 0, 0 },
360 };
361
362 // LGT8 name: lgt8_060_000w6
363 // Self loops: 0.600, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
364 // Edges: 1.000, 1.000, 1.000, 1.000, 1.000, 0.000, 1.000
365 static const tran_high_t lgt8_060_000w6[8][8] = {
366 { 0, 0, 0, 0, 0, 0, 23170, 23170 },
367 { 0, 0, 0, 0, 0, 0, 23170, -23170 },
368 { 6154, 9551, 12487, 14823, 16446, 17277, 0, 0 },
369 { 15149, 17660, 12503, 1917, -9502, -16795, 0, 0 },
370 { 18166, 7740, -11772, -17465, -2656, 15271, 0, 0 },
371 { 16682, -8797, -15561, 10779, 14189, -12586, 0, 0 },
372 { 12436, -18234, 7007, 10763, -18483, 8945, 0, 0 },
373 { 6591, -14172, 18211, -17700, 12766, -4642, 0, 0 },
374 };
375
376 // LGT8 name: lgt8_000w6
377 // Self loops: 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
378 // Edges: 1.000, 1.000, 1.000, 1.000, 1.000, 0.000, 1.000
379 static const tran_high_t lgt8_000w6[8][8] = {
380 { 0, 0, 0, 0, 0, 0, 23170, 23170 },
381 { 0, 0, 0, 0, 0, 0, 23170, -23170 },
382 { 13377, 13377, 13377, 13377, 13377, 13377, 0, 0 },
383 { 18274, 13377, 4896, -4896, -13377, -18274, 0, 0 },
384 { 16384, 0, -16384, -16384, 0, 16384, 0, 0 },
385 { 13377, -13377, -13377, 13377, 13377, -13377, 0, 0 },
386 { 9459, -18919, 9459, 9459, -18919, 9459, 0, 0 },
387 { 4896, -13377, 18274, -18274, 13377, -4896, 0, 0 },
388 };
389
390 // LGT8 name: lgt8_150_000w5
391 // Self loops: 1.500, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
392 // Edges: 1.000, 1.000, 1.000, 1.000, 0.000, 1.000, 1.000
393 static const tran_high_t lgt8_150_000w5[8][8] = {
394 { 0, 0, 0, 0, 0, 18919, 18919, 18919 },
395 { 0, 0, 0, 0, 0, 23170, 0, -23170 },
396 { 0, 0, 0, 0, 0, 13377, -26755, 13377 },
397 { 4109, 9895, 14774, 18299, 20146, 0, 0, 0 },
398 { 11753, 20300, 13161, -4148, -18252, 0, 0, 0 },
399 { 17573, 10921, -16246, -12895, 14679, 0, 0, 0 },
400 { 19760, -9880, -9880, 19760, -9880, 0, 0, 0 },
401 { 14815, -18624, 17909, -12844, 4658, 0, 0, 0 },
402 };
403
404 // LGT8 name: lgt8_100_000w5
405 // Self loops: 1.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
406 // Edges: 1.000, 1.000, 1.000, 1.000, 0.000, 1.000, 1.000
407 static const tran_high_t lgt8_100_000w5[8][8] = {
408 { 0, 0, 0, 0, 0, 18919, 18919, 18919 },
409 { 0, 0, 0, 0, 0, 23170, 0, -23170 },
410 { 0, 0, 0, 0, 0, 13377, -26755, 13377 },
411 { 5567, 10683, 14933, 17974, 19559, 0, 0, 0 },
412 { 14933, 19559, 10683, -5567, -17974, 0, 0, 0 },
413 { 19559, 5567, -17974, -10683, 14933, 0, 0, 0 },
414 { 17974, -14933, -5567, 19559, -10683, 0, 0, 0 },
415 { 10683, -17974, 19559, -14933, 5567, 0, 0, 0 },
416 };
417
418 // LGT8 name: lgt8_060_000w5
419 // Self loops: 0.600, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
420 // Edges: 1.000, 1.000, 1.000, 1.000, 0.000, 1.000, 1.000
421 static const tran_high_t lgt8_060_000w5[8][8] = {
422 { 0, 0, 0, 0, 0, 18919, 18919, 18919 },
423 { 0, 0, 0, 0, 0, 23170, 0, -23170 },
424 { 0, 0, 0, 0, 0, 13377, -26755, 13377 },
425 { 7650, 11741, 15069, 17415, 18628, 0, 0, 0 },
426 { 17824, 18002, 7558, -7345, -17914, 0, 0, 0 },
427 { 19547, 569, -19303, -8852, 15505, 0, 0, 0 },
428 { 15592, -17548, -2862, 19625, -11374, 0, 0, 0 },
429 { 8505, -17423, 20218, -15907, 6006, 0, 0, 0 },
430 };
431
432 // LGT8 name: lgt8_000w5
433 // Self loops: 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
434 // Edges: 1.000, 1.000, 1.000, 1.000, 0.000, 1.000, 1.000
435 static const tran_high_t lgt8_000w5[8][8] = {
436 { 0, 0, 0, 0, 0, 18919, 18919, 18919 },
437 { 0, 0, 0, 0, 0, 23170, 0, -23170 },
438 { 0, 0, 0, 0, 0, 13377, -26755, 13377 },
439 { 14654, 14654, 14654, 14654, 14654, 0, 0, 0 },
440 { 19710, 12181, 0, -12181, -19710, 0, 0, 0 },
441 { 16766, -6404, -20724, -6404, 16766, 0, 0, 0 },
442 { 12181, -19710, 0, 19710, -12181, 0, 0, 0 },
443 { 6404, -16766, 20724, -16766, 6404, 0, 0, 0 },
444 };
445
446 // LGT8 name: lgt8_150_000w4
447 // Self loops: 1.500, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
448 // Edges: 1.000, 1.000, 1.000, 0.000, 1.000, 1.000, 1.000
449 static const tran_high_t lgt8_150_000w4[8][8] = {
450 { 5655, 13343, 19159, 22286, 0, 0, 0, 0 },
451 { 15706, 21362, 2667, -19068, 0, 0, 0, 0 },
452 { 21580, -1459, -20752, 13238, 0, 0, 0, 0 },
453 { 18148, -20910, 16399, -6183, 0, 0, 0, 0 },
454 { 0, 0, 0, 0, 16384, 16384, 16384, 16384 },
455 { 0, 0, 0, 0, 21407, 8867, -8867, -21407 },
456 { 0, 0, 0, 0, 16384, -16384, -16384, 16384 },
457 { 0, 0, 0, 0, 8867, -21407, 21407, -8867 },
458 };
459
460 // LGT8 name: lgt8_100_000w4
461 // Self loops: 1.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
462 // Edges: 1.000, 1.000, 1.000, 0.000, 1.000, 1.000, 1.000
463 static const tran_high_t lgt8_100_000w4[8][8] = {
464 { 7472, 14042, 18919, 21513, 0, 0, 0, 0 },
465 { 18919, 18919, 0, -18919, 0, 0, 0, 0 },
466 { 21513, -7472, -18919, 14042, 0, 0, 0, 0 },
467 { 14042, -21513, 18919, -7472, 0, 0, 0, 0 },
468 { 0, 0, 0, 0, 16384, 16384, 16384, 16384 },
469 { 0, 0, 0, 0, 21407, 8867, -8867, -21407 },
470 { 0, 0, 0, 0, 16384, -16384, -16384, 16384 },
471 { 0, 0, 0, 0, 8867, -21407, 21407, -8867 },
472 };
473
474 // LGT8 name: lgt8_060_000w4
475 // Self loops: 0.600, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
476 // Edges: 1.000, 1.000, 1.000, 0.000, 1.000, 1.000, 1.000
477 static const tran_high_t lgt8_060_000w4[8][8] = {
478 { 9858, 14855, 18470, 20365, 0, 0, 0, 0 },
479 { 21127, 15855, -2886, -19175, 0, 0, 0, 0 },
480 { 19935, -11679, -17764, 14980, 0, 0, 0, 0 },
481 { 11525, -21570, 20217, -8180, 0, 0, 0, 0 },
482 { 0, 0, 0, 0, 16384, 16384, 16384, 16384 },
483 { 0, 0, 0, 0, 21407, 8867, -8867, -21407 },
484 { 0, 0, 0, 0, 16384, -16384, -16384, 16384 },
485 { 0, 0, 0, 0, 8867, -21407, 21407, -8867 },
486 };
487
488 // LGT8 name: lgt8_000w4
489 // Self loops: 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
490 // Edges: 1.000, 1.000, 1.000, 0.000, 1.000, 1.000, 1.000
491 static const tran_high_t lgt8_000w4[8][8] = {
492 { 16384, 16384, 16384, 16384, 0, 0, 0, 0 },
493 { 21407, 8867, -8867, -21407, 0, 0, 0, 0 },
494 { 16384, -16384, -16384, 16384, 0, 0, 0, 0 },
495 { 8867, -21407, 21407, -8867, 0, 0, 0, 0 },
496 { 0, 0, 0, 0, 16384, 16384, 16384, 16384 },
497 { 0, 0, 0, 0, 21407, 8867, -8867, -21407 },
498 { 0, 0, 0, 0, 16384, -16384, -16384, 16384 },
499 { 0, 0, 0, 0, 8867, -21407, 21407, -8867 },
500 };
501
502 // LGT8 name: lgt8_150_000w3
503 // Self loops: 1.500, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
504 // Edges: 1.000, 1.000, 0.000, 1.000, 1.000, 1.000, 1.000
505 static const tran_high_t lgt8_150_000w3[8][8] = {
506 { 8473, 19144, 25209, 0, 0, 0, 0, 0 },
507 { 21942, 15257, -18961, 0, 0, 0, 0, 0 },
508 { 22815, -21783, 8874, 0, 0, 0, 0, 0 },
509 { 0, 0, 0, 14654, 14654, 14654, 14654, 14654 },
510 { 0, 0, 0, 19710, 12181, 0, -12181, -19710 },
511 { 0, 0, 0, 16766, -6404, -20724, -6404, 16766 },
512 { 0, 0, 0, 12181, -19710, 0, 19710, -12181 },
513 { 0, 0, 0, 6404, -16766, 20724, -16766, 6404 },
514 };
515
516 // LGT8 name: lgt8_100_000w3
517 // Self loops: 1.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
518 // Edges: 1.000, 1.000, 0.000, 1.000, 1.000, 1.000, 1.000
519 static const tran_high_t lgt8_100_000w3[8][8] = {
520 { 10747, 19366, 24149, 0, 0, 0, 0, 0 },
521 { 24149, 10747, -19366, 0, 0, 0, 0, 0 },
522 { 19366, -24149, 10747, 0, 0, 0, 0, 0 },
523 { 0, 0, 0, 14654, 14654, 14654, 14654, 14654 },
524 { 0, 0, 0, 19710, 12181, 0, -12181, -19710 },
525 { 0, 0, 0, 16766, -6404, -20724, -6404, 16766 },
526 { 0, 0, 0, 12181, -19710, 0, 19710, -12181 },
527 { 0, 0, 0, 6404, -16766, 20724, -16766, 6404 },
528 };
529
530 // LGT8 name: lgt8_060_000w3
531 // Self loops: 0.600, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
532 // Edges: 1.000, 1.000, 0.000, 1.000, 1.000, 1.000, 1.000
533 static const tran_high_t lgt8_060_000w3[8][8] = {
534 { 13363, 19452, 22733, 0, 0, 0, 0, 0 },
535 { 24815, 6704, -20323, 0, 0, 0, 0, 0 },
536 { 16715, -25503, 11997, 0, 0, 0, 0, 0 },
537 { 0, 0, 0, 14654, 14654, 14654, 14654, 14654 },
538 { 0, 0, 0, 19710, 12181, 0, -12181, -19710 },
539 { 0, 0, 0, 16766, -6404, -20724, -6404, 16766 },
540 { 0, 0, 0, 12181, -19710, 0, 19710, -12181 },
541 { 0, 0, 0, 6404, -16766, 20724, -16766, 6404 },
542 };
543
544 // LGT8 name: lgt8_000w3
545 // Self loops: 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
546 // Edges: 1.000, 1.000, 0.000, 1.000, 1.000, 1.000, 1.000
547 static const tran_high_t lgt8_000w3[8][8] = {
548 { 18919, 18919, 18919, 0, 0, 0, 0, 0 },
549 { 23170, 0, -23170, 0, 0, 0, 0, 0 },
550 { 13377, -26755, 13377, 0, 0, 0, 0, 0 },
551 { 0, 0, 0, 14654, 14654, 14654, 14654, 14654 },
552 { 0, 0, 0, 19710, 12181, 0, -12181, -19710 },
553 { 0, 0, 0, 16766, -6404, -20724, -6404, 16766 },
554 { 0, 0, 0, 12181, -19710, 0, 19710, -12181 },
555 { 0, 0, 0, 6404, -16766, 20724, -16766, 6404 },
556 };
557
558 // LGT8 name: lgt8_150_000w2
559 // Self loops: 1.500, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
560 // Edges: 1.000, 0.000, 1.000, 1.000, 1.000, 1.000, 1.000
561 static const tran_high_t lgt8_150_000w2[8][8] = {
562 { 14654, 29309, 0, 0, 0, 0, 0, 0 },
563 { 29309, -14654, 0, 0, 0, 0, 0, 0 },
564 { 0, 0, 13377, 13377, 13377, 13377, 13377, 13377 },
565 { 0, 0, 18274, 13377, 4896, -4896, -13377, -18274 },
566 { 0, 0, 16384, 0, -16384, -16384, 0, 16384 },
567 { 0, 0, 13377, -13377, -13377, 13377, 13377, -13377 },
568 { 0, 0, 9459, -18919, 9459, 9459, -18919, 9459 },
569 { 0, 0, 4896, -13377, 18274, -18274, 13377, -4896 },
570 };
571
572 // LGT8 name: lgt8_100_000w2
573 // Self loops: 1.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
574 // Edges: 1.000, 0.000, 1.000, 1.000, 1.000, 1.000, 1.000
575 static const tran_high_t lgt8_100_000w2[8][8] = {
576 { 17227, 27874, 0, 0, 0, 0, 0, 0 },
577 { 27874, -17227, 0, 0, 0, 0, 0, 0 },
578 { 0, 0, 13377, 13377, 13377, 13377, 13377, 13377 },
579 { 0, 0, 18274, 13377, 4896, -4896, -13377, -18274 },
580 { 0, 0, 16384, 0, -16384, -16384, 0, 16384 },
581 { 0, 0, 13377, -13377, -13377, 13377, 13377, -13377 },
582 { 0, 0, 9459, -18919, 9459, 9459, -18919, 9459 },
583 { 0, 0, 4896, -13377, 18274, -18274, 13377, -4896 },
584 };
585
586 // LGT8 name: lgt8_060_000w2
587 // Self loops: 0.600, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
588 // Edges: 1.000, 0.000, 1.000, 1.000, 1.000, 1.000, 1.000
589 static const tran_high_t lgt8_060_000w2[8][8] = {
590 { 19560, 26290, 0, 0, 0, 0, 0, 0 },
591 { 26290, -19560, 0, 0, 0, 0, 0, 0 },
592 { 0, 0, 13377, 13377, 13377, 13377, 13377, 13377 },
593 { 0, 0, 18274, 13377, 4896, -4896, -13377, -18274 },
594 { 0, 0, 16384, 0, -16384, -16384, 0, 16384 },
595 { 0, 0, 13377, -13377, -13377, 13377, 13377, -13377 },
596 { 0, 0, 9459, -18919, 9459, 9459, -18919, 9459 },
597 { 0, 0, 4896, -13377, 18274, -18274, 13377, -4896 },
598 };
599
600 // LGT8 name: lgt8_000w2
601 // Self loops: 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
602 // Edges: 1.000, 0.000, 1.000, 1.000, 1.000, 1.000, 1.000
603 static const tran_high_t lgt8_000w2[8][8] = {
604 { 23170, 23170, 0, 0, 0, 0, 0, 0 },
605 { 23170, -23170, 0, 0, 0, 0, 0, 0 },
606 { 0, 0, 13377, 13377, 13377, 13377, 13377, 13377 },
607 { 0, 0, 18274, 13377, 4896, -4896, -13377, -18274 },
608 { 0, 0, 16384, 0, -16384, -16384, 0, 16384 },
609 { 0, 0, 13377, -13377, -13377, 13377, 13377, -13377 },
610 { 0, 0, 9459, -18919, 9459, 9459, -18919, 9459 },
611 { 0, 0, 4896, -13377, 18274, -18274, 13377, -4896 },
612 };
613
614 // LGT8 name: lgt8_150_000w1
615 // Self loops: 1.500, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
616 // Edges: 0.000, 1.000, 1.000, 1.000, 1.000, 1.000, 1.000
617 static const tran_high_t lgt8_150_000w1[8][8] = {
618 { 32768, 0, 0, 0, 0, 0, 0, 0 },
619 { 0, 12385, 12385, 12385, 12385, 12385, 12385, 12385 },
620 { 0, 17076, 13694, 7600, 0, -7600, -13694, -17076 },
621 { 0, 15781, 3898, -10921, -17515, -10921, 3898, 15781 },
622 { 0, 13694, -7600, -17076, 0, 17076, 7600, -13694 },
623 { 0, 10921, -15781, -3898, 17515, -3898, -15781, 10921 },
624 { 0, 7600, -17076, 13694, 0, -13694, 17076, -7600 },
625 { 0, 3898, -10921, 15781, -17515, 15781, -10921, 3898 },
626 };
627
628 // LGT8 name: lgt8_100_000w1
629 // Self loops: 1.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
630 // Edges: 0.000, 1.000, 1.000, 1.000, 1.000, 1.000, 1.000
631 static const tran_high_t lgt8_100_000w1[8][8] = {
632 { 32768, 0, 0, 0, 0, 0, 0, 0 },
633 { 0, 12385, 12385, 12385, 12385, 12385, 12385, 12385 },
634 { 0, 17076, 13694, 7600, 0, -7600, -13694, -17076 },
635 { 0, 15781, 3898, -10921, -17515, -10921, 3898, 15781 },
636 { 0, 13694, -7600, -17076, 0, 17076, 7600, -13694 },
637 { 0, 10921, -15781, -3898, 17515, -3898, -15781, 10921 },
638 { 0, 7600, -17076, 13694, 0, -13694, 17076, -7600 },
639 { 0, 3898, -10921, 15781, -17515, 15781, -10921, 3898 },
640 };
641
642 // LGT8 name: lgt8_060_000w1
643 // Self loops: 0.600, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
644 // Edges: 0.000, 1.000, 1.000, 1.000, 1.000, 1.000, 1.000
645 static const tran_high_t lgt8_060_000w1[8][8] = {
646 { 32768, 0, 0, 0, 0, 0, 0, 0 },
647 { 0, 12385, 12385, 12385, 12385, 12385, 12385, 12385 },
648 { 0, 17076, 13694, 7600, 0, -7600, -13694, -17076 },
649 { 0, 15781, 3898, -10921, -17515, -10921, 3898, 15781 },
650 { 0, 13694, -7600, -17076, 0, 17076, 7600, -13694 },
651 { 0, 10921, -15781, -3898, 17515, -3898, -15781, 10921 },
652 { 0, 7600, -17076, 13694, 0, -13694, 17076, -7600 },
653 { 0, 3898, -10921, 15781, -17515, 15781, -10921, 3898 },
654 };
655
656 // LGT8 name: lgt8_000w1
657 // Self loops: 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
658 // Edges: 0.000, 1.000, 1.000, 1.000, 1.000, 1.000, 1.000
659 static const tran_high_t lgt8_000w1[8][8] = {
660 { 32768, 0, 0, 0, 0, 0, 0, 0 },
661 { 0, 12385, 12385, 12385, 12385, 12385, 12385, 12385 },
662 { 0, 17076, 13694, 7600, 0, -7600, -13694, -17076 },
663 { 0, 15781, 3898, -10921, -17515, -10921, 3898, 15781 },
664 { 0, 13694, -7600, -17076, 0, 17076, 7600, -13694 },
665 { 0, 10921, -15781, -3898, 17515, -3898, -15781, 10921 },
666 { 0, 7600, -17076, 13694, 0, -13694, 17076, -7600 },
667 { 0, 3898, -10921, 15781, -17515, 15781, -10921, 3898 },
668 };
669
670 // LGT8 name: lgt8_060
671 // Self loops: 0.600, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
672 // Edges: 1.000, 1.000, 1.000, 1.000, 1.000, 1.000, 1.000
673 static const tran_high_t lgt8_060[8][8] = {
674 { 4295, 6746, 8999, 10987, 12653, 13947, 14832, 15280 },
675 { 11303, 15101, 14912, 10786, 3812, -4168, -11047, -15010 },
676 { 15051, 13208, 1823, -10879, -15721, -9207, 3959, 14265 },
677 { 15871, 3800, -13441, -12395, 5516, 15922, 4665, -12939 },
678 { 14630, -7269, -13926, 8618, 13091, -9886, -12133, 11062 },
679 { 12008, -14735, 180, 14586, -12245, -4458, 15932, -8720 },
680 { 8472, -15623, 14088, -4721, -7272, 15221, -14708, 6018 },
681 { 4372, -9862, 13927, -15981, 15727, -13202, 8770, -3071 },
682 };
683
684 // LGT8 name: lgt8_100
685 // Self loops: 1.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
686 // Edges: 1.000, 1.000, 1.000, 1.000, 1.000, 1.000, 1.000
687 static const tran_high_t lgt8_100[8][8] = {
688 { 2921, 5742, 8368, 10708, 12684, 14228, 15288, 15827 },
689 { 8368, 14228, 15827, 12684, 5742, -2921, -10708, -15288 },
690 { 12684, 15288, 5742, -8368, -15827, -10708, 2921, 14228 },
691 { 15288, 8368, -10708, -14228, 2921, 15827, 5742, -12684 },
692 { 15827, -2921, -15288, 5742, 14228, -8368, -12684, 10708 },
693 { 14228, -12684, -2921, 15288, -10708, -5742, 15827, -8368 },
694 { 10708, -15827, 12684, -2921, -8368, 15288, -14228, 5742 },
695 { 5742, -10708, 14228, -15827, 15288, -12684, 8368, -2921 },
696 };
697 #endif // CONFIG_LGT_FROM_PRED
698
699 #if CONFIG_LGT || CONFIG_LGT_FROM_PRED
700 // LGT4 name: lgt4_170
701 // Self loops: 1.700, 0.000, 0.000, 0.000
702 // Edges: 1.000, 1.000, 1.000
703 static const tran_high_t lgt4_170[4][4] = {
704 { 3636, 9287, 13584, 15902 },
705 { 10255, 15563, 2470, -13543 },
706 { 14786, 711, -15249, 9231 },
707 { 14138, -14420, 10663, -3920 },
708 };
709
710 // LGT4 name: lgt4_140
711 // Self loops: 1.400, 0.000, 0.000, 0.000
712 // Edges: 1.000, 1.000, 1.000
713 static const tran_high_t lgt4_140[4][4] = {
714 { 4206, 9518, 13524, 15674 },
715 { 11552, 14833, 1560, -13453 },
716 { 15391, -1906, -14393, 9445 },
717 { 12201, -14921, 12016, -4581 },
718 };
719
720 // LGT8 name: lgt8_170
721 // Self loops: 1.700, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
722 // Edges: 1.000, 1.000, 1.000, 1.000, 1.000, 1.000, 1.000
723 static const tran_high_t lgt8_170[8][8] = {
724 { 1858, 4947, 7850, 10458, 12672, 14411, 15607, 16217 },
725 { 5494, 13022, 16256, 14129, 7343, -1864, -10456, -15601 },
726 { 8887, 16266, 9500, -5529, -15749, -12273, 1876, 14394 },
727 { 11870, 13351, -6199, -15984, -590, 15733, 7273, -12644 },
728 { 14248, 5137, -15991, 291, 15893, -5685, -13963, 10425 },
729 { 15716, -5450, -10010, 15929, -6665, -8952, 16036, -7835 },
730 { 15533, -13869, 6559, 3421, -12009, 15707, -13011, 5018 },
731 { 11357, -13726, 14841, -14600, 13025, -10259, 6556, -2254 },
732 };
733
734 // LGT8 name: lgt8_150
735 // Self loops: 1.500, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000
736 // Edges: 1.000, 1.000, 1.000, 1.000, 1.000, 1.000, 1.000
737 static const tran_high_t lgt8_150[8][8] = {
738 { 2075, 5110, 7958, 10511, 12677, 14376, 15544, 16140 },
739 { 6114, 13307, 16196, 13845, 7015, -2084, -10509, -15534 },
740 { 9816, 16163, 8717, -6168, -15790, -11936, 2104, 14348 },
741 { 12928, 12326, -7340, -15653, 242, 15763, 6905, -12632 },
742 { 15124, 3038, -16033, 1758, 15507, -6397, -13593, 10463 },
743 { 15895, -7947, -7947, 15895, -7947, -7947, 15895, -7947 },
744 { 14325, -15057, 9030, 1050, -10659, 15483, -13358, 5236 },
745 { 9054, -12580, 14714, -15220, 14043, -11312, 7330, -2537 },
746 };
747 #endif // CONFIG_LGT || CONFIG_LGT_FROM_PRED
748 #endif // AOM_DSP_TXFM_COMMON_H_
749