1 /*
2  *  Copyright (c) 2014 The WebM project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include <assert.h>
12 #include <immintrin.h>
13 
14 #include "./vp9_rtcd.h"
15 #include "vpx/vpx_integer.h"
16 #include "vpx_dsp/vpx_dsp_common.h"
17 #include "vpx_dsp/x86/bitdepth_conversion_avx2.h"
18 
vp9_block_error_avx2(const tran_low_t * coeff,const tran_low_t * dqcoeff,intptr_t block_size,int64_t * ssz)19 int64_t vp9_block_error_avx2(const tran_low_t *coeff, const tran_low_t *dqcoeff,
20                              intptr_t block_size, int64_t *ssz) {
21   __m256i sse_256, ssz_256;
22   __m256i exp_dqcoeff_lo, exp_dqcoeff_hi, exp_coeff_lo, exp_coeff_hi;
23   __m256i sse_hi, ssz_hi;
24   __m128i sse_128, ssz_128;
25   int64_t sse;
26   const __m256i zero = _mm256_setzero_si256();
27 
28   // If the block size is 16 then the results will fit in 32 bits.
29   if (block_size == 16) {
30     __m256i coeff_256, dqcoeff_256, coeff_hi, dqcoeff_hi;
31     // Load 16 elements for coeff and dqcoeff.
32     coeff_256 = load_tran_low(coeff);
33     dqcoeff_256 = load_tran_low(dqcoeff);
34     // dqcoeff - coeff
35     dqcoeff_256 = _mm256_sub_epi16(dqcoeff_256, coeff_256);
36     // madd (dqcoeff - coeff)
37     dqcoeff_256 = _mm256_madd_epi16(dqcoeff_256, dqcoeff_256);
38     // madd coeff
39     coeff_256 = _mm256_madd_epi16(coeff_256, coeff_256);
40     // Save the higher 64 bit of each 128 bit lane.
41     dqcoeff_hi = _mm256_srli_si256(dqcoeff_256, 8);
42     coeff_hi = _mm256_srli_si256(coeff_256, 8);
43     // Add the higher 64 bit to the low 64 bit.
44     dqcoeff_256 = _mm256_add_epi32(dqcoeff_256, dqcoeff_hi);
45     coeff_256 = _mm256_add_epi32(coeff_256, coeff_hi);
46     // Expand each double word in the lower 64 bits to quad word.
47     sse_256 = _mm256_unpacklo_epi32(dqcoeff_256, zero);
48     ssz_256 = _mm256_unpacklo_epi32(coeff_256, zero);
49   } else {
50     int i;
51     assert(block_size % 32 == 0);
52     sse_256 = zero;
53     ssz_256 = zero;
54 
55     for (i = 0; i < block_size; i += 32) {
56       __m256i coeff_0, coeff_1, dqcoeff_0, dqcoeff_1;
57       // Load 32 elements for coeff and dqcoeff.
58       coeff_0 = load_tran_low(coeff + i);
59       dqcoeff_0 = load_tran_low(dqcoeff + i);
60       coeff_1 = load_tran_low(coeff + i + 16);
61       dqcoeff_1 = load_tran_low(dqcoeff + i + 16);
62       // dqcoeff - coeff
63       dqcoeff_0 = _mm256_sub_epi16(dqcoeff_0, coeff_0);
64       dqcoeff_1 = _mm256_sub_epi16(dqcoeff_1, coeff_1);
65       // madd (dqcoeff - coeff)
66       dqcoeff_0 = _mm256_madd_epi16(dqcoeff_0, dqcoeff_0);
67       dqcoeff_1 = _mm256_madd_epi16(dqcoeff_1, dqcoeff_1);
68       // madd coeff
69       coeff_0 = _mm256_madd_epi16(coeff_0, coeff_0);
70       coeff_1 = _mm256_madd_epi16(coeff_1, coeff_1);
71       // Add the first madd (dqcoeff - coeff) with the second.
72       dqcoeff_0 = _mm256_add_epi32(dqcoeff_0, dqcoeff_1);
73       // Add the first madd (coeff) with the second.
74       coeff_0 = _mm256_add_epi32(coeff_0, coeff_1);
75       // Expand each double word of madd (dqcoeff - coeff) to quad word.
76       exp_dqcoeff_lo = _mm256_unpacklo_epi32(dqcoeff_0, zero);
77       exp_dqcoeff_hi = _mm256_unpackhi_epi32(dqcoeff_0, zero);
78       // expand each double word of madd (coeff) to quad word
79       exp_coeff_lo = _mm256_unpacklo_epi32(coeff_0, zero);
80       exp_coeff_hi = _mm256_unpackhi_epi32(coeff_0, zero);
81       // Add each quad word of madd (dqcoeff - coeff) and madd (coeff).
82       sse_256 = _mm256_add_epi64(sse_256, exp_dqcoeff_lo);
83       ssz_256 = _mm256_add_epi64(ssz_256, exp_coeff_lo);
84       sse_256 = _mm256_add_epi64(sse_256, exp_dqcoeff_hi);
85       ssz_256 = _mm256_add_epi64(ssz_256, exp_coeff_hi);
86     }
87   }
88   // Save the higher 64 bit of each 128 bit lane.
89   sse_hi = _mm256_srli_si256(sse_256, 8);
90   ssz_hi = _mm256_srli_si256(ssz_256, 8);
91   // Add the higher 64 bit to the low 64 bit.
92   sse_256 = _mm256_add_epi64(sse_256, sse_hi);
93   ssz_256 = _mm256_add_epi64(ssz_256, ssz_hi);
94 
95   // Add each 64 bit from each of the 128 bit lane of the 256 bit.
96   sse_128 = _mm_add_epi64(_mm256_castsi256_si128(sse_256),
97                           _mm256_extractf128_si256(sse_256, 1));
98 
99   ssz_128 = _mm_add_epi64(_mm256_castsi256_si128(ssz_256),
100                           _mm256_extractf128_si256(ssz_256, 1));
101 
102   // Store the results.
103   _mm_storel_epi64((__m128i *)(&sse), sse_128);
104 
105   _mm_storel_epi64((__m128i *)(ssz), ssz_128);
106   return sse;
107 }
108 
vp9_block_error_fp_avx2(const tran_low_t * coeff,const tran_low_t * dqcoeff,int block_size)109 int64_t vp9_block_error_fp_avx2(const tran_low_t *coeff,
110                                 const tran_low_t *dqcoeff, int block_size) {
111   int i;
112   const __m256i zero = _mm256_setzero_si256();
113   __m256i sse_256 = zero;
114   __m256i sse_hi;
115   __m128i sse_128;
116   int64_t sse;
117 
118   if (block_size == 16) {
119     // Load 16 elements for coeff and dqcoeff.
120     const __m256i _coeff = load_tran_low(coeff);
121     const __m256i _dqcoeff = load_tran_low(dqcoeff);
122     // dqcoeff - coeff
123     const __m256i diff = _mm256_sub_epi16(_dqcoeff, _coeff);
124     // madd (dqcoeff - coeff)
125     const __m256i error_lo = _mm256_madd_epi16(diff, diff);
126     // Save the higher 64 bit of each 128 bit lane.
127     const __m256i error_hi = _mm256_srli_si256(error_lo, 8);
128     // Add the higher 64 bit to the low 64 bit.
129     const __m256i error = _mm256_add_epi32(error_lo, error_hi);
130     // Expand each double word in the lower 64 bits to quad word.
131     sse_256 = _mm256_unpacklo_epi32(error, zero);
132   } else {
133     for (i = 0; i < block_size; i += 16) {
134       // Load 16 elements for coeff and dqcoeff.
135       const __m256i _coeff = load_tran_low(coeff);
136       const __m256i _dqcoeff = load_tran_low(dqcoeff);
137       const __m256i diff = _mm256_sub_epi16(_dqcoeff, _coeff);
138       const __m256i error = _mm256_madd_epi16(diff, diff);
139       // Expand each double word of madd (dqcoeff - coeff) to quad word.
140       const __m256i exp_error_lo = _mm256_unpacklo_epi32(error, zero);
141       const __m256i exp_error_hi = _mm256_unpackhi_epi32(error, zero);
142       // Add each quad word of madd (dqcoeff - coeff).
143       sse_256 = _mm256_add_epi64(sse_256, exp_error_lo);
144       sse_256 = _mm256_add_epi64(sse_256, exp_error_hi);
145       coeff += 16;
146       dqcoeff += 16;
147     }
148   }
149   // Save the higher 64 bit of each 128 bit lane.
150   sse_hi = _mm256_srli_si256(sse_256, 8);
151   // Add the higher 64 bit to the low 64 bit.
152   sse_256 = _mm256_add_epi64(sse_256, sse_hi);
153 
154   // Add each 64 bit from each of the 128 bit lane of the 256 bit.
155   sse_128 = _mm_add_epi64(_mm256_castsi256_si128(sse_256),
156                           _mm256_extractf128_si256(sse_256, 1));
157 
158   // Store the results.
159   _mm_storel_epi64((__m128i *)&sse, sse_128);
160   return sse;
161 }
162