1 /*
2 * Copyright (c) 2020, Alliance for Open Media. All rights reserved
3 *
4 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10 */
11
12 #include <stdio.h>
13 #include <tmmintrin.h>
14
15 #include "config/aom_config.h"
16 #include "config/aom_dsp_rtcd.h"
17
18 #include "aom_dsp/blend.h"
19 #include "aom/aom_integer.h"
20 #include "aom_dsp/x86/synonyms.h"
21
22 #include "aom_dsp/x86/masked_sad_intrin_ssse3.h"
23
24 #define MASK_SAD16XH_ONE_REF(idx) \
25 a = _mm_loadu_si128((const __m128i *)&ref##idx[x]); \
26 data_l = _mm_unpacklo_epi8(a, b); \
27 mask_l = _mm_unpacklo_epi8(m, m_inv); \
28 pred_l = _mm_maddubs_epi16(data_l, mask_l); \
29 pred_l = xx_roundn_epu16(pred_l, AOM_BLEND_A64_ROUND_BITS); \
30 \
31 data_r = _mm_unpackhi_epi8(a, b); \
32 mask_r = _mm_unpackhi_epi8(m, m_inv); \
33 pred_r = _mm_maddubs_epi16(data_r, mask_r); \
34 pred_r = xx_roundn_epu16(pred_r, AOM_BLEND_A64_ROUND_BITS); \
35 \
36 pred = _mm_packus_epi16(pred_l, pred_r); \
37 res##idx = _mm_add_epi32(res##idx, _mm_sad_epu8(pred, src));
38
masked_sadx4d_ssse3(const uint8_t * src_ptr,int src_stride,const uint8_t * a_ptr[],int a_stride,const uint8_t * b_ptr,int b_stride,const uint8_t * m_ptr,int m_stride,int width,int height,int inv_mask,unsigned sad_array[])39 static INLINE void masked_sadx4d_ssse3(const uint8_t *src_ptr, int src_stride,
40 const uint8_t *a_ptr[], int a_stride,
41 const uint8_t *b_ptr, int b_stride,
42 const uint8_t *m_ptr, int m_stride,
43 int width, int height, int inv_mask,
44 unsigned sad_array[]) {
45 int x, y;
46 __m128i a;
47 __m128i data_l, data_r, mask_l, mask_r, pred_l, pred_r, pred;
48 const __m128i mask_max = _mm_set1_epi8((1 << AOM_BLEND_A64_ROUND_BITS));
49 __m128i res0 = _mm_setzero_si128();
50 __m128i res1 = _mm_setzero_si128();
51 __m128i res2 = _mm_setzero_si128();
52 __m128i res3 = _mm_setzero_si128();
53 const uint8_t *ref0 = a_ptr[0];
54 const uint8_t *ref1 = a_ptr[1];
55 const uint8_t *ref2 = a_ptr[2];
56 const uint8_t *ref3 = a_ptr[3];
57
58 for (y = 0; y < height; y++) {
59 for (x = 0; x < width; x += 16) {
60 const __m128i src = _mm_loadu_si128((const __m128i *)&src_ptr[x]);
61 const __m128i b = _mm_loadu_si128((const __m128i *)&b_ptr[x]);
62 const __m128i m_copy = _mm_loadu_si128((const __m128i *)&m_ptr[x]);
63 __m128i m_inv = _mm_sub_epi8(mask_max, m_copy);
64 __m128i m = inv_mask ? m_inv : m_copy;
65 m_inv = inv_mask ? m_copy : m_inv;
66
67 MASK_SAD16XH_ONE_REF(0)
68 MASK_SAD16XH_ONE_REF(1)
69 MASK_SAD16XH_ONE_REF(2)
70 MASK_SAD16XH_ONE_REF(3)
71 }
72
73 src_ptr += src_stride;
74 ref0 += a_stride;
75 ref1 += a_stride;
76 ref2 += a_stride;
77 ref3 += a_stride;
78 b_ptr += b_stride;
79 m_ptr += m_stride;
80 }
81 res0 = _mm_add_epi32(_mm_unpacklo_epi32(res0, res1),
82 _mm_unpackhi_epi32(res0, res1));
83 res2 = _mm_add_epi32(_mm_unpacklo_epi32(res2, res3),
84 _mm_unpackhi_epi32(res2, res3));
85
86 res0 = _mm_unpacklo_epi64(res0, res2);
87 _mm_storeu_si128((__m128i *)sad_array, res0);
88 }
89
90 #define MASK_SAD8XH_ONE_REF(idx) \
91 const __m128i a##idx##0 = _mm_loadl_epi64((__m128i *)ref##idx); \
92 const __m128i a##idx##1 = _mm_loadl_epi64((__m128i *)(ref##idx + a_stride)); \
93 data_l = _mm_unpacklo_epi8(a##idx##0, b0); \
94 mask_l = _mm_unpacklo_epi8(m, m_inv); \
95 pred_l = _mm_maddubs_epi16(data_l, mask_l); \
96 pred_l = xx_roundn_epu16(pred_l, AOM_BLEND_A64_ROUND_BITS); \
97 \
98 data_r = _mm_unpacklo_epi8(a##idx##1, b1); \
99 mask_r = _mm_unpackhi_epi8(m, m_inv); \
100 pred_r = _mm_maddubs_epi16(data_r, mask_r); \
101 pred_r = xx_roundn_epu16(pred_r, AOM_BLEND_A64_ROUND_BITS); \
102 \
103 pred = _mm_packus_epi16(pred_l, pred_r); \
104 res##idx = _mm_add_epi32(res##idx, _mm_sad_epu8(pred, src));
105
aom_masked_sad8xhx4d_ssse3(const uint8_t * src_ptr,int src_stride,const uint8_t * ref_array[],int a_stride,const uint8_t * b_ptr,int b_stride,const uint8_t * m_ptr,int m_stride,int height,int inv_mask,unsigned sad_array[])106 void aom_masked_sad8xhx4d_ssse3(const uint8_t *src_ptr, int src_stride,
107 const uint8_t *ref_array[], int a_stride,
108 const uint8_t *b_ptr, int b_stride,
109 const uint8_t *m_ptr, int m_stride, int height,
110 int inv_mask, unsigned sad_array[]) {
111 const uint8_t *ref0 = ref_array[0];
112 const uint8_t *ref1 = ref_array[1];
113 const uint8_t *ref2 = ref_array[2];
114 const uint8_t *ref3 = ref_array[3];
115 __m128i data_l, data_r, pred_l, pred_r, mask_l, mask_r, pred;
116 __m128i res0 = _mm_setzero_si128();
117 __m128i res1 = _mm_setzero_si128();
118 __m128i res2 = _mm_setzero_si128();
119 __m128i res3 = _mm_setzero_si128();
120 const __m128i mask_max = _mm_set1_epi8((1 << AOM_BLEND_A64_ROUND_BITS));
121
122 for (int y = 0; y < height; y += 2) {
123 const __m128i src = _mm_unpacklo_epi64(
124 _mm_loadl_epi64((const __m128i *)src_ptr),
125 _mm_loadl_epi64((const __m128i *)(src_ptr + src_stride)));
126 const __m128i b0 = _mm_loadl_epi64((__m128i *)b_ptr);
127 const __m128i b1 = _mm_loadl_epi64((__m128i *)(b_ptr + b_stride));
128 const __m128i m0 = _mm_loadl_epi64((__m128i *)m_ptr);
129 const __m128i m1 = _mm_loadl_epi64((__m128i *)(m_ptr + m_stride));
130 __m128i m_copy = _mm_unpacklo_epi64(m0, m1);
131 __m128i m_inv = _mm_sub_epi8(mask_max, m_copy);
132 __m128i m = inv_mask ? m_inv : m_copy;
133 m_inv = inv_mask ? m_copy : m_inv;
134
135 MASK_SAD8XH_ONE_REF(0)
136 MASK_SAD8XH_ONE_REF(1)
137 MASK_SAD8XH_ONE_REF(2)
138 MASK_SAD8XH_ONE_REF(3)
139
140 ref0 += 2 * a_stride;
141 ref1 += 2 * a_stride;
142 ref2 += 2 * a_stride;
143 ref3 += 2 * a_stride;
144 src_ptr += 2 * src_stride;
145 b_ptr += 2 * b_stride;
146 m_ptr += 2 * m_stride;
147 }
148 res0 = _mm_add_epi32(_mm_unpacklo_epi32(res0, res1),
149 _mm_unpackhi_epi32(res0, res1));
150 res2 = _mm_add_epi32(_mm_unpacklo_epi32(res2, res3),
151 _mm_unpackhi_epi32(res2, res3));
152 res0 = _mm_unpacklo_epi64(res0, res2);
153 _mm_storeu_si128((__m128i *)sad_array, res0);
154 }
155
156 #define MASK_SAD4XH_ONE_REF(idx) \
157 a = _mm_unpacklo_epi32(_mm_cvtsi32_si128(*(uint32_t *)ref##idx), \
158 _mm_cvtsi32_si128(*(uint32_t *)&ref##idx[a_stride])); \
159 data = _mm_unpacklo_epi8(a, b); \
160 mask = _mm_unpacklo_epi8(m, m_inv); \
161 pred = _mm_maddubs_epi16(data, mask); \
162 pred = xx_roundn_epu16(pred, AOM_BLEND_A64_ROUND_BITS); \
163 \
164 pred = _mm_packus_epi16(pred, _mm_setzero_si128()); \
165 res##idx = _mm_add_epi32(res##idx, _mm_sad_epu8(pred, src));
166
aom_masked_sad4xhx4d_ssse3(const uint8_t * src_ptr,int src_stride,const uint8_t * ref_array[],int a_stride,const uint8_t * b_ptr,int b_stride,const uint8_t * m_ptr,int m_stride,int height,int inv_mask,unsigned sad_array[])167 void aom_masked_sad4xhx4d_ssse3(const uint8_t *src_ptr, int src_stride,
168 const uint8_t *ref_array[], int a_stride,
169 const uint8_t *b_ptr, int b_stride,
170 const uint8_t *m_ptr, int m_stride, int height,
171 int inv_mask, unsigned sad_array[]) {
172 const uint8_t *ref0 = ref_array[0];
173 const uint8_t *ref1 = ref_array[1];
174 const uint8_t *ref2 = ref_array[2];
175 const uint8_t *ref3 = ref_array[3];
176 __m128i data, pred, mask;
177 __m128i res0 = _mm_setzero_si128();
178 __m128i res1 = _mm_setzero_si128();
179 __m128i res2 = _mm_setzero_si128();
180 __m128i res3 = _mm_setzero_si128();
181 __m128i a;
182 const __m128i mask_max = _mm_set1_epi8((1 << AOM_BLEND_A64_ROUND_BITS));
183
184 for (int y = 0; y < height; y += 2) {
185 const __m128i src = _mm_unpacklo_epi32(
186 _mm_cvtsi32_si128(*(uint32_t *)src_ptr),
187 _mm_cvtsi32_si128(*(uint32_t *)&src_ptr[src_stride]));
188 const __m128i b =
189 _mm_unpacklo_epi32(_mm_cvtsi32_si128(*(uint32_t *)b_ptr),
190 _mm_cvtsi32_si128(*(uint32_t *)&b_ptr[b_stride]));
191 const __m128i m_copy =
192 _mm_unpacklo_epi32(_mm_cvtsi32_si128(*(uint32_t *)m_ptr),
193 _mm_cvtsi32_si128(*(uint32_t *)&m_ptr[m_stride]));
194
195 __m128i m_inv = _mm_sub_epi8(mask_max, m_copy);
196 __m128i m = inv_mask ? m_inv : m_copy;
197 m_inv = inv_mask ? m_copy : m_inv;
198
199 MASK_SAD4XH_ONE_REF(0)
200 MASK_SAD4XH_ONE_REF(1)
201 MASK_SAD4XH_ONE_REF(2)
202 MASK_SAD4XH_ONE_REF(3)
203
204 ref0 += 2 * a_stride;
205 ref1 += 2 * a_stride;
206 ref2 += 2 * a_stride;
207 ref3 += 2 * a_stride;
208 src_ptr += 2 * src_stride;
209 b_ptr += 2 * b_stride;
210 m_ptr += 2 * m_stride;
211 }
212 res0 = _mm_unpacklo_epi32(res0, res1);
213 res2 = _mm_unpacklo_epi32(res2, res3);
214 res0 = _mm_unpacklo_epi64(res0, res2);
215 _mm_storeu_si128((__m128i *)sad_array, res0);
216 }
217
218 #define MASKSADMXN_SSSE3(m, n) \
219 void aom_masked_sad##m##x##n##x4d_ssse3( \
220 const uint8_t *src, int src_stride, const uint8_t *ref[], \
221 int ref_stride, const uint8_t *second_pred, const uint8_t *msk, \
222 int msk_stride, int inv_mask, unsigned sad_array[]) { \
223 masked_sadx4d_ssse3(src, src_stride, ref, ref_stride, second_pred, m, msk, \
224 msk_stride, m, n, inv_mask, sad_array); \
225 }
226
227 #define MASKSAD8XN_SSSE3(n) \
228 void aom_masked_sad8x##n##x4d_ssse3( \
229 const uint8_t *src, int src_stride, const uint8_t *ref[], \
230 int ref_stride, const uint8_t *second_pred, const uint8_t *msk, \
231 int msk_stride, int inv_mask, unsigned sad_array[]) { \
232 aom_masked_sad8xhx4d_ssse3(src, src_stride, ref, ref_stride, second_pred, \
233 8, msk, msk_stride, n, inv_mask, sad_array); \
234 }
235
236 #define MASKSAD4XN_SSSE3(n) \
237 void aom_masked_sad4x##n##x4d_ssse3( \
238 const uint8_t *src, int src_stride, const uint8_t *ref[], \
239 int ref_stride, const uint8_t *second_pred, const uint8_t *msk, \
240 int msk_stride, int inv_mask, unsigned sad_array[]) { \
241 aom_masked_sad4xhx4d_ssse3(src, src_stride, ref, ref_stride, second_pred, \
242 4, msk, msk_stride, n, inv_mask, sad_array); \
243 }
244
245 MASKSADMXN_SSSE3(128, 128)
246 MASKSADMXN_SSSE3(128, 64)
247 MASKSADMXN_SSSE3(64, 128)
248 MASKSADMXN_SSSE3(64, 64)
249 MASKSADMXN_SSSE3(64, 32)
250 MASKSADMXN_SSSE3(32, 64)
251 MASKSADMXN_SSSE3(32, 32)
252 MASKSADMXN_SSSE3(32, 16)
253 MASKSADMXN_SSSE3(16, 32)
254 MASKSADMXN_SSSE3(16, 16)
255 MASKSADMXN_SSSE3(16, 8)
256 MASKSAD8XN_SSSE3(16)
257 MASKSAD8XN_SSSE3(8)
258 MASKSAD8XN_SSSE3(4)
259 MASKSAD4XN_SSSE3(8)
260 MASKSAD4XN_SSSE3(4)
261 MASKSAD4XN_SSSE3(16)
262 MASKSADMXN_SSSE3(16, 4)
263 MASKSAD8XN_SSSE3(32)
264 MASKSADMXN_SSSE3(32, 8)
265 MASKSADMXN_SSSE3(16, 64)
266 MASKSADMXN_SSSE3(64, 16)
267