1 /*
2 * Copyright (c) 2018 The WebM project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include <arm_neon.h>
12
13 #include <assert.h>
14 #include "./vpx_dsp_rtcd.h"
15
vpx_sum_squares_2d_i16_neon(const int16_t * src,int stride,int size)16 uint64_t vpx_sum_squares_2d_i16_neon(const int16_t *src, int stride, int size) {
17 uint64x1_t s2;
18
19 if (size == 4) {
20 int16x4_t s[4];
21 int32x4_t s0;
22 uint32x2_t s1;
23
24 s[0] = vld1_s16(src + 0 * stride);
25 s[1] = vld1_s16(src + 1 * stride);
26 s[2] = vld1_s16(src + 2 * stride);
27 s[3] = vld1_s16(src + 3 * stride);
28 s0 = vmull_s16(s[0], s[0]);
29 s0 = vmlal_s16(s0, s[1], s[1]);
30 s0 = vmlal_s16(s0, s[2], s[2]);
31 s0 = vmlal_s16(s0, s[3], s[3]);
32 s1 = vpadd_u32(vget_low_u32(vreinterpretq_u32_s32(s0)),
33 vget_high_u32(vreinterpretq_u32_s32(s0)));
34 s2 = vpaddl_u32(s1);
35 } else {
36 int r = size;
37 uint64x2_t s1 = vdupq_n_u64(0);
38
39 do {
40 int c = size;
41 int32x4_t s0 = vdupq_n_s32(0);
42 const int16_t *src_t = src;
43
44 do {
45 int16x8_t s[8];
46
47 s[0] = vld1q_s16(src_t + 0 * stride);
48 s[1] = vld1q_s16(src_t + 1 * stride);
49 s[2] = vld1q_s16(src_t + 2 * stride);
50 s[3] = vld1q_s16(src_t + 3 * stride);
51 s[4] = vld1q_s16(src_t + 4 * stride);
52 s[5] = vld1q_s16(src_t + 5 * stride);
53 s[6] = vld1q_s16(src_t + 6 * stride);
54 s[7] = vld1q_s16(src_t + 7 * stride);
55 s0 = vmlal_s16(s0, vget_low_s16(s[0]), vget_low_s16(s[0]));
56 s0 = vmlal_s16(s0, vget_low_s16(s[1]), vget_low_s16(s[1]));
57 s0 = vmlal_s16(s0, vget_low_s16(s[2]), vget_low_s16(s[2]));
58 s0 = vmlal_s16(s0, vget_low_s16(s[3]), vget_low_s16(s[3]));
59 s0 = vmlal_s16(s0, vget_low_s16(s[4]), vget_low_s16(s[4]));
60 s0 = vmlal_s16(s0, vget_low_s16(s[5]), vget_low_s16(s[5]));
61 s0 = vmlal_s16(s0, vget_low_s16(s[6]), vget_low_s16(s[6]));
62 s0 = vmlal_s16(s0, vget_low_s16(s[7]), vget_low_s16(s[7]));
63 s0 = vmlal_s16(s0, vget_high_s16(s[0]), vget_high_s16(s[0]));
64 s0 = vmlal_s16(s0, vget_high_s16(s[1]), vget_high_s16(s[1]));
65 s0 = vmlal_s16(s0, vget_high_s16(s[2]), vget_high_s16(s[2]));
66 s0 = vmlal_s16(s0, vget_high_s16(s[3]), vget_high_s16(s[3]));
67 s0 = vmlal_s16(s0, vget_high_s16(s[4]), vget_high_s16(s[4]));
68 s0 = vmlal_s16(s0, vget_high_s16(s[5]), vget_high_s16(s[5]));
69 s0 = vmlal_s16(s0, vget_high_s16(s[6]), vget_high_s16(s[6]));
70 s0 = vmlal_s16(s0, vget_high_s16(s[7]), vget_high_s16(s[7]));
71 src_t += 8;
72 c -= 8;
73 } while (c);
74
75 s1 = vaddw_u32(s1, vget_low_u32(vreinterpretq_u32_s32(s0)));
76 s1 = vaddw_u32(s1, vget_high_u32(vreinterpretq_u32_s32(s0)));
77 src += 8 * stride;
78 r -= 8;
79 } while (r);
80
81 s2 = vadd_u64(vget_low_u64(s1), vget_high_u64(s1));
82 }
83
84 return vget_lane_u64(s2, 0);
85 }
86