1 // Copyright 2015 Google Inc. All Rights Reserved.
2 //
3 // Use of this source code is governed by a BSD-style license
4 // that can be found in the COPYING file in the root of the source
5 // tree. An additional intellectual property rights grant can be found
6 // in the file PATENTS. All contributing project authors may
7 // be found in the AUTHORS file in the root of the source tree.
8 // -----------------------------------------------------------------------------
9 //
10 // NEON version of rescaling functions
11 //
12 // Author: Skal (pascal.massimino@gmail.com)
13
14 #include "src/dsp/dsp.h"
15
16 #if defined(WEBP_USE_NEON) && !defined(WEBP_REDUCE_SIZE)
17
18 #include <arm_neon.h>
19 #include <assert.h>
20 #include "src/dsp/neon.h"
21 #include "src/utils/rescaler_utils.h"
22
23 #define ROUNDER (WEBP_RESCALER_ONE >> 1)
24 #define MULT_FIX_C(x, y) (((uint64_t)(x) * (y) + ROUNDER) >> WEBP_RESCALER_RFIX)
25 #define MULT_FIX_FLOOR_C(x, y) (((uint64_t)(x) * (y)) >> WEBP_RESCALER_RFIX)
26
27 #define LOAD_32x4(SRC, DST) const uint32x4_t DST = vld1q_u32((SRC))
28 #define LOAD_32x8(SRC, DST0, DST1) \
29 LOAD_32x4(SRC + 0, DST0); \
30 LOAD_32x4(SRC + 4, DST1)
31
32 #define STORE_32x8(SRC0, SRC1, DST) do { \
33 vst1q_u32((DST) + 0, SRC0); \
34 vst1q_u32((DST) + 4, SRC1); \
35 } while (0);
36
37 #if (WEBP_RESCALER_RFIX == 32)
38 #define MAKE_HALF_CST(C) vdupq_n_s32((int32_t)((C) >> 1))
39 // note: B is actualy scale>>1. See MAKE_HALF_CST
40 #define MULT_FIX(A, B) \
41 vreinterpretq_u32_s32(vqrdmulhq_s32(vreinterpretq_s32_u32((A)), (B)))
42 #define MULT_FIX_FLOOR(A, B) \
43 vreinterpretq_u32_s32(vqdmulhq_s32(vreinterpretq_s32_u32((A)), (B)))
44 #else
45 #error "MULT_FIX/WEBP_RESCALER_RFIX need some more work"
46 #endif
47
Interpolate_NEON(const rescaler_t * const frow,const rescaler_t * const irow,uint32_t A,uint32_t B)48 static uint32x4_t Interpolate_NEON(const rescaler_t* const frow,
49 const rescaler_t* const irow,
50 uint32_t A, uint32_t B) {
51 LOAD_32x4(frow, A0);
52 LOAD_32x4(irow, B0);
53 const uint64x2_t C0 = vmull_n_u32(vget_low_u32(A0), A);
54 const uint64x2_t C1 = vmull_n_u32(vget_high_u32(A0), A);
55 const uint64x2_t D0 = vmlal_n_u32(C0, vget_low_u32(B0), B);
56 const uint64x2_t D1 = vmlal_n_u32(C1, vget_high_u32(B0), B);
57 const uint32x4_t E = vcombine_u32(
58 vrshrn_n_u64(D0, WEBP_RESCALER_RFIX),
59 vrshrn_n_u64(D1, WEBP_RESCALER_RFIX));
60 return E;
61 }
62
RescalerExportRowExpand_NEON(WebPRescaler * const wrk)63 static void RescalerExportRowExpand_NEON(WebPRescaler* const wrk) {
64 int x_out;
65 uint8_t* const dst = wrk->dst;
66 rescaler_t* const irow = wrk->irow;
67 const int x_out_max = wrk->dst_width * wrk->num_channels;
68 const int max_span = x_out_max & ~7;
69 const rescaler_t* const frow = wrk->frow;
70 const uint32_t fy_scale = wrk->fy_scale;
71 const int32x4_t fy_scale_half = MAKE_HALF_CST(fy_scale);
72 assert(!WebPRescalerOutputDone(wrk));
73 assert(wrk->y_accum <= 0);
74 assert(wrk->y_expand);
75 assert(wrk->y_sub != 0);
76 if (wrk->y_accum == 0) {
77 for (x_out = 0; x_out < max_span; x_out += 8) {
78 LOAD_32x4(frow + x_out + 0, A0);
79 LOAD_32x4(frow + x_out + 4, A1);
80 const uint32x4_t B0 = MULT_FIX(A0, fy_scale_half);
81 const uint32x4_t B1 = MULT_FIX(A1, fy_scale_half);
82 const uint16x4_t C0 = vmovn_u32(B0);
83 const uint16x4_t C1 = vmovn_u32(B1);
84 const uint8x8_t D = vmovn_u16(vcombine_u16(C0, C1));
85 vst1_u8(dst + x_out, D);
86 }
87 for (; x_out < x_out_max; ++x_out) {
88 const uint32_t J = frow[x_out];
89 const int v = (int)MULT_FIX_C(J, fy_scale);
90 assert(v >= 0 && v <= 255);
91 dst[x_out] = v;
92 }
93 } else {
94 const uint32_t B = WEBP_RESCALER_FRAC(-wrk->y_accum, wrk->y_sub);
95 const uint32_t A = (uint32_t)(WEBP_RESCALER_ONE - B);
96 for (x_out = 0; x_out < max_span; x_out += 8) {
97 const uint32x4_t C0 =
98 Interpolate_NEON(frow + x_out + 0, irow + x_out + 0, A, B);
99 const uint32x4_t C1 =
100 Interpolate_NEON(frow + x_out + 4, irow + x_out + 4, A, B);
101 const uint32x4_t D0 = MULT_FIX(C0, fy_scale_half);
102 const uint32x4_t D1 = MULT_FIX(C1, fy_scale_half);
103 const uint16x4_t E0 = vmovn_u32(D0);
104 const uint16x4_t E1 = vmovn_u32(D1);
105 const uint8x8_t F = vmovn_u16(vcombine_u16(E0, E1));
106 vst1_u8(dst + x_out, F);
107 }
108 for (; x_out < x_out_max; ++x_out) {
109 const uint64_t I = (uint64_t)A * frow[x_out]
110 + (uint64_t)B * irow[x_out];
111 const uint32_t J = (uint32_t)((I + ROUNDER) >> WEBP_RESCALER_RFIX);
112 const int v = (int)MULT_FIX_C(J, fy_scale);
113 assert(v >= 0 && v <= 255);
114 dst[x_out] = v;
115 }
116 }
117 }
118
RescalerExportRowShrink_NEON(WebPRescaler * const wrk)119 static void RescalerExportRowShrink_NEON(WebPRescaler* const wrk) {
120 int x_out;
121 uint8_t* const dst = wrk->dst;
122 rescaler_t* const irow = wrk->irow;
123 const int x_out_max = wrk->dst_width * wrk->num_channels;
124 const int max_span = x_out_max & ~7;
125 const rescaler_t* const frow = wrk->frow;
126 const uint32_t yscale = wrk->fy_scale * (-wrk->y_accum);
127 const uint32_t fxy_scale = wrk->fxy_scale;
128 const uint32x4_t zero = vdupq_n_u32(0);
129 const int32x4_t yscale_half = MAKE_HALF_CST(yscale);
130 const int32x4_t fxy_scale_half = MAKE_HALF_CST(fxy_scale);
131 assert(!WebPRescalerOutputDone(wrk));
132 assert(wrk->y_accum <= 0);
133 assert(!wrk->y_expand);
134 if (yscale) {
135 for (x_out = 0; x_out < max_span; x_out += 8) {
136 LOAD_32x8(frow + x_out, in0, in1);
137 LOAD_32x8(irow + x_out, in2, in3);
138 const uint32x4_t A0 = MULT_FIX(in0, yscale_half);
139 const uint32x4_t A1 = MULT_FIX(in1, yscale_half);
140 const uint32x4_t B0 = vqsubq_u32(in2, A0);
141 const uint32x4_t B1 = vqsubq_u32(in3, A1);
142 const uint32x4_t C0 = MULT_FIX_FLOOR(B0, fxy_scale_half);
143 const uint32x4_t C1 = MULT_FIX_FLOOR(B1, fxy_scale_half);
144 const uint16x4_t D0 = vmovn_u32(C0);
145 const uint16x4_t D1 = vmovn_u32(C1);
146 const uint8x8_t E = vmovn_u16(vcombine_u16(D0, D1));
147 vst1_u8(dst + x_out, E);
148 STORE_32x8(A0, A1, irow + x_out);
149 }
150 for (; x_out < x_out_max; ++x_out) {
151 const uint32_t frac = (uint32_t)MULT_FIX_C(frow[x_out], yscale);
152 const int v = (int)MULT_FIX_FLOOR_C(irow[x_out] - frac, fxy_scale);
153 assert(v >= 0 && v <= 255);
154 dst[x_out] = v;
155 irow[x_out] = frac; // new fractional start
156 }
157 } else {
158 for (x_out = 0; x_out < max_span; x_out += 8) {
159 LOAD_32x8(irow + x_out, in0, in1);
160 const uint32x4_t A0 = MULT_FIX(in0, fxy_scale_half);
161 const uint32x4_t A1 = MULT_FIX(in1, fxy_scale_half);
162 const uint16x4_t B0 = vmovn_u32(A0);
163 const uint16x4_t B1 = vmovn_u32(A1);
164 const uint8x8_t C = vmovn_u16(vcombine_u16(B0, B1));
165 vst1_u8(dst + x_out, C);
166 STORE_32x8(zero, zero, irow + x_out);
167 }
168 for (; x_out < x_out_max; ++x_out) {
169 const int v = (int)MULT_FIX_C(irow[x_out], fxy_scale);
170 assert(v >= 0 && v <= 255);
171 dst[x_out] = v;
172 irow[x_out] = 0;
173 }
174 }
175 }
176
177 #undef MULT_FIX_FLOOR_C
178 #undef MULT_FIX_C
179 #undef MULT_FIX_FLOOR
180 #undef MULT_FIX
181 #undef ROUNDER
182
183 //------------------------------------------------------------------------------
184
185 extern void WebPRescalerDspInitNEON(void);
186
WebPRescalerDspInitNEON(void)187 WEBP_TSAN_IGNORE_FUNCTION void WebPRescalerDspInitNEON(void) {
188 WebPRescalerExportRowExpand = RescalerExportRowExpand_NEON;
189 WebPRescalerExportRowShrink = RescalerExportRowShrink_NEON;
190 }
191
192 #else // !WEBP_USE_NEON
193
194 WEBP_DSP_INIT_STUB(WebPRescalerDspInitNEON)
195
196 #endif // WEBP_USE_NEON
197