1 // Tencent is pleased to support the open source community by making ncnn available.
2 //
3 // Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
4 //
5 // Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
6 // in compliance with the License. You may obtain a copy of the License at
7 //
8 // https://opensource.org/licenses/BSD-3-Clause
9 //
10 // Unless required by applicable law or agreed to in writing, software distributed
11 // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
12 // CONDITIONS OF ANY KIND, either express or implied. See the License for the
13 // specific language governing permissions and limitations under the License.
14
conv3x3s1_pack1to4_fp16sa_neon(const Mat & bottom_blob,Mat & top_blob,const Mat & kernel,const Mat & _bias,const Option & opt)15 static void conv3x3s1_pack1to4_fp16sa_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt)
16 {
17 int inch = bottom_blob.c;
18 int outw = top_blob.w;
19 int outh = top_blob.h;
20 int outch = top_blob.c;
21
22 const __fp16* bias = _bias;
23
24 #pragma omp parallel for num_threads(opt.num_threads)
25 for (int p = 0; p < outch; p++)
26 {
27 Mat out0 = top_blob.channel(p);
28
29 float16x4_t _bias0 = bias ? vld1_f16(bias + p * 4) : vdup_n_f16((__fp16)0.f);
30 out0.fill(_bias0);
31
32 const __fp16* k0 = kernel.channel(p);
33
34 int q = 0;
35 for (; q < inch; q++)
36 {
37 __fp16* outptr0 = out0;
38
39 const Mat img0 = bottom_blob.channel(q);
40
41 const __fp16* r0 = img0.row<const __fp16>(0);
42 const __fp16* r1 = img0.row<const __fp16>(1);
43 const __fp16* r2 = img0.row<const __fp16>(2);
44
45 float16x4_t _k00 = vld1_f16(k0);
46 float16x4_t _k01 = vld1_f16(k0 + 4);
47 float16x4_t _k02 = vld1_f16(k0 + 8);
48 float16x4_t _k10 = vld1_f16(k0 + 12);
49 float16x4_t _k11 = vld1_f16(k0 + 16);
50 float16x4_t _k12 = vld1_f16(k0 + 20);
51 float16x4_t _k20 = vld1_f16(k0 + 24);
52 float16x4_t _k21 = vld1_f16(k0 + 28);
53 float16x4_t _k22 = vld1_f16(k0 + 32);
54
55 int i = 0;
56 for (; i < outh; i++)
57 {
58 int j = 0;
59 for (; j + 7 < outw; j += 8)
60 {
61 asm volatile(
62 "prfm pldl1keep, [%0, #256] \n"
63 "ld1 {v24.4h, v25.4h, v26.4h, v27.4h}, [%0], #32 \n" // sum0 sum1 sum2 sum3
64
65 "prfm pldl1keep, [%0, #256] \n"
66 "ld1 {v28.4h, v29.4h, v30.4h, v31.4h}, [%0] \n" // sum4 sum5 sum6 sum7
67
68 "sub %0, %0, #32 \n"
69
70 "prfm pldl1keep, [%1, #128] \n"
71 "ld1 {v0.8h}, [%1], #16 \n" // r0
72 "ld1 {v1.4h}, [%1] \n"
73
74 "fmla v24.4h, %8.4h, v0.h[0] \n"
75 "fmla v25.4h, %8.4h, v0.h[1] \n"
76 "fmla v26.4h, %8.4h, v0.h[2] \n"
77 "fmla v27.4h, %8.4h, v0.h[3] \n"
78 "fmla v28.4h, %8.4h, v0.h[4] \n"
79 "fmla v29.4h, %8.4h, v0.h[5] \n"
80 "fmla v30.4h, %8.4h, v0.h[6] \n"
81 "fmla v31.4h, %8.4h, v0.h[7] \n"
82
83 "fmla v24.4h, %9.4h, v0.h[1] \n"
84 "fmla v25.4h, %9.4h, v0.h[2] \n"
85 "fmla v26.4h, %9.4h, v0.h[3] \n"
86 "fmla v27.4h, %9.4h, v0.h[4] \n"
87 "fmla v28.4h, %9.4h, v0.h[5] \n"
88 "fmla v29.4h, %9.4h, v0.h[6] \n"
89 "fmla v30.4h, %9.4h, v0.h[7] \n"
90 "fmla v31.4h, %9.4h, v1.h[0] \n"
91
92 "fmla v24.4h, %10.4h, v0.h[2] \n"
93 "fmla v25.4h, %10.4h, v0.h[3] \n"
94 "fmla v26.4h, %10.4h, v0.h[4] \n"
95 "fmla v27.4h, %10.4h, v0.h[5] \n"
96 "fmla v28.4h, %10.4h, v0.h[6] \n"
97 "fmla v29.4h, %10.4h, v0.h[7] \n"
98 "fmla v30.4h, %10.4h, v1.h[0] \n"
99 "fmla v31.4h, %10.4h, v1.h[1] \n"
100
101 "prfm pldl1keep, [%2, #128] \n"
102 "ld1 {v2.8h}, [%2], #16 \n" // r1
103 "ld1 {v3.4h}, [%2] \n"
104
105 "fmla v24.4h, %11.4h, v2.h[0] \n"
106 "fmla v25.4h, %11.4h, v2.h[1] \n"
107 "fmla v26.4h, %11.4h, v2.h[2] \n"
108 "fmla v27.4h, %11.4h, v2.h[3] \n"
109 "fmla v28.4h, %11.4h, v2.h[4] \n"
110 "fmla v29.4h, %11.4h, v2.h[5] \n"
111 "fmla v30.4h, %11.4h, v2.h[6] \n"
112 "fmla v31.4h, %11.4h, v2.h[7] \n"
113
114 "fmla v24.4h, %12.4h, v2.h[1] \n"
115 "fmla v25.4h, %12.4h, v2.h[2] \n"
116 "fmla v26.4h, %12.4h, v2.h[3] \n"
117 "fmla v27.4h, %12.4h, v2.h[4] \n"
118 "fmla v28.4h, %12.4h, v2.h[5] \n"
119 "fmla v29.4h, %12.4h, v2.h[6] \n"
120 "fmla v30.4h, %12.4h, v2.h[7] \n"
121 "fmla v31.4h, %12.4h, v3.h[0] \n"
122
123 "fmla v24.4h, %13.4h, v2.h[2] \n"
124 "fmla v25.4h, %13.4h, v2.h[3] \n"
125 "fmla v26.4h, %13.4h, v2.h[4] \n"
126 "fmla v27.4h, %13.4h, v2.h[5] \n"
127 "fmla v28.4h, %13.4h, v2.h[6] \n"
128 "fmla v29.4h, %13.4h, v2.h[7] \n"
129 "fmla v30.4h, %13.4h, v3.h[0] \n"
130 "fmla v31.4h, %13.4h, v3.h[1] \n"
131
132 "prfm pldl1keep, [%3, #128] \n"
133 "ld1 {v4.8h}, [%3], #16 \n" // r2
134 "ld1 {v5.4h}, [%3] \n"
135
136 "fmla v24.4h, %14.4h, v4.h[0] \n"
137 "fmla v25.4h, %14.4h, v4.h[1] \n"
138 "fmla v26.4h, %14.4h, v4.h[2] \n"
139 "fmla v27.4h, %14.4h, v4.h[3] \n"
140 "fmla v28.4h, %14.4h, v4.h[4] \n"
141 "fmla v29.4h, %14.4h, v4.h[5] \n"
142 "fmla v30.4h, %14.4h, v4.h[6] \n"
143 "fmla v31.4h, %14.4h, v4.h[7] \n"
144
145 "fmla v24.4h, %15.4h, v4.h[1] \n"
146 "fmla v25.4h, %15.4h, v4.h[2] \n"
147 "fmla v26.4h, %15.4h, v4.h[3] \n"
148 "fmla v27.4h, %15.4h, v4.h[4] \n"
149 "fmla v28.4h, %15.4h, v4.h[5] \n"
150 "fmla v29.4h, %15.4h, v4.h[6] \n"
151 "fmla v30.4h, %15.4h, v4.h[7] \n"
152 "fmla v31.4h, %15.4h, v5.h[0] \n"
153
154 "fmla v24.4h, %16.4h, v4.h[2] \n"
155 "fmla v25.4h, %16.4h, v4.h[3] \n"
156 "fmla v26.4h, %16.4h, v4.h[4] \n"
157 "fmla v27.4h, %16.4h, v4.h[5] \n"
158 "fmla v28.4h, %16.4h, v4.h[6] \n"
159 "fmla v29.4h, %16.4h, v4.h[7] \n"
160 "fmla v30.4h, %16.4h, v5.h[0] \n"
161 "fmla v31.4h, %16.4h, v5.h[1] \n"
162
163 "st1 {v24.4h, v25.4h, v26.4h, v27.4h}, [%0], #32 \n"
164 "st1 {v28.4h, v29.4h, v30.4h, v31.4h}, [%0], #32 \n"
165
166 : "=r"(outptr0), // %0
167 "=r"(r0), // %1
168 "=r"(r1), // %2
169 "=r"(r2) // %3
170 : "0"(outptr0),
171 "1"(r0),
172 "2"(r1),
173 "3"(r2),
174 "w"(_k00), // %8
175 "w"(_k01), // %9
176 "w"(_k02), // %10
177 "w"(_k10), // %11
178 "w"(_k11), // %12
179 "w"(_k12), // %13
180 "w"(_k20), // %14
181 "w"(_k21), // %15
182 "w"(_k22) // %16
183 : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31");
184 }
185 for (; j + 3 < outw; j += 4)
186 {
187 asm volatile(
188 "prfm pldl1keep, [%0, #256] \n"
189 "ld1 {v28.4h, v29.4h, v30.4h, v31.4h}, [%0] \n" // sum0 sum1 sum2 sum3
190
191 "prfm pldl1keep, [%1, #128] \n"
192 "ld1 {v0.8h}, [%1] \n" // r0
193
194 "fmla v28.4h, %8.4h, v0.h[0] \n"
195 "fmla v29.4h, %8.4h, v0.h[1] \n"
196 "fmla v30.4h, %8.4h, v0.h[2] \n"
197 "fmla v31.4h, %8.4h, v0.h[3] \n"
198
199 "fmla v28.4h, %9.4h, v0.h[1] \n"
200 "fmla v29.4h, %9.4h, v0.h[2] \n"
201 "fmla v30.4h, %9.4h, v0.h[3] \n"
202 "fmla v31.4h, %9.4h, v0.h[4] \n"
203
204 "fmla v28.4h, %10.4h, v0.h[2] \n"
205 "fmla v29.4h, %10.4h, v0.h[3] \n"
206 "fmla v30.4h, %10.4h, v0.h[4] \n"
207 "fmla v31.4h, %10.4h, v0.h[5] \n"
208
209 "prfm pldl1keep, [%2, #128] \n"
210 "ld1 {v1.8h}, [%2] \n" // r1
211
212 "fmla v28.4h, %11.4h, v1.h[0] \n"
213 "fmla v29.4h, %11.4h, v1.h[1] \n"
214 "fmla v30.4h, %11.4h, v1.h[2] \n"
215 "fmla v31.4h, %11.4h, v1.h[3] \n"
216
217 "fmla v28.4h, %12.4h, v1.h[1] \n"
218 "fmla v29.4h, %12.4h, v1.h[2] \n"
219 "fmla v30.4h, %12.4h, v1.h[3] \n"
220 "fmla v31.4h, %12.4h, v1.h[4] \n"
221
222 "fmla v28.4h, %13.4h, v1.h[2] \n"
223 "fmla v29.4h, %13.4h, v1.h[3] \n"
224 "fmla v30.4h, %13.4h, v1.h[4] \n"
225 "fmla v31.4h, %13.4h, v1.h[5] \n"
226
227 "prfm pldl1keep, [%3, #128] \n"
228 "ld1 {v2.8h}, [%3] \n" // r2
229
230 "fmla v28.4h, %14.4h, v2.h[0] \n"
231 "fmla v29.4h, %14.4h, v2.h[1] \n"
232 "fmla v30.4h, %14.4h, v2.h[2] \n"
233 "fmla v31.4h, %14.4h, v2.h[3] \n"
234
235 "fmla v28.4h, %15.4h, v2.h[1] \n"
236 "fmla v29.4h, %15.4h, v2.h[2] \n"
237 "fmla v30.4h, %15.4h, v2.h[3] \n"
238 "fmla v31.4h, %15.4h, v2.h[4] \n"
239
240 "fmla v28.4h, %16.4h, v2.h[2] \n"
241 "fmla v29.4h, %16.4h, v2.h[3] \n"
242 "fmla v30.4h, %16.4h, v2.h[4] \n"
243 "fmla v31.4h, %16.4h, v2.h[5] \n"
244
245 "add %1, %1, #8 \n"
246 "add %2, %2, #8 \n"
247 "add %3, %3, #8 \n"
248
249 "st1 {v28.4h, v29.4h, v30.4h, v31.4h}, [%0], #32 \n"
250
251 : "=r"(outptr0), // %0
252 "=r"(r0), // %1
253 "=r"(r1), // %2
254 "=r"(r2) // %3
255 : "0"(outptr0),
256 "1"(r0),
257 "2"(r1),
258 "3"(r2),
259 "w"(_k00), // %8
260 "w"(_k01), // %9
261 "w"(_k02), // %10
262 "w"(_k10), // %11
263 "w"(_k11), // %12
264 "w"(_k12), // %13
265 "w"(_k20), // %14
266 "w"(_k21), // %15
267 "w"(_k22) // %16
268 : "cc", "memory", "v0", "v1", "v2", "v28", "v29", "v30", "v31");
269 }
270 for (; j + 1 < outw; j += 2)
271 {
272 asm volatile(
273 "prfm pldl1keep, [%0, #128] \n"
274 "ld1 {v30.4h, v31.4h}, [%0] \n" // sum0 sum1
275
276 "prfm pldl1keep, [%1, #64] \n"
277 "ld1 {v0.4h}, [%1] \n" // r0
278
279 "fmla v30.4h, %8.4h, v0.h[0] \n"
280 "fmla v31.4h, %8.4h, v0.h[1] \n"
281 "fmla v30.4h, %9.4h, v0.h[1] \n"
282 "fmla v31.4h, %9.4h, v0.h[2] \n"
283 "fmla v30.4h, %10.4h, v0.h[2] \n"
284 "fmla v31.4h, %10.4h, v0.h[3] \n"
285
286 "prfm pldl1keep, [%2, #64] \n"
287 "ld1 {v1.4h}, [%2] \n" // r1
288
289 "fmla v30.4h, %11.4h, v1.h[0] \n"
290 "fmla v31.4h, %11.4h, v1.h[1] \n"
291 "fmla v30.4h, %12.4h, v1.h[1] \n"
292 "fmla v31.4h, %12.4h, v1.h[2] \n"
293 "fmla v30.4h, %13.4h, v1.h[2] \n"
294 "fmla v31.4h, %13.4h, v1.h[3] \n"
295
296 "prfm pldl1keep, [%3, #64] \n"
297 "ld1 {v2.4h}, [%3] \n" // r2
298
299 "fmla v30.4h, %14.4h, v2.h[0] \n"
300 "fmla v31.4h, %14.4h, v2.h[1] \n"
301 "fmla v30.4h, %15.4h, v2.h[1] \n"
302 "fmla v31.4h, %15.4h, v2.h[2] \n"
303 "fmla v30.4h, %16.4h, v2.h[2] \n"
304 "fmla v31.4h, %16.4h, v2.h[3] \n"
305
306 "add %1, %1, #4 \n"
307 "add %2, %2, #4 \n"
308 "add %3, %3, #4 \n"
309
310 "st1 {v30.4h, v31.4h}, [%0], #16 \n"
311
312 : "=r"(outptr0), // %0
313 "=r"(r0), // %1
314 "=r"(r1), // %2
315 "=r"(r2) // %3
316 : "0"(outptr0),
317 "1"(r0),
318 "2"(r1),
319 "3"(r2),
320 "w"(_k00), // %8
321 "w"(_k01), // %9
322 "w"(_k02), // %10
323 "w"(_k10), // %11
324 "w"(_k11), // %12
325 "w"(_k12), // %13
326 "w"(_k20), // %14
327 "w"(_k21), // %15
328 "w"(_k22) // %16
329 : "cc", "memory", "v0", "v1", "v2", "v30", "v31");
330 }
331 for (; j < outw; j++)
332 {
333 asm volatile(
334 "prfm pldl1keep, [%0, #64] \n"
335 "ld1 {v30.4h}, [%0] \n" // sum0
336
337 "prfm pldl1keep, [%1, #64] \n"
338 "ld1 {v0.4h}, [%1] \n" // r0
339
340 "fmla v30.4h, %8.4h, v0.h[0] \n"
341 "fmla v30.4h, %9.4h, v0.h[1] \n"
342 "fmla v30.4h, %10.4h, v0.h[2] \n"
343
344 "prfm pldl1keep, [%2, #64] \n"
345 "ld1 {v1.4h}, [%2] \n" // r1
346
347 "fmla v30.4h, %11.4h, v1.h[0] \n"
348 "fmla v30.4h, %12.4h, v1.h[1] \n"
349 "fmla v30.4h, %13.4h, v1.h[2] \n"
350
351 "prfm pldl1keep, [%3, #64] \n"
352 "ld1 {v2.4h}, [%3] \n" // r2
353
354 "fmla v30.4h, %14.4h, v2.h[0] \n"
355 "fmla v30.4h, %15.4h, v2.h[1] \n"
356 "fmla v30.4h, %16.4h, v2.h[2] \n"
357
358 "add %1, %1, #2 \n"
359 "add %2, %2, #2 \n"
360 "add %3, %3, #2 \n"
361
362 "st1 {v30.4h}, [%0], #8 \n"
363
364 : "=r"(outptr0), // %0
365 "=r"(r0), // %1
366 "=r"(r1), // %2
367 "=r"(r2) // %3
368 : "0"(outptr0),
369 "1"(r0),
370 "2"(r1),
371 "3"(r2),
372 "w"(_k00), // %8
373 "w"(_k01), // %9
374 "w"(_k02), // %10
375 "w"(_k10), // %11
376 "w"(_k11), // %12
377 "w"(_k12), // %13
378 "w"(_k20), // %14
379 "w"(_k21), // %15
380 "w"(_k22) // %16
381 : "cc", "memory", "v0", "v1", "v2", "v30");
382 }
383
384 r0 += 2;
385 r1 += 2;
386 r2 += 2;
387 }
388
389 k0 += 9 * 4;
390 }
391 }
392 }
393
conv3x3s2_pack1to4_fp16sa_neon(const Mat & bottom_blob,Mat & top_blob,const Mat & kernel,const Mat & _bias,const Option & opt)394 static void conv3x3s2_pack1to4_fp16sa_neon(const Mat& bottom_blob, Mat& top_blob, const Mat& kernel, const Mat& _bias, const Option& opt)
395 {
396 int w = bottom_blob.w;
397 int inch = bottom_blob.c;
398 int outw = top_blob.w;
399 int outh = top_blob.h;
400 int outch = top_blob.c;
401
402 const int tailstep = w - 2 * outw + w;
403
404 const __fp16* bias = _bias;
405
406 #pragma omp parallel for num_threads(opt.num_threads)
407 for (int p = 0; p < outch; p++)
408 {
409 Mat out0 = top_blob.channel(p);
410
411 float16x4_t _bias0 = bias ? vld1_f16(bias + p * 4) : vdup_n_f16((__fp16)0.f);
412 out0.fill(_bias0);
413
414 const __fp16* k0 = kernel.channel(p);
415
416 int q = 0;
417 for (; q < inch; q++)
418 {
419 __fp16* outptr0 = out0;
420
421 const Mat img0 = bottom_blob.channel(q);
422
423 const __fp16* r0 = img0.row<const __fp16>(0);
424 const __fp16* r1 = img0.row<const __fp16>(1);
425 const __fp16* r2 = img0.row<const __fp16>(2);
426
427 float16x4_t _k00 = vld1_f16(k0);
428 float16x4_t _k01 = vld1_f16(k0 + 4);
429 float16x4_t _k02 = vld1_f16(k0 + 8);
430 float16x4_t _k10 = vld1_f16(k0 + 12);
431 float16x4_t _k11 = vld1_f16(k0 + 16);
432 float16x4_t _k12 = vld1_f16(k0 + 20);
433 float16x4_t _k20 = vld1_f16(k0 + 24);
434 float16x4_t _k21 = vld1_f16(k0 + 28);
435 float16x4_t _k22 = vld1_f16(k0 + 32);
436
437 int i = 0;
438 for (; i < outh; i++)
439 {
440 int j = 0;
441 for (; j + 3 < outw; j += 4)
442 {
443 asm volatile(
444 "prfm pldl1keep, [%0, #256] \n"
445 "ld1 {v28.4h, v29.4h, v30.4h, v31.4h}, [%0] \n" // sum0 sum1 sum2 sum3
446
447 "prfm pldl1keep, [%1, #128] \n"
448 "ld1 {v0.8h}, [%1], #16 \n" // r0
449 "ld1 {v1.h}[0], [%1] \n"
450
451 "fmla v28.4h, %8.4h, v0.h[0] \n"
452 "fmla v29.4h, %8.4h, v0.h[2] \n"
453 "fmla v30.4h, %8.4h, v0.h[4] \n"
454 "fmla v31.4h, %8.4h, v0.h[6] \n"
455
456 "fmla v28.4h, %9.4h, v0.h[1] \n"
457 "fmla v29.4h, %9.4h, v0.h[3] \n"
458 "fmla v30.4h, %9.4h, v0.h[5] \n"
459 "fmla v31.4h, %9.4h, v0.h[7] \n"
460
461 "fmla v28.4h, %10.4h, v0.h[2] \n"
462 "fmla v29.4h, %10.4h, v0.h[4] \n"
463 "fmla v30.4h, %10.4h, v0.h[6] \n"
464 "fmla v31.4h, %10.4h, v1.h[0] \n"
465
466 "prfm pldl1keep, [%2, #128] \n"
467 "ld1 {v2.8h}, [%2], #16 \n" // r1
468 "ld1 {v3.h}[0], [%2] \n"
469
470 "fmla v28.4h, %11.4h, v2.h[0] \n"
471 "fmla v29.4h, %11.4h, v2.h[2] \n"
472 "fmla v30.4h, %11.4h, v2.h[4] \n"
473 "fmla v31.4h, %11.4h, v2.h[6] \n"
474
475 "fmla v28.4h, %12.4h, v2.h[1] \n"
476 "fmla v29.4h, %12.4h, v2.h[3] \n"
477 "fmla v30.4h, %12.4h, v2.h[5] \n"
478 "fmla v31.4h, %12.4h, v2.h[7] \n"
479
480 "fmla v28.4h, %13.4h, v2.h[2] \n"
481 "fmla v29.4h, %13.4h, v2.h[4] \n"
482 "fmla v30.4h, %13.4h, v2.h[6] \n"
483 "fmla v31.4h, %13.4h, v3.h[0] \n"
484
485 "prfm pldl1keep, [%3, #128] \n"
486 "ld1 {v4.8h}, [%3], #16 \n" // r2
487 "ld1 {v5.h}[0], [%3] \n"
488
489 "fmla v28.4h, %14.4h, v4.h[0] \n"
490 "fmla v29.4h, %14.4h, v4.h[2] \n"
491 "fmla v30.4h, %14.4h, v4.h[4] \n"
492 "fmla v31.4h, %14.4h, v4.h[6] \n"
493
494 "fmla v28.4h, %15.4h, v4.h[1] \n"
495 "fmla v29.4h, %15.4h, v4.h[3] \n"
496 "fmla v30.4h, %15.4h, v4.h[5] \n"
497 "fmla v31.4h, %15.4h, v4.h[7] \n"
498
499 "fmla v28.4h, %16.4h, v4.h[2] \n"
500 "fmla v29.4h, %16.4h, v4.h[4] \n"
501 "fmla v30.4h, %16.4h, v4.h[6] \n"
502 "fmla v31.4h, %16.4h, v5.h[0] \n"
503
504 "st1 {v28.4h, v29.4h, v30.4h, v31.4h}, [%0], #32 \n"
505
506 : "=r"(outptr0), // %0
507 "=r"(r0), // %1
508 "=r"(r1), // %2
509 "=r"(r2) // %3
510 : "0"(outptr0),
511 "1"(r0),
512 "2"(r1),
513 "3"(r2),
514 "w"(_k00), // %8
515 "w"(_k01), // %9
516 "w"(_k02), // %10
517 "w"(_k10), // %11
518 "w"(_k11), // %12
519 "w"(_k12), // %13
520 "w"(_k20), // %14
521 "w"(_k21), // %15
522 "w"(_k22) // %16
523 : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v28", "v29", "v30", "v31");
524 }
525 for (; j + 1 < outw; j += 2)
526 {
527 asm volatile(
528 "prfm pldl1keep, [%0, #128] \n"
529 "ld1 {v30.4h, v31.4h}, [%0] \n" // sum0 sum1
530
531 "prfm pldl1keep, [%1, #64] \n"
532 "ld1 {v0.4h}, [%1], #8 \n" // r0
533 "ld1 {v1.h}[0], [%1] \n"
534
535 "fmla v30.4h, %8.4h, v0.h[0] \n"
536 "fmla v31.4h, %8.4h, v0.h[2] \n"
537 "fmla v30.4h, %9.4h, v0.h[1] \n"
538 "fmla v31.4h, %9.4h, v0.h[3] \n"
539 "fmla v30.4h, %10.4h, v0.h[2] \n"
540 "fmla v31.4h, %10.4h, v1.h[0] \n"
541
542 "prfm pldl1keep, [%2, #64] \n"
543 "ld1 {v2.4h}, [%2], #8 \n" // r1
544 "ld1 {v3.h}[0], [%2] \n"
545
546 "fmla v30.4h, %11.4h, v2.h[0] \n"
547 "fmla v31.4h, %11.4h, v2.h[2] \n"
548 "fmla v30.4h, %12.4h, v2.h[1] \n"
549 "fmla v31.4h, %12.4h, v2.h[3] \n"
550 "fmla v30.4h, %13.4h, v2.h[2] \n"
551 "fmla v31.4h, %13.4h, v3.h[0] \n"
552
553 "prfm pldl1keep, [%3, #64] \n"
554 "ld1 {v4.4h}, [%3], #8 \n" // r2
555 "ld1 {v5.h}[0], [%3] \n"
556
557 "fmla v30.4h, %14.4h, v4.h[0] \n"
558 "fmla v31.4h, %14.4h, v4.h[2] \n"
559 "fmla v30.4h, %15.4h, v4.h[1] \n"
560 "fmla v31.4h, %15.4h, v4.h[3] \n"
561 "fmla v30.4h, %16.4h, v4.h[2] \n"
562 "fmla v31.4h, %16.4h, v5.h[0] \n"
563
564 "st1 {v30.4h, v31.4h}, [%0], #16 \n"
565
566 : "=r"(outptr0), // %0
567 "=r"(r0), // %1
568 "=r"(r1), // %2
569 "=r"(r2) // %3
570 : "0"(outptr0),
571 "1"(r0),
572 "2"(r1),
573 "3"(r2),
574 "w"(_k00), // %8
575 "w"(_k01), // %9
576 "w"(_k02), // %10
577 "w"(_k10), // %11
578 "w"(_k11), // %12
579 "w"(_k12), // %13
580 "w"(_k20), // %14
581 "w"(_k21), // %15
582 "w"(_k22) // %16
583 : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v30", "v31");
584 }
585 for (; j < outw; j++)
586 {
587 asm volatile(
588 "prfm pldl1keep, [%0, #64] \n"
589 "ld1 {v30.4h}, [%0] \n" // sum0
590
591 "prfm pldl1keep, [%1, #64] \n"
592 "ld1 {v0.4h}, [%1] \n" // r0
593
594 "fmla v30.4h, %8.4h, v0.h[0] \n"
595 "fmla v30.4h, %9.4h, v0.h[1] \n"
596 "fmla v30.4h, %10.4h, v0.h[2] \n"
597
598 "prfm pldl1keep, [%2, #64] \n"
599 "ld1 {v1.4h}, [%2] \n" // r1
600
601 "fmla v30.4h, %11.4h, v1.h[0] \n"
602 "fmla v30.4h, %12.4h, v1.h[1] \n"
603 "fmla v30.4h, %13.4h, v1.h[2] \n"
604
605 "prfm pldl1keep, [%3, #64] \n"
606 "ld1 {v2.4h}, [%3] \n" // r2
607
608 "fmla v30.4h, %14.4h, v2.h[0] \n"
609 "fmla v30.4h, %15.4h, v2.h[1] \n"
610 "fmla v30.4h, %16.4h, v2.h[2] \n"
611
612 "add %1, %1, #4 \n"
613 "add %2, %2, #4 \n"
614 "add %3, %3, #4 \n"
615
616 "st1 {v30.4h}, [%0], #8 \n"
617
618 : "=r"(outptr0), // %0
619 "=r"(r0), // %1
620 "=r"(r1), // %2
621 "=r"(r2) // %3
622 : "0"(outptr0),
623 "1"(r0),
624 "2"(r1),
625 "3"(r2),
626 "w"(_k00), // %8
627 "w"(_k01), // %9
628 "w"(_k02), // %10
629 "w"(_k10), // %11
630 "w"(_k11), // %12
631 "w"(_k12), // %13
632 "w"(_k20), // %14
633 "w"(_k21), // %15
634 "w"(_k22) // %16
635 : "cc", "memory", "v0", "v1", "v2", "v30");
636 }
637
638 r0 += tailstep;
639 r1 += tailstep;
640 r2 += tailstep;
641 }
642
643 k0 += 9 * 4;
644 }
645 }
646 }
647