1 .syntax unified 2@ Copyright (c) 2007-2008 CSIRO 3@ Copyright (c) 2007-2009 Xiph.Org Foundation 4@ Copyright (c) 2013 Parrot 5@ Written by Aurélien Zanelli 6@ 7@ Redistribution and use in source and binary forms, with or without 8@ modification, are permitted provided that the following conditions 9@ are met: 10@ 11@ - Redistributions of source code must retain the above copyright 12@ notice, this list of conditions and the following disclaimer. 13@ 14@ - Redistributions in binary form must reproduce the above copyright 15@ notice, this list of conditions and the following disclaimer in the 16@ documentation and/or other materials provided with the distribution. 17@ 18@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19@ ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20@ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21@ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER 22@ OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 23@ EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 24@ PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 25@ PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 26@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 27@ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 28@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30 .text; .p2align 2; .arch armv7-a 31 .fpu neon 32 .object_arch armv4t 33 34 .include "celt/arm/armopts-gnu.S" 35 36 .if OPUS_ARM_MAY_HAVE_EDSP 37 .global celt_pitch_xcorr_edsp 38 .endif 39 40 .if OPUS_ARM_MAY_HAVE_NEON 41 .global celt_pitch_xcorr_neon 42 .endif 43 44 .if OPUS_ARM_MAY_HAVE_NEON 45 46@ Compute sum[k]=sum(x[j]*y[j+k],j=0...len-1), k=0...3 47; xcorr_kernel_neon: @ PROC 48xcorr_kernel_neon_start: 49 @ input: 50 @ r3 = int len 51 @ r4 = opus_val16 *x 52 @ r5 = opus_val16 *y 53 @ q0 = opus_val32 sum[4] 54 @ output: 55 @ q0 = opus_val32 sum[4] 56 @ preserved: r0-r3, r6-r11, d2, q4-q7, q9-q15 57 @ internal usage: 58 @ r12 = int j 59 @ d3 = y_3|y_2|y_1|y_0 60 @ q2 = y_B|y_A|y_9|y_8|y_7|y_6|y_5|y_4 61 @ q3 = x_7|x_6|x_5|x_4|x_3|x_2|x_1|x_0 62 @ q8 = scratch 63 @ 64 @ Load y[0...3] 65 @ This requires len>0 to always be valid (which we assert in the C code). 66 VLD1.16 {d5}, [r5]! 67 SUBS r12, r3, #8 68 BLE xcorr_kernel_neon_process4 69@ Process 8 samples at a time. 70@ This loop loads one y value more than we actually need. Therefore we have to 71@ stop as soon as there are 8 or fewer samples left (instead of 7), to avoid 72@ reading past the end of the array. 73xcorr_kernel_neon_process8: 74 @ This loop has 19 total instructions (10 cycles to issue, minimum), with 75 @ - 2 cycles of ARM insrtuctions, 76 @ - 10 cycles of load/store/byte permute instructions, and 77 @ - 9 cycles of data processing instructions. 78 @ On a Cortex A8, we dual-issue the maximum amount (9 cycles) between the 79 @ latter two categories, meaning the whole loop should run in 10 cycles per 80 @ iteration, barring cache misses. 81 @ 82 @ Load x[0...7] 83 VLD1.16 {d6, d7}, [r4]! 84 @ Unlike VMOV, VAND is a data processsing instruction (and doesn't get 85 @ assembled to VMOV, like VORR would), so it dual-issues with the prior VLD1. 86 VAND d3, d5, d5 87 SUBS r12, r12, #8 88 @ Load y[4...11] 89 VLD1.16 {d4, d5}, [r5]! 90 VMLAL.S16 q0, d3, d6[0] 91 VEXT.16 d16, d3, d4, #1 92 VMLAL.S16 q0, d4, d7[0] 93 VEXT.16 d17, d4, d5, #1 94 VMLAL.S16 q0, d16, d6[1] 95 VEXT.16 d16, d3, d4, #2 96 VMLAL.S16 q0, d17, d7[1] 97 VEXT.16 d17, d4, d5, #2 98 VMLAL.S16 q0, d16, d6[2] 99 VEXT.16 d16, d3, d4, #3 100 VMLAL.S16 q0, d17, d7[2] 101 VEXT.16 d17, d4, d5, #3 102 VMLAL.S16 q0, d16, d6[3] 103 VMLAL.S16 q0, d17, d7[3] 104 BGT xcorr_kernel_neon_process8 105@ Process 4 samples here if we have > 4 left (still reading one extra y value). 106xcorr_kernel_neon_process4: 107 ADDS r12, r12, #4 108 BLE xcorr_kernel_neon_process2 109 @ Load x[0...3] 110 VLD1.16 d6, [r4]! 111 @ Use VAND since it's a data processing instruction again. 112 VAND d4, d5, d5 113 SUB r12, r12, #4 114 @ Load y[4...7] 115 VLD1.16 d5, [r5]! 116 VMLAL.S16 q0, d4, d6[0] 117 VEXT.16 d16, d4, d5, #1 118 VMLAL.S16 q0, d16, d6[1] 119 VEXT.16 d16, d4, d5, #2 120 VMLAL.S16 q0, d16, d6[2] 121 VEXT.16 d16, d4, d5, #3 122 VMLAL.S16 q0, d16, d6[3] 123@ Process 2 samples here if we have > 2 left (still reading one extra y value). 124xcorr_kernel_neon_process2: 125 ADDS r12, r12, #2 126 BLE xcorr_kernel_neon_process1 127 @ Load x[0...1] 128 VLD2.16 {d6[],d7[]}, [r4]! 129 @ Use VAND since it's a data processing instruction again. 130 VAND d4, d5, d5 131 SUB r12, r12, #2 132 @ Load y[4...5] 133 VLD1.32 {d5[]}, [r5]! 134 VMLAL.S16 q0, d4, d6 135 VEXT.16 d16, d4, d5, #1 136 @ Replace bottom copy of {y5,y4} in d5 with {y3,y2} from d4, using VSRI 137 @ instead of VEXT, since it's a data-processing instruction. 138 VSRI.64 d5, d4, #32 139 VMLAL.S16 q0, d16, d7 140@ Process 1 sample using the extra y value we loaded above. 141xcorr_kernel_neon_process1: 142 @ Load next *x 143 VLD1.16 {d6[]}, [r4]! 144 ADDS r12, r12, #1 145 @ y[0...3] are left in d5 from prior iteration(s) (if any) 146 VMLAL.S16 q0, d5, d6 147 MOVLE pc, lr 148@ Now process 1 last sample, not reading ahead. 149 @ Load last *y 150 VLD1.16 {d4[]}, [r5]! 151 VSRI.64 d4, d5, #16 152 @ Load last *x 153 VLD1.16 {d6[]}, [r4]! 154 VMLAL.S16 q0, d4, d6 155 MOV pc, lr 156 .size xcorr_kernel_neon, .-xcorr_kernel_neon @ ENDP 157 158@ opus_val32 celt_pitch_xcorr_neon(opus_val16 *_x, opus_val16 *_y, 159@ opus_val32 *xcorr, int len, int max_pitch) 160; celt_pitch_xcorr_neon: @ PROC 161 @ input: 162 @ r0 = opus_val16 *_x 163 @ r1 = opus_val16 *_y 164 @ r2 = opus_val32 *xcorr 165 @ r3 = int len 166 @ output: 167 @ r0 = int maxcorr 168 @ internal usage: 169 @ r4 = opus_val16 *x (for xcorr_kernel_neon()) 170 @ r5 = opus_val16 *y (for xcorr_kernel_neon()) 171 @ r6 = int max_pitch 172 @ r12 = int j 173 @ q15 = int maxcorr[4] (q15 is not used by xcorr_kernel_neon()) 174 STMFD sp!, {r4-r6, lr} 175 LDR r6, [sp, #16] 176 VMOV.S32 q15, #1 177 @ if (max_pitch < 4) goto celt_pitch_xcorr_neon_process4_done 178 SUBS r6, r6, #4 179 BLT celt_pitch_xcorr_neon_process4_done 180celt_pitch_xcorr_neon_process4: 181 @ xcorr_kernel_neon parameters: 182 @ r3 = len, r4 = _x, r5 = _y, q0 = {0, 0, 0, 0} 183 MOV r4, r0 184 MOV r5, r1 185 VEOR q0, q0, q0 186 @ xcorr_kernel_neon only modifies r4, r5, r12, and q0...q3. 187 @ So we don't save/restore any other registers. 188 BL xcorr_kernel_neon_start 189 SUBS r6, r6, #4 190 VST1.32 {q0}, [r2]! 191 @ _y += 4 192 ADD r1, r1, #8 193 VMAX.S32 q15, q15, q0 194 @ if (max_pitch < 4) goto celt_pitch_xcorr_neon_process4_done 195 BGE celt_pitch_xcorr_neon_process4 196@ We have less than 4 sums left to compute. 197celt_pitch_xcorr_neon_process4_done: 198 ADDS r6, r6, #4 199 @ Reduce maxcorr to a single value 200 VMAX.S32 d30, d30, d31 201 VPMAX.S32 d30, d30, d30 202 @ if (max_pitch <= 0) goto celt_pitch_xcorr_neon_done 203 BLE celt_pitch_xcorr_neon_done 204@ Now compute each remaining sum one at a time. 205celt_pitch_xcorr_neon_process_remaining: 206 MOV r4, r0 207 MOV r5, r1 208 VMOV.I32 q0, #0 209 SUBS r12, r3, #8 210 BLT celt_pitch_xcorr_neon_process_remaining4 211@ Sum terms 8 at a time. 212celt_pitch_xcorr_neon_process_remaining_loop8: 213 @ Load x[0...7] 214 VLD1.16 {q1}, [r4]! 215 @ Load y[0...7] 216 VLD1.16 {q2}, [r5]! 217 SUBS r12, r12, #8 218 VMLAL.S16 q0, d4, d2 219 VMLAL.S16 q0, d5, d3 220 BGE celt_pitch_xcorr_neon_process_remaining_loop8 221@ Sum terms 4 at a time. 222celt_pitch_xcorr_neon_process_remaining4: 223 ADDS r12, r12, #4 224 BLT celt_pitch_xcorr_neon_process_remaining4_done 225 @ Load x[0...3] 226 VLD1.16 {d2}, [r4]! 227 @ Load y[0...3] 228 VLD1.16 {d3}, [r5]! 229 SUB r12, r12, #4 230 VMLAL.S16 q0, d3, d2 231celt_pitch_xcorr_neon_process_remaining4_done: 232 @ Reduce the sum to a single value. 233 VADD.S32 d0, d0, d1 234 VPADDL.S32 d0, d0 235 ADDS r12, r12, #4 236 BLE celt_pitch_xcorr_neon_process_remaining_loop_done 237@ Sum terms 1 at a time. 238celt_pitch_xcorr_neon_process_remaining_loop1: 239 VLD1.16 {d2[]}, [r4]! 240 VLD1.16 {d3[]}, [r5]! 241 SUBS r12, r12, #1 242 VMLAL.S16 q0, d2, d3 243 BGT celt_pitch_xcorr_neon_process_remaining_loop1 244celt_pitch_xcorr_neon_process_remaining_loop_done: 245 VST1.32 {d0[0]}, [r2]! 246 VMAX.S32 d30, d30, d0 247 SUBS r6, r6, #1 248 @ _y++ 249 ADD r1, r1, #2 250 @ if (--max_pitch > 0) goto celt_pitch_xcorr_neon_process_remaining 251 BGT celt_pitch_xcorr_neon_process_remaining 252celt_pitch_xcorr_neon_done: 253 VMOV.32 r0, d30[0] 254 LDMFD sp!, {r4-r6, pc} 255 .size celt_pitch_xcorr_neon, .-celt_pitch_xcorr_neon @ ENDP 256 257 .endif 258 259 .if OPUS_ARM_MAY_HAVE_EDSP 260 261@ This will get used on ARMv7 devices without NEON, so it has been optimized 262@ to take advantage of dual-issuing where possible. 263; xcorr_kernel_edsp: @ PROC 264xcorr_kernel_edsp_start: 265 @ input: 266 @ r3 = int len 267 @ r4 = opus_val16 *_x (must be 32-bit aligned) 268 @ r5 = opus_val16 *_y (must be 32-bit aligned) 269 @ r6...r9 = opus_val32 sum[4] 270 @ output: 271 @ r6...r9 = opus_val32 sum[4] 272 @ preserved: r0-r5 273 @ internal usage 274 @ r2 = int j 275 @ r12,r14 = opus_val16 x[4] 276 @ r10,r11 = opus_val16 y[4] 277 STMFD sp!, {r2,r4,r5,lr} 278 LDR r10, [r5], #4 @ Load y[0...1] 279 SUBS r2, r3, #4 @ j = len-4 280 LDR r11, [r5], #4 @ Load y[2...3] 281 BLE xcorr_kernel_edsp_process4_done 282 LDR r12, [r4], #4 @ Load x[0...1] 283 @ Stall 284xcorr_kernel_edsp_process4: 285 @ The multiplies must issue from pipeline 0, and can't dual-issue with each 286 @ other. Every other instruction here dual-issues with a multiply, and is 287 @ thus "free". There should be no stalls in the body of the loop. 288 SMLABB r6, r12, r10, r6 @ sum[0] = MAC16_16(sum[0],x_0,y_0) 289 LDR r14, [r4], #4 @ Load x[2...3] 290 SMLABT r7, r12, r10, r7 @ sum[1] = MAC16_16(sum[1],x_0,y_1) 291 SUBS r2, r2, #4 @ j-=4 292 SMLABB r8, r12, r11, r8 @ sum[2] = MAC16_16(sum[2],x_0,y_2) 293 SMLABT r9, r12, r11, r9 @ sum[3] = MAC16_16(sum[3],x_0,y_3) 294 SMLATT r6, r12, r10, r6 @ sum[0] = MAC16_16(sum[0],x_1,y_1) 295 LDR r10, [r5], #4 @ Load y[4...5] 296 SMLATB r7, r12, r11, r7 @ sum[1] = MAC16_16(sum[1],x_1,y_2) 297 SMLATT r8, r12, r11, r8 @ sum[2] = MAC16_16(sum[2],x_1,y_3) 298 SMLATB r9, r12, r10, r9 @ sum[3] = MAC16_16(sum[3],x_1,y_4) 299 LDRGT r12, [r4], #4 @ Load x[0...1] 300 SMLABB r6, r14, r11, r6 @ sum[0] = MAC16_16(sum[0],x_2,y_2) 301 SMLABT r7, r14, r11, r7 @ sum[1] = MAC16_16(sum[1],x_2,y_3) 302 SMLABB r8, r14, r10, r8 @ sum[2] = MAC16_16(sum[2],x_2,y_4) 303 SMLABT r9, r14, r10, r9 @ sum[3] = MAC16_16(sum[3],x_2,y_5) 304 SMLATT r6, r14, r11, r6 @ sum[0] = MAC16_16(sum[0],x_3,y_3) 305 LDR r11, [r5], #4 @ Load y[6...7] 306 SMLATB r7, r14, r10, r7 @ sum[1] = MAC16_16(sum[1],x_3,y_4) 307 SMLATT r8, r14, r10, r8 @ sum[2] = MAC16_16(sum[2],x_3,y_5) 308 SMLATB r9, r14, r11, r9 @ sum[3] = MAC16_16(sum[3],x_3,y_6) 309 BGT xcorr_kernel_edsp_process4 310xcorr_kernel_edsp_process4_done: 311 ADDS r2, r2, #4 312 BLE xcorr_kernel_edsp_done 313 LDRH r12, [r4], #2 @ r12 = *x++ 314 SUBS r2, r2, #1 @ j-- 315 @ Stall 316 SMLABB r6, r12, r10, r6 @ sum[0] = MAC16_16(sum[0],x,y_0) 317 LDRHGT r14, [r4], #2 @ r14 = *x++ 318 SMLABT r7, r12, r10, r7 @ sum[1] = MAC16_16(sum[1],x,y_1) 319 SMLABB r8, r12, r11, r8 @ sum[2] = MAC16_16(sum[2],x,y_2) 320 SMLABT r9, r12, r11, r9 @ sum[3] = MAC16_16(sum[3],x,y_3) 321 BLE xcorr_kernel_edsp_done 322 SMLABT r6, r14, r10, r6 @ sum[0] = MAC16_16(sum[0],x,y_1) 323 SUBS r2, r2, #1 @ j-- 324 SMLABB r7, r14, r11, r7 @ sum[1] = MAC16_16(sum[1],x,y_2) 325 LDRH r10, [r5], #2 @ r10 = y_4 = *y++ 326 SMLABT r8, r14, r11, r8 @ sum[2] = MAC16_16(sum[2],x,y_3) 327 LDRHGT r12, [r4], #2 @ r12 = *x++ 328 SMLABB r9, r14, r10, r9 @ sum[3] = MAC16_16(sum[3],x,y_4) 329 BLE xcorr_kernel_edsp_done 330 SMLABB r6, r12, r11, r6 @ sum[0] = MAC16_16(sum[0],tmp,y_2) 331 CMP r2, #1 @ j-- 332 SMLABT r7, r12, r11, r7 @ sum[1] = MAC16_16(sum[1],tmp,y_3) 333 LDRH r2, [r5], #2 @ r2 = y_5 = *y++ 334 SMLABB r8, r12, r10, r8 @ sum[2] = MAC16_16(sum[2],tmp,y_4) 335 LDRHGT r14, [r4] @ r14 = *x 336 SMLABB r9, r12, r2, r9 @ sum[3] = MAC16_16(sum[3],tmp,y_5) 337 BLE xcorr_kernel_edsp_done 338 SMLABT r6, r14, r11, r6 @ sum[0] = MAC16_16(sum[0],tmp,y_3) 339 LDRH r11, [r5] @ r11 = y_6 = *y 340 SMLABB r7, r14, r10, r7 @ sum[1] = MAC16_16(sum[1],tmp,y_4) 341 SMLABB r8, r14, r2, r8 @ sum[2] = MAC16_16(sum[2],tmp,y_5) 342 SMLABB r9, r14, r11, r9 @ sum[3] = MAC16_16(sum[3],tmp,y_6) 343xcorr_kernel_edsp_done: 344 LDMFD sp!, {r2,r4,r5,pc} 345 .size xcorr_kernel_edsp, .-xcorr_kernel_edsp @ ENDP 346 347; celt_pitch_xcorr_edsp: @ PROC 348 @ input: 349 @ r0 = opus_val16 *_x (must be 32-bit aligned) 350 @ r1 = opus_val16 *_y (only needs to be 16-bit aligned) 351 @ r2 = opus_val32 *xcorr 352 @ r3 = int len 353 @ output: 354 @ r0 = maxcorr 355 @ internal usage 356 @ r4 = opus_val16 *x 357 @ r5 = opus_val16 *y 358 @ r6 = opus_val32 sum0 359 @ r7 = opus_val32 sum1 360 @ r8 = opus_val32 sum2 361 @ r9 = opus_val32 sum3 362 @ r1 = int max_pitch 363 @ r12 = int j 364 STMFD sp!, {r4-r11, lr} 365 MOV r5, r1 366 LDR r1, [sp, #36] 367 MOV r4, r0 368 TST r5, #3 369 @ maxcorr = 1 370 MOV r0, #1 371 BEQ celt_pitch_xcorr_edsp_process1u_done 372@ Compute one sum at the start to make y 32-bit aligned. 373 SUBS r12, r3, #4 374 @ r14 = sum = 0 375 MOV r14, #0 376 LDRH r8, [r5], #2 377 BLE celt_pitch_xcorr_edsp_process1u_loop4_done 378 LDR r6, [r4], #4 379 MOV r8, r8, LSL #16 380celt_pitch_xcorr_edsp_process1u_loop4: 381 LDR r9, [r5], #4 382 SMLABT r14, r6, r8, r14 @ sum = MAC16_16(sum, x_0, y_0) 383 LDR r7, [r4], #4 384 SMLATB r14, r6, r9, r14 @ sum = MAC16_16(sum, x_1, y_1) 385 LDR r8, [r5], #4 386 SMLABT r14, r7, r9, r14 @ sum = MAC16_16(sum, x_2, y_2) 387 SUBS r12, r12, #4 @ j-=4 388 SMLATB r14, r7, r8, r14 @ sum = MAC16_16(sum, x_3, y_3) 389 LDRGT r6, [r4], #4 390 BGT celt_pitch_xcorr_edsp_process1u_loop4 391 MOV r8, r8, LSR #16 392celt_pitch_xcorr_edsp_process1u_loop4_done: 393 ADDS r12, r12, #4 394celt_pitch_xcorr_edsp_process1u_loop1: 395 LDRHGE r6, [r4], #2 396 @ Stall 397 SMLABBGE r14, r6, r8, r14 @ sum = MAC16_16(sum, *x, *y) 398 SUBSGE r12, r12, #1 399 LDRHGT r8, [r5], #2 400 BGT celt_pitch_xcorr_edsp_process1u_loop1 401 @ Restore _x 402 SUB r4, r4, r3, LSL #1 403 @ Restore and advance _y 404 SUB r5, r5, r3, LSL #1 405 @ maxcorr = max(maxcorr, sum) 406 CMP r0, r14 407 ADD r5, r5, #2 408 MOVLT r0, r14 409 SUBS r1, r1, #1 410 @ xcorr[i] = sum 411 STR r14, [r2], #4 412 BLE celt_pitch_xcorr_edsp_done 413celt_pitch_xcorr_edsp_process1u_done: 414 @ if (max_pitch < 4) goto celt_pitch_xcorr_edsp_process2 415 SUBS r1, r1, #4 416 BLT celt_pitch_xcorr_edsp_process2 417celt_pitch_xcorr_edsp_process4: 418 @ xcorr_kernel_edsp parameters: 419 @ r3 = len, r4 = _x, r5 = _y, r6...r9 = sum[4] = {0, 0, 0, 0} 420 MOV r6, #0 421 MOV r7, #0 422 MOV r8, #0 423 MOV r9, #0 424 BL xcorr_kernel_edsp_start @ xcorr_kernel_edsp(_x, _y+i, xcorr+i, len) 425 @ maxcorr = max(maxcorr, sum0, sum1, sum2, sum3) 426 CMP r0, r6 427 @ _y+=4 428 ADD r5, r5, #8 429 MOVLT r0, r6 430 CMP r0, r7 431 MOVLT r0, r7 432 CMP r0, r8 433 MOVLT r0, r8 434 CMP r0, r9 435 MOVLT r0, r9 436 STMIA r2!, {r6-r9} 437 SUBS r1, r1, #4 438 BGE celt_pitch_xcorr_edsp_process4 439celt_pitch_xcorr_edsp_process2: 440 ADDS r1, r1, #2 441 BLT celt_pitch_xcorr_edsp_process1a 442 SUBS r12, r3, #4 443 @ {r10, r11} = {sum0, sum1} = {0, 0} 444 MOV r10, #0 445 MOV r11, #0 446 LDR r8, [r5], #4 447 BLE celt_pitch_xcorr_edsp_process2_loop_done 448 LDR r6, [r4], #4 449 LDR r9, [r5], #4 450celt_pitch_xcorr_edsp_process2_loop4: 451 SMLABB r10, r6, r8, r10 @ sum0 = MAC16_16(sum0, x_0, y_0) 452 LDR r7, [r4], #4 453 SMLABT r11, r6, r8, r11 @ sum1 = MAC16_16(sum1, x_0, y_1) 454 SUBS r12, r12, #4 @ j-=4 455 SMLATT r10, r6, r8, r10 @ sum0 = MAC16_16(sum0, x_1, y_1) 456 LDR r8, [r5], #4 457 SMLATB r11, r6, r9, r11 @ sum1 = MAC16_16(sum1, x_1, y_2) 458 LDRGT r6, [r4], #4 459 SMLABB r10, r7, r9, r10 @ sum0 = MAC16_16(sum0, x_2, y_2) 460 SMLABT r11, r7, r9, r11 @ sum1 = MAC16_16(sum1, x_2, y_3) 461 SMLATT r10, r7, r9, r10 @ sum0 = MAC16_16(sum0, x_3, y_3) 462 LDRGT r9, [r5], #4 463 SMLATB r11, r7, r8, r11 @ sum1 = MAC16_16(sum1, x_3, y_4) 464 BGT celt_pitch_xcorr_edsp_process2_loop4 465celt_pitch_xcorr_edsp_process2_loop_done: 466 ADDS r12, r12, #2 467 BLE celt_pitch_xcorr_edsp_process2_1 468 LDR r6, [r4], #4 469 @ Stall 470 SMLABB r10, r6, r8, r10 @ sum0 = MAC16_16(sum0, x_0, y_0) 471 LDR r9, [r5], #4 472 SMLABT r11, r6, r8, r11 @ sum1 = MAC16_16(sum1, x_0, y_1) 473 SUB r12, r12, #2 474 SMLATT r10, r6, r8, r10 @ sum0 = MAC16_16(sum0, x_1, y_1) 475 MOV r8, r9 476 SMLATB r11, r6, r9, r11 @ sum1 = MAC16_16(sum1, x_1, y_2) 477celt_pitch_xcorr_edsp_process2_1: 478 LDRH r6, [r4], #2 479 ADDS r12, r12, #1 480 @ Stall 481 SMLABB r10, r6, r8, r10 @ sum0 = MAC16_16(sum0, x_0, y_0) 482 LDRHGT r7, [r4], #2 483 SMLABT r11, r6, r8, r11 @ sum1 = MAC16_16(sum1, x_0, y_1) 484 BLE celt_pitch_xcorr_edsp_process2_done 485 LDRH r9, [r5], #2 486 SMLABT r10, r7, r8, r10 @ sum0 = MAC16_16(sum0, x_0, y_1) 487 SMLABB r11, r7, r9, r11 @ sum1 = MAC16_16(sum1, x_0, y_2) 488celt_pitch_xcorr_edsp_process2_done: 489 @ Restore _x 490 SUB r4, r4, r3, LSL #1 491 @ Restore and advance _y 492 SUB r5, r5, r3, LSL #1 493 @ maxcorr = max(maxcorr, sum0) 494 CMP r0, r10 495 ADD r5, r5, #2 496 MOVLT r0, r10 497 SUB r1, r1, #2 498 @ maxcorr = max(maxcorr, sum1) 499 CMP r0, r11 500 @ xcorr[i] = sum 501 STR r10, [r2], #4 502 MOVLT r0, r11 503 STR r11, [r2], #4 504celt_pitch_xcorr_edsp_process1a: 505 ADDS r1, r1, #1 506 BLT celt_pitch_xcorr_edsp_done 507 SUBS r12, r3, #4 508 @ r14 = sum = 0 509 MOV r14, #0 510 BLT celt_pitch_xcorr_edsp_process1a_loop_done 511 LDR r6, [r4], #4 512 LDR r8, [r5], #4 513 LDR r7, [r4], #4 514 LDR r9, [r5], #4 515celt_pitch_xcorr_edsp_process1a_loop4: 516 SMLABB r14, r6, r8, r14 @ sum = MAC16_16(sum, x_0, y_0) 517 SUBS r12, r12, #4 @ j-=4 518 SMLATT r14, r6, r8, r14 @ sum = MAC16_16(sum, x_1, y_1) 519 LDRGE r6, [r4], #4 520 SMLABB r14, r7, r9, r14 @ sum = MAC16_16(sum, x_2, y_2) 521 LDRGE r8, [r5], #4 522 SMLATT r14, r7, r9, r14 @ sum = MAC16_16(sum, x_3, y_3) 523 LDRGE r7, [r4], #4 524 LDRGE r9, [r5], #4 525 BGE celt_pitch_xcorr_edsp_process1a_loop4 526celt_pitch_xcorr_edsp_process1a_loop_done: 527 ADDS r12, r12, #2 528 LDRGE r6, [r4], #4 529 LDRGE r8, [r5], #4 530 @ Stall 531 SMLABBGE r14, r6, r8, r14 @ sum = MAC16_16(sum, x_0, y_0) 532 SUBGE r12, r12, #2 533 SMLATTGE r14, r6, r8, r14 @ sum = MAC16_16(sum, x_1, y_1) 534 ADDS r12, r12, #1 535 LDRHGE r6, [r4], #2 536 LDRHGE r8, [r5], #2 537 @ Stall 538 SMLABBGE r14, r6, r8, r14 @ sum = MAC16_16(sum, *x, *y) 539 @ maxcorr = max(maxcorr, sum) 540 CMP r0, r14 541 @ xcorr[i] = sum 542 STR r14, [r2], #4 543 MOVLT r0, r14 544celt_pitch_xcorr_edsp_done: 545 LDMFD sp!, {r4-r11, pc} 546 .size celt_pitch_xcorr_edsp, .-celt_pitch_xcorr_edsp @ ENDP 547 548 .endif 549 550@ END: 551 .section .note.GNU-stack,"",%progbits 552