160662d10Schristos#include "arm_asm.h" 2*1dcdf01fSchristos#include "arm_arch.h" 360662d10Schristos 460662d10Schristos.text 5*1dcdf01fSchristos#if defined(__thumb2__) && !defined(__APPLE__) 6*1dcdf01fSchristos.syntax unified 7*1dcdf01fSchristos.thumb 8*1dcdf01fSchristos#else 960662d10Schristos.code 32 10*1dcdf01fSchristos#undef __thumb2__ 11*1dcdf01fSchristos#endif 1260662d10Schristos 1360662d10Schristos.align 5 14*1dcdf01fSchristos.globl OPENSSL_atomic_add 1560662d10Schristos.type OPENSSL_atomic_add,%function 1660662d10SchristosOPENSSL_atomic_add: 1760662d10Schristos#if __ARM_ARCH__>=6 1860662d10Schristos.Ladd: ldrex r2,[r0] 1960662d10Schristos add r3,r2,r1 2060662d10Schristos strex r2,r3,[r0] 2160662d10Schristos cmp r2,#0 2260662d10Schristos bne .Ladd 2360662d10Schristos mov r0,r3 2460662d10Schristos RET 2560662d10Schristos#else 26*1dcdf01fSchristos stmdb sp!,{r4,r5,r6,lr} 2760662d10Schristos ldr r2,.Lspinlock 2860662d10Schristos adr r3,.Lspinlock 2960662d10Schristos mov r4,r0 3060662d10Schristos mov r5,r1 3160662d10Schristos add r6,r3,r2 @ &spinlock 3260662d10Schristos b .+8 3360662d10Schristos.Lspin: bl sched_yield 3460662d10Schristos mov r0,#-1 3560662d10Schristos swp r0,r0,[r6] 3660662d10Schristos cmp r0,#0 3760662d10Schristos bne .Lspin 3860662d10Schristos 3960662d10Schristos ldr r2,[r4] 4060662d10Schristos add r2,r2,r5 4160662d10Schristos str r2,[r4] 4260662d10Schristos str r0,[r6] @ release spinlock 43*1dcdf01fSchristos ldmia sp!,{r4,r5,r6,lr} 4460662d10Schristos tst lr,#1 4560662d10Schristos moveq pc,lr 46*1dcdf01fSchristos.word 0xe12fff1e @ RET 4760662d10Schristos#endif 4860662d10Schristos.size OPENSSL_atomic_add,.-OPENSSL_atomic_add 4960662d10Schristos 50*1dcdf01fSchristos.globl OPENSSL_cleanse 5160662d10Schristos.type OPENSSL_cleanse,%function 5260662d10SchristosOPENSSL_cleanse: 5360662d10Schristos eor ip,ip,ip 5460662d10Schristos cmp r1,#7 55*1dcdf01fSchristos#ifdef __thumb2__ 56*1dcdf01fSchristos itt hs 57*1dcdf01fSchristos#endif 5860662d10Schristos subhs r1,r1,#4 5960662d10Schristos bhs .Lot 6060662d10Schristos cmp r1,#0 6160662d10Schristos beq .Lcleanse_done 6260662d10Schristos.Little: 6360662d10Schristos strb ip,[r0],#1 6460662d10Schristos subs r1,r1,#1 6560662d10Schristos bhi .Little 6660662d10Schristos b .Lcleanse_done 6760662d10Schristos 6860662d10Schristos.Lot: tst r0,#3 6960662d10Schristos beq .Laligned 7060662d10Schristos strb ip,[r0],#1 7160662d10Schristos sub r1,r1,#1 7260662d10Schristos b .Lot 7360662d10Schristos.Laligned: 7460662d10Schristos str ip,[r0],#4 7560662d10Schristos subs r1,r1,#4 7660662d10Schristos bhs .Laligned 7760662d10Schristos adds r1,r1,#4 7860662d10Schristos bne .Little 7960662d10Schristos.Lcleanse_done: 8060662d10Schristos#if __ARM_ARCH__>=5 8160662d10Schristos RET 8260662d10Schristos#else 8360662d10Schristos tst lr,#1 8460662d10Schristos moveq pc,lr 85*1dcdf01fSchristos.word 0xe12fff1e @ RET 8660662d10Schristos#endif 8760662d10Schristos.size OPENSSL_cleanse,.-OPENSSL_cleanse 8860662d10Schristos 89*1dcdf01fSchristos.globl CRYPTO_memcmp 90*1dcdf01fSchristos.type CRYPTO_memcmp,%function 91*1dcdf01fSchristos.align 4 92*1dcdf01fSchristosCRYPTO_memcmp: 93*1dcdf01fSchristos eor ip,ip,ip 94*1dcdf01fSchristos cmp r2,#0 95*1dcdf01fSchristos beq .Lno_data 96*1dcdf01fSchristos stmdb sp!,{r4,r5} 97*1dcdf01fSchristos 98*1dcdf01fSchristos.Loop_cmp: 99*1dcdf01fSchristos ldrb r4,[r0],#1 100*1dcdf01fSchristos ldrb r5,[r1],#1 101*1dcdf01fSchristos eor r4,r4,r5 102*1dcdf01fSchristos orr ip,ip,r4 103*1dcdf01fSchristos subs r2,r2,#1 104*1dcdf01fSchristos bne .Loop_cmp 105*1dcdf01fSchristos 106*1dcdf01fSchristos ldmia sp!,{r4,r5} 107*1dcdf01fSchristos.Lno_data: 108*1dcdf01fSchristos rsb r0,ip,#0 109*1dcdf01fSchristos mov r0,r0,lsr#31 110*1dcdf01fSchristos#if __ARM_ARCH__>=5 111*1dcdf01fSchristos RET 112*1dcdf01fSchristos#else 113*1dcdf01fSchristos tst lr,#1 114*1dcdf01fSchristos moveq pc,lr 115*1dcdf01fSchristos.word 0xe12fff1e @ RET 116*1dcdf01fSchristos#endif 117*1dcdf01fSchristos.size CRYPTO_memcmp,.-CRYPTO_memcmp 118*1dcdf01fSchristos 11960662d10Schristos#if __ARM_MAX_ARCH__>=7 12060662d10Schristos.arch armv7-a 12160662d10Schristos.fpu neon 12260662d10Schristos 12360662d10Schristos.align 5 124*1dcdf01fSchristos.globl _armv7_neon_probe 12560662d10Schristos.type _armv7_neon_probe,%function 12660662d10Schristos_armv7_neon_probe: 12760662d10Schristos vorr q0,q0,q0 12860662d10Schristos RET 12960662d10Schristos.size _armv7_neon_probe,.-_armv7_neon_probe 13060662d10Schristos 131*1dcdf01fSchristos.globl _armv7_tick 13260662d10Schristos.type _armv7_tick,%function 13360662d10Schristos_armv7_tick: 134*1dcdf01fSchristos#ifdef __APPLE__ 135*1dcdf01fSchristos mrrc p15,0,r0,r1,c14 @ CNTPCT 136*1dcdf01fSchristos#else 13760662d10Schristos mrrc p15,1,r0,r1,c14 @ CNTVCT 138*1dcdf01fSchristos#endif 13960662d10Schristos RET 14060662d10Schristos.size _armv7_tick,.-_armv7_tick 14160662d10Schristos 142*1dcdf01fSchristos.globl _armv8_aes_probe 14360662d10Schristos.type _armv8_aes_probe,%function 14460662d10Schristos_armv8_aes_probe: 145*1dcdf01fSchristos#if defined(__thumb2__) && !defined(__APPLE__) 146*1dcdf01fSchristos.byte 0xb0,0xff,0x00,0x03 @ aese.8 q0,q0 147*1dcdf01fSchristos#else 148*1dcdf01fSchristos.byte 0x00,0x03,0xb0,0xf3 @ aese.8 q0,q0 149*1dcdf01fSchristos#endif 15060662d10Schristos RET 15160662d10Schristos.size _armv8_aes_probe,.-_armv8_aes_probe 15260662d10Schristos 153*1dcdf01fSchristos.globl _armv8_sha1_probe 15460662d10Schristos.type _armv8_sha1_probe,%function 15560662d10Schristos_armv8_sha1_probe: 156*1dcdf01fSchristos#if defined(__thumb2__) && !defined(__APPLE__) 157*1dcdf01fSchristos.byte 0x00,0xef,0x40,0x0c @ sha1c.32 q0,q0,q0 158*1dcdf01fSchristos#else 159*1dcdf01fSchristos.byte 0x40,0x0c,0x00,0xf2 @ sha1c.32 q0,q0,q0 160*1dcdf01fSchristos#endif 16160662d10Schristos RET 16260662d10Schristos.size _armv8_sha1_probe,.-_armv8_sha1_probe 16360662d10Schristos 164*1dcdf01fSchristos.globl _armv8_sha256_probe 16560662d10Schristos.type _armv8_sha256_probe,%function 16660662d10Schristos_armv8_sha256_probe: 167*1dcdf01fSchristos#if defined(__thumb2__) && !defined(__APPLE__) 168*1dcdf01fSchristos.byte 0x00,0xff,0x40,0x0c @ sha256h.32 q0,q0,q0 169*1dcdf01fSchristos#else 170*1dcdf01fSchristos.byte 0x40,0x0c,0x00,0xf3 @ sha256h.32 q0,q0,q0 171*1dcdf01fSchristos#endif 17260662d10Schristos RET 17360662d10Schristos.size _armv8_sha256_probe,.-_armv8_sha256_probe 174*1dcdf01fSchristos.globl _armv8_pmull_probe 17560662d10Schristos.type _armv8_pmull_probe,%function 17660662d10Schristos_armv8_pmull_probe: 177*1dcdf01fSchristos#if defined(__thumb2__) && !defined(__APPLE__) 178*1dcdf01fSchristos.byte 0xa0,0xef,0x00,0x0e @ vmull.p64 q0,d0,d0 179*1dcdf01fSchristos#else 180*1dcdf01fSchristos.byte 0x00,0x0e,0xa0,0xf2 @ vmull.p64 q0,d0,d0 181*1dcdf01fSchristos#endif 18260662d10Schristos RET 18360662d10Schristos.size _armv8_pmull_probe,.-_armv8_pmull_probe 18460662d10Schristos#endif 18560662d10Schristos 186*1dcdf01fSchristos.globl OPENSSL_wipe_cpu 18760662d10Schristos.type OPENSSL_wipe_cpu,%function 18860662d10SchristosOPENSSL_wipe_cpu: 18960662d10Schristos#if __ARM_MAX_ARCH__>=7 19060662d10Schristos ldr r0,.LOPENSSL_armcap 19160662d10Schristos adr r1,.LOPENSSL_armcap 19260662d10Schristos ldr r0,[r1,r0] 193*1dcdf01fSchristos#ifdef __APPLE__ 194*1dcdf01fSchristos ldr r0,[r0] 195*1dcdf01fSchristos#endif 19660662d10Schristos#endif 19760662d10Schristos eor r2,r2,r2 19860662d10Schristos eor r3,r3,r3 19960662d10Schristos eor ip,ip,ip 20060662d10Schristos#if __ARM_MAX_ARCH__>=7 20160662d10Schristos tst r0,#1 20260662d10Schristos beq .Lwipe_done 20360662d10Schristos veor q0, q0, q0 20460662d10Schristos veor q1, q1, q1 20560662d10Schristos veor q2, q2, q2 20660662d10Schristos veor q3, q3, q3 20760662d10Schristos veor q8, q8, q8 20860662d10Schristos veor q9, q9, q9 20960662d10Schristos veor q10, q10, q10 21060662d10Schristos veor q11, q11, q11 21160662d10Schristos veor q12, q12, q12 21260662d10Schristos veor q13, q13, q13 21360662d10Schristos veor q14, q14, q14 21460662d10Schristos veor q15, q15, q15 21560662d10Schristos.Lwipe_done: 21660662d10Schristos#endif 21760662d10Schristos mov r0,sp 21860662d10Schristos#if __ARM_ARCH__>=5 21960662d10Schristos RET 22060662d10Schristos#else 22160662d10Schristos tst lr,#1 22260662d10Schristos moveq pc,lr 223*1dcdf01fSchristos.word 0xe12fff1e @ RET 22460662d10Schristos#endif 22560662d10Schristos.size OPENSSL_wipe_cpu,.-OPENSSL_wipe_cpu 22660662d10Schristos 227*1dcdf01fSchristos.globl OPENSSL_instrument_bus 22860662d10Schristos.type OPENSSL_instrument_bus,%function 22960662d10SchristosOPENSSL_instrument_bus: 23060662d10Schristos eor r0,r0,r0 23160662d10Schristos#if __ARM_ARCH__>=5 23260662d10Schristos RET 23360662d10Schristos#else 23460662d10Schristos tst lr,#1 23560662d10Schristos moveq pc,lr 236*1dcdf01fSchristos.word 0xe12fff1e @ RET 23760662d10Schristos#endif 23860662d10Schristos.size OPENSSL_instrument_bus,.-OPENSSL_instrument_bus 23960662d10Schristos 240*1dcdf01fSchristos.globl OPENSSL_instrument_bus2 24160662d10Schristos.type OPENSSL_instrument_bus2,%function 24260662d10SchristosOPENSSL_instrument_bus2: 24360662d10Schristos eor r0,r0,r0 24460662d10Schristos#if __ARM_ARCH__>=5 24560662d10Schristos RET 24660662d10Schristos#else 24760662d10Schristos tst lr,#1 24860662d10Schristos moveq pc,lr 249*1dcdf01fSchristos.word 0xe12fff1e @ RET 25060662d10Schristos#endif 25160662d10Schristos.size OPENSSL_instrument_bus2,.-OPENSSL_instrument_bus2 25260662d10Schristos 25360662d10Schristos.align 5 25460662d10Schristos#if __ARM_MAX_ARCH__>=7 25560662d10Schristos.LOPENSSL_armcap: 256*1dcdf01fSchristos.word OPENSSL_armcap_P-. 25760662d10Schristos#endif 25860662d10Schristos#if __ARM_ARCH__>=6 25960662d10Schristos.align 5 26060662d10Schristos#else 26160662d10Schristos.Lspinlock: 26260662d10Schristos.word atomic_add_spinlock-.Lspinlock 26360662d10Schristos.align 5 26460662d10Schristos 26560662d10Schristos.data 26660662d10Schristos.align 2 26760662d10Schristosatomic_add_spinlock: 26860662d10Schristos.word 0 26960662d10Schristos#endif 27060662d10Schristos 27160662d10Schristos.comm OPENSSL_armcap_P,4,4 27260662d10Schristos.hidden OPENSSL_armcap_P 273