160662d10Schristos#include "arm_asm.h"
2*1dcdf01fSchristos#include "arm_arch.h"
360662d10Schristos
460662d10Schristos.text
5*1dcdf01fSchristos#if defined(__thumb2__) && !defined(__APPLE__)
6*1dcdf01fSchristos.syntax	unified
7*1dcdf01fSchristos.thumb
8*1dcdf01fSchristos#else
960662d10Schristos.code	32
10*1dcdf01fSchristos#undef	__thumb2__
11*1dcdf01fSchristos#endif
1260662d10Schristos
1360662d10Schristos.align	5
14*1dcdf01fSchristos.globl	OPENSSL_atomic_add
1560662d10Schristos.type	OPENSSL_atomic_add,%function
1660662d10SchristosOPENSSL_atomic_add:
1760662d10Schristos#if __ARM_ARCH__>=6
1860662d10Schristos.Ladd:	ldrex	r2,[r0]
1960662d10Schristos	add	r3,r2,r1
2060662d10Schristos	strex	r2,r3,[r0]
2160662d10Schristos	cmp	r2,#0
2260662d10Schristos	bne	.Ladd
2360662d10Schristos	mov	r0,r3
2460662d10Schristos	RET
2560662d10Schristos#else
26*1dcdf01fSchristos	stmdb	sp!,{r4,r5,r6,lr}
2760662d10Schristos	ldr	r2,.Lspinlock
2860662d10Schristos	adr	r3,.Lspinlock
2960662d10Schristos	mov	r4,r0
3060662d10Schristos	mov	r5,r1
3160662d10Schristos	add	r6,r3,r2	@ &spinlock
3260662d10Schristos	b	.+8
3360662d10Schristos.Lspin:	bl	sched_yield
3460662d10Schristos	mov	r0,#-1
3560662d10Schristos	swp	r0,r0,[r6]
3660662d10Schristos	cmp	r0,#0
3760662d10Schristos	bne	.Lspin
3860662d10Schristos
3960662d10Schristos	ldr	r2,[r4]
4060662d10Schristos	add	r2,r2,r5
4160662d10Schristos	str	r2,[r4]
4260662d10Schristos	str	r0,[r6]		@ release spinlock
43*1dcdf01fSchristos	ldmia	sp!,{r4,r5,r6,lr}
4460662d10Schristos	tst	lr,#1
4560662d10Schristos	moveq	pc,lr
46*1dcdf01fSchristos.word	0xe12fff1e	@ RET
4760662d10Schristos#endif
4860662d10Schristos.size	OPENSSL_atomic_add,.-OPENSSL_atomic_add
4960662d10Schristos
50*1dcdf01fSchristos.globl	OPENSSL_cleanse
5160662d10Schristos.type	OPENSSL_cleanse,%function
5260662d10SchristosOPENSSL_cleanse:
5360662d10Schristos	eor	ip,ip,ip
5460662d10Schristos	cmp	r1,#7
55*1dcdf01fSchristos#ifdef	__thumb2__
56*1dcdf01fSchristos	itt	hs
57*1dcdf01fSchristos#endif
5860662d10Schristos	subhs	r1,r1,#4
5960662d10Schristos	bhs	.Lot
6060662d10Schristos	cmp	r1,#0
6160662d10Schristos	beq	.Lcleanse_done
6260662d10Schristos.Little:
6360662d10Schristos	strb	ip,[r0],#1
6460662d10Schristos	subs	r1,r1,#1
6560662d10Schristos	bhi	.Little
6660662d10Schristos	b	.Lcleanse_done
6760662d10Schristos
6860662d10Schristos.Lot:	tst	r0,#3
6960662d10Schristos	beq	.Laligned
7060662d10Schristos	strb	ip,[r0],#1
7160662d10Schristos	sub	r1,r1,#1
7260662d10Schristos	b	.Lot
7360662d10Schristos.Laligned:
7460662d10Schristos	str	ip,[r0],#4
7560662d10Schristos	subs	r1,r1,#4
7660662d10Schristos	bhs	.Laligned
7760662d10Schristos	adds	r1,r1,#4
7860662d10Schristos	bne	.Little
7960662d10Schristos.Lcleanse_done:
8060662d10Schristos#if __ARM_ARCH__>=5
8160662d10Schristos	RET
8260662d10Schristos#else
8360662d10Schristos	tst	lr,#1
8460662d10Schristos	moveq	pc,lr
85*1dcdf01fSchristos.word	0xe12fff1e	@ RET
8660662d10Schristos#endif
8760662d10Schristos.size	OPENSSL_cleanse,.-OPENSSL_cleanse
8860662d10Schristos
89*1dcdf01fSchristos.globl	CRYPTO_memcmp
90*1dcdf01fSchristos.type	CRYPTO_memcmp,%function
91*1dcdf01fSchristos.align	4
92*1dcdf01fSchristosCRYPTO_memcmp:
93*1dcdf01fSchristos	eor	ip,ip,ip
94*1dcdf01fSchristos	cmp	r2,#0
95*1dcdf01fSchristos	beq	.Lno_data
96*1dcdf01fSchristos	stmdb	sp!,{r4,r5}
97*1dcdf01fSchristos
98*1dcdf01fSchristos.Loop_cmp:
99*1dcdf01fSchristos	ldrb	r4,[r0],#1
100*1dcdf01fSchristos	ldrb	r5,[r1],#1
101*1dcdf01fSchristos	eor	r4,r4,r5
102*1dcdf01fSchristos	orr	ip,ip,r4
103*1dcdf01fSchristos	subs	r2,r2,#1
104*1dcdf01fSchristos	bne	.Loop_cmp
105*1dcdf01fSchristos
106*1dcdf01fSchristos	ldmia	sp!,{r4,r5}
107*1dcdf01fSchristos.Lno_data:
108*1dcdf01fSchristos	rsb	r0,ip,#0
109*1dcdf01fSchristos	mov	r0,r0,lsr#31
110*1dcdf01fSchristos#if __ARM_ARCH__>=5
111*1dcdf01fSchristos	RET
112*1dcdf01fSchristos#else
113*1dcdf01fSchristos	tst	lr,#1
114*1dcdf01fSchristos	moveq	pc,lr
115*1dcdf01fSchristos.word	0xe12fff1e	@ RET
116*1dcdf01fSchristos#endif
117*1dcdf01fSchristos.size	CRYPTO_memcmp,.-CRYPTO_memcmp
118*1dcdf01fSchristos
11960662d10Schristos#if __ARM_MAX_ARCH__>=7
12060662d10Schristos.arch	armv7-a
12160662d10Schristos.fpu	neon
12260662d10Schristos
12360662d10Schristos.align	5
124*1dcdf01fSchristos.globl	_armv7_neon_probe
12560662d10Schristos.type	_armv7_neon_probe,%function
12660662d10Schristos_armv7_neon_probe:
12760662d10Schristos	vorr	q0,q0,q0
12860662d10Schristos	RET
12960662d10Schristos.size	_armv7_neon_probe,.-_armv7_neon_probe
13060662d10Schristos
131*1dcdf01fSchristos.globl	_armv7_tick
13260662d10Schristos.type	_armv7_tick,%function
13360662d10Schristos_armv7_tick:
134*1dcdf01fSchristos#ifdef	__APPLE__
135*1dcdf01fSchristos	mrrc	p15,0,r0,r1,c14		@ CNTPCT
136*1dcdf01fSchristos#else
13760662d10Schristos	mrrc	p15,1,r0,r1,c14		@ CNTVCT
138*1dcdf01fSchristos#endif
13960662d10Schristos	RET
14060662d10Schristos.size	_armv7_tick,.-_armv7_tick
14160662d10Schristos
142*1dcdf01fSchristos.globl	_armv8_aes_probe
14360662d10Schristos.type	_armv8_aes_probe,%function
14460662d10Schristos_armv8_aes_probe:
145*1dcdf01fSchristos#if defined(__thumb2__) && !defined(__APPLE__)
146*1dcdf01fSchristos.byte	0xb0,0xff,0x00,0x03	@ aese.8	q0,q0
147*1dcdf01fSchristos#else
148*1dcdf01fSchristos.byte	0x00,0x03,0xb0,0xf3	@ aese.8	q0,q0
149*1dcdf01fSchristos#endif
15060662d10Schristos	RET
15160662d10Schristos.size	_armv8_aes_probe,.-_armv8_aes_probe
15260662d10Schristos
153*1dcdf01fSchristos.globl	_armv8_sha1_probe
15460662d10Schristos.type	_armv8_sha1_probe,%function
15560662d10Schristos_armv8_sha1_probe:
156*1dcdf01fSchristos#if defined(__thumb2__) && !defined(__APPLE__)
157*1dcdf01fSchristos.byte	0x00,0xef,0x40,0x0c	@ sha1c.32	q0,q0,q0
158*1dcdf01fSchristos#else
159*1dcdf01fSchristos.byte	0x40,0x0c,0x00,0xf2	@ sha1c.32	q0,q0,q0
160*1dcdf01fSchristos#endif
16160662d10Schristos	RET
16260662d10Schristos.size	_armv8_sha1_probe,.-_armv8_sha1_probe
16360662d10Schristos
164*1dcdf01fSchristos.globl	_armv8_sha256_probe
16560662d10Schristos.type	_armv8_sha256_probe,%function
16660662d10Schristos_armv8_sha256_probe:
167*1dcdf01fSchristos#if defined(__thumb2__) && !defined(__APPLE__)
168*1dcdf01fSchristos.byte	0x00,0xff,0x40,0x0c	@ sha256h.32	q0,q0,q0
169*1dcdf01fSchristos#else
170*1dcdf01fSchristos.byte	0x40,0x0c,0x00,0xf3	@ sha256h.32	q0,q0,q0
171*1dcdf01fSchristos#endif
17260662d10Schristos	RET
17360662d10Schristos.size	_armv8_sha256_probe,.-_armv8_sha256_probe
174*1dcdf01fSchristos.globl	_armv8_pmull_probe
17560662d10Schristos.type	_armv8_pmull_probe,%function
17660662d10Schristos_armv8_pmull_probe:
177*1dcdf01fSchristos#if defined(__thumb2__) && !defined(__APPLE__)
178*1dcdf01fSchristos.byte	0xa0,0xef,0x00,0x0e	@ vmull.p64	q0,d0,d0
179*1dcdf01fSchristos#else
180*1dcdf01fSchristos.byte	0x00,0x0e,0xa0,0xf2	@ vmull.p64	q0,d0,d0
181*1dcdf01fSchristos#endif
18260662d10Schristos	RET
18360662d10Schristos.size	_armv8_pmull_probe,.-_armv8_pmull_probe
18460662d10Schristos#endif
18560662d10Schristos
186*1dcdf01fSchristos.globl	OPENSSL_wipe_cpu
18760662d10Schristos.type	OPENSSL_wipe_cpu,%function
18860662d10SchristosOPENSSL_wipe_cpu:
18960662d10Schristos#if __ARM_MAX_ARCH__>=7
19060662d10Schristos	ldr	r0,.LOPENSSL_armcap
19160662d10Schristos	adr	r1,.LOPENSSL_armcap
19260662d10Schristos	ldr	r0,[r1,r0]
193*1dcdf01fSchristos#ifdef	__APPLE__
194*1dcdf01fSchristos	ldr	r0,[r0]
195*1dcdf01fSchristos#endif
19660662d10Schristos#endif
19760662d10Schristos	eor	r2,r2,r2
19860662d10Schristos	eor	r3,r3,r3
19960662d10Schristos	eor	ip,ip,ip
20060662d10Schristos#if __ARM_MAX_ARCH__>=7
20160662d10Schristos	tst	r0,#1
20260662d10Schristos	beq	.Lwipe_done
20360662d10Schristos	veor	q0, q0, q0
20460662d10Schristos	veor	q1, q1, q1
20560662d10Schristos	veor	q2, q2, q2
20660662d10Schristos	veor	q3, q3, q3
20760662d10Schristos	veor	q8, q8, q8
20860662d10Schristos	veor	q9, q9, q9
20960662d10Schristos	veor	q10, q10, q10
21060662d10Schristos	veor	q11, q11, q11
21160662d10Schristos	veor	q12, q12, q12
21260662d10Schristos	veor	q13, q13, q13
21360662d10Schristos	veor	q14, q14, q14
21460662d10Schristos	veor	q15, q15, q15
21560662d10Schristos.Lwipe_done:
21660662d10Schristos#endif
21760662d10Schristos	mov	r0,sp
21860662d10Schristos#if __ARM_ARCH__>=5
21960662d10Schristos	RET
22060662d10Schristos#else
22160662d10Schristos	tst	lr,#1
22260662d10Schristos	moveq	pc,lr
223*1dcdf01fSchristos.word	0xe12fff1e	@ RET
22460662d10Schristos#endif
22560662d10Schristos.size	OPENSSL_wipe_cpu,.-OPENSSL_wipe_cpu
22660662d10Schristos
227*1dcdf01fSchristos.globl	OPENSSL_instrument_bus
22860662d10Schristos.type	OPENSSL_instrument_bus,%function
22960662d10SchristosOPENSSL_instrument_bus:
23060662d10Schristos	eor	r0,r0,r0
23160662d10Schristos#if __ARM_ARCH__>=5
23260662d10Schristos	RET
23360662d10Schristos#else
23460662d10Schristos	tst	lr,#1
23560662d10Schristos	moveq	pc,lr
236*1dcdf01fSchristos.word	0xe12fff1e	@ RET
23760662d10Schristos#endif
23860662d10Schristos.size	OPENSSL_instrument_bus,.-OPENSSL_instrument_bus
23960662d10Schristos
240*1dcdf01fSchristos.globl	OPENSSL_instrument_bus2
24160662d10Schristos.type	OPENSSL_instrument_bus2,%function
24260662d10SchristosOPENSSL_instrument_bus2:
24360662d10Schristos	eor	r0,r0,r0
24460662d10Schristos#if __ARM_ARCH__>=5
24560662d10Schristos	RET
24660662d10Schristos#else
24760662d10Schristos	tst	lr,#1
24860662d10Schristos	moveq	pc,lr
249*1dcdf01fSchristos.word	0xe12fff1e	@ RET
25060662d10Schristos#endif
25160662d10Schristos.size	OPENSSL_instrument_bus2,.-OPENSSL_instrument_bus2
25260662d10Schristos
25360662d10Schristos.align	5
25460662d10Schristos#if __ARM_MAX_ARCH__>=7
25560662d10Schristos.LOPENSSL_armcap:
256*1dcdf01fSchristos.word	OPENSSL_armcap_P-.
25760662d10Schristos#endif
25860662d10Schristos#if __ARM_ARCH__>=6
25960662d10Schristos.align	5
26060662d10Schristos#else
26160662d10Schristos.Lspinlock:
26260662d10Schristos.word	atomic_add_spinlock-.Lspinlock
26360662d10Schristos.align	5
26460662d10Schristos
26560662d10Schristos.data
26660662d10Schristos.align	2
26760662d10Schristosatomic_add_spinlock:
26860662d10Schristos.word	0
26960662d10Schristos#endif
27060662d10Schristos
27160662d10Schristos.comm	OPENSSL_armcap_P,4,4
27260662d10Schristos.hidden	OPENSSL_armcap_P
273