1// Copyright (c) 2020 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5// +build arm64,gc,!purego
6
7#include "textflag.h"
8
9// carryPropagate works exactly like carryPropagateGeneric and uses the
10// same AND, ADD, and LSR+MADD instructions emitted by the compiler, but
11// avoids loading R0-R4 twice and uses LDP and STP.
12//
13// See https://golang.org/issues/43145 for the main compiler issue.
14//
15// func carryPropagate(v *Element)
16TEXT ·carryPropagate(SB),NOFRAME|NOSPLIT,$0-8
17	MOVD v+0(FP), R20
18
19	LDP 0(R20), (R0, R1)
20	LDP 16(R20), (R2, R3)
21	MOVD 32(R20), R4
22
23	AND $0x7ffffffffffff, R0, R10
24	AND $0x7ffffffffffff, R1, R11
25	AND $0x7ffffffffffff, R2, R12
26	AND $0x7ffffffffffff, R3, R13
27	AND $0x7ffffffffffff, R4, R14
28
29	ADD R0>>51, R11, R11
30	ADD R1>>51, R12, R12
31	ADD R2>>51, R13, R13
32	ADD R3>>51, R14, R14
33	// R4>>51 * 19 + R10 -> R10
34	LSR $51, R4, R21
35	MOVD $19, R22
36	MADD R22, R10, R21, R10
37
38	STP (R10, R11), 0(R20)
39	STP (R12, R13), 16(R20)
40	MOVD R14, 32(R20)
41
42	RET
43