1#!/usr/bin/env perl
2
3# ====================================================================
4# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
5# project. The module is, however, dual licensed under OpenSSL and
6# CRYPTOGAMS licenses depending on where you obtain it. For further
7# details see http://www.openssl.org/~appro/cryptogams/.
8# ====================================================================
9
10# September 2010.
11#
12# The module implements "4-bit" GCM GHASH function and underlying
13# single multiplication operation in GF(2^128). "4-bit" means that it
14# uses 256 bytes per-key table [+128 bytes shared table]. Performance
15# was measured to be ~18 cycles per processed byte on z10, which is
16# almost 40% better than gcc-generated code. It should be noted that
17# 18 cycles is worse result than expected: loop is scheduled for 12
18# and the result should be close to 12. In the lack of instruction-
19# level profiling data it's impossible to tell why...
20
21# November 2010.
22#
23# Adapt for -m31 build. If kernel supports what's called "highgprs"
24# feature on Linux [see /proc/cpuinfo], it's possible to use 64-bit
25# instructions and achieve "64-bit" performance even in 31-bit legacy
26# application context. The feature is not specific to any particular
27# processor, as long as it's "z-CPU". Latter implies that the code
28# remains z/Architecture specific. On z990 it was measured to perform
29# 2.8x better than 32-bit code generated by gcc 4.3.
30
31# March 2011.
32#
33# Support for hardware KIMD-GHASH is verified to produce correct
34# result and therefore is engaged. On z196 it was measured to process
35# 8KB buffer ~7 faster than software implementation. It's not as
36# impressive for smaller buffer sizes and for smallest 16-bytes buffer
37# it's actually almost 2 times slower. Which is the reason why
38# KIMD-GHASH is not used in gcm_gmult_4bit.
39
40$flavour = shift;
41
42if ($flavour =~ /3[12]/) {
43	$SIZE_T=4;
44	$g="";
45} else {
46	$SIZE_T=8;
47	$g="g";
48}
49
50while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {}
51open STDOUT,">$output";
52
53$softonly=0;
54
55$Zhi="%r0";
56$Zlo="%r1";
57
58$Xi="%r2";	# argument block
59$Htbl="%r3";
60$inp="%r4";
61$len="%r5";
62
63$rem0="%r6";	# variables
64$rem1="%r7";
65$nlo="%r8";
66$nhi="%r9";
67$xi="%r10";
68$cnt="%r11";
69$tmp="%r12";
70$x78="%r13";
71$rem_4bit="%r14";
72
73$sp="%r15";
74
75$code.=<<___;
76.text
77
78.globl	gcm_gmult_4bit
79.align	32
80gcm_gmult_4bit:
81___
82$code.=<<___ if(!$softonly && 0);	# hardware is slow for single block...
83	larl	%r1,OPENSSL_s390xcap_P
84	lg	%r0,0(%r1)
85	tmhl	%r0,0x4000	# check for message-security-assist
86	jz	.Lsoft_gmult
87	lghi	%r0,0
88	lg	%r1,24(%r1)	# load second word of kimd capabilities vector
89	tmhh	%r1,0x4000	# check for function 65
90	jz	.Lsoft_gmult
91	stg	%r0,16($sp)	# arrange 16 bytes of zero input
92	stg	%r0,24($sp)
93	lghi	%r0,65		# function 65
94	la	%r1,0($Xi)	# H lies right after Xi in gcm128_context
95	la	$inp,16($sp)
96	lghi	$len,16
97	.long	0xb93e0004	# kimd %r0,$inp
98	brc	1,.-4		# pay attention to "partial completion"
99	br	%r14
100.align	32
101.Lsoft_gmult:
102___
103$code.=<<___;
104	stm${g}	%r6,%r14,6*$SIZE_T($sp)
105
106	aghi	$Xi,-1
107	lghi	$len,1
108	lghi	$x78,`0xf<<3`
109	larl	$rem_4bit,rem_4bit
110
111	lg	$Zlo,8+1($Xi)		# Xi
112	j	.Lgmult_shortcut
113.type	gcm_gmult_4bit,\@function
114.size	gcm_gmult_4bit,(.-gcm_gmult_4bit)
115
116.globl	gcm_ghash_4bit
117.align	32
118gcm_ghash_4bit:
119___
120$code.=<<___ if(!$softonly);
121	larl	%r1,OPENSSL_s390xcap_P
122	lg	%r0,0(%r1)
123	tmhl	%r0,0x4000	# check for message-security-assist
124	jz	.Lsoft_ghash
125	lghi	%r0,0
126	la	%r1,16($sp)
127	.long	0xb93e0004	# kimd %r0,%r4
128	lg	%r1,24($sp)
129	tmhh	%r1,0x4000	# check for function 65
130	jz	.Lsoft_ghash
131	lghi	%r0,65		# function 65
132	la	%r1,0($Xi)	# H lies right after Xi in gcm128_context
133	.long	0xb93e0004	# kimd %r0,$inp
134	brc	1,.-4		# pay attention to "partial completion"
135	br	%r14
136.align	32
137.Lsoft_ghash:
138___
139$code.=<<___ if ($flavour =~ /3[12]/);
140	llgfr	$len,$len
141___
142$code.=<<___;
143	stm${g}	%r6,%r14,6*$SIZE_T($sp)
144
145	aghi	$Xi,-1
146	srlg	$len,$len,4
147	lghi	$x78,`0xf<<3`
148	larl	$rem_4bit,rem_4bit
149
150	lg	$Zlo,8+1($Xi)		# Xi
151	lg	$Zhi,0+1($Xi)
152	lghi	$tmp,0
153.Louter:
154	xg	$Zhi,0($inp)		# Xi ^= inp
155	xg	$Zlo,8($inp)
156	xgr	$Zhi,$tmp
157	stg	$Zlo,8+1($Xi)
158	stg	$Zhi,0+1($Xi)
159
160.Lgmult_shortcut:
161	lghi	$tmp,0xf0
162	sllg	$nlo,$Zlo,4
163	srlg	$xi,$Zlo,8		# extract second byte
164	ngr	$nlo,$tmp
165	lgr	$nhi,$Zlo
166	lghi	$cnt,14
167	ngr	$nhi,$tmp
168
169	lg	$Zlo,8($nlo,$Htbl)
170	lg	$Zhi,0($nlo,$Htbl)
171
172	sllg	$nlo,$xi,4
173	sllg	$rem0,$Zlo,3
174	ngr	$nlo,$tmp
175	ngr	$rem0,$x78
176	ngr	$xi,$tmp
177
178	sllg	$tmp,$Zhi,60
179	srlg	$Zlo,$Zlo,4
180	srlg	$Zhi,$Zhi,4
181	xg	$Zlo,8($nhi,$Htbl)
182	xg	$Zhi,0($nhi,$Htbl)
183	lgr	$nhi,$xi
184	sllg	$rem1,$Zlo,3
185	xgr	$Zlo,$tmp
186	ngr	$rem1,$x78
187	sllg	$tmp,$Zhi,60
188	j	.Lghash_inner
189.align	16
190.Lghash_inner:
191	srlg	$Zlo,$Zlo,4
192	srlg	$Zhi,$Zhi,4
193	xg	$Zlo,8($nlo,$Htbl)
194	llgc	$xi,0($cnt,$Xi)
195	xg	$Zhi,0($nlo,$Htbl)
196	sllg	$nlo,$xi,4
197	xg	$Zhi,0($rem0,$rem_4bit)
198	nill	$nlo,0xf0
199	sllg	$rem0,$Zlo,3
200	xgr	$Zlo,$tmp
201	ngr	$rem0,$x78
202	nill	$xi,0xf0
203
204	sllg	$tmp,$Zhi,60
205	srlg	$Zlo,$Zlo,4
206	srlg	$Zhi,$Zhi,4
207	xg	$Zlo,8($nhi,$Htbl)
208	xg	$Zhi,0($nhi,$Htbl)
209	lgr	$nhi,$xi
210	xg	$Zhi,0($rem1,$rem_4bit)
211	sllg	$rem1,$Zlo,3
212	xgr	$Zlo,$tmp
213	ngr	$rem1,$x78
214	sllg	$tmp,$Zhi,60
215	brct	$cnt,.Lghash_inner
216
217	srlg	$Zlo,$Zlo,4
218	srlg	$Zhi,$Zhi,4
219	xg	$Zlo,8($nlo,$Htbl)
220	xg	$Zhi,0($nlo,$Htbl)
221	sllg	$xi,$Zlo,3
222	xg	$Zhi,0($rem0,$rem_4bit)
223	xgr	$Zlo,$tmp
224	ngr	$xi,$x78
225
226	sllg	$tmp,$Zhi,60
227	srlg	$Zlo,$Zlo,4
228	srlg	$Zhi,$Zhi,4
229	xg	$Zlo,8($nhi,$Htbl)
230	xg	$Zhi,0($nhi,$Htbl)
231	xgr	$Zlo,$tmp
232	xg	$Zhi,0($rem1,$rem_4bit)
233
234	lg	$tmp,0($xi,$rem_4bit)
235	la	$inp,16($inp)
236	sllg	$tmp,$tmp,4		# correct last rem_4bit[rem]
237	brctg	$len,.Louter
238
239	xgr	$Zhi,$tmp
240	stg	$Zlo,8+1($Xi)
241	stg	$Zhi,0+1($Xi)
242	lm${g}	%r6,%r14,6*$SIZE_T($sp)
243	br	%r14
244.type	gcm_ghash_4bit,\@function
245.size	gcm_ghash_4bit,(.-gcm_ghash_4bit)
246
247.align	64
248rem_4bit:
249	.long	`0x0000<<12`,0,`0x1C20<<12`,0,`0x3840<<12`,0,`0x2460<<12`,0
250	.long	`0x7080<<12`,0,`0x6CA0<<12`,0,`0x48C0<<12`,0,`0x54E0<<12`,0
251	.long	`0xE100<<12`,0,`0xFD20<<12`,0,`0xD940<<12`,0,`0xC560<<12`,0
252	.long	`0x9180<<12`,0,`0x8DA0<<12`,0,`0xA9C0<<12`,0,`0xB5E0<<12`,0
253.type	rem_4bit,\@object
254.size	rem_4bit,(.-rem_4bit)
255.string	"GHASH for s390x, CRYPTOGAMS by <appro\@openssl.org>"
256___
257
258$code =~ s/\`([^\`]*)\`/eval $1/gem;
259print $code;
260close STDOUT;
261