1#! /usr/bin/env perl
2# Copyright 2010-2020 The OpenSSL Project Authors. All Rights Reserved.
3#
4# Licensed under the Apache License 2.0 (the "License").  You may not use
5# this file except in compliance with the License.  You can obtain a copy
6# in the file LICENSE in the source distribution or at
7# https://www.openssl.org/source/license.html
8
9
10# ====================================================================
11# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
12# project. The module is, however, dual licensed under OpenSSL and
13# CRYPTOGAMS licenses depending on where you obtain it. For further
14# details see http://www.openssl.org/~appro/cryptogams/.
15# ====================================================================
16
17# September 2010.
18#
19# The module implements "4-bit" GCM GHASH function and underlying
20# single multiplication operation in GF(2^128). "4-bit" means that it
21# uses 256 bytes per-key table [+128 bytes shared table]. Performance
22# was measured to be ~18 cycles per processed byte on z10, which is
23# almost 40% better than gcc-generated code. It should be noted that
24# 18 cycles is worse result than expected: loop is scheduled for 12
25# and the result should be close to 12. In the lack of instruction-
26# level profiling data it's impossible to tell why...
27
28# November 2010.
29#
30# Adapt for -m31 build. If kernel supports what's called "highgprs"
31# feature on Linux [see /proc/cpuinfo], it's possible to use 64-bit
32# instructions and achieve "64-bit" performance even in 31-bit legacy
33# application context. The feature is not specific to any particular
34# processor, as long as it's "z-CPU". Latter implies that the code
35# remains z/Architecture specific. On z990 it was measured to perform
36# 2.8x better than 32-bit code generated by gcc 4.3.
37
38# March 2011.
39#
40# Support for hardware KIMD-GHASH is verified to produce correct
41# result and therefore is engaged. On z196 it was measured to process
42# 8KB buffer ~7 faster than software implementation. It's not as
43# impressive for smaller buffer sizes and for smallest 16-bytes buffer
44# it's actually almost 2 times slower. Which is the reason why
45# KIMD-GHASH is not used in gcm_gmult_4bit.
46
47# $output is the last argument if it looks like a file (it has an extension)
48# $flavour is the first argument if it doesn't look like a file
49$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;
50$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;
51
52if ($flavour =~ /3[12]/) {
53	$SIZE_T=4;
54	$g="";
55} else {
56	$SIZE_T=8;
57	$g="g";
58}
59
60$output and open STDOUT,">$output";
61
62$softonly=0;
63
64$Zhi="%r0";
65$Zlo="%r1";
66
67$Xi="%r2";	# argument block
68$Htbl="%r3";
69$inp="%r4";
70$len="%r5";
71
72$rem0="%r6";	# variables
73$rem1="%r7";
74$nlo="%r8";
75$nhi="%r9";
76$xi="%r10";
77$cnt="%r11";
78$tmp="%r12";
79$x78="%r13";
80$rem_4bit="%r14";
81
82$sp="%r15";
83
84$code.=<<___;
85#include "s390x_arch.h"
86
87.text
88
89.globl	gcm_gmult_4bit
90.align	32
91gcm_gmult_4bit:
92___
93$code.=<<___ if(!$softonly && 0);	# hardware is slow for single block...
94	larl	%r1,OPENSSL_s390xcap_P
95	lghi	%r0,0
96	lg	%r1,S390X_KIMD+8(%r1)	# load second word of kimd capabilities
97					#  vector
98	tmhh	%r1,0x4000	# check for function 65
99	jz	.Lsoft_gmult
100	stg	%r0,16($sp)	# arrange 16 bytes of zero input
101	stg	%r0,24($sp)
102	lghi	%r0,S390X_GHASH	# function 65
103	la	%r1,0($Xi)	# H lies right after Xi in gcm128_context
104	la	$inp,16($sp)
105	lghi	$len,16
106	.long	0xb93e0004	# kimd %r0,$inp
107	brc	1,.-4		# pay attention to "partial completion"
108	br	%r14
109.align	32
110.Lsoft_gmult:
111___
112$code.=<<___;
113	stm${g}	%r6,%r14,6*$SIZE_T($sp)
114
115	aghi	$Xi,-1
116	lghi	$len,1
117	lghi	$x78,`0xf<<3`
118	larl	$rem_4bit,rem_4bit
119
120	lg	$Zlo,8+1($Xi)		# Xi
121	j	.Lgmult_shortcut
122.type	gcm_gmult_4bit,\@function
123.size	gcm_gmult_4bit,(.-gcm_gmult_4bit)
124
125.globl	gcm_ghash_4bit
126.align	32
127gcm_ghash_4bit:
128___
129$code.=<<___ if(!$softonly);
130	larl	%r1,OPENSSL_s390xcap_P
131	lg	%r0,S390X_KIMD+8(%r1)	# load second word of kimd capabilities
132					#  vector
133	tmhh	%r0,0x4000	# check for function 65
134	jz	.Lsoft_ghash
135	lghi	%r0,S390X_GHASH	# function 65
136	la	%r1,0($Xi)	# H lies right after Xi in gcm128_context
137	.long	0xb93e0004	# kimd %r0,$inp
138	brc	1,.-4		# pay attention to "partial completion"
139	br	%r14
140.align	32
141.Lsoft_ghash:
142___
143$code.=<<___ if ($flavour =~ /3[12]/);
144	llgfr	$len,$len
145___
146$code.=<<___;
147	stm${g}	%r6,%r14,6*$SIZE_T($sp)
148
149	aghi	$Xi,-1
150	srlg	$len,$len,4
151	lghi	$x78,`0xf<<3`
152	larl	$rem_4bit,rem_4bit
153
154	lg	$Zlo,8+1($Xi)		# Xi
155	lg	$Zhi,0+1($Xi)
156	lghi	$tmp,0
157.Louter:
158	xg	$Zhi,0($inp)		# Xi ^= inp
159	xg	$Zlo,8($inp)
160	xgr	$Zhi,$tmp
161	stg	$Zlo,8+1($Xi)
162	stg	$Zhi,0+1($Xi)
163
164.Lgmult_shortcut:
165	lghi	$tmp,0xf0
166	sllg	$nlo,$Zlo,4
167	srlg	$xi,$Zlo,8		# extract second byte
168	ngr	$nlo,$tmp
169	lgr	$nhi,$Zlo
170	lghi	$cnt,14
171	ngr	$nhi,$tmp
172
173	lg	$Zlo,8($nlo,$Htbl)
174	lg	$Zhi,0($nlo,$Htbl)
175
176	sllg	$nlo,$xi,4
177	sllg	$rem0,$Zlo,3
178	ngr	$nlo,$tmp
179	ngr	$rem0,$x78
180	ngr	$xi,$tmp
181
182	sllg	$tmp,$Zhi,60
183	srlg	$Zlo,$Zlo,4
184	srlg	$Zhi,$Zhi,4
185	xg	$Zlo,8($nhi,$Htbl)
186	xg	$Zhi,0($nhi,$Htbl)
187	lgr	$nhi,$xi
188	sllg	$rem1,$Zlo,3
189	xgr	$Zlo,$tmp
190	ngr	$rem1,$x78
191	sllg	$tmp,$Zhi,60
192	j	.Lghash_inner
193.align	16
194.Lghash_inner:
195	srlg	$Zlo,$Zlo,4
196	srlg	$Zhi,$Zhi,4
197	xg	$Zlo,8($nlo,$Htbl)
198	llgc	$xi,0($cnt,$Xi)
199	xg	$Zhi,0($nlo,$Htbl)
200	sllg	$nlo,$xi,4
201	xg	$Zhi,0($rem0,$rem_4bit)
202	nill	$nlo,0xf0
203	sllg	$rem0,$Zlo,3
204	xgr	$Zlo,$tmp
205	ngr	$rem0,$x78
206	nill	$xi,0xf0
207
208	sllg	$tmp,$Zhi,60
209	srlg	$Zlo,$Zlo,4
210	srlg	$Zhi,$Zhi,4
211	xg	$Zlo,8($nhi,$Htbl)
212	xg	$Zhi,0($nhi,$Htbl)
213	lgr	$nhi,$xi
214	xg	$Zhi,0($rem1,$rem_4bit)
215	sllg	$rem1,$Zlo,3
216	xgr	$Zlo,$tmp
217	ngr	$rem1,$x78
218	sllg	$tmp,$Zhi,60
219	brct	$cnt,.Lghash_inner
220
221	srlg	$Zlo,$Zlo,4
222	srlg	$Zhi,$Zhi,4
223	xg	$Zlo,8($nlo,$Htbl)
224	xg	$Zhi,0($nlo,$Htbl)
225	sllg	$xi,$Zlo,3
226	xg	$Zhi,0($rem0,$rem_4bit)
227	xgr	$Zlo,$tmp
228	ngr	$xi,$x78
229
230	sllg	$tmp,$Zhi,60
231	srlg	$Zlo,$Zlo,4
232	srlg	$Zhi,$Zhi,4
233	xg	$Zlo,8($nhi,$Htbl)
234	xg	$Zhi,0($nhi,$Htbl)
235	xgr	$Zlo,$tmp
236	xg	$Zhi,0($rem1,$rem_4bit)
237
238	lg	$tmp,0($xi,$rem_4bit)
239	la	$inp,16($inp)
240	sllg	$tmp,$tmp,4		# correct last rem_4bit[rem]
241	brctg	$len,.Louter
242
243	xgr	$Zhi,$tmp
244	stg	$Zlo,8+1($Xi)
245	stg	$Zhi,0+1($Xi)
246	lm${g}	%r6,%r14,6*$SIZE_T($sp)
247	br	%r14
248.type	gcm_ghash_4bit,\@function
249.size	gcm_ghash_4bit,(.-gcm_ghash_4bit)
250
251.align	64
252rem_4bit:
253	.long	`0x0000<<12`,0,`0x1C20<<12`,0,`0x3840<<12`,0,`0x2460<<12`,0
254	.long	`0x7080<<12`,0,`0x6CA0<<12`,0,`0x48C0<<12`,0,`0x54E0<<12`,0
255	.long	`0xE100<<12`,0,`0xFD20<<12`,0,`0xD940<<12`,0,`0xC560<<12`,0
256	.long	`0x9180<<12`,0,`0x8DA0<<12`,0,`0xA9C0<<12`,0,`0xB5E0<<12`,0
257.type	rem_4bit,\@object
258.size	rem_4bit,(.-rem_4bit)
259.string	"GHASH for s390x, CRYPTOGAMS by <appro\@openssl.org>"
260___
261
262$code =~ s/\`([^\`]*)\`/eval $1/gem;
263print $code;
264close STDOUT or die "error closing STDOUT: $!";
265