1#!/usr/bin/env perl 2 3# ==================================================================== 4# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL 5# project. The module is, however, dual licensed under OpenSSL and 6# CRYPTOGAMS licenses depending on where you obtain it. For further 7# details see http://www.openssl.org/~appro/cryptogams/. 8# ==================================================================== 9 10# September 2010. 11# 12# The module implements "4-bit" GCM GHASH function and underlying 13# single multiplication operation in GF(2^128). "4-bit" means that it 14# uses 256 bytes per-key table [+128 bytes shared table]. Performance 15# was measured to be ~18 cycles per processed byte on z10, which is 16# almost 40% better than gcc-generated code. It should be noted that 17# 18 cycles is worse result than expected: loop is scheduled for 12 18# and the result should be close to 12. In the lack of instruction- 19# level profiling data it's impossible to tell why... 20 21# November 2010. 22# 23# Adapt for -m31 build. If kernel supports what's called "highgprs" 24# feature on Linux [see /proc/cpuinfo], it's possible to use 64-bit 25# instructions and achieve "64-bit" performance even in 31-bit legacy 26# application context. The feature is not specific to any particular 27# processor, as long as it's "z-CPU". Latter implies that the code 28# remains z/Architecture specific. On z990 it was measured to perform 29# 2.8x better than 32-bit code generated by gcc 4.3. 30 31# March 2011. 32# 33# Support for hardware KIMD-GHASH is verified to produce correct 34# result and therefore is engaged. On z196 it was measured to process 35# 8KB buffer ~7 faster than software implementation. It's not as 36# impressive for smaller buffer sizes and for smallest 16-bytes buffer 37# it's actually almost 2 times slower. Which is the reason why 38# KIMD-GHASH is not used in gcm_gmult_4bit. 39 40$flavour = shift; 41 42if ($flavour =~ /3[12]/) { 43 $SIZE_T=4; 44 $g=""; 45} else { 46 $SIZE_T=8; 47 $g="g"; 48} 49 50while (($output=shift) && ($output!~/^\w[\w\-]*\.\w+$/)) {} 51open STDOUT,">$output"; 52 53$softonly=0; 54 55$Zhi="%r0"; 56$Zlo="%r1"; 57 58$Xi="%r2"; # argument block 59$Htbl="%r3"; 60$inp="%r4"; 61$len="%r5"; 62 63$rem0="%r6"; # variables 64$rem1="%r7"; 65$nlo="%r8"; 66$nhi="%r9"; 67$xi="%r10"; 68$cnt="%r11"; 69$tmp="%r12"; 70$x78="%r13"; 71$rem_4bit="%r14"; 72 73$sp="%r15"; 74 75$code.=<<___; 76.text 77 78.globl gcm_gmult_4bit 79.align 32 80gcm_gmult_4bit: 81___ 82$code.=<<___ if(!$softonly && 0); # hardware is slow for single block... 83 larl %r1,OPENSSL_s390xcap_P 84 lg %r0,0(%r1) 85 tmhl %r0,0x4000 # check for message-security-assist 86 jz .Lsoft_gmult 87 lghi %r0,0 88 lg %r1,24(%r1) # load second word of kimd capabilities vector 89 tmhh %r1,0x4000 # check for function 65 90 jz .Lsoft_gmult 91 stg %r0,16($sp) # arrange 16 bytes of zero input 92 stg %r0,24($sp) 93 lghi %r0,65 # function 65 94 la %r1,0($Xi) # H lies right after Xi in gcm128_context 95 la $inp,16($sp) 96 lghi $len,16 97 .long 0xb93e0004 # kimd %r0,$inp 98 brc 1,.-4 # pay attention to "partial completion" 99 br %r14 100.align 32 101.Lsoft_gmult: 102___ 103$code.=<<___; 104 stm${g} %r6,%r14,6*$SIZE_T($sp) 105 106 aghi $Xi,-1 107 lghi $len,1 108 lghi $x78,`0xf<<3` 109 larl $rem_4bit,rem_4bit 110 111 lg $Zlo,8+1($Xi) # Xi 112 j .Lgmult_shortcut 113.type gcm_gmult_4bit,\@function 114.size gcm_gmult_4bit,(.-gcm_gmult_4bit) 115 116.globl gcm_ghash_4bit 117.align 32 118gcm_ghash_4bit: 119___ 120$code.=<<___ if(!$softonly); 121 larl %r1,OPENSSL_s390xcap_P 122 lg %r0,0(%r1) 123 tmhl %r0,0x4000 # check for message-security-assist 124 jz .Lsoft_ghash 125 lghi %r0,0 126 la %r1,16($sp) 127 .long 0xb93e0004 # kimd %r0,%r4 128 lg %r1,24($sp) 129 tmhh %r1,0x4000 # check for function 65 130 jz .Lsoft_ghash 131 lghi %r0,65 # function 65 132 la %r1,0($Xi) # H lies right after Xi in gcm128_context 133 .long 0xb93e0004 # kimd %r0,$inp 134 brc 1,.-4 # pay attention to "partial completion" 135 br %r14 136.align 32 137.Lsoft_ghash: 138___ 139$code.=<<___ if ($flavour =~ /3[12]/); 140 llgfr $len,$len 141___ 142$code.=<<___; 143 stm${g} %r6,%r14,6*$SIZE_T($sp) 144 145 aghi $Xi,-1 146 srlg $len,$len,4 147 lghi $x78,`0xf<<3` 148 larl $rem_4bit,rem_4bit 149 150 lg $Zlo,8+1($Xi) # Xi 151 lg $Zhi,0+1($Xi) 152 lghi $tmp,0 153.Louter: 154 xg $Zhi,0($inp) # Xi ^= inp 155 xg $Zlo,8($inp) 156 xgr $Zhi,$tmp 157 stg $Zlo,8+1($Xi) 158 stg $Zhi,0+1($Xi) 159 160.Lgmult_shortcut: 161 lghi $tmp,0xf0 162 sllg $nlo,$Zlo,4 163 srlg $xi,$Zlo,8 # extract second byte 164 ngr $nlo,$tmp 165 lgr $nhi,$Zlo 166 lghi $cnt,14 167 ngr $nhi,$tmp 168 169 lg $Zlo,8($nlo,$Htbl) 170 lg $Zhi,0($nlo,$Htbl) 171 172 sllg $nlo,$xi,4 173 sllg $rem0,$Zlo,3 174 ngr $nlo,$tmp 175 ngr $rem0,$x78 176 ngr $xi,$tmp 177 178 sllg $tmp,$Zhi,60 179 srlg $Zlo,$Zlo,4 180 srlg $Zhi,$Zhi,4 181 xg $Zlo,8($nhi,$Htbl) 182 xg $Zhi,0($nhi,$Htbl) 183 lgr $nhi,$xi 184 sllg $rem1,$Zlo,3 185 xgr $Zlo,$tmp 186 ngr $rem1,$x78 187 sllg $tmp,$Zhi,60 188 j .Lghash_inner 189.align 16 190.Lghash_inner: 191 srlg $Zlo,$Zlo,4 192 srlg $Zhi,$Zhi,4 193 xg $Zlo,8($nlo,$Htbl) 194 llgc $xi,0($cnt,$Xi) 195 xg $Zhi,0($nlo,$Htbl) 196 sllg $nlo,$xi,4 197 xg $Zhi,0($rem0,$rem_4bit) 198 nill $nlo,0xf0 199 sllg $rem0,$Zlo,3 200 xgr $Zlo,$tmp 201 ngr $rem0,$x78 202 nill $xi,0xf0 203 204 sllg $tmp,$Zhi,60 205 srlg $Zlo,$Zlo,4 206 srlg $Zhi,$Zhi,4 207 xg $Zlo,8($nhi,$Htbl) 208 xg $Zhi,0($nhi,$Htbl) 209 lgr $nhi,$xi 210 xg $Zhi,0($rem1,$rem_4bit) 211 sllg $rem1,$Zlo,3 212 xgr $Zlo,$tmp 213 ngr $rem1,$x78 214 sllg $tmp,$Zhi,60 215 brct $cnt,.Lghash_inner 216 217 srlg $Zlo,$Zlo,4 218 srlg $Zhi,$Zhi,4 219 xg $Zlo,8($nlo,$Htbl) 220 xg $Zhi,0($nlo,$Htbl) 221 sllg $xi,$Zlo,3 222 xg $Zhi,0($rem0,$rem_4bit) 223 xgr $Zlo,$tmp 224 ngr $xi,$x78 225 226 sllg $tmp,$Zhi,60 227 srlg $Zlo,$Zlo,4 228 srlg $Zhi,$Zhi,4 229 xg $Zlo,8($nhi,$Htbl) 230 xg $Zhi,0($nhi,$Htbl) 231 xgr $Zlo,$tmp 232 xg $Zhi,0($rem1,$rem_4bit) 233 234 lg $tmp,0($xi,$rem_4bit) 235 la $inp,16($inp) 236 sllg $tmp,$tmp,4 # correct last rem_4bit[rem] 237 brctg $len,.Louter 238 239 xgr $Zhi,$tmp 240 stg $Zlo,8+1($Xi) 241 stg $Zhi,0+1($Xi) 242 lm${g} %r6,%r14,6*$SIZE_T($sp) 243 br %r14 244.type gcm_ghash_4bit,\@function 245.size gcm_ghash_4bit,(.-gcm_ghash_4bit) 246 247.align 64 248rem_4bit: 249 .long `0x0000<<12`,0,`0x1C20<<12`,0,`0x3840<<12`,0,`0x2460<<12`,0 250 .long `0x7080<<12`,0,`0x6CA0<<12`,0,`0x48C0<<12`,0,`0x54E0<<12`,0 251 .long `0xE100<<12`,0,`0xFD20<<12`,0,`0xD940<<12`,0,`0xC560<<12`,0 252 .long `0x9180<<12`,0,`0x8DA0<<12`,0,`0xA9C0<<12`,0,`0xB5E0<<12`,0 253.type rem_4bit,\@object 254.size rem_4bit,(.-rem_4bit) 255.string "GHASH for s390x, CRYPTOGAMS by <appro\@openssl.org>" 256___ 257 258$code =~ s/\`([^\`]*)\`/eval $1/gem; 259print $code; 260close STDOUT; 261