868 lines
23 KiB
Diff
868 lines
23 KiB
Diff
From f760137b2144740916afd9ff381451fa16c710de Mon Sep 17 00:00:00 2001
|
|
From: Patrick Steuer <patrick.steuer@de.ibm.com>
|
|
Date: Sat, 4 Aug 2018 00:10:06 +0200
|
|
Subject: [PATCH] crypto/chacha/asm/chacha-s390x.pl: add vx code path.
|
|
|
|
Signed-off-by: Patrick Steuer <patrick.steuer@de.ibm.com>
|
|
|
|
Reviewed-by: Tim Hudson <tjh@openssl.org>
|
|
Reviewed-by: Richard Levitte <levitte@openssl.org>
|
|
(Merged from https://github.com/openssl/openssl/pull/6919)
|
|
---
|
|
crypto/chacha/asm/chacha-s390x.pl | 816 ++++++++++++++++++++----------
|
|
crypto/chacha/build.info | 1 +
|
|
2 files changed, 558 insertions(+), 259 deletions(-)
|
|
|
|
Index: openssl-1.1.1e/crypto/chacha/asm/chacha-s390x.pl
|
|
===================================================================
|
|
--- openssl-1.1.1e.orig/crypto/chacha/asm/chacha-s390x.pl 2020-03-19 11:43:25.650616856 +0100
|
|
+++ openssl-1.1.1e/crypto/chacha/asm/chacha-s390x.pl 2020-03-19 11:43:40.614692484 +0100
|
|
@@ -20,41 +20,46 @@
|
|
#
|
|
# 3 times faster than compiler-generated code.
|
|
|
|
-$flavour = shift;
|
|
+#
|
|
+# August 2018
|
|
+#
|
|
+# Add vx code path.
|
|
+#
|
|
+# Copyright IBM Corp. 2018
|
|
+# Author: Patrick Steuer <patrick.steuer@de.ibm.com>
|
|
+
|
|
+use strict;
|
|
+use FindBin qw($Bin);
|
|
+use lib "$Bin/../..";
|
|
+use perlasm::s390x qw(:DEFAULT :VX AUTOLOAD LABEL INCLUDE);
|
|
+
|
|
+my $flavour = shift;
|
|
|
|
+my ($z,$SIZE_T);
|
|
if ($flavour =~ /3[12]/) {
|
|
+ $z=0; # S/390 ABI
|
|
$SIZE_T=4;
|
|
- $g="";
|
|
} else {
|
|
+ $z=1; # zSeries ABI
|
|
$SIZE_T=8;
|
|
- $g="g";
|
|
}
|
|
|
|
+my $output;
|
|
while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {}
|
|
-open STDOUT,">$output";
|
|
-
|
|
-sub AUTOLOAD() # thunk [simplified] x86-style perlasm
|
|
-{ my $opcode = $AUTOLOAD; $opcode =~ s/.*:://;
|
|
- $code .= "\t$opcode\t".join(',',@_)."\n";
|
|
-}
|
|
|
|
my $sp="%r15";
|
|
-
|
|
my $stdframe=16*$SIZE_T+4*8;
|
|
-my $frame=$stdframe+4*20;
|
|
-
|
|
-my ($out,$inp,$len,$key,$counter)=map("%r$_",(2..6));
|
|
|
|
my @x=map("%r$_",(0..7,"x","x","x","x",(10..13)));
|
|
my @t=map("%r$_",(8,9));
|
|
+my @v=map("%v$_",(16..31));
|
|
|
|
sub ROUND {
|
|
my ($a0,$b0,$c0,$d0)=@_;
|
|
my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
|
|
my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
|
|
my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
|
|
-my ($xc,$xc_)=map("\"$_\"",@t);
|
|
-my @x=map("\"$_\"",@x);
|
|
+my ($xc,$xc_)=map("$_",@t);
|
|
|
|
# Consider order in which variables are addressed by their
|
|
# index:
|
|
@@ -78,249 +83,542 @@ my @x=map("\"$_\"",@x);
|
|
# 'c' stores and loads in the middle, but none in the beginning
|
|
# or end.
|
|
|
|
- (
|
|
- "&alr (@x[$a0],@x[$b0])", # Q1
|
|
- "&alr (@x[$a1],@x[$b1])", # Q2
|
|
- "&xr (@x[$d0],@x[$a0])",
|
|
- "&xr (@x[$d1],@x[$a1])",
|
|
- "&rll (@x[$d0],@x[$d0],16)",
|
|
- "&rll (@x[$d1],@x[$d1],16)",
|
|
-
|
|
- "&alr ($xc,@x[$d0])",
|
|
- "&alr ($xc_,@x[$d1])",
|
|
- "&xr (@x[$b0],$xc)",
|
|
- "&xr (@x[$b1],$xc_)",
|
|
- "&rll (@x[$b0],@x[$b0],12)",
|
|
- "&rll (@x[$b1],@x[$b1],12)",
|
|
-
|
|
- "&alr (@x[$a0],@x[$b0])",
|
|
- "&alr (@x[$a1],@x[$b1])",
|
|
- "&xr (@x[$d0],@x[$a0])",
|
|
- "&xr (@x[$d1],@x[$a1])",
|
|
- "&rll (@x[$d0],@x[$d0],8)",
|
|
- "&rll (@x[$d1],@x[$d1],8)",
|
|
-
|
|
- "&alr ($xc,@x[$d0])",
|
|
- "&alr ($xc_,@x[$d1])",
|
|
- "&xr (@x[$b0],$xc)",
|
|
- "&xr (@x[$b1],$xc_)",
|
|
- "&rll (@x[$b0],@x[$b0],7)",
|
|
- "&rll (@x[$b1],@x[$b1],7)",
|
|
-
|
|
- "&stm ($xc,$xc_,'$stdframe+4*8+4*$c0($sp)')", # reload pair of 'c's
|
|
- "&lm ($xc,$xc_,'$stdframe+4*8+4*$c2($sp)')",
|
|
-
|
|
- "&alr (@x[$a2],@x[$b2])", # Q3
|
|
- "&alr (@x[$a3],@x[$b3])", # Q4
|
|
- "&xr (@x[$d2],@x[$a2])",
|
|
- "&xr (@x[$d3],@x[$a3])",
|
|
- "&rll (@x[$d2],@x[$d2],16)",
|
|
- "&rll (@x[$d3],@x[$d3],16)",
|
|
-
|
|
- "&alr ($xc,@x[$d2])",
|
|
- "&alr ($xc_,@x[$d3])",
|
|
- "&xr (@x[$b2],$xc)",
|
|
- "&xr (@x[$b3],$xc_)",
|
|
- "&rll (@x[$b2],@x[$b2],12)",
|
|
- "&rll (@x[$b3],@x[$b3],12)",
|
|
-
|
|
- "&alr (@x[$a2],@x[$b2])",
|
|
- "&alr (@x[$a3],@x[$b3])",
|
|
- "&xr (@x[$d2],@x[$a2])",
|
|
- "&xr (@x[$d3],@x[$a3])",
|
|
- "&rll (@x[$d2],@x[$d2],8)",
|
|
- "&rll (@x[$d3],@x[$d3],8)",
|
|
-
|
|
- "&alr ($xc,@x[$d2])",
|
|
- "&alr ($xc_,@x[$d3])",
|
|
- "&xr (@x[$b2],$xc)",
|
|
- "&xr (@x[$b3],$xc_)",
|
|
- "&rll (@x[$b2],@x[$b2],7)",
|
|
- "&rll (@x[$b3],@x[$b3],7)"
|
|
- );
|
|
-}
|
|
-
|
|
-$code.=<<___;
|
|
-.text
|
|
-
|
|
-.globl ChaCha20_ctr32
|
|
-.type ChaCha20_ctr32,\@function
|
|
-.align 32
|
|
-ChaCha20_ctr32:
|
|
- lt${g}r $len,$len # $len==0?
|
|
- bzr %r14
|
|
- a${g}hi $len,-64
|
|
- l${g}hi %r1,-$frame
|
|
- stm${g} %r6,%r15,`6*$SIZE_T`($sp)
|
|
- sl${g}r $out,$inp # difference
|
|
- la $len,0($inp,$len) # end of input minus 64
|
|
- larl %r7,.Lsigma
|
|
- lgr %r0,$sp
|
|
- la $sp,0(%r1,$sp)
|
|
- st${g} %r0,0($sp)
|
|
-
|
|
- lmg %r8,%r11,0($key) # load key
|
|
- lmg %r12,%r13,0($counter) # load counter
|
|
- lmg %r6,%r7,0(%r7) # load sigma constant
|
|
-
|
|
- la %r14,0($inp)
|
|
- st${g} $out,$frame+3*$SIZE_T($sp)
|
|
- st${g} $len,$frame+4*$SIZE_T($sp)
|
|
- stmg %r6,%r13,$stdframe($sp) # copy key schedule to stack
|
|
- srlg @x[12],%r12,32 # 32-bit counter value
|
|
- j .Loop_outer
|
|
-
|
|
-.align 16
|
|
-.Loop_outer:
|
|
- lm @x[0],@x[7],$stdframe+4*0($sp) # load x[0]-x[7]
|
|
- lm @t[0],@t[1],$stdframe+4*10($sp) # load x[10]-x[11]
|
|
- lm @x[13],@x[15],$stdframe+4*13($sp) # load x[13]-x[15]
|
|
- stm @t[0],@t[1],$stdframe+4*8+4*10($sp) # offload x[10]-x[11]
|
|
- lm @t[0],@t[1],$stdframe+4*8($sp) # load x[8]-x[9]
|
|
- st @x[12],$stdframe+4*12($sp) # save counter
|
|
- st${g} %r14,$frame+2*$SIZE_T($sp) # save input pointer
|
|
- lhi %r14,10
|
|
- j .Loop
|
|
-
|
|
-.align 4
|
|
-.Loop:
|
|
-___
|
|
- foreach (&ROUND(0, 4, 8,12)) { eval; }
|
|
- foreach (&ROUND(0, 5,10,15)) { eval; }
|
|
-$code.=<<___;
|
|
- brct %r14,.Loop
|
|
-
|
|
- l${g} %r14,$frame+2*$SIZE_T($sp) # pull input pointer
|
|
- stm @t[0],@t[1],$stdframe+4*8+4*8($sp) # offload x[8]-x[9]
|
|
- lm${g} @t[0],@t[1],$frame+3*$SIZE_T($sp)
|
|
-
|
|
- al @x[0],$stdframe+4*0($sp) # accumulate key schedule
|
|
- al @x[1],$stdframe+4*1($sp)
|
|
- al @x[2],$stdframe+4*2($sp)
|
|
- al @x[3],$stdframe+4*3($sp)
|
|
- al @x[4],$stdframe+4*4($sp)
|
|
- al @x[5],$stdframe+4*5($sp)
|
|
- al @x[6],$stdframe+4*6($sp)
|
|
- al @x[7],$stdframe+4*7($sp)
|
|
- lrvr @x[0],@x[0]
|
|
- lrvr @x[1],@x[1]
|
|
- lrvr @x[2],@x[2]
|
|
- lrvr @x[3],@x[3]
|
|
- lrvr @x[4],@x[4]
|
|
- lrvr @x[5],@x[5]
|
|
- lrvr @x[6],@x[6]
|
|
- lrvr @x[7],@x[7]
|
|
- al @x[12],$stdframe+4*12($sp)
|
|
- al @x[13],$stdframe+4*13($sp)
|
|
- al @x[14],$stdframe+4*14($sp)
|
|
- al @x[15],$stdframe+4*15($sp)
|
|
- lrvr @x[12],@x[12]
|
|
- lrvr @x[13],@x[13]
|
|
- lrvr @x[14],@x[14]
|
|
- lrvr @x[15],@x[15]
|
|
-
|
|
- la @t[0],0(@t[0],%r14) # reconstruct output pointer
|
|
- cl${g}r %r14,@t[1]
|
|
- jh .Ltail
|
|
-
|
|
- x @x[0],4*0(%r14) # xor with input
|
|
- x @x[1],4*1(%r14)
|
|
- st @x[0],4*0(@t[0]) # store output
|
|
- x @x[2],4*2(%r14)
|
|
- st @x[1],4*1(@t[0])
|
|
- x @x[3],4*3(%r14)
|
|
- st @x[2],4*2(@t[0])
|
|
- x @x[4],4*4(%r14)
|
|
- st @x[3],4*3(@t[0])
|
|
- lm @x[0],@x[3],$stdframe+4*8+4*8($sp) # load x[8]-x[11]
|
|
- x @x[5],4*5(%r14)
|
|
- st @x[4],4*4(@t[0])
|
|
- x @x[6],4*6(%r14)
|
|
- al @x[0],$stdframe+4*8($sp)
|
|
- st @x[5],4*5(@t[0])
|
|
- x @x[7],4*7(%r14)
|
|
- al @x[1],$stdframe+4*9($sp)
|
|
- st @x[6],4*6(@t[0])
|
|
- x @x[12],4*12(%r14)
|
|
- al @x[2],$stdframe+4*10($sp)
|
|
- st @x[7],4*7(@t[0])
|
|
- x @x[13],4*13(%r14)
|
|
- al @x[3],$stdframe+4*11($sp)
|
|
- st @x[12],4*12(@t[0])
|
|
- x @x[14],4*14(%r14)
|
|
- st @x[13],4*13(@t[0])
|
|
- x @x[15],4*15(%r14)
|
|
- st @x[14],4*14(@t[0])
|
|
- lrvr @x[0],@x[0]
|
|
- st @x[15],4*15(@t[0])
|
|
- lrvr @x[1],@x[1]
|
|
- lrvr @x[2],@x[2]
|
|
- lrvr @x[3],@x[3]
|
|
- lhi @x[12],1
|
|
- x @x[0],4*8(%r14)
|
|
- al @x[12],$stdframe+4*12($sp) # increment counter
|
|
- x @x[1],4*9(%r14)
|
|
- st @x[0],4*8(@t[0])
|
|
- x @x[2],4*10(%r14)
|
|
- st @x[1],4*9(@t[0])
|
|
- x @x[3],4*11(%r14)
|
|
- st @x[2],4*10(@t[0])
|
|
- st @x[3],4*11(@t[0])
|
|
-
|
|
- cl${g}r %r14,@t[1] # done yet?
|
|
- la %r14,64(%r14)
|
|
- jl .Loop_outer
|
|
-
|
|
-.Ldone:
|
|
- xgr %r0,%r0
|
|
- xgr %r1,%r1
|
|
- xgr %r2,%r2
|
|
- xgr %r3,%r3
|
|
- stmg %r0,%r3,$stdframe+4*4($sp) # wipe key copy
|
|
- stmg %r0,%r3,$stdframe+4*12($sp)
|
|
-
|
|
- lm${g} %r6,%r15,`$frame+6*$SIZE_T`($sp)
|
|
- br %r14
|
|
-
|
|
-.align 16
|
|
-.Ltail:
|
|
- la @t[1],64($t[1])
|
|
- stm @x[0],@x[7],$stdframe+4*0($sp)
|
|
- sl${g}r @t[1],%r14
|
|
- lm @x[0],@x[3],$stdframe+4*8+4*8($sp)
|
|
- l${g}hi @x[6],0
|
|
- stm @x[12],@x[15],$stdframe+4*12($sp)
|
|
- al @x[0],$stdframe+4*8($sp)
|
|
- al @x[1],$stdframe+4*9($sp)
|
|
- al @x[2],$stdframe+4*10($sp)
|
|
- al @x[3],$stdframe+4*11($sp)
|
|
- lrvr @x[0],@x[0]
|
|
- lrvr @x[1],@x[1]
|
|
- lrvr @x[2],@x[2]
|
|
- lrvr @x[3],@x[3]
|
|
- stm @x[0],@x[3],$stdframe+4*8($sp)
|
|
-
|
|
-.Loop_tail:
|
|
- llgc @x[4],0(@x[6],%r14)
|
|
- llgc @x[5],$stdframe(@x[6],$sp)
|
|
- xr @x[5],@x[4]
|
|
- stc @x[5],0(@x[6],@t[0])
|
|
- la @x[6],1(@x[6])
|
|
- brct @t[1],.Loop_tail
|
|
-
|
|
- j .Ldone
|
|
-.size ChaCha20_ctr32,.-ChaCha20_ctr32
|
|
-
|
|
-.align 32
|
|
-.Lsigma:
|
|
-.long 0x61707865,0x3320646e,0x79622d32,0x6b206574 # endian-neutral
|
|
-.asciz "ChaCha20 for s390x, CRYPTOGAMS by <appro\@openssl.org>"
|
|
-.align 4
|
|
-___
|
|
+ alr (@x[$a0],@x[$b0]); # Q1
|
|
+ alr (@x[$a1],@x[$b1]); # Q2
|
|
+ xr (@x[$d0],@x[$a0]);
|
|
+ xr (@x[$d1],@x[$a1]);
|
|
+ rll (@x[$d0],@x[$d0],16);
|
|
+ rll (@x[$d1],@x[$d1],16);
|
|
+
|
|
+ alr ($xc,@x[$d0]);
|
|
+ alr ($xc_,@x[$d1]);
|
|
+ xr (@x[$b0],$xc);
|
|
+ xr (@x[$b1],$xc_);
|
|
+ rll (@x[$b0],@x[$b0],12);
|
|
+ rll (@x[$b1],@x[$b1],12);
|
|
+
|
|
+ alr (@x[$a0],@x[$b0]);
|
|
+ alr (@x[$a1],@x[$b1]);
|
|
+ xr (@x[$d0],@x[$a0]);
|
|
+ xr (@x[$d1],@x[$a1]);
|
|
+ rll (@x[$d0],@x[$d0],8);
|
|
+ rll (@x[$d1],@x[$d1],8);
|
|
+
|
|
+ alr ($xc,@x[$d0]);
|
|
+ alr ($xc_,@x[$d1]);
|
|
+ xr (@x[$b0],$xc);
|
|
+ xr (@x[$b1],$xc_);
|
|
+ rll (@x[$b0],@x[$b0],7);
|
|
+ rll (@x[$b1],@x[$b1],7);
|
|
+
|
|
+ stm ($xc,$xc_,"$stdframe+4*8+4*$c0($sp)"); # reload pair of 'c's
|
|
+ lm ($xc,$xc_,"$stdframe+4*8+4*$c2($sp)");
|
|
+
|
|
+ alr (@x[$a2],@x[$b2]); # Q3
|
|
+ alr (@x[$a3],@x[$b3]); # Q4
|
|
+ xr (@x[$d2],@x[$a2]);
|
|
+ xr (@x[$d3],@x[$a3]);
|
|
+ rll (@x[$d2],@x[$d2],16);
|
|
+ rll (@x[$d3],@x[$d3],16);
|
|
+
|
|
+ alr ($xc,@x[$d2]);
|
|
+ alr ($xc_,@x[$d3]);
|
|
+ xr (@x[$b2],$xc);
|
|
+ xr (@x[$b3],$xc_);
|
|
+ rll (@x[$b2],@x[$b2],12);
|
|
+ rll (@x[$b3],@x[$b3],12);
|
|
+
|
|
+ alr (@x[$a2],@x[$b2]);
|
|
+ alr (@x[$a3],@x[$b3]);
|
|
+ xr (@x[$d2],@x[$a2]);
|
|
+ xr (@x[$d3],@x[$a3]);
|
|
+ rll (@x[$d2],@x[$d2],8);
|
|
+ rll (@x[$d3],@x[$d3],8);
|
|
+
|
|
+ alr ($xc,@x[$d2]);
|
|
+ alr ($xc_,@x[$d3]);
|
|
+ xr (@x[$b2],$xc);
|
|
+ xr (@x[$b3],$xc_);
|
|
+ rll (@x[$b2],@x[$b2],7);
|
|
+ rll (@x[$b3],@x[$b3],7);
|
|
+}
|
|
+
|
|
+sub VX_ROUND {
|
|
+my ($a0,$b0,$c0,$d0)=@_;
|
|
+my ($a1,$b1,$c1,$d1)=map(($_&~3)+(($_+1)&3),($a0,$b0,$c0,$d0));
|
|
+my ($a2,$b2,$c2,$d2)=map(($_&~3)+(($_+1)&3),($a1,$b1,$c1,$d1));
|
|
+my ($a3,$b3,$c3,$d3)=map(($_&~3)+(($_+1)&3),($a2,$b2,$c2,$d2));
|
|
+
|
|
+ vaf (@v[$a0],@v[$a0],@v[$b0]);
|
|
+ vaf (@v[$a1],@v[$a1],@v[$b1]);
|
|
+ vaf (@v[$a2],@v[$a2],@v[$b2]);
|
|
+ vaf (@v[$a3],@v[$a3],@v[$b3]);
|
|
+ vx (@v[$d0],@v[$d0],@v[$a0]);
|
|
+ vx (@v[$d1],@v[$d1],@v[$a1]);
|
|
+ vx (@v[$d2],@v[$d2],@v[$a2]);
|
|
+ vx (@v[$d3],@v[$d3],@v[$a3]);
|
|
+ verllf (@v[$d0],@v[$d0],16);
|
|
+ verllf (@v[$d1],@v[$d1],16);
|
|
+ verllf (@v[$d2],@v[$d2],16);
|
|
+ verllf (@v[$d3],@v[$d3],16);
|
|
+
|
|
+ vaf (@v[$c0],@v[$c0],@v[$d0]);
|
|
+ vaf (@v[$c1],@v[$c1],@v[$d1]);
|
|
+ vaf (@v[$c2],@v[$c2],@v[$d2]);
|
|
+ vaf (@v[$c3],@v[$c3],@v[$d3]);
|
|
+ vx (@v[$b0],@v[$b0],@v[$c0]);
|
|
+ vx (@v[$b1],@v[$b1],@v[$c1]);
|
|
+ vx (@v[$b2],@v[$b2],@v[$c2]);
|
|
+ vx (@v[$b3],@v[$b3],@v[$c3]);
|
|
+ verllf (@v[$b0],@v[$b0],12);
|
|
+ verllf (@v[$b1],@v[$b1],12);
|
|
+ verllf (@v[$b2],@v[$b2],12);
|
|
+ verllf (@v[$b3],@v[$b3],12);
|
|
+
|
|
+ vaf (@v[$a0],@v[$a0],@v[$b0]);
|
|
+ vaf (@v[$a1],@v[$a1],@v[$b1]);
|
|
+ vaf (@v[$a2],@v[$a2],@v[$b2]);
|
|
+ vaf (@v[$a3],@v[$a3],@v[$b3]);
|
|
+ vx (@v[$d0],@v[$d0],@v[$a0]);
|
|
+ vx (@v[$d1],@v[$d1],@v[$a1]);
|
|
+ vx (@v[$d2],@v[$d2],@v[$a2]);
|
|
+ vx (@v[$d3],@v[$d3],@v[$a3]);
|
|
+ verllf (@v[$d0],@v[$d0],8);
|
|
+ verllf (@v[$d1],@v[$d1],8);
|
|
+ verllf (@v[$d2],@v[$d2],8);
|
|
+ verllf (@v[$d3],@v[$d3],8);
|
|
+
|
|
+ vaf (@v[$c0],@v[$c0],@v[$d0]);
|
|
+ vaf (@v[$c1],@v[$c1],@v[$d1]);
|
|
+ vaf (@v[$c2],@v[$c2],@v[$d2]);
|
|
+ vaf (@v[$c3],@v[$c3],@v[$d3]);
|
|
+ vx (@v[$b0],@v[$b0],@v[$c0]);
|
|
+ vx (@v[$b1],@v[$b1],@v[$c1]);
|
|
+ vx (@v[$b2],@v[$b2],@v[$c2]);
|
|
+ vx (@v[$b3],@v[$b3],@v[$c3]);
|
|
+ verllf (@v[$b0],@v[$b0],7);
|
|
+ verllf (@v[$b1],@v[$b1],7);
|
|
+ verllf (@v[$b2],@v[$b2],7);
|
|
+ verllf (@v[$b3],@v[$b3],7);
|
|
+}
|
|
+
|
|
+PERLASM_BEGIN($output);
|
|
|
|
-foreach (split("\n",$code)) {
|
|
- s/\`([^\`]*)\`/eval $1/ge;
|
|
+INCLUDE ("s390x_arch.h");
|
|
+TEXT ();
|
|
|
|
- print $_,"\n";
|
|
+################
|
|
+# void ChaCha20_ctr32(unsigned char *out, const unsigned char *inp, size_t len,
|
|
+# const unsigned int key[8], const unsigned int counter[4])
|
|
+{
|
|
+my ($out,$inp,$len,$key,$counter)=map("%r$_",(2..6));
|
|
+
|
|
+# VX CODE PATH
|
|
+{
|
|
+my $off=$z*8*16+8; # offset(initial state)
|
|
+my $frame=$stdframe+4*16+$off;
|
|
+
|
|
+GLOBL ("ChaCha20_ctr32");
|
|
+TYPE ("ChaCha20_ctr32","\@function");
|
|
+ALIGN (32);
|
|
+LABEL ("ChaCha20_ctr32");
|
|
+ larl ("%r1","OPENSSL_s390xcap_P");
|
|
+
|
|
+ lghi ("%r0",64);
|
|
+&{$z? \&cgr:\&cr} ($len,"%r0");
|
|
+ jle ("_s390x_chacha_novx");
|
|
+
|
|
+ lg ("%r0","S390X_STFLE+16(%r1)");
|
|
+ tmhh ("%r0",0x4000); # check for vector facility
|
|
+ jz ("_s390x_chacha_novx");
|
|
+
|
|
+if (!$z) {
|
|
+ llgfr ($len,$len);
|
|
+ std ("%f4","16*$SIZE_T+2*8($sp)");
|
|
+ std ("%f6","16*$SIZE_T+3*8($sp)");
|
|
+}
|
|
+&{$z? \&stmg:\&stm} ("%r6","%r7","6*$SIZE_T($sp)");
|
|
+
|
|
+ lghi ("%r1",-$frame);
|
|
+ lgr ("%r0",$sp);
|
|
+ la ($sp,"0(%r1,$sp)"); # allocate stack frame
|
|
+
|
|
+ larl ("%r7",".Lsigma");
|
|
+&{$z? \&stg:\&st} ("%r0","0($sp)"); # backchain
|
|
+
|
|
+ vstm ("%v8","%v15","8($sp)") if ($z);
|
|
+
|
|
+ vlm ("%v1","%v2","0($key)"); # load key
|
|
+ vl ("%v0","0(%r7)"); # load sigma constant
|
|
+ vl ("%v3","0($counter)"); # load iv (counter||nonce)
|
|
+ l ("%r0","0($counter)"); # load counter
|
|
+ vstm ("%v0","%v3","$off($sp)"); # copy initial state to stack
|
|
+
|
|
+ srlg ("%r1",$len,8);
|
|
+ ltgr ("%r1","%r1");
|
|
+ jz (".Lvx_4x_done");
|
|
+
|
|
+ALIGN (16); # process 4 64-byte blocks
|
|
+LABEL (".Lvx_4x");
|
|
+ vlrepf ("%v$_",($_*4)."+$off($sp)") for (0..15); # load initial
|
|
+ # state
|
|
+ vl ("%v31","16(%r7)");
|
|
+ vaf ("%v12","%v12","%v31"); # increment counter
|
|
+
|
|
+ vlr (@v[$_],"%v$_") for (0..15); # copy initial state
|
|
+
|
|
+ lhi ("%r6",10);
|
|
+ j (".Loop_vx_4x");
|
|
+
|
|
+ALIGN (16);
|
|
+LABEL (".Loop_vx_4x");
|
|
+ VX_ROUND( 0, 4, 8,12); # column round
|
|
+ VX_ROUND( 0, 5,10,15); # diagonal round
|
|
+ brct ("%r6",".Loop_vx_4x");
|
|
+
|
|
+ vaf (@v[$_],@v[$_],"%v$_") for (0..15); # state += initial
|
|
+ # state (mod 32)
|
|
+ vlm ("%v6","%v7","32(%r7)"); # load vperm operands
|
|
+
|
|
+for (0..3) { # blocks 1,2
|
|
+ vmrhf ("%v0",@v[$_*4+0],@v[$_*4+1]); # ks = serialize(state)
|
|
+ vmrhf ("%v1",@v[$_*4+2],@v[$_*4+3]);
|
|
+ vperm ("%v".($_+ 8),"%v0","%v1","%v6");
|
|
+ vperm ("%v".($_+12),"%v0","%v1","%v7");
|
|
+}
|
|
+ vlm ("%v0","%v7","0($inp)"); # load in
|
|
+ vx ("%v$_","%v$_","%v".($_+8)) for (0..7); # out = in ^ ks
|
|
+ vstm ("%v0","%v7","0($out)"); # store out
|
|
+
|
|
+ vlm ("%v6","%v7","32(%r7)"); # restore vperm operands
|
|
+
|
|
+for (0..3) { # blocks 2,3
|
|
+ vmrlf ("%v0",@v[$_*4+0],@v[$_*4+1]); # ks = serialize(state)
|
|
+ vmrlf ("%v1",@v[$_*4+2],@v[$_*4+3]);
|
|
+ vperm ("%v".($_+ 8),"%v0","%v1","%v6");
|
|
+ vperm ("%v".($_+12),"%v0","%v1","%v7");
|
|
+}
|
|
+ vlm ("%v0","%v7","128($inp)"); # load in
|
|
+ vx ("%v$_","%v$_","%v".($_+8)) for (0..7); # out = in ^ ks
|
|
+ vstm ("%v0","%v7","128($out)"); # store out
|
|
+
|
|
+ ahi ("%r0",4);
|
|
+ st ("%r0","48+$off($sp)"); # update initial state
|
|
+
|
|
+ la ($inp,"256($inp)");
|
|
+ la ($out,"256($out)");
|
|
+ brctg ("%r1",".Lvx_4x");
|
|
+
|
|
+ALIGN (16);
|
|
+LABEL (".Lvx_4x_done");
|
|
+ lghi ("%r1",0xff);
|
|
+ ngr ($len,"%r1");
|
|
+ jnz (".Lvx_rem");
|
|
+
|
|
+ALIGN (16);
|
|
+LABEL (".Lvx_done");
|
|
+ vzero ("%v$_") for (16..31); # wipe ks and key copy
|
|
+ vstm ("%v16","%v17","16+$off($sp)");
|
|
+ vlm ("%v8","%v15","8($sp)") if ($z);
|
|
+
|
|
+ la ($sp,"$frame($sp)");
|
|
+&{$z? \&lmg:\&lm} ("%r6","%r7","6*$SIZE_T($sp)");
|
|
+
|
|
+if (!$z) {
|
|
+ ld ("%f4","16*$SIZE_T+2*8($sp)");
|
|
+ ld ("%f6","16*$SIZE_T+3*8($sp)");
|
|
+ vzero ("%v$_") for (8..15);
|
|
+}
|
|
+ br ("%r14");
|
|
+ALIGN (16);
|
|
+LABEL (".Lvx_rem");
|
|
+ lhi ("%r0",64);
|
|
+
|
|
+ sr ($len,"%r0");
|
|
+ brc (2,".Lvx_rem_g64"); # cc==2?
|
|
+
|
|
+ lghi ("%r1",-$stdframe);
|
|
+
|
|
+ la ($counter,"48+$off($sp)"); # load updated iv
|
|
+ ar ($len,"%r0"); # restore len
|
|
+
|
|
+ lgr ("%r7",$counter);
|
|
+&{$z? \&stg:\&st} ("%r14","14*$SIZE_T+$frame($sp)");
|
|
+ la ($sp,"0(%r1,$sp)");
|
|
+
|
|
+ bras ("%r14","_s390x_chacha_novx");
|
|
+
|
|
+ la ($sp,"$stdframe($sp)");
|
|
+&{$z? \&lg:\&l} ("%r14","14*$SIZE_T+$frame($sp)");
|
|
+ lgr ($counter,"%r7");
|
|
+ j (".Lvx_done");
|
|
+
|
|
+ALIGN (16);
|
|
+LABEL (".Lvx_rem_g64");
|
|
+ vlrepf ("%v$_",($_*4)."+$off($sp)") for (0..15); # load initial
|
|
+ # state
|
|
+ vl ("%v31","16(%r7)");
|
|
+ vaf ("%v12","%v12","%v31"); # increment counter
|
|
+
|
|
+ vlr (@v[$_],"%v$_") for (0..15); # state = initial state
|
|
+
|
|
+ lhi ("%r6",10);
|
|
+ j (".Loop_vx_rem");
|
|
+
|
|
+ALIGN (16);
|
|
+LABEL (".Loop_vx_rem");
|
|
+ VX_ROUND( 0, 4, 8,12); # column round
|
|
+ VX_ROUND( 0, 5,10,15); # diagonal round
|
|
+ brct ("%r6",".Loop_vx_rem");
|
|
+
|
|
+ vaf (@v[$_],@v[$_],"%v$_") for (0..15); # state += initial
|
|
+ # state (mod 32)
|
|
+ vlm ("%v6","%v7","32(%r7)"); # load vperm operands
|
|
+
|
|
+for (0..3) { # blocks 1,2
|
|
+ vmrhf ("%v0",@v[$_*4+0],@v[$_*4+1]); # ks = serialize(state)
|
|
+ vmrhf ("%v1",@v[$_*4+2],@v[$_*4+3]);
|
|
+ vperm ("%v".($_+8),"%v0","%v1","%v6");
|
|
+ vperm ("%v".($_+12),"%v0","%v1","%v7");
|
|
+}
|
|
+ vlm ("%v0","%v3","0($inp)"); # load in
|
|
+ vx ("%v$_","%v$_","%v".($_+8)) for (0..3); # out = in ^ ks
|
|
+ vstm ("%v0","%v3","0($out)"); # store out
|
|
+
|
|
+ la ($inp,"64($inp)");
|
|
+ la ($out,"64($out)");
|
|
+
|
|
+ sr ($len,"%r0");
|
|
+ brc (4,".Lvx_tail"); # cc==4?
|
|
+
|
|
+ vlm ("%v0","%v3","0($inp)"); # load in
|
|
+ vx ("%v$_","%v$_","%v".($_+12)) for (0..3); # out = in ^ ks
|
|
+ vstm ("%v0","%v3","0($out)"); # store out
|
|
+ jz (".Lvx_done");
|
|
+
|
|
+for (0..3) { # blocks 3,4
|
|
+ vmrlf ("%v0",@v[$_*4+0],@v[$_*4+1]); # ks = serialize(state)
|
|
+ vmrlf ("%v1",@v[$_*4+2],@v[$_*4+3]);
|
|
+ vperm ("%v".($_+12),"%v0","%v1","%v6");
|
|
+ vperm ("%v".($_+8),"%v0","%v1","%v7");
|
|
+}
|
|
+ la ($inp,"64($inp)");
|
|
+ la ($out,"64($out)");
|
|
+
|
|
+ sr ($len,"%r0");
|
|
+ brc (4,".Lvx_tail"); # cc==4?
|
|
+
|
|
+ vlm ("%v0","%v3","0($inp)"); # load in
|
|
+ vx ("%v$_","%v$_","%v".($_+12)) for (0..3); # out = in ^ ks
|
|
+ vstm ("%v0","%v3","0($out)"); # store out
|
|
+ jz (".Lvx_done");
|
|
+
|
|
+ la ($inp,"64($inp)");
|
|
+ la ($out,"64($out)");
|
|
+
|
|
+ sr ($len,"%r0");
|
|
+ vlr ("%v".($_+4),"%v$_") for (8..11);
|
|
+ j (".Lvx_tail");
|
|
+
|
|
+ALIGN (16);
|
|
+LABEL (".Lvx_tail");
|
|
+ ar ($len,"%r0"); # restore $len
|
|
+ ahi ($len,-1);
|
|
+
|
|
+ lhi ("%r0",16);
|
|
+for (0..2) {
|
|
+ vll ("%v0",$len,($_*16)."($inp)");
|
|
+ vx ("%v0","%v0","%v".($_+12));
|
|
+ vstl ("%v0",$len,($_*16)."($out)");
|
|
+ sr ($len,"%r0");
|
|
+ brc (4,".Lvx_done"); # cc==4?
|
|
+}
|
|
+ vll ("%v0",$len,"3*16($inp)");
|
|
+ vx ("%v0","%v0","%v15");
|
|
+ vstl ("%v0",$len,"3*16($out)");
|
|
+ j (".Lvx_done");
|
|
+SIZE ("ChaCha20_ctr32",".-ChaCha20_ctr32");
|
|
+}
|
|
+
|
|
+# NOVX CODE PATH
|
|
+{
|
|
+my $frame=$stdframe+4*20;
|
|
+
|
|
+TYPE ("_s390x_chacha_novx","\@function");
|
|
+ALIGN (32);
|
|
+LABEL ("_s390x_chacha_novx");
|
|
+&{$z? \<gr:\<r} ($len,$len); # $len==0?
|
|
+ bzr ("%r14");
|
|
+&{$z? \&aghi:\&ahi} ($len,-64);
|
|
+&{$z? \&lghi:\&lhi} ("%r1",-$frame);
|
|
+&{$z? \&stmg:\&stm} ("%r6","%r15","6*$SIZE_T($sp)");
|
|
+&{$z? \&slgr:\&slr} ($out,$inp); # difference
|
|
+ la ($len,"0($inp,$len)"); # end of input minus 64
|
|
+ larl ("%r7",".Lsigma");
|
|
+ lgr ("%r0",$sp);
|
|
+ la ($sp,"0(%r1,$sp)");
|
|
+&{$z? \&stg:\&st} ("%r0","0($sp)");
|
|
+
|
|
+ lmg ("%r8","%r11","0($key)"); # load key
|
|
+ lmg ("%r12","%r13","0($counter)"); # load counter
|
|
+ lmg ("%r6","%r7","0(%r7)"); # load sigma constant
|
|
+
|
|
+ la ("%r14","0($inp)");
|
|
+&{$z? \&stg:\&st} ($out,"$frame+3*$SIZE_T($sp)");
|
|
+&{$z? \&stg:\&st} ($len,"$frame+4*$SIZE_T($sp)");
|
|
+ stmg ("%r6","%r13","$stdframe($sp)");# copy key schedule to stack
|
|
+ srlg (@x[12],"%r12",32); # 32-bit counter value
|
|
+ j (".Loop_outer");
|
|
+
|
|
+ALIGN (16);
|
|
+LABEL (".Loop_outer");
|
|
+ lm (@x[0],@x[7],"$stdframe+4*0($sp)"); # load x[0]-x[7]
|
|
+ lm (@t[0],@t[1],"$stdframe+4*10($sp)"); # load x[10]-x[11]
|
|
+ lm (@x[13],@x[15],"$stdframe+4*13($sp)"); # load x[13]-x[15]
|
|
+ stm (@t[0],@t[1],"$stdframe+4*8+4*10($sp)");# offload x[10]-x[11]
|
|
+ lm (@t[0],@t[1],"$stdframe+4*8($sp)"); # load x[8]-x[9]
|
|
+ st (@x[12],"$stdframe+4*12($sp)"); # save counter
|
|
+&{$z? \&stg:\&st} ("%r14","$frame+2*$SIZE_T($sp)");# save input pointer
|
|
+ lhi ("%r14",10);
|
|
+ j (".Loop");
|
|
+
|
|
+ALIGN (4);
|
|
+LABEL (".Loop");
|
|
+ ROUND (0, 4, 8,12);
|
|
+ ROUND (0, 5,10,15);
|
|
+ brct ("%r14",".Loop");
|
|
+
|
|
+&{$z? \&lg:\&l} ("%r14","$frame+2*$SIZE_T($sp)");# pull input pointer
|
|
+ stm (@t[0],@t[1],"$stdframe+4*8+4*8($sp)"); # offload x[8]-x[9]
|
|
+&{$z? \&lmg:\&lm} (@t[0],@t[1],"$frame+3*$SIZE_T($sp)");
|
|
+
|
|
+ al (@x[0],"$stdframe+4*0($sp)"); # accumulate key schedule
|
|
+ al (@x[1],"$stdframe+4*1($sp)");
|
|
+ al (@x[2],"$stdframe+4*2($sp)");
|
|
+ al (@x[3],"$stdframe+4*3($sp)");
|
|
+ al (@x[4],"$stdframe+4*4($sp)");
|
|
+ al (@x[5],"$stdframe+4*5($sp)");
|
|
+ al (@x[6],"$stdframe+4*6($sp)");
|
|
+ al (@x[7],"$stdframe+4*7($sp)");
|
|
+ lrvr (@x[0],@x[0]);
|
|
+ lrvr (@x[1],@x[1]);
|
|
+ lrvr (@x[2],@x[2]);
|
|
+ lrvr (@x[3],@x[3]);
|
|
+ lrvr (@x[4],@x[4]);
|
|
+ lrvr (@x[5],@x[5]);
|
|
+ lrvr (@x[6],@x[6]);
|
|
+ lrvr (@x[7],@x[7]);
|
|
+ al (@x[12],"$stdframe+4*12($sp)");
|
|
+ al (@x[13],"$stdframe+4*13($sp)");
|
|
+ al (@x[14],"$stdframe+4*14($sp)");
|
|
+ al (@x[15],"$stdframe+4*15($sp)");
|
|
+ lrvr (@x[12],@x[12]);
|
|
+ lrvr (@x[13],@x[13]);
|
|
+ lrvr (@x[14],@x[14]);
|
|
+ lrvr (@x[15],@x[15]);
|
|
+
|
|
+ la (@t[0],"0(@t[0],%r14)"); # reconstruct output pointer
|
|
+&{$z? \&clgr:\&clr} ("%r14",@t[1]);
|
|
+ jh (".Ltail");
|
|
+
|
|
+ x (@x[0],"4*0(%r14)"); # xor with input
|
|
+ x (@x[1],"4*1(%r14)");
|
|
+ st (@x[0],"4*0(@t[0])"); # store output
|
|
+ x (@x[2],"4*2(%r14)");
|
|
+ st (@x[1],"4*1(@t[0])");
|
|
+ x (@x[3],"4*3(%r14)");
|
|
+ st (@x[2],"4*2(@t[0])");
|
|
+ x (@x[4],"4*4(%r14)");
|
|
+ st (@x[3],"4*3(@t[0])");
|
|
+ lm (@x[0],@x[3],"$stdframe+4*8+4*8($sp)"); # load x[8]-x[11]
|
|
+ x (@x[5],"4*5(%r14)");
|
|
+ st (@x[4],"4*4(@t[0])");
|
|
+ x (@x[6],"4*6(%r14)");
|
|
+ al (@x[0],"$stdframe+4*8($sp)");
|
|
+ st (@x[5],"4*5(@t[0])");
|
|
+ x (@x[7],"4*7(%r14)");
|
|
+ al (@x[1],"$stdframe+4*9($sp)");
|
|
+ st (@x[6],"4*6(@t[0])");
|
|
+ x (@x[12],"4*12(%r14)");
|
|
+ al (@x[2],"$stdframe+4*10($sp)");
|
|
+ st (@x[7],"4*7(@t[0])");
|
|
+ x (@x[13],"4*13(%r14)");
|
|
+ al (@x[3],"$stdframe+4*11($sp)");
|
|
+ st (@x[12],"4*12(@t[0])");
|
|
+ x (@x[14],"4*14(%r14)");
|
|
+ st (@x[13],"4*13(@t[0])");
|
|
+ x (@x[15],"4*15(%r14)");
|
|
+ st (@x[14],"4*14(@t[0])");
|
|
+ lrvr (@x[0],@x[0]);
|
|
+ st (@x[15],"4*15(@t[0])");
|
|
+ lrvr (@x[1],@x[1]);
|
|
+ lrvr (@x[2],@x[2]);
|
|
+ lrvr (@x[3],@x[3]);
|
|
+ lhi (@x[12],1);
|
|
+ x (@x[0],"4*8(%r14)");
|
|
+ al (@x[12],"$stdframe+4*12($sp)"); # increment counter
|
|
+ x (@x[1],"4*9(%r14)");
|
|
+ st (@x[0],"4*8(@t[0])");
|
|
+ x (@x[2],"4*10(%r14)");
|
|
+ st (@x[1],"4*9(@t[0])");
|
|
+ x (@x[3],"4*11(%r14)");
|
|
+ st (@x[2],"4*10(@t[0])");
|
|
+ st (@x[3],"4*11(@t[0])");
|
|
+
|
|
+&{$z? \&clgr:\&clr} ("%r14",@t[1]); # done yet?
|
|
+ la ("%r14","64(%r14)");
|
|
+ jl (".Loop_outer");
|
|
+
|
|
+LABEL (".Ldone");
|
|
+ xgr ("%r0","%r0");
|
|
+ xgr ("%r1","%r1");
|
|
+ xgr ("%r2","%r2");
|
|
+ xgr ("%r3","%r3");
|
|
+ stmg ("%r0","%r3","$stdframe+4*4($sp)"); # wipe key copy
|
|
+ stmg ("%r0","%r3","$stdframe+4*12($sp)");
|
|
+
|
|
+&{$z? \&lmg:\&lm} ("%r6","%r15","$frame+6*$SIZE_T($sp)");
|
|
+ br ("%r14");
|
|
+
|
|
+ALIGN (16);
|
|
+LABEL (".Ltail");
|
|
+ la (@t[1],"64($t[1])");
|
|
+ stm (@x[0],@x[7],"$stdframe+4*0($sp)");
|
|
+&{$z? \&slgr:\&slr} (@t[1],"%r14");
|
|
+ lm (@x[0],@x[3],"$stdframe+4*8+4*8($sp)");
|
|
+&{$z? \&lghi:\&lhi} (@x[6],0);
|
|
+ stm (@x[12],@x[15],"$stdframe+4*12($sp)");
|
|
+ al (@x[0],"$stdframe+4*8($sp)");
|
|
+ al (@x[1],"$stdframe+4*9($sp)");
|
|
+ al (@x[2],"$stdframe+4*10($sp)");
|
|
+ al (@x[3],"$stdframe+4*11($sp)");
|
|
+ lrvr (@x[0],@x[0]);
|
|
+ lrvr (@x[1],@x[1]);
|
|
+ lrvr (@x[2],@x[2]);
|
|
+ lrvr (@x[3],@x[3]);
|
|
+ stm (@x[0],@x[3],"$stdframe+4*8($sp)");
|
|
+
|
|
+LABEL (".Loop_tail");
|
|
+ llgc (@x[4],"0(@x[6],%r14)");
|
|
+ llgc (@x[5],"$stdframe(@x[6],$sp)");
|
|
+ xr (@x[5],@x[4]);
|
|
+ stc (@x[5],"0(@x[6],@t[0])");
|
|
+ la (@x[6],"1(@x[6])");
|
|
+ brct (@t[1],".Loop_tail");
|
|
+
|
|
+ j (".Ldone");
|
|
+SIZE ("_s390x_chacha_novx",".-_s390x_chacha_novx");
|
|
+}
|
|
}
|
|
-close STDOUT or die "error closing STDOUT: $!";
|
|
+################
|
|
+
|
|
+ALIGN (64);
|
|
+LABEL (".Lsigma");
|
|
+LONG (0x61707865,0x3320646e,0x79622d32,0x6b206574); # endian-neutral sigma
|
|
+LONG (0x00000000,0x00000001,0x00000002,0x00000003); # vaf counter increment
|
|
+LONG (0x03020100,0x07060504,0x13121110,0x17161514); # vperm serialization
|
|
+LONG (0x0b0a0908,0x0f0e0d0c,0x1b1a1918,0x1f1e1d1c); # vperm serialization
|
|
+ASCIZ ("\"ChaCha20 for s390x, CRYPTOGAMS by <appro\@openssl.org>\"");
|
|
+ALIGN (4);
|
|
+
|
|
+PERLASM_END();
|