1 #!/usr/bin/env perl
2 #
3 # ====================================================================
4 # Written by Andy Polyakov <appro@fy.chalmers.se> for the OpenSSL
5 # project. The module is, however, dual licensed under OpenSSL and
6 # CRYPTOGAMS licenses depending on where you obtain it. For further
7 # details see http://www.openssl.org/~appro/cryptogams/.
8 # ====================================================================
9 #
10 # 2.22x RC4 tune-up:-) It should be noted though that my hand [as in
11 # "hand-coded assembler"] doesn't stand for the whole improvement
12 # coefficient. It turned out that eliminating RC4_CHAR from config
13 # line results in ~40% improvement (yes, even for C implementation).
14 # Presumably it has everything to do with AMD cache architecture and
15 # RAW or whatever penalties. Once again! The module *requires* config
16 # line *without* RC4_CHAR! As for coding "secret," I bet on partial
17 # register arithmetics. For example instead of 'inc %r8; and $255,%r8'
18 # I simply 'inc %r8b'. Even though optimization manual discourages
19 # to operate on partial registers, it turned out to be the best bet.
20 # At least for AMD... How IA32E would perform remains to be seen...
21
22 # As was shown by Marc Bevand reordering of couple of load operations
23 # results in even higher performance gain of 3.3x:-) At least on
24 # Opteron... For reference, 1x in this case is RC4_CHAR C-code
25 # compiled with gcc 3.3.2, which performs at ~54MBps per 1GHz clock.
26 # Latter means that if you want to *estimate* what to expect from
27 # *your* Opteron, then multiply 54 by 3.3 and clock frequency in GHz.
28
29 # Intel P4 EM64T core was found to run the AMD64 code really slow...
30 # The only way to achieve comparable performance on P4 was to keep
31 # RC4_CHAR. Kind of ironic, huh? As it's apparently impossible to
32 # compose blended code, which would perform even within 30% marginal
33 # on either AMD and Intel platforms, I implement both cases. See
34 # rc4_skey.c for further details...
35
36 # P4 EM64T core appears to be "allergic" to 64-bit inc/dec. Replacing
37 # those with add/sub results in 50% performance improvement of folded
38 # loop...
39
40 # As was shown by Zou Nanhai loop unrolling can improve Intel EM64T
41 # performance by >30% [unlike P4 32-bit case that is]. But this is
42 # provided that loads are reordered even more aggressively! Both code
43 # pathes, AMD64 and EM64T, reorder loads in essentially same manner
44 # as my IA-64 implementation. On Opteron this resulted in modest 5%
45 # improvement [I had to test it], while final Intel P4 performance
46 # achieves respectful 432MBps on 2.8GHz processor now. For reference.
47 # If executed on Xeon, current RC4_CHAR code-path is 2.7x faster than
48 # RC4_INT code-path. While if executed on Opteron, it's only 25%
49 # slower than the RC4_INT one [meaning that if CPU µ-arch detection
50 # is not implemented, then this final RC4_CHAR code-path should be
51 # preferred, as it provides better *all-round* performance].
52
53 # Intel Core2 was observed to perform poorly on both code paths:-( It
54 # apparently suffers from some kind of partial register stall, which
55 # occurs in 64-bit mode only [as virtually identical 32-bit loop was
56 # observed to outperform 64-bit one by almost 50%]. Adding two movzb to
57 # cloop1 boosts its performance by 80%! This loop appears to be optimal
58 # fit for Core2 and therefore the code was modified to skip cloop8 on
59 # this CPU.
60
61 #
62 # OpenSolaris OS modifications
63 #
64 # Sun elects to use this software under the BSD license.
65 #
66 # This source originates from OpenSSL file rc4-x86_64.pl at
67 # ftp://ftp.openssl.org/snapshot/openssl-0.9.8-stable-SNAP-20080131.tar.gz
68 # (presumably for future OpenSSL release 0.9.8h), with these changes:
69 #
70 # 1. Added some comments, "use strict", and declared all variables.
71 #
72 # 2. Added OpenSolaris ENTRY_NP/SET_SIZE macros from
73 # /usr/include/sys/asm_linkage.h.
74 #
75 # 3. Changed function name from RC4() to arcfour_crypt_asm() and RC4_set_key()
76 # to arcfour_key_init(), and changed the parameter order for both to that
77 # used by OpenSolaris.
78 #
79 # 4. The current method of using cpuid feature bits 20 (NX) or 28 (HTT) from
80 # function OPENSSL_ia32_cpuid() to distinguish Intel/AMD does not work for
81 # some newer AMD64 processors, as these bits are set on both Intel EM64T
82 # processors and newer AMD64 processors. I replaced this with C code
83 # (function arcfour_crypt_on_intel()) to call cpuid_getvendor()
84 # when executing in the kernel and getisax() when executing in userland.
85 #
86 # 5. Set a new field in the key structure, key->flag to 0 for AMD AMD64
87 # and 1 for Intel EM64T. This is to select the most-efficient arcfour_crypt()
88 # function to use.
89 #
90 # 6. Removed x86_64-xlate.pl script (not needed for as(1) or gas(1) assemblers).
91 #
92 # 7. Removed unused RC4_CHAR, Lcloop1, and Lcloop8 code.
93 #
94 # 8. Added C function definitions for use by lint(1B).
95 #
96
97 use strict;
98 my ($code, $dat, $inp, $out, $len, $idx, $ido, $i, @XX, @TX, $YY, $TY);
99 my $output = shift;
100 open STDOUT,">$output";
101
102 #
103 # Parameters
104 #
105
106 # OpenSSL:
107 # void RC4(RC4_KEY *key, unsigned long len, const unsigned char *indata,
108 # unsigned char *outdata);
109 #$dat="%rdi"; # arg1
110 #$len="%rsi"; # arg2
111 #$inp="%rdx"; # arg3
112 #$out="%rcx"; # arg4
113
114 # OpenSolaris:
115 # void arcfour_crypt_asm(ARCFour_key *key, uchar_t *in, uchar_t *out,
116 # size_t len);
117 $dat="%rdi"; # arg1
118 $inp="%rsi"; # arg2
119 $out="%rdx"; # arg3
120 $len="%rcx"; # arg4
121
122 #
123 # Register variables
124 #
125 # $XX[0] is key->i (aka key->x), $XX[1] is a temporary.
126 # $TX[0] and $TX[1] are temporaries.
127 # $YY is key->j (aka key->y).
128 # $TY is a temporary.
129 #
130 @XX=("%r8","%r10");
131 @TX=("%r9","%r11");
132 $YY="%r12";
133 $TY="%r13";
134
135 $code=<<___;
136 #if defined(lint) || defined(__lint)
137
138 #include "arcfour.h"
139
140 /* ARGSUSED */
141 void
142 arcfour_crypt_asm(ARCFour_key *key, uchar_t *in, uchar_t *out, size_t len)
143 {}
144
145 /* ARGSUSED */
146 void
147 arcfour_key_init(ARCFour_key *key, uchar_t *keyval, int keyvallen)
148 {}
149
150 #else
151 #include <sys/asm_linkage.h>
152
153 ENTRY_NP(arcfour_crypt_asm)
154 /* EXPORT DELETE START */
155
156 or $len,$len # If (len == 0) return
157 jne .Lentry
158 ret
159 .Lentry:
160 push %r12
161 push %r13
162
163 / Set $dat to beginning of array, key->arr[0]
164 add \$8,$dat
165 / Get key->j
166 movl -8($dat),$XX[0]#d
167 / Get key->i
168 movl -4($dat),$YY#d
169
170 /
171 / Use a 4-byte key schedule element array
172 /
173 inc $XX[0]#b
174 movl ($dat,$XX[0],4),$TX[0]#d
175 test \$-8,$len
176 jz .Lloop1
177 jmp .Lloop8
178
179 .align 16
180 .Lloop8:
181 ___
182 for ($i=0;$i<8;$i++) {
183 $code.=<<___;
184 add $TX[0]#b,$YY#b
185 mov $XX[0],$XX[1]
186 movl ($dat,$YY,4),$TY#d
187 ror \$8,%rax # ror is redundant when $i=0
188 inc $XX[1]#b
189 movl ($dat,$XX[1],4),$TX[1]#d
190 cmp $XX[1],$YY
191 movl $TX[0]#d,($dat,$YY,4)
192 cmove $TX[0],$TX[1]
193 movl $TY#d,($dat,$XX[0],4)
194 add $TX[0]#b,$TY#b
195 movb ($dat,$TY,4),%al
196 ___
197 push(@TX,shift(@TX)); push(@XX,shift(@XX)); # "rotate" registers
198 }
199 $code.=<<___;
200 ror \$8,%rax
201 sub \$8,$len
202
203 xor ($inp),%rax
204 add \$8,$inp
205 mov %rax,($out)
206 add \$8,$out
207
208 test \$-8,$len
209 jnz .Lloop8
210 cmp \$0,$len
211 jne .Lloop1
212
213 .Lexit:
214 /
215 / Cleanup and exit code
216 /
217 / --i to undo ++i done at entry
218 sub \$1,$XX[0]#b
219 / set key->i
220 movl $XX[0]#d,-8($dat)
221 / set key->j
222 movl $YY#d,-4($dat)
223
224 pop %r13
225 pop %r12
226 ret
227
228 .align 16
229 .Lloop1:
230 add $TX[0]#b,$YY#b
231 movl ($dat,$YY,4),$TY#d
232 movl $TX[0]#d,($dat,$YY,4)
233 movl $TY#d,($dat,$XX[0],4)
234 add $TY#b,$TX[0]#b
235 inc $XX[0]#b
236 movl ($dat,$TX[0],4),$TY#d
237 movl ($dat,$XX[0],4),$TX[0]#d
238 xorb ($inp),$TY#b
239 inc $inp
240 movb $TY#b,($out)
241 inc $out
242 dec $len
243 jnz .Lloop1
244 jmp .Lexit
245
246 /* EXPORT DELETE END */
247 ret
248 SET_SIZE(arcfour_crypt_asm)
249 ___
250
251
252 #
253 # Parameters
254 #
255
256 # OpenSSL:
257 # void RC4_set_key(RC4_KEY *key, int len, const unsigned char *data);
258 #$dat="%rdi"; # arg1
259 #$len="%rsi"; # arg2
260 #$inp="%rdx"; # arg3
261
262 # OpenSolaris:
263 # void arcfour_key_init(ARCFour_key *key, uchar_t *keyval, int keyvallen);
264 $dat="%rdi"; # arg1
265 $inp="%rsi"; # arg2
266 $len="%rdx"; # arg3
267
268 # Temporaries
269 $idx="%r8";
270 $ido="%r9";
271
272 $code.=<<___;
273 / int arcfour_crypt_on_intel(void);
274 .extern arcfour_crypt_on_intel
275
276 ENTRY_NP(arcfour_key_init)
277 /* EXPORT DELETE START */
278
279 / Find out if we're running on Intel or something else (e.g., AMD64).
280 / This sets %eax to 1 for Intel, otherwise 0.
281 push %rdi / Save arg1
282 push %rsi / Save arg2
283 push %rdx / Save arg3
284 call arcfour_crypt_on_intel
285 pop %rdx / Restore arg3
286 pop %rsi / Restore arg2
287 pop %rdi / Restore arg1
288 / Save return value in key->flag (1=Intel, 0=AMD)
289 movl %eax,1032($dat)
290
291 / Set $dat to beginning of array, key->arr[0]
292 lea 8($dat),$dat
293 lea ($inp,$len),$inp
294 neg $len
295 mov $len,%rcx
296
297 xor %eax,%eax
298 xor $ido,$ido
299 xor %r10,%r10
300 xor %r11,%r11
301
302 / Use a 4-byte data array
303 jmp .Lw1stloop
304
305 .align 16
306 .Lw1stloop:
307 / AMD64 (4-byte array)
308 mov %eax,($dat,%rax,4)
309 add \$1,%al
310 jnc .Lw1stloop
311
312 xor $ido,$ido
313 xor $idx,$idx
314
315 .align 16
316 .Lw2ndloop:
317 mov ($dat,$ido,4),%r10d
318 add ($inp,$len,1),$idx#b
319 add %r10b,$idx#b
320 add \$1,$len
321 mov ($dat,$idx,4),%r11d
322 cmovz %rcx,$len
323 mov %r10d,($dat,$idx,4)
324 mov %r11d,($dat,$ido,4)
325 add \$1,$ido#b
326 jnc .Lw2ndloop
327
328 / Exit code
329 xor %eax,%eax
330 mov %eax,-8($dat)
331 mov %eax,-4($dat)
332
333 /* EXPORT DELETE END */
334 ret
335 SET_SIZE(arcfour_key_init)
336 .asciz "RC4 for x86_64, CRYPTOGAMS by <appro\@openssl.org>"
337 #endif /* !lint && !__lint */
338 ___
339
340 $code =~ s/#([bwd])/$1/gm;
341
342 print $code;
343
344 close STDOUT;