1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License, Version 1.0 only
6 * (the "License"). You may not use this file except in compliance
7 * with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22
23 /*
24 * Copyright 2006 Sun Microsystems, Inc. All rights reserved.
25 * Use is subject to license terms.
26 */
27
28 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
29 /* All Rights Reserved */
30
31 /*
32 * Portions of this source code were derived from Berkeley 4.3 BSD
33 * under license from the Regents of the University of California.
34 */
35
36 #pragma ident "%Z%%M% %I% %E% SMI"
37
38 /*
39 * Hex encryption/decryption and utility routines
40 */
41
42 #include "mt.h"
43 #include <stdio.h>
44 #include <stdlib.h>
45 #include <sys/types.h>
46 #include <rpc/rpc.h>
47 #include <rpc/key_prot.h> /* for KEYCHECKSUMSIZE */
48 #include <rpc/des_crypt.h>
49 #include <string.h>
50 #include <rpcsvc/nis_dhext.h>
51 #include <md5.h>
52
53 #define MD5HEXSIZE 32
54
55 extern int bin2hex(int len, unsigned char *binnum, char *hexnum);
56 extern int hex2bin(int len, char *hexnum, char *binnum);
57 static char hex[]; /* forward */
58 static char hexval();
59
60 int passwd2des(char *, char *);
61 static int weak_DES_key(des_block);
62
63 /*
64 * For export control reasons, we want to limit the maximum size of
65 * data that can be encrypted or decrypted. We limit this to 1024
66 * bits of key data, which amounts to 128 bytes.
67 *
68 * For the extended DH project, we have increased it to
69 * 144 bytes (128key + 16checksum) to accomadate all the 128 bytes
70 * being used by the new 1024bit keys plus 16 bytes MD5 checksum.
71 * We discussed this with Sun's export control office and lawyers
72 * and we have reason to believe this is ok for export.
73 */
74 #define MAX_KEY_CRYPT_LEN 144
75
76 /*
77 * Encrypt a secret key given passwd
78 * The secret key is passed and returned in hex notation.
79 * Its length must be a multiple of 16 hex digits (64 bits).
80 */
81 int
82 xencrypt(secret, passwd)
83 char *secret;
84 char *passwd;
85 {
86 char key[8];
87 char ivec[8];
88 char *buf;
89 int err;
90 int len;
91
92 len = (int)strlen(secret) / 2;
93 if (len > MAX_KEY_CRYPT_LEN)
94 return (0);
95 buf = malloc((unsigned)len);
96 (void) hex2bin(len, secret, buf);
97 (void) passwd2des(passwd, key);
98 (void) memset(ivec, 0, 8);
99
100 err = cbc_crypt(key, buf, len, DES_ENCRYPT | DES_HW, ivec);
101 if (DES_FAILED(err)) {
102 free(buf);
103 return (0);
104 }
105 (void) bin2hex(len, (unsigned char *) buf, secret);
106 free(buf);
107 return (1);
108 }
109
110 /*
111 * Decrypt secret key using passwd
112 * The secret key is passed and returned in hex notation.
113 * Once again, the length is a multiple of 16 hex digits
114 */
115 int
116 xdecrypt(secret, passwd)
117 char *secret;
118 char *passwd;
119 {
120 char key[8];
121 char ivec[8];
122 char *buf;
123 int err;
124 int len;
125
126 len = (int)strlen(secret) / 2;
127 if (len > MAX_KEY_CRYPT_LEN)
128 return (0);
129 buf = malloc((unsigned)len);
130
131 (void) hex2bin(len, secret, buf);
132 (void) passwd2des(passwd, key);
133 (void) memset(ivec, 0, 8);
134
135 err = cbc_crypt(key, buf, len, DES_DECRYPT | DES_HW, ivec);
136 if (DES_FAILED(err)) {
137 free(buf);
138 return (0);
139 }
140 (void) bin2hex(len, (unsigned char *) buf, secret);
141 free(buf);
142 return (1);
143 }
144
145 /*
146 * Turn password into DES key
147 */
148 int
149 passwd2des(pw, key)
150 char *pw;
151 char *key;
152 {
153 int i;
154
155 (void) memset(key, 0, 8);
156 for (i = 0; *pw; i = (i+1) % 8) {
157 key[i] ^= *pw++ << 1;
158 }
159 des_setparity(key);
160 return (1);
161 }
162
163
164 /*
165 * Hex to binary conversion
166 */
167 int
168 hex2bin(len, hexnum, binnum)
169 int len;
170 char *hexnum;
171 char *binnum;
172 {
173 int i;
174
175 for (i = 0; i < len; i++) {
176 *binnum++ = 16 * hexval(hexnum[2 * i]) +
177 hexval(hexnum[2 * i + 1]);
178 }
179 return (1);
180 }
181
182 /*
183 * Binary to hex conversion
184 */
185 int
186 bin2hex(len, binnum, hexnum)
187 int len;
188 unsigned char *binnum;
189 char *hexnum;
190 {
191 int i;
192 unsigned val;
193
194 for (i = 0; i < len; i++) {
195 val = binnum[i];
196 hexnum[i*2] = hex[val >> 4];
197 hexnum[i*2+1] = hex[val & 0xf];
198 }
199 hexnum[len*2] = 0;
200 return (1);
201 }
202
203 static char hex[16] = {
204 '0', '1', '2', '3', '4', '5', '6', '7',
205 '8', '9', 'a', 'b', 'c', 'd', 'e', 'f',
206 };
207
208 static char
209 hexval(c)
210 char c;
211 {
212 if (c >= '0' && c <= '9') {
213 return (c - '0');
214 } else if (c >= 'a' && c <= 'z') {
215 return (c - 'a' + 10);
216 } else if (c >= 'A' && c <= 'Z') {
217 return (c - 'A' + 10);
218 } else {
219 return (-1);
220 }
221 }
222
223 /*
224 * Generic key length/algorithm version of xencrypt().
225 *
226 * Encrypt a secret key given passwd.
227 * The secret key is passed in hex notation.
228 * Arg encrypted_secret will be set to point to the encrypted
229 * secret key (NUL term, hex notation).
230 *
231 * Its length must be a multiple of 16 hex digits (64 bits).
232 *
233 * For 192-0 (AUTH_DES), then encrypt using the same method as xencrypt().
234 *
235 * If arg do_chksum is TRUE, append the checksum before the encrypt.
236 * For 192-0, the checksum is done the same as in xencrypt(). For
237 * bigger keys, MD5 is used.
238 *
239 * Arg netname can be NULL for 192-0.
240 */
241 int
242 xencrypt_g(
243 char *secret, /* in */
244 keylen_t keylen, /* in */
245 algtype_t algtype, /* in */
246 const char *passwd, /* in */
247 const char netname[], /* in */
248 char **encrypted_secret, /* out */
249 bool_t do_chksum) /* in */
250 {
251 des_block key;
252 char ivec[8];
253 char *binkeybuf;
254 int err;
255 const int classic_des = keylen == 192 && algtype == 0;
256 const int hexkeybytes = BITS2NIBBLES(keylen);
257 const int keychecksumsize = classic_des ? KEYCHECKSUMSIZE : MD5HEXSIZE;
258 const int binkeybytes = do_chksum ? keylen/8 + keychecksumsize/2 :
259 keylen/8;
260 const int bufsize = do_chksum ? hexkeybytes + keychecksumsize + 1 :
261 hexkeybytes + 1;
262 char *hexkeybuf;
263
264 if (!secret || !keylen || !passwd || !encrypted_secret)
265 return (0);
266
267 if ((hexkeybuf = malloc(bufsize)) == 0)
268 return (0);
269
270 (void) memcpy(hexkeybuf, secret, hexkeybytes);
271 if (do_chksum)
272 if (classic_des) {
273 (void) memcpy(hexkeybuf + hexkeybytes, secret,
274 keychecksumsize);
275 } else {
276 MD5_CTX md5_ctx;
277 char md5hexbuf[MD5HEXSIZE + 1] = {0};
278 uint8_t digest[MD5HEXSIZE/2];
279
280 MD5Init(&md5_ctx);
281 MD5Update(&md5_ctx, (unsigned char *)hexkeybuf,
282 hexkeybytes);
283 MD5Final(digest, &md5_ctx);
284
285 /* convert md5 binary digest to hex */
286 (void) bin2hex(MD5HEXSIZE/2, digest, md5hexbuf);
287
288 /* append the hex md5 string to the end of the key */
289 (void) memcpy(hexkeybuf + hexkeybytes,
290 (void *)md5hexbuf, MD5HEXSIZE);
291 }
292 hexkeybuf[bufsize - 1] = 0;
293
294 if (binkeybytes > MAX_KEY_CRYPT_LEN) {
295 free(hexkeybuf);
296 return (0);
297 }
298 if ((binkeybuf = malloc((unsigned)binkeybytes)) == 0) {
299 free(hexkeybuf);
300 return (0);
301 }
302
303 (void) hex2bin(binkeybytes, hexkeybuf, binkeybuf);
304 if (classic_des)
305 (void) passwd2des((char *)passwd, key.c);
306 else
307 if (netname)
308 (void) passwd2des_g(passwd, netname,
309 (int)strlen(netname), &key, FALSE);
310 else {
311 free(hexkeybuf);
312 return (0);
313 }
314
315 (void) memset(ivec, 0, 8);
316
317 err = cbc_crypt(key.c, binkeybuf, binkeybytes, DES_ENCRYPT | DES_HW,
318 ivec);
319 if (DES_FAILED(err)) {
320 free(hexkeybuf);
321 free(binkeybuf);
322 return (0);
323 }
324 (void) bin2hex(binkeybytes, (unsigned char *) binkeybuf, hexkeybuf);
325 free(binkeybuf);
326 *encrypted_secret = hexkeybuf;
327 return (1);
328 }
329
330 /*
331 * Generic key len and alg type for version of xdecrypt.
332 *
333 * Decrypt secret key using passwd. The decrypted secret key
334 * *overwrites* the supplied encrypted secret key.
335 * The secret key is passed and returned in hex notation.
336 * Once again, the length is a multiple of 16 hex digits.
337 *
338 * If 'do_chksum' is TRUE, the 'secret' buffer is assumed to contain
339 * a checksum calculated by a call to xencrypt_g().
340 *
341 * If keylen is 192 and algtype is 0, then decrypt the same way
342 * as xdecrypt().
343 *
344 * Arg netname can be NULL for 192-0.
345 */
346 int
347 xdecrypt_g(
348 char *secret, /* out */
349 int keylen, /* in */
350 int algtype, /* in */
351 const char *passwd, /* in */
352 const char netname[], /* in */
353 bool_t do_chksum) /* in */
354 {
355 des_block key;
356 char ivec[8];
357 char *buf;
358 int err;
359 int len;
360 const int classic_des = keylen == 192 && algtype == 0;
361 const int hexkeybytes = BITS2NIBBLES(keylen);
362 const int keychecksumsize = classic_des ? KEYCHECKSUMSIZE : MD5HEXSIZE;
363
364 len = (int)strlen(secret) / 2;
365 if (len > MAX_KEY_CRYPT_LEN)
366 return (0);
367 if ((buf = malloc((unsigned)len)) == 0)
368 return (0);
369
370 (void) hex2bin(len, secret, buf);
371 if (classic_des)
372 (void) passwd2des((char *)passwd, key.c);
373 else
374 if (netname)
375 (void) passwd2des_g(passwd, netname,
376 (int)strlen(netname), &key, FALSE);
377 else {
378 free(buf);
379 return (0);
380 }
381 (void) memset(ivec, 0, 8);
382
383 err = cbc_crypt(key.c, buf, len, DES_DECRYPT | DES_HW, ivec);
384 if (DES_FAILED(err)) {
385 free(buf);
386 return (0);
387 }
388 (void) bin2hex(len, (unsigned char *) buf, secret);
389 free(buf);
390
391 if (do_chksum)
392 if (classic_des) {
393 if (memcmp(secret, &(secret[hexkeybytes]),
394 keychecksumsize) != 0) {
395 secret[0] = 0;
396 return (0);
397 }
398 } else {
399 MD5_CTX md5_ctx;
400 char md5hexbuf[MD5HEXSIZE + 1] = {0};
401 uint8_t digest[MD5HEXSIZE/2];
402
403 MD5Init(&md5_ctx);
404 MD5Update(&md5_ctx, (unsigned char *)secret,
405 hexkeybytes);
406 MD5Final(digest, &md5_ctx);
407
408 /* convert md5 binary digest to hex */
409 (void) bin2hex(MD5HEXSIZE/2, digest, md5hexbuf);
410
411 /* does the digest match the appended one? */
412 if (memcmp(&(secret[hexkeybytes]),
413 md5hexbuf, MD5HEXSIZE) != 0) {
414 secret[0] = 0;
415 return (0);
416 }
417 }
418
419 secret[hexkeybytes] = '\0';
420
421 return (1);
422 }
423
424
425 /*
426 * Modified version of passwd2des(). passwd2des_g() uses the Kerberos
427 * RFC 1510 algorithm to generate a DES key from a user password
428 * and mix-in string. The mix-in is expected to be the netname.
429 * This function to be used only for extended Diffie-Hellman keys.
430 *
431 * If altarg is TRUE, reverse the concat of passwd and mix-in.
432 */
433 int
434 passwd2des_g(
435 const char *pw,
436 const char *mixin,
437 int len,
438 des_block *key, /* out */
439 bool_t altalg)
440 {
441
442 int i, j, incr = 1;
443 des_block ivec, tkey;
444 char *text;
445 int plen, tlen;
446
447 (void) memset(tkey.c, 0, 8);
448 (void) memset(ivec.c, 0, 8);
449
450
451 /*
452 * Concatentate the password and the mix-in string, fan-fold and XOR them
453 * to the required eight byte initial DES key. Since passwords can be
454 * expected to use mostly seven bit ASCII, left shift the password one
455 * bit in order to preserve as much key space as possible.
456 */
457
458 #define KEYLEN sizeof (tkey.c)
459 plen = strlen(pw);
460 tlen = ((plen + len + (KEYLEN-1))/KEYLEN)*KEYLEN;
461 if ((text = malloc(tlen)) == NULL) {
462 return (0);
463 }
464
465 (void) memset(text, 0, tlen);
466
467 if (!altalg) {
468
469 /*
470 * Concatenate the password and the mix-in string, fan-fold and XOR them
471 * to the required eight byte initial DES key. Since passwords can be
472 * expected to use mostly seven bit ASCII, left shift the password one
473 * bit in order to preserve as much key space as possible.
474 */
475 (void) memcpy(text, pw, plen);
476 (void) memcpy(&text[plen], mixin, len);
477
478 for (i = 0, j = 0; pw[j]; j++) {
479 tkey.c[i] ^= pw[j] << 1;
480 i += incr;
481 if (i == 8) {
482 i = 7;
483 incr = -incr;
484 } else if (i == -1) {
485 i = 0;
486 incr = -incr;
487 }
488 }
489
490 for (j = 0; j < len; j++) {
491 tkey.c[i] ^= mixin[j];
492 i += incr;
493 if (i == 8) {
494 i = 7;
495 incr = -incr;
496 } else if (i == -1) {
497 i = 0;
498 incr = -incr;
499 }
500 }
501 } else { /* use alternative algorithm */
502 (void) memcpy(text, mixin, len);
503 (void) memcpy(&text[len], pw, plen);
504
505 for (i = 0, j = 0; j < len; j++) {
506 tkey.c[i] ^= mixin[j];
507 i += incr;
508 if (i == 8) {
509 i = 7;
510 incr = -incr;
511 } else if (i == -1) {
512 i = 0;
513 incr = -incr;
514 }
515 }
516
517 for (j = 0; pw[j]; j++) {
518 tkey.c[i] ^= pw[j] << 1;
519 i += incr;
520 if (i == 8) {
521 i = 7;
522 incr = -incr;
523 } else if (i == -1) {
524 i = 0;
525 incr = -incr;
526 }
527 }
528 }
529 des_setparity_g(&tkey);
530
531 /*
532 * Use the temporary key to produce a DES CBC checksum for the text
533 * string; cbc_crypt returns the checksum in the ivec.
534 */
535 (void) cbc_crypt(tkey.c, text, tlen, DES_ENCRYPT|DES_HW, ivec.c);
536 des_setparity_g(&ivec);
537 free(text);
538
539 if (weak_DES_key(ivec)) {
540 ivec.c[7] ^= 0xf0;
541 /*
542 * XORing with 0xf0 preserves parity, so no need to check
543 * that again.
544 */
545 }
546
547 (void) memcpy((*key).c, ivec.c, sizeof (ivec.c));
548
549 return (1);
550
551 }
552
553 struct DESkey {
554 uint32_t h1;
555 uint32_t h2;
556 };
557
558 /*
559 * Weak and semiweak keys from "Applied Cryptography", second edition,
560 * by Bruce Schneier, Wiley 1996.
561 */
562 static struct DESkey weakDESkeys[] = {
563 /* Weak keys */
564 {0x01010101, 0x01010101},
565 {0x1f1f1f1f, 0x1f1f1f1f},
566 {0xe0e0e0e0, 0xe0e0e0e0},
567 {0xfefefefe, 0xfefefefe},
568 /* Semiweak keys */
569 {0x01fe01fe, 0x01fe01fe},
570 {0x1fe01fe0, 0x0ef10ef1},
571 {0x01e001e0, 0x01f101f1},
572 {0x1ffe1ffe, 0x0efe0efe},
573 {0x011f011f, 0x010e010e},
574 {0xe0fee0fe, 0xf1fef1fe},
575 {0xfe01fe01, 0xfe01fe01},
576 {0xe01fe01f, 0xf10ef10e},
577 {0xe001e001, 0xf101f101},
578 {0xfe1ffe1f, 0xfe0efe0e},
579 {0x1f011f01, 0x0e010e01},
580 {0xfee0fee0, 0xfef1fef1}
581 };
582
583 static int
584 weak_DES_key(des_block db)
585 {
586 int i;
587
588 for (i = 0; i < sizeof (weakDESkeys)/sizeof (struct DESkey); i++) {
589 if (weakDESkeys[i].h1 == db.key.high &&
590 weakDESkeys[i].h2 == db.key.low)
591 return (1);
592 }
593
594 return (0);
595 }