1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  24  */
  25 
  26 /*
  27  * RSA provider for the Kernel Cryptographic Framework (KCF)
  28  */
  29 
  30 #include <sys/types.h>
  31 #include <sys/systm.h>
  32 #include <sys/modctl.h>
  33 #include <sys/cmn_err.h>
  34 #include <sys/ddi.h>
  35 #include <sys/crypto/spi.h>
  36 #include <sys/sysmacros.h>
  37 #include <sys/strsun.h>
  38 #include <sys/md5.h>
  39 #include <sys/sha1.h>
  40 #define _SHA2_IMPL
  41 #include <sys/sha2.h>
  42 #include <sys/random.h>
  43 #include <sys/crypto/impl.h>
  44 #include <sha1/sha1_impl.h>
  45 #include <sha2/sha2_impl.h>
  46 #include <padding/padding.h>
  47 #include <rsa/rsa_impl.h>
  48 
  49 extern struct mod_ops mod_cryptoops;
  50 
  51 /*
  52  * Module linkage information for the kernel.
  53  */
  54 static struct modlcrypto modlcrypto = {
  55         &mod_cryptoops,
  56         "RSA Kernel SW Provider"
  57 };
  58 
  59 static struct modlinkage modlinkage = {
  60         MODREV_1,
  61         (void *)&modlcrypto,
  62         NULL
  63 };
  64 
  65 /*
  66  * CSPI information (entry points, provider info, etc.)
  67  */
  68 typedef enum rsa_mech_type {
  69         RSA_PKCS_MECH_INFO_TYPE,        /* SUN_CKM_RSA_PKCS */
  70         RSA_X_509_MECH_INFO_TYPE,       /* SUN_CKM_RSA_X_509 */
  71         MD5_RSA_PKCS_MECH_INFO_TYPE,    /* SUN_MD5_RSA_PKCS */
  72         SHA1_RSA_PKCS_MECH_INFO_TYPE,   /* SUN_SHA1_RSA_PKCS */
  73         SHA256_RSA_PKCS_MECH_INFO_TYPE, /* SUN_SHA256_RSA_PKCS */
  74         SHA384_RSA_PKCS_MECH_INFO_TYPE, /* SUN_SHA384_RSA_PKCS */
  75         SHA512_RSA_PKCS_MECH_INFO_TYPE  /* SUN_SHA512_RSA_PKCS */
  76 } rsa_mech_type_t;
  77 
  78 /*
  79  * Context for RSA_PKCS and RSA_X_509 mechanisms.
  80  */
  81 typedef struct rsa_ctx {
  82         rsa_mech_type_t mech_type;
  83         crypto_key_t *key;
  84         size_t keychunk_size;
  85 } rsa_ctx_t;
  86 
  87 /*
  88  * Context for MD5_RSA_PKCS and SHA*_RSA_PKCS mechanisms.
  89  */
  90 typedef struct digest_rsa_ctx {
  91         rsa_mech_type_t mech_type;
  92         crypto_key_t *key;
  93         size_t keychunk_size;
  94         union {
  95                 MD5_CTX md5ctx;
  96                 SHA1_CTX sha1ctx;
  97                 SHA2_CTX sha2ctx;
  98         } dctx_u;
  99 } digest_rsa_ctx_t;
 100 
 101 #define md5_ctx         dctx_u.md5ctx
 102 #define sha1_ctx        dctx_u.sha1ctx
 103 #define sha2_ctx        dctx_u.sha2ctx
 104 
 105 /*
 106  * Mechanism info structure passed to KCF during registration.
 107  */
 108 static crypto_mech_info_t rsa_mech_info_tab[] = {
 109         /* RSA_PKCS */
 110         {SUN_CKM_RSA_PKCS, RSA_PKCS_MECH_INFO_TYPE,
 111             CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC |
 112             CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC |
 113             CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
 114             CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC |
 115             CRYPTO_FG_SIGN_RECOVER | CRYPTO_FG_SIGN_RECOVER_ATOMIC |
 116             CRYPTO_FG_VERIFY_RECOVER | CRYPTO_FG_VERIFY_RECOVER_ATOMIC,
 117             RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
 118 
 119         /* RSA_X_509 */
 120         {SUN_CKM_RSA_X_509, RSA_X_509_MECH_INFO_TYPE,
 121             CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC |
 122             CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC |
 123             CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
 124             CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC |
 125             CRYPTO_FG_SIGN_RECOVER | CRYPTO_FG_SIGN_RECOVER_ATOMIC |
 126             CRYPTO_FG_VERIFY_RECOVER | CRYPTO_FG_VERIFY_RECOVER_ATOMIC,
 127             RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
 128 
 129         /* MD5_RSA_PKCS */
 130         {SUN_CKM_MD5_RSA_PKCS, MD5_RSA_PKCS_MECH_INFO_TYPE,
 131             CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
 132             CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
 133             RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
 134 
 135         /* SHA1_RSA_PKCS */
 136         {SUN_CKM_SHA1_RSA_PKCS, SHA1_RSA_PKCS_MECH_INFO_TYPE,
 137             CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
 138             CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
 139             RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
 140 
 141         /* SHA256_RSA_PKCS */
 142         {SUN_CKM_SHA256_RSA_PKCS, SHA256_RSA_PKCS_MECH_INFO_TYPE,
 143             CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
 144             CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
 145             RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
 146 
 147         /* SHA384_RSA_PKCS */
 148         {SUN_CKM_SHA384_RSA_PKCS, SHA384_RSA_PKCS_MECH_INFO_TYPE,
 149             CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
 150             CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
 151             RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
 152 
 153         /* SHA512_RSA_PKCS */
 154         {SUN_CKM_SHA512_RSA_PKCS, SHA512_RSA_PKCS_MECH_INFO_TYPE,
 155             CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
 156             CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
 157             RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS}
 158 
 159 };
 160 
 161 #define RSA_VALID_MECH(mech)                                    \
 162         (((mech)->cm_type == RSA_PKCS_MECH_INFO_TYPE ||              \
 163         (mech)->cm_type == RSA_X_509_MECH_INFO_TYPE ||               \
 164         (mech)->cm_type == MD5_RSA_PKCS_MECH_INFO_TYPE ||    \
 165         (mech)->cm_type == SHA1_RSA_PKCS_MECH_INFO_TYPE ||   \
 166         (mech)->cm_type == SHA256_RSA_PKCS_MECH_INFO_TYPE || \
 167         (mech)->cm_type == SHA384_RSA_PKCS_MECH_INFO_TYPE || \
 168         (mech)->cm_type == SHA512_RSA_PKCS_MECH_INFO_TYPE) ? 1 : 0)
 169 
 170 /* operations are in-place if the output buffer is NULL */
 171 #define RSA_ARG_INPLACE(input, output)                          \
 172         if ((output) == NULL)                                   \
 173                 (output) = (input);
 174 
 175 static void rsa_provider_status(crypto_provider_handle_t, uint_t *);
 176 
 177 static crypto_control_ops_t rsa_control_ops = {
 178         rsa_provider_status
 179 };
 180 
 181 static int rsa_common_init(crypto_ctx_t *, crypto_mechanism_t *,
 182     crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
 183 static int rsaprov_encrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
 184     crypto_req_handle_t);
 185 static int rsa_encrypt_atomic(crypto_provider_handle_t, crypto_session_id_t,
 186     crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
 187     crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
 188 static int rsaprov_decrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
 189     crypto_req_handle_t);
 190 static int rsa_decrypt_atomic(crypto_provider_handle_t, crypto_session_id_t,
 191     crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
 192     crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
 193 
 194 /*
 195  * The RSA mechanisms do not have multiple-part cipher operations.
 196  * So, the update and final routines are set to NULL.
 197  */
 198 static crypto_cipher_ops_t rsa_cipher_ops = {
 199         rsa_common_init,
 200         rsaprov_encrypt,
 201         NULL,
 202         NULL,
 203         rsa_encrypt_atomic,
 204         rsa_common_init,
 205         rsaprov_decrypt,
 206         NULL,
 207         NULL,
 208         rsa_decrypt_atomic
 209 };
 210 
 211 static int rsa_sign_verify_common_init(crypto_ctx_t *, crypto_mechanism_t *,
 212     crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
 213 static int rsaprov_sign(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
 214     crypto_req_handle_t);
 215 static int rsa_sign_update(crypto_ctx_t *, crypto_data_t *,
 216     crypto_req_handle_t);
 217 static int rsa_sign_final(crypto_ctx_t *, crypto_data_t *,
 218     crypto_req_handle_t);
 219 static int rsa_sign_atomic(crypto_provider_handle_t, crypto_session_id_t,
 220     crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *,
 221     crypto_spi_ctx_template_t, crypto_req_handle_t);
 222 
 223 /*
 224  * We use the same routine for sign_init and sign_recover_init fields
 225  * as they do the same thing. Same holds for sign and sign_recover fields,
 226  * and sign_atomic and sign_recover_atomic fields.
 227  */
 228 static crypto_sign_ops_t rsa_sign_ops = {
 229         rsa_sign_verify_common_init,
 230         rsaprov_sign,
 231         rsa_sign_update,
 232         rsa_sign_final,
 233         rsa_sign_atomic,
 234         rsa_sign_verify_common_init,
 235         rsaprov_sign,
 236         rsa_sign_atomic
 237 };
 238 
 239 static int rsaprov_verify(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
 240     crypto_req_handle_t);
 241 static int rsa_verify_update(crypto_ctx_t *, crypto_data_t *,
 242     crypto_req_handle_t);
 243 static int rsa_verify_final(crypto_ctx_t *, crypto_data_t *,
 244     crypto_req_handle_t);
 245 static int rsa_verify_atomic(crypto_provider_handle_t, crypto_session_id_t,
 246     crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
 247     crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
 248 static int rsa_verify_recover(crypto_ctx_t *, crypto_data_t *,
 249     crypto_data_t *, crypto_req_handle_t);
 250 static int rsa_verify_recover_atomic(crypto_provider_handle_t,
 251     crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *,
 252     crypto_data_t *, crypto_data_t *, crypto_spi_ctx_template_t,
 253     crypto_req_handle_t);
 254 
 255 /*
 256  * We use the same routine (rsa_sign_verify_common_init) for verify_init
 257  * and verify_recover_init fields as they do the same thing.
 258  */
 259 static crypto_verify_ops_t rsa_verify_ops = {
 260         rsa_sign_verify_common_init,
 261         rsaprov_verify,
 262         rsa_verify_update,
 263         rsa_verify_final,
 264         rsa_verify_atomic,
 265         rsa_sign_verify_common_init,
 266         rsa_verify_recover,
 267         rsa_verify_recover_atomic
 268 };
 269 
 270 static int rsa_free_context(crypto_ctx_t *);
 271 
 272 static crypto_ctx_ops_t rsa_ctx_ops = {
 273         NULL,
 274         rsa_free_context
 275 };
 276 
 277 static crypto_ops_t rsa_crypto_ops = {
 278         &rsa_control_ops,
 279         NULL,
 280         &rsa_cipher_ops,
 281         NULL,
 282         &rsa_sign_ops,
 283         &rsa_verify_ops,
 284         NULL,
 285         NULL,
 286         NULL,
 287         NULL,
 288         NULL,
 289         NULL,
 290         NULL,
 291         &rsa_ctx_ops,
 292         NULL,
 293         NULL,
 294         NULL,
 295 };
 296 
 297 static crypto_provider_info_t rsa_prov_info = {
 298         CRYPTO_SPI_VERSION_4,
 299         "RSA Software Provider",
 300         CRYPTO_SW_PROVIDER,
 301         {&modlinkage},
 302         NULL,
 303         &rsa_crypto_ops,
 304         sizeof (rsa_mech_info_tab)/sizeof (crypto_mech_info_t),
 305         rsa_mech_info_tab
 306 };
 307 
 308 static int rsa_encrypt_common(rsa_mech_type_t, crypto_key_t *,
 309     crypto_data_t *, crypto_data_t *);
 310 static int rsa_decrypt_common(rsa_mech_type_t, crypto_key_t *,
 311     crypto_data_t *, crypto_data_t *);
 312 static int rsa_sign_common(rsa_mech_type_t, crypto_key_t *,
 313     crypto_data_t *, crypto_data_t *);
 314 static int rsa_verify_common(rsa_mech_type_t, crypto_key_t *,
 315     crypto_data_t *, crypto_data_t *);
 316 static int compare_data(crypto_data_t *, uchar_t *);
 317 
 318 static int core_rsa_encrypt(crypto_key_t *, uchar_t *, int, uchar_t *, int);
 319 static int core_rsa_decrypt(crypto_key_t *, uchar_t *, int, uchar_t *);
 320 
 321 static crypto_kcf_provider_handle_t rsa_prov_handle = NULL;
 322 
 323 int
 324 _init(void)
 325 {
 326         int ret;
 327 
 328         if ((ret = mod_install(&modlinkage)) != 0)
 329                 return (ret);
 330 
 331         /* Register with KCF.  If the registration fails, remove the module. */
 332         if (crypto_register_provider(&rsa_prov_info, &rsa_prov_handle)) {
 333                 (void) mod_remove(&modlinkage);
 334                 return (EACCES);
 335         }
 336 
 337         return (0);
 338 }
 339 
 340 int
 341 _fini(void)
 342 {
 343         /* Unregister from KCF if module is registered */
 344         if (rsa_prov_handle != NULL) {
 345                 if (crypto_unregister_provider(rsa_prov_handle))
 346                         return (EBUSY);
 347 
 348                 rsa_prov_handle = NULL;
 349         }
 350 
 351         return (mod_remove(&modlinkage));
 352 }
 353 
 354 int
 355 _info(struct modinfo *modinfop)
 356 {
 357         return (mod_info(&modlinkage, modinfop));
 358 }
 359 
 360 /* ARGSUSED */
 361 static void
 362 rsa_provider_status(crypto_provider_handle_t provider, uint_t *status)
 363 {
 364         *status = CRYPTO_PROVIDER_READY;
 365 }
 366 
 367 static int
 368 check_mech_and_key(crypto_mechanism_t *mechanism, crypto_key_t *key)
 369 {
 370         int rv = CRYPTO_FAILED;
 371 
 372         uchar_t *modulus;
 373         ssize_t modulus_len; /* In bytes */
 374 
 375         if (!RSA_VALID_MECH(mechanism))
 376                 return (CRYPTO_MECHANISM_INVALID);
 377 
 378         /*
 379          * We only support RSA keys that are passed as a list of
 380          * object attributes.
 381          */
 382         if (key->ck_format != CRYPTO_KEY_ATTR_LIST) {
 383                 return (CRYPTO_KEY_TYPE_INCONSISTENT);
 384         }
 385 
 386         if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
 387             &modulus_len)) != CRYPTO_SUCCESS) {
 388                 return (rv);
 389         }
 390         if (modulus_len < MIN_RSA_KEYLENGTH_IN_BYTES ||
 391             modulus_len > MAX_RSA_KEYLENGTH_IN_BYTES)
 392                 return (CRYPTO_KEY_SIZE_RANGE);
 393 
 394         return (rv);
 395 }
 396 
 397 void
 398 kmemset(uint8_t *buf, char pattern, size_t len)
 399 {
 400         int i = 0;
 401 
 402         while (i < len)
 403                 buf[i++] = pattern;
 404 }
 405 
 406 /*
 407  * This function guarantees to return non-zero random numbers.
 408  * This is needed as the /dev/urandom kernel interface,
 409  * random_get_pseudo_bytes(), may return zeros.
 410  */
 411 int
 412 knzero_random_generator(uint8_t *ran_out, size_t ran_len)
 413 {
 414         int rv;
 415         size_t ebc = 0; /* count of extra bytes in extrarand */
 416         size_t i = 0;
 417         uint8_t extrarand[32];
 418         size_t extrarand_len;
 419 
 420         if ((rv = random_get_pseudo_bytes(ran_out, ran_len)) != 0)
 421                 return (rv);
 422 
 423         /*
 424          * Walk through the returned random numbers pointed by ran_out,
 425          * and look for any random number which is zero.
 426          * If we find zero, call random_get_pseudo_bytes() to generate
 427          * another 32 random numbers pool. Replace any zeros in ran_out[]
 428          * from the random number in pool.
 429          */
 430         while (i < ran_len) {
 431                 if (ran_out[i] != 0) {
 432                         i++;
 433                         continue;
 434                 }
 435 
 436                 /*
 437                  * Note that it is 'while' so we are guaranteed a
 438                  * non-zero value on exit.
 439                  */
 440                 if (ebc == 0) {
 441                         /* refresh extrarand */
 442                         extrarand_len = sizeof (extrarand);
 443                         if ((rv = random_get_pseudo_bytes(extrarand,
 444                             extrarand_len)) != 0) {
 445                                 return (rv);
 446                         }
 447 
 448                         ebc = extrarand_len;
 449                 }
 450                 /* Replace zero with byte from extrarand. */
 451                 -- ebc;
 452 
 453                 /*
 454                  * The new random byte zero/non-zero will be checked in
 455                  * the next pass through the loop.
 456                  */
 457                 ran_out[i] = extrarand[ebc];
 458         }
 459 
 460         return (CRYPTO_SUCCESS);
 461 }
 462 
 463 static int
 464 compare_data(crypto_data_t *data, uchar_t *buf)
 465 {
 466         int len;
 467         uchar_t *dptr;
 468 
 469         len = data->cd_length;
 470         switch (data->cd_format) {
 471         case CRYPTO_DATA_RAW:
 472                 dptr = (uchar_t *)(data->cd_raw.iov_base +
 473                     data->cd_offset);
 474 
 475                 return (bcmp(dptr, buf, len));
 476 
 477         case CRYPTO_DATA_UIO:
 478                 return (crypto_uio_data(data, buf, len,
 479                     COMPARE_TO_DATA, NULL, NULL));
 480 
 481         case CRYPTO_DATA_MBLK:
 482                 return (crypto_mblk_data(data, buf, len,
 483                     COMPARE_TO_DATA, NULL, NULL));
 484         }
 485 
 486         return (CRYPTO_FAILED);
 487 }
 488 
 489 /* ARGSUSED */
 490 static int
 491 rsa_common_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
 492     crypto_key_t *key, crypto_spi_ctx_template_t template,
 493     crypto_req_handle_t req)
 494 {
 495         int rv;
 496         int kmflag;
 497         rsa_ctx_t *ctxp;
 498 
 499         if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
 500                 return (rv);
 501 
 502         /*
 503          * Allocate a RSA context.
 504          */
 505         kmflag = crypto_kmflag(req);
 506         if ((ctxp = kmem_zalloc(sizeof (rsa_ctx_t), kmflag)) == NULL)
 507                 return (CRYPTO_HOST_MEMORY);
 508 
 509         if ((rv = crypto_copy_key_to_ctx(key, &ctxp->key, &ctxp->keychunk_size,
 510             kmflag)) != CRYPTO_SUCCESS) {
 511                 kmem_free(ctxp, sizeof (rsa_ctx_t));
 512                 return (rv);
 513         }
 514         ctxp->mech_type = mechanism->cm_type;
 515 
 516         ctx->cc_provider_private = ctxp;
 517 
 518         return (CRYPTO_SUCCESS);
 519 }
 520 
 521 /* ARGSUSED */
 522 static int
 523 rsaprov_encrypt(crypto_ctx_t *ctx, crypto_data_t *plaintext,
 524     crypto_data_t *ciphertext, crypto_req_handle_t req)
 525 {
 526         int rv;
 527         rsa_ctx_t *ctxp;
 528 
 529         ASSERT(ctx->cc_provider_private != NULL);
 530         ctxp = ctx->cc_provider_private;
 531 
 532         RSA_ARG_INPLACE(plaintext, ciphertext);
 533 
 534         /*
 535          * Note on the KM_SLEEP flag passed to the routine below -
 536          * rsaprov_encrypt() is a single-part encryption routine which is
 537          * currently usable only by /dev/crypto. Since /dev/crypto calls are
 538          * always synchronous, we can safely pass KM_SLEEP here.
 539          */
 540         rv = rsa_encrypt_common(ctxp->mech_type, ctxp->key, plaintext,
 541             ciphertext);
 542 
 543         if (rv != CRYPTO_BUFFER_TOO_SMALL)
 544                 (void) rsa_free_context(ctx);
 545 
 546         return (rv);
 547 }
 548 
 549 /* ARGSUSED */
 550 static int
 551 rsa_encrypt_atomic(crypto_provider_handle_t provider,
 552     crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
 553     crypto_key_t *key, crypto_data_t *plaintext, crypto_data_t *ciphertext,
 554     crypto_spi_ctx_template_t template, crypto_req_handle_t req)
 555 {
 556         int rv;
 557 
 558         if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
 559                 return (rv);
 560         RSA_ARG_INPLACE(plaintext, ciphertext);
 561 
 562         return (rsa_encrypt_common(mechanism->cm_type, key, plaintext,
 563             ciphertext));
 564 }
 565 
 566 static int
 567 rsa_free_context(crypto_ctx_t *ctx)
 568 {
 569         rsa_ctx_t *ctxp = ctx->cc_provider_private;
 570 
 571         if (ctxp != NULL) {
 572                 bzero(ctxp->key, ctxp->keychunk_size);
 573                 kmem_free(ctxp->key, ctxp->keychunk_size);
 574 
 575                 if (ctxp->mech_type == RSA_PKCS_MECH_INFO_TYPE ||
 576                     ctxp->mech_type == RSA_X_509_MECH_INFO_TYPE)
 577                         kmem_free(ctxp, sizeof (rsa_ctx_t));
 578                 else
 579                         kmem_free(ctxp, sizeof (digest_rsa_ctx_t));
 580 
 581                 ctx->cc_provider_private = NULL;
 582         }
 583 
 584         return (CRYPTO_SUCCESS);
 585 }
 586 
 587 static int
 588 rsa_encrypt_common(rsa_mech_type_t mech_type, crypto_key_t *key,
 589     crypto_data_t *plaintext, crypto_data_t *ciphertext)
 590 {
 591         int rv = CRYPTO_FAILED;
 592 
 593         int plen;
 594         uchar_t *ptptr;
 595         uchar_t *modulus;
 596         ssize_t modulus_len;
 597         uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
 598         uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
 599         uchar_t cipher_data[MAX_RSA_KEYLENGTH_IN_BYTES];
 600 
 601         if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
 602             &modulus_len)) != CRYPTO_SUCCESS) {
 603                 return (rv);
 604         }
 605 
 606         plen = plaintext->cd_length;
 607         if (mech_type == RSA_PKCS_MECH_INFO_TYPE) {
 608                 if (plen > (modulus_len - MIN_PKCS1_PADLEN))
 609                         return (CRYPTO_DATA_LEN_RANGE);
 610         } else {
 611                 if (plen > modulus_len)
 612                         return (CRYPTO_DATA_LEN_RANGE);
 613         }
 614 
 615         /*
 616          * Output buf len must not be less than RSA modulus size.
 617          */
 618         if (ciphertext->cd_length < modulus_len) {
 619                 ciphertext->cd_length = modulus_len;
 620                 return (CRYPTO_BUFFER_TOO_SMALL);
 621         }
 622 
 623         ASSERT(plaintext->cd_length <= sizeof (tmp_data));
 624         if ((rv = crypto_get_input_data(plaintext, &ptptr, tmp_data))
 625             != CRYPTO_SUCCESS)
 626                 return (rv);
 627 
 628         if (mech_type == RSA_PKCS_MECH_INFO_TYPE) {
 629                 rv = pkcs1_encode(PKCS1_ENCRYPT, ptptr, plen,
 630                     plain_data, modulus_len);
 631 
 632                 if (rv != CRYPTO_SUCCESS)
 633                         return (rv);
 634         } else {
 635                 bzero(plain_data, modulus_len - plen);
 636                 bcopy(ptptr, &plain_data[modulus_len - plen], plen);
 637         }
 638 
 639         rv = core_rsa_encrypt(key, plain_data, modulus_len, cipher_data, 1);
 640         if (rv == CRYPTO_SUCCESS) {
 641                 /* copy out to ciphertext */
 642                 if ((rv = crypto_put_output_data(cipher_data,
 643                     ciphertext, modulus_len)) != CRYPTO_SUCCESS)
 644                         return (rv);
 645 
 646                 ciphertext->cd_length = modulus_len;
 647         }
 648 
 649         return (rv);
 650 }
 651 
 652 static int
 653 core_rsa_encrypt(crypto_key_t *key, uchar_t *in,
 654     int in_len, uchar_t *out, int is_public)
 655 {
 656         int rv;
 657         uchar_t *expo, *modulus;
 658         ssize_t expo_len;
 659         ssize_t modulus_len;
 660         RSAbytekey k;
 661 
 662         if (is_public) {
 663                 if ((rv = crypto_get_key_attr(key, SUN_CKA_PUBLIC_EXPONENT,
 664                     &expo, &expo_len)) != CRYPTO_SUCCESS)
 665                         return (rv);
 666         } else {
 667                 /*
 668                  * SUN_CKA_PRIVATE_EXPONENT is a required attribute for a
 669                  * RSA secret key. See the comments in core_rsa_decrypt
 670                  * routine which calls this routine with a private key.
 671                  */
 672                 if ((rv = crypto_get_key_attr(key, SUN_CKA_PRIVATE_EXPONENT,
 673                     &expo, &expo_len)) != CRYPTO_SUCCESS)
 674                         return (rv);
 675         }
 676 
 677         if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
 678             &modulus_len)) != CRYPTO_SUCCESS) {
 679                 return (rv);
 680         }
 681 
 682         k.modulus = modulus;
 683         k.modulus_bits = CRYPTO_BYTES2BITS(modulus_len);
 684         k.pubexpo = expo;
 685         k.pubexpo_bytes = expo_len;
 686         k.rfunc = NULL;
 687 
 688         rv = rsa_encrypt(&k, in, in_len, out);
 689 
 690         return (rv);
 691 }
 692 
 693 /* ARGSUSED */
 694 static int
 695 rsaprov_decrypt(crypto_ctx_t *ctx, crypto_data_t *ciphertext,
 696     crypto_data_t *plaintext, crypto_req_handle_t req)
 697 {
 698         int rv;
 699         rsa_ctx_t *ctxp;
 700 
 701         ASSERT(ctx->cc_provider_private != NULL);
 702         ctxp = ctx->cc_provider_private;
 703 
 704         RSA_ARG_INPLACE(ciphertext, plaintext);
 705 
 706         /* See the comments on KM_SLEEP flag in rsaprov_encrypt() */
 707         rv = rsa_decrypt_common(ctxp->mech_type, ctxp->key,
 708             ciphertext, plaintext);
 709 
 710         if (rv != CRYPTO_BUFFER_TOO_SMALL)
 711                 (void) rsa_free_context(ctx);
 712 
 713         return (rv);
 714 }
 715 
 716 /* ARGSUSED */
 717 static int
 718 rsa_decrypt_atomic(crypto_provider_handle_t provider,
 719     crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
 720     crypto_key_t *key, crypto_data_t *ciphertext, crypto_data_t *plaintext,
 721     crypto_spi_ctx_template_t template, crypto_req_handle_t req)
 722 {
 723         int rv;
 724 
 725         if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
 726                 return (rv);
 727         RSA_ARG_INPLACE(ciphertext, plaintext);
 728 
 729         return (rsa_decrypt_common(mechanism->cm_type, key, ciphertext,
 730             plaintext));
 731 }
 732 
 733 static int
 734 rsa_decrypt_common(rsa_mech_type_t mech_type, crypto_key_t *key,
 735     crypto_data_t *ciphertext, crypto_data_t *plaintext)
 736 {
 737         int rv = CRYPTO_FAILED;
 738 
 739         size_t plain_len;
 740         uchar_t *ctptr;
 741         uchar_t *modulus;
 742         ssize_t modulus_len;
 743         uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
 744         uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
 745 
 746         if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
 747             &modulus_len)) != CRYPTO_SUCCESS) {
 748                 return (rv);
 749         }
 750 
 751         /*
 752          * Ciphertext length must be equal to RSA modulus size.
 753          */
 754         if (ciphertext->cd_length != modulus_len)
 755                 return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE);
 756 
 757         ASSERT(ciphertext->cd_length <= sizeof (tmp_data));
 758         if ((rv = crypto_get_input_data(ciphertext, &ctptr, tmp_data))
 759             != CRYPTO_SUCCESS)
 760                 return (rv);
 761 
 762         rv = core_rsa_decrypt(key, ctptr, modulus_len, plain_data);
 763         if (rv == CRYPTO_SUCCESS) {
 764                 plain_len = modulus_len;
 765 
 766                 if (mech_type == RSA_PKCS_MECH_INFO_TYPE) {
 767                         /* Strip off the PKCS block formatting data. */
 768                         rv = pkcs1_decode(PKCS1_DECRYPT, plain_data,
 769                             &plain_len);
 770                         if (rv != CRYPTO_SUCCESS)
 771                                 return (rv);
 772                 }
 773 
 774                 if (plain_len > plaintext->cd_length) {
 775                         plaintext->cd_length = plain_len;
 776                         return (CRYPTO_BUFFER_TOO_SMALL);
 777                 }
 778 
 779                 if ((rv = crypto_put_output_data(
 780                     plain_data + modulus_len - plain_len,
 781                     plaintext, plain_len)) != CRYPTO_SUCCESS)
 782                         return (rv);
 783 
 784                 plaintext->cd_length = plain_len;
 785         }
 786 
 787         return (rv);
 788 }
 789 
 790 static int
 791 core_rsa_decrypt(crypto_key_t *key, uchar_t *in, int in_len, uchar_t *out)
 792 {
 793         int rv;
 794         uchar_t *modulus, *prime1, *prime2, *expo1, *expo2, *coef;
 795         ssize_t modulus_len;
 796         ssize_t prime1_len, prime2_len;
 797         ssize_t expo1_len, expo2_len, coef_len;
 798         RSAbytekey k;
 799 
 800         if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
 801             &modulus_len)) != CRYPTO_SUCCESS) {
 802                 return (rv);
 803         }
 804 
 805         /*
 806          * The following attributes are not required to be
 807          * present in a RSA secret key. If any of them is not present
 808          * we call the encrypt routine with a flag indicating use of
 809          * private exponent (d). Note that SUN_CKA_PRIVATE_EXPONENT is
 810          * a required attribute for a RSA secret key.
 811          */
 812         if ((crypto_get_key_attr(key, SUN_CKA_PRIME_1, &prime1, &prime1_len)
 813             != CRYPTO_SUCCESS) ||
 814             (crypto_get_key_attr(key, SUN_CKA_PRIME_2, &prime2, &prime2_len)
 815             != CRYPTO_SUCCESS) ||
 816             (crypto_get_key_attr(key, SUN_CKA_EXPONENT_1, &expo1, &expo1_len)
 817             != CRYPTO_SUCCESS) ||
 818             (crypto_get_key_attr(key, SUN_CKA_EXPONENT_2, &expo2, &expo2_len)
 819             != CRYPTO_SUCCESS) ||
 820             (crypto_get_key_attr(key, SUN_CKA_COEFFICIENT, &coef, &coef_len)
 821             != CRYPTO_SUCCESS)) {
 822                 return (core_rsa_encrypt(key, in, in_len, out, 0));
 823         }
 824 
 825         k.modulus = modulus;
 826         k.modulus_bits = CRYPTO_BYTES2BITS(modulus_len);
 827         k.prime1 = prime1;
 828         k.prime1_bytes = prime1_len;
 829         k.prime2 = prime2;
 830         k.prime2_bytes = prime2_len;
 831         k.expo1 = expo1;
 832         k.expo1_bytes = expo1_len;
 833         k.expo2 = expo2;
 834         k.expo2_bytes = expo2_len;
 835         k.coeff = coef;
 836         k.coeff_bytes = coef_len;
 837         k.rfunc = NULL;
 838 
 839         rv = rsa_decrypt(&k, in, in_len, out);
 840 
 841         return (rv);
 842 }
 843 
 844 /* ARGSUSED */
 845 static int
 846 rsa_sign_verify_common_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
 847     crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
 848     crypto_req_handle_t req)
 849 {
 850         int rv;
 851         int kmflag;
 852         rsa_ctx_t *ctxp;
 853         digest_rsa_ctx_t *dctxp;
 854 
 855         if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
 856                 return (rv);
 857 
 858         /*
 859          * Allocate a RSA context.
 860          */
 861         kmflag = crypto_kmflag(req);
 862         switch (mechanism->cm_type) {
 863         case MD5_RSA_PKCS_MECH_INFO_TYPE:
 864         case SHA1_RSA_PKCS_MECH_INFO_TYPE:
 865         case SHA256_RSA_PKCS_MECH_INFO_TYPE:
 866         case SHA384_RSA_PKCS_MECH_INFO_TYPE:
 867         case SHA512_RSA_PKCS_MECH_INFO_TYPE:
 868                 dctxp = kmem_zalloc(sizeof (digest_rsa_ctx_t), kmflag);
 869                 ctxp = (rsa_ctx_t *)dctxp;
 870                 break;
 871         default:
 872                 ctxp = kmem_zalloc(sizeof (rsa_ctx_t), kmflag);
 873                 break;
 874         }
 875 
 876         if (ctxp == NULL)
 877                 return (CRYPTO_HOST_MEMORY);
 878 
 879         ctxp->mech_type = mechanism->cm_type;
 880         if ((rv = crypto_copy_key_to_ctx(key, &ctxp->key, &ctxp->keychunk_size,
 881             kmflag)) != CRYPTO_SUCCESS) {
 882                 switch (mechanism->cm_type) {
 883                 case MD5_RSA_PKCS_MECH_INFO_TYPE:
 884                 case SHA1_RSA_PKCS_MECH_INFO_TYPE:
 885                 case SHA256_RSA_PKCS_MECH_INFO_TYPE:
 886                 case SHA384_RSA_PKCS_MECH_INFO_TYPE:
 887                 case SHA512_RSA_PKCS_MECH_INFO_TYPE:
 888                         kmem_free(dctxp, sizeof (digest_rsa_ctx_t));
 889                         break;
 890                 default:
 891                         kmem_free(ctxp, sizeof (rsa_ctx_t));
 892                         break;
 893                 }
 894                 return (rv);
 895         }
 896 
 897         switch (mechanism->cm_type) {
 898         case MD5_RSA_PKCS_MECH_INFO_TYPE:
 899                 MD5Init(&(dctxp->md5_ctx));
 900                 break;
 901 
 902         case SHA1_RSA_PKCS_MECH_INFO_TYPE:
 903                 SHA1Init(&(dctxp->sha1_ctx));
 904                 break;
 905 
 906         case SHA256_RSA_PKCS_MECH_INFO_TYPE:
 907                 SHA2Init(SHA256, &(dctxp->sha2_ctx));
 908                 break;
 909 
 910         case SHA384_RSA_PKCS_MECH_INFO_TYPE:
 911                 SHA2Init(SHA384, &(dctxp->sha2_ctx));
 912                 break;
 913 
 914         case SHA512_RSA_PKCS_MECH_INFO_TYPE:
 915                 SHA2Init(SHA512, &(dctxp->sha2_ctx));
 916                 break;
 917         }
 918 
 919         ctx->cc_provider_private = ctxp;
 920 
 921         return (CRYPTO_SUCCESS);
 922 }
 923 
 924 #define SHA1_DIGEST_SIZE 20
 925 #define MD5_DIGEST_SIZE 16
 926 
 927 #define INIT_RAW_CRYPTO_DATA(data, base, len, cd_len)   \
 928         (data).cd_format = CRYPTO_DATA_RAW;             \
 929         (data).cd_offset = 0;                           \
 930         (data).cd_raw.iov_base = (char *)base;          \
 931         (data).cd_raw.iov_len = len;                    \
 932         (data).cd_length = cd_len;
 933 
 934 static int
 935 rsa_digest_svrfy_common(digest_rsa_ctx_t *ctxp, crypto_data_t *data,
 936     crypto_data_t *signature, uchar_t flag)
 937 {
 938         int rv = CRYPTO_FAILED;
 939 
 940         uchar_t digest[SHA512_DIGEST_LENGTH];
 941         /* The der_data size is enough for MD5 also */
 942         uchar_t der_data[SHA512_DIGEST_LENGTH + SHA2_DER_PREFIX_Len];
 943         ulong_t der_data_len;
 944         crypto_data_t der_cd;
 945         rsa_mech_type_t mech_type;
 946 
 947         ASSERT(flag & CRYPTO_DO_SIGN || flag & CRYPTO_DO_VERIFY);
 948         ASSERT(data != NULL || (flag & CRYPTO_DO_FINAL));
 949 
 950         mech_type = ctxp->mech_type;
 951         if (mech_type == RSA_PKCS_MECH_INFO_TYPE ||
 952             mech_type == RSA_X_509_MECH_INFO_TYPE)
 953                 return (CRYPTO_MECHANISM_INVALID);
 954 
 955         /*
 956          * We need to do the BUFFER_TOO_SMALL check before digesting
 957          * the data. No check is needed for verify as signature is not
 958          * an output argument for verify.
 959          */
 960         if (flag & CRYPTO_DO_SIGN) {
 961                 uchar_t *modulus;
 962                 ssize_t modulus_len;
 963 
 964                 if ((rv = crypto_get_key_attr(ctxp->key, SUN_CKA_MODULUS,
 965                     &modulus, &modulus_len)) != CRYPTO_SUCCESS) {
 966                         return (rv);
 967                 }
 968 
 969                 if (signature->cd_length < modulus_len) {
 970                         signature->cd_length = modulus_len;
 971                         return (CRYPTO_BUFFER_TOO_SMALL);
 972                 }
 973         }
 974 
 975         if (mech_type == MD5_RSA_PKCS_MECH_INFO_TYPE)
 976                 rv = crypto_digest_data(data, &(ctxp->md5_ctx),
 977                     digest, MD5Update, MD5Final, flag | CRYPTO_DO_MD5);
 978 
 979         else if (mech_type == SHA1_RSA_PKCS_MECH_INFO_TYPE)
 980                 rv = crypto_digest_data(data, &(ctxp->sha1_ctx),
 981                     digest, SHA1Update, SHA1Final,  flag | CRYPTO_DO_SHA1);
 982 
 983         else
 984                 rv = crypto_digest_data(data, &(ctxp->sha2_ctx),
 985                     digest, SHA2Update, SHA2Final, flag | CRYPTO_DO_SHA2);
 986 
 987         if (rv != CRYPTO_SUCCESS)
 988                 return (rv);
 989 
 990 
 991         /*
 992          * Prepare the DER encoding of the DigestInfo value as follows:
 993          * MD5:         MD5_DER_PREFIX || H
 994          * SHA-1:       SHA1_DER_PREFIX || H
 995          *
 996          * See rsa_impl.c for more details.
 997          */
 998         switch (mech_type) {
 999         case MD5_RSA_PKCS_MECH_INFO_TYPE:
1000                 bcopy(MD5_DER_PREFIX, der_data, MD5_DER_PREFIX_Len);
1001                 bcopy(digest, der_data + MD5_DER_PREFIX_Len, MD5_DIGEST_SIZE);
1002                 der_data_len = MD5_DER_PREFIX_Len + MD5_DIGEST_SIZE;
1003                 break;
1004 
1005         case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1006                 bcopy(SHA1_DER_PREFIX, der_data, SHA1_DER_PREFIX_Len);
1007                 bcopy(digest, der_data + SHA1_DER_PREFIX_Len,
1008                     SHA1_DIGEST_SIZE);
1009                 der_data_len = SHA1_DER_PREFIX_Len + SHA1_DIGEST_SIZE;
1010                 break;
1011 
1012         case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1013                 bcopy(SHA256_DER_PREFIX, der_data, SHA2_DER_PREFIX_Len);
1014                 bcopy(digest, der_data + SHA2_DER_PREFIX_Len,
1015                     SHA256_DIGEST_LENGTH);
1016                 der_data_len = SHA2_DER_PREFIX_Len + SHA256_DIGEST_LENGTH;
1017                 break;
1018 
1019         case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1020                 bcopy(SHA384_DER_PREFIX, der_data, SHA2_DER_PREFIX_Len);
1021                 bcopy(digest, der_data + SHA2_DER_PREFIX_Len,
1022                     SHA384_DIGEST_LENGTH);
1023                 der_data_len = SHA2_DER_PREFIX_Len + SHA384_DIGEST_LENGTH;
1024                 break;
1025 
1026         case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1027                 bcopy(SHA512_DER_PREFIX, der_data, SHA2_DER_PREFIX_Len);
1028                 bcopy(digest, der_data + SHA2_DER_PREFIX_Len,
1029                     SHA512_DIGEST_LENGTH);
1030                 der_data_len = SHA2_DER_PREFIX_Len + SHA512_DIGEST_LENGTH;
1031                 break;
1032         }
1033 
1034         INIT_RAW_CRYPTO_DATA(der_cd, der_data, der_data_len, der_data_len);
1035         /*
1036          * Now, we are ready to sign or verify the DER_ENCODED data.
1037          */
1038         if (flag & CRYPTO_DO_SIGN)
1039                 rv = rsa_sign_common(mech_type, ctxp->key, &der_cd,
1040                     signature);
1041         else
1042                 rv = rsa_verify_common(mech_type, ctxp->key, &der_cd,
1043                     signature);
1044 
1045         return (rv);
1046 }
1047 
1048 static int
1049 rsa_sign_common(rsa_mech_type_t mech_type, crypto_key_t *key,
1050     crypto_data_t *data, crypto_data_t *signature)
1051 {
1052         int rv = CRYPTO_FAILED;
1053 
1054         int dlen;
1055         uchar_t *dataptr, *modulus;
1056         ssize_t modulus_len;
1057         uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1058         uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1059         uchar_t signed_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1060 
1061         if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
1062             &modulus_len)) != CRYPTO_SUCCESS) {
1063                 return (rv);
1064         }
1065 
1066         dlen = data->cd_length;
1067         switch (mech_type) {
1068         case RSA_PKCS_MECH_INFO_TYPE:
1069                 if (dlen > (modulus_len - MIN_PKCS1_PADLEN))
1070                         return (CRYPTO_DATA_LEN_RANGE);
1071                 break;
1072         case RSA_X_509_MECH_INFO_TYPE:
1073                 if (dlen > modulus_len)
1074                         return (CRYPTO_DATA_LEN_RANGE);
1075                 break;
1076         }
1077 
1078         if (signature->cd_length < modulus_len) {
1079                 signature->cd_length = modulus_len;
1080                 return (CRYPTO_BUFFER_TOO_SMALL);
1081         }
1082 
1083         ASSERT(data->cd_length <= sizeof (tmp_data));
1084         if ((rv = crypto_get_input_data(data, &dataptr, tmp_data))
1085             != CRYPTO_SUCCESS)
1086                 return (rv);
1087 
1088         switch (mech_type) {
1089         case RSA_PKCS_MECH_INFO_TYPE:
1090         case MD5_RSA_PKCS_MECH_INFO_TYPE:
1091         case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1092         case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1093         case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1094         case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1095                 /*
1096                  * Add PKCS padding to the input data to format a block
1097                  * type "01" encryption block.
1098                  */
1099                 rv = pkcs1_encode(PKCS1_SIGN, dataptr, dlen, plain_data,
1100                     modulus_len);
1101                 if (rv != CRYPTO_SUCCESS)
1102                         return (rv);
1103 
1104                 break;
1105 
1106         case RSA_X_509_MECH_INFO_TYPE:
1107                 bzero(plain_data, modulus_len - dlen);
1108                 bcopy(dataptr, &plain_data[modulus_len - dlen], dlen);
1109                 break;
1110         }
1111 
1112         rv = core_rsa_decrypt(key, plain_data, modulus_len, signed_data);
1113         if (rv == CRYPTO_SUCCESS) {
1114                 /* copy out to signature */
1115                 if ((rv = crypto_put_output_data(signed_data,
1116                     signature, modulus_len)) != CRYPTO_SUCCESS)
1117                         return (rv);
1118 
1119                 signature->cd_length = modulus_len;
1120         }
1121 
1122         return (rv);
1123 }
1124 
1125 /* ARGSUSED */
1126 static int
1127 rsaprov_sign(crypto_ctx_t *ctx, crypto_data_t *data, crypto_data_t *signature,
1128     crypto_req_handle_t req)
1129 {
1130         int rv;
1131         rsa_ctx_t *ctxp;
1132 
1133         ASSERT(ctx->cc_provider_private != NULL);
1134         ctxp = ctx->cc_provider_private;
1135 
1136         /* See the comments on KM_SLEEP flag in rsaprov_encrypt() */
1137         switch (ctxp->mech_type) {
1138         case MD5_RSA_PKCS_MECH_INFO_TYPE:
1139         case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1140         case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1141         case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1142         case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1143                 rv = rsa_digest_svrfy_common((digest_rsa_ctx_t *)ctxp, data,
1144                     signature, CRYPTO_DO_SIGN | CRYPTO_DO_UPDATE |
1145                     CRYPTO_DO_FINAL);
1146                 break;
1147         default:
1148                 rv = rsa_sign_common(ctxp->mech_type, ctxp->key, data,
1149                     signature);
1150                 break;
1151         }
1152 
1153         if (rv != CRYPTO_BUFFER_TOO_SMALL)
1154                 (void) rsa_free_context(ctx);
1155 
1156         return (rv);
1157 }
1158 
1159 /* ARGSUSED */
1160 static int
1161 rsa_sign_update(crypto_ctx_t *ctx, crypto_data_t *data, crypto_req_handle_t req)
1162 {
1163         int rv;
1164         digest_rsa_ctx_t *ctxp;
1165         rsa_mech_type_t mech_type;
1166 
1167         ASSERT(ctx->cc_provider_private != NULL);
1168         ctxp = ctx->cc_provider_private;
1169         mech_type = ctxp->mech_type;
1170 
1171         if (mech_type == RSA_PKCS_MECH_INFO_TYPE ||
1172             mech_type == RSA_X_509_MECH_INFO_TYPE)
1173                 return (CRYPTO_MECHANISM_INVALID);
1174 
1175         if (mech_type == MD5_RSA_PKCS_MECH_INFO_TYPE)
1176                 rv = crypto_digest_data(data, &(ctxp->md5_ctx),
1177                     NULL, MD5Update, MD5Final,
1178                     CRYPTO_DO_MD5 | CRYPTO_DO_UPDATE);
1179 
1180         else if (mech_type == SHA1_RSA_PKCS_MECH_INFO_TYPE)
1181                 rv = crypto_digest_data(data, &(ctxp->sha1_ctx),
1182                     NULL, SHA1Update, SHA1Final, CRYPTO_DO_SHA1 |
1183                     CRYPTO_DO_UPDATE);
1184 
1185         else
1186                 rv = crypto_digest_data(data, &(ctxp->sha2_ctx),
1187                     NULL, SHA2Update, SHA2Final, CRYPTO_DO_SHA2 |
1188                     CRYPTO_DO_UPDATE);
1189 
1190         return (rv);
1191 }
1192 
1193 /* ARGSUSED2 */
1194 static int
1195 rsa_sign_final(crypto_ctx_t *ctx, crypto_data_t *signature,
1196     crypto_req_handle_t req)
1197 {
1198         int rv;
1199         digest_rsa_ctx_t *ctxp;
1200 
1201         ASSERT(ctx->cc_provider_private != NULL);
1202         ctxp = ctx->cc_provider_private;
1203 
1204         rv = rsa_digest_svrfy_common(ctxp, NULL, signature,
1205             CRYPTO_DO_SIGN | CRYPTO_DO_FINAL);
1206         if (rv != CRYPTO_BUFFER_TOO_SMALL)
1207                 (void) rsa_free_context(ctx);
1208 
1209         return (rv);
1210 }
1211 
1212 /* ARGSUSED */
1213 static int
1214 rsa_sign_atomic(crypto_provider_handle_t provider,
1215     crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
1216     crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature,
1217     crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
1218 {
1219         int rv;
1220         digest_rsa_ctx_t dctx;
1221 
1222         if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
1223                 return (rv);
1224 
1225         if (mechanism->cm_type == RSA_PKCS_MECH_INFO_TYPE ||
1226             mechanism->cm_type == RSA_X_509_MECH_INFO_TYPE)
1227                 rv = rsa_sign_common(mechanism->cm_type, key, data,
1228                     signature);
1229 
1230         else {
1231                 dctx.mech_type = mechanism->cm_type;
1232                 dctx.key = key;
1233                 switch (mechanism->cm_type) {
1234                 case MD5_RSA_PKCS_MECH_INFO_TYPE:
1235                         MD5Init(&(dctx.md5_ctx));
1236                         break;
1237 
1238                 case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1239                         SHA1Init(&(dctx.sha1_ctx));
1240                         break;
1241 
1242                 case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1243                         SHA2Init(SHA256, &(dctx.sha2_ctx));
1244                         break;
1245 
1246                 case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1247                         SHA2Init(SHA384, &(dctx.sha2_ctx));
1248                         break;
1249 
1250                 case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1251                         SHA2Init(SHA512, &(dctx.sha2_ctx));
1252                         break;
1253                 }
1254 
1255                 rv = rsa_digest_svrfy_common(&dctx, data, signature,
1256                     CRYPTO_DO_SIGN | CRYPTO_DO_UPDATE | CRYPTO_DO_FINAL);
1257         }
1258 
1259         return (rv);
1260 }
1261 
1262 static int
1263 rsa_verify_common(rsa_mech_type_t mech_type, crypto_key_t *key,
1264     crypto_data_t *data, crypto_data_t *signature)
1265 {
1266         int rv = CRYPTO_FAILED;
1267 
1268         uchar_t *sigptr, *modulus;
1269         ssize_t modulus_len;
1270         uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1271         uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1272 
1273         if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
1274             &modulus_len)) != CRYPTO_SUCCESS) {
1275                 return (rv);
1276         }
1277 
1278         if (signature->cd_length != modulus_len)
1279                 return (CRYPTO_SIGNATURE_LEN_RANGE);
1280 
1281         ASSERT(signature->cd_length <= sizeof (tmp_data));
1282         if ((rv = crypto_get_input_data(signature, &sigptr, tmp_data))
1283             != CRYPTO_SUCCESS)
1284                 return (rv);
1285 
1286         rv = core_rsa_encrypt(key, sigptr, modulus_len, plain_data, 1);
1287         if (rv != CRYPTO_SUCCESS)
1288                 return (rv);
1289 
1290         if (mech_type == RSA_X_509_MECH_INFO_TYPE) {
1291                 if (compare_data(data, (plain_data + modulus_len
1292                     - data->cd_length)) != 0)
1293                         rv = CRYPTO_SIGNATURE_INVALID;
1294 
1295         } else {
1296                 size_t data_len = modulus_len;
1297 
1298                 /*
1299                  * Strip off the encoded padding bytes in front of the
1300                  * recovered data, then compare the recovered data with
1301                  * the original data.
1302                  */
1303                 rv = pkcs1_decode(PKCS1_VERIFY, plain_data, &data_len);
1304                 if (rv != CRYPTO_SUCCESS)
1305                         return (rv);
1306 
1307                 if (data_len != data->cd_length)
1308                         return (CRYPTO_SIGNATURE_LEN_RANGE);
1309 
1310                 if (compare_data(data, (plain_data + modulus_len
1311                     - data_len)) != 0)
1312                         rv = CRYPTO_SIGNATURE_INVALID;
1313         }
1314 
1315         return (rv);
1316 }
1317 
1318 /* ARGSUSED */
1319 static int
1320 rsaprov_verify(crypto_ctx_t *ctx, crypto_data_t *data,
1321     crypto_data_t *signature, crypto_req_handle_t req)
1322 {
1323         int rv;
1324         rsa_ctx_t *ctxp;
1325 
1326         ASSERT(ctx->cc_provider_private != NULL);
1327         ctxp = ctx->cc_provider_private;
1328 
1329         /* See the comments on KM_SLEEP flag in rsaprov_encrypt() */
1330         switch (ctxp->mech_type) {
1331         case MD5_RSA_PKCS_MECH_INFO_TYPE:
1332         case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1333         case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1334         case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1335         case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1336                 rv = rsa_digest_svrfy_common((digest_rsa_ctx_t *)ctxp, data,
1337                     signature, CRYPTO_DO_VERIFY | CRYPTO_DO_UPDATE |
1338                     CRYPTO_DO_FINAL);
1339                 break;
1340         default:
1341                 rv = rsa_verify_common(ctxp->mech_type, ctxp->key, data,
1342                     signature);
1343                 break;
1344         }
1345 
1346         if (rv != CRYPTO_BUFFER_TOO_SMALL)
1347                 (void) rsa_free_context(ctx);
1348 
1349         return (rv);
1350 }
1351 
1352 /* ARGSUSED */
1353 static int
1354 rsa_verify_update(crypto_ctx_t *ctx, crypto_data_t *data,
1355     crypto_req_handle_t req)
1356 {
1357         int rv;
1358         digest_rsa_ctx_t *ctxp;
1359 
1360         ASSERT(ctx->cc_provider_private != NULL);
1361         ctxp = ctx->cc_provider_private;
1362 
1363         switch (ctxp->mech_type) {
1364 
1365         case MD5_RSA_PKCS_MECH_INFO_TYPE:
1366                 rv = crypto_digest_data(data, &(ctxp->md5_ctx),
1367                     NULL, MD5Update, MD5Final, CRYPTO_DO_MD5 |
1368                     CRYPTO_DO_UPDATE);
1369                 break;
1370 
1371         case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1372                 rv = crypto_digest_data(data, &(ctxp->sha1_ctx),
1373                     NULL, SHA1Update, SHA1Final, CRYPTO_DO_SHA1 |
1374                     CRYPTO_DO_UPDATE);
1375                 break;
1376 
1377         case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1378         case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1379         case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1380                 rv = crypto_digest_data(data, &(ctxp->sha2_ctx),
1381                     NULL, SHA2Update, SHA2Final, CRYPTO_DO_SHA2 |
1382                     CRYPTO_DO_UPDATE);
1383                 break;
1384 
1385         default:
1386                 return (CRYPTO_MECHANISM_INVALID);
1387         }
1388 
1389         return (rv);
1390 }
1391 
1392 /* ARGSUSED2 */
1393 static int
1394 rsa_verify_final(crypto_ctx_t *ctx, crypto_data_t *signature,
1395     crypto_req_handle_t req)
1396 {
1397         int rv;
1398         digest_rsa_ctx_t *ctxp;
1399 
1400         ASSERT(ctx->cc_provider_private != NULL);
1401         ctxp = ctx->cc_provider_private;
1402 
1403         rv = rsa_digest_svrfy_common(ctxp, NULL, signature,
1404             CRYPTO_DO_VERIFY | CRYPTO_DO_FINAL);
1405         if (rv != CRYPTO_BUFFER_TOO_SMALL)
1406                 (void) rsa_free_context(ctx);
1407 
1408         return (rv);
1409 }
1410 
1411 
1412 /* ARGSUSED */
1413 static int
1414 rsa_verify_atomic(crypto_provider_handle_t provider,
1415     crypto_session_id_t session_id,
1416     crypto_mechanism_t *mechanism, crypto_key_t *key, crypto_data_t *data,
1417     crypto_data_t *signature, crypto_spi_ctx_template_t ctx_template,
1418     crypto_req_handle_t req)
1419 {
1420         int rv;
1421         digest_rsa_ctx_t dctx;
1422 
1423         if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
1424                 return (rv);
1425 
1426         if (mechanism->cm_type == RSA_PKCS_MECH_INFO_TYPE ||
1427             mechanism->cm_type == RSA_X_509_MECH_INFO_TYPE)
1428                 rv = rsa_verify_common(mechanism->cm_type, key, data,
1429                     signature);
1430 
1431         else {
1432                 dctx.mech_type = mechanism->cm_type;
1433                 dctx.key = key;
1434 
1435                 switch (mechanism->cm_type) {
1436                 case MD5_RSA_PKCS_MECH_INFO_TYPE:
1437                         MD5Init(&(dctx.md5_ctx));
1438                         break;
1439 
1440                 case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1441                         SHA1Init(&(dctx.sha1_ctx));
1442                         break;
1443 
1444                 case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1445                         SHA2Init(SHA256, &(dctx.sha2_ctx));
1446                         break;
1447 
1448                 case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1449                         SHA2Init(SHA384, &(dctx.sha2_ctx));
1450                         break;
1451 
1452                 case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1453                         SHA2Init(SHA512, &(dctx.sha2_ctx));
1454                         break;
1455                 }
1456 
1457                 rv = rsa_digest_svrfy_common(&dctx, data, signature,
1458                     CRYPTO_DO_VERIFY | CRYPTO_DO_UPDATE | CRYPTO_DO_FINAL);
1459         }
1460 
1461         return (rv);
1462 }
1463 
1464 static int
1465 rsa_verify_recover_common(rsa_mech_type_t mech_type, crypto_key_t *key,
1466     crypto_data_t *signature, crypto_data_t *data)
1467 {
1468         int rv = CRYPTO_FAILED;
1469 
1470         size_t data_len;
1471         uchar_t *sigptr, *modulus;
1472         ssize_t modulus_len;
1473         uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1474         uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1475 
1476         if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
1477             &modulus_len)) != CRYPTO_SUCCESS) {
1478                 return (rv);
1479         }
1480 
1481         if (signature->cd_length != modulus_len)
1482                 return (CRYPTO_SIGNATURE_LEN_RANGE);
1483 
1484         ASSERT(signature->cd_length <= sizeof (tmp_data));
1485         if ((rv = crypto_get_input_data(signature, &sigptr, tmp_data))
1486             != CRYPTO_SUCCESS)
1487                 return (rv);
1488 
1489         rv = core_rsa_encrypt(key, sigptr, modulus_len, plain_data, 1);
1490         if (rv != CRYPTO_SUCCESS)
1491                 return (rv);
1492 
1493         data_len = modulus_len;
1494 
1495         if (mech_type == RSA_PKCS_MECH_INFO_TYPE) {
1496                 /*
1497                  * Strip off the encoded padding bytes in front of the
1498                  * recovered data, then compare the recovered data with
1499                  * the original data.
1500                  */
1501                 rv = pkcs1_decode(PKCS1_VERIFY, plain_data, &data_len);
1502                 if (rv != CRYPTO_SUCCESS)
1503                         return (rv);
1504         }
1505 
1506         if (data->cd_length < data_len) {
1507                 data->cd_length = data_len;
1508                 return (CRYPTO_BUFFER_TOO_SMALL);
1509         }
1510 
1511         if ((rv = crypto_put_output_data(plain_data + modulus_len - data_len,
1512             data, data_len)) != CRYPTO_SUCCESS)
1513                 return (rv);
1514         data->cd_length = data_len;
1515 
1516         return (rv);
1517 }
1518 
1519 /* ARGSUSED */
1520 static int
1521 rsa_verify_recover(crypto_ctx_t *ctx, crypto_data_t *signature,
1522     crypto_data_t *data, crypto_req_handle_t req)
1523 {
1524         int rv;
1525         rsa_ctx_t *ctxp;
1526 
1527         ASSERT(ctx->cc_provider_private != NULL);
1528         ctxp = ctx->cc_provider_private;
1529 
1530         /* See the comments on KM_SLEEP flag in rsaprov_encrypt() */
1531         rv = rsa_verify_recover_common(ctxp->mech_type, ctxp->key,
1532             signature, data);
1533 
1534         if (rv != CRYPTO_BUFFER_TOO_SMALL)
1535                 (void) rsa_free_context(ctx);
1536 
1537         return (rv);
1538 }
1539 
1540 /* ARGSUSED */
1541 static int
1542 rsa_verify_recover_atomic(crypto_provider_handle_t provider,
1543     crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
1544     crypto_key_t *key, crypto_data_t *signature, crypto_data_t *data,
1545     crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
1546 {
1547         int rv;
1548 
1549         if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
1550                 return (rv);
1551 
1552         return (rsa_verify_recover_common(mechanism->cm_type, key,
1553             signature, data));
1554 }