Index: openssl-ibmca-1.0.0-rc2/e_ibmca.c =================================================================== --- openssl-ibmca-1.0.0-rc2.orig/e_ibmca.c 2005-12-16 14:45:43.000000000 -0600 +++ openssl-ibmca-1.0.0-rc2/e_ibmca.c 2007-01-14 18:03:31.000000000 -0600 @@ -1582,133 +1582,92 @@ } // end ibmca_sha256_cleanup #endif // OPENSSL_NO_SHA256 -static int ibmca_mod_exp(BIGNUM * r, const BIGNUM * a, const BIGNUM * p, - const BIGNUM * m, BN_CTX * ctx) +static int ibmca_mod_exp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p, + const BIGNUM *m, BN_CTX *ctx) { - /* I need somewhere to store temporary serialised values for - * use with the Ibmca API calls. A neat cheat - I'll use - * BIGNUMs from the BN_CTX but access their arrays directly as - * byte arrays . This way I don't have to clean anything - * up. */ - - BIGNUM *argument = NULL; - BIGNUM *result = NULL; - BIGNUM *key = NULL; - int to_return; + char *argument; + char *result; + char *key; + ICA_KEY_RSA_MODEXPO *pubkey; int inLen, outLen, tmpLen; - - - - ICA_KEY_RSA_MODEXPO *publKey = NULL; unsigned int rc; - - to_return = 0; /* expect failure */ + int to_return = 0; /* fail code set by default */ if (!ibmca_dso) { IBMCAerr(IBMCA_F_IBMCA_MOD_EXP, IBMCA_R_NOT_LOADED); goto err; } - /* Prepare the params */ - BN_CTX_start(ctx); - argument = BN_CTX_get(ctx); - result = BN_CTX_get(ctx); - key = BN_CTX_get(ctx); - - if (!argument || !result || !key) { - IBMCAerr(IBMCA_F_IBMCA_MOD_EXP, IBMCA_R_BN_CTX_FULL); - goto err; - } - - - if (!bn_wexpand(argument, m->top) || !bn_wexpand(result, m->top) || - !bn_wexpand(key, sizeof(*publKey) / BN_BYTES)) { - IBMCAerr(IBMCA_F_IBMCA_MOD_EXP, IBMCA_R_BN_EXPAND_FAIL); - goto err; - } - - publKey = (ICA_KEY_RSA_MODEXPO *) key->d; - - if (publKey == NULL) { + outLen = BN_num_bytes(m); + argument = malloc(outLen); + if (!argument) { + IBMCAerr(IBMCA_F_IBMCA_MOD_EXP, IBMCA_R_REQUEST_FAILED); goto err; } - memset(publKey, 0, sizeof(ICA_KEY_RSA_MODEXPO)); - - publKey->keyType = CORRECT_ENDIANNESS(ME_KEY_TYPE); - publKey->keyLength = - CORRECT_ENDIANNESS(sizeof(ICA_KEY_RSA_MODEXPO)); - publKey->expOffset = - (char *) publKey->keyRecord - (char *) publKey; - - /* A quirk of the card: the exponent length has to be the same - as the modulus (key) length */ - - outLen = BN_num_bytes(m); - -/* check for modulus length SAB*/ - if (outLen > 256) { - IBMCAerr(IBMCA_F_IBMCA_MOD_EXP, - IBMCA_R_MEXP_LENGTH_TO_LARGE); + result = malloc(outLen); + if (!result) { + free(argument); + IBMCAerr(IBMCA_F_IBMCA_MOD_EXP, IBMCA_R_REQUEST_FAILED); goto err; } -/* check for modulus length SAB*/ - - - publKey->expLength = publKey->nLength = outLen; -/* SAB Check for underflow condition - the size of the exponent is less than the size of the parameter - then we have a big problem and will underflow the keyRecord - buffer. Bad stuff could happen then -*/ - if (outLen < BN_num_bytes(p)) { - IBMCAerr(IBMCA_F_IBMCA_MOD_EXP, - IBMCA_R_UNDERFLOW_KEYRECORD); + key = malloc(sizeof(*pubkey)); + if (!key) { + free(argument); + free(result); + IBMCAerr(IBMCA_F_IBMCA_MOD_EXP, IBMCA_R_REQUEST_FAILED); goto err; } -/* SAB End check for underflow */ - - - BN_bn2bin(p, &publKey->keyRecord[publKey->expLength - - BN_num_bytes(p)]); - BN_bn2bin(m, &publKey->keyRecord[publKey->expLength]); - - - - publKey->modulusBitLength = - CORRECT_ENDIANNESS(publKey->nLength * 8); - publKey->nOffset = - CORRECT_ENDIANNESS(publKey->expOffset + publKey->expLength); - - publKey->expOffset = - CORRECT_ENDIANNESS((char *) publKey->keyRecord - - (char *) publKey); - + pubkey = (ICA_KEY_RSA_MODEXPO *)key; + memset(pubkey, 0, sizeof(*pubkey)); + pubkey->keyType = CORRECT_ENDIANNESS(ME_KEY_TYPE); + pubkey->keyLength = CORRECT_ENDIANNESS(sizeof(ICA_KEY_RSA_MODEXPO)); + pubkey->expOffset = (char *)pubkey->keyRecord - (char *)pubkey; +#define IBMCA_MAX_EXP_LEN 256 + if (outLen > IBMCA_MAX_EXP_LEN) { + free(argument); + free(result); + free(key); + IBMCAerr(IBMCA_F_IBMCA_MOD_EXP, IBMCA_R_MEXP_LENGTH_TO_LARGE); + goto err; + } + pubkey->expLength = pubkey->nLength = outLen; + if (outLen < BN_num_bytes(p)) { /* Key record underflow check */ + free(argument); + free(result); + free(key); + IBMCAerr(IBMCA_F_IBMCA_MOD_EXP, IBMCA_R_UNDERFLOW_KEYRECORD); + goto err; + } + BN_bn2bin(p, + &pubkey->keyRecord[(pubkey->expLength - BN_num_bytes(p))]); + BN_bn2bin(m, &pubkey->keyRecord[pubkey->expLength]); + pubkey->modulusBitLength = CORRECT_ENDIANNESS((pubkey->nLength * 8)); + pubkey->nOffset = + CORRECT_ENDIANNESS((pubkey->expOffset + pubkey->expLength)); + pubkey->expOffset = CORRECT_ENDIANNESS(((char *)pubkey->keyRecord - + (char *)pubkey)); tmpLen = outLen; - publKey->expLength = publKey->nLength = CORRECT_ENDIANNESS(tmpLen); - - /* Prepare the argument */ - - memset(argument->d, 0, outLen); - BN_bn2bin(a, (unsigned char *) argument->d + outLen - - BN_num_bytes(a)); - + pubkey->expLength = pubkey->nLength = CORRECT_ENDIANNESS(tmpLen); + memset(argument, 0, outLen); + BN_bn2bin(a, ((unsigned char *)argument + + outLen + - BN_num_bytes(a))); inLen = outLen; - - /* Perform the operation */ - if ((rc = p_icaRsaModExpo(ibmca_handle, inLen, - (unsigned char *) argument->d, - publKey, &outLen, - (unsigned char *) result->d)) != 0) { + (unsigned char *)argument, + pubkey, &outLen, + (unsigned char *)result)) != 0) { + free(argument); + free(result); + free(key); IBMCAerr(IBMCA_F_IBMCA_MOD_EXP, IBMCA_R_REQUEST_FAILED); goto err; } - - - /* Convert the response */ - BN_bin2bn((unsigned char *) result->d, outLen, r); + BN_bin2bn((unsigned char *)result, outLen, r); to_return = 1; + free(argument); + free(result); + free(key); err: - BN_CTX_end(ctx); return to_return; }