/src/nss/lib/freebl/rsapkcs.c

Source
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

/*
 * RSA PKCS#1 v2.1 (RFC 3447) operations
 */

#ifdef FREEBL_NO_DEPEND
#include "stubs.h"
#endif

#include "secerr.h"

#include "blapi.h"
#include "secitem.h"
#include "blapii.h"
#include "secmpi.h"

#define RSA_BLOCK_MIN_PAD_LEN 8
#define RSA_BLOCK_FIRST_OCTET 0x00
#define RSA_BLOCK_PRIVATE_PAD_OCTET 0xff
#define RSA_BLOCK_AFTER_PAD_OCTET 0x00

/*
 * RSA block types
 *
 * The values of RSA_BlockPrivate and RSA_BlockPublic are fixed.
 * The value of RSA_BlockRaw isn't fixed by definition, but we are keeping
 * the value that NSS has been using in the past.
 */
typedef enum {
    RSA_BlockPrivate = 1, /* pad for a private-key operation */
    RSA_BlockPublic = 2,  /* pad for a public-key operation */
    RSA_BlockRaw = 4      /* simply justify the block appropriately */
} RSA_BlockType;

/* Needed for RSA-PSS functions */
static const unsigned char eightZeros[] = { 0, 0, 0, 0, 0, 0, 0, 0 };

/* Constant time comparison of a single byte.
 * Returns 1 iff a == b, otherwise returns 0.
 * Note: For ranges of bytes, use constantTimeCompare.
 */
static unsigned char
constantTimeEQ8(unsigned char a, unsigned char b)
{
    unsigned char c = ~((a - b) | (b - a));
    c >>= 7;
    return c;
}

/* Constant time comparison of a range of bytes.
 * Returns 1 iff len bytes of a are identical to len bytes of b, otherwise
 * returns 0.
 */
static unsigned char
constantTimeCompare(const unsigned char *a,
                    const unsigned char *b,
                    unsigned int len)
{
    unsigned char tmp = 0;
    unsigned int i;
    for (i = 0; i < len; ++i, ++a, ++b)
        tmp |= *a ^ *b;
    return constantTimeEQ8(0x00, tmp);
}

/* Constant time conditional.
 * Returns a if c is 1, or b if c is 0. The result is undefined if c is
 * not 0 or 1.
 */
static unsigned int
constantTimeCondition(unsigned int c,
                      unsigned int a,
                      unsigned int b)
{
    return (~(c - 1) & a) | ((c - 1) & b);
}

static unsigned int
rsa_modulusLen(SECItem *modulus)
{
    if (modulus->len == 0) {
        return 0;
    }

    unsigned char byteZero = modulus->data[0];
    unsigned int modLen = modulus->len - !byteZero;
    return modLen;
}

static unsigned int
rsa_modulusBits(SECItem *modulus)
{
    if (modulus->len == 0) {
        return 0;
    }

    unsigned char byteZero = modulus->data[0];
    unsigned int numBits = (modulus->len - 1) * 8;

    if (byteZero == 0 && modulus->len == 1) {
        return 0;
    }

    if (byteZero == 0) {
        numBits -= 8;
        byteZero = modulus->data[1];
    }

    while (byteZero > 0) {
        numBits++;
        byteZero >>= 1;
    }

    return numBits;
}

/*
 * Format one block of data for public/private key encryption using
 * the rules defined in PKCS #1.
 */
static unsigned char *
rsa_FormatOneBlock(unsigned modulusLen,
                   RSA_BlockType blockType,
                   SECItem *data)
{
    unsigned char *block;
    unsigned char *bp;
    unsigned int padLen;
    unsigned int i, j;
    SECStatus rv;

    block = (unsigned char *)PORT_Alloc(modulusLen);
    if (block == NULL)
        return NULL;

    bp = block;

    /*
     * All RSA blocks start with two octets:
     *  0x00 || BlockType
     */
    *bp++ = RSA_BLOCK_FIRST_OCTET;
    *bp++ = (unsigned char)blockType;

    switch (blockType) {

        /*
         * Blocks intended for private-key operation.
         */
        case RSA_BlockPrivate: /* preferred method */
            /*
             * 0x00 || BT || Pad || 0x00 || ActualData
             *   1      1   padLen    1      data->len
             * padLen must be at least RSA_BLOCK_MIN_PAD_LEN (8) bytes.
             * Pad is either all 0x00 or all 0xff bytes, depending on blockType.
             */
            padLen = modulusLen - data->len - 3;
            PORT_Assert(padLen >= RSA_BLOCK_MIN_PAD_LEN);
            if (padLen < RSA_BLOCK_MIN_PAD_LEN) {
                PORT_ZFree(block, modulusLen);
                return NULL;
            }
            PORT_Memset(bp, RSA_BLOCK_PRIVATE_PAD_OCTET, padLen);
            bp += padLen;
            *bp++ = RSA_BLOCK_AFTER_PAD_OCTET;
            PORT_Memcpy(bp, data->data, data->len);
            break;

        /*
         * Blocks intended for public-key operation.
         */
        case RSA_BlockPublic:
            /*
             * 0x00 || BT || Pad || 0x00 || ActualData
             *   1      1   padLen    1      data->len
             * Pad is 8 or more non-zero random bytes.
             *
             * Build the block left to right.
             * Fill the entire block from Pad to the end with random bytes.
             * Use the bytes after Pad as a supply of extra random bytes from
             * which to find replacements for the zero bytes in Pad.
             * If we need more than that, refill the bytes after Pad with
             * new random bytes as necessary.
             */

            padLen = modulusLen - (data->len + 3);
            PORT_Assert(padLen >= RSA_BLOCK_MIN_PAD_LEN);
            if (padLen < RSA_BLOCK_MIN_PAD_LEN) {
                PORT_ZFree(block, modulusLen);
                return NULL;
            }
            j = modulusLen - 2;
            rv = RNG_GenerateGlobalRandomBytes(bp, j);
            if (rv == SECSuccess) {
                for (i = 0; i < padLen;) {
                    unsigned char repl;
                    /* Pad with non-zero random data. */
                    if (bp[i] != RSA_BLOCK_AFTER_PAD_OCTET) {
                        ++i;
                        continue;
                    }
                    if (j <= padLen) {
                        rv = RNG_GenerateGlobalRandomBytes(bp + padLen,
                                                           modulusLen - (2 + padLen));
                        if (rv != SECSuccess)
                            break;
                        j = modulusLen - 2;
                    }
                    do {
                        repl = bp[--j];
                    } while (repl == RSA_BLOCK_AFTER_PAD_OCTET && j > padLen);
                    if (repl != RSA_BLOCK_AFTER_PAD_OCTET) {
                        bp[i++] = repl;
                    }
                }
            }
            if (rv != SECSuccess) {
                PORT_ZFree(block, modulusLen);
                PORT_SetError(SEC_ERROR_LIBRARY_FAILURE);
                return NULL;
            }
            bp += padLen;
            *bp++ = RSA_BLOCK_AFTER_PAD_OCTET;
            PORT_Memcpy(bp, data->data, data->len);
            break;

        default:
            PORT_Assert(0);
            PORT_ZFree(block, modulusLen);
            return NULL;
    }

    return block;
}

/* modulusLen has to be larger than RSA_BLOCK_MIN_PAD_LEN + 3, and data has to be smaller than modulus - (RSA_BLOCK_MIN_PAD_LEN + 3) */
static SECStatus
rsa_FormatBlock(SECItem *result,
                unsigned modulusLen,
                RSA_BlockType blockType,
                SECItem *data)
{
    switch (blockType) {
        case RSA_BlockPrivate:
        case RSA_BlockPublic:
            /*
             * 0x00 || BT || Pad || 0x00 || ActualData
             *
             * The "3" below is the first octet + the second octet + the 0x00
             * octet that always comes just before the ActualData.
             */
            if (modulusLen < (3 + RSA_BLOCK_MIN_PAD_LEN) || data->len > (modulusLen - (3 + RSA_BLOCK_MIN_PAD_LEN))) {
                return SECFailure;
            }
            result->data = rsa_FormatOneBlock(modulusLen, blockType, data);
            if (result->data == NULL) {
                result->len = 0;
                return SECFailure;
            }
            result->len = modulusLen;

            break;

        case RSA_BlockRaw:
            /*
             * Pad || ActualData
             * Pad is zeros. The application is responsible for recovering
             * the actual data.
             */
            if (data->len > modulusLen) {
                return SECFailure;
            }
            result->data = (unsigned char *)PORT_ZAlloc(modulusLen);
            result->len = modulusLen;
            PORT_Memcpy(result->data + (modulusLen - data->len),
                        data->data, data->len);
            break;

        default:
            PORT_Assert(0);
            result->data = NULL;
            result->len = 0;
            return SECFailure;
    }

    return SECSuccess;
}

/*
 * Mask generation function MGF1 as defined in PKCS #1 v2.1 / RFC 3447.
 */
static SECStatus
MGF1(HASH_HashType hashAlg,
     unsigned char *mask,
     unsigned int maskLen,
     const unsigned char *mgfSeed,
     unsigned int mgfSeedLen)
{
    unsigned int digestLen;
    PRUint32 counter;
    PRUint32 rounds;
    unsigned char *tempHash;
    unsigned char *temp;
    const SECHashObject *hash;
    void *hashContext;
    unsigned char C[4];
    SECStatus rv = SECSuccess;

    hash = HASH_GetRawHashObject(hashAlg);
    if (hash == NULL) {
        return SECFailure;
    }

    hashContext = (*hash->create)();
    rounds = (maskLen + hash->length - 1) / hash->length;
    for (counter = 0; counter < rounds; counter++) {
        C[0] = (unsigned char)((counter >> 24) & 0xff);
        C[1] = (unsigned char)((counter >> 16) & 0xff);
        C[2] = (unsigned char)((counter >> 8) & 0xff);
        C[3] = (unsigned char)(counter & 0xff);

        /* This could be optimized when the clone functions in
         * rawhash.c are implemented. */
        (*hash->begin)(hashContext);
        (*hash->update)(hashContext, mgfSeed, mgfSeedLen);
        (*hash->update)(hashContext, C, sizeof C);

        tempHash = mask + counter * hash->length;
        if (counter != (rounds - 1)) {
            (*hash->end)(hashContext, tempHash, &digestLen, hash->length);
        } else { /* we're in the last round and need to cut the hash */
            temp = (unsigned char *)PORT_Alloc(hash->length);
            if (!temp) {
                rv = SECFailure;
                goto done;
            }
            (*hash->end)(hashContext, temp, &digestLen, hash->length);
            PORT_Memcpy(tempHash, temp, maskLen - counter * hash->length);
            PORT_Free(temp);
        }
    }

done:
    (*hash->destroy)(hashContext, PR_TRUE);
    return rv;
}

/* XXX Doesn't set error code */
SECStatus
RSA_SignRaw(RSAPrivateKey *key,
            unsigned char *output,
            unsigned int *outputLen,
            unsigned int maxOutputLen,
            const unsigned char *data,
            unsigned int dataLen)
{
    SECStatus rv = SECSuccess;
    unsigned int modulusLen = rsa_modulusLen(&key->modulus);
    SECItem formatted;
    SECItem unformatted;

    if (maxOutputLen < modulusLen)
        return SECFailure;

    unformatted.len = dataLen;
    unformatted.data = (unsigned char *)data;
    formatted.data = NULL;
    rv = rsa_FormatBlock(&formatted, modulusLen, RSA_BlockRaw, &unformatted);
    if (rv != SECSuccess)
        goto done;

    rv = RSA_PrivateKeyOpDoubleChecked(key, output, formatted.data);
    *outputLen = modulusLen;

done:
    if (formatted.data != NULL)
        PORT_ZFree(formatted.data, modulusLen);
    return rv;
}

/* XXX Doesn't set error code */
SECStatus
RSA_CheckSignRaw(RSAPublicKey *key,
                 const unsigned char *sig,
                 unsigned int sigLen,
                 const unsigned char *hash,
                 unsigned int hashLen)
{
    SECStatus rv;
    unsigned int modulusLen = rsa_modulusLen(&key->modulus);
    unsigned char *buffer;

    if (sigLen != modulusLen)
        goto failure;
    if (hashLen > modulusLen)
        goto failure;

    buffer = (unsigned char *)PORT_Alloc(modulusLen + 1);
    if (!buffer)
        goto failure;

    rv = RSA_PublicKeyOp(key, buffer, sig);
    if (rv != SECSuccess)
        goto loser;

    /*
     * make sure we get the same results
     */
    /* XXX(rsleevi): Constant time */
    /* NOTE: should we verify the leading zeros? */
    if (PORT_Memcmp(buffer + (modulusLen - hashLen), hash, hashLen) != 0)
        goto loser;

    PORT_Free(buffer);
    return SECSuccess;

loser:
    PORT_Free(buffer);
failure:
    return SECFailure;
}

/* XXX Doesn't set error code */
SECStatus
RSA_CheckSignRecoverRaw(RSAPublicKey *key,
                        unsigned char *data,
                        unsigned int *dataLen,
                        unsigned int maxDataLen,
                        const unsigned char *sig,
                        unsigned int sigLen)
{
    SECStatus rv;
    unsigned int modulusLen = rsa_modulusLen(&key->modulus);

    if (sigLen != modulusLen)
        goto failure;
    if (maxDataLen < modulusLen)
        goto failure;

    rv = RSA_PublicKeyOp(key, data, sig);
    if (rv != SECSuccess)
        goto failure;

    *dataLen = modulusLen;
    return SECSuccess;

failure:
    return SECFailure;
}

/* XXX Doesn't set error code */
SECStatus
RSA_EncryptRaw(RSAPublicKey *key,
               unsigned char *output,
               unsigned int *outputLen,
               unsigned int maxOutputLen,
               const unsigned char *input,
               unsigned int inputLen)
{
    SECStatus rv;
    unsigned int modulusLen = rsa_modulusLen(&key->modulus);
    SECItem formatted;
    SECItem unformatted;

    formatted.data = NULL;
    if (maxOutputLen < modulusLen)
        goto failure;

    unformatted.len = inputLen;
    unformatted.data = (unsigned char *)input;
    formatted.data = NULL;
    rv = rsa_FormatBlock(&formatted, modulusLen, RSA_BlockRaw, &unformatted);
    if (rv != SECSuccess)
        goto failure;

    rv = RSA_PublicKeyOp(key, output, formatted.data);
    if (rv != SECSuccess)
        goto failure;

    PORT_ZFree(formatted.data, modulusLen);
    *outputLen = modulusLen;
    return SECSuccess;

failure:
    if (formatted.data != NULL)
        PORT_ZFree(formatted.data, modulusLen);
    return SECFailure;
}

/* XXX Doesn't set error code */
SECStatus
RSA_DecryptRaw(RSAPrivateKey *key,
               unsigned char *output,
               unsigned int *outputLen,
               unsigned int maxOutputLen,
               const unsigned char *input,
               unsigned int inputLen)
{
    SECStatus rv;
    unsigned int modulusLen = rsa_modulusLen(&key->modulus);

    if (modulusLen > maxOutputLen)
        goto failure;
    if (inputLen != modulusLen)
        goto failure;

    rv = RSA_PrivateKeyOp(key, output, input);
    if (rv != SECSuccess)
        goto failure;

    *outputLen = modulusLen;
    return SECSuccess;

failure:
    return SECFailure;
}

/*
 * Decodes an EME-OAEP encoded block, validating the encoding in constant
 * time.
 * Described in RFC 3447, section 7.1.2.
 * input contains the encoded block, after decryption.
 * label is the optional value L that was associated with the message.
 * On success, the original message and message length will be stored in
 * output and outputLen.
 */
static SECStatus
eme_oaep_decode(unsigned char *output,
                unsigned int *outputLen,
                unsigned int maxOutputLen,
                const unsigned char *input,
                unsigned int inputLen,
                HASH_HashType hashAlg,
                HASH_HashType maskHashAlg,
                const unsigned char *label,
                unsigned int labelLen)
{
    const SECHashObject *hash;
    void *hashContext;
    SECStatus rv = SECFailure;
    unsigned char labelHash[HASH_LENGTH_MAX];
    unsigned int i;
    unsigned int maskLen;
    unsigned int paddingOffset;
    unsigned char *mask = NULL;
    unsigned char *tmpOutput = NULL;
    unsigned char isGood;
    unsigned char foundPaddingEnd;

    hash = HASH_GetRawHashObject(hashAlg);

    /* 1.c */
    if (inputLen < (hash->length * 2) + 2) {
        PORT_SetError(SEC_ERROR_INPUT_LEN);
        return SECFailure;
    }

    /* Step 3.a - Generate lHash */
    hashContext = (*hash->create)();
    if (hashContext == NULL) {
        PORT_SetError(SEC_ERROR_NO_MEMORY);
        return SECFailure;
    }
    (*hash->begin)(hashContext);
    if (labelLen > 0)
        (*hash->update)(hashContext, label, labelLen);
    (*hash->end)(hashContext, labelHash, &i, sizeof(labelHash));
    (*hash->destroy)(hashContext, PR_TRUE);

    tmpOutput = (unsigned char *)PORT_Alloc(inputLen);
    if (tmpOutput == NULL) {
        PORT_SetError(SEC_ERROR_NO_MEMORY);
        goto done;
    }

    maskLen = inputLen - hash->length - 1;
    mask = (unsigned char *)PORT_Alloc(maskLen);
    if (mask == NULL) {
        PORT_SetError(SEC_ERROR_NO_MEMORY);
        goto done;
    }

    PORT_Memcpy(tmpOutput, input, inputLen);

    /* 3.c - Generate seedMask */
    MGF1(maskHashAlg, mask, hash->length, &tmpOutput[1 + hash->length],
         inputLen - hash->length - 1);
    /* 3.d - Unmask seed */
    for (i = 0; i < hash->length; ++i)
        tmpOutput[1 + i] ^= mask[i];

    /* 3.e - Generate dbMask */
    MGF1(maskHashAlg, mask, maskLen, &tmpOutput[1], hash->length);
    /* 3.f - Unmask DB */
    for (i = 0; i < maskLen; ++i)
        tmpOutput[1 + hash->length + i] ^= mask[i];

    /* 3.g - Compare Y, lHash, and PS in constant time
     * Warning: This code is timing dependent and must not disclose which of
     * these were invalid.
     */
    paddingOffset = 0;
    isGood = 1;
    foundPaddingEnd = 0;

    /* Compare Y */
    isGood &= constantTimeEQ8(0x00, tmpOutput[0]);

    /* Compare lHash and lHash' */
    isGood &= constantTimeCompare(&labelHash[0],
                                  &tmpOutput[1 + hash->length],
                                  hash->length);

    /* Compare that the padding is zero or more zero octets, followed by a
     * 0x01 octet */
    for (i = 1 + (hash->length * 2); i < inputLen; ++i) {
        unsigned char isZero = constantTimeEQ8(0x00, tmpOutput[i]);
        unsigned char isOne = constantTimeEQ8(0x01, tmpOutput[i]);
        /* non-constant time equivalent:
         * if (tmpOutput[i] == 0x01 && !foundPaddingEnd)
         *     paddingOffset = i;
         */
        paddingOffset = constantTimeCondition(isOne & ~foundPaddingEnd, i,
                                              paddingOffset);
        /* non-constant time equivalent:
         * if (tmpOutput[i] == 0x01)
         *    foundPaddingEnd = true;
         *
         * Note: This may yield false positives, as it will be set whenever
         * a 0x01 byte is encountered. If there was bad padding (eg:
         * 0x03 0x02 0x01), foundPaddingEnd will still be set to true, and
         * paddingOffset will still be set to 2.
         */
        foundPaddingEnd = constantTimeCondition(isOne, 1, foundPaddingEnd);
        /* non-constant time equivalent:
         * if (tmpOutput[i] != 0x00 && tmpOutput[i] != 0x01 &&
         *     !foundPaddingEnd) {
         *    isGood = false;
         * }
         *
         * Note: This may yield false positives, as a message (and padding)
         * that is entirely zeros will result in isGood still being true. Thus
         * it's necessary to check foundPaddingEnd is positive below.
         */
        isGood = constantTimeCondition(~foundPaddingEnd & ~isZero, 0, isGood);
    }

    /* While both isGood and foundPaddingEnd may have false positives, they
     * cannot BOTH have false positives. If both are not true, then an invalid
     * message was received. Note, this comparison must still be done in constant
     * time so as not to leak either condition.
     */
    if (!(isGood & foundPaddingEnd)) {
        PORT_SetError(SEC_ERROR_BAD_DATA);
        goto done;
    }

    /* End timing dependent code */

    ++paddingOffset; /* Skip the 0x01 following the end of PS */

    *outputLen = inputLen - paddingOffset;
    if (*outputLen > maxOutputLen) {
        PORT_SetError(SEC_ERROR_OUTPUT_LEN);
        goto done;
    }

    if (*outputLen)
        PORT_Memcpy(output, &tmpOutput[paddingOffset], *outputLen);
    rv = SECSuccess;

done:
    if (mask)
        PORT_ZFree(mask, maskLen);
    if (tmpOutput)
        PORT_ZFree(tmpOutput, inputLen);
    return rv;
}

/*
 * Generate an EME-OAEP encoded block for encryption
 * Described in RFC 3447, section 7.1.1
 * We use input instead of M for the message to be encrypted
 * label is the optional value L to be associated with the message.
 */
static SECStatus
eme_oaep_encode(unsigned char *em,
                unsigned int emLen,
                const unsigned char *input,
                unsigned int inputLen,
                HASH_HashType hashAlg,
                HASH_HashType maskHashAlg,
                const unsigned char *label,
                unsigned int labelLen,
                const unsigned char *seed,
                unsigned int seedLen)
{
    const SECHashObject *hash;
    void *hashContext;
    SECStatus rv;
    unsigned char *mask;
    unsigned int reservedLen;
    unsigned int dbMaskLen;
    unsigned int i;

    hash = HASH_GetRawHashObject(hashAlg);
    PORT_Assert(seed == NULL || seedLen == hash->length);

    /* Step 1.b */
    reservedLen = (2 * hash->length) + 2;
    if (emLen < reservedLen || inputLen > (emLen - reservedLen)) {
        PORT_SetError(SEC_ERROR_INPUT_LEN);
        return SECFailure;
    }

    /*
     * From RFC 3447, Section 7.1
     *                      +----------+---------+-------+
     *                 DB = |  lHash   |    PS   |   M   |
     *                      +----------+---------+-------+
     *                                     |
     *           +----------+              V
     *           |   seed   |--> MGF ---> xor
     *           +----------+              |
     *                 |                   |
     *        +--+     V                   |
     *        |00|    xor <----- MGF <-----|
     *        +--+     |                   |
     *          |      |                   |
     *          V      V                   V
     *        +--+----------+----------------------------+
     *  EM =  |00|maskedSeed|          maskedDB          |
     *        +--+----------+----------------------------+
     *
     * We use mask to hold the result of the MGF functions, and all other
     * values are generated in their final resting place.
     */
    *em = 0x00;

    /* Step 2.a - Generate lHash */
    hashContext = (*hash->create)();
    if (hashContext == NULL) {
        PORT_SetError(SEC_ERROR_NO_MEMORY);
        return SECFailure;
    }
    (*hash->begin)(hashContext);
    if (labelLen > 0)
        (*hash->update)(hashContext, label, labelLen);
    (*hash->end)(hashContext, &em[1 + hash->length], &i, hash->length);
    (*hash->destroy)(hashContext, PR_TRUE);

    /* Step 2.b - Generate PS */
    if (emLen - reservedLen - inputLen > 0) {
        PORT_Memset(em + 1 + (hash->length * 2), 0x00,
                    emLen - reservedLen - inputLen);
    }

    /* Step 2.c. - Generate DB
     * DB = lHash || PS || 0x01 || M
     * Note that PS and lHash have already been placed into em at their
     * appropriate offsets. This just copies M into place
     */
    em[emLen - inputLen - 1] = 0x01;
    if (inputLen)
        PORT_Memcpy(em + emLen - inputLen, input, inputLen);

    if (seed == NULL) {
        /* Step 2.d - Generate seed */
        rv = RNG_GenerateGlobalRandomBytes(em + 1, hash->length);
        if (rv != SECSuccess) {
            return rv;
        }
    } else {
        /* For Known Answer Tests, copy the supplied seed. */
        PORT_Memcpy(em + 1, seed, seedLen);
    }

    /* Step 2.e - Generate dbMask*/
    dbMaskLen = emLen - hash->length - 1;
    mask = (unsigned char *)PORT_Alloc(dbMaskLen);
    if (mask == NULL) {
        PORT_SetError(SEC_ERROR_NO_MEMORY);
        return SECFailure;
    }
    MGF1(maskHashAlg, mask, dbMaskLen, em + 1, hash->length);
    /* Step 2.f - Compute maskedDB*/
    for (i = 0; i < dbMaskLen; ++i)
        em[1 + hash->length + i] ^= mask[i];

    /* Step 2.g - Generate seedMask */
    MGF1(maskHashAlg, mask, hash->length, &em[1 + hash->length], dbMaskLen);
    /* Step 2.h - Compute maskedSeed */
    for (i = 0; i < hash->length; ++i)
        em[1 + i] ^= mask[i];

    PORT_ZFree(mask, dbMaskLen);
    return SECSuccess;
}

SECStatus
RSA_PartialVerify(RSAPublicKey *key)
{
    mp_int n, e, fact;
    mp_err err;
    SECStatus rv = SECSuccess;
    mp_int small_primes_product;
    /* this string is the NIST string *751, since the requirement is to detect up to prime 751
     * inclusive */
    const char *primes_product_decimal_string = "1090367704589007566035202161494385722019383400119651874476478760664145697976533387343668263403266024900638505861980470638958322454581550203448421495842767449796261170069732921897400657944793163960886418313690808872680411646815934535605444102983629208422567461571568587963766123806881456648026733413053968363611105";

    /* we only need to verify public keys once, and only in FIPS mode */
    if (!key->needVerify) {
        return SECSuccess;
    }
    /* modulus length is check done by the FIPS indicator code */
    /* validate public exponent: this test only succeeds if the exponent e is
     * 2^16 <= e < 2^256. e = 2^16 will be rejected below byte the iseven check */
    if (!key->publicExponent.data[0] || (key->publicExponent.len < 3) || (key->publicExponent.len > 32)) {
        PORT_SetError(SEC_ERROR_INVALID_KEY);
        return SECFailure;
    }
    /* convert to mpi for more detailed tests */
    MP_DIGITS(&n) = 0;
    MP_DIGITS(&e) = 0;
    MP_DIGITS(&fact) = 0;
    MP_DIGITS(&small_primes_product) = 0;
    CHECK_MPI_OK(mp_init(&n));
    CHECK_MPI_OK(mp_init(&e));
    CHECK_MPI_OK(mp_init(&small_primes_product));
    CHECK_MPI_OK(mp_init(&fact));
    SECITEM_TO_MPINT(key->modulus, &n);
    SECITEM_TO_MPINT(key->publicExponent, &e);

    /* reject even exponents and moduluses */
    if (mp_iseven(&e) || mp_iseven(&n)) {
        err = MP_BADARG;
        goto cleanup;
    }
    /* check for componsite or power */
    err = mpp_pprime_or_power_secure(&n, NULL, 4);
    if (err != MP_NOT_POWER) {
        /* prime check succeeded, therefore modulus is not composite */
        /* or we found a factor, indicating that the modulus is probably
         * a power of a prime */
        err = MP_BADARG;
        goto cleanup;
    }
    /* check for small factors using gcd */
    CHECK_MPI_OK(mp_read_radix(&small_primes_product,
                               primes_product_decimal_string, 10));
    CHECK_MPI_OK(mp_gcd(&n, &small_primes_product, &fact));
    if (mp_cmp_d(&fact, 1) != 0) {
        /* factor found, not a good modulus */
        err = MP_BADARG;
        goto cleanup;
    }

cleanup:
    mp_clear(&n);
    mp_clear(&e);
    mp_clear(&small_primes_product);
    mp_clear(&fact);
    if (err) {
        PORT_SetError(SEC_ERROR_INVALID_KEY);
        rv = SECFailure;
    } else {
        /* if we are called again with this key, no need to verify
         * again */
        key->needVerify = PR_FALSE;
    }
    return rv;
}

SECStatus
RSA_EncryptOAEP(RSAPublicKey *key,
                HASH_HashType hashAlg,
                HASH_HashType maskHashAlg,
                const unsigned char *label,
                unsigned int labelLen,
                const unsigned char *seed,
                unsigned int seedLen,
                unsigned char *output,
                unsigned int *outputLen,
                unsigned int maxOutputLen,
                const unsigned char *input,
                unsigned int inputLen)
{
    SECStatus rv = SECFailure;
    unsigned int modulusLen = rsa_modulusLen(&key->modulus);
    unsigned char *oaepEncoded = NULL;

    if (maxOutputLen < modulusLen) {
        PORT_SetError(SEC_ERROR_OUTPUT_LEN);
        return SECFailure;
    }

    if ((hashAlg == HASH_AlgNULL) || (maskHashAlg == HASH_AlgNULL)) {
        PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
        return SECFailure;
    }

    if ((labelLen == 0 && label != NULL) ||
        (labelLen > 0 && label == NULL)) {
        PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
        return SECFailure;
    }

    rv = RSA_PartialVerify(key);
    if (rv != SECSuccess) {
        /* error code set by RSA_PartialVerify() */
        return rv;
    }

    oaepEncoded = (unsigned char *)PORT_Alloc(modulusLen);
    if (oaepEncoded == NULL) {
        PORT_SetError(SEC_ERROR_NO_MEMORY);
        return SECFailure;
    }
    rv = eme_oaep_encode(oaepEncoded, modulusLen, input, inputLen,
                         hashAlg, maskHashAlg, label, labelLen, seed, seedLen);
    if (rv != SECSuccess)
        goto done;

    rv = RSA_PublicKeyOp(key, output, oaepEncoded);
    if (rv != SECSuccess)
        goto done;
    *outputLen = modulusLen;

done:
    PORT_Free(oaepEncoded);
    return rv;
}

SECStatus
RSA_DecryptOAEP(RSAPrivateKey *key,
                HASH_HashType hashAlg,
                HASH_HashType maskHashAlg,
                const unsigned char *label,
                unsigned int labelLen,
                unsigned char *output,
                unsigned int *outputLen,
                unsigned int maxOutputLen,
                const unsigned char *input,
                unsigned int inputLen)
{
    SECStatus rv = SECFailure;
    unsigned int modulusLen = rsa_modulusLen(&key->modulus);
    unsigned char *oaepEncoded = NULL;

    if ((hashAlg == HASH_AlgNULL) || (maskHashAlg == HASH_AlgNULL)) {
        PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
        return SECFailure;
    }

    if (inputLen != modulusLen) {
        PORT_SetError(SEC_ERROR_INPUT_LEN);
        return SECFailure;
    }

    if ((labelLen == 0 && label != NULL) ||
        (labelLen > 0 && label == NULL)) {
        PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
        return SECFailure;
    }

    oaepEncoded = (unsigned char *)PORT_Alloc(modulusLen);
    if (oaepEncoded == NULL) {
        PORT_SetError(SEC_ERROR_NO_MEMORY);
        return SECFailure;
    }

    rv = RSA_PrivateKeyOpDoubleChecked(key, oaepEncoded, input);
    if (rv != SECSuccess) {
        goto done;
    }
    rv = eme_oaep_decode(output, outputLen, maxOutputLen, oaepEncoded,
                         modulusLen, hashAlg, maskHashAlg, label,
                         labelLen);

done:
    if (oaepEncoded)
        PORT_ZFree(oaepEncoded, modulusLen);
    return rv;
}

/* XXX Doesn't set error code */
SECStatus
RSA_EncryptBlock(RSAPublicKey *key,
                 unsigned char *output,
                 unsigned int *outputLen,
                 unsigned int maxOutputLen,
                 const unsigned char *input,
                 unsigned int inputLen)
{
    SECStatus rv;
    unsigned int modulusLen = rsa_modulusLen(&key->modulus);
    SECItem formatted;
    SECItem unformatted;

    formatted.data = NULL;
    if (maxOutputLen < modulusLen)
        goto failure;

    unformatted.len = inputLen;
    unformatted.data = (unsigned char *)input;
    formatted.data = NULL;
    rv = rsa_FormatBlock(&formatted, modulusLen, RSA_BlockPublic,
                         &unformatted);
    if (rv != SECSuccess)
        goto failure;

    rv = RSA_PublicKeyOp(key, output, formatted.data);
    if (rv != SECSuccess)
        goto failure;

    PORT_ZFree(formatted.data, modulusLen);
    *outputLen = modulusLen;
    return SECSuccess;

failure:
    if (formatted.data != NULL)
        PORT_ZFree(formatted.data, modulusLen);
    return SECFailure;
}

static HMACContext *
rsa_GetHMACContext(const SECHashObject *hash, RSAPrivateKey *key,
                   const unsigned char *input, unsigned int inputLen)
{
    unsigned char keyHash[HASH_LENGTH_MAX];
    void *hashContext;
    HMACContext *hmac = NULL;
    unsigned int privKeyLen = key->privateExponent.len;
    unsigned int keyLen;
    SECStatus rv;

    /* first get the key hash (should store in the key structure) */
    PORT_Memset(keyHash, 0, sizeof(keyHash));
    hashContext = (*hash->create)();
    if (hashContext == NULL) {
        return NULL;
    }
    (*hash->begin)(hashContext);
    if (privKeyLen < inputLen) {
        int padLen = inputLen - privKeyLen;
        while (padLen > sizeof(keyHash)) {
            (*hash->update)(hashContext, keyHash, sizeof(keyHash));
            padLen -= sizeof(keyHash);
        }
        (*hash->update)(hashContext, keyHash, padLen);
    }
    (*hash->update)(hashContext, key->privateExponent.data, privKeyLen);
    (*hash->end)(hashContext, keyHash, &keyLen, sizeof(keyHash));
    (*hash->destroy)(hashContext, PR_TRUE);

    /* now create the hmac key */
    hmac = HMAC_Create(hash, keyHash, keyLen, PR_TRUE);
    if (hmac == NULL) {
        PORT_SafeZero(keyHash, sizeof(keyHash));
        return NULL;
    }
    HMAC_Begin(hmac);
    HMAC_Update(hmac, input, inputLen);
    rv = HMAC_Finish(hmac, keyHash, &keyLen, sizeof(keyHash));
    if (rv != SECSuccess) {
        PORT_SafeZero(keyHash, sizeof(keyHash));
        HMAC_Destroy(hmac, PR_TRUE);
        return NULL;
    }
    /* Finally set the new key into the hash context. We
     * reuse the original context allocated above so we don't
     * need to allocate and free another one */
    rv = HMAC_ReInit(hmac, hash, keyHash, keyLen, PR_TRUE);
    PORT_SafeZero(keyHash, sizeof(keyHash));
    if (rv != SECSuccess) {
        HMAC_Destroy(hmac, PR_TRUE);
        return NULL;
    }

    return hmac;
}

static SECStatus
rsa_HMACPrf(HMACContext *hmac, const char *label, int labelLen,
            int hashLength, unsigned char *output, int length)
{
    unsigned char iterator[2] = { 0, 0 };
    unsigned char encodedLen[2] = { 0, 0 };
    unsigned char hmacLast[HASH_LENGTH_MAX];
    unsigned int left = length;
    unsigned int hashReturn;
    SECStatus rv = SECSuccess;

    /* encodedLen is in bits, length is in bytes, thus the shifts
     * do an implied multiply by 8 */
    encodedLen[0] = (length >> 5) & 0xff;
    encodedLen[1] = (length << 3) & 0xff;

    while (left > hashLength) {
        HMAC_Begin(hmac);
        HMAC_Update(hmac, iterator, 2);
        HMAC_Update(hmac, (const unsigned char *)label, labelLen);
        HMAC_Update(hmac, encodedLen, 2);
        rv = HMAC_Finish(hmac, output, &hashReturn, hashLength);
        if (rv != SECSuccess) {
            return rv;
        }
        iterator[1]++;
        if (iterator[1] == 0)
            iterator[0]++;
        left -= hashLength;
        output += hashLength;
    }
    if (left) {
        HMAC_Begin(hmac);
        HMAC_Update(hmac, iterator, 2);
        HMAC_Update(hmac, (const unsigned char *)label, labelLen);
        HMAC_Update(hmac, encodedLen, 2);
        rv = HMAC_Finish(hmac, hmacLast, &hashReturn, sizeof(hmacLast));
        if (rv != SECSuccess) {
            return rv;
        }
        PORT_Memcpy(output, hmacLast, left);
        PORT_SafeZero(hmacLast, sizeof(hmacLast));
    }
    return rv;
}

/* This function takes a 16-bit input number and
 * creates the smallest mask which covers
 * the whole number. Examples:
 *     0x81 -> 0xff
 *     0x1af -> 0x1ff
 *     0x4d1 -> 0x7ff
 */
static int
makeMask16(int len)
{
    // or the high bit in each bit location
    len |= (len >> 1);
    len |= (len >> 2);
    len |= (len >> 4);
    len |= (len >> 8);
    return len;
}

#define STRING_AND_LENGTH(s) s, sizeof(s) - 1
static int
rsa_GetErrorLength(HMACContext *hmac, int hashLen, int maxLegalLen)
{
    unsigned char out[128 * 2];
    unsigned char *outp;
    int outLength = 0;
    int lengthMask;
    SECStatus rv;

    lengthMask = makeMask16(maxLegalLen);
    rv = rsa_HMACPrf(hmac, STRING_AND_LENGTH("length"), hashLen,
                     out, sizeof(out));
    if (rv != SECSuccess) {
        return -1;
    }
    for (outp = out; outp < out + sizeof(out); outp += 2) {
        int candidate = outp[0] << 8 | outp[1];
        candidate = candidate & lengthMask;
        outLength = PORT_CT_SEL(PORT_CT_LT(candidate, maxLegalLen),
                                candidate, outLength);
    }
    PORT_SafeZero(out, sizeof(out));
    return outLength;
}

/*
 * This function can only fail in environmental cases: Programming errors
 * and out of memory situations. It can't fail if the keys are valid and
 * the inputs are the proper size. If the actual RSA decryption fails, a
 * fake value and a fake length, both of which have already been generated
 * based on the key and input, are returned.
 * Applications are expected to detect decryption failures based on the fact
 * that the decrypted value (usually a key) doesn't validate. The prevents
 * Blecheinbaucher style attacks against the key. */
SECStatus
RSA_DecryptBlock(RSAPrivateKey *key,
                 unsigned char *output,
                 unsigned int *outputLen,
                 unsigned int maxOutputLen,
                 const unsigned char *input,
                 unsigned int inputLen)
{
    SECStatus rv;
    PRUint32 fail;
    unsigned int modulusLen = rsa_modulusLen(&key->modulus);
    unsigned int i;
    unsigned char *buffer = NULL;
    unsigned char *errorBuffer = NULL;
    unsigned char *bp = NULL;
    unsigned char *ep = NULL;
    unsigned int outLen = modulusLen;
    unsigned int maxLegalLen = modulusLen - 10;
    unsigned int errorLength;
    const SECHashObject *hashObj;
    HMACContext *hmac = NULL;

    /* failures in the top section indicate failures in the environment
     * (memory) or the library. OK to return errors in these cases because
     * it doesn't provide any oracle information to attackers. */
    if (inputLen != modulusLen || modulusLen < 10) {
        PORT_SetError(SEC_ERROR_INVALID_ARGS);
        return SECFailure;
    }

    /* Allocate enough space to decrypt */
    buffer = PORT_ZAlloc(modulusLen);
    if (!buffer) {
        goto loser;
    }
    errorBuffer = PORT_ZAlloc(modulusLen);
    if (!errorBuffer) {
        goto loser;
    }
    hashObj = HASH_GetRawHashObject(HASH_AlgSHA256);
    if (hashObj == NULL) {
        goto loser;
    }

    /* calculate the values to return in the error case rather than
     * the actual returned values. This data is the same for the
     * same input and private key. */
    hmac = rsa_GetHMACContext(hashObj, key, input, inputLen);
    if (hmac == NULL) {
        goto loser;
    }
    errorLength = rsa_GetErrorLength(hmac, hashObj->length, maxLegalLen);
    if (((int)errorLength) < 0) {
        goto loser;
    }
    /* we always have to generate a full moduluslen error string. Otherwise
     * we create a timing dependency on errorLength, which could be used to
     * determine the difference between errorLength and outputLen and tell
     * us that there was a pkcs1 decryption failure */
    rv = rsa_HMACPrf(hmac, STRING_AND_LENGTH("message"),
                     hashObj->length, errorBuffer, modulusLen);
    if (rv != SECSuccess) {
        goto loser;
    }

    HMAC_Destroy(hmac, PR_TRUE);
    hmac = NULL;

    /* From here on out, we will always return success. If there is
     * an error, we will return deterministic output based on the key
     * and the input data. */
    rv = RSA_PrivateKeyOp(key, buffer, input);

    fail = PORT_CT_NE(rv, SECSuccess);
    fail |= PORT_CT_NE(buffer[0], RSA_BLOCK_FIRST_OCTET) | PORT_CT_NE(buffer[1], RSA_BlockPublic);

    /* There have to be at least 8 bytes of padding. */
    for (i = 2; i < 10; i++) {
        fail |= PORT_CT_EQ(buffer[i], RSA_BLOCK_AFTER_PAD_OCTET);
    }

    for (i = 10; i < modulusLen; i++) {
        unsigned int newLen = modulusLen - i - 1;
        PRUint32 condition = PORT_CT_EQ(buffer[i], RSA_BLOCK_AFTER_PAD_OCTET) & PORT_CT_EQ(outLen, modulusLen);
        outLen = PORT_CT_SEL(condition, newLen, outLen);
    }
    // this can only happen if a zero wasn't found above
    fail |= PORT_CT_GE(outLen, modulusLen);

    outLen = PORT_CT_SEL(fail, errorLength, outLen);

    /* index into the correct buffer. Do it before we truncate outLen if the
     * application was asking for less data than we can return */
    bp = buffer + modulusLen - outLen;
    ep = errorBuffer + modulusLen - outLen;

    /* at this point, outLen returns no information about decryption failures,
     * no need to hide its value. maxOutputLen is how much data the
     * application is expecting, which is also not sensitive. */
    if (outLen > maxOutputLen) {
        outLen = maxOutputLen;
    }

    /* we can't use PORT_Memcpy because caching could create a time dependency
     * on the status of fail. */
    for (i = 0; i < outLen; i++) {
        output[i] = PORT_CT_SEL(fail, ep[i], bp[i]);
    }

    *outputLen = outLen;

    PORT_Free(buffer);
    PORT_Free(errorBuffer);

    return SECSuccess;

loser:
    if (hmac) {
        HMAC_Destroy(hmac, PR_TRUE);
    }
    PORT_Free(buffer);
    PORT_Free(errorBuffer);

    return SECFailure;
}

/*
 * Encode a RSA-PSS signature.
 * Described in RFC 3447, section 9.1.1.
 * We use mHash instead of M as input.
 * emBits from the RFC is just modBits - 1, see section 8.1.1.
 * We only support MGF1 as the MGF.
 */
SECStatus
RSA_EMSAEncodePSS(unsigned char *em,
                  unsigned int emLen,
                  unsigned int emBits,
                  const unsigned char *mHash,
                  unsigned int mHashLen,
                  HASH_HashType hashAlg,
                  HASH_HashType maskHashAlg,
                  const unsigned char *salt,
                  unsigned int saltLen)
{
    const SECHashObject *hash;
    void *hash_context;
    unsigned char *dbMask;
    unsigned int dbMaskLen;
    unsigned int i;
    SECStatus rv;

    hash = HASH_GetRawHashObject(hashAlg);
    PORT_Assert(hash);

    if (mHashLen < hash->length) {
        PORT_SetError(SEC_ERROR_INVALID_ARGS);
        return SECFailure;
    }

    dbMaskLen = emLen - hash->length - 1;

    /* Step 3 */
    if (emLen < hash->length + saltLen + 2) {
        PORT_SetError(SEC_ERROR_OUTPUT_LEN);
        return SECFailure;
    }

    /* Step 4 */
    if (salt == NULL) {
        rv = RNG_GenerateGlobalRandomBytes(&em[dbMaskLen - saltLen], saltLen);
        if (rv != SECSuccess) {
            return rv;
        }
    } else {
        PORT_Memcpy(&em[dbMaskLen - saltLen], salt, saltLen);
    }

    /* Step 5 + 6 */
    /* Compute H and store it at its final location &em[dbMaskLen]. */
    hash_context = (*hash->create)();
    if (hash_context == NULL) {
        PORT_SetError(SEC_ERROR_NO_MEMORY);
        return SECFailure;
    }
    (*hash->begin)(hash_context);
    (*hash->update)(hash_context, eightZeros, 8);
    (*hash->update)(hash_context, mHash, hash->length);
    (*hash->update)(hash_context, &em[dbMaskLen - saltLen], saltLen);
    (*hash->end)(hash_context, &em[dbMaskLen], &i, hash->length);
    (*hash->destroy)(hash_context, PR_TRUE);

    /* Step 7 + 8 */
    PORT_Memset(em, 0, dbMaskLen - saltLen - 1);
    em[dbMaskLen - saltLen - 1] = 0x01;

    /* Step 9 */
    dbMask = (unsigned char *)PORT_Alloc(dbMaskLen);
    if (dbMask == NULL) {
        PORT_SetError(SEC_ERROR_NO_MEMORY);
        return SECFailure;
    }
    MGF1(maskHashAlg, dbMask, dbMaskLen, &em[dbMaskLen], hash->length);

    /* Step 10 */
    for (i = 0; i < dbMaskLen; i++)
        em[i] ^= dbMask[i];
    PORT_Free(dbMask);

    /* Step 11 */
    em[0] &= 0xff >> (8 * emLen - emBits);

    /* Step 12 */
    em[emLen - 1] = 0xbc;

    return SECSuccess;
}

/*
 * Verify a RSA-PSS signature.
 * Described in RFC 3447, section 9.1.2.
 * We use mHash instead of M as input.
 * emBits from the RFC is just modBits - 1, see section 8.1.2.
 * We only support MGF1 as the MGF.
 */
static SECStatus
emsa_pss_verify(const unsigned char *mHash,
                const unsigned char *em,
                unsigned int emLen,
                unsigned int emBits,
                HASH_HashType hashAlg,
                HASH_HashType maskHashAlg,
                unsigned int saltLen)
{
    const SECHashObject *hash;
    void *hash_context;
    unsigned char *db;
    unsigned char *H_; /* H' from the RFC */
    unsigned int i;
    unsigned int dbMaskLen;
    unsigned int zeroBits;
    SECStatus rv;

    hash = HASH_GetRawHashObject(hashAlg);
    dbMaskLen = emLen - hash->length - 1;

    /* Step 3 + 4 */
    if ((emLen < (hash->length + saltLen + 2)) ||
        (em[emLen - 1] != 0xbc)) {
        PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
        return SECFailure;
    }

    /* Step 6 */
    zeroBits = 8 * emLen - emBits;
    if (em[0] >> (8 - zeroBits)) {
        PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
        return SECFailure;
    }

    /* Step 7 */
    db = (unsigned char *)PORT_Alloc(dbMaskLen);
    if (db == NULL) {
        PORT_SetError(SEC_ERROR_NO_MEMORY);
        return SECFailure;
    }
    /* &em[dbMaskLen] points to H, used as mgfSeed */
    MGF1(maskHashAlg, db, dbMaskLen, &em[dbMaskLen], hash->length);

    /* Step 8 */
    for (i = 0; i < dbMaskLen; i++) {
        db[i] ^= em[i];
    }

    /* Step 9 */
    db[0] &= 0xff >> zeroBits;

    /* Step 10 */
    for (i = 0; i < (dbMaskLen - saltLen - 1); i++) {
        if (db[i] != 0) {
            PORT_Free(db);
            PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
            return SECFailure;
        }
    }
    if (db[dbMaskLen - saltLen - 1] != 0x01) {
        PORT_Free(db);
        PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
        return SECFailure;
    }

    /* Step 12 + 13 */
    H_ = (unsigned char *)PORT_Alloc(hash->length);
    if (H_ == NULL) {
        PORT_Free(db);
        PORT_SetError(SEC_ERROR_NO_MEMORY);
        return SECFailure;
    }
    hash_context = (*hash->create)();
    if (hash_context == NULL) {
        PORT_Free(db);
        PORT_Free(H_);
        PORT_SetError(SEC_ERROR_NO_MEMORY);
        return SECFailure;
    }
    (*hash->begin)(hash_context);
    (*hash->update)(hash_context, eightZeros, 8);
    (*hash->update)(hash_context, mHash, hash->length);
    (*hash->update)(hash_context, &db[dbMaskLen - saltLen], saltLen);
    (*hash->end)(hash_context, H_, &i, hash->length);
    (*hash->destroy)(hash_context, PR_TRUE);

    PORT_Free(db);

    /* Step 14 */
    if (PORT_Memcmp(H_, &em[dbMaskLen], hash->length) != 0) {
        PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
        rv = SECFailure;
    } else {
        rv = SECSuccess;
    }

    PORT_Free(H_);
    return rv;
}

SECStatus
RSA_SignPSS(RSAPrivateKey *key,
            HASH_HashType hashAlg,
            HASH_HashType maskHashAlg,
            const unsigned char *salt,
            unsigned int saltLength,
            unsigned char *output,
            unsigned int *outputLen,
            unsigned int maxOutputLen,
            const unsigned char *input,
            unsigned int inputLen)
{
    SECStatus rv = SECSuccess;
    unsigned int modulusLen = rsa_modulusLen(&key->modulus);
    unsigned int modulusBits = rsa_modulusBits(&key->modulus);
    unsigned int emLen = modulusLen;
    unsigned char *pssEncoded, *em;

    if (maxOutputLen < modulusLen) {
        PORT_SetError(SEC_ERROR_OUTPUT_LEN);
        return SECFailure;
    }

    if ((hashAlg == HASH_AlgNULL) || (maskHashAlg == HASH_AlgNULL)) {
        PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
        return SECFailure;
    }

    pssEncoded = em = (unsigned char *)PORT_Alloc(modulusLen);
    if (pssEncoded == NULL) {
        PORT_SetError(SEC_ERROR_NO_MEMORY);
        return SECFailure;
    }

    /* len(em) == ceil((modulusBits - 1) / 8). */
    if (modulusBits % 8 == 1) {
        em[0] = 0;
        emLen--;
        em++;
    }
    rv = RSA_EMSAEncodePSS(em, emLen, modulusBits - 1, input, inputLen, hashAlg,
                           maskHashAlg, salt, saltLength);
    if (rv != SECSuccess)
        goto done;

    // This sets error codes upon failure.
    rv = RSA_PrivateKeyOpDoubleChecked(key, output, pssEncoded);
    *outputLen = modulusLen;

done:
    PORT_Free(pssEncoded);
    return rv;
}

SECStatus
RSA_CheckSignPSS(RSAPublicKey *key,
                 HASH_HashType hashAlg,
                 HASH_HashType maskHashAlg,
                 unsigned int saltLength,
                 const unsigned char *sig,
                 unsigned int sigLen,
                 const unsigned char *hash,
                 unsigned int hashLen)
{
    SECStatus rv;
    unsigned int modulusLen = rsa_modulusLen(&key->modulus);
    unsigned int modulusBits = rsa_modulusBits(&key->modulus);
    unsigned int emLen = modulusLen;
    unsigned char *buffer, *em;

    if (sigLen != modulusLen) {
        PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
        return SECFailure;
    }

    if ((hashAlg == HASH_AlgNULL) || (maskHashAlg == HASH_AlgNULL)) {
        PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
        return SECFailure;
    }

    buffer = em = (unsigned char *)PORT_Alloc(modulusLen);
    if (!buffer) {
        PORT_SetError(SEC_ERROR_NO_MEMORY);
        return SECFailure;
    }

    rv = RSA_PublicKeyOp(key, buffer, sig);
    if (rv != SECSuccess) {
        PORT_Free(buffer);
        PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
        return SECFailure;
    }

    /* len(em) == ceil((modulusBits - 1) / 8). */
    if (modulusBits % 8 == 1) {
        emLen--;
        em++;
    }
    rv = emsa_pss_verify(hash, em, emLen, modulusBits - 1, hashAlg,
                         maskHashAlg, saltLength);

    PORT_Free(buffer);
    return rv;
}

SECStatus
RSA_Sign(RSAPrivateKey *key,
         unsigned char *output,
         unsigned int *outputLen,
         unsigned int maxOutputLen,
         const unsigned char *input,
         unsigned int inputLen)
{
    SECStatus rv = SECFailure;
    unsigned int modulusLen = rsa_modulusLen(&key->modulus);
    SECItem formatted = { siBuffer, NULL, 0 };
    SECItem unformatted = { siBuffer, (unsigned char *)input, inputLen };

    if (maxOutputLen < modulusLen) {
        PORT_SetError(SEC_ERROR_OUTPUT_LEN);
        goto done;
    }

    rv = rsa_FormatBlock(&formatted, modulusLen, RSA_BlockPrivate,
                         &unformatted);
    if (rv != SECSuccess) {
        PORT_SetError(SEC_ERROR_LIBRARY_FAILURE);
        goto done;
    }

    // This sets error codes upon failure.
    rv = RSA_PrivateKeyOpDoubleChecked(key, output, formatted.data);
    *outputLen = modulusLen;

done:
    if (formatted.data != NULL) {
        PORT_ZFree(formatted.data, modulusLen);
    }
    return rv;
}

SECStatus
RSA_CheckSign(RSAPublicKey *key,
              const unsigned char *sig,
              unsigned int sigLen,
              const unsigned char *data,
              unsigned int dataLen)
{
    SECStatus rv = SECFailure;
    unsigned int modulusLen = rsa_modulusLen(&key->modulus);
    unsigned int i;
    unsigned char *buffer = NULL;

    if (sigLen != modulusLen) {
        PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
        goto done;
    }

    /*
     * 0x00 || BT || Pad || 0x00 || ActualData
     *
     * The "3" below is the first octet + the second octet + the 0x00
     * octet that always comes just before the ActualData.
     */
    if (dataLen > modulusLen - (3 + RSA_BLOCK_MIN_PAD_LEN)) {
        PORT_SetError(SEC_ERROR_BAD_DATA);
        goto done;
    }

    buffer = (unsigned char *)PORT_Alloc(modulusLen + 1);
    if (!buffer) {
        PORT_SetError(SEC_ERROR_NO_MEMORY);
        goto done;
    }

    if (RSA_PublicKeyOp(key, buffer, sig) != SECSuccess) {
        PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
        goto done;
    }

    /*
     * check the padding that was used
     */
    if (buffer[0] != RSA_BLOCK_FIRST_OCTET ||
        buffer[1] != (unsigned char)RSA_BlockPrivate) {
        PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
        goto done;
    }
    for (i = 2; i < modulusLen - dataLen - 1; i++) {
        if (buffer[i] != RSA_BLOCK_PRIVATE_PAD_OCTET) {
            PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
            goto done;
        }
    }
    if (buffer[i] != RSA_BLOCK_AFTER_PAD_OCTET) {
        PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
        goto done;
    }

    /*
     * make sure we get the same results
     */
    if (PORT_Memcmp(buffer + modulusLen - dataLen, data, dataLen) == 0) {
        rv = SECSuccess;
    }

done:
    if (buffer) {
        PORT_Free(buffer);
    }
    return rv;
}

SECStatus
RSA_CheckSignRecover(RSAPublicKey *key,
                     unsigned char *output,
                     unsigned int *outputLen,
                     unsigned int maxOutputLen,
                     const unsigned char *sig,
                     unsigned int sigLen)
{
    SECStatus rv = SECFailure;
    unsigned int modulusLen = rsa_modulusLen(&key->modulus);
    unsigned int i;
    unsigned char *buffer = NULL;
    unsigned int padLen;

    if (sigLen != modulusLen) {
        PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
        goto done;
    }

    buffer = (unsigned char *)PORT_Alloc(modulusLen + 1);
    if (!buffer) {
        PORT_SetError(SEC_ERROR_NO_MEMORY);
        goto done;
    }

    if (RSA_PublicKeyOp(key, buffer, sig) != SECSuccess) {
        PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
        goto done;
    }

    *outputLen = 0;

    /*
     * check the padding that was used
     */
    if (buffer[0] != RSA_BLOCK_FIRST_OCTET ||
        buffer[1] != (unsigned char)RSA_BlockPrivate) {
        PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
        goto done;
    }
    for (i = 2; i < modulusLen; i++) {
        if (buffer[i] == RSA_BLOCK_AFTER_PAD_OCTET) {
            *outputLen = modulusLen - i - 1;
            break;
        }
        if (buffer[i] != RSA_BLOCK_PRIVATE_PAD_OCTET) {
            PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
            goto done;
        }
    }
    padLen = i - 2;
    if (padLen < RSA_BLOCK_MIN_PAD_LEN) {
        PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
        goto done;
    }
    if (*outputLen == 0) {
        PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
        goto done;
    }
    if (*outputLen > maxOutputLen) {
        PORT_SetError(SEC_ERROR_OUTPUT_LEN);
        goto done;
    }

    PORT_Memcpy(output, buffer + modulusLen - *outputLen, *outputLen);
    rv = SECSuccess;

done:
    if (buffer) {
        PORT_Free(buffer);
    }
    return rv;
}

Coverage Report

Created: 2026-04-04 06:13