Index: security/nss/lib/softoken/lowkey.c
===================================================================
RCS file: /cvsroot/mozilla/security/nss/lib/softoken/lowkey.c,v
retrieving revision 1.10
diff -u -p -8 -r1.10 lowkey.c
--- security/nss/lib/softoken/lowkey.c	8 Feb 2008 02:50:47 -0000	1.10
+++ security/nss/lib/softoken/lowkey.c	29 Nov 2010 21:55:04 -0000
@@ -292,16 +292,57 @@ nsslowkey_PrivateModulusLen(NSSLOWKEYPri
 	b0 = privk->u.rsa.modulus.data[0];
 	return b0 ? privk->u.rsa.modulus.len : privk->u.rsa.modulus.len - 1;
     default:
 	break;
     }
     return 0;
 }
 
+unsigned int
+nsslowkey_ModulusBits(SECItem *modulus)
+{
+    unsigned char b0;
+    unsigned int bits;
+
+    bits = (modulus->len - 1) * 8;
+    b0 = modulus->data[0];
+    if (b0 == 0) { /* first byte is zero */
+	bits -= 8;
+	b0 = modulus->data[1];
+    }
+    while (b0 != 0) {
+	bits++;
+	b0 >>= 1;
+    }
+    return bits;
+}
+
+unsigned int
+nsslowkey_PublicModulusBits(NSSLOWKEYPublicKey *pubKey)
+{
+    switch (pubKey->keyType) {
+    case NSSLOWKEYRSAKey:
+	return nsslowkey_ModulusBits(&pubKey->u.rsa.modulus);
+    default:
+	return 0;
+    }
+}
+
+unsigned int
+nsslowkey_PrivateModulusBits(NSSLOWKEYPrivateKey *privKey)
+{
+    switch (privKey->keyType) {
+    case NSSLOWKEYRSAKey:
+	return nsslowkey_ModulusBits(&privKey->u.rsa.modulus);
+    default:
+	return 0;
+    }
+}
+
 NSSLOWKEYPublicKey *
 nsslowkey_ConvertToPublicKey(NSSLOWKEYPrivateKey *privk)
 {
     NSSLOWKEYPublicKey *pubk;
     PLArenaPool *arena;
 
 
     arena = PORT_NewArena (DER_DEFAULT_CHUNKSIZE);
Index: security/nss/lib/softoken/lowkeyi.h
===================================================================
RCS file: /cvsroot/mozilla/security/nss/lib/softoken/lowkeyi.h,v
retrieving revision 1.11
diff -u -p -8 -r1.11 lowkeyi.h
--- security/nss/lib/softoken/lowkeyi.h	13 Jun 2007 00:24:56 -0000	1.11
+++ security/nss/lib/softoken/lowkeyi.h	29 Nov 2010 21:55:04 -0000
@@ -74,26 +74,34 @@ extern void nsslowkey_DestroyPrivateKey(
 /*
 ** Destroy a public key object.
 **	"key" the object
 **	"freeit" if PR_TRUE then free the object as well as its sub-objects
 */
 extern void nsslowkey_DestroyPublicKey(NSSLOWKEYPublicKey *key);
 
 /*
-** Return the modulus length of "pubKey".
+** Return the modulus length of "pubKey" in bytes.
 */
 extern unsigned int nsslowkey_PublicModulusLen(NSSLOWKEYPublicKey *pubKey);
 
-
 /*
-** Return the modulus length of "privKey".
+** Return the modulus length of "privKey" in bytes.
 */
 extern unsigned int nsslowkey_PrivateModulusLen(NSSLOWKEYPrivateKey *privKey);
 
+/*
+** Return the modulus length of "pubKey" in bits.
+*/
+extern unsigned int nsslowkey_PublicModulusBits(NSSLOWKEYPublicKey *pubKey);
+
+/*
+** Return the modulus length of "privKey" in bits.
+*/
+extern unsigned int nsslowkey_PrivateModulusBits(NSSLOWKEYPrivateKey *privKey);
 
 /*
 ** Convert a low private key "privateKey" into a public low key
 */
 extern NSSLOWKEYPublicKey 
 		*nsslowkey_ConvertToPublicKey(NSSLOWKEYPrivateKey *privateKey);
 
 /* Make a copy of a low private key in it's own arena.
Index: security/nss/lib/softoken/rsawrapr.c
===================================================================
RCS file: /cvsroot/mozilla/security/nss/lib/softoken/rsawrapr.c,v
retrieving revision 1.17
diff -u -p -8 -r1.17 rsawrapr.c
--- security/nss/lib/softoken/rsawrapr.c	7 Aug 2010 18:10:35 -0000	1.17
+++ security/nss/lib/softoken/rsawrapr.c	29 Nov 2010 21:55:05 -0000
@@ -950,23 +950,24 @@ failure:
  * We use mHash instead of M as input.
  * emBits from the RFC is just modBits - 1, see section 8.1.1.
  * We only support MGF1 as the MGF.
  *
  * NOTE: this code assumes modBits is a multiple of 8.
  */
 static SECStatus
 emsa_pss_encode(unsigned char *em, unsigned int emLen,
-                const unsigned char *mHash, HASH_HashType hashAlg,
-                HASH_HashType maskHashAlg, unsigned int sLen)
+		unsigned int emBits, const unsigned char *mHash,
+		HASH_HashType hashAlg, HASH_HashType maskHashAlg,
+		unsigned int sLen)
 {
     const SECHashObject *hash;
-    void *hash_context;
+    void *hashContext;
     unsigned char *dbMask;
-    unsigned int dbMaskLen, i;
+    unsigned int dbMaskLen, i, zeroBits;
     SECStatus rv;
 
     hash = HASH_GetRawHashObject(hashAlg);
     dbMaskLen = emLen - hash->length - 1;
 
     /* Step 3 */
     if (emLen < hash->length + sLen + 2) {
 	PORT_SetError(SEC_ERROR_OUTPUT_LEN);
@@ -976,27 +977,27 @@ emsa_pss_encode(unsigned char *em, unsig
     /* Step 4 */
     rv = RNG_GenerateGlobalRandomBytes(&em[dbMaskLen - sLen], sLen);
     if (rv != SECSuccess) {
 	return rv;
     }
 
     /* Step 5 + 6 */
     /* Compute H and store it at its final location &em[dbMaskLen]. */
-    hash_context = (*hash->create)();
-    if (hash_context == NULL) {
+    hashContext = (*hash->create)();
+    if (hashContext == NULL) {
 	PORT_SetError(SEC_ERROR_NO_MEMORY);
 	return SECFailure;
     }
-    (*hash->begin)(hash_context);
-    (*hash->update)(hash_context, eightZeros, 8);
-    (*hash->update)(hash_context, mHash, hash->length);
-    (*hash->update)(hash_context, &em[dbMaskLen - sLen], sLen);
-    (*hash->end)(hash_context, &em[dbMaskLen], &i, hash->length);
-    (*hash->destroy)(hash_context, PR_TRUE);
+    (*hash->begin)(hashContext);
+    (*hash->update)(hashContext, eightZeros, 8);
+    (*hash->update)(hashContext, mHash, hash->length);
+    (*hash->update)(hashContext, &em[dbMaskLen - sLen], sLen);
+    (*hash->end)(hashContext, &em[dbMaskLen], &i, hash->length);
+    (*hash->destroy)(hashContext, PR_TRUE);
 
     /* Step 7 + 8 */
     memset(em, 0, dbMaskLen - sLen - 1);
     em[dbMaskLen - sLen - 1] = 0x01;
 
     /* Step 9 */
     dbMask = (unsigned char *)PORT_Alloc(dbMaskLen);
     if (dbMask == NULL) {
@@ -1006,17 +1007,18 @@ emsa_pss_encode(unsigned char *em, unsig
     MGF1(maskHashAlg, dbMask, dbMaskLen, &em[dbMaskLen], hash->length);
 
     /* Step 10 */
     for (i = 0; i < dbMaskLen; i++)
 	em[i] ^= dbMask[i];
     PORT_Free(dbMask);
 
     /* Step 11 */
-    em[0] &= 0x7f;
+    zeroBits = 8 * emLen - emBits;
+    em[0] &= 0xff >> zeroBits;
 
     /* Step 12 */
     em[emLen - 1] = 0xbc;
 
     return SECSuccess;
 }
 
 /*
@@ -1026,33 +1028,39 @@ emsa_pss_encode(unsigned char *em, unsig
  * emBits from the RFC is just modBits - 1, see section 8.1.2.
  * We only support MGF1 as the MGF.
  *
  * NOTE: this code assumes modBits is a multiple of 8.
  */
 static SECStatus
 emsa_pss_verify(const unsigned char *mHash,
                 const unsigned char *em, unsigned int emLen,
-                HASH_HashType hashAlg, HASH_HashType maskHashAlg,
-                unsigned int sLen)
+                unsigned int emBits, HASH_HashType hashAlg,
+                HASH_HashType maskHashAlg, unsigned int sLen)
 {
     const SECHashObject *hash;
-    void *hash_context;
+    void *hashContext;
     unsigned char *db;
     unsigned char *H_;  /* H' from the RFC */
-    unsigned int i, dbMaskLen;
+    unsigned int i, dbMaskLen, zeroBits;
     SECStatus rv;
 
     hash = HASH_GetRawHashObject(hashAlg);
     dbMaskLen = emLen - hash->length - 1;
 
-    /* Step 3 + 4 + 6 */
+    /* Step 3 + 4 */
     if ((emLen < (hash->length + sLen + 2)) ||
-	(em[emLen - 1] != 0xbc) ||
-	((em[0] & 0x80) != 0)) {
+	(em[emLen - 1] != 0xbc)) {
+	PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
+	return SECFailure;
+    }
+
+    /* Step 6 */
+    zeroBits = 8 * emLen - emBits;
+    if ((em[0] >> (8 - zeroBits)) != 0) {
 	PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
 	return SECFailure;
     }
 
     /* Step 7 */
     db = (unsigned char *)PORT_Alloc(dbMaskLen);
     if (db == NULL) {
 	PORT_SetError(SEC_ERROR_NO_MEMORY);
@@ -1062,17 +1070,17 @@ emsa_pss_verify(const unsigned char *mHa
     MGF1(maskHashAlg, db, dbMaskLen, &em[dbMaskLen], hash->length);
 
     /* Step 8 */
     for (i = 0; i < dbMaskLen; i++) {
 	db[i] ^= em[i];
     }
 
     /* Step 9 */
-    db[0] &= 0x7f;
+    db[0] &= 0xff >> zeroBits;
 
     /* Step 10 */
     for (i = 0; i < (dbMaskLen - sLen - 1); i++) {
 	if (db[i] != 0) {
 	    PORT_Free(db);
 	    PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
 	    return SECFailure;
 	}
@@ -1085,29 +1093,29 @@ emsa_pss_verify(const unsigned char *mHa
 
     /* Step 12 + 13 */
     H_ = (unsigned char *)PORT_Alloc(hash->length);
     if (H_ == NULL) {
 	PORT_Free(db);
 	PORT_SetError(SEC_ERROR_NO_MEMORY);
 	return SECFailure;
     }
-    hash_context = (*hash->create)();
-    if (hash_context == NULL) {
+    hashContext = (*hash->create)();
+    if (hashContext == NULL) {
 	PORT_Free(db);
 	PORT_Free(H_);
 	PORT_SetError(SEC_ERROR_NO_MEMORY);
 	return SECFailure;
     }
-    (*hash->begin)(hash_context);
-    (*hash->update)(hash_context, eightZeros, 8);
-    (*hash->update)(hash_context, mHash, hash->length);
-    (*hash->update)(hash_context, &db[dbMaskLen - sLen], sLen);
-    (*hash->end)(hash_context, H_, &i, hash->length);
-    (*hash->destroy)(hash_context, PR_TRUE);
+    (*hash->begin)(hashContext);
+    (*hash->update)(hashContext, eightZeros, 8);
+    (*hash->update)(hashContext, mHash, hash->length);
+    (*hash->update)(hashContext, &db[dbMaskLen - sLen], sLen);
+    (*hash->end)(hashContext, H_, &i, hash->length);
+    (*hash->destroy)(hashContext, PR_TRUE);
 
     PORT_Free(db);
 
     /* Step 14 */
     if (PORT_Memcmp(H_, &em[dbMaskLen], hash->length) != 0) {
 	PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
 	rv = SECFailure;
     } else {
@@ -1146,17 +1154,19 @@ RSA_CheckSignPSS(CK_RSA_PKCS_PSS_PARAMS 
 		 NSSLOWKEYPublicKey *key,
 		 const unsigned char *sign, unsigned int sign_len,
 		 const unsigned char *hash, unsigned int hash_len)
 {
     HASH_HashType hashAlg;
     HASH_HashType maskHashAlg;
     SECStatus rv;
     unsigned int modulus_len = nsslowkey_PublicModulusLen(key);
-    unsigned char *buffer;
+    unsigned int modulus_bits = nsslowkey_PublicModulusBits(key);
+    unsigned char *buffer, *em;
+    unsigned int emLen;
 
     if (sign_len != modulus_len) {
 	PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
 	return SECFailure;
     }
 
     hashAlg = GetHashTypeFromMechanism(pss_params->hashAlg);
     maskHashAlg = GetHashTypeFromMechanism(pss_params->mgf);
@@ -1173,33 +1183,42 @@ RSA_CheckSignPSS(CK_RSA_PKCS_PSS_PARAMS 
 
     rv = RSA_PublicKeyOp(&key->u.rsa, buffer, sign);
     if (rv != SECSuccess) {
 	PORT_Free(buffer);
 	PORT_SetError(SEC_ERROR_BAD_SIGNATURE);
 	return SECFailure;
     }
 
-    rv = emsa_pss_verify(hash, buffer, modulus_len, hashAlg,
+    emLen = modulus_len;
+    em = buffer;
+    if (modulus_bits % 8 == 1) {
+        em++;
+        emLen--;
+    }
+    rv = emsa_pss_verify(hash, em, emLen, modulus_bits - 1, hashAlg,
 			 maskHashAlg, pss_params->sLen);
+
     PORT_Free(buffer);
 
     return rv;
 }
 
 /* MGF1 is the only supported MGF. */
 SECStatus
 RSA_SignPSS(CK_RSA_PKCS_PSS_PARAMS *pss_params, NSSLOWKEYPrivateKey *key,
 	    unsigned char *output, unsigned int *output_len,
 	    unsigned int max_output_len,
 	    const unsigned char *input, unsigned int input_len)
 {
     SECStatus rv = SECSuccess;
     unsigned int modulus_len = nsslowkey_PrivateModulusLen(key);
+    unsigned int modulus_bits = nsslowkey_PrivateModulusBits(key);
     unsigned char *pss_encoded = NULL;
+    unsigned int emLen;
     HASH_HashType hashAlg;
     HASH_HashType maskHashAlg;
 
     if (max_output_len < modulus_len) {
 	PORT_SetError(SEC_ERROR_OUTPUT_LEN);
 	return SECFailure;
     }
     PORT_Assert(key->keyType == NSSLOWKEYRSAKey);
@@ -1210,24 +1229,30 @@ RSA_SignPSS(CK_RSA_PKCS_PSS_PARAMS *pss_
 
     hashAlg = GetHashTypeFromMechanism(pss_params->hashAlg);
     maskHashAlg = GetHashTypeFromMechanism(pss_params->mgf);
     if ((hashAlg == HASH_AlgNULL) || (maskHashAlg == HASH_AlgNULL)) {
 	PORT_SetError(SEC_ERROR_INVALID_ALGORITHM);
 	return SECFailure;
     }
 
-    pss_encoded = (unsigned char *)PORT_Alloc(modulus_len);
+    emLen = modulus_len;
+    if (modulus_bits % 8 == 1) {
+	emLen--;
+    }
+
+    pss_encoded = (unsigned char *)PORT_Alloc(emLen);
     if (pss_encoded == NULL) {
 	PORT_SetError(SEC_ERROR_NO_MEMORY);
 	return SECFailure;
     }
-    rv = emsa_pss_encode(pss_encoded, modulus_len, input, hashAlg,
+
+    rv = emsa_pss_encode(pss_encoded, emLen, modulus_bits-1, input, hashAlg,
 			 maskHashAlg, pss_params->sLen);
-    if (rv != SECSuccess) 
+    if (rv != SECSuccess)
 	goto done;
 
     rv = RSA_PrivateKeyOpDoubleChecked(&key->u.rsa, output, pss_encoded);
     if (rv != SECSuccess && PORT_GetError() == SEC_ERROR_LIBRARY_FAILURE) {
 	sftk_fatalError = PR_TRUE;
     }
     *output_len = modulus_len;
 
