mozilla-nss/nss-fips-tls-allow-md5-prf.patch

# HG changeset patch
# User Hans Petter Jansson <hpj@cl.no>
# Date 1574240734 -3600
#      Wed Nov 20 10:05:34 2019 +0100
# Node ID 0efca22bbafd7575b20461f255c46157c9321822
# Parent  3a2cb65dc157344cdad19e8e16e9c33e36f82d96
[PATCH] 30
From ca3b695ac461eccf4ed97e1b3fe0a311c80a792f Mon Sep 17 00:00:00 2001
---
 nss/lib/freebl/md5.c       | 67 ++++++++++++++++++++++++++------------
 nss/lib/freebl/rawhash.c   | 37 +++++++++++++++++++++
 nss/lib/freebl/tlsprfalg.c |  5 ++-
 nss/lib/softoken/pkcs11c.c |  4 +--
 4 files changed, 90 insertions(+), 23 deletions(-)

diff --git a/lib/freebl/md5.c b/lib/freebl/md5.c
--- a/lib/freebl/md5.c
+++ b/lib/freebl/md5.c
@@ -217,13 +217,11 @@
 }
 
 MD5Context *
-MD5_NewContext(void)
+MD5_NewContext_NonFIPS(void)
 {
     /* no need to ZAlloc, MD5_Begin will init the context */
     MD5Context *cx;
 
-    IN_FIPS_RETURN(NULL);
-
     cx = (MD5Context *)PORT_Alloc(sizeof(MD5Context));
     if (cx == NULL) {
         PORT_SetError(PR_OUT_OF_MEMORY_ERROR);
@@ -232,6 +230,13 @@
     return cx;
 }
 
+MD5Context *
+MD5_NewContext(void)
+{
+    IN_FIPS_RETURN(NULL);
+    return MD5_NewContext_NonFIPS();
+}
+
 void
 MD5_DestroyContext(MD5Context *cx, PRBool freeit)
 {
@@ -243,10 +248,8 @@
 }
 
 void
-MD5_Begin(MD5Context *cx)
+MD5_Begin_NonFIPS(MD5Context *cx)
 {
-    IN_FIPS_RETURN();
-
     cx->lsbInput = 0;
     cx->msbInput = 0;
     /*  memset(cx->inBuf, 0, sizeof(cx->inBuf)); */
@@ -256,6 +259,13 @@
     cx->cv[3] = CV0_4;
 }
 
+void
+MD5_Begin(MD5Context *cx)
+{
+    IN_FIPS_RETURN();
+    MD5_Begin_NonFIPS(cx);
+}
+
 #define cls(i32, s) (tmp = i32, tmp << s | tmp >> (32 - s))
 
 #if defined(SOLARIS) || defined(HPUX)
@@ -431,14 +441,12 @@
 }
 
 void
-MD5_Update(MD5Context *cx, const unsigned char *input, unsigned int inputLen)
+MD5_Update_NonFIPS(MD5Context *cx, const unsigned char *input, unsigned int inputLen)
 {
     PRUint32 bytesToConsume;
     PRUint32 inBufIndex = cx->lsbInput & 63;
     const PRUint32 *wBuf;
 
-    IN_FIPS_RETURN();
-
     /* Add the number of input bytes to the 64-bit input counter. */
     addto64(cx->msbInput, cx->lsbInput, inputLen);
     if (inBufIndex) {
@@ -487,6 +495,13 @@
         memcpy(cx->inBuf, input, inputLen);
 }
 
+void
+MD5_Update(MD5Context *cx, const unsigned char *input, unsigned int inputLen)
+{
+    IN_FIPS_RETURN();
+    MD5_Update_NonFIPS(cx, input, inputLen);
+}
+
 static const unsigned char padbytes[] = {
     0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
@@ -503,8 +518,8 @@
 };
 
 void
-MD5_End(MD5Context *cx, unsigned char *digest,
-        unsigned int *digestLen, unsigned int maxDigestLen)
+MD5_End_NonFIPS(MD5Context *cx, unsigned char *digest,
+                       unsigned int *digestLen, unsigned int maxDigestLen)
 {
 #ifndef IS_LITTLE_ENDIAN
     PRUint32 tmp;
@@ -512,8 +527,6 @@
     PRUint32 lowInput, highInput;
     PRUint32 inBufIndex = cx->lsbInput & 63;
 
-    IN_FIPS_RETURN();
-
     if (maxDigestLen < MD5_HASH_LEN) {
         PORT_SetError(SEC_ERROR_INVALID_ARGS);
         return;
@@ -525,10 +538,10 @@
     lowInput <<= 3;
 
     if (inBufIndex < MD5_END_BUFFER) {
-        MD5_Update(cx, padbytes, MD5_END_BUFFER - inBufIndex);
+        MD5_Update_NonFIPS(cx, padbytes, MD5_END_BUFFER - inBufIndex);
     } else {
-        MD5_Update(cx, padbytes,
-                   MD5_END_BUFFER + MD5_BUFFER_SIZE - inBufIndex);
+        MD5_Update_NonFIPS(cx, padbytes,
+                           MD5_END_BUFFER + MD5_BUFFER_SIZE - inBufIndex);
     }
 
     /* Store the number of bytes input (before padding) in final 64 bits. */
@@ -554,16 +567,22 @@
 }
 
 void
-MD5_EndRaw(MD5Context *cx, unsigned char *digest,
-           unsigned int *digestLen, unsigned int maxDigestLen)
+MD5_End(MD5Context *cx, unsigned char *digest,
+        unsigned int *digestLen, unsigned int maxDigestLen)
+{
+    IN_FIPS_RETURN();
+    MD5_End_NonFIPS(cx, digest, digestLen, maxDigestLen);
+}
+
+void
+MD5_EndRaw_NonFIPS(MD5Context *cx, unsigned char *digest,
+                          unsigned int *digestLen, unsigned int maxDigestLen)
 {
 #ifndef IS_LITTLE_ENDIAN
     PRUint32 tmp;
 #endif
     PRUint32 cv[4];
 
-    IN_FIPS_RETURN();
-
     if (maxDigestLen < MD5_HASH_LEN) {
         PORT_SetError(SEC_ERROR_INVALID_ARGS);
         return;
@@ -581,6 +600,14 @@
         *digestLen = MD5_HASH_LEN;
 }
 
+void
+MD5_EndRaw(MD5Context *cx, unsigned char *digest,
+           unsigned int *digestLen, unsigned int maxDigestLen)
+{
+    IN_FIPS_RETURN();
+    MD5_EndRaw_NonFIPS(cx, digest, digestLen, maxDigestLen);
+}
+
 unsigned int
 MD5_FlattenSize(MD5Context *cx)
 {
diff --git a/lib/freebl/rawhash.c b/lib/freebl/rawhash.c
--- a/lib/freebl/rawhash.c
+++ b/lib/freebl/rawhash.c
@@ -154,3 +154,40 @@
     }
     return &SECRawHashObjects[hashType];
 }
+
+/* Defined in md5.c */
+
+MD5Context *MD5_NewContext_NonFIPS(void);
+void MD5_Begin_NonFIPS(MD5Context *cx);
+void MD5_Update_NonFIPS(MD5Context *cx, const unsigned char *input, unsigned int inputLen);
+void MD5_End_NonFIPS(MD5Context *cx, unsigned char *digest,
+                            unsigned int *digestLen, unsigned int maxDigestLen);
+void MD5_EndRaw_NonFIPS(MD5Context *cx, unsigned char *digest,
+                               unsigned int *digestLen, unsigned int maxDigestLen);
+
+static const SECHashObject SECRawHashObjectMD5NonFIPS = {
+    MD5_LENGTH,
+    (void *(*)(void))MD5_NewContext_NonFIPS,
+    (void *(*)(void *))null_hash_clone_context,
+    (void (*)(void *, PRBool))MD5_DestroyContext,
+    (void (*)(void *))MD5_Begin_NonFIPS,
+    (void (*)(void *, const unsigned char *, unsigned int))MD5_Update_NonFIPS,
+    (void (*)(void *, unsigned char *, unsigned int *, unsigned int))MD5_End_NonFIPS,
+    MD5_BLOCK_LENGTH,
+    HASH_AlgMD5,
+    (void (*)(void *, unsigned char *, unsigned int *, unsigned int))MD5_EndRaw_NonFIPS
+};
+
+const SECHashObject *
+HASH_GetRawHashObjectNonFIPS(HASH_HashType hashType)
+{
+    if (hashType <= HASH_AlgNULL || hashType >= HASH_AlgTOTAL) {
+        PORT_SetError(SEC_ERROR_INVALID_ARGS);
+        return NULL;
+    }
+
+    if (hashType == HASH_AlgMD5)
+      return &SECRawHashObjectMD5NonFIPS;
+
+    return &SECRawHashObjects[hashType];
+}
diff --git a/lib/freebl/tlsprfalg.c b/lib/freebl/tlsprfalg.c
--- a/lib/freebl/tlsprfalg.c
+++ b/lib/freebl/tlsprfalg.c
@@ -12,6 +12,9 @@
 #include "hasht.h"
 #include "alghmac.h"
 
+/* To get valid MD5 object in FIPS mode */
+const SECHashObject *HASH_GetRawHashObjectNonFIPS(HASH_HashType hashType);
+
 #define PHASH_STATE_MAX_LEN HASH_LENGTH_MAX
 
 /* TLS P_hash function */
@@ -27,7 +30,7 @@
     SECStatus status;
     HMACContext *cx;
     SECStatus rv = SECFailure;
-    const SECHashObject *hashObj = HASH_GetRawHashObject(hashType);
+    const SECHashObject *hashObj = HASH_GetRawHashObjectNonFIPS(hashType);
 
     PORT_Assert((secret != NULL) && (secret->data != NULL || !secret->len));
     PORT_Assert((seed != NULL) && (seed->data != NULL));
diff --git a/lib/softoken/pkcs11c.c b/lib/softoken/pkcs11c.c
--- a/lib/softoken/pkcs11c.c
+++ b/lib/softoken/pkcs11c.c
@@ -6953,7 +6953,7 @@
     SFTKAttribute *att2 = NULL;
     unsigned char *buf;
     SHA1Context *sha;
-    MD5Context *md5;
+    MD5Context *md5 = NULL;
     MD2Context *md2;
     CK_ULONG macSize;
     CK_ULONG tmpKeySize;
@@ -7484,7 +7484,7 @@
             }
             sftk_FreeAttribute(att2);
             md5 = MD5_NewContext();
-            if (md5 == NULL) {
+            if (md5 == NULL && !isTLS) {
                 crv = CKR_HOST_MEMORY;
                 break;
             }