rpm/beedigest.diff

--- rpmio/digest.h.orig	2013-08-27 14:26:38.260872601 +0000
+++ rpmio/digest.h	2013-08-27 14:26:47.404872585 +0000
@@ -5,8 +5,7 @@
 
 typedef struct pgpDigAlg_s * pgpDigAlg;
 
-typedef int (*setmpifunc)(pgpDigAlg digp,
-                          int num, const uint8_t *p, const uint8_t *pend);
+typedef int (*setmpifunc)(pgpDigAlg digp, int num, const uint8_t *p);
 typedef int (*verifyfunc)(pgpDigAlg pgpkey, pgpDigAlg pgpsig,
                           uint8_t *hash, size_t hashlen, int hash_algo);
 typedef void (*freefunc)(pgpDigAlg digp);
--- rpmio/digest_beecrypt.c.orig	2013-08-27 14:26:38.260872601 +0000
+++ rpmio/digest_beecrypt.c	2013-08-29 15:45:42.754558355 +0000
@@ -198,66 +198,6 @@ int rpmDigestFinal(DIGEST_CTX ctx, void
     return 0;
 }
 
-/**************************** helpers ************************************/
-
-static inline char * pgpHexCvt(char *t, const byte *s, int nbytes)
-{
-    static char hex[] = "0123456789abcdef";
-    while (nbytes-- > 0) { 
-        unsigned int i;
-        i = *s++;
-        *t++ = hex[ (i >> 4) & 0xf ];
-        *t++ = hex[ (i     ) & 0xf ];
-    }    
-    *t = '\0';
-    return t;
-}
-
-static const char * pgpMpiHex(const byte *p, const byte *pend)
-{
-    static char prbuf[2048];
-    char *t = prbuf;
-    int nbytes =  pgpMpiLen(p) - 2;
-    if (nbytes > 1024 || nbytes > pend - (p + 2))
-	return NULL;
-    t = pgpHexCvt(t, p+2, nbytes);
-    return prbuf;
-}
-
-static int pgpHexSet(int lbits, mpnumber * mpn, const byte * p, const byte * pend)
-{
-    unsigned int mbits = pgpMpiBits(p);
-    unsigned int nbits;
-    unsigned int nbytes;
-    char *t;
-    unsigned int ix;
-
-    nbits = (lbits > mbits ? lbits : mbits);
-    nbytes = ((nbits + 7) >> 3);
-    t = xmalloc(2*nbytes+1);
-    ix = 2 * ((nbits - mbits) >> 3);
-
-    if (ix > 0) memset(t, (int)'0', ix);
-    strcpy(t+ix, pgpMpiHex(p, pend));
-    (void) mpnsethex(mpn, t);
-    t = _free(t);
-    return 0;
-}
-
-static void pgpFreeSigRSADSA(pgpDigAlg sa)
-{
-    if (sa->data)
-	free(sa->data);
-    sa->data = 0;
-}
-
-static void pgpFreeKeyRSADSA(pgpDigAlg sa)
-{
-    if (sa->data)
-	free(sa->data);
-    sa->data = 0;
-}
-
 
 /****************************** RSA **************************************/
 
@@ -267,45 +207,73 @@ struct pgpDigSigRSA_s {
 
 struct pgpDigKeyRSA_s {
     rsapk rsa_pk;
+    int nbytes;
 };
 
-static int pgpSetSigMpiRSA(pgpDigAlg pgpsig, int num,
-                           const uint8_t *p, const uint8_t *pend)
+static int pgpSetSigMpiRSA(pgpDigAlg pgpsig, int num, const uint8_t *p)
 {
     struct pgpDigSigRSA_s *sig = pgpsig->data;
+    int mlen = pgpMpiLen(p) - 2;
     int rc = 1;
 
+    if (!sig)
+	sig = pgpsig->data = xcalloc(1, sizeof(*sig));
+
     switch (num) {
     case 0:
-	sig = pgpsig->data = xcalloc(1, sizeof(*sig));
-	(void) mpnsethex(&sig->c, pgpMpiHex(p, pend));
-	rc = 0;
+	if (!mpnsetbin(&sig->c, p + 2, mlen))
+	    rc = 0;
 	break;
     }
     return rc;
 }
 
-static int pgpSetKeyMpiRSA(pgpDigAlg pgpkey, int num,
-                           const uint8_t *p, const uint8_t *pend)
+static int pgpSetKeyMpiRSA(pgpDigAlg pgpkey, int num, const uint8_t *p)
 {
     struct pgpDigKeyRSA_s *key = pgpkey->data;
+    int mlen = pgpMpiLen(p) - 2;
     int rc = 1;
 
     if (!key)
 	key = pgpkey->data = xcalloc(1, sizeof(*key));
+
     switch (num) {
     case 0:
-	(void) mpbsethex(&key->rsa_pk.n, pgpMpiHex(p, pend));
-	rc = 0;
+	key->nbytes = mlen;
+	if (!mpbsetbin(&key->rsa_pk.n, p + 2, mlen))
+	    rc = 0;
 	break;
     case 1:
-	(void) mpnsethex(&key->rsa_pk.e, pgpMpiHex(p, pend));
-	rc = 0;
+	if (!mpnsetbin(&key->rsa_pk.e, p + 2, mlen))
+	    rc = 0;
 	break;
     }
     return rc;
 }
 
+static int pkcs1pad(mpnumber *rsahm, int nbytes, const char *prefix, uint8_t *hash, size_t hashlen)
+{
+    int datalen = strlen(prefix) / 2 + hashlen;
+    byte *buf, *bp;
+    int rc = 1;
+    
+    if (nbytes < 4 + datalen)
+	return 1;
+    buf = xmalloc(nbytes);
+    memset(buf, 0xff, nbytes);
+    buf[0] = 0x00;
+    buf[1] = 0x01;
+    bp = buf + nbytes - datalen;
+    bp[-1] = 0;
+    for (; *prefix; prefix += 2)
+	*bp++ = (rnibble(prefix[0]) << 4) | rnibble(prefix[1]);
+    memcpy(bp, hash, hashlen);
+    if (!mpnsetbin(rsahm, buf, nbytes))
+	rc = 0;
+    buf = _free(buf);
+    return rc;
+}
+
 static int pgpVerifySigRSA(pgpDigAlg pgpkey, pgpDigAlg pgpsig, uint8_t *hash, size_t hashlen, int hash_algo)
 {
     struct pgpDigKeyRSA_s *key = pgpkey->data;
@@ -340,28 +308,10 @@ static int pgpVerifySigRSA(pgpDigAlg pgp
 	return 1;
     }
 
-    /* Generate RSA modulus parameter. */
-    {   unsigned int nbits = MP_WORDS_TO_BITS(sig->c.size);
-        unsigned int nb = (nbits + 7) >> 3;
-        byte *buf, *bp;
+    memset(&rsahm, 0, sizeof(rsahm));
+    if (pkcs1pad(&rsahm, key->nbytes, prefix, hash, hashlen) != 0)
+	return 1;
 
-	if (nb < 3)
-	    return 1;
-	buf = xmalloc(nb);
-	memset(buf, 0xff, nb);
-	buf[0] = 0x00;
-	buf[1] = 0x01;
-	bp = buf + nb - strlen(prefix)/2 - hashlen - 1;
-	if (bp < buf)
-	    return 1;
-	*bp++ = 0;
-	for (; *prefix; prefix += 2)
-	    *bp++ = (rnibble(prefix[0]) << 4) | rnibble(prefix[1]);
-        memcpy(bp, hash, hashlen);
-        mpnzero(&rsahm);
-        (void) mpnsetbin(&rsahm, buf, nb);
-        buf = _free(buf);
-    }
 #if HAVE_BEECRYPT_API_H
     rc = rsavrfy(&key->rsa_pk.n, &key->rsa_pk.e, &sig->c, &rsahm) == 1 ? 0 : 1;
 #else
@@ -371,6 +321,25 @@ static int pgpVerifySigRSA(pgpDigAlg pgp
     return rc;
 }
 
+static void pgpFreeSigRSA(pgpDigAlg pgpsig)
+{
+    struct pgpDigSigRSA_s *sig = pgpsig->data;
+    if (sig) {
+	mpnfree(&sig->c);
+	pgpsig->data = _free(sig);
+    }
+}
+
+static void pgpFreeKeyRSA(pgpDigAlg pgpkey)
+{
+    struct pgpDigKeyRSA_s *key = pgpkey->data;
+    if (key) {
+	mpbfree(&key->rsa_pk.n);
+	mpnfree(&key->rsa_pk.e);
+	pgpkey->data = _free(key);
+    }
+}
+
 
 /****************************** DSA **************************************/
 
@@ -384,30 +353,35 @@ struct pgpDigKeyDSA_s {
     mpbarrett q;
     mpnumber g;
     mpnumber y;
+    int qbytes;
 };
 
-static int pgpSetSigMpiDSA(pgpDigAlg pgpsig, int num,
-                           const uint8_t *p, const uint8_t *pend)
+static int pgpSetSigMpiDSA(pgpDigAlg pgpsig, int num, const uint8_t *p)
 {
     struct pgpDigSigDSA_s *sig = pgpsig->data;
+    int mlen = pgpMpiLen(p) - 2;
     int rc = 1;
 
+    if (!sig)
+	sig = pgpsig->data = xcalloc(1, sizeof(*sig));
+
     switch (num) {
     case 0:
-	sig = pgpsig->data = xcalloc(1, sizeof(*sig));
-	rc = pgpHexSet(160, &sig->r, p, pend);
+	if (!mpnsetbin(&sig->r, p + 2, mlen))
+	    rc = 0;
 	break;
     case 1:
-	rc = pgpHexSet(160, &sig->s, p, pend);
+	if (!mpnsetbin(&sig->s, p + 2, mlen))
+	    rc = 0;
 	break;
     }
     return rc;
 }
 
-static int pgpSetKeyMpiDSA(pgpDigAlg pgpkey, int num,
-                           const uint8_t *p, const uint8_t *pend)
+static int pgpSetKeyMpiDSA(pgpDigAlg pgpkey, int num, const uint8_t *p)
 {
     struct pgpDigKeyDSA_s *key = pgpkey->data;
+    int mlen = pgpMpiLen(p) - 2;
     int rc = 1;
 
     if (!key)
@@ -415,20 +389,21 @@ static int pgpSetKeyMpiDSA(pgpDigAlg pgp
 
     switch (num) {
     case 0:
-	mpbsethex(&key->p, pgpMpiHex(p, pend));
-	rc = 0;
+	if (!mpbsetbin(&key->p, p + 2, mlen))
+	    rc = 0;
 	break;
     case 1:
-	mpbsethex(&key->q, pgpMpiHex(p, pend));
-	rc = 0;
+	key->qbytes = mlen;
+	if (!mpbsetbin(&key->q, p + 2, mlen))
+	    rc = 0;
 	break;
     case 2:
-	mpnsethex(&key->g, pgpMpiHex(p, pend));
-	rc = 0;
+	if (!mpnsetbin(&key->g, p + 2, mlen))
+	    rc = 0;
 	break;
     case 3:
-	mpnsethex(&key->y, pgpMpiHex(p, pend));
-	rc = 0;
+	if (!mpnsetbin(&key->y, p + 2, mlen))
+	    rc = 0;
 	break;
     }
     return rc;
@@ -441,17 +416,41 @@ static int pgpVerifySigDSA(pgpDigAlg pgp
     mpnumber hm;
     int rc = 1;
 
-    if (sig && key) {
+    if (sig && key && hashlen >= key->qbytes) {
 	mpnzero(&hm);
-	mpnsetbin(&hm, hash, hashlen);
+	mpnsetbin(&hm, hash, key->qbytes);
 	rc = dsavrfy(&key->p, &key->q, &key->g, &hm, &key->y, &sig->r, &sig->s) == 1 ? 0 : 1;
 	mpnfree(&hm);
     }
     return rc;
 }
 
-static int pgpSetMpiNULL(pgpDigAlg pgpkey, int num,
-                         const uint8_t *p, const uint8_t *pend)
+static void pgpFreeSigDSA(pgpDigAlg pgpsig)
+{
+    struct pgpDigSigDSA_s *sig = pgpsig->data;
+    if (sig) {
+	mpnfree(&sig->r);
+	mpnfree(&sig->s);
+	pgpsig->data = _free(sig);
+    }
+}
+
+static void pgpFreeKeyDSA(pgpDigAlg pgpkey)
+{
+    struct pgpDigKeyDSA_s *key = pgpkey->data;
+    if (key) {
+	mpbfree(&key->p);
+	mpbfree(&key->q);
+	mpnfree(&key->g);
+	mpnfree(&key->y);
+	pgpkey->data = _free(key);
+    }
+}
+
+
+/****************************** NULL **************************************/
+
+static int pgpSetMpiNULL(pgpDigAlg pgpkey, int num, const uint8_t *p)
 {
     return 1;
 }
@@ -469,12 +468,12 @@ pgpDigAlg pgpPubkeyNew(int algo)
     switch (algo) {
     case PGPPUBKEYALGO_RSA:
         ka->setmpi = pgpSetKeyMpiRSA;
-        ka->free = pgpFreeKeyRSADSA;
+        ka->free = pgpFreeKeyRSA;
         ka->mpis = 2;
         break;
     case PGPPUBKEYALGO_DSA:
         ka->setmpi = pgpSetKeyMpiDSA;
-        ka->free = pgpFreeKeyRSADSA;
+        ka->free = pgpFreeKeyDSA;
         ka->mpis = 4;
         break;
     default:
@@ -495,13 +494,13 @@ pgpDigAlg pgpSignatureNew(int algo)
     switch (algo) {
     case PGPPUBKEYALGO_RSA:
         sa->setmpi = pgpSetSigMpiRSA;
-        sa->free = pgpFreeSigRSADSA;
+        sa->free = pgpFreeSigRSA;
         sa->verify = pgpVerifySigRSA;
         sa->mpis = 1;
         break;
     case PGPPUBKEYALGO_DSA:
         sa->setmpi = pgpSetSigMpiDSA;
-        sa->free = pgpFreeSigRSADSA;
+        sa->free = pgpFreeSigDSA;
         sa->verify = pgpVerifySigDSA;
         sa->mpis = 2;
         break;
--- rpmio/digest_nss.c.orig	2013-08-27 14:26:38.260872601 +0000
+++ rpmio/digest_nss.c	2013-08-27 14:26:47.405872585 +0000
@@ -224,8 +224,7 @@ static SECOidTag getHashAlg(unsigned int
     return SEC_OID_UNKNOWN;
 }
 
-static int pgpMpiSet(unsigned int lbits, uint8_t *dest,
-		     const uint8_t * p, const uint8_t * pend)
+static int pgpMpiSet(unsigned int lbits, uint8_t *dest, const uint8_t * p)
 {
     unsigned int mbits = pgpMpiBits(p);
     unsigned int nbits;
@@ -233,9 +232,6 @@ static int pgpMpiSet(unsigned int lbits,
     uint8_t *t = dest;
     unsigned int ix;
 
-    if ((p + ((mbits+7) >> 3)) > pend)
-	return 1;
-
     if (mbits > lbits)
 	return 1;
 
@@ -250,14 +246,10 @@ static int pgpMpiSet(unsigned int lbits,
     return 0;
 }
 
-static SECItem *pgpMpiItem(PRArenaPool *arena, SECItem *item,
-			   const uint8_t *p, const uint8_t *pend)
+static SECItem *pgpMpiItem(PRArenaPool *arena, SECItem *item, const uint8_t *p)
 {
     size_t nbytes = pgpMpiLen(p)-2;
 
-    if (p + nbytes + 2 > pend)
-	return NULL;
-
     if (item == NULL) {
     	if ((item=SECITEM_AllocItem(arena, item, nbytes)) == NULL)
     	    return item;
@@ -313,8 +305,7 @@ static SECKEYPublicKey *pgpNewPublicKey(
 #define DSA1_Q_BITS DSA_Q_BITS
 #endif
 
-static int pgpSetSigMpiDSA(pgpDigAlg pgpsig, int num,
-			   const uint8_t *p, const uint8_t *pend)
+static int pgpSetSigMpiDSA(pgpDigAlg pgpsig, int num, const uint8_t *p)
 {
     SECItem *sig = pgpsig->data;
     int lbits = DSA1_Q_BITS;
@@ -325,11 +316,11 @@ static int pgpSetSigMpiDSA(pgpDigAlg pgp
 	sig = pgpsig->data = SECITEM_AllocItem(NULL, NULL, DSA1_SIGNATURE_LEN);
 	if (sig) {
 	    memset(sig->data, 0, DSA1_SIGNATURE_LEN);
-	    rc = pgpMpiSet(lbits, sig->data, p, pend);
+	    rc = pgpMpiSet(lbits, sig->data, p);
 	}
 	break;
     case 1:
-	if (sig && pgpMpiSet(lbits, sig->data+DSA1_SUBPRIME_LEN, p, pend) == 0) {
+	if (sig && pgpMpiSet(lbits, sig->data+DSA1_SUBPRIME_LEN, p) == 0) {
 	    SECItem *signew = SECITEM_AllocItem(NULL, NULL, 0);
 	    if (signew && DSAU_EncodeDerSig(signew, sig) == SECSuccess) {
 		SECITEM_FreeItem(sig, PR_TRUE);
@@ -343,8 +334,7 @@ static int pgpSetSigMpiDSA(pgpDigAlg pgp
     return rc;
 }
 
-static int pgpSetKeyMpiDSA(pgpDigAlg pgpkey, int num,
-			   const uint8_t *p, const uint8_t *pend)
+static int pgpSetKeyMpiDSA(pgpDigAlg pgpkey, int num, const uint8_t *p)
 {
     SECItem *mpi = NULL;
     SECKEYPublicKey *key = pgpkey->data;
@@ -355,16 +345,16 @@ static int pgpSetKeyMpiDSA(pgpDigAlg pgp
     if (key) {
 	switch (num) {
 	case 0:
-	    mpi = pgpMpiItem(key->arena, &key->u.dsa.params.prime, p, pend);
+	    mpi = pgpMpiItem(key->arena, &key->u.dsa.params.prime, p);
 	    break;
 	case 1:
-	    mpi = pgpMpiItem(key->arena, &key->u.dsa.params.subPrime, p, pend);
+	    mpi = pgpMpiItem(key->arena, &key->u.dsa.params.subPrime, p);
 	    break;
 	case 2:
-	    mpi = pgpMpiItem(key->arena, &key->u.dsa.params.base, p, pend);
+	    mpi = pgpMpiItem(key->arena, &key->u.dsa.params.base, p);
 	    break;
 	case 3:
-	    mpi = pgpMpiItem(key->arena, &key->u.dsa.publicValue, p, pend);
+	    mpi = pgpMpiItem(key->arena, &key->u.dsa.publicValue, p);
 	    break;
 	}
     }
@@ -389,21 +379,19 @@ static int pgpVerifySigDSA(pgpDigAlg pgp
     return (rc != SECSuccess);
 }
 
-static int pgpSetSigMpiRSA(pgpDigAlg pgpsig, int num,
-			   const uint8_t *p, const uint8_t *pend)
+static int pgpSetSigMpiRSA(pgpDigAlg pgpsig, int num, const uint8_t *p)
 {
     SECItem *sigitem = NULL;
 
     if (num == 0) {
-       sigitem = pgpMpiItem(NULL, pgpsig->data, p, pend);
+       sigitem = pgpMpiItem(NULL, pgpsig->data, p);
        if (sigitem)
            pgpsig->data = sigitem;
     }
     return (sigitem == NULL);
 }
 
-static int pgpSetKeyMpiRSA(pgpDigAlg pgpkey, int num,
-			   const uint8_t *p, const uint8_t *pend)
+static int pgpSetKeyMpiRSA(pgpDigAlg pgpkey, int num, const uint8_t *p)
 {
     SECItem *kitem = NULL;
     SECKEYPublicKey *key = pgpkey->data;
@@ -414,10 +402,10 @@ static int pgpSetKeyMpiRSA(pgpDigAlg pgp
     if (key) {
 	switch (num) {
 	case 0:
-	    kitem = pgpMpiItem(key->arena, &key->u.rsa.modulus, p, pend);
+	    kitem = pgpMpiItem(key->arena, &key->u.rsa.modulus, p);
 	    break;
 	case 1:
-	    kitem = pgpMpiItem(key->arena, &key->u.rsa.publicExponent, p, pend);
+	    kitem = pgpMpiItem(key->arena, &key->u.rsa.publicExponent, p);
 	    break;
 	}
     }
@@ -472,8 +460,7 @@ static void pgpFreeKeyRSADSA(pgpDigAlg k
     ka->data = NULL;
 }
 
-static int pgpSetMpiNULL(pgpDigAlg pgpkey, int num,
-			 const uint8_t *p, const uint8_t *pend)
+static int pgpSetMpiNULL(pgpDigAlg pgpkey, int num, const uint8_t *p)
 {
     return 1;
 }
--- rpmio/rpmpgp.c.orig	2013-08-27 14:26:38.260872601 +0000
+++ rpmio/rpmpgp.c	2013-08-29 15:48:33.394558053 +0000
@@ -499,11 +499,15 @@ static int pgpPrtSigParams(pgpTag tag, u
     int i;
     pgpDigAlg sigalg = pgpSignatureNew(pubkey_algo);
 
-    for (i = 0; p < pend && i < sigalg->mpis; i++, p += pgpMpiLen(p)) {
+    for (i = 0; i < sigalg->mpis && p + 2 <= pend; i++) {
+        int mpil = pgpMpiLen(p);
+        if (p + mpil > pend)
+	    break;
 	if (sigtype == PGPSIGTYPE_BINARY || sigtype == PGPSIGTYPE_TEXT) {
-	    if (sigalg->setmpi(sigalg, i, p, pend))
+	    if (sigalg->setmpi(sigalg, i, p))
 		break;
 	}
+	p += mpil;
     }
 
     /* Does the size and number of MPI's match our expectations? */
@@ -650,9 +654,13 @@ static int pgpPrtPubkeyParams(uint8_t pu
     int i;
     pgpDigAlg keyalg = pgpPubkeyNew(pubkey_algo);
 
-    for (i = 0; p < pend && i < keyalg->mpis; i++, p += pgpMpiLen(p)) {
-	if (keyalg->setmpi(keyalg, i, p, pend))
+    for (i = 0; i < keyalg->mpis && p + 2 <= pend; i++) {
+        int mpil = pgpMpiLen(p);
+        if (p + mpil > pend)
+	    break;
+	if (keyalg->setmpi(keyalg, i, p))
 	    break;
+	p += mpil;
     }
 
     /* Does the size and number of MPI's match our expectations? */