mono-core/mono-mcs_r85721_biginteger_security_fix.patch

Index: mcs/class/corlib/Mono.Math.Prime.Generator/PrimeGeneratorBase.cs
===================================================================
--- mcs/class/corlib/Mono.Math.Prime.Generator/PrimeGeneratorBase.cs	(revision 85720)
+++ mcs/class/corlib/Mono.Math.Prime.Generator/PrimeGeneratorBase.cs	(revision 85721)
@@ -8,8 +8,6 @@
 //
 
 //
-// Copyright (C) 2004 Novell, Inc (http://www.novell.com)
-//
 // Permission is hereby granted, free of charge, to any person obtaining
 // a copy of this software and associated documentation files (the
 // "Software"), to deal in the Software without restriction, including
Index: mcs/class/corlib/Mono.Math.Prime.Generator/ChangeLog
===================================================================
--- mcs/class/corlib/Mono.Math.Prime.Generator/ChangeLog	(revision 85720)
+++ mcs/class/corlib/Mono.Math.Prime.Generator/ChangeLog	(revision 85721)
@@ -1,3 +1,8 @@
+2007-09-12  Sebastien Pouliot  <sebastien@ximian.com>
+
+	* NextPrimeFinder.cs, PrimeGeneratorBase.cs:  Updated to match 
+	Mono.Security sources.
+
 2004-09-16  Sebastien Pouliot  <sebastien@ximian.com>
 
 	* SequentialSearchPrimeGeneratorBase.cs: Fixed warning (l4) for unused
Index: mcs/class/corlib/Mono.Math.Prime.Generator/NextPrimeFinder.cs
===================================================================
--- mcs/class/corlib/Mono.Math.Prime.Generator/NextPrimeFinder.cs	(revision 85720)
+++ mcs/class/corlib/Mono.Math.Prime.Generator/NextPrimeFinder.cs	(revision 85721)
@@ -8,8 +8,6 @@
 //
 
 //
-// Copyright (C) 2004 Novell, Inc (http://www.novell.com)
-//
 // Permission is hereby granted, free of charge, to any person obtaining
 // a copy of this software and associated documentation files (the
 // "Software"), to deal in the Software without restriction, including
Index: mcs/class/corlib/Mono.Math.Prime/ConfidenceFactor.cs
===================================================================
--- mcs/class/corlib/Mono.Math.Prime/ConfidenceFactor.cs	(revision 85720)
+++ mcs/class/corlib/Mono.Math.Prime/ConfidenceFactor.cs	(revision 85721)
@@ -8,8 +8,6 @@
 //
 
 //
-// Copyright (C) 2004 Novell, Inc (http://www.novell.com)
-//
 // Permission is hereby granted, free of charge, to any person obtaining
 // a copy of this software and associated documentation files (the
 // "Software"), to deal in the Software without restriction, including
Index: mcs/class/corlib/Mono.Math.Prime/PrimalityTests.cs
===================================================================
--- mcs/class/corlib/Mono.Math.Prime/PrimalityTests.cs	(revision 85720)
+++ mcs/class/corlib/Mono.Math.Prime/PrimalityTests.cs	(revision 85721)
@@ -8,8 +8,6 @@
 //
 
 //
-// Copyright (C) 2004 Novell, Inc (http://www.novell.com)
-//
 // Permission is hereby granted, free of charge, to any person obtaining
 // a copy of this software and associated documentation files (the
 // "Software"), to deal in the Software without restriction, including
@@ -94,10 +92,19 @@
 			}
 		}
 
+		public static bool Test (BigInteger n, ConfidenceFactor confidence)
+		{
+			// Rabin-Miller fails with smaller primes (at least with our BigInteger code)
+			if (n.BitCount () < 33)
+				return SmallPrimeSppTest (n, confidence);
+			else
+				return RabinMillerTest (n, confidence);
+		}
+
 		/// <summary>
 		///     Probabilistic prime test based on Rabin-Miller's test
 		/// </summary>
-		/// <param name="bi" type="BigInteger.BigInteger">
+		/// <param name="n" type="BigInteger.BigInteger">
 		///     <para>
 		///         The number to test.
 		///     </para>
@@ -116,67 +123,51 @@
 		///		False if "this" is definitely NOT prime.
 		///	</para>
 		/// </returns>
-		public static bool RabinMillerTest (BigInteger bi, ConfidenceFactor confidence)
+		public static bool RabinMillerTest (BigInteger n, ConfidenceFactor confidence)
 		{
-			int Rounds = GetSPPRounds (bi, confidence);
+			int bits = n.BitCount ();
+			int t = GetSPPRounds (bits, confidence);
 
-			// calculate values of s and t
-			BigInteger p_sub1 = bi - 1;
-			int s = p_sub1.LowestSetBit ();
+			// n - 1 == 2^s * r, r is odd
+			BigInteger n_minus_1 = n - 1;
+			int s = n_minus_1.LowestSetBit ();
+			BigInteger r = n_minus_1 >> s;
 
-			BigInteger t = p_sub1 >> s;
-
-			int bits = bi.BitCount ();
-			BigInteger a = null;
-			BigInteger.ModulusRing mr = new BigInteger.ModulusRing (bi);
+			BigInteger.ModulusRing mr = new BigInteger.ModulusRing (n);
 			
 			// Applying optimization from HAC section 4.50 (base == 2)
 			// not a really random base but an interesting (and speedy) one
-			BigInteger b = mr.Pow (2, t);
-			if (b != 1) {
-				bool result = false;
-				for (int j=0; j < s; j++) {
-					if (b == p_sub1) {         // a^((2^j)*t) mod p = p-1 for some 0 <= j <= s-1
-						result = true;
-						break;
-					}
+			BigInteger y = null;
+			// FIXME - optimization disable for small primes due to bug #81857
+			if (n.BitCount () > 100)
+				y = mr.Pow (2, r);
 
-					b = (b * b) % bi;
-				}
-				if (!result)
-					return false;
-			}
-
 			// still here ? start at round 1 (round 0 was a == 2)
-			for (int round = 1; round < Rounds; round++) {
-				while (true) {		           // generate a < n
-					a = BigInteger.GenerateRandom (bits);
+			for (int round = 0; round < t; round++) {
 
-					// make sure "a" is not 0 (and not 2 as we have already tested that)
-					if (a > 2 && a < bi)
-						break;
-				}
+				if ((round > 0) || (y == null)) {
+					BigInteger a = null;
 
-				if (a.GCD (bi) != 1)
-					return false;
+					// check for 2 <= a <= n - 2
+					// ...but we already did a == 2 previously as an optimization
+					do {
+						a = BigInteger.GenerateRandom (bits);
+					} while ((a <= 2) && (a >= n_minus_1));
 
-				b = mr.Pow (a, t);
+					y = mr.Pow (a, r);
+				}
 
-				if (b == 1)
-					continue;              // a^t mod p = 1
+				if (y == 1)
+					continue;
 
-				bool result = false;
-				for (int j = 0; j < s; j++) {
+				for (int j = 0; ((j < s) && (y != n_minus_1)); j++) {
 
-					if (b == p_sub1) {         // a^((2^j)*t) mod p = p-1 for some 0 <= j <= s-1
-						result = true;
-						break;
-					}
-
-					b = (b * b) % bi;
+					y = mr.Pow (y, 2);
+					if (y == 1)
+						return false;
 				}
 
-				if (!result)
+				if (y != n_minus_1)
 					return false;
 			}
 			return true;
Index: mcs/class/corlib/Mono.Math.Prime/ChangeLog
===================================================================
--- mcs/class/corlib/Mono.Math.Prime/ChangeLog	(revision 85720)
+++ mcs/class/corlib/Mono.Math.Prime/ChangeLog	(revision 85721)
@@ -1,3 +1,8 @@
+2007-09-12  Sebastien Pouliot  <sebastien@ximian.com>
+
+	* ConfidenceFactor.cs, PrimalityTests.cs: Updated to match 
+	Mono.Security sources.
+
 2004-05-07  Sebastien Pouliot  <sebastien@ximian.com>
 
 	* PrimalityTests.cs: In sync with Mono.Security.dll version.
Index: mcs/class/corlib/Mono.Math/BigInteger.cs
===================================================================
--- mcs/class/corlib/Mono.Math/BigInteger.cs	(revision 85720)
+++ mcs/class/corlib/Mono.Math/BigInteger.cs	(revision 85721)
@@ -13,7 +13,7 @@
 // Copyright (c) 2002 Chew Keong TAN
 // All rights reserved.
 //
-// Copyright (C) 2004 Novell, Inc (http://www.novell.com)
+// Copyright (C) 2004, 2007 Novell, Inc (http://www.novell.com)
 //
 // Permission is hereby granted, free of charge, to any person obtaining
 // a copy of this software and associated documentation files (the
@@ -862,10 +862,16 @@
 
 		public override bool Equals (object o)
 		{
-			if (o == null) return false;
-			if (o is int) return (int)o >= 0 && this == (uint)o;
+			if (o == null)
+				return false;
+			if (o is int)
+				return (int)o >= 0 && this == (uint)o;
 
-			return Kernel.Compare (this, (BigInteger)o) == 0;
+			BigInteger bi = o as BigInteger;
+			if (bi == null)
+				return false;
+			
+			return Kernel.Compare (this, bi) == 0;
 		}
 
 		#endregion
@@ -894,19 +900,23 @@
 
 		public bool IsProbablePrime ()
 		{
-			if (this < smallPrimes [smallPrimes.Length - 1]) {
+			// can we use our small-prime table ?
+			if (this <= smallPrimes[smallPrimes.Length - 1]) {
 				for (int p = 0; p < smallPrimes.Length; p++) {
-					if (this == smallPrimes [p])
+					if (this == smallPrimes[p])
 						return true;
 				}
+				// the list is complete, so it's not a prime
+				return false;
 			}
-			else {
-				for (int p = 0; p < smallPrimes.Length; p++) {
-					if (this % smallPrimes [p] == 0)
-						return false;
-				}
+
+			// otherwise check if we can divide by one of the small primes
+			for (int p = 0; p < smallPrimes.Length; p++) {
+				if (this % smallPrimes[p] == 0)
+					return false;
 			}
-			return PrimalityTests.RabinMillerTest (this, Prime.ConfidenceFactor.Medium);
+			// the last step is to confirm the "large" prime with the SPP or Miller-Rabin test
+			return PrimalityTests.Test (this, Prime.ConfidenceFactor.Medium);
 		}
 
 		#endregion
@@ -1038,18 +1048,12 @@
 			{
 				if (a == 0 || b == 0) return 0;
 
-				if (a.length >= mod.length << 1)
+				if (a > mod)
 					a %= mod;
 
-				if (b.length >= mod.length << 1)
+				if (b > mod)
 					b %= mod;
 
-				if (a.length >= mod.length)
-					BarrettReduction (a);
-
-				if (b.length >= mod.length)
-					BarrettReduction (b);
-
 				BigInteger ret = new BigInteger (a * b);
 				BarrettReduction (ret);
 
@@ -1082,7 +1086,25 @@
 					diff = mod - diff;
 				return diff;
 			}
+#if true
+			public BigInteger Pow (BigInteger a, BigInteger k)
+			{
+				BigInteger b = new BigInteger (1);
+				if (k == 0)
+					return b;
 
+				BigInteger A = a;
+				if (k.TestBit (0))
+					b = a;
+
+				for (int i = 1; i < k.BitCount (); i++) {
+					A = Multiply (A, A);
+					if (k.TestBit (i))
+						b = Multiply (A, b);
+				}
+				return b;
+			}
+#else
 			public BigInteger Pow (BigInteger b, BigInteger exp)
 			{
 				if ((mod.data [0] & 1) == 1) return OddPow (b, exp);
@@ -1146,14 +1168,17 @@
 						Montgomery.Reduce (resultNum, mod, mPrime);
 					}
 
-					Kernel.SquarePositive (tempNum, ref wkspace);
-					Montgomery.Reduce (tempNum, mod, mPrime);
+					// the value of tempNum is required in the last loop
+					if (pos < totalBits - 1) {
+						Kernel.SquarePositive (tempNum, ref wkspace);
+						Montgomery.Reduce (tempNum, mod, mPrime);
+					}
 				}
 
 				Montgomery.Reduce (resultNum, mod, mPrime);
 				return resultNum;
 			}
-
+#endif
 			#region Pow Small Base
 
 			// TODO: Make tests for this, not really needed b/c prime stuff
@@ -1161,14 +1186,20 @@
 #if !INSIDE_CORLIB
                         [CLSCompliant (false)]
 #endif 
+#if true
 			public BigInteger Pow (uint b, BigInteger exp)
 			{
+				return Pow (new BigInteger (b), exp);
+			}
+#else
+			public BigInteger Pow (uint b, BigInteger exp)
+			{
 //				if (b != 2) {
 					if ((mod.data [0] & 1) == 1)
 						return OddPow (b, exp);
 					else
 						return EvenPow (b, exp);
-/* buggy in some cases (like the well tested primes)
+/* buggy in some cases (like the well tested primes) 
 				} else {
 					if ((mod.data [0] & 1) == 1)
 						return OddModTwoPow (exp);
@@ -1187,7 +1218,8 @@
 
 				uint mPrime = Montgomery.Inverse (mod.data [0]);
 
-				uint pos = (uint)exp.BitCount () - 2;
+				int bc = exp.BitCount () - 2;
+				uint pos = (bc > 1 ? (uint) bc : 1);
 
 				//
 				// We know that the first itr will make the val b
@@ -1387,8 +1419,9 @@
 
 				return resultNum;
 			}
-
-/* known to be buggy in some cases
+#endif
+/* known to be buggy in some cases */
+#if false
 			private unsafe BigInteger EvenModTwoPow (BigInteger exp)
 			{
 				exp.Normalize ();
@@ -1521,7 +1554,7 @@
 				resultNum = Montgomery.Reduce (resultNum, mod, mPrime);
 				return resultNum;
 			}
-*/			
+#endif
 			#endregion
 		}
 
Index: mcs/class/corlib/Mono.Math/ChangeLog
===================================================================
--- mcs/class/corlib/Mono.Math/ChangeLog	(revision 85720)
+++ mcs/class/corlib/Mono.Math/ChangeLog	(revision 85721)
@@ -1,3 +1,12 @@
+2007-09-12  Sebastien Pouliot  <sebastien@ximian.com>
+
+	* BigInteger.cs: Updated to match Mono.Security sources.
+
+2007-07-28  Miguel de Icaza  <miguel@novell.com>
+
+	* BigInteger.cs: Do not cast inside Equals,instead use the as
+	operator, as reported by Jesse Jones.
+
 2004-12-03  Sebastien Pouliot  <sebastien@ximian.com>
 
 	* BigInteger.cs: Fix issue #70169 in ModPow when modulus is a power of