guard against potential integer overflow in allocators

dll.cpp

@@ -56,18 +56,9 @@ static PDelete s_pDelete = NULL;
 
 static void * CRYPTOPP_CDECL New (size_t size)
 {
-	new_handler newHandler = set_new_handler(NULL);
-	if (newHandler)
-		set_new_handler(newHandler);
-
 	void *p;
 	while (!(p = malloc(size)))
-	{
+		CallNewHandler();
-		if (newHandler)
-			newHandler();
-		else
-			throw std::bad_alloc();
-	}
 
 	return p;
 }

integer.cpp

@@ -46,14 +46,21 @@ static const char s_RunAtStartup = (AssignIntToInteger = FunctionAssignIntToInte
 template <class T>
 CPP_TYPENAME AllocatorBase<T>::pointer AlignedAllocator<T>::allocate(size_type n, const void *)
 {
+	CheckSize(n);
+	if (n == 0)
+		return NULL;
 #ifdef SSE2_INTRINSICS_AVAILABLE
 	if (n >= 4)
+	{
+		void *p;
 	#ifdef __GNUC__
-				return (T *)memalign(16, sizeof(T)*n);
+		while (!(p = memalign(16, sizeof(T)*n)))
 	#else
-				return (T *)_mm_malloc(sizeof(T)*n, 16);
+		while (!(p = _mm_malloc(sizeof(T)*n, 16)))
 	#endif
-	else
+			CallNewHandler();
+		return (T*)p;
+	}
 #endif
 	return new T[n];
 }

misc.cpp

@@ -6,6 +6,7 @@
 
 #include "misc.h"
 #include "words.h"
+#include <new>
 
 NAMESPACE_BEGIN(CryptoPP)
 
@@ -75,6 +76,22 @@ unsigned long Crop(unsigned long value, unsigned int size)
 		return value;
 }
 
+#if !(defined(_MSC_VER) && (_MSC_VER < 1300))
+using std::new_handler;
+#endif
+
+void CallNewHandler()
+{
+	new_handler newHandler = set_new_handler(NULL);
+	if (newHandler)
+		set_new_handler(newHandler);
+
+	if (newHandler)
+		newHandler();
+	else
+		throw std::bad_alloc();
+}
+
 NAMESPACE_END
 
 #endif

misc.h

@@ -59,6 +59,58 @@ private:
     void operator=(const NotCopyable &);
 };
 
+template <class T>
+struct NewObject
+{
+	T* operator()() const {return new T;}
+};
+
+// This function safely initializes a static object in a multithreaded environment without using locks.
+// It may leak memory when two threads try to initialize the static object at the same time
+// but this should be acceptable since each static object is only initialized once per session.
+template <class T, class F = NewObject<T>, int instance=0>
+class Singleton
+{
+public:
+	Singleton(F objectFactory = F()) : m_objectFactory(objectFactory) {}
+
+	// VC60 workaround: use "..." to prevent this function from being inlined
+	const T & Ref(...) const;
+
+private:
+	F m_objectFactory;
+};
+
+template <class T, class F, int instance>
+const T & Singleton<T, F, instance>::Ref(...) const
+{
+	static simple_ptr<T> s_pObject;
+	static char s_objectState = 0;
+
+retry:
+	switch (s_objectState)
+	{
+	case 0:
+		s_objectState = 1;
+		try
+		{
+			s_pObject.m_p = m_objectFactory();
+		}
+		catch(...)
+		{
+			s_objectState = 0;
+			throw;
+		}
+		s_objectState = 2;
+		break;
+	case 1:
+		goto retry;
+	default:
+		break;
+	}
+	return *s_pObject.m_p;
+}
+
 // ************** misc functions ***************
 
 // can't use std::min or std::max in MSVC60 or Cygwin 1.1.0
@@ -209,57 +261,7 @@ inline CipherDir GetCipherDir(const T &obj)
 	return obj.IsForwardTransformation() ? ENCRYPTION : DECRYPTION;
 }
 
-template <class T>
+void CallNewHandler();
-struct NewObject
-{
-	T* operator()() const {return new T;}
-};
-
-// This function safely initializes a static object in a multithreaded environment without using locks.
-// It may leak memory when two threads try to initialize the static object at the same time
-// but this should be acceptable since each static object is only initialized once per session.
-template <class T, class F = NewObject<T>, int instance=0>
-class Singleton
-{
-public:
-	Singleton(F objectFactory = F()) : m_objectFactory(objectFactory) {}
-
-	// VC60 workaround: use "..." to prevent this function from being inlined
-	const T & Ref(...) const;
-
-private:
-	F m_objectFactory;
-};
-
-template <class T, class F, int instance>
-const T & Singleton<T, F, instance>::Ref(...) const
-{
-	static simple_ptr<T> s_pObject;
-	static char s_objectState = 0;
-
-retry:
-	switch (s_objectState)
-	{
-	case 0:
-		s_objectState = 1;
-		try
-		{
-			s_pObject.m_p = m_objectFactory();
-		}
-		catch(...)
-		{
-			s_objectState = 0;
-			throw;
-		}
-		s_objectState = 2;
-		break;
-	case 1:
-		goto retry;
-	default:
-		break;
-	}
-	return *s_pObject.m_p;
-}
 
 // ************** rotate functions ***************
 

secblock.h

@@ -32,7 +32,14 @@ public:
 	const_pointer address(const_reference r) const {return (&r); }
 	void construct(pointer p, const T& val) {new (p) T(val);}
 	void destroy(pointer p) {p->~T();}
-	size_type max_size() const {return size_type(-1)/sizeof(T);}
+	size_type max_size() const {return ~size_type(0)/sizeof(T);}	// switch to std::numeric_limits<T>::max later
+
+protected:
+	static void CheckSize(size_t n)
+	{
+		if (n > ~size_t(0) / sizeof(T))
+			throw InvalidArgument("AllocatorBase: requested size would cause integer overflow");
+	}
 };
 
 #define CRYPTOPP_INHERIT_ALLOCATOR_TYPES	\
@@ -72,10 +79,10 @@ public:
 
 	pointer allocate(size_type n, const void * = NULL)
 	{
-		if (n > 0)
+		CheckSize(n);
-			return new T[n];
+		if (n == 0)
-		else
 			return NULL;
+		return new T[n];
 	}
 
 	void deallocate(void *p, size_type n)