Clear Asan finding in VMAC under 32-bit inline ASM (GH #860)

Second try. The first try cleared the Asan finding but broke at -O3. Eventually we will skin this cat.

vmac.cpp

@@ -178,11 +178,9 @@ unsigned int VMAC_Base::OptimalDataAlignment() const
 #if CRYPTOPP_MSC_VERSION
 # pragma warning(disable: 4731)	// frame pointer register 'ebp' modified by inline assembly code
 #endif
-void
+
-#ifdef __GNUC__
+CRYPTOPP_NOINLINE
-__attribute__ ((noinline))		// Intel Compiler 9.1 workaround
+void VMAC_Base::VHASH_Update_SSE2(const word64 *data, size_t blocksRemainingInWord64, int tagPart)
-#endif
-VMAC_Base::VHASH_Update_SSE2(const word64 *data, size_t blocksRemainingInWord64, int tagPart)
 {
 	const word64 *nhK = m_nhKey();
 	word64 *polyS = (word64*)(void*)m_polyState();
@@ -193,13 +191,27 @@ VMAC_Base::VHASH_Update_SSE2(const word64 *data, size_t blocksRemainingInWord64,
 	CRYPTOPP_UNUSED(L1KeyLength);
 	CRYPTOPP_UNUSED(blocksRemainingInWord64);
 
+	// This inline ASM is tricky, and down right difficult when PIC is
+	// in effect. The ASM uses all the general purpose registers. When
+	// PIC is in effect, GCC uses EBX as a base register. Saving EBX with
+	// 'mov %%ebx, %0' and restoring EBX with 'mov %0, %%ebx' causes GCC
+	// to generate 'mov -0x40(%ebx), %ebx' for the restore. That obviously
+	// won't work. We can push and pop EBX, but then we have to be careful
+	// because GCC references %1 (L1KeyLength) relative to ESP, which is
+	// also used in the function and no longer accurate. Attempting to
+	// sidestep the issues with clobber lists results in "error: ‘asm’
+	// operand has impossible constraints", though we were able to tell
+	// GCC that ESP is dirty. The problems with GCC are the reason for the
+	// pushes and pops rather than the original moves.
 #ifdef __GNUC__
-	word32 temp;
+	// word32 temp;
 	__asm__ __volatile__
 	(
-	AS2(	mov		%%ebx, %0)
+	// AS2(	mov		%%ebx, %0)
-	// AS1(	push	%%ebx)
+	// AS2(	mov		%1, %%ebx)  // L1KeyLength
-	AS2(	mov		%1, %%ebx)
+	AS1(	push	%%ebx)
+	AS1(	push	%0)         // L1KeyLength
+	AS1(	pop		%%ebx)
 	INTEL_NOPREFIX
 #else
 	#if defined(__INTEL_COMPILER)
@@ -419,12 +431,12 @@ VMAC_Base::VHASH_Update_SSE2(const word64 *data, size_t blocksRemainingInWord64,
 	AS1(	emms)
 #ifdef __GNUC__
 	ATT_PREFIX
-	// AS1(	pop		%%ebx)
+	AS1(	pop		%%ebx)
-	AS2(	mov	%0, %%ebx)
+	// AS2(	mov	%0, %%ebx)
-		: "=m" (temp)
+		: // "=m" (temp)
 		: "m" (L1KeyLength), "c" (blocksRemainingInWord64), "S" (data),
 		  "D" (nhK+tagPart*2), "d" (m_isFirstBlock), "a" (polyS+tagPart*4)
-		: "memory", "cc"
+		: "esp", "memory", "cc"
 	);
 #endif
 }