Cleanup signed integer overflow on ppc64 (GH #588)

The code below was flagged by undefined behavior santizier under GCC 8. The offender was the doubling at "r4 = vec_add(r4, r4)". R4 is rcon and an unsigned type. It depends on integer wrap but GCC is generating code that is being flagged for signed overflow. GCC 7 and below is OK.

   for (unsigned int i=0; i<8; ++i)
   {
      r1 = Rijndael_Subkey_POWER8(r1, r4, r5);
      r4 = vec_add(r4, r4);
      skptr = IncrementPointerAndStore(r1, skptr);
   }

   // Final two rounds using table lookup
   ...
pull/589/head
Jeffrey Walton 2018-02-20 02:10:17 -05:00
parent 48033dac0a
commit 5b09d46665
No known key found for this signature in database
GPG Key ID: B36AB348921B1838
1 changed files with 51 additions and 86 deletions

View File

@ -709,40 +709,6 @@ void Rijndael_UncheckedSetKey_POWER8(const byte* userKey, size_t keyLen, word32*
const word32* rc, const byte* Se) const word32* rc, const byte* Se)
{ {
const size_t rounds = keyLen / 4 + 6; const size_t rounds = keyLen / 4 + 6;
if (keyLen == 16)
{
std::memcpy(rk, userKey, keyLen);
uint8_t* skptr = (uint8_t*)rk;
uint8x16_p r1 = (uint8x16_p)VectorLoadKey(skptr);
uint8x16_p r4 = (uint8x16_p)VectorLoadKey(s_rcon[0]);
uint8x16_p r5 = (uint8x16_p)VectorLoadKey(s_mask);
#if defined(CRYPTOPP_LITTLE_ENDIAN)
// Only the user key requires byte reversing.
// The subkeys are stored in proper endianess.
ReverseByteArrayLE(skptr);
#endif
for (unsigned int i=0; i<rounds-2; ++i)
{
r1 = Rijndael_Subkey_POWER8(r1, r4, r5);
r4 = vec_add(r4, r4);
skptr = IncrementPointerAndStore(r1, skptr);
}
/* Round 9 using rcon=0x1b */
r4 = (uint8x16_p)VectorLoadKey(s_rcon[1]);
r1 = Rijndael_Subkey_POWER8(r1, r4, r5);
skptr = IncrementPointerAndStore(r1, skptr);
/* Round 10 using rcon=0x36 */
r4 = (uint8x16_p)VectorLoadKey(s_rcon[2]);
r1 = Rijndael_Subkey_POWER8(r1, r4, r5);
skptr = IncrementPointerAndStore(r1, skptr);
}
else
{
GetUserKey(BIG_ENDIAN_ORDER, rk, keyLen/4, userKey, keyLen); GetUserKey(BIG_ENDIAN_ORDER, rk, keyLen/4, userKey, keyLen);
word32 *rk_saved = rk, temp; word32 *rk_saved = rk, temp;
@ -799,7 +765,6 @@ void Rijndael_UncheckedSetKey_POWER8(const byte* userKey, size_t keyLen, word32*
vec_vsx_st(vec_perm(vec_vsx_ld(0, (uint8_t*)rk), zero, mask), 0, (uint8_t*)rk); vec_vsx_st(vec_perm(vec_vsx_ld(0, (uint8_t*)rk), zero, mask), 0, (uint8_t*)rk);
#endif #endif
} }
}
size_t Rijndael_Enc_AdvancedProcessBlocks128_6x1_ALTIVEC(const word32 *subKeys, size_t rounds, size_t Rijndael_Enc_AdvancedProcessBlocks128_6x1_ALTIVEC(const word32 *subKeys, size_t rounds,
const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags) const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)