Use 12x-4x-1x for Simon and Speck on ARM
Jeffrey Walton
parent
7bc621da62
commit
490701acca
315
simon-simd.cpp
315
simon-simd.cpp
|
|
@ -149,15 +149,15 @@ inline const word64* Ptr64(const T* ptr)
|
|||
return reinterpret_cast<const word64*>(ptr);
|
||||
}
|
||||
|
||||
inline void SIMON64_Enc_Block(uint32x4_t &block0, const word32 *subkeys, unsigned int rounds)
|
||||
inline void SIMON64_Enc_Block(uint32x4_t &block1, uint32x4_t &block0,
|
||||
const word32 *subkeys, unsigned int rounds)
|
||||
{
|
||||
// Rearrange the data for vectorization. The incoming data was read from
|
||||
// a big-endian byte array. Depending on the number of blocks it needs to
|
||||
// be permuted to the following. If only a single block is available then
|
||||
// a Zero block is provided to promote vectorizations.
|
||||
// [A1 A2 A3 A4][B1 B2 B3 B4] ... => [A1 A3 B1 B3][A2 A4 B2 B4] ...
|
||||
const uint32x4_t zero = {0, 0, 0, 0};
|
||||
const uint32x4x2_t t0 = vuzpq_u32(block0, zero);
|
||||
const uint32x4x2_t t0 = vuzpq_u32(block0, block1);
|
||||
uint32x4_t x1 = t0.val[0];
|
||||
uint32x4_t y1 = t0.val[1];
|
||||
|
||||
|
|
@ -185,18 +185,18 @@ inline void SIMON64_Enc_Block(uint32x4_t &block0, const word32 *subkeys, unsigne
|
|||
// [A1 A3 B1 B3][A2 A4 B2 B4] => [A1 A2 A3 A4][B1 B2 B3 B4]
|
||||
const uint32x4x2_t t1 = vzipq_u32(x1, y1);
|
||||
block0 = t1.val[0];
|
||||
// block1 = t1.val[1];
|
||||
block1 = t1.val[1];
|
||||
}
|
||||
|
||||
inline void SIMON64_Dec_Block(uint32x4_t &block0, const word32 *subkeys, unsigned int rounds)
|
||||
inline void SIMON64_Dec_Block(uint32x4_t &block0, uint32x4_t &block1,
|
||||
const word32 *subkeys, unsigned int rounds)
|
||||
{
|
||||
// Rearrange the data for vectorization. The incoming data was read from
|
||||
// a big-endian byte array. Depending on the number of blocks it needs to
|
||||
// be permuted to the following. If only a single block is available then
|
||||
// a Zero block is provided to promote vectorizations.
|
||||
// [A1 A2 A3 A4][B1 B2 B3 B4] ... => [A1 A3 B1 B3][A2 A4 B2 B4] ...
|
||||
const uint32x4_t zero = {0, 0, 0, 0};
|
||||
const uint32x4x2_t t0 = vuzpq_u32(block0, zero);
|
||||
const uint32x4x2_t t0 = vuzpq_u32(block0, block1);
|
||||
uint32x4_t x1 = t0.val[0];
|
||||
uint32x4_t y1 = t0.val[1];
|
||||
|
||||
|
|
@ -225,11 +225,12 @@ inline void SIMON64_Dec_Block(uint32x4_t &block0, const word32 *subkeys, unsigne
|
|||
// [A1 A3 B1 B3][A2 A4 B2 B4] => [A1 A2 A3 A4][B1 B2 B3 B4]
|
||||
const uint32x4x2_t t1 = vzipq_u32(x1, y1);
|
||||
block0 = t1.val[0];
|
||||
// block1 = t1.val[1];
|
||||
block1 = t1.val[1];
|
||||
}
|
||||
|
||||
inline void SIMON64_Enc_4_Blocks(uint32x4_t &block0, uint32x4_t &block1,
|
||||
uint32x4_t &block2, uint32x4_t &block3, const word32 *subkeys, unsigned int rounds)
|
||||
inline void SIMON64_Enc_6_Blocks(uint32x4_t &block0, uint32x4_t &block1,
|
||||
uint32x4_t &block2, uint32x4_t &block3, uint32x4_t &block4, uint32x4_t &block5,
|
||||
const word32 *subkeys, unsigned int rounds)
|
||||
{
|
||||
// Rearrange the data for vectorization. The incoming data was read from
|
||||
// a big-endian byte array. Depending on the number of blocks it needs to
|
||||
|
|
@ -244,18 +245,25 @@ inline void SIMON64_Enc_4_Blocks(uint32x4_t &block0, uint32x4_t &block1,
|
|||
uint32x4_t x2 = t1.val[0];
|
||||
uint32x4_t y2 = t1.val[1];
|
||||
|
||||
const uint32x4x2_t t2 = vuzpq_u32(block4, block5);
|
||||
uint32x4_t x3 = t2.val[0];
|
||||
uint32x4_t y3 = t2.val[1];
|
||||
|
||||
x1 = Shuffle32(x1); y1 = Shuffle32(y1);
|
||||
x2 = Shuffle32(x2); y2 = Shuffle32(y2);
|
||||
x3 = Shuffle32(x3); y3 = Shuffle32(y3);
|
||||
|
||||
for (size_t i = 0; static_cast<int>(i) < (rounds & ~1) - 1; i += 2)
|
||||
{
|
||||
const uint32x4_t rk1 = vld1q_dup_u32(subkeys+i);
|
||||
y1 = veorq_u32(veorq_u32(y1, SIMON64_f(x1)), rk1);
|
||||
y2 = veorq_u32(veorq_u32(y2, SIMON64_f(x2)), rk1);
|
||||
y3 = veorq_u32(veorq_u32(y3, SIMON64_f(x3)), rk1);
|
||||
|
||||
const uint32x4_t rk2 = vld1q_dup_u32(subkeys+i+1);
|
||||
x1 = veorq_u32(veorq_u32(x1, SIMON64_f(y1)), rk2);
|
||||
x2 = veorq_u32(veorq_u32(x2, SIMON64_f(y2)), rk2);
|
||||
x3 = veorq_u32(veorq_u32(x3, SIMON64_f(y3)), rk2);
|
||||
}
|
||||
|
||||
if (rounds & 1)
|
||||
|
|
@ -264,11 +272,13 @@ inline void SIMON64_Enc_4_Blocks(uint32x4_t &block0, uint32x4_t &block1,
|
|||
|
||||
y1 = veorq_u32(veorq_u32(y1, SIMON64_f(x1)), rk);
|
||||
y2 = veorq_u32(veorq_u32(y2, SIMON64_f(x2)), rk);
|
||||
std::swap(x1, y1); std::swap(x2, y2);
|
||||
y3 = veorq_u32(veorq_u32(y3, SIMON64_f(x3)), rk);
|
||||
std::swap(x1, y1); std::swap(x2, y2); std::swap(x3, y3);
|
||||
}
|
||||
|
||||
x1 = Shuffle32(x1); y1 = Shuffle32(y1);
|
||||
x2 = Shuffle32(x2); y2 = Shuffle32(y2);
|
||||
x3 = Shuffle32(x3); y3 = Shuffle32(y3);
|
||||
|
||||
// [A1 A3 B1 B3][A2 A4 B2 B4] => [A1 A2 A3 A4][B1 B2 B3 B4]
|
||||
const uint32x4x2_t t3 = vzipq_u32(x1, y1);
|
||||
|
|
@ -278,10 +288,15 @@ inline void SIMON64_Enc_4_Blocks(uint32x4_t &block0, uint32x4_t &block1,
|
|||
const uint32x4x2_t t4 = vzipq_u32(x2, y2);
|
||||
block2 = t4.val[0];
|
||||
block3 = t4.val[1];
|
||||
|
||||
const uint32x4x2_t t5 = vzipq_u32(x3, y3);
|
||||
block4 = t5.val[0];
|
||||
block5 = t5.val[1];
|
||||
}
|
||||
|
||||
inline void SIMON64_Dec_4_Blocks(uint32x4_t &block0, uint32x4_t &block1,
|
||||
uint32x4_t &block2, uint32x4_t &block3, const word32 *subkeys, unsigned int rounds)
|
||||
inline void SIMON64_Dec_6_Blocks(uint32x4_t &block0, uint32x4_t &block1,
|
||||
uint32x4_t &block2, uint32x4_t &block3, uint32x4_t &block4, uint32x4_t &block5,
|
||||
const word32 *subkeys, unsigned int rounds)
|
||||
{
|
||||
// Rearrange the data for vectorization. The incoming data was read from
|
||||
// a big-endian byte array. Depending on the number of blocks it needs to
|
||||
|
|
@ -296,16 +311,22 @@ inline void SIMON64_Dec_4_Blocks(uint32x4_t &block0, uint32x4_t &block1,
|
|||
uint32x4_t x2 = t1.val[0];
|
||||
uint32x4_t y2 = t1.val[1];
|
||||
|
||||
const uint32x4x2_t t2 = vuzpq_u32(block4, block5);
|
||||
uint32x4_t x3 = t2.val[0];
|
||||
uint32x4_t y3 = t2.val[1];
|
||||
|
||||
x1 = Shuffle32(x1); y1 = Shuffle32(y1);
|
||||
x2 = Shuffle32(x2); y2 = Shuffle32(y2);
|
||||
x3 = Shuffle32(x3); y3 = Shuffle32(y3);
|
||||
|
||||
if (rounds & 1)
|
||||
{
|
||||
std::swap(x1, y1); std::swap(x2, y2);
|
||||
std::swap(x1, y1); std::swap(x2, y2); std::swap(x3, y3);
|
||||
const uint32x4_t rk = vld1q_dup_u32(subkeys + rounds - 1);
|
||||
|
||||
y1 = veorq_u32(veorq_u32(y1, rk), SIMON64_f(x1));
|
||||
y2 = veorq_u32(veorq_u32(y2, rk), SIMON64_f(x2));
|
||||
y3 = veorq_u32(veorq_u32(y3, rk), SIMON64_f(x3));
|
||||
rounds--;
|
||||
}
|
||||
|
||||
|
|
@ -314,14 +335,17 @@ inline void SIMON64_Dec_4_Blocks(uint32x4_t &block0, uint32x4_t &block1,
|
|||
const uint32x4_t rk1 = vld1q_dup_u32(subkeys + i + 1);
|
||||
x1 = veorq_u32(veorq_u32(x1, SIMON64_f(y1)), rk1);
|
||||
x2 = veorq_u32(veorq_u32(x2, SIMON64_f(y2)), rk1);
|
||||
x3 = veorq_u32(veorq_u32(x3, SIMON64_f(y3)), rk1);
|
||||
|
||||
const uint32x4_t rk2 = vld1q_dup_u32(subkeys + i);
|
||||
y1 = veorq_u32(veorq_u32(y1, SIMON64_f(x1)), rk2);
|
||||
y2 = veorq_u32(veorq_u32(y2, SIMON64_f(x2)), rk2);
|
||||
y3 = veorq_u32(veorq_u32(y3, SIMON64_f(x3)), rk2);
|
||||
}
|
||||
|
||||
x1 = Shuffle32(x1); y1 = Shuffle32(y1);
|
||||
x2 = Shuffle32(x2); y2 = Shuffle32(y2);
|
||||
x3 = Shuffle32(x3); y3 = Shuffle32(y3);
|
||||
|
||||
// [A1 A3 B1 B3][A2 A4 B2 B4] => [A1 A2 A3 A4][B1 B2 B3 B4]
|
||||
const uint32x4x2_t t3 = vzipq_u32(x1, y1);
|
||||
|
|
@ -331,10 +355,14 @@ inline void SIMON64_Dec_4_Blocks(uint32x4_t &block0, uint32x4_t &block1,
|
|||
const uint32x4x2_t t4 = vzipq_u32(x2, y2);
|
||||
block2 = t4.val[0];
|
||||
block3 = t4.val[1];
|
||||
|
||||
const uint32x4x2_t t5 = vzipq_u32(x3, y3);
|
||||
block4 = t5.val[0];
|
||||
block5 = t5.val[1];
|
||||
}
|
||||
|
||||
template <typename F1, typename F4>
|
||||
inline size_t SIMON64_AdvancedProcessBlocks_NEON(F1 func1, F4 func4,
|
||||
template <typename F2, typename F6>
|
||||
inline size_t SIMON64_AdvancedProcessBlocks_NEON(F2 func2, F6 func6,
|
||||
const word32 *subKeys, size_t rounds, const byte *inBlocks,
|
||||
const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
|
||||
{
|
||||
|
|
@ -360,9 +388,9 @@ inline size_t SIMON64_AdvancedProcessBlocks_NEON(F1 func1, F4 func4,
|
|||
|
||||
if (flags & BlockTransformation::BT_AllowParallel)
|
||||
{
|
||||
while (length >= 4*neonBlockSize)
|
||||
while (length >= 6*neonBlockSize)
|
||||
{
|
||||
uint32x4_t block0, block1, block2, block3;
|
||||
uint32x4_t block0, block1, block2, block3, block4, block5;
|
||||
block0 = vreinterpretq_u32_u8(vld1q_u8(inBlocks));
|
||||
|
||||
if (flags & BlockTransformation::BT_InBlockIsCounter)
|
||||
|
|
@ -371,8 +399,10 @@ inline size_t SIMON64_AdvancedProcessBlocks_NEON(F1 func1, F4 func4,
|
|||
block1 = vaddq_u32(block0, be1);
|
||||
block2 = vaddq_u32(block1, be1);
|
||||
block3 = vaddq_u32(block2, be1);
|
||||
block4 = vaddq_u32(block3, be1);
|
||||
block5 = vaddq_u32(block4, be1);
|
||||
vst1q_u8(const_cast<byte*>(inBlocks),
|
||||
vreinterpretq_u8_u32(vaddq_u32(block3, be1)));
|
||||
vreinterpretq_u8_u32(vaddq_u32(block5, be1)));
|
||||
}
|
||||
else
|
||||
{
|
||||
|
|
@ -380,7 +410,9 @@ inline size_t SIMON64_AdvancedProcessBlocks_NEON(F1 func1, F4 func4,
|
|||
block1 = vreinterpretq_u32_u8(vld1q_u8(inBlocks+1*inc));
|
||||
block2 = vreinterpretq_u32_u8(vld1q_u8(inBlocks+2*inc));
|
||||
block3 = vreinterpretq_u32_u8(vld1q_u8(inBlocks+3*inc));
|
||||
inBlocks += 4*inc;
|
||||
block4 = vreinterpretq_u32_u8(vld1q_u8(inBlocks+4*inc));
|
||||
block5 = vreinterpretq_u32_u8(vld1q_u8(inBlocks+5*inc));
|
||||
inBlocks += 6*inc;
|
||||
}
|
||||
|
||||
if (flags & BlockTransformation::BT_XorInput)
|
||||
|
|
@ -390,10 +422,12 @@ inline size_t SIMON64_AdvancedProcessBlocks_NEON(F1 func1, F4 func4,
|
|||
block1 = veorq_u32(block1, vreinterpretq_u32_u8(vld1q_u8(xorBlocks+1*inc)));
|
||||
block2 = veorq_u32(block2, vreinterpretq_u32_u8(vld1q_u8(xorBlocks+2*inc)));
|
||||
block3 = veorq_u32(block3, vreinterpretq_u32_u8(vld1q_u8(xorBlocks+3*inc)));
|
||||
xorBlocks += 4*inc;
|
||||
block4 = veorq_u32(block4, vreinterpretq_u32_u8(vld1q_u8(xorBlocks+4*inc)));
|
||||
block5 = veorq_u32(block5, vreinterpretq_u32_u8(vld1q_u8(xorBlocks+5*inc)));
|
||||
xorBlocks += 6*inc;
|
||||
}
|
||||
|
||||
func4(block0, block1, block2, block3, subKeys, static_cast<unsigned int>(rounds));
|
||||
func6(block0, block1, block2, block3, block4, block5, subKeys, static_cast<unsigned int>(rounds));
|
||||
|
||||
if (xorBlocks && !(flags & BlockTransformation::BT_XorInput))
|
||||
{
|
||||
|
|
@ -402,7 +436,9 @@ inline size_t SIMON64_AdvancedProcessBlocks_NEON(F1 func1, F4 func4,
|
|||
block1 = veorq_u32(block1, vreinterpretq_u32_u8(vld1q_u8(xorBlocks+1*inc)));
|
||||
block2 = veorq_u32(block2, vreinterpretq_u32_u8(vld1q_u8(xorBlocks+2*inc)));
|
||||
block3 = veorq_u32(block3, vreinterpretq_u32_u8(vld1q_u8(xorBlocks+3*inc)));
|
||||
xorBlocks += 4*inc;
|
||||
block4 = veorq_u32(block4, vreinterpretq_u32_u8(vld1q_u8(xorBlocks+4*inc)));
|
||||
block5 = veorq_u32(block5, vreinterpretq_u32_u8(vld1q_u8(xorBlocks+5*inc)));
|
||||
xorBlocks += 6*inc;
|
||||
}
|
||||
|
||||
const int inc = static_cast<int>(outIncrement);
|
||||
|
|
@ -410,9 +446,56 @@ inline size_t SIMON64_AdvancedProcessBlocks_NEON(F1 func1, F4 func4,
|
|||
vst1q_u8(outBlocks+1*inc, vreinterpretq_u8_u32(block1));
|
||||
vst1q_u8(outBlocks+2*inc, vreinterpretq_u8_u32(block2));
|
||||
vst1q_u8(outBlocks+3*inc, vreinterpretq_u8_u32(block3));
|
||||
vst1q_u8(outBlocks+4*inc, vreinterpretq_u8_u32(block4));
|
||||
vst1q_u8(outBlocks+5*inc, vreinterpretq_u8_u32(block5));
|
||||
|
||||
outBlocks += 4*inc;
|
||||
length -= 4*neonBlockSize;
|
||||
outBlocks += 6*inc;
|
||||
length -= 6*neonBlockSize;
|
||||
}
|
||||
|
||||
while (length >= 2*neonBlockSize)
|
||||
{
|
||||
uint32x4_t block0, block1;
|
||||
block0 = vreinterpretq_u32_u8(vld1q_u8(inBlocks));
|
||||
|
||||
if (flags & BlockTransformation::BT_InBlockIsCounter)
|
||||
{
|
||||
const uint32x4_t be1 = vld1q_u32(s_one64);
|
||||
block1 = vaddq_u32(block0, be1);
|
||||
vst1q_u8(const_cast<byte*>(inBlocks),
|
||||
vreinterpretq_u8_u32(vaddq_u32(block1, be1)));
|
||||
}
|
||||
else
|
||||
{
|
||||
const int inc = static_cast<int>(inIncrement);
|
||||
block1 = vreinterpretq_u32_u8(vld1q_u8(inBlocks+1*inc));
|
||||
inBlocks += 2*inc;
|
||||
}
|
||||
|
||||
if (flags & BlockTransformation::BT_XorInput)
|
||||
{
|
||||
const int inc = static_cast<int>(xorIncrement);
|
||||
block0 = veorq_u32(block0, vreinterpretq_u32_u8(vld1q_u8(xorBlocks+0*inc)));
|
||||
block1 = veorq_u32(block1, vreinterpretq_u32_u8(vld1q_u8(xorBlocks+1*inc)));
|
||||
xorBlocks += 2*inc;
|
||||
}
|
||||
|
||||
func2(block0, block1, subKeys, static_cast<unsigned int>(rounds));
|
||||
|
||||
if (xorBlocks && !(flags & BlockTransformation::BT_XorInput))
|
||||
{
|
||||
const int inc = static_cast<int>(xorIncrement);
|
||||
block0 = veorq_u32(block0, vreinterpretq_u32_u8(vld1q_u8(xorBlocks+0*inc)));
|
||||
block1 = veorq_u32(block1, vreinterpretq_u32_u8(vld1q_u8(xorBlocks+1*inc)));
|
||||
xorBlocks += 2*inc;
|
||||
}
|
||||
|
||||
const int inc = static_cast<int>(outIncrement);
|
||||
vst1q_u8(outBlocks+0*inc, vreinterpretq_u8_u32(block0));
|
||||
vst1q_u8(outBlocks+1*inc, vreinterpretq_u8_u32(block1));
|
||||
|
||||
outBlocks += 2*inc;
|
||||
length -= 2*neonBlockSize;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -438,7 +521,7 @@ inline size_t SIMON64_AdvancedProcessBlocks_NEON(F1 func1, F4 func4,
|
|||
|
||||
while (length >= blockSize)
|
||||
{
|
||||
uint32x4_t block;
|
||||
uint32x4_t block, zero = {0,0,0,0};
|
||||
block = vsetq_lane_u32(Ptr32(inBlocks)[0], block, 0);
|
||||
block = vsetq_lane_u32(Ptr32(inBlocks)[1], block, 1);
|
||||
|
||||
|
|
@ -453,7 +536,7 @@ inline size_t SIMON64_AdvancedProcessBlocks_NEON(F1 func1, F4 func4,
|
|||
if (flags & BlockTransformation::BT_InBlockIsCounter)
|
||||
const_cast<byte *>(inBlocks)[7]++;
|
||||
|
||||
func1(block, subKeys, static_cast<unsigned int>(rounds));
|
||||
func2(block, zero, subKeys, static_cast<unsigned int>(rounds));
|
||||
|
||||
if (xorBlocks && !(flags & BlockTransformation::BT_XorInput))
|
||||
{
|
||||
|
|
@ -572,12 +655,13 @@ inline uint64x2_t SIMON128_f(const uint64x2_t& val)
|
|||
vandq_u64(RotateLeft64<1>(val), RotateLeft64<8>(val)));
|
||||
}
|
||||
|
||||
inline void SIMON128_Enc_Block(uint64x2_t &block0, const word64 *subkeys, unsigned int rounds)
|
||||
inline void SIMON128_Enc_Block(uint64x2_t &block0, uint64x2_t &block1,
|
||||
const word64 *subkeys, unsigned int rounds)
|
||||
{
|
||||
// Hack ahead... Rearrange the data for vectorization. It is easier to permute
|
||||
// the data in SIMON128_Enc_Blocks then SIMON128_AdvancedProcessBlocks_NEON.
|
||||
// The zero block below is a "don't care". It is present so we can vectorize.
|
||||
uint64x2_t block1 = {0};
|
||||
// Rearrange the data for vectorization. The incoming data was read from
|
||||
// a big-endian byte array. Depending on the number of blocks it needs to
|
||||
// be permuted to the following.
|
||||
// [A1 A2][B1 B2] ... => [A1 B1][A2 B2] ...
|
||||
uint64x2_t x1 = UnpackLow64(block0, block1);
|
||||
uint64x2_t y1 = UnpackHigh64(block0, block1);
|
||||
|
||||
|
|
@ -603,15 +687,17 @@ inline void SIMON128_Enc_Block(uint64x2_t &block0, const word64 *subkeys, unsign
|
|||
x1 = Shuffle64(x1); y1 = Shuffle64(y1);
|
||||
|
||||
block0 = UnpackLow64(x1, y1);
|
||||
// block1 = UnpackHigh64(x1, y1);
|
||||
block1 = UnpackHigh64(x1, y1);
|
||||
}
|
||||
|
||||
inline void SIMON128_Enc_6_Blocks(uint64x2_t &block0, uint64x2_t &block1,
|
||||
uint64x2_t &block2, uint64x2_t &block3, uint64x2_t &block4,
|
||||
uint64x2_t &block5, const word64 *subkeys, unsigned int rounds)
|
||||
uint64x2_t &block2, uint64x2_t &block3, uint64x2_t &block4, uint64x2_t &block5,
|
||||
const word64 *subkeys, unsigned int rounds)
|
||||
{
|
||||
// Hack ahead... Rearrange the data for vectorization. It is easier to permute
|
||||
// the data in SIMON128_Enc_Blocks then SIMON128_AdvancedProcessBlocks_NEON.
|
||||
// Rearrange the data for vectorization. The incoming data was read from
|
||||
// a big-endian byte array. Depending on the number of blocks it needs to
|
||||
// be permuted to the following.
|
||||
// [A1 A2][B1 B2] ... => [A1 B1][A2 B2] ...
|
||||
uint64x2_t x1 = UnpackLow64(block0, block1);
|
||||
uint64x2_t y1 = UnpackHigh64(block0, block1);
|
||||
uint64x2_t x2 = UnpackLow64(block2, block3);
|
||||
|
|
@ -658,12 +744,13 @@ inline void SIMON128_Enc_6_Blocks(uint64x2_t &block0, uint64x2_t &block1,
|
|||
block5 = UnpackHigh64(x3, y3);
|
||||
}
|
||||
|
||||
inline void SIMON128_Dec_Block(uint64x2_t &block0, const word64 *subkeys, unsigned int rounds)
|
||||
inline void SIMON128_Dec_Block(uint64x2_t &block0, uint64x2_t &block1,
|
||||
const word64 *subkeys, unsigned int rounds)
|
||||
{
|
||||
// Hack ahead... Rearrange the data for vectorization. It is easier to permute
|
||||
// the data in SIMON128_Dec_Blocks then SIMON128_AdvancedProcessBlocks_NEON.
|
||||
// The zero block below is a "don't care". It is present so we can vectorize.
|
||||
uint64x2_t block1 = {0};
|
||||
// Rearrange the data for vectorization. The incoming data was read from
|
||||
// a big-endian byte array. Depending on the number of blocks it needs to
|
||||
// be permuted to the following.
|
||||
// [A1 A2][B1 B2] ... => [A1 B1][A2 B2] ...
|
||||
uint64x2_t x1 = UnpackLow64(block0, block1);
|
||||
uint64x2_t y1 = UnpackHigh64(block0, block1);
|
||||
|
||||
|
|
@ -690,15 +777,17 @@ inline void SIMON128_Dec_Block(uint64x2_t &block0, const word64 *subkeys, unsign
|
|||
x1 = Shuffle64(x1); y1 = Shuffle64(y1);
|
||||
|
||||
block0 = UnpackLow64(x1, y1);
|
||||
// block1 = UnpackHigh64(x1, y1);
|
||||
block1 = UnpackHigh64(x1, y1);
|
||||
}
|
||||
|
||||
inline void SIMON128_Dec_6_Blocks(uint64x2_t &block0, uint64x2_t &block1,
|
||||
uint64x2_t &block2, uint64x2_t &block3, uint64x2_t &block4,
|
||||
uint64x2_t &block5, const word64 *subkeys, unsigned int rounds)
|
||||
uint64x2_t &block2, uint64x2_t &block3, uint64x2_t &block4, uint64x2_t &block5,
|
||||
const word64 *subkeys, unsigned int rounds)
|
||||
{
|
||||
// Hack ahead... Rearrange the data for vectorization. It is easier to permute
|
||||
// the data in SIMON128_Dec_Blocks then SIMON128_AdvancedProcessBlocks_NEON.
|
||||
// Rearrange the data for vectorization. The incoming data was read from
|
||||
// a big-endian byte array. Depending on the number of blocks it needs to
|
||||
// be permuted to the following.
|
||||
// [A1 A2][B1 B2] ... => [A1 B1][A2 B2] ...
|
||||
uint64x2_t x1 = UnpackLow64(block0, block1);
|
||||
uint64x2_t y1 = UnpackHigh64(block0, block1);
|
||||
uint64x2_t x2 = UnpackLow64(block2, block3);
|
||||
|
|
@ -746,8 +835,8 @@ inline void SIMON128_Dec_6_Blocks(uint64x2_t &block0, uint64x2_t &block1,
|
|||
block5 = UnpackHigh64(x3, y3);
|
||||
}
|
||||
|
||||
template <typename F1, typename F6>
|
||||
size_t SIMON128_AdvancedProcessBlocks_NEON(F1 func1, F6 func6,
|
||||
template <typename F2, typename F6>
|
||||
size_t SIMON128_AdvancedProcessBlocks_NEON(F2 func2, F6 func6,
|
||||
const word64 *subKeys, size_t rounds, const byte *inBlocks,
|
||||
const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
|
||||
{
|
||||
|
|
@ -812,7 +901,7 @@ size_t SIMON128_AdvancedProcessBlocks_NEON(F1 func1, F6 func6,
|
|||
xorBlocks += 6*inc;
|
||||
}
|
||||
|
||||
func6(block0, block1, block2, block3, block4, block5, subKeys, rounds);
|
||||
func6(block0, block1, block2, block3, block4, block5, subKeys, static_cast<unsigned int>(rounds));
|
||||
|
||||
if (xorBlocks && !(flags & BlockTransformation::BT_XorInput))
|
||||
{
|
||||
|
|
@ -837,11 +926,57 @@ size_t SIMON128_AdvancedProcessBlocks_NEON(F1 func1, F6 func6,
|
|||
outBlocks += 6*inc;
|
||||
length -= 6*blockSize;
|
||||
}
|
||||
|
||||
while (length >= 2*blockSize)
|
||||
{
|
||||
uint64x2_t block0, block1;
|
||||
block0 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
|
||||
|
||||
if (flags & BlockTransformation::BT_InBlockIsCounter)
|
||||
{
|
||||
uint64x2_t be = vreinterpretq_u64_u32(vld1q_u32(s_one128));
|
||||
block1 = vaddq_u64(block0, be);
|
||||
vst1q_u8(const_cast<byte*>(inBlocks),
|
||||
vreinterpretq_u8_u64(vaddq_u64(block1, be)));
|
||||
}
|
||||
else
|
||||
{
|
||||
const int inc = static_cast<int>(inIncrement);
|
||||
block1 = vreinterpretq_u64_u8(vld1q_u8(inBlocks+1*inc));
|
||||
inBlocks += 2*inc;
|
||||
}
|
||||
|
||||
if (flags & BlockTransformation::BT_XorInput)
|
||||
{
|
||||
const int inc = static_cast<int>(xorIncrement);
|
||||
block0 = veorq_u64(block0, vreinterpretq_u64_u8(vld1q_u8(xorBlocks+0*inc)));
|
||||
block1 = veorq_u64(block1, vreinterpretq_u64_u8(vld1q_u8(xorBlocks+1*inc)));
|
||||
xorBlocks += 2*inc;
|
||||
}
|
||||
|
||||
func2(block0, block1, subKeys, static_cast<unsigned int>(rounds));
|
||||
|
||||
if (xorBlocks && !(flags & BlockTransformation::BT_XorInput))
|
||||
{
|
||||
const int inc = static_cast<int>(xorIncrement);
|
||||
block0 = veorq_u64(block0, vreinterpretq_u64_u8(vld1q_u8(xorBlocks+0*inc)));
|
||||
block1 = veorq_u64(block1, vreinterpretq_u64_u8(vld1q_u8(xorBlocks+1*inc)));
|
||||
xorBlocks += 2*inc;
|
||||
}
|
||||
|
||||
const int inc = static_cast<int>(outIncrement);
|
||||
vst1q_u8(outBlocks+0*inc, vreinterpretq_u8_u64(block0));
|
||||
vst1q_u8(outBlocks+1*inc, vreinterpretq_u8_u64(block1));
|
||||
|
||||
outBlocks += 2*inc;
|
||||
length -= 2*blockSize;
|
||||
}
|
||||
}
|
||||
|
||||
while (length >= blockSize)
|
||||
{
|
||||
uint64x2_t block = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
|
||||
uint64x2_t block, zero = {0,0};
|
||||
block = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
|
||||
|
||||
if (flags & BlockTransformation::BT_XorInput)
|
||||
block = veorq_u64(block, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
|
||||
|
|
@ -849,7 +984,7 @@ size_t SIMON128_AdvancedProcessBlocks_NEON(F1 func1, F6 func6,
|
|||
if (flags & BlockTransformation::BT_InBlockIsCounter)
|
||||
const_cast<byte *>(inBlocks)[15]++;
|
||||
|
||||
func1(block, subKeys, rounds);
|
||||
func2(block, zero, subKeys, static_cast<unsigned int>(rounds));
|
||||
|
||||
if (xorBlocks && !(flags & BlockTransformation::BT_XorInput))
|
||||
block = veorq_u64(block, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
|
||||
|
|
@ -940,9 +1075,10 @@ inline __m128i SIMON128_f(const __m128i& v)
|
|||
|
||||
inline void SIMON128_Enc_Block(__m128i &block0, const word64 *subkeys, unsigned int rounds)
|
||||
{
|
||||
// Hack ahead... Rearrange the data for vectorization. It is easier to permute
|
||||
// the data in SIMON128_Enc_Blocks then SIMON128_AdvancedProcessBlocks_SSSE3.
|
||||
// The zero block below is a "don't care". It is present so we can vectorize.
|
||||
// Rearrange the data for vectorization. The incoming data was read from
|
||||
// a big-endian byte array. Depending on the number of blocks it needs to
|
||||
// be permuted to the following.
|
||||
// [A1 A2][B1 B2] ... => [A1 B1][A2 B2] ...
|
||||
__m128i block1 = _mm_setzero_si128();
|
||||
__m128i x1 = _mm_unpacklo_epi64(block0, block1);
|
||||
__m128i y1 = _mm_unpackhi_epi64(block0, block1);
|
||||
|
|
@ -981,8 +1117,10 @@ inline void SIMON128_Enc_Block(__m128i &block0, const word64 *subkeys, unsigned
|
|||
inline void SIMON128_Enc_4_Blocks(__m128i &block0, __m128i &block1,
|
||||
__m128i &block2, __m128i &block3, const word64 *subkeys, unsigned int rounds)
|
||||
{
|
||||
// Hack ahead... Rearrange the data for vectorization. It is easier to permute
|
||||
// the data in SIMON128_Enc_Blocks then SIMON128_AdvancedProcessBlocks_SSSE3.
|
||||
// Rearrange the data for vectorization. The incoming data was read from
|
||||
// a big-endian byte array. Depending on the number of blocks it needs to
|
||||
// be permuted to the following.
|
||||
// [A1 A2][B1 B2] ... => [A1 B1][A2 B2] ...
|
||||
__m128i x1 = _mm_unpacklo_epi64(block0, block1);
|
||||
__m128i y1 = _mm_unpackhi_epi64(block0, block1);
|
||||
__m128i x2 = _mm_unpacklo_epi64(block2, block3);
|
||||
|
|
@ -1029,9 +1167,10 @@ inline void SIMON128_Enc_4_Blocks(__m128i &block0, __m128i &block1,
|
|||
|
||||
inline void SIMON128_Dec_Block(__m128i &block0, const word64 *subkeys, unsigned int rounds)
|
||||
{
|
||||
// Hack ahead... Rearrange the data for vectorization. It is easier to permute
|
||||
// the data in SIMON128_Dec_Blocks then SIMON128_AdvancedProcessBlocks_SSSE3.
|
||||
// The zero block below is a "don't care". It is present so we can vectorize.
|
||||
// Rearrange the data for vectorization. The incoming data was read from
|
||||
// a big-endian byte array. Depending on the number of blocks it needs to
|
||||
// be permuted to the following.
|
||||
// [A1 A2][B1 B2] ... => [A1 B1][A2 B2] ...
|
||||
__m128i block1 = _mm_setzero_si128();
|
||||
__m128i x1 = _mm_unpacklo_epi64(block0, block1);
|
||||
__m128i y1 = _mm_unpackhi_epi64(block0, block1);
|
||||
|
|
@ -1071,8 +1210,10 @@ inline void SIMON128_Dec_Block(__m128i &block0, const word64 *subkeys, unsigned
|
|||
inline void SIMON128_Dec_4_Blocks(__m128i &block0, __m128i &block1,
|
||||
__m128i &block2, __m128i &block3, const word64 *subkeys, unsigned int rounds)
|
||||
{
|
||||
// Hack ahead... Rearrange the data for vectorization. It is easier to permute
|
||||
// the data in SIMON128_Dec_Blocks then SIMON128_AdvancedProcessBlocks_SSSE3.
|
||||
// Rearrange the data for vectorization. The incoming data was read from
|
||||
// a big-endian byte array. Depending on the number of blocks it needs to
|
||||
// be permuted to the following.
|
||||
// [A1 A2][B1 B2] ... => [A1 B1][A2 B2] ...
|
||||
__m128i x1 = _mm_unpacklo_epi64(block0, block1);
|
||||
__m128i y1 = _mm_unpackhi_epi64(block0, block1);
|
||||
__m128i x2 = _mm_unpacklo_epi64(block2, block3);
|
||||
|
|
@ -1275,17 +1416,16 @@ inline __m128i SIMON64_f(const __m128i& v)
|
|||
_mm_and_si128(RotateLeft32<1>(v), RotateLeft32<8>(v)));
|
||||
}
|
||||
|
||||
inline void SIMON64_Enc_Block(__m128i &block0, const word32 *subkeys, unsigned int rounds)
|
||||
inline void SIMON64_Enc_Block(__m128i &block0, __m128i &block1,
|
||||
const word32 *subkeys, unsigned int rounds)
|
||||
{
|
||||
// Rearrange the data for vectorization. The incoming data was read from
|
||||
// a big-endian byte array. Depending on the number of blocks it needs to
|
||||
// be permuted to the following. If only a single block is available then
|
||||
// a Zero block is provided to promote vectorizations. Thanks to Peter
|
||||
// Cordes for help with the SSE permutes below.
|
||||
// be permuted to the following. Thanks to Peter Cordes for help with the
|
||||
// SSE permutes below.
|
||||
// [A1 A2 A3 A4][B1 B2 B3 B4] ... => [A1 A3 B1 B3][A2 A4 B2 B4] ...
|
||||
const __m128i zero = _mm_setzero_si128();
|
||||
const __m128 t0 = _mm_castsi128_ps(block0);
|
||||
const __m128 t1 = _mm_castsi128_ps(zero);
|
||||
const __m128 t1 = _mm_castsi128_ps(block1);
|
||||
__m128i x1 = _mm_castps_si128(_mm_shuffle_ps(t0, t1, _MM_SHUFFLE(2,0,2,0)));
|
||||
__m128i y1 = _mm_castps_si128(_mm_shuffle_ps(t0, t1, _MM_SHUFFLE(3,1,3,1)));
|
||||
|
||||
|
|
@ -1315,20 +1455,19 @@ inline void SIMON64_Enc_Block(__m128i &block0, const word32 *subkeys, unsigned i
|
|||
// The is roughly the SSE equivalent to ARM vzp32
|
||||
// [A1 A3 B1 B3][A2 A4 B2 B4] => [A1 A2 A3 A4][B1 B2 B3 B4]
|
||||
block0 = _mm_unpacklo_epi32(x1, y1);
|
||||
// block1 = _mm_unpackhigh_epi32(x1, y1);
|
||||
block1 = _mm_unpackhi_epi32(x1, y1);
|
||||
}
|
||||
|
||||
inline void SIMON64_Dec_Block(__m128i &block0, const word32 *subkeys, unsigned int rounds)
|
||||
inline void SIMON64_Dec_Block(__m128i &block0, __m128i &block1,
|
||||
const word32 *subkeys, unsigned int rounds)
|
||||
{
|
||||
// Rearrange the data for vectorization. The incoming data was read from
|
||||
// a big-endian byte array. Depending on the number of blocks it needs to
|
||||
// be permuted to the following. If only a single block is available then
|
||||
// a Zero block is provided to promote vectorizations. Thanks to Peter
|
||||
// Cordes for help with the SSE permutes below.
|
||||
// be permuted to the following. Thanks to Peter Cordes for help with the
|
||||
// SSE permutes below.
|
||||
// [A1 A2 A3 A4][B1 B2 B3 B4] ... => [A1 A3 B1 B3][A2 A4 B2 B4] ...
|
||||
const __m128i zero = _mm_setzero_si128();
|
||||
const __m128 t0 = _mm_castsi128_ps(block0);
|
||||
const __m128 t1 = _mm_castsi128_ps(zero);
|
||||
const __m128 t1 = _mm_castsi128_ps(block1);
|
||||
__m128i x1 = _mm_castps_si128(_mm_shuffle_ps(t0, t1, _MM_SHUFFLE(2,0,2,0)));
|
||||
__m128i y1 = _mm_castps_si128(_mm_shuffle_ps(t0, t1, _MM_SHUFFLE(3,1,3,1)));
|
||||
|
||||
|
|
@ -1359,7 +1498,7 @@ inline void SIMON64_Dec_Block(__m128i &block0, const word32 *subkeys, unsigned i
|
|||
// The is roughly the SSE equivalent to ARM vzp32
|
||||
// [A1 A3 B1 B3][A2 A4 B2 B4] => [A1 A2 A3 A4][B1 B2 B3 B4]
|
||||
block0 = _mm_unpacklo_epi32(x1, y1);
|
||||
// block1 = _mm_unpackhigh_epi32(x1, y1);
|
||||
block1 = _mm_unpackhi_epi32(x1, y1);
|
||||
}
|
||||
|
||||
inline void SIMON64_Enc_4_Blocks(__m128i &block0, __m128i &block1, __m128i &block2,
|
||||
|
|
@ -1367,9 +1506,8 @@ inline void SIMON64_Enc_4_Blocks(__m128i &block0, __m128i &block1, __m128i &bloc
|
|||
{
|
||||
// Rearrange the data for vectorization. The incoming data was read from
|
||||
// a big-endian byte array. Depending on the number of blocks it needs to
|
||||
// be permuted to the following. If only a single block is available then
|
||||
// a Zero block is provided to promote vectorizations. Thanks to Peter
|
||||
// Cordes for help with the SSE permutes below.
|
||||
// be permuted to the following. Thanks to Peter Cordes for help with the
|
||||
// SSE permutes below.
|
||||
// [A1 A2 A3 A4][B1 B2 B3 B4] ... => [A1 A3 B1 B3][A2 A4 B2 B4] ...
|
||||
const __m128 t0 = _mm_castsi128_ps(block0);
|
||||
const __m128 t1 = _mm_castsi128_ps(block1);
|
||||
|
|
@ -1424,9 +1562,8 @@ inline void SIMON64_Dec_4_Blocks(__m128i &block0, __m128i &block1, __m128i &bloc
|
|||
{
|
||||
// Rearrange the data for vectorization. The incoming data was read from
|
||||
// a big-endian byte array. Depending on the number of blocks it needs to
|
||||
// be permuted to the following. If only a single block is available then
|
||||
// a Zero block is provided to promote vectorizations. Thanks to Peter
|
||||
// Cordes for help with the SSE permutes below.
|
||||
// be permuted to the following. Thanks to Peter Cordes for help with the
|
||||
// SSE permutes below.
|
||||
// [A1 A2 A3 A4][B1 B2 B3 B4] ... => [A1 A3 B1 B3][A2 A4 B2 B4] ...
|
||||
const __m128 t0 = _mm_castsi128_ps(block0);
|
||||
const __m128 t1 = _mm_castsi128_ps(block1);
|
||||
|
|
@ -1477,8 +1614,8 @@ inline void SIMON64_Dec_4_Blocks(__m128i &block0, __m128i &block1, __m128i &bloc
|
|||
block3 = _mm_unpackhi_epi32(x2, y2);
|
||||
}
|
||||
|
||||
template <typename F1, typename F4>
|
||||
inline size_t SIMON64_AdvancedProcessBlocks_SSE41(F1 func1, F4 func4,
|
||||
template <typename F2, typename F4>
|
||||
inline size_t SIMON64_AdvancedProcessBlocks_SSE41(F2 func2, F4 func4,
|
||||
const word32 *subKeys, size_t rounds, const byte *inBlocks,
|
||||
const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
|
||||
{
|
||||
|
|
@ -1593,7 +1730,8 @@ inline size_t SIMON64_AdvancedProcessBlocks_SSE41(F1 func1, F4 func4,
|
|||
{
|
||||
// temp[] is an aligned array
|
||||
std::memcpy(temp, inBlocks, 8);
|
||||
__m128i block = _mm_load_si128(CONST_M128_CAST(temp));
|
||||
__m128i block, zero = _mm_setzero_si128();
|
||||
block = _mm_load_si128(CONST_M128_CAST(temp));
|
||||
|
||||
if (flags & BlockTransformation::BT_XorInput)
|
||||
{
|
||||
|
|
@ -1604,7 +1742,7 @@ inline size_t SIMON64_AdvancedProcessBlocks_SSE41(F1 func1, F4 func4,
|
|||
if (flags & BlockTransformation::BT_InBlockIsCounter)
|
||||
const_cast<byte *>(inBlocks)[7]++;
|
||||
|
||||
func1(block, subKeys, static_cast<unsigned int>(rounds));
|
||||
func2(block, zero, subKeys, static_cast<unsigned int>(rounds));
|
||||
|
||||
if (xorBlocks && !(flags & BlockTransformation::BT_XorInput))
|
||||
{
|
||||
|
|
@ -1627,7 +1765,6 @@ inline size_t SIMON64_AdvancedProcessBlocks_SSE41(F1 func1, F4 func4,
|
|||
|
||||
#endif // CRYPTOPP_SSE41_AVAILABLE
|
||||
|
||||
|
||||
ANONYMOUS_NAMESPACE_END
|
||||
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
|
|
@ -1640,14 +1777,14 @@ NAMESPACE_BEGIN(CryptoPP)
|
|||
size_t SIMON64_Enc_AdvancedProcessBlocks_NEON(const word32* subKeys, size_t rounds,
|
||||
const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
|
||||
{
|
||||
return SIMON64_AdvancedProcessBlocks_NEON(SIMON64_Enc_Block, SIMON64_Enc_4_Blocks,
|
||||
return SIMON64_AdvancedProcessBlocks_NEON(SIMON64_Enc_Block, SIMON64_Enc_6_Blocks,
|
||||
subKeys, rounds, inBlocks, xorBlocks, outBlocks, length, flags);
|
||||
}
|
||||
|
||||
size_t SIMON64_Dec_AdvancedProcessBlocks_NEON(const word32* subKeys, size_t rounds,
|
||||
const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
|
||||
{
|
||||
return SIMON64_AdvancedProcessBlocks_NEON(SIMON64_Dec_Block, SIMON64_Dec_4_Blocks,
|
||||
return SIMON64_AdvancedProcessBlocks_NEON(SIMON64_Dec_Block, SIMON64_Dec_6_Blocks,
|
||||
subKeys, rounds, inBlocks, xorBlocks, outBlocks, length, flags);
|
||||
}
|
||||
#endif // CRYPTOPP_ARM_NEON_AVAILABLE
|
||||
|
|
|
|||
313
speck-simd.cpp
313
speck-simd.cpp
|
|
@ -141,15 +141,14 @@ inline const word64* Ptr64(const T* ptr)
|
|||
return reinterpret_cast<const word64*>(ptr);
|
||||
}
|
||||
|
||||
inline void SPECK64_Enc_Block(uint32x4_t &block0, const word32 *subkeys, unsigned int rounds)
|
||||
inline void SPECK64_Enc_Block(uint32x4_t &block0, uint32x4_t &block1,
|
||||
const word32 *subkeys, unsigned int rounds)
|
||||
{
|
||||
// Rearrange the data for vectorization. The incoming data was read from
|
||||
// a big-endian byte array. Depending on the number of blocks it needs to
|
||||
// be permuted to the following. If only a single block is available then
|
||||
// a Zero block is provided to promote vectorizations.
|
||||
// be permuted to the following.
|
||||
// [A1 A2 A3 A4][B1 B2 B3 B4] ... => [A1 A3 B1 B3][A2 A4 B2 B4] ...
|
||||
const uint32x4_t zero = {0, 0, 0, 0};
|
||||
const uint32x4x2_t t0 = vuzpq_u32(block0, zero);
|
||||
const uint32x4x2_t t0 = vuzpq_u32(block0, block1);
|
||||
uint32x4_t x1 = t0.val[0];
|
||||
uint32x4_t y1 = t0.val[1];
|
||||
|
||||
|
|
@ -171,18 +170,17 @@ inline void SPECK64_Enc_Block(uint32x4_t &block0, const word32 *subkeys, unsigne
|
|||
// [A1 A3 B1 B3][A2 A4 B2 B4] => [A1 A2 A3 A4][B1 B2 B3 B4]
|
||||
const uint32x4x2_t t1 = vzipq_u32(x1, y1);
|
||||
block0 = t1.val[0];
|
||||
// block1 = t1.val[1];
|
||||
block1 = t1.val[1];
|
||||
}
|
||||
|
||||
inline void SPECK64_Dec_Block(uint32x4_t &block0, const word32 *subkeys, unsigned int rounds)
|
||||
inline void SPECK64_Dec_Block(uint32x4_t &block0, uint32x4_t &block1,
|
||||
const word32 *subkeys, unsigned int rounds)
|
||||
{
|
||||
// Rearrange the data for vectorization. The incoming data was read from
|
||||
// a big-endian byte array. Depending on the number of blocks it needs to
|
||||
// be permuted to the following. If only a single block is available then
|
||||
// a Zero block is provided to promote vectorizations.
|
||||
// be permuted to the following.
|
||||
// [A1 A2 A3 A4][B1 B2 B3 B4] ... => [A1 A3 B1 B3][A2 A4 B2 B4] ...
|
||||
const uint32x4_t zero = {0, 0, 0, 0};
|
||||
const uint32x4x2_t t0 = vuzpq_u32(block0, zero);
|
||||
const uint32x4x2_t t0 = vuzpq_u32(block0, block1);
|
||||
uint32x4_t x1 = t0.val[0];
|
||||
uint32x4_t y1 = t0.val[1];
|
||||
|
||||
|
|
@ -204,11 +202,12 @@ inline void SPECK64_Dec_Block(uint32x4_t &block0, const word32 *subkeys, unsigne
|
|||
// [A1 A3 B1 B3][A2 A4 B2 B4] => [A1 A2 A3 A4][B1 B2 B3 B4]
|
||||
const uint32x4x2_t t1 = vzipq_u32(x1, y1);
|
||||
block0 = t1.val[0];
|
||||
// block1 = t1.val[1];
|
||||
block1 = t1.val[1];
|
||||
}
|
||||
|
||||
inline void SPECK64_Enc_4_Blocks(uint32x4_t &block0, uint32x4_t &block1,
|
||||
uint32x4_t &block2, uint32x4_t &block3, const word32 *subkeys, unsigned int rounds)
|
||||
inline void SPECK64_Enc_6_Blocks(uint32x4_t &block0, uint32x4_t &block1,
|
||||
uint32x4_t &block2, uint32x4_t &block3, uint32x4_t &block4, uint32x4_t &block5,
|
||||
const word32 *subkeys, unsigned int rounds)
|
||||
{
|
||||
// Rearrange the data for vectorization. The incoming data was read from
|
||||
// a big-endian byte array. Depending on the number of blocks it needs to
|
||||
|
|
@ -223,8 +222,13 @@ inline void SPECK64_Enc_4_Blocks(uint32x4_t &block0, uint32x4_t &block1,
|
|||
uint32x4_t x2 = t1.val[0];
|
||||
uint32x4_t y2 = t1.val[1];
|
||||
|
||||
const uint32x4x2_t t2 = vuzpq_u32(block4, block5);
|
||||
uint32x4_t x3 = t2.val[0];
|
||||
uint32x4_t y3 = t2.val[1];
|
||||
|
||||
x1 = Shuffle32(x1); y1 = Shuffle32(y1);
|
||||
x2 = Shuffle32(x2); y2 = Shuffle32(y2);
|
||||
x3 = Shuffle32(x3); y3 = Shuffle32(y3);
|
||||
|
||||
for (size_t i=0; static_cast<int>(i)<rounds; ++i)
|
||||
{
|
||||
|
|
@ -232,18 +236,24 @@ inline void SPECK64_Enc_4_Blocks(uint32x4_t &block0, uint32x4_t &block1,
|
|||
|
||||
x1 = RotateRight32<8>(x1);
|
||||
x2 = RotateRight32<8>(x2);
|
||||
x3 = RotateRight32<8>(x3);
|
||||
x1 = vaddq_u32(x1, y1);
|
||||
x2 = vaddq_u32(x2, y2);
|
||||
x3 = vaddq_u32(x3, y3);
|
||||
x1 = veorq_u32(x1, rk);
|
||||
x2 = veorq_u32(x2, rk);
|
||||
x3 = veorq_u32(x3, rk);
|
||||
y1 = RotateLeft32<3>(y1);
|
||||
y2 = RotateLeft32<3>(y2);
|
||||
y3 = RotateLeft32<3>(y3);
|
||||
y1 = veorq_u32(y1, x1);
|
||||
y2 = veorq_u32(y2, x2);
|
||||
y3 = veorq_u32(y3, x3);
|
||||
}
|
||||
|
||||
x1 = Shuffle32(x1); y1 = Shuffle32(y1);
|
||||
x2 = Shuffle32(x2); y2 = Shuffle32(y2);
|
||||
x3 = Shuffle32(x3); y3 = Shuffle32(y3);
|
||||
|
||||
// [A1 A3 B1 B3][A2 A4 B2 B4] => [A1 A2 A3 A4][B1 B2 B3 B4]
|
||||
const uint32x4x2_t t3 = vzipq_u32(x1, y1);
|
||||
|
|
@ -253,10 +263,15 @@ inline void SPECK64_Enc_4_Blocks(uint32x4_t &block0, uint32x4_t &block1,
|
|||
const uint32x4x2_t t4 = vzipq_u32(x2, y2);
|
||||
block2 = t4.val[0];
|
||||
block3 = t4.val[1];
|
||||
|
||||
const uint32x4x2_t t5 = vzipq_u32(x3, y3);
|
||||
block4 = t5.val[0];
|
||||
block5 = t5.val[1];
|
||||
}
|
||||
|
||||
inline void SPECK64_Dec_4_Blocks(uint32x4_t &block0, uint32x4_t &block1,
|
||||
uint32x4_t &block2, uint32x4_t &block3, const word32 *subkeys, unsigned int rounds)
|
||||
inline void SPECK64_Dec_6_Blocks(uint32x4_t &block0, uint32x4_t &block1,
|
||||
uint32x4_t &block2, uint32x4_t &block3, uint32x4_t &block4, uint32x4_t &block5,
|
||||
const word32 *subkeys, unsigned int rounds)
|
||||
{
|
||||
// Rearrange the data for vectorization. The incoming data was read from
|
||||
// a big-endian byte array. Depending on the number of blocks it needs to
|
||||
|
|
@ -271,8 +286,13 @@ inline void SPECK64_Dec_4_Blocks(uint32x4_t &block0, uint32x4_t &block1,
|
|||
uint32x4_t x2 = t1.val[0];
|
||||
uint32x4_t y2 = t1.val[1];
|
||||
|
||||
const uint32x4x2_t t2 = vuzpq_u32(block4, block5);
|
||||
uint32x4_t x3 = t2.val[0];
|
||||
uint32x4_t y3 = t2.val[1];
|
||||
|
||||
x1 = Shuffle32(x1); y1 = Shuffle32(y1);
|
||||
x2 = Shuffle32(x2); y2 = Shuffle32(y2);
|
||||
x3 = Shuffle32(x3); y3 = Shuffle32(y3);
|
||||
|
||||
for (size_t i=rounds-1; static_cast<int>(i)>=0; --i)
|
||||
{
|
||||
|
|
@ -280,18 +300,24 @@ inline void SPECK64_Dec_4_Blocks(uint32x4_t &block0, uint32x4_t &block1,
|
|||
|
||||
y1 = veorq_u32(y1, x1);
|
||||
y2 = veorq_u32(y2, x2);
|
||||
y3 = veorq_u32(y3, x3);
|
||||
y1 = RotateRight32<3>(y1);
|
||||
y2 = RotateRight32<3>(y2);
|
||||
y3 = RotateRight32<3>(y3);
|
||||
x1 = veorq_u32(x1, rk);
|
||||
x2 = veorq_u32(x2, rk);
|
||||
x3 = veorq_u32(x3, rk);
|
||||
x1 = vsubq_u32(x1, y1);
|
||||
x2 = vsubq_u32(x2, y2);
|
||||
x3 = vsubq_u32(x3, y3);
|
||||
x1 = RotateLeft32<8>(x1);
|
||||
x2 = RotateLeft32<8>(x2);
|
||||
x3 = RotateLeft32<8>(x3);
|
||||
}
|
||||
|
||||
x1 = Shuffle32(x1); y1 = Shuffle32(y1);
|
||||
x2 = Shuffle32(x2); y2 = Shuffle32(y2);
|
||||
x3 = Shuffle32(x3); y3 = Shuffle32(y3);
|
||||
|
||||
// [A1 A3 B1 B3][A2 A4 B2 B4] => [A1 A2 A3 A4][B1 B2 B3 B4]
|
||||
const uint32x4x2_t t3 = vzipq_u32(x1, y1);
|
||||
|
|
@ -301,10 +327,14 @@ inline void SPECK64_Dec_4_Blocks(uint32x4_t &block0, uint32x4_t &block1,
|
|||
const uint32x4x2_t t4 = vzipq_u32(x2, y2);
|
||||
block2 = t4.val[0];
|
||||
block3 = t4.val[1];
|
||||
|
||||
const uint32x4x2_t t5 = vzipq_u32(x3, y3);
|
||||
block4 = t5.val[0];
|
||||
block5 = t5.val[1];
|
||||
}
|
||||
|
||||
template <typename F1, typename F4>
|
||||
inline size_t SPECK64_AdvancedProcessBlocks_NEON(F1 func1, F4 func4,
|
||||
template <typename F2, typename F6>
|
||||
inline size_t SPECK64_AdvancedProcessBlocks_NEON(F2 func2, F6 func6,
|
||||
const word32 *subKeys, size_t rounds, const byte *inBlocks,
|
||||
const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
|
||||
{
|
||||
|
|
@ -330,9 +360,9 @@ inline size_t SPECK64_AdvancedProcessBlocks_NEON(F1 func1, F4 func4,
|
|||
|
||||
if (flags & BlockTransformation::BT_AllowParallel)
|
||||
{
|
||||
while (length >= 4*neonBlockSize)
|
||||
while (length >= 6*neonBlockSize)
|
||||
{
|
||||
uint32x4_t block0, block1, block2, block3;
|
||||
uint32x4_t block0, block1, block2, block3, block4, block5;
|
||||
block0 = vreinterpretq_u32_u8(vld1q_u8(inBlocks));
|
||||
|
||||
if (flags & BlockTransformation::BT_InBlockIsCounter)
|
||||
|
|
@ -341,8 +371,10 @@ inline size_t SPECK64_AdvancedProcessBlocks_NEON(F1 func1, F4 func4,
|
|||
block1 = vaddq_u32(block0, be1);
|
||||
block2 = vaddq_u32(block1, be1);
|
||||
block3 = vaddq_u32(block2, be1);
|
||||
block4 = vaddq_u32(block3, be1);
|
||||
block5 = vaddq_u32(block4, be1);
|
||||
vst1q_u8(const_cast<byte*>(inBlocks),
|
||||
vreinterpretq_u8_u32(vaddq_u32(block3, be1)));
|
||||
vreinterpretq_u8_u32(vaddq_u32(block5, be1)));
|
||||
}
|
||||
else
|
||||
{
|
||||
|
|
@ -350,7 +382,9 @@ inline size_t SPECK64_AdvancedProcessBlocks_NEON(F1 func1, F4 func4,
|
|||
block1 = vreinterpretq_u32_u8(vld1q_u8(inBlocks+1*inc));
|
||||
block2 = vreinterpretq_u32_u8(vld1q_u8(inBlocks+2*inc));
|
||||
block3 = vreinterpretq_u32_u8(vld1q_u8(inBlocks+3*inc));
|
||||
inBlocks += 4*inc;
|
||||
block4 = vreinterpretq_u32_u8(vld1q_u8(inBlocks+4*inc));
|
||||
block5 = vreinterpretq_u32_u8(vld1q_u8(inBlocks+5*inc));
|
||||
inBlocks += 6*inc;
|
||||
}
|
||||
|
||||
if (flags & BlockTransformation::BT_XorInput)
|
||||
|
|
@ -360,10 +394,12 @@ inline size_t SPECK64_AdvancedProcessBlocks_NEON(F1 func1, F4 func4,
|
|||
block1 = veorq_u32(block1, vreinterpretq_u32_u8(vld1q_u8(xorBlocks+1*inc)));
|
||||
block2 = veorq_u32(block2, vreinterpretq_u32_u8(vld1q_u8(xorBlocks+2*inc)));
|
||||
block3 = veorq_u32(block3, vreinterpretq_u32_u8(vld1q_u8(xorBlocks+3*inc)));
|
||||
xorBlocks += 4*inc;
|
||||
block4 = veorq_u32(block4, vreinterpretq_u32_u8(vld1q_u8(xorBlocks+4*inc)));
|
||||
block5 = veorq_u32(block5, vreinterpretq_u32_u8(vld1q_u8(xorBlocks+5*inc)));
|
||||
xorBlocks += 6*inc;
|
||||
}
|
||||
|
||||
func4(block0, block1, block2, block3, subKeys, static_cast<unsigned int>(rounds));
|
||||
func6(block0, block1, block2, block3, block4, block5, subKeys, static_cast<unsigned int>(rounds));
|
||||
|
||||
if (xorBlocks && !(flags & BlockTransformation::BT_XorInput))
|
||||
{
|
||||
|
|
@ -372,7 +408,9 @@ inline size_t SPECK64_AdvancedProcessBlocks_NEON(F1 func1, F4 func4,
|
|||
block1 = veorq_u32(block1, vreinterpretq_u32_u8(vld1q_u8(xorBlocks+1*inc)));
|
||||
block2 = veorq_u32(block2, vreinterpretq_u32_u8(vld1q_u8(xorBlocks+2*inc)));
|
||||
block3 = veorq_u32(block3, vreinterpretq_u32_u8(vld1q_u8(xorBlocks+3*inc)));
|
||||
xorBlocks += 4*inc;
|
||||
block4 = veorq_u32(block4, vreinterpretq_u32_u8(vld1q_u8(xorBlocks+4*inc)));
|
||||
block5 = veorq_u32(block5, vreinterpretq_u32_u8(vld1q_u8(xorBlocks+5*inc)));
|
||||
xorBlocks += 6*inc;
|
||||
}
|
||||
|
||||
const int inc = static_cast<int>(outIncrement);
|
||||
|
|
@ -380,9 +418,56 @@ inline size_t SPECK64_AdvancedProcessBlocks_NEON(F1 func1, F4 func4,
|
|||
vst1q_u8(outBlocks+1*inc, vreinterpretq_u8_u32(block1));
|
||||
vst1q_u8(outBlocks+2*inc, vreinterpretq_u8_u32(block2));
|
||||
vst1q_u8(outBlocks+3*inc, vreinterpretq_u8_u32(block3));
|
||||
vst1q_u8(outBlocks+4*inc, vreinterpretq_u8_u32(block4));
|
||||
vst1q_u8(outBlocks+5*inc, vreinterpretq_u8_u32(block5));
|
||||
|
||||
outBlocks += 4*inc;
|
||||
length -= 4*neonBlockSize;
|
||||
outBlocks += 6*inc;
|
||||
length -= 6*neonBlockSize;
|
||||
}
|
||||
|
||||
while (length >= 2*neonBlockSize)
|
||||
{
|
||||
uint32x4_t block0, block1;
|
||||
block0 = vreinterpretq_u32_u8(vld1q_u8(inBlocks));
|
||||
|
||||
if (flags & BlockTransformation::BT_InBlockIsCounter)
|
||||
{
|
||||
const uint32x4_t be1 = vld1q_u32(s_one64);
|
||||
block1 = vaddq_u32(block0, be1);
|
||||
vst1q_u8(const_cast<byte*>(inBlocks),
|
||||
vreinterpretq_u8_u32(vaddq_u32(block1, be1)));
|
||||
}
|
||||
else
|
||||
{
|
||||
const int inc = static_cast<int>(inIncrement);
|
||||
block1 = vreinterpretq_u32_u8(vld1q_u8(inBlocks+1*inc));
|
||||
inBlocks += 2*inc;
|
||||
}
|
||||
|
||||
if (flags & BlockTransformation::BT_XorInput)
|
||||
{
|
||||
const int inc = static_cast<int>(xorIncrement);
|
||||
block0 = veorq_u32(block0, vreinterpretq_u32_u8(vld1q_u8(xorBlocks+0*inc)));
|
||||
block1 = veorq_u32(block1, vreinterpretq_u32_u8(vld1q_u8(xorBlocks+1*inc)));
|
||||
xorBlocks += 2*inc;
|
||||
}
|
||||
|
||||
func2(block0, block1, subKeys, static_cast<unsigned int>(rounds));
|
||||
|
||||
if (xorBlocks && !(flags & BlockTransformation::BT_XorInput))
|
||||
{
|
||||
const int inc = static_cast<int>(xorIncrement);
|
||||
block0 = veorq_u32(block0, vreinterpretq_u32_u8(vld1q_u8(xorBlocks+0*inc)));
|
||||
block1 = veorq_u32(block1, vreinterpretq_u32_u8(vld1q_u8(xorBlocks+1*inc)));
|
||||
xorBlocks += 2*inc;
|
||||
}
|
||||
|
||||
const int inc = static_cast<int>(outIncrement);
|
||||
vst1q_u8(outBlocks+0*inc, vreinterpretq_u8_u32(block0));
|
||||
vst1q_u8(outBlocks+1*inc, vreinterpretq_u8_u32(block1));
|
||||
|
||||
outBlocks += 2*inc;
|
||||
length -= 2*neonBlockSize;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -408,7 +493,7 @@ inline size_t SPECK64_AdvancedProcessBlocks_NEON(F1 func1, F4 func4,
|
|||
|
||||
while (length >= blockSize)
|
||||
{
|
||||
uint32x4_t block;
|
||||
uint32x4_t block, zero = {0,0,0,0};
|
||||
block = vsetq_lane_u32(Ptr32(inBlocks)[0], block, 0);
|
||||
block = vsetq_lane_u32(Ptr32(inBlocks)[1], block, 1);
|
||||
|
||||
|
|
@ -423,7 +508,7 @@ inline size_t SPECK64_AdvancedProcessBlocks_NEON(F1 func1, F4 func4,
|
|||
if (flags & BlockTransformation::BT_InBlockIsCounter)
|
||||
const_cast<byte *>(inBlocks)[7]++;
|
||||
|
||||
func1(block, subKeys, static_cast<unsigned int>(rounds));
|
||||
func2(block, zero, subKeys, static_cast<unsigned int>(rounds));
|
||||
|
||||
if (xorBlocks && !(flags & BlockTransformation::BT_XorInput))
|
||||
{
|
||||
|
|
@ -536,14 +621,13 @@ inline uint64x2_t Shuffle64(const uint64x2_t& val)
|
|||
#endif
|
||||
}
|
||||
|
||||
inline void SPECK128_Enc_Block(uint64x2_t &block0, const word64 *subkeys, unsigned int rounds)
|
||||
inline void SPECK128_Enc_Block(uint64x2_t &block0, uint64x2_t &block1,
|
||||
const word64 *subkeys, unsigned int rounds)
|
||||
{
|
||||
// Rearrange the data for vectorization. The incoming data was read from
|
||||
// a big-endian byte array. Depending on the number of blocks it needs to
|
||||
// be permuted to the following. If only a single block is available then
|
||||
// a Zero block is provided to promote vectorizations.
|
||||
// be permuted to the following.
|
||||
// [A1 A2][B1 B2] ... => [A1 B1][A2 B2] ...
|
||||
uint64x2_t block1 = {0};
|
||||
uint64x2_t x1 = UnpackLow64(block0, block1);
|
||||
uint64x2_t y1 = UnpackHigh64(block0, block1);
|
||||
|
||||
|
|
@ -564,17 +648,16 @@ inline void SPECK128_Enc_Block(uint64x2_t &block0, const word64 *subkeys, unsign
|
|||
|
||||
// [A1 B1][A2 B2] ... => [A1 A2][B1 B2] ...
|
||||
block0 = UnpackLow64(x1, y1);
|
||||
// block1 = UnpackHigh64(x1, y1);
|
||||
block1 = UnpackHigh64(x1, y1);
|
||||
}
|
||||
|
||||
inline void SPECK128_Enc_6_Blocks(uint64x2_t &block0, uint64x2_t &block1,
|
||||
uint64x2_t &block2, uint64x2_t &block3, uint64x2_t &block4,
|
||||
uint64x2_t &block5, const word64 *subkeys, unsigned int rounds)
|
||||
uint64x2_t &block2, uint64x2_t &block3, uint64x2_t &block4, uint64x2_t &block5,
|
||||
const word64 *subkeys, unsigned int rounds)
|
||||
{
|
||||
// Rearrange the data for vectorization. The incoming data was read from
|
||||
// a big-endian byte array. Depending on the number of blocks it needs to
|
||||
// be permuted to the following. If only a single block is available then
|
||||
// a Zero block is provided to promote vectorizations.
|
||||
// be permuted to the following.
|
||||
// [A1 A2][B1 B2] ... => [A1 B1][A2 B2] ...
|
||||
uint64x2_t x1 = UnpackLow64(block0, block1);
|
||||
uint64x2_t y1 = UnpackHigh64(block0, block1);
|
||||
|
|
@ -621,14 +704,13 @@ inline void SPECK128_Enc_6_Blocks(uint64x2_t &block0, uint64x2_t &block1,
|
|||
block5 = UnpackHigh64(x3, y3);
|
||||
}
|
||||
|
||||
inline void SPECK128_Dec_Block(uint64x2_t &block0, const word64 *subkeys, unsigned int rounds)
|
||||
inline void SPECK128_Dec_Block(uint64x2_t &block0, uint64x2_t &block1,
|
||||
const word64 *subkeys, unsigned int rounds)
|
||||
{
|
||||
// Rearrange the data for vectorization. The incoming data was read from
|
||||
// a big-endian byte array. Depending on the number of blocks it needs to
|
||||
// be permuted to the following. If only a single block is available then
|
||||
// a Zero block is provided to promote vectorizations.
|
||||
// be permuted to the following.
|
||||
// [A1 A2][B1 B2] ... => [A1 B1][A2 B2] ...
|
||||
uint64x2_t block1 = {0};
|
||||
uint64x2_t x1 = UnpackLow64(block0, block1);
|
||||
uint64x2_t y1 = UnpackHigh64(block0, block1);
|
||||
|
||||
|
|
@ -649,17 +731,16 @@ inline void SPECK128_Dec_Block(uint64x2_t &block0, const word64 *subkeys, unsign
|
|||
|
||||
// [A1 B1][A2 B2] ... => [A1 A2][B1 B2] ...
|
||||
block0 = UnpackLow64(x1, y1);
|
||||
// block1 = UnpackHigh64(x1, y1);
|
||||
block1 = UnpackHigh64(x1, y1);
|
||||
}
|
||||
|
||||
inline void SPECK128_Dec_6_Blocks(uint64x2_t &block0, uint64x2_t &block1,
|
||||
uint64x2_t &block2, uint64x2_t &block3, uint64x2_t &block4,
|
||||
uint64x2_t &block5, const word64 *subkeys, unsigned int rounds)
|
||||
uint64x2_t &block2, uint64x2_t &block3, uint64x2_t &block4, uint64x2_t &block5,
|
||||
const word64 *subkeys, unsigned int rounds)
|
||||
{
|
||||
// Rearrange the data for vectorization. The incoming data was read from
|
||||
// a big-endian byte array. Depending on the number of blocks it needs to
|
||||
// be permuted to the following. If only a single block is available then
|
||||
// a Zero block is provided to promote vectorizations.
|
||||
// be permuted to the following.
|
||||
// [A1 A2][B1 B2] ... => [A1 B1][A2 B2] ...
|
||||
uint64x2_t x1 = UnpackLow64(block0, block1);
|
||||
uint64x2_t y1 = UnpackHigh64(block0, block1);
|
||||
|
|
@ -706,8 +787,8 @@ inline void SPECK128_Dec_6_Blocks(uint64x2_t &block0, uint64x2_t &block1,
|
|||
block5 = UnpackHigh64(x3, y3);
|
||||
}
|
||||
|
||||
template <typename F1, typename F6>
|
||||
size_t SPECK128_AdvancedProcessBlocks_NEON(F1 func1, F6 func6,
|
||||
template <typename F2, typename F6>
|
||||
size_t SPECK128_AdvancedProcessBlocks_NEON(F2 func2, F6 func6,
|
||||
const word64 *subKeys, size_t rounds, const byte *inBlocks,
|
||||
const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
|
||||
{
|
||||
|
|
@ -772,7 +853,7 @@ size_t SPECK128_AdvancedProcessBlocks_NEON(F1 func1, F6 func6,
|
|||
xorBlocks += 6*inc;
|
||||
}
|
||||
|
||||
func6(block0, block1, block2, block3, block4, block5, subKeys, rounds);
|
||||
func6(block0, block1, block2, block3, block4, block5, subKeys, static_cast<unsigned int>(rounds));
|
||||
|
||||
if (xorBlocks && !(flags & BlockTransformation::BT_XorInput))
|
||||
{
|
||||
|
|
@ -797,11 +878,57 @@ size_t SPECK128_AdvancedProcessBlocks_NEON(F1 func1, F6 func6,
|
|||
outBlocks += 6*inc;
|
||||
length -= 6*blockSize;
|
||||
}
|
||||
|
||||
while (length >= 2*blockSize)
|
||||
{
|
||||
uint64x2_t block0, block1;
|
||||
block0 = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
|
||||
|
||||
if (flags & BlockTransformation::BT_InBlockIsCounter)
|
||||
{
|
||||
uint64x2_t be = vreinterpretq_u64_u32(vld1q_u32(s_one128));
|
||||
block1 = vaddq_u64(block0, be);
|
||||
vst1q_u8(const_cast<byte*>(inBlocks),
|
||||
vreinterpretq_u8_u64(vaddq_u64(block1, be)));
|
||||
}
|
||||
else
|
||||
{
|
||||
const int inc = static_cast<int>(inIncrement);
|
||||
block1 = vreinterpretq_u64_u8(vld1q_u8(inBlocks+1*inc));
|
||||
inBlocks += 2*inc;
|
||||
}
|
||||
|
||||
if (flags & BlockTransformation::BT_XorInput)
|
||||
{
|
||||
const int inc = static_cast<int>(xorIncrement);
|
||||
block0 = veorq_u64(block0, vreinterpretq_u64_u8(vld1q_u8(xorBlocks+0*inc)));
|
||||
block1 = veorq_u64(block1, vreinterpretq_u64_u8(vld1q_u8(xorBlocks+1*inc)));
|
||||
xorBlocks += 2*inc;
|
||||
}
|
||||
|
||||
func2(block0, block1, subKeys, static_cast<unsigned int>(rounds));
|
||||
|
||||
if (xorBlocks && !(flags & BlockTransformation::BT_XorInput))
|
||||
{
|
||||
const int inc = static_cast<int>(xorIncrement);
|
||||
block0 = veorq_u64(block0, vreinterpretq_u64_u8(vld1q_u8(xorBlocks+0*inc)));
|
||||
block1 = veorq_u64(block1, vreinterpretq_u64_u8(vld1q_u8(xorBlocks+1*inc)));
|
||||
xorBlocks += 2*inc;
|
||||
}
|
||||
|
||||
const int inc = static_cast<int>(outIncrement);
|
||||
vst1q_u8(outBlocks+0*inc, vreinterpretq_u8_u64(block0));
|
||||
vst1q_u8(outBlocks+1*inc, vreinterpretq_u8_u64(block1));
|
||||
|
||||
outBlocks += 2*inc;
|
||||
length -= 2*blockSize;
|
||||
}
|
||||
}
|
||||
|
||||
while (length >= blockSize)
|
||||
{
|
||||
uint64x2_t block = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
|
||||
uint64x2_t block, zero = {0,0};
|
||||
block = vreinterpretq_u64_u8(vld1q_u8(inBlocks));
|
||||
|
||||
if (flags & BlockTransformation::BT_XorInput)
|
||||
block = veorq_u64(block, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
|
||||
|
|
@ -809,7 +936,7 @@ size_t SPECK128_AdvancedProcessBlocks_NEON(F1 func1, F6 func6,
|
|||
if (flags & BlockTransformation::BT_InBlockIsCounter)
|
||||
const_cast<byte *>(inBlocks)[15]++;
|
||||
|
||||
func1(block, subKeys, rounds);
|
||||
func2(block, zero, subKeys, static_cast<unsigned int>(rounds));
|
||||
|
||||
if (xorBlocks && !(flags & BlockTransformation::BT_XorInput))
|
||||
block = veorq_u64(block, vreinterpretq_u64_u8(vld1q_u8(xorBlocks)));
|
||||
|
|
@ -882,14 +1009,13 @@ inline __m128i RotateRight64<8>(const __m128i& val)
|
|||
|
||||
#endif // CRYPTOPP_AVX512_ROTATE
|
||||
|
||||
inline void SPECK128_Enc_Block(__m128i &block0, const word64 *subkeys, unsigned int rounds)
|
||||
inline void SPECK128_Enc_Block(__m128i &block0, __m128i &block1,
|
||||
const word64 *subkeys, unsigned int rounds)
|
||||
{
|
||||
// Rearrange the data for vectorization. The incoming data was read from
|
||||
// a big-endian byte array. Depending on the number of blocks it needs to
|
||||
// be permuted to the following. If only a single block is available then
|
||||
// a Zero block is provided to promote vectorizations.
|
||||
// be permuted to the following.
|
||||
// [A1 A2][B1 B2] ... => [A1 B1][A2 B2] ...
|
||||
__m128i block1 = _mm_setzero_si128();
|
||||
__m128i x1 = _mm_unpacklo_epi64(block0, block1);
|
||||
__m128i y1 = _mm_unpackhi_epi64(block0, block1);
|
||||
|
||||
|
|
@ -914,7 +1040,7 @@ inline void SPECK128_Enc_Block(__m128i &block0, const word64 *subkeys, unsigned
|
|||
|
||||
// [A1 B1][A2 B2] ... => [A1 A2][B1 B2] ...
|
||||
block0 = _mm_unpacklo_epi64(x1, y1);
|
||||
// block1 = _mm_unpackhi_epi64(x1, y1);
|
||||
block1 = _mm_unpackhi_epi64(x1, y1);
|
||||
}
|
||||
|
||||
inline void SPECK128_Enc_4_Blocks(__m128i &block0, __m128i &block1,
|
||||
|
|
@ -922,8 +1048,7 @@ inline void SPECK128_Enc_4_Blocks(__m128i &block0, __m128i &block1,
|
|||
{
|
||||
// Rearrange the data for vectorization. The incoming data was read from
|
||||
// a big-endian byte array. Depending on the number of blocks it needs to
|
||||
// be permuted to the following. If only a single block is available then
|
||||
// a Zero block is provided to promote vectorizations.
|
||||
// be permuted to the following.
|
||||
// [A1 A2][B1 B2] ... => [A1 B1][A2 B2] ...
|
||||
__m128i x1 = _mm_unpacklo_epi64(block0, block1);
|
||||
__m128i y1 = _mm_unpackhi_epi64(block0, block1);
|
||||
|
|
@ -965,14 +1090,13 @@ inline void SPECK128_Enc_4_Blocks(__m128i &block0, __m128i &block1,
|
|||
block3 = _mm_unpackhi_epi64(x2, y2);
|
||||
}
|
||||
|
||||
inline void SPECK128_Dec_Block(__m128i &block0, const word64 *subkeys, unsigned int rounds)
|
||||
inline void SPECK128_Dec_Block(__m128i &block0, __m128i &block1,
|
||||
const word64 *subkeys, unsigned int rounds)
|
||||
{
|
||||
// Rearrange the data for vectorization. The incoming data was read from
|
||||
// a big-endian byte array. Depending on the number of blocks it needs to
|
||||
// be permuted to the following. If only a single block is available then
|
||||
// a Zero block is provided to promote vectorizations.
|
||||
// be permuted to the following.
|
||||
// [A1 A2][B1 B2] ... => [A1 B1][A2 B2] ...
|
||||
__m128i block1 = _mm_setzero_si128();
|
||||
__m128i x1 = _mm_unpacklo_epi64(block0, block1);
|
||||
__m128i y1 = _mm_unpackhi_epi64(block0, block1);
|
||||
|
||||
|
|
@ -997,7 +1121,7 @@ inline void SPECK128_Dec_Block(__m128i &block0, const word64 *subkeys, unsigned
|
|||
|
||||
// [A1 B1][A2 B2] ... => [A1 A2][B1 B2] ...
|
||||
block0 = _mm_unpacklo_epi64(x1, y1);
|
||||
// block1 = _mm_unpackhi_epi64(x1, y1);
|
||||
block1 = _mm_unpackhi_epi64(x1, y1);
|
||||
}
|
||||
|
||||
inline void SPECK128_Dec_4_Blocks(__m128i &block0, __m128i &block1,
|
||||
|
|
@ -1005,8 +1129,7 @@ inline void SPECK128_Dec_4_Blocks(__m128i &block0, __m128i &block1,
|
|||
{
|
||||
// Rearrange the data for vectorization. The incoming data was read from
|
||||
// a big-endian byte array. Depending on the number of blocks it needs to
|
||||
// be permuted to the following. If only a single block is available then
|
||||
// a Zero block is provided to promote vectorizations.
|
||||
// be permuted to the following.
|
||||
// [A1 A2][B1 B2] ... => [A1 B1][A2 B2] ...
|
||||
__m128i x1 = _mm_unpacklo_epi64(block0, block1);
|
||||
__m128i y1 = _mm_unpackhi_epi64(block0, block1);
|
||||
|
|
@ -1048,8 +1171,8 @@ inline void SPECK128_Dec_4_Blocks(__m128i &block0, __m128i &block1,
|
|||
block3 = _mm_unpackhi_epi64(x2, y2);
|
||||
}
|
||||
|
||||
template <typename F1, typename F4>
|
||||
inline size_t SPECK128_AdvancedProcessBlocks_SSSE3(F1 func1, F4 func4,
|
||||
template <typename F2, typename F4>
|
||||
inline size_t SPECK128_AdvancedProcessBlocks_SSSE3(F2 func2, F4 func4,
|
||||
const word64 *subKeys, size_t rounds, const byte *inBlocks,
|
||||
const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
|
||||
{
|
||||
|
|
@ -1140,7 +1263,8 @@ inline size_t SPECK128_AdvancedProcessBlocks_SSSE3(F1 func1, F4 func4,
|
|||
|
||||
while (length >= blockSize)
|
||||
{
|
||||
__m128i block = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
|
||||
__m128i block, zero = _mm_setzero_si128();
|
||||
block = _mm_loadu_si128(CONST_M128_CAST(inBlocks));
|
||||
|
||||
if (flags & BlockTransformation::BT_XorInput)
|
||||
block = _mm_xor_si128(block, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
|
||||
|
|
@ -1148,7 +1272,7 @@ inline size_t SPECK128_AdvancedProcessBlocks_SSSE3(F1 func1, F4 func4,
|
|||
if (flags & BlockTransformation::BT_InBlockIsCounter)
|
||||
const_cast<byte *>(inBlocks)[15]++;
|
||||
|
||||
func1(block, subKeys, static_cast<unsigned int>(rounds));
|
||||
func2(block, zero, subKeys, static_cast<unsigned int>(rounds));
|
||||
|
||||
if (xorBlocks && !(flags & BlockTransformation::BT_XorInput))
|
||||
block = _mm_xor_si128(block, _mm_loadu_si128(CONST_M128_CAST(xorBlocks)));
|
||||
|
|
@ -1198,17 +1322,16 @@ inline __m128i RotateRight32<8>(const __m128i& val)
|
|||
return _mm_shuffle_epi8(val, mask);
|
||||
}
|
||||
|
||||
inline void SPECK64_Enc_Block(__m128i &block0, const word32 *subkeys, unsigned int rounds)
|
||||
inline void SPECK64_Enc_Block(__m128i &block0, __m128i &block1,
|
||||
const word32 *subkeys, unsigned int rounds)
|
||||
{
|
||||
// Rearrange the data for vectorization. The incoming data was read from
|
||||
// a big-endian byte array. Depending on the number of blocks it needs to
|
||||
// be permuted to the following. If only a single block is available then
|
||||
// a Zero block is provided to promote vectorizations. Thanks to Peter
|
||||
// Cordes for help with the SSE permutes below.
|
||||
// be permuted to the following. Thanks to Peter Cordes for help with the
|
||||
// SSE permutes below.
|
||||
// [A1 A2 A3 A4][B1 B2 B3 B4] ... => [A1 A3 B1 B3][A2 A4 B2 B4] ...
|
||||
const __m128i zero = _mm_setzero_si128();
|
||||
const __m128 t0 = _mm_castsi128_ps(block0);
|
||||
const __m128 t1 = _mm_castsi128_ps(zero);
|
||||
const __m128 t1 = _mm_castsi128_ps(block1);
|
||||
__m128i x1 = _mm_castps_si128(_mm_shuffle_ps(t0, t1, _MM_SHUFFLE(2,0,2,0)));
|
||||
__m128i y1 = _mm_castps_si128(_mm_shuffle_ps(t0, t1, _MM_SHUFFLE(3,1,3,1)));
|
||||
|
||||
|
|
@ -1233,20 +1356,19 @@ inline void SPECK64_Enc_Block(__m128i &block0, const word32 *subkeys, unsigned i
|
|||
// The is roughly the SSE equivalent to ARM vzp32
|
||||
// [A1 A3 B1 B3][A2 A4 B2 B4] => [A1 A2 A3 A4][B1 B2 B3 B4]
|
||||
block0 = _mm_unpacklo_epi32(x1, y1);
|
||||
// block1 = _mm_unpackhigh_epi32(x1, y1);
|
||||
block1 = _mm_unpackhi_epi32(x1, y1);
|
||||
}
|
||||
|
||||
inline void SPECK64_Dec_Block(__m128i &block0, const word32 *subkeys, unsigned int rounds)
|
||||
inline void SPECK64_Dec_Block(__m128i &block0, __m128i &block1,
|
||||
const word32 *subkeys, unsigned int rounds)
|
||||
{
|
||||
// Rearrange the data for vectorization. The incoming data was read from
|
||||
// a big-endian byte array. Depending on the number of blocks it needs to
|
||||
// be permuted to the following. If only a single block is available then
|
||||
// a Zero block is provided to promote vectorizations. Thanks to Peter
|
||||
// Cordes for help with the SSE permutes below.
|
||||
// be permuted to the following. Thanks to Peter Cordes for help with the
|
||||
// SSE permutes below.
|
||||
// [A1 A2 A3 A4][B1 B2 B3 B4] ... => [A1 A3 B1 B3][A2 A4 B2 B4] ...
|
||||
const __m128i zero = _mm_setzero_si128();
|
||||
const __m128 t0 = _mm_castsi128_ps(block0);
|
||||
const __m128 t1 = _mm_castsi128_ps(zero);
|
||||
const __m128 t1 = _mm_castsi128_ps(block1);
|
||||
__m128i x1 = _mm_castps_si128(_mm_shuffle_ps(t0, t1, _MM_SHUFFLE(2,0,2,0)));
|
||||
__m128i y1 = _mm_castps_si128(_mm_shuffle_ps(t0, t1, _MM_SHUFFLE(3,1,3,1)));
|
||||
|
||||
|
|
@ -1271,7 +1393,7 @@ inline void SPECK64_Dec_Block(__m128i &block0, const word32 *subkeys, unsigned i
|
|||
// The is roughly the SSE equivalent to ARM vzp32
|
||||
// [A1 A3 B1 B3][A2 A4 B2 B4] => [A1 A2 A3 A4][B1 B2 B3 B4]
|
||||
block0 = _mm_unpacklo_epi32(x1, y1);
|
||||
// block1 = _mm_unpackhigh_epi32(x1, y1);
|
||||
block1 = _mm_unpackhi_epi32(x1, y1);
|
||||
}
|
||||
|
||||
inline void SPECK64_Enc_4_Blocks(__m128i &block0, __m128i &block1, __m128i &block2,
|
||||
|
|
@ -1279,9 +1401,8 @@ inline void SPECK64_Enc_4_Blocks(__m128i &block0, __m128i &block1, __m128i &bloc
|
|||
{
|
||||
// Rearrange the data for vectorization. The incoming data was read from
|
||||
// a big-endian byte array. Depending on the number of blocks it needs to
|
||||
// be permuted to the following. If only a single block is available then
|
||||
// a Zero block is provided to promote vectorizations. Thanks to Peter
|
||||
// Cordes for help with the SSE permutes below.
|
||||
// be permuted to the following. Thanks to Peter Cordes for help with the
|
||||
// SSE permutes below.
|
||||
// [A1 A2 A3 A4][B1 B2 B3 B4] ... => [A1 A3 B1 B3][A2 A4 B2 B4] ...
|
||||
const __m128 t0 = _mm_castsi128_ps(block0);
|
||||
const __m128 t1 = _mm_castsi128_ps(block1);
|
||||
|
|
@ -1333,9 +1454,8 @@ inline void SPECK64_Dec_4_Blocks(__m128i &block0, __m128i &block1, __m128i &bloc
|
|||
{
|
||||
// Rearrange the data for vectorization. The incoming data was read from
|
||||
// a big-endian byte array. Depending on the number of blocks it needs to
|
||||
// be permuted to the following. If only a single block is available then
|
||||
// a Zero block is provided to promote vectorizations. Thanks to Peter
|
||||
// Cordes for help with the SSE permutes below.
|
||||
// be permuted to the following. Thanks to Peter Cordes for help with the
|
||||
// SSE permutes below.
|
||||
// [A1 A2 A3 A4][B1 B2 B3 B4] ... => [A1 A3 B1 B3][A2 A4 B2 B4] ...
|
||||
const __m128 t0 = _mm_castsi128_ps(block0);
|
||||
const __m128 t1 = _mm_castsi128_ps(block1);
|
||||
|
|
@ -1382,8 +1502,8 @@ inline void SPECK64_Dec_4_Blocks(__m128i &block0, __m128i &block1, __m128i &bloc
|
|||
block3 = _mm_unpackhi_epi32(x2, y2);
|
||||
}
|
||||
|
||||
template <typename F1, typename F4>
|
||||
inline size_t SPECK64_AdvancedProcessBlocks_SSE41(F1 func1, F4 func4,
|
||||
template <typename F2, typename F4>
|
||||
inline size_t SPECK64_AdvancedProcessBlocks_SSE41(F2 func2, F4 func4,
|
||||
const word32 *subKeys, size_t rounds, const byte *inBlocks,
|
||||
const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
|
||||
{
|
||||
|
|
@ -1498,7 +1618,8 @@ inline size_t SPECK64_AdvancedProcessBlocks_SSE41(F1 func1, F4 func4,
|
|||
{
|
||||
// temp[] is an aligned array
|
||||
std::memcpy(temp, inBlocks, 8);
|
||||
__m128i block = _mm_load_si128(CONST_M128_CAST(temp));
|
||||
__m128i block, zero = _mm_setzero_si128();
|
||||
block = _mm_load_si128(CONST_M128_CAST(temp));
|
||||
|
||||
if (flags & BlockTransformation::BT_XorInput)
|
||||
{
|
||||
|
|
@ -1509,7 +1630,7 @@ inline size_t SPECK64_AdvancedProcessBlocks_SSE41(F1 func1, F4 func4,
|
|||
if (flags & BlockTransformation::BT_InBlockIsCounter)
|
||||
const_cast<byte *>(inBlocks)[7]++;
|
||||
|
||||
func1(block, subKeys, static_cast<unsigned int>(rounds));
|
||||
func2(block, zero, subKeys, static_cast<unsigned int>(rounds));
|
||||
|
||||
if (xorBlocks && !(flags & BlockTransformation::BT_XorInput))
|
||||
{
|
||||
|
|
@ -1544,14 +1665,14 @@ NAMESPACE_BEGIN(CryptoPP)
|
|||
size_t SPECK64_Enc_AdvancedProcessBlocks_NEON(const word32* subKeys, size_t rounds,
|
||||
const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
|
||||
{
|
||||
return SPECK64_AdvancedProcessBlocks_NEON(SPECK64_Enc_Block, SPECK64_Enc_4_Blocks,
|
||||
return SPECK64_AdvancedProcessBlocks_NEON(SPECK64_Enc_Block, SPECK64_Enc_6_Blocks,
|
||||
subKeys, rounds, inBlocks, xorBlocks, outBlocks, length, flags);
|
||||
}
|
||||
|
||||
size_t SPECK64_Dec_AdvancedProcessBlocks_NEON(const word32* subKeys, size_t rounds,
|
||||
const byte *inBlocks, const byte *xorBlocks, byte *outBlocks, size_t length, word32 flags)
|
||||
{
|
||||
return SPECK64_AdvancedProcessBlocks_NEON(SPECK64_Dec_Block, SPECK64_Dec_4_Blocks,
|
||||
return SPECK64_AdvancedProcessBlocks_NEON(SPECK64_Dec_Block, SPECK64_Dec_6_Blocks,
|
||||
subKeys, rounds, inBlocks, xorBlocks, outBlocks, length, flags);
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
Loading…
Reference in New Issue