// xts.cpp - written and placed in the public domain by Jeffrey Walton

// Aarch32, Aarch64, Altivec and X86_64 include SIMD as part of the

// base architecture. We can use the SIMD code below without an

// architecture option. No runtime tests are required. Unfortunately,

// we can't use it on Altivec because an architecture switch is required.

// The updated XorBuffer gains 0.3 to 1.5 cpb on the architectures for

// 16-byte block sizes.

#include "pch.h"

#include "xts.h"

#include "misc.h"

#include "modes.h"

#include "cpu.h"

#if defined(CRYPTOPP_DEBUG)

# include "aes.h"

# include "threefish.h"

#endif

// 0.3 to 0.4 cpb profit

#if defined(__SSE2__) || defined(_M_X64)

# include <emmintrin.h>

#endif

#if defined(__aarch32__) || defined(__aarch64__) || defined(_M_ARM64)

# if (CRYPTOPP_ARM_NEON_HEADER) || (CRYPTOPP_ARM_ASIMD_AVAILABLE)

#  include <arm_neon.h>

# endif

#endif

#if defined(__ALTIVEC__)

# include "ppc_simd.h"

#endif

ANONYMOUS_NAMESPACE_BEGIN

using namespace CryptoPP;

#if defined(CRYPTOPP_DEBUG) && !defined(CRYPTOPP_DOXYGEN_PROCESSING)

using CryptoPP::AES;

using CryptoPP::XTS_Mode;

using CryptoPP::Threefish512;

void Modes_TestInstantiations()

{

    XTS_Mode<AES>::Encryption m0;

    XTS_Mode<AES>::Decryption m1;

    XTS_Mode<AES>::Encryption m2;

    XTS_Mode<AES>::Decryption m3;

#if CRYPTOPP_XTS_WIDE_BLOCK_CIPHERS

    XTS_Mode<Threefish512>::Encryption m4;

    XTS_Mode<Threefish512>::Decryption m5;

#endif

}

#endif  // CRYPTOPP_DEBUG

inline void XorBuffer(byte *output, const byte *input, const byte *mask, size_t count)

{

    CRYPTOPP_ASSERT(count >= 16 && (count % 16 == 0));

#if defined(CRYPTOPP_DISABLE_ASM)

    xorbuf(output, input, mask, count);

#elif defined(__SSE2__) || defined(_M_X64)

    for (size_t i=0; i<count; i+=16)

        _mm_storeu_si128(M128_CAST(output+i),

            _mm_xor_si128(

                _mm_loadu_si128(CONST_M128_CAST(input+i)),

                _mm_loadu_si128(CONST_M128_CAST(mask+i))));

#elif defined(__aarch32__) || defined(__aarch64__) || defined(_M_ARM64)

    for (size_t i=0; i<count; i+=16)

        vst1q_u8(output+i, veorq_u8(vld1q_u8(input+i), vld1q_u8(mask+i)));

#elif defined(__ALTIVEC__)

    for (size_t i=0; i<count; i+=16)

        VecStore(VecXor(VecLoad(input+i), VecLoad(mask+i)), output+i);

#else

    xorbuf(output, input, mask, count);

#endif

}

inline void XorBuffer(byte *buf, const byte *mask, size_t count)

{

    XorBuffer(buf, buf, mask, count);

}

// Borrowed from CMAC, but little-endian representation

inline void GF_Double(byte *out, const byte* in, unsigned int len)

{

#if defined(CRYPTOPP_WORD128_AVAILABLE)

    word128 carry = 0, x;

    for (size_t i=0, idx=0; i<len/16; ++i, idx+=16)

    {

        x = GetWord<word128>(false, LITTLE_ENDIAN_ORDER, in+idx);

        word128 y = (x >> 127); x = (x << 1) + carry;

        PutWord<word128>(false, LITTLE_ENDIAN_ORDER, out+idx, x);

        carry = y;

    }

#elif defined(_M_X64) || defined(_M_ARM64) || defined(_LP64) || defined(__LP64__)

    word64 carry = 0, x;

    for (size_t i=0, idx=0; i<len/8; ++i, idx+=8)

    {

        x = GetWord<word64>(false, LITTLE_ENDIAN_ORDER, in+idx);

        word64 y = (x >> 63); x = (x << 1) + carry;

        PutWord<word64>(false, LITTLE_ENDIAN_ORDER, out+idx, x);

        carry = y;

    }

#else

    word32 carry = 0, x;

    for (size_t i=0, idx=0; i<len/4; ++i, idx+=4)

    {

        x = GetWord<word32>(false, LITTLE_ENDIAN_ORDER, in+idx);

        word32 y = (x >> 31); x = (x << 1) + carry;

        PutWord<word32>(false, LITTLE_ENDIAN_ORDER, out+idx, x);

        carry = y;

    }

#endif

#if CRYPTOPP_XTS_WIDE_BLOCK_CIPHERS

    CRYPTOPP_ASSERT(IsPowerOf2(len));

    CRYPTOPP_ASSERT(len >= 16);

    CRYPTOPP_ASSERT(len <= 128);

    byte* k = out;

    if (carry)

    {

        switch (len)

        {

        case 16:

        {

            const size_t LEIDX = 16-1;

            k[LEIDX-15] ^= 0x87;

            break;

        }

        case 32:

        {

            // https://crypto.stackexchange.com/q/9815/10496

            // Polynomial x^256 + x^10 + x^5 + x^2 + 1

            const size_t LEIDX = 32-1;

            k[LEIDX-30] ^= 4;

            k[LEIDX-31] ^= 0x25;

            break;

        }

        case 64:

        {

            // https://crypto.stackexchange.com/q/9815/10496

            // Polynomial x^512 + x^8 + x^5 + x^2 + 1

            const size_t LEIDX = 64-1;

            k[LEIDX-62] ^= 1;

            k[LEIDX-63] ^= 0x25;

            break;

        }

        case 128:

        {

            // https://crypto.stackexchange.com/q/9815/10496

            // Polynomial x^1024 + x^19 + x^6 + x + 1

            const size_t LEIDX = 128-1;

            k[LEIDX-125] ^= 8;

            k[LEIDX-126] ^= 0x00;

            k[LEIDX-127] ^= 0x43;

            break;

        }

        default:

            CRYPTOPP_ASSERT(0);

        }

    }

#else

    CRYPTOPP_ASSERT(len == 16);

    byte* k = out;

    if (carry)

    {

        k[0] ^= 0x87;

        return;

    }

#endif  // CRYPTOPP_XTS_WIDE_BLOCK_CIPHERS

}

inline void GF_Double(byte *inout, unsigned int len)

{

    GF_Double(inout, inout, len);

}

ANONYMOUS_NAMESPACE_END

NAMESPACE_BEGIN(CryptoPP)

void XTS_ModeBase::ThrowIfInvalidBlockSize(size_t length)

{

#if CRYPTOPP_XTS_WIDE_BLOCK_CIPHERS

    CRYPTOPP_ASSERT(length >= 16 && length <= 128 && IsPowerOf2(length));

    if (length < 16 || length > 128 || !IsPowerOf2(length))

        throw InvalidArgument(AlgorithmName() + ": block size of underlying block cipher is not valid");

#else

    CRYPTOPP_ASSERT(length == 16);

    if (length != 16)

        throw InvalidArgument(AlgorithmName() + ": block size of underlying block cipher is not 16");

#endif

}

void XTS_ModeBase::ThrowIfInvalidKeyLength(size_t length)

{

    CRYPTOPP_ASSERT(length % 2 == 0);

    if (!GetBlockCipher().IsValidKeyLength((length+1)/2))

        throw InvalidKeyLength(AlgorithmName(), length);

}

void XTS_ModeBase::SetKey(const byte *key, size_t length, const NameValuePairs &params)

{

    ThrowIfInvalidKeyLength(length);

    ThrowIfInvalidBlockSize(BlockSize());

    const size_t klen = length/2;

    AccessBlockCipher().SetKey(key+0, klen, params);

    AccessTweakCipher().SetKey(key+klen, klen, params);

    ResizeBuffers();

    size_t ivLength;

    const byte *iv = GetIVAndThrowIfInvalid(params, ivLength);

    Resynchronize(iv, (int)ivLength);

}

void XTS_ModeBase::Resynchronize(const byte *iv, int ivLength)

{

    BlockOrientedCipherModeBase::Resynchronize(iv, ivLength);

    std::memcpy(m_xregister, m_register, ivLength);

    GetTweakCipher().ProcessBlock(m_xregister);

}

void XTS_ModeBase::Resynchronize(word64 sector, ByteOrder order)

{

    SecByteBlock iv(GetTweakCipher().BlockSize());

    PutWord<word64>(false, order, iv, sector);

    std::memset(iv+8, 0x00, iv.size()-8);

    BlockOrientedCipherModeBase::Resynchronize(iv, (int)iv.size());

    std::memcpy(m_xregister, iv, iv.size());

    GetTweakCipher().ProcessBlock(m_xregister);

}

void XTS_ModeBase::ResizeBuffers()

{

    BlockOrientedCipherModeBase::ResizeBuffers();

    m_xworkspace.New(GetBlockCipher().BlockSize()*ParallelBlocks);

    m_xregister.New(GetBlockCipher().BlockSize()*ParallelBlocks);

}

// ProcessData runs either 12-4-1 blocks, 8-2-1 or 4-1 blocks. Which is

// selected depends on ParallelBlocks in the header file. 12-4-1 or 8-2-1

// can be used on Aarch64 and PowerPC. Intel should use 4-1 due to lack

// of registers. The unneeded code paths should be removed by optimizer.

// The extra gyrations save us 1.8 cpb on Aarch64 and 2.1 cpb on PowerPC.

void XTS_ModeBase::ProcessData(byte *outString, const byte *inString, size_t length)

{

    // data unit is multiple of 16 bytes

    CRYPTOPP_ASSERT(length % BlockSize() == 0);

    enum { lastParallelBlock = ParallelBlocks-1 };

    const unsigned int blockSize = GetBlockCipher().BlockSize();

    const size_t parallelSize = blockSize*ParallelBlocks;

    // encrypt the data unit, optimal size at a time

    while (length >= parallelSize)

    {

        // m_xregister[0] always points to the next tweak.

        GF_Double(m_xregister+1*blockSize, m_xregister+0*blockSize, blockSize);

        GF_Double(m_xregister+2*blockSize, m_xregister+1*blockSize, blockSize);

        GF_Double(m_xregister+3*blockSize, m_xregister+2*blockSize, blockSize);

        if (ParallelBlocks > 4)

        {

            GF_Double(m_xregister+4*blockSize, m_xregister+3*blockSize, blockSize);

            GF_Double(m_xregister+5*blockSize, m_xregister+4*blockSize, blockSize);

            GF_Double(m_xregister+6*blockSize, m_xregister+5*blockSize, blockSize);

            GF_Double(m_xregister+7*blockSize, m_xregister+6*blockSize, blockSize);

        }

        if (ParallelBlocks > 8)

        {

            GF_Double(m_xregister+8*blockSize, m_xregister+7*blockSize, blockSize);

            GF_Double(m_xregister+9*blockSize, m_xregister+8*blockSize, blockSize);

            GF_Double(m_xregister+10*blockSize, m_xregister+9*blockSize, blockSize);

            GF_Double(m_xregister+11*blockSize, m_xregister+10*blockSize, blockSize);

        }

        // merge the tweak into the input block

        XorBuffer(m_xworkspace, inString, m_xregister, parallelSize);

        // encrypt one block, merge the tweak into the output block

        GetBlockCipher().AdvancedProcessBlocks(m_xworkspace, m_xregister,

            outString, parallelSize, BlockTransformation::BT_AllowParallel);

        // m_xregister[0] always points to the next tweak.

        GF_Double(m_xregister+0, m_xregister+lastParallelBlock*blockSize, blockSize);

        inString += parallelSize;

        outString += parallelSize;

        length -= parallelSize;

    }

    // encrypt the data unit, 4 blocks at a time

    while (ParallelBlocks == 12 && length >= blockSize*4)

    {

        // m_xregister[0] always points to the next tweak.

        GF_Double(m_xregister+1*blockSize, m_xregister+0*blockSize, blockSize);

        GF_Double(m_xregister+2*blockSize, m_xregister+1*blockSize, blockSize);

        GF_Double(m_xregister+3*blockSize, m_xregister+2*blockSize, blockSize);

        // merge the tweak into the input block

        XorBuffer(m_xworkspace, inString, m_xregister, blockSize*4);

        // encrypt one block, merge the tweak into the output block

        GetBlockCipher().AdvancedProcessBlocks(m_xworkspace, m_xregister,

            outString, blockSize*4, BlockTransformation::BT_AllowParallel);

        // m_xregister[0] always points to the next tweak.

        GF_Double(m_xregister+0, m_xregister+3*blockSize, blockSize);

        inString += blockSize*4;

        outString += blockSize*4;

        length -= blockSize*4;

    }

    // encrypt the data unit, 2 blocks at a time

    while (ParallelBlocks == 8 && length >= blockSize*2)

    {

        // m_xregister[0] always points to the next tweak.

        GF_Double(m_xregister+1*blockSize, m_xregister+0*blockSize, blockSize);

        // merge the tweak into the input block

        XorBuffer(m_xworkspace, inString, m_xregister, blockSize*2);

        // encrypt one block, merge the tweak into the output block

        GetBlockCipher().AdvancedProcessBlocks(m_xworkspace, m_xregister,

            outString, blockSize*2, BlockTransformation::BT_AllowParallel);

        // m_xregister[0] always points to the next tweak.

        GF_Double(m_xregister+0, m_xregister+1*blockSize, blockSize);

        inString += blockSize*2;

        outString += blockSize*2;

        length -= blockSize*2;

    }

    // encrypt the data unit, blocksize at a time

    while (length)

    {

        // merge the tweak into the input block

        XorBuffer(m_xworkspace, inString, m_xregister, blockSize);

        // encrypt one block

        GetBlockCipher().ProcessBlock(m_xworkspace);

        // merge the tweak into the output block

        XorBuffer(outString, m_xworkspace, m_xregister, blockSize);

        // Multiply T by alpha

        GF_Double(m_xregister, blockSize);

        inString += blockSize;

        outString += blockSize;

        length -= blockSize;

    }

}

size_t XTS_ModeBase::ProcessLastBlock(byte *outString, size_t outLength, const byte *inString, size_t inLength)

{

    // need at least a full AES block

    CRYPTOPP_ASSERT(inLength >= BlockSize());

    if (inLength < BlockSize())

        throw InvalidArgument("XTS: message is too short for ciphertext stealing");

    if (IsForwardTransformation())

        return ProcessLastPlainBlock(outString, outLength, inString, inLength);

    else

        return ProcessLastCipherBlock(outString, outLength, inString, inLength);

}

size_t XTS_ModeBase::ProcessLastPlainBlock(byte *outString, size_t outLength, const byte *inString, size_t inLength)

{

    // ensure output buffer is large enough

    CRYPTOPP_ASSERT(outLength >= inLength);

    const unsigned int blockSize = GetBlockCipher().BlockSize();

    const size_t blocks = inLength / blockSize;

    const size_t tail = inLength % blockSize;

    outLength = inLength;

    if (tail == 0)

    {

        // Allow ProcessData to handle all the full blocks

        ProcessData(outString, inString, inLength);

        return inLength;

    }

    else if (blocks > 1)

    {

        // Allow ProcessData to handle full blocks except one

        const size_t head = (blocks-1)*blockSize;

        ProcessData(outString, inString, inLength-head);

        outString += head;

        inString  += head; inLength -= head;

    }

    ///// handle the full block /////

    // merge the tweak into the input block

    XorBuffer(m_xworkspace, inString, m_xregister, blockSize);

    // encrypt one block

    GetBlockCipher().ProcessBlock(m_xworkspace);

    // merge the tweak into the output block

    XorBuffer(outString, m_xworkspace, m_xregister, blockSize);

    // Multiply T by alpha

    GF_Double(m_xregister, blockSize);

    ///// handle final partial block /////

    inString += blockSize;

    outString += blockSize;

    const size_t len = inLength-blockSize;

    // copy in the final plaintext bytes

    std::memcpy(m_xworkspace, inString, len);

    // and copy out the final ciphertext bytes

    std::memcpy(outString, outString-blockSize, len);

    // "steal" ciphertext to complete the block

    std::memcpy(m_xworkspace+len, outString-blockSize+len, blockSize-len);

    // merge the tweak into the input block

    XorBuffer(m_xworkspace, m_xregister, blockSize);

    // encrypt one block

    GetBlockCipher().ProcessBlock(m_xworkspace);

    // merge the tweak into the previous output block

    XorBuffer(outString-blockSize, m_xworkspace, m_xregister, blockSize);

    return outLength;

}

size_t XTS_ModeBase::ProcessLastCipherBlock(byte *outString, size_t outLength, const byte *inString, size_t inLength)

{

    // ensure output buffer is large enough

    CRYPTOPP_ASSERT(outLength >= inLength);

    const unsigned int blockSize = GetBlockCipher().BlockSize();

    const size_t blocks = inLength / blockSize;

    const size_t tail = inLength % blockSize;

    outLength = inLength;

    if (tail == 0)

    {

        // Allow ProcessData to handle all the full blocks

        ProcessData(outString, inString, inLength);

        return inLength;

    }

    else if (blocks > 1)

    {

        // Allow ProcessData to handle full blocks except one

        const size_t head = (blocks-1)*blockSize;

        ProcessData(outString, inString, inLength-head);

        outString += head;

        inString  += head; inLength -= head;

    }

    #define poly1 (m_xregister+0*blockSize)

    #define poly2 (m_xregister+1*blockSize)

    GF_Double(poly2, poly1, blockSize);

    ///// handle final partial block /////

    inString += blockSize;

    outString += blockSize;

    const size_t len = inLength-blockSize;

    // merge the tweak into the input block

    XorBuffer(m_xworkspace, inString-blockSize, poly2, blockSize);

    // encrypt one block

    GetBlockCipher().ProcessBlock(m_xworkspace);

    // merge the tweak into the output block

    XorBuffer(m_xworkspace, poly2, blockSize);

    // copy in the final plaintext bytes

    std::memcpy(outString-blockSize, inString, len);

    // and copy out the final ciphertext bytes

    std::memcpy(outString, m_xworkspace, len);

    // "steal" ciphertext to complete the block

    std::memcpy(outString-blockSize+len, m_xworkspace+len, blockSize-len);

    ///// handle the full previous block /////

    inString -= blockSize;

    outString -= blockSize;

    // merge the tweak into the input block

    XorBuffer(m_xworkspace, outString, poly1, blockSize);

    // encrypt one block

    GetBlockCipher().ProcessBlock(m_xworkspace);

    // merge the tweak into the output block

    XorBuffer(outString, m_xworkspace, poly1, blockSize);

    return outLength;

}

NAMESPACE_END