//

// Sha256.c

//

// Copyright (c) Microsoft Corporation. Licensed under the MIT license.

//

//

// This module contains the routines to implement SHA2-256 from FIPS 180-2

//

// This revised implementation is based on the older one in RSA32LIB by Scott Field from 2001

//

#include "precomp.h"

//

// See the symcrypt.h file for documentation on what the various functions do.

//

const SYMCRYPT_HASH SymCryptSha256Algorithm_default = {

    &SymCryptSha256Init,

    &SymCryptSha256Append,

    &SymCryptSha256Result,

    &SymCryptSha256AppendBlocks,

    &SymCryptSha256StateCopy,

    sizeof( SYMCRYPT_SHA256_STATE ),

    SYMCRYPT_SHA256_RESULT_SIZE,

    SYMCRYPT_SHA256_INPUT_BLOCK_SIZE,

    SYMCRYPT_FIELD_OFFSET( SYMCRYPT_SHA256_STATE, chain ),

    SYMCRYPT_FIELD_SIZE( SYMCRYPT_SHA256_STATE, chain ),

};

const PCSYMCRYPT_HASH SymCryptSha256Algorithm = &SymCryptSha256Algorithm_default;

//

// SHA-256 uses 64 magic constants of 32 bits each. These are

// referred to as K^{256}_i for i=0...63 by FIPS 180-2.

// This array is also used by the parallel SHA256 implementation

// For performance we align to 256 bytes, which gives optimal cache alignment.

//

SYMCRYPT_ALIGN_AT( 256 ) const  UINT32 SymCryptSha256K[64] = {

    0x428a2f98UL, 0x71374491UL, 0xb5c0fbcfUL, 0xe9b5dba5UL,

    0x3956c25bUL, 0x59f111f1UL, 0x923f82a4UL, 0xab1c5ed5UL,

    0xd807aa98UL, 0x12835b01UL, 0x243185beUL, 0x550c7dc3UL,

    0x72be5d74UL, 0x80deb1feUL, 0x9bdc06a7UL, 0xc19bf174UL,

    0xe49b69c1UL, 0xefbe4786UL, 0x0fc19dc6UL, 0x240ca1ccUL,

    0x2de92c6fUL, 0x4a7484aaUL, 0x5cb0a9dcUL, 0x76f988daUL,

    0x983e5152UL, 0xa831c66dUL, 0xb00327c8UL, 0xbf597fc7UL,

    0xc6e00bf3UL, 0xd5a79147UL, 0x06ca6351UL, 0x14292967UL,

    0x27b70a85UL, 0x2e1b2138UL, 0x4d2c6dfcUL, 0x53380d13UL,

    0x650a7354UL, 0x766a0abbUL, 0x81c2c92eUL, 0x92722c85UL,

    0xa2bfe8a1UL, 0xa81a664bUL, 0xc24b8b70UL, 0xc76c51a3UL,

    0xd192e819UL, 0xd6990624UL, 0xf40e3585UL, 0x106aa070UL,

    0x19a4c116UL, 0x1e376c08UL, 0x2748774cUL, 0x34b0bcb5UL,

    0x391c0cb3UL, 0x4ed8aa4aUL, 0x5b9cca4fUL, 0x682e6ff3UL,

    0x748f82eeUL, 0x78a5636fUL, 0x84c87814UL, 0x8cc70208UL,

    0x90befffaUL, 0xa4506cebUL, 0xbef9a3f7UL, 0xc67178f2UL

};

//

// Initial state

//

static const UINT32 sha256InitialState[8] = {

    0x6a09e667UL,

    0xbb67ae85UL,

    0x3c6ef372UL,

    0xa54ff53aUL,

    0x510e527fUL,

    0x9b05688cUL,

    0x1f83d9abUL,

    0x5be0cd19UL,

};

//

// SymCryptSha256

//

#define ALG SHA256

#define Alg Sha256

#include "hash_pattern.c"

#undef ALG

#undef Alg

//

// SymCryptSha256Init

//

SYMCRYPT_NOINLINE

VOID

SYMCRYPT_CALL

SymCryptSha256Init( _Out_ PSYMCRYPT_SHA256_STATE pState )

{

    SYMCRYPT_SET_MAGIC( pState );

    pState->dataLengthL = 0;

    //pState->dataLengthH = 0;      // not used

    pState->bytesInBuffer = 0;

    memcpy( &pState->chain.H[0], &sha256InitialState[0], sizeof( sha256InitialState ) );

    //

    // There is no need to initialize the buffer part of the state as that will be

    // filled before it is used.

    //

}

//

// SymCryptSha256Append

//

SYMCRYPT_NOINLINE

VOID

SYMCRYPT_CALL

SymCryptSha256Append(

    _Inout_                 PSYMCRYPT_SHA256_STATE  pState,

    _In_reads_( cbData )    PCBYTE                  pbData,

                            SIZE_T                  cbData )

{

    UINT32  bytesInBuffer;

    UINT32  freeInBuffer;

    SIZE_T  tmp;

    SYMCRYPT_CHECK_MAGIC( pState );

    pState->dataLengthL += cbData;      // dataLengthH is not used...

    bytesInBuffer = pState->bytesInBuffer;

    //

    // If previous data in buffer, buffer new input and transform if possible.

    //

    if( bytesInBuffer > 0 )

    {

        SYMCRYPT_ASSERT( SYMCRYPT_SHA256_INPUT_BLOCK_SIZE > bytesInBuffer );

        freeInBuffer = SYMCRYPT_SHA256_INPUT_BLOCK_SIZE - bytesInBuffer;

        if( cbData < freeInBuffer )

        {

            //

            // All the data will fit in the buffer.

            // We don't do anything here.

            // As cbData < inputBlockSize the bulk data processing is skipped,

            // and the data will be copied to the buffer at the end

            // of this code.

        } else {

            //

            // Enough data to fill the whole buffer & process it

            //

            memcpy(&pState->buffer[bytesInBuffer], pbData, freeInBuffer);

            pbData += freeInBuffer;

            cbData -= freeInBuffer;

            SymCryptSha256AppendBlocks( &pState->chain, &pState->buffer[0], SYMCRYPT_SHA256_INPUT_BLOCK_SIZE, &tmp );

            bytesInBuffer = 0;

        }

    }

    //

    // Internal buffer is empty; process all remaining whole blocks in the input

    //

    if( cbData >= SYMCRYPT_SHA256_INPUT_BLOCK_SIZE )

    {

        SymCryptSha256AppendBlocks( &pState->chain, pbData, cbData, &tmp );

        SYMCRYPT_ASSERT( tmp < SYMCRYPT_SHA256_INPUT_BLOCK_SIZE );

        pbData += cbData - tmp;

        cbData = tmp;

    }

    SYMCRYPT_ASSERT( cbData < SYMCRYPT_SHA256_INPUT_BLOCK_SIZE );

    //

    // buffer remaining input if necessary.

    //

    if( cbData > 0 )

    {

        memcpy( &pState->buffer[bytesInBuffer], pbData, cbData );

        bytesInBuffer += (UINT32) cbData;

    }

    pState->bytesInBuffer = bytesInBuffer;

}

//

// SymCryptSha256Result

//

SYMCRYPT_NOINLINE

VOID

SYMCRYPT_CALL

SymCryptSha256Result(

    _Inout_                                     PSYMCRYPT_SHA256_STATE  pState,

    _Out_writes_( SYMCRYPT_SHA256_RESULT_SIZE ) PBYTE                   pbResult )

{

    //

    // We don't use the common padding code as that is slower, and SHA-256 is very frequently used in

    // performance-sensitive areas.

    //

    UINT32 bytesInBuffer;

    SIZE_T tmp;

    SYMCRYPT_CHECK_MAGIC( pState );

    bytesInBuffer = pState->bytesInBuffer;

    //

    // The buffer is never completely full, so we can always put the first

    // padding byte in.

    //

    pState->buffer[bytesInBuffer++] = 0x80;

    if( bytesInBuffer > 64-8 ) {

        //

        // No room for the rest of the padding. Pad with zeroes & process block

        // bytesInBuffer is at most 64, so we do not have an integer underflow

        //

        SymCryptWipe( &pState->buffer[bytesInBuffer], 64-bytesInBuffer );

        SymCryptSha256AppendBlocks( &pState->chain, pState->buffer, 64, &tmp );

        bytesInBuffer = 0;

    }

    //

    // Set rest of padding

    // At this point bytesInBuffer <= 64-8, so we don't have an underflow

    // We wipe to the end of the buffer as it is 16-aligned,

    // and it is faster to wipe to an aligned point

    //

    SymCryptWipe( &pState->buffer[bytesInBuffer], 64-bytesInBuffer );

    SYMCRYPT_STORE_MSBFIRST64( &pState->buffer[64-8], pState->dataLengthL * 8 );

    //

    // Process the final block

    //

    SymCryptSha256AppendBlocks( &pState->chain, pState->buffer, 64, &tmp );

    //

    // Write the output in the correct byte order

    //

    SymCryptUint32ToMsbFirst( &pState->chain.H[0], pbResult, 8 );

    //

    // Wipe & re-initialize

    // We have to wipe the whole state because the Init call

    // might be optimized away by a smart compiler.

    //

    SymCryptWipeKnownSize( pState, sizeof( *pState ) );

    memcpy( &pState->chain.H[0], &sha256InitialState[0], sizeof( sha256InitialState ) );

    SYMCRYPT_SET_MAGIC( pState );

}

VOID

SYMCRYPT_CALL

SymCryptSha256StateExport(

    _In_                                                    PCSYMCRYPT_SHA256_STATE pState,

    _Out_writes_bytes_( SYMCRYPT_SHA256_STATE_EXPORT_SIZE ) PBYTE                   pbBlob )

{

    SYMCRYPT_ALIGN SYMCRYPT_SHA256_STATE_EXPORT_BLOB blob;           // local copy to have proper alignment.

    C_ASSERT( sizeof( blob ) == SYMCRYPT_SHA256_STATE_EXPORT_SIZE );

    SYMCRYPT_CHECK_MAGIC( pState );

    SymCryptWipeKnownSize( &blob, sizeof( blob ) ); // wipe to avoid any data leakage

    blob.header.magic = SYMCRYPT_BLOB_MAGIC;

    blob.header.size = SYMCRYPT_SHA256_STATE_EXPORT_SIZE;

    blob.header.type = SymCryptBlobTypeSha256State;

    //

    // Copy the relevant data. Buffer will be 0-padded.

    //

    SymCryptUint32ToMsbFirst( &pState->chain.H[0], &blob.chain[0], 8 );

    blob.dataLength = pState->dataLengthL;

    memcpy( &blob.buffer[0], &pState->buffer[0], blob.dataLength & 0x3f );

    SYMCRYPT_ASSERT( (PCBYTE) &blob + sizeof( blob ) - sizeof( SYMCRYPT_BLOB_TRAILER ) == (PCBYTE) &blob.trailer );

    SymCryptMarvin32( SymCryptMarvin32DefaultSeed, (PCBYTE) &blob, sizeof( blob ) - sizeof( SYMCRYPT_BLOB_TRAILER ), &blob.trailer.checksum[0] );

    memcpy( pbBlob, &blob, sizeof( blob ) );

//cleanup:

    SymCryptWipeKnownSize( &blob, sizeof( blob ) );

    return;

}

SYMCRYPT_ERROR

SYMCRYPT_CALL

SymCryptSha256StateImport(

    _Out_                                                   PSYMCRYPT_SHA256_STATE  pState,

    _In_reads_bytes_( SYMCRYPT_SHA256_STATE_EXPORT_SIZE)    PCBYTE                  pbBlob )

{

    SYMCRYPT_ERROR                      scError = SYMCRYPT_NO_ERROR;

    SYMCRYPT_ALIGN SYMCRYPT_SHA256_STATE_EXPORT_BLOB   blob;                       // local copy to have proper alignment.

    BYTE                                checksum[8];

    C_ASSERT( sizeof( blob ) == SYMCRYPT_SHA256_STATE_EXPORT_SIZE );

    memcpy( &blob, pbBlob, sizeof( blob ) );

    if( blob.header.magic != SYMCRYPT_BLOB_MAGIC ||

        blob.header.size != SYMCRYPT_SHA256_STATE_EXPORT_SIZE ||

        blob.header.type != SymCryptBlobTypeSha256State )

    {

        scError = SYMCRYPT_INVALID_BLOB;

        goto cleanup;

    }

    SymCryptMarvin32( SymCryptMarvin32DefaultSeed, (PCBYTE) &blob, sizeof( blob ) - sizeof( SYMCRYPT_BLOB_TRAILER ), checksum );

    if( memcmp( checksum, &blob.trailer.checksum[0], 8 ) != 0 )

    {

        scError = SYMCRYPT_INVALID_BLOB;

        goto cleanup;

    }

    SymCryptMsbFirstToUint32( &blob.chain[0], &pState->chain.H[0], 8 );

    pState->dataLengthL = blob.dataLength;

    pState->bytesInBuffer = blob.dataLength & 0x3f;

    memcpy( &pState->buffer[0], &blob.buffer[0], pState->bytesInBuffer );

    SYMCRYPT_SET_MAGIC( pState );

cleanup:

    SymCryptWipeKnownSize( &blob, sizeof(blob) );

    return scError;

}

//

// Simple test vector for FIPS module testing

//

const BYTE SymCryptSha256KATAnswer[ 32 ] = {

    0xba, 0x78, 0x16, 0xbf, 0x8f, 0x01, 0xcf, 0xea,

    0x41, 0x41, 0x40, 0xde, 0x5d, 0xae, 0x22, 0x23,

    0xb0, 0x03, 0x61, 0xa3, 0x96, 0x17, 0x7a, 0x9c,

    0xb4, 0x10, 0xff, 0x61, 0xf2, 0x00, 0x15, 0xad,

    } ;

VOID

SYMCRYPT_CALL

SymCryptSha256Selftest(void)

{

    BYTE result[SYMCRYPT_SHA256_RESULT_SIZE];

    SymCryptSha256( SymCryptTestMsg3, sizeof( SymCryptTestMsg3 ), result );

    SymCryptInjectError( result, sizeof( result ) );

    if( memcmp( result, SymCryptSha256KATAnswer, sizeof( result ) ) != 0 ) {

        SymCryptFatal( 'SH25' );

    }

}

//

// Below are multiple implementations of the SymCryptSha256AppendBlocks function,

// with a compile-time switch about which one to use.

// We keep the multiple implementations here for future reference;

// as CPU architectures evolve we might want to switch to one of the

// other implementations.

// All implementations here have been tested, but some lack production hardening.

//

//

// Enable frame pointer omission to free up an extra register on X86.

//

#if SYMCRYPT_CPU_X86 && SYMCRYPT_MS_VC

#pragma optimize( "y", on )

#endif

//

// For documentation on these function see FIPS 180-2

//

// MAJ and CH are the functions Maj and Ch from the standard.

// CSIGMA0 and CSIGMA1 are the capital sigma functions.

// LSIGMA0 and LSIGMA1 are the lowercase sigma functions.

//

// The canonical definitions of the MAJ and CH functions are:

//#define MAJ( x, y, z )    (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z)))

//#define CH( x, y, z )  (((x) & (y)) ^ ((~(x)) & (z)))

// We use optimized versions defined below

//

#define MAJ( x, y, z )  ((((z) | (y)) & (x) ) | ((z) & (y)))

#define CH( x, y, z )  ((((z) ^ (y)) & (x)) ^ (z))

//

// The four Sigma functions

//

//

// We have two versions of the rotate-and-xor functions.

// one is just a macro that does the rotations and xors.

// This works well on ARM

// For Intel/AMD we have one where we use the rotated value

// from one intermediate result to derive the next rotated

// value from. This removes one register copy from the

// code stream.

//

// In practice, our compiler doesn't take advantage of the

// reduction in the # operations required, and inserts a

// bunch of extra register copies anyway.

// It actually hurts on AMD64.

//

// This should be re-tuned for every release to get the best overall

// SHA-256 performance.

// At the moment we get an improvement from 19.76 c/B to 19.40 c/B on a Core 2 core.

// We should probably tune this to the Atom CPU.

//

#if SYMCRYPT_CPU_X86

#define USE_CSIGMA0_MULTIROT 1

#define USE_CSIGMA1_MULTIROT 0

#define USE_LSIGMA0_MULTIROT 0

#define USE_LSIGMA1_MULTIROT 0

#else

//

// On ARM we have no reason to believe this helps at all.

// on AMD64 it slows our code down.

//

#define USE_CSIGMA0_MULTIROT 0

#define USE_CSIGMA1_MULTIROT 0

#define USE_LSIGMA0_MULTIROT 0

#define USE_LSIGMA1_MULTIROT 0

#endif

#if USE_CSIGMA0_MULTIROT

FORCEINLINE

UINT32

CSIGMA0( UINT32 x )

{

    UINT32 res;

    x = ROR32( x, 2 );

    res = x;

    x = ROR32( x, 11 );

    res ^= x;

    x = ROR32( x, 9 );

    res ^= x;

    return res;

}

#else

#define CSIGMA0( x )    (ROR32((x),  2) ^ ROR32((x), 13) ^ ROR32((x), 22))

#endif

#if USE_CSIGMA1_MULTIROT

FORCEINLINE

UINT32

CSIGMA1( UINT32 x )

{

    UINT32 res;

    x = ROR32( x, 6 );

    res = x;

    x = ROR32( x, 5 );

    res ^= x;

    x = ROR32( x, 14 );

    res ^= x;

    return res;

}

#else

#define CSIGMA1( x )    (ROR32((x),  6) ^ ROR32((x), 11) ^ ROR32((x), 25))

#endif

#if USE_LSIGMA0_MULTIROT

FORCEINLINE

UINT32

LSIGMA0( UINT32 x )

{

    UINT32 res;

    res = x >> 3;

    x = ROR32( x, 7 );

    res ^= x;

    x = ROR32( x, 11 );

    res ^= x;

    return res;

}

#else

#define LSIGMA0( x )    (ROR32((x),  7) ^ ROR32((x), 18) ^ ((x)>> 3))

#endif

#if USE_LSIGMA1_MULTIROT

FORCEINLINE

UINT32

LSIGMA1( UINT32 x )

{

    UINT32 res;

    res = x >> 10;

    x = ROR32( x, 17 );

    res ^= x;

    x = ROR32( x, 2 );

    res ^= x;

    return res;

}

#else

#define LSIGMA1( x )    (ROR32((x), 17) ^ ROR32((x), 19) ^ ((x)>>10))

#endif

//

// The values a-h are stored in an array called ah.

// We have unrolled the loop 16 times. This makes both the indices into

// the ah array constant, and it makes the message addressing constant.

// This provides a significant speed improvement, at the cost of making

// the main loop about 4 kB in code.

//

// The earlier implementation had the loop unrolled 8 times, and is

// around 10 cycles/byte slower. If loading the code from disk takes

// 100 cycles/byte, then we break even once you have hashed 20 kB.

// This is a worthwhile tradeoff as all code is codesigned with SHA-256.

//

//

// Core round macro

//

// r16 is the round number mod 16, r is the round number.

// r16 is a separate macro argument because it is always a compile-time constant

// which allows much better optimisations of the memory accesses.

//

// ah[ r16   &7] = h

// ah[(r16+1)&7] = g;

// ah[(r16+2)&7] = f;

// ah[(r16+3)&7] = e;

// ah[(r16+4)&7] = d;

// ah[(r16+5)&7] = c;

// ah[(r16+6)&7] = b;

// ah[(r16+7)&7] = a;

//

// After that incrementing the round number will automatically map a->b, b->c, etc.

//

// The core round, after the message word has been computed for this round and put in Wt.

// r16 is the round number modulo 16. (Static after loop unrolling)

// r is the round number (dynamic, which is why we don't use (r&0xf) for r16)

// In more readable form this macro does the following:

//      h += CSIGMA( e ) + CH( e, f, g ) + K[round] + W[round];

//      d += h;

//      h += CSIGMA( a ) + MAJ( a, b, c );

//

#define CROUND( r16, r ) {;\

    ah[ r16   &7] += CSIGMA1(ah[(r16+3)&7]) + CH(ah[(r16+3)&7], ah[(r16+2)&7], ah[(r16+1)&7]) + SymCryptSha256K[r] + Wt;\

    ah[(r16+4)&7] += ah[r16 &7];\

    ah[ r16   &7] += CSIGMA0(ah[(r16+7)&7]) + MAJ(ah[(r16+7)&7], ah[(r16+6)&7], ah[(r16+5)&7]);\

}

//

// Initial round that reads the message.

// r is the round number 0..15

//

#define IROUND( r ) {\

    Wt = SYMCRYPT_LOAD_MSBFIRST32( &pbData[ 4*r ] );\

    W[r] = Wt; \

    CROUND(r,r);\

    }

//

// Subsequent rounds.

// r16 is the round number mod 16. rb is the round number minus r16.

//

#define FROUND(r16, rb) {                                      \

    Wt = LSIGMA1( W[(r16-2) & 15] ) +   W[(r16-7) & 15] +    \

         LSIGMA0( W[(r16-15) & 15]) +   W[r16 & 15];       \

    W[r16] = Wt; \

    CROUND( r16, r16+rb ); \

}

//

// UINT32 implementation 1

//

VOID

SYMCRYPT_CALL

SymCryptSha256AppendBlocks_ul1(

    _Inout_                 SYMCRYPT_SHA256_CHAINING_STATE *    pChain,

    _In_reads_( cbData )    PCBYTE                              pbData,

                            SIZE_T                              cbData,

    _Out_                   SIZE_T                            * pcbRemaining )

{

    SYMCRYPT_ALIGN UINT32 W[16];

    SYMCRYPT_ALIGN UINT32 ah[8];

    int round;

    UINT32 Wt;

    while( cbData >= 64 )

    {

        ah[7] = pChain->H[0];

        ah[6] = pChain->H[1];

        ah[5] = pChain->H[2];

        ah[4] = pChain->H[3];

        ah[3] = pChain->H[4];

        ah[2] = pChain->H[5];

        ah[1] = pChain->H[6];

        ah[0] = pChain->H[7];

        //

        // initial rounds 1 to 16

        //

        IROUND(  0 );

        IROUND(  1 );

        IROUND(  2 );

        IROUND(  3 );

        IROUND(  4 );

        IROUND(  5 );

        IROUND(  6 );

        IROUND(  7 );

        IROUND(  8 );

        IROUND(  9 );

        IROUND( 10 );

        IROUND( 11 );

        IROUND( 12 );

        IROUND( 13 );

        IROUND( 14 );

        IROUND( 15 );

        //

        // rounds 16 to 64.

        //

        for( round=16; round<64; round += 16 )

        {

            FROUND(  0, round );

            FROUND(  1, round );

            FROUND(  2, round );

            FROUND(  3, round );

            FROUND(  4, round );

            FROUND(  5, round );

            FROUND(  6, round );

            FROUND(  7, round );

            FROUND(  8, round );

            FROUND(  9, round );

            FROUND( 10, round );

            FROUND( 11, round );

            FROUND( 12, round );

            FROUND( 13, round );

            FROUND( 14, round );

            FROUND( 15, round );

        }

        pChain->H[0] = ah[7] + pChain->H[0];

        pChain->H[1] = ah[6] + pChain->H[1];

        pChain->H[2] = ah[5] + pChain->H[2];

        pChain->H[3] = ah[4] + pChain->H[3];

        pChain->H[4] = ah[3] + pChain->H[4];

        pChain->H[5] = ah[2] + pChain->H[5];

        pChain->H[6] = ah[1] + pChain->H[6];

        pChain->H[7] = ah[0] + pChain->H[7];

        pbData += 64;

        cbData -= 64;

    }

    *pcbRemaining = cbData;

    //

    // Wipe the variables;

    //

    SymCryptWipeKnownSize( ah, sizeof( ah ) );

    SymCryptWipeKnownSize( W, sizeof( W ) );

    SYMCRYPT_FORCE_WRITE32( &Wt, 0 );

}

VOID

SYMCRYPT_CALL

SymCryptSha256AppendBlocks_ul2(

    _Inout_                 SYMCRYPT_SHA256_CHAINING_STATE *    pChain,

    _In_reads_( cbData )    PCBYTE                              pbData,

                            SIZE_T                              cbData,

    _Out_                   SIZE_T                            * pcbRemaining )

{

    //

    // Different arrangement of the code, currently 25 c/B vs 20 c/b for the version above.

    // On Atom: 50 c/B vs 41 c/B for the one above.

    //

    SYMCRYPT_ALIGN UINT32 buf[4 + 8 + 64];    // chaining state concatenated with the expanded input block

    UINT32 * W = &buf[4 + 8];

    UINT32 * ha = &buf[4]; // initial state words, in order h, g, ..., b, a

    UINT32 A, B, C, D, T;

    int r;

    ha[7] = pChain->H[0]; buf[3] = ha[7];

    ha[6] = pChain->H[1]; buf[2] = ha[6];

    ha[5] = pChain->H[2]; buf[1] = ha[5];

    ha[4] = pChain->H[3]; buf[0] = ha[4];

    ha[3] = pChain->H[4];

    ha[2] = pChain->H[5];

    ha[1] = pChain->H[6];

    ha[0] = pChain->H[7];

    while( cbData >= 64 )

    {

        //

        // Capture the input into W[0..15]

        //

        for( r=0; r<16; r++ )

        {

            W[r] = SYMCRYPT_LOAD_MSBFIRST32( &pbData[ 4*r ] );

        }

        //

        // Expand the message

        //

        A = W[15];

        B = W[14];

        D = W[0];

        for( r=16; r<64; r+= 2 )

        {

            // Loop invariant: A=W[r-1], B = W[r-2], D = W[r-16]

            //

            // Macro for one word of message expansion.

            // Invariant:

            // on entry: a = W[r-1], b = W[r-2], d = W[r-16]

            // on exit:  W[r] computed, a = W[r-1], b = W[r], c = W[r-15]

            //

            #define EXPAND( a, b, c, d, r ) \

                        c = W[r-15]; \

                        b =  d + LSIGMA1( b ) + W[r-7] + LSIGMA0( c ); \

                        W[r] = b; \

            EXPAND( A, B, C, D, r );

            EXPAND( B, A, D, C, (r+1));

            #undef EXPAND

        }

        A = ha[7];

        B = ha[6];

        C = ha[5];

        D = ha[4];

        for( r=0; r<64; r += 4 )

        {

            //

            // Loop invariant:

            // A, B, C, and D are the a,b,c,d values of the current state.

            // W[r] is the next expanded message word to be processed.

            // W[r-8 .. r-5] contain the current state words h, g, f, e.

            //

            //

            // Macro to compute one round

            //

            #define DO_ROUND( a, b, c, d, t, r ) \

                t = W[r] + CSIGMA1( W[r-5] ) + W[r-8] + CH( W[r-5], W[r-6], W[r-7] ) + SymCryptSha256K[r]; \

                W[r-4] = t + d; \

                d = t + CSIGMA0( a ) + MAJ( c, b, a );

            DO_ROUND( A, B, C, D, T, r );

            DO_ROUND( D, A, B, C, T, (r+1) );

            DO_ROUND( C, D, A, B, T, (r+2) );

            DO_ROUND( B, C, D, A, T, (r+3) );

            #undef DO_ROUND

        }

        buf[3] = ha[7] = buf[3] + A;

        buf[2] = ha[6] = buf[2] + B;

        buf[1] = ha[5] = buf[1] + C;

        buf[0] = ha[4] = buf[0] + D;

        ha[3] += W[r-5];

        ha[2] += W[r-6];

        ha[1] += W[r-7];

        ha[0] += W[r-8];

        pbData += 64;

        cbData -= 64;

    }

    pChain->H[0] = ha[7];

    pChain->H[1] = ha[6];

    pChain->H[2] = ha[5];

    pChain->H[3] = ha[4];

    pChain->H[4] = ha[3];

    pChain->H[5] = ha[2];

    pChain->H[6] = ha[1];

    pChain->H[7] = ha[0];

    *pcbRemaining = cbData;

    SymCryptWipeKnownSize( buf, sizeof( buf ) );

    SYMCRYPT_FORCE_WRITE32( &A, 0 );

    SYMCRYPT_FORCE_WRITE32( &B, 0 );

    SYMCRYPT_FORCE_WRITE32( &D, 0 );

    SYMCRYPT_FORCE_WRITE32( &T, 0 );

}

#undef CROUND

#undef IROUND

#undef FROUND

#if SYMCRYPT_CPU_X86 | SYMCRYPT_CPU_AMD64

//

// Don't omit frame pointer for XMM code; it isn't register-starved as much

//

#if SYMCRYPT_CPU_X86 && SYMCRYPT_MS_VC

#pragma optimize( "y", off )

#endif

//

// Code that uses the XMM registers.

// This code is currently unused. It was written in case it would provide better performance, but

// it did not. We are retaining it in case it might be useful in a future CPU generation.

//

#if 0

#define MAJXMM( x, y, z ) _mm_or_si128( _mm_and_si128( _mm_or_si128( z, y ), x ), _mm_and_si128( z, y ))

#define CHXMM( x, y, z )  _mm_xor_si128( _mm_and_si128( _mm_xor_si128( z, y ), x ), z )

#define CSIGMA0XMM( x ) \

    _mm_xor_si128( _mm_xor_si128( _mm_xor_si128( _mm_xor_si128( _mm_xor_si128( \

        _mm_slli_epi32(x,30)  , _mm_srli_epi32(x,  2) ),\

        _mm_slli_epi32(x,19) ), _mm_srli_epi32(x, 13) ),\

        _mm_slli_epi32(x,10) ), _mm_srli_epi32(x, 22) )

#define CSIGMA1XMM( x ) \

    _mm_xor_si128( _mm_xor_si128( _mm_xor_si128( _mm_xor_si128( _mm_xor_si128( \

        _mm_slli_epi32(x,26)  , _mm_srli_epi32(x,  6) ),\

        _mm_slli_epi32(x,21) ), _mm_srli_epi32(x, 11) ),\

        _mm_slli_epi32(x,7) ), _mm_srli_epi32(x, 25) )

#define LSIGMA0XMM( x ) \

    _mm_xor_si128( _mm_xor_si128( _mm_xor_si128( _mm_xor_si128( \

        _mm_slli_epi32(x,25)  , _mm_srli_epi32(x,  7) ),\

        _mm_slli_epi32(x,14) ), _mm_srli_epi32(x, 18) ),\

        _mm_srli_epi32(x, 3) )

#define LSIGMA1XMM( x ) \

    _mm_xor_si128( _mm_xor_si128( _mm_xor_si128( _mm_xor_si128( \

        _mm_slli_epi32(x,15)  , _mm_srli_epi32(x, 17) ),\

        _mm_slli_epi32(x,13) ), _mm_srli_epi32(x, 19) ),\

        _mm_srli_epi32(x,10) )

VOID

SYMCRYPT_CALL

SymCryptSha256AppendBlocks_xmm1(

    _Inout_                 SYMCRYPT_SHA256_CHAINING_STATE *    pChain,

    _In_reads_( cbData )    PCBYTE                              pbData,

                            SIZE_T                              cbData,

    _Out_                   SIZE_T                            * pcbRemaining )

{

    //

    // Implementation that has one value in each XMM register.

    // This is significantly slower than the _ul1 implementation

    // but can be extended to compute 4 hash blocks in parallel.

    //

    SYMCRYPT_ALIGN __m128i buf[4 + 8 + 64];    // chaining state concatenated with the expanded input block

    __m128i * W = &buf[4 + 8];

    __m128i * ha = &buf[4]; // initial state words, in order h, g, ..., b, a

    __m128i A, B, C, D, T;

    int r;

    //

    // For 1-input only; set the input buffer to zero so that we have known values in every byte

    //

    //SymCryptWipeKnownSize( buf, sizeof( buf ) );

    //

    // Copy the chaining state into the start of the buffer, order = h,g,f,e,d,c,b,a

    //

    ha[7] = _mm_insert_epi32(ha[7], pChain->H[0], 0);

    ha[6] = _mm_insert_epi32(ha[6], pChain->H[1], 0);

    ha[5] = _mm_insert_epi32(ha[5], pChain->H[2], 0);

    ha[4] = _mm_insert_epi32(ha[4], pChain->H[3], 0);

    ha[3] = _mm_insert_epi32(ha[3], pChain->H[4], 0);

    ha[2] = _mm_insert_epi32(ha[2], pChain->H[5], 0);

    ha[1] = _mm_insert_epi32(ha[1], pChain->H[6], 0);

    ha[0] = _mm_insert_epi32(ha[0], pChain->H[7], 0);

    buf[0] = ha[4];

    buf[1] = ha[5];

    buf[2] = ha[6];

    buf[3] = ha[7];

    while( cbData >= 64 )

    {

        //

        // Capture the input into W[0..15]

        //

        for( r=0; r<16; r++ )

        {

            W[r] = _mm_insert_epi32(W[r], SYMCRYPT_LOAD_MSBFIRST32( &pbData[ 4*r ] ), 0);

        }

        //

        // Expand the message

        //

        A = W[15];

        B = W[14];

        D = W[0];

        for( r=16; r<64; r+= 2 )

        {

            // Loop invariant: A=W[r-1], B = W[r-2], D = W[r-16]

            //

            // Macro for one word of message expansion.

            // Invariant:

            // on entry: a = W[r-1], b = W[r-2], d = W[r-16]

            // on exit:  W[r] computed, a = W[r-1], b = W[r], c = W[r-15]

            //

            #define EXPAND( a, b, c, d, r ) \

                        c = W[r-15]; \

                        b = _mm_add_epi32( _mm_add_epi32( _mm_add_epi32( d, LSIGMA1XMM( b ) ), W[r-7] ), LSIGMA0XMM( c ) ); \

                        W[r] = b; \

            EXPAND( A, B, C, D, r );

            EXPAND( B, A, D, C, (r+1));

            #undef EXPAND

        }

        A = ha[7];

        B = ha[6];

        C = ha[5];

        D = ha[4];

        for( r=0; r<64; r += 4 )

        {

            //

            // Loop invariant:

            // A, B, C, and D are the a,b,c,d values of the current state.

            // W[r] is the next expanded message word to be processed.

            // W[r-8 .. r-5] contain the current state words h, g, f, e.

            //

            //

            // Macro to compute one round

            //

            #define DO_ROUND( a, b, c, d, t, r ) \

                t = W[r]; \

                t = _mm_add_epi32( t, CSIGMA1XMM( W[r-5] ) ); \

                t = _mm_add_epi32( t, W[r-8] ); \

                t = _mm_add_epi32( t, CHXMM( W[r-5], W[r-6], W[r-7] ) ); \

                t = _mm_add_epi32( t, _mm_cvtsi32_si128( SymCryptSha256K[r] ) ); \

                W[r-4] = _mm_add_epi32( t, d ); \

                d = _mm_add_epi32( t, CSIGMA0XMM( a ) ); \

                d = _mm_add_epi32( d, MAJXMM( c, b, a ) );

            DO_ROUND( A, B, C, D, T, r );

            DO_ROUND( D, A, B, C, T, (r+1) );

            DO_ROUND( C, D, A, B, T, (r+2) );

            DO_ROUND( B, C, D, A, T, (r+3) );

            #undef DO_ROUND

        }

        buf[3] = ha[7] = _mm_add_epi32( buf[3], A );

        buf[2] = ha[6] = _mm_add_epi32( buf[2], B );

        buf[1] = ha[5] = _mm_add_epi32( buf[1], C );

        buf[0] = ha[4] = _mm_add_epi32( buf[0], D );

        ha[3] = _mm_add_epi32( ha[3], W[r-5] );

        ha[2] = _mm_add_epi32( ha[2], W[r-6] );

        ha[1] = _mm_add_epi32( ha[1], W[r-7] );

        ha[0] = _mm_add_epi32( ha[0], W[r-8] );

        pbData += 64;

        cbData -= 64;

    }

    //

    // Copy the chaining state back into the hash structure

    //

    pChain->H[0] = _mm_extract_epi32(ha[7], 0);

    pChain->H[1] = _mm_extract_epi32(ha[6], 0);

    pChain->H[2] = _mm_extract_epi32(ha[5], 0);

    pChain->H[3] = _mm_extract_epi32(ha[4], 0);

    pChain->H[4] = _mm_extract_epi32(ha[3], 0);

    pChain->H[5] = _mm_extract_epi32(ha[2], 0);

    pChain->H[6] = _mm_extract_epi32(ha[1], 0);

    pChain->H[7] = _mm_extract_epi32(ha[0], 0);

    *pcbRemaining = cbData;

    SymCryptWipeKnownSize( buf, sizeof( buf ) );

    SymCryptWipeKnownSize( &A, sizeof( A ) );

    SymCryptWipeKnownSize( &B, sizeof( B ) );

    SymCryptWipeKnownSize( &C, sizeof( C ) );

    SymCryptWipeKnownSize( &D, sizeof( D ) );

    SymCryptWipeKnownSize( &T, sizeof( T ) );

}

//

// XMM implementation 2

// We use the XMM registers to compute part of the message schedule.

// The load, BSWAP, and part of the message schedule recursion are done in XMM registers.

// The rest of the work is done using integers.

//

// Core2: 0.1 c/B slower than the _ul1

// Atom: 1.0 c/B slower than _ul1   (42.34 vs 41.39 c/B)

//

VOID

SYMCRYPT_CALL

SymCryptSha256AppendBlocks_xmm2(

    _Inout_                 SYMCRYPT_SHA256_CHAINING_STATE *    pChain,

    _In_reads_( cbData )    PCBYTE                              pbData,

                            SIZE_T                              cbData,

    _Out_                   SIZE_T                            * pcbRemaining )

{

    SYMCRYPT_ALIGN union { UINT32 ul[16]; __m128i xmm[4]; } W;

    SYMCRYPT_ALIGN UINT32 ah[8];

    int round;

    UINT32 Wt;

    const __m128i BYTE_REVERSE_32 = _mm_set_epi8( 12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3 );

    ah[7] = pChain->H[0];

    ah[6] = pChain->H[1];

    ah[5] = pChain->H[2];

    ah[4] = pChain->H[3];

    ah[3] = pChain->H[4];

    ah[2] = pChain->H[5];

    ah[1] = pChain->H[6];

    ah[0] = pChain->H[7];

#define CROUND( r16, r ) {;\

    ah[ r16   &7] += CSIGMA1(ah[(r16+3)&7]) + CH(ah[(r16+3)&7], ah[(r16+2)&7], ah[(r16+1)&7]) + SymCryptSha256K[r] + Wt;\

    ah[(r16+4)&7] += ah[r16 &7];\

    ah[ r16   &7] += CSIGMA0(ah[(r16+7)&7]) + MAJ(ah[(r16+7)&7], ah[(r16+6)&7], ah[(r16+5)&7]);\

}

//

// Initial round that reads the message.

// r is the round number 0..15

//

//    Wt = LOAD_MSBFIRST32( &pbData[ 4*r ] );\

//    W.ul[r] = Wt; \

#define IROUND( r ) {\

    Wt = W.ul[r];\

    CROUND(r,r);\

    }

//

// Subsequent rounds.

// r16 is the round number mod 16. rb is the round number minus r16.

//

#define FROUND(r16, rb) { \

    Wt = W.ul[r16];\

    CROUND( r16, r16+rb ); \

}

    while( cbData >= 64 )

    {

        //

        // The code is faster if we directly access the W.ul array, rather than the W.xmm alias.

        // I think the compiler gets more confused if you use the W.xmm values.

        // We retain them in the union to ensure alignment

        //

        _mm_store_si128( (__m128i *)&W.ul[ 0], _mm_shuffle_epi8( _mm_loadu_si128( (__m128i *)&pbData[   0 ] ), BYTE_REVERSE_32 ));

        _mm_store_si128( (__m128i *)&W.ul[ 4], _mm_shuffle_epi8( _mm_loadu_si128( (__m128i *)&pbData[  16 ] ), BYTE_REVERSE_32 ));

        _mm_store_si128( (__m128i *)&W.ul[ 8], _mm_shuffle_epi8( _mm_loadu_si128( (__m128i *)&pbData[  32 ] ), BYTE_REVERSE_32 ));

        _mm_store_si128( (__m128i *)&W.ul[12], _mm_shuffle_epi8( _mm_loadu_si128( (__m128i *)&pbData[  48 ] ), BYTE_REVERSE_32 ));

        //

        // initial rounds 1 to 16

        //

        IROUND(  0 );

        IROUND(  1 );

        IROUND(  2 );

        IROUND(  3 );

        IROUND(  4 );

        IROUND(  5 );

        IROUND(  6 );

        IROUND(  7 );

        IROUND(  8 );

        IROUND(  9 );