_Z17setup_fake_randomPKhm: 53| 319|setup_fake_random(const unsigned char * seed, const size_t seed_size) { 54| 319| SEED_DATA = seed; 55| 319| SEED_SIZE = seed_size; 56| | 57| 319| int fake_random_set = randombytes_set_implementation(&fake_random); 58| 319| assert(fake_random_set == 0); 59| | 60| 0| assert(strcmp(randombytes_implementation_name(), "fake_random") == 0); 61| 0| int initialized = sodium_init(); 62| 319| assert(initialized >= 0); 63| 319|} secretbox_easy_fuzzer.cc:_ZL24fake_implementation_namev: 28| 319|fake_implementation_name(void) { 29| 319| return "fake_random"; 30| 319|} secretbox_easy_fuzzer.cc:_ZL18fake_random_bufferPvm: 33| 638|fake_random_buffer(void * const buf, const size_t size) { 34| 638| static unsigned char seed[randombytes_SEEDBYTES]; 35| 638| memset(seed, '0', randombytes_SEEDBYTES); ------------------ | | 30| 638|#define randombytes_SEEDBYTES 32U ------------------ 36| | 37| 638| size_t boundary = std::min((size_t) randombytes_SEEDBYTES, SEED_SIZE); ------------------ | | 30| 638|#define randombytes_SEEDBYTES 32U ------------------ 38| 638| memcpy(&seed, SEED_DATA, boundary); 39| | 40| 638| randombytes_buf_deterministic(buf, size, seed); 41| 638|} crypto_core_hsalsa20: 21| 638|{ 22| 638| uint32_t x0, x1, x2, x3, x4, x5, x6, x7, x8, 23| 638| x9, x10, x11, x12, x13, x14, x15; 24| 638| int i; 25| | 26| 638| if (c == NULL) { ------------------ | Branch (26:9): [True: 638, False: 0] ------------------ 27| 638| x0 = U32C(0x61707865); ------------------ | | 14| 638|#define U32C(v) (v##U) ------------------ 28| 638| x5 = U32C(0x3320646e); ------------------ | | 14| 638|#define U32C(v) (v##U) ------------------ 29| 638| x10 = U32C(0x79622d32); ------------------ | | 14| 638|#define U32C(v) (v##U) ------------------ 30| 638| x15 = U32C(0x6b206574); ------------------ | | 14| 638|#define U32C(v) (v##U) ------------------ 31| 638| } else { 32| 0| x0 = LOAD32_LE(c + 0); ------------------ | | 100| 0|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 33| 0| x5 = LOAD32_LE(c + 4); ------------------ | | 100| 0|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 34| 0| x10 = LOAD32_LE(c + 8); ------------------ | | 100| 0|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 35| 0| x15 = LOAD32_LE(c + 12); ------------------ | | 100| 0|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 36| 0| } 37| 638| x1 = LOAD32_LE(k + 0); ------------------ | | 100| 638|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 38| 638| x2 = LOAD32_LE(k + 4); ------------------ | | 100| 638|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 39| 638| x3 = LOAD32_LE(k + 8); ------------------ | | 100| 638|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 40| 638| x4 = LOAD32_LE(k + 12); ------------------ | | 100| 638|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 41| 638| x11 = LOAD32_LE(k + 16); ------------------ | | 100| 638|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 42| 638| x12 = LOAD32_LE(k + 20); ------------------ | | 100| 638|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 43| 638| x13 = LOAD32_LE(k + 24); ------------------ | | 100| 638|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 44| 638| x14 = LOAD32_LE(k + 28); ------------------ | | 100| 638|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 45| 638| x6 = LOAD32_LE(in + 0); ------------------ | | 100| 638|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 46| 638| x7 = LOAD32_LE(in + 4); ------------------ | | 100| 638|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 47| 638| x8 = LOAD32_LE(in + 8); ------------------ | | 100| 638|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 48| 638| x9 = LOAD32_LE(in + 12); ------------------ | | 100| 638|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 49| | 50| 7.01k| for (i = ROUNDS; i > 0; i -= 2) { ------------------ | | 13| 638|#define ROUNDS 20 ------------------ | Branch (50:22): [True: 6.38k, False: 638] ------------------ 51| 6.38k| x4 ^= ROTL32(x0 + x12, 7); ------------------ | | 33| 6.38k|#define ROTL32(X, B) rotl32((X), (B)) ------------------ 52| 6.38k| x8 ^= ROTL32(x4 + x0, 9); ------------------ | | 33| 6.38k|#define ROTL32(X, B) rotl32((X), (B)) ------------------ 53| 6.38k| x12 ^= ROTL32(x8 + x4, 13); ------------------ | | 33| 6.38k|#define ROTL32(X, B) rotl32((X), (B)) ------------------ 54| 6.38k| x0 ^= ROTL32(x12 + x8, 18); ------------------ | | 33| 6.38k|#define ROTL32(X, B) rotl32((X), (B)) ------------------ 55| 6.38k| x9 ^= ROTL32(x5 + x1, 7); ------------------ | | 33| 6.38k|#define ROTL32(X, B) rotl32((X), (B)) ------------------ 56| 6.38k| x13 ^= ROTL32(x9 + x5, 9); ------------------ | | 33| 6.38k|#define ROTL32(X, B) rotl32((X), (B)) ------------------ 57| 6.38k| x1 ^= ROTL32(x13 + x9, 13); ------------------ | | 33| 6.38k|#define ROTL32(X, B) rotl32((X), (B)) ------------------ 58| 6.38k| x5 ^= ROTL32(x1 + x13, 18); ------------------ | | 33| 6.38k|#define ROTL32(X, B) rotl32((X), (B)) ------------------ 59| 6.38k| x14 ^= ROTL32(x10 + x6, 7); ------------------ | | 33| 6.38k|#define ROTL32(X, B) rotl32((X), (B)) ------------------ 60| 6.38k| x2 ^= ROTL32(x14 + x10, 9); ------------------ | | 33| 6.38k|#define ROTL32(X, B) rotl32((X), (B)) ------------------ 61| 6.38k| x6 ^= ROTL32(x2 + x14, 13); ------------------ | | 33| 6.38k|#define ROTL32(X, B) rotl32((X), (B)) ------------------ 62| 6.38k| x10 ^= ROTL32(x6 + x2, 18); ------------------ | | 33| 6.38k|#define ROTL32(X, B) rotl32((X), (B)) ------------------ 63| 6.38k| x3 ^= ROTL32(x15 + x11, 7); ------------------ | | 33| 6.38k|#define ROTL32(X, B) rotl32((X), (B)) ------------------ 64| 6.38k| x7 ^= ROTL32(x3 + x15, 9); ------------------ | | 33| 6.38k|#define ROTL32(X, B) rotl32((X), (B)) ------------------ 65| 6.38k| x11 ^= ROTL32(x7 + x3, 13); ------------------ | | 33| 6.38k|#define ROTL32(X, B) rotl32((X), (B)) ------------------ 66| 6.38k| x15 ^= ROTL32(x11 + x7, 18); ------------------ | | 33| 6.38k|#define ROTL32(X, B) rotl32((X), (B)) ------------------ 67| 6.38k| x1 ^= ROTL32(x0 + x3, 7); ------------------ | | 33| 6.38k|#define ROTL32(X, B) rotl32((X), (B)) ------------------ 68| 6.38k| x2 ^= ROTL32(x1 + x0, 9); ------------------ | | 33| 6.38k|#define ROTL32(X, B) rotl32((X), (B)) ------------------ 69| 6.38k| x3 ^= ROTL32(x2 + x1, 13); ------------------ | | 33| 6.38k|#define ROTL32(X, B) rotl32((X), (B)) ------------------ 70| 6.38k| x0 ^= ROTL32(x3 + x2, 18); ------------------ | | 33| 6.38k|#define ROTL32(X, B) rotl32((X), (B)) ------------------ 71| 6.38k| x6 ^= ROTL32(x5 + x4, 7); ------------------ | | 33| 6.38k|#define ROTL32(X, B) rotl32((X), (B)) ------------------ 72| 6.38k| x7 ^= ROTL32(x6 + x5, 9); ------------------ | | 33| 6.38k|#define ROTL32(X, B) rotl32((X), (B)) ------------------ 73| 6.38k| x4 ^= ROTL32(x7 + x6, 13); ------------------ | | 33| 6.38k|#define ROTL32(X, B) rotl32((X), (B)) ------------------ 74| 6.38k| x5 ^= ROTL32(x4 + x7, 18); ------------------ | | 33| 6.38k|#define ROTL32(X, B) rotl32((X), (B)) ------------------ 75| 6.38k| x11 ^= ROTL32(x10 + x9, 7); ------------------ | | 33| 6.38k|#define ROTL32(X, B) rotl32((X), (B)) ------------------ 76| 6.38k| x8 ^= ROTL32(x11 + x10, 9); ------------------ | | 33| 6.38k|#define ROTL32(X, B) rotl32((X), (B)) ------------------ 77| 6.38k| x9 ^= ROTL32(x8 + x11, 13); ------------------ | | 33| 6.38k|#define ROTL32(X, B) rotl32((X), (B)) ------------------ 78| 6.38k| x10 ^= ROTL32(x9 + x8, 18); ------------------ | | 33| 6.38k|#define ROTL32(X, B) rotl32((X), (B)) ------------------ 79| 6.38k| x12 ^= ROTL32(x15 + x14, 7); ------------------ | | 33| 6.38k|#define ROTL32(X, B) rotl32((X), (B)) ------------------ 80| 6.38k| x13 ^= ROTL32(x12 + x15, 9); ------------------ | | 33| 6.38k|#define ROTL32(X, B) rotl32((X), (B)) ------------------ 81| 6.38k| x14 ^= ROTL32(x13 + x12, 13); ------------------ | | 33| 6.38k|#define ROTL32(X, B) rotl32((X), (B)) ------------------ 82| 6.38k| x15 ^= ROTL32(x14 + x13, 18); ------------------ | | 33| 6.38k|#define ROTL32(X, B) rotl32((X), (B)) ------------------ 83| 6.38k| } 84| | 85| 638| STORE32_LE(out + 0, x0); ------------------ | | 117| 638|#define STORE32_LE(DST, W) store32_le((DST), (W)) ------------------ 86| 638| STORE32_LE(out + 4, x5); ------------------ | | 117| 638|#define STORE32_LE(DST, W) store32_le((DST), (W)) ------------------ 87| 638| STORE32_LE(out + 8, x10); ------------------ | | 117| 638|#define STORE32_LE(DST, W) store32_le((DST), (W)) ------------------ 88| 638| STORE32_LE(out + 12, x15); ------------------ | | 117| 638|#define STORE32_LE(DST, W) store32_le((DST), (W)) ------------------ 89| 638| STORE32_LE(out + 16, x6); ------------------ | | 117| 638|#define STORE32_LE(DST, W) store32_le((DST), (W)) ------------------ 90| 638| STORE32_LE(out + 20, x7); ------------------ | | 117| 638|#define STORE32_LE(DST, W) store32_le((DST), (W)) ------------------ 91| 638| STORE32_LE(out + 24, x8); ------------------ | | 117| 638|#define STORE32_LE(DST, W) store32_le((DST), (W)) ------------------ 92| 638| STORE32_LE(out + 28, x9); ------------------ | | 117| 638|#define STORE32_LE(DST, W) store32_le((DST), (W)) ------------------ 93| | 94| 638| return 0; 95| 638|} blake2b_pick_best_implementation: 412| 1|{ 413| |/* LCOV_EXCL_START */ 414| 1|#if defined(HAVE_AVX2INTRIN_H) && defined(HAVE_TMMINTRIN_H) && \ 415| 1| defined(HAVE_SMMINTRIN_H) 416| 1| if (sodium_runtime_has_avx2()) { ------------------ | Branch (416:9): [True: 1, False: 0] ------------------ 417| 1| blake2b_compress = blake2b_compress_avx2; 418| 1| return 0; 419| 1| } 420| 0|#endif 421| 0|#if defined(HAVE_EMMINTRIN_H) && defined(HAVE_TMMINTRIN_H) && \ 422| 0| defined(HAVE_SMMINTRIN_H) 423| 0| if (sodium_runtime_has_sse41()) { ------------------ | Branch (423:9): [True: 0, False: 0] ------------------ 424| 0| blake2b_compress = blake2b_compress_sse41; 425| 0| return 0; 426| 0| } 427| 0|#endif 428| 0|#if defined(HAVE_EMMINTRIN_H) && defined(HAVE_TMMINTRIN_H) 429| 0| if (sodium_runtime_has_ssse3()) { ------------------ | Branch (429:9): [True: 0, False: 0] ------------------ 430| 0| blake2b_compress = blake2b_compress_ssse3; 431| 0| return 0; 432| 0| } 433| 0|#endif 434| 0| blake2b_compress = blake2b_compress_ref; 435| | 436| 0| return 0; 437| | /* LCOV_EXCL_STOP */ 438| 0|} _crypto_generichash_blake2b_pick_best_implementation: 114| 1|{ 115| 1| return blake2b_pick_best_implementation(); 116| 1|} crypto_onetimeauth_poly1305_verify: 29| 319|{ 30| 319| return implementation->onetimeauth_verify(h, in, inlen, k); 31| 319|} crypto_onetimeauth_poly1305_init: 36| 319|{ 37| 319| return implementation->onetimeauth_init(state, key); 38| 319|} crypto_onetimeauth_poly1305_update: 44| 319|{ 45| 319| return implementation->onetimeauth_update(state, in, inlen); 46| 319|} crypto_onetimeauth_poly1305_final: 51| 319|{ 52| 319| return implementation->onetimeauth_final(state, out); 53| 319|} _crypto_onetimeauth_poly1305_pick_best_implementation: 82| 1|{ 83| 1| implementation = &crypto_onetimeauth_poly1305_donna_implementation; 84| 1|#if defined(HAVE_TI_MODE) && defined(HAVE_EMMINTRIN_H) 85| 1| if (sodium_runtime_has_sse2()) { ------------------ | Branch (85:9): [True: 1, False: 0] ------------------ 86| 1| implementation = &crypto_onetimeauth_poly1305_sse2_implementation; 87| 1| } 88| 1|#endif 89| 1| return 0; 90| 1|} poly1305_sse2.c:crypto_onetimeauth_poly1305_sse2: 909| 319|{ 910| 319| CRYPTO_ALIGN(64) poly1305_state_internal_t st; ------------------ | | 50| 319|# define CRYPTO_ALIGN(x) __attribute__ ((aligned(x))) ------------------ 911| 319| unsigned long long blocks; 912| | 913| 319| poly1305_init_ext(&st, key, inlen); 914| 319| blocks = inlen & ~31; 915| 319| if (blocks > 0) { ------------------ | Branch (915:9): [True: 253, False: 66] ------------------ 916| 253| poly1305_blocks(&st, m, blocks); 917| 253| m += blocks; 918| 253| inlen -= blocks; 919| 253| } 920| 319| poly1305_finish_ext(&st, m, inlen, out); 921| | 922| 319| return 0; 923| 319|} poly1305_sse2.c:poly1305_init_ext: 105| 638|{ 106| 638| uint32_t *R; 107| 638| uint128_t d[3]; 108| 638| uint64_t r0, r1, r2; 109| 638| uint64_t rt0, rt1, rt2, st2, c; 110| 638| uint64_t t0, t1; 111| 638| unsigned long long i; 112| | 113| 638| if (!bytes) { ------------------ | Branch (113:9): [True: 319, False: 319] ------------------ 114| 319| bytes = ~(unsigned long long) 0; 115| 319| } 116| | /* H = 0 */ 117| 638| _mm_storeu_si128((xmmi *) (void *) &st->H.hh[0], _mm_setzero_si128()); 118| 638| _mm_storeu_si128((xmmi *) (void *) &st->H.hh[4], _mm_setzero_si128()); 119| 638| _mm_storeu_si128((xmmi *) (void *) &st->H.hh[8], _mm_setzero_si128()); 120| | 121| | /* clamp key */ 122| 638| memcpy(&t0, key, 8); 123| 638| memcpy(&t1, key + 8, 8); 124| 638| r0 = t0 & 0xffc0fffffff; 125| 638| t0 >>= 44; 126| 638| t0 |= t1 << 20; 127| 638| r1 = t0 & 0xfffffc0ffff; 128| 638| t1 >>= 24; 129| 638| r2 = t1 & 0x00ffffffc0f; 130| | 131| | /* r^1 */ 132| 638| R = st->R; 133| 638| R[0] = (uint32_t)(r0) &0x3ffffff; 134| 638| R[1] = (uint32_t)((r0 >> 26) | (r1 << 18)) & 0x3ffffff; 135| 638| R[2] = (uint32_t)((r1 >> 8)) & 0x3ffffff; 136| 638| R[3] = (uint32_t)((r1 >> 34) | (r2 << 10)) & 0x3ffffff; 137| 638| R[4] = (uint32_t)((r2 >> 16)); 138| | 139| | /* save pad */ 140| 638| memcpy(&st->pad[0], key + 16, 8); 141| 638| memcpy(&st->pad[1], key + 24, 8); 142| | 143| 638| rt0 = r0; 144| 638| rt1 = r1; 145| 638| rt2 = r2; 146| | 147| | /* r^2, r^4 */ 148| 1.78k| for (i = 0; i < 2; i++) { ------------------ | Branch (148:17): [True: 1.23k, False: 551] ------------------ 149| 1.23k| if (i == 0) { ------------------ | Branch (149:13): [True: 638, False: 592] ------------------ 150| 638| R = st->R2; 151| 638| if (bytes <= 16) { ------------------ | Branch (151:17): [True: 46, False: 592] ------------------ 152| 46| break; 153| 46| } 154| 638| } else if (i == 1) { ------------------ | Branch (154:20): [True: 592, False: 0] ------------------ 155| 592| R = st->R4; 156| 592| if (bytes < 96) { ------------------ | Branch (156:17): [True: 41, False: 551] ------------------ 157| 41| break; 158| 41| } 159| 592| } 160| 1.14k| st2 = rt2 * (5 << 2); 161| | 162| 1.14k| d[0] = ((uint128_t) rt0 * rt0) + ((uint128_t)(rt1 * 2) * st2); 163| 1.14k| d[1] = ((uint128_t) rt2 * st2) + ((uint128_t)(rt0 * 2) * rt1); 164| 1.14k| d[2] = ((uint128_t) rt1 * rt1) + ((uint128_t)(rt2 * 2) * rt0); 165| | 166| 1.14k| rt0 = (uint64_t) d[0] & 0xfffffffffff; 167| 1.14k| c = (uint64_t)(d[0] >> 44); 168| 1.14k| d[1] += c; 169| | 170| 1.14k| rt1 = (uint64_t) d[1] & 0xfffffffffff; 171| 1.14k| c = (uint64_t)(d[1] >> 44); 172| 1.14k| d[2] += c; 173| | 174| 1.14k| rt2 = (uint64_t) d[2] & 0x3ffffffffff; 175| 1.14k| c = (uint64_t)(d[2] >> 42); 176| 1.14k| rt0 += c * 5; 177| 1.14k| c = (rt0 >> 44); 178| 1.14k| rt0 = rt0 & 0xfffffffffff; 179| 1.14k| rt1 += c; 180| 1.14k| c = (rt1 >> 44); 181| 1.14k| rt1 = rt1 & 0xfffffffffff; 182| 1.14k| rt2 += c; /* even if rt2 overflows, it will still fit in rp4 safely, and 183| | is safe to multiply with */ 184| | 185| 1.14k| R[0] = (uint32_t)(rt0) &0x3ffffff; 186| 1.14k| R[1] = (uint32_t)((rt0 >> 26) | (rt1 << 18)) & 0x3ffffff; 187| 1.14k| R[2] = (uint32_t)((rt1 >> 8)) & 0x3ffffff; 188| 1.14k| R[3] = (uint32_t)((rt1 >> 34) | (rt2 << 10)) & 0x3ffffff; 189| 1.14k| R[4] = (uint32_t)((rt2 >> 16)); 190| 1.14k| } 191| 638| st->flags = 0; 192| 638| st->leftover = 0U; 193| 638|} poly1305_sse2.c:poly1305_blocks: 198| 1.61k|{ 199| 1.61k| CRYPTO_ALIGN(64) ------------------ | | 50| 1.61k|# define CRYPTO_ALIGN(x) __attribute__ ((aligned(x))) ------------------ 200| 1.61k| xmmi HIBIT = 201| 1.61k| _mm_shuffle_epi32(_mm_cvtsi32_si128(1 << 24), _MM_SHUFFLE(1, 0, 1, 0)); 202| 1.61k| const xmmi MMASK = _mm_shuffle_epi32(_mm_cvtsi32_si128((1 << 26) - 1), 203| 1.61k| _MM_SHUFFLE(1, 0, 1, 0)); 204| 1.61k| const xmmi FIVE = 205| 1.61k| _mm_shuffle_epi32(_mm_cvtsi32_si128(5), _MM_SHUFFLE(1, 0, 1, 0)); 206| 1.61k| xmmi H0, H1, H2, H3, H4; 207| 1.61k| xmmi T0, T1, T2, T3, T4, T5, T6, T7, T8; 208| 1.61k| xmmi M0, M1, M2, M3, M4; 209| 1.61k| xmmi M5, M6, M7, M8; 210| 1.61k| xmmi C1, C2; 211| 1.61k| xmmi R20, R21, R22, R23, R24, S21, S22, S23, S24; 212| 1.61k| xmmi R40, R41, R42, R43, R44, S41, S42, S43, S44; 213| | 214| 1.61k| if (st->flags & poly1305_final_shift8) { ------------------ | Branch (214:9): [True: 444, False: 1.16k] ------------------ 215| 444| HIBIT = _mm_srli_si128(HIBIT, 8); 216| 444| } 217| 1.61k| if (st->flags & poly1305_final_shift16) { ------------------ | Branch (217:9): [True: 488, False: 1.12k] ------------------ 218| 488| HIBIT = _mm_setzero_si128(); 219| 488| } 220| 1.61k| if (!(st->flags & poly1305_started)) { ------------------ | Branch (220:9): [True: 638, False: 972] ------------------ 221| | /* H = [Mx,My] */ 222| 638| T5 = _mm_unpacklo_epi64( 223| 638| _mm_loadl_epi64((const xmmi *) (const void *) (m + 0)), 224| 638| _mm_loadl_epi64((const xmmi *) (const void *) (m + 16))); 225| 638| T6 = _mm_unpacklo_epi64( 226| 638| _mm_loadl_epi64((const xmmi *) (const void *) (m + 8)), 227| 638| _mm_loadl_epi64((const xmmi *) (const void *) (m + 24))); 228| 638| H0 = _mm_and_si128(MMASK, T5); 229| 638| H1 = _mm_and_si128(MMASK, _mm_srli_epi64(T5, 26)); 230| 638| T5 = _mm_or_si128(_mm_srli_epi64(T5, 52), _mm_slli_epi64(T6, 12)); 231| 638| H2 = _mm_and_si128(MMASK, T5); 232| 638| H3 = _mm_and_si128(MMASK, _mm_srli_epi64(T5, 26)); 233| 638| H4 = _mm_srli_epi64(T6, 40); 234| 638| H4 = _mm_or_si128(H4, HIBIT); 235| 638| m += 32; 236| 638| bytes -= 32; 237| 638| st->flags |= poly1305_started; 238| 972| } else { 239| 972| T0 = _mm_loadu_si128((const xmmi *) (const void *) &st->H.hh[0]); 240| 972| T1 = _mm_loadu_si128((const xmmi *) (const void *) &st->H.hh[4]); 241| 972| T2 = _mm_loadu_si128((const xmmi *) (const void *) &st->H.hh[8]); 242| 972| H0 = _mm_shuffle_epi32(T0, _MM_SHUFFLE(1, 1, 0, 0)); 243| 972| H1 = _mm_shuffle_epi32(T0, _MM_SHUFFLE(3, 3, 2, 2)); 244| 972| H2 = _mm_shuffle_epi32(T1, _MM_SHUFFLE(1, 1, 0, 0)); 245| 972| H3 = _mm_shuffle_epi32(T1, _MM_SHUFFLE(3, 3, 2, 2)); 246| 972| H4 = _mm_shuffle_epi32(T2, _MM_SHUFFLE(1, 1, 0, 0)); 247| 972| } 248| 1.61k| if (st->flags & (poly1305_final_r2_r | poly1305_final_r_1)) { ------------------ | Branch (248:9): [True: 638, False: 972] ------------------ 249| 638| if (st->flags & poly1305_final_r2_r) { ------------------ | Branch (249:13): [True: 356, False: 282] ------------------ 250| | /* use [r^2, r] */ 251| 356| T2 = _mm_loadu_si128((const xmmi *) (const void *) &st->R[0]); 252| 356| T3 = _mm_cvtsi32_si128(st->R[4]); 253| 356| T0 = _mm_loadu_si128((const xmmi *) (const void *) &st->R2[0]); 254| 356| T1 = _mm_cvtsi32_si128(st->R2[4]); 255| 356| T4 = _mm_unpacklo_epi32(T0, T2); 256| 356| T5 = _mm_unpackhi_epi32(T0, T2); 257| 356| R24 = _mm_unpacklo_epi64(T1, T3); 258| 356| } else { 259| | /* use [r^1, 1] */ 260| 282| T0 = _mm_loadu_si128((const xmmi *) (const void *) &st->R[0]); 261| 282| T1 = _mm_cvtsi32_si128(st->R[4]); 262| 282| T2 = _mm_cvtsi32_si128(1); 263| 282| T4 = _mm_unpacklo_epi32(T0, T2); 264| 282| T5 = _mm_unpackhi_epi32(T0, T2); 265| 282| R24 = T1; 266| 282| } 267| 638| R20 = _mm_shuffle_epi32(T4, _MM_SHUFFLE(1, 1, 0, 0)); 268| 638| R21 = _mm_shuffle_epi32(T4, _MM_SHUFFLE(3, 3, 2, 2)); 269| 638| R22 = _mm_shuffle_epi32(T5, _MM_SHUFFLE(1, 1, 0, 0)); 270| 638| R23 = _mm_shuffle_epi32(T5, _MM_SHUFFLE(3, 3, 2, 2)); 271| 972| } else { 272| | /* use [r^2, r^2] */ 273| 972| T0 = _mm_loadu_si128((const xmmi *) (const void *) &st->R2[0]); 274| 972| T1 = _mm_cvtsi32_si128(st->R2[4]); 275| 972| R20 = _mm_shuffle_epi32(T0, _MM_SHUFFLE(0, 0, 0, 0)); 276| 972| R21 = _mm_shuffle_epi32(T0, _MM_SHUFFLE(1, 1, 1, 1)); 277| 972| R22 = _mm_shuffle_epi32(T0, _MM_SHUFFLE(2, 2, 2, 2)); 278| 972| R23 = _mm_shuffle_epi32(T0, _MM_SHUFFLE(3, 3, 3, 3)); 279| 972| R24 = _mm_shuffle_epi32(T1, _MM_SHUFFLE(0, 0, 0, 0)); 280| 972| } 281| 1.61k| S21 = _mm_mul_epu32(R21, FIVE); 282| 1.61k| S22 = _mm_mul_epu32(R22, FIVE); 283| 1.61k| S23 = _mm_mul_epu32(R23, FIVE); 284| 1.61k| S24 = _mm_mul_epu32(R24, FIVE); 285| | 286| 1.61k| if (bytes >= 64) { ------------------ | Branch (286:9): [True: 464, False: 1.14k] ------------------ 287| 464| T0 = _mm_loadu_si128((const xmmi *) (const void *) &st->R4[0]); 288| 464| T1 = _mm_cvtsi32_si128(st->R4[4]); 289| 464| R40 = _mm_shuffle_epi32(T0, _MM_SHUFFLE(0, 0, 0, 0)); 290| 464| R41 = _mm_shuffle_epi32(T0, _MM_SHUFFLE(1, 1, 1, 1)); 291| 464| R42 = _mm_shuffle_epi32(T0, _MM_SHUFFLE(2, 2, 2, 2)); 292| 464| R43 = _mm_shuffle_epi32(T0, _MM_SHUFFLE(3, 3, 3, 3)); 293| 464| R44 = _mm_shuffle_epi32(T1, _MM_SHUFFLE(0, 0, 0, 0)); 294| 464| S41 = _mm_mul_epu32(R41, FIVE); 295| 464| S42 = _mm_mul_epu32(R42, FIVE); 296| 464| S43 = _mm_mul_epu32(R43, FIVE); 297| 464| S44 = _mm_mul_epu32(R44, FIVE); 298| | 299| 1.49M| while (bytes >= 64) { ------------------ | Branch (299:16): [True: 1.49M, False: 464] ------------------ 300| 1.49M| xmmi v00, v01, v02, v03, v04; 301| 1.49M| xmmi v10, v11, v12, v13, v14; 302| 1.49M| xmmi v20, v21, v22, v23, v24; 303| 1.49M| xmmi v30, v31, v32, v33, v34; 304| 1.49M| xmmi v40, v41, v42, v43, v44; 305| 1.49M| xmmi T14, T15; 306| | 307| | /* H *= [r^4,r^4], preload [Mx,My] */ 308| 1.49M| T15 = S42; 309| 1.49M| T0 = H4; 310| 1.49M| T0 = _mm_mul_epu32(T0, S41); 311| 1.49M| v01 = H3; 312| 1.49M| v01 = _mm_mul_epu32(v01, T15); 313| 1.49M| T14 = S43; 314| 1.49M| T1 = H4; 315| 1.49M| T1 = _mm_mul_epu32(T1, T15); 316| 1.49M| v11 = H3; 317| 1.49M| v11 = _mm_mul_epu32(v11, T14); 318| 1.49M| T2 = H4; 319| 1.49M| T2 = _mm_mul_epu32(T2, T14); 320| 1.49M| T0 = _mm_add_epi64(T0, v01); 321| 1.49M| T15 = S44; 322| 1.49M| v02 = H2; 323| 1.49M| v02 = _mm_mul_epu32(v02, T14); 324| 1.49M| T3 = H4; 325| 1.49M| T3 = _mm_mul_epu32(T3, T15); 326| 1.49M| T1 = _mm_add_epi64(T1, v11); 327| 1.49M| v03 = H1; 328| 1.49M| v03 = _mm_mul_epu32(v03, T15); 329| 1.49M| v12 = H2; 330| 1.49M| v12 = _mm_mul_epu32(v12, T15); 331| 1.49M| T0 = _mm_add_epi64(T0, v02); 332| 1.49M| T14 = R40; 333| 1.49M| v21 = H3; 334| 1.49M| v21 = _mm_mul_epu32(v21, T15); 335| 1.49M| v31 = H3; 336| 1.49M| v31 = _mm_mul_epu32(v31, T14); 337| 1.49M| T0 = _mm_add_epi64(T0, v03); 338| 1.49M| T4 = H4; 339| 1.49M| T4 = _mm_mul_epu32(T4, T14); 340| 1.49M| T1 = _mm_add_epi64(T1, v12); 341| 1.49M| v04 = H0; 342| 1.49M| v04 = _mm_mul_epu32(v04, T14); 343| 1.49M| T2 = _mm_add_epi64(T2, v21); 344| 1.49M| v13 = H1; 345| 1.49M| v13 = _mm_mul_epu32(v13, T14); 346| 1.49M| T3 = _mm_add_epi64(T3, v31); 347| 1.49M| T15 = R41; 348| 1.49M| v22 = H2; 349| 1.49M| v22 = _mm_mul_epu32(v22, T14); 350| 1.49M| v32 = H2; 351| 1.49M| v32 = _mm_mul_epu32(v32, T15); 352| 1.49M| T0 = _mm_add_epi64(T0, v04); 353| 1.49M| v41 = H3; 354| 1.49M| v41 = _mm_mul_epu32(v41, T15); 355| 1.49M| T1 = _mm_add_epi64(T1, v13); 356| 1.49M| v14 = H0; 357| 1.49M| v14 = _mm_mul_epu32(v14, T15); 358| 1.49M| T2 = _mm_add_epi64(T2, v22); 359| 1.49M| T14 = R42; 360| 1.49M| T5 = _mm_unpacklo_epi64( 361| 1.49M| _mm_loadl_epi64((const xmmi *) (const void *) (m + 0)), 362| 1.49M| _mm_loadl_epi64((const xmmi *) (const void *) (m + 16))); 363| 1.49M| v23 = H1; 364| 1.49M| v23 = _mm_mul_epu32(v23, T15); 365| 1.49M| T3 = _mm_add_epi64(T3, v32); 366| 1.49M| v33 = H1; 367| 1.49M| v33 = _mm_mul_epu32(v33, T14); 368| 1.49M| T4 = _mm_add_epi64(T4, v41); 369| 1.49M| v42 = H2; 370| 1.49M| v42 = _mm_mul_epu32(v42, T14); 371| 1.49M| T1 = _mm_add_epi64(T1, v14); 372| 1.49M| T15 = R43; 373| 1.49M| T6 = _mm_unpacklo_epi64( 374| 1.49M| _mm_loadl_epi64((const xmmi *) (const void *) (m + 8)), 375| 1.49M| _mm_loadl_epi64((const xmmi *) (const void *) (m + 24))); 376| 1.49M| v24 = H0; 377| 1.49M| v24 = _mm_mul_epu32(v24, T14); 378| 1.49M| T2 = _mm_add_epi64(T2, v23); 379| 1.49M| v34 = H0; 380| 1.49M| v34 = _mm_mul_epu32(v34, T15); 381| 1.49M| T3 = _mm_add_epi64(T3, v33); 382| 1.49M| M0 = _mm_and_si128(MMASK, T5); 383| 1.49M| v43 = H1; 384| 1.49M| v43 = _mm_mul_epu32(v43, T15); 385| 1.49M| T4 = _mm_add_epi64(T4, v42); 386| 1.49M| M1 = _mm_and_si128(MMASK, _mm_srli_epi64(T5, 26)); 387| 1.49M| v44 = H0; 388| 1.49M| v44 = _mm_mul_epu32(v44, R44); 389| 1.49M| T2 = _mm_add_epi64(T2, v24); 390| 1.49M| T5 = _mm_or_si128(_mm_srli_epi64(T5, 52), _mm_slli_epi64(T6, 12)); 391| 1.49M| T3 = _mm_add_epi64(T3, v34); 392| 1.49M| M3 = _mm_and_si128(MMASK, _mm_srli_epi64(T6, 14)); 393| 1.49M| T4 = _mm_add_epi64(T4, v43); 394| 1.49M| M2 = _mm_and_si128(MMASK, T5); 395| 1.49M| T4 = _mm_add_epi64(T4, v44); 396| 1.49M| M4 = _mm_or_si128(_mm_srli_epi64(T6, 40), HIBIT); 397| | 398| | /* H += [Mx',My'] */ 399| 1.49M| T5 = _mm_loadu_si128((const xmmi *) (const void *) (m + 32)); 400| 1.49M| T6 = _mm_loadu_si128((const xmmi *) (const void *) (m + 48)); 401| 1.49M| T7 = _mm_unpacklo_epi32(T5, T6); 402| 1.49M| T8 = _mm_unpackhi_epi32(T5, T6); 403| 1.49M| M5 = _mm_unpacklo_epi32(T7, _mm_setzero_si128()); 404| 1.49M| M6 = _mm_unpackhi_epi32(T7, _mm_setzero_si128()); 405| 1.49M| M7 = _mm_unpacklo_epi32(T8, _mm_setzero_si128()); 406| 1.49M| M8 = _mm_unpackhi_epi32(T8, _mm_setzero_si128()); 407| 1.49M| M6 = _mm_slli_epi64(M6, 6); 408| 1.49M| M7 = _mm_slli_epi64(M7, 12); 409| 1.49M| M8 = _mm_slli_epi64(M8, 18); 410| 1.49M| T0 = _mm_add_epi64(T0, M5); 411| 1.49M| T1 = _mm_add_epi64(T1, M6); 412| 1.49M| T2 = _mm_add_epi64(T2, M7); 413| 1.49M| T3 = _mm_add_epi64(T3, M8); 414| 1.49M| T4 = _mm_add_epi64(T4, HIBIT); 415| | 416| | /* H += [Mx,My]*[r^2,r^2] */ 417| 1.49M| T15 = S22; 418| 1.49M| v00 = M4; 419| 1.49M| v00 = _mm_mul_epu32(v00, S21); 420| 1.49M| v01 = M3; 421| 1.49M| v01 = _mm_mul_epu32(v01, T15); 422| 1.49M| T14 = S23; 423| 1.49M| v10 = M4; 424| 1.49M| v10 = _mm_mul_epu32(v10, T15); 425| 1.49M| v11 = M3; 426| 1.49M| v11 = _mm_mul_epu32(v11, T14); 427| 1.49M| T0 = _mm_add_epi64(T0, v00); 428| 1.49M| v20 = M4; 429| 1.49M| v20 = _mm_mul_epu32(v20, T14); 430| 1.49M| T0 = _mm_add_epi64(T0, v01); 431| 1.49M| T15 = S24; 432| 1.49M| v02 = M2; 433| 1.49M| v02 = _mm_mul_epu32(v02, T14); 434| 1.49M| T1 = _mm_add_epi64(T1, v10); 435| 1.49M| v30 = M4; 436| 1.49M| v30 = _mm_mul_epu32(v30, T15); 437| 1.49M| T1 = _mm_add_epi64(T1, v11); 438| 1.49M| v03 = M1; 439| 1.49M| v03 = _mm_mul_epu32(v03, T15); 440| 1.49M| T2 = _mm_add_epi64(T2, v20); 441| 1.49M| v12 = M2; 442| 1.49M| v12 = _mm_mul_epu32(v12, T15); 443| 1.49M| T0 = _mm_add_epi64(T0, v02); 444| 1.49M| T14 = R20; 445| 1.49M| v21 = M3; 446| 1.49M| v21 = _mm_mul_epu32(v21, T15); 447| 1.49M| T3 = _mm_add_epi64(T3, v30); 448| 1.49M| v31 = M3; 449| 1.49M| v31 = _mm_mul_epu32(v31, T14); 450| 1.49M| T0 = _mm_add_epi64(T0, v03); 451| 1.49M| v40 = M4; 452| 1.49M| v40 = _mm_mul_epu32(v40, T14); 453| 1.49M| T1 = _mm_add_epi64(T1, v12); 454| 1.49M| v04 = M0; 455| 1.49M| v04 = _mm_mul_epu32(v04, T14); 456| 1.49M| T2 = _mm_add_epi64(T2, v21); 457| 1.49M| v13 = M1; 458| 1.49M| v13 = _mm_mul_epu32(v13, T14); 459| 1.49M| T3 = _mm_add_epi64(T3, v31); 460| 1.49M| T15 = R21; 461| 1.49M| v22 = M2; 462| 1.49M| v22 = _mm_mul_epu32(v22, T14); 463| 1.49M| T4 = _mm_add_epi64(T4, v40); 464| 1.49M| v32 = M2; 465| 1.49M| v32 = _mm_mul_epu32(v32, T15); 466| 1.49M| T0 = _mm_add_epi64(T0, v04); 467| 1.49M| v41 = M3; 468| 1.49M| v41 = _mm_mul_epu32(v41, T15); 469| 1.49M| T1 = _mm_add_epi64(T1, v13); 470| 1.49M| v14 = M0; 471| 1.49M| v14 = _mm_mul_epu32(v14, T15); 472| 1.49M| T2 = _mm_add_epi64(T2, v22); 473| 1.49M| T14 = R22; 474| 1.49M| v23 = M1; 475| 1.49M| v23 = _mm_mul_epu32(v23, T15); 476| 1.49M| T3 = _mm_add_epi64(T3, v32); 477| 1.49M| v33 = M1; 478| 1.49M| v33 = _mm_mul_epu32(v33, T14); 479| 1.49M| T4 = _mm_add_epi64(T4, v41); 480| 1.49M| v42 = M2; 481| 1.49M| v42 = _mm_mul_epu32(v42, T14); 482| 1.49M| T1 = _mm_add_epi64(T1, v14); 483| 1.49M| T15 = R23; 484| 1.49M| v24 = M0; 485| 1.49M| v24 = _mm_mul_epu32(v24, T14); 486| 1.49M| T2 = _mm_add_epi64(T2, v23); 487| 1.49M| v34 = M0; 488| 1.49M| v34 = _mm_mul_epu32(v34, T15); 489| 1.49M| T3 = _mm_add_epi64(T3, v33); 490| 1.49M| v43 = M1; 491| 1.49M| v43 = _mm_mul_epu32(v43, T15); 492| 1.49M| T4 = _mm_add_epi64(T4, v42); 493| 1.49M| v44 = M0; 494| 1.49M| v44 = _mm_mul_epu32(v44, R24); 495| 1.49M| T2 = _mm_add_epi64(T2, v24); 496| 1.49M| T3 = _mm_add_epi64(T3, v34); 497| 1.49M| T4 = _mm_add_epi64(T4, v43); 498| 1.49M| T4 = _mm_add_epi64(T4, v44); 499| | 500| | /* reduce */ 501| 1.49M| C1 = _mm_srli_epi64(T0, 26); 502| 1.49M| C2 = _mm_srli_epi64(T3, 26); 503| 1.49M| T0 = _mm_and_si128(T0, MMASK); 504| 1.49M| T3 = _mm_and_si128(T3, MMASK); 505| 1.49M| T1 = _mm_add_epi64(T1, C1); 506| 1.49M| T4 = _mm_add_epi64(T4, C2); 507| 1.49M| C1 = _mm_srli_epi64(T1, 26); 508| 1.49M| C2 = _mm_srli_epi64(T4, 26); 509| 1.49M| T1 = _mm_and_si128(T1, MMASK); 510| 1.49M| T4 = _mm_and_si128(T4, MMASK); 511| 1.49M| T2 = _mm_add_epi64(T2, C1); 512| 1.49M| T0 = _mm_add_epi64(T0, _mm_mul_epu32(C2, FIVE)); 513| 1.49M| C1 = _mm_srli_epi64(T2, 26); 514| 1.49M| C2 = _mm_srli_epi64(T0, 26); 515| 1.49M| T2 = _mm_and_si128(T2, MMASK); 516| 1.49M| T0 = _mm_and_si128(T0, MMASK); 517| 1.49M| T3 = _mm_add_epi64(T3, C1); 518| 1.49M| T1 = _mm_add_epi64(T1, C2); 519| 1.49M| C1 = _mm_srli_epi64(T3, 26); 520| 1.49M| T3 = _mm_and_si128(T3, MMASK); 521| 1.49M| T4 = _mm_add_epi64(T4, C1); 522| | 523| | /* Final: H = (H*[r^4,r^4] + [Mx,My]*[r^2,r^2] + [Mx',My']) */ 524| 1.49M| H0 = T0; 525| 1.49M| H1 = T1; 526| 1.49M| H2 = T2; 527| 1.49M| H3 = T3; 528| 1.49M| H4 = T4; 529| | 530| 1.49M| m += 64; 531| 1.49M| bytes -= 64; 532| 1.49M| } 533| 464| } 534| | 535| 1.61k| if (bytes >= 32) { ------------------ | Branch (535:9): [True: 1.20k, False: 408] ------------------ 536| 1.20k| xmmi v01, v02, v03, v04; 537| 1.20k| xmmi v11, v12, v13, v14; 538| 1.20k| xmmi v21, v22, v23, v24; 539| 1.20k| xmmi v31, v32, v33, v34; 540| 1.20k| xmmi v41, v42, v43, v44; 541| 1.20k| xmmi T14, T15; 542| | 543| | /* H *= [r^2,r^2] */ 544| 1.20k| T15 = S22; 545| 1.20k| T0 = H4; 546| 1.20k| T0 = _mm_mul_epu32(T0, S21); 547| 1.20k| v01 = H3; 548| 1.20k| v01 = _mm_mul_epu32(v01, T15); 549| 1.20k| T14 = S23; 550| 1.20k| T1 = H4; 551| 1.20k| T1 = _mm_mul_epu32(T1, T15); 552| 1.20k| v11 = H3; 553| 1.20k| v11 = _mm_mul_epu32(v11, T14); 554| 1.20k| T2 = H4; 555| 1.20k| T2 = _mm_mul_epu32(T2, T14); 556| 1.20k| T0 = _mm_add_epi64(T0, v01); 557| 1.20k| T15 = S24; 558| 1.20k| v02 = H2; 559| 1.20k| v02 = _mm_mul_epu32(v02, T14); 560| 1.20k| T3 = H4; 561| 1.20k| T3 = _mm_mul_epu32(T3, T15); 562| 1.20k| T1 = _mm_add_epi64(T1, v11); 563| 1.20k| v03 = H1; 564| 1.20k| v03 = _mm_mul_epu32(v03, T15); 565| 1.20k| v12 = H2; 566| 1.20k| v12 = _mm_mul_epu32(v12, T15); 567| 1.20k| T0 = _mm_add_epi64(T0, v02); 568| 1.20k| T14 = R20; 569| 1.20k| v21 = H3; 570| 1.20k| v21 = _mm_mul_epu32(v21, T15); 571| 1.20k| v31 = H3; 572| 1.20k| v31 = _mm_mul_epu32(v31, T14); 573| 1.20k| T0 = _mm_add_epi64(T0, v03); 574| 1.20k| T4 = H4; 575| 1.20k| T4 = _mm_mul_epu32(T4, T14); 576| 1.20k| T1 = _mm_add_epi64(T1, v12); 577| 1.20k| v04 = H0; 578| 1.20k| v04 = _mm_mul_epu32(v04, T14); 579| 1.20k| T2 = _mm_add_epi64(T2, v21); 580| 1.20k| v13 = H1; 581| 1.20k| v13 = _mm_mul_epu32(v13, T14); 582| 1.20k| T3 = _mm_add_epi64(T3, v31); 583| 1.20k| T15 = R21; 584| 1.20k| v22 = H2; 585| 1.20k| v22 = _mm_mul_epu32(v22, T14); 586| 1.20k| v32 = H2; 587| 1.20k| v32 = _mm_mul_epu32(v32, T15); 588| 1.20k| T0 = _mm_add_epi64(T0, v04); 589| 1.20k| v41 = H3; 590| 1.20k| v41 = _mm_mul_epu32(v41, T15); 591| 1.20k| T1 = _mm_add_epi64(T1, v13); 592| 1.20k| v14 = H0; 593| 1.20k| v14 = _mm_mul_epu32(v14, T15); 594| 1.20k| T2 = _mm_add_epi64(T2, v22); 595| 1.20k| T14 = R22; 596| 1.20k| v23 = H1; 597| 1.20k| v23 = _mm_mul_epu32(v23, T15); 598| 1.20k| T3 = _mm_add_epi64(T3, v32); 599| 1.20k| v33 = H1; 600| 1.20k| v33 = _mm_mul_epu32(v33, T14); 601| 1.20k| T4 = _mm_add_epi64(T4, v41); 602| 1.20k| v42 = H2; 603| 1.20k| v42 = _mm_mul_epu32(v42, T14); 604| 1.20k| T1 = _mm_add_epi64(T1, v14); 605| 1.20k| T15 = R23; 606| 1.20k| v24 = H0; 607| 1.20k| v24 = _mm_mul_epu32(v24, T14); 608| 1.20k| T2 = _mm_add_epi64(T2, v23); 609| 1.20k| v34 = H0; 610| 1.20k| v34 = _mm_mul_epu32(v34, T15); 611| 1.20k| T3 = _mm_add_epi64(T3, v33); 612| 1.20k| v43 = H1; 613| 1.20k| v43 = _mm_mul_epu32(v43, T15); 614| 1.20k| T4 = _mm_add_epi64(T4, v42); 615| 1.20k| v44 = H0; 616| 1.20k| v44 = _mm_mul_epu32(v44, R24); 617| 1.20k| T2 = _mm_add_epi64(T2, v24); 618| 1.20k| T3 = _mm_add_epi64(T3, v34); 619| 1.20k| T4 = _mm_add_epi64(T4, v43); 620| 1.20k| T4 = _mm_add_epi64(T4, v44); 621| | 622| | /* H += [Mx,My] */ 623| 1.20k| if (m) { ------------------ | Branch (623:13): [True: 564, False: 638] ------------------ 624| 564| T5 = _mm_loadu_si128((const xmmi *) (const void *) (m + 0)); 625| 564| T6 = _mm_loadu_si128((const xmmi *) (const void *) (m + 16)); 626| 564| T7 = _mm_unpacklo_epi32(T5, T6); 627| 564| T8 = _mm_unpackhi_epi32(T5, T6); 628| 564| M0 = _mm_unpacklo_epi32(T7, _mm_setzero_si128()); 629| 564| M1 = _mm_unpackhi_epi32(T7, _mm_setzero_si128()); 630| 564| M2 = _mm_unpacklo_epi32(T8, _mm_setzero_si128()); 631| 564| M3 = _mm_unpackhi_epi32(T8, _mm_setzero_si128()); 632| 564| M1 = _mm_slli_epi64(M1, 6); 633| 564| M2 = _mm_slli_epi64(M2, 12); 634| 564| M3 = _mm_slli_epi64(M3, 18); 635| 564| T0 = _mm_add_epi64(T0, M0); 636| 564| T1 = _mm_add_epi64(T1, M1); 637| 564| T2 = _mm_add_epi64(T2, M2); 638| 564| T3 = _mm_add_epi64(T3, M3); 639| 564| T4 = _mm_add_epi64(T4, HIBIT); 640| 564| } 641| | 642| | /* reduce */ 643| 1.20k| C1 = _mm_srli_epi64(T0, 26); 644| 1.20k| C2 = _mm_srli_epi64(T3, 26); 645| 1.20k| T0 = _mm_and_si128(T0, MMASK); 646| 1.20k| T3 = _mm_and_si128(T3, MMASK); 647| 1.20k| T1 = _mm_add_epi64(T1, C1); 648| 1.20k| T4 = _mm_add_epi64(T4, C2); 649| 1.20k| C1 = _mm_srli_epi64(T1, 26); 650| 1.20k| C2 = _mm_srli_epi64(T4, 26); 651| 1.20k| T1 = _mm_and_si128(T1, MMASK); 652| 1.20k| T4 = _mm_and_si128(T4, MMASK); 653| 1.20k| T2 = _mm_add_epi64(T2, C1); 654| 1.20k| T0 = _mm_add_epi64(T0, _mm_mul_epu32(C2, FIVE)); 655| 1.20k| C1 = _mm_srli_epi64(T2, 26); 656| 1.20k| C2 = _mm_srli_epi64(T0, 26); 657| 1.20k| T2 = _mm_and_si128(T2, MMASK); 658| 1.20k| T0 = _mm_and_si128(T0, MMASK); 659| 1.20k| T3 = _mm_add_epi64(T3, C1); 660| 1.20k| T1 = _mm_add_epi64(T1, C2); 661| 1.20k| C1 = _mm_srli_epi64(T3, 26); 662| 1.20k| T3 = _mm_and_si128(T3, MMASK); 663| 1.20k| T4 = _mm_add_epi64(T4, C1); 664| | 665| | /* H = (H*[r^2,r^2] + [Mx,My]) */ 666| 1.20k| H0 = T0; 667| 1.20k| H1 = T1; 668| 1.20k| H2 = T2; 669| 1.20k| H3 = T3; 670| 1.20k| H4 = T4; 671| 1.20k| } 672| | 673| 1.61k| if (m) { ------------------ | Branch (673:9): [True: 972, False: 638] ------------------ 674| 972| T0 = _mm_shuffle_epi32(H0, _MM_SHUFFLE(0, 0, 2, 0)); 675| 972| T1 = _mm_shuffle_epi32(H1, _MM_SHUFFLE(0, 0, 2, 0)); 676| 972| T2 = _mm_shuffle_epi32(H2, _MM_SHUFFLE(0, 0, 2, 0)); 677| 972| T3 = _mm_shuffle_epi32(H3, _MM_SHUFFLE(0, 0, 2, 0)); 678| 972| T4 = _mm_shuffle_epi32(H4, _MM_SHUFFLE(0, 0, 2, 0)); 679| 972| T0 = _mm_unpacklo_epi64(T0, T1); 680| 972| T1 = _mm_unpacklo_epi64(T2, T3); 681| 972| _mm_storeu_si128((xmmi *) (void *) &st->H.hh[0], T0); 682| 972| _mm_storeu_si128((xmmi *) (void *) &st->H.hh[4], T1); 683| 972| _mm_storel_epi64((xmmi *) (void *) &st->H.hh[8], T4); 684| 972| } else { 685| 638| uint32_t t0, t1, t2, t3, t4, b; 686| 638| uint64_t h0, h1, h2, g0, g1, g2, c, nc; 687| | 688| | /* H = H[0]+H[1] */ 689| 638| T0 = H0; 690| 638| T1 = H1; 691| 638| T2 = H2; 692| 638| T3 = H3; 693| 638| T4 = H4; 694| | 695| 638| T0 = _mm_add_epi64(T0, _mm_srli_si128(T0, 8)); 696| 638| T1 = _mm_add_epi64(T1, _mm_srli_si128(T1, 8)); 697| 638| T2 = _mm_add_epi64(T2, _mm_srli_si128(T2, 8)); 698| 638| T3 = _mm_add_epi64(T3, _mm_srli_si128(T3, 8)); 699| 638| T4 = _mm_add_epi64(T4, _mm_srli_si128(T4, 8)); 700| | 701| 638| t0 = _mm_cvtsi128_si32(T0); 702| 638| b = (t0 >> 26); 703| 638| t0 &= 0x3ffffff; 704| 638| t1 = _mm_cvtsi128_si32(T1) + b; 705| 638| b = (t1 >> 26); 706| 638| t1 &= 0x3ffffff; 707| 638| t2 = _mm_cvtsi128_si32(T2) + b; 708| 638| b = (t2 >> 26); 709| 638| t2 &= 0x3ffffff; 710| 638| t3 = _mm_cvtsi128_si32(T3) + b; 711| 638| b = (t3 >> 26); 712| 638| t3 &= 0x3ffffff; 713| 638| t4 = _mm_cvtsi128_si32(T4) + b; 714| | 715| | /* everything except t4 is in range, so this is all safe */ 716| 638| h0 = (((uint64_t) t0) | ((uint64_t) t1 << 26)) & 0xfffffffffffull; 717| 638| h1 = (((uint64_t) t1 >> 18) | ((uint64_t) t2 << 8) | 718| 638| ((uint64_t) t3 << 34)) & 719| 638| 0xfffffffffffull; 720| 638| h2 = (((uint64_t) t3 >> 10) | ((uint64_t) t4 << 16)); 721| | 722| 638| c = (h2 >> 42); 723| 638| h2 &= 0x3ffffffffff; 724| 638| h0 += c * 5; 725| 638| c = (h0 >> 44); 726| 638| h0 &= 0xfffffffffff; 727| 638| h1 += c; 728| 638| c = (h1 >> 44); 729| 638| h1 &= 0xfffffffffff; 730| 638| h2 += c; 731| 638| c = (h2 >> 42); 732| 638| h2 &= 0x3ffffffffff; 733| 638| h0 += c * 5; 734| 638| c = (h0 >> 44); 735| 638| h0 &= 0xfffffffffff; 736| 638| h1 += c; 737| | 738| 638| g0 = h0 + 5; 739| 638| c = (g0 >> 44); 740| 638| g0 &= 0xfffffffffff; 741| 638| g1 = h1 + c; 742| 638| c = (g1 >> 44); 743| 638| g1 &= 0xfffffffffff; 744| 638| g2 = h2 + c - ((uint64_t) 1 << 42); 745| | 746| 638| c = (g2 >> 63) - 1; 747| 638| nc = ~c; 748| 638| h0 = (h0 & nc) | (g0 & c); 749| 638| h1 = (h1 & nc) | (g1 & c); 750| 638| h2 = (h2 & nc) | (g2 & c); 751| | 752| 638| st->H.h[0] = h0; 753| 638| st->H.h[1] = h1; 754| 638| st->H.h[2] = h2; 755| 638| } 756| 1.61k|} poly1305_sse2.c:poly1305_finish_ext: 805| 638|{ 806| 638| uint64_t h0, h1, h2; 807| | 808| 638| if (leftover) { ------------------ | Branch (808:9): [True: 466, False: 172] ------------------ 809| 466| CRYPTO_ALIGN(16) unsigned char final[32] = { 0 }; ------------------ | | 50| 466|# define CRYPTO_ALIGN(x) __attribute__ ((aligned(x))) ------------------ 810| | 811| 466| poly1305_block_copy31(final, m, leftover); 812| 466| if (leftover != 16) { ------------------ | Branch (812:13): [True: 428, False: 38] ------------------ 813| 428| final[leftover] = 1; 814| 428| } 815| 466| st->flags |= 816| 466| (leftover >= 16) ? poly1305_final_shift8 : poly1305_final_shift16; ------------------ | Branch (816:13): [True: 222, False: 244] ------------------ 817| 466| poly1305_blocks(st, final, 32); 818| 466| } 819| | 820| 638| if (st->flags & poly1305_started) { ------------------ | Branch (820:9): [True: 638, False: 0] ------------------ 821| | /* finalize, H *= [r^2,r], or H *= [r,1] */ 822| 638| if (!leftover || (leftover > 16)) { ------------------ | Branch (822:13): [True: 172, False: 466] | Branch (822:26): [True: 184, False: 282] ------------------ 823| 356| st->flags |= poly1305_final_r2_r; 824| 356| } else { 825| 282| st->flags |= poly1305_final_r_1; 826| 282| } 827| 638| poly1305_blocks(st, NULL, 32); 828| 638| } 829| | 830| 638| h0 = st->H.h[0]; 831| 638| h1 = st->H.h[1]; 832| 638| h2 = st->H.h[2]; 833| | 834| | /* pad */ 835| 638| h0 = ((h0) | (h1 << 44)); 836| 638| h1 = ((h1 >> 20) | (h2 << 24)); 837| 638|#ifdef HAVE_AMD64_ASM 838| 638| __asm__ __volatile__( 839| 638| "addq %2, %0 ;\n" 840| 638| "adcq %3, %1 ;\n" 841| 638| : "+r"(h0), "+r"(h1) 842| 638| : "r"(st->pad[0]), "r"(st->pad[1]) 843| 638| : "flags", "cc"); 844| |#else 845| | { 846| | uint128_t h; 847| | 848| | memcpy(&h, &st->pad[0], 16); 849| | h += ((uint128_t) h1 << 64) | h0; 850| | h0 = (uint64_t) h; 851| | h1 = (uint64_t)(h >> 64); 852| | } 853| |#endif 854| 638| _mm_storeu_si128((xmmi *) (void *) st + 0, _mm_setzero_si128()); 855| 638| _mm_storeu_si128((xmmi *) (void *) st + 1, _mm_setzero_si128()); 856| 638| _mm_storeu_si128((xmmi *) (void *) st + 2, _mm_setzero_si128()); 857| 638| _mm_storeu_si128((xmmi *) (void *) st + 3, _mm_setzero_si128()); 858| 638| _mm_storeu_si128((xmmi *) (void *) st + 4, _mm_setzero_si128()); 859| 638| _mm_storeu_si128((xmmi *) (void *) st + 5, _mm_setzero_si128()); 860| 638| _mm_storeu_si128((xmmi *) (void *) st + 6, _mm_setzero_si128()); 861| 638| _mm_storeu_si128((xmmi *) (void *) st + 7, _mm_setzero_si128()); 862| | 863| 638| memcpy(&mac[0], &h0, 8); 864| 638| memcpy(&mac[8], &h1, 8); 865| | 866| 638| sodium_memzero((void *) st, sizeof *st); 867| 638|} poly1305_sse2.c:poly1305_block_copy31: 75| 466|{ 76| 466| if (bytes & 16) { ------------------ | Branch (76:9): [True: 222, False: 244] ------------------ 77| 222| _mm_store_si128((xmmi *) (void *) dst, 78| 222| _mm_loadu_si128((const xmmi *) (const void *) src)); 79| 222| src += 16; 80| 222| dst += 16; 81| 222| } 82| 466| if (bytes & 8) { ------------------ | Branch (82:9): [True: 252, False: 214] ------------------ 83| 252| memcpy(dst, src, 8); 84| 252| src += 8; 85| 252| dst += 8; 86| 252| } 87| 466| if (bytes & 4) { ------------------ | Branch (87:9): [True: 266, False: 200] ------------------ 88| 266| memcpy(dst, src, 4); 89| 266| src += 4; 90| 266| dst += 4; 91| 266| } 92| 466| if (bytes & 2) { ------------------ | Branch (92:9): [True: 278, False: 188] ------------------ 93| 278| memcpy(dst, src, 2); 94| 278| src += 2; 95| 278| dst += 2; 96| 278| } 97| 466| if (bytes & 1) { ------------------ | Branch (97:9): [True: 326, False: 140] ------------------ 98| 326| *dst = *src; 99| 326| } 100| 466|} poly1305_sse2.c:crypto_onetimeauth_poly1305_sse2_verify: 930| 319|{ 931| 319| unsigned char correct[16]; 932| | 933| 319| crypto_onetimeauth_poly1305_sse2(correct, in, inlen, k); 934| | 935| 319| return crypto_verify_16(h, correct); 936| 319|} poly1305_sse2.c:crypto_onetimeauth_poly1305_sse2_init: 878| 319|{ 879| 319| COMPILER_ASSERT(sizeof(crypto_onetimeauth_poly1305_state) >= ------------------ | | 23| 319|#define COMPILER_ASSERT(X) (void) sizeof(char[(X) ? 1 : -1]) ------------------ 880| 319| sizeof(poly1305_state_internal_t)); 881| 319| poly1305_init_ext((poly1305_state_internal_t *) (void *) state, key, 0U); 882| | 883| 319| return 0; 884| 319|} poly1305_sse2.c:crypto_onetimeauth_poly1305_sse2_update: 890| 319|{ 891| 319| poly1305_update((poly1305_state_internal_t *) (void *) state, in, inlen); 892| | 893| 319| return 0; 894| 319|} poly1305_sse2.c:poly1305_update: 761| 319|{ 762| 319| unsigned long long i; 763| | 764| | /* handle leftover */ 765| 319| if (st->leftover) { ------------------ | Branch (765:9): [True: 0, False: 319] ------------------ 766| 0| unsigned long long want = (poly1305_block_size - st->leftover); ------------------ | | 30| 0|# define poly1305_block_size 32 ------------------ 767| | 768| 0| if (want > bytes) { ------------------ | Branch (768:13): [True: 0, False: 0] ------------------ 769| 0| want = bytes; 770| 0| } 771| 0| for (i = 0; i < want; i++) { ------------------ | Branch (771:21): [True: 0, False: 0] ------------------ 772| 0| st->buffer[st->leftover + i] = m[i]; 773| 0| } 774| 0| bytes -= want; 775| 0| m += want; 776| 0| st->leftover += want; 777| 0| if (st->leftover < poly1305_block_size) { ------------------ | | 30| 0|# define poly1305_block_size 32 ------------------ | Branch (777:13): [True: 0, False: 0] ------------------ 778| 0| return; 779| 0| } 780| 0| poly1305_blocks(st, st->buffer, poly1305_block_size); ------------------ | | 30| 0|# define poly1305_block_size 32 ------------------ 781| 0| st->leftover = 0; 782| 0| } 783| | 784| | /* process full blocks */ 785| 319| if (bytes >= poly1305_block_size) { ------------------ | | 30| 319|# define poly1305_block_size 32 ------------------ | Branch (785:9): [True: 253, False: 66] ------------------ 786| 253| unsigned long long want = (bytes & ~(poly1305_block_size - 1)); ------------------ | | 30| 253|# define poly1305_block_size 32 ------------------ 787| | 788| 253| poly1305_blocks(st, m, want); 789| 253| m += want; 790| 253| bytes -= want; 791| 253| } 792| | 793| | /* store leftover */ 794| 319| if (bytes) { ------------------ | Branch (794:9): [True: 233, False: 86] ------------------ 795| 3.99k| for (i = 0; i < bytes; i++) { ------------------ | Branch (795:21): [True: 3.75k, False: 233] ------------------ 796| 3.75k| st->buffer[st->leftover + i] = m[i]; 797| 3.75k| } 798| 233| st->leftover += bytes; 799| 233| } 800| 319|} poly1305_sse2.c:crypto_onetimeauth_poly1305_sse2_final: 899| 319|{ 900| 319| poly1305_finish((poly1305_state_internal_t *) (void *) state, out); 901| | 902| 319| return 0; 903| 319|} poly1305_sse2.c:poly1305_finish: 871| 319|{ 872| 319| poly1305_finish_ext(st, st->buffer, st->leftover, mac); 873| 319|} _crypto_pwhash_argon2_pick_best_implementation: 554| 1|{ 555| 1| return argon2_pick_best_implementation(); 556| 1|} argon2-core.c:argon2_pick_best_implementation: 523| 1|{ 524| |/* LCOV_EXCL_START */ 525| 1|#if defined(HAVE_AVX512FINTRIN_H) && defined(HAVE_AVX2INTRIN_H) && \ 526| 1| defined(HAVE_TMMINTRIN_H) && defined(HAVE_SMMINTRIN_H) && \ 527| 1| !defined(__APPLE__) 528| 1| if (sodium_runtime_has_avx512f()) { ------------------ | Branch (528:9): [True: 0, False: 1] ------------------ 529| 0| fill_segment = argon2_fill_segment_avx512f; 530| 0| return 0; 531| 0| } 532| 1|#endif 533| 1|#if defined(HAVE_AVX2INTRIN_H) && defined(HAVE_TMMINTRIN_H) && \ 534| 1| defined(HAVE_SMMINTRIN_H) 535| 1| if (sodium_runtime_has_avx2()) { ------------------ | Branch (535:9): [True: 1, False: 0] ------------------ 536| 1| fill_segment = argon2_fill_segment_avx2; 537| 1| return 0; 538| 1| } 539| 0|#endif 540| 0|#if defined(HAVE_EMMINTRIN_H) && defined(HAVE_TMMINTRIN_H) 541| 0| if (sodium_runtime_has_ssse3()) { ------------------ | Branch (541:9): [True: 0, False: 0] ------------------ 542| 0| fill_segment = argon2_fill_segment_ssse3; 543| 0| return 0; 544| 0| } 545| 0|#endif 546| 0| fill_segment = argon2_fill_segment_ref; 547| | 548| 0| return 0; 549| | /* LCOV_EXCL_STOP */ 550| 0|} _crypto_scalarmult_curve25519_pick_best_implementation: 51| 1|{ 52| 1| implementation = &crypto_scalarmult_curve25519_ref10_implementation; 53| | 54| 1|#ifdef HAVE_AVX_ASM 55| 1| if (sodium_runtime_has_avx()) { ------------------ | Branch (55:9): [True: 1, False: 0] ------------------ 56| 1| implementation = &crypto_scalarmult_curve25519_sandy2x_implementation; 57| 1| } 58| 1|#endif 59| 1| return 0; 60| 1|} crypto_secretbox_keygen: 65| 319|{ 66| 319| randombytes_buf(k, crypto_secretbox_KEYBYTES); ------------------ | | 16| 319|#define crypto_secretbox_KEYBYTES crypto_secretbox_xsalsa20poly1305_KEYBYTES | | ------------------ | | | | 15| 319|#define crypto_secretbox_xsalsa20poly1305_KEYBYTES 32U | | ------------------ ------------------ 67| 319|} crypto_secretbox_detached: 21| 319|{ 22| 319| crypto_onetimeauth_poly1305_state state; 23| 319| unsigned char block0[64U]; 24| 319| unsigned char subkey[crypto_stream_salsa20_KEYBYTES]; 25| 319| unsigned long long i; 26| 319| unsigned long long mlen0; 27| | 28| 319| crypto_core_hsalsa20(subkey, n, k, NULL); 29| | 30| 319| if (((uintptr_t) c > (uintptr_t) m && ------------------ | Branch (30:10): [True: 306, False: 13] ------------------ 31| 319| (uintptr_t) c - (uintptr_t) m < mlen) || ------------------ | Branch (31:10): [True: 0, False: 306] ------------------ 32| 319| ((uintptr_t) m > (uintptr_t) c && ------------------ | Branch (32:10): [True: 13, False: 306] ------------------ 33| 319| (uintptr_t) m - (uintptr_t) c < mlen)) { /* LCOV_EXCL_LINE */ ------------------ | Branch (33:10): [True: 0, False: 13] ------------------ 34| 0| memmove(c, m, mlen); 35| 0| m = c; 36| 0| } 37| 319| memset(block0, 0U, crypto_secretbox_ZEROBYTES); ------------------ | | 70| 319|#define crypto_secretbox_ZEROBYTES crypto_secretbox_xsalsa20poly1305_ZEROBYTES | | ------------------ | | | | 60| 319| (crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES + \ | | | | ------------------ | | | | | | 55| 319|#define crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES 16U | | | | ------------------ | | | | 61| 319| crypto_secretbox_xsalsa20poly1305_MACBYTES) | | | | ------------------ | | | | | | 23| 319|#define crypto_secretbox_xsalsa20poly1305_MACBYTES 16U | | | | ------------------ | | ------------------ ------------------ 38| 319| COMPILER_ASSERT(64U >= crypto_secretbox_ZEROBYTES); ------------------ | | 23| 319|#define COMPILER_ASSERT(X) (void) sizeof(char[(X) ? 1 : -1]) ------------------ 39| 319| mlen0 = mlen; 40| 319| if (mlen0 > 64U - crypto_secretbox_ZEROBYTES) { ------------------ | | 70| 319|#define crypto_secretbox_ZEROBYTES crypto_secretbox_xsalsa20poly1305_ZEROBYTES | | ------------------ | | | | 60| 319| (crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES + \ | | | | ------------------ | | | | | | 55| 319|#define crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES 16U | | | | ------------------ | | | | 61| 319| crypto_secretbox_xsalsa20poly1305_MACBYTES) | | | | ------------------ | | | | | | 23| 319|#define crypto_secretbox_xsalsa20poly1305_MACBYTES 16U | | | | ------------------ | | ------------------ ------------------ | Branch (40:9): [True: 252, False: 67] ------------------ 41| 252| mlen0 = 64U - crypto_secretbox_ZEROBYTES; ------------------ | | 70| 252|#define crypto_secretbox_ZEROBYTES crypto_secretbox_xsalsa20poly1305_ZEROBYTES | | ------------------ | | | | 60| 252| (crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES + \ | | | | ------------------ | | | | | | 55| 252|#define crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES 16U | | | | ------------------ | | | | 61| 252| crypto_secretbox_xsalsa20poly1305_MACBYTES) | | | | ------------------ | | | | | | 23| 252|#define crypto_secretbox_xsalsa20poly1305_MACBYTES 16U | | | | ------------------ | | ------------------ ------------------ 42| 252| } 43| 9.08k| for (i = 0U; i < mlen0; i++) { ------------------ | Branch (43:18): [True: 8.76k, False: 319] ------------------ 44| 8.76k| block0[i + crypto_secretbox_ZEROBYTES] = m[i]; ------------------ | | 70| 8.76k|#define crypto_secretbox_ZEROBYTES crypto_secretbox_xsalsa20poly1305_ZEROBYTES | | ------------------ | | | | 60| 8.76k| (crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES + \ | | | | ------------------ | | | | | | 55| 8.76k|#define crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES 16U | | | | ------------------ | | | | 61| 8.76k| crypto_secretbox_xsalsa20poly1305_MACBYTES) | | | | ------------------ | | | | | | 23| 8.76k|#define crypto_secretbox_xsalsa20poly1305_MACBYTES 16U | | | | ------------------ | | ------------------ ------------------ 45| 8.76k| } 46| 319| crypto_stream_salsa20_xor(block0, block0, 47| 319| mlen0 + crypto_secretbox_ZEROBYTES, ------------------ | | 70| 319|#define crypto_secretbox_ZEROBYTES crypto_secretbox_xsalsa20poly1305_ZEROBYTES | | ------------------ | | | | 60| 319| (crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES + \ | | | | ------------------ | | | | | | 55| 319|#define crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES 16U | | | | ------------------ | | | | 61| 319| crypto_secretbox_xsalsa20poly1305_MACBYTES) | | | | ------------------ | | | | | | 23| 319|#define crypto_secretbox_xsalsa20poly1305_MACBYTES 16U | | | | ------------------ | | ------------------ ------------------ 48| 319| n + 16, subkey); 49| 319| COMPILER_ASSERT(crypto_secretbox_ZEROBYTES >= ------------------ | | 23| 319|#define COMPILER_ASSERT(X) (void) sizeof(char[(X) ? 1 : -1]) ------------------ 50| 319| crypto_onetimeauth_poly1305_KEYBYTES); 51| 319| crypto_onetimeauth_poly1305_init(&state, block0); 52| | 53| 9.08k| for (i = 0U; i < mlen0; i++) { ------------------ | Branch (53:18): [True: 8.76k, False: 319] ------------------ 54| 8.76k| c[i] = block0[crypto_secretbox_ZEROBYTES + i]; ------------------ | | 70| 8.76k|#define crypto_secretbox_ZEROBYTES crypto_secretbox_xsalsa20poly1305_ZEROBYTES | | ------------------ | | | | 60| 8.76k| (crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES + \ | | | | ------------------ | | | | | | 55| 8.76k|#define crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES 16U | | | | ------------------ | | | | 61| 8.76k| crypto_secretbox_xsalsa20poly1305_MACBYTES) | | | | ------------------ | | | | | | 23| 8.76k|#define crypto_secretbox_xsalsa20poly1305_MACBYTES 16U | | | | ------------------ | | ------------------ ------------------ 55| 8.76k| } 56| 319| sodium_memzero(block0, sizeof block0); 57| 319| if (mlen > mlen0) { ------------------ | Branch (57:9): [True: 252, False: 67] ------------------ 58| 252| crypto_stream_salsa20_xor_ic(c + mlen0, m + mlen0, mlen - mlen0, 59| 252| n + 16, 1U, subkey); 60| 252| } 61| 319| sodium_memzero(subkey, sizeof subkey); 62| | 63| 319| crypto_onetimeauth_poly1305_update(&state, c, mlen); 64| 319| crypto_onetimeauth_poly1305_final(&state, mac); 65| 319| sodium_memzero(&state, sizeof state); 66| | 67| 319| return 0; 68| 319|} crypto_secretbox_easy: 74| 319|{ 75| 319| if (mlen > crypto_secretbox_MESSAGEBYTES_MAX) { ------------------ | | 32| 319|#define crypto_secretbox_MESSAGEBYTES_MAX crypto_secretbox_xsalsa20poly1305_MESSAGEBYTES_MAX | | ------------------ | | | | 29| 319| (crypto_stream_xsalsa20_MESSAGEBYTES_MAX - crypto_secretbox_xsalsa20poly1305_MACBYTES) | | | | ------------------ | | | | | | 31| 319|#define crypto_stream_xsalsa20_MESSAGEBYTES_MAX SODIUM_SIZE_MAX | | | | | | ------------------ | | | | | | | | 55| 319|#define SODIUM_SIZE_MAX SODIUM_MIN(UINT64_MAX, SIZE_MAX) | | | | | | | | ------------------ | | | | | | | | | | 54| 319|#define SODIUM_MIN(A, B) ((A) < (B) ? (A) : (B)) | | | | | | | | | | ------------------ | | | | | | | | | | | Branch (54:27): [Folded - Ignored] | | | | | | | | | | ------------------ | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | (crypto_stream_xsalsa20_MESSAGEBYTES_MAX - crypto_secretbox_xsalsa20poly1305_MACBYTES) | | | | ------------------ | | | | | | 23| 319|#define crypto_secretbox_xsalsa20poly1305_MACBYTES 16U | | | | ------------------ | | ------------------ ------------------ | Branch (75:9): [True: 0, False: 319] ------------------ 76| 0| sodium_misuse(); 77| 0| } 78| 319| return crypto_secretbox_detached(c + crypto_secretbox_MACBYTES, ------------------ | | 24| 319|#define crypto_secretbox_MACBYTES crypto_secretbox_xsalsa20poly1305_MACBYTES | | ------------------ | | | | 23| 319|#define crypto_secretbox_xsalsa20poly1305_MACBYTES 16U | | ------------------ ------------------ 79| 319| c, m, mlen, n, k); 80| 319|} crypto_secretbox_open_detached: 88| 319|{ 89| 319| unsigned char block0[64U]; 90| 319| unsigned char subkey[crypto_stream_salsa20_KEYBYTES]; 91| 319| unsigned long long i; 92| 319| unsigned long long mlen0; 93| | 94| 319| crypto_core_hsalsa20(subkey, n, k, NULL); 95| 319| crypto_stream_salsa20(block0, crypto_stream_salsa20_KEYBYTES, ------------------ | | 23| 319|#define crypto_stream_salsa20_KEYBYTES 32U ------------------ 96| 319| n + 16, subkey); 97| 319| if (crypto_onetimeauth_poly1305_verify(mac, c, clen, block0) != 0) { ------------------ | Branch (97:9): [True: 0, False: 319] ------------------ 98| 0| sodium_memzero(subkey, sizeof subkey); 99| 0| return -1; 100| 0| } 101| 319| if (m == NULL) { ------------------ | Branch (101:9): [True: 0, False: 319] ------------------ 102| 0| return 0; 103| 0| } 104| 319| if (((uintptr_t) c > (uintptr_t) m && ------------------ | Branch (104:10): [True: 71, False: 248] ------------------ 105| 319| (uintptr_t) c - (uintptr_t) m < clen) || ------------------ | Branch (105:10): [True: 0, False: 71] ------------------ 106| 319| ((uintptr_t) m > (uintptr_t) c && ------------------ | Branch (106:10): [True: 248, False: 71] ------------------ 107| 319| (uintptr_t) m - (uintptr_t) c < clen)) { /* LCOV_EXCL_LINE */ ------------------ | Branch (107:10): [True: 0, False: 248] ------------------ 108| 0| memmove(m, c, clen); 109| 0| c = m; 110| 0| } 111| 319| mlen0 = clen; 112| 319| if (mlen0 > 64U - crypto_secretbox_ZEROBYTES) { ------------------ | | 70| 319|#define crypto_secretbox_ZEROBYTES crypto_secretbox_xsalsa20poly1305_ZEROBYTES | | ------------------ | | | | 60| 319| (crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES + \ | | | | ------------------ | | | | | | 55| 319|#define crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES 16U | | | | ------------------ | | | | 61| 319| crypto_secretbox_xsalsa20poly1305_MACBYTES) | | | | ------------------ | | | | | | 23| 319|#define crypto_secretbox_xsalsa20poly1305_MACBYTES 16U | | | | ------------------ | | ------------------ ------------------ | Branch (112:9): [True: 252, False: 67] ------------------ 113| 252| mlen0 = 64U - crypto_secretbox_ZEROBYTES; ------------------ | | 70| 252|#define crypto_secretbox_ZEROBYTES crypto_secretbox_xsalsa20poly1305_ZEROBYTES | | ------------------ | | | | 60| 252| (crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES + \ | | | | ------------------ | | | | | | 55| 252|#define crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES 16U | | | | ------------------ | | | | 61| 252| crypto_secretbox_xsalsa20poly1305_MACBYTES) | | | | ------------------ | | | | | | 23| 252|#define crypto_secretbox_xsalsa20poly1305_MACBYTES 16U | | | | ------------------ | | ------------------ ------------------ 114| 252| } 115| 9.08k| for (i = 0U; i < mlen0; i++) { ------------------ | Branch (115:18): [True: 8.76k, False: 319] ------------------ 116| 8.76k| block0[crypto_secretbox_ZEROBYTES + i] = c[i]; ------------------ | | 70| 8.76k|#define crypto_secretbox_ZEROBYTES crypto_secretbox_xsalsa20poly1305_ZEROBYTES | | ------------------ | | | | 60| 8.76k| (crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES + \ | | | | ------------------ | | | | | | 55| 8.76k|#define crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES 16U | | | | ------------------ | | | | 61| 8.76k| crypto_secretbox_xsalsa20poly1305_MACBYTES) | | | | ------------------ | | | | | | 23| 8.76k|#define crypto_secretbox_xsalsa20poly1305_MACBYTES 16U | | | | ------------------ | | ------------------ ------------------ 117| 8.76k| } 118| 319| crypto_stream_salsa20_xor(block0, block0, 119| 319| crypto_secretbox_ZEROBYTES + mlen0, ------------------ | | 70| 319|#define crypto_secretbox_ZEROBYTES crypto_secretbox_xsalsa20poly1305_ZEROBYTES | | ------------------ | | | | 60| 319| (crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES + \ | | | | ------------------ | | | | | | 55| 319|#define crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES 16U | | | | ------------------ | | | | 61| 319| crypto_secretbox_xsalsa20poly1305_MACBYTES) | | | | ------------------ | | | | | | 23| 319|#define crypto_secretbox_xsalsa20poly1305_MACBYTES 16U | | | | ------------------ | | ------------------ ------------------ 120| 319| n + 16, subkey); 121| 9.08k| for (i = 0U; i < mlen0; i++) { ------------------ | Branch (121:18): [True: 8.76k, False: 319] ------------------ 122| 8.76k| m[i] = block0[i + crypto_secretbox_ZEROBYTES]; ------------------ | | 70| 8.76k|#define crypto_secretbox_ZEROBYTES crypto_secretbox_xsalsa20poly1305_ZEROBYTES | | ------------------ | | | | 60| 8.76k| (crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES + \ | | | | ------------------ | | | | | | 55| 8.76k|#define crypto_secretbox_xsalsa20poly1305_BOXZEROBYTES 16U | | | | ------------------ | | | | 61| 8.76k| crypto_secretbox_xsalsa20poly1305_MACBYTES) | | | | ------------------ | | | | | | 23| 8.76k|#define crypto_secretbox_xsalsa20poly1305_MACBYTES 16U | | | | ------------------ | | ------------------ ------------------ 123| 8.76k| } 124| 319| sodium_memzero(block0, sizeof block0); 125| 319| if (clen > mlen0) { ------------------ | Branch (125:9): [True: 252, False: 67] ------------------ 126| 252| crypto_stream_salsa20_xor_ic(m + mlen0, c + mlen0, clen - mlen0, 127| 252| n + 16, 1U, subkey); 128| 252| } 129| 319| sodium_memzero(subkey, sizeof subkey); 130| | 131| 319| return 0; 132| 319|} crypto_secretbox_open_easy: 138| 319|{ 139| 319| if (clen < crypto_secretbox_MACBYTES) { ------------------ | | 24| 319|#define crypto_secretbox_MACBYTES crypto_secretbox_xsalsa20poly1305_MACBYTES | | ------------------ | | | | 23| 319|#define crypto_secretbox_xsalsa20poly1305_MACBYTES 16U | | ------------------ ------------------ | Branch (139:9): [True: 0, False: 319] ------------------ 140| 0| return -1; 141| 0| } 142| 319| return crypto_secretbox_open_detached(m, c + crypto_secretbox_MACBYTES, c, ------------------ | | 24| 319|#define crypto_secretbox_MACBYTES crypto_secretbox_xsalsa20poly1305_MACBYTES | | ------------------ | | | | 23| 319|#define crypto_secretbox_xsalsa20poly1305_MACBYTES 16U | | ------------------ ------------------ 143| 319| clen - crypto_secretbox_MACBYTES, ------------------ | | 24| 319|#define crypto_secretbox_MACBYTES crypto_secretbox_xsalsa20poly1305_MACBYTES | | ------------------ | | | | 23| 319|#define crypto_secretbox_xsalsa20poly1305_MACBYTES 16U | | ------------------ ------------------ 144| 319| n, k); 145| 319|} chacha20_dolbeau-avx2.c:chacha_keysetup: 38| 638|{ 39| 638| ctx->input[0] = 0x61707865; 40| 638| ctx->input[1] = 0x3320646e; 41| 638| ctx->input[2] = 0x79622d32; 42| 638| ctx->input[3] = 0x6b206574; 43| 638| ctx->input[4] = LOAD32_LE(k + 0); ------------------ | | 100| 638|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 44| 638| ctx->input[5] = LOAD32_LE(k + 4); ------------------ | | 100| 638|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 45| 638| ctx->input[6] = LOAD32_LE(k + 8); ------------------ | | 100| 638|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 46| 638| ctx->input[7] = LOAD32_LE(k + 12); ------------------ | | 100| 638|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 47| 638| ctx->input[8] = LOAD32_LE(k + 16); ------------------ | | 100| 638|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 48| 638| ctx->input[9] = LOAD32_LE(k + 20); ------------------ | | 100| 638|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 49| 638| ctx->input[10] = LOAD32_LE(k + 24); ------------------ | | 100| 638|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 50| 638| ctx->input[11] = LOAD32_LE(k + 28); ------------------ | | 100| 638|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 51| 638|} chacha20_dolbeau-avx2.c:chacha20_encrypt_bytes: 74| 638|{ 75| 638| uint32_t * const x = &ctx->input[0]; 76| | 77| 638| if (!bytes) { ------------------ | Branch (77:9): [True: 0, False: 638] ------------------ 78| 0| return; /* LCOV_EXCL_LINE */ 79| 0| } 80| 638|# include "u8.h" ------------------ | | 1| 638| | | 2| 638|#define VEC8_ROT(A, IMM) \ | | 3| 638| _mm256_or_si256(_mm256_slli_epi32(A, IMM), _mm256_srli_epi32(A, (32 - IMM))) | | 4| | | | 5| |/* same, but replace 2 of the shift/shift/or "rotation" by byte shuffles (8 & | | 6| | * 16) (better) */ | | 7| 638|#define VEC8_QUARTERROUND_SHUFFLE(A, B, C, D) \ | | 8| 638| x_##A = _mm256_add_epi32(x_##A, x_##B); \ | | 9| 638| t_##A = _mm256_xor_si256(x_##D, x_##A); \ | | 10| 638| x_##D = _mm256_shuffle_epi8(t_##A, rot16); \ | | 11| 638| x_##C = _mm256_add_epi32(x_##C, x_##D); \ | | 12| 638| t_##C = _mm256_xor_si256(x_##B, x_##C); \ | | 13| 638| x_##B = VEC8_ROT(t_##C, 12); \ | | 14| 638| x_##A = _mm256_add_epi32(x_##A, x_##B); \ | | 15| 638| t_##A = _mm256_xor_si256(x_##D, x_##A); \ | | 16| 638| x_##D = _mm256_shuffle_epi8(t_##A, rot8); \ | | 17| 638| x_##C = _mm256_add_epi32(x_##C, x_##D); \ | | 18| 638| t_##C = _mm256_xor_si256(x_##B, x_##C); \ | | 19| 638| x_##B = VEC8_ROT(t_##C, 7) | | 20| | | | 21| 638|#define VEC8_QUARTERROUND(A, B, C, D) VEC8_QUARTERROUND_SHUFFLE(A, B, C, D) | | 22| | | | 23| 638|#define VEC8_LINE1(A, B, C, D) \ | | 24| 638| x_##A = _mm256_add_epi32(x_##A, x_##B); \ | | 25| 638| x_##D = _mm256_shuffle_epi8(_mm256_xor_si256(x_##D, x_##A), rot16) | | 26| 638|#define VEC8_LINE2(A, B, C, D) \ | | 27| 638| x_##C = _mm256_add_epi32(x_##C, x_##D); \ | | 28| 638| x_##B = VEC8_ROT(_mm256_xor_si256(x_##B, x_##C), 12) | | 29| 638|#define VEC8_LINE3(A, B, C, D) \ | | 30| 638| x_##A = _mm256_add_epi32(x_##A, x_##B); \ | | 31| 638| x_##D = _mm256_shuffle_epi8(_mm256_xor_si256(x_##D, x_##A), rot8) | | 32| 638|#define VEC8_LINE4(A, B, C, D) \ | | 33| 638| x_##C = _mm256_add_epi32(x_##C, x_##D); \ | | 34| 638| x_##B = VEC8_ROT(_mm256_xor_si256(x_##B, x_##C), 7) | | 35| | | | 36| 638|#define VEC8_ROUND_SEQ(A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, B4, \ | | 37| 638| C4, D4) \ | | 38| 638| VEC8_LINE1(A1, B1, C1, D1); \ | | 39| 638| VEC8_LINE1(A2, B2, C2, D2); \ | | 40| 638| VEC8_LINE1(A3, B3, C3, D3); \ | | 41| 638| VEC8_LINE1(A4, B4, C4, D4); \ | | 42| 638| VEC8_LINE2(A1, B1, C1, D1); \ | | 43| 638| VEC8_LINE2(A2, B2, C2, D2); \ | | 44| 638| VEC8_LINE2(A3, B3, C3, D3); \ | | 45| 638| VEC8_LINE2(A4, B4, C4, D4); \ | | 46| 638| VEC8_LINE3(A1, B1, C1, D1); \ | | 47| 638| VEC8_LINE3(A2, B2, C2, D2); \ | | 48| 638| VEC8_LINE3(A3, B3, C3, D3); \ | | 49| 638| VEC8_LINE3(A4, B4, C4, D4); \ | | 50| 638| VEC8_LINE4(A1, B1, C1, D1); \ | | 51| 638| VEC8_LINE4(A2, B2, C2, D2); \ | | 52| 638| VEC8_LINE4(A3, B3, C3, D3); \ | | 53| 638| VEC8_LINE4(A4, B4, C4, D4) | | 54| | | | 55| 638|#define VEC8_ROUND_HALF(A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, \ | | 56| 638| B4, C4, D4) \ | | 57| 638| VEC8_LINE1(A1, B1, C1, D1); \ | | 58| 638| VEC8_LINE1(A2, B2, C2, D2); \ | | 59| 638| VEC8_LINE2(A1, B1, C1, D1); \ | | 60| 638| VEC8_LINE2(A2, B2, C2, D2); \ | | 61| 638| VEC8_LINE3(A1, B1, C1, D1); \ | | 62| 638| VEC8_LINE3(A2, B2, C2, D2); \ | | 63| 638| VEC8_LINE4(A1, B1, C1, D1); \ | | 64| 638| VEC8_LINE4(A2, B2, C2, D2); \ | | 65| 638| VEC8_LINE1(A3, B3, C3, D3); \ | | 66| 638| VEC8_LINE1(A4, B4, C4, D4); \ | | 67| 638| VEC8_LINE2(A3, B3, C3, D3); \ | | 68| 638| VEC8_LINE2(A4, B4, C4, D4); \ | | 69| 638| VEC8_LINE3(A3, B3, C3, D3); \ | | 70| 638| VEC8_LINE3(A4, B4, C4, D4); \ | | 71| 638| VEC8_LINE4(A3, B3, C3, D3); \ | | 72| 638| VEC8_LINE4(A4, B4, C4, D4) | | 73| | | | 74| 638|#define VEC8_ROUND_HALFANDHALF(A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, \ | | 75| 638| A4, B4, C4, D4) \ | | 76| 638| VEC8_LINE1(A1, B1, C1, D1); \ | | 77| 638| VEC8_LINE1(A2, B2, C2, D2); \ | | 78| 638| VEC8_LINE2(A1, B1, C1, D1); \ | | 79| 638| VEC8_LINE2(A2, B2, C2, D2); \ | | 80| 638| VEC8_LINE1(A3, B3, C3, D3); \ | | 81| 638| VEC8_LINE1(A4, B4, C4, D4); \ | | 82| 638| VEC8_LINE2(A3, B3, C3, D3); \ | | 83| 638| VEC8_LINE2(A4, B4, C4, D4); \ | | 84| 638| VEC8_LINE3(A1, B1, C1, D1); \ | | 85| 638| VEC8_LINE3(A2, B2, C2, D2); \ | | 86| 638| VEC8_LINE4(A1, B1, C1, D1); \ | | 87| 638| VEC8_LINE4(A2, B2, C2, D2); \ | | 88| 638| VEC8_LINE3(A3, B3, C3, D3); \ | | 89| 638| VEC8_LINE3(A4, B4, C4, D4); \ | | 90| 638| VEC8_LINE4(A3, B3, C3, D3); \ | | 91| 638| VEC8_LINE4(A4, B4, C4, D4) | | 92| | | | 93| 638|#define VEC8_ROUND(A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, B4, C4, \ | | 94| 638| D4) \ | | 95| 638| VEC8_ROUND_SEQ(A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, B4, C4, \ | | 96| 638| D4) | | 97| | | | 98| 638|if (bytes >= 512) { | | ------------------ | | | Branch (98:5): [True: 0, False: 638] | | ------------------ | | 99| | /* constant for shuffling bytes (replacing multiple-of-8 rotates) */ | | 100| 0| __m256i rot16 = | | 101| 0| _mm256_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2, | | 102| 0| 13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); | | 103| 0| __m256i rot8 = | | 104| 0| _mm256_set_epi8(14, 13, 12, 15, 10, 9, 8, 11, 6, 5, 4, 7, 2, 1, 0, 3, | | 105| 0| 14, 13, 12, 15, 10, 9, 8, 11, 6, 5, 4, 7, 2, 1, 0, 3); | | 106| 0| uint32_t in12, in13; | | 107| | | | 108| | /* the naive way seems as fast (if not a bit faster) than the vector way */ | | 109| 0| __m256i x_0 = _mm256_set1_epi32(x[0]); | | 110| 0| __m256i x_1 = _mm256_set1_epi32(x[1]); | | 111| 0| __m256i x_2 = _mm256_set1_epi32(x[2]); | | 112| 0| __m256i x_3 = _mm256_set1_epi32(x[3]); | | 113| 0| __m256i x_4 = _mm256_set1_epi32(x[4]); | | 114| 0| __m256i x_5 = _mm256_set1_epi32(x[5]); | | 115| 0| __m256i x_6 = _mm256_set1_epi32(x[6]); | | 116| 0| __m256i x_7 = _mm256_set1_epi32(x[7]); | | 117| 0| __m256i x_8 = _mm256_set1_epi32(x[8]); | | 118| 0| __m256i x_9 = _mm256_set1_epi32(x[9]); | | 119| 0| __m256i x_10 = _mm256_set1_epi32(x[10]); | | 120| 0| __m256i x_11 = _mm256_set1_epi32(x[11]); | | 121| 0| __m256i x_12; | | 122| 0| __m256i x_13; | | 123| 0| __m256i x_14 = _mm256_set1_epi32(x[14]); | | 124| 0| __m256i x_15 = _mm256_set1_epi32(x[15]); | | 125| | | | 126| 0| __m256i orig0 = x_0; | | 127| 0| __m256i orig1 = x_1; | | 128| 0| __m256i orig2 = x_2; | | 129| 0| __m256i orig3 = x_3; | | 130| 0| __m256i orig4 = x_4; | | 131| 0| __m256i orig5 = x_5; | | 132| 0| __m256i orig6 = x_6; | | 133| 0| __m256i orig7 = x_7; | | 134| 0| __m256i orig8 = x_8; | | 135| 0| __m256i orig9 = x_9; | | 136| 0| __m256i orig10 = x_10; | | 137| 0| __m256i orig11 = x_11; | | 138| 0| __m256i orig12; | | 139| 0| __m256i orig13; | | 140| 0| __m256i orig14 = x_14; | | 141| 0| __m256i orig15 = x_15; | | 142| 0| __m256i t_0, t_1, t_2, t_3, t_4, t_5, t_6, t_7, t_8, t_9, t_10, t_11, t_12, | | 143| 0| t_13, t_14, t_15; | | 144| | | | 145| 0| while (bytes >= 512) { | | ------------------ | | | Branch (145:12): [True: 0, False: 0] | | ------------------ | | 146| 0| const __m256i addv12 = _mm256_set_epi64x(3, 2, 1, 0); | | 147| 0| const __m256i addv13 = _mm256_set_epi64x(7, 6, 5, 4); | | 148| 0| const __m256i permute = _mm256_set_epi32(7, 6, 3, 2, 5, 4, 1, 0); | | 149| 0| __m256i t12, t13; | | 150| | | | 151| 0| uint64_t in1213; | | 152| 0| int i; | | 153| | | | 154| 0| x_0 = orig0; | | 155| 0| x_1 = orig1; | | 156| 0| x_2 = orig2; | | 157| 0| x_3 = orig3; | | 158| 0| x_4 = orig4; | | 159| 0| x_5 = orig5; | | 160| 0| x_6 = orig6; | | 161| 0| x_7 = orig7; | | 162| 0| x_8 = orig8; | | 163| 0| x_9 = orig9; | | 164| 0| x_10 = orig10; | | 165| 0| x_11 = orig11; | | 166| 0| x_14 = orig14; | | 167| 0| x_15 = orig15; | | 168| | | | 169| 0| in12 = x[12]; | | 170| 0| in13 = x[13]; | | 171| 0| in1213 = ((uint64_t) in12) | (((uint64_t) in13) << 32); | | 172| 0| x_12 = x_13 = _mm256_broadcastq_epi64(_mm_cvtsi64_si128(in1213)); | | 173| | | | 174| 0| t12 = _mm256_add_epi64(addv12, x_12); | | 175| 0| t13 = _mm256_add_epi64(addv13, x_13); | | 176| | | | 177| 0| x_12 = _mm256_unpacklo_epi32(t12, t13); | | 178| 0| x_13 = _mm256_unpackhi_epi32(t12, t13); | | 179| | | | 180| 0| t12 = _mm256_unpacklo_epi32(x_12, x_13); | | 181| 0| t13 = _mm256_unpackhi_epi32(x_12, x_13); | | 182| | | | 183| | /* required because unpack* are intra-lane */ | | 184| 0| x_12 = _mm256_permutevar8x32_epi32(t12, permute); | | 185| 0| x_13 = _mm256_permutevar8x32_epi32(t13, permute); | | 186| | | | 187| 0| orig12 = x_12; | | 188| 0| orig13 = x_13; | | 189| | | | 190| 0| in1213 += 8; | | 191| | | | 192| 0| x[12] = in1213 & 0xFFFFFFFF; | | 193| 0| x[13] = (in1213 >> 32) & 0xFFFFFFFF; | | 194| | | | 195| 0| for (i = 0; i < ROUNDS; i += 2) { | | ------------------ | | | | 30| 0|# define ROUNDS 20 | | ------------------ | | | Branch (195:21): [True: 0, False: 0] | | ------------------ | | 196| 0| VEC8_ROUND(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); | | ------------------ | | | | 95| 0| VEC8_ROUND_SEQ(A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, B4, C4, \ | | | | ------------------ | | | | | | 38| 0| VEC8_LINE1(A1, B1, C1, D1); \ | | | | | | ------------------ | | | | | | | | 24| 0| x_##A = _mm256_add_epi32(x_##A, x_##B); \ | | | | | | | | 25| 0| x_##D = _mm256_shuffle_epi8(_mm256_xor_si256(x_##D, x_##A), rot16) | | | | | | ------------------ | | | | | | 39| 0| VEC8_LINE1(A2, B2, C2, D2); \ | | | | | | ------------------ | | | | | | | | 24| 0| x_##A = _mm256_add_epi32(x_##A, x_##B); \ | | | | | | | | 25| 0| x_##D = _mm256_shuffle_epi8(_mm256_xor_si256(x_##D, x_##A), rot16) | | | | | | ------------------ | | | | | | 40| 0| VEC8_LINE1(A3, B3, C3, D3); \ | | | | | | ------------------ | | | | | | | | 24| 0| x_##A = _mm256_add_epi32(x_##A, x_##B); \ | | | | | | | | 25| 0| x_##D = _mm256_shuffle_epi8(_mm256_xor_si256(x_##D, x_##A), rot16) | | | | | | ------------------ | | | | | | 41| 0| VEC8_LINE1(A4, B4, C4, D4); \ | | | | | | ------------------ | | | | | | | | 24| 0| x_##A = _mm256_add_epi32(x_##A, x_##B); \ | | | | | | | | 25| 0| x_##D = _mm256_shuffle_epi8(_mm256_xor_si256(x_##D, x_##A), rot16) | | | | | | ------------------ | | | | | | 42| 0| VEC8_LINE2(A1, B1, C1, D1); \ | | | | | | ------------------ | | | | | | | | 27| 0| x_##C = _mm256_add_epi32(x_##C, x_##D); \ | | | | | | | | 28| 0| x_##B = VEC8_ROT(_mm256_xor_si256(x_##B, x_##C), 12) | | | | | | | | ------------------ | | | | | | | | | | 3| 0| _mm256_or_si256(_mm256_slli_epi32(A, IMM), _mm256_srli_epi32(A, (32 - IMM))) | | | | | | | | ------------------ | | | | | | ------------------ | | | | | | 43| 0| VEC8_LINE2(A2, B2, C2, D2); \ | | | | | | ------------------ | | | | | | | | 27| 0| x_##C = _mm256_add_epi32(x_##C, x_##D); \ | | | | | | | | 28| 0| x_##B = VEC8_ROT(_mm256_xor_si256(x_##B, x_##C), 12) | | | | | | | | ------------------ | | | | | | | | | | 3| 0| _mm256_or_si256(_mm256_slli_epi32(A, IMM), _mm256_srli_epi32(A, (32 - IMM))) | | | | | | | | ------------------ | | | | | | ------------------ | | | | | | 44| 0| VEC8_LINE2(A3, B3, C3, D3); \ | | | | | | ------------------ | | | | | | | | 27| 0| x_##C = _mm256_add_epi32(x_##C, x_##D); \ | | | | | | | | 28| 0| x_##B = VEC8_ROT(_mm256_xor_si256(x_##B, x_##C), 12) | | | | | | | | ------------------ | | | | | | | | | | 3| 0| _mm256_or_si256(_mm256_slli_epi32(A, IMM), _mm256_srli_epi32(A, (32 - IMM))) | | | | | | | | ------------------ | | | | | | ------------------ | | | | | | 45| 0| VEC8_LINE2(A4, B4, C4, D4); \ | | | | | | ------------------ | | | | | | | | 27| 0| x_##C = _mm256_add_epi32(x_##C, x_##D); \ | | | | | | | | 28| 0| x_##B = VEC8_ROT(_mm256_xor_si256(x_##B, x_##C), 12) | | | | | | | | ------------------ | | | | | | | | | | 3| 0| _mm256_or_si256(_mm256_slli_epi32(A, IMM), _mm256_srli_epi32(A, (32 - IMM))) | | | | | | | | ------------------ | | | | | | ------------------ | | | | | | 46| 0| VEC8_LINE3(A1, B1, C1, D1); \ | | | | | | ------------------ | | | | | | | | 30| 0| x_##A = _mm256_add_epi32(x_##A, x_##B); \ | | | | | | | | 31| 0| x_##D = _mm256_shuffle_epi8(_mm256_xor_si256(x_##D, x_##A), rot8) | | | | | | ------------------ | | | | | | 47| 0| VEC8_LINE3(A2, B2, C2, D2); \ | | | | | | ------------------ | | | | | | | | 30| 0| x_##A = _mm256_add_epi32(x_##A, x_##B); \ | | | | | | | | 31| 0| x_##D = _mm256_shuffle_epi8(_mm256_xor_si256(x_##D, x_##A), rot8) | | | | | | ------------------ | | | | | | 48| 0| VEC8_LINE3(A3, B3, C3, D3); \ | | | | | | ------------------ | | | | | | | | 30| 0| x_##A = _mm256_add_epi32(x_##A, x_##B); \ | | | | | | | | 31| 0| x_##D = _mm256_shuffle_epi8(_mm256_xor_si256(x_##D, x_##A), rot8) | | | | | | ------------------ | | | | | | 49| 0| VEC8_LINE3(A4, B4, C4, D4); \ | | | | | | ------------------ | | | | | | | | 30| 0| x_##A = _mm256_add_epi32(x_##A, x_##B); \ | | | | | | | | 31| 0| x_##D = _mm256_shuffle_epi8(_mm256_xor_si256(x_##D, x_##A), rot8) | | | | | | ------------------ | | | | | | 50| 0| VEC8_LINE4(A1, B1, C1, D1); \ | | | | | | ------------------ | | | | | | | | 33| 0| x_##C = _mm256_add_epi32(x_##C, x_##D); \ | | | | | | | | 34| 0| x_##B = VEC8_ROT(_mm256_xor_si256(x_##B, x_##C), 7) | | | | | | | | ------------------ | | | | | | | | | | 3| 0| _mm256_or_si256(_mm256_slli_epi32(A, IMM), _mm256_srli_epi32(A, (32 - IMM))) | | | | | | | | ------------------ | | | | | | ------------------ | | | | | | 51| 0| VEC8_LINE4(A2, B2, C2, D2); \ | | | | | | ------------------ | | | | | | | | 33| 0| x_##C = _mm256_add_epi32(x_##C, x_##D); \ | | | | | | | | 34| 0| x_##B = VEC8_ROT(_mm256_xor_si256(x_##B, x_##C), 7) | | | | | | | | ------------------ | | | | | | | | | | 3| 0| _mm256_or_si256(_mm256_slli_epi32(A, IMM), _mm256_srli_epi32(A, (32 - IMM))) | | | | | | | | ------------------ | | | | | | ------------------ | | | | | | 52| 0| VEC8_LINE4(A3, B3, C3, D3); \ | | | | | | ------------------ | | | | | | | | 33| 0| x_##C = _mm256_add_epi32(x_##C, x_##D); \ | | | | | | | | 34| 0| x_##B = VEC8_ROT(_mm256_xor_si256(x_##B, x_##C), 7) | | | | | | | | ------------------ | | | | | | | | | | 3| 0| _mm256_or_si256(_mm256_slli_epi32(A, IMM), _mm256_srli_epi32(A, (32 - IMM))) | | | | | | | | ------------------ | | | | | | ------------------ | | | | | | 53| 0| VEC8_LINE4(A4, B4, C4, D4) | | | | | | ------------------ | | | | | | | | 33| 0| x_##C = _mm256_add_epi32(x_##C, x_##D); \ | | | | | | | | 34| 0| x_##B = VEC8_ROT(_mm256_xor_si256(x_##B, x_##C), 7) | | | | | | | | ------------------ | | | | | | | | | | 3| 0| _mm256_or_si256(_mm256_slli_epi32(A, IMM), _mm256_srli_epi32(A, (32 - IMM))) | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 96| 0| D4) | | ------------------ | | 197| 0| VEC8_ROUND(0, 5, 10, 15, 1, 6, 11, 12, 2, 7, 8, 13, 3, 4, 9, 14); | | ------------------ | | | | 95| 0| VEC8_ROUND_SEQ(A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, B4, C4, \ | | | | ------------------ | | | | | | 38| 0| VEC8_LINE1(A1, B1, C1, D1); \ | | | | | | ------------------ | | | | | | | | 24| 0| x_##A = _mm256_add_epi32(x_##A, x_##B); \ | | | | | | | | 25| 0| x_##D = _mm256_shuffle_epi8(_mm256_xor_si256(x_##D, x_##A), rot16) | | | | | | ------------------ | | | | | | 39| 0| VEC8_LINE1(A2, B2, C2, D2); \ | | | | | | ------------------ | | | | | | | | 24| 0| x_##A = _mm256_add_epi32(x_##A, x_##B); \ | | | | | | | | 25| 0| x_##D = _mm256_shuffle_epi8(_mm256_xor_si256(x_##D, x_##A), rot16) | | | | | | ------------------ | | | | | | 40| 0| VEC8_LINE1(A3, B3, C3, D3); \ | | | | | | ------------------ | | | | | | | | 24| 0| x_##A = _mm256_add_epi32(x_##A, x_##B); \ | | | | | | | | 25| 0| x_##D = _mm256_shuffle_epi8(_mm256_xor_si256(x_##D, x_##A), rot16) | | | | | | ------------------ | | | | | | 41| 0| VEC8_LINE1(A4, B4, C4, D4); \ | | | | | | ------------------ | | | | | | | | 24| 0| x_##A = _mm256_add_epi32(x_##A, x_##B); \ | | | | | | | | 25| 0| x_##D = _mm256_shuffle_epi8(_mm256_xor_si256(x_##D, x_##A), rot16) | | | | | | ------------------ | | | | | | 42| 0| VEC8_LINE2(A1, B1, C1, D1); \ | | | | | | ------------------ | | | | | | | | 27| 0| x_##C = _mm256_add_epi32(x_##C, x_##D); \ | | | | | | | | 28| 0| x_##B = VEC8_ROT(_mm256_xor_si256(x_##B, x_##C), 12) | | | | | | | | ------------------ | | | | | | | | | | 3| 0| _mm256_or_si256(_mm256_slli_epi32(A, IMM), _mm256_srli_epi32(A, (32 - IMM))) | | | | | | | | ------------------ | | | | | | ------------------ | | | | | | 43| 0| VEC8_LINE2(A2, B2, C2, D2); \ | | | | | | ------------------ | | | | | | | | 27| 0| x_##C = _mm256_add_epi32(x_##C, x_##D); \ | | | | | | | | 28| 0| x_##B = VEC8_ROT(_mm256_xor_si256(x_##B, x_##C), 12) | | | | | | | | ------------------ | | | | | | | | | | 3| 0| _mm256_or_si256(_mm256_slli_epi32(A, IMM), _mm256_srli_epi32(A, (32 - IMM))) | | | | | | | | ------------------ | | | | | | ------------------ | | | | | | 44| 0| VEC8_LINE2(A3, B3, C3, D3); \ | | | | | | ------------------ | | | | | | | | 27| 0| x_##C = _mm256_add_epi32(x_##C, x_##D); \ | | | | | | | | 28| 0| x_##B = VEC8_ROT(_mm256_xor_si256(x_##B, x_##C), 12) | | | | | | | | ------------------ | | | | | | | | | | 3| 0| _mm256_or_si256(_mm256_slli_epi32(A, IMM), _mm256_srli_epi32(A, (32 - IMM))) | | | | | | | | ------------------ | | | | | | ------------------ | | | | | | 45| 0| VEC8_LINE2(A4, B4, C4, D4); \ | | | | | | ------------------ | | | | | | | | 27| 0| x_##C = _mm256_add_epi32(x_##C, x_##D); \ | | | | | | | | 28| 0| x_##B = VEC8_ROT(_mm256_xor_si256(x_##B, x_##C), 12) | | | | | | | | ------------------ | | | | | | | | | | 3| 0| _mm256_or_si256(_mm256_slli_epi32(A, IMM), _mm256_srli_epi32(A, (32 - IMM))) | | | | | | | | ------------------ | | | | | | ------------------ | | | | | | 46| 0| VEC8_LINE3(A1, B1, C1, D1); \ | | | | | | ------------------ | | | | | | | | 30| 0| x_##A = _mm256_add_epi32(x_##A, x_##B); \ | | | | | | | | 31| 0| x_##D = _mm256_shuffle_epi8(_mm256_xor_si256(x_##D, x_##A), rot8) | | | | | | ------------------ | | | | | | 47| 0| VEC8_LINE3(A2, B2, C2, D2); \ | | | | | | ------------------ | | | | | | | | 30| 0| x_##A = _mm256_add_epi32(x_##A, x_##B); \ | | | | | | | | 31| 0| x_##D = _mm256_shuffle_epi8(_mm256_xor_si256(x_##D, x_##A), rot8) | | | | | | ------------------ | | | | | | 48| 0| VEC8_LINE3(A3, B3, C3, D3); \ | | | | | | ------------------ | | | | | | | | 30| 0| x_##A = _mm256_add_epi32(x_##A, x_##B); \ | | | | | | | | 31| 0| x_##D = _mm256_shuffle_epi8(_mm256_xor_si256(x_##D, x_##A), rot8) | | | | | | ------------------ | | | | | | 49| 0| VEC8_LINE3(A4, B4, C4, D4); \ | | | | | | ------------------ | | | | | | | | 30| 0| x_##A = _mm256_add_epi32(x_##A, x_##B); \ | | | | | | | | 31| 0| x_##D = _mm256_shuffle_epi8(_mm256_xor_si256(x_##D, x_##A), rot8) | | | | | | ------------------ | | | | | | 50| 0| VEC8_LINE4(A1, B1, C1, D1); \ | | | | | | ------------------ | | | | | | | | 33| 0| x_##C = _mm256_add_epi32(x_##C, x_##D); \ | | | | | | | | 34| 0| x_##B = VEC8_ROT(_mm256_xor_si256(x_##B, x_##C), 7) | | | | | | | | ------------------ | | | | | | | | | | 3| 0| _mm256_or_si256(_mm256_slli_epi32(A, IMM), _mm256_srli_epi32(A, (32 - IMM))) | | | | | | | | ------------------ | | | | | | ------------------ | | | | | | 51| 0| VEC8_LINE4(A2, B2, C2, D2); \ | | | | | | ------------------ | | | | | | | | 33| 0| x_##C = _mm256_add_epi32(x_##C, x_##D); \ | | | | | | | | 34| 0| x_##B = VEC8_ROT(_mm256_xor_si256(x_##B, x_##C), 7) | | | | | | | | ------------------ | | | | | | | | | | 3| 0| _mm256_or_si256(_mm256_slli_epi32(A, IMM), _mm256_srli_epi32(A, (32 - IMM))) | | | | | | | | ------------------ | | | | | | ------------------ | | | | | | 52| 0| VEC8_LINE4(A3, B3, C3, D3); \ | | | | | | ------------------ | | | | | | | | 33| 0| x_##C = _mm256_add_epi32(x_##C, x_##D); \ | | | | | | | | 34| 0| x_##B = VEC8_ROT(_mm256_xor_si256(x_##B, x_##C), 7) | | | | | | | | ------------------ | | | | | | | | | | 3| 0| _mm256_or_si256(_mm256_slli_epi32(A, IMM), _mm256_srli_epi32(A, (32 - IMM))) | | | | | | | | ------------------ | | | | | | ------------------ | | | | | | 53| 0| VEC8_LINE4(A4, B4, C4, D4) | | | | | | ------------------ | | | | | | | | 33| 0| x_##C = _mm256_add_epi32(x_##C, x_##D); \ | | | | | | | | 34| 0| x_##B = VEC8_ROT(_mm256_xor_si256(x_##B, x_##C), 7) | | | | | | | | ------------------ | | | | | | | | | | 3| 0| _mm256_or_si256(_mm256_slli_epi32(A, IMM), _mm256_srli_epi32(A, (32 - IMM))) | | | | | | | | ------------------ | | | | | | ------------------ | | | | ------------------ | | | | 96| 0| D4) | | ------------------ | | 198| 0| } | | 199| | | | 200| 0|#define ONEQUAD_TRANSPOSE(A, B, C, D) \ | | 201| 0| { \ | | 202| 0| __m128i t0, t1, t2, t3; \ | | 203| 0| x_##A = _mm256_add_epi32(x_##A, orig##A); \ | | 204| 0| x_##B = _mm256_add_epi32(x_##B, orig##B); \ | | 205| 0| x_##C = _mm256_add_epi32(x_##C, orig##C); \ | | 206| 0| x_##D = _mm256_add_epi32(x_##D, orig##D); \ | | 207| 0| t_##A = _mm256_unpacklo_epi32(x_##A, x_##B); \ | | 208| 0| t_##B = _mm256_unpacklo_epi32(x_##C, x_##D); \ | | 209| 0| t_##C = _mm256_unpackhi_epi32(x_##A, x_##B); \ | | 210| 0| t_##D = _mm256_unpackhi_epi32(x_##C, x_##D); \ | | 211| 0| x_##A = _mm256_unpacklo_epi64(t_##A, t_##B); \ | | 212| 0| x_##B = _mm256_unpackhi_epi64(t_##A, t_##B); \ | | 213| 0| x_##C = _mm256_unpacklo_epi64(t_##C, t_##D); \ | | 214| 0| x_##D = _mm256_unpackhi_epi64(t_##C, t_##D); \ | | 215| 0| t0 = _mm_xor_si128(_mm256_extracti128_si256(x_##A, 0), \ | | 216| 0| _mm_loadu_si128((const __m128i*) (m + 0))); \ | | 217| 0| _mm_storeu_si128((__m128i*) (c + 0), t0); \ | | 218| 0| t1 = _mm_xor_si128(_mm256_extracti128_si256(x_##B, 0), \ | | 219| 0| _mm_loadu_si128((const __m128i*) (m + 64))); \ | | 220| 0| _mm_storeu_si128((__m128i*) (c + 64), t1); \ | | 221| 0| t2 = _mm_xor_si128(_mm256_extracti128_si256(x_##C, 0), \ | | 222| 0| _mm_loadu_si128((const __m128i*) (m + 128))); \ | | 223| 0| _mm_storeu_si128((__m128i*) (c + 128), t2); \ | | 224| 0| t3 = _mm_xor_si128(_mm256_extracti128_si256(x_##D, 0), \ | | 225| 0| _mm_loadu_si128((const __m128i*) (m + 192))); \ | | 226| 0| _mm_storeu_si128((__m128i*) (c + 192), t3); \ | | 227| 0| t0 = _mm_xor_si128(_mm256_extracti128_si256(x_##A, 1), \ | | 228| 0| _mm_loadu_si128((const __m128i*) (m + 256))); \ | | 229| 0| _mm_storeu_si128((__m128i*) (c + 256), t0); \ | | 230| 0| t1 = _mm_xor_si128(_mm256_extracti128_si256(x_##B, 1), \ | | 231| 0| _mm_loadu_si128((const __m128i*) (m + 320))); \ | | 232| 0| _mm_storeu_si128((__m128i*) (c + 320), t1); \ | | 233| 0| t2 = _mm_xor_si128(_mm256_extracti128_si256(x_##C, 1), \ | | 234| 0| _mm_loadu_si128((const __m128i*) (m + 384))); \ | | 235| 0| _mm_storeu_si128((__m128i*) (c + 384), t2); \ | | 236| 0| t3 = _mm_xor_si128(_mm256_extracti128_si256(x_##D, 1), \ | | 237| 0| _mm_loadu_si128((const __m128i*) (m + 448))); \ | | 238| 0| _mm_storeu_si128((__m128i*) (c + 448), t3); \ | | 239| 0| } | | 240| | | | 241| 0|#define ONEQUAD(A, B, C, D) ONEQUAD_TRANSPOSE(A, B, C, D) | | 242| | | | 243| 0|#define ONEQUAD_UNPCK(A, B, C, D) \ | | 244| 0| { \ | | 245| 0| x_##A = _mm256_add_epi32(x_##A, orig##A); \ | | 246| 0| x_##B = _mm256_add_epi32(x_##B, orig##B); \ | | 247| 0| x_##C = _mm256_add_epi32(x_##C, orig##C); \ | | 248| 0| x_##D = _mm256_add_epi32(x_##D, orig##D); \ | | 249| 0| t_##A = _mm256_unpacklo_epi32(x_##A, x_##B); \ | | 250| 0| t_##B = _mm256_unpacklo_epi32(x_##C, x_##D); \ | | 251| 0| t_##C = _mm256_unpackhi_epi32(x_##A, x_##B); \ | | 252| 0| t_##D = _mm256_unpackhi_epi32(x_##C, x_##D); \ | | 253| 0| x_##A = _mm256_unpacklo_epi64(t_##A, t_##B); \ | | 254| 0| x_##B = _mm256_unpackhi_epi64(t_##A, t_##B); \ | | 255| 0| x_##C = _mm256_unpacklo_epi64(t_##C, t_##D); \ | | 256| 0| x_##D = _mm256_unpackhi_epi64(t_##C, t_##D); \ | | 257| 0| } | | 258| | | | 259| 0|#define ONEOCTO(A, B, C, D, A2, B2, C2, D2) \ | | 260| 0| { \ | | 261| 0| ONEQUAD_UNPCK(A, B, C, D); \ | | 262| 0| ONEQUAD_UNPCK(A2, B2, C2, D2); \ | | 263| 0| t_##A = _mm256_permute2x128_si256(x_##A, x_##A2, 0x20); \ | | 264| 0| t_##A2 = _mm256_permute2x128_si256(x_##A, x_##A2, 0x31); \ | | 265| 0| t_##B = _mm256_permute2x128_si256(x_##B, x_##B2, 0x20); \ | | 266| 0| t_##B2 = _mm256_permute2x128_si256(x_##B, x_##B2, 0x31); \ | | 267| 0| t_##C = _mm256_permute2x128_si256(x_##C, x_##C2, 0x20); \ | | 268| 0| t_##C2 = _mm256_permute2x128_si256(x_##C, x_##C2, 0x31); \ | | 269| 0| t_##D = _mm256_permute2x128_si256(x_##D, x_##D2, 0x20); \ | | 270| 0| t_##D2 = _mm256_permute2x128_si256(x_##D, x_##D2, 0x31); \ | | 271| 0| t_##A = _mm256_xor_si256( \ | | 272| 0| t_##A, _mm256_loadu_si256((const __m256i*) (m + 0))); \ | | 273| 0| t_##B = _mm256_xor_si256( \ | | 274| 0| t_##B, _mm256_loadu_si256((const __m256i*) (m + 64))); \ | | 275| 0| t_##C = _mm256_xor_si256( \ | | 276| 0| t_##C, _mm256_loadu_si256((const __m256i*) (m + 128))); \ | | 277| 0| t_##D = _mm256_xor_si256( \ | | 278| 0| t_##D, _mm256_loadu_si256((const __m256i*) (m + 192))); \ | | 279| 0| t_##A2 = _mm256_xor_si256( \ | | 280| 0| t_##A2, _mm256_loadu_si256((const __m256i*) (m + 256))); \ | | 281| 0| t_##B2 = _mm256_xor_si256( \ | | 282| 0| t_##B2, _mm256_loadu_si256((const __m256i*) (m + 320))); \ | | 283| 0| t_##C2 = _mm256_xor_si256( \ | | 284| 0| t_##C2, _mm256_loadu_si256((const __m256i*) (m + 384))); \ | | 285| 0| t_##D2 = _mm256_xor_si256( \ | | 286| 0| t_##D2, _mm256_loadu_si256((const __m256i*) (m + 448))); \ | | 287| 0| _mm256_storeu_si256((__m256i*) (c + 0), t_##A); \ | | 288| 0| _mm256_storeu_si256((__m256i*) (c + 64), t_##B); \ | | 289| 0| _mm256_storeu_si256((__m256i*) (c + 128), t_##C); \ | | 290| 0| _mm256_storeu_si256((__m256i*) (c + 192), t_##D); \ | | 291| 0| _mm256_storeu_si256((__m256i*) (c + 256), t_##A2); \ | | 292| 0| _mm256_storeu_si256((__m256i*) (c + 320), t_##B2); \ | | 293| 0| _mm256_storeu_si256((__m256i*) (c + 384), t_##C2); \ | | 294| 0| _mm256_storeu_si256((__m256i*) (c + 448), t_##D2); \ | | 295| 0| } | | 296| | | | 297| 0| ONEOCTO(0, 1, 2, 3, 4, 5, 6, 7); | | ------------------ | | | | 260| 0| { \ | | | | 261| 0| ONEQUAD_UNPCK(A, B, C, D); \ | | | | ------------------ | | | | | | 244| 0| { \ | | | | | | 245| 0| x_##A = _mm256_add_epi32(x_##A, orig##A); \ | | | | | | 246| 0| x_##B = _mm256_add_epi32(x_##B, orig##B); \ | | | | | | 247| 0| x_##C = _mm256_add_epi32(x_##C, orig##C); \ | | | | | | 248| 0| x_##D = _mm256_add_epi32(x_##D, orig##D); \ | | | | | | 249| 0| t_##A = _mm256_unpacklo_epi32(x_##A, x_##B); \ | | | | | | 250| 0| t_##B = _mm256_unpacklo_epi32(x_##C, x_##D); \ | | | | | | 251| 0| t_##C = _mm256_unpackhi_epi32(x_##A, x_##B); \ | | | | | | 252| 0| t_##D = _mm256_unpackhi_epi32(x_##C, x_##D); \ | | | | | | 253| 0| x_##A = _mm256_unpacklo_epi64(t_##A, t_##B); \ | | | | | | 254| 0| x_##B = _mm256_unpackhi_epi64(t_##A, t_##B); \ | | | | | | 255| 0| x_##C = _mm256_unpacklo_epi64(t_##C, t_##D); \ | | | | | | 256| 0| x_##D = _mm256_unpackhi_epi64(t_##C, t_##D); \ | | | | | | 257| 0| } | | | | ------------------ | | | | 262| 0| ONEQUAD_UNPCK(A2, B2, C2, D2); \ | | | | ------------------ | | | | | | 244| 0| { \ | | | | | | 245| 0| x_##A = _mm256_add_epi32(x_##A, orig##A); \ | | | | | | 246| 0| x_##B = _mm256_add_epi32(x_##B, orig##B); \ | | | | | | 247| 0| x_##C = _mm256_add_epi32(x_##C, orig##C); \ | | | | | | 248| 0| x_##D = _mm256_add_epi32(x_##D, orig##D); \ | | | | | | 249| 0| t_##A = _mm256_unpacklo_epi32(x_##A, x_##B); \ | | | | | | 250| 0| t_##B = _mm256_unpacklo_epi32(x_##C, x_##D); \ | | | | | | 251| 0| t_##C = _mm256_unpackhi_epi32(x_##A, x_##B); \ | | | | | | 252| 0| t_##D = _mm256_unpackhi_epi32(x_##C, x_##D); \ | | | | | | 253| 0| x_##A = _mm256_unpacklo_epi64(t_##A, t_##B); \ | | | | | | 254| 0| x_##B = _mm256_unpackhi_epi64(t_##A, t_##B); \ | | | | | | 255| 0| x_##C = _mm256_unpacklo_epi64(t_##C, t_##D); \ | | | | | | 256| 0| x_##D = _mm256_unpackhi_epi64(t_##C, t_##D); \ | | | | | | 257| 0| } | | | | ------------------ | | | | 263| 0| t_##A = _mm256_permute2x128_si256(x_##A, x_##A2, 0x20); \ | | | | 264| 0| t_##A2 = _mm256_permute2x128_si256(x_##A, x_##A2, 0x31); \ | | | | 265| 0| t_##B = _mm256_permute2x128_si256(x_##B, x_##B2, 0x20); \ | | | | 266| 0| t_##B2 = _mm256_permute2x128_si256(x_##B, x_##B2, 0x31); \ | | | | 267| 0| t_##C = _mm256_permute2x128_si256(x_##C, x_##C2, 0x20); \ | | | | 268| 0| t_##C2 = _mm256_permute2x128_si256(x_##C, x_##C2, 0x31); \ | | | | 269| 0| t_##D = _mm256_permute2x128_si256(x_##D, x_##D2, 0x20); \ | | | | 270| 0| t_##D2 = _mm256_permute2x128_si256(x_##D, x_##D2, 0x31); \ | | | | 271| 0| t_##A = _mm256_xor_si256( \ | | | | 272| 0| t_##A, _mm256_loadu_si256((const __m256i*) (m + 0))); \ | | | | 273| 0| t_##B = _mm256_xor_si256( \ | | | | 274| 0| t_##B, _mm256_loadu_si256((const __m256i*) (m + 64))); \ | | | | 275| 0| t_##C = _mm256_xor_si256( \ | | | | 276| 0| t_##C, _mm256_loadu_si256((const __m256i*) (m + 128))); \ | | | | 277| 0| t_##D = _mm256_xor_si256( \ | | | | 278| 0| t_##D, _mm256_loadu_si256((const __m256i*) (m + 192))); \ | | | | 279| 0| t_##A2 = _mm256_xor_si256( \ | | | | 280| 0| t_##A2, _mm256_loadu_si256((const __m256i*) (m + 256))); \ | | | | 281| 0| t_##B2 = _mm256_xor_si256( \ | | | | 282| 0| t_##B2, _mm256_loadu_si256((const __m256i*) (m + 320))); \ | | | | 283| 0| t_##C2 = _mm256_xor_si256( \ | | | | 284| 0| t_##C2, _mm256_loadu_si256((const __m256i*) (m + 384))); \ | | | | 285| 0| t_##D2 = _mm256_xor_si256( \ | | | | 286| 0| t_##D2, _mm256_loadu_si256((const __m256i*) (m + 448))); \ | | | | 287| 0| _mm256_storeu_si256((__m256i*) (c + 0), t_##A); \ | | | | 288| 0| _mm256_storeu_si256((__m256i*) (c + 64), t_##B); \ | | | | 289| 0| _mm256_storeu_si256((__m256i*) (c + 128), t_##C); \ | | | | 290| 0| _mm256_storeu_si256((__m256i*) (c + 192), t_##D); \ | | | | 291| 0| _mm256_storeu_si256((__m256i*) (c + 256), t_##A2); \ | | | | 292| 0| _mm256_storeu_si256((__m256i*) (c + 320), t_##B2); \ | | | | 293| 0| _mm256_storeu_si256((__m256i*) (c + 384), t_##C2); \ | | | | 294| 0| _mm256_storeu_si256((__m256i*) (c + 448), t_##D2); \ | | | | 295| 0| } | | ------------------ | | 298| 0| m += 32; | | 299| 0| c += 32; | | 300| 0| ONEOCTO(8, 9, 10, 11, 12, 13, 14, 15); | | ------------------ | | | | 260| 0| { \ | | | | 261| 0| ONEQUAD_UNPCK(A, B, C, D); \ | | | | ------------------ | | | | | | 244| 0| { \ | | | | | | 245| 0| x_##A = _mm256_add_epi32(x_##A, orig##A); \ | | | | | | 246| 0| x_##B = _mm256_add_epi32(x_##B, orig##B); \ | | | | | | 247| 0| x_##C = _mm256_add_epi32(x_##C, orig##C); \ | | | | | | 248| 0| x_##D = _mm256_add_epi32(x_##D, orig##D); \ | | | | | | 249| 0| t_##A = _mm256_unpacklo_epi32(x_##A, x_##B); \ | | | | | | 250| 0| t_##B = _mm256_unpacklo_epi32(x_##C, x_##D); \ | | | | | | 251| 0| t_##C = _mm256_unpackhi_epi32(x_##A, x_##B); \ | | | | | | 252| 0| t_##D = _mm256_unpackhi_epi32(x_##C, x_##D); \ | | | | | | 253| 0| x_##A = _mm256_unpacklo_epi64(t_##A, t_##B); \ | | | | | | 254| 0| x_##B = _mm256_unpackhi_epi64(t_##A, t_##B); \ | | | | | | 255| 0| x_##C = _mm256_unpacklo_epi64(t_##C, t_##D); \ | | | | | | 256| 0| x_##D = _mm256_unpackhi_epi64(t_##C, t_##D); \ | | | | | | 257| 0| } | | | | ------------------ | | | | 262| 0| ONEQUAD_UNPCK(A2, B2, C2, D2); \ | | | | ------------------ | | | | | | 244| 0| { \ | | | | | | 245| 0| x_##A = _mm256_add_epi32(x_##A, orig##A); \ | | | | | | 246| 0| x_##B = _mm256_add_epi32(x_##B, orig##B); \ | | | | | | 247| 0| x_##C = _mm256_add_epi32(x_##C, orig##C); \ | | | | | | 248| 0| x_##D = _mm256_add_epi32(x_##D, orig##D); \ | | | | | | 249| 0| t_##A = _mm256_unpacklo_epi32(x_##A, x_##B); \ | | | | | | 250| 0| t_##B = _mm256_unpacklo_epi32(x_##C, x_##D); \ | | | | | | 251| 0| t_##C = _mm256_unpackhi_epi32(x_##A, x_##B); \ | | | | | | 252| 0| t_##D = _mm256_unpackhi_epi32(x_##C, x_##D); \ | | | | | | 253| 0| x_##A = _mm256_unpacklo_epi64(t_##A, t_##B); \ | | | | | | 254| 0| x_##B = _mm256_unpackhi_epi64(t_##A, t_##B); \ | | | | | | 255| 0| x_##C = _mm256_unpacklo_epi64(t_##C, t_##D); \ | | | | | | 256| 0| x_##D = _mm256_unpackhi_epi64(t_##C, t_##D); \ | | | | | | 257| 0| } | | | | ------------------ | | | | 263| 0| t_##A = _mm256_permute2x128_si256(x_##A, x_##A2, 0x20); \ | | | | 264| 0| t_##A2 = _mm256_permute2x128_si256(x_##A, x_##A2, 0x31); \ | | | | 265| 0| t_##B = _mm256_permute2x128_si256(x_##B, x_##B2, 0x20); \ | | | | 266| 0| t_##B2 = _mm256_permute2x128_si256(x_##B, x_##B2, 0x31); \ | | | | 267| 0| t_##C = _mm256_permute2x128_si256(x_##C, x_##C2, 0x20); \ | | | | 268| 0| t_##C2 = _mm256_permute2x128_si256(x_##C, x_##C2, 0x31); \ | | | | 269| 0| t_##D = _mm256_permute2x128_si256(x_##D, x_##D2, 0x20); \ | | | | 270| 0| t_##D2 = _mm256_permute2x128_si256(x_##D, x_##D2, 0x31); \ | | | | 271| 0| t_##A = _mm256_xor_si256( \ | | | | 272| 0| t_##A, _mm256_loadu_si256((const __m256i*) (m + 0))); \ | | | | 273| 0| t_##B = _mm256_xor_si256( \ | | | | 274| 0| t_##B, _mm256_loadu_si256((const __m256i*) (m + 64))); \ | | | | 275| 0| t_##C = _mm256_xor_si256( \ | | | | 276| 0| t_##C, _mm256_loadu_si256((const __m256i*) (m + 128))); \ | | | | 277| 0| t_##D = _mm256_xor_si256( \ | | | | 278| 0| t_##D, _mm256_loadu_si256((const __m256i*) (m + 192))); \ | | | | 279| 0| t_##A2 = _mm256_xor_si256( \ | | | | 280| 0| t_##A2, _mm256_loadu_si256((const __m256i*) (m + 256))); \ | | | | 281| 0| t_##B2 = _mm256_xor_si256( \ | | | | 282| 0| t_##B2, _mm256_loadu_si256((const __m256i*) (m + 320))); \ | | | | 283| 0| t_##C2 = _mm256_xor_si256( \ | | | | 284| 0| t_##C2, _mm256_loadu_si256((const __m256i*) (m + 384))); \ | | | | 285| 0| t_##D2 = _mm256_xor_si256( \ | | | | 286| 0| t_##D2, _mm256_loadu_si256((const __m256i*) (m + 448))); \ | | | | 287| 0| _mm256_storeu_si256((__m256i*) (c + 0), t_##A); \ | | | | 288| 0| _mm256_storeu_si256((__m256i*) (c + 64), t_##B); \ | | | | 289| 0| _mm256_storeu_si256((__m256i*) (c + 128), t_##C); \ | | | | 290| 0| _mm256_storeu_si256((__m256i*) (c + 192), t_##D); \ | | | | 291| 0| _mm256_storeu_si256((__m256i*) (c + 256), t_##A2); \ | | | | 292| 0| _mm256_storeu_si256((__m256i*) (c + 320), t_##B2); \ | | | | 293| 0| _mm256_storeu_si256((__m256i*) (c + 384), t_##C2); \ | | | | 294| 0| _mm256_storeu_si256((__m256i*) (c + 448), t_##D2); \ | | | | 295| 0| } | | ------------------ | | 301| 0| m -= 32; | | 302| 0| c -= 32; | | 303| | | | 304| 0|#undef ONEQUAD | | 305| 0|#undef ONEQUAD_TRANSPOSE | | 306| 0|#undef ONEQUAD_UNPCK | | 307| 0|#undef ONEOCTO | | 308| | | | 309| 0| bytes -= 512; | | 310| 0| c += 512; | | 311| 0| m += 512; | | 312| 0| } | | 313| 0|} | | 314| 638|#undef VEC8_ROT | | 315| 638|#undef VEC8_QUARTERROUND | | 316| 638|#undef VEC8_QUARTERROUND_NAIVE | | 317| 638|#undef VEC8_QUARTERROUND_SHUFFLE | | 318| 638|#undef VEC8_QUARTERROUND_SHUFFLE2 | | 319| 638|#undef VEC8_LINE1 | | 320| 638|#undef VEC8_LINE2 | | 321| 638|#undef VEC8_LINE3 | | 322| 638|#undef VEC8_LINE4 | | 323| 638|#undef VEC8_ROUND | | 324| 638|#undef VEC8_ROUND_SEQ | | 325| 638|#undef VEC8_ROUND_HALF | | 326| 638|#undef VEC8_ROUND_HALFANDHALF ------------------ 81| 638|# include "u4.h" ------------------ | | 1| 638| | | 2| 638|#define VEC4_ROT(A, IMM) \ | | 3| 638| _mm_or_si128(_mm_slli_epi32(A, IMM), _mm_srli_epi32(A, (32 - IMM))) | | 4| | | | 5| |/* same, but replace 2 of the shift/shift/or "rotation" by byte shuffles (8 & | | 6| | * 16) (better) */ | | 7| 638|#define VEC4_QUARTERROUND_SHUFFLE(A, B, C, D) \ | | 8| 638| x_##A = _mm_add_epi32(x_##A, x_##B); \ | | 9| 638| t_##A = _mm_xor_si128(x_##D, x_##A); \ | | 10| 638| x_##D = _mm_shuffle_epi8(t_##A, rot16); \ | | 11| 638| x_##C = _mm_add_epi32(x_##C, x_##D); \ | | 12| 638| t_##C = _mm_xor_si128(x_##B, x_##C); \ | | 13| 638| x_##B = VEC4_ROT(t_##C, 12); \ | | 14| 638| x_##A = _mm_add_epi32(x_##A, x_##B); \ | | 15| 638| t_##A = _mm_xor_si128(x_##D, x_##A); \ | | 16| 638| x_##D = _mm_shuffle_epi8(t_##A, rot8); \ | | 17| 638| x_##C = _mm_add_epi32(x_##C, x_##D); \ | | 18| 638| t_##C = _mm_xor_si128(x_##B, x_##C); \ | | 19| 638| x_##B = VEC4_ROT(t_##C, 7) | | 20| | | | 21| 638|#define VEC4_QUARTERROUND(A, B, C, D) VEC4_QUARTERROUND_SHUFFLE(A, B, C, D) | | 22| | | | 23| 638|if (bytes >= 256) { | | ------------------ | | | Branch (23:5): [True: 0, False: 638] | | ------------------ | | 24| | /* constant for shuffling bytes (replacing multiple-of-8 rotates) */ | | 25| 0| __m128i rot16 = | | 26| 0| _mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); | | 27| 0| __m128i rot8 = | | 28| 0| _mm_set_epi8(14, 13, 12, 15, 10, 9, 8, 11, 6, 5, 4, 7, 2, 1, 0, 3); | | 29| | | | 30| 0| __m128i x_0 = _mm_set1_epi32(x[0]); | | 31| 0| __m128i x_1 = _mm_set1_epi32(x[1]); | | 32| 0| __m128i x_2 = _mm_set1_epi32(x[2]); | | 33| 0| __m128i x_3 = _mm_set1_epi32(x[3]); | | 34| 0| __m128i x_4 = _mm_set1_epi32(x[4]); | | 35| 0| __m128i x_5 = _mm_set1_epi32(x[5]); | | 36| 0| __m128i x_6 = _mm_set1_epi32(x[6]); | | 37| 0| __m128i x_7 = _mm_set1_epi32(x[7]); | | 38| 0| __m128i x_8 = _mm_set1_epi32(x[8]); | | 39| 0| __m128i x_9 = _mm_set1_epi32(x[9]); | | 40| 0| __m128i x_10 = _mm_set1_epi32(x[10]); | | 41| 0| __m128i x_11 = _mm_set1_epi32(x[11]); | | 42| 0| __m128i x_12; | | 43| 0| __m128i x_13; | | 44| 0| __m128i x_14 = _mm_set1_epi32(x[14]); | | 45| 0| __m128i x_15 = _mm_set1_epi32(x[15]); | | 46| 0| __m128i orig0 = x_0; | | 47| 0| __m128i orig1 = x_1; | | 48| 0| __m128i orig2 = x_2; | | 49| 0| __m128i orig3 = x_3; | | 50| 0| __m128i orig4 = x_4; | | 51| 0| __m128i orig5 = x_5; | | 52| 0| __m128i orig6 = x_6; | | 53| 0| __m128i orig7 = x_7; | | 54| 0| __m128i orig8 = x_8; | | 55| 0| __m128i orig9 = x_9; | | 56| 0| __m128i orig10 = x_10; | | 57| 0| __m128i orig11 = x_11; | | 58| 0| __m128i orig12; | | 59| 0| __m128i orig13; | | 60| 0| __m128i orig14 = x_14; | | 61| 0| __m128i orig15 = x_15; | | 62| 0| __m128i t_0, t_1, t_2, t_3, t_4, t_5, t_6, t_7, t_8, t_9, t_10, t_11, t_12, | | 63| 0| t_13, t_14, t_15; | | 64| | | | 65| 0| uint32_t in12, in13; | | 66| 0| int i; | | 67| | | | 68| 0| while (bytes >= 256) { | | ------------------ | | | Branch (68:12): [True: 0, False: 0] | | ------------------ | | 69| 0| const __m128i addv12 = _mm_set_epi64x(1, 0); | | 70| 0| const __m128i addv13 = _mm_set_epi64x(3, 2); | | 71| 0| __m128i t12, t13; | | 72| 0| uint64_t in1213; | | 73| | | | 74| 0| x_0 = orig0; | | 75| 0| x_1 = orig1; | | 76| 0| x_2 = orig2; | | 77| 0| x_3 = orig3; | | 78| 0| x_4 = orig4; | | 79| 0| x_5 = orig5; | | 80| 0| x_6 = orig6; | | 81| 0| x_7 = orig7; | | 82| 0| x_8 = orig8; | | 83| 0| x_9 = orig9; | | 84| 0| x_10 = orig10; | | 85| 0| x_11 = orig11; | | 86| 0| x_14 = orig14; | | 87| 0| x_15 = orig15; | | 88| | | | 89| 0| in12 = x[12]; | | 90| 0| in13 = x[13]; | | 91| 0| in1213 = ((uint64_t) in12) | (((uint64_t) in13) << 32); | | 92| 0| t12 = _mm_set1_epi64x(in1213); | | 93| 0| t13 = _mm_set1_epi64x(in1213); | | 94| | | | 95| 0| x_12 = _mm_add_epi64(addv12, t12); | | 96| 0| x_13 = _mm_add_epi64(addv13, t13); | | 97| | | | 98| 0| t12 = _mm_unpacklo_epi32(x_12, x_13); | | 99| 0| t13 = _mm_unpackhi_epi32(x_12, x_13); | | 100| | | | 101| 0| x_12 = _mm_unpacklo_epi32(t12, t13); | | 102| 0| x_13 = _mm_unpackhi_epi32(t12, t13); | | 103| | | | 104| 0| orig12 = x_12; | | 105| 0| orig13 = x_13; | | 106| | | | 107| 0| in1213 += 4; | | 108| | | | 109| 0| x[12] = in1213 & 0xFFFFFFFF; | | 110| 0| x[13] = (in1213 >> 32) & 0xFFFFFFFF; | | 111| | | | 112| 0| for (i = 0; i < ROUNDS; i += 2) { | | ------------------ | | | | 30| 0|# define ROUNDS 20 | | ------------------ | | | Branch (112:21): [True: 0, False: 0] | | ------------------ | | 113| 0| VEC4_QUARTERROUND(0, 4, 8, 12); | | ------------------ | | | | 21| 0|#define VEC4_QUARTERROUND(A, B, C, D) VEC4_QUARTERROUND_SHUFFLE(A, B, C, D) | | | | ------------------ | | | | | | 8| 0| x_##A = _mm_add_epi32(x_##A, x_##B); \ | | | | | | 9| 0| t_##A = _mm_xor_si128(x_##D, x_##A); \ | | | | | | 10| 0| x_##D = _mm_shuffle_epi8(t_##A, rot16); \ | | | | | | 11| 0| x_##C = _mm_add_epi32(x_##C, x_##D); \ | | | | | | 12| 0| t_##C = _mm_xor_si128(x_##B, x_##C); \ | | | | | | 13| 0| x_##B = VEC4_ROT(t_##C, 12); \ | | | | | | ------------------ | | | | | | | | 3| 0| _mm_or_si128(_mm_slli_epi32(A, IMM), _mm_srli_epi32(A, (32 - IMM))) | | | | | | ------------------ | | | | | | 14| 0| x_##A = _mm_add_epi32(x_##A, x_##B); \ | | | | | | 15| 0| t_##A = _mm_xor_si128(x_##D, x_##A); \ | | | | | | 16| 0| x_##D = _mm_shuffle_epi8(t_##A, rot8); \ | | | | | | 17| 0| x_##C = _mm_add_epi32(x_##C, x_##D); \ | | | | | | 18| 0| t_##C = _mm_xor_si128(x_##B, x_##C); \ | | | | | | 19| 0| x_##B = VEC4_ROT(t_##C, 7) | | | | | | ------------------ | | | | | | | | 3| 0| _mm_or_si128(_mm_slli_epi32(A, IMM), _mm_srli_epi32(A, (32 - IMM))) | | | | | | ------------------ | | | | ------------------ | | ------------------ | | 114| 0| VEC4_QUARTERROUND(1, 5, 9, 13); | | ------------------ | | | | 21| 0|#define VEC4_QUARTERROUND(A, B, C, D) VEC4_QUARTERROUND_SHUFFLE(A, B, C, D) | | | | ------------------ | | | | | | 8| 0| x_##A = _mm_add_epi32(x_##A, x_##B); \ | | | | | | 9| 0| t_##A = _mm_xor_si128(x_##D, x_##A); \ | | | | | | 10| 0| x_##D = _mm_shuffle_epi8(t_##A, rot16); \ | | | | | | 11| 0| x_##C = _mm_add_epi32(x_##C, x_##D); \ | | | | | | 12| 0| t_##C = _mm_xor_si128(x_##B, x_##C); \ | | | | | | 13| 0| x_##B = VEC4_ROT(t_##C, 12); \ | | | | | | ------------------ | | | | | | | | 3| 0| _mm_or_si128(_mm_slli_epi32(A, IMM), _mm_srli_epi32(A, (32 - IMM))) | | | | | | ------------------ | | | | | | 14| 0| x_##A = _mm_add_epi32(x_##A, x_##B); \ | | | | | | 15| 0| t_##A = _mm_xor_si128(x_##D, x_##A); \ | | | | | | 16| 0| x_##D = _mm_shuffle_epi8(t_##A, rot8); \ | | | | | | 17| 0| x_##C = _mm_add_epi32(x_##C, x_##D); \ | | | | | | 18| 0| t_##C = _mm_xor_si128(x_##B, x_##C); \ | | | | | | 19| 0| x_##B = VEC4_ROT(t_##C, 7) | | | | | | ------------------ | | | | | | | | 3| 0| _mm_or_si128(_mm_slli_epi32(A, IMM), _mm_srli_epi32(A, (32 - IMM))) | | | | | | ------------------ | | | | ------------------ | | ------------------ | | 115| 0| VEC4_QUARTERROUND(2, 6, 10, 14); | | ------------------ | | | | 21| 0|#define VEC4_QUARTERROUND(A, B, C, D) VEC4_QUARTERROUND_SHUFFLE(A, B, C, D) | | | | ------------------ | | | | | | 8| 0| x_##A = _mm_add_epi32(x_##A, x_##B); \ | | | | | | 9| 0| t_##A = _mm_xor_si128(x_##D, x_##A); \ | | | | | | 10| 0| x_##D = _mm_shuffle_epi8(t_##A, rot16); \ | | | | | | 11| 0| x_##C = _mm_add_epi32(x_##C, x_##D); \ | | | | | | 12| 0| t_##C = _mm_xor_si128(x_##B, x_##C); \ | | | | | | 13| 0| x_##B = VEC4_ROT(t_##C, 12); \ | | | | | | ------------------ | | | | | | | | 3| 0| _mm_or_si128(_mm_slli_epi32(A, IMM), _mm_srli_epi32(A, (32 - IMM))) | | | | | | ------------------ | | | | | | 14| 0| x_##A = _mm_add_epi32(x_##A, x_##B); \ | | | | | | 15| 0| t_##A = _mm_xor_si128(x_##D, x_##A); \ | | | | | | 16| 0| x_##D = _mm_shuffle_epi8(t_##A, rot8); \ | | | | | | 17| 0| x_##C = _mm_add_epi32(x_##C, x_##D); \ | | | | | | 18| 0| t_##C = _mm_xor_si128(x_##B, x_##C); \ | | | | | | 19| 0| x_##B = VEC4_ROT(t_##C, 7) | | | | | | ------------------ | | | | | | | | 3| 0| _mm_or_si128(_mm_slli_epi32(A, IMM), _mm_srli_epi32(A, (32 - IMM))) | | | | | | ------------------ | | | | ------------------ | | ------------------ | | 116| 0| VEC4_QUARTERROUND(3, 7, 11, 15); | | ------------------ | | | | 21| 0|#define VEC4_QUARTERROUND(A, B, C, D) VEC4_QUARTERROUND_SHUFFLE(A, B, C, D) | | | | ------------------ | | | | | | 8| 0| x_##A = _mm_add_epi32(x_##A, x_##B); \ | | | | | | 9| 0| t_##A = _mm_xor_si128(x_##D, x_##A); \ | | | | | | 10| 0| x_##D = _mm_shuffle_epi8(t_##A, rot16); \ | | | | | | 11| 0| x_##C = _mm_add_epi32(x_##C, x_##D); \ | | | | | | 12| 0| t_##C = _mm_xor_si128(x_##B, x_##C); \ | | | | | | 13| 0| x_##B = VEC4_ROT(t_##C, 12); \ | | | | | | ------------------ | | | | | | | | 3| 0| _mm_or_si128(_mm_slli_epi32(A, IMM), _mm_srli_epi32(A, (32 - IMM))) | | | | | | ------------------ | | | | | | 14| 0| x_##A = _mm_add_epi32(x_##A, x_##B); \ | | | | | | 15| 0| t_##A = _mm_xor_si128(x_##D, x_##A); \ | | | | | | 16| 0| x_##D = _mm_shuffle_epi8(t_##A, rot8); \ | | | | | | 17| 0| x_##C = _mm_add_epi32(x_##C, x_##D); \ | | | | | | 18| 0| t_##C = _mm_xor_si128(x_##B, x_##C); \ | | | | | | 19| 0| x_##B = VEC4_ROT(t_##C, 7) | | | | | | ------------------ | | | | | | | | 3| 0| _mm_or_si128(_mm_slli_epi32(A, IMM), _mm_srli_epi32(A, (32 - IMM))) | | | | | | ------------------ | | | | ------------------ | | ------------------ | | 117| 0| VEC4_QUARTERROUND(0, 5, 10, 15); | | ------------------ | | | | 21| 0|#define VEC4_QUARTERROUND(A, B, C, D) VEC4_QUARTERROUND_SHUFFLE(A, B, C, D) | | | | ------------------ | | | | | | 8| 0| x_##A = _mm_add_epi32(x_##A, x_##B); \ | | | | | | 9| 0| t_##A = _mm_xor_si128(x_##D, x_##A); \ | | | | | | 10| 0| x_##D = _mm_shuffle_epi8(t_##A, rot16); \ | | | | | | 11| 0| x_##C = _mm_add_epi32(x_##C, x_##D); \ | | | | | | 12| 0| t_##C = _mm_xor_si128(x_##B, x_##C); \ | | | | | | 13| 0| x_##B = VEC4_ROT(t_##C, 12); \ | | | | | | ------------------ | | | | | | | | 3| 0| _mm_or_si128(_mm_slli_epi32(A, IMM), _mm_srli_epi32(A, (32 - IMM))) | | | | | | ------------------ | | | | | | 14| 0| x_##A = _mm_add_epi32(x_##A, x_##B); \ | | | | | | 15| 0| t_##A = _mm_xor_si128(x_##D, x_##A); \ | | | | | | 16| 0| x_##D = _mm_shuffle_epi8(t_##A, rot8); \ | | | | | | 17| 0| x_##C = _mm_add_epi32(x_##C, x_##D); \ | | | | | | 18| 0| t_##C = _mm_xor_si128(x_##B, x_##C); \ | | | | | | 19| 0| x_##B = VEC4_ROT(t_##C, 7) | | | | | | ------------------ | | | | | | | | 3| 0| _mm_or_si128(_mm_slli_epi32(A, IMM), _mm_srli_epi32(A, (32 - IMM))) | | | | | | ------------------ | | | | ------------------ | | ------------------ | | 118| 0| VEC4_QUARTERROUND(1, 6, 11, 12); | | ------------------ | | | | 21| 0|#define VEC4_QUARTERROUND(A, B, C, D) VEC4_QUARTERROUND_SHUFFLE(A, B, C, D) | | | | ------------------ | | | | | | 8| 0| x_##A = _mm_add_epi32(x_##A, x_##B); \ | | | | | | 9| 0| t_##A = _mm_xor_si128(x_##D, x_##A); \ | | | | | | 10| 0| x_##D = _mm_shuffle_epi8(t_##A, rot16); \ | | | | | | 11| 0| x_##C = _mm_add_epi32(x_##C, x_##D); \ | | | | | | 12| 0| t_##C = _mm_xor_si128(x_##B, x_##C); \ | | | | | | 13| 0| x_##B = VEC4_ROT(t_##C, 12); \ | | | | | | ------------------ | | | | | | | | 3| 0| _mm_or_si128(_mm_slli_epi32(A, IMM), _mm_srli_epi32(A, (32 - IMM))) | | | | | | ------------------ | | | | | | 14| 0| x_##A = _mm_add_epi32(x_##A, x_##B); \ | | | | | | 15| 0| t_##A = _mm_xor_si128(x_##D, x_##A); \ | | | | | | 16| 0| x_##D = _mm_shuffle_epi8(t_##A, rot8); \ | | | | | | 17| 0| x_##C = _mm_add_epi32(x_##C, x_##D); \ | | | | | | 18| 0| t_##C = _mm_xor_si128(x_##B, x_##C); \ | | | | | | 19| 0| x_##B = VEC4_ROT(t_##C, 7) | | | | | | ------------------ | | | | | | | | 3| 0| _mm_or_si128(_mm_slli_epi32(A, IMM), _mm_srli_epi32(A, (32 - IMM))) | | | | | | ------------------ | | | | ------------------ | | ------------------ | | 119| 0| VEC4_QUARTERROUND(2, 7, 8, 13); | | ------------------ | | | | 21| 0|#define VEC4_QUARTERROUND(A, B, C, D) VEC4_QUARTERROUND_SHUFFLE(A, B, C, D) | | | | ------------------ | | | | | | 8| 0| x_##A = _mm_add_epi32(x_##A, x_##B); \ | | | | | | 9| 0| t_##A = _mm_xor_si128(x_##D, x_##A); \ | | | | | | 10| 0| x_##D = _mm_shuffle_epi8(t_##A, rot16); \ | | | | | | 11| 0| x_##C = _mm_add_epi32(x_##C, x_##D); \ | | | | | | 12| 0| t_##C = _mm_xor_si128(x_##B, x_##C); \ | | | | | | 13| 0| x_##B = VEC4_ROT(t_##C, 12); \ | | | | | | ------------------ | | | | | | | | 3| 0| _mm_or_si128(_mm_slli_epi32(A, IMM), _mm_srli_epi32(A, (32 - IMM))) | | | | | | ------------------ | | | | | | 14| 0| x_##A = _mm_add_epi32(x_##A, x_##B); \ | | | | | | 15| 0| t_##A = _mm_xor_si128(x_##D, x_##A); \ | | | | | | 16| 0| x_##D = _mm_shuffle_epi8(t_##A, rot8); \ | | | | | | 17| 0| x_##C = _mm_add_epi32(x_##C, x_##D); \ | | | | | | 18| 0| t_##C = _mm_xor_si128(x_##B, x_##C); \ | | | | | | 19| 0| x_##B = VEC4_ROT(t_##C, 7) | | | | | | ------------------ | | | | | | | | 3| 0| _mm_or_si128(_mm_slli_epi32(A, IMM), _mm_srli_epi32(A, (32 - IMM))) | | | | | | ------------------ | | | | ------------------ | | ------------------ | | 120| 0| VEC4_QUARTERROUND(3, 4, 9, 14); | | ------------------ | | | | 21| 0|#define VEC4_QUARTERROUND(A, B, C, D) VEC4_QUARTERROUND_SHUFFLE(A, B, C, D) | | | | ------------------ | | | | | | 8| 0| x_##A = _mm_add_epi32(x_##A, x_##B); \ | | | | | | 9| 0| t_##A = _mm_xor_si128(x_##D, x_##A); \ | | | | | | 10| 0| x_##D = _mm_shuffle_epi8(t_##A, rot16); \ | | | | | | 11| 0| x_##C = _mm_add_epi32(x_##C, x_##D); \ | | | | | | 12| 0| t_##C = _mm_xor_si128(x_##B, x_##C); \ | | | | | | 13| 0| x_##B = VEC4_ROT(t_##C, 12); \ | | | | | | ------------------ | | | | | | | | 3| 0| _mm_or_si128(_mm_slli_epi32(A, IMM), _mm_srli_epi32(A, (32 - IMM))) | | | | | | ------------------ | | | | | | 14| 0| x_##A = _mm_add_epi32(x_##A, x_##B); \ | | | | | | 15| 0| t_##A = _mm_xor_si128(x_##D, x_##A); \ | | | | | | 16| 0| x_##D = _mm_shuffle_epi8(t_##A, rot8); \ | | | | | | 17| 0| x_##C = _mm_add_epi32(x_##C, x_##D); \ | | | | | | 18| 0| t_##C = _mm_xor_si128(x_##B, x_##C); \ | | | | | | 19| 0| x_##B = VEC4_ROT(t_##C, 7) | | | | | | ------------------ | | | | | | | | 3| 0| _mm_or_si128(_mm_slli_epi32(A, IMM), _mm_srli_epi32(A, (32 - IMM))) | | | | | | ------------------ | | | | ------------------ | | ------------------ | | 121| 0| } | | 122| | | | 123| 0|#define ONEQUAD_TRANSPOSE(A, B, C, D) \ | | 124| 0| { \ | | 125| 0| __m128i t0, t1, t2, t3; \ | | 126| 0| \ | | 127| 0| x_##A = _mm_add_epi32(x_##A, orig##A); \ | | 128| 0| x_##B = _mm_add_epi32(x_##B, orig##B); \ | | 129| 0| x_##C = _mm_add_epi32(x_##C, orig##C); \ | | 130| 0| x_##D = _mm_add_epi32(x_##D, orig##D); \ | | 131| 0| t_##A = _mm_unpacklo_epi32(x_##A, x_##B); \ | | 132| 0| t_##B = _mm_unpacklo_epi32(x_##C, x_##D); \ | | 133| 0| t_##C = _mm_unpackhi_epi32(x_##A, x_##B); \ | | 134| 0| t_##D = _mm_unpackhi_epi32(x_##C, x_##D); \ | | 135| 0| x_##A = _mm_unpacklo_epi64(t_##A, t_##B); \ | | 136| 0| x_##B = _mm_unpackhi_epi64(t_##A, t_##B); \ | | 137| 0| x_##C = _mm_unpacklo_epi64(t_##C, t_##D); \ | | 138| 0| x_##D = _mm_unpackhi_epi64(t_##C, t_##D); \ | | 139| 0| \ | | 140| 0| t0 = _mm_xor_si128(x_##A, _mm_loadu_si128((const __m128i*) (m + 0))); \ | | 141| 0| _mm_storeu_si128((__m128i*) (c + 0), t0); \ | | 142| 0| t1 = _mm_xor_si128(x_##B, _mm_loadu_si128((const __m128i*) (m + 64))); \ | | 143| 0| _mm_storeu_si128((__m128i*) (c + 64), t1); \ | | 144| 0| t2 = \ | | 145| 0| _mm_xor_si128(x_##C, _mm_loadu_si128((const __m128i*) (m + 128))); \ | | 146| 0| _mm_storeu_si128((__m128i*) (c + 128), t2); \ | | 147| 0| t3 = \ | | 148| 0| _mm_xor_si128(x_##D, _mm_loadu_si128((const __m128i*) (m + 192))); \ | | 149| 0| _mm_storeu_si128((__m128i*) (c + 192), t3); \ | | 150| 0| } | | 151| | | | 152| 0|#define ONEQUAD(A, B, C, D) ONEQUAD_TRANSPOSE(A, B, C, D) | | 153| | | | 154| 0| ONEQUAD(0, 1, 2, 3); | | ------------------ | | | | 152| 0|#define ONEQUAD(A, B, C, D) ONEQUAD_TRANSPOSE(A, B, C, D) | | | | ------------------ | | | | | | 124| 0| { \ | | | | | | 125| 0| __m128i t0, t1, t2, t3; \ | | | | | | 126| 0| \ | | | | | | 127| 0| x_##A = _mm_add_epi32(x_##A, orig##A); \ | | | | | | 128| 0| x_##B = _mm_add_epi32(x_##B, orig##B); \ | | | | | | 129| 0| x_##C = _mm_add_epi32(x_##C, orig##C); \ | | | | | | 130| 0| x_##D = _mm_add_epi32(x_##D, orig##D); \ | | | | | | 131| 0| t_##A = _mm_unpacklo_epi32(x_##A, x_##B); \ | | | | | | 132| 0| t_##B = _mm_unpacklo_epi32(x_##C, x_##D); \ | | | | | | 133| 0| t_##C = _mm_unpackhi_epi32(x_##A, x_##B); \ | | | | | | 134| 0| t_##D = _mm_unpackhi_epi32(x_##C, x_##D); \ | | | | | | 135| 0| x_##A = _mm_unpacklo_epi64(t_##A, t_##B); \ | | | | | | 136| 0| x_##B = _mm_unpackhi_epi64(t_##A, t_##B); \ | | | | | | 137| 0| x_##C = _mm_unpacklo_epi64(t_##C, t_##D); \ | | | | | | 138| 0| x_##D = _mm_unpackhi_epi64(t_##C, t_##D); \ | | | | | | 139| 0| \ | | | | | | 140| 0| t0 = _mm_xor_si128(x_##A, _mm_loadu_si128((const __m128i*) (m + 0))); \ | | | | | | 141| 0| _mm_storeu_si128((__m128i*) (c + 0), t0); \ | | | | | | 142| 0| t1 = _mm_xor_si128(x_##B, _mm_loadu_si128((const __m128i*) (m + 64))); \ | | | | | | 143| 0| _mm_storeu_si128((__m128i*) (c + 64), t1); \ | | | | | | 144| 0| t2 = \ | | | | | | 145| 0| _mm_xor_si128(x_##C, _mm_loadu_si128((const __m128i*) (m + 128))); \ | | | | | | 146| 0| _mm_storeu_si128((__m128i*) (c + 128), t2); \ | | | | | | 147| 0| t3 = \ | | | | | | 148| 0| _mm_xor_si128(x_##D, _mm_loadu_si128((const __m128i*) (m + 192))); \ | | | | | | 149| 0| _mm_storeu_si128((__m128i*) (c + 192), t3); \ | | | | | | 150| 0| } | | | | ------------------ | | ------------------ | | 155| 0| m += 16; | | 156| 0| c += 16; | | 157| 0| ONEQUAD(4, 5, 6, 7); | | ------------------ | | | | 152| 0|#define ONEQUAD(A, B, C, D) ONEQUAD_TRANSPOSE(A, B, C, D) | | | | ------------------ | | | | | | 124| 0| { \ | | | | | | 125| 0| __m128i t0, t1, t2, t3; \ | | | | | | 126| 0| \ | | | | | | 127| 0| x_##A = _mm_add_epi32(x_##A, orig##A); \ | | | | | | 128| 0| x_##B = _mm_add_epi32(x_##B, orig##B); \ | | | | | | 129| 0| x_##C = _mm_add_epi32(x_##C, orig##C); \ | | | | | | 130| 0| x_##D = _mm_add_epi32(x_##D, orig##D); \ | | | | | | 131| 0| t_##A = _mm_unpacklo_epi32(x_##A, x_##B); \ | | | | | | 132| 0| t_##B = _mm_unpacklo_epi32(x_##C, x_##D); \ | | | | | | 133| 0| t_##C = _mm_unpackhi_epi32(x_##A, x_##B); \ | | | | | | 134| 0| t_##D = _mm_unpackhi_epi32(x_##C, x_##D); \ | | | | | | 135| 0| x_##A = _mm_unpacklo_epi64(t_##A, t_##B); \ | | | | | | 136| 0| x_##B = _mm_unpackhi_epi64(t_##A, t_##B); \ | | | | | | 137| 0| x_##C = _mm_unpacklo_epi64(t_##C, t_##D); \ | | | | | | 138| 0| x_##D = _mm_unpackhi_epi64(t_##C, t_##D); \ | | | | | | 139| 0| \ | | | | | | 140| 0| t0 = _mm_xor_si128(x_##A, _mm_loadu_si128((const __m128i*) (m + 0))); \ | | | | | | 141| 0| _mm_storeu_si128((__m128i*) (c + 0), t0); \ | | | | | | 142| 0| t1 = _mm_xor_si128(x_##B, _mm_loadu_si128((const __m128i*) (m + 64))); \ | | | | | | 143| 0| _mm_storeu_si128((__m128i*) (c + 64), t1); \ | | | | | | 144| 0| t2 = \ | | | | | | 145| 0| _mm_xor_si128(x_##C, _mm_loadu_si128((const __m128i*) (m + 128))); \ | | | | | | 146| 0| _mm_storeu_si128((__m128i*) (c + 128), t2); \ | | | | | | 147| 0| t3 = \ | | | | | | 148| 0| _mm_xor_si128(x_##D, _mm_loadu_si128((const __m128i*) (m + 192))); \ | | | | | | 149| 0| _mm_storeu_si128((__m128i*) (c + 192), t3); \ | | | | | | 150| 0| } | | | | ------------------ | | ------------------ | | 158| 0| m += 16; | | 159| 0| c += 16; | | 160| 0| ONEQUAD(8, 9, 10, 11); | | ------------------ | | | | 152| 0|#define ONEQUAD(A, B, C, D) ONEQUAD_TRANSPOSE(A, B, C, D) | | | | ------------------ | | | | | | 124| 0| { \ | | | | | | 125| 0| __m128i t0, t1, t2, t3; \ | | | | | | 126| 0| \ | | | | | | 127| 0| x_##A = _mm_add_epi32(x_##A, orig##A); \ | | | | | | 128| 0| x_##B = _mm_add_epi32(x_##B, orig##B); \ | | | | | | 129| 0| x_##C = _mm_add_epi32(x_##C, orig##C); \ | | | | | | 130| 0| x_##D = _mm_add_epi32(x_##D, orig##D); \ | | | | | | 131| 0| t_##A = _mm_unpacklo_epi32(x_##A, x_##B); \ | | | | | | 132| 0| t_##B = _mm_unpacklo_epi32(x_##C, x_##D); \ | | | | | | 133| 0| t_##C = _mm_unpackhi_epi32(x_##A, x_##B); \ | | | | | | 134| 0| t_##D = _mm_unpackhi_epi32(x_##C, x_##D); \ | | | | | | 135| 0| x_##A = _mm_unpacklo_epi64(t_##A, t_##B); \ | | | | | | 136| 0| x_##B = _mm_unpackhi_epi64(t_##A, t_##B); \ | | | | | | 137| 0| x_##C = _mm_unpacklo_epi64(t_##C, t_##D); \ | | | | | | 138| 0| x_##D = _mm_unpackhi_epi64(t_##C, t_##D); \ | | | | | | 139| 0| \ | | | | | | 140| 0| t0 = _mm_xor_si128(x_##A, _mm_loadu_si128((const __m128i*) (m + 0))); \ | | | | | | 141| 0| _mm_storeu_si128((__m128i*) (c + 0), t0); \ | | | | | | 142| 0| t1 = _mm_xor_si128(x_##B, _mm_loadu_si128((const __m128i*) (m + 64))); \ | | | | | | 143| 0| _mm_storeu_si128((__m128i*) (c + 64), t1); \ | | | | | | 144| 0| t2 = \ | | | | | | 145| 0| _mm_xor_si128(x_##C, _mm_loadu_si128((const __m128i*) (m + 128))); \ | | | | | | 146| 0| _mm_storeu_si128((__m128i*) (c + 128), t2); \ | | | | | | 147| 0| t3 = \ | | | | | | 148| 0| _mm_xor_si128(x_##D, _mm_loadu_si128((const __m128i*) (m + 192))); \ | | | | | | 149| 0| _mm_storeu_si128((__m128i*) (c + 192), t3); \ | | | | | | 150| 0| } | | | | ------------------ | | ------------------ | | 161| 0| m += 16; | | 162| 0| c += 16; | | 163| 0| ONEQUAD(12, 13, 14, 15); | | ------------------ | | | | 152| 0|#define ONEQUAD(A, B, C, D) ONEQUAD_TRANSPOSE(A, B, C, D) | | | | ------------------ | | | | | | 124| 0| { \ | | | | | | 125| 0| __m128i t0, t1, t2, t3; \ | | | | | | 126| 0| \ | | | | | | 127| 0| x_##A = _mm_add_epi32(x_##A, orig##A); \ | | | | | | 128| 0| x_##B = _mm_add_epi32(x_##B, orig##B); \ | | | | | | 129| 0| x_##C = _mm_add_epi32(x_##C, orig##C); \ | | | | | | 130| 0| x_##D = _mm_add_epi32(x_##D, orig##D); \ | | | | | | 131| 0| t_##A = _mm_unpacklo_epi32(x_##A, x_##B); \ | | | | | | 132| 0| t_##B = _mm_unpacklo_epi32(x_##C, x_##D); \ | | | | | | 133| 0| t_##C = _mm_unpackhi_epi32(x_##A, x_##B); \ | | | | | | 134| 0| t_##D = _mm_unpackhi_epi32(x_##C, x_##D); \ | | | | | | 135| 0| x_##A = _mm_unpacklo_epi64(t_##A, t_##B); \ | | | | | | 136| 0| x_##B = _mm_unpackhi_epi64(t_##A, t_##B); \ | | | | | | 137| 0| x_##C = _mm_unpacklo_epi64(t_##C, t_##D); \ | | | | | | 138| 0| x_##D = _mm_unpackhi_epi64(t_##C, t_##D); \ | | | | | | 139| 0| \ | | | | | | 140| 0| t0 = _mm_xor_si128(x_##A, _mm_loadu_si128((const __m128i*) (m + 0))); \ | | | | | | 141| 0| _mm_storeu_si128((__m128i*) (c + 0), t0); \ | | | | | | 142| 0| t1 = _mm_xor_si128(x_##B, _mm_loadu_si128((const __m128i*) (m + 64))); \ | | | | | | 143| 0| _mm_storeu_si128((__m128i*) (c + 64), t1); \ | | | | | | 144| 0| t2 = \ | | | | | | 145| 0| _mm_xor_si128(x_##C, _mm_loadu_si128((const __m128i*) (m + 128))); \ | | | | | | 146| 0| _mm_storeu_si128((__m128i*) (c + 128), t2); \ | | | | | | 147| 0| t3 = \ | | | | | | 148| 0| _mm_xor_si128(x_##D, _mm_loadu_si128((const __m128i*) (m + 192))); \ | | | | | | 149| 0| _mm_storeu_si128((__m128i*) (c + 192), t3); \ | | | | | | 150| 0| } | | | | ------------------ | | ------------------ | | 164| 0| m -= 48; | | 165| 0| c -= 48; | | 166| | | | 167| 0|#undef ONEQUAD | | 168| 0|#undef ONEQUAD_TRANSPOSE | | 169| | | | 170| 0| bytes -= 256; | | 171| 0| c += 256; | | 172| 0| m += 256; | | 173| 0| } | | 174| 0|} | | 175| 638|#undef VEC4_ROT | | 176| 638|#undef VEC4_QUARTERROUND | | 177| 638|#undef VEC4_QUARTERROUND_SHUFFLE ------------------ 82| 638|# include "u1.h" ------------------ | | 1| 638|while (bytes >= 64) { | | ------------------ | | | Branch (1:8): [True: 0, False: 638] | | ------------------ | | 2| 0| __m128i x_0, x_1, x_2, x_3; | | 3| 0| __m128i t_1; | | 4| 0| const __m128i rot16 = | | 5| 0| _mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); | | 6| 0| const __m128i rot8 = | | 7| 0| _mm_set_epi8(14, 13, 12, 15, 10, 9, 8, 11, 6, 5, 4, 7, 2, 1, 0, 3); | | 8| | | | 9| 0| uint32_t in12; | | 10| 0| uint32_t in13; | | 11| 0| int i; | | 12| | | | 13| 0| x_0 = _mm_loadu_si128((const __m128i*) (x + 0)); | | 14| 0| x_1 = _mm_loadu_si128((const __m128i*) (x + 4)); | | 15| 0| x_2 = _mm_loadu_si128((const __m128i*) (x + 8)); | | 16| 0| x_3 = _mm_loadu_si128((const __m128i*) (x + 12)); | | 17| | | | 18| 0| for (i = 0; i < ROUNDS; i += 2) { | | ------------------ | | | | 30| 0|# define ROUNDS 20 | | ------------------ | | | Branch (18:17): [True: 0, False: 0] | | ------------------ | | 19| 0| x_0 = _mm_add_epi32(x_0, x_1); | | 20| 0| x_3 = _mm_xor_si128(x_3, x_0); | | 21| 0| x_3 = _mm_shuffle_epi8(x_3, rot16); | | 22| | | | 23| 0| x_2 = _mm_add_epi32(x_2, x_3); | | 24| 0| x_1 = _mm_xor_si128(x_1, x_2); | | 25| | | | 26| 0| t_1 = x_1; | | 27| 0| x_1 = _mm_slli_epi32(x_1, 12); | | 28| 0| t_1 = _mm_srli_epi32(t_1, 20); | | 29| 0| x_1 = _mm_xor_si128(x_1, t_1); | | 30| | | | 31| 0| x_0 = _mm_add_epi32(x_0, x_1); | | 32| 0| x_3 = _mm_xor_si128(x_3, x_0); | | 33| 0| x_0 = _mm_shuffle_epi32(x_0, 0x93); | | 34| 0| x_3 = _mm_shuffle_epi8(x_3, rot8); | | 35| | | | 36| 0| x_2 = _mm_add_epi32(x_2, x_3); | | 37| 0| x_3 = _mm_shuffle_epi32(x_3, 0x4e); | | 38| 0| x_1 = _mm_xor_si128(x_1, x_2); | | 39| 0| x_2 = _mm_shuffle_epi32(x_2, 0x39); | | 40| | | | 41| 0| t_1 = x_1; | | 42| 0| x_1 = _mm_slli_epi32(x_1, 7); | | 43| 0| t_1 = _mm_srli_epi32(t_1, 25); | | 44| 0| x_1 = _mm_xor_si128(x_1, t_1); | | 45| | | | 46| 0| x_0 = _mm_add_epi32(x_0, x_1); | | 47| 0| x_3 = _mm_xor_si128(x_3, x_0); | | 48| 0| x_3 = _mm_shuffle_epi8(x_3, rot16); | | 49| | | | 50| 0| x_2 = _mm_add_epi32(x_2, x_3); | | 51| 0| x_1 = _mm_xor_si128(x_1, x_2); | | 52| | | | 53| 0| t_1 = x_1; | | 54| 0| x_1 = _mm_slli_epi32(x_1, 12); | | 55| 0| t_1 = _mm_srli_epi32(t_1, 20); | | 56| 0| x_1 = _mm_xor_si128(x_1, t_1); | | 57| | | | 58| 0| x_0 = _mm_add_epi32(x_0, x_1); | | 59| 0| x_3 = _mm_xor_si128(x_3, x_0); | | 60| 0| x_0 = _mm_shuffle_epi32(x_0, 0x39); | | 61| 0| x_3 = _mm_shuffle_epi8(x_3, rot8); | | 62| | | | 63| 0| x_2 = _mm_add_epi32(x_2, x_3); | | 64| 0| x_3 = _mm_shuffle_epi32(x_3, 0x4e); | | 65| 0| x_1 = _mm_xor_si128(x_1, x_2); | | 66| 0| x_2 = _mm_shuffle_epi32(x_2, 0x93); | | 67| | | | 68| 0| t_1 = x_1; | | 69| 0| x_1 = _mm_slli_epi32(x_1, 7); | | 70| 0| t_1 = _mm_srli_epi32(t_1, 25); | | 71| 0| x_1 = _mm_xor_si128(x_1, t_1); | | 72| 0| } | | 73| 0| x_0 = _mm_add_epi32(x_0, _mm_loadu_si128((const __m128i*) (x + 0))); | | 74| 0| x_1 = _mm_add_epi32(x_1, _mm_loadu_si128((const __m128i*) (x + 4))); | | 75| 0| x_2 = _mm_add_epi32(x_2, _mm_loadu_si128((const __m128i*) (x + 8))); | | 76| 0| x_3 = _mm_add_epi32(x_3, _mm_loadu_si128((const __m128i*) (x + 12))); | | 77| 0| x_0 = _mm_xor_si128(x_0, _mm_loadu_si128((const __m128i*) (m + 0))); | | 78| 0| x_1 = _mm_xor_si128(x_1, _mm_loadu_si128((const __m128i*) (m + 16))); | | 79| 0| x_2 = _mm_xor_si128(x_2, _mm_loadu_si128((const __m128i*) (m + 32))); | | 80| 0| x_3 = _mm_xor_si128(x_3, _mm_loadu_si128((const __m128i*) (m + 48))); | | 81| 0| _mm_storeu_si128((__m128i*) (c + 0), x_0); | | 82| 0| _mm_storeu_si128((__m128i*) (c + 16), x_1); | | 83| 0| _mm_storeu_si128((__m128i*) (c + 32), x_2); | | 84| 0| _mm_storeu_si128((__m128i*) (c + 48), x_3); | | 85| | | | 86| 0| in12 = x[12]; | | 87| 0| in13 = x[13]; | | 88| 0| in12++; | | 89| 0| if (in12 == 0) { | | ------------------ | | | Branch (89:9): [True: 0, False: 0] | | ------------------ | | 90| 0| in13++; | | 91| 0| } | | 92| 0| x[12] = in12; | | 93| 0| x[13] = in13; | | 94| | | | 95| 0| bytes -= 64; | | 96| 0| c += 64; | | 97| 0| m += 64; | | 98| 0|} ------------------ 83| 638|# include "u0.h" ------------------ | | 1| 638|if (bytes > 0) { | | ------------------ | | | Branch (1:5): [True: 638, False: 0] | | ------------------ | | 2| 638| __m128i x_0, x_1, x_2, x_3; | | 3| 638| __m128i t_1; | | 4| 638| const __m128i rot16 = | | 5| 638| _mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); | | 6| 638| const __m128i rot8 = | | 7| 638| _mm_set_epi8(14, 13, 12, 15, 10, 9, 8, 11, 6, 5, 4, 7, 2, 1, 0, 3); | | 8| 638| uint8_t partialblock[64]; | | 9| | | | 10| 638| unsigned int i; | | 11| | | | 12| 638| x_0 = _mm_loadu_si128((const __m128i*) (x + 0)); | | 13| 638| x_1 = _mm_loadu_si128((const __m128i*) (x + 4)); | | 14| 638| x_2 = _mm_loadu_si128((const __m128i*) (x + 8)); | | 15| 638| x_3 = _mm_loadu_si128((const __m128i*) (x + 12)); | | 16| | | | 17| 7.01k| for (i = 0; i < ROUNDS; i += 2) { | | ------------------ | | | | 30| 7.01k|# define ROUNDS 20 | | ------------------ | | | Branch (17:17): [True: 6.38k, False: 638] | | ------------------ | | 18| 6.38k| x_0 = _mm_add_epi32(x_0, x_1); | | 19| 6.38k| x_3 = _mm_xor_si128(x_3, x_0); | | 20| 6.38k| x_3 = _mm_shuffle_epi8(x_3, rot16); | | 21| | | | 22| 6.38k| x_2 = _mm_add_epi32(x_2, x_3); | | 23| 6.38k| x_1 = _mm_xor_si128(x_1, x_2); | | 24| | | | 25| 6.38k| t_1 = x_1; | | 26| 6.38k| x_1 = _mm_slli_epi32(x_1, 12); | | 27| 6.38k| t_1 = _mm_srli_epi32(t_1, 20); | | 28| 6.38k| x_1 = _mm_xor_si128(x_1, t_1); | | 29| | | | 30| 6.38k| x_0 = _mm_add_epi32(x_0, x_1); | | 31| 6.38k| x_3 = _mm_xor_si128(x_3, x_0); | | 32| 6.38k| x_0 = _mm_shuffle_epi32(x_0, 0x93); | | 33| 6.38k| x_3 = _mm_shuffle_epi8(x_3, rot8); | | 34| | | | 35| 6.38k| x_2 = _mm_add_epi32(x_2, x_3); | | 36| 6.38k| x_3 = _mm_shuffle_epi32(x_3, 0x4e); | | 37| 6.38k| x_1 = _mm_xor_si128(x_1, x_2); | | 38| 6.38k| x_2 = _mm_shuffle_epi32(x_2, 0x39); | | 39| | | | 40| 6.38k| t_1 = x_1; | | 41| 6.38k| x_1 = _mm_slli_epi32(x_1, 7); | | 42| 6.38k| t_1 = _mm_srli_epi32(t_1, 25); | | 43| 6.38k| x_1 = _mm_xor_si128(x_1, t_1); | | 44| | | | 45| 6.38k| x_0 = _mm_add_epi32(x_0, x_1); | | 46| 6.38k| x_3 = _mm_xor_si128(x_3, x_0); | | 47| 6.38k| x_3 = _mm_shuffle_epi8(x_3, rot16); | | 48| | | | 49| 6.38k| x_2 = _mm_add_epi32(x_2, x_3); | | 50| 6.38k| x_1 = _mm_xor_si128(x_1, x_2); | | 51| | | | 52| 6.38k| t_1 = x_1; | | 53| 6.38k| x_1 = _mm_slli_epi32(x_1, 12); | | 54| 6.38k| t_1 = _mm_srli_epi32(t_1, 20); | | 55| 6.38k| x_1 = _mm_xor_si128(x_1, t_1); | | 56| | | | 57| 6.38k| x_0 = _mm_add_epi32(x_0, x_1); | | 58| 6.38k| x_3 = _mm_xor_si128(x_3, x_0); | | 59| 6.38k| x_0 = _mm_shuffle_epi32(x_0, 0x39); | | 60| 6.38k| x_3 = _mm_shuffle_epi8(x_3, rot8); | | 61| | | | 62| 6.38k| x_2 = _mm_add_epi32(x_2, x_3); | | 63| 6.38k| x_3 = _mm_shuffle_epi32(x_3, 0x4e); | | 64| 6.38k| x_1 = _mm_xor_si128(x_1, x_2); | | 65| 6.38k| x_2 = _mm_shuffle_epi32(x_2, 0x93); | | 66| | | | 67| 6.38k| t_1 = x_1; | | 68| 6.38k| x_1 = _mm_slli_epi32(x_1, 7); | | 69| 6.38k| t_1 = _mm_srli_epi32(t_1, 25); | | 70| 6.38k| x_1 = _mm_xor_si128(x_1, t_1); | | 71| 6.38k| } | | 72| 638| x_0 = _mm_add_epi32(x_0, _mm_loadu_si128((const __m128i*) (x + 0))); | | 73| 638| x_1 = _mm_add_epi32(x_1, _mm_loadu_si128((const __m128i*) (x + 4))); | | 74| 638| x_2 = _mm_add_epi32(x_2, _mm_loadu_si128((const __m128i*) (x + 8))); | | 75| 638| x_3 = _mm_add_epi32(x_3, _mm_loadu_si128((const __m128i*) (x + 12))); | | 76| 638| _mm_storeu_si128((__m128i*) (partialblock + 0), x_0); | | 77| 638| _mm_storeu_si128((__m128i*) (partialblock + 16), x_1); | | 78| 638| _mm_storeu_si128((__m128i*) (partialblock + 32), x_2); | | 79| 638| _mm_storeu_si128((__m128i*) (partialblock + 48), x_3); | | 80| | | | 81| 18.5k| for (i = 0; i < bytes; i++) { | | ------------------ | | | Branch (81:17): [True: 17.8k, False: 638] | | ------------------ | | 82| 17.8k| c[i] = m[i] ^ partialblock[i]; | | 83| 17.8k| } | | 84| | | | 85| 638| sodium_memzero(partialblock, sizeof partialblock); | | 86| 638|} ------------------ 84| 638|} chacha20_dolbeau-avx2.c:stream_ietf_ext_ref: 108| 638|{ 109| 638| struct chacha_ctx ctx; 110| | 111| 638| if (!clen) { ------------------ | Branch (111:9): [True: 0, False: 638] ------------------ 112| 0| return 0; 113| 0| } 114| 638| COMPILER_ASSERT(crypto_stream_chacha20_KEYBYTES == 256 / 8); ------------------ | | 23| 638|#define COMPILER_ASSERT(X) (void) sizeof(char[(X) ? 1 : -1]) ------------------ 115| 638| chacha_keysetup(&ctx, k); 116| 638| chacha_ietf_ivsetup(&ctx, n, NULL); 117| 638| memset(c, 0, clen); 118| 638| chacha20_encrypt_bytes(&ctx, c, c, clen); 119| 638| sodium_memzero(&ctx, sizeof ctx); 120| | 121| 638| return 0; 122| 638|} chacha20_dolbeau-avx2.c:chacha_ietf_ivsetup: 64| 638|{ 65| 638| ctx->input[12] = counter == NULL ? 0 : LOAD32_LE(counter); ------------------ | | 100| 638|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ | Branch (65:22): [True: 638, False: 0] ------------------ 66| 638| ctx->input[13] = LOAD32_LE(iv + 0); ------------------ | | 100| 638|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 67| 638| ctx->input[14] = LOAD32_LE(iv + 4); ------------------ | | 100| 638|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 68| 638| ctx->input[15] = LOAD32_LE(iv + 8); ------------------ | | 100| 638|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 69| 638|} crypto_stream_chacha20_ietf_ext: 90| 638|{ 91| 638| if (clen > crypto_stream_chacha20_MESSAGEBYTES_MAX) { ------------------ | | 31| 638|#define crypto_stream_chacha20_MESSAGEBYTES_MAX SODIUM_SIZE_MAX | | ------------------ | | | | 55| 638|#define SODIUM_SIZE_MAX SODIUM_MIN(UINT64_MAX, SIZE_MAX) | | | | ------------------ | | | | | | 54| 638|#define SODIUM_MIN(A, B) ((A) < (B) ? (A) : (B)) | | | | | | ------------------ | | | | | | | Branch (54:27): [Folded - Ignored] | | | | | | ------------------ | | | | ------------------ | | ------------------ ------------------ | Branch (91:9): [True: 0, False: 638] ------------------ 92| 0| sodium_misuse(); 93| 0| } 94| 638| return implementation->stream_ietf_ext(c, clen, n, k); 95| 638|} crypto_stream_chacha20_ietf: 123| 638|{ 124| 638| if (clen > crypto_stream_chacha20_ietf_MESSAGEBYTES_MAX) { ------------------ | | 70| 638| SODIUM_MIN(SODIUM_SIZE_MAX, 64ULL * (1ULL << 32)) | | ------------------ | | | | 54| 1.27k|#define SODIUM_MIN(A, B) ((A) < (B) ? (A) : (B)) | | | | ------------------ | | | | | Branch (54:27): [Folded - Ignored] | | | | | Branch (54:28): [Folded - Ignored] | | | | | Branch (54:40): [Folded - Ignored] | | | | ------------------ | | ------------------ ------------------ | Branch (124:9): [True: 0, False: 638] ------------------ 125| 0| sodium_misuse(); 126| 0| } 127| 638| return crypto_stream_chacha20_ietf_ext(c, clen, n, k); 128| 638|} _crypto_stream_chacha20_pick_best_implementation: 168| 1|{ 169| 1| implementation = &crypto_stream_chacha20_ref_implementation; 170| 1|#if defined(HAVE_AVX2INTRIN_H) && defined(HAVE_EMMINTRIN_H) && \ 171| 1| defined(HAVE_TMMINTRIN_H) && defined(HAVE_SMMINTRIN_H) 172| 1| if (sodium_runtime_has_avx2()) { ------------------ | Branch (172:9): [True: 1, False: 0] ------------------ 173| 1| implementation = &crypto_stream_chacha20_dolbeau_avx2_implementation; 174| 1| return 0; 175| 1| } 176| 0|#endif 177| 0|#if defined(HAVE_EMMINTRIN_H) && defined(HAVE_TMMINTRIN_H) 178| 0| if (sodium_runtime_has_ssse3()) { ------------------ | Branch (178:9): [True: 0, False: 0] ------------------ 179| 0| implementation = &crypto_stream_chacha20_dolbeau_ssse3_implementation; 180| 0| return 0; 181| 0| } 182| 0|#endif 183| 0| return 0; 184| 0|} crypto_stream_salsa20: 50| 319|{ 51| 319| return implementation->stream(c, clen, n, k); 52| 319|} crypto_stream_salsa20_xor_ic: 59| 504|{ 60| 504| return implementation->stream_xor_ic(c, m, mlen, n, ic, k); 61| 504|} crypto_stream_salsa20_xor: 67| 638|{ 68| 638| return implementation->stream_xor_ic(c, m, mlen, n, 0U, k); 69| 638|} _crypto_stream_salsa20_pick_best_implementation: 79| 1|{ 80| 1|#ifdef HAVE_AMD64_ASM 81| 1| implementation = &crypto_stream_salsa20_xmm6_implementation; 82| |#else 83| | implementation = &crypto_stream_salsa20_ref_implementation; 84| |#endif 85| | 86| 1|#if defined(HAVE_AVX2INTRIN_H) && defined(HAVE_EMMINTRIN_H) && \ 87| 1| defined(HAVE_TMMINTRIN_H) && defined(HAVE_SMMINTRIN_H) 88| 1| if (sodium_runtime_has_avx2()) { ------------------ | Branch (88:9): [True: 1, False: 0] ------------------ 89| 1| implementation = &crypto_stream_salsa20_xmm6int_avx2_implementation; 90| 1| return 0; 91| 1| } 92| 0|#endif 93| |#if !defined(HAVE_AMD64_ASM) && defined(HAVE_EMMINTRIN_H) 94| | if (sodium_runtime_has_sse2()) { 95| | implementation = &crypto_stream_salsa20_xmm6int_sse2_implementation; 96| | return 0; 97| | } 98| |#endif 99| 0| return 0; /* LCOV_EXCL_LINE */ 100| 1|} salsa20_xmm6int-avx2.c:stream_avx2: 84| 319|{ 85| 319| struct salsa_ctx ctx; 86| | 87| 319| if (!clen) { ------------------ | Branch (87:9): [True: 0, False: 319] ------------------ 88| 0| return 0; 89| 0| } 90| 319| COMPILER_ASSERT(crypto_stream_salsa20_KEYBYTES == 256 / 8); ------------------ | | 23| 319|#define COMPILER_ASSERT(X) (void) sizeof(char[(X) ? 1 : -1]) ------------------ 91| 319| salsa_keysetup(&ctx, k); 92| 319| salsa_ivsetup(&ctx, n, NULL); 93| 319| memset(c, 0, clen); 94| 319| salsa20_encrypt_bytes(&ctx, c, c, clen); 95| 319| sodium_memzero(&ctx, sizeof ctx); 96| | 97| 319| return 0; 98| 319|} salsa20_xmm6int-avx2.c:salsa_keysetup: 41| 1.46k|{ 42| 1.46k| ctx->input[TR[1]] = LOAD32_LE(k + 0); ------------------ | | 100| 1.46k|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 43| 1.46k| ctx->input[TR[2]] = LOAD32_LE(k + 4); ------------------ | | 100| 1.46k|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 44| 1.46k| ctx->input[TR[3]] = LOAD32_LE(k + 8); ------------------ | | 100| 1.46k|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 45| 1.46k| ctx->input[TR[4]] = LOAD32_LE(k + 12); ------------------ | | 100| 1.46k|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 46| 1.46k| ctx->input[TR[11]] = LOAD32_LE(k + 16); ------------------ | | 100| 1.46k|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 47| 1.46k| ctx->input[TR[12]] = LOAD32_LE(k + 20); ------------------ | | 100| 1.46k|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 48| 1.46k| ctx->input[TR[13]] = LOAD32_LE(k + 24); ------------------ | | 100| 1.46k|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 49| 1.46k| ctx->input[TR[14]] = LOAD32_LE(k + 28); ------------------ | | 100| 1.46k|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 50| 1.46k| ctx->input[TR[0]] = 0x61707865; 51| 1.46k| ctx->input[TR[5]] = 0x3320646e; 52| 1.46k| ctx->input[TR[10]] = 0x79622d32; 53| 1.46k| ctx->input[TR[15]] = 0x6b206574; 54| 1.46k|} salsa20_xmm6int-avx2.c:salsa_ivsetup: 58| 1.46k|{ 59| 1.46k| ctx->input[TR[6]] = LOAD32_LE(iv + 0); ------------------ | | 100| 1.46k|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 60| 1.46k| ctx->input[TR[7]] = LOAD32_LE(iv + 4); ------------------ | | 100| 1.46k|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ 61| 1.46k| ctx->input[TR[8]] = counter == NULL ? 0 : LOAD32_LE(counter + 0); ------------------ | | 100| 2.60k|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ | Branch (61:25): [True: 319, False: 1.14k] ------------------ 62| 1.46k| ctx->input[TR[9]] = counter == NULL ? 0 : LOAD32_LE(counter + 4); ------------------ | | 100| 2.60k|#define LOAD32_LE(SRC) load32_le(SRC) ------------------ | Branch (62:25): [True: 319, False: 1.14k] ------------------ 63| 1.46k|} salsa20_xmm6int-avx2.c:salsa20_encrypt_bytes: 68| 1.46k|{ 69| 1.46k| uint32_t * const x = &ctx->input[0]; 70| | 71| 1.46k| if (!bytes) { ------------------ | Branch (71:9): [True: 0, False: 1.46k] ------------------ 72| 0| return; /* LCOV_EXCL_LINE */ 73| 0| } 74| | 75| 1.46k|#include "u8.h" ------------------ | | 1| 1.46k|if (bytes >= 512) { | | ------------------ | | | Branch (1:5): [True: 364, False: 1.09k] | | ------------------ | | 2| 364| __m256i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, | | 3| 364| y15; | | 4| | | | 5| | /* the naive way seems as fast (if not a bit faster) than the vector way */ | | 6| 364| __m256i z0 = _mm256_set1_epi32(x[0]); | | 7| 364| __m256i z5 = _mm256_set1_epi32(x[1]); | | 8| 364| __m256i z10 = _mm256_set1_epi32(x[2]); | | 9| 364| __m256i z15 = _mm256_set1_epi32(x[3]); | | 10| 364| __m256i z12 = _mm256_set1_epi32(x[4]); | | 11| 364| __m256i z1 = _mm256_set1_epi32(x[5]); | | 12| 364| __m256i z6 = _mm256_set1_epi32(x[6]); | | 13| 364| __m256i z11 = _mm256_set1_epi32(x[7]); | | 14| 364| __m256i z8; /* useless */ | | 15| 364| __m256i z13 = _mm256_set1_epi32(x[9]); | | 16| 364| __m256i z2 = _mm256_set1_epi32(x[10]); | | 17| 364| __m256i z7 = _mm256_set1_epi32(x[11]); | | 18| 364| __m256i z4 = _mm256_set1_epi32(x[12]); | | 19| 364| __m256i z9; /* useless */ | | 20| 364| __m256i z14 = _mm256_set1_epi32(x[14]); | | 21| 364| __m256i z3 = _mm256_set1_epi32(x[15]); | | 22| | | | 23| 364| __m256i orig0 = z0; | | 24| 364| __m256i orig1 = z1; | | 25| 364| __m256i orig2 = z2; | | 26| 364| __m256i orig3 = z3; | | 27| 364| __m256i orig4 = z4; | | 28| 364| __m256i orig5 = z5; | | 29| 364| __m256i orig6 = z6; | | 30| 364| __m256i orig7 = z7; | | 31| 364| __m256i orig8; | | 32| 364| __m256i orig9; | | 33| 364| __m256i orig10 = z10; | | 34| 364| __m256i orig11 = z11; | | 35| 364| __m256i orig12 = z12; | | 36| 364| __m256i orig13 = z13; | | 37| 364| __m256i orig14 = z14; | | 38| 364| __m256i orig15 = z15; | | 39| | | | 40| 364| uint32_t in8; | | 41| 364| uint32_t in9; | | 42| 364| int i; | | 43| | | | 44| 186k| while (bytes >= 512) { | | ------------------ | | | Branch (44:12): [True: 186k, False: 364] | | ------------------ | | 45| | /* vector implementation for z8 and z9 */ | | 46| | /* faster than the naive version for 8 blocks */ | | 47| 186k| const __m256i addv8 = _mm256_set_epi64x(3, 2, 1, 0); | | 48| 186k| const __m256i addv9 = _mm256_set_epi64x(7, 6, 5, 4); | | 49| 186k| const __m256i permute = _mm256_set_epi32(7, 6, 3, 2, 5, 4, 1, 0); | | 50| | | | 51| 186k| __m256i t8, t9; | | 52| 186k| uint64_t in89; | | 53| | | | 54| 186k| in8 = x[8]; | | 55| 186k| in9 = x[13]; /* see arrays above for the address translation */ | | 56| 186k| in89 = ((uint64_t) in8) | (((uint64_t) in9) << 32); | | 57| | | | 58| 186k| z8 = z9 = _mm256_broadcastq_epi64(_mm_cvtsi64_si128(in89)); | | 59| | | | 60| 186k| t8 = _mm256_add_epi64(addv8, z8); | | 61| 186k| t9 = _mm256_add_epi64(addv9, z9); | | 62| | | | 63| 186k| z8 = _mm256_unpacklo_epi32(t8, t9); | | 64| 186k| z9 = _mm256_unpackhi_epi32(t8, t9); | | 65| | | | 66| 186k| t8 = _mm256_unpacklo_epi32(z8, z9); | | 67| 186k| t9 = _mm256_unpackhi_epi32(z8, z9); | | 68| | | | 69| | /* required because unpack* are intra-lane */ | | 70| 186k| z8 = _mm256_permutevar8x32_epi32(t8, permute); | | 71| 186k| z9 = _mm256_permutevar8x32_epi32(t9, permute); | | 72| | | | 73| 186k| orig8 = z8; | | 74| 186k| orig9 = z9; | | 75| | | | 76| 186k| in89 += 8; | | 77| | | | 78| 186k| x[8] = in89 & 0xFFFFFFFF; | | 79| 186k| x[13] = (in89 >> 32) & 0xFFFFFFFF; | | 80| | | | 81| 186k| z5 = orig5; | | 82| 186k| z10 = orig10; | | 83| 186k| z15 = orig15; | | 84| 186k| z14 = orig14; | | 85| 186k| z3 = orig3; | | 86| 186k| z6 = orig6; | | 87| 186k| z11 = orig11; | | 88| 186k| z1 = orig1; | | 89| | | | 90| 186k| z7 = orig7; | | 91| 186k| z13 = orig13; | | 92| 186k| z2 = orig2; | | 93| 186k| z9 = orig9; | | 94| 186k| z0 = orig0; | | 95| 186k| z12 = orig12; | | 96| 186k| z4 = orig4; | | 97| 186k| z8 = orig8; | | 98| | | | 99| 2.05M| for (i = 0; i < ROUNDS; i += 2) { | | ------------------ | | | | 29| 2.05M|# define ROUNDS 20 | | ------------------ | | | Branch (99:21): [True: 1.86M, False: 186k] | | ------------------ | | 100| | /* the inner loop is a direct translation (regexp search/replace) | | 101| | * from the amd64-xmm6 ASM */ | | 102| 1.86M| __m256i r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, r13, | | 103| 1.86M| r14, r15; | | 104| | | | 105| 1.86M| y4 = z12; | | 106| 1.86M| y4 = _mm256_add_epi32(y4, z0); | | 107| 1.86M| r4 = y4; | | 108| 1.86M| y4 = _mm256_slli_epi32(y4, 7); | | 109| 1.86M| z4 = _mm256_xor_si256(z4, y4); | | 110| 1.86M| r4 = _mm256_srli_epi32(r4, 25); | | 111| 1.86M| z4 = _mm256_xor_si256(z4, r4); | | 112| | | | 113| 1.86M| y9 = z1; | | 114| 1.86M| y9 = _mm256_add_epi32(y9, z5); | | 115| 1.86M| r9 = y9; | | 116| 1.86M| y9 = _mm256_slli_epi32(y9, 7); | | 117| 1.86M| z9 = _mm256_xor_si256(z9, y9); | | 118| 1.86M| r9 = _mm256_srli_epi32(r9, 25); | | 119| 1.86M| z9 = _mm256_xor_si256(z9, r9); | | 120| | | | 121| 1.86M| y8 = z0; | | 122| 1.86M| y8 = _mm256_add_epi32(y8, z4); | | 123| 1.86M| r8 = y8; | | 124| 1.86M| y8 = _mm256_slli_epi32(y8, 9); | | 125| 1.86M| z8 = _mm256_xor_si256(z8, y8); | | 126| 1.86M| r8 = _mm256_srli_epi32(r8, 23); | | 127| 1.86M| z8 = _mm256_xor_si256(z8, r8); | | 128| | | | 129| 1.86M| y13 = z5; | | 130| 1.86M| y13 = _mm256_add_epi32(y13, z9); | | 131| 1.86M| r13 = y13; | | 132| 1.86M| y13 = _mm256_slli_epi32(y13, 9); | | 133| 1.86M| z13 = _mm256_xor_si256(z13, y13); | | 134| 1.86M| r13 = _mm256_srli_epi32(r13, 23); | | 135| 1.86M| z13 = _mm256_xor_si256(z13, r13); | | 136| | | | 137| 1.86M| y12 = z4; | | 138| 1.86M| y12 = _mm256_add_epi32(y12, z8); | | 139| 1.86M| r12 = y12; | | 140| 1.86M| y12 = _mm256_slli_epi32(y12, 13); | | 141| 1.86M| z12 = _mm256_xor_si256(z12, y12); | | 142| 1.86M| r12 = _mm256_srli_epi32(r12, 19); | | 143| 1.86M| z12 = _mm256_xor_si256(z12, r12); | | 144| | | | 145| 1.86M| y1 = z9; | | 146| 1.86M| y1 = _mm256_add_epi32(y1, z13); | | 147| 1.86M| r1 = y1; | | 148| 1.86M| y1 = _mm256_slli_epi32(y1, 13); | | 149| 1.86M| z1 = _mm256_xor_si256(z1, y1); | | 150| 1.86M| r1 = _mm256_srli_epi32(r1, 19); | | 151| 1.86M| z1 = _mm256_xor_si256(z1, r1); | | 152| | | | 153| 1.86M| y0 = z8; | | 154| 1.86M| y0 = _mm256_add_epi32(y0, z12); | | 155| 1.86M| r0 = y0; | | 156| 1.86M| y0 = _mm256_slli_epi32(y0, 18); | | 157| 1.86M| z0 = _mm256_xor_si256(z0, y0); | | 158| 1.86M| r0 = _mm256_srli_epi32(r0, 14); | | 159| 1.86M| z0 = _mm256_xor_si256(z0, r0); | | 160| | | | 161| 1.86M| y5 = z13; | | 162| 1.86M| y5 = _mm256_add_epi32(y5, z1); | | 163| 1.86M| r5 = y5; | | 164| 1.86M| y5 = _mm256_slli_epi32(y5, 18); | | 165| 1.86M| z5 = _mm256_xor_si256(z5, y5); | | 166| 1.86M| r5 = _mm256_srli_epi32(r5, 14); | | 167| 1.86M| z5 = _mm256_xor_si256(z5, r5); | | 168| | | | 169| 1.86M| y14 = z6; | | 170| 1.86M| y14 = _mm256_add_epi32(y14, z10); | | 171| 1.86M| r14 = y14; | | 172| 1.86M| y14 = _mm256_slli_epi32(y14, 7); | | 173| 1.86M| z14 = _mm256_xor_si256(z14, y14); | | 174| 1.86M| r14 = _mm256_srli_epi32(r14, 25); | | 175| 1.86M| z14 = _mm256_xor_si256(z14, r14); | | 176| | | | 177| 1.86M| y3 = z11; | | 178| 1.86M| y3 = _mm256_add_epi32(y3, z15); | | 179| 1.86M| r3 = y3; | | 180| 1.86M| y3 = _mm256_slli_epi32(y3, 7); | | 181| 1.86M| z3 = _mm256_xor_si256(z3, y3); | | 182| 1.86M| r3 = _mm256_srli_epi32(r3, 25); | | 183| 1.86M| z3 = _mm256_xor_si256(z3, r3); | | 184| | | | 185| 1.86M| y2 = z10; | | 186| 1.86M| y2 = _mm256_add_epi32(y2, z14); | | 187| 1.86M| r2 = y2; | | 188| 1.86M| y2 = _mm256_slli_epi32(y2, 9); | | 189| 1.86M| z2 = _mm256_xor_si256(z2, y2); | | 190| 1.86M| r2 = _mm256_srli_epi32(r2, 23); | | 191| 1.86M| z2 = _mm256_xor_si256(z2, r2); | | 192| | | | 193| 1.86M| y7 = z15; | | 194| 1.86M| y7 = _mm256_add_epi32(y7, z3); | | 195| 1.86M| r7 = y7; | | 196| 1.86M| y7 = _mm256_slli_epi32(y7, 9); | | 197| 1.86M| z7 = _mm256_xor_si256(z7, y7); | | 198| 1.86M| r7 = _mm256_srli_epi32(r7, 23); | | 199| 1.86M| z7 = _mm256_xor_si256(z7, r7); | | 200| | | | 201| 1.86M| y6 = z14; | | 202| 1.86M| y6 = _mm256_add_epi32(y6, z2); | | 203| 1.86M| r6 = y6; | | 204| 1.86M| y6 = _mm256_slli_epi32(y6, 13); | | 205| 1.86M| z6 = _mm256_xor_si256(z6, y6); | | 206| 1.86M| r6 = _mm256_srli_epi32(r6, 19); | | 207| 1.86M| z6 = _mm256_xor_si256(z6, r6); | | 208| | | | 209| 1.86M| y11 = z3; | | 210| 1.86M| y11 = _mm256_add_epi32(y11, z7); | | 211| 1.86M| r11 = y11; | | 212| 1.86M| y11 = _mm256_slli_epi32(y11, 13); | | 213| 1.86M| z11 = _mm256_xor_si256(z11, y11); | | 214| 1.86M| r11 = _mm256_srli_epi32(r11, 19); | | 215| 1.86M| z11 = _mm256_xor_si256(z11, r11); | | 216| | | | 217| 1.86M| y10 = z2; | | 218| 1.86M| y10 = _mm256_add_epi32(y10, z6); | | 219| 1.86M| r10 = y10; | | 220| 1.86M| y10 = _mm256_slli_epi32(y10, 18); | | 221| 1.86M| z10 = _mm256_xor_si256(z10, y10); | | 222| 1.86M| r10 = _mm256_srli_epi32(r10, 14); | | 223| 1.86M| z10 = _mm256_xor_si256(z10, r10); | | 224| | | | 225| 1.86M| y1 = z3; | | 226| 1.86M| y1 = _mm256_add_epi32(y1, z0); | | 227| 1.86M| r1 = y1; | | 228| 1.86M| y1 = _mm256_slli_epi32(y1, 7); | | 229| 1.86M| z1 = _mm256_xor_si256(z1, y1); | | 230| 1.86M| r1 = _mm256_srli_epi32(r1, 25); | | 231| 1.86M| z1 = _mm256_xor_si256(z1, r1); | | 232| | | | 233| 1.86M| y15 = z7; | | 234| 1.86M| y15 = _mm256_add_epi32(y15, z11); | | 235| 1.86M| r15 = y15; | | 236| 1.86M| y15 = _mm256_slli_epi32(y15, 18); | | 237| 1.86M| z15 = _mm256_xor_si256(z15, y15); | | 238| 1.86M| r15 = _mm256_srli_epi32(r15, 14); | | 239| 1.86M| z15 = _mm256_xor_si256(z15, r15); | | 240| | | | 241| 1.86M| y6 = z4; | | 242| 1.86M| y6 = _mm256_add_epi32(y6, z5); | | 243| 1.86M| r6 = y6; | | 244| 1.86M| y6 = _mm256_slli_epi32(y6, 7); | | 245| 1.86M| z6 = _mm256_xor_si256(z6, y6); | | 246| 1.86M| r6 = _mm256_srli_epi32(r6, 25); | | 247| 1.86M| z6 = _mm256_xor_si256(z6, r6); | | 248| | | | 249| 1.86M| y2 = z0; | | 250| 1.86M| y2 = _mm256_add_epi32(y2, z1); | | 251| 1.86M| r2 = y2; | | 252| 1.86M| y2 = _mm256_slli_epi32(y2, 9); | | 253| 1.86M| z2 = _mm256_xor_si256(z2, y2); | | 254| 1.86M| r2 = _mm256_srli_epi32(r2, 23); | | 255| 1.86M| z2 = _mm256_xor_si256(z2, r2); | | 256| | | | 257| 1.86M| y7 = z5; | | 258| 1.86M| y7 = _mm256_add_epi32(y7, z6); | | 259| 1.86M| r7 = y7; | | 260| 1.86M| y7 = _mm256_slli_epi32(y7, 9); | | 261| 1.86M| z7 = _mm256_xor_si256(z7, y7); | | 262| 1.86M| r7 = _mm256_srli_epi32(r7, 23); | | 263| 1.86M| z7 = _mm256_xor_si256(z7, r7); | | 264| | | | 265| 1.86M| y3 = z1; | | 266| 1.86M| y3 = _mm256_add_epi32(y3, z2); | | 267| 1.86M| r3 = y3; | | 268| 1.86M| y3 = _mm256_slli_epi32(y3, 13); | | 269| 1.86M| z3 = _mm256_xor_si256(z3, y3); | | 270| 1.86M| r3 = _mm256_srli_epi32(r3, 19); | | 271| 1.86M| z3 = _mm256_xor_si256(z3, r3); | | 272| | | | 273| 1.86M| y4 = z6; | | 274| 1.86M| y4 = _mm256_add_epi32(y4, z7); | | 275| 1.86M| r4 = y4; | | 276| 1.86M| y4 = _mm256_slli_epi32(y4, 13); | | 277| 1.86M| z4 = _mm256_xor_si256(z4, y4); | | 278| 1.86M| r4 = _mm256_srli_epi32(r4, 19); | | 279| 1.86M| z4 = _mm256_xor_si256(z4, r4); | | 280| | | | 281| 1.86M| y0 = z2; | | 282| 1.86M| y0 = _mm256_add_epi32(y0, z3); | | 283| 1.86M| r0 = y0; | | 284| 1.86M| y0 = _mm256_slli_epi32(y0, 18); | | 285| 1.86M| z0 = _mm256_xor_si256(z0, y0); | | 286| 1.86M| r0 = _mm256_srli_epi32(r0, 14); | | 287| 1.86M| z0 = _mm256_xor_si256(z0, r0); | | 288| | | | 289| 1.86M| y5 = z7; | | 290| 1.86M| y5 = _mm256_add_epi32(y5, z4); | | 291| 1.86M| r5 = y5; | | 292| 1.86M| y5 = _mm256_slli_epi32(y5, 18); | | 293| 1.86M| z5 = _mm256_xor_si256(z5, y5); | | 294| 1.86M| r5 = _mm256_srli_epi32(r5, 14); | | 295| 1.86M| z5 = _mm256_xor_si256(z5, r5); | | 296| | | | 297| 1.86M| y11 = z9; | | 298| 1.86M| y11 = _mm256_add_epi32(y11, z10); | | 299| 1.86M| r11 = y11; | | 300| 1.86M| y11 = _mm256_slli_epi32(y11, 7); | | 301| 1.86M| z11 = _mm256_xor_si256(z11, y11); | | 302| 1.86M| r11 = _mm256_srli_epi32(r11, 25); | | 303| 1.86M| z11 = _mm256_xor_si256(z11, r11); | | 304| | | | 305| 1.86M| y12 = z14; | | 306| 1.86M| y12 = _mm256_add_epi32(y12, z15); | | 307| 1.86M| r12 = y12; | | 308| 1.86M| y12 = _mm256_slli_epi32(y12, 7); | | 309| 1.86M| z12 = _mm256_xor_si256(z12, y12); | | 310| 1.86M| r12 = _mm256_srli_epi32(r12, 25); | | 311| 1.86M| z12 = _mm256_xor_si256(z12, r12); | | 312| | | | 313| 1.86M| y8 = z10; | | 314| 1.86M| y8 = _mm256_add_epi32(y8, z11); | | 315| 1.86M| r8 = y8; | | 316| 1.86M| y8 = _mm256_slli_epi32(y8, 9); | | 317| 1.86M| z8 = _mm256_xor_si256(z8, y8); | | 318| 1.86M| r8 = _mm256_srli_epi32(r8, 23); | | 319| 1.86M| z8 = _mm256_xor_si256(z8, r8); | | 320| | | | 321| 1.86M| y13 = z15; | | 322| 1.86M| y13 = _mm256_add_epi32(y13, z12); | | 323| 1.86M| r13 = y13; | | 324| 1.86M| y13 = _mm256_slli_epi32(y13, 9); | | 325| 1.86M| z13 = _mm256_xor_si256(z13, y13); | | 326| 1.86M| r13 = _mm256_srli_epi32(r13, 23); | | 327| 1.86M| z13 = _mm256_xor_si256(z13, r13); | | 328| | | | 329| 1.86M| y9 = z11; | | 330| 1.86M| y9 = _mm256_add_epi32(y9, z8); | | 331| 1.86M| r9 = y9; | | 332| 1.86M| y9 = _mm256_slli_epi32(y9, 13); | | 333| 1.86M| z9 = _mm256_xor_si256(z9, y9); | | 334| 1.86M| r9 = _mm256_srli_epi32(r9, 19); | | 335| 1.86M| z9 = _mm256_xor_si256(z9, r9); | | 336| | | | 337| 1.86M| y14 = z12; | | 338| 1.86M| y14 = _mm256_add_epi32(y14, z13); | | 339| 1.86M| r14 = y14; | | 340| 1.86M| y14 = _mm256_slli_epi32(y14, 13); | | 341| 1.86M| z14 = _mm256_xor_si256(z14, y14); | | 342| 1.86M| r14 = _mm256_srli_epi32(r14, 19); | | 343| 1.86M| z14 = _mm256_xor_si256(z14, r14); | | 344| | | | 345| 1.86M| y10 = z8; | | 346| 1.86M| y10 = _mm256_add_epi32(y10, z9); | | 347| 1.86M| r10 = y10; | | 348| 1.86M| y10 = _mm256_slli_epi32(y10, 18); | | 349| 1.86M| z10 = _mm256_xor_si256(z10, y10); | | 350| 1.86M| r10 = _mm256_srli_epi32(r10, 14); | | 351| 1.86M| z10 = _mm256_xor_si256(z10, r10); | | 352| | | | 353| 1.86M| y15 = z13; | | 354| 1.86M| y15 = _mm256_add_epi32(y15, z14); | | 355| 1.86M| r15 = y15; | | 356| 1.86M| y15 = _mm256_slli_epi32(y15, 18); | | 357| 1.86M| z15 = _mm256_xor_si256(z15, y15); | | 358| 1.86M| r15 = _mm256_srli_epi32(r15, 14); | | 359| 1.86M| z15 = _mm256_xor_si256(z15, r15); | | 360| 1.86M| } | | 361| | | | 362| |/* store data ; this macro first transpose data in-registers, and then store | | 363| | * them in memory. much faster with icc. */ | | 364| 186k|#define ONEQUAD_TRANSPOSE(A, B, C, D) \ | | 365| 186k| { \ | | 366| 186k| __m128i t0, t1, t2, t3; \ | | 367| 186k| z##A = _mm256_add_epi32(z##A, orig##A); \ | | 368| 186k| z##B = _mm256_add_epi32(z##B, orig##B); \ | | 369| 186k| z##C = _mm256_add_epi32(z##C, orig##C); \ | | 370| 186k| z##D = _mm256_add_epi32(z##D, orig##D); \ | | 371| 186k| y##A = _mm256_unpacklo_epi32(z##A, z##B); \ | | 372| 186k| y##B = _mm256_unpacklo_epi32(z##C, z##D); \ | | 373| 186k| y##C = _mm256_unpackhi_epi32(z##A, z##B); \ | | 374| 186k| y##D = _mm256_unpackhi_epi32(z##C, z##D); \ | | 375| 186k| z##A = _mm256_unpacklo_epi64(y##A, y##B); \ | | 376| 186k| z##B = _mm256_unpackhi_epi64(y##A, y##B); \ | | 377| 186k| z##C = _mm256_unpacklo_epi64(y##C, y##D); \ | | 378| 186k| z##D = _mm256_unpackhi_epi64(y##C, y##D); \ | | 379| 186k| t0 = _mm_xor_si128(_mm256_extracti128_si256(z##A, 0), \ | | 380| 186k| _mm_loadu_si128((const __m128i*) (m + 0))); \ | | 381| 186k| _mm_storeu_si128((__m128i*) (c + 0), t0); \ | | 382| 186k| t1 = _mm_xor_si128(_mm256_extracti128_si256(z##B, 0), \ | | 383| 186k| _mm_loadu_si128((const __m128i*) (m + 64))); \ | | 384| 186k| _mm_storeu_si128((__m128i*) (c + 64), t1); \ | | 385| 186k| t2 = _mm_xor_si128(_mm256_extracti128_si256(z##C, 0), \ | | 386| 186k| _mm_loadu_si128((const __m128i*) (m + 128))); \ | | 387| 186k| _mm_storeu_si128((__m128i*) (c + 128), t2); \ | | 388| 186k| t3 = _mm_xor_si128(_mm256_extracti128_si256(z##D, 0), \ | | 389| 186k| _mm_loadu_si128((const __m128i*) (m + 192))); \ | | 390| 186k| _mm_storeu_si128((__m128i*) (c + 192), t3); \ | | 391| 186k| t0 = _mm_xor_si128(_mm256_extracti128_si256(z##A, 1), \ | | 392| 186k| _mm_loadu_si128((const __m128i*) (m + 256))); \ | | 393| 186k| _mm_storeu_si128((__m128i*) (c + 256), t0); \ | | 394| 186k| t1 = _mm_xor_si128(_mm256_extracti128_si256(z##B, 1), \ | | 395| 186k| _mm_loadu_si128((const __m128i*) (m + 320))); \ | | 396| 186k| _mm_storeu_si128((__m128i*) (c + 320), t1); \ | | 397| 186k| t2 = _mm_xor_si128(_mm256_extracti128_si256(z##C, 1), \ | | 398| 186k| _mm_loadu_si128((const __m128i*) (m + 384))); \ | | 399| 186k| _mm_storeu_si128((__m128i*) (c + 384), t2); \ | | 400| 186k| t3 = _mm_xor_si128(_mm256_extracti128_si256(z##D, 1), \ | | 401| 186k| _mm_loadu_si128((const __m128i*) (m + 448))); \ | | 402| 186k| _mm_storeu_si128((__m128i*) (c + 448), t3); \ | | 403| 186k| } | | 404| | | | 405| 186k|#define ONEQUAD(A, B, C, D) ONEQUAD_TRANSPOSE(A, B, C, D) | | 406| | | | 407| 186k|#define ONEQUAD_UNPCK(A, B, C, D) \ | | 408| 186k| { \ | | 409| 186k| z##A = _mm256_add_epi32(z##A, orig##A); \ | | 410| 186k| z##B = _mm256_add_epi32(z##B, orig##B); \ | | 411| 186k| z##C = _mm256_add_epi32(z##C, orig##C); \ | | 412| 186k| z##D = _mm256_add_epi32(z##D, orig##D); \ | | 413| 186k| y##A = _mm256_unpacklo_epi32(z##A, z##B); \ | | 414| 186k| y##B = _mm256_unpacklo_epi32(z##C, z##D); \ | | 415| 186k| y##C = _mm256_unpackhi_epi32(z##A, z##B); \ | | 416| 186k| y##D = _mm256_unpackhi_epi32(z##C, z##D); \ | | 417| 186k| z##A = _mm256_unpacklo_epi64(y##A, y##B); \ | | 418| 186k| z##B = _mm256_unpackhi_epi64(y##A, y##B); \ | | 419| 186k| z##C = _mm256_unpacklo_epi64(y##C, y##D); \ | | 420| 186k| z##D = _mm256_unpackhi_epi64(y##C, y##D); \ | | 421| 186k| } | | 422| | | | 423| 186k|#define ONEOCTO(A, B, C, D, A2, B2, C2, D2) \ | | 424| 186k| { \ | | 425| 186k| ONEQUAD_UNPCK(A, B, C, D); \ | | 426| 186k| ONEQUAD_UNPCK(A2, B2, C2, D2); \ | | 427| 186k| y##A = _mm256_permute2x128_si256(z##A, z##A2, 0x20); \ | | 428| 186k| y##A2 = _mm256_permute2x128_si256(z##A, z##A2, 0x31); \ | | 429| 186k| y##B = _mm256_permute2x128_si256(z##B, z##B2, 0x20); \ | | 430| 186k| y##B2 = _mm256_permute2x128_si256(z##B, z##B2, 0x31); \ | | 431| 186k| y##C = _mm256_permute2x128_si256(z##C, z##C2, 0x20); \ | | 432| 186k| y##C2 = _mm256_permute2x128_si256(z##C, z##C2, 0x31); \ | | 433| 186k| y##D = _mm256_permute2x128_si256(z##D, z##D2, 0x20); \ | | 434| 186k| y##D2 = _mm256_permute2x128_si256(z##D, z##D2, 0x31); \ | | 435| 186k| y##A = _mm256_xor_si256(y##A, \ | | 436| 186k| _mm256_loadu_si256((const __m256i*) (m + 0))); \ | | 437| 186k| y##B = _mm256_xor_si256( \ | | 438| 186k| y##B, _mm256_loadu_si256((const __m256i*) (m + 64))); \ | | 439| 186k| y##C = _mm256_xor_si256( \ | | 440| 186k| y##C, _mm256_loadu_si256((const __m256i*) (m + 128))); \ | | 441| 186k| y##D = _mm256_xor_si256( \ | | 442| 186k| y##D, _mm256_loadu_si256((const __m256i*) (m + 192))); \ | | 443| 186k| y##A2 = _mm256_xor_si256( \ | | 444| 186k| y##A2, _mm256_loadu_si256((const __m256i*) (m + 256))); \ | | 445| 186k| y##B2 = _mm256_xor_si256( \ | | 446| 186k| y##B2, _mm256_loadu_si256((const __m256i*) (m + 320))); \ | | 447| 186k| y##C2 = _mm256_xor_si256( \ | | 448| 186k| y##C2, _mm256_loadu_si256((const __m256i*) (m + 384))); \ | | 449| 186k| y##D2 = _mm256_xor_si256( \ | | 450| 186k| y##D2, _mm256_loadu_si256((const __m256i*) (m + 448))); \ | | 451| 186k| _mm256_storeu_si256((__m256i*) (c + 0), y##A); \ | | 452| 186k| _mm256_storeu_si256((__m256i*) (c + 64), y##B); \ | | 453| 186k| _mm256_storeu_si256((__m256i*) (c + 128), y##C); \ | | 454| 186k| _mm256_storeu_si256((__m256i*) (c + 192), y##D); \ | | 455| 186k| _mm256_storeu_si256((__m256i*) (c + 256), y##A2); \ | | 456| 186k| _mm256_storeu_si256((__m256i*) (c + 320), y##B2); \ | | 457| 186k| _mm256_storeu_si256((__m256i*) (c + 384), y##C2); \ | | 458| 186k| _mm256_storeu_si256((__m256i*) (c + 448), y##D2); \ | | 459| 186k| } | | 460| | | | 461| 186k| ONEOCTO(0, 1, 2, 3, 4, 5, 6, 7); | | ------------------ | | | | 424| 186k| { \ | | | | 425| 186k| ONEQUAD_UNPCK(A, B, C, D); \ | | | | ------------------ | | | | | | 408| 186k| { \ | | | | | | 409| 186k| z##A = _mm256_add_epi32(z##A, orig##A); \ | | | | | | 410| 186k| z##B = _mm256_add_epi32(z##B, orig##B); \ | | | | | | 411| 186k| z##C = _mm256_add_epi32(z##C, orig##C); \ | | | | | | 412| 186k| z##D = _mm256_add_epi32(z##D, orig##D); \ | | | | | | 413| 186k| y##A = _mm256_unpacklo_epi32(z##A, z##B); \ | | | | | | 414| 186k| y##B = _mm256_unpacklo_epi32(z##C, z##D); \ | | | | | | 415| 186k| y##C = _mm256_unpackhi_epi32(z##A, z##B); \ | | | | | | 416| 186k| y##D = _mm256_unpackhi_epi32(z##C, z##D); \ | | | | | | 417| 186k| z##A = _mm256_unpacklo_epi64(y##A, y##B); \ | | | | | | 418| 186k| z##B = _mm256_unpackhi_epi64(y##A, y##B); \ | | | | | | 419| 186k| z##C = _mm256_unpacklo_epi64(y##C, y##D); \ | | | | | | 420| 186k| z##D = _mm256_unpackhi_epi64(y##C, y##D); \ | | | | | | 421| 186k| } | | | | ------------------ | | | | 426| 186k| ONEQUAD_UNPCK(A2, B2, C2, D2); \ | | | | ------------------ | | | | | | 408| 186k| { \ | | | | | | 409| 186k| z##A = _mm256_add_epi32(z##A, orig##A); \ | | | | | | 410| 186k| z##B = _mm256_add_epi32(z##B, orig##B); \ | | | | | | 411| 186k| z##C = _mm256_add_epi32(z##C, orig##C); \ | | | | | | 412| 186k| z##D = _mm256_add_epi32(z##D, orig##D); \ | | | | | | 413| 186k| y##A = _mm256_unpacklo_epi32(z##A, z##B); \ | | | | | | 414| 186k| y##B = _mm256_unpacklo_epi32(z##C, z##D); \ | | | | | | 415| 186k| y##C = _mm256_unpackhi_epi32(z##A, z##B); \ | | | | | | 416| 186k| y##D = _mm256_unpackhi_epi32(z##C, z##D); \ | | | | | | 417| 186k| z##A = _mm256_unpacklo_epi64(y##A, y##B); \ | | | | | | 418| 186k| z##B = _mm256_unpackhi_epi64(y##A, y##B); \ | | | | | | 419| 186k| z##C = _mm256_unpacklo_epi64(y##C, y##D); \ | | | | | | 420| 186k| z##D = _mm256_unpackhi_epi64(y##C, y##D); \ | | | | | | 421| 186k| } | | | | ------------------ | | | | 427| 186k| y##A = _mm256_permute2x128_si256(z##A, z##A2, 0x20); \ | | | | 428| 186k| y##A2 = _mm256_permute2x128_si256(z##A, z##A2, 0x31); \ | | | | 429| 186k| y##B = _mm256_permute2x128_si256(z##B, z##B2, 0x20); \ | | | | 430| 186k| y##B2 = _mm256_permute2x128_si256(z##B, z##B2, 0x31); \ | | | | 431| 186k| y##C = _mm256_permute2x128_si256(z##C, z##C2, 0x20); \ | | | | 432| 186k| y##C2 = _mm256_permute2x128_si256(z##C, z##C2, 0x31); \ | | | | 433| 186k| y##D = _mm256_permute2x128_si256(z##D, z##D2, 0x20); \ | | | | 434| 186k| y##D2 = _mm256_permute2x128_si256(z##D, z##D2, 0x31); \ | | | | 435| 186k| y##A = _mm256_xor_si256(y##A, \ | | | | 436| 186k| _mm256_loadu_si256((const __m256i*) (m + 0))); \ | | | | 437| 186k| y##B = _mm256_xor_si256( \ | | | | 438| 186k| y##B, _mm256_loadu_si256((const __m256i*) (m + 64))); \ | | | | 439| 186k| y##C = _mm256_xor_si256( \ | | | | 440| 186k| y##C, _mm256_loadu_si256((const __m256i*) (m + 128))); \ | | | | 441| 186k| y##D = _mm256_xor_si256( \ | | | | 442| 186k| y##D, _mm256_loadu_si256((const __m256i*) (m + 192))); \ | | | | 443| 186k| y##A2 = _mm256_xor_si256( \ | | | | 444| 186k| y##A2, _mm256_loadu_si256((const __m256i*) (m + 256))); \ | | | | 445| 186k| y##B2 = _mm256_xor_si256( \ | | | | 446| 186k| y##B2, _mm256_loadu_si256((const __m256i*) (m + 320))); \ | | | | 447| 186k| y##C2 = _mm256_xor_si256( \ | | | | 448| 186k| y##C2, _mm256_loadu_si256((const __m256i*) (m + 384))); \ | | | | 449| 186k| y##D2 = _mm256_xor_si256( \ | | | | 450| 186k| y##D2, _mm256_loadu_si256((const __m256i*) (m + 448))); \ | | | | 451| 186k| _mm256_storeu_si256((__m256i*) (c + 0), y##A); \ | | | | 452| 186k| _mm256_storeu_si256((__m256i*) (c + 64), y##B); \ | | | | 453| 186k| _mm256_storeu_si256((__m256i*) (c + 128), y##C); \ | | | | 454| 186k| _mm256_storeu_si256((__m256i*) (c + 192), y##D); \ | | | | 455| 186k| _mm256_storeu_si256((__m256i*) (c + 256), y##A2); \ | | | | 456| 186k| _mm256_storeu_si256((__m256i*) (c + 320), y##B2); \ | | | | 457| 186k| _mm256_storeu_si256((__m256i*) (c + 384), y##C2); \ | | | | 458| 186k| _mm256_storeu_si256((__m256i*) (c + 448), y##D2); \ | | | | 459| 186k| } | | ------------------ | | 462| 186k| m += 32; | | 463| 186k| c += 32; | | 464| 186k| ONEOCTO(8, 9, 10, 11, 12, 13, 14, 15); | | ------------------ | | | | 424| 186k| { \ | | | | 425| 186k| ONEQUAD_UNPCK(A, B, C, D); \ | | | | ------------------ | | | | | | 408| 186k| { \ | | | | | | 409| 186k| z##A = _mm256_add_epi32(z##A, orig##A); \ | | | | | | 410| 186k| z##B = _mm256_add_epi32(z##B, orig##B); \ | | | | | | 411| 186k| z##C = _mm256_add_epi32(z##C, orig##C); \ | | | | | | 412| 186k| z##D = _mm256_add_epi32(z##D, orig##D); \ | | | | | | 413| 186k| y##A = _mm256_unpacklo_epi32(z##A, z##B); \ | | | | | | 414| 186k| y##B = _mm256_unpacklo_epi32(z##C, z##D); \ | | | | | | 415| 186k| y##C = _mm256_unpackhi_epi32(z##A, z##B); \ | | | | | | 416| 186k| y##D = _mm256_unpackhi_epi32(z##C, z##D); \ | | | | | | 417| 186k| z##A = _mm256_unpacklo_epi64(y##A, y##B); \ | | | | | | 418| 186k| z##B = _mm256_unpackhi_epi64(y##A, y##B); \ | | | | | | 419| 186k| z##C = _mm256_unpacklo_epi64(y##C, y##D); \ | | | | | | 420| 186k| z##D = _mm256_unpackhi_epi64(y##C, y##D); \ | | | | | | 421| 186k| } | | | | ------------------ | | | | 426| 186k| ONEQUAD_UNPCK(A2, B2, C2, D2); \ | | | | ------------------ | | | | | | 408| 186k| { \ | | | | | | 409| 186k| z##A = _mm256_add_epi32(z##A, orig##A); \ | | | | | | 410| 186k| z##B = _mm256_add_epi32(z##B, orig##B); \ | | | | | | 411| 186k| z##C = _mm256_add_epi32(z##C, orig##C); \ | | | | | | 412| 186k| z##D = _mm256_add_epi32(z##D, orig##D); \ | | | | | | 413| 186k| y##A = _mm256_unpacklo_epi32(z##A, z##B); \ | | | | | | 414| 186k| y##B = _mm256_unpacklo_epi32(z##C, z##D); \ | | | | | | 415| 186k| y##C = _mm256_unpackhi_epi32(z##A, z##B); \ | | | | | | 416| 186k| y##D = _mm256_unpackhi_epi32(z##C, z##D); \ | | | | | | 417| 186k| z##A = _mm256_unpacklo_epi64(y##A, y##B); \ | | | | | | 418| 186k| z##B = _mm256_unpackhi_epi64(y##A, y##B); \ | | | | | | 419| 186k| z##C = _mm256_unpacklo_epi64(y##C, y##D); \ | | | | | | 420| 186k| z##D = _mm256_unpackhi_epi64(y##C, y##D); \ | | | | | | 421| 186k| } | | | | ------------------ | | | | 427| 186k| y##A = _mm256_permute2x128_si256(z##A, z##A2, 0x20); \ | | | | 428| 186k| y##A2 = _mm256_permute2x128_si256(z##A, z##A2, 0x31); \ | | | | 429| 186k| y##B = _mm256_permute2x128_si256(z##B, z##B2, 0x20); \ | | | | 430| 186k| y##B2 = _mm256_permute2x128_si256(z##B, z##B2, 0x31); \ | | | | 431| 186k| y##C = _mm256_permute2x128_si256(z##C, z##C2, 0x20); \ | | | | 432| 186k| y##C2 = _mm256_permute2x128_si256(z##C, z##C2, 0x31); \ | | | | 433| 186k| y##D = _mm256_permute2x128_si256(z##D, z##D2, 0x20); \ | | | | 434| 186k| y##D2 = _mm256_permute2x128_si256(z##D, z##D2, 0x31); \ | | | | 435| 186k| y##A = _mm256_xor_si256(y##A, \ | | | | 436| 186k| _mm256_loadu_si256((const __m256i*) (m + 0))); \ | | | | 437| 186k| y##B = _mm256_xor_si256( \ | | | | 438| 186k| y##B, _mm256_loadu_si256((const __m256i*) (m + 64))); \ | | | | 439| 186k| y##C = _mm256_xor_si256( \ | | | | 440| 186k| y##C, _mm256_loadu_si256((const __m256i*) (m + 128))); \ | | | | 441| 186k| y##D = _mm256_xor_si256( \ | | | | 442| 186k| y##D, _mm256_loadu_si256((const __m256i*) (m + 192))); \ | | | | 443| 186k| y##A2 = _mm256_xor_si256( \ | | | | 444| 186k| y##A2, _mm256_loadu_si256((const __m256i*) (m + 256))); \ | | | | 445| 186k| y##B2 = _mm256_xor_si256( \ | | | | 446| 186k| y##B2, _mm256_loadu_si256((const __m256i*) (m + 320))); \ | | | | 447| 186k| y##C2 = _mm256_xor_si256( \ | | | | 448| 186k| y##C2, _mm256_loadu_si256((const __m256i*) (m + 384))); \ | | | | 449| 186k| y##D2 = _mm256_xor_si256( \ | | | | 450| 186k| y##D2, _mm256_loadu_si256((const __m256i*) (m + 448))); \ | | | | 451| 186k| _mm256_storeu_si256((__m256i*) (c + 0), y##A); \ | | | | 452| 186k| _mm256_storeu_si256((__m256i*) (c + 64), y##B); \ | | | | 453| 186k| _mm256_storeu_si256((__m256i*) (c + 128), y##C); \ | | | | 454| 186k| _mm256_storeu_si256((__m256i*) (c + 192), y##D); \ | | | | 455| 186k| _mm256_storeu_si256((__m256i*) (c + 256), y##A2); \ | | | | 456| 186k| _mm256_storeu_si256((__m256i*) (c + 320), y##B2); \ | | | | 457| 186k| _mm256_storeu_si256((__m256i*) (c + 384), y##C2); \ | | | | 458| 186k| _mm256_storeu_si256((__m256i*) (c + 448), y##D2); \ | | | | 459| 186k| } | | ------------------ | | 465| 186k| m -= 32; | | 466| 186k| c -= 32; | | 467| | | | 468| 186k|#undef ONEQUAD | | 469| 186k|#undef ONEQUAD_TRANSPOSE | | 470| 186k|#undef ONEQUAD_UNPCK | | 471| 186k|#undef ONEOCTO | | 472| | | | 473| 186k| bytes -= 512; | | 474| 186k| c += 512; | | 475| 186k| m += 512; | | 476| 186k| } | | 477| 364|} ------------------ 76| 1.46k|#include "u4.h" ------------------ | | 1| 1.46k|if (bytes >= 256) { | | ------------------ | | | Branch (1:5): [True: 344, False: 1.11k] | | ------------------ | | 2| 344| __m128i y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, y10, y11, y12, y13, y14, | | 3| 344| y15; | | 4| 344| __m128i z0, z1, z2, z3, z4, z5, z6, z7, z8, z9, z10, z11, z12, z13, z14, | | 5| 344| z15; | | 6| 344| __m128i orig0, orig1, orig2, orig3, orig4, orig5, orig6, orig7, orig8, | | 7| 344| orig9, orig10, orig11, orig12, orig13, orig14, orig15; | | 8| | | | 9| 344| uint32_t in8; | | 10| 344| uint32_t in9; | | 11| 344| int i; | | 12| | | | 13| | /* element broadcast immediate for _mm_shuffle_epi32 are in order: | | 14| | 0x00, 0x55, 0xaa, 0xff */ | | 15| 344| z0 = _mm_loadu_si128((const __m128i *) (x + 0)); | | 16| 344| z5 = _mm_shuffle_epi32(z0, 0x55); | | 17| 344| z10 = _mm_shuffle_epi32(z0, 0xaa); | | 18| 344| z15 = _mm_shuffle_epi32(z0, 0xff); | | 19| 344| z0 = _mm_shuffle_epi32(z0, 0x00); | | 20| 344| z1 = _mm_loadu_si128((const __m128i *) (x + 4)); | | 21| 344| z6 = _mm_shuffle_epi32(z1, 0xaa); | | 22| 344| z11 = _mm_shuffle_epi32(z1, 0xff); | | 23| 344| z12 = _mm_shuffle_epi32(z1, 0x00); | | 24| 344| z1 = _mm_shuffle_epi32(z1, 0x55); | | 25| 344| z2 = _mm_loadu_si128((const __m128i *) (x + 8)); | | 26| 344| z7 = _mm_shuffle_epi32(z2, 0xff); | | 27| 344| z13 = _mm_shuffle_epi32(z2, 0x55); | | 28| 344| z2 = _mm_shuffle_epi32(z2, 0xaa); | | 29| | /* no z8 -> first half of the nonce, will fill later */ | | 30| 344| z3 = _mm_loadu_si128((const __m128i *) (x + 12)); | | 31| 344| z4 = _mm_shuffle_epi32(z3, 0x00); | | 32| 344| z14 = _mm_shuffle_epi32(z3, 0xaa); | | 33| 344| z3 = _mm_shuffle_epi32(z3, 0xff); | | 34| | /* no z9 -> second half of the nonce, will fill later */ | | 35| 344| orig0 = z0; | | 36| 344| orig1 = z1; | | 37| 344| orig2 = z2; | | 38| 344| orig3 = z3; | | 39| 344| orig4 = z4; | | 40| 344| orig5 = z5; | | 41| 344| orig6 = z6; | | 42| 344| orig7 = z7; | | 43| 344| orig10 = z10; | | 44| 344| orig11 = z11; | | 45| 344| orig12 = z12; | | 46| 344| orig13 = z13; | | 47| 344| orig14 = z14; | | 48| 344| orig15 = z15; | | 49| | | | 50| 688| while (bytes >= 256) { | | ------------------ | | | Branch (50:12): [True: 344, False: 344] | | ------------------ | | 51| | /* vector implementation for z8 and z9 */ | | 52| | /* not sure if it helps for only 4 blocks */ | | 53| 344| const __m128i addv8 = _mm_set_epi64x(1, 0); | | 54| 344| const __m128i addv9 = _mm_set_epi64x(3, 2); | | 55| 344| __m128i t8, t9; | | 56| 344| uint64_t in89; | | 57| | | | 58| 344| in8 = x[8]; | | 59| 344| in9 = x[13]; | | 60| 344| in89 = ((uint64_t) in8) | (((uint64_t) in9) << 32); | | 61| 344| t8 = _mm_set1_epi64x(in89); | | 62| 344| t9 = _mm_set1_epi64x(in89); | | 63| | | | 64| 344| z8 = _mm_add_epi64(addv8, t8); | | 65| 344| z9 = _mm_add_epi64(addv9, t9); | | 66| | | | 67| 344| t8 = _mm_unpacklo_epi32(z8, z9); | | 68| 344| t9 = _mm_unpackhi_epi32(z8, z9); | | 69| | | | 70| 344| z8 = _mm_unpacklo_epi32(t8, t9); | | 71| 344| z9 = _mm_unpackhi_epi32(t8, t9); | | 72| | | | 73| 344| orig8 = z8; | | 74| 344| orig9 = z9; | | 75| | | | 76| 344| in89 += 4; | | 77| | | | 78| 344| x[8] = in89 & 0xFFFFFFFF; | | 79| 344| x[13] = (in89 >> 32) & 0xFFFFFFFF; | | 80| | | | 81| 344| z5 = orig5; | | 82| 344| z10 = orig10; | | 83| 344| z15 = orig15; | | 84| 344| z14 = orig14; | | 85| 344| z3 = orig3; | | 86| 344| z6 = orig6; | | 87| 344| z11 = orig11; | | 88| 344| z1 = orig1; | | 89| | | | 90| 344| z7 = orig7; | | 91| 344| z13 = orig13; | | 92| 344| z2 = orig2; | | 93| 344| z9 = orig9; | | 94| 344| z0 = orig0; | | 95| 344| z12 = orig12; | | 96| 344| z4 = orig4; | | 97| 344| z8 = orig8; | | 98| | | | 99| 3.78k| for (i = 0; i < ROUNDS; i += 2) { | | ------------------ | | | | 29| 3.78k|# define ROUNDS 20 | | ------------------ | | | Branch (99:21): [True: 3.44k, False: 344] | | ------------------ | | 100| | /* the inner loop is a direct translation (regexp search/replace) | | 101| | * from the amd64-xmm6 ASM */ | | 102| 3.44k| __m128i r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, r13, | | 103| 3.44k| r14, r15; | | 104| | | | 105| 3.44k| y4 = z12; | | 106| 3.44k| y4 = _mm_add_epi32(y4, z0); | | 107| 3.44k| r4 = y4; | | 108| 3.44k| y4 = _mm_slli_epi32(y4, 7); | | 109| 3.44k| z4 = _mm_xor_si128(z4, y4); | | 110| 3.44k| r4 = _mm_srli_epi32(r4, 25); | | 111| 3.44k| z4 = _mm_xor_si128(z4, r4); | | 112| | | | 113| 3.44k| y9 = z1; | | 114| 3.44k| y9 = _mm_add_epi32(y9, z5); | | 115| 3.44k| r9 = y9; | | 116| 3.44k| y9 = _mm_slli_epi32(y9, 7); | | 117| 3.44k| z9 = _mm_xor_si128(z9, y9); | | 118| 3.44k| r9 = _mm_srli_epi32(r9, 25); | | 119| 3.44k| z9 = _mm_xor_si128(z9, r9); | | 120| | | | 121| 3.44k| y8 = z0; | | 122| 3.44k| y8 = _mm_add_epi32(y8, z4); | | 123| 3.44k| r8 = y8; | | 124| 3.44k| y8 = _mm_slli_epi32(y8, 9); | | 125| 3.44k| z8 = _mm_xor_si128(z8, y8); | | 126| 3.44k| r8 = _mm_srli_epi32(r8, 23); | | 127| 3.44k| z8 = _mm_xor_si128(z8, r8); | | 128| | | | 129| 3.44k| y13 = z5; | | 130| 3.44k| y13 = _mm_add_epi32(y13, z9); | | 131| 3.44k| r13 = y13; | | 132| 3.44k| y13 = _mm_slli_epi32(y13, 9); | | 133| 3.44k| z13 = _mm_xor_si128(z13, y13); | | 134| 3.44k| r13 = _mm_srli_epi32(r13, 23); | | 135| 3.44k| z13 = _mm_xor_si128(z13, r13); | | 136| | | | 137| 3.44k| y12 = z4; | | 138| 3.44k| y12 = _mm_add_epi32(y12, z8); | | 139| 3.44k| r12 = y12; | | 140| 3.44k| y12 = _mm_slli_epi32(y12, 13); | | 141| 3.44k| z12 = _mm_xor_si128(z12, y12); | | 142| 3.44k| r12 = _mm_srli_epi32(r12, 19); | | 143| 3.44k| z12 = _mm_xor_si128(z12, r12); | | 144| | | | 145| 3.44k| y1 = z9; | | 146| 3.44k| y1 = _mm_add_epi32(y1, z13); | | 147| 3.44k| r1 = y1; | | 148| 3.44k| y1 = _mm_slli_epi32(y1, 13); | | 149| 3.44k| z1 = _mm_xor_si128(z1, y1); | | 150| 3.44k| r1 = _mm_srli_epi32(r1, 19); | | 151| 3.44k| z1 = _mm_xor_si128(z1, r1); | | 152| | | | 153| 3.44k| y0 = z8; | | 154| 3.44k| y0 = _mm_add_epi32(y0, z12); | | 155| 3.44k| r0 = y0; | | 156| 3.44k| y0 = _mm_slli_epi32(y0, 18); | | 157| 3.44k| z0 = _mm_xor_si128(z0, y0); | | 158| 3.44k| r0 = _mm_srli_epi32(r0, 14); | | 159| 3.44k| z0 = _mm_xor_si128(z0, r0); | | 160| | | | 161| 3.44k| y5 = z13; | | 162| 3.44k| y5 = _mm_add_epi32(y5, z1); | | 163| 3.44k| r5 = y5; | | 164| 3.44k| y5 = _mm_slli_epi32(y5, 18); | | 165| 3.44k| z5 = _mm_xor_si128(z5, y5); | | 166| 3.44k| r5 = _mm_srli_epi32(r5, 14); | | 167| 3.44k| z5 = _mm_xor_si128(z5, r5); | | 168| | | | 169| 3.44k| y14 = z6; | | 170| 3.44k| y14 = _mm_add_epi32(y14, z10); | | 171| 3.44k| r14 = y14; | | 172| 3.44k| y14 = _mm_slli_epi32(y14, 7); | | 173| 3.44k| z14 = _mm_xor_si128(z14, y14); | | 174| 3.44k| r14 = _mm_srli_epi32(r14, 25); | | 175| 3.44k| z14 = _mm_xor_si128(z14, r14); | | 176| | | | 177| 3.44k| y3 = z11; | | 178| 3.44k| y3 = _mm_add_epi32(y3, z15); | | 179| 3.44k| r3 = y3; | | 180| 3.44k| y3 = _mm_slli_epi32(y3, 7); | | 181| 3.44k| z3 = _mm_xor_si128(z3, y3); | | 182| 3.44k| r3 = _mm_srli_epi32(r3, 25); | | 183| 3.44k| z3 = _mm_xor_si128(z3, r3); | | 184| | | | 185| 3.44k| y2 = z10; | | 186| 3.44k| y2 = _mm_add_epi32(y2, z14); | | 187| 3.44k| r2 = y2; | | 188| 3.44k| y2 = _mm_slli_epi32(y2, 9); | | 189| 3.44k| z2 = _mm_xor_si128(z2, y2); | | 190| 3.44k| r2 = _mm_srli_epi32(r2, 23); | | 191| 3.44k| z2 = _mm_xor_si128(z2, r2); | | 192| | | | 193| 3.44k| y7 = z15; | | 194| 3.44k| y7 = _mm_add_epi32(y7, z3); | | 195| 3.44k| r7 = y7; | | 196| 3.44k| y7 = _mm_slli_epi32(y7, 9); | | 197| 3.44k| z7 = _mm_xor_si128(z7, y7); | | 198| 3.44k| r7 = _mm_srli_epi32(r7, 23); | | 199| 3.44k| z7 = _mm_xor_si128(z7, r7); | | 200| | | | 201| 3.44k| y6 = z14; | | 202| 3.44k| y6 = _mm_add_epi32(y6, z2); | | 203| 3.44k| r6 = y6; | | 204| 3.44k| y6 = _mm_slli_epi32(y6, 13); | | 205| 3.44k| z6 = _mm_xor_si128(z6, y6); | | 206| 3.44k| r6 = _mm_srli_epi32(r6, 19); | | 207| 3.44k| z6 = _mm_xor_si128(z6, r6); | | 208| | | | 209| 3.44k| y11 = z3; | | 210| 3.44k| y11 = _mm_add_epi32(y11, z7); | | 211| 3.44k| r11 = y11; | | 212| 3.44k| y11 = _mm_slli_epi32(y11, 13); | | 213| 3.44k| z11 = _mm_xor_si128(z11, y11); | | 214| 3.44k| r11 = _mm_srli_epi32(r11, 19); | | 215| 3.44k| z11 = _mm_xor_si128(z11, r11); | | 216| | | | 217| 3.44k| y10 = z2; | | 218| 3.44k| y10 = _mm_add_epi32(y10, z6); | | 219| 3.44k| r10 = y10; | | 220| 3.44k| y10 = _mm_slli_epi32(y10, 18); | | 221| 3.44k| z10 = _mm_xor_si128(z10, y10); | | 222| 3.44k| r10 = _mm_srli_epi32(r10, 14); | | 223| 3.44k| z10 = _mm_xor_si128(z10, r10); | | 224| | | | 225| 3.44k| y1 = z3; | | 226| 3.44k| y1 = _mm_add_epi32(y1, z0); | | 227| 3.44k| r1 = y1; | | 228| 3.44k| y1 = _mm_slli_epi32(y1, 7); | | 229| 3.44k| z1 = _mm_xor_si128(z1, y1); | | 230| 3.44k| r1 = _mm_srli_epi32(r1, 25); | | 231| 3.44k| z1 = _mm_xor_si128(z1, r1); | | 232| | | | 233| 3.44k| y15 = z7; | | 234| 3.44k| y15 = _mm_add_epi32(y15, z11); | | 235| 3.44k| r15 = y15; | | 236| 3.44k| y15 = _mm_slli_epi32(y15, 18); | | 237| 3.44k| z15 = _mm_xor_si128(z15, y15); | | 238| 3.44k| r15 = _mm_srli_epi32(r15, 14); | | 239| 3.44k| z15 = _mm_xor_si128(z15, r15); | | 240| | | | 241| 3.44k| y6 = z4; | | 242| 3.44k| y6 = _mm_add_epi32(y6, z5); | | 243| 3.44k| r6 = y6; | | 244| 3.44k| y6 = _mm_slli_epi32(y6, 7); | | 245| 3.44k| z6 = _mm_xor_si128(z6, y6); | | 246| 3.44k| r6 = _mm_srli_epi32(r6, 25); | | 247| 3.44k| z6 = _mm_xor_si128(z6, r6); | | 248| | | | 249| 3.44k| y2 = z0; | | 250| 3.44k| y2 = _mm_add_epi32(y2, z1); | | 251| 3.44k| r2 = y2; | | 252| 3.44k| y2 = _mm_slli_epi32(y2, 9); | | 253| 3.44k| z2 = _mm_xor_si128(z2, y2); | | 254| 3.44k| r2 = _mm_srli_epi32(r2, 23); | | 255| 3.44k| z2 = _mm_xor_si128(z2, r2); | | 256| | | | 257| 3.44k| y7 = z5; | | 258| 3.44k| y7 = _mm_add_epi32(y7, z6); | | 259| 3.44k| r7 = y7; | | 260| 3.44k| y7 = _mm_slli_epi32(y7, 9); | | 261| 3.44k| z7 = _mm_xor_si128(z7, y7); | | 262| 3.44k| r7 = _mm_srli_epi32(r7, 23); | | 263| 3.44k| z7 = _mm_xor_si128(z7, r7); | | 264| | | | 265| 3.44k| y3 = z1; | | 266| 3.44k| y3 = _mm_add_epi32(y3, z2); | | 267| 3.44k| r3 = y3; | | 268| 3.44k| y3 = _mm_slli_epi32(y3, 13); | | 269| 3.44k| z3 = _mm_xor_si128(z3, y3); | | 270| 3.44k| r3 = _mm_srli_epi32(r3, 19); | | 271| 3.44k| z3 = _mm_xor_si128(z3, r3); | | 272| | | | 273| 3.44k| y4 = z6; | | 274| 3.44k| y4 = _mm_add_epi32(y4, z7); | | 275| 3.44k| r4 = y4; | | 276| 3.44k| y4 = _mm_slli_epi32(y4, 13); | | 277| 3.44k| z4 = _mm_xor_si128(z4, y4); | | 278| 3.44k| r4 = _mm_srli_epi32(r4, 19); | | 279| 3.44k| z4 = _mm_xor_si128(z4, r4); | | 280| | | | 281| 3.44k| y0 = z2; | | 282| 3.44k| y0 = _mm_add_epi32(y0, z3); | | 283| 3.44k| r0 = y0; | | 284| 3.44k| y0 = _mm_slli_epi32(y0, 18); | | 285| 3.44k| z0 = _mm_xor_si128(z0, y0); | | 286| 3.44k| r0 = _mm_srli_epi32(r0, 14); | | 287| 3.44k| z0 = _mm_xor_si128(z0, r0); | | 288| | | | 289| 3.44k| y5 = z7; | | 290| 3.44k| y5 = _mm_add_epi32(y5, z4); | | 291| 3.44k| r5 = y5; | | 292| 3.44k| y5 = _mm_slli_epi32(y5, 18); | | 293| 3.44k| z5 = _mm_xor_si128(z5, y5); | | 294| 3.44k| r5 = _mm_srli_epi32(r5, 14); | | 295| 3.44k| z5 = _mm_xor_si128(z5, r5); | | 296| | | | 297| 3.44k| y11 = z9; | | 298| 3.44k| y11 = _mm_add_epi32(y11, z10); | | 299| 3.44k| r11 = y11; | | 300| 3.44k| y11 = _mm_slli_epi32(y11, 7); | | 301| 3.44k| z11 = _mm_xor_si128(z11, y11); | | 302| 3.44k| r11 = _mm_srli_epi32(r11, 25); | | 303| 3.44k| z11 = _mm_xor_si128(z11, r11); | | 304| | | | 305| 3.44k| y12 = z14; | | 306| 3.44k| y12 = _mm_add_epi32(y12, z15); | | 307| 3.44k| r12 = y12; | | 308| 3.44k| y12 = _mm_slli_epi32(y12, 7); | | 309| 3.44k| z12 = _mm_xor_si128(z12, y12); | | 310| 3.44k| r12 = _mm_srli_epi32(r12, 25); | | 311| 3.44k| z12 = _mm_xor_si128(z12, r12); | | 312| | | | 313| 3.44k| y8 = z10; | | 314| 3.44k| y8 = _mm_add_epi32(y8, z11); | | 315| 3.44k| r8 = y8; | | 316| 3.44k| y8 = _mm_slli_epi32(y8, 9); | | 317| 3.44k| z8 = _mm_xor_si128(z8, y8); | | 318| 3.44k| r8 = _mm_srli_epi32(r8, 23); | | 319| 3.44k| z8 = _mm_xor_si128(z8, r8); | | 320| | | | 321| 3.44k| y13 = z15; | | 322| 3.44k| y13 = _mm_add_epi32(y13, z12); | | 323| 3.44k| r13 = y13; | | 324| 3.44k| y13 = _mm_slli_epi32(y13, 9); | | 325| 3.44k| z13 = _mm_xor_si128(z13, y13); | | 326| 3.44k| r13 = _mm_srli_epi32(r13, 23); | | 327| 3.44k| z13 = _mm_xor_si128(z13, r13); | | 328| | | | 329| 3.44k| y9 = z11; | | 330| 3.44k| y9 = _mm_add_epi32(y9, z8); | | 331| 3.44k| r9 = y9; | | 332| 3.44k| y9 = _mm_slli_epi32(y9, 13); | | 333| 3.44k| z9 = _mm_xor_si128(z9, y9); | | 334| 3.44k| r9 = _mm_srli_epi32(r9, 19); | | 335| 3.44k| z9 = _mm_xor_si128(z9, r9); | | 336| | | | 337| 3.44k| y14 = z12; | | 338| 3.44k| y14 = _mm_add_epi32(y14, z13); | | 339| 3.44k| r14 = y14; | | 340| 3.44k| y14 = _mm_slli_epi32(y14, 13); | | 341| 3.44k| z14 = _mm_xor_si128(z14, y14); | | 342| 3.44k| r14 = _mm_srli_epi32(r14, 19); | | 343| 3.44k| z14 = _mm_xor_si128(z14, r14); | | 344| | | | 345| 3.44k| y10 = z8; | | 346| 3.44k| y10 = _mm_add_epi32(y10, z9); | | 347| 3.44k| r10 = y10; | | 348| 3.44k| y10 = _mm_slli_epi32(y10, 18); | | 349| 3.44k| z10 = _mm_xor_si128(z10, y10); | | 350| 3.44k| r10 = _mm_srli_epi32(r10, 14); | | 351| 3.44k| z10 = _mm_xor_si128(z10, r10); | | 352| | | | 353| 3.44k| y15 = z13; | | 354| 3.44k| y15 = _mm_add_epi32(y15, z14); | | 355| 3.44k| r15 = y15; | | 356| 3.44k| y15 = _mm_slli_epi32(y15, 18); | | 357| 3.44k| z15 = _mm_xor_si128(z15, y15); | | 358| 3.44k| r15 = _mm_srli_epi32(r15, 14); | | 359| 3.44k| z15 = _mm_xor_si128(z15, r15); | | 360| 3.44k| } | | 361| | | | 362| |/* store data ; this macro replicates the original amd64-xmm6 code */ | | 363| 344|#define ONEQUAD_SHUFFLE(A, B, C, D) \ | | 364| 344| z##A = _mm_add_epi32(z##A, orig##A); \ | | 365| 344| z##B = _mm_add_epi32(z##B, orig##B); \ | | 366| 344| z##C = _mm_add_epi32(z##C, orig##C); \ | | 367| 344| z##D = _mm_add_epi32(z##D, orig##D); \ | | 368| 344| in##A = _mm_cvtsi128_si32(z##A); \ | | 369| 344| in##B = _mm_cvtsi128_si32(z##B); \ | | 370| 344| in##C = _mm_cvtsi128_si32(z##C); \ | | 371| 344| in##D = _mm_cvtsi128_si32(z##D); \ | | 372| 344| z##A = _mm_shuffle_epi32(z##A, 0x39); \ | | 373| 344| z##B = _mm_shuffle_epi32(z##B, 0x39); \ | | 374| 344| z##C = _mm_shuffle_epi32(z##C, 0x39); \ | | 375| 344| z##D = _mm_shuffle_epi32(z##D, 0x39); \ | | 376| 344| \ | | 377| 344| in##A ^= *(uint32_t *) (m + 0); \ | | 378| 344| in##B ^= *(uint32_t *) (m + 4); \ | | 379| 344| in##C ^= *(uint32_t *) (m + 8); \ | | 380| 344| in##D ^= *(uint32_t *) (m + 12); \ | | 381| 344| \ | | 382| 344| *(uint32_t *) (c + 0) = in##A; \ | | 383| 344| *(uint32_t *) (c + 4) = in##B; \ | | 384| 344| *(uint32_t *) (c + 8) = in##C; \ | | 385| 344| *(uint32_t *) (c + 12) = in##D; \ | | 386| 344| \ | | 387| 344| in##A = _mm_cvtsi128_si32(z##A); \ | | 388| 344| in##B = _mm_cvtsi128_si32(z##B); \ | | 389| 344| in##C = _mm_cvtsi128_si32(z##C); \ | | 390| 344| in##D = _mm_cvtsi128_si32(z##D); \ | | 391| 344| z##A = _mm_shuffle_epi32(z##A, 0x39); \ | | 392| 344| z##B = _mm_shuffle_epi32(z##B, 0x39); \ | | 393| 344| z##C = _mm_shuffle_epi32(z##C, 0x39); \ | | 394| 344| z##D = _mm_shuffle_epi32(z##D, 0x39); \ | | 395| 344| \ | | 396| 344| in##A ^= *(uint32_t *) (m + 64); \ | | 397| 344| in##B ^= *(uint32_t *) (m + 68); \ | | 398| 344| in##C ^= *(uint32_t *) (m + 72); \ | | 399| 344| in##D ^= *(uint32_t *) (m + 76); \ | | 400| 344| *(uint32_t *) (c + 64) = in##A; \ | | 401| 344| *(uint32_t *) (c + 68) = in##B; \ | | 402| 344| *(uint32_t *) (c + 72) = in##C; \ | | 403| 344| *(uint32_t *) (c + 76) = in##D; \ | | 404| 344| \ | | 405| 344| in##A = _mm_cvtsi128_si32(z##A); \ | | 406| 344| in##B = _mm_cvtsi128_si32(z##B); \ | | 407| 344| in##C = _mm_cvtsi128_si32(z##C); \ | | 408| 344| in##D = _mm_cvtsi128_si32(z##D); \ | | 409| 344| z##A = _mm_shuffle_epi32(z##A, 0x39); \ | | 410| 344| z##B = _mm_shuffle_epi32(z##B, 0x39); \ | | 411| 344| z##C = _mm_shuffle_epi32(z##C, 0x39); \ | | 412| 344| z##D = _mm_shuffle_epi32(z##D, 0x39); \ | | 413| 344| \ | | 414| 344| in##A ^= *(uint32_t *) (m + 128); \ | | 415| 344| in##B ^= *(uint32_t *) (m + 132); \ | | 416| 344| in##C ^= *(uint32_t *) (m + 136); \ | | 417| 344| in##D ^= *(uint32_t *) (m + 140); \ | | 418| 344| *(uint32_t *) (c + 128) = in##A; \ | | 419| 344| *(uint32_t *) (c + 132) = in##B; \ | | 420| 344| *(uint32_t *) (c + 136) = in##C; \ | | 421| 344| *(uint32_t *) (c + 140) = in##D; \ | | 422| 344| \ | | 423| 344| in##A = _mm_cvtsi128_si32(z##A); \ | | 424| 344| in##B = _mm_cvtsi128_si32(z##B); \ | | 425| 344| in##C = _mm_cvtsi128_si32(z##C); \ | | 426| 344| in##D = _mm_cvtsi128_si32(z##D); \ | | 427| 344| \ | | 428| 344| in##A ^= *(uint32_t *) (m + 192); \ | | 429| 344| in##B ^= *(uint32_t *) (m + 196); \ | | 430| 344| in##C ^= *(uint32_t *) (m + 200); \ | | 431| 344| in##D ^= *(uint32_t *) (m + 204); \ | | 432| 344| *(uint32_t *) (c + 192) = in##A; \ | | 433| 344| *(uint32_t *) (c + 196) = in##B; \ | | 434| 344| *(uint32_t *) (c + 200) = in##C; \ | | 435| 344| *(uint32_t *) (c + 204) = in##D | | 436| | | | 437| |/* store data ; this macro replaces shuffle+mov by a direct extract; not much | | 438| | * difference */ | | 439| 344|#define ONEQUAD_EXTRACT(A, B, C, D) \ | | 440| 344| z##A = _mm_add_epi32(z##A, orig##A); \ | | 441| 344| z##B = _mm_add_epi32(z##B, orig##B); \ | | 442| 344| z##C = _mm_add_epi32(z##C, orig##C); \ | | 443| 344| z##D = _mm_add_epi32(z##D, orig##D); \ | | 444| 344| in##A = _mm_cvtsi128_si32(z##A); \ | | 445| 344| in##B = _mm_cvtsi128_si32(z##B); \ | | 446| 344| in##C = _mm_cvtsi128_si32(z##C); \ | | 447| 344| in##D = _mm_cvtsi128_si32(z##D); \ | | 448| 344| in##A ^= *(uint32_t *) (m + 0); \ | | 449| 344| in##B ^= *(uint32_t *) (m + 4); \ | | 450| 344| in##C ^= *(uint32_t *) (m + 8); \ | | 451| 344| in##D ^= *(uint32_t *) (m + 12); \ | | 452| 344| *(uint32_t *) (c + 0) = in##A; \ | | 453| 344| *(uint32_t *) (c + 4) = in##B; \ | | 454| 344| *(uint32_t *) (c + 8) = in##C; \ | | 455| 344| *(uint32_t *) (c + 12) = in##D; \ | | 456| 344| \ | | 457| 344| in##A = _mm_extract_epi32(z##A, 1); \ | | 458| 344| in##B = _mm_extract_epi32(z##B, 1); \ | | 459| 344| in##C = _mm_extract_epi32(z##C, 1); \ | | 460| 344| in##D = _mm_extract_epi32(z##D, 1); \ | | 461| 344| \ | | 462| 344| in##A ^= *(uint32_t *) (m + 64); \ | | 463| 344| in##B ^= *(uint32_t *) (m + 68); \ | | 464| 344| in##C ^= *(uint32_t *) (m + 72); \ | | 465| 344| in##D ^= *(uint32_t *) (m + 76); \ | | 466| 344| *(uint32_t *) (c + 64) = in##A; \ | | 467| 344| *(uint32_t *) (c + 68) = in##B; \ | | 468| 344| *(uint32_t *) (c + 72) = in##C; \ | | 469| 344| *(uint32_t *) (c + 76) = in##D; \ | | 470| 344| \ | | 471| 344| in##A = _mm_extract_epi32(z##A, 2); \ | | 472| 344| in##B = _mm_extract_epi32(z##B, 2); \ | | 473| 344| in##C = _mm_extract_epi32(z##C, 2); \ | | 474| 344| in##D = _mm_extract_epi32(z##D, 2); \ | | 475| 344| \ | | 476| 344| in##A ^= *(uint32_t *) (m + 128); \ | | 477| 344| in##B ^= *(uint32_t *) (m + 132); \ | | 478| 344| in##C ^= *(uint32_t *) (m + 136); \ | | 479| 344| in##D ^= *(uint32_t *) (m + 140); \ | | 480| 344| *(uint32_t *) (c + 128) = in##A; \ | | 481| 344| *(uint32_t *) (c + 132) = in##B; \ | | 482| 344| *(uint32_t *) (c + 136) = in##C; \ | | 483| 344| *(uint32_t *) (c + 140) = in##D; \ | | 484| 344| \ | | 485| 344| in##A = _mm_extract_epi32(z##A, 3); \ | | 486| 344| in##B = _mm_extract_epi32(z##B, 3); \ | | 487| 344| in##C = _mm_extract_epi32(z##C, 3); \ | | 488| 344| in##D = _mm_extract_epi32(z##D, 3); \ | | 489| 344| \ | | 490| 344| in##A ^= *(uint32_t *) (m + 192); \ | | 491| 344| in##B ^= *(uint32_t *) (m + 196); \ | | 492| 344| in##C ^= *(uint32_t *) (m + 200); \ | | 493| 344| in##D ^= *(uint32_t *) (m + 204); \ | | 494| 344| *(uint32_t *) (c + 192) = in##A; \ | | 495| 344| *(uint32_t *) (c + 196) = in##B; \ | | 496| 344| *(uint32_t *) (c + 200) = in##C; \ | | 497| 344| *(uint32_t *) (c + 204) = in##D | | 498| | | | 499| |/* store data ; this macro first transpose data in-registers, and then store | | 500| | * them in memory. much faster with icc. */ | | 501| 344|#define ONEQUAD_TRANSPOSE(A, B, C, D) \ | | 502| 344| z##A = _mm_add_epi32(z##A, orig##A); \ | | 503| 344| z##B = _mm_add_epi32(z##B, orig##B); \ | | 504| 344| z##C = _mm_add_epi32(z##C, orig##C); \ | | 505| 344| z##D = _mm_add_epi32(z##D, orig##D); \ | | 506| 344| y##A = _mm_unpacklo_epi32(z##A, z##B); \ | | 507| 344| y##B = _mm_unpacklo_epi32(z##C, z##D); \ | | 508| 344| y##C = _mm_unpackhi_epi32(z##A, z##B); \ | | 509| 344| y##D = _mm_unpackhi_epi32(z##C, z##D); \ | | 510| 344| z##A = _mm_unpacklo_epi64(y##A, y##B); \ | | 511| 344| z##B = _mm_unpackhi_epi64(y##A, y##B); \ | | 512| 344| z##C = _mm_unpacklo_epi64(y##C, y##D); \ | | 513| 344| z##D = _mm_unpackhi_epi64(y##C, y##D); \ | | 514| 344| y##A = _mm_xor_si128(z##A, _mm_loadu_si128((const __m128i *) (m + 0))); \ | | 515| 344| _mm_storeu_si128((__m128i *) (c + 0), y##A); \ | | 516| 344| y##B = _mm_xor_si128(z##B, _mm_loadu_si128((const __m128i *) (m + 64))); \ | | 517| 344| _mm_storeu_si128((__m128i *) (c + 64), y##B); \ | | 518| 344| y##C = _mm_xor_si128(z##C, _mm_loadu_si128((const __m128i *) (m + 128))); \ | | 519| 344| _mm_storeu_si128((__m128i *) (c + 128), y##C); \ | | 520| 344| y##D = _mm_xor_si128(z##D, _mm_loadu_si128((const __m128i *) (m + 192))); \ | | 521| 344| _mm_storeu_si128((__m128i *) (c + 192), y##D) | | 522| | | | 523| 344|#define ONEQUAD(A, B, C, D) ONEQUAD_TRANSPOSE(A, B, C, D) | | 524| | | | 525| 344| ONEQUAD(0, 1, 2, 3); | | ------------------ | | | | 523| 344|#define ONEQUAD(A, B, C, D) ONEQUAD_TRANSPOSE(A, B, C, D) | | | | ------------------ | | | | | | 502| 344| z##A = _mm_add_epi32(z##A, orig##A); \ | | | | | | 503| 344| z##B = _mm_add_epi32(z##B, orig##B); \ | | | | | | 504| 344| z##C = _mm_add_epi32(z##C, orig##C); \ | | | | | | 505| 344| z##D = _mm_add_epi32(z##D, orig##D); \ | | | | | | 506| 344| y##A = _mm_unpacklo_epi32(z##A, z##B); \ | | | | | | 507| 344| y##B = _mm_unpacklo_epi32(z##C, z##D); \ | | | | | | 508| 344| y##C = _mm_unpackhi_epi32(z##A, z##B); \ | | | | | | 509| 344| y##D = _mm_unpackhi_epi32(z##C, z##D); \ | | | | | | 510| 344| z##A = _mm_unpacklo_epi64(y##A, y##B); \ | | | | | | 511| 344| z##B = _mm_unpackhi_epi64(y##A, y##B); \ | | | | | | 512| 344| z##C = _mm_unpacklo_epi64(y##C, y##D); \ | | | | | | 513| 344| z##D = _mm_unpackhi_epi64(y##C, y##D); \ | | | | | | 514| 344| y##A = _mm_xor_si128(z##A, _mm_loadu_si128((const __m128i *) (m + 0))); \ | | | | | | 515| 344| _mm_storeu_si128((__m128i *) (c + 0), y##A); \ | | | | | | 516| 344| y##B = _mm_xor_si128(z##B, _mm_loadu_si128((const __m128i *) (m + 64))); \ | | | | | | 517| 344| _mm_storeu_si128((__m128i *) (c + 64), y##B); \ | | | | | | 518| 344| y##C = _mm_xor_si128(z##C, _mm_loadu_si128((const __m128i *) (m + 128))); \ | | | | | | 519| 344| _mm_storeu_si128((__m128i *) (c + 128), y##C); \ | | | | | | 520| 344| y##D = _mm_xor_si128(z##D, _mm_loadu_si128((const __m128i *) (m + 192))); \ | | | | | | 521| 344| _mm_storeu_si128((__m128i *) (c + 192), y##D) | | | | ------------------ | | ------------------ | | 526| 344| m += 16; | | 527| 344| c += 16; | | 528| 344| ONEQUAD(4, 5, 6, 7); | | ------------------ | | | | 523| 344|#define ONEQUAD(A, B, C, D) ONEQUAD_TRANSPOSE(A, B, C, D) | | | | ------------------ | | | | | | 502| 344| z##A = _mm_add_epi32(z##A, orig##A); \ | | | | | | 503| 344| z##B = _mm_add_epi32(z##B, orig##B); \ | | | | | | 504| 344| z##C = _mm_add_epi32(z##C, orig##C); \ | | | | | | 505| 344| z##D = _mm_add_epi32(z##D, orig##D); \ | | | | | | 506| 344| y##A = _mm_unpacklo_epi32(z##A, z##B); \ | | | | | | 507| 344| y##B = _mm_unpacklo_epi32(z##C, z##D); \ | | | | | | 508| 344| y##C = _mm_unpackhi_epi32(z##A, z##B); \ | | | | | | 509| 344| y##D = _mm_unpackhi_epi32(z##C, z##D); \ | | | | | | 510| 344| z##A = _mm_unpacklo_epi64(y##A, y##B); \ | | | | | | 511| 344| z##B = _mm_unpackhi_epi64(y##A, y##B); \ | | | | | | 512| 344| z##C = _mm_unpacklo_epi64(y##C, y##D); \ | | | | | | 513| 344| z##D = _mm_unpackhi_epi64(y##C, y##D); \ | | | | | | 514| 344| y##A = _mm_xor_si128(z##A, _mm_loadu_si128((const __m128i *) (m + 0))); \ | | | | | | 515| 344| _mm_storeu_si128((__m128i *) (c + 0), y##A); \ | | | | | | 516| 344| y##B = _mm_xor_si128(z##B, _mm_loadu_si128((const __m128i *) (m + 64))); \ | | | | | | 517| 344| _mm_storeu_si128((__m128i *) (c + 64), y##B); \ | | | | | | 518| 344| y##C = _mm_xor_si128(z##C, _mm_loadu_si128((const __m128i *) (m + 128))); \ | | | | | | 519| 344| _mm_storeu_si128((__m128i *) (c + 128), y##C); \ | | | | | | 520| 344| y##D = _mm_xor_si128(z##D, _mm_loadu_si128((const __m128i *) (m + 192))); \ | | | | | | 521| 344| _mm_storeu_si128((__m128i *) (c + 192), y##D) | | | | ------------------ | | ------------------ | | 529| 344| m += 16; | | 530| 344| c += 16; | | 531| 344| ONEQUAD(8, 9, 10, 11); | | ------------------ | | | | 523| 344|#define ONEQUAD(A, B, C, D) ONEQUAD_TRANSPOSE(A, B, C, D) | | | | ------------------ | | | | | | 502| 344| z##A = _mm_add_epi32(z##A, orig##A); \ | | | | | | 503| 344| z##B = _mm_add_epi32(z##B, orig##B); \ | | | | | | 504| 344| z##C = _mm_add_epi32(z##C, orig##C); \ | | | | | | 505| 344| z##D = _mm_add_epi32(z##D, orig##D); \ | | | | | | 506| 344| y##A = _mm_unpacklo_epi32(z##A, z##B); \ | | | | | | 507| 344| y##B = _mm_unpacklo_epi32(z##C, z##D); \ | | | | | | 508| 344| y##C = _mm_unpackhi_epi32(z##A, z##B); \ | | | | | | 509| 344| y##D = _mm_unpackhi_epi32(z##C, z##D); \ | | | | | | 510| 344| z##A = _mm_unpacklo_epi64(y##A, y##B); \ | | | | | | 511| 344| z##B = _mm_unpackhi_epi64(y##A, y##B); \ | | | | | | 512| 344| z##C = _mm_unpacklo_epi64(y##C, y##D); \ | | | | | | 513| 344| z##D = _mm_unpackhi_epi64(y##C, y##D); \ | | | | | | 514| 344| y##A = _mm_xor_si128(z##A, _mm_loadu_si128((const __m128i *) (m + 0))); \ | | | | | | 515| 344| _mm_storeu_si128((__m128i *) (c + 0), y##A); \ | | | | | | 516| 344| y##B = _mm_xor_si128(z##B, _mm_loadu_si128((const __m128i *) (m + 64))); \ | | | | | | 517| 344| _mm_storeu_si128((__m128i *) (c + 64), y##B); \ | | | | | | 518| 344| y##C = _mm_xor_si128(z##C, _mm_loadu_si128((const __m128i *) (m + 128))); \ | | | | | | 519| 344| _mm_storeu_si128((__m128i *) (c + 128), y##C); \ | | | | | | 520| 344| y##D = _mm_xor_si128(z##D, _mm_loadu_si128((const __m128i *) (m + 192))); \ | | | | | | 521| 344| _mm_storeu_si128((__m128i *) (c + 192), y##D) | | | | ------------------ | | ------------------ | | 532| 344| m += 16; | | 533| 344| c += 16; | | 534| 344| ONEQUAD(12, 13, 14, 15); | | ------------------ | | | | 523| 344|#define ONEQUAD(A, B, C, D) ONEQUAD_TRANSPOSE(A, B, C, D) | | | | ------------------ | | | | | | 502| 344| z##A = _mm_add_epi32(z##A, orig##A); \ | | | | | | 503| 344| z##B = _mm_add_epi32(z##B, orig##B); \ | | | | | | 504| 344| z##C = _mm_add_epi32(z##C, orig##C); \ | | | | | | 505| 344| z##D = _mm_add_epi32(z##D, orig##D); \ | | | | | | 506| 344| y##A = _mm_unpacklo_epi32(z##A, z##B); \ | | | | | | 507| 344| y##B = _mm_unpacklo_epi32(z##C, z##D); \ | | | | | | 508| 344| y##C = _mm_unpackhi_epi32(z##A, z##B); \ | | | | | | 509| 344| y##D = _mm_unpackhi_epi32(z##C, z##D); \ | | | | | | 510| 344| z##A = _mm_unpacklo_epi64(y##A, y##B); \ | | | | | | 511| 344| z##B = _mm_unpackhi_epi64(y##A, y##B); \ | | | | | | 512| 344| z##C = _mm_unpacklo_epi64(y##C, y##D); \ | | | | | | 513| 344| z##D = _mm_unpackhi_epi64(y##C, y##D); \ | | | | | | 514| 344| y##A = _mm_xor_si128(z##A, _mm_loadu_si128((const __m128i *) (m + 0))); \ | | | | | | 515| 344| _mm_storeu_si128((__m128i *) (c + 0), y##A); \ | | | | | | 516| 344| y##B = _mm_xor_si128(z##B, _mm_loadu_si128((const __m128i *) (m + 64))); \ | | | | | | 517| 344| _mm_storeu_si128((__m128i *) (c + 64), y##B); \ | | | | | | 518| 344| y##C = _mm_xor_si128(z##C, _mm_loadu_si128((const __m128i *) (m + 128))); \ | | | | | | 519| 344| _mm_storeu_si128((__m128i *) (c + 128), y##C); \ | | | | | | 520| 344| y##D = _mm_xor_si128(z##D, _mm_loadu_si128((const __m128i *) (m + 192))); \ | | | | | | 521| 344| _mm_storeu_si128((__m128i *) (c + 192), y##D) | | | | ------------------ | | ------------------ | | 535| 344| m -= 48; | | 536| 344| c -= 48; | | 537| | | | 538| 344|#undef ONEQUAD | | 539| 344|#undef ONEQUAD_TRANSPOSE | | 540| 344|#undef ONEQUAD_EXTRACT | | 541| 344|#undef ONEQUAD_SHUFFLE | | 542| | | | 543| 344| bytes -= 256; | | 544| 344| c += 256; | | 545| 344| m += 256; | | 546| 344| } | | 547| 344|} ------------------ 77| 1.46k|#include "u1.h" ------------------ | | 1| 2.98k|while (bytes >= 64) { | | ------------------ | | | Branch (1:8): [True: 1.52k, False: 1.46k] | | ------------------ | | 2| 1.52k| __m128i diag0 = _mm_loadu_si128((const __m128i *) (x + 0)); | | 3| 1.52k| __m128i diag1 = _mm_loadu_si128((const __m128i *) (x + 4)); | | 4| 1.52k| __m128i diag2 = _mm_loadu_si128((const __m128i *) (x + 8)); | | 5| 1.52k| __m128i diag3 = _mm_loadu_si128((const __m128i *) (x + 12)); | | 6| 1.52k| __m128i a0, a1, a2, a3, a4, a5, a6, a7; | | 7| 1.52k| __m128i b0, b1, b2, b3, b4, b5, b6, b7; | | 8| | | | 9| 1.52k| uint32_t in8; | | 10| 1.52k| uint32_t in9; | | 11| 1.52k| int i; | | 12| | | | 13| 1.52k| a0 = diag1; | | 14| 9.16k| for (i = 0; i < ROUNDS; i += 4) { | | ------------------ | | | | 29| 9.16k|# define ROUNDS 20 | | ------------------ | | | Branch (14:17): [True: 7.64k, False: 1.52k] | | ------------------ | | 15| 7.64k| a0 = _mm_add_epi32(a0, diag0); | | 16| 7.64k| a1 = diag0; | | 17| 7.64k| b0 = a0; | | 18| 7.64k| a0 = _mm_slli_epi32(a0, 7); | | 19| 7.64k| b0 = _mm_srli_epi32(b0, 25); | | 20| 7.64k| diag3 = _mm_xor_si128(diag3, a0); | | 21| | | | 22| 7.64k| diag3 = _mm_xor_si128(diag3, b0); | | 23| | | | 24| 7.64k| a1 = _mm_add_epi32(a1, diag3); | | 25| 7.64k| a2 = diag3; | | 26| 7.64k| b1 = a1; | | 27| 7.64k| a1 = _mm_slli_epi32(a1, 9); | | 28| 7.64k| b1 = _mm_srli_epi32(b1, 23); | | 29| 7.64k| diag2 = _mm_xor_si128(diag2, a1); | | 30| 7.64k| diag3 = _mm_shuffle_epi32(diag3, 0x93); | | 31| 7.64k| diag2 = _mm_xor_si128(diag2, b1); | | 32| | | | 33| 7.64k| a2 = _mm_add_epi32(a2, diag2); | | 34| 7.64k| a3 = diag2; | | 35| 7.64k| b2 = a2; | | 36| 7.64k| a2 = _mm_slli_epi32(a2, 13); | | 37| 7.64k| b2 = _mm_srli_epi32(b2, 19); | | 38| 7.64k| diag1 = _mm_xor_si128(diag1, a2); | | 39| 7.64k| diag2 = _mm_shuffle_epi32(diag2, 0x4e); | | 40| 7.64k| diag1 = _mm_xor_si128(diag1, b2); | | 41| | | | 42| 7.64k| a3 = _mm_add_epi32(a3, diag1); | | 43| 7.64k| a4 = diag3; | | 44| 7.64k| b3 = a3; | | 45| 7.64k| a3 = _mm_slli_epi32(a3, 18); | | 46| 7.64k| b3 = _mm_srli_epi32(b3, 14); | | 47| 7.64k| diag0 = _mm_xor_si128(diag0, a3); | | 48| 7.64k| diag1 = _mm_shuffle_epi32(diag1, 0x39); | | 49| 7.64k| diag0 = _mm_xor_si128(diag0, b3); | | 50| | | | 51| 7.64k| a4 = _mm_add_epi32(a4, diag0); | | 52| 7.64k| a5 = diag0; | | 53| 7.64k| b4 = a4; | | 54| 7.64k| a4 = _mm_slli_epi32(a4, 7); | | 55| 7.64k| b4 = _mm_srli_epi32(b4, 25); | | 56| 7.64k| diag1 = _mm_xor_si128(diag1, a4); | | 57| | | | 58| 7.64k| diag1 = _mm_xor_si128(diag1, b4); | | 59| | | | 60| 7.64k| a5 = _mm_add_epi32(a5, diag1); | | 61| 7.64k| a6 = diag1; | | 62| 7.64k| b5 = a5; | | 63| 7.64k| a5 = _mm_slli_epi32(a5, 9); | | 64| 7.64k| b5 = _mm_srli_epi32(b5, 23); | | 65| 7.64k| diag2 = _mm_xor_si128(diag2, a5); | | 66| 7.64k| diag1 = _mm_shuffle_epi32(diag1, 0x93); | | 67| 7.64k| diag2 = _mm_xor_si128(diag2, b5); | | 68| | | | 69| 7.64k| a6 = _mm_add_epi32(a6, diag2); | | 70| 7.64k| a7 = diag2; | | 71| 7.64k| b6 = a6; | | 72| 7.64k| a6 = _mm_slli_epi32(a6, 13); | | 73| 7.64k| b6 = _mm_srli_epi32(b6, 19); | | 74| 7.64k| diag3 = _mm_xor_si128(diag3, a6); | | 75| 7.64k| diag2 = _mm_shuffle_epi32(diag2, 0x4e); | | 76| 7.64k| diag3 = _mm_xor_si128(diag3, b6); | | 77| | | | 78| 7.64k| a7 = _mm_add_epi32(a7, diag3); | | 79| 7.64k| a0 = diag1; | | 80| 7.64k| b7 = a7; | | 81| 7.64k| a7 = _mm_slli_epi32(a7, 18); | | 82| 7.64k| b7 = _mm_srli_epi32(b7, 14); | | 83| 7.64k| diag0 = _mm_xor_si128(diag0, a7); | | 84| 7.64k| diag3 = _mm_shuffle_epi32(diag3, 0x39); | | 85| 7.64k| diag0 = _mm_xor_si128(diag0, b7); | | 86| | | | 87| 7.64k| a0 = _mm_add_epi32(a0, diag0); | | 88| 7.64k| a1 = diag0; | | 89| 7.64k| b0 = a0; | | 90| 7.64k| a0 = _mm_slli_epi32(a0, 7); | | 91| 7.64k| b0 = _mm_srli_epi32(b0, 25); | | 92| 7.64k| diag3 = _mm_xor_si128(diag3, a0); | | 93| | | | 94| 7.64k| diag3 = _mm_xor_si128(diag3, b0); | | 95| | | | 96| 7.64k| a1 = _mm_add_epi32(a1, diag3); | | 97| 7.64k| a2 = diag3; | | 98| 7.64k| b1 = a1; | | 99| 7.64k| a1 = _mm_slli_epi32(a1, 9); | | 100| 7.64k| b1 = _mm_srli_epi32(b1, 23); | | 101| 7.64k| diag2 = _mm_xor_si128(diag2, a1); | | 102| 7.64k| diag3 = _mm_shuffle_epi32(diag3, 0x93); | | 103| 7.64k| diag2 = _mm_xor_si128(diag2, b1); | | 104| | | | 105| 7.64k| a2 = _mm_add_epi32(a2, diag2); | | 106| 7.64k| a3 = diag2; | | 107| 7.64k| b2 = a2; | | 108| 7.64k| a2 = _mm_slli_epi32(a2, 13); | | 109| 7.64k| b2 = _mm_srli_epi32(b2, 19); | | 110| 7.64k| diag1 = _mm_xor_si128(diag1, a2); | | 111| 7.64k| diag2 = _mm_shuffle_epi32(diag2, 0x4e); | | 112| 7.64k| diag1 = _mm_xor_si128(diag1, b2); | | 113| | | | 114| 7.64k| a3 = _mm_add_epi32(a3, diag1); | | 115| 7.64k| a4 = diag3; | | 116| 7.64k| b3 = a3; | | 117| 7.64k| a3 = _mm_slli_epi32(a3, 18); | | 118| 7.64k| b3 = _mm_srli_epi32(b3, 14); | | 119| 7.64k| diag0 = _mm_xor_si128(diag0, a3); | | 120| 7.64k| diag1 = _mm_shuffle_epi32(diag1, 0x39); | | 121| 7.64k| diag0 = _mm_xor_si128(diag0, b3); | | 122| | | | 123| 7.64k| a4 = _mm_add_epi32(a4, diag0); | | 124| 7.64k| a5 = diag0; | | 125| 7.64k| b4 = a4; | | 126| 7.64k| a4 = _mm_slli_epi32(a4, 7); | | 127| 7.64k| b4 = _mm_srli_epi32(b4, 25); | | 128| 7.64k| diag1 = _mm_xor_si128(diag1, a4); | | 129| | | | 130| 7.64k| diag1 = _mm_xor_si128(diag1, b4); | | 131| | | | 132| 7.64k| a5 = _mm_add_epi32(a5, diag1); | | 133| 7.64k| a6 = diag1; | | 134| 7.64k| b5 = a5; | | 135| 7.64k| a5 = _mm_slli_epi32(a5, 9); | | 136| 7.64k| b5 = _mm_srli_epi32(b5, 23); | | 137| 7.64k| diag2 = _mm_xor_si128(diag2, a5); | | 138| 7.64k| diag1 = _mm_shuffle_epi32(diag1, 0x93); | | 139| 7.64k| diag2 = _mm_xor_si128(diag2, b5); | | 140| | | | 141| 7.64k| a6 = _mm_add_epi32(a6, diag2); | | 142| 7.64k| a7 = diag2; | | 143| 7.64k| b6 = a6; | | 144| 7.64k| a6 = _mm_slli_epi32(a6, 13); | | 145| 7.64k| b6 = _mm_srli_epi32(b6, 19); | | 146| 7.64k| diag3 = _mm_xor_si128(diag3, a6); | | 147| 7.64k| diag2 = _mm_shuffle_epi32(diag2, 0x4e); | | 148| 7.64k| diag3 = _mm_xor_si128(diag3, b6); | | 149| | | | 150| 7.64k| a7 = _mm_add_epi32(a7, diag3); | | 151| 7.64k| a0 = diag1; | | 152| 7.64k| b7 = a7; | | 153| 7.64k| a7 = _mm_slli_epi32(a7, 18); | | 154| 7.64k| b7 = _mm_srli_epi32(b7, 14); | | 155| 7.64k| diag0 = _mm_xor_si128(diag0, a7); | | 156| 7.64k| diag3 = _mm_shuffle_epi32(diag3, 0x39); | | 157| 7.64k| diag0 = _mm_xor_si128(diag0, b7); | | 158| 7.64k| } | | 159| | | | 160| 1.52k| diag0 = _mm_add_epi32(diag0, _mm_loadu_si128((const __m128i *) (x + 0))); | | 161| 1.52k| diag1 = _mm_add_epi32(diag1, _mm_loadu_si128((const __m128i *) (x + 4))); | | 162| 1.52k| diag2 = _mm_add_epi32(diag2, _mm_loadu_si128((const __m128i *) (x + 8))); | | 163| 1.52k| diag3 = _mm_add_epi32(diag3, _mm_loadu_si128((const __m128i *) (x + 12))); | | 164| | | | 165| 1.52k|#define ONEQUAD_SHUFFLE(A, B, C, D) \ | | 166| 1.52k| do { \ | | 167| 1.52k| uint32_t in##A = _mm_cvtsi128_si32(diag0); \ | | 168| 1.52k| uint32_t in##B = _mm_cvtsi128_si32(diag1); \ | | 169| 1.52k| uint32_t in##C = _mm_cvtsi128_si32(diag2); \ | | 170| 1.52k| uint32_t in##D = _mm_cvtsi128_si32(diag3); \ | | 171| 1.52k| diag0 = _mm_shuffle_epi32(diag0, 0x39); \ | | 172| 1.52k| diag1 = _mm_shuffle_epi32(diag1, 0x39); \ | | 173| 1.52k| diag2 = _mm_shuffle_epi32(diag2, 0x39); \ | | 174| 1.52k| diag3 = _mm_shuffle_epi32(diag3, 0x39); \ | | 175| 1.52k| in##A ^= *(const uint32_t *) (m + (A * 4)); \ | | 176| 1.52k| in##B ^= *(const uint32_t *) (m + (B * 4)); \ | | 177| 1.52k| in##C ^= *(const uint32_t *) (m + (C * 4)); \ | | 178| 1.52k| in##D ^= *(const uint32_t *) (m + (D * 4)); \ | | 179| 1.52k| *(uint32_t *) (c + (A * 4)) = in##A; \ | | 180| 1.52k| *(uint32_t *) (c + (B * 4)) = in##B; \ | | 181| 1.52k| *(uint32_t *) (c + (C * 4)) = in##C; \ | | 182| 1.52k| *(uint32_t *) (c + (D * 4)) = in##D; \ | | 183| 1.52k| } while (0) | | 184| | | | 185| 1.52k|#define ONEQUAD(A, B, C, D) ONEQUAD_SHUFFLE(A, B, C, D) | | 186| | | | 187| 1.52k| ONEQUAD(0, 12, 8, 4); | | ------------------ | | | | 185| 1.52k|#define ONEQUAD(A, B, C, D) ONEQUAD_SHUFFLE(A, B, C, D) | | | | ------------------ | | | | | | 166| 1.52k| do { \ | | | | | | 167| 1.52k| uint32_t in##A = _mm_cvtsi128_si32(diag0); \ | | | | | | 168| 1.52k| uint32_t in##B = _mm_cvtsi128_si32(diag1); \ | | | | | | 169| 1.52k| uint32_t in##C = _mm_cvtsi128_si32(diag2); \ | | | | | | 170| 1.52k| uint32_t in##D = _mm_cvtsi128_si32(diag3); \ | | | | | | 171| 1.52k| diag0 = _mm_shuffle_epi32(diag0, 0x39); \ | | | | | | 172| 1.52k| diag1 = _mm_shuffle_epi32(diag1, 0x39); \ | | | | | | 173| 1.52k| diag2 = _mm_shuffle_epi32(diag2, 0x39); \ | | | | | | 174| 1.52k| diag3 = _mm_shuffle_epi32(diag3, 0x39); \ | | | | | | 175| 1.52k| in##A ^= *(const uint32_t *) (m + (A * 4)); \ | | | | | | 176| 1.52k| in##B ^= *(const uint32_t *) (m + (B * 4)); \ | | | | | | 177| 1.52k| in##C ^= *(const uint32_t *) (m + (C * 4)); \ | | | | | | 178| 1.52k| in##D ^= *(const uint32_t *) (m + (D * 4)); \ | | | | | | 179| 1.52k| *(uint32_t *) (c + (A * 4)) = in##A; \ | | | | | | 180| 1.52k| *(uint32_t *) (c + (B * 4)) = in##B; \ | | | | | | 181| 1.52k| *(uint32_t *) (c + (C * 4)) = in##C; \ | | | | | | 182| 1.52k| *(uint32_t *) (c + (D * 4)) = in##D; \ | | | | | | 183| 1.52k| } while (0) | | | | | | ------------------ | | | | | | | Branch (183:14): [Folded - Ignored] | | | | | | ------------------ | | | | ------------------ | | ------------------ | | 188| 1.52k| ONEQUAD(5, 1, 13, 9); | | ------------------ | | | | 185| 1.52k|#define ONEQUAD(A, B, C, D) ONEQUAD_SHUFFLE(A, B, C, D) | | | | ------------------ | | | | | | 166| 1.52k| do { \ | | | | | | 167| 1.52k| uint32_t in##A = _mm_cvtsi128_si32(diag0); \ | | | | | | 168| 1.52k| uint32_t in##B = _mm_cvtsi128_si32(diag1); \ | | | | | | 169| 1.52k| uint32_t in##C = _mm_cvtsi128_si32(diag2); \ | | | | | | 170| 1.52k| uint32_t in##D = _mm_cvtsi128_si32(diag3); \ | | | | | | 171| 1.52k| diag0 = _mm_shuffle_epi32(diag0, 0x39); \ | | | | | | 172| 1.52k| diag1 = _mm_shuffle_epi32(diag1, 0x39); \ | | | | | | 173| 1.52k| diag2 = _mm_shuffle_epi32(diag2, 0x39); \ | | | | | | 174| 1.52k| diag3 = _mm_shuffle_epi32(diag3, 0x39); \ | | | | | | 175| 1.52k| in##A ^= *(const uint32_t *) (m + (A * 4)); \ | | | | | | 176| 1.52k| in##B ^= *(const uint32_t *) (m + (B * 4)); \ | | | | | | 177| 1.52k| in##C ^= *(const uint32_t *) (m + (C * 4)); \ | | | | | | 178| 1.52k| in##D ^= *(const uint32_t *) (m + (D * 4)); \ | | | | | | 179| 1.52k| *(uint32_t *) (c + (A * 4)) = in##A; \ | | | | | | 180| 1.52k| *(uint32_t *) (c + (B * 4)) = in##B; \ | | | | | | 181| 1.52k| *(uint32_t *) (c + (C * 4)) = in##C; \ | | | | | | 182| 1.52k| *(uint32_t *) (c + (D * 4)) = in##D; \ | | | | | | 183| 1.52k| } while (0) | | | | | | ------------------ | | | | | | | Branch (183:14): [Folded - Ignored] | | | | | | ------------------ | | | | ------------------ | | ------------------ | | 189| 1.52k| ONEQUAD(10, 6, 2, 14); | | ------------------ | | | | 185| 1.52k|#define ONEQUAD(A, B, C, D) ONEQUAD_SHUFFLE(A, B, C, D) | | | | ------------------ | | | | | | 166| 1.52k| do { \ | | | | | | 167| 1.52k| uint32_t in##A = _mm_cvtsi128_si32(diag0); \ | | | | | | 168| 1.52k| uint32_t in##B = _mm_cvtsi128_si32(diag1); \ | | | | | | 169| 1.52k| uint32_t in##C = _mm_cvtsi128_si32(diag2); \ | | | | | | 170| 1.52k| uint32_t in##D = _mm_cvtsi128_si32(diag3); \ | | | | | | 171| 1.52k| diag0 = _mm_shuffle_epi32(diag0, 0x39); \ | | | | | | 172| 1.52k| diag1 = _mm_shuffle_epi32(diag1, 0x39); \ | | | | | | 173| 1.52k| diag2 = _mm_shuffle_epi32(diag2, 0x39); \ | | | | | | 174| 1.52k| diag3 = _mm_shuffle_epi32(diag3, 0x39); \ | | | | | | 175| 1.52k| in##A ^= *(const uint32_t *) (m + (A * 4)); \ | | | | | | 176| 1.52k| in##B ^= *(const uint32_t *) (m + (B * 4)); \ | | | | | | 177| 1.52k| in##C ^= *(const uint32_t *) (m + (C * 4)); \ | | | | | | 178| 1.52k| in##D ^= *(const uint32_t *) (m + (D * 4)); \ | | | | | | 179| 1.52k| *(uint32_t *) (c + (A * 4)) = in##A; \ | | | | | | 180| 1.52k| *(uint32_t *) (c + (B * 4)) = in##B; \ | | | | | | 181| 1.52k| *(uint32_t *) (c + (C * 4)) = in##C; \ | | | | | | 182| 1.52k| *(uint32_t *) (c + (D * 4)) = in##D; \ | | | | | | 183| 1.52k| } while (0) | | | | | | ------------------ | | | | | | | Branch (183:14): [Folded - Ignored] | | | | | | ------------------ | | | | ------------------ | | ------------------ | | 190| 1.52k| ONEQUAD(15, 11, 7, 3); | | ------------------ | | | | 185| 1.52k|#define ONEQUAD(A, B, C, D) ONEQUAD_SHUFFLE(A, B, C, D) | | | | ------------------ | | | | | | 166| 1.52k| do { \ | | | | | | 167| 1.52k| uint32_t in##A = _mm_cvtsi128_si32(diag0); \ | | | | | | 168| 1.52k| uint32_t in##B = _mm_cvtsi128_si32(diag1); \ | | | | | | 169| 1.52k| uint32_t in##C = _mm_cvtsi128_si32(diag2); \ | | | | | | 170| 1.52k| uint32_t in##D = _mm_cvtsi128_si32(diag3); \ | | | | | | 171| 1.52k| diag0 = _mm_shuffle_epi32(diag0, 0x39); \ | | | | | | 172| 1.52k| diag1 = _mm_shuffle_epi32(diag1, 0x39); \ | | | | | | 173| 1.52k| diag2 = _mm_shuffle_epi32(diag2, 0x39); \ | | | | | | 174| 1.52k| diag3 = _mm_shuffle_epi32(diag3, 0x39); \ | | | | | | 175| 1.52k| in##A ^= *(const uint32_t *) (m + (A * 4)); \ | | | | | | 176| 1.52k| in##B ^= *(const uint32_t *) (m + (B * 4)); \ | | | | | | 177| 1.52k| in##C ^= *(const uint32_t *) (m + (C * 4)); \ | | | | | | 178| 1.52k| in##D ^= *(const uint32_t *) (m + (D * 4)); \ | | | | | | 179| 1.52k| *(uint32_t *) (c + (A * 4)) = in##A; \ | | | | | | 180| 1.52k| *(uint32_t *) (c + (B * 4)) = in##B; \ | | | | | | 181| 1.52k| *(uint32_t *) (c + (C * 4)) = in##C; \ | | | | | | 182| 1.52k| *(uint32_t *) (c + (D * 4)) = in##D; \ | | | | | | 183| 1.52k| } while (0) | | | | | | ------------------ | | | | | | | Branch (183:14): [Folded - Ignored] | | | | | | ------------------ | | | | ------------------ | | ------------------ | | 191| | | | 192| 1.52k|#undef ONEQUAD | | 193| 1.52k|#undef ONEQUAD_SHUFFLE | | 194| | | | 195| 1.52k| in8 = x[8]; | | 196| 1.52k| in9 = x[13]; | | 197| 1.52k| in8++; | | 198| 1.52k| if (in8 == 0) { | | ------------------ | | | Branch (198:9): [True: 0, False: 1.52k] | | ------------------ | | 199| 0| in9++; | | 200| 0| } | | 201| 1.52k| x[8] = in8; | | 202| 1.52k| x[13] = in9; | | 203| | | | 204| 1.52k| c += 64; | | 205| 1.52k| m += 64; | | 206| 1.52k| bytes -= 64; | | 207| 1.52k|} ------------------ 78| 1.46k|#include "u0.h" ------------------ | | 1| 1.46k|if (bytes > 0) { | | ------------------ | | | Branch (1:5): [True: 837, False: 624] | | ------------------ | | 2| 837| __m128i diag0 = _mm_loadu_si128((const __m128i *) (x + 0)); | | 3| 837| __m128i diag1 = _mm_loadu_si128((const __m128i *) (x + 4)); | | 4| 837| __m128i diag2 = _mm_loadu_si128((const __m128i *) (x + 8)); | | 5| 837| __m128i diag3 = _mm_loadu_si128((const __m128i *) (x + 12)); | | 6| 837| __m128i a0, a1, a2, a3, a4, a5, a6, a7; | | 7| 837| __m128i b0, b1, b2, b3, b4, b5, b6, b7; | | 8| 837| uint8_t partialblock[64] = { 0 }; | | 9| | | | 10| 837| unsigned int i; | | 11| | | | 12| 837| a0 = diag1; | | 13| 5.02k| for (i = 0; i < ROUNDS; i += 4) { | | ------------------ | | | | 29| 5.02k|# define ROUNDS 20 | | ------------------ | | | Branch (13:17): [True: 4.18k, False: 837] | | ------------------ | | 14| 4.18k| a0 = _mm_add_epi32(a0, diag0); | | 15| 4.18k| a1 = diag0; | | 16| 4.18k| b0 = a0; | | 17| 4.18k| a0 = _mm_slli_epi32(a0, 7); | | 18| 4.18k| b0 = _mm_srli_epi32(b0, 25); | | 19| 4.18k| diag3 = _mm_xor_si128(diag3, a0); | | 20| | | | 21| 4.18k| diag3 = _mm_xor_si128(diag3, b0); | | 22| | | | 23| 4.18k| a1 = _mm_add_epi32(a1, diag3); | | 24| 4.18k| a2 = diag3; | | 25| 4.18k| b1 = a1; | | 26| 4.18k| a1 = _mm_slli_epi32(a1, 9); | | 27| 4.18k| b1 = _mm_srli_epi32(b1, 23); | | 28| 4.18k| diag2 = _mm_xor_si128(diag2, a1); | | 29| 4.18k| diag3 = _mm_shuffle_epi32(diag3, 0x93); | | 30| 4.18k| diag2 = _mm_xor_si128(diag2, b1); | | 31| | | | 32| 4.18k| a2 = _mm_add_epi32(a2, diag2); | | 33| 4.18k| a3 = diag2; | | 34| 4.18k| b2 = a2; | | 35| 4.18k| a2 = _mm_slli_epi32(a2, 13); | | 36| 4.18k| b2 = _mm_srli_epi32(b2, 19); | | 37| 4.18k| diag1 = _mm_xor_si128(diag1, a2); | | 38| 4.18k| diag2 = _mm_shuffle_epi32(diag2, 0x4e); | | 39| 4.18k| diag1 = _mm_xor_si128(diag1, b2); | | 40| | | | 41| 4.18k| a3 = _mm_add_epi32(a3, diag1); | | 42| 4.18k| a4 = diag3; | | 43| 4.18k| b3 = a3; | | 44| 4.18k| a3 = _mm_slli_epi32(a3, 18); | | 45| 4.18k| b3 = _mm_srli_epi32(b3, 14); | | 46| 4.18k| diag0 = _mm_xor_si128(diag0, a3); | | 47| 4.18k| diag1 = _mm_shuffle_epi32(diag1, 0x39); | | 48| 4.18k| diag0 = _mm_xor_si128(diag0, b3); | | 49| | | | 50| 4.18k| a4 = _mm_add_epi32(a4, diag0); | | 51| 4.18k| a5 = diag0; | | 52| 4.18k| b4 = a4; | | 53| 4.18k| a4 = _mm_slli_epi32(a4, 7); | | 54| 4.18k| b4 = _mm_srli_epi32(b4, 25); | | 55| 4.18k| diag1 = _mm_xor_si128(diag1, a4); | | 56| | | | 57| 4.18k| diag1 = _mm_xor_si128(diag1, b4); | | 58| | | | 59| 4.18k| a5 = _mm_add_epi32(a5, diag1); | | 60| 4.18k| a6 = diag1; | | 61| 4.18k| b5 = a5; | | 62| 4.18k| a5 = _mm_slli_epi32(a5, 9); | | 63| 4.18k| b5 = _mm_srli_epi32(b5, 23); | | 64| 4.18k| diag2 = _mm_xor_si128(diag2, a5); | | 65| 4.18k| diag1 = _mm_shuffle_epi32(diag1, 0x93); | | 66| 4.18k| diag2 = _mm_xor_si128(diag2, b5); | | 67| | | | 68| 4.18k| a6 = _mm_add_epi32(a6, diag2); | | 69| 4.18k| a7 = diag2; | | 70| 4.18k| b6 = a6; | | 71| 4.18k| a6 = _mm_slli_epi32(a6, 13); | | 72| 4.18k| b6 = _mm_srli_epi32(b6, 19); | | 73| 4.18k| diag3 = _mm_xor_si128(diag3, a6); | | 74| 4.18k| diag2 = _mm_shuffle_epi32(diag2, 0x4e); | | 75| 4.18k| diag3 = _mm_xor_si128(diag3, b6); | | 76| | | | 77| 4.18k| a7 = _mm_add_epi32(a7, diag3); | | 78| 4.18k| a0 = diag1; | | 79| 4.18k| b7 = a7; | | 80| 4.18k| a7 = _mm_slli_epi32(a7, 18); | | 81| 4.18k| b7 = _mm_srli_epi32(b7, 14); | | 82| 4.18k| diag0 = _mm_xor_si128(diag0, a7); | | 83| 4.18k| diag3 = _mm_shuffle_epi32(diag3, 0x39); | | 84| 4.18k| diag0 = _mm_xor_si128(diag0, b7); | | 85| | | | 86| 4.18k| a0 = _mm_add_epi32(a0, diag0); | | 87| 4.18k| a1 = diag0; | | 88| 4.18k| b0 = a0; | | 89| 4.18k| a0 = _mm_slli_epi32(a0, 7); | | 90| 4.18k| b0 = _mm_srli_epi32(b0, 25); | | 91| 4.18k| diag3 = _mm_xor_si128(diag3, a0); | | 92| | | | 93| 4.18k| diag3 = _mm_xor_si128(diag3, b0); | | 94| | | | 95| 4.18k| a1 = _mm_add_epi32(a1, diag3); | | 96| 4.18k| a2 = diag3; | | 97| 4.18k| b1 = a1; | | 98| 4.18k| a1 = _mm_slli_epi32(a1, 9); | | 99| 4.18k| b1 = _mm_srli_epi32(b1, 23); | | 100| 4.18k| diag2 = _mm_xor_si128(diag2, a1); | | 101| 4.18k| diag3 = _mm_shuffle_epi32(diag3, 0x93); | | 102| 4.18k| diag2 = _mm_xor_si128(diag2, b1); | | 103| | | | 104| 4.18k| a2 = _mm_add_epi32(a2, diag2); | | 105| 4.18k| a3 = diag2; | | 106| 4.18k| b2 = a2; | | 107| 4.18k| a2 = _mm_slli_epi32(a2, 13); | | 108| 4.18k| b2 = _mm_srli_epi32(b2, 19); | | 109| 4.18k| diag1 = _mm_xor_si128(diag1, a2); | | 110| 4.18k| diag2 = _mm_shuffle_epi32(diag2, 0x4e); | | 111| 4.18k| diag1 = _mm_xor_si128(diag1, b2); | | 112| | | | 113| 4.18k| a3 = _mm_add_epi32(a3, diag1); | | 114| 4.18k| a4 = diag3; | | 115| 4.18k| b3 = a3; | | 116| 4.18k| a3 = _mm_slli_epi32(a3, 18); | | 117| 4.18k| b3 = _mm_srli_epi32(b3, 14); | | 118| 4.18k| diag0 = _mm_xor_si128(diag0, a3); | | 119| 4.18k| diag1 = _mm_shuffle_epi32(diag1, 0x39); | | 120| 4.18k| diag0 = _mm_xor_si128(diag0, b3); | | 121| | | | 122| 4.18k| a4 = _mm_add_epi32(a4, diag0); | | 123| 4.18k| a5 = diag0; | | 124| 4.18k| b4 = a4; | | 125| 4.18k| a4 = _mm_slli_epi32(a4, 7); | | 126| 4.18k| b4 = _mm_srli_epi32(b4, 25); | | 127| 4.18k| diag1 = _mm_xor_si128(diag1, a4); | | 128| | | | 129| 4.18k| diag1 = _mm_xor_si128(diag1, b4); | | 130| | | | 131| 4.18k| a5 = _mm_add_epi32(a5, diag1); | | 132| 4.18k| a6 = diag1; | | 133| 4.18k| b5 = a5; | | 134| 4.18k| a5 = _mm_slli_epi32(a5, 9); | | 135| 4.18k| b5 = _mm_srli_epi32(b5, 23); | | 136| 4.18k| diag2 = _mm_xor_si128(diag2, a5); | | 137| 4.18k| diag1 = _mm_shuffle_epi32(diag1, 0x93); | | 138| 4.18k| diag2 = _mm_xor_si128(diag2, b5); | | 139| | | | 140| 4.18k| a6 = _mm_add_epi32(a6, diag2); | | 141| 4.18k| a7 = diag2; | | 142| 4.18k| b6 = a6; | | 143| 4.18k| a6 = _mm_slli_epi32(a6, 13); | | 144| 4.18k| b6 = _mm_srli_epi32(b6, 19); | | 145| 4.18k| diag3 = _mm_xor_si128(diag3, a6); | | 146| 4.18k| diag2 = _mm_shuffle_epi32(diag2, 0x4e); | | 147| 4.18k| diag3 = _mm_xor_si128(diag3, b6); | | 148| | | | 149| 4.18k| a7 = _mm_add_epi32(a7, diag3); | | 150| 4.18k| a0 = diag1; | | 151| 4.18k| b7 = a7; | | 152| 4.18k| a7 = _mm_slli_epi32(a7, 18); | | 153| 4.18k| b7 = _mm_srli_epi32(b7, 14); | | 154| 4.18k| diag0 = _mm_xor_si128(diag0, a7); | | 155| 4.18k| diag3 = _mm_shuffle_epi32(diag3, 0x39); | | 156| 4.18k| diag0 = _mm_xor_si128(diag0, b7); | | 157| 4.18k| } | | 158| | | | 159| 837| diag0 = _mm_add_epi32(diag0, _mm_loadu_si128((const __m128i *) (x + 0))); | | 160| 837| diag1 = _mm_add_epi32(diag1, _mm_loadu_si128((const __m128i *) (x + 4))); | | 161| 837| diag2 = _mm_add_epi32(diag2, _mm_loadu_si128((const __m128i *) (x + 8))); | | 162| 837| diag3 = _mm_add_epi32(diag3, _mm_loadu_si128((const __m128i *) (x + 12))); | | 163| | | | 164| 837|#define ONEQUAD_SHUFFLE(A, B, C, D) \ | | 165| 837| do { \ | | 166| 837| uint32_t in##A = _mm_cvtsi128_si32(diag0); \ | | 167| 837| uint32_t in##B = _mm_cvtsi128_si32(diag1); \ | | 168| 837| uint32_t in##C = _mm_cvtsi128_si32(diag2); \ | | 169| 837| uint32_t in##D = _mm_cvtsi128_si32(diag3); \ | | 170| 837| diag0 = _mm_shuffle_epi32(diag0, 0x39); \ | | 171| 837| diag1 = _mm_shuffle_epi32(diag1, 0x39); \ | | 172| 837| diag2 = _mm_shuffle_epi32(diag2, 0x39); \ | | 173| 837| diag3 = _mm_shuffle_epi32(diag3, 0x39); \ | | 174| 837| *(uint32_t *) (partialblock + (A * 4)) = in##A; \ | | 175| 837| *(uint32_t *) (partialblock + (B * 4)) = in##B; \ | | 176| 837| *(uint32_t *) (partialblock + (C * 4)) = in##C; \ | | 177| 837| *(uint32_t *) (partialblock + (D * 4)) = in##D; \ | | 178| 837| } while (0) | | 179| | | | 180| 837|#define ONEQUAD(A, B, C, D) ONEQUAD_SHUFFLE(A, B, C, D) | | 181| | | | 182| 837| ONEQUAD(0, 12, 8, 4); | | ------------------ | | | | 180| 837|#define ONEQUAD(A, B, C, D) ONEQUAD_SHUFFLE(A, B, C, D) | | | | ------------------ | | | | | | 165| 837| do { \ | | | | | | 166| 837| uint32_t in##A = _mm_cvtsi128_si32(diag0); \ | | | | | | 167| 837| uint32_t in##B = _mm_cvtsi128_si32(diag1); \ | | | | | | 168| 837| uint32_t in##C = _mm_cvtsi128_si32(diag2); \ | | | | | | 169| 837| uint32_t in##D = _mm_cvtsi128_si32(diag3); \ | | | | | | 170| 837| diag0 = _mm_shuffle_epi32(diag0, 0x39); \ | | | | | | 171| 837| diag1 = _mm_shuffle_epi32(diag1, 0x39); \ | | | | | | 172| 837| diag2 = _mm_shuffle_epi32(diag2, 0x39); \ | | | | | | 173| 837| diag3 = _mm_shuffle_epi32(diag3, 0x39); \ | | | | | | 174| 837| *(uint32_t *) (partialblock + (A * 4)) = in##A; \ | | | | | | 175| 837| *(uint32_t *) (partialblock + (B * 4)) = in##B; \ | | | | | | 176| 837| *(uint32_t *) (partialblock + (C * 4)) = in##C; \ | | | | | | 177| 837| *(uint32_t *) (partialblock + (D * 4)) = in##D; \ | | | | | | 178| 837| } while (0) | | | | | | ------------------ | | | | | | | Branch (178:14): [Folded - Ignored] | | | | | | ------------------ | | | | ------------------ | | ------------------ | | 183| 837| ONEQUAD(5, 1, 13, 9); | | ------------------ | | | | 180| 837|#define ONEQUAD(A, B, C, D) ONEQUAD_SHUFFLE(A, B, C, D) | | | | ------------------ | | | | | | 165| 837| do { \ | | | | | | 166| 837| uint32_t in##A = _mm_cvtsi128_si32(diag0); \ | | | | | | 167| 837| uint32_t in##B = _mm_cvtsi128_si32(diag1); \ | | | | | | 168| 837| uint32_t in##C = _mm_cvtsi128_si32(diag2); \ | | | | | | 169| 837| uint32_t in##D = _mm_cvtsi128_si32(diag3); \ | | | | | | 170| 837| diag0 = _mm_shuffle_epi32(diag0, 0x39); \ | | | | | | 171| 837| diag1 = _mm_shuffle_epi32(diag1, 0x39); \ | | | | | | 172| 837| diag2 = _mm_shuffle_epi32(diag2, 0x39); \ | | | | | | 173| 837| diag3 = _mm_shuffle_epi32(diag3, 0x39); \ | | | | | | 174| 837| *(uint32_t *) (partialblock + (A * 4)) = in##A; \ | | | | | | 175| 837| *(uint32_t *) (partialblock + (B * 4)) = in##B; \ | | | | | | 176| 837| *(uint32_t *) (partialblock + (C * 4)) = in##C; \ | | | | | | 177| 837| *(uint32_t *) (partialblock + (D * 4)) = in##D; \ | | | | | | 178| 837| } while (0) | | | | | | ------------------ | | | | | | | Branch (178:14): [Folded - Ignored] | | | | | | ------------------ | | | | ------------------ | | ------------------ | | 184| 837| ONEQUAD(10, 6, 2, 14); | | ------------------ | | | | 180| 837|#define ONEQUAD(A, B, C, D) ONEQUAD_SHUFFLE(A, B, C, D) | | | | ------------------ | | | | | | 165| 837| do { \ | | | | | | 166| 837| uint32_t in##A = _mm_cvtsi128_si32(diag0); \ | | | | | | 167| 837| uint32_t in##B = _mm_cvtsi128_si32(diag1); \ | | | | | | 168| 837| uint32_t in##C = _mm_cvtsi128_si32(diag2); \ | | | | | | 169| 837| uint32_t in##D = _mm_cvtsi128_si32(diag3); \ | | | | | | 170| 837| diag0 = _mm_shuffle_epi32(diag0, 0x39); \ | | | | | | 171| 837| diag1 = _mm_shuffle_epi32(diag1, 0x39); \ | | | | | | 172| 837| diag2 = _mm_shuffle_epi32(diag2, 0x39); \ | | | | | | 173| 837| diag3 = _mm_shuffle_epi32(diag3, 0x39); \ | | | | | | 174| 837| *(uint32_t *) (partialblock + (A * 4)) = in##A; \ | | | | | | 175| 837| *(uint32_t *) (partialblock + (B * 4)) = in##B; \ | | | | | | 176| 837| *(uint32_t *) (partialblock + (C * 4)) = in##C; \ | | | | | | 177| 837| *(uint32_t *) (partialblock + (D * 4)) = in##D; \ | | | | | | 178| 837| } while (0) | | | | | | ------------------ | | | | | | | Branch (178:14): [Folded - Ignored] | | | | | | ------------------ | | | | ------------------ | | ------------------ | | 185| 837| ONEQUAD(15, 11, 7, 3); | | ------------------ | | | | 180| 837|#define ONEQUAD(A, B, C, D) ONEQUAD_SHUFFLE(A, B, C, D) | | | | ------------------ | | | | | | 165| 837| do { \ | | | | | | 166| 837| uint32_t in##A = _mm_cvtsi128_si32(diag0); \ | | | | | | 167| 837| uint32_t in##B = _mm_cvtsi128_si32(diag1); \ | | | | | | 168| 837| uint32_t in##C = _mm_cvtsi128_si32(diag2); \ | | | | | | 169| 837| uint32_t in##D = _mm_cvtsi128_si32(diag3); \ | | | | | | 170| 837| diag0 = _mm_shuffle_epi32(diag0, 0x39); \ | | | | | | 171| 837| diag1 = _mm_shuffle_epi32(diag1, 0x39); \ | | | | | | 172| 837| diag2 = _mm_shuffle_epi32(diag2, 0x39); \ | | | | | | 173| 837| diag3 = _mm_shuffle_epi32(diag3, 0x39); \ | | | | | | 174| 837| *(uint32_t *) (partialblock + (A * 4)) = in##A; \ | | | | | | 175| 837| *(uint32_t *) (partialblock + (B * 4)) = in##B; \ | | | | | | 176| 837| *(uint32_t *) (partialblock + (C * 4)) = in##C; \ | | | | | | 177| 837| *(uint32_t *) (partialblock + (D * 4)) = in##D; \ | | | | | | 178| 837| } while (0) | | | | | | ------------------ | | | | | | | Branch (178:14): [Folded - Ignored] | | | | | | ------------------ | | | | ------------------ | | ------------------ | | 186| | | | 187| 837|#undef ONEQUAD | | 188| 837|#undef ONEQUAD_SHUFFLE | | 189| | | | 190| 30.1k| for (i = 0; i < bytes; i++) { | | ------------------ | | | Branch (190:17): [True: 29.3k, False: 837] | | ------------------ | | 191| 29.3k| c[i] = m[i] ^ partialblock[i]; | | 192| 29.3k| } | | 193| | | | 194| 837| sodium_memzero(partialblock, sizeof partialblock); | | 195| 837|} ------------------ 79| 1.46k|} salsa20_xmm6int-avx2.c:stream_avx2_xor_ic: 104| 1.14k|{ 105| 1.14k| struct salsa_ctx ctx; 106| 1.14k| uint8_t ic_bytes[8]; 107| 1.14k| uint32_t ic_high; 108| 1.14k| uint32_t ic_low; 109| | 110| 1.14k| if (!mlen) { ------------------ | Branch (110:9): [True: 0, False: 1.14k] ------------------ 111| 0| return 0; 112| 0| } 113| 1.14k| ic_high = (uint32_t) (ic >> 32); 114| 1.14k| ic_low = (uint32_t) ic; 115| 1.14k| STORE32_LE(&ic_bytes[0], ic_low); ------------------ | | 117| 1.14k|#define STORE32_LE(DST, W) store32_le((DST), (W)) ------------------ 116| 1.14k| STORE32_LE(&ic_bytes[4], ic_high); ------------------ | | 117| 1.14k|#define STORE32_LE(DST, W) store32_le((DST), (W)) ------------------ 117| 1.14k| salsa_keysetup(&ctx, k); 118| 1.14k| salsa_ivsetup(&ctx, n, ic_bytes); 119| 1.14k| salsa20_encrypt_bytes(&ctx, m, c, mlen); 120| 1.14k| sodium_memzero(&ctx, sizeof ctx); 121| | 122| 1.14k| return 0; 123| 1.14k|} crypto_verify_16: 84| 319|{ 85| 319| return crypto_verify_n(x, y, crypto_verify_16_BYTES); ------------------ | | 11| 319|#define crypto_verify_16_BYTES 16U ------------------ 86| 319|} verify.c:crypto_verify_n: 37| 319|{ 38| 319| const __m128i zero = _mm_setzero_si128(); 39| 319| volatile __m128i v1, v2, z; 40| 319| volatile int m; 41| 319| int i; 42| | 43| 319| const volatile __m128i *volatile x = 44| 319| (const volatile __m128i *volatile) (const void *) x_; 45| 319| const volatile __m128i *volatile y = 46| 319| (const volatile __m128i *volatile) (const void *) y_; 47| 319| v1 = _mm_loadu_si128((const __m128i *) &x[0]); 48| 319| v2 = _mm_loadu_si128((const __m128i *) &y[0]); 49| 319| z = _mm_xor_si128(v1, v2); 50| 319| for (i = 1; i < n / 16; i++) { ------------------ | Branch (50:17): [True: 0, False: 319] ------------------ 51| 0| v1 = _mm_loadu_si128((const __m128i *) &x[i]); 52| 0| v2 = _mm_loadu_si128((const __m128i *) &y[i]); 53| 0| z = _mm_or_si128(z, _mm_xor_si128(v1, v2)); 54| 0| } 55| 319| m = _mm_movemask_epi8(_mm_cmpeq_epi32(z, zero)); 56| 319| v1 = zero; v2 = zero; z = zero; 57| | 58| 319| return (int) (((uint32_t) m + 1U) >> 16) - 1; 59| 319|} salsa20_xmm6int-avx2.c:load32_le: 103| 16.8k|{ 104| 16.8k|#ifdef NATIVE_LITTLE_ENDIAN 105| 16.8k| uint32_t w; 106| 16.8k| memcpy(&w, src, sizeof w); 107| 16.8k| return w; 108| |#else 109| | uint32_t w = (uint32_t) src[0]; 110| | w |= (uint32_t) src[1] << 8; 111| | w |= (uint32_t) src[2] << 16; 112| | w |= (uint32_t) src[3] << 24; 113| | return w; 114| |#endif 115| 16.8k|} salsa20_xmm6int-avx2.c:store32_le: 120| 2.28k|{ 121| 2.28k|#ifdef NATIVE_LITTLE_ENDIAN 122| 2.28k| memcpy(dst, &w, sizeof w); 123| |#else 124| | dst[0] = (uint8_t) w; w >>= 8; 125| | dst[1] = (uint8_t) w; w >>= 8; 126| | dst[2] = (uint8_t) w; w >>= 8; 127| | dst[3] = (uint8_t) w; 128| |#endif 129| 2.28k|} core_hsalsa20_ref2.c:load32_le: 103| 7.65k|{ 104| 7.65k|#ifdef NATIVE_LITTLE_ENDIAN 105| 7.65k| uint32_t w; 106| 7.65k| memcpy(&w, src, sizeof w); 107| 7.65k| return w; 108| |#else 109| | uint32_t w = (uint32_t) src[0]; 110| | w |= (uint32_t) src[1] << 8; 111| | w |= (uint32_t) src[2] << 16; 112| | w |= (uint32_t) src[3] << 24; 113| | return w; 114| |#endif 115| 7.65k|} core_hsalsa20_ref2.c:rotl32: 36| 204k|{ 37| 204k| return (x << b) | (x >> (32 - b)); 38| 204k|} core_hsalsa20_ref2.c:store32_le: 120| 5.10k|{ 121| 5.10k|#ifdef NATIVE_LITTLE_ENDIAN 122| 5.10k| memcpy(dst, &w, sizeof w); 123| |#else 124| | dst[0] = (uint8_t) w; w >>= 8; 125| | dst[1] = (uint8_t) w; w >>= 8; 126| | dst[2] = (uint8_t) w; w >>= 8; 127| | dst[3] = (uint8_t) w; 128| |#endif 129| 5.10k|} chacha20_dolbeau-avx2.c:load32_le: 103| 7.01k|{ 104| 7.01k|#ifdef NATIVE_LITTLE_ENDIAN 105| 7.01k| uint32_t w; 106| 7.01k| memcpy(&w, src, sizeof w); 107| 7.01k| return w; 108| |#else 109| | uint32_t w = (uint32_t) src[0]; 110| | w |= (uint32_t) src[1] << 8; 111| | w |= (uint32_t) src[2] << 16; 112| | w |= (uint32_t) src[3] << 24; 113| | return w; 114| |#endif 115| 7.01k|} randombytes_set_implementation: 48| 319|{ 49| 319| implementation = impl; 50| | 51| 319| return 0; 52| 319|} randombytes_implementation_name: 56| 319|{ 57| 319|#ifndef __EMSCRIPTEN__ 58| 319| randombytes_init_if_needed(); 59| 319| return implementation->implementation_name(); 60| |#else 61| | return "js"; 62| |#endif 63| 319|} randombytes_stir: 80| 2|{ 81| 2|#ifndef __EMSCRIPTEN__ 82| 2| randombytes_init_if_needed(); 83| 2| if (implementation->stir != NULL) { ------------------ | Branch (83:9): [True: 2, False: 0] ------------------ 84| 2| implementation->stir(); 85| 2| } 86| |#else 87| | EM_ASM({ 88| | if (Module.getRandomValue === undefined) { 89| | try { 90| | var window_ = 'object' === typeof window ? window : self; 91| | var crypto_ = typeof window_.crypto !== 'undefined' ? window_.crypto : window_.msCrypto; 92| | var randomValuesStandard = function() { 93| | var buf = new Uint32Array(1); 94| | crypto_.getRandomValues(buf); 95| | return buf[0] >>> 0; 96| | }; 97| | randomValuesStandard(); 98| | Module.getRandomValue = randomValuesStandard; 99| | } catch (e) { 100| | try { 101| | var crypto = require('crypto'); 102| | var randomValueNodeJS = function() { 103| | var buf = crypto['randomBytes'](4); 104| | return (buf[0] << 24 | buf[1] << 16 | buf[2] << 8 | buf[3]) >>> 0; 105| | }; 106| | randomValueNodeJS(); 107| | Module.getRandomValue = randomValueNodeJS; 108| | } catch (e) { 109| | throw 'No secure random number generator found'; 110| | } 111| | } 112| | } 113| | }); 114| |#endif 115| 2|} randombytes_buf: 144| 639|{ 145| 639|#ifndef __EMSCRIPTEN__ 146| 639| randombytes_init_if_needed(); 147| 639| if (size > (size_t) 0U) { ------------------ | Branch (147:9): [True: 639, False: 0] ------------------ 148| 639| implementation->buf(buf, size); 149| 639| } 150| |#else 151| | unsigned char *p = (unsigned char *) buf; 152| | size_t i; 153| | 154| | for (i = (size_t) 0U; i < size; i++) { 155| | p[i] = (unsigned char) randombytes_random(); 156| | } 157| |#endif 158| 639|} randombytes_buf_deterministic: 163| 638|{ 164| 638| static const unsigned char nonce[crypto_stream_chacha20_ietf_NONCEBYTES] = { 165| 638| 'L', 'i', 'b', 's', 'o', 'd', 'i', 'u', 'm', 'D', 'R', 'G' 166| 638| }; 167| | 168| 638| COMPILER_ASSERT(randombytes_SEEDBYTES == crypto_stream_chacha20_ietf_KEYBYTES); ------------------ | | 23| 638|#define COMPILER_ASSERT(X) (void) sizeof(char[(X) ? 1 : -1]) ------------------ 169| 638|#if SIZE_MAX > 0x4000000000ULL 170| 638| COMPILER_ASSERT(randombytes_BYTES_MAX <= 0x4000000000ULL); ------------------ | | 23| 638|#define COMPILER_ASSERT(X) (void) sizeof(char[(X) ? 1 : -1]) ------------------ 171| 638| if (size > 0x4000000000ULL) { ------------------ | Branch (171:9): [True: 0, False: 638] ------------------ 172| 0| sodium_misuse(); 173| 0| } 174| 638|#endif 175| 638| crypto_stream_chacha20_ietf((unsigned char *) buf, (unsigned long long) size, 176| 638| nonce, seed); 177| 638|} randombytes.c:randombytes_init_if_needed: 39| 960|{ 40| 960| if (implementation == NULL) { ------------------ | Branch (40:9): [True: 1, False: 959] ------------------ 41| 1| implementation = RANDOMBYTES_DEFAULT_IMPLEMENTATION; ------------------ | | 33| 1|# define RANDOMBYTES_DEFAULT_IMPLEMENTATION &randombytes_sysrandom_implementation; ------------------ 42| 1| randombytes_stir(); 43| 1| } 44| 960|} randombytes_sysrandom.c:randombytes_sysrandom_stir: 297| 2|{ 298| 2| if (stream.initialized == 0) { ------------------ | Branch (298:9): [True: 1, False: 1] ------------------ 299| 1| randombytes_sysrandom_init(); 300| 1| stream.initialized = 1; 301| 1| } 302| 2|} randombytes_sysrandom.c:randombytes_sysrandom_init: 264| 1|{ 265| 1| const int errno_save = errno; 266| | 267| 1|# ifdef HAVE_LINUX_COMPATIBLE_GETRANDOM 268| 1| { 269| 1| unsigned char fodder[16]; 270| | 271| 1| if (randombytes_linux_getrandom(fodder, sizeof fodder) == 0) { ------------------ | Branch (271:13): [True: 1, False: 0] ------------------ 272| 1| stream.getrandom_available = 1; 273| 1| errno = errno_save; 274| 1| return; 275| 1| } 276| 0| stream.getrandom_available = 0; 277| 0| } 278| 0|# endif 279| | 280| 0| if ((stream.random_data_source_fd = ------------------ | Branch (280:9): [True: 0, False: 0] ------------------ 281| 0| randombytes_sysrandom_random_dev_open()) == -1) { 282| 0| sodium_misuse(); /* LCOV_EXCL_LINE */ 283| 0| } 284| 0| errno = errno_save; 285| 0|} randombytes_sysrandom.c:randombytes_linux_getrandom: 242| 2|{ 243| 2| unsigned char *buf = (unsigned char *) buf_; 244| 2| size_t chunk_size = 256U; 245| | 246| 2| do { 247| 2| if (size < chunk_size) { ------------------ | Branch (247:13): [True: 2, False: 0] ------------------ 248| 2| chunk_size = size; 249| 2| assert(chunk_size > (size_t) 0U); 250| 2| } 251| 2| if (_randombytes_linux_getrandom(buf, chunk_size) != 0) { ------------------ | Branch (251:13): [True: 0, False: 2] ------------------ 252| 0| return -1; 253| 0| } 254| 2| size -= chunk_size; 255| 2| buf += chunk_size; 256| 2| } while (size > (size_t) 0U); ------------------ | Branch (256:14): [True: 0, False: 2] ------------------ 257| | 258| 2| return 0; 259| 2|} randombytes_sysrandom.c:_randombytes_linux_getrandom: 229| 2|{ 230| 2| int readnb; 231| | 232| 2| assert(size <= 256U); 233| 2| do { 234| 2| readnb = getrandom(buf, size, 0); 235| 2| } while (readnb < 0 && (errno == EINTR || errno == EAGAIN)); ------------------ | Branch (235:14): [True: 0, False: 2] | Branch (235:29): [True: 0, False: 0] | Branch (235:47): [True: 0, False: 0] ------------------ 236| | 237| 2| return (readnb == (int) size) - 1; 238| 2|} randombytes_sysrandom.c:randombytes_sysrandom_buf: 340| 1|{ 341| 1| randombytes_sysrandom_stir_if_needed(); 342| 1|# if defined(ULLONG_MAX) && defined(SIZE_MAX) 343| |# if SIZE_MAX > ULLONG_MAX 344| | /* coverity[result_independent_of_operands] */ 345| | assert(size <= ULLONG_MAX); 346| |# endif 347| 1|# endif 348| 1|# ifndef _WIN32 349| 1|# ifdef HAVE_LINUX_COMPATIBLE_GETRANDOM 350| 1| if (stream.getrandom_available != 0) { ------------------ | Branch (350:9): [True: 1, False: 0] ------------------ 351| 1| if (randombytes_linux_getrandom(buf, size) != 0) { ------------------ | Branch (351:13): [True: 0, False: 1] ------------------ 352| 0| sodium_misuse(); /* LCOV_EXCL_LINE */ 353| 0| } 354| 1| return; 355| 1| } 356| 0|# endif 357| 0| if (stream.random_data_source_fd == -1 || ------------------ | Branch (357:9): [True: 0, False: 0] ------------------ 358| 0| safe_read(stream.random_data_source_fd, buf, size) != (ssize_t) size) { ------------------ | Branch (358:9): [True: 0, False: 0] ------------------ 359| 0| sodium_misuse(); /* LCOV_EXCL_LINE */ 360| 0| } 361| |# else /* _WIN32 */ 362| | COMPILER_ASSERT(randombytes_BYTES_MAX <= 0xffffffffUL); 363| | if (size > (size_t) 0xffffffffUL) { 364| | sodium_misuse(); /* LCOV_EXCL_LINE */ 365| | } 366| | if (! RtlGenRandom((PVOID) buf, (ULONG) size)) { 367| | sodium_misuse(); /* LCOV_EXCL_LINE */ 368| | } 369| |# endif /* _WIN32 */ 370| 0|} randombytes_sysrandom.c:randombytes_sysrandom_stir_if_needed: 306| 1|{ 307| 1| if (stream.initialized == 0) { ------------------ | Branch (307:9): [True: 0, False: 1] ------------------ 308| 0| randombytes_sysrandom_stir(); 309| 0| } 310| 1|} sodium_init: 29| 638|{ 30| 638| if (sodium_crit_enter() != 0) { ------------------ | Branch (30:9): [True: 0, False: 638] ------------------ 31| 0| return -1; /* LCOV_EXCL_LINE */ 32| 0| } 33| 638| if (initialized != 0) { ------------------ | Branch (33:9): [True: 637, False: 1] ------------------ 34| 637| if (sodium_crit_leave() != 0) { ------------------ | Branch (34:13): [True: 0, False: 637] ------------------ 35| 0| return -1; /* LCOV_EXCL_LINE */ 36| 0| } 37| 637| return 1; 38| 637| } 39| 1| _sodium_runtime_get_cpu_features(); 40| 1| randombytes_stir(); 41| 1| _sodium_alloc_init(); 42| 1| _crypto_pwhash_argon2_pick_best_implementation(); 43| 1| _crypto_generichash_blake2b_pick_best_implementation(); 44| 1| _crypto_onetimeauth_poly1305_pick_best_implementation(); 45| 1| _crypto_scalarmult_curve25519_pick_best_implementation(); 46| 1| _crypto_stream_chacha20_pick_best_implementation(); 47| 1| _crypto_stream_salsa20_pick_best_implementation(); 48| 1| initialized = 1; 49| 1| if (sodium_crit_leave() != 0) { ------------------ | Branch (49:9): [True: 0, False: 1] ------------------ 50| 0| return -1; /* LCOV_EXCL_LINE */ 51| 0| } 52| 1| return 0; 53| 1|} sodium_crit_enter: 115| 638|{ 116| 638| int ret; 117| | 118| 638| if ((ret = pthread_mutex_lock(&_sodium_lock)) == 0) { ------------------ | Branch (118:9): [True: 638, False: 0] ------------------ 119| 638| assert(locked == 0); 120| 638| locked = 1; 121| 638| } 122| 638| return ret; 123| 638|} sodium_crit_leave: 127| 638|{ 128| 638| if (locked == 0) { ------------------ | Branch (128:9): [True: 0, False: 638] ------------------ 129| |# ifdef EPERM 130| | errno = EPERM; 131| |# endif 132| 0| return -1; 133| 0| } 134| 638| locked = 0; 135| | 136| 638| return pthread_mutex_unlock(&_sodium_lock); 137| 638|} _sodium_runtime_get_cpu_features: 235| 1|{ 236| 1| int ret = -1; 237| | 238| 1| ret &= _sodium_runtime_arm_cpu_features(&_cpu_features); 239| 1| ret &= _sodium_runtime_intel_cpu_features(&_cpu_features); 240| 1| _cpu_features.initialized = 1; 241| | 242| 1| return ret; 243| 1|} sodium_runtime_has_sse2: 253| 1|{ 254| 1| return _cpu_features.has_sse2; 255| 1|} sodium_runtime_has_avx: 277| 1|{ 278| 1| return _cpu_features.has_avx; 279| 1|} sodium_runtime_has_avx2: 283| 4|{ 284| 4| return _cpu_features.has_avx2; 285| 4|} sodium_runtime_has_avx512f: 289| 1|{ 290| 1| return _cpu_features.has_avx512f; 291| 1|} runtime.c:_sodium_runtime_arm_cpu_features: 50| 1|{ 51| 1|#ifndef __arm__ 52| 1| cpu_features->has_neon = 0; 53| 1| return -1; 54| |#else 55| |# ifdef __APPLE__ 56| |# ifdef __ARM_NEON__ 57| | cpu_features->has_neon = 1; 58| |# else 59| | cpu_features->has_neon = 0; 60| |# endif 61| |# elif defined(HAVE_ANDROID_GETCPUFEATURES) && \ 62| | defined(ANDROID_CPU_ARM_FEATURE_NEON) 63| | cpu_features->has_neon = 64| | (android_getCpuFeatures() & ANDROID_CPU_ARM_FEATURE_NEON) != 0x0; 65| |# else 66| | cpu_features->has_neon = 0; 67| |# endif 68| | return 0; 69| |#endif 70| 1|} runtime.c:_sodium_runtime_intel_cpu_features: 117| 1|{ 118| 1| unsigned int cpu_info[4]; 119| 1| unsigned int id; 120| 1| uint32_t xcr0 = 0U; 121| | 122| 1| _cpuid(cpu_info, 0x0); 123| 1| if ((id = cpu_info[0]) == 0U) { ------------------ | Branch (123:9): [True: 0, False: 1] ------------------ 124| 0| return -1; /* LCOV_EXCL_LINE */ 125| 0| } 126| 1| _cpuid(cpu_info, 0x00000001); 127| 1|#ifdef HAVE_EMMINTRIN_H 128| 1| cpu_features->has_sse2 = ((cpu_info[3] & CPUID_EDX_SSE2) != 0x0); ------------------ | | 40| 1|#define CPUID_EDX_SSE2 0x04000000 ------------------ 129| |#else 130| | cpu_features->has_sse2 = 0; 131| |#endif 132| | 133| 1|#ifdef HAVE_PMMINTRIN_H 134| 1| cpu_features->has_sse3 = ((cpu_info[2] & CPUID_ECX_SSE3) != 0x0); ------------------ | | 30| 1|#define CPUID_ECX_SSE3 0x00000001 ------------------ 135| |#else 136| | cpu_features->has_sse3 = 0; 137| |#endif 138| | 139| 1|#ifdef HAVE_TMMINTRIN_H 140| 1| cpu_features->has_ssse3 = ((cpu_info[2] & CPUID_ECX_SSSE3) != 0x0); ------------------ | | 32| 1|#define CPUID_ECX_SSSE3 0x00000200 ------------------ 141| |#else 142| | cpu_features->has_ssse3 = 0; 143| |#endif 144| | 145| 1|#ifdef HAVE_SMMINTRIN_H 146| 1| cpu_features->has_sse41 = ((cpu_info[2] & CPUID_ECX_SSE41) != 0x0); ------------------ | | 33| 1|#define CPUID_ECX_SSE41 0x00080000 ------------------ 147| |#else 148| | cpu_features->has_sse41 = 0; 149| |#endif 150| | 151| 1| cpu_features->has_avx = 0; 152| | 153| 1| (void) xcr0; 154| 1|#ifdef HAVE_AVXINTRIN_H 155| 1| if ((cpu_info[2] & (CPUID_ECX_AVX | CPUID_ECX_XSAVE | CPUID_ECX_OSXSAVE)) == ------------------ | | 37| 1|#define CPUID_ECX_AVX 0x10000000 ------------------ if ((cpu_info[2] & (CPUID_ECX_AVX | CPUID_ECX_XSAVE | CPUID_ECX_OSXSAVE)) == ------------------ | | 35| 1|#define CPUID_ECX_XSAVE 0x04000000 ------------------ if ((cpu_info[2] & (CPUID_ECX_AVX | CPUID_ECX_XSAVE | CPUID_ECX_OSXSAVE)) == ------------------ | | 36| 1|#define CPUID_ECX_OSXSAVE 0x08000000 ------------------ | Branch (155:9): [True: 1, False: 0] ------------------ 156| 1| (CPUID_ECX_AVX | CPUID_ECX_XSAVE | CPUID_ECX_OSXSAVE)) { ------------------ | | 37| 1|#define CPUID_ECX_AVX 0x10000000 ------------------ (CPUID_ECX_AVX | CPUID_ECX_XSAVE | CPUID_ECX_OSXSAVE)) { ------------------ | | 35| 1|#define CPUID_ECX_XSAVE 0x04000000 ------------------ (CPUID_ECX_AVX | CPUID_ECX_XSAVE | CPUID_ECX_OSXSAVE)) { ------------------ | | 36| 1|#define CPUID_ECX_OSXSAVE 0x08000000 ------------------ 157| 1| xcr0 = 0U; 158| |# if defined(HAVE__XGETBV) || \ 159| | (defined(_MSC_VER) && defined(_XCR_XFEATURE_ENABLED_MASK) && _MSC_FULL_VER >= 160040219) 160| | xcr0 = (uint32_t) _xgetbv(0); 161| |# elif defined(_MSC_VER) && defined(_M_IX86) 162| | /* 163| | * Visual Studio documentation states that eax/ecx/edx don't need to 164| | * be preserved in inline assembly code. But that doesn't seem to 165| | * always hold true on Visual Studio 2010. 166| | */ 167| | __asm { 168| | push eax 169| | push ecx 170| | push edx 171| | xor ecx, ecx 172| | _asm _emit 0x0f _asm _emit 0x01 _asm _emit 0xd0 173| | mov xcr0, eax 174| | pop edx 175| | pop ecx 176| | pop eax 177| | } 178| |# elif defined(HAVE_AVX_ASM) 179| 1| __asm__ __volatile__(".byte 0x0f, 0x01, 0xd0" /* XGETBV */ 180| 1| : "=a"(xcr0) 181| 1| : "c"((uint32_t) 0U) 182| 1| : "%edx"); 183| 1|# endif 184| 1| if ((xcr0 & (XCR0_SSE | XCR0_AVX)) == (XCR0_SSE | XCR0_AVX)) { ------------------ | | 42| 1|#define XCR0_SSE 0x00000002 ------------------ if ((xcr0 & (XCR0_SSE | XCR0_AVX)) == (XCR0_SSE | XCR0_AVX)) { ------------------ | | 43| 1|#define XCR0_AVX 0x00000004 ------------------ if ((xcr0 & (XCR0_SSE | XCR0_AVX)) == (XCR0_SSE | XCR0_AVX)) { ------------------ | | 42| 1|#define XCR0_SSE 0x00000002 ------------------ if ((xcr0 & (XCR0_SSE | XCR0_AVX)) == (XCR0_SSE | XCR0_AVX)) { ------------------ | | 43| 1|#define XCR0_AVX 0x00000004 ------------------ | Branch (184:13): [True: 1, False: 0] ------------------ 185| 1| cpu_features->has_avx = 1; 186| 1| } 187| 1| } 188| 1|#endif 189| | 190| 1| cpu_features->has_avx2 = 0; 191| 1|#ifdef HAVE_AVX2INTRIN_H 192| 1| if (cpu_features->has_avx) { ------------------ | Branch (192:9): [True: 1, False: 0] ------------------ 193| 1| unsigned int cpu_info7[4]; 194| | 195| 1| _cpuid(cpu_info7, 0x00000007); 196| 1| cpu_features->has_avx2 = ((cpu_info7[1] & CPUID_EBX_AVX2) != 0x0); ------------------ | | 27| 1|#define CPUID_EBX_AVX2 0x00000020 ------------------ 197| 1| } 198| 1|#endif 199| | 200| 1| cpu_features->has_avx512f = 0; 201| 1|#ifdef HAVE_AVX512FINTRIN_H 202| 1| if (cpu_features->has_avx2) { ------------------ | Branch (202:9): [True: 1, False: 0] ------------------ 203| 1| unsigned int cpu_info7[4]; 204| | 205| 1| _cpuid(cpu_info7, 0x00000007); 206| | /* LCOV_EXCL_START */ 207| 1| if ((cpu_info7[1] & CPUID_EBX_AVX512F) == CPUID_EBX_AVX512F && ------------------ | | 28| 1|#define CPUID_EBX_AVX512F 0x00010000 ------------------ if ((cpu_info7[1] & CPUID_EBX_AVX512F) == CPUID_EBX_AVX512F && ------------------ | | 28| 2|#define CPUID_EBX_AVX512F 0x00010000 ------------------ | Branch (207:13): [True: 0, False: 1] ------------------ 208| 1| (xcr0 & (XCR0_OPMASK | XCR0_ZMM_HI256 | XCR0_HI16_ZMM)) ------------------ | | 44| 0|#define XCR0_OPMASK 0x00000020 ------------------ (xcr0 & (XCR0_OPMASK | XCR0_ZMM_HI256 | XCR0_HI16_ZMM)) ------------------ | | 45| 0|#define XCR0_ZMM_HI256 0x00000040 ------------------ (xcr0 & (XCR0_OPMASK | XCR0_ZMM_HI256 | XCR0_HI16_ZMM)) ------------------ | | 46| 0|#define XCR0_HI16_ZMM 0x00000080 ------------------ | Branch (208:13): [True: 0, False: 0] ------------------ 209| 0| == (XCR0_OPMASK | XCR0_ZMM_HI256 | XCR0_HI16_ZMM)) { ------------------ | | 44| 0|#define XCR0_OPMASK 0x00000020 ------------------ == (XCR0_OPMASK | XCR0_ZMM_HI256 | XCR0_HI16_ZMM)) { ------------------ | | 45| 0|#define XCR0_ZMM_HI256 0x00000040 ------------------ == (XCR0_OPMASK | XCR0_ZMM_HI256 | XCR0_HI16_ZMM)) { ------------------ | | 46| 0|#define XCR0_HI16_ZMM 0x00000080 ------------------ 210| 0| cpu_features->has_avx512f = 1; 211| 0| } 212| | /* LCOV_EXCL_STOP */ 213| 1| } 214| 1|#endif 215| | 216| 1|#ifdef HAVE_WMMINTRIN_H 217| 1| cpu_features->has_pclmul = ((cpu_info[2] & CPUID_ECX_PCLMUL) != 0x0); ------------------ | | 31| 1|#define CPUID_ECX_PCLMUL 0x00000002 ------------------ 218| 1| cpu_features->has_aesni = ((cpu_info[2] & CPUID_ECX_AESNI) != 0x0); ------------------ | | 34| 1|#define CPUID_ECX_AESNI 0x02000000 ------------------ 219| |#else 220| | cpu_features->has_pclmul = 0; 221| | cpu_features->has_aesni = 0; 222| |#endif 223| | 224| 1|#ifdef HAVE_RDRAND 225| 1| cpu_features->has_rdrand = ((cpu_info[2] & CPUID_ECX_RDRAND) != 0x0); ------------------ | | 38| 1|#define CPUID_ECX_RDRAND 0x40000000 ------------------ 226| |#else 227| | cpu_features->has_rdrand = 0; 228| |#endif 229| | 230| 1| return 0; 231| 1|} runtime.c:_cpuid: 74| 4|{ 75| |#if defined(_MSC_VER) && \ 76| | (defined(_M_X64) || defined(_M_AMD64) || defined(_M_IX86)) 77| | __cpuid((int *) cpu_info, cpu_info_type); 78| |#elif defined(HAVE_CPUID) 79| 4| cpu_info[0] = cpu_info[1] = cpu_info[2] = cpu_info[3] = 0; 80| |# ifdef __i386__ 81| | __asm__ __volatile__( 82| | "pushfl; pushfl; " 83| | "popl %0; " 84| | "movl %0, %1; xorl %2, %0; " 85| | "pushl %0; " 86| | "popfl; pushfl; popl %0; popfl" 87| | : "=&r"(cpu_info[0]), "=&r"(cpu_info[1]) 88| | : "i"(0x200000)); 89| | if (((cpu_info[0] ^ cpu_info[1]) & 0x200000) == 0x0) { 90| | return; /* LCOV_EXCL_LINE */ 91| | } 92| |# endif 93| |# ifdef __i386__ 94| | __asm__ __volatile__("xchgl %%ebx, %k1; cpuid; xchgl %%ebx, %k1" 95| | : "=a"(cpu_info[0]), "=&r"(cpu_info[1]), 96| | "=c"(cpu_info[2]), "=d"(cpu_info[3]) 97| | : "0"(cpu_info_type), "2"(0U)); 98| |# elif defined(__x86_64__) 99| 4| __asm__ __volatile__("xchgq %%rbx, %q1; cpuid; xchgq %%rbx, %q1" 100| 4| : "=a"(cpu_info[0]), "=&r"(cpu_info[1]), 101| 4| "=c"(cpu_info[2]), "=d"(cpu_info[3]) 102| 4| : "0"(cpu_info_type), "2"(0U)); 103| |# else 104| | __asm__ __volatile__("cpuid" 105| | : "=a"(cpu_info[0]), "=b"(cpu_info[1]), 106| | "=c"(cpu_info[2]), "=d"(cpu_info[3]) 107| | : "0"(cpu_info_type), "2"(0U)); 108| |# endif 109| |#else 110| | (void) cpu_info_type; 111| | cpu_info[0] = cpu_info[1] = cpu_info[2] = cpu_info[3] = 0; 112| |#endif 113| 4|} sodium_memzero: 123| 5.80k|{ 124| |#if defined(_WIN32) && !defined(__CRT_INLINE) 125| | SecureZeroMemory(pnt, len); 126| |#elif defined(HAVE_MEMSET_S) 127| | if (len > 0U && memset_s(pnt, (rsize_t) len, 0, (rsize_t) len) != 0) { 128| | sodium_misuse(); /* LCOV_EXCL_LINE */ 129| | } 130| |#elif defined(HAVE_EXPLICIT_BZERO) 131| 5.80k| explicit_bzero(pnt, len); 132| |#elif defined(HAVE_MEMSET_EXPLICIT) 133| | memset_explicit(pnt, 0, len); 134| |#elif defined(HAVE_EXPLICIT_MEMSET) 135| | explicit_memset(pnt, 0, len); 136| |#elif HAVE_WEAK_SYMBOLS 137| | if (len > 0U) { 138| | memset(pnt, 0, len); 139| | _sodium_dummy_symbol_to_prevent_memzero_lto(pnt, len); 140| | } 141| |# ifdef HAVE_INLINE_ASM 142| | __asm__ __volatile__ ("" : : "r"(pnt) : "memory"); 143| |# endif 144| |#else 145| | volatile unsigned char *volatile pnt_ = 146| | (volatile unsigned char *volatile) pnt; 147| | size_t i = (size_t) 0U; 148| | 149| | while (i < len) { 150| | pnt_[i++] = 0U; 151| | } 152| |#endif 153| 5.80k|} _sodium_alloc_init: 406| 1|{ 407| 1|#ifdef HAVE_ALIGNED_MALLOC 408| 1|# if defined(_SC_PAGESIZE) && defined(HAVE_SYSCONF) 409| 1| long page_size_ = sysconf(_SC_PAGESIZE); 410| 1| if (page_size_ > 0L) { ------------------ | Branch (410:9): [True: 1, False: 0] ------------------ 411| 1| page_size = (size_t) page_size_; 412| 1| } 413| |# elif defined(WINAPI_DESKTOP) 414| | SYSTEM_INFO si; 415| | GetSystemInfo(&si); 416| | page_size = (size_t) si.dwPageSize; 417| |# elif !defined(PAGE_SIZE) 418| |# warning Unknown page size 419| |# endif 420| 1| if (page_size < CANARY_SIZE || page_size < sizeof(size_t)) { ------------------ | | 65| 2|#define CANARY_SIZE 16U ------------------ | Branch (420:9): [True: 0, False: 1] | Branch (420:36): [True: 0, False: 1] ------------------ 421| 0| sodium_misuse(); /* LCOV_EXCL_LINE */ 422| 0| } 423| 1|#endif 424| 1| randombytes_buf(canary, sizeof canary); 425| | 426| 1| return 0; 427| 1|} LLVMFuzzerTestOneInput: 21| 319|extern "C" int LLVMFuzzerTestOneInput(const unsigned char *data, size_t size) { 22| 319| int initialized = sodium_init(); 23| 319| assert(initialized >= 0); 24| | 25| 0| setup_fake_random(data, size); 26| | 27| 319| unsigned char key[crypto_secretbox_KEYBYTES]; 28| 319| unsigned char nonce[crypto_secretbox_NONCEBYTES]; 29| | 30| | // these use a deterministic generator 31| 319| crypto_secretbox_keygen(key); 32| 319| randombytes_buf(nonce, sizeof nonce); 33| | 34| 319| size_t ciphertext_len = crypto_secretbox_MACBYTES + size; ------------------ | | 24| 319|#define crypto_secretbox_MACBYTES crypto_secretbox_xsalsa20poly1305_MACBYTES | | ------------------ | | | | 23| 319|#define crypto_secretbox_xsalsa20poly1305_MACBYTES 16U | | ------------------ ------------------ 35| 319| unsigned char *ciphertext = (unsigned char *) malloc(ciphertext_len); 36| | 37| 319| crypto_secretbox_easy(ciphertext, data, size, nonce, key); 38| | 39| 319| unsigned char *decrypted = (unsigned char *) malloc(size); 40| 319| int err = crypto_secretbox_open_easy(decrypted, ciphertext, ciphertext_len, nonce, key); 41| 319| assert(err == 0); 42| | 43| 0| free((void *) ciphertext); 44| 319| free((void *) decrypted); 45| | 46| 319| return 0; 47| 319|}