{

  return _mm_loadu_si128((const __m128i *)p);

}

{

  return _mm_loadu_si128((const __m128i *)p);

}

{

  return _mm_and_si128(my_load128((vmasks + count)), v);

}

{

  return _mm_and_si128(my_load128((vmasks + count)), v);

}

{

  return _mm_shuffle_epi8(v, my_load128((vmasks + 32 - amount)));

}

{

  return _mm_shuffle_epi8(v, my_load128((vmasks + 32 - amount)));

}

{

  return _mm_shuffle_epi8(v, my_load128((vmasks + 32 + amount)));

}

{

  return _mm_shuffle_epi8(v, my_load128((vmasks + 32 + amount)));

}

{

  __m128i a = _mm_clmulepi64_si128(v, k, 0x00);

  __m128i b = _mm_clmulepi64_si128(v, k, 0x11);

  return _mm_xor_si128(a, b);

}

{

  __m128i a = _mm_clmulepi64_si128(v, k, 0x00);

  __m128i b = _mm_clmulepi64_si128(v, k, 0x11);

  return _mm_xor_si128(a, b);

}

{

  return _mm_xor_si128(my_load128(buf), fold(v, k));

}

{

  return _mm_xor_si128(my_load128(buf), fold(v, k));

}

{

  // We will assume that there is at least one byte of input.

  if (size == 0)

    return crc;

  // See crc_clmul_consts_gen.c.

#if BUILDING_CRC_CLMUL == 32

  const __m128i fold512 = _mm_set_epi64x(0x1d9513d7, 0x8f352d95);

  const __m128i fold128 = _mm_set_epi64x(0xccaa009e, 0xae689191);

  const __m128i mu_p = _mm_set_epi64x(

    (int64_t)0xb4e5b025f7011641, 0x1db710640);

#else

  const __m128i fold512 = _mm_set_epi64x(

    (int64_t)0x081f6054a7842df4, (int64_t)0x6ae3efbb9dd441f3);

  const __m128i fold128 = _mm_set_epi64x(

    (int64_t)0xdabe95afc7875f40, (int64_t)0xe05dd497ca393ae4);

  const __m128i mu_p = _mm_set_epi64x(

    (int64_t)0x9c3e466c172963d5, (int64_t)0x92d8af2baf0e1e84);

#endif

  __m128i v0, v1, v2, v3;

  crc = ~crc;

  if (size < 8) {

    uint64_t x = crc;

    size_t i = 0;

    // Checking the bit instead of comparing the size means

    // that we don't need to update the size between the steps.

    if (size & 4) {

      x ^= read32le(buf);

      buf += 4;

      i = 32;

    }

    if (size & 2) {

      x ^= (uint64_t)read16le(buf) << i;

      buf += 2;

      i += 16;

    }

    if (size & 1)

      x ^= (uint64_t)*buf << i;

    v0 = my_set_low64((int64_t)x);

    v0 = shift_left(v0, 8 - size);

  } else if (size < 16) {

    v0 = my_set_low64((int64_t)(crc ^ read64le(buf)));

    // NOTE: buf is intentionally left 8 bytes behind so that

    // we can read the last 1-7 bytes with read64le(buf + size).

    size -= 8;

    // Handling 8-byte input specially is a speed optimization

    // as the clmul can be skipped. A branch is also needed to

    // avoid a too high shift amount.

    if (size > 0) {

      const size_t padding = 8 - size;

      uint64_t high = read64le(buf + size) >> (padding * 8);

#if defined(__i386__) || defined(_M_IX86)

      // Simple but likely not the best code for 32-bit x86.

      v0 = _mm_insert_epi32(v0, (int32_t)high, 2);

      v0 = _mm_insert_epi32(v0, (int32_t)(high >> 32), 3);

#else

      v0 = _mm_insert_epi64(v0, (int64_t)high, 1);

#endif

      v0 = shift_left(v0, padding);

      v1 = _mm_srli_si128(v0, 8);

      v0 = _mm_clmulepi64_si128(v0, fold128, 0x10);

      v0 = _mm_xor_si128(v0, v1);

    }

  } else {

    v0 = my_set_low64((int64_t)crc);

    // To align or not to align the buf pointer? If the end of

    // the buffer isn't aligned, aligning the pointer here would

    // make us do an extra folding step with the associated byte

    // shuffling overhead. The cost of that would need to be

    // lower than the benefit of aligned reads. Testing on an old

    // Intel Ivy Bridge processor suggested that aligning isn't

    // worth the cost but it likely depends on the processor and

    // buffer size. Unaligned loads (MOVDQU) should be fast on

    // x86 processors that support PCLMULQDQ, so we don't align

    // the buf pointer here.

    // Read the first (and possibly the only) full 16 bytes.

    v0 = _mm_xor_si128(v0, my_load128(buf));

    buf += 16;

    size -= 16;

    if (size >= 48) {

      v1 = my_load128(buf);

      v2 = my_load128(buf + 16);

      v3 = my_load128(buf + 32);

      buf += 48;

      size -= 48;

      while (size >= 64) {

        v0 = fold_xor(v0, fold512, buf);

        v1 = fold_xor(v1, fold512, buf + 16);

        v2 = fold_xor(v2, fold512, buf + 32);

        v3 = fold_xor(v3, fold512, buf + 48);

        buf += 64;

        size -= 64;

      }

      v0 = _mm_xor_si128(v1, fold(v0, fold128));

      v0 = _mm_xor_si128(v2, fold(v0, fold128));

      v0 = _mm_xor_si128(v3, fold(v0, fold128));

    }

    while (size >= 16) {

      v0 = fold_xor(v0, fold128, buf);

      buf += 16;

      size -= 16;

    }

    if (size > 0) {

      // We want the last "size" number of input bytes to

      // be at the high bits of v1. First do a full 16-byte

      // load and then mask the low bytes to zeros.

      v1 = my_load128(buf + size - 16);

      v1 = keep_high_bytes(v1, size);

      // Shift high bytes from v0 to the low bytes of v1.

      //

      // Alternatively we could replace the combination

      // keep_high_bytes + shift_right + _mm_or_si128 with

      // _mm_shuffle_epi8 + _mm_blendv_epi8 but that would

      // require larger tables for the masks. Now there are

      // three loads (instead of two) from the mask tables

      // but they all are from the same cache line.

      v1 = _mm_or_si128(v1, shift_right(v0, size));

      // Shift high bytes of v0 away, padding the

      // low bytes with zeros.

      v0 = shift_left(v0, 16 - size);

      v0 = _mm_xor_si128(v1, fold(v0, fold128));

    }

    v1 = _mm_srli_si128(v0, 8);

    v0 = _mm_clmulepi64_si128(v0, fold128, 0x10);

    v0 = _mm_xor_si128(v0, v1);

  }

  // Barrett reduction

#if BUILDING_CRC_CLMUL == 32

  v1 = _mm_clmulepi64_si128(v0, mu_p, 0x10); // v0 * mu

  v1 = _mm_clmulepi64_si128(v1, mu_p, 0x00); // v1 * p

  v0 = _mm_xor_si128(v0, v1);

  return ~(uint32_t)_mm_extract_epi32(v0, 2);

#else

  // Because p is 65 bits but one bit doesn't fit into the 64-bit

  // half of __m128i, finish the second clmul by shifting v1 left

  // by 64 bits and xorring it to the final result.

  v1 = _mm_clmulepi64_si128(v0, mu_p, 0x10); // v0 * mu

  v2 = _mm_slli_si128(v1, 8);

  v1 = _mm_clmulepi64_si128(v1, mu_p, 0x00); // v1 * p

  v0 = _mm_xor_si128(v0, v2);

  v0 = _mm_xor_si128(v0, v1);

#if defined(__i386__) || defined(_M_IX86)

  return ~(((uint64_t)(uint32_t)_mm_extract_epi32(v0, 3) << 32) |

      (uint64_t)(uint32_t)_mm_extract_epi32(v0, 2));

#else

  return ~(uint64_t)_mm_extract_epi64(v0, 1);

#endif

#endif

}

{

  // We will assume that there is at least one byte of input.

  if (size == 0)

    return crc;

  // See crc_clmul_consts_gen.c.

#if BUILDING_CRC_CLMUL == 32

  const __m128i fold512 = _mm_set_epi64x(0x1d9513d7, 0x8f352d95);

  const __m128i fold128 = _mm_set_epi64x(0xccaa009e, 0xae689191);

  const __m128i mu_p = _mm_set_epi64x(

    (int64_t)0xb4e5b025f7011641, 0x1db710640);

#else

  const __m128i fold512 = _mm_set_epi64x(

    (int64_t)0x081f6054a7842df4, (int64_t)0x6ae3efbb9dd441f3);

  const __m128i fold128 = _mm_set_epi64x(

    (int64_t)0xdabe95afc7875f40, (int64_t)0xe05dd497ca393ae4);

  const __m128i mu_p = _mm_set_epi64x(

    (int64_t)0x9c3e466c172963d5, (int64_t)0x92d8af2baf0e1e84);

#endif

  __m128i v0, v1, v2, v3;

  crc = ~crc;

  if (size < 8) {

    uint64_t x = crc;

    size_t i = 0;

    // Checking the bit instead of comparing the size means

    // that we don't need to update the size between the steps.

    if (size & 4) {

      x ^= read32le(buf);

      buf += 4;

      i = 32;

    }

    if (size & 2) {

      x ^= (uint64_t)read16le(buf) << i;

      buf += 2;

      i += 16;

    }

    if (size & 1)

      x ^= (uint64_t)*buf << i;

    v0 = my_set_low64((int64_t)x);

    v0 = shift_left(v0, 8 - size);

  } else if (size < 16) {

    v0 = my_set_low64((int64_t)(crc ^ read64le(buf)));

    // NOTE: buf is intentionally left 8 bytes behind so that

    // we can read the last 1-7 bytes with read64le(buf + size).

    size -= 8;

    // Handling 8-byte input specially is a speed optimization

    // as the clmul can be skipped. A branch is also needed to

    // avoid a too high shift amount.

    if (size > 0) {

      const size_t padding = 8 - size;

      uint64_t high = read64le(buf + size) >> (padding * 8);

#if defined(__i386__) || defined(_M_IX86)

      // Simple but likely not the best code for 32-bit x86.

      v0 = _mm_insert_epi32(v0, (int32_t)high, 2);

      v0 = _mm_insert_epi32(v0, (int32_t)(high >> 32), 3);

#else

      v0 = _mm_insert_epi64(v0, (int64_t)high, 1);

#endif

      v0 = shift_left(v0, padding);

      v1 = _mm_srli_si128(v0, 8);

      v0 = _mm_clmulepi64_si128(v0, fold128, 0x10);

      v0 = _mm_xor_si128(v0, v1);

    }

  } else {

    v0 = my_set_low64((int64_t)crc);

    // To align or not to align the buf pointer? If the end of

    // the buffer isn't aligned, aligning the pointer here would

    // make us do an extra folding step with the associated byte

    // shuffling overhead. The cost of that would need to be

    // lower than the benefit of aligned reads. Testing on an old

    // Intel Ivy Bridge processor suggested that aligning isn't

    // worth the cost but it likely depends on the processor and

    // buffer size. Unaligned loads (MOVDQU) should be fast on

    // x86 processors that support PCLMULQDQ, so we don't align

    // the buf pointer here.

    // Read the first (and possibly the only) full 16 bytes.

    v0 = _mm_xor_si128(v0, my_load128(buf));

    buf += 16;

    size -= 16;

    if (size >= 48) {

      v1 = my_load128(buf);

      v2 = my_load128(buf + 16);

      v3 = my_load128(buf + 32);

      buf += 48;

      size -= 48;

      while (size >= 64) {

        v0 = fold_xor(v0, fold512, buf);

        v1 = fold_xor(v1, fold512, buf + 16);

        v2 = fold_xor(v2, fold512, buf + 32);

        v3 = fold_xor(v3, fold512, buf + 48);

        buf += 64;

        size -= 64;

      }

      v0 = _mm_xor_si128(v1, fold(v0, fold128));

      v0 = _mm_xor_si128(v2, fold(v0, fold128));

      v0 = _mm_xor_si128(v3, fold(v0, fold128));

    }

    while (size >= 16) {

      v0 = fold_xor(v0, fold128, buf);

      buf += 16;

      size -= 16;

    }

    if (size > 0) {

      // We want the last "size" number of input bytes to

      // be at the high bits of v1. First do a full 16-byte

      // load and then mask the low bytes to zeros.

      v1 = my_load128(buf + size - 16);

      v1 = keep_high_bytes(v1, size);

      // Shift high bytes from v0 to the low bytes of v1.

      //

      // Alternatively we could replace the combination

      // keep_high_bytes + shift_right + _mm_or_si128 with

      // _mm_shuffle_epi8 + _mm_blendv_epi8 but that would

      // require larger tables for the masks. Now there are

      // three loads (instead of two) from the mask tables

      // but they all are from the same cache line.

      v1 = _mm_or_si128(v1, shift_right(v0, size));

      // Shift high bytes of v0 away, padding the

      // low bytes with zeros.

      v0 = shift_left(v0, 16 - size);

      v0 = _mm_xor_si128(v1, fold(v0, fold128));

    }

    v1 = _mm_srli_si128(v0, 8);

    v0 = _mm_clmulepi64_si128(v0, fold128, 0x10);

    v0 = _mm_xor_si128(v0, v1);

  }

  // Barrett reduction

#if BUILDING_CRC_CLMUL == 32

  v1 = _mm_clmulepi64_si128(v0, mu_p, 0x10); // v0 * mu

  v1 = _mm_clmulepi64_si128(v1, mu_p, 0x00); // v1 * p

  v0 = _mm_xor_si128(v0, v1);

  return ~(uint32_t)_mm_extract_epi32(v0, 2);

#else

  // Because p is 65 bits but one bit doesn't fit into the 64-bit

  // half of __m128i, finish the second clmul by shifting v1 left

  // by 64 bits and xorring it to the final result.

  v1 = _mm_clmulepi64_si128(v0, mu_p, 0x10); // v0 * mu

  v2 = _mm_slli_si128(v1, 8);

  v1 = _mm_clmulepi64_si128(v1, mu_p, 0x00); // v1 * p

  v0 = _mm_xor_si128(v0, v2);

  v0 = _mm_xor_si128(v0, v1);

#if defined(__i386__) || defined(_M_IX86)

  return ~(((uint64_t)(uint32_t)_mm_extract_epi32(v0, 3) << 32) |

      (uint64_t)(uint32_t)_mm_extract_epi32(v0, 2));

#else

  return ~(uint64_t)_mm_extract_epi64(v0, 1);

#endif

#endif

}

{

  int success = 1;

  uint32_t r[4]; // eax, ebx, ecx, edx

#if defined(_MSC_VER)

  // This needs <intrin.h> with MSVC. ICC has it as a built-in

  // on all platforms.

  __cpuid((int *)r, 1);

#elif defined(HAVE_CPUID_H)

  // Compared to just using __asm__ to run CPUID, this also checks

  // that CPUID is supported and saves and restores ebx as that is

  // needed with GCC < 5 with position-independent code (PIC).

  success = __get_cpuid(1, &r[0], &r[1], &r[2], &r[3]);

#else

  // Just a fallback that shouldn't be needed.

  __asm__("cpuid\n\t"

      : "=a"(r[0]), "=b"(r[1]), "=c"(r[2]), "=d"(r[3])

      : "a"(1), "c"(0));

#endif

  // Returns true if these are supported:

  // CLMUL (bit 1 in ecx)

  // SSSE3 (bit 9 in ecx)

  // SSE4.1 (bit 19 in ecx)

  const uint32_t ecx_mask = (1 << 1) | (1 << 9) | (1 << 19);

  return success && (r[2] & ecx_mask) == ecx_mask;

  // Alternative methods that weren't used:

  //   - ICC's _may_i_use_cpu_feature: the other methods should work too.

  //   - GCC >= 6 / Clang / ICX __builtin_cpu_supports("pclmul")

  //

  // CPUID decoding is needed with MSVC anyway and older GCC. This keeps

  // the feature checks in the build system simpler too. The nice thing

  // about __builtin_cpu_supports would be that it generates very short

  // code as is it only reads a variable set at startup but a few bytes

  // doesn't matter here.

}

{

  int success = 1;

  uint32_t r[4]; // eax, ebx, ecx, edx

#if defined(_MSC_VER)

  // This needs <intrin.h> with MSVC. ICC has it as a built-in

  // on all platforms.

  __cpuid((int *)r, 1);

#elif defined(HAVE_CPUID_H)

  // Compared to just using __asm__ to run CPUID, this also checks

  // that CPUID is supported and saves and restores ebx as that is

  // needed with GCC < 5 with position-independent code (PIC).

  success = __get_cpuid(1, &r[0], &r[1], &r[2], &r[3]);

#else

  // Just a fallback that shouldn't be needed.

  __asm__("cpuid\n\t"

      : "=a"(r[0]), "=b"(r[1]), "=c"(r[2]), "=d"(r[3])

      : "a"(1), "c"(0));

#endif

  // Returns true if these are supported:

  // CLMUL (bit 1 in ecx)

  // SSSE3 (bit 9 in ecx)

  // SSE4.1 (bit 19 in ecx)

  const uint32_t ecx_mask = (1 << 1) | (1 << 9) | (1 << 19);

  return success && (r[2] & ecx_mask) == ecx_mask;

  // Alternative methods that weren't used:

  //   - ICC's _may_i_use_cpu_feature: the other methods should work too.

  //   - GCC >= 6 / Clang / ICX __builtin_cpu_supports("pclmul")

  //

  // CPUID decoding is needed with MSVC anyway and older GCC. This keeps

  // the feature checks in the build system simpler too. The nice thing

  // about __builtin_cpu_supports would be that it generates very short

  // code as is it only reads a variable set at startup but a few bytes

  // doesn't matter here.

}