/src/php-src/ext/standard/crc32_x86.c

Source
/*
  +----------------------------------------------------------------------+
  | Copyright © The PHP Group and Contributors.                          |
  +----------------------------------------------------------------------+
  | This source file is subject to the Modified BSD License that is      |
  | bundled with this package in the file LICENSE, and is available      |
  | through the World Wide Web at <https://www.php.net/license/>.        |
  |                                                                      |
  | SPDX-License-Identifier: BSD-3-Clause                                |
  +----------------------------------------------------------------------+
  | Author: Frank Du <frank.du@intel.com>                                |
  +----------------------------------------------------------------------+
  | Compute the crc32 of the buffer. Based on:                           |
  | "Fast CRC Computation for Generic Polynomials Using PCLMULQDQ"       |
  |  V. Gopal, E. Ozturk, et al., 2009, http://intel.ly/2ySEwL0          |
*/

#include "crc32_x86.h"

#if defined(ZEND_INTRIN_SSE4_2_PCLMUL_NATIVE) || defined(ZEND_INTRIN_SSE4_2_PCLMUL_RESOLVER)
# include <nmmintrin.h>
# include <wmmintrin.h>
#endif

#ifdef ZEND_INTRIN_SSE4_2_PCLMUL_RESOLVER
# include "Zend/zend_cpuinfo.h"
#endif

#if defined(ZEND_INTRIN_SSE4_2_PCLMUL_NATIVE) || defined(ZEND_INTRIN_SSE4_2_PCLMUL_RESOLVER)

typedef struct _crc32_pclmul_bit_consts {
  uint64_t k1k2[2];
  uint64_t k3k4[2];
  uint64_t k5k6[2];
  uint64_t uPx[2];
} crc32_pclmul_consts;

static const crc32_pclmul_consts crc32_pclmul_consts_maps[X86_CRC32_MAX] = {
  { /* X86_CRC32, polynomial: 0x04C11DB7 */
    {0x00e6228b11, 0x008833794c}, /* endianness swap */
    {0x00e8a45605, 0x00c5b9cd4c}, /* endianness swap */
    {0x00490d678d, 0x00f200aa66}, /* endianness swap */
    {0x0104d101df, 0x0104c11db7}
  },
  { /* X86_CRC32B, polynomial: 0x04C11DB7 with reversed ordering */
    {0x0154442bd4, 0x01c6e41596},
    {0x01751997d0, 0x00ccaa009e},
    {0x0163cd6124, 0x01db710640},
    {0x01f7011641, 0x01db710641},
  },
  { /* X86_CRC32C, polynomial: 0x1EDC6F41 with reversed ordering */
    {0x00740eef02, 0x009e4addf8},
    {0x00f20c0dfe, 0x014cd00bd6},
    {0x00dd45aab8, 0x0000000000},
    {0x00dea713f1, 0x0105ec76f0}
  }
};

static uint8_t pclmul_shuf_mask_table[16] = {
  0x0f, 0x0e, 0x0d, 0x0c, 0x0b, 0x0a, 0x09, 0x08,
  0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00,
};

/* Folding of 128-bit data chunks */
#define CRC32_FOLDING_BLOCK_SIZE (16)

/* PCLMUL version of non-reflected crc32 */
ZEND_INTRIN_SSE4_2_PCLMUL_FUNC_DECL(size_t crc32_pclmul_batch(uint32_t *crc, const unsigned char *p, size_t nr, const crc32_pclmul_consts *consts));
size_t crc32_pclmul_batch(uint32_t *crc, const unsigned char *p, size_t nr, const crc32_pclmul_consts *consts)
{
  size_t nr_in = nr;
  __m128i x0, x1, x2, k, shuf_mask;

  if (nr < CRC32_FOLDING_BLOCK_SIZE) {
    return 0;
  }

  shuf_mask = _mm_loadu_si128((__m128i *)(pclmul_shuf_mask_table));
  x0 = _mm_cvtsi32_si128(*crc);
  x1 = _mm_loadu_si128((__m128i *)(p + 0x00));
  x0 = _mm_slli_si128(x0, 12);
  x1 = _mm_shuffle_epi8(x1, shuf_mask); /* endianness swap */
  x0 = _mm_xor_si128(x1, x0);
  p += CRC32_FOLDING_BLOCK_SIZE;
  nr -= CRC32_FOLDING_BLOCK_SIZE;

  if (nr >= (CRC32_FOLDING_BLOCK_SIZE * 3)) {
    __m128i x3, x4;

    x1 = _mm_loadu_si128((__m128i *)(p + 0x00));
    x1 = _mm_shuffle_epi8(x1, shuf_mask); /* endianness swap */
    x2 = _mm_loadu_si128((__m128i *)(p + 0x10));
    x2 = _mm_shuffle_epi8(x2, shuf_mask); /* endianness swap */
    x3 = _mm_loadu_si128((__m128i *)(p + 0x20));
    x3 = _mm_shuffle_epi8(x3, shuf_mask); /* endianness swap */
    p += CRC32_FOLDING_BLOCK_SIZE * 3;
    nr -= CRC32_FOLDING_BLOCK_SIZE * 3;

    k = _mm_loadu_si128((__m128i *)consts->k1k2);
    /* parallel folding by 4 */
    while (nr >= (CRC32_FOLDING_BLOCK_SIZE * 4)) {
      __m128i x5, x6, x7, x8, x9, x10, x11;
      x4 = _mm_clmulepi64_si128(x0, k, 0x00);
      x5 = _mm_clmulepi64_si128(x1, k, 0x00);
      x6 = _mm_clmulepi64_si128(x2, k, 0x00);
      x7 = _mm_clmulepi64_si128(x3, k, 0x00);
      x0 = _mm_clmulepi64_si128(x0, k, 0x11);
      x1 = _mm_clmulepi64_si128(x1, k, 0x11);
      x2 = _mm_clmulepi64_si128(x2, k, 0x11);
      x3 = _mm_clmulepi64_si128(x3, k, 0x11);
      x8 = _mm_loadu_si128((__m128i *)(p + 0x00));
      x8 = _mm_shuffle_epi8(x8, shuf_mask); /* endianness swap */
      x9 = _mm_loadu_si128((__m128i *)(p + 0x10));
      x9 = _mm_shuffle_epi8(x9, shuf_mask); /* endianness swap */
      x10 = _mm_loadu_si128((__m128i *)(p + 0x20));
      x10 = _mm_shuffle_epi8(x10, shuf_mask); /* endianness swap */
      x11 = _mm_loadu_si128((__m128i *)(p + 0x30));
      x11 = _mm_shuffle_epi8(x11, shuf_mask); /* endianness swap */
      x0 = _mm_xor_si128(x0, x4);
      x1 = _mm_xor_si128(x1, x5);
      x2 = _mm_xor_si128(x2, x6);
      x3 = _mm_xor_si128(x3, x7);
      x0 = _mm_xor_si128(x0, x8);
      x1 = _mm_xor_si128(x1, x9);
      x2 = _mm_xor_si128(x2, x10);
      x3 = _mm_xor_si128(x3, x11);

      p += CRC32_FOLDING_BLOCK_SIZE * 4;
      nr -= CRC32_FOLDING_BLOCK_SIZE * 4;
    }

    k = _mm_loadu_si128((__m128i *)consts->k3k4);
    /* fold 4 to 1, [x1, x2, x3] -> x0 */
    x4 = _mm_clmulepi64_si128(x0, k, 0x00);
    x0 = _mm_clmulepi64_si128(x0, k, 0x11);
    x0 = _mm_xor_si128(x0, x1);
    x0 = _mm_xor_si128(x0, x4);
    x4 = _mm_clmulepi64_si128(x0, k, 0x00);
    x0 = _mm_clmulepi64_si128(x0, k, 0x11);
    x0 = _mm_xor_si128(x0, x2);
    x0 = _mm_xor_si128(x0, x4);
    x4 = _mm_clmulepi64_si128(x0, k, 0x00);
    x0 = _mm_clmulepi64_si128(x0, k, 0x11);
    x0 = _mm_xor_si128(x0, x3);
    x0 = _mm_xor_si128(x0, x4);
  }

  k = _mm_loadu_si128((__m128i *)consts->k3k4);
  /* folding by 1 */
  while (nr >= CRC32_FOLDING_BLOCK_SIZE) {
    /* load next to x2, fold to x0, x1 */
    x2 = _mm_loadu_si128((__m128i *)(p + 0x00));
    x2 = _mm_shuffle_epi8(x2, shuf_mask); /* endianness swap */
    x1 = _mm_clmulepi64_si128(x0, k, 0x00);
    x0 = _mm_clmulepi64_si128(x0, k, 0x11);
    x0 = _mm_xor_si128(x0, x2);
    x0 = _mm_xor_si128(x0, x1);
    p += CRC32_FOLDING_BLOCK_SIZE;
    nr -= CRC32_FOLDING_BLOCK_SIZE;
  }

  /* reduce 128-bits(final fold) to 96-bits */
  k = _mm_loadu_si128((__m128i*)consts->k5k6);
  x1 = _mm_clmulepi64_si128(x0, k, 0x11);
  x0 = _mm_slli_si128(x0, 8);
  x0 = _mm_srli_si128(x0, 4);
  x0 = _mm_xor_si128(x0, x1);
  /* reduce 96-bits to 64-bits */
  x1 = _mm_clmulepi64_si128(x0, k, 0x01);
  x0 = _mm_xor_si128(x0, x1);

  /* barrett reduction */
  k = _mm_loadu_si128((__m128i*)consts->uPx);
  x1 = _mm_move_epi64(x0);
  x1 = _mm_srli_si128(x1, 4);
  x1 = _mm_clmulepi64_si128(x1, k, 0x00);
  x1 = _mm_srli_si128(x1, 4);
  x1 = _mm_clmulepi64_si128(x1, k, 0x10);
  x0 = _mm_xor_si128(x1, x0);
  *crc =  _mm_extract_epi32(x0, 0);
  return (nr_in - nr); /* the nr processed */
}

/* PCLMUL version of reflected crc32 */
ZEND_INTRIN_SSE4_2_PCLMUL_FUNC_DECL(size_t crc32_pclmul_reflected_batch(uint32_t *crc, const unsigned char *p, size_t nr, const crc32_pclmul_consts *consts));
size_t crc32_pclmul_reflected_batch(uint32_t *crc, const unsigned char *p, size_t nr, const crc32_pclmul_consts *consts)
{
  size_t nr_in = nr;
  __m128i x0, x1, x2, k;

  if (nr < CRC32_FOLDING_BLOCK_SIZE) {
    return 0;
  }

  x0 = _mm_loadu_si128((__m128i *)(p + 0x00));
  x0 = _mm_xor_si128(x0, _mm_cvtsi32_si128(*crc));
  p += CRC32_FOLDING_BLOCK_SIZE;
  nr -= CRC32_FOLDING_BLOCK_SIZE;
  if (nr >= (CRC32_FOLDING_BLOCK_SIZE * 3)) {
    __m128i x3, x4;

    x1 = _mm_loadu_si128((__m128i *)(p + 0x00));
    x2 = _mm_loadu_si128((__m128i *)(p + 0x10));
    x3 = _mm_loadu_si128((__m128i *)(p + 0x20));
    p += CRC32_FOLDING_BLOCK_SIZE * 3;
    nr -= CRC32_FOLDING_BLOCK_SIZE * 3;

    k = _mm_loadu_si128((__m128i *)consts->k1k2);
    /* parallel folding by 4 */
    while (nr >= (CRC32_FOLDING_BLOCK_SIZE * 4)) {
      __m128i x5, x6, x7, x8, x9, x10, x11;
      x4 = _mm_clmulepi64_si128(x0, k, 0x00);
      x5 = _mm_clmulepi64_si128(x1, k, 0x00);
      x6 = _mm_clmulepi64_si128(x2, k, 0x00);
      x7 = _mm_clmulepi64_si128(x3, k, 0x00);
      x0 = _mm_clmulepi64_si128(x0, k, 0x11);
      x1 = _mm_clmulepi64_si128(x1, k, 0x11);
      x2 = _mm_clmulepi64_si128(x2, k, 0x11);
      x3 = _mm_clmulepi64_si128(x3, k, 0x11);
      x8 = _mm_loadu_si128((__m128i *)(p + 0x00));
      x9 = _mm_loadu_si128((__m128i *)(p + 0x10));
      x10 = _mm_loadu_si128((__m128i *)(p + 0x20));
      x11 = _mm_loadu_si128((__m128i *)(p + 0x30));
      x0 = _mm_xor_si128(x0, x4);
      x1 = _mm_xor_si128(x1, x5);
      x2 = _mm_xor_si128(x2, x6);
      x3 = _mm_xor_si128(x3, x7);
      x0 = _mm_xor_si128(x0, x8);
      x1 = _mm_xor_si128(x1, x9);
      x2 = _mm_xor_si128(x2, x10);
      x3 = _mm_xor_si128(x3, x11);

      p += CRC32_FOLDING_BLOCK_SIZE * 4;
      nr -= CRC32_FOLDING_BLOCK_SIZE * 4;
    }

    k = _mm_loadu_si128((__m128i *)consts->k3k4);
    /* fold 4 to 1, [x1, x2, x3] -> x0 */
    x4 = _mm_clmulepi64_si128(x0, k, 0x00);
    x0 = _mm_clmulepi64_si128(x0, k, 0x11);
    x0 = _mm_xor_si128(x0, x1);
    x0 = _mm_xor_si128(x0, x4);
    x4 = _mm_clmulepi64_si128(x0, k, 0x00);
    x0 = _mm_clmulepi64_si128(x0, k, 0x11);
    x0 = _mm_xor_si128(x0, x2);
    x0 = _mm_xor_si128(x0, x4);
    x4 = _mm_clmulepi64_si128(x0, k, 0x00);
    x0 = _mm_clmulepi64_si128(x0, k, 0x11);
    x0 = _mm_xor_si128(x0, x3);
    x0 = _mm_xor_si128(x0, x4);
  }

  k = _mm_loadu_si128((__m128i *)consts->k3k4);
  /* folding by 1 */
  while (nr >= CRC32_FOLDING_BLOCK_SIZE) {
    /* load next to x2, fold to x0, x1 */
    x2 = _mm_loadu_si128((__m128i *)(p + 0x00));
    x1 = _mm_clmulepi64_si128(x0, k, 0x00);
    x0 = _mm_clmulepi64_si128(x0, k, 0x11);
    x0 = _mm_xor_si128(x0, x2);
    x0 = _mm_xor_si128(x0, x1);
    p += CRC32_FOLDING_BLOCK_SIZE;
    nr -= CRC32_FOLDING_BLOCK_SIZE;
  }

  /* reduce 128-bits(final fold) to 96-bits */
  x1 = _mm_clmulepi64_si128(x0, k, 0x10);
  x0 = _mm_srli_si128(x0, 8);
  x0 = _mm_xor_si128(x0, x1);
  /* reduce 96-bits to 64-bits */
  x1 = _mm_shuffle_epi32(x0, 0xfc);
  x0 = _mm_shuffle_epi32(x0, 0xf9);
  k = _mm_loadu_si128((__m128i*)consts->k5k6);
  x1 = _mm_clmulepi64_si128(x1, k, 0x00);
  x0 = _mm_xor_si128(x0, x1);

  /* barrett reduction */
  x1 = _mm_shuffle_epi32(x0, 0xf3);
  x0 = _mm_slli_si128(x0, 4);
  k = _mm_loadu_si128((__m128i*)consts->uPx);
  x1 = _mm_clmulepi64_si128(x1, k, 0x00);
  x1 = _mm_clmulepi64_si128(x1, k, 0x10);
  x0 = _mm_xor_si128(x1, x0);
  *crc =  _mm_extract_epi32(x0, 2);
  return (nr_in - nr); /* the nr processed */
}

# if defined(ZEND_INTRIN_SSE4_2_PCLMUL_NATIVE)
size_t crc32_x86_simd_update(X86_CRC32_TYPE type, uint32_t *crc, const unsigned char *p, size_t nr)
# else /* ZEND_INTRIN_SSE4_2_PCLMUL_RESOLVER */
size_t crc32_sse42_pclmul_update(X86_CRC32_TYPE type, uint32_t *crc, const unsigned char *p, size_t nr)
# endif
{
  if (type > X86_CRC32_MAX) {
    return 0;
  }
  const crc32_pclmul_consts *consts = &crc32_pclmul_consts_maps[type];

  switch (type) {
  case X86_CRC32:
    return crc32_pclmul_batch(crc, p, nr, consts);
  case X86_CRC32B:
  case X86_CRC32C:
    return crc32_pclmul_reflected_batch(crc, p, nr, consts);
  default:
    return 0;
  }
}
#endif

#ifdef ZEND_INTRIN_SSE4_2_PCLMUL_RESOLVER
static size_t crc32_x86_simd_update_default(X86_CRC32_TYPE type, uint32_t *crc, const unsigned char *p, size_t nr)
{
  return 0;
}

# ifdef ZEND_INTRIN_SSE4_2_PCLMUL_FUNC_PROTO
size_t crc32_x86_simd_update(X86_CRC32_TYPE type, uint32_t *crc, const unsigned char *p, size_t nr) __attribute__((ifunc("resolve_crc32_x86_simd_update")));

typedef size_t (*crc32_x86_simd_func_t)(X86_CRC32_TYPE type, uint32_t *crc, const unsigned char *p, size_t nr);

ZEND_NO_SANITIZE_ADDRESS
ZEND_ATTRIBUTE_UNUSED /* clang mistakenly warns about this */
static crc32_x86_simd_func_t resolve_crc32_x86_simd_update(void) {
  if (zend_cpu_supports_sse42() && zend_cpu_supports_pclmul()) {
    return crc32_sse42_pclmul_update;
  }
  return crc32_x86_simd_update_default;
}
# else /* ZEND_INTRIN_SSE4_2_PCLMUL_FUNC_PTR */
static size_t (*crc32_x86_simd_ptr)(X86_CRC32_TYPE type, uint32_t *crc, const unsigned char *p, size_t nr) = crc32_x86_simd_update_default;

size_t crc32_x86_simd_update(X86_CRC32_TYPE type, uint32_t *crc, const unsigned char *p, size_t nr) {
  return crc32_x86_simd_ptr(type, crc, p, nr);
}

/* {{{ PHP_MINIT_FUNCTION */
PHP_MINIT_FUNCTION(crc32_x86_intrin)
{
  if (zend_cpu_supports_sse42() && zend_cpu_supports_pclmul()) {
    crc32_x86_simd_ptr = crc32_sse42_pclmul_update;
  }
  return SUCCESS;
}
/* }}} */
# endif
#endif

Coverage Report

Created: 2026-09-14 06:25

Line	Count	Source
1		/*
2		+----------------------------------------------------------------------+
3		\| Copyright © The PHP Group and Contributors. \|
4		+----------------------------------------------------------------------+
5		\| This source file is subject to the Modified BSD License that is \|
6		\| bundled with this package in the file LICENSE, and is available \|
7		\| through the World Wide Web at <https://www.php.net/license/>. \|
8		\| \|
9		\| SPDX-License-Identifier: BSD-3-Clause \|
10		+----------------------------------------------------------------------+
11		\| Author: Frank Du <frank.du@intel.com> \|
12		+----------------------------------------------------------------------+
13		\| Compute the crc32 of the buffer. Based on: \|
14		\| "Fast CRC Computation for Generic Polynomials Using PCLMULQDQ" \|
15		\| V. Gopal, E. Ozturk, et al., 2009, http://intel.ly/2ySEwL0 \|
16		*/
17
18		#include "crc32_x86.h"
19
20		#if defined(ZEND_INTRIN_SSE4_2_PCLMUL_NATIVE) \|\| defined(ZEND_INTRIN_SSE4_2_PCLMUL_RESOLVER)
21		# include <nmmintrin.h>
22		# include <wmmintrin.h>
23		#endif
24
25		#ifdef ZEND_INTRIN_SSE4_2_PCLMUL_RESOLVER
26		# include "Zend/zend_cpuinfo.h"
27		#endif
28
29		#if defined(ZEND_INTRIN_SSE4_2_PCLMUL_NATIVE) \|\| defined(ZEND_INTRIN_SSE4_2_PCLMUL_RESOLVER)
30
31		typedef struct _crc32_pclmul_bit_consts {
32		uint64_t k1k2[2];
33		uint64_t k3k4[2];
34		uint64_t k5k6[2];
35		uint64_t uPx[2];
36		} crc32_pclmul_consts;
37
38		static const crc32_pclmul_consts crc32_pclmul_consts_maps[X86_CRC32_MAX] = {
39		{ /* X86_CRC32, polynomial: 0x04C11DB7 */
40		{0x00e6228b11, 0x008833794c}, /* endianness swap */
41		{0x00e8a45605, 0x00c5b9cd4c}, /* endianness swap */
42		{0x00490d678d, 0x00f200aa66}, /* endianness swap */
43		{0x0104d101df, 0x0104c11db7}
44		},
45		{ /* X86_CRC32B, polynomial: 0x04C11DB7 with reversed ordering */
46		{0x0154442bd4, 0x01c6e41596},
47		{0x01751997d0, 0x00ccaa009e},
48		{0x0163cd6124, 0x01db710640},
49		{0x01f7011641, 0x01db710641},
50		},
51		{ /* X86_CRC32C, polynomial: 0x1EDC6F41 with reversed ordering */
52		{0x00740eef02, 0x009e4addf8},
53		{0x00f20c0dfe, 0x014cd00bd6},
54		{0x00dd45aab8, 0x0000000000},
55		{0x00dea713f1, 0x0105ec76f0}
56		}
57		};
58
59		static uint8_t pclmul_shuf_mask_table[16] = {
60		0x0f, 0x0e, 0x0d, 0x0c, 0x0b, 0x0a, 0x09, 0x08,
61		0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00,
62		};
63
64		/* Folding of 128-bit data chunks */
65	60.0k	#define CRC32_FOLDING_BLOCK_SIZE (16)
66
67		/* PCLMUL version of non-reflected crc32 */
68		ZEND_INTRIN_SSE4_2_PCLMUL_FUNC_DECL(size_t crc32_pclmul_batch(uint32_t crc, const unsigned char p, size_t nr, const crc32_pclmul_consts *consts));
69		size_t crc32_pclmul_batch(uint32_t crc, const unsigned char p, size_t nr, const crc32_pclmul_consts *consts)
70	30	{
71	30	size_t nr_in = nr;
72	30	__m128i x0, x1, x2, k, shuf_mask;
73
74	30	if (nr < CRC32_FOLDING_BLOCK_SIZE) {
75	3	return 0;
76	3	}
77
78	27	shuf_mask = _mm_loadu_si128((__m128i *)(pclmul_shuf_mask_table));
79	27	x0 = _mm_cvtsi32_si128(*crc);
80	27	x1 = _mm_loadu_si128((__m128i *)(p + 0x00));
81	27	x0 = _mm_slli_si128(x0, 12);
82	27	x1 = _mm_shuffle_epi8(x1, shuf_mask); /* endianness swap */
83	27	x0 = _mm_xor_si128(x1, x0);
84	27	p += CRC32_FOLDING_BLOCK_SIZE;
85	27	nr -= CRC32_FOLDING_BLOCK_SIZE;
86
87	27	if (nr >= (CRC32_FOLDING_BLOCK_SIZE * 3)) {
88	22	__m128i x3, x4;
89
90	22	x1 = _mm_loadu_si128((__m128i *)(p + 0x00));
91	22	x1 = _mm_shuffle_epi8(x1, shuf_mask); /* endianness swap */
92	22	x2 = _mm_loadu_si128((__m128i *)(p + 0x10));
93	22	x2 = _mm_shuffle_epi8(x2, shuf_mask); /* endianness swap */
94	22	x3 = _mm_loadu_si128((__m128i *)(p + 0x20));
95	22	x3 = _mm_shuffle_epi8(x3, shuf_mask); /* endianness swap */
96	22	p += CRC32_FOLDING_BLOCK_SIZE * 3;
97	22	nr -= CRC32_FOLDING_BLOCK_SIZE * 3;
98
99	22	k = _mm_loadu_si128((__m128i *)consts->k1k2);
100		/* parallel folding by 4 */
101	8.35k	while (nr >= (CRC32_FOLDING_BLOCK_SIZE * 4)) {
102	8.33k	__m128i x5, x6, x7, x8, x9, x10, x11;
103	8.33k	x4 = _mm_clmulepi64_si128(x0, k, 0x00);
104	8.33k	x5 = _mm_clmulepi64_si128(x1, k, 0x00);
105	8.33k	x6 = _mm_clmulepi64_si128(x2, k, 0x00);
106	8.33k	x7 = _mm_clmulepi64_si128(x3, k, 0x00);
107	8.33k	x0 = _mm_clmulepi64_si128(x0, k, 0x11);
108	8.33k	x1 = _mm_clmulepi64_si128(x1, k, 0x11);
109	8.33k	x2 = _mm_clmulepi64_si128(x2, k, 0x11);
110	8.33k	x3 = _mm_clmulepi64_si128(x3, k, 0x11);
111	8.33k	x8 = _mm_loadu_si128((__m128i *)(p + 0x00));
112	8.33k	x8 = _mm_shuffle_epi8(x8, shuf_mask); /* endianness swap */
113	8.33k	x9 = _mm_loadu_si128((__m128i *)(p + 0x10));
114	8.33k	x9 = _mm_shuffle_epi8(x9, shuf_mask); /* endianness swap */
115	8.33k	x10 = _mm_loadu_si128((__m128i *)(p + 0x20));
116	8.33k	x10 = _mm_shuffle_epi8(x10, shuf_mask); /* endianness swap */
117	8.33k	x11 = _mm_loadu_si128((__m128i *)(p + 0x30));
118	8.33k	x11 = _mm_shuffle_epi8(x11, shuf_mask); /* endianness swap */
119	8.33k	x0 = _mm_xor_si128(x0, x4);
120	8.33k	x1 = _mm_xor_si128(x1, x5);
121	8.33k	x2 = _mm_xor_si128(x2, x6);
122	8.33k	x3 = _mm_xor_si128(x3, x7);
123	8.33k	x0 = _mm_xor_si128(x0, x8);
124	8.33k	x1 = _mm_xor_si128(x1, x9);
125	8.33k	x2 = _mm_xor_si128(x2, x10);
126	8.33k	x3 = _mm_xor_si128(x3, x11);
127
128	8.33k	p += CRC32_FOLDING_BLOCK_SIZE * 4;
129	8.33k	nr -= CRC32_FOLDING_BLOCK_SIZE * 4;
130	8.33k	}
131
132	22	k = _mm_loadu_si128((__m128i *)consts->k3k4);
133		/* fold 4 to 1, [x1, x2, x3] -> x0 */
134	22	x4 = _mm_clmulepi64_si128(x0, k, 0x00);
135	22	x0 = _mm_clmulepi64_si128(x0, k, 0x11);
136	22	x0 = _mm_xor_si128(x0, x1);
137	22	x0 = _mm_xor_si128(x0, x4);
138	22	x4 = _mm_clmulepi64_si128(x0, k, 0x00);
139	22	x0 = _mm_clmulepi64_si128(x0, k, 0x11);
140	22	x0 = _mm_xor_si128(x0, x2);
141	22	x0 = _mm_xor_si128(x0, x4);
142	22	x4 = _mm_clmulepi64_si128(x0, k, 0x00);
143	22	x0 = _mm_clmulepi64_si128(x0, k, 0x11);
144	22	x0 = _mm_xor_si128(x0, x3);
145	22	x0 = _mm_xor_si128(x0, x4);
146	22	}
147
148	27	k = _mm_loadu_si128((__m128i *)consts->k3k4);
149		/* folding by 1 */
150	42	while (nr >= CRC32_FOLDING_BLOCK_SIZE) {
151		/* load next to x2, fold to x0, x1 */
152	15	x2 = _mm_loadu_si128((__m128i *)(p + 0x00));
153	15	x2 = _mm_shuffle_epi8(x2, shuf_mask); /* endianness swap */
154	15	x1 = _mm_clmulepi64_si128(x0, k, 0x00);
155	15	x0 = _mm_clmulepi64_si128(x0, k, 0x11);
156	15	x0 = _mm_xor_si128(x0, x2);
157	15	x0 = _mm_xor_si128(x0, x1);
158	15	p += CRC32_FOLDING_BLOCK_SIZE;
159	15	nr -= CRC32_FOLDING_BLOCK_SIZE;
160	15	}
161
162		/* reduce 128-bits(final fold) to 96-bits */
163	27	k = _mm_loadu_si128((__m128i*)consts->k5k6);
164	27	x1 = _mm_clmulepi64_si128(x0, k, 0x11);
165	27	x0 = _mm_slli_si128(x0, 8);
166	27	x0 = _mm_srli_si128(x0, 4);
167	27	x0 = _mm_xor_si128(x0, x1);
168		/* reduce 96-bits to 64-bits */
169	27	x1 = _mm_clmulepi64_si128(x0, k, 0x01);
170	27	x0 = _mm_xor_si128(x0, x1);
171
172		/* barrett reduction */
173	27	k = _mm_loadu_si128((__m128i*)consts->uPx);
174	27	x1 = _mm_move_epi64(x0);
175	27	x1 = _mm_srli_si128(x1, 4);
176	27	x1 = _mm_clmulepi64_si128(x1, k, 0x00);
177	27	x1 = _mm_srli_si128(x1, 4);
178	27	x1 = _mm_clmulepi64_si128(x1, k, 0x10);
179	27	x0 = _mm_xor_si128(x1, x0);
180	27	*crc = _mm_extract_epi32(x0, 0);
181	27	return (nr_in - nr); /* the nr processed */
182	30	}
183
184		/* PCLMUL version of reflected crc32 */
185		ZEND_INTRIN_SSE4_2_PCLMUL_FUNC_DECL(size_t crc32_pclmul_reflected_batch(uint32_t crc, const unsigned char p, size_t nr, const crc32_pclmul_consts *consts));
186		size_t crc32_pclmul_reflected_batch(uint32_t crc, const unsigned char p, size_t nr, const crc32_pclmul_consts *consts)
187	206	{
188	206	size_t nr_in = nr;
189	206	__m128i x0, x1, x2, k;
190
191	206	if (nr < CRC32_FOLDING_BLOCK_SIZE) {
192	76	return 0;
193	76	}
194
195	130	x0 = _mm_loadu_si128((__m128i *)(p + 0x00));
196	130	x0 = _mm_xor_si128(x0, _mm_cvtsi32_si128(*crc));
197	130	p += CRC32_FOLDING_BLOCK_SIZE;
198	130	nr -= CRC32_FOLDING_BLOCK_SIZE;
199	130	if (nr >= (CRC32_FOLDING_BLOCK_SIZE * 3)) {
200	50	__m128i x3, x4;
201
202	50	x1 = _mm_loadu_si128((__m128i *)(p + 0x00));
203	50	x2 = _mm_loadu_si128((__m128i *)(p + 0x10));
204	50	x3 = _mm_loadu_si128((__m128i *)(p + 0x20));
205	50	p += CRC32_FOLDING_BLOCK_SIZE * 3;
206	50	nr -= CRC32_FOLDING_BLOCK_SIZE * 3;
207
208	50	k = _mm_loadu_si128((__m128i *)consts->k1k2);
209		/* parallel folding by 4 */
210	11.2k	while (nr >= (CRC32_FOLDING_BLOCK_SIZE * 4)) {
211	11.1k	__m128i x5, x6, x7, x8, x9, x10, x11;
212	11.1k	x4 = _mm_clmulepi64_si128(x0, k, 0x00);
213	11.1k	x5 = _mm_clmulepi64_si128(x1, k, 0x00);
214	11.1k	x6 = _mm_clmulepi64_si128(x2, k, 0x00);
215	11.1k	x7 = _mm_clmulepi64_si128(x3, k, 0x00);
216	11.1k	x0 = _mm_clmulepi64_si128(x0, k, 0x11);
217	11.1k	x1 = _mm_clmulepi64_si128(x1, k, 0x11);
218	11.1k	x2 = _mm_clmulepi64_si128(x2, k, 0x11);
219	11.1k	x3 = _mm_clmulepi64_si128(x3, k, 0x11);
220	11.1k	x8 = _mm_loadu_si128((__m128i *)(p + 0x00));
221	11.1k	x9 = _mm_loadu_si128((__m128i *)(p + 0x10));
222	11.1k	x10 = _mm_loadu_si128((__m128i *)(p + 0x20));
223	11.1k	x11 = _mm_loadu_si128((__m128i *)(p + 0x30));
224	11.1k	x0 = _mm_xor_si128(x0, x4);
225	11.1k	x1 = _mm_xor_si128(x1, x5);
226	11.1k	x2 = _mm_xor_si128(x2, x6);
227	11.1k	x3 = _mm_xor_si128(x3, x7);
228	11.1k	x0 = _mm_xor_si128(x0, x8);
229	11.1k	x1 = _mm_xor_si128(x1, x9);
230	11.1k	x2 = _mm_xor_si128(x2, x10);
231	11.1k	x3 = _mm_xor_si128(x3, x11);
232
233	11.1k	p += CRC32_FOLDING_BLOCK_SIZE * 4;
234	11.1k	nr -= CRC32_FOLDING_BLOCK_SIZE * 4;
235	11.1k	}
236
237	50	k = _mm_loadu_si128((__m128i *)consts->k3k4);
238		/* fold 4 to 1, [x1, x2, x3] -> x0 */
239	50	x4 = _mm_clmulepi64_si128(x0, k, 0x00);
240	50	x0 = _mm_clmulepi64_si128(x0, k, 0x11);
241	50	x0 = _mm_xor_si128(x0, x1);
242	50	x0 = _mm_xor_si128(x0, x4);
243	50	x4 = _mm_clmulepi64_si128(x0, k, 0x00);
244	50	x0 = _mm_clmulepi64_si128(x0, k, 0x11);
245	50	x0 = _mm_xor_si128(x0, x2);
246	50	x0 = _mm_xor_si128(x0, x4);
247	50	x4 = _mm_clmulepi64_si128(x0, k, 0x00);
248	50	x0 = _mm_clmulepi64_si128(x0, k, 0x11);
249	50	x0 = _mm_xor_si128(x0, x3);
250	50	x0 = _mm_xor_si128(x0, x4);
251	50	}
252
253	130	k = _mm_loadu_si128((__m128i *)consts->k3k4);
254		/* folding by 1 */
255	251	while (nr >= CRC32_FOLDING_BLOCK_SIZE) {
256		/* load next to x2, fold to x0, x1 */
257	121	x2 = _mm_loadu_si128((__m128i *)(p + 0x00));
258	121	x1 = _mm_clmulepi64_si128(x0, k, 0x00);
259	121	x0 = _mm_clmulepi64_si128(x0, k, 0x11);
260	121	x0 = _mm_xor_si128(x0, x2);
261	121	x0 = _mm_xor_si128(x0, x1);
262	121	p += CRC32_FOLDING_BLOCK_SIZE;
263	121	nr -= CRC32_FOLDING_BLOCK_SIZE;
264	121	}
265
266		/* reduce 128-bits(final fold) to 96-bits */
267	130	x1 = _mm_clmulepi64_si128(x0, k, 0x10);
268	130	x0 = _mm_srli_si128(x0, 8);
269	130	x0 = _mm_xor_si128(x0, x1);
270		/* reduce 96-bits to 64-bits */
271	130	x1 = _mm_shuffle_epi32(x0, 0xfc);
272	130	x0 = _mm_shuffle_epi32(x0, 0xf9);
273	130	k = _mm_loadu_si128((__m128i*)consts->k5k6);
274	130	x1 = _mm_clmulepi64_si128(x1, k, 0x00);
275	130	x0 = _mm_xor_si128(x0, x1);
276
277		/* barrett reduction */
278	130	x1 = _mm_shuffle_epi32(x0, 0xf3);
279	130	x0 = _mm_slli_si128(x0, 4);
280	130	k = _mm_loadu_si128((__m128i*)consts->uPx);
281	130	x1 = _mm_clmulepi64_si128(x1, k, 0x00);
282	130	x1 = _mm_clmulepi64_si128(x1, k, 0x10);
283	130	x0 = _mm_xor_si128(x1, x0);
284	130	*crc = _mm_extract_epi32(x0, 2);
285	130	return (nr_in - nr); /* the nr processed */
286	206	}
287
288		# if defined(ZEND_INTRIN_SSE4_2_PCLMUL_NATIVE)
289		size_t crc32_x86_simd_update(X86_CRC32_TYPE type, uint32_t crc, const unsigned char p, size_t nr)
290		# else /* ZEND_INTRIN_SSE4_2_PCLMUL_RESOLVER */
291		size_t crc32_sse42_pclmul_update(X86_CRC32_TYPE type, uint32_t crc, const unsigned char p, size_t nr)
292		# endif
293	236	{
294	236	if (type > X86_CRC32_MAX) {
295	0	return 0;
296	0	}
297	236	const crc32_pclmul_consts *consts = &crc32_pclmul_consts_maps[type];
298
299	236	switch (type) {
300	30	case X86_CRC32:
301	30	return crc32_pclmul_batch(crc, p, nr, consts);
302	168	case X86_CRC32B:
303	206	case X86_CRC32C:
304	206	return crc32_pclmul_reflected_batch(crc, p, nr, consts);
305	0	default:
306	0	return 0;
307	236	}
308	236	}
309		#endif
310
311		#ifdef ZEND_INTRIN_SSE4_2_PCLMUL_RESOLVER
312		static size_t crc32_x86_simd_update_default(X86_CRC32_TYPE type, uint32_t crc, const unsigned char p, size_t nr)
313	0	{
314	0	return 0;
315	0	}
316
317		# ifdef ZEND_INTRIN_SSE4_2_PCLMUL_FUNC_PROTO
318		size_t crc32_x86_simd_update(X86_CRC32_TYPE type, uint32_t crc, const unsigned char p, size_t nr) __attribute__((ifunc("resolve_crc32_x86_simd_update")));
319
320		typedef size_t (crc32_x86_simd_func_t)(X86_CRC32_TYPE type, uint32_t crc, const unsigned char *p, size_t nr);
321
322		ZEND_NO_SANITIZE_ADDRESS
323		ZEND_ATTRIBUTE_UNUSED /* clang mistakenly warns about this */
324	16	static crc32_x86_simd_func_t resolve_crc32_x86_simd_update(void) {
325	16	if (zend_cpu_supports_sse42() && zend_cpu_supports_pclmul()) {
326	16	return crc32_sse42_pclmul_update;
327	16	}
328	0	return crc32_x86_simd_update_default;
329	16	}
330		# else /* ZEND_INTRIN_SSE4_2_PCLMUL_FUNC_PTR */
331		static size_t (crc32_x86_simd_ptr)(X86_CRC32_TYPE type, uint32_t crc, const unsigned char *p, size_t nr) = crc32_x86_simd_update_default;
332
333		size_t crc32_x86_simd_update(X86_CRC32_TYPE type, uint32_t crc, const unsigned char p, size_t nr) {
334		return crc32_x86_simd_ptr(type, crc, p, nr);
335		}
336
337		/* {{{ PHP_MINIT_FUNCTION */
338		PHP_MINIT_FUNCTION(crc32_x86_intrin)
339		{
340		if (zend_cpu_supports_sse42() && zend_cpu_supports_pclmul()) {
341		crc32_x86_simd_ptr = crc32_sse42_pclmul_update;
342		}
343		return SUCCESS;
344		}
345		/* }}} */
346		# endif
347		#endif