Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/chardet/_kernel.py: 29%

1"""The hot scoring kernel, shared by the pure-Python and compiled builds. 

2 

3This module is ordinary Python with no third-party imports: it runs as-is on 

4PyPy and in pure-Python installs. When a compiled wheel is built, 

5``_kernel.pxd`` supplies C type declarations for these same functions and 

6Cython compiles this file to native code — the ``.py`` stays the one and only 

7implementation. 

8 

9Note: ``from __future__ import annotations`` is intentionally omitted to match 

10the modules mypyc compiles, which import from here. 

11 

12``dot_packed`` reads the profile's parallel ``array('i')`` buffers rather than 

13its dense 65536-entry table. Compiled, that is the difference between a gather 

14through a Python list and a C loop over two contiguous int32 buffers. The 

15interpreter pays for it — ``array`` indexing boxes an int where a list returns 

16a cached one — which is the trade this build makes deliberately. 

17""" 

18 

19import array 

20 

21 

22def dot_packed(idx: array.array, vals: array.array, model: bytes) -> int: 

23 """Return the dot product of a packed bigram profile with a model table. 

24 

25 :param idx: ``array('i')`` of bigram indices, in first-encounter order. 

26 Deliberately *not* sorted --- do not add a binary search or an 

27 early exit over it. 

28 :param vals: ``array('i')`` of weights, parallel to *idx*. 

29 :param model: 65536-byte model lookup table. 

30 :returns: Sum of ``model[idx[k]] * vals[k]`` over all ``k``. 

31 """ 

32 dot = 0 

33 n = len(idx) 

34 for i in range(n): 

35 dot += model[idx[i]] * vals[i] 

36 return dot 

37 

38 

39def pack_profile(nonzero: list, freq: list) -> tuple: 

40 """Return parallel ``array('i')`` index/value buffers for a dense profile. 

41 

42 ``int32`` holds any weight a truncated input can produce: statistical 

43 scoring caps its input at 16384 bytes, so no weight exceeds ``255 * 16383`` 

44 (about 4.2 million) against an int32 ceiling of 2.1 billion. The bound is 

45 the caller's to keep --- see :class:`~chardet.models.BigramProfile`, which 

46 documents the input limit that makes it hold. 

47 

48 :param nonzero: Bigram indices with non-zero weight. 

49 :param freq: Dense 65536-entry weight table. 

50 :returns: An ``(idx, vals)`` tuple of ``array('i')`` buffers. 

51 """ 

52 vals = array.array("i", [0]) * len(nonzero) 

53 n = len(nonzero) 

54 for i in range(n): 

55 vals[i] = freq[nonzero[i]] 

56 return array.array("i", nonzero), vals