Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/bitstring/bits.py: 23%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3import numbers
4import pathlib
5import sys
6import mmap
7import struct
8import array
9import io
10from collections import abc
11import functools
12from typing import Tuple, Union, List, Iterable, Any, Optional, BinaryIO, TextIO, overload, Iterator, Type, TypeVar
13import bitarray
14import bitstring
15from bitstring import utils
16from bitstring.dtypes import Dtype, dtype_register
17from bitstring.fp8 import p4binary_fmt, p3binary_fmt
18from bitstring.mxfp import e3m2mxfp_fmt, e2m3mxfp_fmt, e2m1mxfp_fmt, e4m3mxfp_saturate_fmt, e5m2mxfp_saturate_fmt
19from bitstring.bitstring_options import Colour
21ConstBitStore = bitstring.bitstore.ConstBitStore
22MutableBitStore = bitstring.bitstore.MutableBitStore
23helpers = bitstring.bitstore_helpers
24common_helpers = bitstring.bitstore_common_helpers
27# Things that can be converted to Bits when a Bits type is needed
28BitsType = Union['Bits', str, Iterable[Any], bool, BinaryIO, bytearray, bytes, memoryview, bitarray.bitarray]
30TBits = TypeVar("TBits", bound='Bits')
32# Maximum number of digits to use in __str__ and __repr__.
33MAX_CHARS: int = 250
36class Bits:
37 """A container holding an immutable sequence of bits.
39 For a mutable container use the BitArray class instead.
41 Methods:
43 all() -- Check if all specified bits are set to 1 or 0.
44 any() -- Check if any of specified bits are set to 1 or 0.
45 copy() - Return a copy of the bitstring.
46 count() -- Count the number of bits set to 1 or 0.
47 cut() -- Create generator of constant sized chunks.
48 endswith() -- Return whether the bitstring ends with a sub-string.
49 find() -- Find a sub-bitstring in the current bitstring.
50 findall() -- Find all occurrences of a sub-bitstring in the current bitstring.
51 fromstring() -- Create a bitstring from a formatted string.
52 join() -- Join bitstrings together using current bitstring.
53 pp() -- Pretty print the bitstring.
54 rfind() -- Seek backwards to find a sub-bitstring.
55 split() -- Create generator of chunks split by a delimiter.
56 startswith() -- Return whether the bitstring starts with a sub-bitstring.
57 tobitarray() -- Return bitstring as a bitarray from the bitarray package.
58 tobytes() -- Return bitstring as bytes, padding if needed.
59 tofile() -- Write bitstring to file, padding if needed.
60 unpack() -- Interpret bits using format string.
62 Special methods:
64 Also available are the operators [], ==, !=, +, *, ~, <<, >>, &, |, ^.
66 Properties:
68 [GENERATED_PROPERTY_DESCRIPTIONS]
70 len -- Length of the bitstring in bits.
72 """
73 __slots__ = ('_bitstore', '_filename')
75 def __init__(self, auto: Optional[Union[BitsType, int]] = None, /, length: Optional[int] = None,
76 offset: Optional[int] = None, **kwargs) -> None:
77 """Either specify an 'auto' initialiser:
78 A string of comma separated tokens, an integer, a file object,
79 a bytearray, a boolean iterable, an array or another bitstring.
81 Or initialise via **kwargs with one (and only one) of:
82 bin -- binary string representation, e.g. '0b001010'.
83 hex -- hexadecimal string representation, e.g. '0x2ef'
84 oct -- octal string representation, e.g. '0o777'.
85 bytes -- raw data as a bytes object, for example read from a binary file.
86 int -- a signed integer.
87 uint -- an unsigned integer.
88 float / floatbe -- a big-endian floating point number.
89 bool -- a boolean (True or False).
90 se -- a signed exponential-Golomb code.
91 ue -- an unsigned exponential-Golomb code.
92 sie -- a signed interleaved exponential-Golomb code.
93 uie -- an unsigned interleaved exponential-Golomb code.
94 floatle -- a little-endian floating point number.
95 floatne -- a native-endian floating point number.
96 bfloat / bfloatbe - a big-endian bfloat format 16-bit floating point number.
97 bfloatle -- a little-endian bfloat format 16-bit floating point number.
98 bfloatne -- a native-endian bfloat format 16-bit floating point number.
99 intbe -- a signed big-endian whole byte integer.
100 intle -- a signed little-endian whole byte integer.
101 intne -- a signed native-endian whole byte integer.
102 uintbe -- an unsigned big-endian whole byte integer.
103 uintle -- an unsigned little-endian whole byte integer.
104 uintne -- an unsigned native-endian whole byte integer.
105 filename -- the path of a file which will be opened in binary read-only mode.
107 Other keyword arguments:
108 length -- length of the bitstring in bits, if needed and appropriate.
109 It must be supplied for all integer and float initialisers.
110 offset -- bit offset to the data. These offset bits are
111 ignored and this is mainly intended for use when
112 initialising using 'bytes' or 'filename'.
114 """
115 pass
117 def __new__(cls: Type[TBits], auto: Optional[Union[BitsType, int]] = None, /, length: Optional[int] = None,
118 offset: Optional[int] = None, pos: Optional[int] = None, **kwargs) -> TBits:
119 x = super().__new__(cls)
120 if auto is None and not kwargs:
121 # No initialiser so fill with zero bits up to length
122 if length is not None:
123 x._bitstore = ConstBitStore.from_zeros(length)
124 else:
125 x._bitstore = ConstBitStore()
126 return x
127 x._initialise(auto, length, offset, immutable=True, **kwargs)
128 return x
130 @classmethod
131 def _create_from_bitstype(cls: Type[TBits], auto: BitsType, /) -> TBits:
132 if isinstance(auto, cls):
133 return auto
134 b = super().__new__(cls)
135 b._setauto_no_length_or_offset(auto)
136 return b
138 def _initialise(self, auto: Any, /, length: Optional[int], offset: Optional[int], immutable: bool, **kwargs) -> None:
139 if auto is not None:
140 if isinstance(auto, numbers.Integral):
141 # Initialise with s zero bits.
142 if auto < 0:
143 raise bitstring.CreationError(f"Can't create bitstring of negative length {auto}.")
144 if immutable:
145 self._bitstore = ConstBitStore.from_zeros(int(auto))
146 else:
147 self._bitstore = MutableBitStore.from_zeros(int(auto))
148 return
149 self._setauto(auto, length, offset)
150 else:
151 k, v = kwargs.popitem()
152 if k == 'bytes':
153 # Special case for bytes as we want to allow offsets and lengths to work only on creation.
154 self._setbytes_with_truncation(v, length, offset)
155 elif k == 'filename':
156 self._setfile(v, length, offset)
157 elif k == 'bitarray':
158 self._setbitarray(v, length, offset)
159 elif k == 'auto':
160 raise bitstring.CreationError(
161 f"The 'auto' parameter should not be given explicitly - just use the first positional argument. "
162 f"Instead of '{self.__class__.__name__}(auto=x)' use '{self.__class__.__name__}(x)'.")
163 else:
164 if offset is not None:
165 raise bitstring.CreationError(f"offset cannot be used when initialising with '{k}'.")
166 try:
167 Dtype(k, length).set_fn(self, v)
168 except ValueError as e:
169 raise bitstring.CreationError(e)
170 if not immutable:
171 # TODO: This copy is not a good idea.
172 self._bitstore = self._bitstore._mutable_copy()
174 def __getattr__(self, attribute: str) -> Any:
175 # Support for arbitrary attributes like u16 or f64.
176 try:
177 d = Dtype(attribute)
178 except ValueError:
179 raise AttributeError(f"'{self.__class__.__name__}' object has no attribute '{attribute}'.")
180 if d.bitlength is not None and len(self) != d.bitlength:
181 raise ValueError(f"bitstring length {len(self)} doesn't match length {d.bitlength} of property '{attribute}'.")
182 return d.get_fn(self)
184 def __iter__(self) -> Iterable[bool]:
185 return iter(self._bitstore)
187 def __copy__(self: TBits) -> TBits:
188 """Return a new copy of the Bits for the copy module."""
189 # Note that if you want a new copy (different ID), use _copy instead.
190 # The copy can return self as it's immutable.
191 return self
193 def __lt__(self, other: Any) -> bool:
194 # bitstrings can't really be ordered.
195 return NotImplemented
197 def __gt__(self, other: Any) -> bool:
198 return NotImplemented
200 def __le__(self, other: Any) -> bool:
201 return NotImplemented
203 def __ge__(self, other: Any) -> bool:
204 return NotImplemented
206 def __add__(self: TBits, bs: BitsType) -> TBits:
207 """Concatenate bitstrings and return new bitstring.
209 bs -- the bitstring to append.
211 """
212 bs = self.__class__._create_from_bitstype(bs)
213 s = self._copy() if len(bs) <= len(self) else bs._copy()
214 if len(bs) <= len(self):
215 s._addright(bs)
216 else:
217 s._addleft(self)
218 return s
220 def __radd__(self: TBits, bs: BitsType) -> TBits:
221 """Append current bitstring to bs and return new bitstring.
223 bs -- An object that can be 'auto' initialised as a bitstring that will be appended to.
225 """
226 bs = self.__class__._create_from_bitstype(bs)
227 return bs.__add__(self)
229 @overload
230 def __getitem__(self: TBits, key: slice, /) -> TBits:
231 ...
233 @overload
234 def __getitem__(self, key: int, /) -> bool:
235 ...
237 def __getitem__(self: TBits, key: Union[slice, int], /) -> Union[TBits, bool]:
238 """Return a new bitstring representing a slice of the current bitstring.
240 >>> print(Bits('0b00110')[1:4])
241 '0b011'
243 """
244 if isinstance(key, numbers.Integral):
245 return bool(self._bitstore.getindex(key))
246 bs = super().__new__(self.__class__)
247 bs._bitstore = self._bitstore.getslice_withstep(key)
248 return bs
250 def __len__(self) -> int:
251 """Return the length of the bitstring in bits."""
252 return self._getlength()
254 def __bytes__(self) -> bytes:
255 return self.tobytes()
257 def __str__(self) -> str:
258 """Return approximate string representation of bitstring for printing.
260 Short strings will be given wholly in hexadecimal or binary. Longer
261 strings may be part hexadecimal and part binary. Very long strings will
262 be truncated with '...'.
264 """
265 length = len(self)
266 if not length:
267 return ''
268 if length > MAX_CHARS * 4:
269 # Too long for hex. Truncate...
270 return ''.join(('0x', self[0:MAX_CHARS*4]._gethex(), '...'))
271 # If it's quite short and we can't do hex then use bin
272 if length < 32 and length % 4 != 0:
273 return '0b' + self.bin
274 # If we can use hex then do so
275 if not length % 4:
276 return '0x' + self.hex
277 # Otherwise first we do as much as we can in hex
278 # then add on 1, 2 or 3 bits on at the end
279 bits_at_end = length % 4
280 return ''.join(('0x', self[0:length - bits_at_end]._gethex(),
281 ', ', '0b', self[length - bits_at_end:]._getbin()))
283 def _repr(self, classname: str, length: int, pos: int):
284 pos_string = f', pos={pos}' if pos else ''
285 if hasattr(self, '_filename') and self._filename:
286 return f"{classname}(filename={self._filename!r}, length={length}{pos_string})"
287 else:
288 s = self.__str__()
289 lengthstring = ''
290 if s.endswith('...'):
291 lengthstring = f' # length={length}'
292 return f"{classname}('{s}'{pos_string}){lengthstring}"
294 def __repr__(self) -> str:
295 """Return representation that could be used to recreate the bitstring.
297 If the returned string is too long it will be truncated. See __str__().
299 """
300 return self._repr(self.__class__.__name__, len(self), 0)
302 def __eq__(self, bs: Any, /) -> bool:
303 """Return True if two bitstrings have the same binary representation.
305 >>> BitArray('0b1110') == '0xe'
306 True
308 """
309 try:
310 return self._bitstore == Bits._create_from_bitstype(bs)._bitstore
311 except TypeError:
312 return False
314 def __ne__(self, bs: Any, /) -> bool:
315 """Return False if two bitstrings have the same binary representation.
317 >>> BitArray('0b111') == '0x7'
318 False
320 """
321 return not self.__eq__(bs)
323 def __invert__(self: TBits) -> TBits:
324 """Return bitstring with every bit inverted.
326 Raises Error if the bitstring is empty.
328 """
329 if len(self) == 0:
330 raise bitstring.Error("Cannot invert empty bitstring.")
331 s = self.__class__.__new__(self.__class__)
332 s._bitstore = ~self._bitstore
333 return s
335 def __lshift__(self: TBits, n: int, /) -> TBits:
336 """Return bitstring with bits shifted by n to the left.
338 n -- the number of bits to shift. Must be >= 0.
340 """
341 if n < 0:
342 raise ValueError("Cannot shift by a negative amount.")
343 if len(self) == 0:
344 raise ValueError("Cannot shift an empty bitstring.")
345 n = min(n, len(self))
346 s = self._absolute_slice(n, len(self))
347 s._addright(Bits(n))
348 return s
350 def __rshift__(self: TBits, n: int, /) -> TBits:
351 """Return bitstring with bits shifted by n to the right.
353 n -- the number of bits to shift. Must be >= 0.
355 """
356 if n < 0:
357 raise ValueError("Cannot shift by a negative amount.")
358 if len(self) == 0:
359 raise ValueError("Cannot shift an empty bitstring.")
360 if not n:
361 return self._copy()
362 s = self.__class__(length=min(n, len(self)))
363 n = min(n, len(self))
364 s._addright(self._absolute_slice(0, len(self) - n))
365 return s
367 def __mul__(self: TBits, n: int, /) -> TBits:
368 """Return bitstring consisting of n concatenations of self.
370 Called for expression of the form 'a = b*3'.
371 n -- The number of concatenations. Must be >= 0.
373 """
374 if n < 0:
375 raise ValueError("Cannot multiply by a negative integer.")
376 if n == 0:
377 return self.__class__()
378 s = self._copy()
379 s._imul(n)
380 return s
382 def _imul(self: TBits, n: int, /) -> TBits:
383 """Concatenate n copies of self in place. Return self."""
384 self._bitstore.__imul__(n)
385 return self
387 def __rmul__(self: TBits, n: int, /) -> TBits:
388 """Return bitstring consisting of n concatenations of self.
390 Called for expressions of the form 'a = 3*b'.
391 n -- The number of concatenations. Must be >= 0.
393 """
394 return self.__mul__(n)
396 def __and__(self: TBits, bs: BitsType, /) -> TBits:
397 """Bit-wise 'and' between two bitstrings. Returns new bitstring.
399 bs -- The bitstring to '&' with.
401 Raises ValueError if the two bitstrings have differing lengths.
403 """
404 if bs is self:
405 return self.copy()
406 bs = Bits._create_from_bitstype(bs)
407 s = object.__new__(self.__class__)
408 s._bitstore = self._bitstore & bs._bitstore
409 return s
411 def __rand__(self: TBits, bs: BitsType, /) -> TBits:
412 """Bit-wise 'and' between two bitstrings. Returns new bitstring.
414 bs -- the bitstring to '&' with.
416 Raises ValueError if the two bitstrings have differing lengths.
418 """
419 return self.__and__(bs)
421 def __or__(self: TBits, bs: BitsType, /) -> TBits:
422 """Bit-wise 'or' between two bitstrings. Returns new bitstring.
424 bs -- The bitstring to '|' with.
426 Raises ValueError if the two bitstrings have differing lengths.
428 """
429 if bs is self:
430 return self.copy()
431 bs = Bits._create_from_bitstype(bs)
432 s = object.__new__(self.__class__)
433 s._bitstore = self._bitstore | bs._bitstore
434 return s
436 def __ror__(self: TBits, bs: BitsType, /) -> TBits:
437 """Bit-wise 'or' between two bitstrings. Returns new bitstring.
439 bs -- The bitstring to '|' with.
441 Raises ValueError if the two bitstrings have differing lengths.
443 """
444 return self.__or__(bs)
446 def __xor__(self: TBits, bs: BitsType, /) -> TBits:
447 """Bit-wise 'xor' between two bitstrings. Returns new bitstring.
449 bs -- The bitstring to '^' with.
451 Raises ValueError if the two bitstrings have differing lengths.
453 """
454 bs = Bits._create_from_bitstype(bs)
455 s = object.__new__(self.__class__)
456 s._bitstore = self._bitstore ^ bs._bitstore
457 return s
459 def __rxor__(self: TBits, bs: BitsType, /) -> TBits:
460 """Bit-wise 'xor' between two bitstrings. Returns new bitstring.
462 bs -- The bitstring to '^' with.
464 Raises ValueError if the two bitstrings have differing lengths.
466 """
467 return self.__xor__(bs)
469 def __contains__(self, bs: BitsType, /) -> bool:
470 """Return whether bs is contained in the current bitstring.
472 bs -- The bitstring to search for.
474 """
475 found = Bits.find(self, bs, bytealigned=False)
476 return bool(found)
478 def __hash__(self) -> int:
479 """Return an integer hash of the object."""
480 # Only requirement is that equal bitstring should return the same hash.
481 # For equal bitstrings the bytes at the start/end will be the same and they will have the same length
482 # (need to check the length as there could be zero padding when getting the bytes). We do not check any
483 # bit position inside the bitstring as that does not feature in the __eq__ operation.
484 if len(self) <= 2000:
485 # Use the whole bitstring.
486 return hash((self.tobytes(), len(self)))
487 else:
488 # We can't in general hash the whole bitstring (it could take hours!)
489 # So instead take some bits from the start and end.
490 return hash(((self[:800] + self[-800:]).tobytes(), len(self)))
492 def __bool__(self) -> bool:
493 """Return False if bitstring is empty, otherwise return True."""
494 return len(self) != 0
496 def _clear(self) -> None:
497 """Reset the bitstring to an empty state."""
498 self._bitstore.clear()
500 def _setauto_no_length_or_offset(self, s: BitsType, /) -> None:
501 """Set bitstring from a bitstring, file, bool, array, iterable or string."""
502 if isinstance(s, str):
503 self._bitstore = common_helpers.str_to_bitstore(s)
504 elif isinstance(s, Bits):
505 self._bitstore = s._bitstore.copy()
506 elif isinstance(s, (bytes, bytearray, memoryview)):
507 self._bitstore = ConstBitStore.from_bytes(bytearray(s))
508 elif isinstance(s, io.BytesIO):
509 self._bitstore = ConstBitStore.from_bytes(s.getvalue())
510 elif isinstance(s, io.BufferedReader):
511 self._setfile(s.name)
512 elif isinstance(s, bitarray.bitarray):
513 self._bitstore = ConstBitStore(s)
514 elif isinstance(s, array.array):
515 self._bitstore = ConstBitStore.from_bytes(s.tobytes())
516 elif isinstance(s, abc.Iterable):
517 # Evaluate each item as True or False and set bits to 1 or 0.
518 self._setbin(''.join(str(int(bool(x))) for x in s))
519 elif isinstance(s, numbers.Integral):
520 raise TypeError(f"It's no longer possible to auto initialise a bitstring from an integer."
521 f" Use '{self.__class__.__name__}({s})' instead of just '{s}' as this makes it "
522 f"clearer that a bitstring of {int(s)} zero bits will be created.")
523 else:
524 raise TypeError(f"Cannot initialise bitstring from type '{type(s)}'.")
526 def _setauto(self, s: BitsType, length: Optional[int], offset: Optional[int], /) -> None:
527 """Set bitstring from a bitstring, file, bool, array, iterable or string."""
528 # As s can be so many different things it's important to do the checks
529 # in the correct order, as some types are also other allowed types.
530 if offset is None and length is None:
531 self._setauto_no_length_or_offset(s)
532 return
533 if offset is None:
534 offset = 0
536 if isinstance(s, io.BytesIO):
537 if length is None:
538 length = s.seek(0, 2) * 8 - offset
539 byteoffset, offset = divmod(offset, 8)
540 bytelength = (length + byteoffset * 8 + offset + 7) // 8 - byteoffset
541 if length + byteoffset * 8 + offset > s.seek(0, 2) * 8:
542 raise bitstring.CreationError("BytesIO object is not long enough for specified length and offset.")
543 self._bitstore = ConstBitStore.from_bytes(s.getvalue()[byteoffset: byteoffset + bytelength]).getslice(
544 offset, offset + length)
545 return
547 if isinstance(s, io.BufferedReader):
548 self._setfile(s.name, length, offset)
549 return
551 if isinstance(s, (str, Bits, bytes, bytearray, memoryview, io.BytesIO, io.BufferedReader,
552 bitarray.bitarray, array.array, abc.Iterable)):
553 raise bitstring.CreationError(f"Cannot initialise bitstring from type '{type(s)}' when using explicit lengths or offsets.")
554 raise TypeError(f"Cannot initialise bitstring from type '{type(s)}'.")
556 def _setfile(self, filename: str, length: Optional[int] = None, offset: Optional[int] = None) -> None:
557 """Use file as source of bits."""
558 with open(pathlib.Path(filename), 'rb') as source:
559 if offset is None:
560 offset = 0
561 m = mmap.mmap(source.fileno(), 0, access=mmap.ACCESS_READ)
562 if offset == 0:
563 self._filename = source.name
564 self._bitstore = ConstBitStore.frombuffer(m, length=length)
565 else:
566 # If offset is given then always read into memory.
567 temp = ConstBitStore.frombuffer(m)
568 if length is None:
569 if offset > len(temp):
570 raise bitstring.CreationError(f"The offset of {offset} bits is greater than the file length ({len(temp)} bits).")
571 self._bitstore = temp.getslice(offset, None)
572 else:
573 self._bitstore = temp.getslice(offset, offset + length)
574 if len(self) != length:
575 raise bitstring.CreationError(f"Can't use a length of {length} bits and an offset of {offset} bits as file length is only {len(temp)} bits.")
577 def _setbitarray(self, ba: bitarray.bitarray, length: Optional[int], offset: Optional[int]) -> None:
578 if offset is None:
579 offset = 0
580 if offset > len(ba):
581 raise bitstring.CreationError(f"Offset of {offset} too large for bitarray of length {len(ba)}.")
582 if length is None:
583 self._bitstore = ConstBitStore(ba[offset:])
584 else:
585 if offset + length > len(ba):
586 raise bitstring.CreationError(
587 f"Offset of {offset} and length of {length} too large for bitarray of length {len(ba)}.")
588 self._bitstore = ConstBitStore(ba[offset: offset + length])
590 def _setbits(self, bs: BitsType, length: None = None) -> None:
591 bs = Bits._create_from_bitstype(bs)
592 self._bitstore = bs._bitstore
594 def _setp3binary(self, f: float) -> None:
595 self._bitstore = common_helpers.p3binary2bitstore(f)
597 def _setp4binary(self, f: float) -> None:
598 self._bitstore = common_helpers.p4binary2bitstore(f)
600 def _sete4m3mxfp(self, f: float) -> None:
601 self._bitstore = common_helpers.e4m3mxfp2bitstore(f)
603 def _sete5m2mxfp(self, f: float) -> None:
604 self._bitstore = common_helpers.e5m2mxfp2bitstore(f)
606 def _sete3m2mxfp(self, f: float) -> None:
607 self._bitstore = common_helpers.e3m2mxfp2bitstore(f)
609 def _sete2m3mxfp(self, f: float) -> None:
610 self._bitstore = common_helpers.e2m3mxfp2bitstore(f)
612 def _sete2m1mxfp(self, f: float) -> None:
613 self._bitstore = common_helpers.e2m1mxfp2bitstore(f)
615 def _sete8m0mxfp(self, f: float) -> None:
616 self._bitstore = common_helpers.e8m0mxfp2bitstore(f)
618 def _setmxint(self, f: float) -> None:
619 self._bitstore = common_helpers.mxint2bitstore(f)
621 def _setbytes(self, data: Union[bytearray, bytes, List], length:None = None) -> None:
622 """Set the data from a bytes or bytearray object."""
623 self._bitstore = ConstBitStore.from_bytes(bytes(data))
625 def _setbytes_with_truncation(self, data: Union[bytearray, bytes], length: Optional[int] = None, offset: Optional[int] = None) -> None:
626 """Set the data from a bytes or bytearray object, with optional offset and length truncations."""
627 if offset is None and length is None:
628 return self._setbytes(data)
629 data = bytearray(data)
630 if offset is None:
631 offset = 0
632 if length is None:
633 # Use to the end of the data
634 length = len(data) * 8 - offset
635 else:
636 if length + offset > len(data) * 8:
637 raise bitstring.CreationError(f"Not enough data present. Need {length + offset} bits, have {len(data) * 8}.")
638 self._bitstore = ConstBitStore.from_bytes(data).getslice_msb0(offset, offset + length)
640 def _getbytes(self) -> bytes:
641 """Return the data as an ordinary bytes object."""
642 if len(self) % 8:
643 raise bitstring.InterpretError("Cannot interpret as bytes unambiguously - not multiple of 8 bits.")
644 return self._bitstore.to_bytes()
646 _unprintable = list(range(0x00, 0x20)) # ASCII control characters
647 _unprintable.extend(range(0x7f, 0xff)) # DEL char + non-ASCII
649 def _getbytes_printable(self) -> str:
650 """Return an approximation of the data as a string of printable characters."""
651 bytes_ = self._getbytes()
652 # For everything that isn't printable ASCII, use value from 'Latin Extended-A' unicode block.
653 string = ''.join(chr(0x100 + x) if x in Bits._unprintable else chr(x) for x in bytes_)
654 return string
656 def _setuint(self, uint: int, length: Optional[int] = None) -> None:
657 """Reset the bitstring to have given unsigned int interpretation."""
658 # If no length given, and we've previously been given a length, use it.
659 if length is None and hasattr(self, 'len') and len(self) != 0:
660 length = len(self)
661 if length is None or length == 0:
662 raise bitstring.CreationError("A non-zero length must be specified with a uint initialiser.")
663 self._bitstore = helpers.int2bitstore(uint, length, False)
665 def _getuint(self) -> int:
666 """Return data as an unsigned int."""
667 if len(self) == 0:
668 raise bitstring.InterpretError("Cannot interpret a zero length bitstring as an integer.")
669 return self._bitstore.to_u()
671 def _setint(self, int_: int, length: Optional[int] = None) -> None:
672 """Reset the bitstring to have given signed int interpretation."""
673 # If no length given, and we've previously been given a length, use it.
674 if length is None and hasattr(self, 'len') and len(self) != 0:
675 length = len(self)
676 if length is None or length == 0:
677 raise bitstring.CreationError("A non-zero length must be specified with an int initialiser.")
678 self._bitstore = helpers.int2bitstore(int_, length, True)
680 def _getint(self) -> int:
681 """Return data as a two's complement signed int."""
682 if len(self) == 0:
683 raise bitstring.InterpretError("Cannot interpret bitstring without a length as an integer.")
684 return self._bitstore.to_i()
686 def _setuintbe(self, uintbe: int, length: Optional[int] = None) -> None:
687 """Set the bitstring to a big-endian unsigned int interpretation."""
688 if length is None and hasattr(self, 'len') and len(self) != 0:
689 length = len(self)
690 if length is None or length == 0:
691 raise bitstring.CreationError("A non-zero length must be specified with a uintbe initialiser.")
692 self._bitstore = helpers.int2bitstore(uintbe, length, False)
694 def _getuintbe(self) -> int:
695 """Return data as a big-endian two's complement unsigned int."""
696 if len(self) % 8:
697 raise bitstring.InterpretError(f"Big-endian integers must be whole-byte. Length = {len(self)} bits.")
698 return self._getuint()
700 def _setintbe(self, intbe: int, length: Optional[int] = None) -> None:
701 """Set bitstring to a big-endian signed int interpretation."""
702 if length is None and hasattr(self, 'len') and len(self) != 0:
703 length = len(self)
704 if length is None or length == 0:
705 raise bitstring.CreationError("A non-zero length must be specified with a intbe initialiser.")
706 self._bitstore = helpers.int2bitstore(intbe, length, True)
708 def _getintbe(self) -> int:
709 """Return data as a big-endian two's complement signed int."""
710 if len(self) % 8:
711 raise bitstring.InterpretError(f"Big-endian integers must be whole-byte. Length = {len(self)} bits.")
712 return self._getint()
714 def _setuintle(self, uintle: int, length: Optional[int] = None) -> None:
715 if length is None and hasattr(self, 'len') and len(self) != 0:
716 length = len(self)
717 if length is None or length == 0:
718 raise bitstring.CreationError("A non-zero length must be specified with a uintle initialiser.")
719 self._bitstore = helpers.intle2bitstore(uintle, length, False)
721 def _getuintle(self) -> int:
722 """Interpret as a little-endian unsigned int."""
723 if len(self) % 8:
724 raise bitstring.InterpretError(f"Little-endian integers must be whole-byte. Length = {len(self)} bits.")
725 bs = ConstBitStore.from_bytes(self._bitstore.to_bytes()[::-1])
726 return bs.to_u()
728 def _setintle(self, intle: int, length: Optional[int] = None) -> None:
729 if length is None and hasattr(self, 'len') and len(self) != 0:
730 length = len(self)
731 if length is None or length == 0:
732 raise bitstring.CreationError("A non-zero length must be specified with an intle initialiser.")
733 self._bitstore = helpers.intle2bitstore(intle, length, True)
735 def _getintle(self) -> int:
736 """Interpret as a little-endian signed int."""
737 if len(self) % 8:
738 raise bitstring.InterpretError(f"Little-endian integers must be whole-byte. Length = {len(self)} bits.")
739 bs = ConstBitStore.from_bytes(self._bitstore.to_bytes()[::-1])
740 return bs.to_i()
742 def _getp4binary(self) -> float:
743 u = self._getuint()
744 return p4binary_fmt.lut_binary8_to_float[u]
746 def _getp3binary(self) -> float:
747 u = self._getuint()
748 return p3binary_fmt.lut_binary8_to_float[u]
750 def _gete4m3mxfp(self) -> float:
751 u = self._getuint()
752 return e4m3mxfp_saturate_fmt.lut_int_to_float[u]
754 def _gete5m2mxfp(self) -> float:
755 u = self._getuint()
756 return e5m2mxfp_saturate_fmt.lut_int_to_float[u]
758 def _gete3m2mxfp(self) -> float:
759 u = self._getuint()
760 return e3m2mxfp_fmt.lut_int_to_float[u]
762 def _gete2m3mxfp(self) -> float:
763 u = self._getuint()
764 return e2m3mxfp_fmt.lut_int_to_float[u]
766 def _gete2m1mxfp(self) -> float:
767 u = self._getuint()
768 return e2m1mxfp_fmt.lut_int_to_float[u]
770 def _gete8m0mxfp(self) -> float:
771 u = self._getuint() - 127
772 if u == 128:
773 return float('nan')
774 return 2.0 ** u
776 def _getmxint(self) -> float:
777 u = self._getint()
778 return float(u) * 2 ** -6
780 def _setfloat(self, f: float, length: Optional[int], big_endian: bool) -> None:
781 if length is None and hasattr(self, 'len') and len(self) != 0:
782 length = len(self)
783 if length is None or length not in [16, 32, 64]:
784 raise bitstring.CreationError("A length of 16, 32, or 64 must be specified with a float initialiser.")
785 self._bitstore = helpers.float2bitstore(f, length, big_endian)
787 def _setfloatbe(self, f: float, length: Optional[int] = None) -> None:
788 self._setfloat(f, length, True)
790 def _getfloatbe(self) -> float:
791 """Interpret the whole bitstring as a big-endian float."""
792 fmt = {16: '>e', 32: '>f', 64: '>d'}[len(self)]
793 return struct.unpack(fmt, self._bitstore.to_bytes())[0]
795 def _setfloatle(self, f: float, length: Optional[int] = None) -> None:
796 self._setfloat(f, length, False)
798 def _getfloatle(self) -> float:
799 """Interpret the whole bitstring as a little-endian float."""
800 fmt = {16: '<e', 32: '<f', 64: '<d'}[len(self)]
801 return struct.unpack(fmt, self._bitstore.to_bytes())[0]
803 def _getbfloatbe(self) -> float:
804 zero_padded = self + Bits(16)
805 return zero_padded._getfloatbe()
807 def _setbfloatbe(self, f: Union[float, str], length: Optional[int] = None) -> None:
808 if length is not None and length != 16:
809 raise bitstring.CreationError(f"bfloats must be length 16, received a length of {length} bits.")
810 self._bitstore = common_helpers.bfloat2bitstore(f, True)
812 def _getbfloatle(self) -> float:
813 zero_padded = Bits(16) + self
814 return zero_padded._getfloatle()
816 def _setbfloatle(self, f: Union[float, str], length: Optional[int] = None) -> None:
817 if length is not None and length != 16:
818 raise bitstring.CreationError(f"bfloats must be length 16, received a length of {length} bits.")
819 self._bitstore = common_helpers.bfloat2bitstore(f, False)
821 def _setue(self, i: int) -> None:
822 """Initialise bitstring with unsigned exponential-Golomb code for integer i.
824 Raises CreationError if i < 0.
826 """
827 if bitstring.options.lsb0:
828 raise bitstring.CreationError("Exp-Golomb codes cannot be used in lsb0 mode.")
829 self._bitstore = common_helpers.ue2bitstore(i)
831 def _readue(self, pos: int) -> Tuple[int, int]:
832 """Return interpretation of next bits as unsigned exponential-Golomb code.
834 Raises ReadError if the end of the bitstring is encountered while
835 reading the code.
837 """
838 if bitstring.options.lsb0:
839 raise bitstring.ReadError("Exp-Golomb codes cannot be read in lsb0 mode.")
840 oldpos = pos
841 try:
842 while not self[pos]:
843 pos += 1
844 except IndexError:
845 raise bitstring.ReadError("Read off end of bitstring trying to read code.")
846 leadingzeros = pos - oldpos
847 codenum = (1 << leadingzeros) - 1
848 if leadingzeros > 0:
849 if pos + leadingzeros + 1 > len(self):
850 raise bitstring.ReadError("Read off end of bitstring trying to read code.")
851 codenum += self[pos + 1:pos + 1 + leadingzeros]._getuint()
852 pos += leadingzeros + 1
853 else:
854 assert codenum == 0
855 pos += 1
856 return codenum, pos
858 def _getue(self) -> Tuple[int, int]:
859 try:
860 return self._readue(0)
861 except bitstring.ReadError:
862 raise bitstring.InterpretError
864 def _getse(self) -> Tuple[int, int]:
865 try:
866 return self._readse(0)
867 except bitstring.ReadError:
868 raise bitstring.InterpretError
870 def _getuie(self) -> Tuple[int, int]:
871 try:
872 return self._readuie(0)
873 except bitstring.ReadError:
874 raise bitstring.InterpretError
876 def _getsie(self) -> Tuple[int, int]:
877 try:
878 return self._readsie(0)
879 except bitstring.ReadError:
880 raise bitstring.InterpretError
882 def _setse(self, i: int) -> None:
883 """Initialise bitstring with signed exponential-Golomb code for integer i."""
884 if bitstring.options.lsb0:
885 raise bitstring.CreationError("Exp-Golomb codes cannot be used in lsb0 mode.")
886 self._bitstore = common_helpers.se2bitstore(i)
888 def _readse(self, pos: int) -> Tuple[int, int]:
889 """Return interpretation of next bits as a signed exponential-Golomb code.
891 Advances position to after the read code.
893 Raises ReadError if the end of the bitstring is encountered while
894 reading the code.
896 """
897 codenum, pos = self._readue(pos)
898 m = (codenum + 1) // 2
899 return (m, pos) if codenum % 2 else (-m, pos)
901 def _setuie(self, i: int) -> None:
902 """Initialise bitstring with unsigned interleaved exponential-Golomb code for integer i.
904 Raises CreationError if i < 0.
906 """
907 if bitstring.options.lsb0:
908 raise bitstring.CreationError("Exp-Golomb codes cannot be used in lsb0 mode.")
909 self._bitstore = common_helpers.uie2bitstore(i)
911 def _readuie(self, pos: int) -> Tuple[int, int]:
912 """Return interpretation of next bits as unsigned interleaved exponential-Golomb code.
914 Raises ReadError if the end of the bitstring is encountered while
915 reading the code.
917 """
918 if bitstring.options.lsb0:
919 raise bitstring.ReadError("Exp-Golomb codes cannot be read in lsb0 mode.")
920 try:
921 codenum: int = 1
922 while not self[pos]:
923 pos += 1
924 codenum <<= 1
925 codenum += self[pos]
926 pos += 1
927 pos += 1
928 except IndexError:
929 raise bitstring.ReadError("Read off end of bitstring trying to read code.")
930 return codenum - 1, pos
932 def _setsie(self, i: int, ) -> None:
933 """Initialise bitstring with signed interleaved exponential-Golomb code for integer i."""
934 if bitstring.options.lsb0:
935 raise bitstring.CreationError("Exp-Golomb codes cannot be used in lsb0 mode.")
936 self._bitstore = common_helpers.sie2bitstore(i)
938 def _readsie(self, pos: int) -> Tuple[int, int]:
939 """Return interpretation of next bits as a signed interleaved exponential-Golomb code.
941 Advances position to after the read code.
943 Raises ReadError if the end of the bitstring is encountered while
944 reading the code.
946 """
947 codenum, pos = self._readuie(pos)
948 if not codenum:
949 return 0, pos
950 try:
951 return (-codenum, pos + 1) if self[pos] else (codenum, pos + 1)
952 except IndexError:
953 raise bitstring.ReadError("Read off end of bitstring trying to read code.")
955 def _setbool(self, value: Union[bool, str]) -> None:
956 # We deliberately don't want to have implicit conversions to bool here.
957 # If we did then it would be difficult to deal with the 'False' string.
958 if value in (1, 'True', '1'):
959 self._bitstore = ConstBitStore.from_bin('1')
960 elif value in (0, 'False', '0'):
961 self._bitstore = ConstBitStore.from_bin('0')
962 else:
963 raise bitstring.CreationError(f"Cannot initialise boolean with {value}.")
965 def _getbool(self) -> bool:
966 return self[0]
968 def _getpad(self) -> None:
969 return None
971 def _setpad(self, value: None, length: int) -> None:
972 self._bitstore = ConstBitStore.from_zeros(length)
974 def _setbin(self, binstring: str, length: None = None) -> None:
975 """Reset the bitstring to the value given in binstring."""
976 self._bitstore = helpers.bin2bitstore(binstring)
978 def _getbin(self) -> str:
979 """Return interpretation as a binary string."""
980 return self._bitstore.to_bin()
982 def _setoct(self, octstring: str, length: None = None) -> None:
983 """Reset the bitstring to have the value given in octstring."""
984 self._bitstore = helpers.oct2bitstore(octstring)
986 def _getoct(self) -> str:
987 """Return interpretation as an octal string."""
988 return self._bitstore.to_oct()
990 def _sethex(self, hexstring: str, length: None = None) -> None:
991 """Reset the bitstring to have the value given in hexstring."""
992 self._bitstore = helpers.hex2bitstore(hexstring)
994 def _gethex(self) -> str:
995 """Return the hexadecimal representation as a string.
997 Raises an InterpretError if the bitstring's length is not a multiple of 4.
999 """
1000 return self._bitstore.to_hex()
1002 def _getlength(self) -> int:
1003 """Return the length of the bitstring in bits."""
1004 return len(self._bitstore)
1006 def _copy(self: TBits) -> TBits:
1007 """Create and return a new copy of the Bits (always in memory)."""
1008 # Note that __copy__ may choose to return self if it's immutable. This method always makes a copy.
1009 s_copy = self.__class__()
1010 s_copy._bitstore = self._bitstore._mutable_copy()
1011 return s_copy
1013 def _slice(self: TBits, start: int, end: int) -> TBits:
1014 """Used internally to get a slice, without error checking."""
1015 bs = self.__class__()
1016 bs._bitstore = self._bitstore.getslice(start, end)
1017 return bs
1019 def _absolute_slice(self: TBits, start: int, end: int) -> TBits:
1020 """Used internally to get a slice, without error checking.
1021 Uses MSB0 bit numbering even if LSB0 is set."""
1022 if end == start:
1023 return self.__class__()
1024 assert start < end, f"start={start}, end={end}"
1025 bs = self.__class__()
1026 bs._bitstore = self._bitstore.getslice_msb0(start, end)
1027 return bs
1029 def _readtoken(self, name: str, pos: int, length: Optional[int]) -> Tuple[Union[float, int, str, None, Bits], int]:
1030 """Reads a token from the bitstring and returns the result."""
1031 dtype = dtype_register.get_dtype(name, length)
1032 if dtype.bitlength is not None and dtype.bitlength > len(self) - pos:
1033 raise bitstring.ReadError("Reading off the end of the data. "
1034 f"Tried to read {dtype.bitlength} bits when only {len(self) - pos} available.")
1035 try:
1036 val = dtype.read_fn(self, pos)
1037 if isinstance(val, tuple):
1038 return val
1039 else:
1040 assert length is not None
1041 return val, pos + dtype.bitlength
1042 except KeyError:
1043 raise ValueError(f"Can't parse token {name}:{length}")
1045 def _addright(self, bs: Bits, /) -> None:
1046 """Add a bitstring to the RHS of the current bitstring."""
1047 self._bitstore += bs._bitstore
1049 def _addleft(self, bs: Bits, /) -> None:
1050 """Prepend a bitstring to the current bitstring."""
1051 self._bitstore.extend_left(bs._bitstore)
1053 def _insert(self, bs: Bits, pos: int, /) -> None:
1054 """Insert bs at pos."""
1055 assert 0 <= pos <= len(self)
1056 self._bitstore[pos: pos] = bs._bitstore
1057 return
1059 def _overwrite(self, bs: Bits, pos: int, /) -> None:
1060 """Overwrite with bs at pos."""
1061 assert 0 <= pos <= len(self)
1062 if bs is self:
1063 # Just overwriting with self, so do nothing.
1064 assert pos == 0
1065 return
1066 self._bitstore[pos: pos + len(bs)] = bs._bitstore
1068 def _delete(self, bits: int, pos: int, /) -> None:
1069 """Delete bits at pos."""
1070 assert 0 <= pos <= len(self)
1071 assert pos + bits <= len(self), f"pos={pos}, bits={bits}, len={len(self)}"
1072 del self._bitstore[pos: pos + bits]
1074 def _reversebytes(self, start: int, end: int) -> None:
1075 """Reverse bytes in-place."""
1076 assert (end - start) % 8 == 0
1077 self._bitstore[start:end] = ConstBitStore.from_bytes(self._bitstore.getslice(start, end).to_bytes()[::-1])
1079 def _invert(self, pos: int, /) -> None:
1080 """Flip bit at pos 1<->0."""
1081 assert 0 <= pos < len(self)
1082 self._bitstore.invert(pos)
1084 def _ilshift(self: TBits, n: int, /) -> TBits:
1085 """Shift bits by n to the left in place. Return self."""
1086 self._bitstore.__ilshift__(n)
1087 return self
1089 def _irshift(self: TBits, n: int, /) -> TBits:
1090 """Shift bits by n to the right in place. Return self."""
1091 self._bitstore.__irshift__(n)
1092 return self
1094 def _getbits(self: TBits):
1095 return self._copy()
1097 def _validate_slice(self, start: Optional[int], end: Optional[int]) -> Tuple[int, int]:
1098 """Validate start and end and return them as positive bit positions."""
1099 start = 0 if start is None else (start + len(self) if start < 0 else start)
1100 end = len(self) if end is None else (end + len(self) if end < 0 else end)
1101 if not 0 <= start <= end <= len(self):
1102 raise ValueError(f"Invalid slice positions for bitstring length {len(self)}: start={start}, end={end}.")
1103 return start, end
1105 def unpack(self, fmt: Union[str, List[Union[str, int]]], **kwargs) -> List[Union[int, float, str, Bits, bool, bytes, None]]:
1106 """Interpret the whole bitstring using fmt and return list.
1108 fmt -- A single string or a list of strings with comma separated tokens
1109 describing how to interpret the bits in the bitstring. Items
1110 can also be integers, for reading new bitstring of the given length.
1111 kwargs -- A dictionary or keyword-value pairs - the keywords used in the
1112 format string will be replaced with their given value.
1114 Raises ValueError if the format is not understood. If not enough bits
1115 are available then all bits to the end of the bitstring will be used.
1117 See the docstring for 'read' for token examples.
1119 """
1120 return self._readlist(fmt, 0, **kwargs)[0]
1122 def _readlist(self, fmt: Union[str, List[Union[str, int, Dtype]]], pos: int, **kwargs) \
1123 -> Tuple[List[Union[int, float, str, Bits, bool, bytes, None]], int]:
1124 if isinstance(fmt, str):
1125 fmt = [fmt]
1126 # Convert to a flat list of Dtypes
1127 dtype_list = []
1128 for f_item in fmt:
1129 if isinstance(f_item, numbers.Integral):
1130 dtype_list.append(Dtype('bits', f_item))
1131 elif isinstance(f_item, Dtype):
1132 dtype_list.append(f_item)
1133 else:
1134 token_list = utils.preprocess_tokens(f_item)
1135 for t in token_list:
1136 try:
1137 name, length = utils.parse_name_length_token(t, **kwargs)
1138 except ValueError:
1139 dtype_list.append(Dtype('bits', int(t)))
1140 else:
1141 dtype_list.append(Dtype(name, length))
1142 return self._read_dtype_list(dtype_list, pos)
1144 def _read_dtype_list(self, dtypes: List[Dtype], pos: int) -> Tuple[List[Union[int, float, str, Bits, bool, bytes, None]], int]:
1145 has_stretchy_token = False
1146 bits_after_stretchy_token = 0
1147 for dtype in dtypes:
1148 stretchy = dtype.bitlength is None and not dtype.variable_length
1149 if stretchy:
1150 if has_stretchy_token:
1151 raise bitstring.Error("It's not possible to have more than one 'filler' token.")
1152 has_stretchy_token = True
1153 elif has_stretchy_token:
1154 if dtype.variable_length:
1155 raise bitstring.Error(f"It's not possible to parse a variable length token '{dtype}' after a 'filler' token.")
1156 bits_after_stretchy_token += dtype.bitlength
1158 # We should have precisely zero or one stretchy token
1159 vals = []
1160 for dtype in dtypes:
1161 stretchy = dtype.bitlength is None and not dtype.variable_length
1162 if stretchy:
1163 bits_remaining = len(self) - pos
1164 # Set length to the remaining bits
1165 bitlength = max(bits_remaining - bits_after_stretchy_token, 0)
1166 items, remainder = divmod(bitlength, dtype.bits_per_item)
1167 if remainder != 0:
1168 raise ValueError(
1169 f"The '{dtype.name}' type must have a bit length that is a multiple of {dtype.bits_per_item}"
1170 f" so cannot be created from the {bitlength} bits that are available for this stretchy token.")
1171 dtype = Dtype(dtype.name, items)
1172 if dtype.bitlength is not None:
1173 val = dtype.read_fn(self, pos)
1174 pos += dtype.bitlength
1175 else:
1176 val, pos = dtype.read_fn(self, pos)
1177 if val is not None: # Don't append pad tokens
1178 vals.append(val)
1179 return vals, pos
1181 def find(self, bs: BitsType, /, start: Optional[int] = None, end: Optional[int] = None,
1182 bytealigned: Optional[bool] = None) -> Union[Tuple[int], Tuple[()]]:
1183 """Find first occurrence of substring bs.
1185 Returns a single item tuple with the bit position if found, or an
1186 empty tuple if not found. The bit position (pos property) will
1187 also be set to the start of the substring if it is found.
1189 bs -- The bitstring to find.
1190 start -- The bit position to start the search. Defaults to 0.
1191 end -- The bit position one past the last bit to search.
1192 Defaults to len(self).
1193 bytealigned -- If True the bitstring will only be
1194 found on byte boundaries.
1196 Raises ValueError if bs is empty, if start < 0, if end > len(self) or
1197 if end < start.
1199 >>> BitArray('0xc3e').find('0b1111')
1200 (6,)
1202 """
1203 bs = Bits._create_from_bitstype(bs)
1204 if len(bs) == 0:
1205 raise ValueError("Cannot find an empty bitstring.")
1206 start, end = self._validate_slice(start, end)
1207 ba = bitstring.options.bytealigned if bytealigned is None else bytealigned
1208 p = self._find(bs, start, end, ba)
1209 return p
1211 def _find_lsb0(self, bs: Bits, start: int, end: int, bytealigned: bool) -> Union[Tuple[int], Tuple[()]]:
1212 # A forward find in lsb0 is very like a reverse find in msb0.
1213 assert start <= end
1214 assert bitstring.options.lsb0
1216 new_slice = bitstring.helpers.offset_slice_indices_lsb0(slice(start, end, None), len(self))
1217 msb0_start, msb0_end = self._validate_slice(new_slice.start, new_slice.stop)
1218 p = self._rfind_msb0(bs, msb0_start, msb0_end, bytealigned)
1220 if p:
1221 return (len(self) - p[0] - len(bs),)
1222 else:
1223 return ()
1225 def _find_msb0(self, bs: Bits, start: int, end: int, bytealigned: bool) -> Union[Tuple[int], Tuple[()]]:
1226 """Find first occurrence of a binary string."""
1227 p = self._bitstore.find(bs._bitstore, start, end, bytealigned)
1228 return () if p is None else (p,)
1230 def findall(self, bs: BitsType, start: Optional[int] = None, end: Optional[int] = None, count: Optional[int] = None,
1231 bytealigned: Optional[bool] = None) -> Iterable[int]:
1232 """Find all occurrences of bs. Return generator of bit positions.
1234 bs -- The bitstring to find.
1235 start -- The bit position to start the search. Defaults to 0.
1236 end -- The bit position one past the last bit to search.
1237 Defaults to len(self).
1238 count -- The maximum number of occurrences to find.
1239 bytealigned -- If True the bitstring will only be found on
1240 byte boundaries.
1242 Raises ValueError if bs is empty, if start < 0, if end > len(self) or
1243 if end < start.
1245 Note that all occurrences of bs are found, even if they overlap.
1247 """
1248 if count is not None and count < 0:
1249 raise ValueError("In findall, count must be >= 0.")
1250 bs = Bits._create_from_bitstype(bs)
1251 start, end = self._validate_slice(start, end)
1252 ba = bitstring.options.bytealigned if bytealigned is None else bytealigned
1253 return self._findall(bs, start, end, count, ba)
1255 def _findall_msb0(self, bs: Bits, start: int, end: int, count: Optional[int],
1256 bytealigned: bool) -> Iterable[int]:
1257 c = 0
1258 for i in self._bitstore.findall_msb0(bs._bitstore, start, end, bytealigned):
1259 if count is not None and c >= count:
1260 return
1261 c += 1
1262 yield i
1263 return
1265 def _findall_lsb0(self, bs: Bits, start: int, end: int, count: Optional[int],
1266 bytealigned: bool) -> Iterable[int]:
1267 assert start <= end
1268 assert bitstring.options.lsb0
1270 new_slice = bitstring.helpers.offset_slice_indices_lsb0(slice(start, end, None), len(self))
1271 msb0_start, msb0_end = self._validate_slice(new_slice.start, new_slice.stop)
1273 # Search chunks starting near the end and then moving back.
1274 c = 0
1275 increment = max(8192, len(bs) * 80)
1276 buffersize = min(increment + len(bs), msb0_end - msb0_start)
1277 pos = max(msb0_start, msb0_end - buffersize)
1278 while True:
1279 found = list(self._findall_msb0(bs, start=pos, end=pos + buffersize, count=None, bytealigned=False))
1280 if not found:
1281 if pos == msb0_start:
1282 return
1283 pos = max(msb0_start, pos - increment)
1284 continue
1285 while found:
1286 if count is not None and c >= count:
1287 return
1288 c += 1
1289 lsb0_pos = len(self) - found.pop() - len(bs)
1290 if not bytealigned or lsb0_pos % 8 == 0:
1291 yield lsb0_pos
1293 pos = max(msb0_start, pos - increment)
1294 if pos == msb0_start:
1295 return
1297 def rfind(self, bs: BitsType, /, start: Optional[int] = None, end: Optional[int] = None,
1298 bytealigned: Optional[bool] = None) -> Union[Tuple[int], Tuple[()]]:
1299 """Find final occurrence of substring bs.
1301 Returns a single item tuple with the bit position if found, or an
1302 empty tuple if not found. The bit position (pos property) will
1303 also be set to the start of the substring if it is found.
1305 bs -- The bitstring to find.
1306 start -- The bit position to end the reverse search. Defaults to 0.
1307 end -- The bit position one past the first bit to reverse search.
1308 Defaults to len(self).
1309 bytealigned -- If True the bitstring will only be found on byte
1310 boundaries.
1312 Raises ValueError if bs is empty, if start < 0, if end > len(self) or
1313 if end < start.
1315 """
1316 bs = Bits._create_from_bitstype(bs)
1317 start, end = self._validate_slice(start, end)
1318 ba = bitstring.options.bytealigned if bytealigned is None else bytealigned
1319 if len(bs) == 0:
1320 raise ValueError("Cannot find an empty bitstring.")
1321 p = self._rfind(bs, start, end, ba)
1322 return p
1324 def _rfind_msb0(self, bs: Bits, start: int, end: int, bytealigned: bool) -> Union[Tuple[int], Tuple[()]]:
1325 """Find final occurrence of a binary string."""
1326 p = self._bitstore.rfind(bs._bitstore, start, end, bytealigned)
1327 return () if p is None else (p,)
1329 def _rfind_lsb0(self, bs: Bits, start: int, end: int, bytealigned: bool) -> Union[Tuple[int], Tuple[()]]:
1330 # A reverse find in lsb0 is very like a forward find in msb0.
1331 assert start <= end
1332 assert bitstring.options.lsb0
1333 new_slice = bitstring.helpers.offset_slice_indices_lsb0(slice(start, end, None), len(self))
1334 msb0_start, msb0_end = self._validate_slice(new_slice.start, new_slice.stop)
1336 p = self._find_msb0(bs, msb0_start, msb0_end, bytealigned)
1337 if p:
1338 return (len(self) - p[0] - len(bs),)
1339 else:
1340 return ()
1342 def cut(self, bits: int, start: Optional[int] = None, end: Optional[int] = None,
1343 count: Optional[int] = None) -> Iterator[Bits]:
1344 """Return bitstring generator by cutting into bits sized chunks.
1346 bits -- The size in bits of the bitstring chunks to generate.
1347 start -- The bit position to start the first cut. Defaults to 0.
1348 end -- The bit position one past the last bit to use in the cut.
1349 Defaults to len(self).
1350 count -- If specified then at most count items are generated.
1351 Default is to cut as many times as possible.
1353 """
1354 start_, end_ = self._validate_slice(start, end)
1355 if count is not None and count < 0:
1356 raise ValueError("Cannot cut - count must be >= 0.")
1357 if bits <= 0:
1358 raise ValueError("Cannot cut - bits must be >= 0.")
1359 c = 0
1360 while count is None or c < count:
1361 c += 1
1362 nextchunk = self._slice(start_, min(start_ + bits, end_))
1363 if len(nextchunk) == 0:
1364 return
1365 yield nextchunk
1366 if len(nextchunk) != bits:
1367 return
1368 start_ += bits
1369 return
1371 def split(self, delimiter: BitsType, start: Optional[int] = None, end: Optional[int] = None,
1372 count: Optional[int] = None, bytealigned: Optional[bool] = None) -> Iterable[Bits]:
1373 """Return bitstring generator by splitting using a delimiter.
1375 The first item returned is the initial bitstring before the delimiter,
1376 which may be an empty bitstring.
1378 delimiter -- The bitstring used as the divider.
1379 start -- The bit position to start the split. Defaults to 0.
1380 end -- The bit position one past the last bit to use in the split.
1381 Defaults to len(self).
1382 count -- If specified then at most count items are generated.
1383 Default is to split as many times as possible.
1384 bytealigned -- If True splits will only occur on byte boundaries.
1386 Raises ValueError if the delimiter is empty.
1388 """
1389 delimiter = Bits._create_from_bitstype(delimiter)
1390 if len(delimiter) == 0:
1391 raise ValueError("split delimiter cannot be empty.")
1392 start, end = self._validate_slice(start, end)
1393 bytealigned_: bool = bitstring.options.bytealigned if bytealigned is None else bytealigned
1394 if count is not None and count < 0:
1395 raise ValueError("Cannot split - count must be >= 0.")
1396 if count == 0:
1397 return
1398 f = functools.partial(self._find_msb0, bs=delimiter, bytealigned=bytealigned_)
1399 found = f(start=start, end=end)
1400 if not found:
1401 # Initial bits are the whole bitstring being searched
1402 yield self._slice(start, end)
1403 return
1404 # yield the bytes before the first occurrence of the delimiter, even if empty
1405 yield self._slice(start, found[0])
1406 startpos = pos = found[0]
1407 c = 1
1408 while count is None or c < count:
1409 pos += len(delimiter)
1410 found = f(start=pos, end=end)
1411 if not found:
1412 # No more occurrences, so return the rest of the bitstring
1413 yield self._slice(startpos, end)
1414 return
1415 c += 1
1416 yield self._slice(startpos, found[0])
1417 startpos = pos = found[0]
1418 # Have generated count bitstrings, so time to quit.
1419 return
1421 def join(self: TBits, sequence: Iterable[Any]) -> TBits:
1422 """Return concatenation of bitstrings joined by self.
1424 sequence -- A sequence of bitstrings.
1426 """
1427 bs = MutableBitStore()
1428 if len(self) == 0:
1429 # Optimised version that doesn't need to add self between every item
1430 for item in sequence:
1431 bs += Bits._create_from_bitstype(item)._bitstore
1432 else:
1433 sequence_iter = iter(sequence)
1434 try:
1435 bs += Bits._create_from_bitstype(next(sequence_iter))._bitstore
1436 except StopIteration:
1437 pass
1438 else:
1439 for item in sequence_iter:
1440 bs += self._bitstore
1441 bs += Bits._create_from_bitstype(item)._bitstore
1442 s = self.__class__()
1443 s._bitstore = bs
1444 return s
1446 def tobytes(self) -> bytes:
1447 """Return the bitstring as bytes, padding with zero bits if needed.
1449 Up to seven zero bits will be added at the end to byte align.
1451 """
1452 return self._bitstore.to_bytes()
1454 def tobitarray(self) -> bitarray.bitarray:
1455 """Convert the bitstring to a bitarray object."""
1456 return self._bitstore.tobitarray()
1458 def tofile(self, f: BinaryIO) -> None:
1459 """Write the bitstring to a file object, padding with zero bits if needed.
1461 Up to seven zero bits will be added at the end to byte align.
1463 """
1464 # If the bitstring is file based then we don't want to read it all in to memory first.
1465 chunk_size = 8 * 100 * 1024 * 1024 # 100 MiB
1466 for chunk in self.cut(chunk_size):
1467 f.write(chunk.tobytes())
1469 def startswith(self, prefix: BitsType, start: Optional[int] = None, end: Optional[int] = None) -> bool:
1470 """Return whether the current bitstring starts with prefix.
1472 prefix -- The bitstring to search for.
1473 start -- The bit position to start from. Defaults to 0.
1474 end -- The bit position to end at. Defaults to len(self).
1476 """
1477 prefix = self._create_from_bitstype(prefix)
1478 start, end = self._validate_slice(start, end)
1479 return self._slice(start, start + len(prefix)) == prefix if end >= start + len(prefix) else False
1481 def endswith(self, suffix: BitsType, start: Optional[int] = None, end: Optional[int] = None) -> bool:
1482 """Return whether the current bitstring ends with suffix.
1484 suffix -- The bitstring to search for.
1485 start -- The bit position to start from. Defaults to 0.
1486 end -- The bit position to end at. Defaults to len(self).
1488 """
1489 suffix = self._create_from_bitstype(suffix)
1490 start, end = self._validate_slice(start, end)
1491 return self._slice(end - len(suffix), end) == suffix if start + len(suffix) <= end else False
1493 def all(self, value: Any, pos: Optional[Iterable[int]] = None) -> bool:
1494 """Return True if one or many bits are all set to bool(value).
1496 value -- If value is True then checks for bits set to 1, otherwise
1497 checks for bits set to 0.
1498 pos -- An iterable of bit positions. Negative numbers are treated in
1499 the same way as slice indices. Defaults to the whole bitstring.
1501 """
1502 value = 1 if bool(value) else 0
1503 if pos is None:
1504 return self._bitstore.all() if value else not self._bitstore.any()
1505 for p in pos:
1506 if self._bitstore.getindex(p) != value:
1507 return False
1508 return True
1510 def any(self, value: Any, pos: Optional[Iterable[int]] = None) -> bool:
1511 """Return True if any of one or many bits are set to bool(value).
1513 value -- If value is True then checks for bits set to 1, otherwise
1514 checks for bits set to 0.
1515 pos -- An iterable of bit positions. Negative numbers are treated in
1516 the same way as slice indices. Defaults to the whole bitstring.
1518 """
1519 value = 1 if bool(value) else 0
1520 if pos is None:
1521 return self._bitstore.any() if value else not self._bitstore.all()
1522 for p in pos:
1523 if self._bitstore.getindex(p) == value:
1524 return True
1525 return False
1527 def count(self, value: Any) -> int:
1528 """Return count of total number of either zero or one bits.
1530 value -- If bool(value) is True then bits set to 1 are counted, otherwise bits set
1531 to 0 are counted.
1533 >>> Bits('0xef').count(1)
1534 7
1536 """
1537 # count the number of 1s (from which it's easy to work out the 0s).
1538 count = self._bitstore.count(1)
1539 return count if value else len(self) - count
1541 @staticmethod
1542 def _format_bits(bits: Bits, bits_per_group: int, sep: str, dtype: Dtype,
1543 colour_start: str, colour_end: str, width: Optional[int]=None) -> Tuple[str, int]:
1544 get_fn = dtype.get_fn
1545 if dtype.name == 'bytes': # Special case for bytes to print one character each.
1546 get_fn = Bits._getbytes_printable
1547 if dtype.name == 'bool': # Special case for bool to print '1' or '0' instead of `True` or `False`.
1548 get_fn = dtype_register.get_dtype('uint', bits_per_group).get_fn
1549 if bits_per_group == 0:
1550 x = str(get_fn(bits))
1551 else:
1552 # Left-align for fixed width types when msb0, otherwise right-align.
1553 align = '<' if dtype.name in ['bin', 'oct', 'hex', 'bits', 'bytes'] and not bitstring.options.lsb0 else '>'
1554 chars_per_group = 0
1555 if dtype_register[dtype.name].bitlength2chars_fn is not None:
1556 chars_per_group = dtype_register[dtype.name].bitlength2chars_fn(bits_per_group)
1557 x = sep.join(f"{str(get_fn(b)): {align}{chars_per_group}}" for b in bits.cut(bits_per_group))
1559 chars_used = len(x)
1560 padding_spaces = 0 if width is None else max(width - len(x), 0)
1561 x = colour_start + x + colour_end
1562 # Pad final line with spaces to align it
1563 if bitstring.options.lsb0:
1564 x = ' ' * padding_spaces + x
1565 else:
1566 x += ' ' * padding_spaces
1567 return x, chars_used
1569 @staticmethod
1570 def _chars_per_group(bits_per_group: int, fmt: Optional[str]):
1571 """How many characters are needed to represent a number of bits with a given format."""
1572 if fmt is None or dtype_register[fmt].bitlength2chars_fn is None:
1573 return 0
1574 return dtype_register[fmt].bitlength2chars_fn(bits_per_group)
1576 @staticmethod
1577 def _bits_per_char(fmt: str):
1578 """How many bits are represented by each character of a given format."""
1579 if fmt not in ['bin', 'oct', 'hex', 'bytes']:
1580 raise ValueError
1581 return 24 // dtype_register[fmt].bitlength2chars_fn(24)
1583 def _pp(self, dtype1: Dtype, dtype2: Optional[Dtype], bits_per_group: int, width: int, sep: str, format_sep: str,
1584 show_offset: bool, stream: TextIO, lsb0: bool, offset_factor: int) -> None:
1585 """Internal pretty print method."""
1586 colour = Colour(not bitstring.options.no_color)
1587 name1 = dtype1.name
1588 name2 = dtype2.name if dtype2 is not None else None
1589 if dtype1.variable_length:
1590 raise ValueError(f"Can't use Dtype '{dtype1}' in pp() as it has a variable length.")
1591 if dtype2 is not None and dtype2.variable_length:
1592 raise ValueError(f"Can't use Dtype '{dtype2}' in pp() as it has a variable length.")
1593 offset_width = 0
1594 offset_sep = ' :' if lsb0 else ': '
1595 if show_offset:
1596 # This could be 1 too large in some circumstances. Slightly recurrent logic needed to fix it...
1597 offset_width = len(str(len(self))) + len(offset_sep)
1598 if bits_per_group > 0:
1599 group_chars1 = Bits._chars_per_group(bits_per_group, name1)
1600 group_chars2 = Bits._chars_per_group(bits_per_group, name2)
1601 # The number of characters that get added when we add an extra group (after the first one)
1602 total_group_chars = group_chars1 + group_chars2 + len(sep) + len(sep) * bool(group_chars2)
1603 width_excluding_offset_and_final_group = width - offset_width - group_chars1 - group_chars2 - len(
1604 format_sep) * bool(group_chars2)
1605 width_excluding_offset_and_final_group = max(width_excluding_offset_and_final_group, 0)
1606 groups_per_line = 1 + width_excluding_offset_and_final_group // total_group_chars
1607 max_bits_per_line = groups_per_line * bits_per_group # Number of bits represented on each line
1608 else:
1609 assert bits_per_group == 0 # Don't divide into groups
1610 width_available = width - offset_width - len(format_sep) * (name2 is not None)
1611 width_available = max(width_available, 1)
1612 if name2 is None:
1613 max_bits_per_line = width_available * Bits._bits_per_char(name1)
1614 else:
1615 chars_per_24_bits = dtype_register[name1].bitlength2chars_fn(24) + dtype_register[name2].bitlength2chars_fn(24)
1616 max_bits_per_line = 24 * (width_available // chars_per_24_bits)
1617 if max_bits_per_line == 0:
1618 max_bits_per_line = 24 # We can't fit into the width asked for. Show something small.
1619 assert max_bits_per_line > 0
1621 bitpos = 0
1622 first_fb_width = second_fb_width = None
1623 for bits in self.cut(max_bits_per_line):
1624 offset_str = ''
1625 if show_offset:
1626 offset = bitpos // offset_factor
1627 bitpos += len(bits)
1628 if bitstring.options.lsb0:
1629 offset_str = colour.green + offset_sep + f'{offset: <{offset_width - len(offset_sep)}}' + colour.off
1630 else:
1631 offset_str = colour.green + f'{offset: >{offset_width - len(offset_sep)}}' + offset_sep + colour.off
1633 fb1, chars_used = Bits._format_bits(bits, bits_per_group, sep, dtype1, colour.purple, colour.off, first_fb_width)
1634 if first_fb_width is None:
1635 first_fb_width = chars_used
1637 fb2 = ''
1638 if dtype2 is not None:
1639 fb2, chars_used = Bits._format_bits(bits, bits_per_group, sep, dtype2, colour.blue, colour.off, second_fb_width)
1640 if second_fb_width is None:
1641 second_fb_width = chars_used
1642 fb2 = format_sep + fb2
1644 if bitstring.options.lsb0 is True:
1645 line_fmt = fb1 + fb2 + offset_str + '\n'
1646 else:
1647 line_fmt = offset_str + fb1 + fb2 + '\n'
1648 stream.write(line_fmt)
1649 return
1651 @staticmethod
1652 def _process_pp_tokens(token_list, fmt):
1653 if len(token_list) not in [1, 2]:
1654 raise ValueError(
1655 f"Only one or two tokens can be used in an pp() format - '{fmt}' has {len(token_list)} tokens.")
1656 has_length_in_fmt = True
1657 name1, length1 = utils.parse_name_length_token(token_list[0])
1658 dtype1 = Dtype(name1, length1)
1659 bits_per_group = dtype1.bitlength
1660 dtype2 = None
1662 if len(token_list) == 2:
1663 name2, length2 = utils.parse_name_length_token(token_list[1])
1664 dtype2 = Dtype(name2, length2)
1665 if None not in {dtype1.bitlength, dtype2.bitlength} and dtype1.bitlength != dtype2.bitlength:
1666 raise ValueError(
1667 f"Differing bit lengths of {dtype1.bitlength} and {dtype2.bitlength} in format string '{fmt}'.")
1668 if bits_per_group is None:
1669 bits_per_group = dtype2.bitlength
1671 if bits_per_group is None:
1672 has_length_in_fmt = False
1673 if len(token_list) == 1:
1674 bits_per_group = {'bin': 8, 'hex': 8, 'oct': 12, 'bytes': 32}.get(dtype1.name)
1675 if bits_per_group is None:
1676 raise ValueError(f"No length or default length available for pp() format '{fmt}'.")
1677 else:
1678 try:
1679 bits_per_group = 2 * Bits._bits_per_char(dtype1.name) * Bits._bits_per_char(dtype2.name)
1680 except ValueError:
1681 raise ValueError(f"Can't find a default bitlength to use for pp() format '{fmt}'.")
1682 if bits_per_group >= 24:
1683 bits_per_group //= 2
1684 return dtype1, dtype2, bits_per_group, has_length_in_fmt
1686 def pp(self, fmt: Optional[str] = None, width: int = 120, sep: str = ' ',
1687 show_offset: bool = True, stream: TextIO = sys.stdout) -> None:
1688 """Pretty print the bitstring's value.
1690 fmt -- Printed data format. One or two of 'bin', 'oct', 'hex' or 'bytes'.
1691 The number of bits represented in each printed group defaults to 8 for hex and bin,
1692 12 for oct and 32 for bytes. This can be overridden with an explicit length, e.g. 'hex:64'.
1693 Use a length of 0 to not split into groups, e.g. `bin:0`.
1694 width -- Max width of printed lines. Defaults to 120. A single group will always be printed
1695 per line even if it exceeds the max width.
1696 sep -- A separator string to insert between groups. Defaults to a single space.
1697 show_offset -- If True (the default) shows the bit offset in the first column of each line.
1698 stream -- A TextIO object with a write() method. Defaults to sys.stdout.
1700 >>> s.pp('hex16')
1701 >>> s.pp('b, h', sep='_', show_offset=False)
1703 """
1704 colour = Colour(not bitstring.options.no_color)
1705 if fmt is None:
1706 fmt = 'bin, hex' if len(self) % 8 == 0 and len(self) >= 8 else 'bin'
1707 token_list = utils.preprocess_tokens(fmt)
1708 dtype1, dtype2, bits_per_group, has_length_in_fmt = Bits._process_pp_tokens(token_list, fmt)
1709 trailing_bit_length = len(self) % bits_per_group if has_length_in_fmt and bits_per_group else 0
1710 data = self if trailing_bit_length == 0 else self[0: -trailing_bit_length]
1711 format_sep = " : " # String to insert on each line between multiple formats
1712 tidy_fmt = colour.purple + str(dtype1) + colour.off
1713 if dtype2 is not None:
1714 tidy_fmt += ', ' + colour.blue + str(dtype2) + colour.off
1715 output_stream = io.StringIO()
1716 len_str = colour.green + str(len(self)) + colour.off
1717 output_stream.write(f"<{self.__class__.__name__}, fmt='{tidy_fmt}', length={len_str} bits> [\n")
1718 data._pp(dtype1, dtype2, bits_per_group, width, sep, format_sep, show_offset,
1719 output_stream, bitstring.options.lsb0, 1)
1720 output_stream.write("]")
1721 if trailing_bit_length != 0:
1722 output_stream.write(" + trailing_bits = " + str(self[-trailing_bit_length:]))
1723 output_stream.write("\n")
1724 stream.write(output_stream.getvalue())
1725 return
1727 def copy(self: TBits) -> TBits:
1728 """Return a copy of the bitstring."""
1729 # Note that if you want a new copy (different ID), use _copy instead.
1730 # The copy can return self as it's immutable.
1731 return self
1733 @classmethod
1734 def fromstring(cls: TBits, s: str, /) -> TBits:
1735 """Create a new bitstring from a formatted string."""
1736 x = super().__new__(cls)
1737 x._bitstore = common_helpers.str_to_bitstore(s)
1738 return x
1740 len = length = property(_getlength, doc="The length of the bitstring in bits. Read only.")