Coverage for /pythoncovmergedfiles/medio/medio/src/pdfminer.six/pdfminer/utils.py: 75%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""Miscellaneous Routines."""
3import io
4import pathlib
5import string
6import struct
7from html import escape
8from typing import (
9 TYPE_CHECKING,
10 Any,
11 BinaryIO,
12 Callable,
13 Dict,
14 Generic,
15 Iterable,
16 Iterator,
17 List,
18 Optional,
19 Set,
20 TextIO,
21 Tuple,
22 TypeVar,
23 Union,
24 cast,
25)
27from pdfminer.pdfexceptions import PDFTypeError, PDFValueError
29if TYPE_CHECKING:
30 from pdfminer.layout import LTComponent
32import charset_normalizer # For str encoding detection
34# from sys import maxint as INF doesn't work anymore under Python3, but PDF
35# still uses 32 bits ints
36INF = (1 << 31) - 1
39FileOrName = Union[pathlib.PurePath, str, io.IOBase]
40AnyIO = Union[TextIO, BinaryIO]
43class open_filename:
44 """Context manager that allows opening a filename
45 (str or pathlib.PurePath type is supported) and closes it on exit,
46 (just like `open`), but does nothing for file-like objects.
47 """
49 def __init__(self, filename: FileOrName, *args: Any, **kwargs: Any) -> None:
50 if isinstance(filename, pathlib.PurePath):
51 filename = str(filename)
52 if isinstance(filename, str):
53 self.file_handler: AnyIO = open(filename, *args, **kwargs)
54 self.closing = True
55 elif isinstance(filename, io.IOBase):
56 self.file_handler = cast(AnyIO, filename)
57 self.closing = False
58 else:
59 raise PDFTypeError("Unsupported input type: %s" % type(filename))
61 def __enter__(self) -> AnyIO:
62 return self.file_handler
64 def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None:
65 if self.closing:
66 self.file_handler.close()
69def make_compat_bytes(in_str: str) -> bytes:
70 """Converts to bytes, encoding to unicode."""
71 assert isinstance(in_str, str), str(type(in_str))
72 return in_str.encode()
75def make_compat_str(o: object) -> str:
76 """Converts everything to string, if bytes guessing the encoding."""
77 if isinstance(o, bytes):
78 enc = charset_normalizer.detect(o)
79 try:
80 return o.decode(enc["encoding"])
81 except UnicodeDecodeError:
82 return str(o)
83 else:
84 return str(o)
87def shorten_str(s: str, size: int) -> str:
88 if size < 7:
89 return s[:size]
90 if len(s) > size:
91 length = (size - 5) // 2
92 return f"{s[:length]} ... {s[-length:]}"
93 else:
94 return s
97def compatible_encode_method(
98 bytesorstring: Union[bytes, str],
99 encoding: str = "utf-8",
100 erraction: str = "ignore",
101) -> str:
102 """When Py2 str.encode is called, it often means bytes.encode in Py3.
104 This does either.
105 """
106 if isinstance(bytesorstring, str):
107 return bytesorstring
108 assert isinstance(bytesorstring, bytes), str(type(bytesorstring))
109 return bytesorstring.decode(encoding, erraction)
112def paeth_predictor(left: int, above: int, upper_left: int) -> int:
113 # From http://www.libpng.org/pub/png/spec/1.2/PNG-Filters.html
114 # Initial estimate
115 p = left + above - upper_left
116 # Distances to a,b,c
117 pa = abs(p - left)
118 pb = abs(p - above)
119 pc = abs(p - upper_left)
121 # Return nearest of a,b,c breaking ties in order a,b,c
122 if pa <= pb and pa <= pc:
123 return left
124 elif pb <= pc:
125 return above
126 else:
127 return upper_left
130def apply_png_predictor(
131 pred: int,
132 colors: int,
133 columns: int,
134 bitspercomponent: int,
135 data: bytes,
136) -> bytes:
137 """Reverse the effect of the PNG predictor
139 Documentation: http://www.libpng.org/pub/png/spec/1.2/PNG-Filters.html
140 """
141 if bitspercomponent not in [8, 1]:
142 msg = "Unsupported `bitspercomponent': %d" % bitspercomponent
143 raise PDFValueError(msg)
145 nbytes = colors * columns * bitspercomponent // 8
146 bpp = colors * bitspercomponent // 8 # number of bytes per complete pixel
147 buf = []
148 line_above = list(b"\x00" * columns)
149 for scanline_i in range(0, len(data), nbytes + 1):
150 filter_type = data[scanline_i]
151 line_encoded = data[scanline_i + 1 : scanline_i + 1 + nbytes]
152 raw = []
154 if filter_type == 0:
155 # Filter type 0: None
156 raw = list(line_encoded)
158 elif filter_type == 1:
159 # Filter type 1: Sub
160 # To reverse the effect of the Sub() filter after decompression,
161 # output the following value:
162 # Raw(x) = Sub(x) + Raw(x - bpp)
163 # (computed mod 256), where Raw() refers to the bytes already
164 # decoded.
165 for j, sub_x in enumerate(line_encoded):
166 if j - bpp < 0:
167 raw_x_bpp = 0
168 else:
169 raw_x_bpp = int(raw[j - bpp])
170 raw_x = (sub_x + raw_x_bpp) & 255
171 raw.append(raw_x)
173 elif filter_type == 2:
174 # Filter type 2: Up
175 # To reverse the effect of the Up() filter after decompression,
176 # output the following value:
177 # Raw(x) = Up(x) + Prior(x)
178 # (computed mod 256), where Prior() refers to the decoded bytes of
179 # the prior scanline.
180 for up_x, prior_x in zip(line_encoded, line_above):
181 raw_x = (up_x + prior_x) & 255
182 raw.append(raw_x)
184 elif filter_type == 3:
185 # Filter type 3: Average
186 # To reverse the effect of the Average() filter after
187 # decompression, output the following value:
188 # Raw(x) = Average(x) + floor((Raw(x-bpp)+Prior(x))/2)
189 # where the result is computed mod 256, but the prediction is
190 # calculated in the same way as for encoding. Raw() refers to the
191 # bytes already decoded, and Prior() refers to the decoded bytes of
192 # the prior scanline.
193 for j, average_x in enumerate(line_encoded):
194 if j - bpp < 0:
195 raw_x_bpp = 0
196 else:
197 raw_x_bpp = int(raw[j - bpp])
198 prior_x = int(line_above[j])
199 raw_x = (average_x + (raw_x_bpp + prior_x) // 2) & 255
200 raw.append(raw_x)
202 elif filter_type == 4:
203 # Filter type 4: Paeth
204 # To reverse the effect of the Paeth() filter after decompression,
205 # output the following value:
206 # Raw(x) = Paeth(x)
207 # + PaethPredictor(Raw(x-bpp), Prior(x), Prior(x-bpp))
208 # (computed mod 256), where Raw() and Prior() refer to bytes
209 # already decoded. Exactly the same PaethPredictor() function is
210 # used by both encoder and decoder.
211 for j, paeth_x in enumerate(line_encoded):
212 if j - bpp < 0:
213 raw_x_bpp = 0
214 prior_x_bpp = 0
215 else:
216 raw_x_bpp = int(raw[j - bpp])
217 prior_x_bpp = int(line_above[j - bpp])
218 prior_x = int(line_above[j])
219 paeth = paeth_predictor(raw_x_bpp, prior_x, prior_x_bpp)
220 raw_x = (paeth_x + paeth) & 255
221 raw.append(raw_x)
223 else:
224 raise PDFValueError("Unsupported predictor value: %d" % filter_type)
226 buf.extend(raw)
227 line_above = raw
228 return bytes(buf)
231Point = Tuple[float, float]
232Rect = Tuple[float, float, float, float]
233Matrix = Tuple[float, float, float, float, float, float]
234PathSegment = Union[
235 Tuple[str], # Literal['h']
236 Tuple[str, float, float], # Literal['m', 'l']
237 Tuple[str, float, float, float, float], # Literal['v', 'y']
238 Tuple[str, float, float, float, float, float, float],
239] # Literal['c']
241# Matrix operations
242MATRIX_IDENTITY: Matrix = (1, 0, 0, 1, 0, 0)
245def parse_rect(o: Any) -> Rect:
246 try:
247 (x0, y0, x1, y1) = o
248 return float(x0), float(y0), float(x1), float(y1)
249 except ValueError:
250 raise PDFValueError("Could not parse rectangle")
253def mult_matrix(m1: Matrix, m0: Matrix) -> Matrix:
254 (a1, b1, c1, d1, e1, f1) = m1
255 (a0, b0, c0, d0, e0, f0) = m0
256 """Returns the multiplication of two matrices."""
257 return (
258 a0 * a1 + c0 * b1,
259 b0 * a1 + d0 * b1,
260 a0 * c1 + c0 * d1,
261 b0 * c1 + d0 * d1,
262 a0 * e1 + c0 * f1 + e0,
263 b0 * e1 + d0 * f1 + f0,
264 )
267def translate_matrix(m: Matrix, v: Point) -> Matrix:
268 """Translates a matrix by (x, y)."""
269 (a, b, c, d, e, f) = m
270 (x, y) = v
271 return a, b, c, d, x * a + y * c + e, x * b + y * d + f
274def apply_matrix_pt(m: Matrix, v: Point) -> Point:
275 (a, b, c, d, e, f) = m
276 (x, y) = v
277 """Applies a matrix to a point."""
278 return a * x + c * y + e, b * x + d * y + f
281def apply_matrix_norm(m: Matrix, v: Point) -> Point:
282 """Equivalent to apply_matrix_pt(M, (p,q)) - apply_matrix_pt(M, (0,0))"""
283 (a, b, c, d, e, f) = m
284 (p, q) = v
285 return a * p + c * q, b * p + d * q
288# Utility functions
291def isnumber(x: object) -> bool:
292 return isinstance(x, (int, float))
295_T = TypeVar("_T")
298def uniq(objs: Iterable[_T]) -> Iterator[_T]:
299 """Eliminates duplicated elements."""
300 done = set()
301 for obj in objs:
302 if obj in done:
303 continue
304 done.add(obj)
305 yield obj
308def fsplit(pred: Callable[[_T], bool], objs: Iterable[_T]) -> Tuple[List[_T], List[_T]]:
309 """Split a list into two classes according to the predicate."""
310 t = []
311 f = []
312 for obj in objs:
313 if pred(obj):
314 t.append(obj)
315 else:
316 f.append(obj)
317 return t, f
320def drange(v0: float, v1: float, d: int) -> range:
321 """Returns a discrete range."""
322 return range(int(v0) // d, int(v1 + d) // d)
325def get_bound(pts: Iterable[Point]) -> Rect:
326 """Compute a minimal rectangle that covers all the points."""
327 limit: Rect = (INF, INF, -INF, -INF)
328 (x0, y0, x1, y1) = limit
329 for x, y in pts:
330 x0 = min(x0, x)
331 y0 = min(y0, y)
332 x1 = max(x1, x)
333 y1 = max(y1, y)
334 return x0, y0, x1, y1
337def pick(
338 seq: Iterable[_T],
339 func: Callable[[_T], float],
340 maxobj: Optional[_T] = None,
341) -> Optional[_T]:
342 """Picks the object obj where func(obj) has the highest value."""
343 maxscore = None
344 for obj in seq:
345 score = func(obj)
346 if maxscore is None or maxscore < score:
347 (maxscore, maxobj) = (score, obj)
348 return maxobj
351def choplist(n: int, seq: Iterable[_T]) -> Iterator[Tuple[_T, ...]]:
352 """Groups every n elements of the list."""
353 r = []
354 for x in seq:
355 r.append(x)
356 if len(r) == n:
357 yield tuple(r)
358 r = []
361def nunpack(s: bytes, default: int = 0) -> int:
362 """Unpacks 1 to 4 or 8 byte integers (big endian)."""
363 length = len(s)
364 if not length:
365 return default
366 elif length == 1:
367 return ord(s)
368 elif length == 2:
369 return cast(int, struct.unpack(">H", s)[0])
370 elif length == 3:
371 return cast(int, struct.unpack(">L", b"\x00" + s)[0])
372 elif length == 4:
373 return cast(int, struct.unpack(">L", s)[0])
374 elif length == 8:
375 return cast(int, struct.unpack(">Q", s)[0])
376 else:
377 raise PDFTypeError("invalid length: %d" % length)
380PDFDocEncoding = "".join(
381 chr(x)
382 for x in (
383 0x0000,
384 0x0001,
385 0x0002,
386 0x0003,
387 0x0004,
388 0x0005,
389 0x0006,
390 0x0007,
391 0x0008,
392 0x0009,
393 0x000A,
394 0x000B,
395 0x000C,
396 0x000D,
397 0x000E,
398 0x000F,
399 0x0010,
400 0x0011,
401 0x0012,
402 0x0013,
403 0x0014,
404 0x0015,
405 0x0017,
406 0x0017,
407 0x02D8,
408 0x02C7,
409 0x02C6,
410 0x02D9,
411 0x02DD,
412 0x02DB,
413 0x02DA,
414 0x02DC,
415 0x0020,
416 0x0021,
417 0x0022,
418 0x0023,
419 0x0024,
420 0x0025,
421 0x0026,
422 0x0027,
423 0x0028,
424 0x0029,
425 0x002A,
426 0x002B,
427 0x002C,
428 0x002D,
429 0x002E,
430 0x002F,
431 0x0030,
432 0x0031,
433 0x0032,
434 0x0033,
435 0x0034,
436 0x0035,
437 0x0036,
438 0x0037,
439 0x0038,
440 0x0039,
441 0x003A,
442 0x003B,
443 0x003C,
444 0x003D,
445 0x003E,
446 0x003F,
447 0x0040,
448 0x0041,
449 0x0042,
450 0x0043,
451 0x0044,
452 0x0045,
453 0x0046,
454 0x0047,
455 0x0048,
456 0x0049,
457 0x004A,
458 0x004B,
459 0x004C,
460 0x004D,
461 0x004E,
462 0x004F,
463 0x0050,
464 0x0051,
465 0x0052,
466 0x0053,
467 0x0054,
468 0x0055,
469 0x0056,
470 0x0057,
471 0x0058,
472 0x0059,
473 0x005A,
474 0x005B,
475 0x005C,
476 0x005D,
477 0x005E,
478 0x005F,
479 0x0060,
480 0x0061,
481 0x0062,
482 0x0063,
483 0x0064,
484 0x0065,
485 0x0066,
486 0x0067,
487 0x0068,
488 0x0069,
489 0x006A,
490 0x006B,
491 0x006C,
492 0x006D,
493 0x006E,
494 0x006F,
495 0x0070,
496 0x0071,
497 0x0072,
498 0x0073,
499 0x0074,
500 0x0075,
501 0x0076,
502 0x0077,
503 0x0078,
504 0x0079,
505 0x007A,
506 0x007B,
507 0x007C,
508 0x007D,
509 0x007E,
510 0x0000,
511 0x2022,
512 0x2020,
513 0x2021,
514 0x2026,
515 0x2014,
516 0x2013,
517 0x0192,
518 0x2044,
519 0x2039,
520 0x203A,
521 0x2212,
522 0x2030,
523 0x201E,
524 0x201C,
525 0x201D,
526 0x2018,
527 0x2019,
528 0x201A,
529 0x2122,
530 0xFB01,
531 0xFB02,
532 0x0141,
533 0x0152,
534 0x0160,
535 0x0178,
536 0x017D,
537 0x0131,
538 0x0142,
539 0x0153,
540 0x0161,
541 0x017E,
542 0x0000,
543 0x20AC,
544 0x00A1,
545 0x00A2,
546 0x00A3,
547 0x00A4,
548 0x00A5,
549 0x00A6,
550 0x00A7,
551 0x00A8,
552 0x00A9,
553 0x00AA,
554 0x00AB,
555 0x00AC,
556 0x0000,
557 0x00AE,
558 0x00AF,
559 0x00B0,
560 0x00B1,
561 0x00B2,
562 0x00B3,
563 0x00B4,
564 0x00B5,
565 0x00B6,
566 0x00B7,
567 0x00B8,
568 0x00B9,
569 0x00BA,
570 0x00BB,
571 0x00BC,
572 0x00BD,
573 0x00BE,
574 0x00BF,
575 0x00C0,
576 0x00C1,
577 0x00C2,
578 0x00C3,
579 0x00C4,
580 0x00C5,
581 0x00C6,
582 0x00C7,
583 0x00C8,
584 0x00C9,
585 0x00CA,
586 0x00CB,
587 0x00CC,
588 0x00CD,
589 0x00CE,
590 0x00CF,
591 0x00D0,
592 0x00D1,
593 0x00D2,
594 0x00D3,
595 0x00D4,
596 0x00D5,
597 0x00D6,
598 0x00D7,
599 0x00D8,
600 0x00D9,
601 0x00DA,
602 0x00DB,
603 0x00DC,
604 0x00DD,
605 0x00DE,
606 0x00DF,
607 0x00E0,
608 0x00E1,
609 0x00E2,
610 0x00E3,
611 0x00E4,
612 0x00E5,
613 0x00E6,
614 0x00E7,
615 0x00E8,
616 0x00E9,
617 0x00EA,
618 0x00EB,
619 0x00EC,
620 0x00ED,
621 0x00EE,
622 0x00EF,
623 0x00F0,
624 0x00F1,
625 0x00F2,
626 0x00F3,
627 0x00F4,
628 0x00F5,
629 0x00F6,
630 0x00F7,
631 0x00F8,
632 0x00F9,
633 0x00FA,
634 0x00FB,
635 0x00FC,
636 0x00FD,
637 0x00FE,
638 0x00FF,
639 )
640)
643def decode_text(s: bytes) -> str:
644 """Decodes a PDFDocEncoding string to Unicode."""
645 if s.startswith(b"\xfe\xff"):
646 return str(s[2:], "utf-16be", "ignore")
647 else:
648 return "".join(PDFDocEncoding[c] for c in s)
651def enc(x: str) -> str:
652 """Encodes a string for SGML/XML/HTML"""
653 if isinstance(x, bytes):
654 return ""
655 return escape(x)
658def bbox2str(bbox: Rect) -> str:
659 (x0, y0, x1, y1) = bbox
660 return f"{x0:.3f},{y0:.3f},{x1:.3f},{y1:.3f}"
663def matrix2str(m: Matrix) -> str:
664 (a, b, c, d, e, f) = m
665 return f"[{a:.2f},{b:.2f},{c:.2f},{d:.2f}, ({e:.2f},{f:.2f})]"
668def vecBetweenBoxes(obj1: "LTComponent", obj2: "LTComponent") -> Point:
669 """A distance function between two TextBoxes.
671 Consider the bounding rectangle for obj1 and obj2.
672 Return vector between 2 boxes boundaries if they don't overlap, otherwise
673 returns vector betweeen boxes centers
675 +------+..........+ (x1, y1)
676 | obj1 | :
677 +------+www+------+
678 : | obj2 |
679 (x0, y0) +..........+------+
680 """
681 (x0, y0) = (min(obj1.x0, obj2.x0), min(obj1.y0, obj2.y0))
682 (x1, y1) = (max(obj1.x1, obj2.x1), max(obj1.y1, obj2.y1))
683 (ow, oh) = (x1 - x0, y1 - y0)
684 (iw, ih) = (ow - obj1.width - obj2.width, oh - obj1.height - obj2.height)
685 if iw < 0 and ih < 0:
686 # if one is inside another we compute euclidean distance
687 (xc1, yc1) = ((obj1.x0 + obj1.x1) / 2, (obj1.y0 + obj1.y1) / 2)
688 (xc2, yc2) = ((obj2.x0 + obj2.x1) / 2, (obj2.y0 + obj2.y1) / 2)
689 return xc1 - xc2, yc1 - yc2
690 else:
691 return max(0, iw), max(0, ih)
694LTComponentT = TypeVar("LTComponentT", bound="LTComponent")
697class Plane(Generic[LTComponentT]):
698 """A set-like data structure for objects placed on a plane.
700 Can efficiently find objects in a certain rectangular area.
701 It maintains two parallel lists of objects, each of
702 which is sorted by its x or y coordinate.
703 """
705 def __init__(self, bbox: Rect, gridsize: int = 50) -> None:
706 self._seq: List[LTComponentT] = [] # preserve the object order.
707 self._objs: Set[LTComponentT] = set()
708 self._grid: Dict[Point, List[LTComponentT]] = {}
709 self.gridsize = gridsize
710 (self.x0, self.y0, self.x1, self.y1) = bbox
712 def __repr__(self) -> str:
713 return "<Plane objs=%r>" % list(self)
715 def __iter__(self) -> Iterator[LTComponentT]:
716 return (obj for obj in self._seq if obj in self._objs)
718 def __len__(self) -> int:
719 return len(self._objs)
721 def __contains__(self, obj: object) -> bool:
722 return obj in self._objs
724 def _getrange(self, bbox: Rect) -> Iterator[Point]:
725 (x0, y0, x1, y1) = bbox
726 if x1 <= self.x0 or self.x1 <= x0 or y1 <= self.y0 or self.y1 <= y0:
727 return
728 x0 = max(self.x0, x0)
729 y0 = max(self.y0, y0)
730 x1 = min(self.x1, x1)
731 y1 = min(self.y1, y1)
732 for grid_y in drange(y0, y1, self.gridsize):
733 for grid_x in drange(x0, x1, self.gridsize):
734 yield (grid_x, grid_y)
736 def extend(self, objs: Iterable[LTComponentT]) -> None:
737 for obj in objs:
738 self.add(obj)
740 def add(self, obj: LTComponentT) -> None:
741 """Place an object."""
742 for k in self._getrange((obj.x0, obj.y0, obj.x1, obj.y1)):
743 if k not in self._grid:
744 r: List[LTComponentT] = []
745 self._grid[k] = r
746 else:
747 r = self._grid[k]
748 r.append(obj)
749 self._seq.append(obj)
750 self._objs.add(obj)
752 def remove(self, obj: LTComponentT) -> None:
753 """Displace an object."""
754 for k in self._getrange((obj.x0, obj.y0, obj.x1, obj.y1)):
755 try:
756 self._grid[k].remove(obj)
757 except (KeyError, ValueError):
758 pass
759 self._objs.remove(obj)
761 def find(self, bbox: Rect) -> Iterator[LTComponentT]:
762 """Finds objects that are in a certain area."""
763 (x0, y0, x1, y1) = bbox
764 done = set()
765 for k in self._getrange(bbox):
766 if k not in self._grid:
767 continue
768 for obj in self._grid[k]:
769 if obj in done:
770 continue
771 done.add(obj)
772 if obj.x1 <= x0 or x1 <= obj.x0 or obj.y1 <= y0 or y1 <= obj.y0:
773 continue
774 yield obj
777ROMAN_ONES = ["i", "x", "c", "m"]
778ROMAN_FIVES = ["v", "l", "d"]
781def format_int_roman(value: int) -> str:
782 """Format a number as lowercase Roman numerals."""
783 assert 0 < value < 4000
784 result: List[str] = []
785 index = 0
787 while value != 0:
788 value, remainder = divmod(value, 10)
789 if remainder == 9:
790 result.insert(0, ROMAN_ONES[index])
791 result.insert(1, ROMAN_ONES[index + 1])
792 elif remainder == 4:
793 result.insert(0, ROMAN_ONES[index])
794 result.insert(1, ROMAN_FIVES[index])
795 else:
796 over_five = remainder >= 5
797 if over_five:
798 result.insert(0, ROMAN_FIVES[index])
799 remainder -= 5
800 result.insert(1 if over_five else 0, ROMAN_ONES[index] * remainder)
801 index += 1
803 return "".join(result)
806def format_int_alpha(value: int) -> str:
807 """Format a number as lowercase letters a-z, aa-zz, etc."""
808 assert value > 0
809 result: List[str] = []
811 while value != 0:
812 value, remainder = divmod(value - 1, len(string.ascii_lowercase))
813 result.append(string.ascii_lowercase[remainder])
815 result.reverse()
816 return "".join(result)