Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pypdf/_cmap.py: 11%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from binascii import Error as BinasciiError
2from binascii import unhexlify
3from collections.abc import Callable
4from functools import partial
5from hashlib import sha256
6from io import BytesIO
7from typing import Any, Union, cast
9from ._codecs import adobe_glyphs, charset_encoding
10from ._utils import logger_error, logger_warning
11from .errors import LimitReachedError
12from .generic import (
13 ArrayObject,
14 DecodedStreamObject,
15 DictionaryObject,
16 NullObject,
17 StreamObject,
18)
19from .generic._data_structures import _FontFileCharacterMap
21_predefined_cmap: dict[str, str] = {
22 "/Identity-H": "utf-16-be",
23 "/Identity-V": "utf-16-be",
24 "/GB-EUC-H": "gbk",
25 "/GB-EUC-V": "gbk",
26 "/GBpc-EUC-H": "gb2312",
27 "/GBpc-EUC-V": "gb2312",
28 "/GBK-EUC-H": "gbk",
29 "/GBK-EUC-V": "gbk",
30 "/GBK2K-H": "gb18030",
31 "/GBK2K-V": "gb18030",
32 "/ETen-B5-H": "cp950",
33 "/ETen-B5-V": "cp950",
34 "/ETenms-B5-H": "cp950",
35 "/ETenms-B5-V": "cp950",
36 "/UniCNS-UTF16-H": "utf-16-be",
37 "/UniCNS-UTF16-V": "utf-16-be",
38 "/UniGB-UTF16-H": "gb18030",
39 "/UniGB-UTF16-V": "gb18030",
40 # Japanese CMaps (PDF Reference 1.7, Appendix H)
41 "/90ms-RKSJ-H": "cp932", # Shift-JIS (JIS X 0208-1990), horizontal
42 "/90ms-RKSJ-V": "cp932", # Shift-JIS (JIS X 0208-1990), vertical
43 "/UniJIS-UTF16-H": "utf-16-be", # Unicode UTF-16BE -> JIS, horizontal
44 "/UniJIS-UTF16-V": "utf-16-be", # Unicode UTF-16BE -> JIS, vertical
45 # UCS2 in code
46}
49def get_encoding(
50 ft: DictionaryObject
51) -> tuple[Union[str, dict[int, str]], dict[Any, Any]]:
52 encoding = _parse_encoding(ft)
53 map_dict, int_entry = _parse_to_unicode(ft)
55 # Apply rule from PDF ref 1.7 §5.9.1, 1st bullet:
56 # if cmap not empty encoding should be discarded
57 # (here transformed into identity for those characters)
58 # If encoding is a string, it is expected to be an identity translation.
59 if isinstance(encoding, dict):
60 for x in int_entry:
61 if x <= 255:
62 encoding[x] = chr(x)
64 return encoding, map_dict
67def _parse_encoding(
68 ft: DictionaryObject
69) -> Union[str, dict[int, str]]:
70 encoding: Union[str, list[str], dict[int, str]] = []
71 # If ft["/Encoding"] exists, then use that for encoding. Otherwise, use StandardEncoding as a basis,
72 # and add what the embedded font file says, if present. See Table 114, PDF Reference 1.7 / 2.0
73 if "/Encoding" not in ft:
74 if "/BaseFont" in ft and cast(str, ft["/BaseFont"]) in charset_encoding:
75 # This will match Symbol and ZapfDingBats
76 return dict(
77 zip(range(256), charset_encoding[cast(str, ft["/BaseFont"])])
78 )
80 # Return StandardEncoding as fallback option. Note that a font's internal encoding can be used
81 # to overwrite this, which we do for Type1 fonts in _derive_character_map_from_(cff_)type1_font_file.
82 return dict(
83 zip(range(256), charset_encoding["/StandardEncoding"])
84 )
86 enc: Union[str, DictionaryObject, NullObject] = cast(
87 Union[str, DictionaryObject, NullObject], ft["/Encoding"].get_object()
88 )
89 if isinstance(enc, str):
90 try:
91 # already done : enc = NameObject.unnumber(enc.encode()).decode()
92 # for #xx decoding
93 if enc in charset_encoding:
94 encoding = charset_encoding[enc].copy()
95 elif enc in _predefined_cmap:
96 encoding = _predefined_cmap[enc]
97 elif "-UCS2-" in enc:
98 encoding = "utf-16-be"
99 else:
100 raise Exception("not found")
101 except Exception:
102 encoding = (
103 "utf-16-be" if ft.get("/Subtype", "") == "/Type0" else charset_encoding["/StandardEncoding"].copy()
104 )
105 logger_error(
106 "Advanced encoding %(encoding)s not implemented yet, using %(new_encoding)s instead.",
107 source=__name__,
108 encoding=enc,
109 new_encoding=encoding
110 )
111 elif isinstance(enc, DictionaryObject) and "/BaseEncoding" in enc:
112 try:
113 encoding = charset_encoding[cast(str, enc["/BaseEncoding"])].copy()
114 except Exception:
115 logger_error(
116 "Advanced encoding %(encoding)s not implemented yet",
117 source=__name__, encoding=encoding
118 )
119 encoding = charset_encoding["/StandardEncoding"].copy()
120 else:
121 encoding = charset_encoding["/StandardEncoding"].copy()
122 if isinstance(enc, DictionaryObject) and "/Differences" in enc:
123 x: int = 0
124 o: Union[int, str]
125 differences = enc["/Differences"].get_object()
126 if not isinstance(differences, ArrayObject):
127 logger_warning(
128 "Font encoding differences are not an array: %(differences)s",
129 source=__name__,
130 differences=differences,
131 )
132 differences = ArrayObject()
133 for o in differences:
134 if isinstance(o, int):
135 x = o
136 else: # isinstance(o, str):
137 try:
138 if x < len(encoding):
139 encoding[x] = adobe_glyphs[o] # type: ignore[index]
140 except Exception:
141 encoding[x] = o # type: ignore[index]
142 x += 1
143 if isinstance(encoding, list):
144 encoding = dict(zip(range(256), encoding))
145 return encoding
148def _parse_to_unicode(
149 ft: DictionaryObject
150) -> tuple[dict[Any, Any], list[int]]:
151 from .generic._font import HAS_FONTTOOLS # noqa: PLC0415
153 # We store all character mappings in map_dict. In map_dict[-1] we store the byte length
154 # of the character codes (or CIDs) encoded inside the ToUnicode stream.
155 map_dict: dict[Any, Any] = {}
157 # We provide the list of cmap keys in int_entry to correct encoding later on in get_encoding().
158 int_entry: list[int] = []
160 if "/ToUnicode" not in ft:
161 if ft.get("/Subtype", "") == "/Type1":
162 font_descriptor = ft.get("/FontDescriptor")
163 if not font_descriptor:
164 return map_dict, int_entry
166 # We try to read encoding from an embedded font file, if we can. See Table 126 about embedded font
167 # file organization in the PDF specification 1.7 for details.
168 font_file_handlers = (
169 # A normal Type1 font file, can be part of a Type1 or MMType1 font dictionary.
170 (
171 "/FontFile",
172 lambda _: True,
173 _derive_character_map_from_type1_font_file
174 ),
175 # A CFF Type1 font file, as part of a Type1 or MMType1 font dictionary, when subtype is Type1C.
176 (
177 "/FontFile3",
178 lambda stream: stream.get("/Subtype") == "/Type1C",
179 _derive_character_map_from_cff_type1_font_file,
180 )
181 )
182 for font_file, condition, font_file_processor in font_file_handlers:
183 if (
184 font_file in font_descriptor and
185 isinstance(font_file_dict := font_descriptor[font_file], StreamObject) and
186 condition(font_file_dict)
187 ):
188 if font_file == "/FontFile3" and not HAS_FONTTOOLS:
189 logger_warning(
190 (
191 "fontTools is required to fully parse the encoding of a CFF Type1 font in font "
192 "dictionary %(ft)s, but is not installed. Consider installing fontTools if you "
193 "encounter encoding problems."
194 ),
195 source=__name__,
196 ft=ft,
197 )
198 return map_dict, int_entry
200 font_file_data = font_file_dict.get_data()
201 if not font_file_data:
202 return map_dict, int_entry
204 return _derive_character_map_from_font_file(
205 font_file_dict, font_file_data, font_file_processor, map_dict, int_entry
206 )
208 return map_dict, int_entry
210 return {}, []
212 process_rg: bool = False
213 process_char: bool = False
214 multiline_rg: Union[
215 tuple[int, int], None
216 ] = None # tuple = (current_char, remaining size) ; cf #1285 for example of file
217 cm = prepare_cm(ft)
218 for line in cm.split(b"\n"):
219 process_rg, process_char, multiline_rg = process_cm_line(
220 line.strip(b" \t"),
221 process_rg,
222 process_char,
223 multiline_rg,
224 map_dict,
225 int_entry,
226 )
228 map_dict.pop(-1, None) # Don't pass the -1 key, we only used it to temporarily store encoding length
230 return map_dict, int_entry
233def prepare_cm(ft: DictionaryObject) -> bytes:
234 tu = ft["/ToUnicode"]
235 cm: bytes
236 if isinstance(tu, StreamObject):
237 cm = cast(DecodedStreamObject, ft["/ToUnicode"]).get_data()
238 else: # if (tu is None) or cast(str, tu).startswith("/Identity"):
239 # the full range 0000-FFFF will be processed
240 cm = b"beginbfrange\n<0000> <0001> <0000>\nendbfrange"
241 if isinstance(cm, str):
242 cm = cm.encode()
243 # we need to prepare cm before due to missing return line in pdf printed
244 # to pdf from word
245 cm = (
246 cm.strip()
247 .replace(b"beginbfchar", b"\nbeginbfchar\n")
248 .replace(b"endbfchar", b"\nendbfchar\n")
249 .replace(b"beginbfrange", b"\nbeginbfrange\n")
250 .replace(b"endbfrange", b"\nendbfrange\n")
251 .replace(b"<<", b"\n{\n") # text between << and >> not used but
252 .replace(b">>", b"\n}\n") # some solution to find it back
253 )
254 ll = cm.split(b"<")
255 for i in range(len(ll)):
256 j = ll[i].find(b">")
257 if j >= 0:
258 if j == 0:
259 # string is empty: stash a placeholder here (see below)
260 # see https://github.com/py-pdf/pypdf/issues/1111
261 content = b"."
262 else:
263 content = ll[i][:j].replace(b" ", b"")
264 ll[i] = content + b" " + ll[i][j + 1 :]
265 cm = (
266 (b" ".join(ll))
267 .replace(b"[", b" [ ")
268 .replace(b"]", b" ]\n ")
269 .replace(b"\r", b"\n")
270 )
271 return cm
274def process_cm_line(
275 line: bytes,
276 process_rg: bool,
277 process_char: bool,
278 multiline_rg: Union[tuple[int, int], None],
279 map_dict: dict[Any, Any],
280 int_entry: list[int],
281) -> tuple[bool, bool, Union[tuple[int, int], None]]:
282 if line == b"" or line[0] == 37: # 37 = %
283 return process_rg, process_char, multiline_rg
284 line = line.replace(b"\t", b" ")
285 if b"beginbfrange" in line:
286 process_rg = True
287 elif b"endbfrange" in line:
288 process_rg = False
289 elif b"beginbfchar" in line:
290 process_char = True
291 elif b"endbfchar" in line:
292 process_char = False
293 elif process_rg:
294 try:
295 multiline_rg = parse_bfrange(line, map_dict, int_entry, multiline_rg)
296 except (ValueError, IndexError) as error:
297 logger_warning("Skipping broken line %(line)r: %(error)s", source=__name__, line=line, error=error)
298 elif process_char:
299 try:
300 parse_bfchar(line, map_dict, int_entry)
301 except ValueError as error:
302 logger_warning("Skipping broken line %(line)r: %(error)s", source=__name__, line=line, error=error)
303 return process_rg, process_char, multiline_rg
306# Usual values should be up to 65_536.
307MAPPING_DICTIONARY_SIZE_LIMIT = 100_000
309# Typical /ToUnicode CMaps use 1-4 byte source codes.
310# This is intentionally generous.
311# The actual limit is doubled, as each byte is represented by two hex characters.
312MAX_CMAP_CODE_BYTES = 8
313MAX_CMAP_STRING_BYTES = 512
314MAX_CMAP_CODE_BYTES_LIMIT = MAX_CMAP_CODE_BYTES * 2
315MAX_CMAP_STRING_BYTES_LIMIT = MAX_CMAP_STRING_BYTES * 2
318def _check_mapping_size(size: int) -> None:
319 if size > MAPPING_DICTIONARY_SIZE_LIMIT:
320 raise LimitReachedError(f"Maximum /ToUnicode size limit reached: {size} > {MAPPING_DICTIONARY_SIZE_LIMIT}.")
323def _check_token_length(token: bytes, limit: int) -> None:
324 token_length = len(token)
325 if token_length > limit:
326 description = {
327 MAX_CMAP_CODE_BYTES_LIMIT: "code",
328 MAX_CMAP_STRING_BYTES_LIMIT: "string",
329 }.get(limit, "token")
331 raise LimitReachedError(
332 f"Maximum /ToUnicode {description} length exceeded: {token_length} > {limit}."
333 )
336def __parse_bfrange__decode(map_dict: dict[Any, Any], code: int) -> str:
337 # `map_dict[-1]` is the number of bytes each source code occupies. Building
338 # the bytes directly with `int.to_bytes` avoids the hex round-trip of
339 # `unhexlify(b"%%0%dX" % (map_dict[-1] * 2) % code)` (format to hex, parse
340 # the hex back to bytes), which is measurably cheaper for large maps.
341 return code.to_bytes(map_dict[-1], "big").decode(
342 "charmap" if map_dict[-1] == 1 else "utf-16-be",
343 "surrogatepass",
344 )
347def parse_bfrange(
348 line: bytes,
349 map_dict: dict[Any, Any],
350 int_entry: list[int],
351 multiline_rg: Union[tuple[int, int], None],
352) -> Union[tuple[int, int], None]:
353 lst = line.split()
354 closure_found = False
355 entry_count = len(int_entry)
356 _check_mapping_size(entry_count)
357 decode_utf16 = partial(bytes.decode, encoding="utf-16-be", errors="surrogatepass")
358 if multiline_rg is not None:
359 a = multiline_rg[0] # a, b not in the current line
360 b = multiline_rg[1]
361 for sq in lst:
362 if sq == b"]":
363 closure_found = True
364 break
365 _check_token_length(sq, limit=MAX_CMAP_STRING_BYTES_LIMIT)
366 entry_count += 1
367 _check_mapping_size(entry_count)
368 map_dict[
369 __parse_bfrange__decode(map_dict=map_dict, code=a)
370 ] = decode_utf16(unhexlify(sq))
371 int_entry.append(a)
372 a += 1
373 else:
374 _check_token_length(lst[0], limit=MAX_CMAP_CODE_BYTES_LIMIT)
375 _check_token_length(lst[1], limit=MAX_CMAP_CODE_BYTES_LIMIT)
376 a = int(lst[0], 16)
377 b = int(lst[1], 16)
378 nbi = max(len(lst[0]), len(lst[1]))
379 map_dict[-1] = (nbi + 1) // 2
380 if lst[2] == b"[":
381 for sq in lst[3:]:
382 if sq == b"]":
383 closure_found = True
384 break
385 _check_token_length(sq, limit=MAX_CMAP_STRING_BYTES_LIMIT)
386 entry_count += 1
387 _check_mapping_size(entry_count)
388 map_dict[
389 __parse_bfrange__decode(map_dict=map_dict, code=a)
390 ] = decode_utf16(unhexlify(sq))
391 int_entry.append(a)
392 a += 1
393 else: # case without list
394 _check_token_length(lst[2], limit=MAX_CMAP_STRING_BYTES_LIMIT)
395 c = int(lst[2], 16)
396 fmt2 = b"%%0%dX" % max(4, len(lst[2]))
397 closure_found = True
398 range_size = max(0, b - a + 1)
399 _check_mapping_size(entry_count + range_size) # This can be checked beforehand.
400 while a <= b:
401 destination = unhexlify(fmt2 % c)
402 _check_token_length(destination, limit=MAX_CMAP_CODE_BYTES_LIMIT)
403 map_dict[
404 __parse_bfrange__decode(map_dict=map_dict, code=a)
405 ] = decode_utf16(destination)
406 int_entry.append(a)
407 a += 1
408 c += 1
409 return None if closure_found else (a, b)
412def parse_bfchar(line: bytes, map_dict: dict[Any, Any], int_entry: list[int]) -> None:
413 lst = [x for x in line.split(b" ") if x]
414 lst_length = len(lst)
415 if lst_length == 0:
416 logger_warning("Skipping broken line %(line)r: Line is empty.", source=__name__, line=line)
417 return
418 if lst_length % 2:
419 logger_warning("Ignoring final token of odd-length line %(line)r.", source=__name__, line=line)
421 new_count = lst_length // 2
422 _check_mapping_size(len(int_entry) + new_count) # This can be checked beforehand.
423 map_dict[-1] = len(lst[0]) // 2
425 while len(lst) > 1:
426 source = lst[0]
427 destination = lst[1]
429 _check_token_length(source, limit=MAX_CMAP_CODE_BYTES_LIMIT)
431 map_to = ""
432 # placeholder (see above) means empty string
433 if destination != b".":
434 _check_token_length(destination, limit=MAX_CMAP_STRING_BYTES_LIMIT)
435 try:
436 map_to = unhexlify(destination).decode(
437 "charmap" if len(destination) < 4 else "utf-16-be", "surrogatepass"
438 ) # join is here as some cases where the code was split
439 except BinasciiError as exception:
440 logger_warning(
441 "Got invalid hex string: %(exception)s (%(destination)r)",
442 source=__name__,
443 exception=exception,
444 destination=destination,
445 )
446 map_dict[
447 unhexlify(source).decode(
448 "charmap" if map_dict[-1] == 1 else "utf-16-be", "surrogatepass"
449 )
450 ] = map_to
451 int_entry.append(int(source, 16))
452 lst = lst[2:]
455def _glyph_name_to_unicode(glyph_name: str) -> Union[str, None]:
456 try:
457 return adobe_glyphs[glyph_name]
458 except KeyError:
459 if not glyph_name.startswith("/uni"):
460 return None
461 try:
462 return chr(int(glyph_name[4:], 16))
463 except ValueError: # pragma: no cover
464 return None
467def _derive_character_map_from_font_file(
468 font_file: StreamObject,
469 font_data: bytes,
470 font_file_processor: Callable[[bytes, dict[Any, Any], list[int]], tuple[dict[Any, Any], list[int]]],
471 map_dict: dict[Any, Any],
472 int_entry: list[int],
473) -> tuple[dict[Any, Any], list[int]]:
474 """
475 Derive the character map from an embedded font program, caching the result on its stream.
477 The character map depends only on the decoded font data, and the same font program is reached from every
478 font dictionary, resource name and page referencing its stream. Parsing a CFF font program with fontTools is
479 expensive, so each stream is parsed once and reused until its data changes, which a digest of the data
480 detects (a writer may replace the font program). See #4156.
481 """
482 digest = sha256(font_data).digest()
483 cached = font_file._font_file_character_map
484 if cached is None or cached.digest != digest:
485 cached_map_dict, cached_int_entry = font_file_processor(font_data, {}, [])
486 cached = _FontFileCharacterMap(digest, cached_map_dict, cached_int_entry)
487 font_file._font_file_character_map = cached
488 map_dict.update(cached.map_dict)
489 int_entry.extend(cached.int_entry)
490 return map_dict, int_entry
493def _derive_character_map_from_cff_type1_font_file(
494 font_data: bytes,
495 map_dict: dict[Any, Any],
496 int_entry: list[int],
497) -> tuple[dict[Any, Any], list[int]]:
498 try:
499 from fontTools.cffLib import CFFFontSet # noqa: PLC0415
500 cff_set = CFFFontSet()
501 cff_set.decompile(BytesIO(font_data), None)
502 cff_font = cff_set.topDictIndex[0]
503 cff_encoding = cff_font.Encoding
504 # Encoding can fall back to literal strings "StandardEncoding" or "ExpertEncoding", which we do not parse.
505 if isinstance(cff_encoding, str):
506 return map_dict, int_entry
507 for i in range(min(len(cff_encoding), 256)):
508 glyph_name = cff_encoding[i]
509 if not glyph_name or glyph_name == ".notdef":
510 continue
511 if unipoint := _glyph_name_to_unicode(f"/{glyph_name}"):
512 map_dict[chr(i)] = unipoint
513 int_entry.append(i)
514 return map_dict, int_entry
516 except Exception:
517 return map_dict, int_entry
520def _derive_character_map_from_type1_font_file(
521 font_data: bytes,
522 map_dict: dict[Any, Any],
523 int_entry: list[int],
524) -> tuple[dict[Any, Any], list[int]]:
525 txt = font_data.split(b"eexec\n")[0] # Only the clear part
526 encoding_part = txt.split(b"/Encoding")
527 if len(encoding_part) < 2:
528 return map_dict, int_entry
529 txt = encoding_part[1] # To get the encoding part
530 lines = txt.replace(b"\r", b"\n").split(b"\n")
531 for li in lines:
532 if li.startswith(b"dup"):
533 words = [_w for _w in li.split(b" ") if _w != b""]
534 if len(words) < 3 or (len(words) > 3 and words[3] != b"put"):
535 continue
536 try:
537 i = int(words[1])
538 except ValueError:
539 continue
541 unipoint = _glyph_name_to_unicode(words[2].decode())
542 if unipoint:
543 map_dict[chr(i)] = unipoint
544 int_entry.append(i)
546 return map_dict, int_entry