Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pypdf/_cmap.py: 11%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from binascii import Error as BinasciiError
2from binascii import unhexlify
3from functools import partial
4from typing import Any, Union, cast
6from ._codecs import adobe_glyphs, charset_encoding
7from ._utils import logger_error, logger_warning
8from .errors import LimitReachedError
9from .generic import (
10 DecodedStreamObject,
11 DictionaryObject,
12 NullObject,
13 StreamObject,
14 is_null_or_none,
15)
17_predefined_cmap: dict[str, str] = {
18 "/Identity-H": "utf-16-be",
19 "/Identity-V": "utf-16-be",
20 "/GB-EUC-H": "gbk",
21 "/GB-EUC-V": "gbk",
22 "/GBpc-EUC-H": "gb2312",
23 "/GBpc-EUC-V": "gb2312",
24 "/GBK-EUC-H": "gbk",
25 "/GBK-EUC-V": "gbk",
26 "/GBK2K-H": "gb18030",
27 "/GBK2K-V": "gb18030",
28 "/ETen-B5-H": "cp950",
29 "/ETen-B5-V": "cp950",
30 "/ETenms-B5-H": "cp950",
31 "/ETenms-B5-V": "cp950",
32 "/UniCNS-UTF16-H": "utf-16-be",
33 "/UniCNS-UTF16-V": "utf-16-be",
34 "/UniGB-UTF16-H": "gb18030",
35 "/UniGB-UTF16-V": "gb18030",
36 # Japanese CMaps (PDF Reference 1.7, Appendix H)
37 "/90ms-RKSJ-H": "cp932", # Shift-JIS (JIS X 0208-1990), horizontal
38 "/90ms-RKSJ-V": "cp932", # Shift-JIS (JIS X 0208-1990), vertical
39 "/UniJIS-UTF16-H": "utf-16-be", # Unicode UTF-16BE -> JIS, horizontal
40 "/UniJIS-UTF16-V": "utf-16-be", # Unicode UTF-16BE -> JIS, vertical
41 # UCS2 in code
42}
45def get_encoding(
46 ft: DictionaryObject
47) -> tuple[Union[str, dict[int, str]], dict[Any, Any]]:
48 encoding = _parse_encoding(ft)
49 map_dict, int_entry = _parse_to_unicode(ft)
51 # Apply rule from PDF ref 1.7 §5.9.1, 1st bullet:
52 # if cmap not empty encoding should be discarded
53 # (here transformed into identity for those characters)
54 # If encoding is a string, it is expected to be an identity translation.
55 if isinstance(encoding, dict):
56 for x in int_entry:
57 if x <= 255:
58 encoding[x] = chr(x)
60 return encoding, map_dict
63def _parse_encoding(
64 ft: DictionaryObject
65) -> Union[str, dict[int, str]]:
66 encoding: Union[str, list[str], dict[int, str]] = []
67 # If ft["/Encoding"] exists, then use that for encoding. Otherwise, use StandardEncoding as a basis,
68 # and add what the embedded font file says, if present. See Table 114, PDF Reference 1.7 / 2.0
69 if "/Encoding" not in ft:
70 if "/BaseFont" in ft and cast(str, ft["/BaseFont"]) in charset_encoding:
71 # This will match Symbol and ZapfDingBats
72 return dict(
73 zip(range(256), charset_encoding[cast(str, ft["/BaseFont"])])
74 )
76 # Return StandardEncoding as fallback option. Note that a font's internal encoding can be used
77 # to overwrite this, which we do for Type1 fonts in _type1_alternative.
78 return dict(
79 zip(range(256), charset_encoding["/StandardEncoding"])
80 )
82 enc: Union[str, DictionaryObject, NullObject] = cast(
83 Union[str, DictionaryObject, NullObject], ft["/Encoding"].get_object()
84 )
85 if isinstance(enc, str):
86 try:
87 # already done : enc = NameObject.unnumber(enc.encode()).decode()
88 # for #xx decoding
89 if enc in charset_encoding:
90 encoding = charset_encoding[enc].copy()
91 elif enc in _predefined_cmap:
92 encoding = _predefined_cmap[enc]
93 elif "-UCS2-" in enc:
94 encoding = "utf-16-be"
95 else:
96 raise Exception("not found")
97 except Exception:
98 logger_error("Advanced encoding %(encoding)s not implemented yet", source=__name__, encoding=enc)
99 encoding = enc
100 elif isinstance(enc, DictionaryObject) and "/BaseEncoding" in enc:
101 try:
102 encoding = charset_encoding[cast(str, enc["/BaseEncoding"])].copy()
103 except Exception:
104 logger_error(
105 "Advanced encoding %(encoding)s not implemented yet",
106 source=__name__, encoding=encoding
107 )
108 encoding = charset_encoding["/StandardEncoding"].copy()
109 else:
110 encoding = charset_encoding["/StandardEncoding"].copy()
111 if isinstance(enc, DictionaryObject) and "/Differences" in enc:
112 x: int = 0
113 o: Union[int, str]
114 for o in cast(DictionaryObject, enc["/Differences"]):
115 if isinstance(o, int):
116 x = o
117 else: # isinstance(o, str):
118 try:
119 if x < len(encoding):
120 encoding[x] = adobe_glyphs[o] # type: ignore[index]
121 except Exception:
122 encoding[x] = o # type: ignore[index]
123 x += 1
124 if isinstance(encoding, list):
125 encoding = dict(zip(range(256), encoding))
126 return encoding
129def _parse_to_unicode(
130 ft: DictionaryObject
131) -> tuple[dict[Any, Any], list[int]]:
132 # will store all translation code
133 # and map_dict[-1] we will have the number of bytes to convert
134 map_dict: dict[Any, Any] = {}
136 # will provide the list of cmap keys as int to correct encoding
137 int_entry: list[int] = []
139 if "/ToUnicode" not in ft:
140 if ft.get("/Subtype", "") == "/Type1":
141 return _type1_alternative(ft, map_dict, int_entry)
142 return {}, []
143 process_rg: bool = False
144 process_char: bool = False
145 multiline_rg: Union[
146 tuple[int, int], None
147 ] = None # tuple = (current_char, remaining size) ; cf #1285 for example of file
148 cm = prepare_cm(ft)
149 for line in cm.split(b"\n"):
150 process_rg, process_char, multiline_rg = process_cm_line(
151 line.strip(b" \t"),
152 process_rg,
153 process_char,
154 multiline_rg,
155 map_dict,
156 int_entry,
157 )
159 map_dict.pop(-1, None) # Don't pass the -1 key, we only used it to temporarily store encoding length
161 return map_dict, int_entry
164def prepare_cm(ft: DictionaryObject) -> bytes:
165 tu = ft["/ToUnicode"]
166 cm: bytes
167 if isinstance(tu, StreamObject):
168 cm = cast(DecodedStreamObject, ft["/ToUnicode"]).get_data()
169 else: # if (tu is None) or cast(str, tu).startswith("/Identity"):
170 # the full range 0000-FFFF will be processed
171 cm = b"beginbfrange\n<0000> <0001> <0000>\nendbfrange"
172 if isinstance(cm, str):
173 cm = cm.encode()
174 # we need to prepare cm before due to missing return line in pdf printed
175 # to pdf from word
176 cm = (
177 cm.strip()
178 .replace(b"beginbfchar", b"\nbeginbfchar\n")
179 .replace(b"endbfchar", b"\nendbfchar\n")
180 .replace(b"beginbfrange", b"\nbeginbfrange\n")
181 .replace(b"endbfrange", b"\nendbfrange\n")
182 .replace(b"<<", b"\n{\n") # text between << and >> not used but
183 .replace(b">>", b"\n}\n") # some solution to find it back
184 )
185 ll = cm.split(b"<")
186 for i in range(len(ll)):
187 j = ll[i].find(b">")
188 if j >= 0:
189 if j == 0:
190 # string is empty: stash a placeholder here (see below)
191 # see https://github.com/py-pdf/pypdf/issues/1111
192 content = b"."
193 else:
194 content = ll[i][:j].replace(b" ", b"")
195 ll[i] = content + b" " + ll[i][j + 1 :]
196 cm = (
197 (b" ".join(ll))
198 .replace(b"[", b" [ ")
199 .replace(b"]", b" ]\n ")
200 .replace(b"\r", b"\n")
201 )
202 return cm
205def process_cm_line(
206 line: bytes,
207 process_rg: bool,
208 process_char: bool,
209 multiline_rg: Union[tuple[int, int], None],
210 map_dict: dict[Any, Any],
211 int_entry: list[int],
212) -> tuple[bool, bool, Union[tuple[int, int], None]]:
213 if line == b"" or line[0] == 37: # 37 = %
214 return process_rg, process_char, multiline_rg
215 line = line.replace(b"\t", b" ")
216 if b"beginbfrange" in line:
217 process_rg = True
218 elif b"endbfrange" in line:
219 process_rg = False
220 elif b"beginbfchar" in line:
221 process_char = True
222 elif b"endbfchar" in line:
223 process_char = False
224 elif process_rg:
225 try:
226 multiline_rg = parse_bfrange(line, map_dict, int_entry, multiline_rg)
227 except (ValueError, IndexError) as error:
228 logger_warning("Skipping broken line %(line)r: %(error)s", source=__name__, line=line, error=error)
229 elif process_char:
230 try:
231 parse_bfchar(line, map_dict, int_entry)
232 except (ValueError, IndexError) as error:
233 logger_warning("Skipping broken line %(line)r: %(error)s", source=__name__, line=line, error=error)
234 return process_rg, process_char, multiline_rg
237# Usual values should be up to 65_536.
238MAPPING_DICTIONARY_SIZE_LIMIT = 100_000
240# Typical /ToUnicode CMaps use 1-4 byte source codes.
241# This is intentionally generous.
242# The actual limit is doubled, as each byte is represented by two hex characters.
243MAX_CMAP_CODE_BYTES = 8
244MAX_CMAP_STRING_BYTES = 512
245MAX_CMAP_CODE_BYTES_LIMIT = MAX_CMAP_CODE_BYTES * 2
246MAX_CMAP_STRING_BYTES_LIMIT = MAX_CMAP_STRING_BYTES * 2
249def _check_mapping_size(size: int) -> None:
250 if size > MAPPING_DICTIONARY_SIZE_LIMIT:
251 raise LimitReachedError(f"Maximum /ToUnicode size limit reached: {size} > {MAPPING_DICTIONARY_SIZE_LIMIT}.")
254def _check_token_length(token: bytes, limit: int) -> None:
255 token_length = len(token)
256 if token_length > limit:
257 description = {
258 MAX_CMAP_CODE_BYTES_LIMIT: "code",
259 MAX_CMAP_STRING_BYTES_LIMIT: "string",
260 }.get(limit, "token")
262 raise LimitReachedError(
263 f"Maximum /ToUnicode {description} length exceeded: {token_length} > {limit}."
264 )
267def __parse_bfrange__decode(map_dict: dict[Any, Any], code: int) -> str:
268 # `map_dict[-1]` is the number of bytes each source code occupies. Building
269 # the bytes directly with `int.to_bytes` avoids the hex round-trip of
270 # `unhexlify(b"%%0%dX" % (map_dict[-1] * 2) % code)` (format to hex, parse
271 # the hex back to bytes), which is measurably cheaper for large maps.
272 return code.to_bytes(map_dict[-1], "big").decode(
273 "charmap" if map_dict[-1] == 1 else "utf-16-be",
274 "surrogatepass",
275 )
278def parse_bfrange(
279 line: bytes,
280 map_dict: dict[Any, Any],
281 int_entry: list[int],
282 multiline_rg: Union[tuple[int, int], None],
283) -> Union[tuple[int, int], None]:
284 lst = line.split()
285 closure_found = False
286 entry_count = len(int_entry)
287 _check_mapping_size(entry_count)
288 decode_utf16 = partial(bytes.decode, encoding="utf-16-be", errors="surrogatepass")
289 if multiline_rg is not None:
290 a = multiline_rg[0] # a, b not in the current line
291 b = multiline_rg[1]
292 for sq in lst:
293 if sq == b"]":
294 closure_found = True
295 break
296 _check_token_length(sq, limit=MAX_CMAP_STRING_BYTES_LIMIT)
297 entry_count += 1
298 _check_mapping_size(entry_count)
299 map_dict[
300 __parse_bfrange__decode(map_dict=map_dict, code=a)
301 ] = decode_utf16(unhexlify(sq))
302 int_entry.append(a)
303 a += 1
304 else:
305 _check_token_length(lst[0], limit=MAX_CMAP_CODE_BYTES_LIMIT)
306 _check_token_length(lst[1], limit=MAX_CMAP_CODE_BYTES_LIMIT)
307 a = int(lst[0], 16)
308 b = int(lst[1], 16)
309 nbi = max(len(lst[0]), len(lst[1]))
310 map_dict[-1] = (nbi + 1) // 2
311 if lst[2] == b"[":
312 for sq in lst[3:]:
313 if sq == b"]":
314 closure_found = True
315 break
316 _check_token_length(sq, limit=MAX_CMAP_STRING_BYTES_LIMIT)
317 entry_count += 1
318 _check_mapping_size(entry_count)
319 map_dict[
320 __parse_bfrange__decode(map_dict=map_dict, code=a)
321 ] = decode_utf16(unhexlify(sq))
322 int_entry.append(a)
323 a += 1
324 else: # case without list
325 _check_token_length(lst[2], limit=MAX_CMAP_STRING_BYTES_LIMIT)
326 c = int(lst[2], 16)
327 fmt2 = b"%%0%dX" % max(4, len(lst[2]))
328 closure_found = True
329 range_size = max(0, b - a + 1)
330 _check_mapping_size(entry_count + range_size) # This can be checked beforehand.
331 while a <= b:
332 destination = unhexlify(fmt2 % c)
333 _check_token_length(destination, limit=MAX_CMAP_CODE_BYTES_LIMIT)
334 map_dict[
335 __parse_bfrange__decode(map_dict=map_dict, code=a)
336 ] = decode_utf16(destination)
337 int_entry.append(a)
338 a += 1
339 c += 1
340 return None if closure_found else (a, b)
343def parse_bfchar(line: bytes, map_dict: dict[Any, Any], int_entry: list[int]) -> None:
344 lst = [x for x in line.split(b" ") if x]
345 new_count = len(lst) // 2
346 _check_mapping_size(len(int_entry) + new_count) # This can be checked beforehand.
347 map_dict[-1] = len(lst[0]) // 2
348 while len(lst) > 1:
349 map_to = ""
350 # placeholder (see above) means empty string
351 if lst[1] != b".":
352 try:
353 map_to = unhexlify(lst[1]).decode(
354 "charmap" if len(lst[1]) < 4 else "utf-16-be", "surrogatepass"
355 ) # join is here as some cases where the code was split
356 except BinasciiError as exception:
357 logger_warning(
358 "Got invalid hex string: %(exception)s (%(lst_value)r)",
359 source=__name__,
360 exception=exception,
361 lst_value=lst[1],
362 )
363 map_dict[
364 unhexlify(lst[0]).decode(
365 "charmap" if map_dict[-1] == 1 else "utf-16-be", "surrogatepass"
366 )
367 ] = map_to
368 int_entry.append(int(lst[0], 16))
369 lst = lst[2:]
372def _type1_alternative(
373 ft: DictionaryObject,
374 map_dict: dict[Any, Any],
375 int_entry: list[int],
376) -> tuple[dict[Any, Any], list[int]]:
377 if "/FontDescriptor" not in ft:
378 return map_dict, int_entry
379 ft_desc = cast(DictionaryObject, ft["/FontDescriptor"]).get("/FontFile")
380 if is_null_or_none(ft_desc):
381 return map_dict, int_entry
382 assert ft_desc is not None, "mypy"
383 txt = ft_desc.get_object().get_data()
384 txt = txt.split(b"eexec\n")[0] # only clear part
385 encoding_part = txt.split(b"/Encoding")
386 if len(encoding_part) < 2:
387 return map_dict, int_entry
388 txt = encoding_part[1] # to get the encoding part
389 lines = txt.replace(b"\r", b"\n").split(b"\n")
390 for li in lines:
391 if li.startswith(b"dup"):
392 words = [_w for _w in li.split(b" ") if _w != b""]
393 if len(words) < 3 or (len(words) > 3 and words[3] != b"put"):
394 continue
395 try:
396 i = int(words[1])
397 except ValueError: # pragma: no cover
398 continue
399 try:
400 v = adobe_glyphs[words[2].decode()]
401 except KeyError:
402 if words[2].startswith(b"/uni"):
403 try:
404 v = chr(int(words[2][4:], 16))
405 except ValueError: # pragma: no cover
406 continue
407 else:
408 continue
409 map_dict[chr(i)] = v
410 int_entry.append(i)
411 return map_dict, int_entry