Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pypdf/_cmap.py: 9%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from binascii import Error as BinasciiError
2from binascii import unhexlify
3from math import ceil
4from typing import Any, Union, cast
6from ._codecs import adobe_glyphs, charset_encoding
7from ._utils import logger_error, logger_warning
8from .errors import LimitReachedError
9from .generic import (
10 DecodedStreamObject,
11 DictionaryObject,
12 NullObject,
13 StreamObject,
14 is_null_or_none,
15)
17_predefined_cmap: dict[str, str] = {
18 "/Identity-H": "utf-16-be",
19 "/Identity-V": "utf-16-be",
20 "/GB-EUC-H": "gbk",
21 "/GB-EUC-V": "gbk",
22 "/GBpc-EUC-H": "gb2312",
23 "/GBpc-EUC-V": "gb2312",
24 "/GBK-EUC-H": "gbk",
25 "/GBK-EUC-V": "gbk",
26 "/GBK2K-H": "gb18030",
27 "/GBK2K-V": "gb18030",
28 "/ETen-B5-H": "cp950",
29 "/ETen-B5-V": "cp950",
30 "/ETenms-B5-H": "cp950",
31 "/ETenms-B5-V": "cp950",
32 "/UniCNS-UTF16-H": "utf-16-be",
33 "/UniCNS-UTF16-V": "utf-16-be",
34 "/UniGB-UTF16-H": "gb18030",
35 "/UniGB-UTF16-V": "gb18030",
36 # Japanese CMaps (PDF Reference 1.7, Appendix H)
37 "/90ms-RKSJ-H": "cp932", # Shift-JIS (JIS X 0208-1990), horizontal
38 "/90ms-RKSJ-V": "cp932", # Shift-JIS (JIS X 0208-1990), vertical
39 "/UniJIS-UTF16-H": "utf-16-be", # Unicode UTF-16BE -> JIS, horizontal
40 "/UniJIS-UTF16-V": "utf-16-be", # Unicode UTF-16BE -> JIS, vertical
41 # UCS2 in code
42}
45def get_encoding(
46 ft: DictionaryObject
47) -> tuple[Union[str, dict[int, str]], dict[Any, Any]]:
48 encoding = _parse_encoding(ft)
49 map_dict, int_entry = _parse_to_unicode(ft)
51 # Apply rule from PDF ref 1.7 §5.9.1, 1st bullet:
52 # if cmap not empty encoding should be discarded
53 # (here transformed into identity for those characters)
54 # If encoding is a string, it is expected to be an identity translation.
55 if isinstance(encoding, dict):
56 for x in int_entry:
57 if x <= 255:
58 encoding[x] = chr(x)
60 return encoding, map_dict
63def _parse_encoding(
64 ft: DictionaryObject
65) -> Union[str, dict[int, str]]:
66 encoding: Union[str, list[str], dict[int, str]] = []
67 if "/Encoding" not in ft:
68 if "/BaseFont" in ft and cast(str, ft["/BaseFont"]) in charset_encoding:
69 encoding = dict(
70 zip(range(256), charset_encoding[cast(str, ft["/BaseFont"])])
71 )
72 else:
73 encoding = "charmap"
74 return encoding
75 enc: Union[str, DictionaryObject, NullObject] = cast(
76 Union[str, DictionaryObject, NullObject], ft["/Encoding"].get_object()
77 )
78 if isinstance(enc, str):
79 try:
80 # already done : enc = NameObject.unnumber(enc.encode()).decode()
81 # for #xx decoding
82 if enc in charset_encoding:
83 encoding = charset_encoding[enc].copy()
84 elif enc in _predefined_cmap:
85 encoding = _predefined_cmap[enc]
86 elif "-UCS2-" in enc:
87 encoding = "utf-16-be"
88 else:
89 raise Exception("not found")
90 except Exception:
91 logger_error("Advanced encoding %(encoding)s not implemented yet", source=__name__, encoding=enc)
92 encoding = enc
93 elif isinstance(enc, DictionaryObject) and "/BaseEncoding" in enc:
94 try:
95 encoding = charset_encoding[cast(str, enc["/BaseEncoding"])].copy()
96 except Exception:
97 logger_error(
98 "Advanced encoding %(encoding)s not implemented yet",
99 source=__name__, encoding=encoding
100 )
101 encoding = charset_encoding["/StandardEncoding"].copy()
102 else:
103 encoding = charset_encoding["/StandardEncoding"].copy()
104 if isinstance(enc, DictionaryObject) and "/Differences" in enc:
105 x: int = 0
106 o: Union[int, str]
107 for o in cast(DictionaryObject, enc["/Differences"]):
108 if isinstance(o, int):
109 x = o
110 else: # isinstance(o, str):
111 try:
112 if x < len(encoding):
113 encoding[x] = adobe_glyphs[o] # type: ignore[index]
114 except Exception:
115 encoding[x] = o # type: ignore[index]
116 x += 1
117 if isinstance(encoding, list):
118 encoding = dict(zip(range(256), encoding))
119 return encoding
122def _parse_to_unicode(
123 ft: DictionaryObject
124) -> tuple[dict[Any, Any], list[int]]:
125 # will store all translation code
126 # and map_dict[-1] we will have the number of bytes to convert
127 map_dict: dict[Any, Any] = {}
129 # will provide the list of cmap keys as int to correct encoding
130 int_entry: list[int] = []
132 if "/ToUnicode" not in ft:
133 if ft.get("/Subtype", "") == "/Type1":
134 return _type1_alternative(ft, map_dict, int_entry)
135 return {}, []
136 process_rg: bool = False
137 process_char: bool = False
138 multiline_rg: Union[
139 None, tuple[int, int]
140 ] = None # tuple = (current_char, remaining size) ; cf #1285 for example of file
141 cm = prepare_cm(ft)
142 for line in cm.split(b"\n"):
143 process_rg, process_char, multiline_rg = process_cm_line(
144 line.strip(b" \t"),
145 process_rg,
146 process_char,
147 multiline_rg,
148 map_dict,
149 int_entry,
150 )
152 map_dict.pop(-1, None) # Don't pass the -1 key, we only used it to temporarily store encoding length
154 return map_dict, int_entry
157def prepare_cm(ft: DictionaryObject) -> bytes:
158 tu = ft["/ToUnicode"]
159 cm: bytes
160 if isinstance(tu, StreamObject):
161 cm = cast(DecodedStreamObject, ft["/ToUnicode"]).get_data()
162 else: # if (tu is None) or cast(str, tu).startswith("/Identity"):
163 # the full range 0000-FFFF will be processed
164 cm = b"beginbfrange\n<0000> <0001> <0000>\nendbfrange"
165 if isinstance(cm, str):
166 cm = cm.encode()
167 # we need to prepare cm before due to missing return line in pdf printed
168 # to pdf from word
169 cm = (
170 cm.strip()
171 .replace(b"beginbfchar", b"\nbeginbfchar\n")
172 .replace(b"endbfchar", b"\nendbfchar\n")
173 .replace(b"beginbfrange", b"\nbeginbfrange\n")
174 .replace(b"endbfrange", b"\nendbfrange\n")
175 .replace(b"<<", b"\n{\n") # text between << and >> not used but
176 .replace(b">>", b"\n}\n") # some solution to find it back
177 )
178 ll = cm.split(b"<")
179 for i in range(len(ll)):
180 j = ll[i].find(b">")
181 if j >= 0:
182 if j == 0:
183 # string is empty: stash a placeholder here (see below)
184 # see https://github.com/py-pdf/pypdf/issues/1111
185 content = b"."
186 else:
187 content = ll[i][:j].replace(b" ", b"")
188 ll[i] = content + b" " + ll[i][j + 1 :]
189 cm = (
190 (b" ".join(ll))
191 .replace(b"[", b" [ ")
192 .replace(b"]", b" ]\n ")
193 .replace(b"\r", b"\n")
194 )
195 return cm
198def process_cm_line(
199 line: bytes,
200 process_rg: bool,
201 process_char: bool,
202 multiline_rg: Union[None, tuple[int, int]],
203 map_dict: dict[Any, Any],
204 int_entry: list[int],
205) -> tuple[bool, bool, Union[None, tuple[int, int]]]:
206 if line == b"" or line[0] == 37: # 37 = %
207 return process_rg, process_char, multiline_rg
208 line = line.replace(b"\t", b" ")
209 if b"beginbfrange" in line:
210 process_rg = True
211 elif b"endbfrange" in line:
212 process_rg = False
213 elif b"beginbfchar" in line:
214 process_char = True
215 elif b"endbfchar" in line:
216 process_char = False
217 elif process_rg:
218 try:
219 multiline_rg = parse_bfrange(line, map_dict, int_entry, multiline_rg)
220 except (ValueError, IndexError) as error:
221 logger_warning("Skipping broken line %(line)r: %(error)s", source=__name__, line=line, error=error)
222 elif process_char:
223 try:
224 parse_bfchar(line, map_dict, int_entry)
225 except (ValueError, IndexError) as error:
226 logger_warning("Skipping broken line %(line)r: %(error)s", source=__name__, line=line, error=error)
227 return process_rg, process_char, multiline_rg
230# Usual values should be up to 65_536.
231MAPPING_DICTIONARY_SIZE_LIMIT = 100_000
234def _check_mapping_size(size: int) -> None:
235 if size > MAPPING_DICTIONARY_SIZE_LIMIT:
236 raise LimitReachedError(f"Maximum /ToUnicode size limit reached: {size} > {MAPPING_DICTIONARY_SIZE_LIMIT}.")
239def parse_bfrange(
240 line: bytes,
241 map_dict: dict[Any, Any],
242 int_entry: list[int],
243 multiline_rg: Union[None, tuple[int, int]],
244) -> Union[None, tuple[int, int]]:
245 lst = [x for x in line.split(b" ") if x]
246 closure_found = False
247 entry_count = len(int_entry)
248 _check_mapping_size(entry_count)
249 if multiline_rg is not None:
250 fmt = b"%%0%dX" % (map_dict[-1] * 2)
251 a = multiline_rg[0] # a, b not in the current line
252 b = multiline_rg[1]
253 for sq in lst:
254 if sq == b"]":
255 closure_found = True
256 break
257 entry_count += 1
258 _check_mapping_size(entry_count)
259 map_dict[
260 unhexlify(fmt % a).decode(
261 "charmap" if map_dict[-1] == 1 else "utf-16-be",
262 "surrogatepass",
263 )
264 ] = unhexlify(sq).decode("utf-16-be", "surrogatepass")
265 int_entry.append(a)
266 a += 1
267 else:
268 a = int(lst[0], 16)
269 b = int(lst[1], 16)
270 nbi = max(len(lst[0]), len(lst[1]))
271 map_dict[-1] = ceil(nbi / 2)
272 fmt = b"%%0%dX" % (map_dict[-1] * 2)
273 if lst[2] == b"[":
274 for sq in lst[3:]:
275 if sq == b"]":
276 closure_found = True
277 break
278 entry_count += 1
279 _check_mapping_size(entry_count)
280 map_dict[
281 unhexlify(fmt % a).decode(
282 "charmap" if map_dict[-1] == 1 else "utf-16-be",
283 "surrogatepass",
284 )
285 ] = unhexlify(sq).decode("utf-16-be", "surrogatepass")
286 int_entry.append(a)
287 a += 1
288 else: # case without list
289 c = int(lst[2], 16)
290 fmt2 = b"%%0%dX" % max(4, len(lst[2]))
291 closure_found = True
292 range_size = max(0, b - a + 1)
293 _check_mapping_size(entry_count + range_size) # This can be checked beforehand.
294 while a <= b:
295 map_dict[
296 unhexlify(fmt % a).decode(
297 "charmap" if map_dict[-1] == 1 else "utf-16-be",
298 "surrogatepass",
299 )
300 ] = unhexlify(fmt2 % c).decode("utf-16-be", "surrogatepass")
301 int_entry.append(a)
302 a += 1
303 c += 1
304 return None if closure_found else (a, b)
307def parse_bfchar(line: bytes, map_dict: dict[Any, Any], int_entry: list[int]) -> None:
308 lst = [x for x in line.split(b" ") if x]
309 new_count = len(lst) // 2
310 _check_mapping_size(len(int_entry) + new_count) # This can be checked beforehand.
311 map_dict[-1] = len(lst[0]) // 2
312 while len(lst) > 1:
313 map_to = ""
314 # placeholder (see above) means empty string
315 if lst[1] != b".":
316 try:
317 map_to = unhexlify(lst[1]).decode(
318 "charmap" if len(lst[1]) < 4 else "utf-16-be", "surrogatepass"
319 ) # join is here as some cases where the code was split
320 except BinasciiError as exception:
321 logger_warning(
322 "Got invalid hex string: %(exception)s (%(lst_value)r)",
323 source=__name__,
324 exception=exception,
325 lst_value=lst[1],
326 )
327 map_dict[
328 unhexlify(lst[0]).decode(
329 "charmap" if map_dict[-1] == 1 else "utf-16-be", "surrogatepass"
330 )
331 ] = map_to
332 int_entry.append(int(lst[0], 16))
333 lst = lst[2:]
336def _type1_alternative(
337 ft: DictionaryObject,
338 map_dict: dict[Any, Any],
339 int_entry: list[int],
340) -> tuple[dict[Any, Any], list[int]]:
341 if "/FontDescriptor" not in ft:
342 return map_dict, int_entry
343 ft_desc = cast(DictionaryObject, ft["/FontDescriptor"]).get("/FontFile")
344 if is_null_or_none(ft_desc):
345 return map_dict, int_entry
346 assert ft_desc is not None, "mypy"
347 txt = ft_desc.get_object().get_data()
348 txt = txt.split(b"eexec\n")[0] # only clear part
349 encoding_part = txt.split(b"/Encoding")
350 if len(encoding_part) < 2:
351 return map_dict, int_entry
352 txt = encoding_part[1] # to get the encoding part
353 lines = txt.replace(b"\r", b"\n").split(b"\n")
354 for li in lines:
355 if li.startswith(b"dup"):
356 words = [_w for _w in li.split(b" ") if _w != b""]
357 if len(words) < 3 or (len(words) > 3 and words[3] != b"put"):
358 continue
359 try:
360 i = int(words[1])
361 except ValueError: # pragma: no cover
362 continue
363 try:
364 v = adobe_glyphs[words[2].decode()]
365 except KeyError:
366 if words[2].startswith(b"/uni"):
367 try:
368 v = chr(int(words[2][4:], 16))
369 except ValueError: # pragma: no cover
370 continue
371 else:
372 continue
373 map_dict[chr(i)] = v
374 int_entry.append(i)
375 return map_dict, int_entry