Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
3 webencodings
4 ~~~~~~~~~~~~
6 This is a Python implementation of the `WHATWG Encoding standard
7 <http://encoding.spec.whatwg.org/>`. See README for details.
9 :copyright: Copyright 2012 by Simon Sapin
10 :license: BSD, see LICENSE for details.
12"""
14import codecs
16from .custom import replacement_codec_info, user_codec_info
17from .labels import LABELS
19VERSION = __version__ = '0.6.1'
22PYTHON_NAMES = {
23 # Some names in Encoding are not valid Python aliases. Remap these:
24 'iso-8859-8-i': 'iso8859-8',
25 'x-mac-cyrillic': 'mac-cyrillic',
26 'macintosh': 'mac-roman',
27 'windows-874': 'cp874',
28 # Some WHATWG-defined names conflict with a Python alias for an
29 # incompatible codec. These should be remapped to the correct one:
30 'shift_jis': 'cp932',
31 'big5': 'big5hkscs',
32 'euc-kr': 'cp949',
33}
35CACHE = {}
38def ascii_lower(string):
39 """Transform (only) ASCII letters to lower case: A-Z is mapped to a-z.
41 :param string: An Unicode string.
42 :returns: A new Unicode string.
44 This is used for `ASCII case-insensitive
45 <http://encoding.spec.whatwg.org/#ascii-case-insensitive>`_
46 matching of encoding labels.
47 The same matching is also used, among other things,
48 for `CSS keywords <http://dev.w3.org/csswg/css-values/#keywords>`_.
50 This is different from the :meth:`str.lower` method of Unicode strings
51 which also affect non-ASCII characters,
52 sometimes mapping them into the ASCII range:
54 >>> keyword = 'Bac\\N{KELVIN SIGN}ground'
55 >>> assert keyword.lower() == 'background'
56 >>> assert ascii_lower(keyword) != keyword.lower()
57 >>> assert ascii_lower(keyword) == 'bac\\N{KELVIN SIGN}ground'
59 """
60 # This turns out to be faster than unicode.translate()
61 return string.encode().lower().decode()
64def lookup(label):
65 """Look for an encoding by its label.
67 This is the spec's `get an encoding
68 <http://encoding.spec.whatwg.org/#concept-encoding-get>`_ algorithm.
69 Supported labels are listed there.
71 :param label: A string.
72 :returns:
73 An :class:`Encoding` object, or :obj:`None` for an unknown label.
75 """
76 # Only strip ASCII whitespace: U+0009, U+000A, U+000C, U+000D, and U+0020.
77 label = ascii_lower(label.strip('\t\n\f\r '))
78 name = LABELS.get(label)
79 if name is None:
80 return None
81 encoding = CACHE.get(name)
82 if encoding is None:
83 if name == 'x-user-defined':
84 codec_info = user_codec_info
85 elif name == 'replacement':
86 codec_info = replacement_codec_info
87 else:
88 python_name = PYTHON_NAMES.get(name, name)
89 # Any python_name value that gets to here should be valid.
90 codec_info = codecs.lookup(python_name)
91 encoding = Encoding(name, codec_info)
92 CACHE[name] = encoding
93 return encoding
96def _get_encoding(encoding_or_label):
97 """Accept either an encoding object or label.
99 :param encoding: An :class:`Encoding` object or a label string.
100 :returns: An :class:`Encoding` object.
101 :raises: :exc:`LookupError` for an unknown label.
103 """
104 if hasattr(encoding_or_label, 'codec_info'):
105 return encoding_or_label
107 encoding = lookup(encoding_or_label)
108 if encoding is None:
109 raise LookupError(f'Unknown encoding label: {encoding_or_label!r}')
110 return encoding
113class Encoding:
114 """A character encoding that can be used for decoding or encoding.
116 .. attribute:: name
118 Canonical name of the encoding
120 .. attribute:: codec_info
122 The actual implementation of the encoding,
123 a stdlib :class:`~codecs.CodecInfo` object.
124 See :func:`codecs.register`.
126 """
127 def __init__(self, name, codec_info):
128 self.name = name
129 self.codec_info = codec_info
131 def __repr__(self):
132 return f'<Encoding {self.name}>'
135#: The UTF-8 encoding. Should be used for new content and formats.
136UTF8 = lookup('utf-8')
138_UTF16LE = lookup('utf-16le')
139_UTF16BE = lookup('utf-16be')
142def decode(input, fallback_encoding, errors='replace'):
143 """Decode a single string.
145 :param input: A byte string
146 :param fallback_encoding:
147 An :class:`Encoding` object or a label string.
148 The encoding to use if :obj:`input` does not have a BOM.
149 :param errors: Type of error handling. See :func:`codecs.register`.
150 :raises: :exc:`LookupError` for an unknown encoding label.
151 :return:
152 A ``(output, encoding)`` tuple of an Unicode string
153 and an :obj:`Encoding`.
155 """
156 # Fail early if `encoding` is an invalid label.
157 fallback_encoding = _get_encoding(fallback_encoding)
158 bom_encoding, input = _detect_bom(input)
159 encoding = bom_encoding or fallback_encoding
160 return encoding.codec_info.decode(input, errors)[0], encoding
163def _detect_bom(input):
164 """Return (bom_encoding, input), with any BOM removed from the input."""
165 if input.startswith(b'\xFF\xFE'):
166 return _UTF16LE, input[2:]
167 if input.startswith(b'\xFE\xFF'):
168 return _UTF16BE, input[2:]
169 if input.startswith(b'\xEF\xBB\xBF'):
170 return UTF8, input[3:]
171 return None, input
174def encode(input, encoding=UTF8, errors='strict'):
175 """Encode a single string.
177 :param input: An Unicode string.
178 :param encoding: An :class:`Encoding` object or a label string.
179 :param errors: Type of error handling. See :func:`codecs.register`.
180 :raises: :exc:`LookupError` for an unknown encoding label.
181 :return: A byte string.
183 """
184 return _get_encoding(encoding).codec_info.encode(input, errors)[0]
187def iter_decode(input, fallback_encoding, errors='replace'):
188 """"Pull"-based decoder.
190 :param input:
191 An iterable of byte strings.
193 The input is first consumed just enough to determine the encoding
194 based on the precense of a BOM,
195 then consumed on demand when the return value is.
196 :param fallback_encoding:
197 An :class:`Encoding` object or a label string.
198 The encoding to use if :obj:`input` does not have a BOM.
199 :param errors: Type of error handling. See :func:`codecs.register`.
200 :raises: :exc:`LookupError` for an unknown encoding label.
201 :returns:
202 An ``(output, encoding)`` tuple.
203 ``output`` is an iterable of Unicode strings,
204 ``encoding`` is the :obj:`Encoding` that is being used.
206 """
208 decoder = IncrementalDecoder(fallback_encoding, errors)
209 generator = _iter_decode_generator(input, decoder)
210 encoding = next(generator)
211 return generator, encoding
214def _iter_decode_generator(input, decoder):
215 """Return a decode generator.
217 It first yields the :obj:`Encoding`, then yields output chunks as Unicode
218 strings.
220 """
221 decode = decoder.decode
222 input = iter(input)
223 for chunck in input:
224 output = decode(chunck)
225 if output:
226 assert decoder.encoding is not None
227 yield decoder.encoding
228 yield output
229 break
230 else:
231 # Input exhausted without determining the encoding
232 output = decode(b'', final=True)
233 assert decoder.encoding is not None
234 yield decoder.encoding
235 if output:
236 yield output
237 return
239 for chunck in input:
240 output = decode(chunck)
241 if output:
242 yield output
243 output = decode(b'', final=True)
244 if output:
245 yield output
248def iter_encode(input, encoding=UTF8, errors='strict'):
249 """"Pull"-based encoder.
251 :param input: An iterable of Unicode strings.
252 :param encoding: An :class:`Encoding` object or a label string.
253 :param errors: Type of error handling. See :func:`codecs.register`.
254 :raises: :exc:`LookupError` for an unknown encoding label.
255 :returns: An iterable of byte strings.
257 """
258 # Fail early if `encoding` is an invalid label.
259 encode = IncrementalEncoder(encoding, errors).encode
260 return _iter_encode_generator(input, encode)
263def _iter_encode_generator(input, encode):
264 """Return an encode generator."""
265 for chunck in input:
266 output = encode(chunck)
267 if output:
268 yield output
269 output = encode('', final=True)
270 if output:
271 yield output
274class IncrementalDecoder:
275 """"Push"-based decoder.
277 :param fallback_encoding:
278 An :class:`Encoding` object or a label string.
279 The encoding to use if :obj:`input` does not have a BOM.
280 :param errors: Type of error handling. See :func:`codecs.register`.
281 :raises: :exc:`LookupError` for an unknown encoding label.
283 """
284 def __init__(self, fallback_encoding, errors='replace'):
285 # Fail early if `encoding` is an invalid label.
286 self._fallback_encoding = _get_encoding(fallback_encoding)
287 self._errors = errors
288 self._buffer = b''
289 self._decoder = None
290 #: The actual :class:`Encoding` that is being used,
291 #: or :obj:`None` if that is not determined yet.
292 #: (Ie. if there is not enough input yet to determine
293 #: if there is a BOM.)
294 self.encoding = None # Not known yet.
296 def decode(self, input, final=False):
297 """Decode one chunk of the input.
299 :param input: A byte string.
300 :param final:
301 Indicate that no more input is available.
302 Must be :obj:`True` if this is the last call.
303 :returns: An Unicode string.
305 """
306 decoder = self._decoder
307 if decoder is not None:
308 return decoder(input, final)
310 input = self._buffer + input
311 encoding, input = _detect_bom(input)
312 if encoding is None:
313 if len(input) < 3 and not final: # Not enough data yet.
314 self._buffer = input
315 return ''
316 else: # No BOM
317 encoding = self._fallback_encoding
318 decoder = encoding.codec_info.incrementaldecoder(self._errors).decode
319 self._decoder = decoder
320 self.encoding = encoding
321 return decoder(input, final)
324class IncrementalEncoder:
325 """"Push"-based encoder.
327 :param encoding: An :class:`Encoding` object or a label string.
328 :param errors: Type of error handling. See :func:`codecs.register`.
329 :raises: :exc:`LookupError` for an unknown encoding label.
331 .. method:: encode(input, final=False)
333 :param input: An Unicode string.
334 :param final:
335 Indicate that no more input is available.
336 Must be :obj:`True` if this is the last call.
337 :returns: A byte string.
339 """
340 def __init__(self, encoding=UTF8, errors='strict'):
341 encoding = _get_encoding(encoding)
342 self.encode = encoding.codec_info.incrementalencoder(errors).encode