Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/dns/tokenizer.py: 51%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license
3# Copyright (C) 2003-2017 Nominum, Inc.
4#
5# Permission to use, copy, modify, and distribute this software and its
6# documentation for any purpose with or without fee is hereby granted,
7# provided that the above copyright notice and this permission notice
8# appear in all copies.
9#
10# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES
11# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR
13# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
16# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
18"""Tokenize DNS zone file format"""
20import io
21import sys
22from typing import Any
24import dns._text_util
25import dns.exception
26import dns.name
27import dns.ttl
29_DELIMITERS = {" ", "\t", "\n", ";", "(", ")", '"'}
30_QUOTING_DELIMITERS = {'"'}
32EOF = 0
33EOL = 1
34WHITESPACE = 2
35IDENTIFIER = 3
36QUOTED_STRING = 4
37COMMENT = 5
38DELIMITER = 6
41class UngetBufferFull(dns.exception.DNSException):
42 """An attempt was made to unget a token when the unget buffer was full."""
45class Token:
46 """A DNS zone file format token.
48 ttype: The token type
49 value: The token value
50 has_escape: Does the token value contain escapes?
51 """
53 def __init__(
54 self,
55 ttype: int,
56 value: Any = "",
57 has_escape: bool = False,
58 comment: str | None = None,
59 ):
60 """Initialize a token instance."""
62 self.ttype = ttype
63 self.value = value
64 self.has_escape = has_escape
65 self.comment = comment
67 def is_eof(self) -> bool:
68 return self.ttype == EOF
70 def is_eol(self) -> bool:
71 return self.ttype == EOL
73 def is_whitespace(self) -> bool:
74 return self.ttype == WHITESPACE
76 def is_identifier(self) -> bool:
77 return self.ttype == IDENTIFIER
79 def is_quoted_string(self) -> bool:
80 return self.ttype == QUOTED_STRING
82 def is_comment(self) -> bool:
83 return self.ttype == COMMENT
85 def is_delimiter(self) -> bool: # pragma: no cover (we don't return delimiters yet)
86 return self.ttype == DELIMITER
88 def is_eol_or_eof(self) -> bool:
89 return self.ttype == EOL or self.ttype == EOF
91 def __eq__(self, other):
92 if not isinstance(other, Token):
93 return False
94 return self.ttype == other.ttype and self.value == other.value
96 def __ne__(self, other):
97 if not isinstance(other, Token):
98 return True
99 return self.ttype != other.ttype or self.value != other.value
101 def __str__(self):
102 return f'{self.ttype} "{self.value}"'
104 def unescape(self) -> "Token":
105 if not self.has_escape:
106 return self
107 unescaped = ""
108 l = len(self.value)
109 i = 0
110 while i < l:
111 c = self.value[i]
112 i += 1
113 if c == "\\":
114 if i >= l: # pragma: no cover (can't happen via get())
115 raise dns.exception.UnexpectedEnd
116 c = self.value[i]
117 i += 1
118 if dns._text_util.is_ascii_digit(c):
119 if i >= l:
120 raise dns.exception.UnexpectedEnd
121 c2 = self.value[i]
122 i += 1
123 if i >= l:
124 raise dns.exception.UnexpectedEnd
125 c3 = self.value[i]
126 i += 1
127 if not (
128 dns._text_util.is_ascii_digit(c2)
129 and dns._text_util.is_ascii_digit(c3)
130 ):
131 raise dns.exception.SyntaxError
132 codepoint = int(c) * 100 + int(c2) * 10 + int(c3)
133 if codepoint > 255:
134 raise dns.exception.SyntaxError
135 c = chr(codepoint)
136 unescaped += c
137 return Token(self.ttype, unescaped)
139 def unescape_to_bytes(self) -> "Token":
140 # We used to use unescape() for TXT-like records, but this
141 # caused problems as we'd process DNS escapes into Unicode code
142 # points instead of byte values, and then a to_text() of the
143 # processed data would not equal the original input. For
144 # example, \226 in the TXT record would have a to_text() of
145 # \195\162 because we applied UTF-8 encoding to Unicode code
146 # point 226.
147 #
148 # We now apply escapes while converting directly to bytes,
149 # avoiding this double encoding.
150 #
151 # This code also handles cases where the unicode input has
152 # non-ASCII code-points in it by converting it to UTF-8. TXT
153 # records aren't defined for Unicode, but this is the best we
154 # can do to preserve meaning. For example,
155 #
156 # foo\u200bbar
157 #
158 # (where \u200b is Unicode code point 0x200b) will be treated
159 # as if the input had been the UTF-8 encoding of that string,
160 # namely:
161 #
162 # foo\226\128\139bar
163 #
164 unescaped = bytearray()
165 l = len(self.value)
166 i = 0
167 while i < l:
168 c = self.value[i]
169 i += 1
170 if c == "\\":
171 if i >= l: # pragma: no cover (can't happen via get())
172 raise dns.exception.UnexpectedEnd
173 c = self.value[i]
174 i += 1
175 if dns._text_util.is_ascii_digit(c):
176 if i >= l:
177 raise dns.exception.UnexpectedEnd
178 c2 = self.value[i]
179 i += 1
180 if i >= l:
181 raise dns.exception.UnexpectedEnd
182 c3 = self.value[i]
183 i += 1
184 if not (
185 dns._text_util.is_ascii_digit(c2)
186 and dns._text_util.is_ascii_digit(c3)
187 ):
188 raise dns.exception.SyntaxError
189 codepoint = int(c) * 100 + int(c2) * 10 + int(c3)
190 if codepoint > 255:
191 raise dns.exception.SyntaxError
192 unescaped += b"%c" % (codepoint)
193 else:
194 # Note that as mentioned above, if c is a Unicode
195 # code point outside of the ASCII range, then this
196 # += is converting that code point to its UTF-8
197 # encoding and appending multiple bytes to
198 # unescaped.
199 unescaped += c.encode()
200 else:
201 unescaped += c.encode()
202 return Token(self.ttype, bytes(unescaped))
205class Tokenizer:
206 """A DNS zone file format tokenizer.
208 A token object is basically a (type, value) tuple. The valid
209 types are EOF, EOL, WHITESPACE, IDENTIFIER, QUOTED_STRING,
210 COMMENT, and DELIMITER.
212 file: The file to tokenize
214 ungotten_char: The most recently ungotten character, or None.
216 ungotten_token: The most recently ungotten token, or None.
218 multiline: The current multiline level. This value is increased
219 by one every time a '(' delimiter is read, and decreased by one every time
220 a ')' delimiter is read.
222 quoting: This variable is true if the tokenizer is currently
223 reading a quoted string.
225 eof: This variable is true if the tokenizer has encountered EOF.
227 after_cr: This variable is true if the most recent character read was
228 a carriage return, which is read as a newline.
230 delimiters: The current delimiter dictionary.
232 line_number: The current line number
234 filename: A filename that will be returned by the where() method.
236 idna_codec: A dns.name.IDNACodec, specifies the IDNA
237 encoder/decoder. If None, the default IDNA
238 encoder/decoder is used.
239 """
241 def __init__(
242 self,
243 f: Any = sys.stdin,
244 filename: str | None = None,
245 idna_codec: dns.name.IDNACodec | None = None,
246 ):
247 """Initialize a tokenizer instance.
249 f: The file to tokenize. The default is sys.stdin.
250 This parameter may also be a string, in which case the tokenizer
251 will take its input from the contents of the string.
253 filename: the name of the filename that the where() method
254 will return.
256 idna_codec: A dns.name.IDNACodec, specifies the IDNA
257 encoder/decoder. If None, the default IDNA
258 encoder/decoder is used.
259 """
261 if isinstance(f, str):
262 f = io.StringIO(f)
263 if filename is None:
264 filename = "<string>"
265 elif isinstance(f, bytes):
266 f = io.StringIO(f.decode())
267 if filename is None:
268 filename = "<string>"
269 else:
270 if filename is None:
271 if f is sys.stdin:
272 filename = "<stdin>"
273 else:
274 filename = "<file>"
275 self.file = f
276 self.ungotten_char: str | None = None
277 self.ungotten_token: Token | None = None
278 self.multiline = 0
279 self.quoting = False
280 self.eof = False
281 self.after_cr = False
282 self.delimiters = _DELIMITERS
283 self.line_number = 1
284 assert filename is not None
285 self.filename = filename
286 if idna_codec is None:
287 self.idna_codec: dns.name.IDNACodec = dns.name.IDNA_DEFAULT
288 else:
289 self.idna_codec = idna_codec
291 def _get_char(self) -> str:
292 """Read a character from input."""
294 if self.ungotten_char is None:
295 if self.eof:
296 c = ""
297 else:
298 c = self.file.read(1)
299 if self.after_cr:
300 self.after_cr = False
301 if c == "\n":
302 # The newline of a CRLF pair; the CR already ended the line.
303 c = self.file.read(1)
304 if c == "\r" and not self.quoting:
305 # Outside of a quoted string, a CR ends the line like a newline.
306 self.after_cr = True
307 c = "\n"
308 if c == "":
309 self.eof = True
310 elif c == "\n":
311 self.line_number += 1
312 else:
313 c = self.ungotten_char
314 self.ungotten_char = None
315 return c
317 def where(self) -> tuple[str, int]:
318 """Return the current location in the input.
320 Returns a (string, int) tuple. The first item is the filename of
321 the input, the second is the current line number.
322 """
324 return (self.filename, self.line_number)
326 def _unget_char(self, c: str) -> None:
327 """Unget a character.
329 The unget buffer for characters is only one character large; it is
330 an error to try to unget a character when the unget buffer is not
331 empty.
333 c: the character to unget
334 raises UngetBufferFull: there is already an ungotten char
335 """
337 if self.ungotten_char is not None:
338 # this should never happen!
339 raise UngetBufferFull # pragma: no cover
340 self.ungotten_char = c
342 def skip_whitespace(self) -> int:
343 """Consume input until a non-whitespace character is encountered.
345 The non-whitespace character is then ungotten, and the number of
346 whitespace characters consumed is returned.
348 If the tokenizer is in multiline mode, then newlines are whitespace.
350 Returns the number of characters skipped.
351 """
353 skipped = 0
354 while True:
355 c = self._get_char()
356 if c != " " and c != "\t":
357 if (c != "\n") or not self.multiline:
358 self._unget_char(c)
359 return skipped
360 skipped += 1
362 def get(self, want_leading: bool = False, want_comment: bool = False) -> Token:
363 """Get the next token.
365 want_leading: If True, return a WHITESPACE token if the
366 first character read is whitespace. The default is False.
368 want_comment: If True, return a COMMENT token if the
369 first token read is a comment. The default is False.
371 Raises dns.exception.UnexpectedEnd: input ended prematurely
373 Raises dns.exception.SyntaxError: input was badly formed
375 Returns a Token.
376 """
378 if self.ungotten_token is not None:
379 utoken = self.ungotten_token
380 self.ungotten_token = None
381 if utoken.is_whitespace():
382 if want_leading:
383 return utoken
384 elif utoken.is_comment():
385 if want_comment:
386 return utoken
387 else:
388 return utoken
389 skipped = self.skip_whitespace()
390 if want_leading and skipped > 0:
391 return Token(WHITESPACE, " ")
392 token = ""
393 ttype = IDENTIFIER
394 has_escape = False
395 while True:
396 c = self._get_char()
397 if c == "" or c in self.delimiters:
398 if c == "" and self.quoting:
399 raise dns.exception.UnexpectedEnd
400 if token == "" and ttype != QUOTED_STRING:
401 if c == "(":
402 self.multiline += 1
403 self.skip_whitespace()
404 continue
405 elif c == ")":
406 if self.multiline <= 0:
407 raise dns.exception.SyntaxError
408 self.multiline -= 1
409 self.skip_whitespace()
410 continue
411 elif c == '"':
412 if not self.quoting:
413 self.quoting = True
414 self.delimiters = _QUOTING_DELIMITERS
415 ttype = QUOTED_STRING
416 continue
417 else:
418 self.quoting = False
419 self.delimiters = _DELIMITERS
420 self.skip_whitespace()
421 continue
422 elif c == "\n":
423 return Token(EOL, "\n")
424 elif c == ";":
425 while 1:
426 c = self._get_char()
427 if c == "\n" or c == "":
428 break
429 token += c
430 if want_comment:
431 self._unget_char(c)
432 return Token(COMMENT, token)
433 elif c == "":
434 if self.multiline:
435 raise dns.exception.SyntaxError(
436 "unbalanced parentheses"
437 )
438 return Token(EOF, comment=token)
439 elif self.multiline:
440 self.skip_whitespace()
441 token = ""
442 continue
443 else:
444 return Token(EOL, "\n", comment=token)
445 else:
446 # This code exists in case we ever want a
447 # delimiter to be returned. It never produces
448 # a token currently.
449 token = c
450 ttype = DELIMITER
451 else:
452 self._unget_char(c)
453 break
454 elif self.quoting and c == "\n":
455 raise dns.exception.SyntaxError("newline in quoted string")
456 elif c == "\\":
457 #
458 # It's an escape. Put it and the next character into
459 # the token; it will be checked later for goodness.
460 #
461 token += c
462 has_escape = True
463 c = self._get_char()
464 if c == "" or (c == "\n" and not self.quoting):
465 raise dns.exception.UnexpectedEnd
466 token += c
467 if token == "" and ttype != QUOTED_STRING:
468 if self.multiline:
469 raise dns.exception.SyntaxError("unbalanced parentheses")
470 ttype = EOF
471 return Token(ttype, token, has_escape)
473 def unget(self, token: Token) -> None:
474 """Unget a token.
476 The unget buffer for tokens is only one token large; it is
477 an error to try to unget a token when the unget buffer is not
478 empty.
480 token: the token to unget
482 Raises UngetBufferFull: there is already an ungotten token
483 """
485 if self.ungotten_token is not None:
486 raise UngetBufferFull
487 self.ungotten_token = token
489 def next(self):
490 """Return the next item in an iteration.
492 Returns a Token.
493 """
495 token = self.get()
496 if token.is_eof():
497 raise StopIteration
498 return token
500 __next__ = next
502 def __iter__(self):
503 return self
505 # Helpers
507 def get_int(self, base: int = 10) -> int:
508 """Read the next token and interpret it as an unsigned integer.
510 Raises dns.exception.SyntaxError if not an unsigned integer.
512 Returns an int.
513 """
514 return self.as_int(self.get().unescape(), base)
516 def get_uint8(self) -> int:
517 """Read the next token and interpret it as an 8-bit unsigned
518 integer.
520 Raises dns.exception.SyntaxError if not an 8-bit unsigned integer.
522 Returns an int.
523 """
525 return self.as_uint8(self.get().unescape())
527 def get_uint16(self, base: int = 10) -> int:
528 """Read the next token and interpret it as a 16-bit unsigned
529 integer.
531 Raises dns.exception.SyntaxError if not a 16-bit unsigned integer.
533 Returns an int.
534 """
536 return self.as_uint16(self.get().unescape(), base)
538 def get_uint32(self, base: int = 10) -> int:
539 """Read the next token and interpret it as a 32-bit unsigned
540 integer.
542 Raises dns.exception.SyntaxError if not a 32-bit unsigned integer.
544 Returns an int.
545 """
547 return self.as_uint32(self.get().unescape(), base)
549 def get_uint48(self, base: int = 10) -> int:
550 """Read the next token and interpret it as a 48-bit unsigned
551 integer.
553 Raises dns.exception.SyntaxError if not a 48-bit unsigned integer.
555 Returns an int.
556 """
558 return self.as_uint48(self.get().unescape(), base)
560 def get_string(self, max_length: int | None = None) -> str:
561 """Read the next token and interpret it as a string.
563 Raises dns.exception.SyntaxError if not a string.
564 Raises dns.exception.SyntaxError if token value length
565 exceeds max_length (if specified).
567 Returns a string.
568 """
570 return self.as_string(self.get().unescape(), max_length)
572 def get_bytes(self, max_length: int | None = None) -> bytes:
573 """Read the next token and interpret it as a byte string,
574 applying any DNS escapes directly to bytes.
576 Raises dns.exception.SyntaxError if not a string.
577 Raises dns.exception.SyntaxError if the byte length
578 exceeds max_length (if specified).
580 Returns bytes.
581 """
583 return self.as_bytes(self.get().unescape_to_bytes(), max_length)
585 def get_identifier(self) -> str:
586 """Read the next token, which should be an identifier.
588 Raises dns.exception.SyntaxError if not an identifier.
590 Returns a string.
591 """
593 return self.as_identifier(self.get().unescape())
595 def get_remaining(self, max_tokens: int | None = None) -> list[Token]:
596 """Return the remaining tokens on the line, until an EOL or EOF is seen.
598 max_tokens: If not None, stop after this number of tokens.
600 Returns a list of tokens.
601 """
603 tokens = []
604 while True:
605 token = self.get()
606 if token.is_eol_or_eof():
607 self.unget(token)
608 break
609 tokens.append(token)
610 if len(tokens) == max_tokens:
611 break
612 return tokens
614 def concatenate_remaining_identifiers(self, allow_empty: bool = False) -> str:
615 """Read the remaining tokens on the line, which should be identifiers.
617 Raises dns.exception.SyntaxError if there are no remaining tokens,
618 unless `allow_empty=True` is given.
620 Raises dns.exception.SyntaxError if a token is seen that is not an
621 identifier.
623 Returns a string containing a concatenation of the remaining
624 identifiers.
625 """
626 s = ""
627 while True:
628 token = self.get().unescape()
629 if token.is_eol_or_eof():
630 self.unget(token)
631 break
632 if not token.is_identifier():
633 raise dns.exception.SyntaxError
634 s += token.value
635 if not (allow_empty or s):
636 raise dns.exception.SyntaxError("expecting another identifier")
637 return s
639 def as_name(
640 self,
641 token: Token,
642 origin: dns.name.Name | None = None,
643 relativize: bool = False,
644 relativize_to: dns.name.Name | None = None,
645 ) -> dns.name.Name:
646 """Try to interpret the token as a DNS name.
648 Raises dns.exception.SyntaxError if not a name.
650 Returns a dns.name.Name.
651 """
652 if not token.is_identifier():
653 raise dns.exception.SyntaxError("expecting an identifier")
654 name = dns.name.from_text(token.value, origin, self.idna_codec)
655 return name.choose_relativity(relativize_to or origin, relativize)
657 def as_int(self, token: Token, base: int = 10) -> int:
658 """Try to interpret the token as an unsigned integer.
660 The token must consist only of ASCII digits valid in *base*, which
661 must be between 2 and 36 inclusive. A sign, whitespace, underscores,
662 and base prefixes such as ``0x`` are not allowed.
664 Raises dns.exception.SyntaxError if not an unsigned integer.
666 Returns an int.
667 """
669 if not token.is_identifier():
670 raise dns.exception.SyntaxError("expecting an identifier")
671 if not dns._text_util.is_ascii_digits(token.value, base):
672 raise dns.exception.SyntaxError("expecting an integer")
673 return int(token.value, base)
675 def as_uint8(self, token: Token) -> int:
676 """Try to interpret the token as an unsigned 8-bit integer.
678 Raises dns.exception.SyntaxError if not 8-bit unsigned integer.
680 Returns an int.
681 """
683 value = self.as_int(token=token)
684 if value < 0 or value > 255:
685 raise dns.exception.SyntaxError(f"{value} is not an unsigned 8-bit integer")
686 return value
688 def as_uint16(self, token: Token, base: int = 10) -> int:
689 """Try to interpret the token as an unsigned 16-bit integer.
691 Raises dns.exception.SyntaxError if not a 16-bit unsigned integer.
693 Returns an int.
694 """
696 value = self.as_int(token=token, base=base)
697 if value < 0 or value > 65535:
698 if base == 8:
699 raise dns.exception.SyntaxError(
700 f"{value:o} is not an octal unsigned 16-bit integer"
701 )
702 else:
703 raise dns.exception.SyntaxError(
704 f"{value} is not an unsigned 16-bit integer"
705 )
706 return value
708 def as_uint32(self, token: Token, base: int = 10) -> int:
709 """Try to interpret the token as an unsigned 32-bit integer.
711 Raises dns.exception.SyntaxError if not a 32-bit unsigned integer.
713 Returns an int.
714 """
716 value = self.as_int(token=token, base=base)
717 if value < 0 or value > 4294967295:
718 raise dns.exception.SyntaxError(
719 f"{value} is not an unsigned 32-bit integer"
720 )
721 return value
723 def as_uint48(self, token: Token, base: int = 10) -> int:
724 """Try to interpret the token as an unsigned 48-bit integer.
726 Raises dns.exception.SyntaxError if not a 48-bit unsigned integer.
728 Returns an int.
729 """
731 value = self.as_int(token=token, base=base)
732 if value < 0 or value > 281474976710655:
733 raise dns.exception.SyntaxError(
734 f"{value} is not an unsigned 48-bit integer"
735 )
736 return value
738 def as_string(self, token: Token, max_length: int | None = None) -> str:
739 """Try to interpret the token as a string.
741 Raises dns.exception.SyntaxError if not a string.
742 Raises dns.exception.SyntaxError if token value length
743 exceeds max_length (if specified).
745 Returns a string.
746 """
748 if not (token.is_identifier() or token.is_quoted_string()):
749 raise dns.exception.SyntaxError("expecting a string")
750 if max_length and len(token.value) > max_length:
751 raise dns.exception.SyntaxError("string too long")
752 return token.value
754 def as_bytes(self, token: Token, max_length: int | None = None) -> bytes:
755 """Try to interpret the token as a byte string.
757 Raises dns.exception.SyntaxError if not a string.
758 Raises dns.exception.SyntaxError if the byte length
759 exceeds max_length (if specified).
761 Returns bytes.
762 """
764 if not (token.is_identifier() or token.is_quoted_string()):
765 raise dns.exception.SyntaxError("expecting a string")
766 value = token.value
767 if isinstance(value, str):
768 value = value.encode()
769 if max_length and len(value) > max_length:
770 raise dns.exception.SyntaxError("string too long")
771 return value
773 def as_identifier(self, token: Token) -> str:
774 """Try to interpret the token as an identifier.
776 Raises dns.exception.SyntaxError if not an identifier.
778 Returns a string.
779 """
781 if not token.is_identifier():
782 raise dns.exception.SyntaxError("expecting an identifier")
783 return token.value
785 def get_name(
786 self,
787 origin: dns.name.Name | None = None,
788 relativize: bool = False,
789 relativize_to: dns.name.Name | None = None,
790 ) -> dns.name.Name:
791 """Read the next token and interpret it as a DNS name.
793 Raises dns.exception.SyntaxError if not a name.
795 Returns a dns.name.Name.
796 """
798 token = self.get()
799 return self.as_name(token, origin, relativize, relativize_to)
801 def get_eol_as_token(self) -> Token:
802 """Read the next token and raise an exception if it isn't EOL or
803 EOF.
805 Returns a string.
806 """
808 token = self.get()
809 if not token.is_eol_or_eof():
810 raise dns.exception.SyntaxError(
811 f'expected EOL or EOF, got {token.ttype} "{token.value}"'
812 )
813 return token
815 def get_eol(self) -> str:
816 return self.get_eol_as_token().value
818 def get_ttl(self) -> int:
819 """Read the next token and interpret it as a DNS TTL.
821 Raises dns.exception.SyntaxError or dns.ttl.BadTTL if not an
822 identifier or badly formed.
824 Returns an int.
825 """
827 token = self.get().unescape()
828 if not token.is_identifier():
829 raise dns.exception.SyntaxError("expecting an identifier")
830 return dns.ttl.from_text(token.value)