Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/asttokens/line_numbers.py: 90%

1# Copyright 2016 Grist Labs, Inc. 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); 

4# you may not use this file except in compliance with the License. 

5# You may obtain a copy of the License at 

6# 

7# https://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, 

11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

12# See the License for the specific language governing permissions and 

13# limitations under the License. 

14 

15import bisect 

16import re 

17from typing import Dict, List, Tuple 

18 

19# Matches the end-of-line sequences that Python treats as line boundaries in source code, i.e. 

20# "\r\n", "\r", or "\n". Using this (rather than a plain `re.M` `^`) means we recognise a lone 

21# "\r" as a line separator, matching how the tokenizer and ast module number lines. See issue #105. 

22_line_end_re = re.compile(r'\r\n|\r|\n') 

23 

24class LineNumbers: 

25 """ 

26 Class to convert between character offsets in a text string, and pairs (line, column) of 1-based 

27 line and 0-based column numbers, as used by tokens and AST nodes. 

28 

29 This class expects unicode for input and stores positions in unicode. But it supports 

30 translating to and from utf8 offsets, which are used by ast parsing. 

31 """ 

32 def __init__(self, text: str) -> None: 

33 # A list of character offsets of each line's first character. The first line always starts at 

34 # offset 0, and each subsequent line starts right after an end-of-line sequence. 

35 self._line_offsets = [0] + [m.end(0) for m in _line_end_re.finditer(text)] 

36 self._text = text 

37 self._text_len = len(text) 

38 self._utf8_offset_cache: Dict[int, List[int]] = {} # maps line num to list of char offset for each byte in line 

39 

40 def from_utf8_col(self, line: int, utf8_column: int) -> int: 

41 """ 

42 Given a 1-based line number and 0-based utf8 column, returns a 0-based unicode column. 

43 """ 

44 offsets = self._utf8_offset_cache.get(line) 

45 if offsets is None: 

46 end_offset = self._line_offsets[line] if line < len(self._line_offsets) else self._text_len 

47 line_text = self._text[self._line_offsets[line - 1] : end_offset] 

48 

49 offsets = [i for i,c in enumerate(line_text) for byte in c.encode('utf8')] 

50 offsets.append(len(line_text)) 

51 self._utf8_offset_cache[line] = offsets 

52 

53 return offsets[max(0, min(len(offsets)-1, utf8_column))] 

54 

55 def line_to_offset(self, line: int, column: int) -> int: 

56 """ 

57 Converts 1-based line number and 0-based column to 0-based character offset into text. 

58 """ 

59 line -= 1 

60 if line >= len(self._line_offsets): 

61 return self._text_len 

62 elif line < 0: 

63 return 0 

64 else: 

65 return min(self._line_offsets[line] + max(0, column), self._text_len) 

66 

67 def offset_to_line(self, offset: int) -> Tuple[int, int]: 

68 """ 

69 Converts 0-based character offset to pair (line, col) of 1-based line and 0-based column 

70 numbers. 

71 """ 

72 offset = max(0, min(self._text_len, offset)) 

73 line_index = bisect.bisect_right(self._line_offsets, offset) - 1 

74 return (line_index + 1, offset - self._line_offsets[line_index])