/src/cpython/Parser/lexer/lexer_internal.h
Line | Count | Source |
1 | | #ifndef _PY_LEXER_INTERNAL_H_ |
2 | | #define _PY_LEXER_INTERNAL_H_ |
3 | | |
4 | | #include "errcode.h" |
5 | | #include "state.h" |
6 | | |
7 | 1.85M | #define is_potential_identifier_start(c) (\ |
8 | 1.85M | (c >= 'a' && c <= 'z')\ |
9 | 1.85M | || (c >= 'A' && c <= 'Z')\ |
10 | 1.85M | || c == '_'\ |
11 | 1.85M | || (c >= 128)) |
12 | | |
13 | 2.70M | #define is_potential_identifier_char(c) (\ |
14 | 2.70M | (c >= 'a' && c <= 'z')\ |
15 | 2.70M | || (c >= 'A' && c <= 'Z')\ |
16 | 2.70M | || (c >= '0' && c <= '9')\ |
17 | 2.70M | || c == '_'\ |
18 | 2.70M | || (c >= 128)) |
19 | | |
20 | | #define FTSTRING_MIDDLE(state) \ |
21 | | (_PyLexer_IsTString((state)->kind) ? TSTRING_MIDDLE : FSTRING_MIDDLE) |
22 | | #define FTSTRING_END(state) \ |
23 | 13.5k | (_PyLexer_IsTString((state)->kind) ? TSTRING_END : FSTRING_END) |
24 | 3.94M | #define tok_backup _PyLexer_backup |
25 | | |
26 | | static inline int |
27 | | tok_failed(const struct tok_state *tok) |
28 | 1.87M | { |
29 | 1.87M | return tok->done != E_OK && tok->done != E_EOF && |
30 | 2.57k | tok->done != E_INTERACT_STOP; |
31 | 1.87M | } Line | Count | Source | 28 | 1.87M | { | 29 | 1.87M | return tok->done != E_OK && tok->done != E_EOF && | 30 | 2.57k | tok->done != E_INTERACT_STOP; | 31 | 1.87M | } |
Unexecuted instantiation: layout.c:tok_failed Unexecuted instantiation: number.c:tok_failed Line | Count | Source | 28 | 710 | { | 29 | 710 | return tok->done != E_OK && tok->done != E_EOF && | 30 | 0 | tok->done != E_INTERACT_STOP; | 31 | 710 | } |
|
32 | | |
33 | | int _PyLexer_refill(struct tok_state *); |
34 | | |
35 | | static inline int |
36 | | tok_nextc(struct tok_state *tok) |
37 | 10.2M | { |
38 | 10.4M | while (tok->cur == tok->inp) { |
39 | 435k | if (!_PyLexer_refill(tok)) { |
40 | 179k | return EOF; |
41 | 179k | } |
42 | 435k | } |
43 | 10.0M | if (tok->cur - tok->line_start >= INT_MAX) { |
44 | 0 | tok->done = E_COLUMNOVERFLOW; |
45 | 0 | return EOF; |
46 | 0 | } |
47 | 10.0M | return _PyTok_SourceByte(&tok->source, tok->cur++); |
48 | 10.0M | } Line | Count | Source | 37 | 8.36M | { | 38 | 8.36M | while (tok->cur == tok->inp) { | 39 | 119k | if (!_PyLexer_refill(tok)) { | 40 | 119k | return EOF; | 41 | 119k | } | 42 | 119k | } | 43 | 8.24M | if (tok->cur - tok->line_start >= INT_MAX) { | 44 | 0 | tok->done = E_COLUMNOVERFLOW; | 45 | 0 | return EOF; | 46 | 0 | } | 47 | 8.24M | return _PyTok_SourceByte(&tok->source, tok->cur++); | 48 | 8.24M | } |
Line | Count | Source | 37 | 576k | { | 38 | 821k | while (tok->cur == tok->inp) { | 39 | 304k | if (!_PyLexer_refill(tok)) { | 40 | 59.7k | return EOF; | 41 | 59.7k | } | 42 | 304k | } | 43 | 516k | if (tok->cur - tok->line_start >= INT_MAX) { | 44 | 0 | tok->done = E_COLUMNOVERFLOW; | 45 | 0 | return EOF; | 46 | 0 | } | 47 | 516k | return _PyTok_SourceByte(&tok->source, tok->cur++); | 48 | 516k | } |
Line | Count | Source | 37 | 538k | { | 38 | 538k | while (tok->cur == tok->inp) { | 39 | 0 | if (!_PyLexer_refill(tok)) { | 40 | 0 | return EOF; | 41 | 0 | } | 42 | 0 | } | 43 | 538k | if (tok->cur - tok->line_start >= INT_MAX) { | 44 | 0 | tok->done = E_COLUMNOVERFLOW; | 45 | 0 | return EOF; | 46 | 0 | } | 47 | 538k | return _PyTok_SourceByte(&tok->source, tok->cur++); | 48 | 538k | } |
Line | Count | Source | 37 | 757k | { | 38 | 769k | while (tok->cur == tok->inp) { | 39 | 11.2k | if (!_PyLexer_refill(tok)) { | 40 | 76 | return EOF; | 41 | 76 | } | 42 | 11.2k | } | 43 | 757k | if (tok->cur - tok->line_start >= INT_MAX) { | 44 | 0 | tok->done = E_COLUMNOVERFLOW; | 45 | 0 | return EOF; | 46 | 0 | } | 47 | 757k | return _PyTok_SourceByte(&tok->source, tok->cur++); | 48 | 757k | } |
|
49 | | |
50 | | /* Return -1 on error, otherwise whether the line is blank. */ |
51 | | int _PyLexer_BeginLine(struct tok_state *); |
52 | | int _PyLexer_ContinueLine(struct tok_state *); |
53 | | int _PyLexer_IndentationToken(struct tok_state *, struct token *); |
54 | | /* Return zero when the newline is suppressed, otherwise its token type. */ |
55 | | int _PyLexer_Newline(struct tok_state *, struct token *, int); |
56 | | void _PyLexer_backup(struct tok_state *, int); |
57 | | int _PyLexer_record_ftstring_comment( |
58 | | struct tok_state *, ftstring_state *, _PyTok_Off, _PyTok_Off); |
59 | | int _PyLexer_ftstring_punctuation( |
60 | | struct tok_state *, ftstring_state *, struct token *, int); |
61 | | int _PyLexer_close_ftstring_expr( |
62 | | struct tok_state *, ftstring_state *, int); |
63 | | void _PyLexer_mark_ftstring_debug(struct tok_state *, ftstring_state *); |
64 | | int _PyLexer_check_string_prefixes(struct tok_state *, int, int, int, int, int); |
65 | | int _PyLexer_scan_number(struct tok_state *, struct token *, int, int); |
66 | | int _PyLexer_scan_fstring_start( |
67 | | struct tok_state *, struct token *, int, ftstring_kind); |
68 | | int _PyLexer_scan_string(struct tok_state *, struct token *, int); |
69 | | int _PyLexer_get_normal(struct tok_state *, ftstring_state *, struct token *); |
70 | | int _PyLexer_get_ftstring(struct tok_state *, ftstring_state *, struct token *); |
71 | | |
72 | | #endif |