/src/cpython/Parser/tokenizer/utf8_tokenizer.c
Line | Count | Source (jump to first uncovered line) |
1 | | #include "Python.h" |
2 | | #include "errcode.h" |
3 | | |
4 | | #include "helpers.h" |
5 | | #include "../lexer/state.h" |
6 | | |
7 | | static int |
8 | 512 | tok_underflow_string(struct tok_state *tok) { |
9 | 512 | char *end = strchr(tok->inp, '\n'); |
10 | 512 | if (end != NULL) { |
11 | 368 | end++; |
12 | 368 | } |
13 | 144 | else { |
14 | 144 | end = strchr(tok->inp, '\0'); |
15 | 144 | if (end == tok->inp) { |
16 | 78 | tok->done = E_EOF; |
17 | 78 | return 0; |
18 | 78 | } |
19 | 144 | } |
20 | 434 | if (tok->start == NULL) { |
21 | 434 | tok->buf = tok->cur; |
22 | 434 | } |
23 | 434 | tok->line_start = tok->cur; |
24 | 434 | ADVANCE_LINENO(); |
25 | 434 | tok->inp = end; |
26 | 434 | return 1; |
27 | 512 | } |
28 | | |
29 | | /* Set up tokenizer for UTF-8 string */ |
30 | | struct tok_state * |
31 | | _PyTokenizer_FromUTF8(const char *str, int exec_input, int preserve_crlf) |
32 | 78 | { |
33 | 78 | struct tok_state *tok = _PyTokenizer_tok_new(); |
34 | 78 | char *translated; |
35 | 78 | if (tok == NULL) |
36 | 0 | return NULL; |
37 | 78 | tok->input = translated = _PyTokenizer_translate_newlines(str, exec_input, preserve_crlf, tok); |
38 | 78 | if (translated == NULL) { |
39 | 0 | _PyTokenizer_Free(tok); |
40 | 0 | return NULL; |
41 | 0 | } |
42 | 78 | tok->decoding_state = STATE_NORMAL; |
43 | 78 | tok->enc = NULL; |
44 | 78 | tok->str = translated; |
45 | 78 | tok->encoding = _PyTokenizer_new_string("utf-8", 5, tok); |
46 | 78 | if (!tok->encoding) { |
47 | 0 | _PyTokenizer_Free(tok); |
48 | 0 | return NULL; |
49 | 0 | } |
50 | | |
51 | 78 | tok->buf = tok->cur = tok->inp = translated; |
52 | 78 | tok->end = translated; |
53 | 78 | tok->underflow = &tok_underflow_string; |
54 | 78 | return tok; |
55 | 78 | } |