/src/cpython/Parser/tokenizer/source.h
Line | Count | Source |
1 | | #ifndef Py_TOKENIZER_SOURCE_H |
2 | | #define Py_TOKENIZER_SOURCE_H |
3 | | |
4 | | #include "Python.h" |
5 | | |
6 | | #include "types.h" |
7 | | |
8 | | typedef struct { |
9 | | char *bytes; |
10 | | _PyTok_Off base_offset; |
11 | | _PyTok_Off len; |
12 | | _PyTok_Off cap; |
13 | | int nlines; |
14 | | } _PyTok_SourceText; |
15 | | |
16 | | static inline const char * |
17 | | _PyTok_SourceData(const _PyTok_SourceText *source) |
18 | 2.53M | { |
19 | 2.53M | return source->bytes != NULL ? source->bytes : ""; |
20 | 2.53M | } reader.c:_PyTok_SourceData Line | Count | Source | 18 | 566k | { | 19 | 566k | return source->bytes != NULL ? source->bytes : ""; | 20 | 566k | } |
source.c:_PyTok_SourceData Line | Count | Source | 18 | 257 | { | 19 | 257 | return source->bytes != NULL ? source->bytes : ""; | 20 | 257 | } |
helpers.c:_PyTok_SourceData Line | Count | Source | 18 | 1.25k | { | 19 | 1.25k | return source->bytes != NULL ? source->bytes : ""; | 20 | 1.25k | } |
Unexecuted instantiation: pegen.c:_PyTok_SourceData lexer.c:_PyTok_SourceData Line | Count | Source | 18 | 269k | { | 19 | 269k | return source->bytes != NULL ? source->bytes : ""; | 20 | 269k | } |
Unexecuted instantiation: layout.c:_PyTok_SourceData Unexecuted instantiation: number.c:_PyTok_SourceData Unexecuted instantiation: state.c:_PyTok_SourceData string.c:_PyTok_SourceData Line | Count | Source | 18 | 13.1k | { | 19 | 13.1k | return source->bytes != NULL ? source->bytes : ""; | 20 | 13.1k | } |
decoder.c:_PyTok_SourceData Line | Count | Source | 18 | 17.2k | { | 19 | 17.2k | return source->bytes != NULL ? source->bytes : ""; | 20 | 17.2k | } |
Line | Count | Source | 18 | 1.66M | { | 19 | 1.66M | return source->bytes != NULL ? source->bytes : ""; | 20 | 1.66M | } |
Unexecuted instantiation: action_helpers.c:_PyTok_SourceData |
21 | | |
22 | | /* Convert positions within the retained source window. Pointers and views |
23 | | are borrowed; append, discard, and clear invalidate them. */ |
24 | | static inline const char * |
25 | | _PyTok_SourcePointer(const _PyTok_SourceText *source, _PyTok_Off offset) |
26 | 2.45M | { |
27 | 2.45M | assert(offset >= source->base_offset); |
28 | 2.45M | assert(offset - source->base_offset <= source->len); |
29 | 2.45M | return _PyTok_SourceData(source) + (offset - source->base_offset); |
30 | 2.45M | } reader.c:_PyTok_SourcePointer Line | Count | Source | 26 | 511k | { | 27 | 511k | assert(offset >= source->base_offset); | 28 | | assert(offset - source->base_offset <= source->len); | 29 | 511k | return _PyTok_SourceData(source) + (offset - source->base_offset); | 30 | 511k | } |
Unexecuted instantiation: source.c:_PyTok_SourcePointer helpers.c:_PyTok_SourcePointer Line | Count | Source | 26 | 1.25k | { | 27 | 1.25k | assert(offset >= source->base_offset); | 28 | | assert(offset - source->base_offset <= source->len); | 29 | 1.25k | return _PyTok_SourceData(source) + (offset - source->base_offset); | 30 | 1.25k | } |
Unexecuted instantiation: pegen.c:_PyTok_SourcePointer lexer.c:_PyTok_SourcePointer Line | Count | Source | 26 | 269k | { | 27 | 269k | assert(offset >= source->base_offset); | 28 | | assert(offset - source->base_offset <= source->len); | 29 | 269k | return _PyTok_SourceData(source) + (offset - source->base_offset); | 30 | 269k | } |
Unexecuted instantiation: layout.c:_PyTok_SourcePointer Unexecuted instantiation: number.c:_PyTok_SourcePointer Unexecuted instantiation: state.c:_PyTok_SourcePointer string.c:_PyTok_SourcePointer Line | Count | Source | 26 | 13.1k | { | 27 | 13.1k | assert(offset >= source->base_offset); | 28 | | assert(offset - source->base_offset <= source->len); | 29 | 13.1k | return _PyTok_SourceData(source) + (offset - source->base_offset); | 30 | 13.1k | } |
Unexecuted instantiation: decoder.c:_PyTok_SourcePointer api.c:_PyTok_SourcePointer Line | Count | Source | 26 | 1.66M | { | 27 | 1.66M | assert(offset >= source->base_offset); | 28 | | assert(offset - source->base_offset <= source->len); | 29 | 1.66M | return _PyTok_SourceData(source) + (offset - source->base_offset); | 30 | 1.66M | } |
Unexecuted instantiation: action_helpers.c:_PyTok_SourcePointer |
31 | | |
32 | | static inline unsigned char |
33 | | _PyTok_SourceByte(const _PyTok_SourceText *source, _PyTok_Off offset) |
34 | 13.8M | { |
35 | 13.8M | assert(offset >= source->base_offset); |
36 | 13.8M | assert(offset - source->base_offset < source->len); |
37 | 13.8M | return (unsigned char)source->bytes[offset - source->base_offset]; |
38 | 13.8M | } Unexecuted instantiation: reader.c:_PyTok_SourceByte Unexecuted instantiation: source.c:_PyTok_SourceByte Unexecuted instantiation: helpers.c:_PyTok_SourceByte Unexecuted instantiation: pegen.c:_PyTok_SourceByte lexer.c:_PyTok_SourceByte Line | Count | Source | 34 | 12.0M | { | 35 | 12.0M | assert(offset >= source->base_offset); | 36 | | assert(offset - source->base_offset < source->len); | 37 | 12.0M | return (unsigned char)source->bytes[offset - source->base_offset]; | 38 | 12.0M | } |
layout.c:_PyTok_SourceByte Line | Count | Source | 34 | 516k | { | 35 | 516k | assert(offset >= source->base_offset); | 36 | | assert(offset - source->base_offset < source->len); | 37 | 516k | return (unsigned char)source->bytes[offset - source->base_offset]; | 38 | 516k | } |
number.c:_PyTok_SourceByte Line | Count | Source | 34 | 538k | { | 35 | 538k | assert(offset >= source->base_offset); | 36 | | assert(offset - source->base_offset < source->len); | 37 | 538k | return (unsigned char)source->bytes[offset - source->base_offset]; | 38 | 538k | } |
Unexecuted instantiation: state.c:_PyTok_SourceByte string.c:_PyTok_SourceByte Line | Count | Source | 34 | 757k | { | 35 | 757k | assert(offset >= source->base_offset); | 36 | | assert(offset - source->base_offset < source->len); | 37 | 757k | return (unsigned char)source->bytes[offset - source->base_offset]; | 38 | 757k | } |
Unexecuted instantiation: decoder.c:_PyTok_SourceByte Unexecuted instantiation: api.c:_PyTok_SourceByte Unexecuted instantiation: action_helpers.c:_PyTok_SourceByte |
39 | | |
40 | | static inline const char * |
41 | | _PyTok_SourceSpanView(const _PyTok_SourceText *source, _PyTok_Span span, |
42 | | Py_ssize_t *length) |
43 | 2.44M | { |
44 | 2.44M | assert(length != NULL); |
45 | 2.44M | assert(_PyTok_SpanIsValid(span)); |
46 | 2.44M | assert(span.end - source->base_offset <= source->len); |
47 | 2.44M | *length = span.end - span.start; |
48 | 2.44M | return _PyTok_SourcePointer(source, span.start); |
49 | 2.44M | } reader.c:_PyTok_SourceSpanView Line | Count | Source | 43 | 511k | { | 44 | 511k | assert(length != NULL); | 45 | 511k | assert(_PyTok_SpanIsValid(span)); | 46 | | assert(span.end - source->base_offset <= source->len); | 47 | 511k | *length = span.end - span.start; | 48 | 511k | return _PyTok_SourcePointer(source, span.start); | 49 | 511k | } |
Unexecuted instantiation: source.c:_PyTok_SourceSpanView Unexecuted instantiation: helpers.c:_PyTok_SourceSpanView Unexecuted instantiation: pegen.c:_PyTok_SourceSpanView lexer.c:_PyTok_SourceSpanView Line | Count | Source | 43 | 256k | { | 44 | 256k | assert(length != NULL); | 45 | 256k | assert(_PyTok_SpanIsValid(span)); | 46 | | assert(span.end - source->base_offset <= source->len); | 47 | 256k | *length = span.end - span.start; | 48 | 256k | return _PyTok_SourcePointer(source, span.start); | 49 | 256k | } |
Unexecuted instantiation: layout.c:_PyTok_SourceSpanView Unexecuted instantiation: number.c:_PyTok_SourceSpanView Unexecuted instantiation: state.c:_PyTok_SourceSpanView string.c:_PyTok_SourceSpanView Line | Count | Source | 43 | 12.5k | { | 44 | 12.5k | assert(length != NULL); | 45 | 12.5k | assert(_PyTok_SpanIsValid(span)); | 46 | | assert(span.end - source->base_offset <= source->len); | 47 | 12.5k | *length = span.end - span.start; | 48 | 12.5k | return _PyTok_SourcePointer(source, span.start); | 49 | 12.5k | } |
Unexecuted instantiation: decoder.c:_PyTok_SourceSpanView api.c:_PyTok_SourceSpanView Line | Count | Source | 43 | 1.66M | { | 44 | 1.66M | assert(length != NULL); | 45 | 1.66M | assert(_PyTok_SpanIsValid(span)); | 46 | | assert(span.end - source->base_offset <= source->len); | 47 | 1.66M | *length = span.end - span.start; | 48 | 1.66M | return _PyTok_SourcePointer(source, span.start); | 49 | 1.66M | } |
Unexecuted instantiation: action_helpers.c:_PyTok_SourceSpanView |
50 | | |
51 | | /* Return the first matching offset within span, or -1 if absent. */ |
52 | | static inline _PyTok_Off |
53 | | _PyTok_SourceFindByte(const _PyTok_SourceText *source, _PyTok_Span span, |
54 | | int byte) |
55 | 512k | { |
56 | 512k | Py_ssize_t length; |
57 | 512k | const char *data = _PyTok_SourceSpanView(source, span, &length); |
58 | 512k | const char *found = memchr(data, byte, length); |
59 | 512k | return found != NULL ? span.start + (found - data) : -1; |
60 | 512k | } reader.c:_PyTok_SourceFindByte Line | Count | Source | 55 | 255k | { | 56 | 255k | Py_ssize_t length; | 57 | 255k | const char *data = _PyTok_SourceSpanView(source, span, &length); | 58 | 255k | const char *found = memchr(data, byte, length); | 59 | 255k | return found != NULL ? span.start + (found - data) : -1; | 60 | 255k | } |
Unexecuted instantiation: source.c:_PyTok_SourceFindByte Unexecuted instantiation: helpers.c:_PyTok_SourceFindByte Unexecuted instantiation: pegen.c:_PyTok_SourceFindByte lexer.c:_PyTok_SourceFindByte Line | Count | Source | 55 | 256k | { | 56 | 256k | Py_ssize_t length; | 57 | 256k | const char *data = _PyTok_SourceSpanView(source, span, &length); | 58 | 256k | const char *found = memchr(data, byte, length); | 59 | 256k | return found != NULL ? span.start + (found - data) : -1; | 60 | 256k | } |
Unexecuted instantiation: layout.c:_PyTok_SourceFindByte Unexecuted instantiation: number.c:_PyTok_SourceFindByte Unexecuted instantiation: state.c:_PyTok_SourceFindByte Unexecuted instantiation: string.c:_PyTok_SourceFindByte Unexecuted instantiation: decoder.c:_PyTok_SourceFindByte Unexecuted instantiation: api.c:_PyTok_SourceFindByte Unexecuted instantiation: action_helpers.c:_PyTok_SourceFindByte |
61 | | |
62 | | PyAPI_FUNC(void) _PyTok_SourceInit(_PyTok_SourceText *); |
63 | | /* Clear invalidates all spans and views for the source. */ |
64 | | PyAPI_FUNC(void) _PyTok_SourceClear(_PyTok_SourceText *); |
65 | | /* Discard the retained window and invalidate its spans and views. |
66 | | Keep its allocation and advance the logical base to the end of the window. */ |
67 | | PyAPI_FUNC(void) _PyTok_SourceDiscard(_PyTok_SourceText *); |
68 | | /* Append one nonempty logical line and return its start offset. The input may |
69 | | contain one newline, as its final byte. An unterminated line must be the |
70 | | final line. The input must not point into source storage. */ |
71 | | PyAPI_FUNC(_PyTok_Off) _PyTok_SourceAppendLine( |
72 | | _PyTok_SourceText *source, const char *bytes, Py_ssize_t len); |
73 | | /* Return borrowed bytes excluding '\n', writing the byte length to *len. |
74 | | Line numbers are 1-based and clamp to the first or final line; a trailing |
75 | | '\n' adds an empty final line. The view need not be NUL-terminated. |
76 | | This does not set an exception. Append, discard, and clear invalidate the view. */ |
77 | | PyAPI_FUNC(const char *) _PyTok_SourceLineView( |
78 | | const _PyTok_SourceText *source, Py_ssize_t lineno, Py_ssize_t *len); |
79 | | |
80 | | #endif |