#ifndef Py_TOKENIZER_H

#define Py_TOKENIZER_H

#include "Python.h"

#include "types.h"

struct tok_state;

typedef enum {

    _PyTokenizer_KEEP_NEWLINE,   // Preserve the original line terminator.

    _PyTokenizer_IMPLY_NEWLINE,  // Add a missing line terminator.

    _PyTokenizer_APPEND_NEWLINE, // Also append a blank line to terminated input.

} _PyTokenizer_Newline;

/* Reasons that the appended newline can change the token stream. */

enum {

    _PyTokenizer_NEWLINE_STATEMENT = 1,

    _PyTokenizer_NEWLINE_INDENT = 2,

    _PyTokenizer_NEWLINE_CONTINUATION = 4,

};

/* Initialize before use. metadata owns a reference released by _PyToken_Free;

   a consumer taking that reference must set metadata to NULL. */

struct token {

    int type;

    int level;

    unsigned char is_raw;

    unsigned char is_incomplete;

    _PyTok_Span span;

    _PyTok_Loc start_loc;

    _PyTok_Loc end_loc;

    PyObject *metadata;

};

typedef struct {

    const char *text;

    Py_ssize_t length;

    _PyTok_Span line_span;

    _PyTok_Off end_line_start;

    int implicit_newline;

    int at_eof;

} _PyToken_View;

/* Supplemental source context for a terminal error. location is the reporting

   cursor, independent of the scanner cursor; lineno == 0 means absent.

   The text span may cover multiple physical lines. */

typedef struct {

    _PyTok_Loc location;

    _PyTok_Span text_span;

} _PyTokenizer_Diagnostic;

typedef struct {

    int status;

    unsigned char implicit_newline;  // _PyTokenizer_Newline

    _PyTokenizer_Diagnostic diagnostic;

    _PyTok_Loc location;

    _PyTok_Off cursor;

    _PyTok_Span input_span;

    _PyTok_Span line_span;

    int level;

    char delimiter;

    _PyTok_Loc delimiter_loc;

    int in_formatted_string;

    int is_interactive;

    int is_file;

    PyObject *filename;

    PyObject *module;

    const char *encoding;

} _PyTokenizer_Info;

/* Get replaces the initialized token, releasing its previous metadata.

   Errors are returned as ERRORTOKEN, with or without a Python exception.

   In incomplete-input mode, an unfinished string retains its token type,

   sets is_incomplete and has E_INCOMPLETE status. Only f-string and t-string

   middle tokens retain their text span. No Python exception is set. */

void _PyTokenizer_Get(struct tok_state *, struct token *);

void _PyTokenizer_Free(struct tok_state *);

int _PyTokenizer_Traverse(struct tok_state *, visitproc, void *);

void _PyTokenizer_raise_init_error(PyObject *filename);

void _PyToken_Init(struct token *);

static inline void

_PyToken_Free(struct token *token)

{

    Py_CLEAR(token->metadata);

}

Unexecuted instantiation: errors.c:_PyToken_Free

Unexecuted instantiation: traceback.c:_PyToken_Free

Unexecuted instantiation: peg_api.c:_PyToken_Free

Unexecuted instantiation: reader.c:_PyToken_Free

Unexecuted instantiation: helpers.c:_PyToken_Free

Python-tokenize.c:_PyToken_Free

Line

Count

Source

83

2.00k

{

84

    Py_CLEAR(token->metadata);

85

2.00k

}

pegen.c:_PyToken_Free

Line

Count

Source

83

1.74M

{

84

    Py_CLEAR(token->metadata);

85

1.74M

}

pegen_errors.c:_PyToken_Free

Line

Count

Source

83

61.0k

{

84

    Py_CLEAR(token->metadata);

85

61.0k

}

Unexecuted instantiation: parser.c:_PyToken_Free

Unexecuted instantiation: parser_invalid.c:_PyToken_Free

lexer.c:_PyToken_Free

Line

Count

Source

83

2.01M

{

84

    Py_CLEAR(token->metadata);

85

2.01M

}

Unexecuted instantiation: layout.c:_PyToken_Free

Unexecuted instantiation: number.c:_PyToken_Free

Unexecuted instantiation: state.c:_PyToken_Free

Unexecuted instantiation: string.c:_PyToken_Free

Unexecuted instantiation: decoder.c:_PyToken_Free

Unexecuted instantiation: api.c:_PyToken_Free

Unexecuted instantiation: action_helpers.c:_PyToken_Free

Unexecuted instantiation: string_parser.c:_PyToken_Free

/* Views and borrowed snapshot references remain valid until the tokenizer is

   mutated or freed. Source spans may be discarded when reading more input. */

_PyTokenizer_Info _PyTokenizer_GetInfo(const struct tok_state *);

int _PyTokenizer_GetStatus(const struct tok_state *);

int _PyTokenizer_GetNewlineRequirements(const struct tok_state *);

/* An absent token span has a nonnull empty text view. */

const char *_PyToken_TextView(

    const struct tok_state *, const struct token *, Py_ssize_t *);

/* Use the token from the most recent Get. An absent span has nonnull empty text.

   line_span covers the token's complete physical line range. end_line_start

   is the logical offset of its final physical line. */

void _PyToken_GetView(

    const struct tok_state *tok, const struct token *token,

    _PyToken_View *view);

const char *_PyTokenizer_SpanView(

    const struct tok_state *, _PyTok_Span, Py_ssize_t *);

/* For retained input: 1-based lines clamp to the first or final line.

   The view excludes the newline and need not be NUL-terminated. */

const char *_PyTokenizer_LineView(

    const struct tok_state *, Py_ssize_t, Py_ssize_t *);

/* Return NUL-terminated retained input, or NULL without setting an exception

   for streaming input or interactive input before its first line. */

const char *_PyTokenizer_RetainedSource(const struct tok_state *);

void _PyTokenizer_SetContext(

    struct tok_state *tok, PyObject *filename, PyObject *module);

void _PyTokenizer_SetOptions(

    struct tok_state *tok, int extra_tokens, int type_comments);

void _PyTokenizer_AllowIncompleteInput(struct tok_state *, int);

void _PyTokenizer_ImplyDedents(struct tok_state *);

int _PyTokenizer_HasTrailingStatement(const struct tok_state *);

int _PyTokenizer_IsInteractive(const struct tok_state *);

void _PyTokenizer_StopInteractive(struct tok_state *);

struct tok_state *_PyTokenizer_FromString(

    const char *input, int exec_input, int preserve_crlf);

struct tok_state *_PyTokenizer_FromUTF8(

    const char *input, int exec_input, int preserve_crlf);

struct tok_state *_PyTokenizer_FromReadline(

    PyObject *readline, const char *encoding);

struct tok_state *_PyTokenizer_FromFile(

    FILE *fp, const char *encoding, const char *ps1, const char *ps2);

/* Return the declared encoding in PyMem-allocated storage, or NULL.

   An exception is set on error. */

char *_PyTokenizer_FindEncodingFilename(int, PyObject *);

#endif /* !Py_TOKENIZER_H */