/src/cpython/Python/ceval.h

Source
#define _PY_INTERPRETER

#include "Python.h"
#include "pycore_abstract.h"      // _PyIndex_Check()
#include "pycore_audit.h"         // _PySys_Audit()
#include "pycore_backoff.h"
#include "pycore_call.h"          // _PyObject_CallNoArgs()
#include "pycore_cell.h"          // PyCell_GetRef()
#include "pycore_ceval.h"         // SPECIAL___ENTER__
#include "pycore_code.h"
#include "pycore_dict.h"
#include "pycore_emscripten_signal.h"  // _Py_CHECK_EMSCRIPTEN_SIGNALS
#include "pycore_floatobject.h"   // _PyFloat_ExactDealloc()
#include "pycore_frame.h"
#include "pycore_function.h"
#include "pycore_genobject.h"     // _PyCoro_GetAwaitableIter()
#include "pycore_import.h"        // _PyImport_IsDefaultImportFunc()
#include "pycore_instruments.h"
#include "pycore_interpframe.h"   // _PyFrame_SetStackPointer()
#include "pycore_interpolation.h" // _PyInterpolation_Build()
#include "pycore_intrinsics.h"
#include "pycore_jit.h"
#include "pycore_lazyimportobject.h"
#include "pycore_list.h"          // _PyList_GetItemRef()
#include "pycore_long.h"          // _PyLong_GetZero()
#include "pycore_moduleobject.h"  // PyModuleObject
#include "pycore_object.h"        // _PyObject_GC_TRACK()
#include "pycore_opcode_metadata.h" // EXTRA_CASES
#include "pycore_opcode_utils.h"  // MAKE_FUNCTION_*
#include "pycore_optimizer.h"     // _PyUOpExecutor_Type
#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_*
#include "pycore_pyerrors.h"      // _PyErr_GetRaisedException()
#include "pycore_pystate.h"       // _PyInterpreterState_GET()
#include "pycore_range.h"         // _PyRangeIterObject
#include "pycore_setobject.h"     // _PySet_Update()
#include "pycore_sliceobject.h"   // _PyBuildSlice_ConsumeRefs
#include "pycore_sysmodule.h"     // _PySys_GetOptionalAttrString()
#include "pycore_template.h"      // _PyTemplate_Build()
#include "pycore_traceback.h"     // _PyTraceBack_FromFrame
#include "pycore_tuple.h"         // _PyTuple_ITEMS()
#include "pycore_uop_ids.h"       // Uops

#include "dictobject.h"
#include "frameobject.h"          // _PyInterpreterFrame_GetLine
#include "opcode.h"
#include "pydtrace.h"
#include "setobject.h"
#include "pycore_stackref.h"

#include <stdbool.h>              // bool

#if !defined(Py_BUILD_CORE)
#  error "ceval.c must be build with Py_BUILD_CORE define for best performance"
#endif

#if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS)
// GH-89279: The MSVC compiler does not inline these static inline functions
// in PGO build in _PyEval_EvalFrameDefault(), because this function is over
// the limit of PGO, and that limit cannot be configured.
// Define them as macros to make sure that they are always inlined by the
// preprocessor.

#undef Py_IS_TYPE
#define Py_IS_TYPE(ob, type) \
    (_PyObject_CAST(ob)->ob_type == (type))

#undef Py_XDECREF
#define Py_XDECREF(arg) \
    do { \
        PyObject *xop = _PyObject_CAST(arg); \
        if (xop != NULL) { \
            Py_DECREF(xop); \
        } \
    } while (0)

#ifndef Py_GIL_DISABLED

#undef Py_DECREF
#define Py_DECREF(arg) \
    do { \
        PyObject *op = _PyObject_CAST(arg); \
        if (_Py_IsImmortal(op)) { \
            _Py_DECREF_IMMORTAL_STAT_INC(); \
            break; \
        } \
        _Py_DECREF_STAT_INC(); \
        if (--op->ob_refcnt == 0) { \
            _PyReftracerTrack(op, PyRefTracer_DESTROY); \
            destructor dealloc = Py_TYPE(op)->tp_dealloc; \
            (*dealloc)(op); \
        } \
    } while (0)

#undef _Py_DECREF_SPECIALIZED
#define _Py_DECREF_SPECIALIZED(arg, dealloc) \
    do { \
        PyObject *op = _PyObject_CAST(arg); \
        if (_Py_IsImmortal(op)) { \
            _Py_DECREF_IMMORTAL_STAT_INC(); \
            break; \
        } \
        _Py_DECREF_STAT_INC(); \
        if (--op->ob_refcnt == 0) { \
            _PyReftracerTrack(op, PyRefTracer_DESTROY); \
            destructor d = (destructor)(dealloc); \
            d(op); \
        } \
    } while (0)

#else // Py_GIL_DISABLED

#undef Py_DECREF
#define Py_DECREF(arg) \
    do { \
        PyObject *op = _PyObject_CAST(arg); \
        uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); \
        if (local == _Py_IMMORTAL_REFCNT_LOCAL) { \
            _Py_DECREF_IMMORTAL_STAT_INC(); \
            break; \
        } \
        _Py_DECREF_STAT_INC(); \
        if (_Py_IsOwnedByCurrentThread(op)) { \
            local--; \
            _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); \
            if (local == 0) { \
                _Py_MergeZeroLocalRefcount(op); \
            } \
        } \
        else { \
            _Py_DecRefShared(op); \
        } \
    } while (0)

#undef _Py_DECREF_SPECIALIZED
#define _Py_DECREF_SPECIALIZED(arg, dealloc) Py_DECREF(arg)

#endif
#endif

static void
check_invalid_reentrancy(void)
{
#if defined(Py_DEBUG) && defined(Py_GIL_DISABLED)
    // In the free-threaded build, the interpreter must not be re-entered if
    // the world-is-stopped.  If so, that's a bug somewhere (quite likely in
    // the painfully complex typeobject code).
    PyInterpreterState *interp = _PyInterpreterState_GET();
    assert(!interp->stoptheworld.world_stopped);
#endif
}


#ifdef Py_DEBUG
static void
dump_item(_PyStackRef item)
{
    if (PyStackRef_IsNull(item)) {
        printf("<NULL>");
        return;
    }
    if (PyStackRef_IsMalformed(item)) {
        printf("<INVALID>");
        return;
    }
    if (PyStackRef_IsTaggedInt(item)) {
        printf("%" PRId64, (int64_t)PyStackRef_UntagInt(item));
        return;
    }
    PyObject *obj = PyStackRef_AsPyObjectBorrow(item);
    if (obj == NULL) {
        printf("<nil>");
        return;
    }
    // Don't call __repr__(), it might recurse into the interpreter.
    printf("<%s at %p>", Py_TYPE(obj)->tp_name, (void *)obj);
}

static void
dump_stack(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer)
{
    _PyFrame_SetStackPointer(frame, stack_pointer);
    _PyStackRef *locals_base = _PyFrame_GetLocalsArray(frame);
    _PyStackRef *stack_base = _PyFrame_Stackbase(frame);
    PyObject *exc = PyErr_GetRaisedException();
    printf("    locals=[");
    for (_PyStackRef *ptr = locals_base; ptr < stack_base; ptr++) {
        if (ptr != locals_base) {
            printf(", ");
        }
        dump_item(*ptr);
    }
    printf("]\n");
    if (stack_pointer < stack_base) {
        printf("    stack=%d\n", (int)(stack_pointer-stack_base));
    }
    else {
        printf("    stack=[");
        for (_PyStackRef *ptr = stack_base; ptr < stack_pointer; ptr++) {
            if (ptr != stack_base) {
                printf(", ");
            }
            dump_item(*ptr);
        }
        printf("]\n");
    }
    fflush(stdout);
    PyErr_SetRaisedException(exc);
    _PyFrame_GetStackPointer(frame);
}

#if defined(_Py_TIER2) && !defined(_Py_JIT) && defined(Py_DEBUG)
static void
dump_cache_item(_PyStackRef cache, int position, int depth)
{
    if (position < depth) {
        dump_item(cache);
    }
    else {
        printf("---");
    }
}
#endif

static void
lltrace_instruction(_PyInterpreterFrame *frame,
                    _PyStackRef *stack_pointer,
                    _Py_CODEUNIT *next_instr,
                    int opcode,
                    int oparg)
{
    int offset = 0;
    if (frame->owner < FRAME_OWNED_BY_INTERPRETER) {
        dump_stack(frame, stack_pointer);
        offset = (int)(next_instr - _PyFrame_GetBytecode(frame));
    }
    const char *opname = _PyOpcode_OpName[opcode];
    assert(opname != NULL);
    if (OPCODE_HAS_ARG((int)_PyOpcode_Deopt[opcode])) {
        printf("%d: %s %d\n", offset * 2, opname, oparg);
    }
    else {
        printf("%d: %s\n", offset * 2, opname);
    }
    fflush(stdout);
}

static void
lltrace_resume_frame(_PyInterpreterFrame *frame)
{
    PyObject *fobj = PyStackRef_AsPyObjectBorrow(frame->f_funcobj);
    if (!PyStackRef_CodeCheck(frame->f_executable) ||
        fobj == NULL ||
        !PyFunction_Check(fobj)
    ) {
        printf("\nResuming frame.\n");
        return;
    }
    PyFunctionObject *f = (PyFunctionObject *)fobj;
    PyObject *exc = PyErr_GetRaisedException();
    PyObject *name = f->func_qualname;
    if (name == NULL) {
        name = f->func_name;
    }
    printf("\nResuming frame");
    if (name) {
        printf(" for ");
        if (PyObject_Print(name, stdout, 0) < 0) {
            PyErr_Clear();
        }
    }
    if (f->func_module) {
        printf(" in module ");
        if (PyObject_Print(f->func_module, stdout, 0) < 0) {
            PyErr_Clear();
        }
    }
    printf("\n");
    fflush(stdout);
    PyErr_SetRaisedException(exc);
}

static int
maybe_lltrace_resume_frame(_PyInterpreterFrame *frame, PyObject *globals)
{
    if (globals == NULL) {
        return 0;
    }
    if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) {
        return 0;
    }
    int r = PyDict_Contains(globals, &_Py_ID(__lltrace__));
    if (r < 0) {
        PyErr_Clear();
        return 0;
    }
    int lltrace = r * 5;  // Levels 1-4 only trace uops
    if (!lltrace) {
        // Can also be controlled by environment variable
        char *python_lltrace = Py_GETENV("PYTHON_LLTRACE");
        if (python_lltrace != NULL && *python_lltrace >= '0') {
            lltrace = *python_lltrace - '0';  // TODO: Parse an int and all that
        }
    }
    if (lltrace >= 5) {
        lltrace_resume_frame(frame);
    }
    return lltrace;
}

#endif

static void monitor_reraise(PyThreadState *tstate,
                 _PyInterpreterFrame *frame,
                 _Py_CODEUNIT *instr);
static int monitor_stop_iteration(PyThreadState *tstate,
                 _PyInterpreterFrame *frame,
                 _Py_CODEUNIT *instr,
                 PyObject *value);
static void monitor_unwind(PyThreadState *tstate,
                 _PyInterpreterFrame *frame,
                 _Py_CODEUNIT *instr);
static int monitor_handled(PyThreadState *tstate,
                 _PyInterpreterFrame *frame,
                 _Py_CODEUNIT *instr, PyObject *exc);
static void monitor_throw(PyThreadState *tstate,
                 _PyInterpreterFrame *frame,
                 _Py_CODEUNIT *instr);

static int get_exception_handler(PyCodeObject *, int, int*, int*, int*);

#ifdef HAVE_ERRNO_H
#include <errno.h>
#endif

typedef struct {
    _PyInterpreterFrame frame;
    _PyStackRef stack[1];
} _PyEntryFrame;

static int
do_monitor_exc(PyThreadState *tstate, _PyInterpreterFrame *frame,
               _Py_CODEUNIT *instr, int event)
{
    assert(event < _PY_MONITORING_UNGROUPED_EVENTS);
    if (_PyFrame_GetCode(frame)->co_flags & CO_NO_MONITORING_EVENTS) {
        return 0;
    }
    PyObject *exc = PyErr_GetRaisedException();
    assert(exc != NULL);
    int err = _Py_call_instrumentation_arg(tstate, event, frame, instr, exc);
    if (err == 0) {
        PyErr_SetRaisedException(exc);
    }
    else {
        assert(PyErr_Occurred());
        Py_DECREF(exc);
    }
    return err;
}

static inline bool
no_tools_for_global_event(PyThreadState *tstate, int event)
{
    return tstate->interp->monitors.tools[event] == 0;
}

static inline bool
no_tools_for_local_event(PyThreadState *tstate, _PyInterpreterFrame *frame, int event)
{
    assert(event < _PY_MONITORING_LOCAL_EVENTS);
    _PyCoMonitoringData *data = _PyFrame_GetCode(frame)->_co_monitoring;
    if (data) {
        return data->active_monitors.tools[event] == 0;
    }
    else {
        return no_tools_for_global_event(tstate, event);
    }
}

static int
monitor_handled(PyThreadState *tstate,
                _PyInterpreterFrame *frame,
                _Py_CODEUNIT *instr, PyObject *exc)
{
    if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED)) {
        return 0;
    }
    return _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED, frame, instr, exc);
}

static void
monitor_throw(PyThreadState *tstate,
              _PyInterpreterFrame *frame,
              _Py_CODEUNIT *instr)
{
    if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_THROW)) {
        return;
    }
    do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_THROW);
}

static void
monitor_reraise(PyThreadState *tstate, _PyInterpreterFrame *frame,
              _Py_CODEUNIT *instr)
{
    if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_RERAISE)) {
        return;
    }
    do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RERAISE);
}

static int
monitor_stop_iteration(PyThreadState *tstate, _PyInterpreterFrame *frame,
                       _Py_CODEUNIT *instr, PyObject *value)
{
    if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_STOP_ITERATION)) {
        return 0;
    }
    assert(!PyErr_Occurred());
    PyErr_SetObject(PyExc_StopIteration, value);
    int res = do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_STOP_ITERATION);
    if (res < 0) {
        return res;
    }
    PyErr_SetRaisedException(NULL);
    return 0;
}

static void
monitor_unwind(PyThreadState *tstate,
               _PyInterpreterFrame *frame,
               _Py_CODEUNIT *instr)
{
    if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_UNWIND)) {
        return;
    }
    do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_UNWIND);
}

static inline unsigned char *
scan_back_to_entry_start(unsigned char *p) {
    for (; (p[0]&128) == 0; p--);
    return p;
}

static inline unsigned char *
skip_to_next_entry(unsigned char *p, unsigned char *end) {
    while (p < end && ((p[0] & 128) == 0)) {
        p++;
    }
    return p;
}


#define MAX_LINEAR_SEARCH 40

static Py_NO_INLINE int
get_exception_handler(PyCodeObject *code, int index, int *level, int *handler, int *lasti)
{
    unsigned char *start = (unsigned char *)PyBytes_AS_STRING(code->co_exceptiontable);
    unsigned char *end = start + PyBytes_GET_SIZE(code->co_exceptiontable);
    /* Invariants:
     * start_table == end_table OR
     * start_table points to a legal entry and end_table points
     * beyond the table or to a legal entry that is after index.
     */
    if (end - start > MAX_LINEAR_SEARCH) {
        int offset;
        parse_varint(start, &offset);
        if (offset > index) {
            return 0;
        }
        do {
            unsigned char * mid = start + ((end-start)>>1);
            mid = scan_back_to_entry_start(mid);
            parse_varint(mid, &offset);
            if (offset > index) {
                end = mid;
            }
            else {
                start = mid;
            }

        } while (end - start > MAX_LINEAR_SEARCH);
    }
    unsigned char *scan = start;
    while (scan < end) {
        int start_offset, size;
        scan = parse_varint(scan, &start_offset);
        if (start_offset > index) {
            break;
        }
        scan = parse_varint(scan, &size);
        if (start_offset + size > index) {
            scan = parse_varint(scan, handler);
            int depth_and_lasti;
            parse_varint(scan, &depth_and_lasti);
            *level = depth_and_lasti >> 1;
            *lasti = depth_and_lasti & 1;
            return 1;
        }
        scan = skip_to_next_entry(scan, end);
    }
    return 0;
}


#ifdef Py_DEBUG
#define ASSERT_WITHIN_STACK_BOUNDS(F, L) _Py_assert_within_stack_bounds(frame, stack_pointer, (F), (L))
#else
#define ASSERT_WITHIN_STACK_BOUNDS(F, L) (void)0
#endif

/* Logic for the raise statement (too complicated for inlining).
   This *consumes* a reference count to each of its arguments. */
static int
do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause)
{
    PyObject *type = NULL, *value = NULL;

    if (exc == NULL) {
        /* Reraise */
        _PyErr_StackItem *exc_info = _PyErr_GetTopmostException(tstate);
        exc = exc_info->exc_value;
        if (Py_IsNone(exc) || exc == NULL) {
            _PyErr_SetString(tstate, PyExc_RuntimeError,
                             "No active exception to reraise");
            return 0;
        }
        Py_INCREF(exc);
        assert(PyExceptionInstance_Check(exc));
        _PyErr_SetRaisedException(tstate, exc);
        return 1;
    }

    /* We support the following forms of raise:
       raise
       raise <instance>
       raise <type> */

    if (PyExceptionClass_Check(exc)) {
        type = exc;
        value = _PyObject_CallNoArgs(exc);
        if (value == NULL)
            goto raise_error;
        if (!PyExceptionInstance_Check(value)) {
            _PyErr_Format(tstate, PyExc_TypeError,
                          "calling %R should have returned an instance of "
                          "BaseException, not %R",
                          type, Py_TYPE(value));
             goto raise_error;
        }
    }
    else if (PyExceptionInstance_Check(exc)) {
        value = exc;
        type = PyExceptionInstance_Class(exc);
        Py_INCREF(type);
    }
    else {
        /* Not something you can raise.  You get an exception
           anyway, just not what you specified :-) */
        Py_DECREF(exc);
        _PyErr_SetString(tstate, PyExc_TypeError,
                         "exceptions must derive from BaseException");
        goto raise_error;
    }

    assert(type != NULL);
    assert(value != NULL);

    if (cause) {
        PyObject *fixed_cause;
        if (PyExceptionClass_Check(cause)) {
            fixed_cause = _PyObject_CallNoArgs(cause);
            if (fixed_cause == NULL)
                goto raise_error;
            if (!PyExceptionInstance_Check(fixed_cause)) {
                _PyErr_Format(tstate, PyExc_TypeError,
                              "calling %R should have returned an instance of "
                              "BaseException, not %R",
                              cause, Py_TYPE(fixed_cause));
                Py_DECREF(fixed_cause);
                goto raise_error;
            }
            Py_DECREF(cause);
        }
        else if (PyExceptionInstance_Check(cause)) {
            fixed_cause = cause;
        }
        else if (Py_IsNone(cause)) {
            Py_DECREF(cause);
            fixed_cause = NULL;
        }
        else {
            _PyErr_SetString(tstate, PyExc_TypeError,
                             "exception causes must derive from "
                             "BaseException");
            goto raise_error;
        }
        PyException_SetCause(value, fixed_cause);
    }

    _PyErr_SetObject(tstate, type, value);
    /* _PyErr_SetObject incref's its arguments */
    Py_DECREF(value);
    Py_DECREF(type);
    return 0;

raise_error:
    Py_XDECREF(value);
    Py_XDECREF(type);
    Py_XDECREF(cause);
    return 0;
}

/* Disable unused label warnings.  They are handy for debugging, even
   if computed gotos aren't used. */

/* TBD - what about other compilers? */
#if defined(__GNUC__) || defined(__clang__)
#  pragma GCC diagnostic push
#  pragma GCC diagnostic ignored "-Wunused-label"
#elif defined(_MSC_VER) /* MS_WINDOWS */
#  pragma warning(push)
#  pragma warning(disable:4102)
#endif

Coverage Report

Created: 2026-02-26 06:53

Line	Count	Source
1		#define _PY_INTERPRETER
2
3		#include "Python.h"
4		#include "pycore_abstract.h" // _PyIndex_Check()
5		#include "pycore_audit.h" // _PySys_Audit()
6		#include "pycore_backoff.h"
7		#include "pycore_call.h" // _PyObject_CallNoArgs()
8		#include "pycore_cell.h" // PyCell_GetRef()
9		#include "pycore_ceval.h" // SPECIAL___ENTER__
10		#include "pycore_code.h"
11		#include "pycore_dict.h"
12		#include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS
13		#include "pycore_floatobject.h" // _PyFloat_ExactDealloc()
14		#include "pycore_frame.h"
15		#include "pycore_function.h"
16		#include "pycore_genobject.h" // _PyCoro_GetAwaitableIter()
17		#include "pycore_import.h" // _PyImport_IsDefaultImportFunc()
18		#include "pycore_instruments.h"
19		#include "pycore_interpframe.h" // _PyFrame_SetStackPointer()
20		#include "pycore_interpolation.h" // _PyInterpolation_Build()
21		#include "pycore_intrinsics.h"
22		#include "pycore_jit.h"
23		#include "pycore_lazyimportobject.h"
24		#include "pycore_list.h" // _PyList_GetItemRef()
25		#include "pycore_long.h" // _PyLong_GetZero()
26		#include "pycore_moduleobject.h" // PyModuleObject
27		#include "pycore_object.h" // _PyObject_GC_TRACK()
28		#include "pycore_opcode_metadata.h" // EXTRA_CASES
29		#include "pycore_opcode_utils.h" // MAKE_FUNCTION_*
30		#include "pycore_optimizer.h" // _PyUOpExecutor_Type
31		#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_*
32		#include "pycore_pyerrors.h" // _PyErr_GetRaisedException()
33		#include "pycore_pystate.h" // _PyInterpreterState_GET()
34		#include "pycore_range.h" // _PyRangeIterObject
35		#include "pycore_setobject.h" // _PySet_Update()
36		#include "pycore_sliceobject.h" // _PyBuildSlice_ConsumeRefs
37		#include "pycore_sysmodule.h" // _PySys_GetOptionalAttrString()
38		#include "pycore_template.h" // _PyTemplate_Build()
39		#include "pycore_traceback.h" // _PyTraceBack_FromFrame
40		#include "pycore_tuple.h" // _PyTuple_ITEMS()
41		#include "pycore_uop_ids.h" // Uops
42
43		#include "dictobject.h"
44		#include "frameobject.h" // _PyInterpreterFrame_GetLine
45		#include "opcode.h"
46		#include "pydtrace.h"
47		#include "setobject.h"
48		#include "pycore_stackref.h"
49
50		#include <stdbool.h> // bool
51
52		#if !defined(Py_BUILD_CORE)
53		# error "ceval.c must be build with Py_BUILD_CORE define for best performance"
54		#endif
55
56		#if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS)
57		// GH-89279: The MSVC compiler does not inline these static inline functions
58		// in PGO build in _PyEval_EvalFrameDefault(), because this function is over
59		// the limit of PGO, and that limit cannot be configured.
60		// Define them as macros to make sure that they are always inlined by the
61		// preprocessor.
62
63		#undef Py_IS_TYPE
64		#define Py_IS_TYPE(ob, type) \
65	13.5G	(_PyObject_CAST(ob)->ob_type == (type))
66
67		#undef Py_XDECREF
68		#define Py_XDECREF(arg) \
69	458M	do { \
70	458M	PyObject *xop = _PyObject_CAST(arg); \
71	458M	if (xop != NULL) { \
72	325M	Py_DECREF(xop); \
73	325M	} \
74	458M	} while (0)
75
76		#ifndef Py_GIL_DISABLED
77
78		#undef Py_DECREF
79		#define Py_DECREF(arg) \
80	513M	do { \
81	513M	PyObject *op = _PyObject_CAST(arg); \
82	513M	if (_Py_IsImmortal(op)) { \
83	196M	_Py_DECREF_IMMORTAL_STAT_INC(); \
84	196M	break; \
85	196M	} \
86	513M	_Py_DECREF_STAT_INC(); \
87	317M	if (--op->ob_refcnt == 0) { \
88	135M	_PyReftracerTrack(op, PyRefTracer_DESTROY); \
89	135M	destructor dealloc = Py_TYPE(op)->tp_dealloc; \
90	135M	(*dealloc)(op); \
91	135M	} \
92	317M	} while (0)
93
94		#undef _Py_DECREF_SPECIALIZED
95		#define _Py_DECREF_SPECIALIZED(arg, dealloc) \
96	5.56M	do { \
97	5.56M	PyObject *op = _PyObject_CAST(arg); \
98	5.56M	if (_Py_IsImmortal(op)) { \
99	4.72M	_Py_DECREF_IMMORTAL_STAT_INC(); \
100	4.72M	break; \
101	4.72M	} \
102	5.56M	_Py_DECREF_STAT_INC(); \
103	835k	if (--op->ob_refcnt == 0) { \
104	609k	_PyReftracerTrack(op, PyRefTracer_DESTROY); \
105	609k	destructor d = (destructor)(dealloc); \
106	609k	d(op); \
107	609k	} \
108	835k	} while (0)
109
110		#else // Py_GIL_DISABLED
111
112		#undef Py_DECREF
113		#define Py_DECREF(arg) \
114		do { \
115		PyObject *op = _PyObject_CAST(arg); \
116		uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); \
117		if (local == _Py_IMMORTAL_REFCNT_LOCAL) { \
118		_Py_DECREF_IMMORTAL_STAT_INC(); \
119		break; \
120		} \
121		_Py_DECREF_STAT_INC(); \
122		if (_Py_IsOwnedByCurrentThread(op)) { \
123		local--; \
124		_Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); \
125		if (local == 0) { \
126		_Py_MergeZeroLocalRefcount(op); \
127		} \
128		} \
129		else { \
130		_Py_DecRefShared(op); \
131		} \
132		} while (0)
133
134		#undef _Py_DECREF_SPECIALIZED
135		#define _Py_DECREF_SPECIALIZED(arg, dealloc) Py_DECREF(arg)
136
137		#endif
138		#endif
139
140		static void
141		check_invalid_reentrancy(void)
142	279M	{
143		#if defined(Py_DEBUG) && defined(Py_GIL_DISABLED)
144		// In the free-threaded build, the interpreter must not be re-entered if
145		// the world-is-stopped. If so, that's a bug somewhere (quite likely in
146		// the painfully complex typeobject code).
147		PyInterpreterState *interp = _PyInterpreterState_GET();
148		assert(!interp->stoptheworld.world_stopped);
149		#endif
150	279M	}
151
152
153		#ifdef Py_DEBUG
154		static void
155		dump_item(_PyStackRef item)
156		{
157		if (PyStackRef_IsNull(item)) {
158		printf("<NULL>");
159		return;
160		}
161		if (PyStackRef_IsMalformed(item)) {
162		printf("<INVALID>");
163		return;
164		}
165		if (PyStackRef_IsTaggedInt(item)) {
166		printf("%" PRId64, (int64_t)PyStackRef_UntagInt(item));
167		return;
168		}
169		PyObject *obj = PyStackRef_AsPyObjectBorrow(item);
170		if (obj == NULL) {
171		printf("<nil>");
172		return;
173		}
174		// Don't call __repr__(), it might recurse into the interpreter.
175		printf("<%s at %p>", Py_TYPE(obj)->tp_name, (void *)obj);
176		}
177
178		static void
179		dump_stack(_PyInterpreterFrame frame, _PyStackRef stack_pointer)
180		{
181		_PyFrame_SetStackPointer(frame, stack_pointer);
182		_PyStackRef *locals_base = _PyFrame_GetLocalsArray(frame);
183		_PyStackRef *stack_base = _PyFrame_Stackbase(frame);
184		PyObject *exc = PyErr_GetRaisedException();
185		printf(" locals=[");
186		for (_PyStackRef *ptr = locals_base; ptr < stack_base; ptr++) {
187		if (ptr != locals_base) {
188		printf(", ");
189		}
190		dump_item(*ptr);
191		}
192		printf("]\n");
193		if (stack_pointer < stack_base) {
194		printf(" stack=%d\n", (int)(stack_pointer-stack_base));
195		}
196		else {
197		printf(" stack=[");
198		for (_PyStackRef *ptr = stack_base; ptr < stack_pointer; ptr++) {
199		if (ptr != stack_base) {
200		printf(", ");
201		}
202		dump_item(*ptr);
203		}
204		printf("]\n");
205		}
206		fflush(stdout);
207		PyErr_SetRaisedException(exc);
208		_PyFrame_GetStackPointer(frame);
209		}
210
211		#if defined(_Py_TIER2) && !defined(_Py_JIT) && defined(Py_DEBUG)
212		static void
213		dump_cache_item(_PyStackRef cache, int position, int depth)
214		{
215		if (position < depth) {
216		dump_item(cache);
217		}
218		else {
219		printf("---");
220		}
221		}
222		#endif
223
224		static void
225		lltrace_instruction(_PyInterpreterFrame *frame,
226		_PyStackRef *stack_pointer,
227		_Py_CODEUNIT *next_instr,
228		int opcode,
229		int oparg)
230		{
231		int offset = 0;
232		if (frame->owner < FRAME_OWNED_BY_INTERPRETER) {
233		dump_stack(frame, stack_pointer);
234		offset = (int)(next_instr - _PyFrame_GetBytecode(frame));
235		}
236		const char *opname = _PyOpcode_OpName[opcode];
237		assert(opname != NULL);
238		if (OPCODE_HAS_ARG((int)_PyOpcode_Deopt[opcode])) {
239		printf("%d: %s %d\n", offset * 2, opname, oparg);
240		}
241		else {
242		printf("%d: %s\n", offset * 2, opname);
243		}
244		fflush(stdout);
245		}
246
247		static void
248		lltrace_resume_frame(_PyInterpreterFrame *frame)
249		{
250		PyObject *fobj = PyStackRef_AsPyObjectBorrow(frame->f_funcobj);
251		if (!PyStackRef_CodeCheck(frame->f_executable) \|\|
252		fobj == NULL \|\|
253		!PyFunction_Check(fobj)
254		) {
255		printf("\nResuming frame.\n");
256		return;
257		}
258		PyFunctionObject f = (PyFunctionObject )fobj;
259		PyObject *exc = PyErr_GetRaisedException();
260		PyObject *name = f->func_qualname;
261		if (name == NULL) {
262		name = f->func_name;
263		}
264		printf("\nResuming frame");
265		if (name) {
266		printf(" for ");
267		if (PyObject_Print(name, stdout, 0) < 0) {
268		PyErr_Clear();
269		}
270		}
271		if (f->func_module) {
272		printf(" in module ");
273		if (PyObject_Print(f->func_module, stdout, 0) < 0) {
274		PyErr_Clear();
275		}
276		}
277		printf("\n");
278		fflush(stdout);
279		PyErr_SetRaisedException(exc);
280		}
281
282		static int
283		maybe_lltrace_resume_frame(_PyInterpreterFrame frame, PyObject globals)
284		{
285		if (globals == NULL) {
286		return 0;
287		}
288		if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) {
289		return 0;
290		}
291		int r = PyDict_Contains(globals, &_Py_ID(__lltrace__));
292		if (r < 0) {
293		PyErr_Clear();
294		return 0;
295		}
296		int lltrace = r * 5; // Levels 1-4 only trace uops
297		if (!lltrace) {
298		// Can also be controlled by environment variable
299		char *python_lltrace = Py_GETENV("PYTHON_LLTRACE");
300		if (python_lltrace != NULL && *python_lltrace >= '0') {
301		lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that
302		}
303		}
304		if (lltrace >= 5) {
305		lltrace_resume_frame(frame);
306		}
307		return lltrace;
308		}
309
310		#endif
311
312		static void monitor_reraise(PyThreadState *tstate,
313		_PyInterpreterFrame *frame,
314		_Py_CODEUNIT *instr);
315		static int monitor_stop_iteration(PyThreadState *tstate,
316		_PyInterpreterFrame *frame,
317		_Py_CODEUNIT *instr,
318		PyObject *value);
319		static void monitor_unwind(PyThreadState *tstate,
320		_PyInterpreterFrame *frame,
321		_Py_CODEUNIT *instr);
322		static int monitor_handled(PyThreadState *tstate,
323		_PyInterpreterFrame *frame,
324		_Py_CODEUNIT instr, PyObject exc);
325		static void monitor_throw(PyThreadState *tstate,
326		_PyInterpreterFrame *frame,
327		_Py_CODEUNIT *instr);
328
329		static int get_exception_handler(PyCodeObject , int, int, int, int);
330
331		#ifdef HAVE_ERRNO_H
332		#include <errno.h>
333		#endif
334
335		typedef struct {
336		_PyInterpreterFrame frame;
337		_PyStackRef stack[1];
338		} _PyEntryFrame;
339
340		static int
341		do_monitor_exc(PyThreadState tstate, _PyInterpreterFrame frame,
342		_Py_CODEUNIT *instr, int event)
343	0	{
344	0	assert(event < _PY_MONITORING_UNGROUPED_EVENTS);
345	0	if (_PyFrame_GetCode(frame)->co_flags & CO_NO_MONITORING_EVENTS) {
346	0	return 0;
347	0	}
348	0	PyObject *exc = PyErr_GetRaisedException();
349	0	assert(exc != NULL);
350	0	int err = _Py_call_instrumentation_arg(tstate, event, frame, instr, exc);
351	0	if (err == 0) {
352	0	PyErr_SetRaisedException(exc);
353	0	}
354	0	else {
355	0	assert(PyErr_Occurred());
356	0	Py_DECREF(exc);
357	0	}
358	0	return err;
359	0	}
360
361		static inline bool
362		no_tools_for_global_event(PyThreadState *tstate, int event)
363	138M	{
364	138M	return tstate->interp->monitors.tools[event] == 0;
365	138M	}
366
367		static inline bool
368		no_tools_for_local_event(PyThreadState tstate, _PyInterpreterFrame frame, int event)
369	0	{
370	0	assert(event < _PY_MONITORING_LOCAL_EVENTS);
371	0	_PyCoMonitoringData *data = _PyFrame_GetCode(frame)->_co_monitoring;
372	0	if (data) {
373	0	return data->active_monitors.tools[event] == 0;
374	0	}
375	0	else {
376	0	return no_tools_for_global_event(tstate, event);
377	0	}
378	0	}
379
380		static int
381		monitor_handled(PyThreadState *tstate,
382		_PyInterpreterFrame *frame,
383		_Py_CODEUNIT instr, PyObject exc)
384	29.8M	{
385	29.8M	if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED)) {
386	29.8M	return 0;
387	29.8M	}
388	0	return _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED, frame, instr, exc);
389	29.8M	}
390
391		static void
392		monitor_throw(PyThreadState *tstate,
393		_PyInterpreterFrame *frame,
394		_Py_CODEUNIT *instr)
395	1.50k	{
396	1.50k	if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_THROW)) {
397	1.50k	return;
398	1.50k	}
399	0	do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_THROW);
400	0	}
401
402		static void
403		monitor_reraise(PyThreadState tstate, _PyInterpreterFrame frame,
404		_Py_CODEUNIT *instr)
405	4.89M	{
406	4.89M	if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_RERAISE)) {
407	4.89M	return;
408	4.89M	}
409	0	do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RERAISE);
410	0	}
411
412		static int
413		monitor_stop_iteration(PyThreadState tstate, _PyInterpreterFrame frame,
414		_Py_CODEUNIT instr, PyObject value)
415	0	{
416	0	if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_STOP_ITERATION)) {
417	0	return 0;
418	0	}
419	0	assert(!PyErr_Occurred());
420	0	PyErr_SetObject(PyExc_StopIteration, value);
421	0	int res = do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_STOP_ITERATION);
422	0	if (res < 0) {
423	0	return res;
424	0	}
425	0	PyErr_SetRaisedException(NULL);
426	0	return 0;
427	0	}
428
429		static void
430		monitor_unwind(PyThreadState *tstate,
431		_PyInterpreterFrame *frame,
432		_Py_CODEUNIT *instr)
433	33.8M	{
434	33.8M	if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_UNWIND)) {
435	33.8M	return;
436	33.8M	}
437	0	do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_UNWIND);
438	0	}
439
440		static inline unsigned char *
441	19.9M	scan_back_to_entry_start(unsigned char *p) {
442	54.1M	for (; (p[0]&128) == 0; p--);
443	19.9M	return p;
444	19.9M	}
445
446		static inline unsigned char *
447	38.2M	skip_to_next_entry(unsigned char p, unsigned char end) {
448	153M	while (p < end && ((p[0] & 128) == 0)) {
449	114M	p++;
450	114M	}
451	38.2M	return p;
452	38.2M	}
453
454
455	83.5M	#define MAX_LINEAR_SEARCH 40
456
457		static Py_NO_INLINE int
458		get_exception_handler(PyCodeObject code, int index, int level, int handler, int lasti)
459	63.6M	{
460	63.6M	unsigned char start = (unsigned char )PyBytes_AS_STRING(code->co_exceptiontable);
461	63.6M	unsigned char *end = start + PyBytes_GET_SIZE(code->co_exceptiontable);
462		/* Invariants:
463		* start_table == end_table OR
464		* start_table points to a legal entry and end_table points
465		* beyond the table or to a legal entry that is after index.
466		*/
467	63.6M	if (end - start > MAX_LINEAR_SEARCH) {
468	7.41M	int offset;
469	7.41M	parse_varint(start, &offset);
470	7.41M	if (offset > index) {
471	104k	return 0;
472	104k	}
473	19.9M	do {
474	19.9M	unsigned char * mid = start + ((end-start)>>1);
475	19.9M	mid = scan_back_to_entry_start(mid);
476	19.9M	parse_varint(mid, &offset);
477	19.9M	if (offset > index) {
478	19.3M	end = mid;
479	19.3M	}
480	554k	else {
481	554k	start = mid;
482	554k	}
483
484	19.9M	} while (end - start > MAX_LINEAR_SEARCH);
485	7.31M	}
486	63.5M	unsigned char *scan = start;
487	101M	while (scan < end) {
488	74.5M	int start_offset, size;
489	74.5M	scan = parse_varint(scan, &start_offset);
490	74.5M	if (start_offset > index) {
491	6.44M	break;
492	6.44M	}
493	68.1M	scan = parse_varint(scan, &size);
494	68.1M	if (start_offset + size > index) {
495	29.8M	scan = parse_varint(scan, handler);
496	29.8M	int depth_and_lasti;
497	29.8M	parse_varint(scan, &depth_and_lasti);
498	29.8M	*level = depth_and_lasti >> 1;
499	29.8M	*lasti = depth_and_lasti & 1;
500	29.8M	return 1;
501	29.8M	}
502	38.2M	scan = skip_to_next_entry(scan, end);
503	38.2M	}
504	33.7M	return 0;
505	63.5M	}
506
507
508		#ifdef Py_DEBUG
509		#define ASSERT_WITHIN_STACK_BOUNDS(F, L) _Py_assert_within_stack_bounds(frame, stack_pointer, (F), (L))
510		#else
511	59.4G	#define ASSERT_WITHIN_STACK_BOUNDS(F, L) (void)0
512		#endif
513
514		/* Logic for the raise statement (too complicated for inlining).
515		This consumes a reference count to each of its arguments. */
516		static int
517		do_raise(PyThreadState tstate, PyObject exc, PyObject *cause)
518	17.1M	{
519	17.1M	PyObject type = NULL, value = NULL;
520
521	17.1M	if (exc == NULL) {
522		/* Reraise */
523	15.8k	_PyErr_StackItem *exc_info = _PyErr_GetTopmostException(tstate);
524	15.8k	exc = exc_info->exc_value;
525	15.8k	if (Py_IsNone(exc) \|\| exc == NULL) {
526	0	_PyErr_SetString(tstate, PyExc_RuntimeError,
527	0	"No active exception to reraise");
528	0	return 0;
529	0	}
530	15.8k	Py_INCREF(exc);
531	15.8k	assert(PyExceptionInstance_Check(exc));
532	15.8k	_PyErr_SetRaisedException(tstate, exc);
533	15.8k	return 1;
534	15.8k	}
535
536		/* We support the following forms of raise:
537		raise
538		raise <instance>
539		raise <type> */
540
541	17.1M	if (PyExceptionClass_Check(exc)) {
542	10.6M	type = exc;
543	10.6M	value = _PyObject_CallNoArgs(exc);
544	10.6M	if (value == NULL)
545	0	goto raise_error;
546	10.6M	if (!PyExceptionInstance_Check(value)) {
547	0	_PyErr_Format(tstate, PyExc_TypeError,
548	0	"calling %R should have returned an instance of "
549	0	"BaseException, not %R",
550	0	type, Py_TYPE(value));
551	0	goto raise_error;
552	0	}
553	10.6M	}
554	6.55M	else if (PyExceptionInstance_Check(exc)) {
555	6.55M	value = exc;
556	6.55M	type = PyExceptionInstance_Class(exc);
557	6.55M	Py_INCREF(type);
558	6.55M	}
559	0	else {
560		/* Not something you can raise. You get an exception
561		anyway, just not what you specified :-) */
562	0	Py_DECREF(exc);
563	0	_PyErr_SetString(tstate, PyExc_TypeError,
564	0	"exceptions must derive from BaseException");
565	0	goto raise_error;
566	0	}
567
568	17.1M	assert(type != NULL);
569	17.1M	assert(value != NULL);
570
571	17.1M	if (cause) {
572	13.0k	PyObject *fixed_cause;
573	13.0k	if (PyExceptionClass_Check(cause)) {
574	0	fixed_cause = _PyObject_CallNoArgs(cause);
575	0	if (fixed_cause == NULL)
576	0	goto raise_error;
577	0	if (!PyExceptionInstance_Check(fixed_cause)) {
578	0	_PyErr_Format(tstate, PyExc_TypeError,
579	0	"calling %R should have returned an instance of "
580	0	"BaseException, not %R",
581	0	cause, Py_TYPE(fixed_cause));
582	0	Py_DECREF(fixed_cause);
583	0	goto raise_error;
584	0	}
585	0	Py_DECREF(cause);
586	0	}
587	13.0k	else if (PyExceptionInstance_Check(cause)) {
588	6.02k	fixed_cause = cause;
589	6.02k	}
590	7.03k	else if (Py_IsNone(cause)) {
591	7.03k	Py_DECREF(cause);
592	7.03k	fixed_cause = NULL;
593	7.03k	}
594	0	else {
595	0	_PyErr_SetString(tstate, PyExc_TypeError,
596	0	"exception causes must derive from "
597	0	"BaseException");
598	0	goto raise_error;
599	0	}
600	13.0k	PyException_SetCause(value, fixed_cause);
601	13.0k	}
602
603	17.1M	_PyErr_SetObject(tstate, type, value);
604		/* _PyErr_SetObject incref's its arguments */
605	17.1M	Py_DECREF(value);
606	17.1M	Py_DECREF(type);
607	17.1M	return 0;
608
609	0	raise_error:
610	0	Py_XDECREF(value);
611	0	Py_XDECREF(type);
612		Py_XDECREF(cause);
613	0	return 0;
614	17.1M	}
615
616		/* Disable unused label warnings. They are handy for debugging, even
617		if computed gotos aren't used. */
618
619		/* TBD - what about other compilers? */
620		#if defined(__GNUC__) \|\| defined(__clang__)
621		# pragma GCC diagnostic push
622		# pragma GCC diagnostic ignored "-Wunused-label"
623		#elif defined(_MSC_VER) /* MS_WINDOWS */
624		# pragma warning(push)
625		# pragma warning(disable:4102)
626		#endif