/src/cpython/Python/ceval.h

Source
#define _PY_INTERPRETER

#include "Python.h"
#include "pycore_abstract.h"      // _PyIndex_Check()
#include "pycore_audit.h"         // _PySys_Audit()
#include "pycore_backoff.h"
#include "pycore_call.h"          // _PyObject_CallNoArgs()
#include "pycore_cell.h"          // PyCell_GetRef()
#include "pycore_ceval.h"         // SPECIAL___ENTER__
#include "pycore_code.h"
#include "pycore_dict.h"
#include "pycore_emscripten_signal.h"  // _Py_CHECK_EMSCRIPTEN_SIGNALS
#include "pycore_floatobject.h"   // _PyFloat_ExactDealloc()
#include "pycore_frame.h"
#include "pycore_function.h"
#include "pycore_genobject.h"     // _PyCoro_GetAwaitableIter()
#include "pycore_import.h"        // _PyImport_IsDefaultImportFunc()
#include "pycore_instruments.h"
#include "pycore_interpframe.h"   // _PyFrame_SetStackPointer()
#include "pycore_interpolation.h" // _PyInterpolation_Build()
#include "pycore_intrinsics.h"
#include "pycore_jit.h"
#include "pycore_list.h"          // _PyList_GetItemRef()
#include "pycore_long.h"          // _PyLong_GetZero()
#include "pycore_moduleobject.h"  // PyModuleObject
#include "pycore_object.h"        // _PyObject_GC_TRACK()
#include "pycore_opcode_metadata.h" // EXTRA_CASES
#include "pycore_opcode_utils.h"  // MAKE_FUNCTION_*
#include "pycore_optimizer.h"     // _PyUOpExecutor_Type
#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_*
#include "pycore_pyerrors.h"      // _PyErr_GetRaisedException()
#include "pycore_pystate.h"       // _PyInterpreterState_GET()
#include "pycore_range.h"         // _PyRangeIterObject
#include "pycore_setobject.h"     // _PySet_Update()
#include "pycore_sliceobject.h"   // _PyBuildSlice_ConsumeRefs
#include "pycore_sysmodule.h"     // _PySys_GetOptionalAttrString()
#include "pycore_template.h"      // _PyTemplate_Build()
#include "pycore_traceback.h"     // _PyTraceBack_FromFrame
#include "pycore_tuple.h"         // _PyTuple_ITEMS()
#include "pycore_uop_ids.h"       // Uops

#include "dictobject.h"
#include "frameobject.h"          // _PyInterpreterFrame_GetLine
#include "opcode.h"
#include "pydtrace.h"
#include "setobject.h"
#include "pycore_stackref.h"

#include <stdbool.h>              // bool

#if !defined(Py_BUILD_CORE)
#  error "ceval.c must be build with Py_BUILD_CORE define for best performance"
#endif

#if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS)
// GH-89279: The MSVC compiler does not inline these static inline functions
// in PGO build in _PyEval_EvalFrameDefault(), because this function is over
// the limit of PGO, and that limit cannot be configured.
// Define them as macros to make sure that they are always inlined by the
// preprocessor.

#undef Py_IS_TYPE
#define Py_IS_TYPE(ob, type) \
    (_PyObject_CAST(ob)->ob_type == (type))

#undef Py_XDECREF
#define Py_XDECREF(arg) \
    do { \
        PyObject *xop = _PyObject_CAST(arg); \
        if (xop != NULL) { \
            Py_DECREF(xop); \
        } \
    } while (0)

#ifndef Py_GIL_DISABLED

#undef Py_DECREF
#define Py_DECREF(arg) \
    do { \
        PyObject *op = _PyObject_CAST(arg); \
        if (_Py_IsImmortal(op)) { \
            _Py_DECREF_IMMORTAL_STAT_INC(); \
            break; \
        } \
        _Py_DECREF_STAT_INC(); \
        if (--op->ob_refcnt == 0) { \
            _PyReftracerTrack(op, PyRefTracer_DESTROY); \
            destructor dealloc = Py_TYPE(op)->tp_dealloc; \
            (*dealloc)(op); \
        } \
    } while (0)

#undef _Py_DECREF_SPECIALIZED
#define _Py_DECREF_SPECIALIZED(arg, dealloc) \
    do { \
        PyObject *op = _PyObject_CAST(arg); \
        if (_Py_IsImmortal(op)) { \
            _Py_DECREF_IMMORTAL_STAT_INC(); \
            break; \
        } \
        _Py_DECREF_STAT_INC(); \
        if (--op->ob_refcnt == 0) { \
            _PyReftracerTrack(op, PyRefTracer_DESTROY); \
            destructor d = (destructor)(dealloc); \
            d(op); \
        } \
    } while (0)

#else // Py_GIL_DISABLED

#undef Py_DECREF
#define Py_DECREF(arg) \
    do { \
        PyObject *op = _PyObject_CAST(arg); \
        uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); \
        if (local == _Py_IMMORTAL_REFCNT_LOCAL) { \
            _Py_DECREF_IMMORTAL_STAT_INC(); \
            break; \
        } \
        _Py_DECREF_STAT_INC(); \
        if (_Py_IsOwnedByCurrentThread(op)) { \
            local--; \
            _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); \
            if (local == 0) { \
                _Py_MergeZeroLocalRefcount(op); \
            } \
        } \
        else { \
            _Py_DecRefShared(op); \
        } \
    } while (0)

#undef _Py_DECREF_SPECIALIZED
#define _Py_DECREF_SPECIALIZED(arg, dealloc) Py_DECREF(arg)

#endif
#endif

static void
check_invalid_reentrancy(void)
{
#if defined(Py_DEBUG) && defined(Py_GIL_DISABLED)
    // In the free-threaded build, the interpreter must not be re-entered if
    // the world-is-stopped.  If so, that's a bug somewhere (quite likely in
    // the painfully complex typeobject code).
    PyInterpreterState *interp = _PyInterpreterState_GET();
    assert(!interp->stoptheworld.world_stopped);
#endif
}


#ifdef Py_DEBUG
static void
dump_item(_PyStackRef item)
{
    if (PyStackRef_IsNull(item)) {
        printf("<NULL>");
        return;
    }
    if (PyStackRef_IsMalformed(item)) {
        printf("<INVALID>");
        return;
    }
    if (PyStackRef_IsTaggedInt(item)) {
        printf("%" PRId64, (int64_t)PyStackRef_UntagInt(item));
        return;
    }
    PyObject *obj = PyStackRef_AsPyObjectBorrow(item);
    if (obj == NULL) {
        printf("<nil>");
        return;
    }
    // Don't call __repr__(), it might recurse into the interpreter.
    printf("<%s at %p>", Py_TYPE(obj)->tp_name, (void *)obj);
}

static void
dump_stack(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer)
{
    _PyFrame_SetStackPointer(frame, stack_pointer);
    _PyStackRef *locals_base = _PyFrame_GetLocalsArray(frame);
    _PyStackRef *stack_base = _PyFrame_Stackbase(frame);
    PyObject *exc = PyErr_GetRaisedException();
    printf("    locals=[");
    for (_PyStackRef *ptr = locals_base; ptr < stack_base; ptr++) {
        if (ptr != locals_base) {
            printf(", ");
        }
        dump_item(*ptr);
    }
    printf("]\n");
    if (stack_pointer < stack_base) {
        printf("    stack=%d\n", (int)(stack_pointer-stack_base));
    }
    else {
        printf("    stack=[");
        for (_PyStackRef *ptr = stack_base; ptr < stack_pointer; ptr++) {
            if (ptr != stack_base) {
                printf(", ");
            }
            dump_item(*ptr);
        }
        printf("]\n");
    }
    fflush(stdout);
    PyErr_SetRaisedException(exc);
    _PyFrame_GetStackPointer(frame);
}

#if defined(_Py_TIER2) && !defined(_Py_JIT) && defined(Py_DEBUG)
static void
dump_cache_item(_PyStackRef cache, int position, int depth)
{
    if (position < depth) {
        dump_item(cache);
    }
    else {
        printf("---");
    }
}
#endif

static void
lltrace_instruction(_PyInterpreterFrame *frame,
                    _PyStackRef *stack_pointer,
                    _Py_CODEUNIT *next_instr,
                    int opcode,
                    int oparg)
{
    int offset = 0;
    if (frame->owner < FRAME_OWNED_BY_INTERPRETER) {
        dump_stack(frame, stack_pointer);
        offset = (int)(next_instr - _PyFrame_GetBytecode(frame));
    }
    const char *opname = _PyOpcode_OpName[opcode];
    assert(opname != NULL);
    if (OPCODE_HAS_ARG((int)_PyOpcode_Deopt[opcode])) {
        printf("%d: %s %d\n", offset * 2, opname, oparg);
    }
    else {
        printf("%d: %s\n", offset * 2, opname);
    }
    fflush(stdout);
}

static void
lltrace_resume_frame(_PyInterpreterFrame *frame)
{
    PyObject *fobj = PyStackRef_AsPyObjectBorrow(frame->f_funcobj);
    if (!PyStackRef_CodeCheck(frame->f_executable) ||
        fobj == NULL ||
        !PyFunction_Check(fobj)
    ) {
        printf("\nResuming frame.\n");
        return;
    }
    PyFunctionObject *f = (PyFunctionObject *)fobj;
    PyObject *exc = PyErr_GetRaisedException();
    PyObject *name = f->func_qualname;
    if (name == NULL) {
        name = f->func_name;
    }
    printf("\nResuming frame");
    if (name) {
        printf(" for ");
        if (PyObject_Print(name, stdout, 0) < 0) {
            PyErr_Clear();
        }
    }
    if (f->func_module) {
        printf(" in module ");
        if (PyObject_Print(f->func_module, stdout, 0) < 0) {
            PyErr_Clear();
        }
    }
    printf("\n");
    fflush(stdout);
    PyErr_SetRaisedException(exc);
}

static int
maybe_lltrace_resume_frame(_PyInterpreterFrame *frame, PyObject *globals)
{
    if (globals == NULL) {
        return 0;
    }
    if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) {
        return 0;
    }
    int r = PyDict_Contains(globals, &_Py_ID(__lltrace__));
    if (r < 0) {
        PyErr_Clear();
        return 0;
    }
    int lltrace = r * 5;  // Levels 1-4 only trace uops
    if (!lltrace) {
        // Can also be controlled by environment variable
        char *python_lltrace = Py_GETENV("PYTHON_LLTRACE");
        if (python_lltrace != NULL && *python_lltrace >= '0') {
            lltrace = *python_lltrace - '0';  // TODO: Parse an int and all that
        }
    }
    if (lltrace >= 5) {
        lltrace_resume_frame(frame);
    }
    return lltrace;
}

#endif

static void monitor_reraise(PyThreadState *tstate,
                 _PyInterpreterFrame *frame,
                 _Py_CODEUNIT *instr);
static int monitor_stop_iteration(PyThreadState *tstate,
                 _PyInterpreterFrame *frame,
                 _Py_CODEUNIT *instr,
                 PyObject *value);
static void monitor_unwind(PyThreadState *tstate,
                 _PyInterpreterFrame *frame,
                 _Py_CODEUNIT *instr);
static int monitor_handled(PyThreadState *tstate,
                 _PyInterpreterFrame *frame,
                 _Py_CODEUNIT *instr, PyObject *exc);
static void monitor_throw(PyThreadState *tstate,
                 _PyInterpreterFrame *frame,
                 _Py_CODEUNIT *instr);

static int get_exception_handler(PyCodeObject *, int, int*, int*, int*);

#ifdef HAVE_ERRNO_H
#include <errno.h>
#endif

typedef struct {
    _PyInterpreterFrame frame;
    _PyStackRef stack[1];
} _PyEntryFrame;

static int
do_monitor_exc(PyThreadState *tstate, _PyInterpreterFrame *frame,
               _Py_CODEUNIT *instr, int event)
{
    assert(event < _PY_MONITORING_UNGROUPED_EVENTS);
    if (_PyFrame_GetCode(frame)->co_flags & CO_NO_MONITORING_EVENTS) {
        return 0;
    }
    PyObject *exc = PyErr_GetRaisedException();
    assert(exc != NULL);
    int err = _Py_call_instrumentation_arg(tstate, event, frame, instr, exc);
    if (err == 0) {
        PyErr_SetRaisedException(exc);
    }
    else {
        assert(PyErr_Occurred());
        Py_DECREF(exc);
    }
    return err;
}

static inline bool
no_tools_for_global_event(PyThreadState *tstate, int event)
{
    return tstate->interp->monitors.tools[event] == 0;
}

static inline bool
no_tools_for_local_event(PyThreadState *tstate, _PyInterpreterFrame *frame, int event)
{
    assert(event < _PY_MONITORING_LOCAL_EVENTS);
    _PyCoMonitoringData *data = _PyFrame_GetCode(frame)->_co_monitoring;
    if (data) {
        return data->active_monitors.tools[event] == 0;
    }
    else {
        return no_tools_for_global_event(tstate, event);
    }
}

static int
monitor_handled(PyThreadState *tstate,
                _PyInterpreterFrame *frame,
                _Py_CODEUNIT *instr, PyObject *exc)
{
    if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED)) {
        return 0;
    }
    return _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED, frame, instr, exc);
}

static void
monitor_throw(PyThreadState *tstate,
              _PyInterpreterFrame *frame,
              _Py_CODEUNIT *instr)
{
    if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_THROW)) {
        return;
    }
    do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_THROW);
}

static void
monitor_reraise(PyThreadState *tstate, _PyInterpreterFrame *frame,
              _Py_CODEUNIT *instr)
{
    if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_RERAISE)) {
        return;
    }
    do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RERAISE);
}

static int
monitor_stop_iteration(PyThreadState *tstate, _PyInterpreterFrame *frame,
                       _Py_CODEUNIT *instr, PyObject *value)
{
    if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_STOP_ITERATION)) {
        return 0;
    }
    assert(!PyErr_Occurred());
    PyErr_SetObject(PyExc_StopIteration, value);
    int res = do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_STOP_ITERATION);
    if (res < 0) {
        return res;
    }
    PyErr_SetRaisedException(NULL);
    return 0;
}

static void
monitor_unwind(PyThreadState *tstate,
               _PyInterpreterFrame *frame,
               _Py_CODEUNIT *instr)
{
    if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_UNWIND)) {
        return;
    }
    do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_UNWIND);
}

static inline unsigned char *
scan_back_to_entry_start(unsigned char *p) {
    for (; (p[0]&128) == 0; p--);
    return p;
}

static inline unsigned char *
skip_to_next_entry(unsigned char *p, unsigned char *end) {
    while (p < end && ((p[0] & 128) == 0)) {
        p++;
    }
    return p;
}


#define MAX_LINEAR_SEARCH 40

static Py_NO_INLINE int
get_exception_handler(PyCodeObject *code, int index, int *level, int *handler, int *lasti)
{
    unsigned char *start = (unsigned char *)PyBytes_AS_STRING(code->co_exceptiontable);
    unsigned char *end = start + PyBytes_GET_SIZE(code->co_exceptiontable);
    /* Invariants:
     * start_table == end_table OR
     * start_table points to a legal entry and end_table points
     * beyond the table or to a legal entry that is after index.
     */
    if (end - start > MAX_LINEAR_SEARCH) {
        int offset;
        parse_varint(start, &offset);
        if (offset > index) {
            return 0;
        }
        do {
            unsigned char * mid = start + ((end-start)>>1);
            mid = scan_back_to_entry_start(mid);
            parse_varint(mid, &offset);
            if (offset > index) {
                end = mid;
            }
            else {
                start = mid;
            }

        } while (end - start > MAX_LINEAR_SEARCH);
    }
    unsigned char *scan = start;
    while (scan < end) {
        int start_offset, size;
        scan = parse_varint(scan, &start_offset);
        if (start_offset > index) {
            break;
        }
        scan = parse_varint(scan, &size);
        if (start_offset + size > index) {
            scan = parse_varint(scan, handler);
            int depth_and_lasti;
            parse_varint(scan, &depth_and_lasti);
            *level = depth_and_lasti >> 1;
            *lasti = depth_and_lasti & 1;
            return 1;
        }
        scan = skip_to_next_entry(scan, end);
    }
    return 0;
}


#ifdef Py_DEBUG
#define ASSERT_WITHIN_STACK_BOUNDS(F, L) _Py_assert_within_stack_bounds(frame, stack_pointer, (F), (L))
#else
#define ASSERT_WITHIN_STACK_BOUNDS(F, L) (void)0
#endif

/* Logic for the raise statement (too complicated for inlining).
   This *consumes* a reference count to each of its arguments. */
static int
do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause)
{
    PyObject *type = NULL, *value = NULL;

    if (exc == NULL) {
        /* Reraise */
        _PyErr_StackItem *exc_info = _PyErr_GetTopmostException(tstate);
        exc = exc_info->exc_value;
        if (Py_IsNone(exc) || exc == NULL) {
            _PyErr_SetString(tstate, PyExc_RuntimeError,
                             "No active exception to reraise");
            return 0;
        }
        Py_INCREF(exc);
        assert(PyExceptionInstance_Check(exc));
        _PyErr_SetRaisedException(tstate, exc);
        return 1;
    }

    /* We support the following forms of raise:
       raise
       raise <instance>
       raise <type> */

    if (PyExceptionClass_Check(exc)) {
        type = exc;
        value = _PyObject_CallNoArgs(exc);
        if (value == NULL)
            goto raise_error;
        if (!PyExceptionInstance_Check(value)) {
            _PyErr_Format(tstate, PyExc_TypeError,
                          "calling %R should have returned an instance of "
                          "BaseException, not %R",
                          type, Py_TYPE(value));
             goto raise_error;
        }
    }
    else if (PyExceptionInstance_Check(exc)) {
        value = exc;
        type = PyExceptionInstance_Class(exc);
        Py_INCREF(type);
    }
    else {
        /* Not something you can raise.  You get an exception
           anyway, just not what you specified :-) */
        Py_DECREF(exc);
        _PyErr_SetString(tstate, PyExc_TypeError,
                         "exceptions must derive from BaseException");
        goto raise_error;
    }

    assert(type != NULL);
    assert(value != NULL);

    if (cause) {
        PyObject *fixed_cause;
        if (PyExceptionClass_Check(cause)) {
            fixed_cause = _PyObject_CallNoArgs(cause);
            if (fixed_cause == NULL)
                goto raise_error;
            if (!PyExceptionInstance_Check(fixed_cause)) {
                _PyErr_Format(tstate, PyExc_TypeError,
                              "calling %R should have returned an instance of "
                              "BaseException, not %R",
                              cause, Py_TYPE(fixed_cause));
                Py_DECREF(fixed_cause);
                goto raise_error;
            }
            Py_DECREF(cause);
        }
        else if (PyExceptionInstance_Check(cause)) {
            fixed_cause = cause;
        }
        else if (Py_IsNone(cause)) {
            Py_DECREF(cause);
            fixed_cause = NULL;
        }
        else {
            _PyErr_SetString(tstate, PyExc_TypeError,
                             "exception causes must derive from "
                             "BaseException");
            goto raise_error;
        }
        PyException_SetCause(value, fixed_cause);
    }

    _PyErr_SetObject(tstate, type, value);
    /* _PyErr_SetObject incref's its arguments */
    Py_DECREF(value);
    Py_DECREF(type);
    return 0;

raise_error:
    Py_XDECREF(value);
    Py_XDECREF(type);
    Py_XDECREF(cause);
    return 0;
}

/* Disable unused label warnings.  They are handy for debugging, even
   if computed gotos aren't used. */

/* TBD - what about other compilers? */
#if defined(__GNUC__) || defined(__clang__)
#  pragma GCC diagnostic push
#  pragma GCC diagnostic ignored "-Wunused-label"
#elif defined(_MSC_VER) /* MS_WINDOWS */
#  pragma warning(push)
#  pragma warning(disable:4102)
#endif

Coverage Report

Created: 2026-02-09 07:07

Line	Count	Source
1		#define _PY_INTERPRETER
2
3		#include "Python.h"
4		#include "pycore_abstract.h" // _PyIndex_Check()
5		#include "pycore_audit.h" // _PySys_Audit()
6		#include "pycore_backoff.h"
7		#include "pycore_call.h" // _PyObject_CallNoArgs()
8		#include "pycore_cell.h" // PyCell_GetRef()
9		#include "pycore_ceval.h" // SPECIAL___ENTER__
10		#include "pycore_code.h"
11		#include "pycore_dict.h"
12		#include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS
13		#include "pycore_floatobject.h" // _PyFloat_ExactDealloc()
14		#include "pycore_frame.h"
15		#include "pycore_function.h"
16		#include "pycore_genobject.h" // _PyCoro_GetAwaitableIter()
17		#include "pycore_import.h" // _PyImport_IsDefaultImportFunc()
18		#include "pycore_instruments.h"
19		#include "pycore_interpframe.h" // _PyFrame_SetStackPointer()
20		#include "pycore_interpolation.h" // _PyInterpolation_Build()
21		#include "pycore_intrinsics.h"
22		#include "pycore_jit.h"
23		#include "pycore_list.h" // _PyList_GetItemRef()
24		#include "pycore_long.h" // _PyLong_GetZero()
25		#include "pycore_moduleobject.h" // PyModuleObject
26		#include "pycore_object.h" // _PyObject_GC_TRACK()
27		#include "pycore_opcode_metadata.h" // EXTRA_CASES
28		#include "pycore_opcode_utils.h" // MAKE_FUNCTION_*
29		#include "pycore_optimizer.h" // _PyUOpExecutor_Type
30		#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_*
31		#include "pycore_pyerrors.h" // _PyErr_GetRaisedException()
32		#include "pycore_pystate.h" // _PyInterpreterState_GET()
33		#include "pycore_range.h" // _PyRangeIterObject
34		#include "pycore_setobject.h" // _PySet_Update()
35		#include "pycore_sliceobject.h" // _PyBuildSlice_ConsumeRefs
36		#include "pycore_sysmodule.h" // _PySys_GetOptionalAttrString()
37		#include "pycore_template.h" // _PyTemplate_Build()
38		#include "pycore_traceback.h" // _PyTraceBack_FromFrame
39		#include "pycore_tuple.h" // _PyTuple_ITEMS()
40		#include "pycore_uop_ids.h" // Uops
41
42		#include "dictobject.h"
43		#include "frameobject.h" // _PyInterpreterFrame_GetLine
44		#include "opcode.h"
45		#include "pydtrace.h"
46		#include "setobject.h"
47		#include "pycore_stackref.h"
48
49		#include <stdbool.h> // bool
50
51		#if !defined(Py_BUILD_CORE)
52		# error "ceval.c must be build with Py_BUILD_CORE define for best performance"
53		#endif
54
55		#if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS)
56		// GH-89279: The MSVC compiler does not inline these static inline functions
57		// in PGO build in _PyEval_EvalFrameDefault(), because this function is over
58		// the limit of PGO, and that limit cannot be configured.
59		// Define them as macros to make sure that they are always inlined by the
60		// preprocessor.
61
62		#undef Py_IS_TYPE
63		#define Py_IS_TYPE(ob, type) \
64	8.05G	(_PyObject_CAST(ob)->ob_type == (type))
65
66		#undef Py_XDECREF
67		#define Py_XDECREF(arg) \
68	347M	do { \
69	347M	PyObject *xop = _PyObject_CAST(arg); \
70	347M	if (xop != NULL) { \
71	229M	Py_DECREF(xop); \
72	229M	} \
73	347M	} while (0)
74
75		#ifndef Py_GIL_DISABLED
76
77		#undef Py_DECREF
78		#define Py_DECREF(arg) \
79	381M	do { \
80	381M	PyObject *op = _PyObject_CAST(arg); \
81	381M	if (_Py_IsImmortal(op)) { \
82	141M	_Py_DECREF_IMMORTAL_STAT_INC(); \
83	141M	break; \
84	141M	} \
85	381M	_Py_DECREF_STAT_INC(); \
86	240M	if (--op->ob_refcnt == 0) { \
87	113M	_PyReftracerTrack(op, PyRefTracer_DESTROY); \
88	113M	destructor dealloc = Py_TYPE(op)->tp_dealloc; \
89	113M	(*dealloc)(op); \
90	113M	} \
91	240M	} while (0)
92
93		#undef _Py_DECREF_SPECIALIZED
94		#define _Py_DECREF_SPECIALIZED(arg, dealloc) \
95	5.31M	do { \
96	5.31M	PyObject *op = _PyObject_CAST(arg); \
97	5.31M	if (_Py_IsImmortal(op)) { \
98	4.75M	_Py_DECREF_IMMORTAL_STAT_INC(); \
99	4.75M	break; \
100	4.75M	} \
101	5.31M	_Py_DECREF_STAT_INC(); \
102	564k	if (--op->ob_refcnt == 0) { \
103	460k	_PyReftracerTrack(op, PyRefTracer_DESTROY); \
104	460k	destructor d = (destructor)(dealloc); \
105	460k	d(op); \
106	460k	} \
107	564k	} while (0)
108
109		#else // Py_GIL_DISABLED
110
111		#undef Py_DECREF
112		#define Py_DECREF(arg) \
113		do { \
114		PyObject *op = _PyObject_CAST(arg); \
115		uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); \
116		if (local == _Py_IMMORTAL_REFCNT_LOCAL) { \
117		_Py_DECREF_IMMORTAL_STAT_INC(); \
118		break; \
119		} \
120		_Py_DECREF_STAT_INC(); \
121		if (_Py_IsOwnedByCurrentThread(op)) { \
122		local--; \
123		_Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); \
124		if (local == 0) { \
125		_Py_MergeZeroLocalRefcount(op); \
126		} \
127		} \
128		else { \
129		_Py_DecRefShared(op); \
130		} \
131		} while (0)
132
133		#undef _Py_DECREF_SPECIALIZED
134		#define _Py_DECREF_SPECIALIZED(arg, dealloc) Py_DECREF(arg)
135
136		#endif
137		#endif
138
139		static void
140		check_invalid_reentrancy(void)
141	214M	{
142		#if defined(Py_DEBUG) && defined(Py_GIL_DISABLED)
143		// In the free-threaded build, the interpreter must not be re-entered if
144		// the world-is-stopped. If so, that's a bug somewhere (quite likely in
145		// the painfully complex typeobject code).
146		PyInterpreterState *interp = _PyInterpreterState_GET();
147		assert(!interp->stoptheworld.world_stopped);
148		#endif
149	214M	}
150
151
152		#ifdef Py_DEBUG
153		static void
154		dump_item(_PyStackRef item)
155		{
156		if (PyStackRef_IsNull(item)) {
157		printf("<NULL>");
158		return;
159		}
160		if (PyStackRef_IsMalformed(item)) {
161		printf("<INVALID>");
162		return;
163		}
164		if (PyStackRef_IsTaggedInt(item)) {
165		printf("%" PRId64, (int64_t)PyStackRef_UntagInt(item));
166		return;
167		}
168		PyObject *obj = PyStackRef_AsPyObjectBorrow(item);
169		if (obj == NULL) {
170		printf("<nil>");
171		return;
172		}
173		// Don't call __repr__(), it might recurse into the interpreter.
174		printf("<%s at %p>", Py_TYPE(obj)->tp_name, (void *)obj);
175		}
176
177		static void
178		dump_stack(_PyInterpreterFrame frame, _PyStackRef stack_pointer)
179		{
180		_PyFrame_SetStackPointer(frame, stack_pointer);
181		_PyStackRef *locals_base = _PyFrame_GetLocalsArray(frame);
182		_PyStackRef *stack_base = _PyFrame_Stackbase(frame);
183		PyObject *exc = PyErr_GetRaisedException();
184		printf(" locals=[");
185		for (_PyStackRef *ptr = locals_base; ptr < stack_base; ptr++) {
186		if (ptr != locals_base) {
187		printf(", ");
188		}
189		dump_item(*ptr);
190		}
191		printf("]\n");
192		if (stack_pointer < stack_base) {
193		printf(" stack=%d\n", (int)(stack_pointer-stack_base));
194		}
195		else {
196		printf(" stack=[");
197		for (_PyStackRef *ptr = stack_base; ptr < stack_pointer; ptr++) {
198		if (ptr != stack_base) {
199		printf(", ");
200		}
201		dump_item(*ptr);
202		}
203		printf("]\n");
204		}
205		fflush(stdout);
206		PyErr_SetRaisedException(exc);
207		_PyFrame_GetStackPointer(frame);
208		}
209
210		#if defined(_Py_TIER2) && !defined(_Py_JIT) && defined(Py_DEBUG)
211		static void
212		dump_cache_item(_PyStackRef cache, int position, int depth)
213		{
214		if (position < depth) {
215		dump_item(cache);
216		}
217		else {
218		printf("---");
219		}
220		}
221		#endif
222
223		static void
224		lltrace_instruction(_PyInterpreterFrame *frame,
225		_PyStackRef *stack_pointer,
226		_Py_CODEUNIT *next_instr,
227		int opcode,
228		int oparg)
229		{
230		int offset = 0;
231		if (frame->owner < FRAME_OWNED_BY_INTERPRETER) {
232		dump_stack(frame, stack_pointer);
233		offset = (int)(next_instr - _PyFrame_GetBytecode(frame));
234		}
235		const char *opname = _PyOpcode_OpName[opcode];
236		assert(opname != NULL);
237		if (OPCODE_HAS_ARG((int)_PyOpcode_Deopt[opcode])) {
238		printf("%d: %s %d\n", offset * 2, opname, oparg);
239		}
240		else {
241		printf("%d: %s\n", offset * 2, opname);
242		}
243		fflush(stdout);
244		}
245
246		static void
247		lltrace_resume_frame(_PyInterpreterFrame *frame)
248		{
249		PyObject *fobj = PyStackRef_AsPyObjectBorrow(frame->f_funcobj);
250		if (!PyStackRef_CodeCheck(frame->f_executable) \|\|
251		fobj == NULL \|\|
252		!PyFunction_Check(fobj)
253		) {
254		printf("\nResuming frame.\n");
255		return;
256		}
257		PyFunctionObject f = (PyFunctionObject )fobj;
258		PyObject *exc = PyErr_GetRaisedException();
259		PyObject *name = f->func_qualname;
260		if (name == NULL) {
261		name = f->func_name;
262		}
263		printf("\nResuming frame");
264		if (name) {
265		printf(" for ");
266		if (PyObject_Print(name, stdout, 0) < 0) {
267		PyErr_Clear();
268		}
269		}
270		if (f->func_module) {
271		printf(" in module ");
272		if (PyObject_Print(f->func_module, stdout, 0) < 0) {
273		PyErr_Clear();
274		}
275		}
276		printf("\n");
277		fflush(stdout);
278		PyErr_SetRaisedException(exc);
279		}
280
281		static int
282		maybe_lltrace_resume_frame(_PyInterpreterFrame frame, PyObject globals)
283		{
284		if (globals == NULL) {
285		return 0;
286		}
287		if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) {
288		return 0;
289		}
290		int r = PyDict_Contains(globals, &_Py_ID(__lltrace__));
291		if (r < 0) {
292		PyErr_Clear();
293		return 0;
294		}
295		int lltrace = r * 5; // Levels 1-4 only trace uops
296		if (!lltrace) {
297		// Can also be controlled by environment variable
298		char *python_lltrace = Py_GETENV("PYTHON_LLTRACE");
299		if (python_lltrace != NULL && *python_lltrace >= '0') {
300		lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that
301		}
302		}
303		if (lltrace >= 5) {
304		lltrace_resume_frame(frame);
305		}
306		return lltrace;
307		}
308
309		#endif
310
311		static void monitor_reraise(PyThreadState *tstate,
312		_PyInterpreterFrame *frame,
313		_Py_CODEUNIT *instr);
314		static int monitor_stop_iteration(PyThreadState *tstate,
315		_PyInterpreterFrame *frame,
316		_Py_CODEUNIT *instr,
317		PyObject *value);
318		static void monitor_unwind(PyThreadState *tstate,
319		_PyInterpreterFrame *frame,
320		_Py_CODEUNIT *instr);
321		static int monitor_handled(PyThreadState *tstate,
322		_PyInterpreterFrame *frame,
323		_Py_CODEUNIT instr, PyObject exc);
324		static void monitor_throw(PyThreadState *tstate,
325		_PyInterpreterFrame *frame,
326		_Py_CODEUNIT *instr);
327
328		static int get_exception_handler(PyCodeObject , int, int, int, int);
329
330		#ifdef HAVE_ERRNO_H
331		#include <errno.h>
332		#endif
333
334		typedef struct {
335		_PyInterpreterFrame frame;
336		_PyStackRef stack[1];
337		} _PyEntryFrame;
338
339		static int
340		do_monitor_exc(PyThreadState tstate, _PyInterpreterFrame frame,
341		_Py_CODEUNIT *instr, int event)
342	0	{
343	0	assert(event < _PY_MONITORING_UNGROUPED_EVENTS);
344	0	if (_PyFrame_GetCode(frame)->co_flags & CO_NO_MONITORING_EVENTS) {
345	0	return 0;
346	0	}
347	0	PyObject *exc = PyErr_GetRaisedException();
348	0	assert(exc != NULL);
349	0	int err = _Py_call_instrumentation_arg(tstate, event, frame, instr, exc);
350	0	if (err == 0) {
351	0	PyErr_SetRaisedException(exc);
352	0	}
353	0	else {
354	0	assert(PyErr_Occurred());
355	0	Py_DECREF(exc);
356	0	}
357	0	return err;
358	0	}
359
360		static inline bool
361		no_tools_for_global_event(PyThreadState *tstate, int event)
362	108M	{
363	108M	return tstate->interp->monitors.tools[event] == 0;
364	108M	}
365
366		static inline bool
367		no_tools_for_local_event(PyThreadState tstate, _PyInterpreterFrame frame, int event)
368	0	{
369	0	assert(event < _PY_MONITORING_LOCAL_EVENTS);
370	0	_PyCoMonitoringData *data = _PyFrame_GetCode(frame)->_co_monitoring;
371	0	if (data) {
372	0	return data->active_monitors.tools[event] == 0;
373	0	}
374	0	else {
375	0	return no_tools_for_global_event(tstate, event);
376	0	}
377	0	}
378
379		static int
380		monitor_handled(PyThreadState *tstate,
381		_PyInterpreterFrame *frame,
382		_Py_CODEUNIT instr, PyObject exc)
383	21.7M	{
384	21.7M	if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED)) {
385	21.7M	return 0;
386	21.7M	}
387	0	return _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED, frame, instr, exc);
388	21.7M	}
389
390		static void
391		monitor_throw(PyThreadState *tstate,
392		_PyInterpreterFrame *frame,
393		_Py_CODEUNIT *instr)
394	988	{
395	988	if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_THROW)) {
396	988	return;
397	988	}
398	0	do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_THROW);
399	0	}
400
401		static void
402		monitor_reraise(PyThreadState tstate, _PyInterpreterFrame frame,
403		_Py_CODEUNIT *instr)
404	3.19M	{
405	3.19M	if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_RERAISE)) {
406	3.19M	return;
407	3.19M	}
408	0	do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RERAISE);
409	0	}
410
411		static int
412		monitor_stop_iteration(PyThreadState tstate, _PyInterpreterFrame frame,
413		_Py_CODEUNIT instr, PyObject value)
414	0	{
415	0	if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_STOP_ITERATION)) {
416	0	return 0;
417	0	}
418	0	assert(!PyErr_Occurred());
419	0	PyErr_SetObject(PyExc_StopIteration, value);
420	0	int res = do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_STOP_ITERATION);
421	0	if (res < 0) {
422	0	return res;
423	0	}
424	0	PyErr_SetRaisedException(NULL);
425	0	return 0;
426	0	}
427
428		static void
429		monitor_unwind(PyThreadState *tstate,
430		_PyInterpreterFrame *frame,
431		_Py_CODEUNIT *instr)
432	26.8M	{
433	26.8M	if (no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_UNWIND)) {
434	26.8M	return;
435	26.8M	}
436	0	do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_UNWIND);
437	0	}
438
439		static inline unsigned char *
440	13.5M	scan_back_to_entry_start(unsigned char *p) {
441	37.0M	for (; (p[0]&128) == 0; p--);
442	13.5M	return p;
443	13.5M	}
444
445		static inline unsigned char *
446	25.5M	skip_to_next_entry(unsigned char p, unsigned char end) {
447	102M	while (p < end && ((p[0] & 128) == 0)) {
448	76.6M	p++;
449	76.6M	}
450	25.5M	return p;
451	25.5M	}
452
453
454	62.1M	#define MAX_LINEAR_SEARCH 40
455
456		static Py_NO_INLINE int
457		get_exception_handler(PyCodeObject code, int index, int level, int handler, int lasti)
458	48.5M	{
459	48.5M	unsigned char start = (unsigned char )PyBytes_AS_STRING(code->co_exceptiontable);
460	48.5M	unsigned char *end = start + PyBytes_GET_SIZE(code->co_exceptiontable);
461		/* Invariants:
462		* start_table == end_table OR
463		* start_table points to a legal entry and end_table points
464		* beyond the table or to a legal entry that is after index.
465		*/
466	48.5M	if (end - start > MAX_LINEAR_SEARCH) {
467	5.14M	int offset;
468	5.14M	parse_varint(start, &offset);
469	5.14M	if (offset > index) {
470	83.6k	return 0;
471	83.6k	}
472	13.5M	do {
473	13.5M	unsigned char * mid = start + ((end-start)>>1);
474	13.5M	mid = scan_back_to_entry_start(mid);
475	13.5M	parse_varint(mid, &offset);
476	13.5M	if (offset > index) {
477	13.0M	end = mid;
478	13.0M	}
479	492k	else {
480	492k	start = mid;
481	492k	}
482
483	13.5M	} while (end - start > MAX_LINEAR_SEARCH);
484	5.06M	}
485	48.4M	unsigned char *scan = start;
486	74.0M	while (scan < end) {
487	51.6M	int start_offset, size;
488	51.6M	scan = parse_varint(scan, &start_offset);
489	51.6M	if (start_offset > index) {
490	4.37M	break;
491	4.37M	}
492	47.2M	scan = parse_varint(scan, &size);
493	47.2M	if (start_offset + size > index) {
494	21.7M	scan = parse_varint(scan, handler);
495	21.7M	int depth_and_lasti;
496	21.7M	parse_varint(scan, &depth_and_lasti);
497	21.7M	*level = depth_and_lasti >> 1;
498	21.7M	*lasti = depth_and_lasti & 1;
499	21.7M	return 1;
500	21.7M	}
501	25.5M	scan = skip_to_next_entry(scan, end);
502	25.5M	}
503	26.7M	return 0;
504	48.4M	}
505
506
507		#ifdef Py_DEBUG
508		#define ASSERT_WITHIN_STACK_BOUNDS(F, L) _Py_assert_within_stack_bounds(frame, stack_pointer, (F), (L))
509		#else
510	37.1G	#define ASSERT_WITHIN_STACK_BOUNDS(F, L) (void)0
511		#endif
512
513		/* Logic for the raise statement (too complicated for inlining).
514		This consumes a reference count to each of its arguments. */
515		static int
516		do_raise(PyThreadState tstate, PyObject exc, PyObject *cause)
517	16.1M	{
518	16.1M	PyObject type = NULL, value = NULL;
519
520	16.1M	if (exc == NULL) {
521		/* Reraise */
522	15.5k	_PyErr_StackItem *exc_info = _PyErr_GetTopmostException(tstate);
523	15.5k	exc = exc_info->exc_value;
524	15.5k	if (Py_IsNone(exc) \|\| exc == NULL) {
525	0	_PyErr_SetString(tstate, PyExc_RuntimeError,
526	0	"No active exception to reraise");
527	0	return 0;
528	0	}
529	15.5k	Py_INCREF(exc);
530	15.5k	assert(PyExceptionInstance_Check(exc));
531	15.5k	_PyErr_SetRaisedException(tstate, exc);
532	15.5k	return 1;
533	15.5k	}
534
535		/* We support the following forms of raise:
536		raise
537		raise <instance>
538		raise <type> */
539
540	16.1M	if (PyExceptionClass_Check(exc)) {
541	11.7M	type = exc;
542	11.7M	value = _PyObject_CallNoArgs(exc);
543	11.7M	if (value == NULL)
544	0	goto raise_error;
545	11.7M	if (!PyExceptionInstance_Check(value)) {
546	0	_PyErr_Format(tstate, PyExc_TypeError,
547	0	"calling %R should have returned an instance of "
548	0	"BaseException, not %R",
549	0	type, Py_TYPE(value));
550	0	goto raise_error;
551	0	}
552	11.7M	}
553	4.45M	else if (PyExceptionInstance_Check(exc)) {
554	4.45M	value = exc;
555	4.45M	type = PyExceptionInstance_Class(exc);
556	4.45M	Py_INCREF(type);
557	4.45M	}
558	0	else {
559		/* Not something you can raise. You get an exception
560		anyway, just not what you specified :-) */
561	0	Py_DECREF(exc);
562	0	_PyErr_SetString(tstate, PyExc_TypeError,
563	0	"exceptions must derive from BaseException");
564	0	goto raise_error;
565	0	}
566
567	16.1M	assert(type != NULL);
568	16.1M	assert(value != NULL);
569
570	16.1M	if (cause) {
571	13.2k	PyObject *fixed_cause;
572	13.2k	if (PyExceptionClass_Check(cause)) {
573	0	fixed_cause = _PyObject_CallNoArgs(cause);
574	0	if (fixed_cause == NULL)
575	0	goto raise_error;
576	0	if (!PyExceptionInstance_Check(fixed_cause)) {
577	0	_PyErr_Format(tstate, PyExc_TypeError,
578	0	"calling %R should have returned an instance of "
579	0	"BaseException, not %R",
580	0	cause, Py_TYPE(fixed_cause));
581	0	Py_DECREF(fixed_cause);
582	0	goto raise_error;
583	0	}
584	0	Py_DECREF(cause);
585	0	}
586	13.2k	else if (PyExceptionInstance_Check(cause)) {
587	6.13k	fixed_cause = cause;
588	6.13k	}
589	7.08k	else if (Py_IsNone(cause)) {
590	7.08k	Py_DECREF(cause);
591	7.08k	fixed_cause = NULL;
592	7.08k	}
593	0	else {
594	0	_PyErr_SetString(tstate, PyExc_TypeError,
595	0	"exception causes must derive from "
596	0	"BaseException");
597	0	goto raise_error;
598	0	}
599	13.2k	PyException_SetCause(value, fixed_cause);
600	13.2k	}
601
602	16.1M	_PyErr_SetObject(tstate, type, value);
603		/* _PyErr_SetObject incref's its arguments */
604	16.1M	Py_DECREF(value);
605	16.1M	Py_DECREF(type);
606	16.1M	return 0;
607
608	0	raise_error:
609	0	Py_XDECREF(value);
610	0	Py_XDECREF(type);
611		Py_XDECREF(cause);
612	0	return 0;
613	16.1M	}
614
615		/* Disable unused label warnings. They are handy for debugging, even
616		if computed gotos aren't used. */
617
618		/* TBD - what about other compilers? */
619		#if defined(__GNUC__) \|\| defined(__clang__)
620		# pragma GCC diagnostic push
621		# pragma GCC diagnostic ignored "-Wunused-label"
622		#elif defined(_MSC_VER) /* MS_WINDOWS */
623		# pragma warning(push)
624		# pragma warning(disable:4102)
625		#endif