/src/cpython/Python/ceval.h

Source
#define _PY_INTERPRETER

#include "Python.h"
#include "pycore_abstract.h"      // _PyIndex_Check()
#include "pycore_audit.h"         // _PySys_Audit()
#include "pycore_backoff.h"
#include "pycore_call.h"          // _PyObject_CallNoArgs()
#include "pycore_cell.h"          // PyCell_GetRef()
#include "pycore_ceval.h"         // SPECIAL___ENTER__
#include "pycore_code.h"
#include "pycore_dict.h"
#include "pycore_emscripten_signal.h"  // _Py_CHECK_EMSCRIPTEN_SIGNALS
#include "pycore_floatobject.h"   // _PyFloat_ExactDealloc()
#include "pycore_frame.h"
#include "pycore_function.h"
#include "pycore_genobject.h"     // _PyCoro_GetAwaitableIter()
#include "pycore_import.h"        // _PyImport_IsDefaultImportFunc()
#include "pycore_instruments.h"
#include "pycore_interpframe.h"   // _PyFrame_SetStackPointer()
#include "pycore_interpolation.h" // _PyInterpolation_Build()
#include "pycore_intrinsics.h"
#include "pycore_jit.h"
#include "pycore_lazyimportobject.h"
#include "pycore_list.h"          // _PyList_GetItemRef()
#include "pycore_long.h"          // _PyLong_GetZero()
#include "pycore_moduleobject.h"  // PyModuleObject
#include "pycore_object.h"        // _PyObject_GC_TRACK()
#include "pycore_opcode_metadata.h" // EXTRA_CASES
#include "pycore_opcode_utils.h"  // MAKE_FUNCTION_*
#include "pycore_optimizer.h"     // _PyUOpExecutor_Type
#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_*
#include "pycore_pyerrors.h"      // _PyErr_GetRaisedException()
#include "pycore_pystate.h"       // _PyInterpreterState_GET()
#include "pycore_range.h"         // _PyRangeIterObject
#include "pycore_setobject.h"     // _PySet_Update()
#include "pycore_sliceobject.h"   // _PyBuildSlice_ConsumeRefs
#include "pycore_sysmodule.h"     // _PySys_GetOptionalAttrString()
#include "pycore_template.h"      // _PyTemplate_Build()
#include "pycore_traceback.h"     // _PyTraceBack_FromFrame
#include "pycore_tuple.h"         // _PyTuple_ITEMS()
#include "pycore_uop_ids.h"       // Uops

#include "dictobject.h"
#include "frameobject.h"          // _PyInterpreterFrame_GetLine
#include "opcode.h"
#include "pydtrace.h"
#include "setobject.h"
#include "pycore_stackref.h"

#include <stdbool.h>              // bool

#if !defined(Py_BUILD_CORE)
#  error "ceval.c must be build with Py_BUILD_CORE define for best performance"
#endif

#if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS)
// GH-89279: The MSVC compiler does not inline these static inline functions
// in PGO build in _PyEval_EvalFrameDefault(), because this function is over
// the limit of PGO, and that limit cannot be configured.
// Define them as macros to make sure that they are always inlined by the
// preprocessor.

#undef Py_IS_TYPE
#define Py_IS_TYPE(ob, type) \
    (_PyObject_CAST(ob)->ob_type == (type))

#undef Py_XDECREF
#define Py_XDECREF(arg) \
    do { \
        PyObject *xop = _PyObject_CAST(arg); \
        if (xop != NULL) { \
            Py_DECREF(xop); \
        } \
    } while (0)

#ifndef Py_GIL_DISABLED

#undef Py_DECREF
#define Py_DECREF(arg) \
    do { \
        PyObject *op = _PyObject_CAST(arg); \
        if (_Py_IsImmortal(op)) { \
            _Py_DECREF_IMMORTAL_STAT_INC(); \
            break; \
        } \
        _Py_DECREF_STAT_INC(); \
        if (--op->ob_refcnt == 0) { \
            _PyReftracerTrack(op, PyRefTracer_DESTROY); \
            destructor dealloc = Py_TYPE(op)->tp_dealloc; \
            (*dealloc)(op); \
        } \
    } while (0)

#undef _Py_DECREF_SPECIALIZED
#define _Py_DECREF_SPECIALIZED(arg, dealloc) \
    do { \
        PyObject *op = _PyObject_CAST(arg); \
        if (_Py_IsImmortal(op)) { \
            _Py_DECREF_IMMORTAL_STAT_INC(); \
            break; \
        } \
        _Py_DECREF_STAT_INC(); \
        if (--op->ob_refcnt == 0) { \
            _PyReftracerTrack(op, PyRefTracer_DESTROY); \
            destructor d = (destructor)(dealloc); \
            d(op); \
        } \
    } while (0)

#else // Py_GIL_DISABLED

#undef Py_DECREF
#define Py_DECREF(arg) \
    do { \
        PyObject *op = _PyObject_CAST(arg); \
        uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); \
        if (local == _Py_IMMORTAL_REFCNT_LOCAL) { \
            _Py_DECREF_IMMORTAL_STAT_INC(); \
            break; \
        } \
        _Py_DECREF_STAT_INC(); \
        if (_Py_IsOwnedByCurrentThread(op)) { \
            local--; \
            _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); \
            if (local == 0) { \
                _Py_MergeZeroLocalRefcount(op); \
            } \
        } \
        else { \
            _Py_DecRefShared(op); \
        } \
    } while (0)

#undef _Py_DECREF_SPECIALIZED
#define _Py_DECREF_SPECIALIZED(arg, dealloc) Py_DECREF(arg)

#endif
#endif

static void
check_invalid_reentrancy(void)
{
#if defined(Py_DEBUG) && defined(Py_GIL_DISABLED)
    // In the free-threaded build, the interpreter must not be re-entered if
    // the world-is-stopped.  If so, that's a bug somewhere (quite likely in
    // the painfully complex typeobject code).
    PyInterpreterState *interp = _PyInterpreterState_GET();
    assert(!interp->stoptheworld.world_stopped);
#endif
}


#ifdef Py_DEBUG
static void
dump_item(_PyStackRef item)
{
    if (PyStackRef_IsNull(item)) {
        printf("<NULL>");
        return;
    }
    if (PyStackRef_IsMalformed(item)) {
        printf("<INVALID>");
        return;
    }
    if (PyStackRef_IsTaggedInt(item)) {
        printf("%" PRId64, (int64_t)PyStackRef_UntagInt(item));
        return;
    }
    PyObject *obj = PyStackRef_AsPyObjectBorrow(item);
    if (obj == NULL) {
        printf("<nil>");
        return;
    }
    // Don't call __repr__(), it might recurse into the interpreter.
    printf("<%s at %p>", Py_TYPE(obj)->tp_name, (void *)obj);
}

static void
dump_stack(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer)
{
    _PyFrame_SetStackPointer(frame, stack_pointer);
    _PyStackRef *locals_base = _PyFrame_GetLocalsArray(frame);
    _PyStackRef *stack_base = _PyFrame_Stackbase(frame);
    PyObject *exc = PyErr_GetRaisedException();
    printf("    locals=[");
    for (_PyStackRef *ptr = locals_base; ptr < stack_base; ptr++) {
        if (ptr != locals_base) {
            printf(", ");
        }
        dump_item(*ptr);
    }
    printf("]\n");
    if (stack_pointer < stack_base) {
        printf("    stack=%d\n", (int)(stack_pointer-stack_base));
    }
    else {
        printf("    stack=[");
        for (_PyStackRef *ptr = stack_base; ptr < stack_pointer; ptr++) {
            if (ptr != stack_base) {
                printf(", ");
            }
            dump_item(*ptr);
        }
        printf("]\n");
    }
    fflush(stdout);
    PyErr_SetRaisedException(exc);
    _PyFrame_GetStackPointer(frame);
}

#if defined(_Py_TIER2) && !defined(_Py_JIT) && defined(Py_DEBUG)
static void
dump_cache_item(_PyStackRef cache, int position, int depth)
{
    if (position < depth) {
        dump_item(cache);
    }
    else {
        printf("---");
    }
}
#endif

static void
lltrace_instruction(_PyInterpreterFrame *frame,
                    _PyStackRef *stack_pointer,
                    _Py_CODEUNIT *next_instr,
                    int opcode,
                    int oparg)
{
    int offset = 0;
    if (frame->owner < FRAME_OWNED_BY_INTERPRETER) {
        dump_stack(frame, stack_pointer);
        offset = (int)(next_instr - _PyFrame_GetBytecode(frame));
    }
    const char *opname = _PyOpcode_OpName[opcode];
    assert(opname != NULL);
    if (OPCODE_HAS_ARG((int)_PyOpcode_Deopt[opcode])) {
        printf("%d: %s %d\n", offset * 2, opname, oparg);
    }
    else {
        printf("%d: %s\n", offset * 2, opname);
    }
    fflush(stdout);
}

static void
lltrace_resume_frame(_PyInterpreterFrame *frame)
{
    PyObject *fobj = PyStackRef_AsPyObjectBorrow(frame->f_funcobj);
    if (!PyStackRef_CodeCheck(frame->f_executable) ||
        fobj == NULL ||
        !PyFunction_Check(fobj)
    ) {
        printf("\nResuming frame.\n");
        return;
    }
    PyFunctionObject *f = (PyFunctionObject *)fobj;
    PyObject *exc = PyErr_GetRaisedException();
    PyObject *name = Py_XNewRef(FT_ATOMIC_LOAD_PTR_ACQUIRE(f->func_qualname));
    if (name == NULL) {
        name = Py_XNewRef(FT_ATOMIC_LOAD_PTR_ACQUIRE(f->func_name));
    }
    printf("\nResuming frame");
    if (name) {
        printf(" for ");
        if (PyObject_Print(name, stdout, 0) < 0) {
            PyErr_Clear();
        }
        Py_DECREF(name);
    }
    PyObject *module = Py_XNewRef(FT_ATOMIC_LOAD_PTR_ACQUIRE(f->func_module));
    if (module) {
        printf(" in module ");
        if (PyObject_Print(module, stdout, 0) < 0) {
            PyErr_Clear();
        }
        Py_DECREF(module);
    }
    printf("\n");
    fflush(stdout);
    PyErr_SetRaisedException(exc);
}

static int
maybe_lltrace_resume_frame(_PyInterpreterFrame *frame, PyObject *globals)
{
    if (globals == NULL) {
        return 0;
    }
    if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) {
        return 0;
    }
    int r = PyDict_Contains(globals, &_Py_ID(__lltrace__));
    if (r < 0) {
        PyErr_Clear();
        return 0;
    }
    int lltrace = r * 5;  // Levels 1-4 only trace uops
    if (!lltrace) {
        // Can also be controlled by environment variable
        char *python_lltrace = Py_GETENV("PYTHON_LLTRACE");
        if (python_lltrace != NULL && *python_lltrace >= '0') {
            lltrace = *python_lltrace - '0';  // TODO: Parse an int and all that
        }
    }
    if (lltrace >= 5) {
        lltrace_resume_frame(frame);
    }
    return lltrace;
}

#endif

static void monitor_reraise(PyThreadState *tstate,
                 _PyInterpreterFrame *frame,
                 _Py_CODEUNIT *instr);
static int monitor_stop_iteration(PyThreadState *tstate,
                 _PyInterpreterFrame *frame,
                 _Py_CODEUNIT *instr,
                 PyObject *value);
static void monitor_unwind(PyThreadState *tstate,
                 _PyInterpreterFrame *frame,
                 _Py_CODEUNIT *instr);
static int monitor_handled(PyThreadState *tstate,
                 _PyInterpreterFrame *frame,
                 _Py_CODEUNIT *instr, PyObject *exc);
static void monitor_throw(PyThreadState *tstate,
                 _PyInterpreterFrame *frame,
                 _Py_CODEUNIT *instr);

static int get_exception_handler(PyCodeObject *, int, int*, int*, int*);

#ifdef HAVE_ERRNO_H
#include <errno.h>
#endif

typedef struct {
    _PyInterpreterFrame frame;
    _PyStackRef stack[1];
} _PyEntryFrame;

static int
do_monitor_exc(PyThreadState *tstate, _PyInterpreterFrame *frame,
               _Py_CODEUNIT *instr, int event)
{
    assert(event < _PY_MONITORING_UNGROUPED_EVENTS);
    if (_PyFrame_GetCode(frame)->co_flags & CO_NO_MONITORING_EVENTS) {
        return 0;
    }
    PyObject *exc = PyErr_GetRaisedException();
    assert(exc != NULL);
    int err = _Py_call_instrumentation_arg(tstate, event, frame, instr, exc);
    if (err == 0) {
        PyErr_SetRaisedException(exc);
    }
    else {
        assert(PyErr_Occurred());
        Py_DECREF(exc);
    }
    return err;
}

static inline bool
no_tools_for_global_event(PyThreadState *tstate, int event)
{
    return tstate->interp->monitors.tools[event] == 0;
}

static inline bool
no_tools_for_local_event(PyThreadState *tstate, _PyInterpreterFrame *frame, int event)
{
    assert(event < _PY_MONITORING_UNGROUPED_EVENTS);
    _PyCoMonitoringData *data = _PyFrame_GetCode(frame)->_co_monitoring;
    if (data) {
        return data->active_monitors.tools[event] == 0;
    }
    else {
        return no_tools_for_global_event(tstate, event);
    }
}

static int
monitor_handled(PyThreadState *tstate,
                _PyInterpreterFrame *frame,
                _Py_CODEUNIT *instr, PyObject *exc)
{
    if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_EXCEPTION_HANDLED)) {
        return 0;
    }
    return _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED, frame, instr, exc);
}

static void
monitor_throw(PyThreadState *tstate,
              _PyInterpreterFrame *frame,
              _Py_CODEUNIT *instr)
{
    if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_PY_THROW)) {
        return;
    }
    do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_THROW);
}

static void
monitor_reraise(PyThreadState *tstate, _PyInterpreterFrame *frame,
              _Py_CODEUNIT *instr)
{
    if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_RERAISE)) {
        return;
    }
    do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RERAISE);
}

static int
monitor_stop_iteration(PyThreadState *tstate, _PyInterpreterFrame *frame,
                       _Py_CODEUNIT *instr, PyObject *value)
{
    if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_STOP_ITERATION)) {
        return 0;
    }
    assert(!PyErr_Occurred());
    PyErr_SetObject(PyExc_StopIteration, value);
    int res = do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_STOP_ITERATION);
    if (res < 0) {
        return res;
    }
    PyErr_SetRaisedException(NULL);
    return 0;
}

static void
monitor_unwind(PyThreadState *tstate,
               _PyInterpreterFrame *frame,
               _Py_CODEUNIT *instr)
{
    if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_PY_UNWIND)) {
        return;
    }
    do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_UNWIND);
}

static inline unsigned char *
scan_back_to_entry_start(unsigned char *p) {
    for (; (p[0]&128) == 0; p--);
    return p;
}

static inline unsigned char *
skip_to_next_entry(unsigned char *p, unsigned char *end) {
    while (p < end && ((p[0] & 128) == 0)) {
        p++;
    }
    return p;
}


#define MAX_LINEAR_SEARCH 40

static Py_NO_INLINE int
get_exception_handler(PyCodeObject *code, int index, int *level, int *handler, int *lasti)
{
    unsigned char *start = (unsigned char *)PyBytes_AS_STRING(code->co_exceptiontable);
    unsigned char *end = start + PyBytes_GET_SIZE(code->co_exceptiontable);
    /* Invariants:
     * start_table == end_table OR
     * start_table points to a legal entry and end_table points
     * beyond the table or to a legal entry that is after index.
     */
    if (end - start > MAX_LINEAR_SEARCH) {
        int offset;
        parse_varint(start, &offset);
        if (offset > index) {
            return 0;
        }
        do {
            unsigned char * mid = start + ((end-start)>>1);
            mid = scan_back_to_entry_start(mid);
            parse_varint(mid, &offset);
            if (offset > index) {
                end = mid;
            }
            else {
                start = mid;
            }

        } while (end - start > MAX_LINEAR_SEARCH);
    }
    unsigned char *scan = start;
    while (scan < end) {
        int start_offset, size;
        scan = parse_varint(scan, &start_offset);
        if (start_offset > index) {
            break;
        }
        scan = parse_varint(scan, &size);
        if (start_offset + size > index) {
            scan = parse_varint(scan, handler);
            int depth_and_lasti;
            parse_varint(scan, &depth_and_lasti);
            *level = depth_and_lasti >> 1;
            *lasti = depth_and_lasti & 1;
            return 1;
        }
        scan = skip_to_next_entry(scan, end);
    }
    return 0;
}


#ifdef Py_DEBUG
#define ASSERT_WITHIN_STACK_BOUNDS(F, L) _Py_assert_within_stack_bounds(frame, stack_pointer, (F), (L))
#else
#define ASSERT_WITHIN_STACK_BOUNDS(F, L) (void)0
#endif

/* Logic for the raise statement (too complicated for inlining).
   This *consumes* a reference count to each of its arguments. */
static int
do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause)
{
    PyObject *type = NULL, *value = NULL;

    if (exc == NULL) {
        /* Reraise */
        _PyErr_StackItem *exc_info = _PyErr_GetTopmostException(tstate);
        exc = exc_info->exc_value;
        if (Py_IsNone(exc) || exc == NULL) {
            _PyErr_SetString(tstate, PyExc_RuntimeError,
                             "No active exception to reraise");
            return 0;
        }
        Py_INCREF(exc);
        assert(PyExceptionInstance_Check(exc));
        _PyErr_SetRaisedException(tstate, exc);
        return 1;
    }

    /* We support the following forms of raise:
       raise
       raise <instance>
       raise <type> */

    if (PyExceptionClass_Check(exc)) {
        type = exc;
        value = _PyObject_CallNoArgs(exc);
        if (value == NULL)
            goto raise_error;
        if (!PyExceptionInstance_Check(value)) {
            _PyErr_Format(tstate, PyExc_TypeError,
                          "calling %R should have returned an instance of "
                          "BaseException, not %R",
                          type, Py_TYPE(value));
             goto raise_error;
        }
    }
    else if (PyExceptionInstance_Check(exc)) {
        value = exc;
        type = PyExceptionInstance_Class(exc);
        Py_INCREF(type);
    }
    else {
        /* Not something you can raise.  You get an exception
           anyway, just not what you specified :-) */
        Py_DECREF(exc);
        _PyErr_SetString(tstate, PyExc_TypeError,
                         "exceptions must derive from BaseException");
        goto raise_error;
    }

    assert(type != NULL);
    assert(value != NULL);

    if (cause) {
        PyObject *fixed_cause;
        if (PyExceptionClass_Check(cause)) {
            fixed_cause = _PyObject_CallNoArgs(cause);
            if (fixed_cause == NULL)
                goto raise_error;
            if (!PyExceptionInstance_Check(fixed_cause)) {
                _PyErr_Format(tstate, PyExc_TypeError,
                              "calling %R should have returned an instance of "
                              "BaseException, not %R",
                              cause, Py_TYPE(fixed_cause));
                Py_DECREF(fixed_cause);
                goto raise_error;
            }
            Py_DECREF(cause);
        }
        else if (PyExceptionInstance_Check(cause)) {
            fixed_cause = cause;
        }
        else if (Py_IsNone(cause)) {
            Py_DECREF(cause);
            fixed_cause = NULL;
        }
        else {
            _PyErr_SetString(tstate, PyExc_TypeError,
                             "exception causes must derive from "
                             "BaseException");
            goto raise_error;
        }
        PyException_SetCause(value, fixed_cause);
    }

    _PyErr_SetObject(tstate, type, value);
    /* _PyErr_SetObject incref's its arguments */
    Py_DECREF(value);
    Py_DECREF(type);
    return 0;

raise_error:
    Py_XDECREF(value);
    Py_XDECREF(type);
    Py_XDECREF(cause);
    return 0;
}

/* Disable unused label warnings.  They are handy for debugging, even
   if computed gotos aren't used. */

/* TBD - what about other compilers? */
#if defined(__GNUC__) || defined(__clang__)
#  pragma GCC diagnostic push
#  pragma GCC diagnostic ignored "-Wunused-label"
#elif defined(_MSC_VER) /* MS_WINDOWS */
#  pragma warning(push)
#  pragma warning(disable:4102)
#endif

Coverage Report

Created: 2026-10-17 06:15

Line	Count	Source
1		#define _PY_INTERPRETER
2
3		#include "Python.h"
4		#include "pycore_abstract.h" // _PyIndex_Check()
5		#include "pycore_audit.h" // _PySys_Audit()
6		#include "pycore_backoff.h"
7		#include "pycore_call.h" // _PyObject_CallNoArgs()
8		#include "pycore_cell.h" // PyCell_GetRef()
9		#include "pycore_ceval.h" // SPECIAL___ENTER__
10		#include "pycore_code.h"
11		#include "pycore_dict.h"
12		#include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS
13		#include "pycore_floatobject.h" // _PyFloat_ExactDealloc()
14		#include "pycore_frame.h"
15		#include "pycore_function.h"
16		#include "pycore_genobject.h" // _PyCoro_GetAwaitableIter()
17		#include "pycore_import.h" // _PyImport_IsDefaultImportFunc()
18		#include "pycore_instruments.h"
19		#include "pycore_interpframe.h" // _PyFrame_SetStackPointer()
20		#include "pycore_interpolation.h" // _PyInterpolation_Build()
21		#include "pycore_intrinsics.h"
22		#include "pycore_jit.h"
23		#include "pycore_lazyimportobject.h"
24		#include "pycore_list.h" // _PyList_GetItemRef()
25		#include "pycore_long.h" // _PyLong_GetZero()
26		#include "pycore_moduleobject.h" // PyModuleObject
27		#include "pycore_object.h" // _PyObject_GC_TRACK()
28		#include "pycore_opcode_metadata.h" // EXTRA_CASES
29		#include "pycore_opcode_utils.h" // MAKE_FUNCTION_*
30		#include "pycore_optimizer.h" // _PyUOpExecutor_Type
31		#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_*
32		#include "pycore_pyerrors.h" // _PyErr_GetRaisedException()
33		#include "pycore_pystate.h" // _PyInterpreterState_GET()
34		#include "pycore_range.h" // _PyRangeIterObject
35		#include "pycore_setobject.h" // _PySet_Update()
36		#include "pycore_sliceobject.h" // _PyBuildSlice_ConsumeRefs
37		#include "pycore_sysmodule.h" // _PySys_GetOptionalAttrString()
38		#include "pycore_template.h" // _PyTemplate_Build()
39		#include "pycore_traceback.h" // _PyTraceBack_FromFrame
40		#include "pycore_tuple.h" // _PyTuple_ITEMS()
41		#include "pycore_uop_ids.h" // Uops
42
43		#include "dictobject.h"
44		#include "frameobject.h" // _PyInterpreterFrame_GetLine
45		#include "opcode.h"
46		#include "pydtrace.h"
47		#include "setobject.h"
48		#include "pycore_stackref.h"
49
50		#include <stdbool.h> // bool
51
52		#if !defined(Py_BUILD_CORE)
53		# error "ceval.c must be build with Py_BUILD_CORE define for best performance"
54		#endif
55
56		#if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS)
57		// GH-89279: The MSVC compiler does not inline these static inline functions
58		// in PGO build in _PyEval_EvalFrameDefault(), because this function is over
59		// the limit of PGO, and that limit cannot be configured.
60		// Define them as macros to make sure that they are always inlined by the
61		// preprocessor.
62
63		#undef Py_IS_TYPE
64		#define Py_IS_TYPE(ob, type) \
65	10.7G	(_PyObject_CAST(ob)->ob_type == (type))
66
67		#undef Py_XDECREF
68		#define Py_XDECREF(arg) \
69	693M	do { \
70	693M	PyObject *xop = _PyObject_CAST(arg); \
71	693M	if (xop != NULL) { \
72	305M	Py_DECREF(xop); \
73	305M	} \
74	693M	} while (0)
75
76		#ifndef Py_GIL_DISABLED
77
78		#undef Py_DECREF
79		#define Py_DECREF(arg) \
80	453M	do { \
81	453M	PyObject *op = _PyObject_CAST(arg); \
82	453M	if (_Py_IsImmortal(op)) { \
83	173M	_Py_DECREF_IMMORTAL_STAT_INC(); \
84	173M	break; \
85	173M	} \
86	453M	_Py_DECREF_STAT_INC(); \
87	280M	if (--op->ob_refcnt == 0) { \
88	110M	_PyReftracerTrack(op, PyRefTracer_DESTROY); \
89	110M	destructor dealloc = Py_TYPE(op)->tp_dealloc; \
90	110M	(*dealloc)(op); \
91	110M	} \
92	280M	} while (0)
93
94		#undef _Py_DECREF_SPECIALIZED
95		#define _Py_DECREF_SPECIALIZED(arg, dealloc) \
96	8.38M	do { \
97	8.38M	PyObject *op = _PyObject_CAST(arg); \
98	8.38M	if (_Py_IsImmortal(op)) { \
99	7.61M	_Py_DECREF_IMMORTAL_STAT_INC(); \
100	7.61M	break; \
101	7.61M	} \
102	8.38M	_Py_DECREF_STAT_INC(); \
103	771k	if (--op->ob_refcnt == 0) { \
104	257k	_PyReftracerTrack(op, PyRefTracer_DESTROY); \
105	257k	destructor d = (destructor)(dealloc); \
106	257k	d(op); \
107	257k	} \
108	771k	} while (0)
109
110		#else // Py_GIL_DISABLED
111
112		#undef Py_DECREF
113		#define Py_DECREF(arg) \
114		do { \
115		PyObject *op = _PyObject_CAST(arg); \
116		uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); \
117		if (local == _Py_IMMORTAL_REFCNT_LOCAL) { \
118		_Py_DECREF_IMMORTAL_STAT_INC(); \
119		break; \
120		} \
121		_Py_DECREF_STAT_INC(); \
122		if (_Py_IsOwnedByCurrentThread(op)) { \
123		local--; \
124		_Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); \
125		if (local == 0) { \
126		_Py_MergeZeroLocalRefcount(op); \
127		} \
128		} \
129		else { \
130		_Py_DecRefShared(op); \
131		} \
132		} while (0)
133
134		#undef _Py_DECREF_SPECIALIZED
135		#define _Py_DECREF_SPECIALIZED(arg, dealloc) Py_DECREF(arg)
136
137		#endif
138		#endif
139
140		static void
141		check_invalid_reentrancy(void)
142	261M	{
143		#if defined(Py_DEBUG) && defined(Py_GIL_DISABLED)
144		// In the free-threaded build, the interpreter must not be re-entered if
145		// the world-is-stopped. If so, that's a bug somewhere (quite likely in
146		// the painfully complex typeobject code).
147		PyInterpreterState *interp = _PyInterpreterState_GET();
148		assert(!interp->stoptheworld.world_stopped);
149		#endif
150	261M	}
151
152
153		#ifdef Py_DEBUG
154		static void
155		dump_item(_PyStackRef item)
156		{
157		if (PyStackRef_IsNull(item)) {
158		printf("<NULL>");
159		return;
160		}
161		if (PyStackRef_IsMalformed(item)) {
162		printf("<INVALID>");
163		return;
164		}
165		if (PyStackRef_IsTaggedInt(item)) {
166		printf("%" PRId64, (int64_t)PyStackRef_UntagInt(item));
167		return;
168		}
169		PyObject *obj = PyStackRef_AsPyObjectBorrow(item);
170		if (obj == NULL) {
171		printf("<nil>");
172		return;
173		}
174		// Don't call __repr__(), it might recurse into the interpreter.
175		printf("<%s at %p>", Py_TYPE(obj)->tp_name, (void *)obj);
176		}
177
178		static void
179		dump_stack(_PyInterpreterFrame frame, _PyStackRef stack_pointer)
180		{
181		_PyFrame_SetStackPointer(frame, stack_pointer);
182		_PyStackRef *locals_base = _PyFrame_GetLocalsArray(frame);
183		_PyStackRef *stack_base = _PyFrame_Stackbase(frame);
184		PyObject *exc = PyErr_GetRaisedException();
185		printf(" locals=[");
186		for (_PyStackRef *ptr = locals_base; ptr < stack_base; ptr++) {
187		if (ptr != locals_base) {
188		printf(", ");
189		}
190		dump_item(*ptr);
191		}
192		printf("]\n");
193		if (stack_pointer < stack_base) {
194		printf(" stack=%d\n", (int)(stack_pointer-stack_base));
195		}
196		else {
197		printf(" stack=[");
198		for (_PyStackRef *ptr = stack_base; ptr < stack_pointer; ptr++) {
199		if (ptr != stack_base) {
200		printf(", ");
201		}
202		dump_item(*ptr);
203		}
204		printf("]\n");
205		}
206		fflush(stdout);
207		PyErr_SetRaisedException(exc);
208		_PyFrame_GetStackPointer(frame);
209		}
210
211		#if defined(_Py_TIER2) && !defined(_Py_JIT) && defined(Py_DEBUG)
212		static void
213		dump_cache_item(_PyStackRef cache, int position, int depth)
214		{
215		if (position < depth) {
216		dump_item(cache);
217		}
218		else {
219		printf("---");
220		}
221		}
222		#endif
223
224		static void
225		lltrace_instruction(_PyInterpreterFrame *frame,
226		_PyStackRef *stack_pointer,
227		_Py_CODEUNIT *next_instr,
228		int opcode,
229		int oparg)
230		{
231		int offset = 0;
232		if (frame->owner < FRAME_OWNED_BY_INTERPRETER) {
233		dump_stack(frame, stack_pointer);
234		offset = (int)(next_instr - _PyFrame_GetBytecode(frame));
235		}
236		const char *opname = _PyOpcode_OpName[opcode];
237		assert(opname != NULL);
238		if (OPCODE_HAS_ARG((int)_PyOpcode_Deopt[opcode])) {
239		printf("%d: %s %d\n", offset * 2, opname, oparg);
240		}
241		else {
242		printf("%d: %s\n", offset * 2, opname);
243		}
244		fflush(stdout);
245		}
246
247		static void
248		lltrace_resume_frame(_PyInterpreterFrame *frame)
249		{
250		PyObject *fobj = PyStackRef_AsPyObjectBorrow(frame->f_funcobj);
251		if (!PyStackRef_CodeCheck(frame->f_executable) \|\|
252		fobj == NULL \|\|
253		!PyFunction_Check(fobj)
254		) {
255		printf("\nResuming frame.\n");
256		return;
257		}
258		PyFunctionObject f = (PyFunctionObject )fobj;
259		PyObject *exc = PyErr_GetRaisedException();
260		PyObject *name = Py_XNewRef(FT_ATOMIC_LOAD_PTR_ACQUIRE(f->func_qualname));
261		if (name == NULL) {
262		name = Py_XNewRef(FT_ATOMIC_LOAD_PTR_ACQUIRE(f->func_name));
263		}
264		printf("\nResuming frame");
265		if (name) {
266		printf(" for ");
267		if (PyObject_Print(name, stdout, 0) < 0) {
268		PyErr_Clear();
269		}
270		Py_DECREF(name);
271		}
272		PyObject *module = Py_XNewRef(FT_ATOMIC_LOAD_PTR_ACQUIRE(f->func_module));
273		if (module) {
274		printf(" in module ");
275		if (PyObject_Print(module, stdout, 0) < 0) {
276		PyErr_Clear();
277		}
278		Py_DECREF(module);
279		}
280		printf("\n");
281		fflush(stdout);
282		PyErr_SetRaisedException(exc);
283		}
284
285		static int
286		maybe_lltrace_resume_frame(_PyInterpreterFrame frame, PyObject globals)
287		{
288		if (globals == NULL) {
289		return 0;
290		}
291		if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) {
292		return 0;
293		}
294		int r = PyDict_Contains(globals, &_Py_ID(__lltrace__));
295		if (r < 0) {
296		PyErr_Clear();
297		return 0;
298		}
299		int lltrace = r * 5; // Levels 1-4 only trace uops
300		if (!lltrace) {
301		// Can also be controlled by environment variable
302		char *python_lltrace = Py_GETENV("PYTHON_LLTRACE");
303		if (python_lltrace != NULL && *python_lltrace >= '0') {
304		lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that
305		}
306		}
307		if (lltrace >= 5) {
308		lltrace_resume_frame(frame);
309		}
310		return lltrace;
311		}
312
313		#endif
314
315		static void monitor_reraise(PyThreadState *tstate,
316		_PyInterpreterFrame *frame,
317		_Py_CODEUNIT *instr);
318		static int monitor_stop_iteration(PyThreadState *tstate,
319		_PyInterpreterFrame *frame,
320		_Py_CODEUNIT *instr,
321		PyObject *value);
322		static void monitor_unwind(PyThreadState *tstate,
323		_PyInterpreterFrame *frame,
324		_Py_CODEUNIT *instr);
325		static int monitor_handled(PyThreadState *tstate,
326		_PyInterpreterFrame *frame,
327		_Py_CODEUNIT instr, PyObject exc);
328		static void monitor_throw(PyThreadState *tstate,
329		_PyInterpreterFrame *frame,
330		_Py_CODEUNIT *instr);
331
332		static int get_exception_handler(PyCodeObject , int, int, int, int);
333
334		#ifdef HAVE_ERRNO_H
335		#include <errno.h>
336		#endif
337
338		typedef struct {
339		_PyInterpreterFrame frame;
340		_PyStackRef stack[1];
341		} _PyEntryFrame;
342
343		static int
344		do_monitor_exc(PyThreadState tstate, _PyInterpreterFrame frame,
345		_Py_CODEUNIT *instr, int event)
346	0	{
347	0	assert(event < _PY_MONITORING_UNGROUPED_EVENTS);
348	0	if (_PyFrame_GetCode(frame)->co_flags & CO_NO_MONITORING_EVENTS) {
349	0	return 0;
350	0	}
351	0	PyObject *exc = PyErr_GetRaisedException();
352	0	assert(exc != NULL);
353	0	int err = _Py_call_instrumentation_arg(tstate, event, frame, instr, exc);
354	0	if (err == 0) {
355	0	PyErr_SetRaisedException(exc);
356	0	}
357	0	else {
358	0	assert(PyErr_Occurred());
359	0	Py_DECREF(exc);
360	0	}
361	0	return err;
362	0	}
363
364		static inline bool
365		no_tools_for_global_event(PyThreadState *tstate, int event)
366	153M	{
367	153M	return tstate->interp->monitors.tools[event] == 0;
368	153M	}
369
370		static inline bool
371		no_tools_for_local_event(PyThreadState tstate, _PyInterpreterFrame frame, int event)
372	153M	{
373	153M	assert(event < _PY_MONITORING_UNGROUPED_EVENTS);
374	153M	_PyCoMonitoringData *data = _PyFrame_GetCode(frame)->_co_monitoring;
375	153M	if (data) {
376	0	return data->active_monitors.tools[event] == 0;
377	0	}
378	153M	else {
379	153M	return no_tools_for_global_event(tstate, event);
380	153M	}
381	153M	}
382
383		static int
384		monitor_handled(PyThreadState *tstate,
385		_PyInterpreterFrame *frame,
386		_Py_CODEUNIT instr, PyObject exc)
387	29.0M	{
388	29.0M	if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_EXCEPTION_HANDLED)) {
389	29.0M	return 0;
390	29.0M	}
391	0	return _Py_call_instrumentation_arg(tstate, PY_MONITORING_EVENT_EXCEPTION_HANDLED, frame, instr, exc);
392	29.0M	}
393
394		static void
395		monitor_throw(PyThreadState *tstate,
396		_PyInterpreterFrame *frame,
397		_Py_CODEUNIT *instr)
398	36.6k	{
399	36.6k	if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_PY_THROW)) {
400	36.6k	return;
401	36.6k	}
402	0	do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_THROW);
403	0	}
404
405		static void
406		monitor_reraise(PyThreadState tstate, _PyInterpreterFrame frame,
407		_Py_CODEUNIT *instr)
408	4.93M	{
409	4.93M	if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_RERAISE)) {
410	4.93M	return;
411	4.93M	}
412	0	do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_RERAISE);
413	0	}
414
415		static int
416		monitor_stop_iteration(PyThreadState tstate, _PyInterpreterFrame frame,
417		_Py_CODEUNIT instr, PyObject value)
418	0	{
419	0	if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_STOP_ITERATION)) {
420	0	return 0;
421	0	}
422	0	assert(!PyErr_Occurred());
423	0	PyErr_SetObject(PyExc_StopIteration, value);
424	0	int res = do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_STOP_ITERATION);
425	0	if (res < 0) {
426	0	return res;
427	0	}
428	0	PyErr_SetRaisedException(NULL);
429	0	return 0;
430	0	}
431
432		static void
433		monitor_unwind(PyThreadState *tstate,
434		_PyInterpreterFrame *frame,
435		_Py_CODEUNIT *instr)
436	40.3M	{
437	40.3M	if (no_tools_for_local_event(tstate, frame, PY_MONITORING_EVENT_PY_UNWIND)) {
438	40.3M	return;
439	40.3M	}
440	0	do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_UNWIND);
441	0	}
442
443		static inline unsigned char *
444	20.2M	scan_back_to_entry_start(unsigned char *p) {
445	51.9M	for (; (p[0]&128) == 0; p--);
446	20.2M	return p;
447	20.2M	}
448
449		static inline unsigned char *
450	35.7M	skip_to_next_entry(unsigned char p, unsigned char end) {
451	140M	while (p < end && ((p[0] & 128) == 0)) {
452	104M	p++;
453	104M	}
454	35.7M	return p;
455	35.7M	}
456
457
458	89.6M	#define MAX_LINEAR_SEARCH 40
459
460		static Py_NO_INLINE int
461		get_exception_handler(PyCodeObject code, int index, int level, int handler, int lasti)
462	69.4M	{
463	69.4M	unsigned char start = (unsigned char )PyBytes_AS_STRING(code->co_exceptiontable);
464	69.4M	unsigned char *end = start + PyBytes_GET_SIZE(code->co_exceptiontable);
465		/* Invariants:
466		* start_table == end_table OR
467		* start_table points to a legal entry and end_table points
468		* beyond the table or to a legal entry that is after index.
469		*/
470	69.4M	if (end - start > MAX_LINEAR_SEARCH) {
471	9.37M	int offset;
472	9.37M	parse_varint(start, &offset);
473	9.37M	if (offset > index) {
474	231k	return 0;
475	231k	}
476	20.2M	do {
477	20.2M	unsigned char * mid = start + ((end-start)>>1);
478	20.2M	mid = scan_back_to_entry_start(mid);
479	20.2M	parse_varint(mid, &offset);
480	20.2M	if (offset > index) {
481	18.1M	end = mid;
482	18.1M	}
483	2.07M	else {
484	2.07M	start = mid;
485	2.07M	}
486
487	20.2M	} while (end - start > MAX_LINEAR_SEARCH);
488	9.14M	}
489	69.2M	unsigned char *scan = start;
490	104M	while (scan < end) {
491	70.2M	int start_offset, size;
492	70.2M	scan = parse_varint(scan, &start_offset);
493	70.2M	if (start_offset > index) {
494	5.41M	break;
495	5.41M	}
496	64.7M	scan = parse_varint(scan, &size);
497	64.7M	if (start_offset + size > index) {
498	29.0M	scan = parse_varint(scan, handler);
499	29.0M	int depth_and_lasti;
500	29.0M	parse_varint(scan, &depth_and_lasti);
501	29.0M	*level = depth_and_lasti >> 1;
502	29.0M	*lasti = depth_and_lasti & 1;
503	29.0M	return 1;
504	29.0M	}
505	35.7M	scan = skip_to_next_entry(scan, end);
506	35.7M	}
507	40.1M	return 0;
508	69.2M	}
509
510
511		#ifdef Py_DEBUG
512		#define ASSERT_WITHIN_STACK_BOUNDS(F, L) _Py_assert_within_stack_bounds(frame, stack_pointer, (F), (L))
513		#else
514	45.9G	#define ASSERT_WITHIN_STACK_BOUNDS(F, L) (void)0
515		#endif
516
517		/* Logic for the raise statement (too complicated for inlining).
518		This consumes a reference count to each of its arguments. */
519		static int
520		do_raise(PyThreadState tstate, PyObject exc, PyObject *cause)
521	19.7M	{
522	19.7M	PyObject type = NULL, value = NULL;
523
524	19.7M	if (exc == NULL) {
525		/* Reraise */
526	13.3k	_PyErr_StackItem *exc_info = _PyErr_GetTopmostException(tstate);
527	13.3k	exc = exc_info->exc_value;
528	13.3k	if (Py_IsNone(exc) \|\| exc == NULL) {
529	0	_PyErr_SetString(tstate, PyExc_RuntimeError,
530	0	"No active exception to reraise");
531	0	return 0;
532	0	}
533	13.3k	Py_INCREF(exc);
534	13.3k	assert(PyExceptionInstance_Check(exc));
535	13.3k	_PyErr_SetRaisedException(tstate, exc);
536	13.3k	return 1;
537	13.3k	}
538
539		/* We support the following forms of raise:
540		raise
541		raise <instance>
542		raise <type> */
543
544	19.7M	if (PyExceptionClass_Check(exc)) {
545	14.1M	type = exc;
546	14.1M	value = _PyObject_CallNoArgs(exc);
547	14.1M	if (value == NULL)
548	0	goto raise_error;
549	14.1M	if (!PyExceptionInstance_Check(value)) {
550	0	_PyErr_Format(tstate, PyExc_TypeError,
551	0	"calling %R should have returned an instance of "
552	0	"BaseException, not %R",
553	0	type, Py_TYPE(value));
554	0	goto raise_error;
555	0	}
556	14.1M	}
557	5.58M	else if (PyExceptionInstance_Check(exc)) {
558	5.58M	value = exc;
559	5.58M	type = PyExceptionInstance_Class(exc);
560	5.58M	Py_INCREF(type);
561	5.58M	}
562	0	else {
563		/* Not something you can raise. You get an exception
564		anyway, just not what you specified :-) */
565	0	Py_DECREF(exc);
566	0	_PyErr_SetString(tstate, PyExc_TypeError,
567	0	"exceptions must derive from BaseException");
568	0	goto raise_error;
569	0	}
570
571	19.7M	assert(type != NULL);
572	19.7M	assert(value != NULL);
573
574	19.7M	if (cause) {
575	8.90k	PyObject *fixed_cause;
576	8.90k	if (PyExceptionClass_Check(cause)) {
577	0	fixed_cause = _PyObject_CallNoArgs(cause);
578	0	if (fixed_cause == NULL)
579	0	goto raise_error;
580	0	if (!PyExceptionInstance_Check(fixed_cause)) {
581	0	_PyErr_Format(tstate, PyExc_TypeError,
582	0	"calling %R should have returned an instance of "
583	0	"BaseException, not %R",
584	0	cause, Py_TYPE(fixed_cause));
585	0	Py_DECREF(fixed_cause);
586	0	goto raise_error;
587	0	}
588	0	Py_DECREF(cause);
589	0	}
590	8.90k	else if (PyExceptionInstance_Check(cause)) {
591	2.93k	fixed_cause = cause;
592	2.93k	}
593	5.97k	else if (Py_IsNone(cause)) {
594	5.97k	Py_DECREF(cause);
595	5.97k	fixed_cause = NULL;
596	5.97k	}
597	0	else {
598	0	_PyErr_SetString(tstate, PyExc_TypeError,
599	0	"exception causes must derive from "
600	0	"BaseException");
601	0	goto raise_error;
602	0	}
603	8.90k	PyException_SetCause(value, fixed_cause);
604	8.90k	}
605
606	19.7M	_PyErr_SetObject(tstate, type, value);
607		/* _PyErr_SetObject incref's its arguments */
608	19.7M	Py_DECREF(value);
609	19.7M	Py_DECREF(type);
610	19.7M	return 0;
611
612	0	raise_error:
613	0	Py_XDECREF(value);
614	0	Py_XDECREF(type);
615		Py_XDECREF(cause);
616	0	return 0;
617	19.7M	}
618
619		/* Disable unused label warnings. They are handy for debugging, even
620		if computed gotos aren't used. */
621
622		/* TBD - what about other compilers? */
623		#if defined(__GNUC__) \|\| defined(__clang__)
624		# pragma GCC diagnostic push
625		# pragma GCC diagnostic ignored "-Wunused-label"
626		#elif defined(_MSC_VER) /* MS_WINDOWS */
627		# pragma warning(push)
628		# pragma warning(disable:4102)
629		#endif