/*

   +----------------------------------------------------------------------+

   | Zend JIT                                                             |

   +----------------------------------------------------------------------+

   | Copyright © The PHP Group and Contributors.                          |

   +----------------------------------------------------------------------+

   | This source file is subject to the Modified BSD License that is      |

   | bundled with this package in the file LICENSE, and is available      |

   | through the World Wide Web at <https://www.php.net/license/>.        |

   |                                                                      |

   | SPDX-License-Identifier: BSD-3-Clause                                |

   +----------------------------------------------------------------------+

   | Authors: Dmitry Stogov <dmitry@php.net>                              |

   +----------------------------------------------------------------------+

*/

#include "Zend/zend_cpuinfo.h"

#include "Zend/zend_types.h"

#include "Zend/zend_type_info.h"

#include "jit/ir/ir.h"

#include "jit/ir/ir_builder.h"

#include "jit/tls/zend_jit_tls.h"

#if defined(IR_TARGET_X86)

# define IR_REG_SP            4 /* IR_REG_RSP */

# define IR_REG_FP            5 /* IR_REG_RBP */

# define ZREG_FP              6 /* IR_REG_RSI */

# define ZREG_IP              7 /* IR_REG_RDI */

# define ZREG_FIRST_FPR       8

# define IR_REGSET_PRESERVED ((1<<3) | (1<<5) | (1<<6) | (1<<7)) /* all preserved registers */

# if ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL

#  error

# endif

#elif defined(IR_TARGET_X64)

# define IR_REG_SP            4 /* IR_REG_RSP */

# define IR_REG_FP            5 /* IR_REG_RBP */

# if ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL

/* Use the first two arg registers of the preserve_none calling convention for FP/IP

 * https://github.com/llvm/llvm-project/blob/68bfe91b5a34f80dbcc4f0a7fa5d7aa1cdf959c2/llvm/lib/Target/X86/X86CallingConv.td#L1029 */

#  define ZREG_FP             12 /* IR_REG_R12 */

#  define ZREG_IP             13 /* IR_REG_R13 */

# else

#  define ZREG_FP             14 /* IR_REG_R14 */

#  define ZREG_IP             15 /* IR_REG_R15 */

# endif

# define ZREG_FIRST_FPR      16

# if defined(_WIN64)

#  define IR_REGSET_PRESERVED ((1<<3) | (1<<5) | (1<<6) | (1<<7) | (1<<12) | (1<<13) | (1<<14) | (1<<15))

/*

#  define IR_REGSET_PRESERVED ((1<<3) | (1<<5) | (1<<6) | (1<<7) | (1<<12) | (1<<13) | (1<<14) | (1<<15) | \

                               (1<<(16+6)) | (1<<(16+7)) | (1<<(16+8)) | (1<<(16+9)) | (1<<(16+10)) | \

                               (1<<(16+11)) | (1<<(16+12)) | (1<<(16+13)) | (1<<(16+14)) | (1<<(16+15)))

*/

#  define IR_SHADOW_ARGS     32

# else

#  define IR_REGSET_PRESERVED ((1<<3) | (1<<5) | (1<<12) | (1<<13) | (1<<14) | (1<<15)) /* all preserved registers */

# endif

#elif defined(IR_TARGET_AARCH64)

# define IR_REG_SP           31 /* IR_REG_RSP */

# define IR_REG_LR           30 /* IR_REG_X30 */

# define IR_REG_FP           29 /* IR_REG_X29 */

# if ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL

/* Use the first two arg registers of the preserve_none calling convention for FP/IP

 * https://github.com/llvm/llvm-project/blob/68bfe91b5a34f80dbcc4f0a7fa5d7aa1cdf959c2/llvm/lib/Target/AArch64/AArch64CallingConvention.td#L541 */

#  define ZREG_FP             20 /* IR_REG_X20 */

#  define ZREG_IP             21 /* IR_REG_X21 */

# else

#  define ZREG_FP             27 /* IR_REG_X27 */

#  define ZREG_IP             28 /* IR_REG_X28 */

# endif

# define ZREG_FIRST_FPR      32

# define IR_REGSET_PRESERVED ((1<<19) | (1<<20) | (1<<21) | (1<<22) | (1<<23) | \

                              (1<<24) | (1<<25) | (1<<26) | (1<<27) | (1<<28)) /* all preserved registers */

#else

# error "Unknown IR target"

#endif

#define ZREG_RX ZREG_IP

#define OPTIMIZE_FOR_SIZE 0

/* IR builder defines */

#undef  _ir_CTX

#define _ir_CTX                 (&jit->ctx)

#if GCC_GLOBAL_REGS

# define IR_OPCODE_HANDLER_RET IR_VOID

#else

# define IR_OPCODE_HANDLER_RET IR_ADDR

#endif

#define ir_CONST_FUNC(_addr)    jit_CONST_FUNC(jit, (uintptr_t)(_addr), 0)

#define ir_CONST_FC_FUNC(_addr) jit_CONST_FUNC(jit, (uintptr_t)(_addr), IR_FASTCALL_FUNC)

#define ir_CAST_FC_FUNC(_addr)  ir_fold2(_ir_CTX, IR_OPT(IR_PROTO, IR_ADDR), (_addr), \

  ir_proto_0(_ir_CTX, IR_FASTCALL_FUNC, IR_I32))

# define ir_CONST_OPCODE_HANDLER_FUNC(_addr) \

  jit_CONST_OPCODE_HANDLER_FUNC(jit, _addr)

# define ir_CAST_OPCODE_HANDLER_FUNC(_addr)  ir_fold2(_ir_CTX, IR_OPT(IR_PROTO, IR_ADDR), (_addr), \

  ir_proto_0(_ir_CTX, IR_FASTCALL_FUNC, IR_OPCODE_HANDLER_RET))

#define ir_CONST_FUNC_PROTO(_addr, _proto) \

  ir_const_func_addr(_ir_CTX, (uintptr_t)(_addr), (_proto))

#undef  ir_ADD_OFFSET

#define ir_ADD_OFFSET(_addr, _offset) \

  jit_ADD_OFFSET(jit, _addr, _offset)

#ifdef ZEND_ENABLE_ZVAL_LONG64

# define IR_LONG           IR_I64

# define ir_CONST_LONG     ir_CONST_I64

# define ir_UNARY_OP_L     ir_UNARY_OP_I64

# define ir_BINARY_OP_L    ir_BINARY_OP_I64

# define ir_ADD_L          ir_ADD_I64

# define ir_SUB_L          ir_SUB_I64

# define ir_MUL_L          ir_MUL_I64

# define ir_DIV_L          ir_DIV_I64

# define ir_MOD_L          ir_MOD_I64

# define ir_NEG_L          ir_NEG_I64

# define ir_ABS_L          ir_ABS_I64

# define ir_SEXT_L         ir_SEXT_I64

# define ir_ZEXT_L         ir_ZEXT_I64

# define ir_TRUNC_L        ir_TRUNC_I64

# define ir_BITCAST_L      ir_BITCAST_I64

# define ir_FP2L           ir_FP2I64

# define ir_ADD_OV_L       ir_ADD_OV_I64

# define ir_SUB_OV_L       ir_SUB_OV_I64

# define ir_MUL_OV_L       ir_MUL_OV_I64

# define ir_NOT_L          ir_NOT_I64

# define ir_OR_L           ir_OR_I64

# define ir_AND_L          ir_AND_I64

# define ir_XOR_L          ir_XOR_I64

# define ir_SHL_L          ir_SHL_I64

# define ir_SHR_L          ir_SHR_I64

# define ir_SAR_L          ir_SAR_I64

# define ir_ROL_L          ir_ROL_I64

# define ir_ROR_L          ir_ROR_I64

# define ir_MIN_L          ir_MIN_I64

# define ir_MAX_L          ir_MAX_I64

# define ir_LOAD_L         ir_LOAD_I64

#else

# define IR_LONG           IR_I32

# define ir_CONST_LONG     ir_CONST_I32

# define ir_UNARY_OP_L     ir_UNARY_OP_I32

# define ir_BINARY_OP_L    ir_BINARY_OP_I32

# define ir_ADD_L          ir_ADD_I32

# define ir_SUB_L          ir_SUB_I32

# define ir_MUL_L          ir_MUL_I32

# define ir_DIV_L          ir_DIV_I32

# define ir_MOD_L          ir_MOD_I32

# define ir_NEG_L          ir_NEG_I32

# define ir_ABS_L          ir_ABS_I32

# define ir_SEXT_L         ir_SEXT_I32

# define ir_ZEXT_L         ir_ZEXT_I32

# define ir_TRUNC_L        ir_TRUNC_I32

# define ir_BITCAST_L      ir_BITCAST_I32

# define ir_FP2L           ir_FP2I32

# define ir_ADD_OV_L       ir_ADD_OV_I32

# define ir_SUB_OV_L       ir_SUB_OV_I32

# define ir_MUL_OV_L       ir_MUL_OV_I32

# define ir_NOT_L          ir_NOT_I32

# define ir_OR_L           ir_OR_I32

# define ir_AND_L          ir_AND_I32

# define ir_XOR_L          ir_XOR_I32

# define ir_SHL_L          ir_SHL_I32

# define ir_SHR_L          ir_SHR_I32

# define ir_SAR_L          ir_SAR_I32

# define ir_ROL_L          ir_ROL_I32

# define ir_ROR_L          ir_ROR_I32

# define ir_MIN_L          ir_MIN_I32

# define ir_MAX_L          ir_MAX_I32

# define ir_LOAD_L         ir_LOAD_I32

#endif

/* A helper structure to collect IT rers for the following use in (MERGE/PHI)_N */

typedef struct _ir_refs {

  uint32_t count;

  uint32_t limit;

  ir_ref   refs[] ZEND_ELEMENT_COUNT(count);

} ir_refs;

#define ir_refs_size(_n)          (offsetof(ir_refs, refs) + sizeof(ir_ref) * (_n))

#define ir_refs_init(_name, _n)   _name = alloca(ir_refs_size(_n)); \

                                  do {_name->count = 0; _name->limit = (_n);} while (0)

static void ir_refs_add(ir_refs *refs, ir_ref ref)

{

  ir_ref *ptr;

  ZEND_ASSERT(refs->count < refs->limit);

  ptr = refs->refs;

  ptr[refs->count++] = ref;

}

static size_t zend_jit_trace_prologue_size = (size_t)-1;

#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)

static uint32_t allowed_opt_flags = 0;

static uint32_t default_mflags = 0;

#endif

static bool delayed_call_chain = false; // TODO: remove this var (use jit->delayed_call_level) ???

#ifdef ZTS

static size_t tsrm_ls_cache_tcb_offset = 0;

static size_t tsrm_tls_index = -1;

static size_t tsrm_tls_offset = -1;

# define EG_TLS_OFFSET(field) \

  (executor_globals_offset + offsetof(zend_executor_globals, field))

# define CG_TLS_OFFSET(field) \

  (compiler_globals_offset + offsetof(zend_compiler_globals, field))

# define jit_EG(_field) \

  ir_ADD_OFFSET(jit_TLS(jit), EG_TLS_OFFSET(_field))

# define jit_CG(_field) \

  ir_ADD_OFFSET(jit_TLS(jit), CG_TLS_OFFSET(_field))

#else

# define jit_EG(_field) \

  ir_CONST_ADDR(&EG(_field))

# define jit_CG(_field) \

  ir_CONST_ADDR(&CG(_field))

#endif

#define jit_CALL(_call, _field) \

  ir_ADD_OFFSET(_call, offsetof(zend_execute_data, _field))

#define jit_EX(_field) \

  jit_CALL(jit_FP(jit), _field)

#define jit_RX(_field) \

  jit_CALL(jit_IP(jit), _field)

#define JIT_STUBS(_) \

  _(exception_handler,              IR_SKIP_PROLOGUE) \

  _(exception_handler_undef,        IR_SKIP_PROLOGUE) \

  _(exception_handler_free_op2,     IR_SKIP_PROLOGUE) \

  _(exception_handler_free_op1_op2, IR_SKIP_PROLOGUE) \

  _(interrupt_handler,              IR_SKIP_PROLOGUE) \

  _(leave_function_handler,         IR_SKIP_PROLOGUE) \

  _(negative_shift,                 IR_SKIP_PROLOGUE) \

  _(mod_by_zero,                    IR_SKIP_PROLOGUE) \

  _(invalid_this,                   IR_SKIP_PROLOGUE) \

  _(undefined_function,             IR_SKIP_PROLOGUE) \

  _(throw_cannot_pass_by_ref,       IR_SKIP_PROLOGUE) \

  _(icall_throw,                    IR_SKIP_PROLOGUE) \

  _(leave_throw,                    IR_SKIP_PROLOGUE) \

  _(hybrid_runtime_jit,             IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \

  _(hybrid_profile_jit,             IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \

  _(hybrid_func_hot_counter,        IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \

  _(hybrid_loop_hot_counter,        IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \

  _(hybrid_func_trace_counter,      IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \

  _(hybrid_ret_trace_counter,       IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \

  _(hybrid_loop_trace_counter,      IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \

  _(trace_halt,                     IR_SKIP_PROLOGUE) \

  _(trace_escape,                   IR_SKIP_PROLOGUE) \

  _(trace_exit,                     IR_SKIP_PROLOGUE) \

  _(undefined_offset,               IR_FUNCTION | IR_FASTCALL_FUNC) \

  _(undefined_key,                  IR_FUNCTION | IR_FASTCALL_FUNC) \

  _(cannot_add_element,             IR_FUNCTION | IR_FASTCALL_FUNC) \

  _(assign_const,                   IR_FUNCTION | IR_FASTCALL_FUNC) \

  _(assign_tmp,                     IR_FUNCTION | IR_FASTCALL_FUNC) \

  _(assign_var,                     IR_FUNCTION | IR_FASTCALL_FUNC) \

  _(assign_cv_noref,                IR_FUNCTION | IR_FASTCALL_FUNC) \

  _(assign_cv,                      IR_FUNCTION | IR_FASTCALL_FUNC) \

  _(new_array,                      IR_FUNCTION | IR_FASTCALL_FUNC) \

#define JIT_STUB_ID(name, flags) \

  jit_stub_ ## name,

#define JIT_STUB_FORWARD(name, flags) \

  static int zend_jit_ ## name ## _stub(zend_jit_ctx *jit);

#define JIT_STUB(name, flags) \

  {JIT_STUB_PREFIX #name, zend_jit_ ## name ## _stub, flags},

typedef enum _jit_stub_id {

  JIT_STUBS(JIT_STUB_ID)

  jit_last_stub

} jit_stub_id;

typedef struct _zend_jit_reg_var {

  ir_ref   ref;

  uint32_t flags;

} zend_jit_reg_var;

typedef struct _zend_jit_ctx {

  ir_ctx               ctx;

  const zend_op       *last_valid_opline;

  bool                 use_last_valid_opline;

  bool                 track_last_valid_opline;

  bool                 reuse_ip;

  uint32_t             delayed_call_level;

  int                  b;           /* current basic block number or -1 */

#ifdef ZTS

  ir_ref               tls;

#endif

  ir_ref               fp;

  ir_ref               poly_func_ref; /* restored from parent trace snapshot */

  ir_ref               poly_this_ref; /* restored from parent trace snapshot */

  ir_ref               trace_loop_ref;

  ir_ref               return_inputs;

  const zend_op_array *op_array;

  const zend_op_array *current_op_array;

  zend_ssa            *ssa;

  zend_string         *name;

  ir_ref              *bb_start_ref;      /* PHP BB -> IR ref mapping */

  ir_ref              *bb_predecessors;   /* PHP BB -> index in bb_edges -> IR refs of predessors */

  ir_ref              *bb_edges;

  zend_jit_trace_info *trace;

  zend_jit_reg_var    *ra;

  int                  delay_var;

  ir_refs             *delay_refs;

  ir_ref               eg_exception_addr;

  ir_ref               stub_addr[jit_last_stub];

} zend_jit_ctx;

typedef int8_t zend_reg;

typedef struct _zend_jit_registers_buf {

#if defined(IR_TARGET_X64)

  uint64_t gpr[16]; /* general purpose integer register */

  double   fpr[16]; /* floating point registers */

#elif defined(IR_TARGET_X86)

  uint32_t gpr[8]; /* general purpose integer register */

  double   fpr[8]; /* floating point registers */

#elif defined (IR_TARGET_AARCH64)

  uint64_t gpr[32]; /* general purpose integer register */

  double   fpr[32]; /* floating point registers */

#else

# error "Unknown IR target"

#endif

} zend_jit_registers_buf;

/* Keep 32 exit points in a single code block */

#define ZEND_JIT_EXIT_POINTS_SPACING   4  // push byte + short jmp = bytes

#define ZEND_JIT_EXIT_POINTS_PER_GROUP 32 // number of continuous exit points

static uint32_t zend_jit_exit_point_by_addr(const void *addr);

int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs);

static int zend_jit_assign_to_variable(zend_jit_ctx   *jit,

                                       const zend_op  *opline,

                                       zend_jit_addr   var_use_addr,

                                       zend_jit_addr   var_addr,

                                       uint32_t        var_info,

                                       uint32_t        var_def_info,

                                       uint8_t         val_type,

                                       zend_jit_addr   val_addr,

                                       uint32_t        val_info,

                                       zend_jit_addr   res_addr,

                                       zend_jit_addr   ref_addr,

                                       bool       check_exception);

static ir_ref jit_CONST_FUNC(zend_jit_ctx *jit, uintptr_t addr, uint16_t flags);

static ir_ref jit_ADD_OFFSET(zend_jit_ctx *jit, ir_ref addr, uintptr_t offset);

static void zend_jit_preserve_parent_regs(zend_jit_ctx *jit,

                                          zend_ssa *ssa,

                                          zend_jit_trace_info *parent,

                                          uint32_t exit_num);

typedef struct _zend_jit_stub {

  const char *name;

  int (*stub)(zend_jit_ctx *jit);

  uint32_t flags;

} zend_jit_stub;

JIT_STUBS(JIT_STUB_FORWARD)

static const zend_jit_stub zend_jit_stubs[] = {

  JIT_STUBS(JIT_STUB)

};

#if defined(_WIN32) || defined(IR_TARGET_AARCH64)

/* We keep addresses in SHM to share them between sepaeate processes (on Windows) or to support veneers (on AArch64) */

static void** zend_jit_stub_handlers = NULL;

#else

static void* zend_jit_stub_handlers[sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0])];

#endif

#if defined(IR_TARGET_AARCH64)

# ifdef __FreeBSD__

/* https://github.com/freebsd/freebsd-src/blob/c52ca7dd09066648b1cc40f758289404d68ab886/libexec/rtld-elf/aarch64/reloc.c#L180-L184 */

typedef struct TLSDescriptor {

  void*   thunk;

  int     index;

  size_t  offset;

} TLSDescriptor;

# endif

#define IR_HAS_VENEERS (1U<<31) /* IR_RESERVED_FLAG_1 */

static const void *zend_jit_get_veneer(ir_ctx *ctx, const void *addr)

{

  int i, count = sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0]);

  for (i = 0; i < count; i++) {

    if (zend_jit_stub_handlers[i] == addr) {

      return zend_jit_stub_handlers[count + i];

    }

  }

  if (((zend_jit_ctx*)ctx)->trace

   && (void*)addr >= dasm_buf && (void*)addr < dasm_end) {

    uint32_t exit_point = zend_jit_exit_point_by_addr(addr);

    if (exit_point != (uint32_t)-1) {

      zend_jit_trace_info *t = ((zend_jit_ctx*)ctx)->trace;

      ZEND_ASSERT(exit_point < t->exit_count);

      return (const void*)((char*)ctx->deoptimization_exits_base + (exit_point * 4));

    }

  }

  return NULL;

}

static bool zend_jit_set_veneer(ir_ctx *ctx, const void *addr, const void *veneer)

{

  int i, count = sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0]);

  uint32_t exit_point = zend_jit_exit_point_by_addr(addr);

  if (exit_point != (uint32_t)-1) {

    return true;

  }

  for (i = 0; i < count; i++) {

    if (zend_jit_stub_handlers[i] == addr) {

      const void **ptr = (const void**)&zend_jit_stub_handlers[count + i];

      *ptr = veneer;

      ctx->flags2 |= IR_HAS_VENEERS;

#ifdef HAVE_CAPSTONE

      int64_t offset;

        if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) {

        const char *name = ir_disasm_find_symbol((uint64_t)(uintptr_t)addr, &offset);

        if (name && !offset) {

          if (strstr(name, "@veneer") == NULL) {

            char *new_name;

            zend_spprintf(&new_name, 0, "%s@veneer", name);

            ir_disasm_add_symbol(new_name, (uint64_t)(uintptr_t)veneer, 4);

            efree(new_name);

          } else {

            ir_disasm_add_symbol(name, (uint64_t)(uintptr_t)veneer, 4);

          }

        }

      }

#endif

      return true;

    }

  }

  return false;

}

static void zend_jit_commit_veneers(void)

{

  int i, count = sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0]);

  for (i = 0; i < count; i++) {

    if (zend_jit_stub_handlers[count + i]) {

      zend_jit_stub_handlers[i] = zend_jit_stub_handlers[count + i];

      zend_jit_stub_handlers[count + i] = NULL;

    }

  }

}

static void zend_jit_discard_veneers(void)

{

  int i, count = sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0]);

  for (i = 0; i < count; i++) {

    zend_jit_stub_handlers[count + i] = NULL;

  }

}

#endif

static bool zend_jit_prefer_const_addr_load(zend_jit_ctx *jit, uintptr_t addr)

{

#if defined(IR_TARGET_X86)

  return false; /* always use immediate value */

#elif defined(IR_TARGET_X64)

  return addr > 0xffffffff; /* prefer loading long constant from memory */

#elif defined(IR_TARGET_AARCH64)

  return addr > 0xffff;

#else

# error "Unknown IR target"

#endif

}

static const char* zend_reg_name(int8_t reg)

{

  return ir_reg_name(reg, ir_reg_is_int(reg) ? IR_LONG : IR_DOUBLE);

}

/* IR helpers */

#ifdef ZTS

static void * ZEND_FASTCALL zend_jit_get_tsrm_ls_cache(void)

{

  return _tsrm_ls_cache;

}

static ir_ref jit_TLS(zend_jit_ctx *jit)

{

  ZEND_ASSERT(jit->ctx.control);

  if (jit->tls) {

    /* Emit "TLS" once for basic block */

    ir_insn *insn;

    ir_ref ref = jit->ctx.control;

    while (1) {

      if (ref == jit->tls) {

        return jit->tls;

      }

      insn = &jit->ctx.ir_base[ref];

      if (insn->op >= IR_START || insn->op == IR_CALL) {

        break;

      }

      ref = insn->op1;

    }

  }

  if (tsrm_ls_cache_tcb_offset == 0 && tsrm_tls_index == -1) {

    jit->tls = ir_CALL(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_get_tsrm_ls_cache));

  } else {

# ifdef ZEND_OPCACHE_SHM_REATTACHMENT

    /* Different processes may load TLS variables in a different order, so

     * they may have a different module index. Fetch the module index at

     * runtime from tsrm_tls_index. Its address is stable as it's in the same

     * module as execute_ex (whose address is checked before SHM reattachment). */

#  ifdef _WIN64

    const int offset = 0x58;

#  else

    const int offset = 0x2c;

#  endif

    /* NtCurrentTeb()->ThreadLocalStoragePointer */

    ir_ref tls_array = ir_TLS(-1, offset);

    ir_ref index = ir_LOAD_A(ir_CONST_ADDR(&tsrm_tls_index));

    jit->tls = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(ir_ADD_A(tls_array, index)), tsrm_tls_offset));

# else

    jit->tls = ir_TLS(

        tsrm_ls_cache_tcb_offset ? tsrm_ls_cache_tcb_offset : tsrm_tls_index,

        tsrm_ls_cache_tcb_offset ? IR_NULL : tsrm_tls_offset);

# endif

  }

  return jit->tls;

}

#endif

static ir_ref jit_CONST_FUNC(zend_jit_ctx *jit, uintptr_t addr, uint16_t flags)

{

#if defined(IR_TARGET_X86)

  /* TODO: dummy prototype (only flags matter) ??? */

  ir_ref proto = flags ? ir_proto_0(&jit->ctx, flags, IR_I32) : 0;

#else

  ir_ref proto = 0;

#endif

  return ir_const_func_addr(&jit->ctx, addr, proto);

}

static ir_ref jit_CONST_OPCODE_HANDLER_FUNC(zend_jit_ctx *jit, zend_vm_opcode_handler_t handler)

{

  return jit_CONST_FUNC(jit, (uintptr_t)handler, IR_FASTCALL_FUNC);

}

static ir_ref jit_ADD_OFFSET(zend_jit_ctx *jit, ir_ref addr, uintptr_t offset)

{

  if (offset) {

    addr = ir_ADD_A(addr, ir_CONST_ADDR(offset));

  }

  return addr;

}

static ir_ref jit_EG_exception(zend_jit_ctx *jit)

{

#ifdef ZTS

  return jit_EG(exception);

#else

  ir_ref ref = jit->eg_exception_addr;

  if (UNEXPECTED(!ref)) {

    ref = ir_CONST_ADDR(&EG(exception));

    jit->eg_exception_addr = ref;

  }

  return ref;

#endif

}

static ir_ref jit_STUB_ADDR(zend_jit_ctx *jit, jit_stub_id id)

{

  ir_ref ref = jit->stub_addr[id];

  if (UNEXPECTED(!ref)) {

    ref = ir_CONST_ADDR(zend_jit_stub_handlers[id]);

    jit->stub_addr[id] = ref;

  }

  return ref;

}

static ir_ref jit_STUB_FUNC_ADDR(zend_jit_ctx *jit, jit_stub_id id, uint16_t flags)

{

  ir_ref ref = jit->stub_addr[id];

  if (UNEXPECTED(!ref)) {

    ref = jit_CONST_FUNC(jit, (uintptr_t)zend_jit_stub_handlers[id], flags);

    jit->stub_addr[id] = ref;

  }

  return ref;

}

static void jit_SNAPSHOT(zend_jit_ctx *jit, ir_ref addr)

{

  if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame)) {

    const void *ptr = (const void*)jit->ctx.ir_base[addr].val.addr;

    const zend_op_array *op_array = &JIT_G(current_frame)->func->op_array;

    uint32_t stack_size = op_array->last_var + op_array->T;

    if (ptr == zend_jit_stub_handlers[jit_stub_exception_handler]

       || ptr == zend_jit_stub_handlers[jit_stub_exception_handler_undef]

       || ptr == zend_jit_stub_handlers[jit_stub_exception_handler_free_op1_op2]

       || ptr == zend_jit_stub_handlers[jit_stub_exception_handler_free_op2]

       || ptr == zend_jit_stub_handlers[jit_stub_interrupt_handler]

       || ptr == zend_jit_stub_handlers[jit_stub_leave_function_handler]

       || ptr == zend_jit_stub_handlers[jit_stub_negative_shift]

       || ptr == zend_jit_stub_handlers[jit_stub_mod_by_zero]

       || ptr == zend_jit_stub_handlers[jit_stub_invalid_this]

       || ptr == zend_jit_stub_handlers[jit_stub_undefined_function]

       || ptr == zend_jit_stub_handlers[jit_stub_throw_cannot_pass_by_ref]

       || ptr == zend_jit_stub_handlers[jit_stub_icall_throw]

       || ptr == zend_jit_stub_handlers[jit_stub_leave_throw]

       || ptr == zend_jit_stub_handlers[jit_stub_trace_halt]

       || ptr == zend_jit_stub_handlers[jit_stub_trace_escape]) {

      /* This is a GUARD that trigger exit through a stub code (without deoptimization) */

      return;

    }

    /* Check if we need snapshot entries for polymorphic method call */

    zend_jit_trace_info *t = jit->trace;

    uint32_t exit_point = 0, n = 0;

    if (addr < 0) {

      /* addr is not always the address of the *last* exit point,

       * so we can not optimize this to 'exit_point = t->exit_count-1' */

      exit_point = zend_jit_exit_point_by_addr(ptr);

      ZEND_ASSERT(exit_point != -1);

      if (t->exit_info[exit_point].flags & ZEND_JIT_EXIT_METHOD_CALL) {

        n = 2;

      }

    }

    if (stack_size || n) {

      zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;

      uint32_t snapshot_size, i;

      snapshot_size = stack_size;

      while (snapshot_size > 0) {

        ir_ref ref = STACK_REF(stack, snapshot_size - 1);

        if (!ref || ref == IR_NULL || (STACK_FLAGS(stack, snapshot_size - 1) & (/*ZREG_LOAD|*/ZREG_STORE))) {

          snapshot_size--;

        } else {

          break;

        }

      }

      if (snapshot_size || n) {

        ir_ref snapshot;

        snapshot = ir_SNAPSHOT(snapshot_size + n);

        for (i = 0; i < snapshot_size; i++) {

          ir_ref ref = STACK_REF(stack, i);

          if (!ref || ref == IR_NULL || (STACK_FLAGS(stack, i) & (/*ZREG_LOAD|*/ZREG_STORE))) {

            ref = IR_UNUSED;

          }

          ir_SNAPSHOT_SET_OP(snapshot, i + 1, ref);

        }

        if (n) {

          ir_SNAPSHOT_SET_OP(snapshot, snapshot_size + 1, t->exit_info[exit_point].poly_func.ref);

          ir_SNAPSHOT_SET_OP(snapshot, snapshot_size + 2, t->exit_info[exit_point].poly_this.ref);

        }

      }

    }

  }

}

static int32_t _add_trace_const(zend_jit_trace_info *t, int64_t val)

{

  int32_t i;

  for (i = 0; i < t->consts_count; i++) {

    if (t->constants[i].i == val) {

      return i;

    }

  }

  ZEND_ASSERT(i < 0x7fffffff);

  t->consts_count = i + 1;

  t->constants = erealloc(t->constants, (i + 1) * sizeof(zend_jit_exit_const));

  t->constants[i].i = val;

  return i;

}

uint32_t zend_jit_duplicate_exit_point(ir_ctx *ctx, zend_jit_trace_info *t, uint32_t exit_point, ir_ref snapshot_ref)

{

  uint32_t stack_size, stack_offset;

  uint32_t new_exit_point = t->exit_count;

  if (new_exit_point >= ZEND_JIT_TRACE_MAX_EXITS) {

    ctx->status = -ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS;

    return exit_point;

  }

  t->exit_count++;

  memcpy(&t->exit_info[new_exit_point], &t->exit_info[exit_point], sizeof(zend_jit_trace_exit_info));

  stack_size = t->exit_info[new_exit_point].stack_size;

  if (stack_size != 0) {

    stack_offset = t->stack_map_size;

    t->stack_map_size += stack_size;

    // TODO: reduce number of reallocations ???

    t->stack_map = erealloc(t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));

    memcpy(t->stack_map + stack_offset, t->stack_map + t->exit_info[new_exit_point].stack_offset, stack_size * sizeof(zend_jit_trace_stack));

    t->exit_info[new_exit_point].stack_offset = stack_offset;

  }

  t->exit_info[new_exit_point].flags &= ~ZEND_JIT_EXIT_FIXED;

  return new_exit_point;

}

static void zend_jit_resolve_ref_snapshot(zend_jit_ref_snapshot *dest, ir_ctx *ctx, ir_ref snapshot_ref, ir_insn *snapshot, int op)

{

  int8_t *reg_ops = ctx->regs[snapshot_ref];

  ZEND_ASSERT(reg_ops[op] != ZREG_NONE);

  int8_t reg = reg_ops[op];

  int32_t offset;

  if (IR_REG_SPILLED(reg)) {

    reg = ((ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FP : IR_REG_SP) | IR_REG_SPILL_LOAD;

    offset = ir_get_spill_slot_offset(ctx, ir_insn_op(snapshot, op));

  } else {

    offset = 0;

  }

  dest->reg = reg;

  dest->offset = offset;

}

static bool zend_jit_ref_snapshot_equals(const zend_jit_ref_snapshot *a, const zend_jit_ref_snapshot *b)

{

  return a->reg == b->reg

    && (!IR_REG_SPILLED(a->reg) || (a->offset == b->offset));

}

void *zend_jit_snapshot_handler(ir_ctx *ctx, ir_ref snapshot_ref, ir_insn *snapshot, void *addr)

{

  zend_jit_trace_info *t = ((zend_jit_ctx*)ctx)->trace;

  uint32_t exit_point, exit_flags;

  ir_ref n = snapshot->inputs_count;

  ir_ref i;

  exit_point = zend_jit_exit_point_by_addr(addr);

  ZEND_ASSERT(exit_point < t->exit_count);

  exit_flags = t->exit_info[exit_point].flags;

  if (exit_flags & ZEND_JIT_EXIT_METHOD_CALL) {

    zend_jit_ref_snapshot func, this;

    zend_jit_resolve_ref_snapshot(&func, ctx, snapshot_ref, snapshot, n - 1);

    zend_jit_resolve_ref_snapshot(&this, ctx, snapshot_ref, snapshot, n);

    if ((exit_flags & ZEND_JIT_EXIT_FIXED)

     && (!zend_jit_ref_snapshot_equals(&t->exit_info[exit_point].poly_func, &func)

       || !zend_jit_ref_snapshot_equals(&t->exit_info[exit_point].poly_this, &this))) {

      exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);

      addr = (void*)zend_jit_trace_get_exit_addr(exit_point);

      exit_flags &= ~ZEND_JIT_EXIT_FIXED;

    }

    t->exit_info[exit_point].poly_func = func;

    t->exit_info[exit_point].poly_this = this;

    n -= 2;

  }

  for (i = 2; i <= n; i++) {

    ir_ref ref = ir_insn_op(snapshot, i);

    if (ref) {

      int8_t *reg_ops = ctx->regs[snapshot_ref];

      int8_t reg = reg_ops[i];

      ir_ref var = i - 2;

      ZEND_ASSERT(var < t->exit_info[exit_point].stack_size);

      if (t->stack_map[t->exit_info[exit_point].stack_offset + var].flags == ZREG_ZVAL_COPY) {

        ZEND_ASSERT(reg != ZREG_NONE);

        if ((exit_flags & ZEND_JIT_EXIT_FIXED)

         && t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != IR_REG_NUM(reg)) {

          exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);

          addr = (void*)zend_jit_trace_get_exit_addr(exit_point);

          exit_flags &= ~ZEND_JIT_EXIT_FIXED;

        }

        t->stack_map[t->exit_info[exit_point].stack_offset + var].reg = IR_REG_NUM(reg);

      } else if (t->stack_map[t->exit_info[exit_point].stack_offset + var].flags != ZREG_CONST) {

        ZEND_ASSERT(t->stack_map[t->exit_info[exit_point].stack_offset + var].type == IS_LONG ||

          t->stack_map[t->exit_info[exit_point].stack_offset + var].type == IS_DOUBLE);

        if (ref > 0) {

          if (reg != ZREG_NONE) {

            if (reg & IR_REG_SPILL_LOAD) {

              ZEND_ASSERT(!(reg & IR_REG_SPILL_SPECIAL));

              /* spill slot on a CPU stack */

              if ((exit_flags & ZEND_JIT_EXIT_FIXED)

               && (t->stack_map[t->exit_info[exit_point].stack_offset + var].ref != ref

                || t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != ZREG_NONE

                || !(t->stack_map[t->exit_info[exit_point].stack_offset + var].flags & ZREG_SPILL_SLOT))) {

                exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);

                addr = (void*)zend_jit_trace_get_exit_addr(exit_point);

                exit_flags &= ~ZEND_JIT_EXIT_FIXED;

              }

              t->stack_map[t->exit_info[exit_point].stack_offset + var].ref = ref;

              t->stack_map[t->exit_info[exit_point].stack_offset + var].reg = ZREG_NONE;

              t->stack_map[t->exit_info[exit_point].stack_offset + var].flags |= ZREG_SPILL_SLOT;

            } else if (reg & IR_REG_SPILL_SPECIAL) {

              /* spill slot on a VM stack */

              if ((exit_flags & ZEND_JIT_EXIT_FIXED)

               && (t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != ZREG_NONE

               || t->stack_map[t->exit_info[exit_point].stack_offset + var].flags != ZREG_TYPE_ONLY)) {

                exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);

                addr = (void*)zend_jit_trace_get_exit_addr(exit_point);

                exit_flags &= ~ZEND_JIT_EXIT_FIXED;

              }

              t->stack_map[t->exit_info[exit_point].stack_offset + var].reg = ZREG_NONE;

              t->stack_map[t->exit_info[exit_point].stack_offset + var].flags = ZREG_TYPE_ONLY;

            } else {

              if ((exit_flags & ZEND_JIT_EXIT_FIXED)

               && (t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != IR_REG_NUM(reg)

               || (t->stack_map[t->exit_info[exit_point].stack_offset + var].flags & ~(ZREG_LOAD|ZREG_STORE|ZREG_LAST_USE)))) {

                exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);

                addr = (void*)zend_jit_trace_get_exit_addr(exit_point);

                exit_flags &= ~ZEND_JIT_EXIT_FIXED;

              }

              t->stack_map[t->exit_info[exit_point].stack_offset + var].reg = IR_REG_NUM(reg);

              t->stack_map[t->exit_info[exit_point].stack_offset + var].flags &= (ZREG_LOAD|ZREG_STORE|ZREG_LAST_USE);

            }

          } else {

            if ((exit_flags & ZEND_JIT_EXIT_FIXED)

             && (t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != ZREG_NONE

             || t->stack_map[t->exit_info[exit_point].stack_offset + var].flags != ZREG_TYPE_ONLY)) {

              exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);

              addr = (void*)zend_jit_trace_get_exit_addr(exit_point);

              exit_flags &= ~ZEND_JIT_EXIT_FIXED;

            }

            t->stack_map[t->exit_info[exit_point].stack_offset + var].reg = ZREG_NONE;

            t->stack_map[t->exit_info[exit_point].stack_offset + var].flags = ZREG_TYPE_ONLY;

          }

        } else if (!(exit_flags & ZEND_JIT_EXIT_FIXED)) {

          int32_t idx = _add_trace_const(t, ctx->ir_base[ref].val.i64);

          t->stack_map[t->exit_info[exit_point].stack_offset + var].flags = ZREG_CONST;

          t->stack_map[t->exit_info[exit_point].stack_offset + var].ref = idx;

        }

      }

    }

  }

  t->exit_info[exit_point].flags |= ZEND_JIT_EXIT_FIXED;

  return addr;

}

static void jit_SIDE_EXIT(zend_jit_ctx *jit, ir_ref addr)

{

  jit_SNAPSHOT(jit, addr);

  ir_IJMP(addr);

}

/* PHP JIT helpers */

static ir_ref jit_EMALLOC(zend_jit_ctx *jit, size_t size, const zend_op_array *op_array, const zend_op *opline)

{

#if ZEND_DEBUG

  return ir_CALL_5(IR_ADDR, ir_CONST_FC_FUNC(_emalloc),

    ir_CONST_ADDR(size),

    op_array->filename ? ir_CONST_ADDR(op_array->filename->val) : IR_NULL,

    ir_CONST_U32(opline ? opline->lineno : 0),

    IR_NULL,

    ir_CONST_U32(0));

#elif defined(HAVE_BUILTIN_CONSTANT_P)

  if (size > 24 && size <= 32) {

    return ir_CALL(IR_ADDR, ir_CONST_FC_FUNC(_emalloc_32));

  } else {

    return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_emalloc), ir_CONST_ADDR(size));

  }

#else

  return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_emalloc), ir_CONST_ADDR(size));

#endif

}

static ir_ref jit_EFREE(zend_jit_ctx *jit, ir_ref ptr, size_t size, const zend_op_array *op_array, const zend_op *opline)

{

#if ZEND_DEBUG

  return ir_CALL_5(IR_ADDR, ir_CONST_FC_FUNC(_efree),

    ptr,

    op_array && op_array->filename ? ir_CONST_ADDR(op_array->filename->val) : IR_NULL,

    ir_CONST_U32(opline ? opline->lineno : 0),

    IR_NULL,

    ir_CONST_U32(0));

#elif defined(HAVE_BUILTIN_CONSTANT_P)

  if (size > 24 && size <= 32) {

    return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_efree_32), ptr);

  } else {

    return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_efree), ptr);

  }

#else

  return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_efree), ptr);

#endif

}

static ir_ref jit_FP(zend_jit_ctx *jit)

{

  ZEND_ASSERT(jit->ctx.control);

  if (jit->fp == IR_UNUSED) {

    /* Emit "RLOAD FP" once for basic block */

    jit->fp = ir_RLOAD_A(ZREG_FP);

  } else {

    ir_insn *insn;

    ir_ref ref = jit->ctx.control;

    while (1) {

      if (ref == jit->fp) {

        break;

      }

      insn = &jit->ctx.ir_base[ref];

      if (insn->op >= IR_START || insn->op == IR_CALL) {

        jit->fp = ir_RLOAD_A(ZREG_FP);

        break;

      }

      ref = insn->op1;

    }

  }

  return jit->fp;

}

static void jit_STORE_FP(zend_jit_ctx *jit, ir_ref ref)

{

  ir_RSTORE(ZREG_FP, ref);

  jit->fp = IR_UNUSED;

}

static ir_ref jit_IP(zend_jit_ctx *jit)

{

  return ir_RLOAD_A(ZREG_IP);

}

static void jit_STORE_IP(zend_jit_ctx *jit, ir_ref ref)

{

  ir_RSTORE(ZREG_IP, ref);

}

static ir_ref jit_IP32(zend_jit_ctx *jit)

{

  return ir_RLOAD_U32(ZREG_IP);

}

static void jit_LOAD_IP_ADDR(zend_jit_ctx *jit, const zend_op *target)

{

  jit_STORE_IP(jit, ir_CONST_ADDR(target));

}

static void zend_jit_track_last_valid_opline(zend_jit_ctx *jit)

{

  jit->use_last_valid_opline = false;

  jit->track_last_valid_opline = true;

}

static void zend_jit_use_last_valid_opline(zend_jit_ctx *jit)

{

  if (jit->track_last_valid_opline) {

    jit->use_last_valid_opline = true;

    jit->track_last_valid_opline = false;

  }

}

static bool zend_jit_trace_uses_initial_ip(zend_jit_ctx *jit)

{

  return jit->use_last_valid_opline;

}

static void zend_jit_set_last_valid_opline(zend_jit_ctx *jit, const zend_op *opline)

{

  if (!jit->reuse_ip) {

    jit->track_last_valid_opline = true;

    jit->last_valid_opline = opline;

  }

}

static void zend_jit_reset_last_valid_opline(zend_jit_ctx *jit)

{

  jit->track_last_valid_opline = false;

  jit->last_valid_opline = NULL;

}

static void zend_jit_start_reuse_ip(zend_jit_ctx *jit)

{

  zend_jit_reset_last_valid_opline(jit);

  jit->reuse_ip = true;

}

static int zend_jit_reuse_ip(zend_jit_ctx *jit)

{

  if (!jit->reuse_ip) {

    zend_jit_start_reuse_ip(jit);

    // RX = EX(call);

    jit_STORE_IP(jit, ir_LOAD_A(jit_EX(call)));

  }

  return 1;

}

static void zend_jit_stop_reuse_ip(zend_jit_ctx *jit)

{

  jit->reuse_ip = false;

}

static int zend_jit_save_call_chain(zend_jit_ctx *jit, uint32_t call_level)

{

  ir_ref rx, call;

  if (call_level == 1) {

    // JIT: call = NULL;

    call = IR_NULL;

  } else {

    // JIT: call = EX(call);

    call = ir_LOAD_A(jit_EX(call));

  }

  rx = jit_IP(jit);

  // JIT: call->prev_execute_data = call;

  ir_STORE(jit_CALL(rx, prev_execute_data), call);

  // JIT: EX(call) = call;

  ir_STORE(jit_EX(call), rx);

  jit->delayed_call_level = 0;

  delayed_call_chain = false;

  return 1;

}

static int zend_jit_set_ip(zend_jit_ctx *jit, const zend_op *target)

{

  ir_ref ref;