/src/php-src/ext/opcache/jit/zend_jit.c

Source
/*
   +----------------------------------------------------------------------+
   | Zend JIT                                                             |
   +----------------------------------------------------------------------+
   | Copyright © The PHP Group and Contributors.                          |
   +----------------------------------------------------------------------+
   | This source file is subject to the Modified BSD License that is      |
   | bundled with this package in the file LICENSE, and is available      |
   | through the World Wide Web at <https://www.php.net/license/>.        |
   |                                                                      |
   | SPDX-License-Identifier: BSD-3-Clause                                |
   +----------------------------------------------------------------------+
   | Authors: Dmitry Stogov <dmitry@php.net>                              |
   +----------------------------------------------------------------------+
*/

#include "main/php.h"
#include "main/SAPI.h"
#include "php_version.h"
#include <ZendAccelerator.h>
#include "zend_shared_alloc.h"
#include "Zend/zend_execute.h"
#include "Zend/zend_vm.h"
#include "Zend/zend_exceptions.h"
#include "Zend/zend_constants.h"
#include "Zend/zend_closures.h"
#include "Zend/zend_ini.h"
#include "Zend/zend_observer.h"
#include "zend_smart_str.h"
#include "jit/zend_jit.h"

#ifdef HAVE_JIT

#include "Optimizer/zend_func_info.h"
#include "Optimizer/zend_ssa.h"
#include "Optimizer/zend_inference.h"
#include "Optimizer/zend_call_graph.h"
#include "Optimizer/zend_dump.h"
#include "Optimizer/zend_worklist.h"

#include "jit/zend_jit_internal.h"

#ifdef ZEND_JIT_USE_APPLE_MAP_JIT
#include <mach/vm_inherit.h>
#include <pthread.h>
#include <sys/mman.h>
#endif

#ifdef ZTS
int jit_globals_id;
size_t jit_globals_offset;
#else
zend_jit_globals jit_globals;
#endif

//#define CONTEXT_THREADED_JIT
#define ZEND_JIT_USE_RC_INFERENCE

#ifdef ZEND_JIT_USE_RC_INFERENCE
# define ZEND_SSA_RC_INFERENCE_FLAG ZEND_SSA_RC_INFERENCE
# define RC_MAY_BE_1(info)          (((info) & (MAY_BE_RC1|MAY_BE_REF)) != 0)
# define RC_MAY_BE_N(info)          (((info) & (MAY_BE_RCN|MAY_BE_REF)) != 0)
#else
# define ZEND_SSA_RC_INFERENCE_FLAG 0
# define RC_MAY_BE_1(info)          1
# define RC_MAY_BE_N(info)          1
#endif

#define JIT_PREFIX      "JIT$"
#define JIT_STUB_PREFIX "JIT$$"
#define TRACE_PREFIX    "TRACE-"

bool zend_jit_startup_ok = false;

zend_ulong zend_jit_profile_counter = 0;
int zend_jit_profile_counter_rid = -1;

int16_t zend_jit_hot_counters[ZEND_HOT_COUNTERS_COUNT];

const zend_op *zend_jit_halt_op = NULL;
const zend_op *zend_jit_interrupt_op = NULL;

static void *dasm_buf = NULL;
static void *dasm_end = NULL;
static void **dasm_ptr = NULL;

static size_t dasm_size = 0;

static zend_long jit_bisect_pos = 0;

static zend_vm_opcode_handler_t zend_jit_runtime_jit_handler = NULL;
static zend_vm_opcode_handler_t zend_jit_profile_jit_handler = NULL;
static zend_vm_opcode_handler_t zend_jit_func_hot_counter_handler = NULL;
static zend_vm_opcode_handler_t zend_jit_loop_hot_counter_handler = NULL;
static zend_vm_opcode_handler_t zend_jit_func_trace_counter_handler = NULL;
static zend_vm_opcode_handler_t zend_jit_ret_trace_counter_handler = NULL;
static zend_vm_opcode_handler_t zend_jit_loop_trace_counter_handler = NULL;

#if ZEND_VM_KIND == ZEND_VM_KIND_CALL || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL
static ZEND_OPCODE_HANDLER_RET ZEND_OPCODE_HANDLER_CCONV zend_runtime_jit(ZEND_OPCODE_HANDLER_ARGS);
#else
static ZEND_OPCODE_HANDLER_RET ZEND_OPCODE_HANDLER_FUNC_CCONV zend_runtime_jit(ZEND_OPCODE_HANDLER_ARGS);
#endif

static int zend_jit_trace_op_len(const zend_op *opline);
static int zend_jit_trace_may_exit(const zend_op_array *op_array, const zend_op *opline);
static uint32_t _zend_jit_trace_get_exit_point(const zend_op *to_opline, uint32_t flags ZEND_FILE_LINE_DC);
#define zend_jit_trace_get_exit_point(to_opline, flags) _zend_jit_trace_get_exit_point(to_opline, flags ZEND_FILE_LINE_CC)
static const void *zend_jit_trace_get_exit_addr(uint32_t n);
static void zend_jit_trace_add_code(const void *start, uint32_t size);
static zend_string *zend_jit_func_name(const zend_op_array *op_array);

static bool zend_jit_needs_arg_dtor(const zend_function *func, uint32_t arg_num, zend_call_info *call_info);
static bool zend_jit_supported_binary_op(uint8_t op, uint32_t op1_info, uint32_t op2_info);

static bool dominates(const zend_basic_block *blocks, uint32_t a, uint32_t b) {
  while (blocks[b].level > blocks[a].level) {
    b = blocks[b].idom;
  }
  return a == b;
}

static bool zend_ssa_is_last_use(const zend_op_array *op_array, const zend_ssa *ssa, int var, int use)
{
  int next_use;

  if (ssa->vars[var].phi_use_chain) {
    zend_ssa_phi *phi = ssa->vars[var].phi_use_chain;
    do {
      if (!ssa->vars[phi->ssa_var].no_val) {
        return 0;
      }
      phi = zend_ssa_next_use_phi(ssa, var, phi);
    } while (phi);
  }

  if (ssa->cfg.blocks[ssa->cfg.map[use]].loop_header > 0
   || (ssa->cfg.blocks[ssa->cfg.map[use]].flags & ZEND_BB_LOOP_HEADER)) {
    uint32_t b = ssa->cfg.map[use];
    int prev_use = ssa->vars[var].use_chain;
    uint32_t def_block;

    if (ssa->vars[var].definition >= 0) {
      def_block =ssa->cfg.map[ssa->vars[var].definition];
    } else {
      ZEND_ASSERT(ssa->vars[var].definition_phi);
      def_block = ssa->vars[var].definition_phi->block;
    }
    if (dominates(ssa->cfg.blocks, def_block,
        (ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) ? b : ssa->cfg.blocks[b].loop_header)) {
      return 0;
    }

    while (prev_use >= 0 && prev_use != use) {
      if (b != ssa->cfg.map[prev_use]
       && dominates(ssa->cfg.blocks, b, ssa->cfg.map[prev_use])
       && !zend_ssa_is_no_val_use(op_array->opcodes + prev_use, ssa->ops + prev_use, var)) {
        return 0;
      }
      prev_use = zend_ssa_next_use(ssa->ops, var, prev_use);
    }
  }

  next_use = zend_ssa_next_use(ssa->ops, var, use);
  if (next_use < 0) {
    return 1;
  } else if (zend_ssa_is_no_val_use(op_array->opcodes + next_use, ssa->ops + next_use, var)) {
    return 1;
  }
  return 0;
}

static int zend_jit_is_constant_cmp_long_long(const zend_op  *opline,
                                              zend_ssa_range *op1_range,
                                              zend_jit_addr   op1_addr,
                                              zend_ssa_range *op2_range,
                                              zend_jit_addr   op2_addr,
                                              bool           *result)
{
  zend_long op1_min;
  zend_long op1_max;
  zend_long op2_min;
  zend_long op2_max;

  if (op1_range) {
    op1_min = op1_range->min;
    op1_max = op1_range->max;
  } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL) {
    ZEND_ASSERT(Z_TYPE_P(Z_ZV(op1_addr)) == IS_LONG);
    op1_min = op1_max = Z_LVAL_P(Z_ZV(op1_addr));
  } else {
    return 0;
  }

  if (op2_range) {
    op2_min = op2_range->min;
    op2_max = op2_range->max;
  } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
    ZEND_ASSERT(Z_TYPE_P(Z_ZV(op2_addr)) == IS_LONG);
    op2_min = op2_max = Z_LVAL_P(Z_ZV(op2_addr));
  } else {
    return 0;
  }

  switch (opline->opcode) {
    case ZEND_IS_EQUAL:
    case ZEND_IS_IDENTICAL:
    case ZEND_CASE:
    case ZEND_CASE_STRICT:
      if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) {
        *result = true;
        return 1;
      } else if (op1_max < op2_min || op1_min > op2_max) {
        *result = false;
        return 1;
      }
      return 0;
    case ZEND_IS_NOT_EQUAL:
    case ZEND_IS_NOT_IDENTICAL:
      if (op1_min == op1_max && op2_min == op2_max && op1_min == op2_min) {
        *result = false;
        return 1;
      } else if (op1_max < op2_min || op1_min > op2_max) {
        *result = true;
        return 1;
      }
      return 0;
    case ZEND_IS_SMALLER:
      if (op1_max < op2_min) {
        *result = true;
        return 1;
      } else if (op1_min >= op2_max) {
        *result = false;
        return 1;
      }
      return 0;
    case ZEND_IS_SMALLER_OR_EQUAL:
      if (op1_max <= op2_min) {
        *result = true;
        return 1;
      } else if (op1_min > op2_max) {
        *result = false;
        return 1;
      }
      return 0;
    default:
      ZEND_UNREACHABLE();
  }
  return 0;
}

#define ADVANCE_SSA_OP(ssa_op, offset) \
  do { \
    if (ssa_op) ssa_op += offset; \
  } while (0)

static int zend_jit_needs_call_chain(zend_call_info *call_info, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, const zend_op *opline, int call_level, zend_jit_trace_rec *trace)
{
  int skip;

  if (trace) {
    zend_jit_trace_rec *p = trace;

    ADVANCE_SSA_OP(ssa_op, 1);
    while (1) {
      if (p->op == ZEND_JIT_TRACE_VM) {
        switch (p->opline->opcode) {
          case ZEND_SEND_ARRAY:
          case ZEND_SEND_USER:
          case ZEND_SEND_UNPACK:
          case ZEND_INIT_FCALL:
          case ZEND_INIT_METHOD_CALL:
          case ZEND_INIT_STATIC_METHOD_CALL:
          case ZEND_INIT_PARENT_PROPERTY_HOOK_CALL:
          case ZEND_INIT_FCALL_BY_NAME:
          case ZEND_INIT_NS_FCALL_BY_NAME:
          case ZEND_INIT_DYNAMIC_CALL:
          case ZEND_NEW:
          case ZEND_INIT_USER_CALL:
          case ZEND_FAST_CALL:
          case ZEND_JMP:
          case ZEND_JMPZ:
          case ZEND_JMPNZ:
          case ZEND_JMPZ_EX:
          case ZEND_JMPNZ_EX:
          case ZEND_FE_RESET_R:
          case ZEND_FE_RESET_RW:
          case ZEND_JMP_SET:
          case ZEND_COALESCE:
          case ZEND_JMP_NULL:
          case ZEND_ASSERT_CHECK:
          case ZEND_CATCH:
          case ZEND_DECLARE_ANON_CLASS:
          case ZEND_FE_FETCH_R:
          case ZEND_FE_FETCH_RW:
          case ZEND_BIND_INIT_STATIC_OR_JMP:
          case ZEND_JMP_FRAMELESS:
            return 1;
          case ZEND_DO_ICALL:
          case ZEND_DO_UCALL:
          case ZEND_DO_FCALL_BY_NAME:
          case ZEND_DO_FCALL:
          case ZEND_CALLABLE_CONVERT:
          case ZEND_CALLABLE_CONVERT_PARTIAL:
            return 0;
          case ZEND_SEND_VAL:
          case ZEND_SEND_VAR:
          case ZEND_SEND_VAL_EX:
          case ZEND_SEND_VAR_EX:
          case ZEND_SEND_FUNC_ARG:
          case ZEND_SEND_REF:
          case ZEND_SEND_VAR_NO_REF:
          case ZEND_SEND_VAR_NO_REF_EX:
            /* skip */
            break;
          default:
            if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
              return 1;
            }
        }
        ADVANCE_SSA_OP(ssa_op, zend_jit_trace_op_len(opline));
      } else if (p->op == ZEND_JIT_TRACE_ENTER ||
                 p->op == ZEND_JIT_TRACE_BACK ||
                 p->op == ZEND_JIT_TRACE_END) {
        return 1;
      }
      p++;
    }
  }

  if (!call_info) {
    const zend_op *end = op_array->opcodes + op_array->last;

    opline++;
    ADVANCE_SSA_OP(ssa_op, 1);
    skip = (call_level == 1);
    while (opline != end) {
      if (!skip) {
        if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
          return 1;
        }
      }
      switch (opline->opcode) {
        case ZEND_SEND_VAL:
        case ZEND_SEND_VAR:
        case ZEND_SEND_VAL_EX:
        case ZEND_SEND_VAR_EX:
        case ZEND_SEND_FUNC_ARG:
        case ZEND_SEND_REF:
        case ZEND_SEND_VAR_NO_REF:
        case ZEND_SEND_VAR_NO_REF_EX:
          skip = 0;
          break;
        case ZEND_SEND_ARRAY:
        case ZEND_SEND_USER:
        case ZEND_SEND_UNPACK:
        case ZEND_INIT_FCALL:
        case ZEND_INIT_METHOD_CALL:
        case ZEND_INIT_STATIC_METHOD_CALL:
        case ZEND_INIT_PARENT_PROPERTY_HOOK_CALL:
        case ZEND_INIT_FCALL_BY_NAME:
        case ZEND_INIT_NS_FCALL_BY_NAME:
        case ZEND_INIT_DYNAMIC_CALL:
        case ZEND_NEW:
        case ZEND_INIT_USER_CALL:
        case ZEND_FAST_CALL:
        case ZEND_JMP:
        case ZEND_JMPZ:
        case ZEND_JMPNZ:
        case ZEND_JMPZ_EX:
        case ZEND_JMPNZ_EX:
        case ZEND_FE_RESET_R:
        case ZEND_FE_RESET_RW:
        case ZEND_JMP_SET:
        case ZEND_COALESCE:
        case ZEND_JMP_NULL:
        case ZEND_ASSERT_CHECK:
        case ZEND_CATCH:
        case ZEND_DECLARE_ANON_CLASS:
        case ZEND_FE_FETCH_R:
        case ZEND_FE_FETCH_RW:
        case ZEND_BIND_INIT_STATIC_OR_JMP:
        case ZEND_JMP_FRAMELESS:
          return 1;
        case ZEND_DO_ICALL:
        case ZEND_DO_UCALL:
        case ZEND_DO_FCALL_BY_NAME:
        case ZEND_DO_FCALL:
        case ZEND_CALLABLE_CONVERT:
        case ZEND_CALLABLE_CONVERT_PARTIAL:
          end = opline;
          if (end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) {
            /* INIT_FCALL and DO_FCALL in different BasicBlocks */
            return 1;
          }
          return 0;
      }
      opline++;
      ADVANCE_SSA_OP(ssa_op, 1);
    }

    return 1;
  } else {
    const zend_op *end = call_info->caller_call_opline;

    /* end may be null if an opcode like EXIT is part of the argument list. */
    if (!end || end - op_array->opcodes >= ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len) {
      /* INIT_FCALL and DO_FCALL in different BasicBlocks */
      return 1;
    }

    opline++;
    ADVANCE_SSA_OP(ssa_op, 1);
    skip = (call_level == 1);
    while (opline != end) {
      if (skip) {
        switch (opline->opcode) {
          case ZEND_SEND_VAL:
          case ZEND_SEND_VAR:
          case ZEND_SEND_VAL_EX:
          case ZEND_SEND_VAR_EX:
          case ZEND_SEND_FUNC_ARG:
          case ZEND_SEND_REF:
          case ZEND_SEND_VAR_NO_REF:
          case ZEND_SEND_VAR_NO_REF_EX:
            skip = 0;
            break;
          case ZEND_SEND_ARRAY:
          case ZEND_SEND_USER:
          case ZEND_SEND_UNPACK:
            return 1;
        }
      } else {
        if (zend_may_throw(opline, ssa_op, op_array, ssa)) {
          return 1;
        }
      }
      opline++;
      ADVANCE_SSA_OP(ssa_op, 1);
    }

    return 0;
  }
}

static uint32_t skip_valid_arguments(const zend_op_array *op_array, zend_ssa *ssa, const zend_call_info *call_info)
{
  uint32_t num_args = 0;
  zend_function *func = call_info->callee_func;

  /* It's okay to handle prototypes here, because they can only increase the accepted arguments.
   * Anything legal for the parent method is also legal for the parent method. */
  while (num_args < call_info->num_args) {
    zend_arg_info *arg_info = func->op_array.arg_info + num_args;

    if (ZEND_TYPE_IS_SET(arg_info->type)) {
      if (ZEND_TYPE_IS_ONLY_MASK(arg_info->type)) {
        zend_op *opline = call_info->arg_info[num_args].opline;
        zend_ssa_op *ssa_op = ssa->ops ? &ssa->ops[opline - op_array->opcodes] : NULL;
        uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type);
        if ((OP1_INFO() & (MAY_BE_ANY|MAY_BE_UNDEF)) & ~type_mask) {
          break;
        }
      } else {
        break;
      }
    }
    num_args++;
  }
  return num_args;
}

static uint32_t zend_ssa_cv_info(const zend_op_array *op_array, zend_ssa *ssa, uint32_t var)
{
  uint32_t j, info;

  if (ssa->vars && ssa->var_info) {
    info = ssa->var_info[var].type;
    for (j = op_array->last_var; j < ssa->vars_count; j++) {
      if (ssa->vars[j].var == var) {
        info |= ssa->var_info[j].type;
      }
    }
  } else {
    info = MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_REF | MAY_BE_ANY | MAY_BE_UNDEF |
      MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
  }

#ifdef ZEND_JIT_USE_RC_INFERENCE
  /* Refcount may be increased by RETURN opcode */
  if ((info & MAY_BE_RC1) && !(info & MAY_BE_RCN)) {
    for (j = 0; j < ssa->cfg.blocks_count; j++) {
      if ((ssa->cfg.blocks[j].flags & ZEND_BB_REACHABLE) &&
          ssa->cfg.blocks[j].len > 0) {
        const zend_op *opline = op_array->opcodes + ssa->cfg.blocks[j].start + ssa->cfg.blocks[j].len - 1;

        if (opline->opcode == ZEND_RETURN) {
          if (opline->op1_type == IS_CV && opline->op1.var == EX_NUM_TO_VAR(var)) {
            info |= MAY_BE_RCN;
            break;
          }
        }
      }
    }
  }
#endif

  return info;
}

static bool zend_jit_may_avoid_refcounting(const zend_op *opline, uint32_t op1_info)
{
  switch (opline->opcode) {
    case ZEND_FETCH_OBJ_FUNC_ARG:
      if (!JIT_G(current_frame) ||
          !JIT_G(current_frame)->call->func ||
          !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
        return 0;
      }
      /* break missing intentionally */
    case ZEND_FETCH_OBJ_R:
    case ZEND_FETCH_OBJ_IS:
      if ((op1_info & MAY_BE_OBJECT)
       && opline->op2_type == IS_CONST
       && Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_STRING
       && Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] != '\0') {
        return 1;
      }
      break;
    case ZEND_FETCH_DIM_FUNC_ARG:
      if (!JIT_G(current_frame) ||
          !JIT_G(current_frame)->call->func ||
          !TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
        return 0;
      }
      /* break missing intentionally */
    case ZEND_FETCH_DIM_R:
    case ZEND_FETCH_DIM_IS:
      return 1;
    case ZEND_ISSET_ISEMPTY_DIM_OBJ:
      if (!(opline->extended_value & ZEND_ISEMPTY)) {
        return 1;
      }
      break;
  }
  return 0;
}

static bool zend_jit_is_persistent_constant(zval *key, uint32_t flags)
{
  zval *zv;
  zend_constant *c = NULL;

  /* null/true/false are resolved during compilation, so don't check for them here. */
  zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key));
  if (zv) {
    c = (zend_constant*)Z_PTR_P(zv);
  } else if (flags & IS_CONSTANT_UNQUALIFIED_IN_NAMESPACE) {
    key++;
    zv = zend_hash_find_known_hash(EG(zend_constants), Z_STR_P(key));
    if (zv) {
      c = (zend_constant*)Z_PTR_P(zv);
    }
  }
  return c && (ZEND_CONSTANT_FLAGS(c) & CONST_PERSISTENT);
}

static zend_class_entry* zend_get_known_class(const zend_op_array *op_array, const zend_op *opline, uint8_t op_type, znode_op op)
{
  zend_class_entry *ce = NULL;

  if (op_type == IS_CONST) {
    zval *zv = RT_CONSTANT(opline, op);
    zend_string *class_name;

    ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING);
    class_name = Z_STR_P(zv);
    ce = zend_lookup_class_ex(class_name, NULL, ZEND_FETCH_CLASS_NO_AUTOLOAD);
    if (ce && (ce->type == ZEND_INTERNAL_CLASS || ce->info.user.filename != op_array->filename)) {
      ce = NULL;
    }
  } else {
    ZEND_ASSERT(op_type == IS_UNUSED);
    if ((op.num & ZEND_FETCH_CLASS_MASK) == ZEND_FETCH_CLASS_SELF) {
      ce = op_array->scope;
    } else {
      ZEND_ASSERT((op.num & ZEND_FETCH_CLASS_MASK) == ZEND_FETCH_CLASS_PARENT);
      ce = op_array->scope;
      if (ce) {
        if (ce->parent) {
          ce = ce->parent;
          if (ce->type == ZEND_INTERNAL_CLASS || ce->info.user.filename != op_array->filename) {
            ce = NULL;
          }
        } else {
          ce = NULL;
        }
      }
    }
  }

  return ce;
}

static zend_property_info* zend_get_known_property_info(const zend_op_array *op_array, zend_class_entry *ce, zend_string *member, bool on_this, zend_string *filename)
{
  zend_property_info *info = NULL;

  if ((on_this && (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE)) ||
      !ce ||
      !(ce->ce_flags & ZEND_ACC_LINKED) ||
      (ce->ce_flags & ZEND_ACC_TRAIT) ||
      ce->create_object) {
    return NULL;
  }

  if (!(ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
    if (ce->info.user.filename != filename) {
      /* class declaration might be changed independently */
      return NULL;
    }

    if (ce->parent) {
      zend_class_entry *parent = ce->parent;

      do {
        if (parent->type == ZEND_INTERNAL_CLASS) {
          break;
        } else if (parent->info.user.filename != filename) {
          /* some of parents class declarations might be changed independently */
          /* TODO: this check may be not enough, because even
           * in the same it's possible to conditionally define
           * few classes with the same name, and "parent" may
           * change from request to request.
           */
          return NULL;
        }
        parent = parent->parent;
      } while (parent);
    }
  }

  // TODO: Treat property hooks more precisely.
  info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member);
  if (info == NULL ||
      !IS_VALID_PROPERTY_OFFSET(info->offset) ||
      (info->flags & ZEND_ACC_STATIC) ||
      info->hooks) {
    return NULL;
  }

  if (info->flags & ZEND_ACC_PUBLIC) {
    if ((info->flags & ZEND_ACC_CHANGED)
     && op_array->scope
     && op_array->scope != ce) {
      return NULL;
    }
    return info;
  } else if (on_this) {
    if (ce == info->ce) {
      if (ce == op_array->scope) {
        return info;
      } else {
        return NULL;
      }
    } else if ((info->flags & ZEND_ACC_PROTECTED)
        && instanceof_function_slow(ce, info->ce)) {
      return info;
    }
  }

  return NULL;
}

static bool zend_may_be_dynamic_property(zend_class_entry *ce, zend_string *member, bool on_this, const zend_op_array *op_array)
{
  zend_property_info *info;

  if (!ce || (ce->ce_flags & ZEND_ACC_TRAIT) || (op_array->fn_flags & ZEND_ACC_TRAIT_CLONE)) {
    return 1;
  }

  if (!(ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
    if (ce->info.user.filename != op_array->filename) {
      /* class declaration might be changed independently */
      return 1;
    }
  }

  // TODO: Treat property hooks more precisely.
  info = (zend_property_info*)zend_hash_find_ptr(&ce->properties_info, member);
  if (info == NULL ||
      !IS_VALID_PROPERTY_OFFSET(info->offset) ||
      (info->flags & ZEND_ACC_STATIC) ||
      info->hooks) {
    return 1;
  }

  if (!(info->flags & ZEND_ACC_PUBLIC) &&
      (!on_this || info->ce != ce)) {
    return 1;
  }

  return 0;
}

static bool zend_jit_class_may_be_modified(const zend_class_entry *ce, const zend_op_array *called_from)
{
  uint32_t i;

  if (ce->type == ZEND_INTERNAL_CLASS) {
#ifdef ZEND_OPCACHE_SHM_REATTACHMENT
    /* ASLR */
    return 1;
#else
    return 0;
#endif
  } else if (ce->type == ZEND_USER_CLASS) {
    if (ce->ce_flags & ZEND_ACC_PRELOADED) {
      return 0;
    }
    if (ce->info.user.filename == called_from->filename) {
      if (ce->parent
       && (!(ce->ce_flags & ZEND_ACC_LINKED)
        || zend_jit_class_may_be_modified(ce->parent, called_from))) {
        return 1;
      }
      if (ce->num_interfaces) {
        if (!(ce->ce_flags & ZEND_ACC_LINKED)) {
          return 1;
        }
        for (i = 0; i < ce->num_interfaces; i++) {
          if (zend_jit_class_may_be_modified(ce->interfaces[i], called_from)) {
            return 1;
          }
        }
      }
      if (ce->num_traits) {
        if (!(ce->ce_flags & ZEND_ACC_LINKED)) {
          return 1;
        }
        for (i=0; i < ce->num_traits; i++) {
          zend_class_entry *trait = zend_fetch_class_by_name(ce->trait_names[i].name,
            ce->trait_names[i].lc_name,
            ZEND_FETCH_CLASS_TRAIT | ZEND_FETCH_CLASS_NO_AUTOLOAD | ZEND_FETCH_CLASS_SILENT);
          if (!trait || zend_jit_class_may_be_modified(trait, called_from)) {
            return 1;
          }
        }
      }
      return 0;
    }
  }
  return 1;
}

static bool zend_jit_may_be_modified(const zend_function *func, const zend_op_array *called_from)
{
  if (func->type == ZEND_INTERNAL_FUNCTION) {
#ifdef ZEND_OPCACHE_SHM_REATTACHMENT
    /* ASLR */
    return 1;
#else
    return 0;
#endif
  } else if (func->type == ZEND_USER_FUNCTION) {
    if (func->common.fn_flags & ZEND_ACC_PRELOADED) {
      return 0;
    }
    if (func->op_array.filename == called_from->filename
     && (!func->op_array.scope
      || !zend_jit_class_may_be_modified(func->op_array.scope, called_from))) {
      return 0;
    }
  }
  return 1;
}

#define OP_RANGE(ssa_op, opN) \
  (((opline->opN##_type & (IS_TMP_VAR|IS_VAR|IS_CV)) && \
    ssa->var_info && \
    (ssa_op)->opN##_use >= 0 && \
    ssa->var_info[(ssa_op)->opN##_use].has_range) ? \
   &ssa->var_info[(ssa_op)->opN##_use].range : NULL)

#define OP1_RANGE()      OP_RANGE(ssa_op, op1)
#define OP2_RANGE()      OP_RANGE(ssa_op, op2)
#define OP1_DATA_RANGE() OP_RANGE(ssa_op + 1, op1)

#include "jit/zend_jit_helpers.c"
#include "Zend/zend_cpuinfo.h"

#ifdef HAVE_GCC_GLOBAL_REGS
# define GCC_GLOBAL_REGS 1
#else
# define GCC_GLOBAL_REGS 0
#endif

/* By default avoid JITing inline handlers if it does not seem profitable due to lack of
 * type information. Disabling this option allows testing some JIT handlers in the
 * presence of try/catch blocks, which prevent SSA construction. */
#ifndef PROFITABILITY_CHECKS
# define PROFITABILITY_CHECKS 1
#endif

#define BP_JIT_IS 6 /* Used for ISSET_ISEMPTY_DIM_OBJ. see BP_VAR_*defines in Zend/zend_compile.h */

/* The generated code may contain tautological comparisons, ignore them. */
#if defined(__clang__)
# pragma clang diagnostic push
# pragma clang diagnostic ignored "-Wtautological-compare"
# pragma clang diagnostic ignored "-Wstring-compare"
#endif

#include "jit/zend_jit_ir.c"

#if defined(__clang__)
# pragma clang diagnostic pop
#endif

#ifdef _WIN32
# include <Windows.h>
#else
# include <sys/mman.h>
# if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
#   define MAP_ANONYMOUS MAP_ANON
# endif
#endif

ZEND_EXT_API void zend_jit_status(zval *ret)
{
  zval stats;
  array_init(&stats);
  add_assoc_bool(&stats, "enabled", JIT_G(enabled));
  add_assoc_bool(&stats, "on", JIT_G(on));
  add_assoc_long(&stats, "kind", JIT_G(trigger));
  add_assoc_long(&stats, "opt_level", JIT_G(opt_level));
  add_assoc_long(&stats, "opt_flags", JIT_G(opt_flags));
  if (dasm_buf) {
    add_assoc_long(&stats, "buffer_size", (char*)dasm_end - (char*)dasm_buf);
    add_assoc_long(&stats, "buffer_free", (char*)dasm_end - (char*)*dasm_ptr);
  } else {
    add_assoc_long(&stats, "buffer_size", 0);
    add_assoc_long(&stats, "buffer_free", 0);
  }
  add_assoc_zval(ret, "jit", &stats);
}

static bool zend_jit_inc_call_level(uint8_t opcode)
{
  switch (opcode) {
    case ZEND_INIT_FCALL:
    case ZEND_INIT_FCALL_BY_NAME:
    case ZEND_INIT_NS_FCALL_BY_NAME:
    case ZEND_INIT_METHOD_CALL:
    case ZEND_INIT_DYNAMIC_CALL:
    case ZEND_INIT_STATIC_METHOD_CALL:
    case ZEND_INIT_PARENT_PROPERTY_HOOK_CALL:
    case ZEND_INIT_USER_CALL:
    case ZEND_NEW:
      return true;
    default:
      return false;
  }
}

static bool zend_jit_dec_call_level(uint8_t opcode)
{
  switch (opcode) {
    case ZEND_DO_FCALL:
    case ZEND_DO_ICALL:
    case ZEND_DO_UCALL:
    case ZEND_DO_FCALL_BY_NAME:
    case ZEND_CALLABLE_CONVERT:
    case ZEND_CALLABLE_CONVERT_PARTIAL:
      return true;
    default:
      return false;
  }
}

static zend_string *zend_jit_func_name(const zend_op_array *op_array)
{
  smart_str buf = {0};

  if (op_array->function_name) {
    smart_str_appends(&buf, JIT_PREFIX);
    if (op_array->scope) {
      smart_str_append(&buf, op_array->scope->name);
      smart_str_appends(&buf, "::");
    }
    smart_str_append(&buf, op_array->function_name);
    if (op_array->fn_flags & ZEND_ACC_CLOSURE) {
      smart_str_appends(&buf, ":");
      smart_str_append(&buf, op_array->filename);
      smart_str_appends(&buf, ":");
      smart_str_append_long(&buf, op_array->line_start);
    }
    smart_str_0(&buf);
    return buf.s;
  } else if (op_array->filename) {
    smart_str_appends(&buf, JIT_PREFIX);
    smart_str_append(&buf, op_array->filename);
    smart_str_0(&buf);
    return buf.s;
  } else {
    return NULL;
  }
}

static int zend_may_overflow(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa)
{
  int res;
  zend_long op1_min, op1_max, op2_min, op2_max;

  if (!ssa->ops || !ssa->var_info) {
    return 1;
  }
  switch (opline->opcode) {
    case ZEND_PRE_INC:
    case ZEND_POST_INC:
      res = ssa_op->op1_def;
      if (res < 0
       || !ssa->var_info[res].has_range
       || ssa->var_info[res].range.overflow) {
        if (!OP1_HAS_RANGE()) {
          return 1;
        }
        op1_max = OP1_MAX_RANGE();
        if (op1_max == ZEND_LONG_MAX) {
          return 1;
        }
      }
      return 0;
    case ZEND_PRE_DEC:
    case ZEND_POST_DEC:
      res = ssa_op->op1_def;
      if (res < 0
       || !ssa->var_info[res].has_range
       || ssa->var_info[res].range.underflow) {
        if (!OP1_HAS_RANGE()) {
          return 1;
        }
        op1_min = OP1_MIN_RANGE();
        if (op1_min == ZEND_LONG_MIN) {
          return 1;
        }
      }
      return 0;
    case ZEND_ADD:
      res = ssa_op->result_def;
      if (res < 0
       || !ssa->var_info[res].has_range
       || ssa->var_info[res].range.underflow) {
        if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
          return 1;
        }
        op1_min = OP1_MIN_RANGE();
        op2_min = OP2_MIN_RANGE();
        if (zend_add_will_overflow(op1_min, op2_min)) {
          return 1;
        }
      }
      if (res < 0
       || !ssa->var_info[res].has_range
       || ssa->var_info[res].range.overflow) {
        if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
          return 1;
        }
        op1_max = OP1_MAX_RANGE();
        op2_max = OP2_MAX_RANGE();
        if (zend_add_will_overflow(op1_max, op2_max)) {
          return 1;
        }
      }
      return 0;
    case ZEND_SUB:
      res = ssa_op->result_def;
      if (res < 0
       || !ssa->var_info[res].has_range
       || ssa->var_info[res].range.underflow) {
        if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
          return 1;
        }
        op1_min = OP1_MIN_RANGE();
        op2_max = OP2_MAX_RANGE();
        if (zend_sub_will_overflow(op1_min, op2_max)) {
          return 1;
        }
      }
      if (res < 0
       || !ssa->var_info[res].has_range
       || ssa->var_info[res].range.overflow) {
        if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
          return 1;
        }
        op1_max = OP1_MAX_RANGE();
        op2_min = OP2_MIN_RANGE();
        if (zend_sub_will_overflow(op1_max, op2_min)) {
          return 1;
        }
      }
      return 0;
    case ZEND_MUL:
      res = ssa_op->result_def;
      return (res < 0 ||
        !ssa->var_info[res].has_range ||
        ssa->var_info[res].range.underflow ||
        ssa->var_info[res].range.overflow);
    case ZEND_ASSIGN_OP:
      if (opline->extended_value == ZEND_ADD) {
        res = ssa_op->op1_def;
        if (res < 0
         || !ssa->var_info[res].has_range
         || ssa->var_info[res].range.underflow) {
          if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
            return 1;
          }
          op1_min = OP1_MIN_RANGE();
          op2_min = OP2_MIN_RANGE();
          if (zend_add_will_overflow(op1_min, op2_min)) {
            return 1;
          }
        }
        if (res < 0
         || !ssa->var_info[res].has_range
         || ssa->var_info[res].range.overflow) {
          if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
            return 1;
          }
          op1_max = OP1_MAX_RANGE();
          op2_max = OP2_MAX_RANGE();
          if (zend_add_will_overflow(op1_max, op2_max)) {
            return 1;
          }
        }
        return 0;
      } else if (opline->extended_value == ZEND_SUB) {
        res = ssa_op->op1_def;
        if (res < 0
         || !ssa->var_info[res].has_range
         || ssa->var_info[res].range.underflow) {
          if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
            return 1;
          }
          op1_min = OP1_MIN_RANGE();
          op2_max = OP2_MAX_RANGE();
          if (zend_sub_will_overflow(op1_min, op2_max)) {
            return 1;
          }
        }
        if (res < 0
         || !ssa->var_info[res].has_range
         || ssa->var_info[res].range.overflow) {
          if (!OP1_HAS_RANGE() || !OP2_HAS_RANGE()) {
            return 1;
          }
          op1_max = OP1_MAX_RANGE();
          op2_min = OP2_MIN_RANGE();
          if (zend_sub_will_overflow(op1_max, op2_min)) {
            return 1;
          }
        }
        return 0;
      } else if (opline->extended_value == ZEND_MUL) {
        res = ssa_op->op1_def;
        return (res < 0 ||
          !ssa->var_info[res].has_range ||
          ssa->var_info[res].range.underflow ||
          ssa->var_info[res].range.overflow);
      }
      ZEND_FALLTHROUGH;
    default:
      return 1;
  }
}

static int zend_jit_build_cfg(const zend_op_array *op_array, zend_cfg *cfg)
{
  uint32_t flags;

  flags = ZEND_CFG_STACKLESS | ZEND_CFG_NO_ENTRY_PREDECESSORS | ZEND_SSA_RC_INFERENCE_FLAG | ZEND_SSA_USE_CV_RESULTS | ZEND_CFG_RECV_ENTRY;

  zend_build_cfg(&CG(arena), op_array, flags, cfg);

  /* Don't JIT huge functions. Apart from likely being detrimental due to the amount of
   * generated code, some of our analysis is recursive and will stack overflow with many
   * blocks. */
  if (cfg->blocks_count > 100000) {
    return FAILURE;
  }

  zend_cfg_build_predecessors(&CG(arena), cfg);

  /* Compute Dominators Tree */
  zend_cfg_compute_dominators_tree(op_array, cfg);

  /* Identify reducible and irreducible loops */
  zend_cfg_identify_loops(op_array, cfg);

  return SUCCESS;
}

static int zend_jit_op_array_analyze1(const zend_op_array *op_array, zend_script *script, zend_ssa *ssa)
{
  if (zend_jit_build_cfg(op_array, &ssa->cfg) != SUCCESS) {
    return FAILURE;
  }

#if 0
  /* TODO: debugger and profiler supports? */
  if ((ssa->cfg.flags & ZEND_FUNC_HAS_EXTENDED_INFO)) {
    return FAILURE;
  }
#endif

  /* TODO: move this to zend_cfg.c ? */
  if (!op_array->function_name) {
    ssa->cfg.flags |= ZEND_FUNC_INDIRECT_VAR_ACCESS;
  }

  if ((JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC)
   && ssa->cfg.blocks
   && op_array->last_try_catch == 0
   && !(op_array->fn_flags & ZEND_ACC_GENERATOR)
   && !(ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) {
    if (zend_build_ssa(&CG(arena), script, op_array, ZEND_SSA_RC_INFERENCE | ZEND_SSA_USE_CV_RESULTS, ssa) != SUCCESS) {
      return FAILURE;
    }

    zend_ssa_compute_use_def_chains(&CG(arena), op_array, ssa);

    zend_ssa_find_false_dependencies(op_array, ssa);

    zend_ssa_find_sccs(op_array, ssa);
  }

  return SUCCESS;
}

static int zend_jit_op_array_analyze2(const zend_op_array *op_array, zend_script *script, zend_ssa *ssa, uint32_t optimization_level)
{
  if ((JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC)
   && ssa->cfg.blocks
   && op_array->last_try_catch == 0
   && !(op_array->fn_flags & ZEND_ACC_GENERATOR)
   && !(ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS)) {
    if (zend_ssa_inference(&CG(arena), op_array, script, ssa,
        optimization_level & ~ZEND_OPTIMIZER_NARROW_TO_DOUBLE) != SUCCESS) {
      return FAILURE;
    }
  }

  return SUCCESS;
}

static void zend_jit_allocate_registers(zend_jit_ctx *ctx, const zend_op_array *op_array, zend_ssa *ssa)
{
  void *checkpoint;
  int candidates_count, i;
  zend_jit_reg_var *ra;

  checkpoint = zend_arena_checkpoint(CG(arena));
  ra = zend_arena_calloc(&CG(arena), ssa->vars_count, sizeof(zend_jit_reg_var));
  candidates_count = 0;
  for (i = 0; i < ssa->vars_count; i++) {
    if (zend_jit_may_be_in_reg(op_array, ssa, i)) {
      ra[i].ref = IR_NULL;
      candidates_count++;
    }
  }
  if (!candidates_count) {
    zend_arena_release(&CG(arena), checkpoint);
    return;
  }

  if (JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) {
    /* Naive SSA resolution */
    for (i = 0; i < ssa->vars_count; i++) {
      if (ssa->vars[i].definition_phi && !ssa->vars[i].no_val) {
        zend_ssa_phi *phi = ssa->vars[i].definition_phi;
        int src;

        if (phi->pi >= 0) {
          src = phi->sources[0];
          if (ra[i].ref) {
            if (!ra[src].ref) {
              ra[i].flags |= ZREG_LOAD;
            } else {
              ra[i].flags |= ZREG_PI;
            }
          } else if (ra[src].ref) {
            ra[src].flags |= ZREG_STORE;
          }
        } else {
          int need_move = 0;
          uint32_t k;

          for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) {
            src = phi->sources[k];
            if (src >= 0) {
              if (ssa->vars[src].definition_phi
               && ssa->vars[src].definition_phi->pi >= 0
               && phi->block == ssa->vars[src].definition_phi->block) {
                /* Skip zero-length interval for Pi variable */
                src = ssa->vars[src].definition_phi->sources[0];
              }
              if (ra[i].ref) {
                if (!ra[src].ref) {
                  need_move = 1;
                }
              } else if (ra[src].ref) {
                need_move = 1;
              }
            }
          }
          if (need_move) {
            if (ra[i].ref) {
              ra[i].flags |= ZREG_LOAD;
            }
            for (k = 0; k < ssa->cfg.blocks[phi->block].predecessors_count; k++) {
              src = phi->sources[k];
              if (src >= 0) {
                if (ssa->vars[src].definition_phi
                 && ssa->vars[src].definition_phi->pi >= 0
                 && phi->block == ssa->vars[src].definition_phi->block) {
                  /* Skip zero-length interval for Pi variable */
                  src = ssa->vars[src].definition_phi->sources[0];
                }
                if (ra[src].ref) {
                  ra[src].flags |= ZREG_STORE;
                }
              }
            }
          } else {
            ra[i].flags |= ZREG_PHI;
          }
        }
      }
    }

    /* Remove useless register allocation */
    for (i = 0; i < ssa->vars_count; i++) {
      if (ra[i].ref &&
          ((ra[i].flags & ZREG_LOAD) ||
           ((ra[i].flags & ZREG_STORE) && ssa->vars[i].definition >= 0)) &&
          ssa->vars[i].use_chain < 0) {
          bool may_remove = true;
        zend_ssa_phi *phi = ssa->vars[i].phi_use_chain;

        while (phi) {
          if (ra[phi->ssa_var].ref &&
              !(ra[phi->ssa_var].flags & ZREG_LOAD)) {
            may_remove = false;
            break;
          }
          phi = zend_ssa_next_use_phi(ssa, i, phi);
        }
        if (may_remove) {
          ra[i].ref = IR_UNUSED;
        }
      }
    }

    /* Remove intervals used once */
    for (i = 0; i < ssa->vars_count; i++) {
      if (ra[i].ref) {
        if (!(ra[i].flags & (ZREG_LOAD|ZREG_STORE))) {
          uint32_t var_num = ssa->vars[i].var;
          uint32_t op_num = ssa->vars[i].definition;

          /* Check if a tempoary variable may be freed by exception handler */
          if (op_array->last_live_range
           && var_num >= op_array->last_var
           && ssa->vars[i].definition >= 0
           && ssa->ops[op_num].result_def == i) {
            const zend_live_range *range = op_array->live_range;

            op_num++;
            if (op_array->opcodes[op_num].opcode == ZEND_OP_DATA) {
              op_num++;
            }
            for (uint32_t j = 0; j < op_array->last_live_range; range++, j++) {
              if (range->start > op_num) {
                /* further blocks will not be relevant... */
                break;
              } else if (op_num < range->end && var_num == (range->var & ~ZEND_LIVE_MASK)) {
                /* check if opcodes in range may throw */
                do {
                  if (zend_may_throw(op_array->opcodes + op_num, ssa->ops + op_num, op_array, ssa)) {
                    ra[i].flags |= ZREG_STORE;
                    break;
                  }
                  op_num++;
                  if (op_array->opcodes[op_num].opcode == ZEND_OP_DATA) {
                    op_num++;
                  }
                } while (op_num < range->end);
                break;
              }
            }
          }
        }
        if ((ra[i].flags & ZREG_LOAD)
         && (ra[i].flags & ZREG_STORE)
         && (ssa->vars[i].use_chain < 0
          || zend_ssa_next_use(ssa->ops, i, ssa->vars[i].use_chain) < 0)) {
          bool may_remove = true;
          zend_ssa_phi *phi = ssa->vars[i].phi_use_chain;

          while (phi) {
            if (ra[phi->ssa_var].ref &&
                !(ra[phi->ssa_var].flags & ZREG_LOAD)) {
              may_remove = false;
              break;
            }
            phi = zend_ssa_next_use_phi(ssa, i, phi);
          }
          if (may_remove) {
            ra[i].ref = IR_UNUSED;
          }
        }
      }
    }
  }

  if (JIT_G(debug) & ZEND_JIT_DEBUG_REG_ALLOC) {
    fprintf(stderr, "Live Ranges \"%s\"\n", op_array->function_name ? ZSTR_VAL(op_array->function_name) : "[main]");
    for (i = 0; i < ssa->vars_count; i++) {
      if (ra[i].ref) {
        fprintf(stderr, "#%d.", i);
        uint32_t var_num = ssa->vars[i].var;
        zend_dump_var(op_array, (var_num < op_array->last_var ? IS_CV : 0), var_num);
        if (ra[i].flags & ZREG_LOAD) {
          fprintf(stderr, " load");
        }
        if (ra[i].flags & ZREG_STORE) {
          fprintf(stderr, " store");
        }
        fprintf(stderr, "\n");
      }
    }
    fprintf(stderr, "\n");
  }

  ctx->ra = ra;
}

static int zend_jit_compute_post_order(zend_cfg *cfg, int start, int *post_order)
{
  int count = 0;
  int b, *p;
  zend_basic_block *bb;
  zend_worklist worklist;
  ALLOCA_FLAG(use_heap)

  ZEND_WORKLIST_ALLOCA(&worklist, cfg->blocks_count, use_heap);
  zend_worklist_push(&worklist, start);

  while (zend_worklist_len(&worklist) != 0) {
next:
    b = zend_worklist_peek(&worklist);
    bb = &cfg->blocks[b];
    uint32_t n = bb->successors_count;
    if (n > 0) {
      p = bb->successors;
      do {
        if (cfg->blocks[*p].flags & (ZEND_BB_CATCH|ZEND_BB_FINALLY|ZEND_BB_FINALLY_END)) {
          /* skip */
        } else if (zend_worklist_push(&worklist, *p)) {
          goto next;
        }
        p++;
        n--;
      } while (n > 0);
    }
    zend_worklist_pop(&worklist);
    post_order[count++] = b;
  }
  ZEND_WORKLIST_FREE_ALLOCA(&worklist, use_heap);
  return count;
}

static bool zend_jit_next_is_send_result(const zend_op *opline)
{
  if (opline->result_type == IS_TMP_VAR
   && (opline+1)->opcode == ZEND_SEND_VAL
   && (opline+1)->op1_type == IS_TMP_VAR
   && (opline+1)->op2_type != IS_CONST
   && (opline+1)->op1.var == opline->result.var) {
    return 1;
  }
  return 0;
}

static bool zend_jit_supported_binary_op(uint8_t op, uint32_t op1_info, uint32_t op2_info)
{
  if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
    return false;
  }
  switch (op) {
    case ZEND_POW:
    case ZEND_DIV:
      // TODO: check for division by zero ???
      return false;
    case ZEND_ADD:
    case ZEND_SUB:
    case ZEND_MUL:
      return (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))
        && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE));
    case ZEND_BW_OR:
    case ZEND_BW_AND:
    case ZEND_BW_XOR:
    case ZEND_SL:
    case ZEND_SR:
    case ZEND_MOD:
      return (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG);
    case ZEND_CONCAT:
      return (op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING);
    default: ZEND_UNREACHABLE();
  }
}

static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *rt_opline)
{
  int b, i, end;
  zend_op *opline;
  zend_jit_ctx ctx;
  zend_jit_ctx *jit = &ctx;
  zend_jit_reg_var *ra = NULL;
  zend_vm_opcode_handler_t handler;
  int call_level = 0;
  void *checkpoint = NULL;
  bool recv_emitted = false;   /* emitted at least one RECV opcode */
  uint8_t smart_branch_opcode;
  uint32_t target_label, target_label2;
  uint32_t op1_info, op1_def_info, op2_info, res_info, res_use_info, op1_mem_info;
  zend_jit_addr op1_addr, op1_def_addr, op2_addr, op2_def_addr, res_addr;
  zend_class_entry *ce = NULL;
  bool ce_is_instanceof;
  bool on_this;

  ZEND_ASSERT(!(op_array->fn_flags & ZEND_ACC_CLOSURE) || !(op_array->scope));

  if (JIT_G(bisect_limit)) {
    jit_bisect_pos++;
    if (jit_bisect_pos >= JIT_G(bisect_limit)) {
      if (jit_bisect_pos == JIT_G(bisect_limit)) {
        fprintf(stderr, "Not JITing %s%s%s in %s:%d and after due to jit_bisect_limit\n",
          op_array->scope ? ZSTR_VAL(op_array->scope->name) : "",
          op_array->scope ? "::" : "",
          op_array->function_name ? ZSTR_VAL(op_array->function_name) : "{main}",
          ZSTR_VAL(op_array->filename), op_array->line_start);
      }
      return FAILURE;
    }
  }

  if (ssa->cfg.flags & ZEND_FUNC_IRREDUCIBLE) {
    /* We can't order blocks properly */
    return FAILURE;
  }

  if (rt_opline) {
    /* Set BB_ENTRY flag to limit register usage across the OSR ENTRY point */
    ssa->cfg.blocks[ssa->cfg.map[rt_opline - op_array->opcodes]].flags |= ZEND_BB_ENTRY;
  }

  zend_jit_start(&ctx, op_array, ssa);
  if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
    checkpoint = zend_arena_checkpoint(CG(arena));
    zend_jit_allocate_registers(&ctx, op_array, ssa);
    ra = ctx.ra;
  }

  /* Process blocks in Reverse Post Order */
  int *sorted_blocks = alloca(sizeof(int) * ssa->cfg.blocks_count);
  int n = zend_jit_compute_post_order(&ssa->cfg, 0, sorted_blocks);

  while (n > 0) {
    b = sorted_blocks[--n];
    if ((ssa->cfg.blocks[b].flags & ZEND_BB_REACHABLE) == 0) {
      continue;
    }

    if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) {
      opline = op_array->opcodes + ssa->cfg.blocks[b].start;
      if (ssa->cfg.flags & ZEND_CFG_RECV_ENTRY) {
        if (opline->opcode == ZEND_RECV_INIT) {
          if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
            if (opline != op_array->opcodes && (opline-1)->opcode != ZEND_RECV_INIT) {
              zend_jit_recv_entry(&ctx, b);
            }
          } else {
            if (opline != op_array->opcodes && recv_emitted) {
              zend_jit_recv_entry(&ctx, b);
            }
          }
          recv_emitted = true;
        } else if (opline->opcode == ZEND_RECV) {
          if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
            /* skip */
            zend_jit_bb_start(&ctx, b);
            zend_jit_bb_end(&ctx, b);
            continue;
          } else if (recv_emitted) {
            zend_jit_recv_entry(&ctx, b);
          } else {
            recv_emitted = true;
          }
        } else {
          if (recv_emitted) {
            zend_jit_recv_entry(&ctx, b);
          } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE &&
                     ssa->cfg.blocks[b].len == 1 &&
                     (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT)) {
            /* don't generate code for BB with single opcode */
            zend_jit_free_ctx(&ctx);

            if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
              zend_arena_release(&CG(arena), checkpoint);
            }
            return SUCCESS;
          }
        }
      } else if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE &&
                 ssa->cfg.blocks[b].len == 1 &&
                 (ssa->cfg.blocks[b].flags & ZEND_BB_EXIT)) {
        /* don't generate code for BB with single opcode */
        zend_jit_free_ctx(&ctx);

        if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
          zend_arena_release(&CG(arena), checkpoint);
        }
        return SUCCESS;
      }
    }

    zend_jit_bb_start(&ctx, b);

    if ((JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) && ctx.ra) {
      zend_ssa_phi *phi = ssa->blocks[b].phis;

      /* First try to insert IR Phi */
      while (phi) {
        zend_jit_reg_var *ival = &ctx.ra[phi->ssa_var];

        if (ival->ref) {
          if (ival->flags & ZREG_PI) {
            zend_jit_gen_pi(jit, phi);
          } else if (ival->flags & ZREG_PHI) {
            zend_jit_gen_phi(jit, phi);
          }
        }
        phi = phi->next;
      }
    }

    if (rt_opline
     && (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_RECV_ENTRY)) == 0
     && rt_opline == op_array->opcodes + ssa->cfg.blocks[b].start) {
      zend_jit_osr_entry(&ctx, b); /* OSR (On-Stack-Replacement) Entry-Point */
    }

    if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
      if ((ssa->cfg.blocks[b].flags & ZEND_BB_FOLLOW)
        && ssa->cfg.blocks[b].start != 0
        && (op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_NOP
         || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_SWITCH_LONG
         || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_SWITCH_STRING
         || op_array->opcodes[ssa->cfg.blocks[b].start - 1].opcode == ZEND_MATCH)) {
        zend_jit_reset_last_valid_opline(&ctx);
      } else {
        zend_jit_set_last_valid_opline(&ctx, op_array->opcodes + ssa->cfg.blocks[b].start);
      }
    } else if (ssa->cfg.blocks[b].flags & ZEND_BB_TARGET) {
      zend_jit_reset_last_valid_opline(&ctx);
    } else if (ssa->cfg.blocks[b].flags & ZEND_BB_RECV_ENTRY) {
      zend_jit_reset_last_valid_opline(&ctx);
    } else if (ssa->cfg.blocks[b].flags & (ZEND_BB_START|ZEND_BB_ENTRY)) {
      zend_jit_set_last_valid_opline(&ctx, op_array->opcodes + ssa->cfg.blocks[b].start);
    }
    if (ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) {
      zend_jit_check_timeout(&ctx, op_array->opcodes + ssa->cfg.blocks[b].start, NULL);
    }
    if (!ssa->cfg.blocks[b].len) {
      zend_jit_bb_end(&ctx, b);
      continue;
    }
    if ((JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL) && ra) {
      zend_ssa_phi *phi = ssa->blocks[b].phis;

      while (phi) {
        zend_jit_reg_var *ival = &ra[phi->ssa_var];

        if (ival->ref) {
          if (ival->flags & ZREG_LOAD) {
            ZEND_ASSERT(ival->ref == IR_NULL);

            if (!zend_jit_load_var(&ctx, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, phi->ssa_var)) {
              goto jit_failure;
            }
          } else if (ival->flags & ZREG_STORE) {
            ZEND_ASSERT(ival->ref != IR_NULL);

            if (!zend_jit_store_var(&ctx, ssa->var_info[phi->ssa_var].type, ssa->vars[phi->ssa_var].var, phi->ssa_var, 1)) {
              goto jit_failure;
            }
          }
        }
        phi = phi->next;
      }
    }
    end = ssa->cfg.blocks[b].start + ssa->cfg.blocks[b].len - 1;
    for (i = ssa->cfg.blocks[b].start; i <= end; i++) {
      zend_ssa_op *ssa_op = ssa->ops ? &ssa->ops[i] : NULL;
      opline = op_array->opcodes + i;
      if (zend_jit_inc_call_level(opline->opcode)) {
        call_level++;
      }

      if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_INLINE) {
        switch (opline->opcode) {
          case ZEND_PRE_INC:
          case ZEND_PRE_DEC:
          case ZEND_POST_INC:
          case ZEND_POST_DEC:
            if (opline->op1_type != IS_CV) {
              break;
            }
            op1_info = OP1_INFO();
            if (!(op1_info & MAY_BE_LONG)) {
              break;
            }
            if (opline->result_type != IS_UNUSED) {
              res_use_info = -1;

              if (opline->result_type == IS_CV
               && ssa->vars
               && ssa_op->result_use >= 0
               && !ssa->vars[ssa_op->result_use].no_val) {
                zend_jit_addr res_use_addr = RES_USE_REG_ADDR();

                if (Z_MODE(res_use_addr) != IS_REG
                 || Z_LOAD(res_use_addr)
                 || Z_STORE(res_use_addr)) {
                  res_use_info = RES_USE_INFO();
                }
              }
              res_info = RES_INFO();
              res_addr = RES_REG_ADDR();
            } else {
              res_use_info = -1;
              res_info = -1;
              res_addr = 0;
            }
            op1_def_info = OP1_DEF_INFO();
            if (!zend_jit_inc_dec(&ctx, opline,
                op1_info, OP1_REG_ADDR(),
                op1_def_info, OP1_DEF_REG_ADDR(),
                res_use_info, res_info,
                res_addr,
                (op1_info & MAY_BE_LONG) && (op1_def_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
                zend_may_throw(opline, ssa_op, op_array, ssa))) {
              goto jit_failure;
            }
            goto done;
          case ZEND_BW_OR:
          case ZEND_BW_AND:
          case ZEND_BW_XOR:
          case ZEND_SL:
          case ZEND_SR:
          case ZEND_MOD:
            if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
              break;
            }
            op1_info = OP1_INFO();
            op2_info = OP2_INFO();
            if (!(op1_info & MAY_BE_LONG)
             || !(op2_info & MAY_BE_LONG)) {
              break;
            }
            res_addr = RES_REG_ADDR();
            if (Z_MODE(res_addr) != IS_REG
             && (i + 1) <= end
             && zend_jit_next_is_send_result(opline)) {
              i++;
              res_use_info = -1;
              res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
              if (!zend_jit_reuse_ip(&ctx)) {
                goto jit_failure;
              }
            } else {
              res_use_info = -1;

              if (opline->result_type == IS_CV
               && ssa->vars
               && ssa_op->result_use >= 0
               && !ssa->vars[ssa_op->result_use].no_val) {
                zend_jit_addr res_use_addr = RES_USE_REG_ADDR();

                if (Z_MODE(res_use_addr) != IS_REG
                 || Z_LOAD(res_use_addr)
                 || Z_STORE(res_use_addr)) {
                  res_use_info = RES_USE_INFO();
                }
              }
            }
            if (!zend_jit_long_math(&ctx, opline,
                op1_info, OP1_RANGE(), OP1_REG_ADDR(),
                op2_info, OP2_RANGE(), OP2_REG_ADDR(),
                res_use_info, RES_INFO(), res_addr,
                zend_may_throw(opline, ssa_op, op_array, ssa))) {
              goto jit_failure;
            }
            goto done;
          case ZEND_ADD:
          case ZEND_SUB:
          case ZEND_MUL:
//          case ZEND_DIV: // TODO: check for division by zero ???
            if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
              break;
            }
            op1_info = OP1_INFO();
            op2_info = OP2_INFO();
            if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
              break;
            }
            if (opline->opcode == ZEND_ADD &&
                (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
                (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
              /* pass */
            } else if (!(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) ||
                !(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
              break;
            }
            res_addr = RES_REG_ADDR();
            if (Z_MODE(res_addr) != IS_REG
             && (i + 1) <= end
             && zend_jit_next_is_send_result(opline)) {
              i++;
              res_use_info = -1;
              res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
              if (!zend_jit_reuse_ip(&ctx)) {
                goto jit_failure;
              }
            } else {
              res_use_info = -1;

              if (opline->result_type == IS_CV
               && ssa->vars
               && ssa_op->result_use >= 0
               && !ssa->vars[ssa_op->result_use].no_val) {
                zend_jit_addr res_use_addr = RES_USE_REG_ADDR();

                if (Z_MODE(res_use_addr) != IS_REG
                 || Z_LOAD(res_use_addr)
                 || Z_STORE(res_use_addr)) {
                  res_use_info = RES_USE_INFO();
                }
              }
            }
            res_info = RES_INFO();
            if (opline->opcode == ZEND_ADD &&
                (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY &&
                (op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
              if (!zend_jit_add_arrays(&ctx, opline, op1_info, OP1_REG_ADDR(), op2_info, OP2_REG_ADDR(), res_addr)) {
                goto jit_failure;
              }
            } else {
              if (!zend_jit_math(&ctx, opline,
                  op1_info, OP1_REG_ADDR(),
                  op2_info, OP2_REG_ADDR(),
                  res_use_info, res_info, res_addr,
                  (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
                  zend_may_throw(opline, ssa_op, op_array, ssa))) {
                goto jit_failure;
              }
            }
            goto done;
          case ZEND_CONCAT:
          case ZEND_FAST_CONCAT:
            if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
              break;
            }
            op1_info = OP1_INFO();
            op2_info = OP2_INFO();
            if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
              break;
            }
            if (!(op1_info & MAY_BE_STRING) ||
                !(op2_info & MAY_BE_STRING)) {
              break;
            }
            res_addr = RES_REG_ADDR();
            if ((i + 1) <= end
             && zend_jit_next_is_send_result(opline)) {
              i++;
              res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
              if (!zend_jit_reuse_ip(&ctx)) {
                goto jit_failure;
              }
            }
            if (!zend_jit_concat(&ctx, opline,
                op1_info, op2_info, res_addr,
                zend_may_throw(opline, ssa_op, op_array, ssa))) {
              goto jit_failure;
            }
            goto done;
          case ZEND_ASSIGN_OP:
            if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
              break;
            }
            if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
              break;
            }
            op1_info = OP1_INFO();
            op2_info = OP2_INFO();
            if (!zend_jit_supported_binary_op(
                opline->extended_value, op1_info, op2_info)) {
              break;
            }
            op1_addr = OP1_REG_ADDR();
            op1_mem_info = -1;
            if (Z_MODE(op1_addr) != IS_REG
             || Z_LOAD(op1_addr)
             || Z_STORE(op1_addr)) {
              op1_mem_info = op1_info;
            }
            op1_def_info = OP1_DEF_INFO();
            if (!zend_jit_assign_op(&ctx, opline,
                op1_info, op1_addr, OP1_RANGE(),
                op1_def_info, OP1_DEF_REG_ADDR(), op1_mem_info,
                op2_info, OP2_REG_ADDR(), OP2_RANGE(),
                (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (op1_def_info & MAY_BE_DOUBLE) && zend_may_overflow(opline, ssa_op, op_array, ssa),
                zend_may_throw(opline, ssa_op, op_array, ssa))) {
              goto jit_failure;
            }
            goto done;
          case ZEND_ASSIGN_DIM_OP:
            if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
              break;
            }
            if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
              break;
            }
            if (!zend_jit_supported_binary_op(
                opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
              break;
            }
            if (!zend_jit_assign_dim_op(&ctx, opline,
                OP1_INFO(), OP1_DEF_INFO(), OP1_REG_ADDR(), 0,
                OP2_INFO(), (opline->op2_type != IS_UNUSED) ? OP2_REG_ADDR() : 0,
                (opline->op2_type != IS_UNUSED) ? OP2_RANGE() : NULL,
                OP1_DATA_INFO(), OP1_DATA_REG_ADDR(), OP1_DATA_RANGE(), IS_UNKNOWN,
                zend_may_throw(opline, ssa_op, op_array, ssa))) {
              goto jit_failure;
            }
            goto done;
          case ZEND_ASSIGN_DIM:
            if (opline->op1_type != IS_CV) {
              break;
            }
            if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
              break;
            }
            if (!zend_jit_assign_dim(&ctx, opline,
                OP1_INFO(), OP1_REG_ADDR(), 0,
                OP2_INFO(), (opline->op2_type != IS_UNUSED) ? OP2_REG_ADDR() : 0,
                (opline->op2_type != IS_UNUSED) ? OP2_RANGE() : NULL,
                OP1_DATA_INFO(), OP1_DATA_REG_ADDR(),
                (ctx.ra && (ssa_op+1)->op1_def >= 0) ? OP1_DATA_DEF_REG_ADDR() : 0,
                (opline->result_type != IS_UNUSED) ? RES_REG_ADDR() : 0,
                IS_UNKNOWN,
                zend_may_throw(opline, ssa_op, op_array, ssa))) {
              goto jit_failure;
            }
            goto done;
          case ZEND_PRE_INC_OBJ:
          case ZEND_PRE_DEC_OBJ:
          case ZEND_POST_INC_OBJ:
          case ZEND_POST_DEC_OBJ:
            if (opline->op2_type != IS_CONST
             || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
             || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
              break;
            }
            if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
              break;
            }
            ce = NULL;
            ce_is_instanceof = false;
            on_this = false;
            if (opline->op1_type == IS_UNUSED) {
              op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
              ce = op_array->scope;
              /* scope is NULL for closures. */
              if (ce) {
                ce_is_instanceof = !(ce->ce_flags & ZEND_ACC_FINAL);
              }
              op1_addr = 0;
              on_this = true;
            } else {
              op1_info = OP1_INFO();
              if (!(op1_info & MAY_BE_OBJECT)) {
                break;
              }
              op1_addr = OP1_REG_ADDR();
              if (ssa->var_info && ssa->ops) {
                zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
                if (ssa_op->op1_use >= 0) {
                  zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
                  if (op1_ssa->ce && !op1_ssa->ce->create_object) {
                    ce = op1_ssa->ce;
                    ce_is_instanceof = op1_ssa->is_instanceof;
                  }
                }
              }
            }
            if (!zend_jit_incdec_obj(&ctx, opline, op_array, ssa, ssa_op,
                op1_info, op1_addr,
                0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN)) {
              goto jit_failure;
            }
            goto done;
          case ZEND_ASSIGN_OBJ_OP:
            if (opline->result_type != IS_UNUSED) {
              break;
            }
            if (opline->op2_type != IS_CONST
             || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
             || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
              break;
            }
            if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
              break;
            }
            if (!zend_jit_supported_binary_op(
                opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
              break;
            }
            ce = NULL;
            ce_is_instanceof = false;
            on_this = false;
            if (opline->op1_type == IS_UNUSED) {
              op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
              ce = op_array->scope;
              /* scope is NULL for closures. */
              if (ce) {
                ce_is_instanceof = !(ce->ce_flags & ZEND_ACC_FINAL);
              }
              op1_addr = 0;
              on_this = true;
            } else {
              op1_info = OP1_INFO();
              if (!(op1_info & MAY_BE_OBJECT)) {
                break;
              }
              op1_addr = OP1_REG_ADDR();
              if (ssa->var_info && ssa->ops) {
                zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
                if (ssa_op->op1_use >= 0) {
                  zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
                  if (op1_ssa->ce && !op1_ssa->ce->create_object) {
                    ce = op1_ssa->ce;
                    ce_is_instanceof = op1_ssa->is_instanceof;
                  }
                }
              }
            }
            if (!zend_jit_assign_obj_op(&ctx, opline, op_array, ssa, ssa_op,
                op1_info, op1_addr, OP1_DATA_INFO(), OP1_DATA_REG_ADDR(), OP1_DATA_RANGE(),
                0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN)) {
              goto jit_failure;
            }
            goto done;
          case ZEND_ASSIGN_OBJ:
            if (opline->op2_type != IS_CONST
             || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
             || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
              break;
            }
            if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
              break;
            }
            ce = NULL;
            ce_is_instanceof = false;
            on_this = false;
            if (opline->op1_type == IS_UNUSED) {
              op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
              ce = op_array->scope;
              /* scope is NULL for closures. */
              if (ce) {
                ce_is_instanceof = !(ce->ce_flags & ZEND_ACC_FINAL);
              }
              op1_addr = 0;
              on_this = true;
            } else {
              op1_info = OP1_INFO();
              if (!(op1_info & MAY_BE_OBJECT)) {
                break;
              }
              op1_addr = OP1_REG_ADDR();
              if (ssa->var_info && ssa->ops) {
                zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
                if (ssa_op->op1_use >= 0) {
                  zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
                  if (op1_ssa->ce && !op1_ssa->ce->create_object) {
                    ce = op1_ssa->ce;
                    ce_is_instanceof = op1_ssa->is_instanceof;
                  }
                }
              }
            }
            if (!zend_jit_assign_obj(&ctx, opline, op_array, ssa, ssa_op,
                op1_info, op1_addr, OP1_DATA_INFO(), OP1_DATA_REG_ADDR(), OP1_DATA_DEF_REG_ADDR(),
                (opline->result_type != IS_UNUSED) ? RES_REG_ADDR() : 0,
                0, ce, ce_is_instanceof, on_this, 0, NULL, IS_UNKNOWN,
                zend_may_throw(opline, ssa_op, op_array, ssa))) {
              goto jit_failure;
            }
            goto done;
          case ZEND_ASSIGN:
            if (opline->op1_type != IS_CV) {
              break;
            }
            if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
              break;
            }
            op2_addr = OP2_REG_ADDR();
            if (ra
             && ssa->ops[opline - op_array->opcodes].op2_def >= 0
             && !ssa->vars[ssa->ops[opline - op_array->opcodes].op2_def].no_val) {
              op2_def_addr = OP2_DEF_REG_ADDR();
            } else {
              op2_def_addr = op2_addr;
            }
            op1_info = OP1_INFO();
            if (ra && ssa->vars[ssa_op->op1_use].no_val) {
              op1_info |= MAY_BE_UNDEF; // requires type assignment
            }
            if (opline->result_type == IS_UNUSED) {
              res_addr = 0;
              res_info = -1;
            } else {
              res_addr = RES_REG_ADDR();
              res_info = RES_INFO();
              if (Z_MODE(res_addr) != IS_REG
               && (i + 1) <= end
               && zend_jit_next_is_send_result(opline)
               && (!(op1_info & MAY_HAVE_DTOR) || !(op1_info & MAY_BE_RC1))) {
                i++;
                res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, (opline+1)->result.var);
                if (!zend_jit_reuse_ip(&ctx)) {
                  goto jit_failure;
                }
              }
            }
            if (!zend_jit_assign(&ctx, opline,
                op1_info, OP1_REG_ADDR(),
                OP1_DEF_INFO(), OP1_DEF_REG_ADDR(),
                OP2_INFO(), op2_addr, op2_def_addr,
                res_info, res_addr,
                0,
                zend_may_throw(opline, ssa_op, op_array, ssa))) {
              goto jit_failure;
            }
            goto done;
          case ZEND_QM_ASSIGN:
            op1_addr = OP1_REG_ADDR();
            if (ra
             && ssa->ops[opline - op_array->opcodes].op1_def >= 0
             && !ssa->vars[ssa->ops[opline - op_array->opcodes].op1_def].no_val) {
              op1_def_addr = OP1_DEF_REG_ADDR();
            } else {
              op1_def_addr = op1_addr;
            }
            if (!zend_jit_qm_assign(&ctx, opline,
                OP1_INFO(), op1_addr, op1_def_addr,
                -1, RES_INFO(), RES_REG_ADDR())) {
              goto jit_failure;
            }
            goto done;
          case ZEND_INIT_FCALL:
          case ZEND_INIT_FCALL_BY_NAME:
          case ZEND_INIT_NS_FCALL_BY_NAME:
            if (!zend_jit_init_fcall(&ctx, opline, b, op_array, ssa, ssa_op, call_level, NULL, 0)) {
              goto jit_failure;
            }
            goto done;
          case ZEND_SEND_VAL:
          case ZEND_SEND_VAL_EX:
            if (opline->op2_type == IS_CONST) {
              /* Named parameters not supported in JIT (yet) */
              break;
            }
            if (opline->opcode == ZEND_SEND_VAL_EX
             && opline->op2.num > MAX_ARG_FLAG_NUM) {
              break;
            }
            if (!zend_jit_send_val(&ctx, opline,
                OP1_INFO(), OP1_REG_ADDR())) {
              goto jit_failure;
            }
            goto done;
          case ZEND_SEND_REF:
            if (opline->op2_type == IS_CONST) {
              /* Named parameters not supported in JIT (yet) */
              break;
            }
            if (!zend_jit_send_ref(&ctx, opline, op_array,
                OP1_INFO(), 0)) {
              goto jit_failure;
            }
            goto done;
          case ZEND_SEND_VAR:
          case ZEND_SEND_VAR_EX:
          case ZEND_SEND_VAR_NO_REF:
          case ZEND_SEND_VAR_NO_REF_EX:
          case ZEND_SEND_FUNC_ARG:
            if (opline->op2_type == IS_CONST) {
              /* Named parameters not supported in JIT (yet) */
              break;
            }
            if ((opline->opcode == ZEND_SEND_VAR_EX
              || opline->opcode == ZEND_SEND_VAR_NO_REF_EX)
             && opline->op2.num > MAX_ARG_FLAG_NUM) {
              break;
            }
            op1_addr = OP1_REG_ADDR();
            if (ra
             && ssa->ops[opline - op_array->opcodes].op1_def >= 0
             && !ssa->vars[ssa->ops[opline - op_array->opcodes].op1_def].no_val) {
              op1_def_addr = OP1_DEF_REG_ADDR();
            } else {
              op1_def_addr = op1_addr;
            }
            if (!zend_jit_send_var(&ctx, opline, op_array,
                OP1_INFO(), op1_addr, op1_def_addr)) {
              goto jit_failure;
            }
            goto done;
          case ZEND_CHECK_FUNC_ARG:
            if (opline->op2_type == IS_CONST) {
              /* Named parameters not supported in JIT (yet) */
              break;
            }
            if (opline->op2.num > MAX_ARG_FLAG_NUM) {
              break;
            }
            if (!zend_jit_check_func_arg(&ctx, opline)) {
              goto jit_failure;
            }
            goto done;
          case ZEND_CHECK_UNDEF_ARGS:
            if (!zend_jit_check_undef_args(&ctx, opline)) {
              goto jit_failure;
            }
            goto done;
          case ZEND_DO_UCALL:
            ZEND_FALLTHROUGH;
          case ZEND_DO_ICALL:
          case ZEND_DO_FCALL_BY_NAME:
          case ZEND_DO_FCALL:
            if (!zend_jit_do_fcall(&ctx, opline, op_array, ssa, call_level, b + 1, NULL)) {
              goto jit_failure;
            }
            goto done;
          case ZEND_IS_EQUAL:
          case ZEND_IS_NOT_EQUAL:
          case ZEND_IS_SMALLER:
          case ZEND_IS_SMALLER_OR_EQUAL:
          case ZEND_CASE: {
            res_addr = RES_REG_ADDR();
            if ((opline->result_type & IS_TMP_VAR)
             && (i + 1) <= end
             && ((opline+1)->opcode == ZEND_JMPZ
              || (opline+1)->opcode == ZEND_JMPNZ
              || (opline+1)->opcode == ZEND_JMPZ_EX
              || (opline+1)->opcode == ZEND_JMPNZ_EX)
             && (opline+1)->op1_type == IS_TMP_VAR
             && (opline+1)->op1.var == opline->result.var) {
              i++;
              smart_branch_opcode = (opline+1)->opcode;
              target_label = ssa->cfg.blocks[b].successors[0];
              target_label2 = ssa->cfg.blocks[b].successors[1];
              /* For EX variant write into the result of EX opcode. */
              if ((opline+1)->opcode == ZEND_JMPZ_EX
                  || (opline+1)->opcode == ZEND_JMPNZ_EX) {
                res_addr = OP_REG_ADDR(opline + 1, ssa_op + 1, result_type, result, result_def);
              }
            } else {
              smart_branch_opcode = 0;
              target_label = target_label2 = (uint32_t)-1;
            }
            if (!zend_jit_cmp(&ctx, opline,
                OP1_INFO(), OP1_RANGE(), OP1_REG_ADDR(),
                OP2_INFO(), OP2_RANGE(), OP2_REG_ADDR(),
                res_addr,
                zend_may_throw(opline, ssa_op, op_array, ssa),
                smart_branch_opcode, target_label, target_label2,
                NULL, 0)) {
              goto jit_failure;
            }
            goto done;
          }
          case ZEND_IS_IDENTICAL:
          case ZEND_IS_NOT_IDENTICAL:
          case ZEND_CASE_STRICT:
            res_addr = RES_REG_ADDR();
            if ((opline->result_type & IS_TMP_VAR)
             && (i + 1) <= end
             && ((opline+1)->opcode == ZEND_JMPZ
              || (opline+1)->opcode == ZEND_JMPZ_EX
              || (opline+1)->opcode == ZEND_JMPNZ_EX
              || (opline+1)->opcode == ZEND_JMPNZ)
             && (opline+1)->op1_type == IS_TMP_VAR
             && (opline+1)->op1.var == opline->result.var) {
              i++;
              smart_branch_opcode = (opline+1)->opcode;
              target_label = ssa->cfg.blocks[b].successors[0];
              target_label2 = ssa->cfg.blocks[b].successors[1];
              /* For EX variant write into the result of EX opcode. */
              if ((opline+1)->opcode == ZEND_JMPZ_EX
                  || (opline+1)->opcode == ZEND_JMPNZ_EX) {
                res_addr = OP_REG_ADDR(opline + 1, ssa_op + 1, result_type, result, result_def);
              }
            } else {
              smart_branch_opcode = 0;
              target_label = target_label2 = (uint32_t)-1;
            }
            if (!zend_jit_identical(&ctx, opline,
                OP1_INFO(), OP1_RANGE(), OP1_REG_ADDR(),
                OP2_INFO(), OP2_RANGE(), OP2_REG_ADDR(),
                res_addr,
                zend_may_throw(opline, ssa_op, op_array, ssa),
                smart_branch_opcode, target_label, target_label2,
                NULL, 0)) {
              goto jit_failure;
            }
            goto done;
          case ZEND_DEFINED:
            if ((opline->result_type & IS_TMP_VAR)
             && (i + 1) <= end
             && ((opline+1)->opcode == ZEND_JMPZ
              || (opline+1)->opcode == ZEND_JMPNZ)
             && (opline+1)->op1_type == IS_TMP_VAR
             && (opline+1)->op1.var == opline->result.var) {
              i++;
              smart_branch_opcode = (opline+1)->opcode;
              target_label = ssa->cfg.blocks[b].successors[0];
              target_label2 = ssa->cfg.blocks[b].successors[1];
            } else {
              smart_branch_opcode = 0;
              target_label = target_label2 = (uint32_t)-1;
            }
            if (!zend_jit_defined(&ctx, opline, smart_branch_opcode, target_label, target_label2, NULL)) {
              goto jit_failure;
            }
            goto done;
          case ZEND_TYPE_CHECK:
            if (opline->extended_value == MAY_BE_RESOURCE) {
              // TODO: support for is_resource() ???
              break;
            }
            if ((opline->result_type & IS_TMP_VAR)
             && (i + 1) <= end
             && ((opline+1)->opcode == ZEND_JMPZ
              || (opline+1)->opcode == ZEND_JMPNZ)
             && (opline+1)->op1_type == IS_TMP_VAR
             && (opline+1)->op1.var == opline->result.var) {
              i++;
              smart_branch_opcode = (opline+1)->opcode;
              target_label = ssa->cfg.blocks[b].successors[0];
              target_label2 = ssa->cfg.blocks[b].successors[1];
            } else {
              smart_branch_opcode = 0;
              target_label = target_label2 = (uint32_t)-1;
            }
            if (!zend_jit_type_check(&ctx, opline, OP1_INFO(), smart_branch_opcode, target_label, target_label2, NULL)) {
              goto jit_failure;
            }
            goto done;
          case ZEND_RETURN:
            op1_info = OP1_INFO();
            if ((PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info))
             || op_array->type == ZEND_EVAL_CODE
             // TODO: support for top-level code
             || !op_array->function_name
             // TODO: support for IS_UNDEF ???
             || (op1_info & MAY_BE_UNDEF)) {
              if (!zend_jit_tail_handler(&ctx, opline)) {
                goto jit_failure;
              }
            } else {
              if (!zend_jit_return(&ctx, opline, op_array,
                  op1_info, OP1_REG_ADDR())) {
                goto jit_failure;
              }
            }
            goto done;
          case ZEND_BOOL:
          case ZEND_BOOL_NOT:
            if (!zend_jit_bool_jmpznz(&ctx, opline,
                OP1_INFO(), OP1_REG_ADDR(), RES_REG_ADDR(),
                -1, -1,
                zend_may_throw(opline, ssa_op, op_array, ssa),
                opline->opcode, NULL, true)) {
              goto jit_failure;
            }
            goto done;
          case ZEND_JMPZ:
          case ZEND_JMPNZ:
            if (opline > op_array->opcodes + ssa->cfg.blocks[b].start &&
                ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
              /* smart branch */
              if (!zend_jit_cond_jmp(&ctx, opline + 1, ssa->cfg.blocks[b].successors[0])) {
                goto jit_failure;
              }
              goto done;
            }
            ZEND_FALLTHROUGH;
          case ZEND_JMPZ_EX:
          case ZEND_JMPNZ_EX:
            if (opline->result_type == IS_UNDEF) {
              res_addr = 0;
            } else {
              res_addr = RES_REG_ADDR();
            }
            if (!zend_jit_bool_jmpznz(&ctx, opline,
                OP1_INFO(), OP1_REG_ADDR(), res_addr,
                ssa->cfg.blocks[b].successors[0], ssa->cfg.blocks[b].successors[1],
                zend_may_throw(opline, ssa_op, op_array, ssa),
                opline->opcode, NULL, true)) {
              goto jit_failure;
            }
            goto done;
          case ZEND_ISSET_ISEMPTY_CV:
            if ((opline->result_type & IS_TMP_VAR)
             && (i + 1) <= end
             && ((opline+1)->opcode == ZEND_JMPZ
              || (opline+1)->opcode == ZEND_JMPNZ)
             && (opline+1)->op1_type == IS_TMP_VAR
             && (opline+1)->op1.var == opline->result.var) {
              i++;
              smart_branch_opcode = (opline+1)->opcode;
              target_label = ssa->cfg.blocks[b].successors[0];
              target_label2 = ssa->cfg.blocks[b].successors[1];
            } else {
              smart_branch_opcode = 0;
              target_label = target_label2 = (uint32_t)-1;
            }
            if (!zend_jit_isset_isempty_cv(&ctx, opline,
                OP1_INFO(), OP1_REG_ADDR(),
                smart_branch_opcode, target_label, target_label2,
                NULL)) {
              goto jit_failure;
            }
            goto done;
          case ZEND_IN_ARRAY:
            if (opline->op1_type == IS_VAR || opline->op1_type == IS_TMP_VAR) {
              break;
            }
            op1_info = OP1_INFO();
            if ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_STRING) {
              break;
            }
            if ((opline->result_type & IS_TMP_VAR)
             && (i + 1) <= end
             && ((opline+1)->opcode == ZEND_JMPZ
              || (opline+1)->opcode == ZEND_JMPNZ)
             && (opline+1)->op1_type == IS_TMP_VAR
             && (opline+1)->op1.var == opline->result.var) {
              i++;
              smart_branch_opcode = (opline+1)->opcode;
              target_label = ssa->cfg.blocks[b].successors[0];
              target_label2 = ssa->cfg.blocks[b].successors[1];
            } else {
              smart_branch_opcode = 0;
              target_label = target_label2 = (uint32_t)-1;
            }
            if (!zend_jit_in_array(&ctx, opline,
                op1_info, OP1_REG_ADDR(),
                smart_branch_opcode, target_label, target_label2,
                NULL)) {
              goto jit_failure;
            }
            goto done;
          case ZEND_FETCH_DIM_R:
          case ZEND_FETCH_DIM_IS:
          case ZEND_FETCH_LIST_R:
            if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
              break;
            }
            if (!zend_jit_fetch_dim_read(&ctx, opline, ssa, ssa_op,
                OP1_INFO(), OP1_REG_ADDR(), 0,
                OP2_INFO(), OP2_REG_ADDR(), OP2_RANGE(),
                RES_INFO(), RES_REG_ADDR(), IS_UNKNOWN)) {
              goto jit_failure;
            }
            goto done;
          case ZEND_FETCH_DIM_W:
          case ZEND_FETCH_DIM_RW:
//          case ZEND_FETCH_DIM_UNSET:
          case ZEND_FETCH_LIST_W:
            if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
              break;
            }
            if (opline->op1_type != IS_CV) {
              break;
            }
            if (!zend_jit_fetch_dim(&ctx, opline,
                OP1_INFO(), OP1_REG_ADDR(),
                OP2_INFO(), (opline->op2_type != IS_UNUSED) ? OP2_REG_ADDR() : 0,
                (opline->op2_type != IS_UNUSED) ? OP2_RANGE() : 0,
                RES_REG_ADDR(), IS_UNKNOWN)) {
              goto jit_failure;
            }
            goto done;
          case ZEND_ISSET_ISEMPTY_DIM_OBJ:
            if ((opline->extended_value & ZEND_ISEMPTY)) {
              // TODO: support for empty() ???
              break;
            }
            if (PROFITABILITY_CHECKS && (!ssa->ops || !ssa->var_info)) {
              break;
            }
            if ((opline->result_type & IS_TMP_VAR)
             && (i + 1) <= end
             && ((opline+1)->opcode == ZEND_JMPZ
              || (opline+1)->opcode == ZEND_JMPNZ)
             && (opline+1)->op1_type == IS_TMP_VAR
             && (opline+1)->op1.var == opline->result.var) {
              i++;
              smart_branch_opcode = (opline+1)->opcode;
              target_label = ssa->cfg.blocks[b].successors[0];
              target_label2 = ssa->cfg.blocks[b].successors[1];
            } else {
              smart_branch_opcode = 0;
              target_label = target_label2 = (uint32_t)-1;
            }
            if (!zend_jit_isset_isempty_dim(&ctx, opline,
                OP1_INFO(), OP1_REG_ADDR(), 0,
                OP2_INFO(), OP2_REG_ADDR(), OP2_RANGE(), IS_UNKNOWN,
                zend_may_throw(opline, ssa_op, op_array, ssa),
                smart_branch_opcode, target_label, target_label2,
                NULL)) {
              goto jit_failure;
            }
            goto done;
          case ZEND_FETCH_OBJ_FUNC_ARG:
          case ZEND_FETCH_OBJ_R:
          case ZEND_FETCH_OBJ_IS:
          case ZEND_FETCH_OBJ_W:
            ce = NULL;
            ce_is_instanceof = false;
            on_this = false;
            if (opline->op1_type == IS_UNUSED) {
              op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
              op1_addr = 0;
              ce = op_array->scope;
              /* scope is NULL for closures. */
              if (ce) {
                ce_is_instanceof = !(ce->ce_flags & ZEND_ACC_FINAL);
              }
              on_this = true;
            } else {
              op1_info = OP1_INFO();
              if (!(op1_info & MAY_BE_OBJECT)) {
                break;
              }
              op1_addr = OP1_REG_ADDR();
              if (ssa->var_info && ssa->ops) {
                zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
                if (ssa_op->op1_use >= 0) {
                  zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
                  if (op1_ssa->ce && !op1_ssa->ce->create_object) {
                    ce = op1_ssa->ce;
                    ce_is_instanceof = op1_ssa->is_instanceof;
                  }
                }
              }
            }
            if (opline->op2_type != IS_CONST
             || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
             || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
              break;
            }
            if (opline->opcode == ZEND_FETCH_OBJ_FUNC_ARG) {
              /* FETCH_OBJ_FUNC_ARG's by-value fetch dispatches into the
               * FETCH_OBJ_R handler, which may take the SIMPLE_GET hook fast
               * path and push a getter frame; by-ref dispatches into
               * FETCH_OBJ_W. The function JIT may keep values solely in
               * registers, so we must NOT exit to the VM (stale stack slots).
               * Inline the by-value path through zend_jit_fetch_obj, which runs
               * the hook getter inside a helper and keeps all registers live.
               * The by-ref path has no SIMPLE_GET fast path, so the generic
               * handler (a full C call, safe under register allocation) is used.
               * This mirrors the tracing JIT fix for GH-21006 (GH-21369); the
               * runtime by-ref check is required because the passing mode is
               * only known once the callee is resolved via namespace fallback.
               * See GH-22857. */
              if (!zend_jit_fetch_obj_func_arg(jit, opline, op_array, ssa, ssa_op,
                  op1_info, op1_addr, ce, ce_is_instanceof, on_this, RES_REG_ADDR())) {
                goto jit_failure;
              }
            } else {
              if (!zend_jit_fetch_obj(&ctx, opline, op_array, ssa, ssa_op,
                  op1_info, op1_addr, 0, ce, ce_is_instanceof, on_this, 0, 0, NULL,
                  RES_REG_ADDR(), IS_UNKNOWN,
                  zend_may_throw(opline, ssa_op, op_array, ssa))) {
                goto jit_failure;
              }
            }
            goto done;
          case ZEND_FETCH_STATIC_PROP_R:
          case ZEND_FETCH_STATIC_PROP_IS:
          case ZEND_FETCH_STATIC_PROP_W:
          case ZEND_FETCH_STATIC_PROP_RW:
          case ZEND_FETCH_STATIC_PROP_UNSET:
            if (!(opline->op1_type == IS_CONST
             && (opline->op2_type == IS_CONST
              || (opline->op2_type == IS_UNUSED
               && ((opline->op2.num & ZEND_FETCH_CLASS_MASK) == ZEND_FETCH_CLASS_SELF
                || (opline->op2.num & ZEND_FETCH_CLASS_MASK) == ZEND_FETCH_CLASS_PARENT))))) {
              break;
            }
            if (!zend_jit_fetch_static_prop(&ctx, opline, op_array)) {
              goto jit_failure;
            }
            goto done;
          case ZEND_BIND_GLOBAL:
            if (!ssa->ops || !ssa->var_info) {
              op1_info = MAY_BE_ANY|MAY_BE_REF;
            } else {
              op1_info = OP1_INFO();
            }
            if (!zend_jit_bind_global(&ctx, opline, op1_info)) {
              goto jit_failure;
            }
            goto done;
          case ZEND_RECV:
            if (!zend_jit_recv(&ctx, opline, op_array, NULL)) {
              goto jit_failure;
            }
            goto done;
          case ZEND_RECV_INIT:
            if (!zend_jit_recv_init(&ctx, opline, op_array,
                (opline + 1)->opcode != ZEND_RECV_INIT,
                zend_may_throw(opline, ssa_op, op_array, ssa))) {
              goto jit_failure;
            }
            goto done;
          case ZEND_FREE:
          case ZEND_FE_FREE:
            if (!zend_jit_free(&ctx, opline, OP1_INFO(),
                zend_may_throw(opline, ssa_op, op_array, ssa))) {
              goto jit_failure;
            }
            goto done;
          case ZEND_ECHO:
            op1_info = OP1_INFO();
            if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
              break;
            }
            if (!zend_jit_echo(&ctx, opline, op1_info)) {
              goto jit_failure;
            }
            goto done;
          case ZEND_STRLEN:
            op1_info = OP1_INFO();
            if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
              break;
            }
            if (!zend_jit_strlen(&ctx, opline, op1_info, OP1_REG_ADDR(), RES_REG_ADDR())) {
              goto jit_failure;
            }
            goto done;
          case ZEND_COUNT:
            op1_info = OP1_INFO();
            if ((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_ARRAY) {
              break;
            }
            if (!zend_jit_count(&ctx, opline, op1_info, OP1_REG_ADDR(), RES_REG_ADDR(), zend_may_throw(opline, ssa_op, op_array, ssa))) {
              goto jit_failure;
            }
            goto done;
          case ZEND_FETCH_THIS:
            if (!zend_jit_fetch_this(&ctx, opline, op_array, 0)) {
              goto jit_failure;
            }
            goto done;
          case ZEND_SWITCH_LONG:
          case ZEND_SWITCH_STRING:
          case ZEND_MATCH:
            if (!zend_jit_switch(&ctx, opline, op_array, ssa, NULL, NULL)) {
              goto jit_failure;
            }
            goto done;
          case ZEND_VERIFY_RETURN_TYPE:
            if (opline->op1_type == IS_UNUSED) {
              /* Always throws */
              break;
            }
            if (opline->op1_type == IS_CONST) {
              /* TODO Different instruction format, has return value */
              break;
            }
            if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) {
              /* Not worth bothering with */
              break;
            }
            if (OP1_INFO() & MAY_BE_REF) {
              /* TODO May need reference unwrapping. */
              break;
            }
            if (!zend_jit_verify_return_type(&ctx, opline, op_array, OP1_INFO(), NULL)) {
              goto jit_failure;
            }
            goto done;
          case ZEND_FE_RESET_R:
            op1_info = OP1_INFO();
            if ((op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) != MAY_BE_ARRAY) {
              break;
            }
            if (!zend_jit_fe_reset(&ctx, opline, op1_info)) {
              goto jit_failure;
            }
            goto done;
          case ZEND_FE_FETCH_R:
            op1_info = OP1_INFO();
            if ((op1_info & MAY_BE_ANY) != MAY_BE_ARRAY) {
              break;
            }
            if (!zend_jit_fe_fetch(&ctx, opline, op1_info, OP2_INFO(),
                ssa->cfg.blocks[b].successors[0], opline->opcode, NULL)) {
              goto jit_failure;
            }
            goto done;
          case ZEND_FETCH_CONSTANT:
            if (!zend_jit_fetch_constant(&ctx, opline, op_array, ssa, ssa_op, RES_REG_ADDR())) {
              goto jit_failure;
            }
            goto done;
          case ZEND_JMP_FRAMELESS:
            if (!zend_jit_jmp_frameless(&ctx, opline, /* exit_addr */ NULL, /* guard */ 0)) {
              goto jit_failure;
            }
            goto done;
          case ZEND_INIT_METHOD_CALL:
            if (opline->op2_type != IS_CONST
             || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING) {
              break;
            }
            ce = NULL;
            ce_is_instanceof = false;
            on_this = false;
            if (opline->op1_type == IS_UNUSED) {
              op1_info = MAY_BE_OBJECT|MAY_BE_RC1|MAY_BE_RCN;
              op1_addr = 0;
              ce = op_array->scope;
              /* scope is NULL for closures. */
              if (ce) {
                ce_is_instanceof = !(ce->ce_flags & ZEND_ACC_FINAL);
              }
              on_this = true;
            } else {
              op1_info = OP1_INFO();
              if (!(op1_info & MAY_BE_OBJECT)) {
                break;
              }
              op1_addr = OP1_REG_ADDR();
              if (ssa->var_info && ssa->ops) {
                zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
                if (ssa_op->op1_use >= 0) {
                  zend_ssa_var_info *op1_ssa = ssa->var_info + ssa_op->op1_use;
                  if (op1_ssa->ce && !op1_ssa->ce->create_object) {
                    ce = op1_ssa->ce;
                    ce_is_instanceof = op1_ssa->is_instanceof;
                  }
                }
              }
            }
            if (!zend_jit_init_method_call(&ctx, opline, b, op_array, ssa, ssa_op, call_level,
                op1_info, op1_addr, ce, ce_is_instanceof, on_this, 0, NULL,
                NULL, 0,
                -1, -1,
                0)) {
              goto jit_failure;
            }
            goto done;
          case ZEND_INIT_STATIC_METHOD_CALL:
            if (!(opline->op2_type == IS_CONST
             && (opline->op1_type == IS_CONST
              || (opline->op1_type == IS_UNUSED
               && ((opline->op1.num & ZEND_FETCH_CLASS_MASK) == ZEND_FETCH_CLASS_SELF
                || (opline->op1.num & ZEND_FETCH_CLASS_MASK) == ZEND_FETCH_CLASS_PARENT))))) {
              break;
            }
            if (!zend_jit_init_static_method_call(&ctx, opline, b, op_array, ssa, ssa_op, call_level,
                NULL, 0)) {
              goto jit_failure;
            }
            goto done;
          case ZEND_ROPE_INIT:
          case ZEND_ROPE_ADD:
          case ZEND_ROPE_END:
            op2_info = OP2_INFO();
            if ((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) != MAY_BE_STRING) {
              break;
            }
            if (!zend_jit_rope(&ctx, opline, op2_info)) {
              goto jit_failure;
            }
            goto done;
          case ZEND_FRAMELESS_ICALL_0:
            if (!jit_frameless_icall0(jit, opline)) {
              goto jit_failure;
            }
            goto done;
          case ZEND_FRAMELESS_ICALL_1:
            op1_info = OP1_INFO();
            if (!jit_frameless_icall1(jit, opline, op1_info)) {
              goto jit_failure;
            }
            goto done;
          case ZEND_FRAMELESS_ICALL_2:
            op1_info = OP1_INFO();
            op2_info = OP2_INFO();
            if (!jit_frameless_icall2(jit, opline, op1_info, op2_info)) {
              goto jit_failure;
            }
            goto done;
          case ZEND_FRAMELESS_ICALL_3:
            op1_info = OP1_INFO();
            op2_info = OP2_INFO();
            if (!jit_frameless_icall3(jit, opline, op1_info, op2_info, OP1_DATA_INFO())) {
              goto jit_failure;
            }
            goto done;
          default:
            break;
        }
      }

      switch (opline->opcode) {
        case ZEND_RECV_INIT:
        case ZEND_BIND_GLOBAL:
          if (opline == op_array->opcodes ||
              opline->opcode != op_array->opcodes[i-1].opcode) {
            /* repeatable opcodes */
            if (!zend_jit_handler(&ctx, opline,
                zend_may_throw(opline, ssa_op, op_array, ssa))) {
              goto jit_failure;
            }
          }
          zend_jit_set_last_valid_opline(&ctx, opline+1);
          break;
        case ZEND_NOP:
        case ZEND_OP_DATA:
        case ZEND_SWITCH_LONG:
        case ZEND_SWITCH_STRING:
          break;
        case ZEND_MATCH:
          /* We have to exit to the VM because the MATCH handler performs an N-way jump for
           * which we can't generate simple (opcache.jit=1201) JIT code. */
          if (!zend_jit_tail_handler(&ctx, opline)) {
            goto jit_failure;
          }
          break;
        case ZEND_JMP:
          if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
            const zend_op *target = OP_JMP_ADDR(opline, opline->op1);

            if (!zend_jit_set_ip(&ctx, target)) {
              goto jit_failure;
            }
          }
          break;
        case ZEND_CATCH:
        case ZEND_FAST_CALL:
        case ZEND_FAST_RET:
        case ZEND_GENERATOR_CREATE:
        case ZEND_GENERATOR_RETURN:
        case ZEND_RETURN_BY_REF:
        case ZEND_RETURN:
        case ZEND_MATCH_ERROR:
        /* switch through trampoline */
        case ZEND_YIELD:
        case ZEND_YIELD_FROM:
        case ZEND_THROW:
        case ZEND_VERIFY_NEVER_TYPE:
          if (!zend_jit_tail_handler(&ctx, opline)) {
            goto jit_failure;
          }
          /* THROW and EXIT may be used in the middle of BB */
          /* don't generate code for the rest of BB */

          /* Skip current opline for call_level computation because it does not influence call_level.
           * Don't include last opline because end of loop already checks call level of last opline. */
          i++;
          for (; i < end; i++) {
            opline = op_array->opcodes + i;
            if (zend_jit_inc_call_level(opline->opcode)) {
              call_level++;
            } else if (zend_jit_dec_call_level(opline->opcode)) {
              call_level--;
            }
          }
          opline = op_array->opcodes + end;
          break;
        /* stackless execution */
        case ZEND_INCLUDE_OR_EVAL:
        case ZEND_DO_FCALL:
        case ZEND_DO_UCALL:
        case ZEND_DO_FCALL_BY_NAME:
          if (!zend_jit_call(&ctx, opline, b + 1)) {
            goto jit_failure;
          }
          break;
        case ZEND_JMPZ:
        case ZEND_JMPNZ:
          if (opline > op_array->opcodes + ssa->cfg.blocks[b].start &&
              ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
            /* smart branch */
            if (!zend_jit_cond_jmp(&ctx, opline + 1, ssa->cfg.blocks[b].successors[0])) {
              goto jit_failure;
            }
            goto done;
          }
          ZEND_FALLTHROUGH;
        case ZEND_JMPZ_EX:
        case ZEND_JMPNZ_EX:
        case ZEND_JMP_SET:
        case ZEND_COALESCE:
        case ZEND_JMP_NULL:
        case ZEND_FE_RESET_R:
        case ZEND_FE_RESET_RW:
        case ZEND_ASSERT_CHECK:
        case ZEND_FE_FETCH_R:
        case ZEND_FE_FETCH_RW:
        case ZEND_BIND_INIT_STATIC_OR_JMP:
        case ZEND_JMP_FRAMELESS:
          if (!zend_jit_handler(&ctx, opline,
              zend_may_throw(opline, ssa_op, op_array, ssa)) ||
              !zend_jit_cond_jmp(&ctx, opline + 1, ssa->cfg.blocks[b].successors[0])) {
            goto jit_failure;
          }
          break;
        case ZEND_NEW:
          if (!zend_jit_handler(&ctx, opline, 1)) {
            return 0;
          }
          if (opline->extended_value == 0 && (opline+1)->opcode == ZEND_DO_FCALL) {
            zend_class_entry *ce = NULL;

            if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNC) {
              if (ssa->ops && ssa->var_info) {
                zend_ssa_var_info *res_ssa = &ssa->var_info[ssa->ops[opline - op_array->opcodes].result_def];
                if (res_ssa->ce && !res_ssa->is_instanceof) {
                  ce = res_ssa->ce;
                }
              }
            } else {
              if (opline->op1_type == IS_CONST) {
                zval *zv = RT_CONSTANT(opline, opline->op1);
                if (Z_TYPE_P(zv) == IS_STRING) {
                  zval *lc = zv + 1;
                  ce = (zend_class_entry*)zend_hash_find_ptr(EG(class_table), Z_STR_P(lc));
                }
              }
            }

            i++;

            if (!ce || !(ce->ce_flags & ZEND_ACC_LINKED) || ce->constructor) {
              const zend_op *next_opline = opline + 1;

              ZEND_ASSERT(b + 1 == ssa->cfg.blocks[b].successors[0]);
              zend_jit_constructor(&ctx, next_opline, op_array, ssa, call_level, b + 1);
            }

            /* We skip over the DO_FCALL, so decrement call_level ourselves. */
            call_level--;
          }
          break;
        /* FETCH_OBJ_FUNC_ARG may dispatch into the FETCH_OBJ_R handler (if sent by value). */
        case ZEND_FETCH_OBJ_FUNC_ARG:
        case ZEND_FETCH_OBJ_R:
          if (!zend_jit_handler(&ctx, opline,
            zend_may_throw(opline, ssa_op, op_array, ssa))) {
            goto jit_failure;
          }

          /* Cache slot is only used for IS_CONST op2, so only that can result in hook fast path. */
          if (opline->op2_type == IS_CONST) {
            if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
              if (opline->op1_type == IS_UNUSED) {
                ce = op_array->scope;
              } else {
                ce = NULL;
              }
            }

            if (!ce || !(ce->ce_flags & ZEND_ACC_FINAL) || ce->num_hooked_props > 0) {
              /* If a simple hook is called, exit to the VM. */
              ir_ref if_hook_enter = ir_IF(jit_CMP_IP(jit, IR_EQ, opline + 1));
              ir_IF_FALSE(if_hook_enter);
              if (GCC_GLOBAL_REGS) {
                ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
              } else {
                zend_jit_vm_enter(jit, jit_IP(jit));
              }
              ir_IF_TRUE(if_hook_enter);
            }
          }

          break;
        default:
          if (!zend_jit_handler(&ctx, opline,
              zend_may_throw(opline, ssa_op, op_array, ssa))) {
            goto jit_failure;
          }
          if (i == end
           && (opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
            /* smart branch split across basic blocks */
            if (!zend_jit_set_cond(&ctx, opline, opline + 2, opline->result.var)) {
              goto jit_failure;
            }
          }
      }
done:
      if (zend_jit_dec_call_level(opline->opcode)) {
        call_level--;
      }
    }
    zend_jit_bb_end(&ctx, b);
  }

  if (jit->return_inputs) {
    zend_jit_common_return(jit);

    bool left_frame = false;
    if (op_array->last_var > 100) {
      /* Too many CVs to unroll */
      if (!zend_jit_free_cvs(&ctx)) {
        goto jit_failure;
      }
      left_frame = true;
    }
    if (!left_frame) {
      int j;

      for (j = 0 ; j < op_array->last_var; j++) {
        uint32_t info = zend_ssa_cv_info(op_array, ssa, j);

        if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
          if (!left_frame) {
            left_frame = true;
              if (!zend_jit_leave_frame(&ctx)) {
              goto jit_failure;
              }
          }
          if (!zend_jit_free_cv(&ctx, info, j)) {
            goto jit_failure;
          }
        }
      }
    }
    if (!zend_jit_leave_func(&ctx, op_array, NULL, MAY_BE_ANY, left_frame,
        NULL, NULL, (ssa->cfg.flags & ZEND_FUNC_INDIRECT_VAR_ACCESS) != 0, 1)) {
      goto jit_failure;
    }
  }

  handler = zend_jit_finish(&ctx);
  if (!handler) {
    goto jit_failure;
  }
  zend_jit_free_ctx(&ctx);

  if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
    zend_arena_release(&CG(arena), checkpoint);
  }
  return SUCCESS;

jit_failure:
  zend_jit_free_ctx(&ctx);
  if (JIT_G(opt_flags) & (ZEND_JIT_REG_ALLOC_LOCAL|ZEND_JIT_REG_ALLOC_GLOBAL)) {
    zend_arena_release(&CG(arena), checkpoint);
  }
  return FAILURE;
}

static void zend_jit_collect_calls(zend_op_array *op_array, zend_script *script)
{
  zend_func_info *func_info;

  if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
      JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
      JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
      func_info = ZEND_FUNC_INFO(op_array);
  } else {
    func_info = zend_arena_calloc(&CG(arena), 1, sizeof(zend_func_info));
    ZEND_SET_FUNC_INFO(op_array, func_info);
  }
  zend_analyze_calls(&CG(arena), script, ZEND_CALL_TREE, op_array, func_info);
}

static void zend_jit_cleanup_func_info(zend_op_array *op_array)
{
  zend_func_info *func_info = ZEND_FUNC_INFO(op_array);
  zend_call_info *caller_info, *callee_info;

  if (func_info) {
    caller_info = func_info->caller_info;
    callee_info = func_info->callee_info;

    if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
        JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
        JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
      func_info->num = 0;
      func_info->flags &= ZEND_FUNC_JIT_ON_FIRST_EXEC
        | ZEND_FUNC_JIT_ON_PROF_REQUEST
        | ZEND_FUNC_JIT_ON_HOT_COUNTERS
        | ZEND_FUNC_JIT_ON_HOT_TRACE;
      memset(&func_info->ssa, 0, sizeof(zend_func_info) - offsetof(zend_func_info, ssa));
    } else {
      ZEND_SET_FUNC_INFO(op_array, NULL);
    }

    while (caller_info) {
      if (caller_info->caller_op_array) {
        zend_jit_cleanup_func_info(caller_info->caller_op_array);
      }
      caller_info = caller_info->next_caller;
    }
    while (callee_info) {
      if (callee_info->callee_func && callee_info->callee_func->type == ZEND_USER_FUNCTION) {
        zend_jit_cleanup_func_info(&callee_info->callee_func->op_array);
      }
      callee_info = callee_info->next_callee;
    }
  }
}

static int zend_real_jit_func(zend_op_array *op_array, zend_script *script, const zend_op *rt_opline, uint8_t trigger)
{
  zend_ssa ssa;
  void *checkpoint;
  zend_func_info *func_info;
  uint8_t orig_trigger;

  if (*dasm_ptr == dasm_end) {
    return FAILURE;
  }

  orig_trigger = JIT_G(trigger);
  JIT_G(trigger) = trigger;
  checkpoint = zend_arena_checkpoint(CG(arena));

  /* Build SSA */
  memset(&ssa, 0, sizeof(zend_ssa));

  if (op_array->fn_flags & ZEND_ACC_CLOSURE) {
    if (trigger == ZEND_JIT_ON_FIRST_EXEC) {
      zend_jit_op_array_extension *jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
      op_array = (zend_op_array*) jit_extension->op_array;
    } else if (trigger == ZEND_JIT_ON_HOT_COUNTERS) {
      zend_jit_op_array_hot_extension *jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array);
      op_array = (zend_op_array*) jit_extension->op_array;
    } else {
      ZEND_ASSERT(!op_array->scope);
    }
  }

  if (zend_jit_op_array_analyze1(op_array, script, &ssa) != SUCCESS) {
    goto jit_failure;
  }

  if (JIT_G(opt_level) >= ZEND_JIT_LEVEL_OPT_FUNCS) {
    zend_jit_collect_calls(op_array, script);
    func_info = ZEND_FUNC_INFO(op_array);
    func_info->call_map = zend_build_call_map(&CG(arena), func_info, op_array);
    if (op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
      zend_init_func_return_info(op_array, script, &func_info->return_info);
    }
  }

  if (zend_jit_op_array_analyze2(op_array, script, &ssa, ZCG(accel_directives).optimization_level) != SUCCESS) {
    goto jit_failure;
  }

  if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
    zend_dump_op_array(op_array, ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", &ssa);
  }

  if (zend_jit(op_array, &ssa, rt_opline) != SUCCESS) {
    goto jit_failure;
  }

  zend_jit_cleanup_func_info(op_array);
  zend_arena_release(&CG(arena), checkpoint);
  JIT_G(trigger) = orig_trigger;
  return SUCCESS;

jit_failure:
  zend_jit_cleanup_func_info(op_array);
  zend_arena_release(&CG(arena), checkpoint);
  JIT_G(trigger) = orig_trigger;
  return FAILURE;
}

/* GCC cannot tail-call from a function that uses setjmp. */
static zend_never_inline void zend_runtime_jit_compile(zend_op_array *op_array)
{
  const zend_op *opline = op_array->opcodes;
  zend_jit_op_array_extension *jit_extension;
  bool do_bailout = 0;

  zend_shared_alloc_lock();
  jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);

  if (jit_extension && !(jit_extension->func_info.flags & ZEND_FUNC_JITED)) {

    SHM_UNPROTECT();
    zend_jit_unprotect();

    zend_try {
      /* restore original opcode handlers */
      if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
        while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
          opline++;
        }
      }
      ((zend_op*)opline)->handler = jit_extension->orig_handler;

      /* perform real JIT for this function */
      zend_real_jit_func(op_array, NULL, NULL, ZEND_JIT_ON_FIRST_EXEC);

      jit_extension->func_info.flags |= ZEND_FUNC_JITED;
    } zend_catch {
      do_bailout = true;
    } zend_end_try();

    zend_jit_protect();
    SHM_PROTECT();
  }

  zend_shared_alloc_unlock();

  if (do_bailout) {
    zend_bailout();
  }
}

/* Run-time JIT handler */
#if ZEND_VM_KIND == ZEND_VM_KIND_CALL || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL
static ZEND_OPCODE_HANDLER_RET ZEND_OPCODE_HANDLER_CCONV zend_runtime_jit(ZEND_OPCODE_HANDLER_ARGS)
#else
static ZEND_OPCODE_HANDLER_RET ZEND_OPCODE_HANDLER_FUNC_CCONV zend_runtime_jit(ZEND_OPCODE_HANDLER_ARGS)
#endif
{
#if GCC_GLOBAL_REGS
  zend_execute_data *execute_data;
#else
  const zend_op *orig_opline = opline;
#endif

  execute_data = EG(current_execute_data);
  zend_runtime_jit_compile(&EX(func)->op_array);

  /* JIT-ed code is going to be called by VM */
#if GCC_GLOBAL_REGS
  return; // ZEND_VM_CONTINUE
#else
  opline = orig_opline;
  ZEND_OPCODE_RETURN();
#endif
}

void zend_jit_check_funcs(HashTable *function_table, bool is_method) {
  zend_op *opline;
  zend_function *func;
  zend_op_array *op_array;
  uintptr_t counter;
  zend_jit_op_array_extension *jit_extension;

  ZEND_HASH_MAP_REVERSE_FOREACH_PTR(function_table, func) {
    if (func->type == ZEND_INTERNAL_FUNCTION) {
      break;
    }
    op_array = &func->op_array;
    opline = op_array->opcodes;
    if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
      while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
        opline++;
      }
    }
    if (opline->handler == zend_jit_profile_jit_handler) {
      if (!RUN_TIME_CACHE(op_array)) {
        continue;
      }
      counter = (uintptr_t)ZEND_COUNTER_INFO(op_array);
      ZEND_COUNTER_INFO(op_array) = 0;
      jit_extension = (zend_jit_op_array_extension*)ZEND_FUNC_INFO(op_array);
      opline->handler = jit_extension->orig_handler;
      if (((double)counter / (double)zend_jit_profile_counter) > JIT_G(prof_threshold)) {
        zend_real_jit_func(op_array, NULL, NULL, ZEND_JIT_ON_PROF_REQUEST);
      }
    }
  } ZEND_HASH_FOREACH_END();
}

void ZEND_FASTCALL zend_jit_hot_func(zend_execute_data *execute_data, const zend_op *opline)
{
  zend_op_array *op_array = &EX(func)->op_array;
  zend_jit_op_array_hot_extension *jit_extension;
  uint32_t i;
  bool do_bailout = 0;

  zend_shared_alloc_lock();
  jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array);

  if (jit_extension && !(jit_extension->func_info.flags & ZEND_FUNC_JITED)) {
    SHM_UNPROTECT();
    zend_jit_unprotect();

    zend_try {
      for (i = 0; i < op_array->last; i++) {
        op_array->opcodes[i].handler = jit_extension->orig_handlers[i];
      }

      EX(opline) = opline;

      /* perform real JIT for this function */
      zend_real_jit_func(op_array, NULL, opline, ZEND_JIT_ON_HOT_COUNTERS);

      jit_extension->func_info.flags |= ZEND_FUNC_JITED;
    } zend_catch {
      do_bailout = 1;
    } zend_end_try();

    zend_jit_protect();
    SHM_PROTECT();
  }

  zend_shared_alloc_unlock();

  if (do_bailout) {
    zend_bailout();
  }
  /* JIT-ed code is going to be called by VM */
}

static void zend_jit_setup_hot_counters_ex(zend_op_array *op_array, zend_cfg *cfg)
{
  if (JIT_G(hot_func)) {
    zend_op *opline = op_array->opcodes;

    if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
      while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
        opline++;
      }
    }

    opline->handler = zend_jit_func_hot_counter_handler;
  }

  if (JIT_G(hot_loop)) {
    uint32_t i;

    for (i = 0; i < cfg->blocks_count; i++) {
      if ((cfg->blocks[i].flags & ZEND_BB_REACHABLE) &&
          (cfg->blocks[i].flags & ZEND_BB_LOOP_HEADER)) {
          op_array->opcodes[cfg->blocks[i].start].handler =
          zend_jit_loop_hot_counter_handler;
      }
    }
  }
}

static int zend_jit_restart_hot_counters(zend_op_array *op_array)
{
  zend_jit_op_array_hot_extension *jit_extension;
  zend_cfg cfg;
  uint32_t i;

  jit_extension = (zend_jit_op_array_hot_extension*)ZEND_FUNC_INFO(op_array);
  for (i = 0; i < op_array->last; i++) {
    op_array->opcodes[i].handler = jit_extension->orig_handlers[i];
  }

  if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
    return FAILURE;
  }

  zend_jit_setup_hot_counters_ex(op_array, &cfg);

  return SUCCESS;
}

static int zend_jit_setup_hot_counters(zend_op_array *op_array)
{
  zend_jit_op_array_hot_extension *jit_extension;
  zend_cfg cfg;
  uint32_t i;

  ZEND_ASSERT(!JIT_G(hot_func) || zend_jit_func_hot_counter_handler != NULL);
  ZEND_ASSERT(!JIT_G(hot_loop) || zend_jit_loop_hot_counter_handler != NULL);

  if (zend_jit_build_cfg(op_array, &cfg) != SUCCESS) {
    return FAILURE;
  }

  jit_extension = (zend_jit_op_array_hot_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_hot_extension) + (op_array->last - 1) * sizeof(void*));
  if (!jit_extension) {
    return FAILURE;
  }
  memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
  jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_HOT_COUNTERS;
  jit_extension->op_array = op_array;
  jit_extension->counter = &zend_jit_hot_counters[zend_jit_op_array_hash(op_array) & (ZEND_HOT_COUNTERS_COUNT - 1)];
  for (i = 0; i < op_array->last; i++) {
    jit_extension->orig_handlers[i] = op_array->opcodes[i].handler;
  }
  ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);

  zend_jit_setup_hot_counters_ex(op_array, &cfg);

  zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);

  return SUCCESS;
}

#include "jit/zend_jit_trace.c"

int zend_jit_op_array(zend_op_array *op_array, zend_script *script)
{
  if (dasm_ptr == NULL) {
    return FAILURE;
  }

  if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC) {
    zend_jit_op_array_extension *jit_extension;
    zend_op *opline = op_array->opcodes;

    if (CG(compiler_options) & ZEND_COMPILE_PRELOAD) {
      ZEND_SET_FUNC_INFO(op_array, NULL);
      zend_error(E_WARNING, "Preloading is incompatible with first-exec and profile triggered JIT");
      return SUCCESS;
    }

    /* Set run-time JIT handler */
    ZEND_ASSERT(zend_jit_runtime_jit_handler != NULL);
    if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
      while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
        opline++;
      }
    }
    jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension));
    if (!jit_extension) {
      return FAILURE;
    }
    memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
    jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_FIRST_EXEC;
    jit_extension->op_array = op_array;
    jit_extension->orig_handler = opline->handler;
    ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
    opline->handler = zend_jit_runtime_jit_handler;
    zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);

    return SUCCESS;
  } else if (JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST) {
    zend_jit_op_array_extension *jit_extension;
    zend_op *opline = op_array->opcodes;

    if (CG(compiler_options) & ZEND_COMPILE_PRELOAD) {
      ZEND_SET_FUNC_INFO(op_array, NULL);
      zend_error(E_WARNING, "Preloading is incompatible with first-exec and profile triggered JIT");
      return SUCCESS;
    }

    ZEND_ASSERT(zend_jit_profile_jit_handler != NULL);
    if (op_array->function_name) {
      if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
        while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
          opline++;
        }
      }
      jit_extension = (zend_jit_op_array_extension*)zend_shared_alloc(sizeof(zend_jit_op_array_extension));
      if (!jit_extension) {
        return FAILURE;
      }
      memset(&jit_extension->func_info, 0, sizeof(zend_func_info));
      jit_extension->func_info.flags = ZEND_FUNC_JIT_ON_PROF_REQUEST;
      jit_extension->op_array = op_array;
      jit_extension->orig_handler = opline->handler;
      ZEND_SET_FUNC_INFO(op_array, (void*)jit_extension);
      opline->handler = zend_jit_profile_jit_handler;
      zend_shared_alloc_register_xlat_entry(op_array->opcodes, jit_extension);
    }

    return SUCCESS;
  } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
    return zend_jit_setup_hot_counters(op_array);
  } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
    return zend_jit_setup_hot_trace_counters(op_array);
  } else if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
    return zend_real_jit_func(op_array, script, NULL, ZEND_JIT_ON_SCRIPT_LOAD);
  } else {
    ZEND_UNREACHABLE();
  }
  return FAILURE;
}

static void zend_jit_link_func_info(zend_op_array *op_array)
{
  if (!ZEND_FUNC_INFO(op_array)) {
    void *jit_extension = zend_shared_alloc_get_xlat_entry(op_array->opcodes);

    if (jit_extension) {
      ZEND_SET_FUNC_INFO(op_array, jit_extension);
    }
  }
}

int zend_jit_script(zend_script *script)
{
  void *checkpoint;
  zend_call_graph call_graph;
  zend_func_info *info;
  int i;

  if (dasm_ptr == NULL || *dasm_ptr == dasm_end) {
    return FAILURE;
  }

  checkpoint = zend_arena_checkpoint(CG(arena));

  call_graph.op_arrays_count = 0;
  zend_build_call_graph(&CG(arena), script, &call_graph);

  zend_analyze_call_graph(&CG(arena), script, &call_graph);

  if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC ||
      JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST ||
      JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS ||
      JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
    for (i = 0; i < call_graph.op_arrays_count; i++) {
      if (zend_jit_op_array(call_graph.op_arrays[i], script) != SUCCESS) {
        goto jit_failure;
      }
    }
  } else if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
    for (i = 0; i < call_graph.op_arrays_count; i++) {
      info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
      if (info) {
        if (zend_jit_op_array_analyze1(call_graph.op_arrays[i], script, &info->ssa) != SUCCESS) {
          goto jit_failure;
        }
        info->ssa.cfg.flags |= info->flags;
        info->flags = info->ssa.cfg.flags;
      }
    }

    for (i = 0; i < call_graph.op_arrays_count; i++) {
      info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
      if (info) {
        info->call_map = zend_build_call_map(&CG(arena), info, call_graph.op_arrays[i]);
        if (call_graph.op_arrays[i]->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
          zend_init_func_return_info(call_graph.op_arrays[i], script, &info->return_info);
        }
      }
    }

    for (i = 0; i < call_graph.op_arrays_count; i++) {
      info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
      if (info) {
        if (zend_jit_op_array_analyze2(call_graph.op_arrays[i], script, &info->ssa, ZCG(accel_directives).optimization_level) != SUCCESS) {
          goto jit_failure;
        }
        info->flags = info->ssa.cfg.flags;
      }
    }

    for (i = 0; i < call_graph.op_arrays_count; i++) {
      info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
      if (info) {
        if (JIT_G(debug) & ZEND_JIT_DEBUG_SSA) {
          zend_dump_op_array(call_graph.op_arrays[i], ZEND_DUMP_HIDE_UNREACHABLE|ZEND_DUMP_RC_INFERENCE|ZEND_DUMP_SSA, "JIT", &info->ssa);
        }
        if (zend_jit(call_graph.op_arrays[i], &info->ssa, NULL) != SUCCESS) {
          goto jit_failure;
        }
      }
    }

    for (i = 0; i < call_graph.op_arrays_count; i++) {
      ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL);
    }
  } else {
    ZEND_UNREACHABLE();
  }

  zend_arena_release(&CG(arena), checkpoint);

  if (JIT_G(trigger) == ZEND_JIT_ON_FIRST_EXEC
   || JIT_G(trigger) == ZEND_JIT_ON_PROF_REQUEST
   || JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS
   || JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
    zend_class_entry *ce;
    zend_op_array *op_array;
    zval *zv;
    zend_property_info *prop;

    ZEND_HASH_MAP_FOREACH_VAL(&script->class_table, zv) {
      if (Z_TYPE_P(zv) == IS_ALIAS_PTR) {
        continue;
      }

      ce = Z_PTR_P(zv);
      ZEND_ASSERT(ce->type == ZEND_USER_CLASS);

      ZEND_HASH_MAP_FOREACH_PTR(&ce->function_table, op_array) {
        zend_jit_link_func_info(op_array);
      } ZEND_HASH_FOREACH_END();

      if (ce->num_hooked_props > 0) {
        ZEND_HASH_MAP_FOREACH_PTR(&ce->properties_info, prop) {
          if (prop->hooks) {
            for (uint32_t i = 0; i < ZEND_PROPERTY_HOOK_COUNT; i++) {
              if (prop->hooks[i]) {
                op_array = &prop->hooks[i]->op_array;
                zend_jit_link_func_info(op_array);
              }
            }
          }
        } ZEND_HASH_FOREACH_END();
      }
    } ZEND_HASH_FOREACH_END();
  }

  return SUCCESS;

jit_failure:
  if (JIT_G(trigger) == ZEND_JIT_ON_SCRIPT_LOAD) {
    for (i = 0; i < call_graph.op_arrays_count; i++) {
      ZEND_SET_FUNC_INFO(call_graph.op_arrays[i], NULL);
    }
  }
  zend_arena_release(&CG(arena), checkpoint);
  return FAILURE;
}

void zend_jit_unprotect(void)
{
#ifdef ZEND_JIT_USE_APPLE_MAP_JIT
  pthread_jit_write_protect_np(0);
#elif defined(HAVE_MPROTECT)
  if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
    int opts = PROT_READ | PROT_WRITE;
# ifdef ZTS
    opts |= PROT_EXEC;
# endif
    if (mprotect(dasm_buf, dasm_size, opts) != 0) {
      zend_accel_error_noreturn(ACCEL_LOG_FATAL, "mprotect() failed [%d] %s\n", errno, strerror(errno));
    }
  }
#elif defined(_WIN32)
  if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
    DWORD old, new;
# ifdef ZTS
    new = PAGE_EXECUTE_READWRITE;
# else
    new = PAGE_READWRITE;
# endif
    if (!VirtualProtect(dasm_buf, dasm_size, new, &old)) {
      DWORD err = GetLastError();
      char *msg = php_win32_error_to_msg(err);
      zend_accel_error_noreturn(ACCEL_LOG_FATAL, "VirtualProtect() failed [%lu] %s\n", err, msg);
    }
  }
#endif
}

void zend_jit_protect(void)
{
#ifdef ZEND_JIT_USE_APPLE_MAP_JIT
  pthread_jit_write_protect_np(1);
#elif defined(HAVE_MPROTECT)
  if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
    if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_EXEC) != 0) {
      zend_accel_error_noreturn(ACCEL_LOG_FATAL, "mprotect() failed [%d] %s\n", errno, strerror(errno));
    }
  }
#elif defined(_WIN32)
  if (!(JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP))) {
    DWORD old;

    if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READ, &old)) {
      DWORD err = GetLastError();
      char *msg = php_win32_error_to_msg(err);
      zend_accel_error_noreturn(ACCEL_LOG_FATAL, "VirtualProtect() failed [%lu] %s\n", err, msg);
    }
  }
#endif
}

static void zend_jit_init_handlers(void)
{
#if ZEND_VM_KIND == ZEND_VM_KIND_HYBRID
    zend_jit_runtime_jit_handler = (zend_vm_opcode_handler_t)zend_jit_stub_handlers[jit_stub_hybrid_runtime_jit];
    zend_jit_profile_jit_handler = (zend_vm_opcode_handler_t)zend_jit_stub_handlers[jit_stub_hybrid_profile_jit];
    zend_jit_func_hot_counter_handler = (zend_vm_opcode_handler_t)zend_jit_stub_handlers[jit_stub_hybrid_func_hot_counter];
    zend_jit_loop_hot_counter_handler = (zend_vm_opcode_handler_t)zend_jit_stub_handlers[jit_stub_hybrid_loop_hot_counter];
    zend_jit_func_trace_counter_handler = (zend_vm_opcode_handler_t)zend_jit_stub_handlers[jit_stub_hybrid_func_trace_counter];
    zend_jit_ret_trace_counter_handler = (zend_vm_opcode_handler_t)zend_jit_stub_handlers[jit_stub_hybrid_ret_trace_counter];
    zend_jit_loop_trace_counter_handler = (zend_vm_opcode_handler_t)zend_jit_stub_handlers[jit_stub_hybrid_loop_trace_counter];
#else
    zend_jit_runtime_jit_handler = zend_runtime_jit;
    zend_jit_profile_jit_handler = zend_jit_profile_helper;
    zend_jit_func_hot_counter_handler = zend_jit_func_counter_helper;
    zend_jit_loop_hot_counter_handler = zend_jit_loop_counter_helper;
    zend_jit_func_trace_counter_handler = zend_jit_func_trace_helper;
    zend_jit_ret_trace_counter_handler = zend_jit_ret_trace_helper;
    zend_jit_loop_trace_counter_handler = zend_jit_loop_trace_helper;
#endif
}

static void zend_jit_globals_ctor(zend_jit_globals *jit_globals)
{
  memset(jit_globals, 0, sizeof(zend_jit_globals));
  zend_jit_trace_init_caches();
}

#ifdef ZTS
static void zend_jit_globals_dtor(zend_jit_globals *jit_globals)
{
  zend_jit_trace_free_caches(jit_globals);
}
#endif

static int zend_jit_parse_config_num(zend_long jit)
{
  if (jit == 0) {
    JIT_G(on) = 0;
    return SUCCESS;
  }

  if (jit < 0) return FAILURE;

  if (jit % 10 == 0 || jit % 10 > 5) return FAILURE;
  JIT_G(opt_level) = jit % 10;

  jit /= 10;
  if (jit % 10 > 5 || jit % 10 == 4) return FAILURE;
  JIT_G(trigger) = jit % 10;

  jit /= 10;
  if (jit % 10 > 2) return FAILURE;
  JIT_G(opt_flags) = jit % 10;

  jit /= 10;
  if (jit % 10 > 1) return FAILURE;
  JIT_G(opt_flags) |= ((jit % 10) ? ZEND_JIT_CPU_AVX : 0);

  if (jit / 10 != 0) return FAILURE;

  JIT_G(on) = 1;

  return SUCCESS;
}

int zend_jit_config(zend_string *jit, int stage)
{
  if (stage != ZEND_INI_STAGE_STARTUP && !JIT_G(enabled)) {
    if (stage == ZEND_INI_STAGE_RUNTIME) {
      zend_error(E_WARNING, "Cannot change opcache.jit setting at run-time (JIT is disabled)");
    }
    return FAILURE;
  }

  if (zend_string_equals_literal_ci(jit, "disable")) {
    JIT_G(enabled) = 0;
    JIT_G(on) = 0;
    return SUCCESS;
  } else if (ZSTR_LEN(jit) == 0
      || zend_string_equals_literal_ci(jit, "0")
      || zend_string_equals_literal_ci(jit, "off")
      || zend_string_equals_literal_ci(jit, "no")
      || zend_string_equals_literal_ci(jit, "false")) {
    JIT_G(enabled) = 1;
    JIT_G(on) = 0;
    return SUCCESS;
  } else if (zend_string_equals_literal_ci(jit, "1")
      || zend_string_equals_literal_ci(jit, "on")
      || zend_string_equals_literal_ci(jit, "yes")
      || zend_string_equals_literal_ci(jit, "true")
      || zend_string_equals_literal_ci(jit, "tracing")) {
    JIT_G(enabled) = 1;
    JIT_G(on) = 1;
    JIT_G(opt_level) = ZEND_JIT_LEVEL_OPT_FUNCS;
    JIT_G(trigger) = ZEND_JIT_ON_HOT_TRACE;
    JIT_G(opt_flags) = ZEND_JIT_REG_ALLOC_GLOBAL | ZEND_JIT_CPU_AVX;
    return SUCCESS;
  } else if (zend_string_equals_ci(jit, ZSTR_KNOWN(ZEND_STR_FUNCTION))) {
    JIT_G(enabled) = 1;
    JIT_G(on) = 1;
    JIT_G(opt_level) = ZEND_JIT_LEVEL_OPT_SCRIPT;
    JIT_G(trigger) = ZEND_JIT_ON_SCRIPT_LOAD;
    JIT_G(opt_flags) = ZEND_JIT_REG_ALLOC_GLOBAL | ZEND_JIT_CPU_AVX;
    return SUCCESS;
  } else  {
    char *end;
    zend_long num = ZEND_STRTOL(ZSTR_VAL(jit), &end, 10);
    if (end != ZSTR_VAL(jit) + ZSTR_LEN(jit) || zend_jit_parse_config_num(num) != SUCCESS) {
      goto failure;
    }
    JIT_G(enabled) = 1;
    return SUCCESS;
  }

failure:
  zend_error(E_WARNING, "Invalid \"opcache.jit\" setting. Should be \"disable\", \"on\", \"off\", \"tracing\", \"function\" or 4-digit number");
  JIT_G(enabled) = 0;
  JIT_G(on) = 0;
  return FAILURE;
}

int zend_jit_debug_config(zend_long old_val, zend_long new_val, int stage)
{
  if (stage != ZEND_INI_STAGE_STARTUP) {
    if (((old_val ^ new_val) & ZEND_JIT_DEBUG_PERSISTENT) != 0) {
      if (stage == ZEND_INI_STAGE_RUNTIME) {
        zend_error(E_WARNING, "Some opcache.jit_debug bits cannot be changed after startup");
      }
      return FAILURE;
    }
  }
  return SUCCESS;
}

void zend_jit_init(void)
{
#ifdef ZTS
  jit_globals_id = ts_allocate_fast_id(&jit_globals_id, &jit_globals_offset, sizeof(zend_jit_globals), (ts_allocate_ctor) zend_jit_globals_ctor, (ts_allocate_dtor) zend_jit_globals_dtor);
#else
  zend_jit_globals_ctor(&jit_globals);
#endif
}

#if ZEND_VM_KIND != ZEND_VM_KIND_CALL && ZEND_VM_KIND != ZEND_VM_KIND_TAILCALL && ZEND_VM_KIND != ZEND_VM_KIND_HYBRID
# error JIT is compatible only with CALL and HYBRID VM
#endif

int zend_jit_check_support(void)
{
  int i;

#ifdef ZEND_JIT_USE_APPLE_MAP_JIT
  if (!pthread_jit_write_protect_supported_np()) {
    zend_accel_error(ACCEL_LOG_WARNING,
      "Apple Silicon ZTS JIT requires pthread_jit_write_protect_np() support. JIT disabled.");
    JIT_G(enabled) = 0;
    JIT_G(on) = 0;
    return FAILURE;
  }
#endif

  if (zend_execute_ex != execute_ex) {
    if (zend_dtrace_enabled) {
      zend_error(E_WARNING, "JIT is incompatible with DTrace. JIT disabled.");
    } else if (strcmp(sapi_module.name, "phpdbg") != 0) {
      zend_error(E_WARNING, "JIT is incompatible with third party extensions that override zend_execute_ex(). JIT disabled.");
    }
    JIT_G(enabled) = 0;
    JIT_G(on) = 0;
    return FAILURE;
  }

  for (i = 0; i <= 256; i++) {
    switch (i) {
      /* JIT has no effect on these opcodes */
      case ZEND_BEGIN_SILENCE:
      case ZEND_END_SILENCE:
        break;
      default:
        if (zend_get_user_opcode_handler(i) != NULL) {
          zend_error(E_WARNING, "JIT is incompatible with third party extensions that setup user opcode handlers. JIT disabled.");
          JIT_G(enabled) = 0;
          JIT_G(on) = 0;
          return FAILURE;
        }
    }
  }

#if defined(IR_TARGET_AARCH64)
  if (JIT_G(buffer_size) > 128*1024*1024) {
    zend_error(E_WARNING, "JIT on AArch64 doesn't support opcache.jit_buffer_size above 128M.");
    JIT_G(enabled) = 0;
    JIT_G(on) = 0;
    return FAILURE;
  }
#elif defined(IR_TARGET_X64)
  if (JIT_G(buffer_size) > 2 * Z_L(1024*1024*1024)) {
    zend_error(E_WARNING, "JIT on x86_64 doesn't support opcache.jit_buffer_size above 2G.");
    JIT_G(enabled) = 0;
    JIT_G(on) = 0;
    return FAILURE;
  }
#endif

  return SUCCESS;
}

void zend_jit_startup(void *buf, size_t size, bool reattached)
{
  zend_jit_halt_op = zend_get_halt_op();
  zend_jit_interrupt_op = zend_get_interrupt_op();
  zend_jit_profile_counter_rid = zend_get_op_array_extension_handle(ACCELERATOR_PRODUCT_NAME);

#ifdef ZEND_JIT_USE_APPLE_MAP_JIT
  buf = mmap(NULL, size, PROT_READ | PROT_WRITE | PROT_EXEC,
    MAP_PRIVATE | MAP_ANON | MAP_JIT, -1, 0);
  if (buf == MAP_FAILED) {
    int error = errno;
    zend_accel_error_noreturn(ACCEL_LOG_FATAL,
      "Unable to allocate %zu bytes for JIT buffer using MAP_JIT: %s (%d)",
      size, strerror(error), error);
  }
  if (minherit(buf, size, VM_INHERIT_SHARE) != 0) {
    int error = errno;
    munmap(buf, size);
    zend_accel_error_noreturn(ACCEL_LOG_FATAL,
      "Unable to share JIT buffer across fork using minherit(): %s (%d)",
      strerror(error), error);
  }
#endif

  dasm_buf = buf;
  dasm_size = size;
  dasm_ptr = dasm_end = (void*)(((char*)dasm_buf) + size - sizeof(*dasm_ptr) * 2);

#ifdef ZEND_JIT_USE_APPLE_MAP_JIT
  pthread_jit_write_protect_np(1);
#elif defined(HAVE_MPROTECT)
  if (JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) {
    if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_WRITE | PROT_EXEC) != 0) {
      zend_accel_error_noreturn(ACCEL_LOG_FATAL, "mprotect() failed [%d] %s\n", errno, strerror(errno));
    }
  } else {
    if (mprotect(dasm_buf, dasm_size, PROT_READ | PROT_EXEC) != 0) {
      zend_accel_error_noreturn(ACCEL_LOG_FATAL, "mprotect() failed [%d] %s\n", errno, strerror(errno));
    }
  }
#elif defined(_WIN32)
  if (JIT_G(debug) & (ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF_DUMP)) {
    DWORD old;

    if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READWRITE, &old)) {
      DWORD err = GetLastError();
      char *msg = php_win32_error_to_msg(err);
      zend_accel_error_noreturn(ACCEL_LOG_FATAL, "VirtualProtect() failed [%lu] %s\n", err, msg);
    }
  } else {
    DWORD old;

    if (!VirtualProtect(dasm_buf, dasm_size, PAGE_EXECUTE_READ, &old)) {
      DWORD err = GetLastError();
      char *msg = php_win32_error_to_msg(err);
      zend_accel_error_noreturn(ACCEL_LOG_FATAL, "VirtualProtect() failed [%lu] %s\n", err, msg);
    }
  }
#endif

  if (!reattached) {
    zend_jit_unprotect();
    *dasm_ptr = dasm_buf;
#if defined(_WIN32)
    zend_jit_stub_handlers = dasm_buf;
    *dasm_ptr = (void**)*dasm_ptr + sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0]);
#elif defined(IR_TARGET_AARCH64)
    zend_jit_stub_handlers = dasm_buf;
    *dasm_ptr = (void**)*dasm_ptr + (sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0])) * 2;
    memset(zend_jit_stub_handlers, 0, (sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0])) * 2 * sizeof(void*));
#endif
    *dasm_ptr = (void*)ZEND_MM_ALIGNED_SIZE_EX(((size_t)(*dasm_ptr)), 16);
    zend_jit_protect();
  } else {
#if defined(_WIN32) || defined(IR_TARGET_AARCH64)
    zend_jit_stub_handlers = dasm_buf;
    zend_jit_init_handlers();
#endif
  }

  zend_jit_unprotect();
  zend_jit_setup(reattached);
  zend_jit_protect();
  if (!reattached) {
    zend_jit_init_handlers();
  }

  zend_jit_trace_startup(reattached);

  zend_jit_unprotect();
  /* save JIT buffer pos */
  dasm_ptr[1] = dasm_ptr[0];
  zend_jit_protect();
}

void zend_jit_shutdown(void)
{
  if (JIT_G(debug) & ZEND_JIT_DEBUG_SIZE && dasm_ptr != NULL) {
    fprintf(stderr, "\nJIT memory usage: %td\n", (ptrdiff_t)((char*)*dasm_ptr - (char*)dasm_buf));
  }

  zend_jit_shutdown_ir();

#ifdef ZTS
  ts_free_id(jit_globals_id);
#else
  zend_jit_trace_free_caches(&jit_globals);
#endif

#ifdef ZEND_JIT_USE_APPLE_MAP_JIT
  if (dasm_buf != NULL) {
    munmap(dasm_buf, dasm_size);
  }
#endif

  /* Reset global pointers to prevent use-after-free in `zend_jit_status()`
   * after gracefully restarting Apache with mod_php, see:
   * https://github.com/php/php-src/pull/19212 */
  dasm_ptr = NULL;
  dasm_buf = NULL;
  dasm_end = NULL;
  dasm_size = 0;
}

static void zend_jit_reset_counters(void)
{
  int i;

  for (i = 0; i < ZEND_HOT_COUNTERS_COUNT; i++) {
    zend_jit_hot_counters[i] = ZEND_JIT_COUNTER_INIT;
  }
}

void zend_jit_activate(void)
{
#ifdef ZTS
  if (!zend_jit_startup_ok) {
    JIT_G(enabled) = 0;
    JIT_G(on) = 0;
    return;
  }
#endif
  zend_jit_profile_counter = 0;
  if (JIT_G(on)) {
    if (JIT_G(trigger) == ZEND_JIT_ON_HOT_COUNTERS) {
      zend_jit_reset_counters();
    } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
      zend_jit_reset_counters();
      zend_jit_trace_reset_caches();
    }
  }
}

void zend_jit_deactivate(void)
{
  if (zend_jit_profile_counter && !CG(unclean_shutdown)) {
    zend_class_entry *ce;

    zend_shared_alloc_lock();
    SHM_UNPROTECT();
    zend_jit_unprotect();

    zend_jit_check_funcs(EG(function_table), false);
    ZEND_HASH_MAP_REVERSE_FOREACH_PTR(EG(class_table), ce) {
      if (ce->type == ZEND_INTERNAL_CLASS) {
        break;
      }
      zend_jit_check_funcs(&ce->function_table, true);
    } ZEND_HASH_FOREACH_END();

    zend_jit_protect();
    SHM_PROTECT();
    zend_shared_alloc_unlock();
  }

  zend_jit_profile_counter = 0;
}

static void zend_jit_restart_preloaded_op_array(zend_op_array *op_array, void *context)
{
  ZEND_IGNORE_VALUE(context);

  zend_func_info *func_info = ZEND_FUNC_INFO(op_array);

  if (!func_info) {
    return;
  }

  if (func_info->flags & ZEND_FUNC_JIT_ON_HOT_TRACE) {
    zend_jit_restart_hot_trace_counters(op_array);
  } else if (func_info->flags & ZEND_FUNC_JIT_ON_HOT_COUNTERS) {
    zend_jit_restart_hot_counters(op_array);
#if 0
  // TODO: We have to restore handlers for some inner basic-blocks, but we didn't store them ???
  } else if (func_info->flags & (ZEND_FUNC_JIT_ON_FIRST_EXEC|ZEND_FUNC_JIT_ON_PROF_REQUEST)) {
    zend_op *opline = op_array->opcodes;
    zend_jit_op_array_extension *jit_extension =
      (zend_jit_op_array_extension*)func_info;

    if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
      while (opline->opcode == ZEND_RECV || opline->opcode == ZEND_RECV_INIT) {
        opline++;
      }
    }
    if (func_info->flags & ZEND_FUNC_JIT_ON_FIRST_EXEC) {
      opline->handler = zend_jit_runtime_jit_handler;
    } else {
      opline->handler = zend_jit_profile_jit_handler;
    }
#endif
  }
}

static void zend_jit_restart_preloaded_script(zend_persistent_script *script)
{
  zend_foreach_op_array(&script->script, zend_jit_restart_preloaded_op_array, NULL);
}

void zend_jit_restart(void)
{
  if (dasm_buf) {
    zend_jit_unprotect();

    /* restore JIT buffer pos */
    dasm_ptr[0] = dasm_ptr[1];

    zend_jit_trace_restart();

    if (ZCSG(preload_script)) {
      zend_jit_restart_preloaded_script(ZCSG(preload_script));
      if (ZCSG(saved_scripts)) {
        zend_persistent_script **p = ZCSG(saved_scripts);

        while (*p) {
          zend_jit_restart_preloaded_script(*p);
          p++;
        }
      }
    }

    zend_jit_protect();
  }
}

#endif /* HAVE_JIT */

Coverage Report

Created: 2026-10-28 06:20