/*

 *    Stack-less Just-In-Time compiler

 *

 *    Copyright Zoltan Herczeg (hzmester@freemail.hu). All rights reserved.

 *

 * Redistribution and use in source and binary forms, with or without modification, are

 * permitted provided that the following conditions are met:

 *

 *   1. Redistributions of source code must retain the above copyright notice, this list of

 *      conditions and the following disclaimer.

 *

 *   2. Redistributions in binary form must reproduce the above copyright notice, this list

 *      of conditions and the following disclaimer in the documentation and/or other materials

 *      provided with the distribution.

 *

 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) AND CONTRIBUTORS ``AS IS'' AND ANY

 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES

 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT

 * SHALL THE COPYRIGHT HOLDER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,

 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED

 * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR

 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN

 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

 */

#include "sljitLir.h"

#ifdef _WIN32

#include <windows.h>

#endif /* _WIN32 */

#if !(defined SLJIT_STD_MACROS_DEFINED && SLJIT_STD_MACROS_DEFINED)

/* These libraries are needed for the macros below. */

#include <stdlib.h>

#include <string.h>

#endif /* SLJIT_STD_MACROS_DEFINED */

#define CHECK_ERROR() \

  do { \

    if (SLJIT_UNLIKELY(compiler->error)) \

      return compiler->error; \

  } while (0)

#define CHECK_ERROR_PTR() \

  do { \

    if (SLJIT_UNLIKELY(compiler->error)) \

      return NULL; \

  } while (0)

#define FAIL_IF(expr) \

  do { \

    if (SLJIT_UNLIKELY(expr)) \

      return compiler->error; \

  } while (0)

#define PTR_FAIL_IF(expr) \

  do { \

    if (SLJIT_UNLIKELY(expr)) \

      return NULL; \

  } while (0)

#define FAIL_IF_NULL(ptr) \

  do { \

    if (SLJIT_UNLIKELY(!(ptr))) { \

      compiler->error = SLJIT_ERR_ALLOC_FAILED; \

      return SLJIT_ERR_ALLOC_FAILED; \

    } \

  } while (0)

#define PTR_FAIL_IF_NULL(ptr) \

  do { \

    if (SLJIT_UNLIKELY(!(ptr))) { \

      compiler->error = SLJIT_ERR_ALLOC_FAILED; \

      return NULL; \

    } \

  } while (0)

#define PTR_FAIL_WITH_EXEC_IF(ptr) \

  do { \

    if (SLJIT_UNLIKELY(!(ptr))) { \

      compiler->error = SLJIT_ERR_EX_ALLOC_FAILED; \

      return NULL; \

    } \

  } while (0)

#if !(defined SLJIT_CONFIG_UNSUPPORTED && SLJIT_CONFIG_UNSUPPORTED)

#define SSIZE_OF(type) ((sljit_s32)sizeof(sljit_ ## type))

#define VARIABLE_FLAG_SHIFT (10)

/* All variable flags are even. */

#define VARIABLE_FLAG_MASK (0x3e << VARIABLE_FLAG_SHIFT)

#define GET_FLAG_TYPE(op) ((op) >> VARIABLE_FLAG_SHIFT)

#define GET_OPCODE(op) \

  ((op) & ~(SLJIT_32 | SLJIT_SET_Z | VARIABLE_FLAG_MASK))

#define HAS_FLAGS(op) \

  ((op) & (SLJIT_SET_Z | VARIABLE_FLAG_MASK))

#define GET_ALL_FLAGS(op) \

  ((op) & (SLJIT_32 | SLJIT_SET_Z | VARIABLE_FLAG_MASK))

#if (defined SLJIT_64BIT_ARCHITECTURE && SLJIT_64BIT_ARCHITECTURE)

#define TYPE_CAST_NEEDED(op) \

  ((op) >= SLJIT_MOV_U8 && (op) <= SLJIT_MOV_S32)

#else /* !SLJIT_64BIT_ARCHITECTURE */

#define TYPE_CAST_NEEDED(op) \

  ((op) >= SLJIT_MOV_U8 && (op) <= SLJIT_MOV_S16)

#endif /* SLJIT_64BIT_ARCHITECTURE */

#define BUF_SIZE  4096

#if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE)

#define ABUF_SIZE 2048

#else

#define ABUF_SIZE 4096

#endif

/* Parameter parsing. */

#define REG_MASK    0x7f

#define OFFS_REG(reg)   (((reg) >> 8) & REG_MASK)

#define OFFS_REG_MASK   (REG_MASK << 8)

#define TO_OFFS_REG(reg)  ((reg) << 8)

#define FAST_IS_REG(reg)  ((reg) < REG_MASK)

/* Mask for argument types. */

#define SLJIT_ARG_MASK    0x7

#define SLJIT_ARG_FULL_MASK (SLJIT_ARG_MASK | SLJIT_ARG_TYPE_SCRATCH_REG)

/* Mask for register pairs. */

#define REG_PAIR_MASK   0x7f00

#define REG_PAIR_FIRST(reg) ((reg) & 0x7f)

#define REG_PAIR_SECOND(reg)  ((reg) >> 8)

/* Mask for sljit_emit_enter. */

#define SLJIT_KEPT_SAVEDS_COUNT(options) ((options) & 0x3)

/* Getters for simd operations, which returns with log2(size). */

#define SLJIT_SIMD_GET_OPCODE(type)   ((type) & 0xff)

#define SLJIT_SIMD_GET_REG_SIZE(type)   (((type) >> 12) & 0x3f)

#define SLJIT_SIMD_GET_ELEM_SIZE(type)    (((type) >> 18) & 0x3f)

#define SLJIT_SIMD_GET_ELEM2_SIZE(type)   (((type) >> 24) & 0x3f)

#define SLJIT_SIMD_CHECK_REG(type) (((type) & 0x3f000) >= SLJIT_SIMD_REG_64 && ((type) & 0x3f000) <= SLJIT_SIMD_REG_512)

#define SLJIT_SIMD_TYPE_MASK(m) ((sljit_s32)0xff000fff & ~(SLJIT_SIMD_FLOAT | SLJIT_SIMD_TEST | (m)))

#define SLJIT_SIMD_TYPE_MASK2(m) ((sljit_s32)0xc0000fff & ~(SLJIT_SIMD_FLOAT | SLJIT_SIMD_TEST | (m)))

/* Jump flags. */

#define JUMP_ADDR 0x1

#define JUMP_MOV_ADDR 0x2

/* SLJIT_REWRITABLE_JUMP is 0x1000. */

#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)

# define PATCH_MB   0x04

# define PATCH_MW   0x08

#if (defined SLJIT_CONFIG_X86_64 && SLJIT_CONFIG_X86_64)

# define PATCH_MD   0x10

# define MOV_ADDR_HI  0x20

# define JUMP_MAX_SIZE  ((sljit_uw)(10 + 3))

# define CJUMP_MAX_SIZE ((sljit_uw)(2 + 10 + 3))

#else /* !SLJIT_CONFIG_X86_64 */

# define JUMP_MAX_SIZE  ((sljit_uw)5)

# define CJUMP_MAX_SIZE ((sljit_uw)6)

#endif /* SLJIT_CONFIG_X86_64 */

# define TYPE_SHIFT 13

#if (defined SLJIT_DEBUG && SLJIT_DEBUG)

/* Bits 7..12 is for debug jump size, SLJIT_REWRITABLE_JUMP is 0x1000 */

# define JUMP_SIZE_SHIFT  7

#endif /* SLJIT_DEBUG */

#endif /* SLJIT_CONFIG_X86 */

#if (defined SLJIT_CONFIG_ARM_V6 && SLJIT_CONFIG_ARM_V6) || (defined SLJIT_CONFIG_ARM_V7 && SLJIT_CONFIG_ARM_V7)

# define IS_BL    0x04

# define PATCH_B    0x08

#endif /* SLJIT_CONFIG_ARM_V6 || SLJIT_CONFIG_ARM_V7 */

#if (defined SLJIT_CONFIG_ARM_V6 && SLJIT_CONFIG_ARM_V6)

# define CPOOL_SIZE 512

#endif /* SLJIT_CONFIG_ARM_V6 */

#if (defined SLJIT_CONFIG_ARM_V7 && SLJIT_CONFIG_ARM_V7)

# define JUMP_SIZE_SHIFT  26

# define JUMP_MAX_SIZE  ((sljit_uw)3)

#endif /* SLJIT_CONFIG_ARM_V7 */

#if (defined SLJIT_CONFIG_ARM_THUMB2 && SLJIT_CONFIG_ARM_THUMB2)

# define IS_COND    0x04

# define IS_BL    0x08

  /* conditional + imm8 */

# define PATCH_TYPE1  0x10

  /* conditional + imm20 */

# define PATCH_TYPE2  0x20

  /* imm11 */

# define PATCH_TYPE3  0x30

  /* imm24 */

# define PATCH_TYPE4  0x40

  /* BL + imm24 */

# define PATCH_TYPE5  0x50

  /* addwi/subwi */

# define PATCH_TYPE6  0x60

  /* 0xf00 cc code for branches */

# define JUMP_SIZE_SHIFT  26

# define JUMP_MAX_SIZE  ((sljit_uw)5)

#endif /* SLJIT_CONFIG_ARM_THUMB2 */

#if (defined SLJIT_CONFIG_ARM_64 && SLJIT_CONFIG_ARM_64)

# define IS_COND    0x004

# define IS_CBZ   0x008

# define IS_BL    0x010

# define PATCH_COND 0x020

# define PATCH_B    0x040

# define PATCH_B32  0x080

# define PATCH_ABS48  0x100

# define PATCH_ABS64  0x200

# define JUMP_SIZE_SHIFT  58

# define JUMP_MAX_SIZE  ((sljit_uw)5)

#endif /* SLJIT_CONFIG_ARM_64 */

#if (defined SLJIT_CONFIG_PPC && SLJIT_CONFIG_PPC)

# define IS_COND    0x004

# define IS_CALL    0x008

# define PATCH_B    0x010

# define PATCH_ABS_B  0x020

#if (defined SLJIT_CONFIG_PPC_64 && SLJIT_CONFIG_PPC_64)

# define PATCH_ABS32  0x040

# define PATCH_ABS48  0x080

# define JUMP_SIZE_SHIFT  58

# define JUMP_MAX_SIZE  ((sljit_uw)7)

#else /* !SLJIT_CONFIG_PPC_64 */

# define JUMP_SIZE_SHIFT  26

# define JUMP_MAX_SIZE  ((sljit_uw)4)

#endif /* SLJIT_CONFIG_PPC_64 */

#endif /* SLJIT_CONFIG_PPC */

#if (defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS)

# define IS_MOVABLE 0x004

# define IS_JAL   0x008

# define IS_CALL    0x010

# define IS_BIT26_COND  0x020

# define IS_BIT16_COND  0x040

# define IS_BIT23_COND  0x080

# define IS_COND    (IS_BIT26_COND | IS_BIT16_COND | IS_BIT23_COND)

# define PATCH_B    0x100

# define PATCH_J    0x200

#if (defined SLJIT_CONFIG_MIPS_64 && SLJIT_CONFIG_MIPS_64)

# define PATCH_ABS32  0x400

# define PATCH_ABS48  0x800

#endif /* SLJIT_CONFIG_MIPS_64 */

  /* instruction types */

# define MOVABLE_INS  0

  /* 1 - 31 last destination register */

  /* no destination (i.e: store) */

# define UNMOVABLE_INS  32

  /* FPU status register */

# define FCSR_FCC   33

#endif /* SLJIT_CONFIG_MIPS */

#if (defined SLJIT_CONFIG_RISCV && SLJIT_CONFIG_RISCV)

# define IS_COND    0x004

# define IS_CALL    0x008

# define PATCH_B    0x010

# define PATCH_J    0x020

#if (defined SLJIT_CONFIG_RISCV_64 && SLJIT_CONFIG_RISCV_64)

# define PATCH_REL32  0x040

# define PATCH_ABS32  0x080

# define PATCH_ABS44  0x100

# define PATCH_ABS52  0x200

# define JUMP_SIZE_SHIFT  58

# define JUMP_MAX_SIZE  ((sljit_uw)6)

#else /* !SLJIT_CONFIG_RISCV_64 */

# define JUMP_SIZE_SHIFT  26

# define JUMP_MAX_SIZE  ((sljit_uw)2)

#endif /* SLJIT_CONFIG_RISCV_64 */

#endif /* SLJIT_CONFIG_RISCV */

#if (defined SLJIT_CONFIG_LOONGARCH && SLJIT_CONFIG_LOONGARCH)

# define IS_COND    0x004

# define IS_CALL    0x008

# define PATCH_B    0x010

# define PATCH_J    0x020

# define PATCH_REL32  0x040

# define PATCH_ABS32  0x080

# define PATCH_ABS52  0x100

# define JUMP_SIZE_SHIFT  58

# define JUMP_MAX_SIZE  ((sljit_uw)4)

#endif /* SLJIT_CONFIG_LOONGARCH */

/* Stack management. */

#define GET_SAVED_REGISTERS_SIZE(scratches, saveds, extra) \

  (((scratches < SLJIT_NUMBER_OF_SCRATCH_REGISTERS ? 0 : (scratches - SLJIT_NUMBER_OF_SCRATCH_REGISTERS)) + \

    (saveds) + (sljit_s32)(extra)) * (sljit_s32)sizeof(sljit_sw))

#define GET_SAVED_FLOAT_REGISTERS_SIZE(fscratches, fsaveds, type) \

  (((fscratches < SLJIT_NUMBER_OF_SCRATCH_FLOAT_REGISTERS ? 0 : (fscratches - SLJIT_NUMBER_OF_SCRATCH_FLOAT_REGISTERS)) + \

    (fsaveds)) * SSIZE_OF(type))

#define ADJUST_LOCAL_OFFSET(p, i) \

  if ((p) == (SLJIT_MEM1(SLJIT_SP))) \

    (i) += SLJIT_LOCALS_OFFSET;

#endif /* !(defined SLJIT_CONFIG_UNSUPPORTED && SLJIT_CONFIG_UNSUPPORTED) */

/* Utils can still be used even if SLJIT_CONFIG_UNSUPPORTED is set. */

#include "sljitUtils.c"

#if (defined SLJIT_CONFIG_ARM_THUMB2 && SLJIT_CONFIG_ARM_THUMB2)

#define SLJIT_CODE_TO_PTR(code) ((void*)((sljit_up)(code) & ~(sljit_up)0x1))

#elif (defined SLJIT_INDIRECT_CALL && SLJIT_INDIRECT_CALL)

#define SLJIT_CODE_TO_PTR(code) ((void*)(*(sljit_up*)code))

#else /* !SLJIT_CONFIG_ARM_THUMB2 && !SLJIT_INDIRECT_CALL */

#define SLJIT_CODE_TO_PTR(code) ((void*)(code))

#endif /* SLJIT_CONFIG_ARM_THUMB2 || SLJIT_INDIRECT_CALL */

#if !(defined SLJIT_CONFIG_UNSUPPORTED && SLJIT_CONFIG_UNSUPPORTED)

#if (defined SLJIT_EXECUTABLE_ALLOCATOR && SLJIT_EXECUTABLE_ALLOCATOR)

#if (defined SLJIT_PROT_EXECUTABLE_ALLOCATOR && SLJIT_PROT_EXECUTABLE_ALLOCATOR)

#if defined(__NetBSD__)

#include "allocator_src/sljitProtExecAllocatorNetBSD.c"

#else

#include "allocator_src/sljitProtExecAllocatorPosix.c"

#endif

#elif (defined SLJIT_WX_EXECUTABLE_ALLOCATOR && SLJIT_WX_EXECUTABLE_ALLOCATOR)

#if defined(_WIN32)

#include "allocator_src/sljitWXExecAllocatorWindows.c"

#else

#include "allocator_src/sljitWXExecAllocatorPosix.c"

#endif

#else

#if defined(_WIN32)

#include "allocator_src/sljitExecAllocatorWindows.c"

#elif defined(__APPLE__)

#include "allocator_src/sljitExecAllocatorApple.c"

#elif defined(__FreeBSD__)

#include "allocator_src/sljitExecAllocatorFreeBSD.c"

#else

#include "allocator_src/sljitExecAllocatorPosix.c"

#endif

#endif

#else /* !SLJIT_EXECUTABLE_ALLOCATOR */

#ifndef SLJIT_UPDATE_WX_FLAGS

#define SLJIT_UPDATE_WX_FLAGS(from, to, enable_exec)

#endif

#endif /* SLJIT_EXECUTABLE_ALLOCATOR */

#if (defined SLJIT_PROT_EXECUTABLE_ALLOCATOR && SLJIT_PROT_EXECUTABLE_ALLOCATOR)

#define SLJIT_ADD_EXEC_OFFSET(ptr, exec_offset) ((sljit_u8 *)(ptr) + (exec_offset))

#else

#define SLJIT_ADD_EXEC_OFFSET(ptr, exec_offset) ((sljit_u8 *)(ptr))

#endif

/* Argument checking features. */

#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)

/* Returns with error when an invalid argument is passed. */

#define CHECK_ARGUMENT(x) \

  do { \

    if (SLJIT_UNLIKELY(!(x))) \

      return 1; \

  } while (0)

#define CHECK_RETURN_TYPE sljit_s32

#define CHECK_RETURN_OK return 0

#define CHECK(x) \

  do { \

    if (SLJIT_UNLIKELY(x)) { \

      compiler->error = SLJIT_ERR_BAD_ARGUMENT; \

      return SLJIT_ERR_BAD_ARGUMENT; \

    } \

  } while (0)

#define CHECK_PTR(x) \

  do { \

    if (SLJIT_UNLIKELY(x)) { \

      compiler->error = SLJIT_ERR_BAD_ARGUMENT; \

      return NULL; \

    } \

  } while (0)

#define CHECK_REG_INDEX(x) \

  do { \

    if (SLJIT_UNLIKELY(x)) { \

      return -2; \

    } \

  } while (0)

#elif (defined SLJIT_DEBUG && SLJIT_DEBUG)

/* Assertion failure occures if an invalid argument is passed. */

#undef SLJIT_ARGUMENT_CHECKS

#define SLJIT_ARGUMENT_CHECKS 1

#define CHECK_ARGUMENT(x) SLJIT_ASSERT(x)

#define CHECK_RETURN_TYPE void

#define CHECK_RETURN_OK return

#define CHECK(x) x

#define CHECK_PTR(x) x

#define CHECK_REG_INDEX(x) x

#elif (defined SLJIT_VERBOSE && SLJIT_VERBOSE)

/* Arguments are not checked. */

#define CHECK_RETURN_TYPE void

#define CHECK_RETURN_OK return

#define CHECK(x) x

#define CHECK_PTR(x) x

#define CHECK_REG_INDEX(x) x

#else

/* Arguments are not checked. */

#define CHECK(x)

#define CHECK_PTR(x)

#define CHECK_REG_INDEX(x)

#endif /* SLJIT_ARGUMENT_CHECKS */

/* --------------------------------------------------------------------- */

/*  Public functions                                                     */

/* --------------------------------------------------------------------- */

#if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)

#define SLJIT_NEEDS_COMPILER_INIT 1

static sljit_s32 compiler_initialized = 0;

/* A thread safe initialization. */

static void init_compiler(void);

#endif

SLJIT_API_FUNC_ATTRIBUTE struct sljit_compiler* sljit_create_compiler(void *allocator_data)

{

  struct sljit_compiler *compiler = (struct sljit_compiler*)SLJIT_MALLOC(sizeof(struct sljit_compiler), allocator_data);

  if (!compiler)

    return NULL;

  SLJIT_ZEROMEM(compiler, sizeof(struct sljit_compiler));

  SLJIT_COMPILE_ASSERT(

    sizeof(sljit_s8) == 1 && sizeof(sljit_u8) == 1

    && sizeof(sljit_s16) == 2 && sizeof(sljit_u16) == 2

    && sizeof(sljit_s32) == 4 && sizeof(sljit_u32) == 4

    && (sizeof(sljit_up) == 4 || sizeof(sljit_up) == 8)

    && sizeof(sljit_up) <= sizeof(sljit_sw)

    && sizeof(sljit_up) == sizeof(sljit_sp)

    && (sizeof(sljit_sw) == 4 || sizeof(sljit_sw) == 8)

    && (sizeof(sljit_uw) == sizeof(sljit_sw)),

    invalid_integer_types);

  SLJIT_COMPILE_ASSERT(SLJIT_REWRITABLE_JUMP != SLJIT_32,

    rewritable_jump_and_single_op_must_not_be_the_same);

  SLJIT_COMPILE_ASSERT(!(SLJIT_EQUAL & 0x1) && !(SLJIT_LESS & 0x1) && !(SLJIT_F_EQUAL & 0x1) && !(SLJIT_JUMP & 0x1),

    conditional_flags_must_be_even_numbers);

  /* Only the non-zero members must be set. */

  compiler->error = SLJIT_SUCCESS;

  compiler->allocator_data = allocator_data;

  compiler->buf = (struct sljit_memory_fragment*)SLJIT_MALLOC(BUF_SIZE, allocator_data);

  compiler->abuf = (struct sljit_memory_fragment*)SLJIT_MALLOC(ABUF_SIZE, allocator_data);

  if (!compiler->buf || !compiler->abuf) {

    if (compiler->buf)

      SLJIT_FREE(compiler->buf, allocator_data);

    if (compiler->abuf)

      SLJIT_FREE(compiler->abuf, allocator_data);

    SLJIT_FREE(compiler, allocator_data);

    return NULL;

  }

  compiler->buf->next = NULL;

  compiler->buf->used_size = 0;

  compiler->abuf->next = NULL;

  compiler->abuf->used_size = 0;

  compiler->scratches = -1;

  compiler->saveds = -1;

  compiler->fscratches = -1;

  compiler->fsaveds = -1;

  compiler->local_size = -1;

#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)

  compiler->args_size = -1;

#endif /* SLJIT_CONFIG_X86_32 */

#if (defined SLJIT_CONFIG_ARM_V6 && SLJIT_CONFIG_ARM_V6)

  compiler->cpool = (sljit_uw*)SLJIT_MALLOC(CPOOL_SIZE * sizeof(sljit_uw)

    + CPOOL_SIZE * sizeof(sljit_u8), allocator_data);

  if (!compiler->cpool) {

    SLJIT_FREE(compiler->buf, allocator_data);

    SLJIT_FREE(compiler->abuf, allocator_data);

    SLJIT_FREE(compiler, allocator_data);

    return NULL;

  }

  compiler->cpool_unique = (sljit_u8*)(compiler->cpool + CPOOL_SIZE);

  compiler->cpool_diff = 0xffffffff;

#endif /* SLJIT_CONFIG_ARM_V6 */

#if (defined SLJIT_CONFIG_MIPS && SLJIT_CONFIG_MIPS)

  compiler->delay_slot = UNMOVABLE_INS;

#endif /* SLJIT_CONFIG_MIPS */

#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS) \

    || (defined SLJIT_DEBUG && SLJIT_DEBUG)

  compiler->last_flags = 0;

  compiler->last_return = -1;

  compiler->logical_local_size = 0;

#endif /* SLJIT_ARGUMENT_CHECKS || SLJIT_DEBUG */

#if (defined SLJIT_NEEDS_COMPILER_INIT && SLJIT_NEEDS_COMPILER_INIT)

  if (!compiler_initialized) {

    init_compiler();

    compiler_initialized = 1;

  }

#endif

  return compiler;

}

SLJIT_API_FUNC_ATTRIBUTE void sljit_free_compiler(struct sljit_compiler *compiler)

{

  struct sljit_memory_fragment *buf;

  struct sljit_memory_fragment *curr;

  void *allocator_data = compiler->allocator_data;

  SLJIT_UNUSED_ARG(allocator_data);

  buf = compiler->buf;

  while (buf) {

    curr = buf;

    buf = buf->next;

    SLJIT_FREE(curr, allocator_data);

  }

  buf = compiler->abuf;

  while (buf) {

    curr = buf;

    buf = buf->next;

    SLJIT_FREE(curr, allocator_data);

  }

#if (defined SLJIT_CONFIG_ARM_V6 && SLJIT_CONFIG_ARM_V6)

  SLJIT_FREE(compiler->cpool, allocator_data);

#endif

  SLJIT_FREE(compiler, allocator_data);

}

SLJIT_API_FUNC_ATTRIBUTE void sljit_set_compiler_memory_error(struct sljit_compiler *compiler)

{

  if (compiler->error == SLJIT_SUCCESS)

    compiler->error = SLJIT_ERR_ALLOC_FAILED;

}

SLJIT_API_FUNC_ATTRIBUTE void sljit_free_code(void* code, void *exec_allocator_data)

{

  SLJIT_UNUSED_ARG(exec_allocator_data);

  SLJIT_FREE_EXEC(SLJIT_CODE_TO_PTR(code), exec_allocator_data);

}

SLJIT_API_FUNC_ATTRIBUTE void sljit_set_label(struct sljit_jump *jump, struct sljit_label* label)

{

  if (SLJIT_LIKELY(!!jump) && SLJIT_LIKELY(!!label)) {

    jump->flags &= (sljit_uw)~JUMP_ADDR;

    jump->u.label = label;

  }

}

SLJIT_API_FUNC_ATTRIBUTE void sljit_set_target(struct sljit_jump *jump, sljit_uw target)

{

  if (SLJIT_LIKELY(!!jump)) {

    jump->flags |= JUMP_ADDR;

    jump->u.target = target;

  }

}

#define SLJIT_CURRENT_FLAGS_ALL \

  (SLJIT_CURRENT_FLAGS_32 | SLJIT_CURRENT_FLAGS_ADD | SLJIT_CURRENT_FLAGS_SUB | SLJIT_CURRENT_FLAGS_COMPARE)

SLJIT_API_FUNC_ATTRIBUTE void sljit_set_current_flags(struct sljit_compiler *compiler, sljit_s32 current_flags)

{

  SLJIT_UNUSED_ARG(compiler);

  SLJIT_UNUSED_ARG(current_flags);

#if (defined SLJIT_HAS_STATUS_FLAGS_STATE && SLJIT_HAS_STATUS_FLAGS_STATE)

  compiler->status_flags_state = current_flags;

#endif

#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)

  compiler->last_flags = 0;

  if ((current_flags & ~(VARIABLE_FLAG_MASK | SLJIT_SET_Z | SLJIT_CURRENT_FLAGS_ALL)) == 0) {

    compiler->last_flags = GET_FLAG_TYPE(current_flags) | (current_flags & (SLJIT_32 | SLJIT_SET_Z));

  }

#endif

}

/* --------------------------------------------------------------------- */

/*  Private functions                                                    */

/* --------------------------------------------------------------------- */

static void* ensure_buf(struct sljit_compiler *compiler, sljit_uw size)

{

  sljit_u8 *ret;

  struct sljit_memory_fragment *new_frag;

  SLJIT_ASSERT(size <= 256);

  if (compiler->buf->used_size + size <= (BUF_SIZE - (sljit_uw)SLJIT_OFFSETOF(struct sljit_memory_fragment, memory))) {

    ret = compiler->buf->memory + compiler->buf->used_size;

    compiler->buf->used_size += size;

    return ret;

  }

  new_frag = (struct sljit_memory_fragment*)SLJIT_MALLOC(BUF_SIZE, compiler->allocator_data);

  PTR_FAIL_IF_NULL(new_frag);

  new_frag->next = compiler->buf;

  compiler->buf = new_frag;

  new_frag->used_size = size;

  return new_frag->memory;

}

static void* ensure_abuf(struct sljit_compiler *compiler, sljit_uw size)

{

  sljit_u8 *ret;

  struct sljit_memory_fragment *new_frag;

  SLJIT_ASSERT(size <= 256);

  if (compiler->abuf->used_size + size <= (ABUF_SIZE - (sljit_uw)SLJIT_OFFSETOF(struct sljit_memory_fragment, memory))) {

    ret = compiler->abuf->memory + compiler->abuf->used_size;

    compiler->abuf->used_size += size;

    return ret;

  }

  new_frag = (struct sljit_memory_fragment*)SLJIT_MALLOC(ABUF_SIZE, compiler->allocator_data);

  PTR_FAIL_IF_NULL(new_frag);

  new_frag->next = compiler->abuf;

  compiler->abuf = new_frag;

  new_frag->used_size = size;

  return new_frag->memory;

}

SLJIT_API_FUNC_ATTRIBUTE void* sljit_alloc_memory(struct sljit_compiler *compiler, sljit_s32 size)

{

  CHECK_ERROR_PTR();

#if (defined SLJIT_64BIT_ARCHITECTURE && SLJIT_64BIT_ARCHITECTURE)

  if (size <= 0 || size > 128)

    return NULL;

  size = (size + 7) & ~7;

#else

  if (size <= 0 || size > 64)

    return NULL;

  size = (size + 3) & ~3;

#endif

  return ensure_abuf(compiler, (sljit_uw)size);

}

static SLJIT_INLINE void reverse_buf(struct sljit_compiler *compiler)

{

  struct sljit_memory_fragment *buf = compiler->buf;

  struct sljit_memory_fragment *prev = NULL;

  struct sljit_memory_fragment *tmp;

  do {

    tmp = buf->next;

    buf->next = prev;

    prev = buf;

    buf = tmp;

  } while (buf != NULL);

  compiler->buf = prev;

}

static SLJIT_INLINE void* allocate_executable_memory(sljit_uw size, sljit_s32 options,

  void *exec_allocator_data, sljit_sw *executable_offset)

{

  void *code;

  struct sljit_generate_code_buffer *buffer;

  if (SLJIT_LIKELY(!(options & SLJIT_GENERATE_CODE_BUFFER))) {

    code = SLJIT_MALLOC_EXEC(size, exec_allocator_data);

    *executable_offset = SLJIT_EXEC_OFFSET(code);

    return code;

  }

  buffer = (struct sljit_generate_code_buffer*)exec_allocator_data;

  if (size <= buffer->size) {

    *executable_offset = buffer->executable_offset;

    return buffer->buffer;

  }

  return NULL;

}

#define SLJIT_MAX_ADDRESS ~(sljit_uw)0

#define SLJIT_GET_NEXT_SIZE(ptr) (ptr != NULL) ? ((ptr)->size) : SLJIT_MAX_ADDRESS

#define SLJIT_GET_NEXT_ADDRESS(ptr) (ptr != NULL) ? ((ptr)->addr) : SLJIT_MAX_ADDRESS

#if !(defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86)

#define SLJIT_NEXT_DEFINE_TYPES \

  sljit_uw next_label_size; \

  sljit_uw next_jump_addr; \

  sljit_uw next_const_addr; \

  sljit_uw next_min_addr

#define SLJIT_NEXT_INIT_TYPES() \

  next_label_size = SLJIT_GET_NEXT_SIZE(label); \

  next_jump_addr = SLJIT_GET_NEXT_ADDRESS(jump); \

  next_const_addr = SLJIT_GET_NEXT_ADDRESS(const_);

#define SLJIT_GET_NEXT_MIN() \

  next_min_addr = sljit_get_next_min(next_label_size, next_jump_addr, next_const_addr);

static SLJIT_INLINE sljit_uw sljit_get_next_min(sljit_uw next_label_size,

  sljit_uw next_jump_addr, sljit_uw next_const_addr)

{

  sljit_uw result = next_jump_addr;

  SLJIT_ASSERT(result == SLJIT_MAX_ADDRESS || result != next_const_addr);

  if (next_const_addr < result)

    result = next_const_addr;

  if (next_label_size < result)

    result = next_label_size;

  return result;

}

#endif /* !SLJIT_CONFIG_X86 */

static SLJIT_INLINE void set_emit_enter(struct sljit_compiler *compiler,

  sljit_s32 options, sljit_s32 args, sljit_s32 scratches, sljit_s32 saveds,

  sljit_s32 fscratches, sljit_s32 fsaveds, sljit_s32 local_size)

{

  SLJIT_UNUSED_ARG(args);

  SLJIT_UNUSED_ARG(local_size);

  compiler->options = options;

  compiler->scratches = scratches;

  compiler->saveds = saveds;

  compiler->fscratches = fscratches;

  compiler->fsaveds = fsaveds;

#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)

  compiler->last_return = args & SLJIT_ARG_MASK;

  compiler->logical_local_size = local_size;

#endif

}

static SLJIT_INLINE void set_set_context(struct sljit_compiler *compiler,

  sljit_s32 options, sljit_s32 args, sljit_s32 scratches, sljit_s32 saveds,

  sljit_s32 fscratches, sljit_s32 fsaveds, sljit_s32 local_size)

{

  SLJIT_UNUSED_ARG(args);

  SLJIT_UNUSED_ARG(local_size);

  compiler->options = options;

  compiler->scratches = scratches;

  compiler->saveds = saveds;

  compiler->fscratches = fscratches;

  compiler->fsaveds = fsaveds;

#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)

  compiler->last_return = args & SLJIT_ARG_MASK;

  compiler->logical_local_size = local_size;

#endif

}

static SLJIT_INLINE void set_label(struct sljit_label *label, struct sljit_compiler *compiler)

{

  label->next = NULL;

  label->u.index = compiler->label_count++;

  label->size = compiler->size;

  if (compiler->last_label != NULL)

    compiler->last_label->next = label;

  else

    compiler->labels = label;

  compiler->last_label = label;

}

static SLJIT_INLINE void set_jump(struct sljit_jump *jump, struct sljit_compiler *compiler, sljit_u32 flags)

{

  jump->next = NULL;

  jump->flags = flags;

  jump->u.label = NULL;

  if (compiler->last_jump != NULL)

    compiler->last_jump->next = jump;

  else

    compiler->jumps = jump;

  compiler->last_jump = jump;

}

static SLJIT_INLINE void set_mov_addr(struct sljit_jump *jump, struct sljit_compiler *compiler, sljit_uw offset)

{

  jump->next = NULL;

  jump->addr = compiler->size - offset;

  jump->flags = JUMP_MOV_ADDR;

  jump->u.label = NULL;

  if (compiler->last_jump != NULL)

    compiler->last_jump->next = jump;

  else

    compiler->jumps = jump;

  compiler->last_jump = jump;

}

static SLJIT_INLINE void set_const(struct sljit_const *const_, struct sljit_compiler *compiler)

{

  const_->next = NULL;

  const_->addr = compiler->size;

  if (compiler->last_const != NULL)

    compiler->last_const->next = const_;

  else

    compiler->consts = const_;

  compiler->last_const = const_;

}

#define ADDRESSING_DEPENDS_ON(exp, reg) \

  (((exp) & SLJIT_MEM) && (((exp) & REG_MASK) == reg || OFFS_REG(exp) == reg))

#if (defined SLJIT_ARGUMENT_CHECKS && SLJIT_ARGUMENT_CHECKS)

static sljit_s32 function_check_arguments(sljit_s32 arg_types, sljit_s32 scratches, sljit_s32 saveds, sljit_s32 fscratches)

{

  sljit_s32 word_arg_count, scratch_arg_end, saved_arg_count, float_arg_count, curr_type;

  curr_type = (arg_types & SLJIT_ARG_FULL_MASK);

  if (curr_type >= SLJIT_ARG_TYPE_F64) {

    if (curr_type > SLJIT_ARG_TYPE_F32 || fscratches == 0)

      return 0;

  } else if (curr_type >= SLJIT_ARG_TYPE_W) {

    if (scratches == 0)

      return 0;

  }

  arg_types >>= SLJIT_ARG_SHIFT;

  word_arg_count = 0;

  scratch_arg_end = 0;

  saved_arg_count = 0;

  float_arg_count = 0;

  while (arg_types != 0) {

    if (word_arg_count + float_arg_count >= 4)

      return 0;

    curr_type = (arg_types & SLJIT_ARG_MASK);

    if (arg_types & SLJIT_ARG_TYPE_SCRATCH_REG) {

      if (saveds == -1 || curr_type < SLJIT_ARG_TYPE_W || curr_type > SLJIT_ARG_TYPE_P)

        return 0;

      word_arg_count++;

      scratch_arg_end = word_arg_count;

    } else {

      if (curr_type < SLJIT_ARG_TYPE_W || curr_type > SLJIT_ARG_TYPE_F32)

        return 0;

      if (curr_type < SLJIT_ARG_TYPE_F64) {

        word_arg_count++;

        saved_arg_count++;

      } else

        float_arg_count++;

    }

    arg_types >>= SLJIT_ARG_SHIFT;

  }

  if (saveds == -1)

    return (word_arg_count <= scratches && float_arg_count <= fscratches);

  return (saved_arg_count <= saveds && scratch_arg_end <= scratches && float_arg_count <= fscratches);

}

#define FUNCTION_CHECK_IS_REG(r) \

  (((r) >= SLJIT_R0 && (r) < (SLJIT_R0 + compiler->scratches)) \

  || ((r) > (SLJIT_S0 - compiler->saveds) && (r) <= SLJIT_S0) \

  || ((r) >= SLJIT_TMP_REGISTER_BASE && (r) < (SLJIT_TMP_REGISTER_BASE + SLJIT_NUMBER_OF_TEMPORARY_REGISTERS)))

#if (defined SLJIT_CONFIG_X86_32 && SLJIT_CONFIG_X86_32)

#define CHECK_IF_VIRTUAL_REGISTER(p) ((p) <= SLJIT_S3 && (p) >= SLJIT_S8)

#else

#define CHECK_IF_VIRTUAL_REGISTER(p) 0

#endif

static sljit_s32 function_check_src_mem(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)

{

  if (compiler->scratches == -1)

    return 0;

  if (!(p & SLJIT_MEM))

    return 0;

  if (p == SLJIT_MEM1(SLJIT_SP))

    return (i >= 0 && i < compiler->logical_local_size);

  if (!(!(p & REG_MASK) || FUNCTION_CHECK_IS_REG(p & REG_MASK)))

    return 0;

  if (CHECK_IF_VIRTUAL_REGISTER(p & REG_MASK))

    return 0;

  if (p & OFFS_REG_MASK) {

    if (!(p & REG_MASK))

      return 0;

    if (!(FUNCTION_CHECK_IS_REG(OFFS_REG(p))))

      return 0;

    if (CHECK_IF_VIRTUAL_REGISTER(OFFS_REG(p)))

      return 0;

    if ((i & ~0x3) != 0)

      return 0;

  }

  return (p & ~(SLJIT_MEM | REG_MASK | OFFS_REG_MASK)) == 0;

}

#define FUNCTION_CHECK_SRC_MEM(p, i) \

  CHECK_ARGUMENT(function_check_src_mem(compiler, p, i));

static sljit_s32 function_check_src(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)

{

  if (compiler->scratches == -1)

    return 0;

  if (FUNCTION_CHECK_IS_REG(p))

    return (i == 0);

  if (p == SLJIT_IMM)

    return 1;

  return function_check_src_mem(compiler, p, i);

}

#define FUNCTION_CHECK_SRC(p, i) \

  CHECK_ARGUMENT(function_check_src(compiler, p, i));

static sljit_s32 function_check_dst(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i)

{

  if (compiler->scratches == -1)

    return 0;

  if (FUNCTION_CHECK_IS_REG(p))

    return (i == 0);

  return function_check_src_mem(compiler, p, i);

}

#define FUNCTION_CHECK_DST(p, i) \

  CHECK_ARGUMENT(function_check_dst(compiler, p, i));

#if (defined SLJIT_CONFIG_ARM_32 && SLJIT_CONFIG_ARM_32) \

  || (defined SLJIT_CONFIG_MIPS_32 && SLJIT_CONFIG_MIPS_32)

#define FUNCTION_CHECK_IS_FREG(fr, is_32) \

  function_check_is_freg(compiler, (fr), (is_32))

static sljit_s32 function_check_is_freg(struct sljit_compiler *compiler, sljit_s32 fr, sljit_s32 is_32);

#define FUNCTION_FCHECK(p, i, is_32) \

  CHECK_ARGUMENT(function_fcheck(compiler, (p), (i), (is_32)));

static sljit_s32 function_fcheck(struct sljit_compiler *compiler, sljit_s32 p, sljit_sw i, sljit_s32 is_32)

{

  if (compiler->scratches == -1)

    return 0;

  if (FUNCTION_CHECK_IS_FREG(p, is_32))

    return (i == 0);

  return function_check_src_mem(compiler, p, i);

}

#else /* !SLJIT_CONFIG_ARM_32 && !SLJIT_CONFIG_MIPS_32 */

#define FUNCTION_CHECK_IS_FREG(fr, is_32) \

  function_check_is_freg(compiler, (fr))

static sljit_s32 function_check_is_freg(struct sljit_compiler *compiler, sljit_s32 fr)

{

  if (compiler->scratches == -1)

    return 0;

  return (fr >= SLJIT_FR0 && fr < (SLJIT_FR0 + compiler->fscratches))

    || (fr > (SLJIT_FS0 - compiler->fsaveds) && fr <= SLJIT_FS0)

    || (fr >= SLJIT_TMP_FREGISTER_BASE && fr < (SLJIT_TMP_FREGISTER_BASE + SLJIT_NUMBER_OF_TEMPORARY_FLOAT_REGISTERS));

}