/*

 * fast architecture specific locking

 *

 * Copyright (C) 2001-2003 FhG Fokus

 *

 * Copyright (C) 2017 Core Network Dynamics for ARM8 (aarch64) support

 *

 * Permission to use, copy, modify, and distribute this software for any

 * purpose with or without fee is hereby granted, provided that the above

 * copyright notice and this permission notice appear in all copies.

 *

 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES

 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF

 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR

 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES

 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN

 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF

 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

 */

/*!

* \file

* \brief Kamailio core :: fast architecture specific locking

* \author andrei

* \ingroup core

* Module: \ref core

 * WARNING: the code was not tested on the following architectures:

 *           - alpha (cross-compiles ok, no test)

 *           - mips64 (cross-compiles ok)

 *           - ppc64 (compiles ok)

 *           - sparc32 (tested on a sparc64)

 */

#ifndef fastlock_h

#define fastlock_h

#include "sched_yield.h"

#define SPIN_OPTIMIZE /* if defined optimize spining on the lock:

                         try first the lock with non-atomic/non memory locking

                         operations, and only if the lock appears to be free

                         switch to the more expensive version */

typedef volatile int fl_lock_t;

#define init_lock(l) (l) = 0

/* what membar to use (if any) after taking a lock. This

 *  was separated from the lock code to allow better optimizations.

 *  e.g.: use the membar_getlock only after getting the lock and don't use

 *  it if lock_get fails / when spinning on tsl.

 *  There is no corresponding membar_release_lock (because lock_release

 *  must always include the needed memory barrier).

 *  WARNING: this is intended only for internal fastlock use*/

#if defined(__CPU_i386) || defined(__CPU_x86_64)

#define membar_getlock() /* not needed on x86 */

#elif defined(__CPU_sparc64)

#ifndef NOSMP

#define membar_getlock() \

  asm volatile("membar #StoreStore | #StoreLoad \n\t" : : : "memory")

/* can be either StoreStore|StoreLoad or LoadStore|LoadLoad

   * since ldstub acts both as a store and as a load */

#else

/* no need for a compiler barrier, that is already included in lock_get/tsl*/

#define membar_getlock() /* not needed if no smp*/

#endif           /* NOSMP */

#elif defined(__CPU_sparc)

#define membar_getlock() /* no need for a compiler barrier, already included */

#elif defined __CPU_arm

#ifndef NOSMP

#warning smp not supported on arm < 6 (no membars), try compiling with -DNOSMP

#endif /* NOSMP */

#define membar_getlock()

#elif defined __CPU_arm6

#ifndef NOSMP

#define membar_getlock() \

  asm volatile("mcr p15, 0, %0, c7, c10, 5" : : "r"(0) : "memory")

#else /* NOSMP */

#define membar_getlock()

#endif /* NOSMP */

#elif defined __CPU_arm7

#ifndef NOSMP

#define membar_getlock() asm volatile("dmb" : : : "memory")

#else /* NOSMP */

#define membar_getlock()

#endif /* NOSMP */

#elif defined(__CPU_aarch64)

#ifndef NOSMP

#warning smp not supported on arm* (no membars), try compiling with -DNOSMP

#endif /* NOSMP */

#define membar_getlock()

#elif defined(__CPU_ppc) || defined(__CPU_ppc64)

#ifndef NOSMP

#define membar_getlock() asm volatile("lwsync \n\t" : : : "memory");

#else

#define membar_getlock()

#endif /* NOSMP */

#elif defined __CPU_mips2 || defined __CPU_mips64

#ifndef NOSMP

#define membar_getlock() asm volatile("sync \n\t" : : : "memory");

#else

#define membar_getlock()

#endif /* NOSMP */

#elif defined __CPU_mips

#ifndef NOSMP

#warning smp not supported on mips1 (no membars), try compiling with -DNOSMP

#endif

#define membar_getlock()

#elif defined __CPU_alpha

#ifndef NOSMP

#define membar_getlock() asm volatile("mb \n\t" : : : "memory");

#else

#define membar_getlock()

#endif /* NOSMP */

#else /* __CPU_xxx */

#error "unknown architecture"

#endif

/*test and set lock, ret !=0 if lock held by someone else, 0 otherwise

 * WARNING: no memory barriers included, if you use this function directly

 *          (not recommended) and it gets the lock (ret==0), you should call

 *          membar_getlock() after it */

inline static int tsl(fl_lock_t *lock)

{

  int val;

#if defined(__CPU_i386) || defined(__CPU_x86_64)

#ifdef NOSMP

  asm volatile(" xor %0, %0 \n\t"

         " btsl $0, %2 \n\t"

         " setc %b0 \n\t"

         : "=&q"(val), "=m"(*lock)

         : "m"(*lock)

         : "memory", "cc");

#else

  asm volatile(

#ifdef SPIN_OPTIMIZE

      " cmpb $0, %2 \n\t"

      " mov $1, %0 \n\t"

      " jnz 1f \n\t"

#else

      " mov $1, %0 \n\t"

#endif

      " xchgb %2, %b0 \n\t"

      "1: \n\t"

      : "=&q"(val), "=m"(*lock)

      : "m"(*lock)

      : "memory"

#ifdef SPIN_OPTIMIZE

      ,

      "cc"

#endif

  );

#endif /*NOSMP*/

#elif defined(__CPU_sparc64)

  asm volatile(

#ifdef SPIN_OPTIMIZE

      "   ldub [%2], %0 \n\t"

      "   brnz,a,pn %0, 1f \n\t"

      "   nop \n\t"

#endif

      "   ldstub [%2], %0 \n\t"

      "1: \n\t"

      /* membar_getlock must be  called outside this function */

      : "=&r"(val), "=m"(*lock)

      : "r"(lock)

      : "memory");

#elif defined(__CPU_sparc)

  asm volatile(

#ifdef SPIN_OPTIMIZE

      "   ldub [%2], %0 \n\t"

      "   tst %0 \n\t"

      "   bne,a  1f \n\t"

      "   nop \n\t"

#endif

      "   ldstub [%2], %0 \n\t"

      "1: \n\t"

      /* membar_getlock must be  called outside this function */

      : "=&r"(val), "=m"(*lock)

      : "r"(lock)

      : "memory"

#ifdef SPIN_OPTIMIZE

      ,

      "cc"

#endif

  );

#elif defined __CPU_arm

  asm volatile("swp %0, %2, [%3] \n\t"

         : "=&r"(val), "=m"(*lock)

         : "r"(1), "r"(lock)

         : "memory");

#elif defined __CPU_arm6 || defined __CPU_arm7

  asm volatile("   ldrex %0, [%2] \n\t"

         "   cmp %0, #0 \n\t"

         "   strexeq %0, %3, [%2] \n\t" /* executed only if Z=1 */

         /* if %0!=0 => either it was 1 initially or was 0

       * and somebody changed it just before the strexeq (so the

       * lock is taken) => it's safe to return %0 */

         /* membar_getlock must be  called outside this function */

         : "=&r"(val), "=m"(*lock)

         : "r"(lock), "r"(1)

         : "cc");

#elif defined __CPU_aarch64

  int res = 0;

  int one = 1;

  asm volatile("1: ldaxr %w0, %2 \n\t"

         "   stlxr %w1, %w3, %2 \n\t"

         "   cbnz %w1, 1b \n\t"

         : "=&r"(val), "=&r"(res), "+Q"(*lock)

         : "r"(one)

         : "cc", "memory");

#elif defined(__CPU_ppc) || defined(__CPU_ppc64)

  asm volatile("1: \n\t"

#ifdef SPIN_OPTIMIZE

         "   lwzx %0, 0, %2 \n\t"

         "   cmpwi %0, 0 \n\t"

         "   bne- 2f \n\t" /* predict: not taken */

#endif

         "   lwarx  %0, 0, %2\n\t"

         "   cmpwi  %0, 0\n\t"

         "   bne-    2f\n\t"

         "   stwcx. %3, 0, %2\n\t"

         "   bne-   1b\n\t"

         /* membar_getlock must be  called outside this function */

         "2:\n\t"

         : "=&r"(val), "=m"(*lock)

         : "r"(lock), "r"(1)

         : "memory", "cc");

#elif defined __CPU_mips2 || (defined __CPU_mips && defined MIPS_HAS_LLSC) \

    || defined __CPU_mips64

  long tmp;

  asm volatile(".set push \n\t"

         ".set noreorder\n\t"

         ".set mips2 \n\t"

#ifdef SPIN_OPTIMIZE

         "    lw %1, %2 \n\t"

         "    bne %1, $0, 2f \n\t"

         "    nop \n\t"

#endif

         "1:  ll %1, %2   \n\t"

         "    bne %1, $0, 2f \n\t"

         "    li %0, 1 \n\t" /* delay slot */

         "    sc %0, %2  \n\t"

         "    beqz %0, 1b \n\t"

         "    nop \n\t"

         "2: \n\t"

         /* membar_getlock must be called outside this function */

         ".set pop\n\t"

         : "=&r"(tmp), "=&r"(val), "=m"(*lock)

         : "m"(*lock)

         : "memory");

#elif defined __CPU_alpha

  long tmp;

  tmp = 0;

  /* lock low bit set to 1 when the lock is hold and to 0 otherwise */

  asm volatile(

      "1:  ldl %0, %1   \n\t"

      "    blbs %0, 2f  \n\t" /* optimization if locked */

      "    ldl_l %0, %1 \n\t"

      "    blbs %0, 2f  \n\t"

      "    lda %2, 1    \n\t" /* or: or $31, 1, %2 ??? */

      "    stl_c %2, %1 \n\t"

      "    beq %2, 3f   \n\t" /* back cond. jumps are always predicted to be

                   taken => make forward jump */

      /* membar_getlock must be called outside this function */

      "2:               \n\t"

      ".subsection 2 \n\t"

      "3:  br 1b \n\t"

      ".previous \n\t"

      : "=&r"(val), "=m"(*lock), "=&r"(tmp)

      : "m"(*lock)

      : "memory");

#else

#error "unknown architecture"

#endif

  return val;

}

Unexecuted instantiation: fuzz_parse_msg.c:tsl

Unexecuted instantiation: fuzz_uri.c:tsl

inline static void get_lock(fl_lock_t *lock)

{

#ifdef ADAPTIVE_WAIT

  int i = ADAPTIVE_WAIT_LOOPS;

#endif

  while(tsl(lock)) {

#ifdef BUSY_WAIT

#elif defined ADAPTIVE_WAIT

    if(i > 0)

      i--;

    else

      sched_yield();

#else

    sched_yield();

#endif

  }

  membar_getlock();

}

Unexecuted instantiation: fuzz_parse_msg.c:get_lock

Unexecuted instantiation: fuzz_uri.c:get_lock

/* like get_lock, but it doesn't wait. If it gets the lock returns 0,

 *  <0  otherwise (-1) */

inline static int try_lock(fl_lock_t *lock)

{

  if(tsl(lock)) {

    return -1;

  }

  membar_getlock();

  return 0;

}

Unexecuted instantiation: fuzz_parse_msg.c:try_lock

Unexecuted instantiation: fuzz_uri.c:try_lock

inline static void release_lock(fl_lock_t *lock)

{

#if defined(__CPU_i386)

#ifdef NOSMP

  asm volatile(" movb $0, %0 \n\t" : "=m"(*lock) : : "memory");

#else  /* ! NOSMP */

  int val;

  /* a simple mov $0, (lock) does not force StoreStore ordering on all

     x86 versions and it doesn't seem to force LoadStore either */

  asm volatile(" xchgb %b0, %1 \n\t"

         : "=q"(val), "=m"(*lock)

         : "0"(0)

         : "memory");

#endif /* NOSMP */

#elif defined(__CPU_x86_64)

  asm volatile(

      " movb $0, %0 \n\t" /* on amd64 membar StoreStore | LoadStore is

                 implicit (at least on the same mem. type) */

      : "=m"(*lock)

      :

      : "memory");

#elif defined(__CPU_sparc64) || defined(__CPU_sparc)

  asm volatile(

#ifndef NOSMP

#ifdef __CPU_sparc64

      "membar #LoadStore | #StoreStore \n\t"

#else  /* __CPU_sparc */

      "stbar \n\t"

#endif /* __CPU_sparc64 */

#endif

      "stb %%g0, [%1] \n\t"

      : "=m"(*lock)

      : "r"(lock)

      : "memory");

#elif defined __CPU_arm || defined __CPU_arm6 || defined __CPU_arm7

#if !defined NOSMP && defined __CPU_arm

#warning arm* smp mode not supported (no membars), try compiling with -DNOSMP

#endif

  /* missuse membar_getlock */

  membar_getlock();

  asm volatile(" str %1, [%2] \n\r"

         : "=m"(*lock)

         : "r"(0), "r"(lock)

         : "cc", "memory");

#ifdef __CPU_arm6

  /* drain store buffer: drain the per processor buffer into the L1 cache

     making all the changes visible to other processors */

  asm volatile("mcr p15, 0, %0, c7, c10, 4 \n\r" /* DSB equiv. on arm6*/

         :

         : "r"(0)

         : "memory");

#elif defined __CPU_arm7

  /* drain store buffer: drain the per processor buffer into the L1 cache

     making all the changes visible to other processors */

  asm volatile("dsb \n\r" : : : "memory");

#endif /* __CPU_arm6 / __CPU_arm7 */

#elif defined __CPU_aarch64

#ifndef NOSMP

#warning arm* smp mode not supported (no membars), try compiling with -DNOSMP

#endif

  asm volatile(" stlr %w1, %0 \n\t" : "=Q"(*lock) : "r"(0) : "memory");

#elif defined(__CPU_ppc) || defined(__CPU_ppc64)

  asm volatile(

      /* "sync\n\t"  lwsync is faster and will work

       *             here too

       *             [IBM Prgramming Environments Manual, D.4.2.2]

       */

      "lwsync\n\t"

      "stwx %1, 0, %2\n\t"

      : "=m"(*lock)

      : "r"(0), "r"(lock)

      : "memory");

#elif defined __CPU_mips2 || (defined __CPU_mips && defined MIPS_HAS_LLSC) \

    || defined __CPU_mips64

  asm volatile(".set push \n\t"

         ".set noreorder \n\t"

         ".set mips2 \n\t"

#ifndef NOSMP

#ifdef __CPU_mips

#warning mips1 smp mode not supported (no membars), try compiling with -DNOSMP

#else

         "    sync \n\t"

#endif

#endif

         "    sw $0, %0 \n\t"

         ".set pop \n\t"

         : "=m"(*lock)

         : /* no input */

         : "memory");

#elif defined __CPU_alpha

  asm volatile(

#ifndef NOSMP

      "    mb          \n\t"

#endif

      "    stl $31, %0 \n\t"

      : "=m"(*lock)

      :      /* no input*/

      : "memory" /* because of the mb */

  );

#else

#error "unknown architecture"

#endif

}

Unexecuted instantiation: fuzz_parse_msg.c:release_lock

Unexecuted instantiation: fuzz_uri.c:release_lock

#endif