/src/postgres/src/include/port/atomics/generic-gcc.h

Source
/*-------------------------------------------------------------------------
 *
 * generic-gcc.h
 *    Atomic operations, implemented using gcc (or compatible) intrinsics.
 *
 * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
 * Portions Copyright (c) 1994, Regents of the University of California
 *
 * NOTES:
 *
 * Documentation:
 * * Legacy __sync Built-in Functions for Atomic Memory Access
 *   https://gcc.gnu.org/onlinedocs/gcc-4.8.2/gcc/_005f_005fsync-Builtins.html
 * * Built-in functions for memory model aware atomic operations
 *   https://gcc.gnu.org/onlinedocs/gcc-4.8.2/gcc/_005f_005fatomic-Builtins.html
 *
 * src/include/port/atomics/generic-gcc.h
 *
 *-------------------------------------------------------------------------
 */

/* intentionally no include guards, should only be included by atomics.h */
#ifndef INSIDE_ATOMICS_H
#error "should be included via atomics.h"
#endif

/*
 * An empty asm block should be a sufficient compiler barrier.
 */
#define pg_compiler_barrier_impl()  __asm__ __volatile__("" ::: "memory")

/*
 * If we're on GCC, we should be able to get a memory barrier
 * out of this compiler built-in.  But we prefer to rely on platform specific
 * definitions where possible, and use this only as a fallback.
 */
#if !defined(pg_memory_barrier_impl)
# if defined(HAVE_GCC__ATOMIC_INT32_CAS)
#   define pg_memory_barrier_impl()   __atomic_thread_fence(__ATOMIC_SEQ_CST)
# elif defined(__GNUC__)
#   define pg_memory_barrier_impl()   __sync_synchronize()
# endif
#endif /* !defined(pg_memory_barrier_impl) */

#if !defined(pg_read_barrier_impl) && defined(HAVE_GCC__ATOMIC_INT32_CAS)
/* acquire semantics include read barrier semantics */
#   define pg_read_barrier_impl()   __atomic_thread_fence(__ATOMIC_ACQUIRE)
#endif

#if !defined(pg_write_barrier_impl) && defined(HAVE_GCC__ATOMIC_INT32_CAS)
/* release semantics include write barrier semantics */
#   define pg_write_barrier_impl()    __atomic_thread_fence(__ATOMIC_RELEASE)
#endif


/* generic gcc based atomic flag implementation */
#if !defined(PG_HAVE_ATOMIC_FLAG_SUPPORT) \
  && (defined(HAVE_GCC__SYNC_INT32_TAS) || defined(HAVE_GCC__SYNC_CHAR_TAS))

#define PG_HAVE_ATOMIC_FLAG_SUPPORT
typedef struct pg_atomic_flag
{
  /*
   * If we have a choice, use int-width TAS, because that is more efficient
   * and/or more reliably implemented on most non-Intel platforms.  (Note
   * that this code isn't used on x86[_64]; see arch-x86.h for that.)
   */
#ifdef HAVE_GCC__SYNC_INT32_TAS
  volatile int value;
#else
  volatile char value;
#endif
} pg_atomic_flag;

#endif /* !ATOMIC_FLAG_SUPPORT && SYNC_INT32_TAS */

/* generic gcc based atomic uint32 implementation */
#if !defined(PG_HAVE_ATOMIC_U32_SUPPORT) \
  && (defined(HAVE_GCC__ATOMIC_INT32_CAS) || defined(HAVE_GCC__SYNC_INT32_CAS))

#define PG_HAVE_ATOMIC_U32_SUPPORT
typedef struct pg_atomic_uint32
{
  volatile uint32 value;
} pg_atomic_uint32;

#endif /* defined(HAVE_GCC__ATOMIC_INT32_CAS) || defined(HAVE_GCC__SYNC_INT32_CAS) */

/* generic gcc based atomic uint64 implementation */
#if !defined(PG_HAVE_ATOMIC_U64_SUPPORT) \
  && !defined(PG_DISABLE_64_BIT_ATOMICS) \
  && (defined(HAVE_GCC__ATOMIC_INT64_CAS) || defined(HAVE_GCC__SYNC_INT64_CAS))

#define PG_HAVE_ATOMIC_U64_SUPPORT

typedef struct pg_atomic_uint64
{
  volatile uint64 value pg_attribute_aligned(8);
} pg_atomic_uint64;

#endif /* defined(HAVE_GCC__ATOMIC_INT64_CAS) || defined(HAVE_GCC__SYNC_INT64_CAS) */

#ifdef PG_HAVE_ATOMIC_FLAG_SUPPORT

#if defined(HAVE_GCC__SYNC_CHAR_TAS) || defined(HAVE_GCC__SYNC_INT32_TAS)

#ifndef PG_HAVE_ATOMIC_TEST_SET_FLAG
#define PG_HAVE_ATOMIC_TEST_SET_FLAG
static inline bool
pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr)
{
  /* NB: only an acquire barrier, not a full one */
  /* some platform only support a 1 here */
  return __sync_lock_test_and_set(&ptr->value, 1) == 0;
}
#endif

#endif /* defined(HAVE_GCC__SYNC_*_TAS) */

#ifndef PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG
#define PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG
static inline bool
pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag *ptr)
{
  return ptr->value == 0;
}
#endif

#ifndef PG_HAVE_ATOMIC_CLEAR_FLAG
#define PG_HAVE_ATOMIC_CLEAR_FLAG
static inline void
pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr)
{
  __sync_lock_release(&ptr->value);
}
#endif

#ifndef PG_HAVE_ATOMIC_INIT_FLAG
#define PG_HAVE_ATOMIC_INIT_FLAG
static inline void
pg_atomic_init_flag_impl(volatile pg_atomic_flag *ptr)
{
  pg_atomic_clear_flag_impl(ptr);
}
#endif

#endif /* defined(PG_HAVE_ATOMIC_FLAG_SUPPORT) */

/* prefer __atomic, it has a better API */
#if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) && defined(HAVE_GCC__ATOMIC_INT32_CAS)
#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32
static inline bool
pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr,
                  uint32 *expected, uint32 newval)
{
  /* FIXME: we can probably use a lower consistency model */
  return __atomic_compare_exchange_n(&ptr->value, expected, newval, false,
                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
}
#endif

#if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) && defined(HAVE_GCC__SYNC_INT32_CAS)
#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32
static inline bool
pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr,
                  uint32 *expected, uint32 newval)
{
  bool  ret;
  uint32  current;
  current = __sync_val_compare_and_swap(&ptr->value, *expected, newval);
  ret = current == *expected;
  *expected = current;
  return ret;
}
#endif

/*
 * __sync_lock_test_and_set() only supports setting the value to 1 on some
 * platforms, so we only provide an __atomic implementation for
 * pg_atomic_exchange.
 *
 * We assume the availability of 32-bit __atomic_compare_exchange_n() implies
 * the availability of 32-bit __atomic_exchange_n().
 */
#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U32) && defined(HAVE_GCC__ATOMIC_INT32_CAS)
#define PG_HAVE_ATOMIC_EXCHANGE_U32
static inline uint32
pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 newval)
{
  return __atomic_exchange_n(&ptr->value, newval, __ATOMIC_SEQ_CST);
}
#endif

/* if we have 32-bit __sync_val_compare_and_swap, assume we have these too: */

#if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U32) && defined(HAVE_GCC__SYNC_INT32_CAS)
#define PG_HAVE_ATOMIC_FETCH_ADD_U32
static inline uint32
pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_)
{
  return __sync_fetch_and_add(&ptr->value, add_);
}
#endif

#if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U32) && defined(HAVE_GCC__SYNC_INT32_CAS)
#define PG_HAVE_ATOMIC_FETCH_SUB_U32
static inline uint32
pg_atomic_fetch_sub_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_)
{
  return __sync_fetch_and_sub(&ptr->value, sub_);
}
#endif

#if !defined(PG_HAVE_ATOMIC_FETCH_AND_U32) && defined(HAVE_GCC__SYNC_INT32_CAS)
#define PG_HAVE_ATOMIC_FETCH_AND_U32
static inline uint32
pg_atomic_fetch_and_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 and_)
{
  return __sync_fetch_and_and(&ptr->value, and_);
}
#endif

#if !defined(PG_HAVE_ATOMIC_FETCH_OR_U32) && defined(HAVE_GCC__SYNC_INT32_CAS)
#define PG_HAVE_ATOMIC_FETCH_OR_U32
static inline uint32
pg_atomic_fetch_or_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 or_)
{
  return __sync_fetch_and_or(&ptr->value, or_);
}
#endif


#if !defined(PG_DISABLE_64_BIT_ATOMICS)

#if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) && defined(HAVE_GCC__ATOMIC_INT64_CAS)
#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64
static inline bool
pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr,
                  uint64 *expected, uint64 newval)
{
  AssertPointerAlignment(expected, 8);
  return __atomic_compare_exchange_n(&ptr->value, expected, newval, false,
                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
}
#endif

#if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) && defined(HAVE_GCC__SYNC_INT64_CAS)
#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64
static inline bool
pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr,
                  uint64 *expected, uint64 newval)
{
  bool  ret;
  uint64  current;

  AssertPointerAlignment(expected, 8);
  current = __sync_val_compare_and_swap(&ptr->value, *expected, newval);
  ret = current == *expected;
  *expected = current;
  return ret;
}
#endif

/*
 * __sync_lock_test_and_set() only supports setting the value to 1 on some
 * platforms, so we only provide an __atomic implementation for
 * pg_atomic_exchange.
 *
 * We assume the availability of 64-bit __atomic_compare_exchange_n() implies
 * the availability of 64-bit __atomic_exchange_n().
 */
#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U64) && defined(HAVE_GCC__ATOMIC_INT64_CAS)
#define PG_HAVE_ATOMIC_EXCHANGE_U64
static inline uint64
pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 newval)
{
  return __atomic_exchange_n(&ptr->value, newval, __ATOMIC_SEQ_CST);
}
#endif

/* if we have 64-bit __sync_val_compare_and_swap, assume we have these too: */

#if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U64) && defined(HAVE_GCC__SYNC_INT64_CAS)
#define PG_HAVE_ATOMIC_FETCH_ADD_U64
static inline uint64
pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_)
{
  return __sync_fetch_and_add(&ptr->value, add_);
}
#endif

#if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U64) && defined(HAVE_GCC__SYNC_INT64_CAS)
#define PG_HAVE_ATOMIC_FETCH_SUB_U64
static inline uint64
pg_atomic_fetch_sub_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_)
{
  return __sync_fetch_and_sub(&ptr->value, sub_);
}
#endif

#if !defined(PG_HAVE_ATOMIC_FETCH_AND_U64) && defined(HAVE_GCC__SYNC_INT64_CAS)
#define PG_HAVE_ATOMIC_FETCH_AND_U64
static inline uint64
pg_atomic_fetch_and_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 and_)
{
  return __sync_fetch_and_and(&ptr->value, and_);
}
#endif

#if !defined(PG_HAVE_ATOMIC_FETCH_OR_U64) && defined(HAVE_GCC__SYNC_INT64_CAS)
#define PG_HAVE_ATOMIC_FETCH_OR_U64
static inline uint64
pg_atomic_fetch_or_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 or_)
{
  return __sync_fetch_and_or(&ptr->value, or_);
}
#endif

#endif /* !defined(PG_DISABLE_64_BIT_ATOMICS) */

Coverage Report

Created: 2025-10-09 06:07

Line	Count	Source
1		/*-------------------------------------------------------------------------
2		*
3		* generic-gcc.h
4		* Atomic operations, implemented using gcc (or compatible) intrinsics.
5		*
6		* Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
7		* Portions Copyright (c) 1994, Regents of the University of California
8		*
9		* NOTES:
10		*
11		* Documentation:
12		* * Legacy __sync Built-in Functions for Atomic Memory Access
13		* https://gcc.gnu.org/onlinedocs/gcc-4.8.2/gcc/_005f_005fsync-Builtins.html
14		* * Built-in functions for memory model aware atomic operations
15		* https://gcc.gnu.org/onlinedocs/gcc-4.8.2/gcc/_005f_005fatomic-Builtins.html
16		*
17		* src/include/port/atomics/generic-gcc.h
18		*
19		*-------------------------------------------------------------------------
20		*/
21
22		/* intentionally no include guards, should only be included by atomics.h */
23		#ifndef INSIDE_ATOMICS_H
24		#error "should be included via atomics.h"
25		#endif
26
27		/*
28		* An empty asm block should be a sufficient compiler barrier.
29		*/
30	0	#define pg_compiler_barrier_impl() __asm__ __volatile__("" ::: "memory")
31
32		/*
33		* If we're on GCC, we should be able to get a memory barrier
34		* out of this compiler built-in. But we prefer to rely on platform specific
35		* definitions where possible, and use this only as a fallback.
36		*/
37		#if !defined(pg_memory_barrier_impl)
38		# if defined(HAVE_GCC__ATOMIC_INT32_CAS)
39		# define pg_memory_barrier_impl() __atomic_thread_fence(__ATOMIC_SEQ_CST)