/src/usrsctp/usrsctplib/user_atomic.h
Line | Count | Source (jump to first uncovered line) |
1 | | /*- |
2 | | * Copyright (c) 2009-2010 Brad Penoff |
3 | | * Copyright (c) 2009-2010 Humaira Kamal |
4 | | * Copyright (c) 2011-2012 Irene Ruengeler |
5 | | * Copyright (c) 2011-2012 Michael Tuexen |
6 | | * |
7 | | * All rights reserved. |
8 | | * |
9 | | * Redistribution and use in source and binary forms, with or without |
10 | | * modification, are permitted provided that the following conditions |
11 | | * are met: |
12 | | * 1. Redistributions of source code must retain the above copyright |
13 | | * notice, this list of conditions and the following disclaimer. |
14 | | * 2. Redistributions in binary form must reproduce the above copyright |
15 | | * notice, this list of conditions and the following disclaimer in the |
16 | | * documentation and/or other materials provided with the distribution. |
17 | | * |
18 | | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND |
19 | | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
20 | | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE |
21 | | * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE |
22 | | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
23 | | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS |
24 | | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) |
25 | | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
26 | | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY |
27 | | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF |
28 | | * SUCH DAMAGE. |
29 | | */ |
30 | | |
31 | | #ifndef _USER_ATOMIC_H_ |
32 | | #define _USER_ATOMIC_H_ |
33 | | |
34 | | /* __Userspace__ version of sys/i386/include/atomic.h goes here */ |
35 | | |
36 | | /* TODO In the future, might want to not use i386 specific assembly. |
37 | | * The options include: |
38 | | * - implement them generically (but maybe not truly atomic?) in userspace |
39 | | * - have ifdef's for __Userspace_arch_ perhaps (OS isn't enough...) |
40 | | */ |
41 | | |
42 | | #include <stdio.h> |
43 | | #include <sys/types.h> |
44 | | |
45 | | #if defined(__APPLE__) || defined(_WIN32) |
46 | | #if defined(_WIN32) |
47 | | #define atomic_add_int(addr, val) InterlockedExchangeAdd((LPLONG)addr, (LONG)val) |
48 | | #define atomic_fetchadd_int(addr, val) InterlockedExchangeAdd((LPLONG)addr, (LONG)val) |
49 | | #define atomic_subtract_int(addr, val) InterlockedExchangeAdd((LPLONG)addr,-((LONG)val)) |
50 | | #define atomic_cmpset_int(dst, exp, src) InterlockedCompareExchange((LPLONG)dst, src, exp) |
51 | | #define SCTP_DECREMENT_AND_CHECK_REFCOUNT(addr) (InterlockedExchangeAdd((LPLONG)addr, (-1L)) == 1) |
52 | | #else |
53 | | #include <libkern/OSAtomic.h> |
54 | | #define atomic_add_int(addr, val) OSAtomicAdd32Barrier(val, (int32_t *)addr) |
55 | | #define atomic_fetchadd_int(addr, val) OSAtomicAdd32Barrier(val, (int32_t *)addr) |
56 | | #define atomic_subtract_int(addr, val) OSAtomicAdd32Barrier(-val, (int32_t *)addr) |
57 | | #define atomic_cmpset_int(dst, exp, src) OSAtomicCompareAndSwapIntBarrier(exp, src, (int *)dst) |
58 | | #define SCTP_DECREMENT_AND_CHECK_REFCOUNT(addr) (atomic_fetchadd_int(addr, -1) == 0) |
59 | | #endif |
60 | | |
61 | | #if defined(INVARIANTS) |
62 | | #define SCTP_SAVE_ATOMIC_DECREMENT(addr, val) \ |
63 | | { \ |
64 | | int32_t newval; \ |
65 | | newval = atomic_fetchadd_int(addr, -val); \ |
66 | | if (newval < 0) { \ |
67 | | panic("Counter goes negative"); \ |
68 | | } \ |
69 | | } |
70 | | #else |
71 | | #define SCTP_SAVE_ATOMIC_DECREMENT(addr, val) \ |
72 | | { \ |
73 | | int32_t newval; \ |
74 | | newval = atomic_fetchadd_int(addr, -val); \ |
75 | | if (newval < 0) { \ |
76 | | *addr = 0; \ |
77 | | } \ |
78 | | } |
79 | | #endif |
80 | | #if defined(_WIN32) |
81 | | static void atomic_init(void) {} /* empty when we are not using atomic_mtx */ |
82 | | #else |
83 | | static inline void atomic_init(void) {} /* empty when we are not using atomic_mtx */ |
84 | | #endif |
85 | | |
86 | | #else |
87 | | /* Using gcc built-in functions for atomic memory operations |
88 | | Reference: http://gcc.gnu.org/onlinedocs/gcc-4.1.0/gcc/Atomic-Builtins.html |
89 | | Requires gcc version 4.1.0 |
90 | | compile with -march=i486 |
91 | | */ |
92 | | |
93 | | /*Atomically add V to *P.*/ |
94 | 60.6M | #define atomic_add_int(P, V) (void) __sync_fetch_and_add(P, V) |
95 | | |
96 | | /*Atomically subtrace V from *P.*/ |
97 | 10.4M | #define atomic_subtract_int(P, V) (void) __sync_fetch_and_sub(P, V) |
98 | | |
99 | | /* |
100 | | * Atomically add the value of v to the integer pointed to by p and return |
101 | | * the previous value of *p. |
102 | | */ |
103 | 99.8k | #define atomic_fetchadd_int(p, v) __sync_fetch_and_add(p, v) |
104 | | |
105 | | /* Following explanation from src/sys/i386/include/atomic.h, |
106 | | * for atomic compare and set |
107 | | * |
108 | | * if (*dst == exp) *dst = src (all 32 bit words) |
109 | | * |
110 | | * Returns 0 on failure, non-zero on success |
111 | | */ |
112 | | |
113 | 25.0k | #define atomic_cmpset_int(dst, exp, src) __sync_bool_compare_and_swap(dst, exp, src) |
114 | | |
115 | 86.9k | #define SCTP_DECREMENT_AND_CHECK_REFCOUNT(addr) (atomic_fetchadd_int(addr, -1) == 1) |
116 | | #if defined(INVARIANTS) |
117 | 12.8k | #define SCTP_SAVE_ATOMIC_DECREMENT(addr, val) \ |
118 | 12.8k | { \ |
119 | 12.8k | int32_t oldval; \ |
120 | 12.8k | oldval = atomic_fetchadd_int(addr, -val); \ |
121 | 12.8k | if (oldval < val) { \ |
122 | 0 | panic("Counter goes negative"); \ |
123 | 0 | } \ |
124 | 12.8k | } |
125 | | #else |
126 | | #define SCTP_SAVE_ATOMIC_DECREMENT(addr, val) \ |
127 | | { \ |
128 | | int32_t oldval; \ |
129 | | oldval = atomic_fetchadd_int(addr, -val); \ |
130 | | if (oldval < val) { \ |
131 | | *addr = 0; \ |
132 | | } \ |
133 | | } |
134 | | #endif |
135 | 1 | static inline void atomic_init(void) {} /* empty when we are not using atomic_mtx */ Unexecuted instantiation: user_socket.c:atomic_init Unexecuted instantiation: sctp_bsd_addr.c:atomic_init Unexecuted instantiation: sctp_callout.c:atomic_init Unexecuted instantiation: sctp_crc32.c:atomic_init Unexecuted instantiation: sctp_input.c:atomic_init Unexecuted instantiation: sctp_output.c:atomic_init Line | Count | Source | 135 | 1 | static inline void atomic_init(void) {} /* empty when we are not using atomic_mtx */ |
Unexecuted instantiation: sctp_peeloff.c:atomic_init Unexecuted instantiation: sctp_timer.c:atomic_init Unexecuted instantiation: sctp_userspace.c:atomic_init Unexecuted instantiation: sctp_usrreq.c:atomic_init Unexecuted instantiation: sctputil.c:atomic_init Unexecuted instantiation: sctp6_usrreq.c:atomic_init Unexecuted instantiation: user_environment.c:atomic_init Unexecuted instantiation: user_mbuf.c:atomic_init Unexecuted instantiation: user_recv_thread.c:atomic_init Unexecuted instantiation: sctp_asconf.c:atomic_init Unexecuted instantiation: sctp_auth.c:atomic_init Unexecuted instantiation: sctp_cc_functions.c:atomic_init Unexecuted instantiation: sctp_indata.c:atomic_init Unexecuted instantiation: sctp_ss_functions.c:atomic_init Unexecuted instantiation: sctp_sysctl.c:atomic_init |
136 | | #endif |
137 | | |
138 | | #if 0 /* using libatomic_ops */ |
139 | | #include "user_include/atomic_ops.h" |
140 | | |
141 | | /*Atomically add incr to *P, and return the original value of *P.*/ |
142 | | #define atomic_add_int(P, V) AO_fetch_and_add((AO_t*)P, V) |
143 | | |
144 | | #define atomic_subtract_int(P, V) AO_fetch_and_add((AO_t*)P, -(V)) |
145 | | |
146 | | /* |
147 | | * Atomically add the value of v to the integer pointed to by p and return |
148 | | * the previous value of *p. |
149 | | */ |
150 | | #define atomic_fetchadd_int(p, v) AO_fetch_and_add((AO_t*)p, v) |
151 | | |
152 | | /* Atomically compare *addr to old_val, and replace *addr by new_val |
153 | | if the first comparison succeeds. Returns nonzero if the comparison |
154 | | succeeded and *addr was updated. |
155 | | */ |
156 | | /* Following Explanation from src/sys/i386/include/atomic.h, which |
157 | | matches that of AO_compare_and_swap above. |
158 | | * Atomic compare and set, used by the mutex functions |
159 | | * |
160 | | * if (*dst == exp) *dst = src (all 32 bit words) |
161 | | * |
162 | | * Returns 0 on failure, non-zero on success |
163 | | */ |
164 | | |
165 | | #define atomic_cmpset_int(dst, exp, src) AO_compare_and_swap((AO_t*)dst, exp, src) |
166 | | |
167 | | static inline void atomic_init() {} /* empty when we are not using atomic_mtx */ |
168 | | #endif /* closing #if for libatomic */ |
169 | | |
170 | | #if 0 /* using atomic_mtx */ |
171 | | |
172 | | #include <pthread.h> |
173 | | |
174 | | extern userland_mutex_t atomic_mtx; |
175 | | |
176 | | #if defined(_WIN32) |
177 | | static inline void atomic_init() { |
178 | | InitializeCriticalSection(&atomic_mtx); |
179 | | } |
180 | | static inline void atomic_destroy() { |
181 | | DeleteCriticalSection(&atomic_mtx); |
182 | | } |
183 | | static inline void atomic_lock() { |
184 | | EnterCriticalSection(&atomic_mtx); |
185 | | } |
186 | | static inline void atomic_unlock() { |
187 | | LeaveCriticalSection(&atomic_mtx); |
188 | | } |
189 | | #else |
190 | | static inline void atomic_init() { |
191 | | pthread_mutexattr_t mutex_attr; |
192 | | |
193 | | pthread_mutexattr_init(&mutex_attr); |
194 | | #ifdef INVARIANTS |
195 | | pthread_mutexattr_settype(&mutex_attr, PTHREAD_MUTEX_ERRORCHECK); |
196 | | #endif |
197 | | pthread_mutex_init(&accept_mtx, &mutex_attr); |
198 | | pthread_mutexattr_destroy(&mutex_attr); |
199 | | } |
200 | | static inline void atomic_destroy() { |
201 | | (void)pthread_mutex_destroy(&atomic_mtx); |
202 | | } |
203 | | static inline void atomic_lock() { |
204 | | #ifdef INVARIANTS |
205 | | KASSERT(pthread_mutex_lock(&atomic_mtx) == 0, ("atomic_lock: atomic_mtx already locked")) |
206 | | #else |
207 | | (void)pthread_mutex_lock(&atomic_mtx); |
208 | | #endif |
209 | | } |
210 | | static inline void atomic_unlock() { |
211 | | #ifdef INVARIANTS |
212 | | KASSERT(pthread_mutex_unlock(&atomic_mtx) == 0, ("atomic_unlock: atomic_mtx not locked")) |
213 | | #else |
214 | | (void)pthread_mutex_unlock(&atomic_mtx); |
215 | | #endif |
216 | | } |
217 | | #endif |
218 | | /* |
219 | | * For userland, always use lock prefixes so that the binaries will run |
220 | | * on both SMP and !SMP systems. |
221 | | */ |
222 | | |
223 | | #define MPLOCKED "lock ; " |
224 | | |
225 | | /* |
226 | | * Atomically add the value of v to the integer pointed to by p and return |
227 | | * the previous value of *p. |
228 | | */ |
229 | | static __inline u_int |
230 | | atomic_fetchadd_int(volatile void *n, u_int v) |
231 | | { |
232 | | int *p = (int *) n; |
233 | | atomic_lock(); |
234 | | __asm __volatile( |
235 | | " " MPLOCKED " " |
236 | | " xaddl %0, %1 ; " |
237 | | "# atomic_fetchadd_int" |
238 | | : "+r" (v), /* 0 (result) */ |
239 | | "=m" (*p) /* 1 */ |
240 | | : "m" (*p)); /* 2 */ |
241 | | atomic_unlock(); |
242 | | |
243 | | return (v); |
244 | | } |
245 | | |
246 | | |
247 | | #ifdef CPU_DISABLE_CMPXCHG |
248 | | |
249 | | static __inline int |
250 | | atomic_cmpset_int(volatile u_int *dst, u_int exp, u_int src) |
251 | | { |
252 | | u_char res; |
253 | | |
254 | | atomic_lock(); |
255 | | __asm __volatile( |
256 | | " pushfl ; " |
257 | | " cli ; " |
258 | | " cmpl %3,%4 ; " |
259 | | " jne 1f ; " |
260 | | " movl %2,%1 ; " |
261 | | "1: " |
262 | | " sete %0 ; " |
263 | | " popfl ; " |
264 | | "# atomic_cmpset_int" |
265 | | : "=q" (res), /* 0 */ |
266 | | "=m" (*dst) /* 1 */ |
267 | | : "r" (src), /* 2 */ |
268 | | "r" (exp), /* 3 */ |
269 | | "m" (*dst) /* 4 */ |
270 | | : "memory"); |
271 | | atomic_unlock(); |
272 | | |
273 | | return (res); |
274 | | } |
275 | | |
276 | | #else /* !CPU_DISABLE_CMPXCHG */ |
277 | | |
278 | | static __inline int |
279 | | atomic_cmpset_int(volatile u_int *dst, u_int exp, u_int src) |
280 | | { |
281 | | atomic_lock(); |
282 | | u_char res; |
283 | | |
284 | | __asm __volatile( |
285 | | " " MPLOCKED " " |
286 | | " cmpxchgl %2,%1 ; " |
287 | | " sete %0 ; " |
288 | | "1: " |
289 | | "# atomic_cmpset_int" |
290 | | : "=a" (res), /* 0 */ |
291 | | "=m" (*dst) /* 1 */ |
292 | | : "r" (src), /* 2 */ |
293 | | "a" (exp), /* 3 */ |
294 | | "m" (*dst) /* 4 */ |
295 | | : "memory"); |
296 | | atomic_unlock(); |
297 | | |
298 | | return (res); |
299 | | } |
300 | | |
301 | | #endif /* CPU_DISABLE_CMPXCHG */ |
302 | | |
303 | | #define atomic_add_int(P, V) do { \ |
304 | | atomic_lock(); \ |
305 | | (*(u_int *)(P) += (V)); \ |
306 | | atomic_unlock(); \ |
307 | | } while(0) |
308 | | #define atomic_subtract_int(P, V) do { \ |
309 | | atomic_lock(); \ |
310 | | (*(u_int *)(P) -= (V)); \ |
311 | | atomic_unlock(); \ |
312 | | } while(0) |
313 | | |
314 | | #endif |
315 | | #endif |