Print this page
5044 define static inlines for most often used atomic functions
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/intel/asm/atomic.h
+++ new/usr/src/uts/intel/asm/atomic.h
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License, Version 1.0 only
6 6 * (the "License"). You may not use this file except in compliance
7 7 * with the License.
8 8 *
9 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 10 * or http://www.opensolaris.org/os/licensing.
11 11 * See the License for the specific language governing permissions
12 12 * and limitations under the License.
13 13 *
14 14 * When distributing Covered Code, include this CDDL HEADER in each
15 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 16 * If applicable, add the following below this CDDL HEADER, with the
17 17 * fields enclosed by brackets "[]" replaced with your own identifying
18 18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 19 *
20 20 * CDDL HEADER END
21 21 */
22 22 /*
23 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
24 24 * Use is subject to license terms.
25 25 */
26 26
27 27 #ifndef _ASM_ATOMIC_H
28 28 #define _ASM_ATOMIC_H
|
↓ open down ↓ |
28 lines elided |
↑ open up ↑ |
29 29
30 30 #include <sys/ccompile.h>
31 31 #include <sys/types.h>
32 32
33 33 #ifdef __cplusplus
34 34 extern "C" {
35 35 #endif
36 36
37 37 #if !defined(__lint) && defined(__GNUC__)
38 38
39 -#if defined(__amd64)
39 +/*
40 + * This file contains a number of static inline functions implementing
41 + * various atomic variable functions. Note that these are *not* all of the
42 + * atomic_* functions as defined in usr/src/uts/common/sys/atomic.h. All
43 + * possible atomic_* functions are implemented in usr/src/common/atomic in
44 + * pure assembly. In the absence of an identically named function in this
45 + * header file, any use of the function will result in the compiler emitting
46 + * a function call as usual. On the other hand, if an identically named
47 + * function exists in this header as a static inline, the compiler will
48 + * inline its contents and the linker never sees the symbol reference. We
49 + * use this to avoid implementing some of the more complex and less used
50 + * functions and instead falling back to function calls. Note that in some
51 + * cases (e.g., atomic_inc_64) we implement a static inline only on AMD64
52 + * but not i386.
53 + */
54 +
55 +/*
56 + * Instruction suffixes for various operand sizes (assuming AMD64)
57 + */
58 +#define SUF_8 "b"
59 +#define SUF_16 "w"
60 +#define SUF_32 "l"
61 +#define SUF_64 "q"
40 62
63 +#if defined(__amd64)
64 +#define SUF_LONG SUF_64
65 +#define SUF_PTR SUF_64
66 +#define __ATOMIC_OP64(...) __ATOMIC_OPXX(__VA_ARGS__)
41 67 #elif defined(__i386)
68 +#define SUF_LONG SUF_32
69 +#define SUF_PTR SUF_32
70 +#define __ATOMIC_OP64(...)
71 +#else
72 +#error "port me"
73 +#endif
74 +
75 +#if defined(__amd64) || defined(__i386)
76 +
77 +#define __ATOMIC_OPXX(fxn, type, op) \
78 +extern __GNU_INLINE void \
79 +fxn(volatile type *target) \
80 +{ \
81 + __asm__ __volatile__( \
82 + "lock; " op " %0" \
83 + : "+m" (*target)); \
84 +}
85 +
86 +__ATOMIC_OPXX(atomic_inc_8, uint8_t, "inc" SUF_8)
87 +__ATOMIC_OPXX(atomic_inc_16, uint16_t, "inc" SUF_16)
88 +__ATOMIC_OPXX(atomic_inc_32, uint32_t, "inc" SUF_32)
89 +__ATOMIC_OP64(atomic_inc_64, uint64_t, "inc" SUF_64)
90 +__ATOMIC_OPXX(atomic_inc_uchar, uchar_t, "inc" SUF_8)
91 +__ATOMIC_OPXX(atomic_inc_ushort, ushort_t, "inc" SUF_16)
92 +__ATOMIC_OPXX(atomic_inc_uint, uint_t, "inc" SUF_32)
93 +__ATOMIC_OPXX(atomic_inc_ulong, ulong_t, "inc" SUF_LONG)
94 +
95 +__ATOMIC_OPXX(atomic_dec_8, uint8_t, "dec" SUF_8)
96 +__ATOMIC_OPXX(atomic_dec_16, uint16_t, "dec" SUF_16)
97 +__ATOMIC_OPXX(atomic_dec_32, uint32_t, "dec" SUF_32)
98 +__ATOMIC_OP64(atomic_dec_64, uint64_t, "dec" SUF_64)
99 +__ATOMIC_OPXX(atomic_dec_uchar, uchar_t, "dec" SUF_8)
100 +__ATOMIC_OPXX(atomic_dec_ushort, ushort_t, "dec" SUF_16)
101 +__ATOMIC_OPXX(atomic_dec_uint, uint_t, "dec" SUF_32)
102 +__ATOMIC_OPXX(atomic_dec_ulong, ulong_t, "dec" SUF_LONG)
103 +
104 +#undef __ATOMIC_OPXX
105 +
106 +#define __ATOMIC_OPXX(fxn, type1, type2, op) \
107 +extern __GNU_INLINE void \
108 +fxn(volatile type1 *target, type2 delta) \
109 +{ \
110 + __asm__ __volatile__( \
111 + "lock; " op " %1,%0" \
112 + : "+m" (*target) \
113 + : "ir" (delta)); \
114 +}
115 +
116 +__ATOMIC_OPXX(atomic_add_8, uint8_t, int8_t, "add" SUF_8)
117 +__ATOMIC_OPXX(atomic_add_16, uint16_t, int16_t, "add" SUF_16)
118 +__ATOMIC_OPXX(atomic_add_32, uint32_t, int32_t, "add" SUF_32)
119 +__ATOMIC_OP64(atomic_add_64, uint64_t, int64_t, "add" SUF_64)
120 +__ATOMIC_OPXX(atomic_add_char, uchar_t, signed char, "add" SUF_8)
121 +__ATOMIC_OPXX(atomic_add_short, ushort_t, short, "add" SUF_16)
122 +__ATOMIC_OPXX(atomic_add_int, uint_t, int, "add" SUF_32)
123 +__ATOMIC_OPXX(atomic_add_long, ulong_t, long, "add" SUF_LONG)
124 +
125 +/*
126 + * We don't use the above macro here because atomic_add_ptr has an
127 + * inconsistent type. The first argument should really be a 'volatile void
128 + * **'.
129 + */
130 +extern __GNU_INLINE void
131 +atomic_add_ptr(volatile void *target, ssize_t delta)
132 +{
133 + volatile void **tmp = (volatile void **)target;
134 +
135 + __asm__ __volatile__(
136 + "lock; add" SUF_PTR " %1,%0"
137 + : "+m" (*tmp)
138 + : "ir" (delta));
139 +}
140 +
141 +__ATOMIC_OPXX(atomic_or_8, uint8_t, uint8_t, "or" SUF_8)
142 +__ATOMIC_OPXX(atomic_or_16, uint16_t, uint16_t, "or" SUF_16)
143 +__ATOMIC_OPXX(atomic_or_32, uint32_t, uint32_t, "or" SUF_32)
144 +__ATOMIC_OP64(atomic_or_64, uint64_t, uint64_t, "or" SUF_64)
145 +__ATOMIC_OPXX(atomic_or_uchar, uchar_t, uchar_t, "or" SUF_8)
146 +__ATOMIC_OPXX(atomic_or_ushort, ushort_t, ushort_t, "or" SUF_16)
147 +__ATOMIC_OPXX(atomic_or_uint, uint_t, uint_t, "or" SUF_32)
148 +__ATOMIC_OPXX(atomic_or_ulong, ulong_t, ulong_t, "or" SUF_LONG)
149 +
150 +__ATOMIC_OPXX(atomic_and_8, uint8_t, uint8_t, "and" SUF_8)
151 +__ATOMIC_OPXX(atomic_and_16, uint16_t, uint16_t, "and" SUF_16)
152 +__ATOMIC_OPXX(atomic_and_32, uint32_t, uint32_t, "and" SUF_32)
153 +__ATOMIC_OP64(atomic_and_64, uint64_t, uint64_t, "and" SUF_64)
154 +__ATOMIC_OPXX(atomic_and_uchar, uchar_t, uchar_t, "and" SUF_8)
155 +__ATOMIC_OPXX(atomic_and_ushort, ushort_t, ushort_t, "and" SUF_16)
156 +__ATOMIC_OPXX(atomic_and_uint, uint_t, uint_t, "and" SUF_32)
157 +__ATOMIC_OPXX(atomic_and_ulong, ulong_t, ulong_t, "and" SUF_LONG)
158 +
159 +#undef __ATOMIC_OPXX
160 +
161 +#define __ATOMIC_OPXX(fxn, type, op, reg) \
162 +extern __GNU_INLINE type \
163 +fxn(volatile type *target, type cmp, type new) \
164 +{ \
165 + type ret; \
166 + __asm__ __volatile__( \
167 + "lock; " op " %2,%0" \
168 + : "+m" (*target), "=a" (ret) \
169 + : reg (new), "1" (cmp) \
170 + : "cc"); \
171 + return (ret); \
172 +}
173 +
174 +__ATOMIC_OPXX(atomic_cas_8, uint8_t, "cmpxchg" SUF_8, "q")
175 +__ATOMIC_OPXX(atomic_cas_16, uint16_t, "cmpxchg" SUF_16, "r")
176 +__ATOMIC_OPXX(atomic_cas_32, uint32_t, "cmpxchg" SUF_32, "r")
177 +__ATOMIC_OP64(atomic_cas_64, uint64_t, "cmpxchg" SUF_64, "r")
178 +__ATOMIC_OPXX(atomic_cas_uchar, uchar_t, "cmpxchg" SUF_8, "q")
179 +__ATOMIC_OPXX(atomic_cas_ushort, ushort_t, "cmpxchg" SUF_16, "r")
180 +__ATOMIC_OPXX(atomic_cas_uint, uint_t, "cmpxchg" SUF_32, "r")
181 +__ATOMIC_OPXX(atomic_cas_ulong, ulong_t, "cmpxchg" SUF_LONG, "r")
182 +
183 +#undef __ATOMIC_OPXX
184 +
185 +/*
186 + * We don't use the above macro here because atomic_cas_ptr has an
187 + * inconsistent type. The first argument should really be a 'volatile void
188 + * **'.
189 + */
190 +extern __GNU_INLINE void *
191 +atomic_cas_ptr(volatile void *target, void *cmp, void *new)
192 +{
193 + volatile void **tmp = (volatile void **)target;
194 + void *ret;
195 +
196 + __asm__ __volatile__(
197 + "lock; cmpxchg" SUF_PTR " %2,%0"
198 + : "+m" (*tmp), "=a" (ret)
199 + : "r" (new), "1" (cmp)
200 + : "cc");
201 +
202 + return (ret);
203 +}
204 +
205 +#define __ATOMIC_OPXX(fxn, type, op, reg) \
206 +extern __GNU_INLINE type \
207 +fxn(volatile type *target, type val) \
208 +{ \
209 + __asm__ __volatile__( \
210 + op " %1,%0" \
211 + : "+m" (*target), "+" reg (val)); \
212 + return (val); \
213 +}
214 +
215 +__ATOMIC_OPXX(atomic_swap_8, uint8_t, "xchg" SUF_8, "q")
216 +__ATOMIC_OPXX(atomic_swap_16, uint16_t, "xchg" SUF_16, "r")
217 +__ATOMIC_OPXX(atomic_swap_32, uint32_t, "xchg" SUF_32, "r")
218 +__ATOMIC_OP64(atomic_swap_64, uint64_t, "xchg" SUF_64, "r")
219 +__ATOMIC_OPXX(atomic_swap_uchar, uchar_t, "xchg" SUF_8, "q")
220 +__ATOMIC_OPXX(atomic_swap_ushort, ushort_t, "xchg" SUF_16, "r")
221 +__ATOMIC_OPXX(atomic_swap_uint, uint_t, "xchg" SUF_32, "r")
222 +__ATOMIC_OPXX(atomic_swap_ulong, ulong_t, "xchg" SUF_LONG, "r")
223 +
224 +#undef __ATOMIC_OPXX
225 +
226 +/*
227 + * We don't use the above macro here because atomic_swap_ptr has an
228 + * inconsistent type. The first argument should really be a 'volatile void
229 + * **'.
230 + */
231 +extern __GNU_INLINE void *
232 +atomic_swap_ptr(volatile void *target, void *val)
233 +{
234 + volatile void **tmp = (volatile void **)target;
235 +
236 + __asm__ __volatile__(
237 + "xchg" SUF_PTR " %1,%0"
238 + : "+m" (*tmp), "+r" (val));
239 +
240 + return (val);
241 +}
42 242
43 243 #else
44 244 #error "port me"
45 245 #endif
46 246
247 +#undef SUF_8
248 +#undef SUF_16
249 +#undef SUF_32
250 +#undef SUF_64
251 +#undef SUF_LONG
252 +#undef SUF_PTR
253 +
254 +#undef __ATOMIC_OP64
255 +
47 256 #endif /* !__lint && __GNUC__ */
48 257
49 258 #ifdef __cplusplus
50 259 }
51 260 #endif
52 261
53 262 #endif /* _ASM_ATOMIC_H */
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX