6 * (the "License"). You may not use this file except in compliance
7 * with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22 /*
23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 * Copyright 2014 Nexenta Systems, Inc. All rights reserved.
26 */
27
28 #ifndef _ASM_ATOMIC_H
29 #define _ASM_ATOMIC_H
30
31 #include <sys/ccompile.h>
32 #include <sys/types.h>
33
34 #ifdef __cplusplus
35 extern "C" {
36 #endif
37
38 #if !defined(__lint) && defined(__GNUC__)
39
40 /* BEGIN CSTYLED */
41 /*
42 * This file contains a number of static inline functions implementing
43 * various atomic variable functions. Note that these are *not* all of the
44 * atomic_* functions as defined in usr/src/uts/common/sys/atomic.h. All
45 * possible atomic_* functions are implemented in usr/src/common/atomic in
65 #if defined(__amd64)
66 #define SUF_LONG SUF_64
67 #define SUF_PTR SUF_64
68 #define __ATOMIC_OP64(...) __ATOMIC_OPXX(__VA_ARGS__)
69 #elif defined(__i386)
70 #define SUF_LONG SUF_32
71 #define SUF_PTR SUF_32
72 #define __ATOMIC_OP64(...)
73 #else
74 #error "port me"
75 #endif
76
77 #if defined(__amd64) || defined(__i386)
78
79 #define __ATOMIC_OPXX(fxn, type, op) \
80 extern __GNU_INLINE void \
81 fxn(volatile type *target) \
82 { \
83 __asm__ __volatile__( \
84 "lock; " op " %0" \
85 : "+m" (*target)); \
86 }
87
88 __ATOMIC_OPXX(atomic_inc_8, uint8_t, "inc" SUF_8)
89 __ATOMIC_OPXX(atomic_inc_16, uint16_t, "inc" SUF_16)
90 __ATOMIC_OPXX(atomic_inc_32, uint32_t, "inc" SUF_32)
91 __ATOMIC_OP64(atomic_inc_64, uint64_t, "inc" SUF_64)
92 __ATOMIC_OPXX(atomic_inc_uchar, uchar_t, "inc" SUF_8)
93 __ATOMIC_OPXX(atomic_inc_ushort, ushort_t, "inc" SUF_16)
94 __ATOMIC_OPXX(atomic_inc_uint, uint_t, "inc" SUF_32)
95 __ATOMIC_OPXX(atomic_inc_ulong, ulong_t, "inc" SUF_LONG)
96
97 __ATOMIC_OPXX(atomic_dec_8, uint8_t, "dec" SUF_8)
98 __ATOMIC_OPXX(atomic_dec_16, uint16_t, "dec" SUF_16)
99 __ATOMIC_OPXX(atomic_dec_32, uint32_t, "dec" SUF_32)
100 __ATOMIC_OP64(atomic_dec_64, uint64_t, "dec" SUF_64)
101 __ATOMIC_OPXX(atomic_dec_uchar, uchar_t, "dec" SUF_8)
102 __ATOMIC_OPXX(atomic_dec_ushort, ushort_t, "dec" SUF_16)
103 __ATOMIC_OPXX(atomic_dec_uint, uint_t, "dec" SUF_32)
104 __ATOMIC_OPXX(atomic_dec_ulong, ulong_t, "dec" SUF_LONG)
105
106 #undef __ATOMIC_OPXX
107
108 #define __ATOMIC_OPXX(fxn, type1, type2, op, reg) \
109 extern __GNU_INLINE void \
110 fxn(volatile type1 *target, type2 delta) \
111 { \
112 __asm__ __volatile__( \
113 "lock; " op " %1,%0" \
114 : "+m" (*target) \
115 : "i" reg (delta)); \
116 }
117
118 __ATOMIC_OPXX(atomic_add_8, uint8_t, int8_t, "add" SUF_8, "q")
119 __ATOMIC_OPXX(atomic_add_16, uint16_t, int16_t, "add" SUF_16, "r")
120 __ATOMIC_OPXX(atomic_add_32, uint32_t, int32_t, "add" SUF_32, "r")
121 __ATOMIC_OP64(atomic_add_64, uint64_t, int64_t, "add" SUF_64, "r")
122 __ATOMIC_OPXX(atomic_add_char, uchar_t, signed char, "add" SUF_8, "q")
123 __ATOMIC_OPXX(atomic_add_short, ushort_t, short, "add" SUF_16, "r")
124 __ATOMIC_OPXX(atomic_add_int, uint_t, int, "add" SUF_32, "r")
125 __ATOMIC_OPXX(atomic_add_long, ulong_t, long, "add" SUF_LONG, "r")
126
127 /*
128 * We don't use the above macro here because atomic_add_ptr has an
129 * inconsistent type. The first argument should really be a 'volatile void
130 * **'.
131 */
132 extern __GNU_INLINE void
133 atomic_add_ptr(volatile void *target, ssize_t delta)
134 {
135 volatile void **tmp = (volatile void **)target;
136
137 __asm__ __volatile__(
138 "lock; add" SUF_PTR " %1,%0"
139 : "+m" (*tmp)
140 : "ir" (delta));
141 }
142
143 __ATOMIC_OPXX(atomic_or_8, uint8_t, uint8_t, "or" SUF_8, "q")
144 __ATOMIC_OPXX(atomic_or_16, uint16_t, uint16_t, "or" SUF_16, "r")
145 __ATOMIC_OPXX(atomic_or_32, uint32_t, uint32_t, "or" SUF_32, "r")
146 __ATOMIC_OP64(atomic_or_64, uint64_t, uint64_t, "or" SUF_64, "r")
147 __ATOMIC_OPXX(atomic_or_uchar, uchar_t, uchar_t, "or" SUF_8, "q")
148 __ATOMIC_OPXX(atomic_or_ushort, ushort_t, ushort_t, "or" SUF_16, "r")
149 __ATOMIC_OPXX(atomic_or_uint, uint_t, uint_t, "or" SUF_32, "r")
150 __ATOMIC_OPXX(atomic_or_ulong, ulong_t, ulong_t, "or" SUF_LONG, "r")
151
152 __ATOMIC_OPXX(atomic_and_8, uint8_t, uint8_t, "and" SUF_8, "q")
153 __ATOMIC_OPXX(atomic_and_16, uint16_t, uint16_t, "and" SUF_16, "r")
154 __ATOMIC_OPXX(atomic_and_32, uint32_t, uint32_t, "and" SUF_32, "r")
155 __ATOMIC_OP64(atomic_and_64, uint64_t, uint64_t, "and" SUF_64, "r")
156 __ATOMIC_OPXX(atomic_and_uchar, uchar_t, uchar_t, "and" SUF_8, "q")
157 __ATOMIC_OPXX(atomic_and_ushort, ushort_t, ushort_t, "and" SUF_16, "r")
158 __ATOMIC_OPXX(atomic_and_uint, uint_t, uint_t, "and" SUF_32, "r")
159 __ATOMIC_OPXX(atomic_and_ulong, ulong_t, ulong_t, "and" SUF_LONG, "r")
160
|
6 * (the "License"). You may not use this file except in compliance
7 * with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22 /*
23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 * Copyright 2014 Nexenta Systems, Inc. All rights reserved.
26 * Copyright 2015 Josef 'Jeff' Sipek <jeffpc@josefsipek.net>
27 */
28
29 #ifndef _ASM_ATOMIC_H
30 #define _ASM_ATOMIC_H
31
32 #include <sys/ccompile.h>
33 #include <sys/types.h>
34
35 #ifdef __cplusplus
36 extern "C" {
37 #endif
38
39 #if !defined(__lint) && defined(__GNUC__)
40
41 /* BEGIN CSTYLED */
42 /*
43 * This file contains a number of static inline functions implementing
44 * various atomic variable functions. Note that these are *not* all of the
45 * atomic_* functions as defined in usr/src/uts/common/sys/atomic.h. All
46 * possible atomic_* functions are implemented in usr/src/common/atomic in
66 #if defined(__amd64)
67 #define SUF_LONG SUF_64
68 #define SUF_PTR SUF_64
69 #define __ATOMIC_OP64(...) __ATOMIC_OPXX(__VA_ARGS__)
70 #elif defined(__i386)
71 #define SUF_LONG SUF_32
72 #define SUF_PTR SUF_32
73 #define __ATOMIC_OP64(...)
74 #else
75 #error "port me"
76 #endif
77
78 #if defined(__amd64) || defined(__i386)
79
80 #define __ATOMIC_OPXX(fxn, type, op) \
81 extern __GNU_INLINE void \
82 fxn(volatile type *target) \
83 { \
84 __asm__ __volatile__( \
85 "lock; " op " %0" \
86 : "+m" (*target) \
87 : /* no inputs */ \
88 : "cc"); \
89 }
90
91 __ATOMIC_OPXX(atomic_inc_8, uint8_t, "inc" SUF_8)
92 __ATOMIC_OPXX(atomic_inc_16, uint16_t, "inc" SUF_16)
93 __ATOMIC_OPXX(atomic_inc_32, uint32_t, "inc" SUF_32)
94 __ATOMIC_OP64(atomic_inc_64, uint64_t, "inc" SUF_64)
95 __ATOMIC_OPXX(atomic_inc_uchar, uchar_t, "inc" SUF_8)
96 __ATOMIC_OPXX(atomic_inc_ushort, ushort_t, "inc" SUF_16)
97 __ATOMIC_OPXX(atomic_inc_uint, uint_t, "inc" SUF_32)
98 __ATOMIC_OPXX(atomic_inc_ulong, ulong_t, "inc" SUF_LONG)
99
100 __ATOMIC_OPXX(atomic_dec_8, uint8_t, "dec" SUF_8)
101 __ATOMIC_OPXX(atomic_dec_16, uint16_t, "dec" SUF_16)
102 __ATOMIC_OPXX(atomic_dec_32, uint32_t, "dec" SUF_32)
103 __ATOMIC_OP64(atomic_dec_64, uint64_t, "dec" SUF_64)
104 __ATOMIC_OPXX(atomic_dec_uchar, uchar_t, "dec" SUF_8)
105 __ATOMIC_OPXX(atomic_dec_ushort, ushort_t, "dec" SUF_16)
106 __ATOMIC_OPXX(atomic_dec_uint, uint_t, "dec" SUF_32)
107 __ATOMIC_OPXX(atomic_dec_ulong, ulong_t, "dec" SUF_LONG)
108
109 #undef __ATOMIC_OPXX
110
111 #define __ATOMIC_OPXX(fxn, type1, type2, op, reg) \
112 extern __GNU_INLINE void \
113 fxn(volatile type1 *target, type2 delta) \
114 { \
115 __asm__ __volatile__( \
116 "lock; " op " %1,%0" \
117 : "+m" (*target) \
118 : "i" reg (delta) \
119 : "cc"); \
120 }
121
122 __ATOMIC_OPXX(atomic_add_8, uint8_t, int8_t, "add" SUF_8, "q")
123 __ATOMIC_OPXX(atomic_add_16, uint16_t, int16_t, "add" SUF_16, "r")
124 __ATOMIC_OPXX(atomic_add_32, uint32_t, int32_t, "add" SUF_32, "r")
125 __ATOMIC_OP64(atomic_add_64, uint64_t, int64_t, "add" SUF_64, "r")
126 __ATOMIC_OPXX(atomic_add_char, uchar_t, signed char, "add" SUF_8, "q")
127 __ATOMIC_OPXX(atomic_add_short, ushort_t, short, "add" SUF_16, "r")
128 __ATOMIC_OPXX(atomic_add_int, uint_t, int, "add" SUF_32, "r")
129 __ATOMIC_OPXX(atomic_add_long, ulong_t, long, "add" SUF_LONG, "r")
130
131 /*
132 * We don't use the above macro here because atomic_add_ptr has an
133 * inconsistent type. The first argument should really be a 'volatile void
134 * **'.
135 */
136 extern __GNU_INLINE void
137 atomic_add_ptr(volatile void *target, ssize_t delta)
138 {
139 volatile void **tmp = (volatile void **)target;
140
141 __asm__ __volatile__(
142 "lock; add" SUF_PTR " %1,%0"
143 : "+m" (*tmp)
144 : "ir" (delta)
145 : "cc");
146 }
147
148 __ATOMIC_OPXX(atomic_or_8, uint8_t, uint8_t, "or" SUF_8, "q")
149 __ATOMIC_OPXX(atomic_or_16, uint16_t, uint16_t, "or" SUF_16, "r")
150 __ATOMIC_OPXX(atomic_or_32, uint32_t, uint32_t, "or" SUF_32, "r")
151 __ATOMIC_OP64(atomic_or_64, uint64_t, uint64_t, "or" SUF_64, "r")
152 __ATOMIC_OPXX(atomic_or_uchar, uchar_t, uchar_t, "or" SUF_8, "q")
153 __ATOMIC_OPXX(atomic_or_ushort, ushort_t, ushort_t, "or" SUF_16, "r")
154 __ATOMIC_OPXX(atomic_or_uint, uint_t, uint_t, "or" SUF_32, "r")
155 __ATOMIC_OPXX(atomic_or_ulong, ulong_t, ulong_t, "or" SUF_LONG, "r")
156
157 __ATOMIC_OPXX(atomic_and_8, uint8_t, uint8_t, "and" SUF_8, "q")
158 __ATOMIC_OPXX(atomic_and_16, uint16_t, uint16_t, "and" SUF_16, "r")
159 __ATOMIC_OPXX(atomic_and_32, uint32_t, uint32_t, "and" SUF_32, "r")
160 __ATOMIC_OP64(atomic_and_64, uint64_t, uint64_t, "and" SUF_64, "r")
161 __ATOMIC_OPXX(atomic_and_uchar, uchar_t, uchar_t, "and" SUF_8, "q")
162 __ATOMIC_OPXX(atomic_and_ushort, ushort_t, ushort_t, "and" SUF_16, "r")
163 __ATOMIC_OPXX(atomic_and_uint, uint_t, uint_t, "and" SUF_32, "r")
164 __ATOMIC_OPXX(atomic_and_ulong, ulong_t, ulong_t, "and" SUF_LONG, "r")
165
|