1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26 
  27 #ifndef _SYS_ATOMIC_H
  28 #define _SYS_ATOMIC_H
  29 
  30 #include <sys/types.h>
  31 #include <sys/inttypes.h>
  32 
  33 #ifdef  __cplusplus
  34 extern "C" {
  35 #endif
  36 
  37 #if defined(_KERNEL) && defined(__GNUC__) && defined(_ASM_INLINES) && \
  38         (defined(__i386) || defined(__amd64))
  39 #include <asm/atomic.h>
  40 #endif
  41 
  42 #if defined(_KERNEL) || defined(__STDC__)
  43 /*
  44  * Increment target.
  45  */
  46 extern void atomic_inc_8(volatile uint8_t *);
  47 extern void atomic_inc_uchar(volatile uchar_t *);
  48 extern void atomic_inc_16(volatile uint16_t *);
  49 extern void atomic_inc_ushort(volatile ushort_t *);
  50 extern void atomic_inc_32(volatile uint32_t *);
  51 extern void atomic_inc_uint(volatile uint_t *);
  52 extern void atomic_inc_ulong(volatile ulong_t *);
  53 #if defined(_KERNEL) || defined(_INT64_TYPE)
  54 extern void atomic_inc_64(volatile uint64_t *);
  55 #endif
  56 
  57 /*
  58  * Decrement target
  59  */
  60 extern void atomic_dec_8(volatile uint8_t *);
  61 extern void atomic_dec_uchar(volatile uchar_t *);
  62 extern void atomic_dec_16(volatile uint16_t *);
  63 extern void atomic_dec_ushort(volatile ushort_t *);
  64 extern void atomic_dec_32(volatile uint32_t *);
  65 extern void atomic_dec_uint(volatile uint_t *);
  66 extern void atomic_dec_ulong(volatile ulong_t *);
  67 #if defined(_KERNEL) || defined(_INT64_TYPE)
  68 extern void atomic_dec_64(volatile uint64_t *);
  69 #endif
  70 
  71 /*
  72  * Add delta to target
  73  */
  74 extern void atomic_add_8(volatile uint8_t *, int8_t);
  75 extern void atomic_add_char(volatile uchar_t *, signed char);
  76 extern void atomic_add_16(volatile uint16_t *, int16_t);
  77 extern void atomic_add_short(volatile ushort_t *, short);
  78 extern void atomic_add_32(volatile uint32_t *, int32_t);
  79 extern void atomic_add_int(volatile uint_t *, int);
  80 extern void atomic_add_ptr(volatile void *, ssize_t);
  81 extern void atomic_add_long(volatile ulong_t *, long);
  82 #if defined(_KERNEL) || defined(_INT64_TYPE)
  83 extern void atomic_add_64(volatile uint64_t *, int64_t);
  84 #endif
  85 
  86 /*
  87  * logical OR bits with target
  88  */
  89 extern void atomic_or_8(volatile uint8_t *, uint8_t);
  90 extern void atomic_or_uchar(volatile uchar_t *, uchar_t);
  91 extern void atomic_or_16(volatile uint16_t *, uint16_t);
  92 extern void atomic_or_ushort(volatile ushort_t *, ushort_t);
  93 extern void atomic_or_32(volatile uint32_t *, uint32_t);
  94 extern void atomic_or_uint(volatile uint_t *, uint_t);
  95 extern void atomic_or_ulong(volatile ulong_t *, ulong_t);
  96 #if defined(_KERNEL) || defined(_INT64_TYPE)
  97 extern void atomic_or_64(volatile uint64_t *, uint64_t);
  98 #endif
  99 
 100 /*
 101  * logical AND bits with target
 102  */
 103 extern void atomic_and_8(volatile uint8_t *, uint8_t);
 104 extern void atomic_and_uchar(volatile uchar_t *, uchar_t);
 105 extern void atomic_and_16(volatile uint16_t *, uint16_t);
 106 extern void atomic_and_ushort(volatile ushort_t *, ushort_t);
 107 extern void atomic_and_32(volatile uint32_t *, uint32_t);
 108 extern void atomic_and_uint(volatile uint_t *, uint_t);
 109 extern void atomic_and_ulong(volatile ulong_t *, ulong_t);
 110 #if defined(_KERNEL) || defined(_INT64_TYPE)
 111 extern void atomic_and_64(volatile uint64_t *, uint64_t);
 112 #endif
 113 
 114 /*
 115  * As above, but return the new value.  Note that these _nv() variants are
 116  * substantially more expensive on some platforms than the no-return-value
 117  * versions above, so don't use them unless you really need to know the
 118  * new value *atomically* (e.g. when decrementing a reference count and
 119  * checking whether it went to zero).
 120  */
 121 
 122 /*
 123  * Increment target and return new value.
 124  */
 125 extern uint8_t atomic_inc_8_nv(volatile uint8_t *);
 126 extern uchar_t atomic_inc_uchar_nv(volatile uchar_t *);
 127 extern uint16_t atomic_inc_16_nv(volatile uint16_t *);
 128 extern ushort_t atomic_inc_ushort_nv(volatile ushort_t *);
 129 extern uint32_t atomic_inc_32_nv(volatile uint32_t *);
 130 extern uint_t atomic_inc_uint_nv(volatile uint_t *);
 131 extern ulong_t atomic_inc_ulong_nv(volatile ulong_t *);
 132 #if defined(_KERNEL) || defined(_INT64_TYPE)
 133 extern uint64_t atomic_inc_64_nv(volatile uint64_t *);
 134 #endif
 135 
 136 /*
 137  * Decrement target and return new value.
 138  */
 139 extern uint8_t atomic_dec_8_nv(volatile uint8_t *);
 140 extern uchar_t atomic_dec_uchar_nv(volatile uchar_t *);
 141 extern uint16_t atomic_dec_16_nv(volatile uint16_t *);
 142 extern ushort_t atomic_dec_ushort_nv(volatile ushort_t *);
 143 extern uint32_t atomic_dec_32_nv(volatile uint32_t *);
 144 extern uint_t atomic_dec_uint_nv(volatile uint_t *);
 145 extern ulong_t atomic_dec_ulong_nv(volatile ulong_t *);
 146 #if defined(_KERNEL) || defined(_INT64_TYPE)
 147 extern uint64_t atomic_dec_64_nv(volatile uint64_t *);
 148 #endif
 149 
 150 /*
 151  * Add delta to target
 152  */
 153 extern uint8_t atomic_add_8_nv(volatile uint8_t *, int8_t);
 154 extern uchar_t atomic_add_char_nv(volatile uchar_t *, signed char);
 155 extern uint16_t atomic_add_16_nv(volatile uint16_t *, int16_t);
 156 extern ushort_t atomic_add_short_nv(volatile ushort_t *, short);
 157 extern uint32_t atomic_add_32_nv(volatile uint32_t *, int32_t);
 158 extern uint_t atomic_add_int_nv(volatile uint_t *, int);
 159 extern void *atomic_add_ptr_nv(volatile void *, ssize_t);
 160 extern ulong_t atomic_add_long_nv(volatile ulong_t *, long);
 161 #if defined(_KERNEL) || defined(_INT64_TYPE)
 162 extern uint64_t atomic_add_64_nv(volatile uint64_t *, int64_t);
 163 #endif
 164 
 165 /*
 166  * logical OR bits with target and return new value.
 167  */
 168 extern uint8_t atomic_or_8_nv(volatile uint8_t *, uint8_t);
 169 extern uchar_t atomic_or_uchar_nv(volatile uchar_t *, uchar_t);
 170 extern uint16_t atomic_or_16_nv(volatile uint16_t *, uint16_t);
 171 extern ushort_t atomic_or_ushort_nv(volatile ushort_t *, ushort_t);
 172 extern uint32_t atomic_or_32_nv(volatile uint32_t *, uint32_t);
 173 extern uint_t atomic_or_uint_nv(volatile uint_t *, uint_t);
 174 extern ulong_t atomic_or_ulong_nv(volatile ulong_t *, ulong_t);
 175 #if defined(_KERNEL) || defined(_INT64_TYPE)
 176 extern uint64_t atomic_or_64_nv(volatile uint64_t *, uint64_t);
 177 #endif
 178 
 179 /*
 180  * logical AND bits with target and return new value.
 181  */
 182 extern uint8_t atomic_and_8_nv(volatile uint8_t *, uint8_t);
 183 extern uchar_t atomic_and_uchar_nv(volatile uchar_t *, uchar_t);
 184 extern uint16_t atomic_and_16_nv(volatile uint16_t *, uint16_t);
 185 extern ushort_t atomic_and_ushort_nv(volatile ushort_t *, ushort_t);
 186 extern uint32_t atomic_and_32_nv(volatile uint32_t *, uint32_t);
 187 extern uint_t atomic_and_uint_nv(volatile uint_t *, uint_t);
 188 extern ulong_t atomic_and_ulong_nv(volatile ulong_t *, ulong_t);
 189 #if defined(_KERNEL) || defined(_INT64_TYPE)
 190 extern uint64_t atomic_and_64_nv(volatile uint64_t *, uint64_t);
 191 #endif
 192 
 193 /*
 194  * If *arg1 == arg2, set *arg1 = arg3; return old value
 195  */
 196 extern uint8_t atomic_cas_8(volatile uint8_t *, uint8_t, uint8_t);
 197 extern uchar_t atomic_cas_uchar(volatile uchar_t *, uchar_t, uchar_t);
 198 extern uint16_t atomic_cas_16(volatile uint16_t *, uint16_t, uint16_t);
 199 extern ushort_t atomic_cas_ushort(volatile ushort_t *, ushort_t, ushort_t);
 200 extern uint32_t atomic_cas_32(volatile uint32_t *, uint32_t, uint32_t);
 201 extern uint_t atomic_cas_uint(volatile uint_t *, uint_t, uint_t);
 202 extern void *atomic_cas_ptr(volatile void *, void *, void *);
 203 extern ulong_t atomic_cas_ulong(volatile ulong_t *, ulong_t, ulong_t);
 204 #if defined(_KERNEL) || defined(_INT64_TYPE)
 205 extern uint64_t atomic_cas_64(volatile uint64_t *, uint64_t, uint64_t);
 206 #endif
 207 
 208 /*
 209  * Swap target and return old value
 210  */
 211 extern uint8_t atomic_swap_8(volatile uint8_t *, uint8_t);
 212 extern uchar_t atomic_swap_uchar(volatile uchar_t *, uchar_t);
 213 extern uint16_t atomic_swap_16(volatile uint16_t *, uint16_t);
 214 extern ushort_t atomic_swap_ushort(volatile ushort_t *, ushort_t);
 215 extern uint32_t atomic_swap_32(volatile uint32_t *, uint32_t);
 216 extern uint_t atomic_swap_uint(volatile uint_t *, uint_t);
 217 extern void *atomic_swap_ptr(volatile void *, void *);
 218 extern ulong_t atomic_swap_ulong(volatile ulong_t *, ulong_t);
 219 #if defined(_KERNEL) || defined(_INT64_TYPE)
 220 extern uint64_t atomic_swap_64(volatile uint64_t *, uint64_t);
 221 #endif
 222 
 223 /*
 224  * Perform an exclusive atomic bit set/clear on a target.
 225  * Returns 0 if bit was sucessfully set/cleared, or -1
 226  * if the bit was already set/cleared.
 227  */
 228 extern int atomic_set_long_excl(volatile ulong_t *, uint_t);
 229 extern int atomic_clear_long_excl(volatile ulong_t *, uint_t);
 230 
 231 /*
 232  * Generic memory barrier used during lock entry, placed after the
 233  * memory operation that acquires the lock to guarantee that the lock
 234  * protects its data.  No stores from after the memory barrier will
 235  * reach visibility, and no loads from after the barrier will be
 236  * resolved, before the lock acquisition reaches global visibility.
 237  */
 238 extern void membar_enter(void);
 239 
 240 /*
 241  * Generic memory barrier used during lock exit, placed before the
 242  * memory operation that releases the lock to guarantee that the lock
 243  * protects its data.  All loads and stores issued before the barrier
 244  * will be resolved before the subsequent lock update reaches visibility.
 245  */
 246 extern void membar_exit(void);
 247 
 248 /*
 249  * Arrange that all stores issued before this point in the code reach
 250  * global visibility before any stores that follow; useful in producer
 251  * modules that update a data item, then set a flag that it is available.
 252  * The memory barrier guarantees that the available flag is not visible
 253  * earlier than the updated data, i.e. it imposes store ordering.
 254  */
 255 extern void membar_producer(void);
 256 
 257 /*
 258  * Arrange that all loads issued before this point in the code are
 259  * completed before any subsequent loads; useful in consumer modules
 260  * that check to see if data is available and read the data.
 261  * The memory barrier guarantees that the data is not sampled until
 262  * after the available flag has been seen, i.e. it imposes load ordering.
 263  */
 264 extern void membar_consumer(void);
 265 #endif
 266 
 267 #if !defined(_KERNEL) && !defined(__STDC__)
 268 extern void atomic_inc_8();
 269 extern void atomic_inc_uchar();
 270 extern void atomic_inc_16();
 271 extern void atomic_inc_ushort();
 272 extern void atomic_inc_32();
 273 extern void atomic_inc_uint();
 274 extern void atomic_inc_ulong();
 275 #if defined(_INT64_TYPE)
 276 extern void atomic_inc_64();
 277 #endif /* defined(_INT64_TYPE) */
 278 extern void atomic_dec_8();
 279 extern void atomic_dec_uchar();
 280 extern void atomic_dec_16();
 281 extern void atomic_dec_ushort();
 282 extern void atomic_dec_32();
 283 extern void atomic_dec_uint();
 284 extern void atomic_dec_ulong();
 285 #if defined(_INT64_TYPE)
 286 extern void atomic_dec_64();
 287 #endif /* defined(_INT64_TYPE) */
 288 extern void atomic_add_8();
 289 extern void atomic_add_char();
 290 extern void atomic_add_16();
 291 extern void atomic_add_short();
 292 extern void atomic_add_32();
 293 extern void atomic_add_int();
 294 extern void atomic_add_ptr();
 295 extern void atomic_add_long();
 296 #if defined(_INT64_TYPE)
 297 extern void atomic_add_64();
 298 #endif /* defined(_INT64_TYPE) */
 299 extern void atomic_or_8();
 300 extern void atomic_or_uchar();
 301 extern void atomic_or_16();
 302 extern void atomic_or_ushort();
 303 extern void atomic_or_32();
 304 extern void atomic_or_uint();
 305 extern void atomic_or_ulong();
 306 #if defined(_INT64_TYPE)
 307 extern void atomic_or_64();
 308 #endif /* defined(_INT64_TYPE) */
 309 extern void atomic_and_8();
 310 extern void atomic_and_uchar();
 311 extern void atomic_and_16();
 312 extern void atomic_and_ushort();
 313 extern void atomic_and_32();
 314 extern void atomic_and_uint();
 315 extern void atomic_and_ulong();
 316 #if defined(_INT64_TYPE)
 317 extern void atomic_and_64();
 318 #endif /* defined(_INT64_TYPE) */
 319 extern uint8_t atomic_inc_8_nv();
 320 extern uchar_t atomic_inc_uchar_nv();
 321 extern uint16_t atomic_inc_16_nv();
 322 extern ushort_t atomic_inc_ushort_nv();
 323 extern uint32_t atomic_inc_32_nv();
 324 extern uint_t atomic_inc_uint_nv();
 325 extern ulong_t atomic_inc_ulong_nv();
 326 #if defined(_INT64_TYPE)
 327 extern uint64_t atomic_inc_64_nv();
 328 #endif /* defined(_INT64_TYPE) */
 329 extern uint8_t atomic_dec_8_nv();
 330 extern uchar_t atomic_dec_uchar_nv();
 331 extern uint16_t atomic_dec_16_nv();
 332 extern ushort_t atomic_dec_ushort_nv();
 333 extern uint32_t atomic_dec_32_nv();
 334 extern uint_t atomic_dec_uint_nv();
 335 extern ulong_t atomic_dec_ulong_nv();
 336 #if defined(_INT64_TYPE)
 337 extern uint64_t atomic_dec_64_nv();
 338 #endif /* defined(_INT64_TYPE) */
 339 extern uint8_t atomic_add_8_nv();
 340 extern uchar_t atomic_add_char_nv();
 341 extern uint16_t atomic_add_16_nv();
 342 extern ushort_t atomic_add_short_nv();
 343 extern uint32_t atomic_add_32_nv();
 344 extern uint_t atomic_add_int_nv();
 345 extern void *atomic_add_ptr_nv();
 346 extern ulong_t atomic_add_long_nv();
 347 #if defined(_INT64_TYPE)
 348 extern uint64_t atomic_add_64_nv();
 349 #endif /* defined(_INT64_TYPE) */
 350 extern uint8_t atomic_or_8_nv();
 351 extern uchar_t atomic_or_uchar_nv();
 352 extern uint16_t atomic_or_16_nv();
 353 extern ushort_t atomic_or_ushort_nv();
 354 extern uint32_t atomic_or_32_nv();
 355 extern uint_t atomic_or_uint_nv();
 356 extern ulong_t atomic_or_ulong_nv();
 357 #if defined(_INT64_TYPE)
 358 extern uint64_t atomic_or_64_nv();
 359 #endif /* defined(_INT64_TYPE) */
 360 extern uint8_t atomic_and_8_nv();
 361 extern uchar_t atomic_and_uchar_nv();
 362 extern uint16_t atomic_and_16_nv();
 363 extern ushort_t atomic_and_ushort_nv();
 364 extern uint32_t atomic_and_32_nv();
 365 extern uint_t atomic_and_uint_nv();
 366 extern ulong_t atomic_and_ulong_nv();
 367 #if defined(_INT64_TYPE)
 368 extern uint64_t atomic_and_64_nv();
 369 #endif /* defined(_INT64_TYPE) */
 370 extern uint8_t atomic_cas_8();
 371 extern uchar_t atomic_cas_uchar();
 372 extern uint16_t atomic_cas_16();
 373 extern ushort_t atomic_cas_ushort();
 374 extern uint32_t atomic_cas_32();
 375 extern uint_t atomic_cas_uint();
 376 extern void *atomic_cas_ptr();
 377 extern ulong_t atomic_cas_ulong();
 378 #if defined(_INT64_TYPE)
 379 extern uint64_t atomic_cas_64();
 380 #endif /* defined(_INT64_TYPE) */
 381 extern uint8_t atomic_swap_8();
 382 extern uchar_t atomic_swap_uchar();
 383 extern uint16_t atomic_swap_16();
 384 extern ushort_t atomic_swap_ushort();
 385 extern uint32_t atomic_swap_32();
 386 extern uint_t atomic_swap_uint();
 387 extern void *atomic_swap_ptr();
 388 extern ulong_t atomic_swap_ulong();
 389 #if defined(_INT64_TYPE)
 390 extern uint64_t atomic_swap_64();
 391 #endif /* defined(_INT64_TYPE) */
 392 
 393 
 394 extern int atomic_set_long_excl();
 395 extern int atomic_clear_long_excl();
 396 
 397 extern void membar_enter();
 398 extern void membar_exit();
 399 extern void membar_producer();
 400 extern void membar_consumer();
 401 
 402 #endif
 403 
 404 #if defined(_KERNEL)
 405 
 406 #if defined(_LP64) || defined(_ILP32)
 407 #define atomic_add_ip           atomic_add_long
 408 #define atomic_add_ip_nv        atomic_add_long_nv
 409 #define casip                   atomic_cas_ulong
 410 #endif
 411 
 412 #if defined(__sparc)
 413 extern uint8_t ldstub(uint8_t *);
 414 #endif
 415 
 416 #endif  /* _KERNEL */
 417 
 418 #ifdef  __cplusplus
 419 }
 420 #endif
 421 
 422 #endif  /* _SYS_ATOMIC_H */