1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26 
  27         .file   "atomic.s"
  28 
  29 #include <sys/asm_linkage.h>
  30 
  31 #if defined(_KERNEL)
  32         /*
  33          * Legacy kernel interfaces; they will go away (eventually).
  34          */
  35         ANSI_PRAGMA_WEAK2(cas8,atomic_cas_8,function)
  36         ANSI_PRAGMA_WEAK2(cas32,atomic_cas_32,function)
  37         ANSI_PRAGMA_WEAK2(cas64,atomic_cas_64,function)
  38         ANSI_PRAGMA_WEAK2(caslong,atomic_cas_ulong,function)
  39         ANSI_PRAGMA_WEAK2(casptr,atomic_cas_ptr,function)
  40         ANSI_PRAGMA_WEAK2(atomic_and_long,atomic_and_ulong,function)
  41         ANSI_PRAGMA_WEAK2(atomic_or_long,atomic_or_ulong,function)
  42         ANSI_PRAGMA_WEAK2(swapl,atomic_swap_32,function)
  43 #endif
  44 
  45         /*
  46          * NOTE: If atomic_inc_8 and atomic_inc_8_nv are ever
  47          * separated, you need to also edit the libc sparc platform
  48          * specific mapfile and remove the NODYNSORT attribute
  49          * from atomic_inc_8_nv.
  50          */
  51         ENTRY(atomic_inc_8)
  52         ALTENTRY(atomic_inc_8_nv)
  53         ALTENTRY(atomic_inc_uchar)
  54         ALTENTRY(atomic_inc_uchar_nv)
  55         ba      add_8
  56           add   %g0, 1, %o1
  57         SET_SIZE(atomic_inc_uchar_nv)
  58         SET_SIZE(atomic_inc_uchar)
  59         SET_SIZE(atomic_inc_8_nv)
  60         SET_SIZE(atomic_inc_8)
  61 
  62         /*
  63          * NOTE: If atomic_dec_8 and atomic_dec_8_nv are ever
  64          * separated, you need to also edit the libc sparc platform
  65          * specific mapfile and remove the NODYNSORT attribute
  66          * from atomic_dec_8_nv.
  67          */
  68         ENTRY(atomic_dec_8)
  69         ALTENTRY(atomic_dec_8_nv)
  70         ALTENTRY(atomic_dec_uchar)
  71         ALTENTRY(atomic_dec_uchar_nv)
  72         ba      add_8
  73           sub   %g0, 1, %o1
  74         SET_SIZE(atomic_dec_uchar_nv)
  75         SET_SIZE(atomic_dec_uchar)
  76         SET_SIZE(atomic_dec_8_nv)
  77         SET_SIZE(atomic_dec_8)
  78 
  79         /*
  80          * NOTE: If atomic_add_8 and atomic_add_8_nv are ever
  81          * separated, you need to also edit the libc sparc platform
  82          * specific mapfile and remove the NODYNSORT attribute
  83          * from atomic_add_8_nv.
  84          */
  85         ENTRY(atomic_add_8)
  86         ALTENTRY(atomic_add_8_nv)
  87         ALTENTRY(atomic_add_char)
  88         ALTENTRY(atomic_add_char_nv)
  89 add_8:
  90         and     %o0, 0x3, %o4           ! %o4 = byte offset, left-to-right
  91         xor     %o4, 0x3, %g1           ! %g1 = byte offset, right-to-left
  92         sll     %g1, 3, %g1             ! %g1 = bit offset, right-to-left
  93         set     0xff, %o3               ! %o3 = mask
  94         sll     %o3, %g1, %o3           ! %o3 = shifted to bit offset
  95         sll     %o1, %g1, %o1           ! %o1 = shifted to bit offset
  96         and     %o1, %o3, %o1           ! %o1 = single byte value
  97         andn    %o0, 0x3, %o0           ! %o0 = word address
  98         ld      [%o0], %o2              ! read old value
  99 1:
 100         add     %o2, %o1, %o5           ! add value to the old value
 101         and     %o5, %o3, %o5           ! clear other bits
 102         andn    %o2, %o3, %o4           ! clear target bits
 103         or      %o4, %o5, %o5           ! insert the new value
 104         cas     [%o0], %o2, %o5
 105         cmp     %o2, %o5
 106         bne,a,pn %icc, 1b
 107           mov   %o5, %o2                ! %o2 = old value
 108         add     %o2, %o1, %o5
 109         and     %o5, %o3, %o5
 110         retl
 111         srl     %o5, %g1, %o0           ! %o0 = new value
 112         SET_SIZE(atomic_add_char_nv)
 113         SET_SIZE(atomic_add_char)
 114         SET_SIZE(atomic_add_8_nv)
 115         SET_SIZE(atomic_add_8)
 116 
 117         /*
 118          * NOTE: If atomic_inc_16 and atomic_inc_16_nv are ever
 119          * separated, you need to also edit the libc sparc platform
 120          * specific mapfile and remove the NODYNSORT attribute
 121          * from atomic_inc_16_nv.
 122          */
 123         ENTRY(atomic_inc_16)
 124         ALTENTRY(atomic_inc_16_nv)
 125         ALTENTRY(atomic_inc_ushort)
 126         ALTENTRY(atomic_inc_ushort_nv)
 127         ba      add_16
 128           add   %g0, 1, %o1
 129         SET_SIZE(atomic_inc_ushort_nv)
 130         SET_SIZE(atomic_inc_ushort)
 131         SET_SIZE(atomic_inc_16_nv)
 132         SET_SIZE(atomic_inc_16)
 133 
 134         /*
 135          * NOTE: If atomic_dec_16 and atomic_dec_16_nv are ever
 136          * separated, you need to also edit the libc sparc platform
 137          * specific mapfile and remove the NODYNSORT attribute
 138          * from atomic_dec_16_nv.
 139          */
 140         ENTRY(atomic_dec_16)
 141         ALTENTRY(atomic_dec_16_nv)
 142         ALTENTRY(atomic_dec_ushort)
 143         ALTENTRY(atomic_dec_ushort_nv)
 144         ba      add_16
 145           sub   %g0, 1, %o1
 146         SET_SIZE(atomic_dec_ushort_nv)
 147         SET_SIZE(atomic_dec_ushort)
 148         SET_SIZE(atomic_dec_16_nv)
 149         SET_SIZE(atomic_dec_16)
 150 
 151         /*
 152          * NOTE: If atomic_add_16 and atomic_add_16_nv are ever
 153          * separated, you need to also edit the libc sparc platform
 154          * specific mapfile and remove the NODYNSORT attribute
 155          * from atomic_add_16_nv.
 156          */
 157         ENTRY(atomic_add_16)
 158         ALTENTRY(atomic_add_16_nv)
 159         ALTENTRY(atomic_add_short)
 160         ALTENTRY(atomic_add_short_nv)
 161 add_16:
 162         and     %o0, 0x2, %o4           ! %o4 = byte offset, left-to-right
 163         xor     %o4, 0x2, %g1           ! %g1 = byte offset, right-to-left
 164         sll     %o4, 3, %o4             ! %o4 = bit offset, left-to-right
 165         sll     %g1, 3, %g1             ! %g1 = bit offset, right-to-left
 166         sethi   %hi(0xffff0000), %o3    ! %o3 = mask
 167         srl     %o3, %o4, %o3           ! %o3 = shifted to bit offset
 168         sll     %o1, %g1, %o1           ! %o1 = shifted to bit offset
 169         and     %o1, %o3, %o1           ! %o1 = single short value
 170         andn    %o0, 0x2, %o0           ! %o0 = word address
 171         ! if low-order bit is 1, we will properly get an alignment fault here
 172         ld      [%o0], %o2              ! read old value
 173 1:
 174         add     %o1, %o2, %o5           ! add value to the old value
 175         and     %o5, %o3, %o5           ! clear other bits
 176         andn    %o2, %o3, %o4           ! clear target bits
 177         or      %o4, %o5, %o5           ! insert the new value
 178         cas     [%o0], %o2, %o5
 179         cmp     %o2, %o5
 180         bne,a,pn %icc, 1b
 181           mov   %o5, %o2                ! %o2 = old value
 182         add     %o1, %o2, %o5
 183         and     %o5, %o3, %o5
 184         retl
 185         srl     %o5, %g1, %o0           ! %o0 = new value
 186         SET_SIZE(atomic_add_short_nv)
 187         SET_SIZE(atomic_add_short)
 188         SET_SIZE(atomic_add_16_nv)
 189         SET_SIZE(atomic_add_16)
 190 
 191         /*
 192          * NOTE: If atomic_inc_32 and atomic_inc_32_nv are ever
 193          * separated, you need to also edit the libc sparc platform
 194          * specific mapfile and remove the NODYNSORT attribute
 195          * from atomic_inc_32_nv.
 196          */
 197         ENTRY(atomic_inc_32)
 198         ALTENTRY(atomic_inc_32_nv)
 199         ALTENTRY(atomic_inc_uint)
 200         ALTENTRY(atomic_inc_uint_nv)
 201         ALTENTRY(atomic_inc_ulong)
 202         ALTENTRY(atomic_inc_ulong_nv)
 203         ba      add_32
 204           add   %g0, 1, %o1
 205         SET_SIZE(atomic_inc_ulong_nv)
 206         SET_SIZE(atomic_inc_ulong)
 207         SET_SIZE(atomic_inc_uint_nv)
 208         SET_SIZE(atomic_inc_uint)
 209         SET_SIZE(atomic_inc_32_nv)
 210         SET_SIZE(atomic_inc_32)
 211 
 212         /*
 213          * NOTE: If atomic_dec_32 and atomic_dec_32_nv are ever
 214          * separated, you need to also edit the libc sparc platform
 215          * specific mapfile and remove the NODYNSORT attribute
 216          * from atomic_dec_32_nv.
 217          */
 218         ENTRY(atomic_dec_32)
 219         ALTENTRY(atomic_dec_32_nv)
 220         ALTENTRY(atomic_dec_uint)
 221         ALTENTRY(atomic_dec_uint_nv)
 222         ALTENTRY(atomic_dec_ulong)
 223         ALTENTRY(atomic_dec_ulong_nv)
 224         ba      add_32
 225           sub   %g0, 1, %o1
 226         SET_SIZE(atomic_dec_ulong_nv)
 227         SET_SIZE(atomic_dec_ulong)
 228         SET_SIZE(atomic_dec_uint_nv)
 229         SET_SIZE(atomic_dec_uint)
 230         SET_SIZE(atomic_dec_32_nv)
 231         SET_SIZE(atomic_dec_32)
 232 
 233         /*
 234          * NOTE: If atomic_add_32 and atomic_add_32_nv are ever
 235          * separated, you need to also edit the libc sparc platform
 236          * specific mapfile and remove the NODYNSORT attribute
 237          * from atomic_add_32_nv.
 238          */
 239         ENTRY(atomic_add_32)
 240         ALTENTRY(atomic_add_32_nv)
 241         ALTENTRY(atomic_add_int)
 242         ALTENTRY(atomic_add_int_nv)
 243         ALTENTRY(atomic_add_ptr)
 244         ALTENTRY(atomic_add_ptr_nv)
 245         ALTENTRY(atomic_add_long)
 246         ALTENTRY(atomic_add_long_nv)
 247 add_32:
 248         ld      [%o0], %o2
 249 1:
 250         add     %o2, %o1, %o3
 251         cas     [%o0], %o2, %o3
 252         cmp     %o2, %o3
 253         bne,a,pn %icc, 1b
 254           mov   %o3, %o2
 255         retl
 256         add     %o2, %o1, %o0           ! return new value
 257         SET_SIZE(atomic_add_long_nv)
 258         SET_SIZE(atomic_add_long)
 259         SET_SIZE(atomic_add_ptr_nv)
 260         SET_SIZE(atomic_add_ptr)
 261         SET_SIZE(atomic_add_int_nv)
 262         SET_SIZE(atomic_add_int)
 263         SET_SIZE(atomic_add_32_nv)
 264         SET_SIZE(atomic_add_32)
 265 
 266         /*
 267          * NOTE: If atomic_inc_64 and atomic_inc_64_nv are ever
 268          * separated, you need to also edit the libc sparc platform
 269          * specific mapfile and remove the NODYNSORT attribute
 270          * from atomic_inc_64_nv.
 271          */
 272         ENTRY(atomic_inc_64)
 273         ALTENTRY(atomic_inc_64_nv)
 274         ba      add_64
 275           add   %g0, 1, %o1
 276         SET_SIZE(atomic_inc_64_nv)
 277         SET_SIZE(atomic_inc_64)
 278 
 279         /*
 280          * NOTE: If atomic_dec_64 and atomic_dec_64_nv are ever
 281          * separated, you need to also edit the libc sparc platform
 282          * specific mapfile and remove the NODYNSORT attribute
 283          * from atomic_dec_64_nv.
 284          */
 285         ENTRY(atomic_dec_64)
 286         ALTENTRY(atomic_dec_64_nv)
 287         ba      add_64
 288           sub   %g0, 1, %o1
 289         SET_SIZE(atomic_dec_64_nv)
 290         SET_SIZE(atomic_dec_64)
 291 
 292         /*
 293          * NOTE: If atomic_add_64 and atomic_add_64_nv are ever
 294          * separated, you need to also edit the libc sparc platform
 295          * specific mapfile and remove the NODYNSORT attribute
 296          * from atomic_add_64_nv.
 297          */
 298         ENTRY(atomic_add_64)
 299         ALTENTRY(atomic_add_64_nv)
 300         sllx    %o1, 32, %o1            ! upper 32 in %o1, lower in %o2
 301         srl     %o2, 0, %o2
 302         add     %o1, %o2, %o1           ! convert 2 32-bit args into 1 64-bit
 303 add_64:
 304         ldx     [%o0], %o2
 305 1:
 306         add     %o2, %o1, %o3
 307         casx    [%o0], %o2, %o3
 308         cmp     %o2, %o3
 309         bne,a,pn %xcc, 1b
 310           mov   %o3, %o2
 311         add     %o2, %o1, %o1           ! return lower 32-bits in %o1
 312         retl
 313         srlx    %o1, 32, %o0            ! return upper 32-bits in %o0
 314         SET_SIZE(atomic_add_64_nv)
 315         SET_SIZE(atomic_add_64)
 316 
 317         /*
 318          * NOTE: If atomic_or_8 and atomic_or_8_nv are ever
 319          * separated, you need to also edit the libc sparc platform
 320          * specific mapfile and remove the NODYNSORT attribute
 321          * from atomic_or_8_nv.
 322          */
 323         ENTRY(atomic_or_8)
 324         ALTENTRY(atomic_or_8_nv)
 325         ALTENTRY(atomic_or_uchar)
 326         ALTENTRY(atomic_or_uchar_nv)
 327         and     %o0, 0x3, %o4           ! %o4 = byte offset, left-to-right
 328         xor     %o4, 0x3, %g1           ! %g1 = byte offset, right-to-left
 329         sll     %g1, 3, %g1             ! %g1 = bit offset, right-to-left
 330         set     0xff, %o3               ! %o3 = mask
 331         sll     %o3, %g1, %o3           ! %o3 = shifted to bit offset
 332         sll     %o1, %g1, %o1           ! %o1 = shifted to bit offset
 333         and     %o1, %o3, %o1           ! %o1 = single byte value
 334         andn    %o0, 0x3, %o0           ! %o0 = word address
 335         ld      [%o0], %o2              ! read old value
 336 1:
 337         or      %o2, %o1, %o5           ! or in the new value
 338         cas     [%o0], %o2, %o5
 339         cmp     %o2, %o5
 340         bne,a,pn %icc, 1b
 341           mov   %o5, %o2                ! %o2 = old value
 342         or      %o2, %o1, %o5
 343         and     %o5, %o3, %o5
 344         retl
 345         srl     %o5, %g1, %o0           ! %o0 = new value
 346         SET_SIZE(atomic_or_uchar_nv)
 347         SET_SIZE(atomic_or_uchar)
 348         SET_SIZE(atomic_or_8_nv)
 349         SET_SIZE(atomic_or_8)
 350 
 351         /*
 352          * NOTE: If atomic_or_16 and atomic_or_16_nv are ever
 353          * separated, you need to also edit the libc sparc platform
 354          * specific mapfile and remove the NODYNSORT attribute
 355          * from atomic_or_16_nv.
 356          */
 357         ENTRY(atomic_or_16)
 358         ALTENTRY(atomic_or_16_nv)
 359         ALTENTRY(atomic_or_ushort)
 360         ALTENTRY(atomic_or_ushort_nv)
 361         and     %o0, 0x2, %o4           ! %o4 = byte offset, left-to-right
 362         xor     %o4, 0x2, %g1           ! %g1 = byte offset, right-to-left
 363         sll     %o4, 3, %o4             ! %o4 = bit offset, left-to-right
 364         sll     %g1, 3, %g1             ! %g1 = bit offset, right-to-left
 365         sethi   %hi(0xffff0000), %o3    ! %o3 = mask
 366         srl     %o3, %o4, %o3           ! %o3 = shifted to bit offset
 367         sll     %o1, %g1, %o1           ! %o1 = shifted to bit offset
 368         and     %o1, %o3, %o1           ! %o1 = single short value
 369         andn    %o0, 0x2, %o0           ! %o0 = word address
 370         ! if low-order bit is 1, we will properly get an alignment fault here
 371         ld      [%o0], %o2              ! read old value
 372 1:
 373         or      %o2, %o1, %o5           ! or in the new value
 374         cas     [%o0], %o2, %o5
 375         cmp     %o2, %o5
 376         bne,a,pn %icc, 1b
 377           mov   %o5, %o2                ! %o2 = old value
 378         or      %o2, %o1, %o5           ! or in the new value
 379         and     %o5, %o3, %o5
 380         retl
 381         srl     %o5, %g1, %o0           ! %o0 = new value
 382         SET_SIZE(atomic_or_ushort_nv)
 383         SET_SIZE(atomic_or_ushort)
 384         SET_SIZE(atomic_or_16_nv)
 385         SET_SIZE(atomic_or_16)
 386 
 387         /*
 388          * NOTE: If atomic_or_32 and atomic_or_32_nv are ever
 389          * separated, you need to also edit the libc sparc platform
 390          * specific mapfile and remove the NODYNSORT attribute
 391          * from atomic_or_32_nv.
 392          */
 393         ENTRY(atomic_or_32)
 394         ALTENTRY(atomic_or_32_nv)
 395         ALTENTRY(atomic_or_uint)
 396         ALTENTRY(atomic_or_uint_nv)
 397         ALTENTRY(atomic_or_ulong)
 398         ALTENTRY(atomic_or_ulong_nv)
 399         ld      [%o0], %o2
 400 1:
 401         or      %o2, %o1, %o3
 402         cas     [%o0], %o2, %o3
 403         cmp     %o2, %o3
 404         bne,a,pn %icc, 1b
 405           mov   %o3, %o2
 406         retl
 407         or      %o2, %o1, %o0           ! return new value
 408         SET_SIZE(atomic_or_ulong_nv)
 409         SET_SIZE(atomic_or_ulong)
 410         SET_SIZE(atomic_or_uint_nv)
 411         SET_SIZE(atomic_or_uint)
 412         SET_SIZE(atomic_or_32_nv)
 413         SET_SIZE(atomic_or_32)
 414 
 415         /*
 416          * NOTE: If atomic_or_64 and atomic_or_64_nv are ever
 417          * separated, you need to also edit the libc sparc platform
 418          * specific mapfile and remove the NODYNSORT attribute
 419          * from atomic_or_64_nv.
 420          */
 421         ENTRY(atomic_or_64)
 422         ALTENTRY(atomic_or_64_nv)
 423         sllx    %o1, 32, %o1            ! upper 32 in %o1, lower in %o2
 424         srl     %o2, 0, %o2
 425         add     %o1, %o2, %o1           ! convert 2 32-bit args into 1 64-bit
 426         ldx     [%o0], %o2
 427 1:
 428         or      %o2, %o1, %o3
 429         casx    [%o0], %o2, %o3
 430         cmp     %o2, %o3
 431         bne,a,pn %xcc, 1b
 432           mov   %o3, %o2
 433         or      %o2, %o1, %o1           ! return lower 32-bits in %o1
 434         retl
 435         srlx    %o1, 32, %o0            ! return upper 32-bits in %o0
 436         SET_SIZE(atomic_or_64_nv)
 437         SET_SIZE(atomic_or_64)
 438 
 439         /*
 440          * NOTE: If atomic_and_8 and atomic_and_8_nv are ever
 441          * separated, you need to also edit the libc sparc platform
 442          * specific mapfile and remove the NODYNSORT attribute
 443          * from atomic_and_8_nv.
 444          */
 445         ENTRY(atomic_and_8)
 446         ALTENTRY(atomic_and_8_nv)
 447         ALTENTRY(atomic_and_uchar)
 448         ALTENTRY(atomic_and_uchar_nv)
 449         and     %o0, 0x3, %o4           ! %o4 = byte offset, left-to-right
 450         xor     %o4, 0x3, %g1           ! %g1 = byte offset, right-to-left
 451         sll     %g1, 3, %g1             ! %g1 = bit offset, right-to-left
 452         set     0xff, %o3               ! %o3 = mask
 453         sll     %o3, %g1, %o3           ! %o3 = shifted to bit offset
 454         sll     %o1, %g1, %o1           ! %o1 = shifted to bit offset
 455         orn     %o1, %o3, %o1           ! all ones in other bytes
 456         andn    %o0, 0x3, %o0           ! %o0 = word address
 457         ld      [%o0], %o2              ! read old value
 458 1:
 459         and     %o2, %o1, %o5           ! and in the new value
 460         cas     [%o0], %o2, %o5
 461         cmp     %o2, %o5
 462         bne,a,pn %icc, 1b
 463           mov   %o5, %o2                ! %o2 = old value
 464         and     %o2, %o1, %o5
 465         and     %o5, %o3, %o5
 466         retl
 467         srl     %o5, %g1, %o0           ! %o0 = new value
 468         SET_SIZE(atomic_and_uchar_nv)
 469         SET_SIZE(atomic_and_uchar)
 470         SET_SIZE(atomic_and_8_nv)
 471         SET_SIZE(atomic_and_8)
 472 
 473         /*
 474          * NOTE: If atomic_and_16 and atomic_and_16_nv are ever
 475          * separated, you need to also edit the libc sparc platform
 476          * specific mapfile and remove the NODYNSORT attribute
 477          * from atomic_and_16_nv.
 478          */
 479         ENTRY(atomic_and_16)
 480         ALTENTRY(atomic_and_16_nv)
 481         ALTENTRY(atomic_and_ushort)
 482         ALTENTRY(atomic_and_ushort_nv)
 483         and     %o0, 0x2, %o4           ! %o4 = byte offset, left-to-right
 484         xor     %o4, 0x2, %g1           ! %g1 = byte offset, right-to-left
 485         sll     %o4, 3, %o4             ! %o4 = bit offset, left-to-right
 486         sll     %g1, 3, %g1             ! %g1 = bit offset, right-to-left
 487         sethi   %hi(0xffff0000), %o3    ! %o3 = mask
 488         srl     %o3, %o4, %o3           ! %o3 = shifted to bit offset
 489         sll     %o1, %g1, %o1           ! %o1 = shifted to bit offset
 490         orn     %o1, %o3, %o1           ! all ones in the other half
 491         andn    %o0, 0x2, %o0           ! %o0 = word address
 492         ! if low-order bit is 1, we will properly get an alignment fault here
 493         ld      [%o0], %o2              ! read old value
 494 1:
 495         and     %o2, %o1, %o5           ! and in the new value
 496         cas     [%o0], %o2, %o5
 497         cmp     %o2, %o5
 498         bne,a,pn %icc, 1b
 499           mov   %o5, %o2                ! %o2 = old value
 500         and     %o2, %o1, %o5
 501         and     %o5, %o3, %o5
 502         retl
 503         srl     %o5, %g1, %o0           ! %o0 = new value
 504         SET_SIZE(atomic_and_ushort_nv)
 505         SET_SIZE(atomic_and_ushort)
 506         SET_SIZE(atomic_and_16_nv)
 507         SET_SIZE(atomic_and_16)
 508 
 509         /*
 510          * NOTE: If atomic_and_32 and atomic_and_32_nv are ever
 511          * separated, you need to also edit the libc sparc platform
 512          * specific mapfile and remove the NODYNSORT attribute
 513          * from atomic_and_32_nv.
 514          */
 515         ENTRY(atomic_and_32)
 516         ALTENTRY(atomic_and_32_nv)
 517         ALTENTRY(atomic_and_uint)
 518         ALTENTRY(atomic_and_uint_nv)
 519         ALTENTRY(atomic_and_ulong)
 520         ALTENTRY(atomic_and_ulong_nv)
 521         ld      [%o0], %o2
 522 1:
 523         and     %o2, %o1, %o3
 524         cas     [%o0], %o2, %o3
 525         cmp     %o2, %o3
 526         bne,a,pn %icc, 1b
 527           mov   %o3, %o2
 528         retl
 529         and     %o2, %o1, %o0           ! return new value
 530         SET_SIZE(atomic_and_ulong_nv)
 531         SET_SIZE(atomic_and_ulong)
 532         SET_SIZE(atomic_and_uint_nv)
 533         SET_SIZE(atomic_and_uint)
 534         SET_SIZE(atomic_and_32_nv)
 535         SET_SIZE(atomic_and_32)
 536 
 537         /*
 538          * NOTE: If atomic_and_64 and atomic_and_64_nv are ever
 539          * separated, you need to also edit the libc sparc platform
 540          * specific mapfile and remove the NODYNSORT attribute
 541          * from atomic_and_64_nv.
 542          */
 543         ENTRY(atomic_and_64)
 544         ALTENTRY(atomic_and_64_nv)
 545         sllx    %o1, 32, %o1            ! upper 32 in %o1, lower in %o2
 546         srl     %o2, 0, %o2
 547         add     %o1, %o2, %o1           ! convert 2 32-bit args into 1 64-bit
 548         ldx     [%o0], %o2
 549 1:
 550         and     %o2, %o1, %o3
 551         casx    [%o0], %o2, %o3
 552         cmp     %o2, %o3
 553         bne,a,pn %xcc, 1b
 554           mov   %o3, %o2
 555         and     %o2, %o1, %o1           ! return lower 32-bits in %o1
 556         retl
 557         srlx    %o1, 32, %o0            ! return upper 32-bits in %o0
 558         SET_SIZE(atomic_and_64_nv)
 559         SET_SIZE(atomic_and_64)
 560 
 561         ENTRY(atomic_cas_8)
 562         ALTENTRY(atomic_cas_uchar)
 563         and     %o0, 0x3, %o4           ! %o4 = byte offset, left-to-right
 564         xor     %o4, 0x3, %g1           ! %g1 = byte offset, right-to-left
 565         sll     %g1, 3, %g1             ! %g1 = bit offset, right-to-left
 566         set     0xff, %o3               ! %o3 = mask
 567         sll     %o3, %g1, %o3           ! %o3 = shifted to bit offset
 568         sll     %o1, %g1, %o1           ! %o1 = shifted to bit offset
 569         and     %o1, %o3, %o1           ! %o1 = single byte value
 570         sll     %o2, %g1, %o2           ! %o2 = shifted to bit offset
 571         and     %o2, %o3, %o2           ! %o2 = single byte value
 572         andn    %o0, 0x3, %o0           ! %o0 = word address
 573         ld      [%o0], %o4              ! read old value
 574 1:
 575         andn    %o4, %o3, %o4           ! clear target bits
 576         or      %o4, %o2, %o5           ! insert the new value
 577         or      %o4, %o1, %o4           ! insert the comparison value
 578         cas     [%o0], %o4, %o5
 579         cmp     %o4, %o5                ! did we succeed?
 580         be,pt   %icc, 2f
 581           and   %o5, %o3, %o4           ! isolate the old value
 582         cmp     %o1, %o4                ! should we have succeeded?
 583         be,a,pt %icc, 1b                ! yes, try again
 584           mov   %o5, %o4                ! %o4 = old value
 585 2:
 586         retl
 587         srl     %o4, %g1, %o0           ! %o0 = old value
 588         SET_SIZE(atomic_cas_uchar)
 589         SET_SIZE(atomic_cas_8)
 590 
 591         ENTRY(atomic_cas_16)
 592         ALTENTRY(atomic_cas_ushort)
 593         and     %o0, 0x2, %o4           ! %o4 = byte offset, left-to-right
 594         xor     %o4, 0x2, %g1           ! %g1 = byte offset, right-to-left
 595         sll     %o4, 3, %o4             ! %o4 = bit offset, left-to-right
 596         sll     %g1, 3, %g1             ! %g1 = bit offset, right-to-left
 597         sethi   %hi(0xffff0000), %o3    ! %o3 = mask
 598         srl     %o3, %o4, %o3           ! %o3 = shifted to bit offset
 599         sll     %o1, %g1, %o1           ! %o1 = shifted to bit offset
 600         and     %o1, %o3, %o1           ! %o1 = single short value
 601         sll     %o2, %g1, %o2           ! %o2 = shifted to bit offset
 602         and     %o2, %o3, %o2           ! %o2 = single short value
 603         andn    %o0, 0x2, %o0           ! %o0 = word address
 604         ! if low-order bit is 1, we will properly get an alignment fault here
 605         ld      [%o0], %o4              ! read old value
 606 1:
 607         andn    %o4, %o3, %o4           ! clear target bits
 608         or      %o4, %o2, %o5           ! insert the new value
 609         or      %o4, %o1, %o4           ! insert the comparison value
 610         cas     [%o0], %o4, %o5
 611         cmp     %o4, %o5                ! did we succeed?
 612         be,pt   %icc, 2f
 613           and   %o5, %o3, %o4           ! isolate the old value
 614         cmp     %o1, %o4                ! should we have succeeded?
 615         be,a,pt %icc, 1b                ! yes, try again
 616           mov   %o5, %o4                ! %o4 = old value
 617 2:
 618         retl
 619         srl     %o4, %g1, %o0           ! %o0 = old value
 620         SET_SIZE(atomic_cas_ushort)
 621         SET_SIZE(atomic_cas_16)
 622 
 623         ENTRY(atomic_cas_32)
 624         ALTENTRY(atomic_cas_uint)
 625         ALTENTRY(atomic_cas_ptr)
 626         ALTENTRY(atomic_cas_ulong)
 627         cas     [%o0], %o1, %o2
 628         retl
 629         mov     %o2, %o0
 630         SET_SIZE(atomic_cas_ulong)
 631         SET_SIZE(atomic_cas_ptr)
 632         SET_SIZE(atomic_cas_uint)
 633         SET_SIZE(atomic_cas_32)
 634 
 635         ENTRY(atomic_cas_64)
 636         sllx    %o1, 32, %o1            ! cmp's upper 32 in %o1, lower in %o2
 637         srl     %o2, 0, %o2             ! convert 2 32-bit args into 1 64-bit
 638         add     %o1, %o2, %o1
 639         sllx    %o3, 32, %o2            ! newval upper 32 in %o3, lower in %o4
 640         srl     %o4, 0, %o4             ! setup %o2 to have newval
 641         add     %o2, %o4, %o2
 642         casx    [%o0], %o1, %o2
 643         srl     %o2, 0, %o1             ! return lower 32-bits in %o1
 644         retl
 645         srlx    %o2, 32, %o0            ! return upper 32-bits in %o0
 646         SET_SIZE(atomic_cas_64)
 647 
 648         ENTRY(atomic_swap_8)
 649         ALTENTRY(atomic_swap_uchar)
 650         and     %o0, 0x3, %o4           ! %o4 = byte offset, left-to-right
 651         xor     %o4, 0x3, %g1           ! %g1 = byte offset, right-to-left
 652         sll     %g1, 3, %g1             ! %g1 = bit offset, right-to-left
 653         set     0xff, %o3               ! %o3 = mask
 654         sll     %o3, %g1, %o3           ! %o3 = shifted to bit offset
 655         sll     %o1, %g1, %o1           ! %o1 = shifted to bit offset
 656         and     %o1, %o3, %o1           ! %o1 = single byte value
 657         andn    %o0, 0x3, %o0           ! %o0 = word address
 658         ld      [%o0], %o2              ! read old value
 659 1:
 660         andn    %o2, %o3, %o5           ! clear target bits
 661         or      %o5, %o1, %o5           ! insert the new value
 662         cas     [%o0], %o2, %o5
 663         cmp     %o2, %o5
 664         bne,a,pn %icc, 1b
 665           mov   %o5, %o2                ! %o2 = old value
 666         and     %o5, %o3, %o5
 667         retl
 668         srl     %o5, %g1, %o0           ! %o0 = old value
 669         SET_SIZE(atomic_swap_uchar)
 670         SET_SIZE(atomic_swap_8)
 671 
 672         ENTRY(atomic_swap_16)
 673         ALTENTRY(atomic_swap_ushort)
 674         and     %o0, 0x2, %o4           ! %o4 = byte offset, left-to-right
 675         xor     %o4, 0x2, %g1           ! %g1 = byte offset, right-to-left
 676         sll     %o4, 3, %o4             ! %o4 = bit offset, left-to-right
 677         sll     %g1, 3, %g1             ! %g1 = bit offset, right-to-left
 678         sethi   %hi(0xffff0000), %o3    ! %o3 = mask
 679         srl     %o3, %o4, %o3           ! %o3 = shifted to bit offset
 680         sll     %o1, %g1, %o1           ! %o1 = shifted to bit offset
 681         and     %o1, %o3, %o1           ! %o1 = single short value
 682         andn    %o0, 0x2, %o0           ! %o0 = word address
 683         ! if low-order bit is 1, we will properly get an alignment fault here
 684         ld      [%o0], %o2              ! read old value
 685 1:
 686         andn    %o2, %o3, %o5           ! clear target bits
 687         or      %o5, %o1, %o5           ! insert the new value
 688         cas     [%o0], %o2, %o5
 689         cmp     %o2, %o5
 690         bne,a,pn %icc, 1b
 691           mov   %o5, %o2                ! %o2 = old value
 692         and     %o5, %o3, %o5
 693         retl
 694         srl     %o5, %g1, %o0           ! %o0 = old value
 695         SET_SIZE(atomic_swap_ushort)
 696         SET_SIZE(atomic_swap_16)
 697 
 698         ENTRY(atomic_swap_32)
 699         ALTENTRY(atomic_swap_uint)
 700         ALTENTRY(atomic_swap_ptr)
 701         ALTENTRY(atomic_swap_ulong)
 702         ld      [%o0], %o2
 703 1:
 704         mov     %o1, %o3
 705         cas     [%o0], %o2, %o3
 706         cmp     %o2, %o3
 707         bne,a,pn %icc, 1b
 708           mov   %o3, %o2
 709         retl
 710         mov     %o3, %o0
 711         SET_SIZE(atomic_swap_ulong)
 712         SET_SIZE(atomic_swap_ptr)
 713         SET_SIZE(atomic_swap_uint)
 714         SET_SIZE(atomic_swap_32)
 715 
 716         ENTRY(atomic_swap_64)
 717         sllx    %o1, 32, %o1            ! upper 32 in %o1, lower in %o2
 718         srl     %o2, 0, %o2
 719         add     %o1, %o2, %o1           ! convert 2 32-bit args into 1 64-bit
 720         ldx     [%o0], %o2
 721 1:
 722         mov     %o1, %o3
 723         casx    [%o0], %o2, %o3
 724         cmp     %o2, %o3
 725         bne,a,pn %xcc, 1b
 726           mov   %o3, %o2
 727         srl     %o3, 0, %o1             ! return lower 32-bits in %o1
 728         retl
 729         srlx    %o3, 32, %o0            ! return upper 32-bits in %o0
 730         SET_SIZE(atomic_swap_64)
 731 
 732         ENTRY(atomic_set_long_excl)
 733         mov     1, %o3
 734         slln    %o3, %o1, %o3
 735         ldn     [%o0], %o2
 736 1:
 737         andcc   %o2, %o3, %g0           ! test if the bit is set
 738         bnz,a,pn %ncc, 2f               ! if so, then fail out
 739           mov   -1, %o0
 740         or      %o2, %o3, %o4           ! set the bit, and try to commit it
 741         casn    [%o0], %o2, %o4
 742         cmp     %o2, %o4
 743         bne,a,pn %ncc, 1b               ! failed to commit, try again
 744           mov   %o4, %o2
 745         mov     %g0, %o0
 746 2:
 747         retl
 748         nop
 749         SET_SIZE(atomic_set_long_excl)
 750 
 751         ENTRY(atomic_clear_long_excl)
 752         mov     1, %o3
 753         slln    %o3, %o1, %o3
 754         ldn     [%o0], %o2
 755 1:
 756         andncc  %o3, %o2, %g0           ! test if the bit is clear
 757         bnz,a,pn %ncc, 2f               ! if so, then fail out
 758           mov   -1, %o0
 759         andn    %o2, %o3, %o4           ! clear the bit, and try to commit it
 760         casn    [%o0], %o2, %o4
 761         cmp     %o2, %o4
 762         bne,a,pn %ncc, 1b               ! failed to commit, try again
 763           mov   %o4, %o2
 764         mov     %g0, %o0
 765 2:
 766         retl
 767         nop
 768         SET_SIZE(atomic_clear_long_excl)
 769 
 770 #if !defined(_KERNEL)
 771 
 772         /*
 773          * Spitfires and Blackbirds have a problem with membars in the
 774          * delay slot (SF_ERRATA_51).  For safety's sake, we assume
 775          * that the whole world needs the workaround.
 776          */
 777         ENTRY(membar_enter)
 778         membar  #StoreLoad|#StoreStore
 779         retl
 780         nop
 781         SET_SIZE(membar_enter)
 782 
 783         ENTRY(membar_exit)
 784         membar  #LoadStore|#StoreStore
 785         retl
 786         nop
 787         SET_SIZE(membar_exit)
 788 
 789         ENTRY(membar_producer)
 790         membar  #StoreStore
 791         retl
 792         nop
 793         SET_SIZE(membar_producer)
 794 
 795         ENTRY(membar_consumer)
 796         membar  #LoadLoad
 797         retl
 798         nop
 799         SET_SIZE(membar_consumer)
 800 
 801 #endif  /* !_KERNEL */