Print this page
patch keith-feedback
5291 x86 {high,low}bit rely on undefined behavior
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/intel/ia32/ml/i86_subr.s
+++ new/usr/src/uts/intel/ia32/ml/i86_subr.s
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
24 24 * Copyright 2014 Nexenta Systems, Inc. All rights reserved.
25 25 * Copyright (c) 2014 by Delphix. All rights reserved.
26 26 */
27 27
28 28 /*
29 29 * Copyright (c) 1990, 1991 UNIX System Laboratories, Inc.
30 30 * Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T
31 31 * All Rights Reserved
32 32 */
33 33
34 34 /*
35 35 * Copyright (c) 2009, Intel Corporation.
36 36 * All rights reserved.
37 37 */
38 38
39 39 /*
40 40 * General assembly language routines.
41 41 * It is the intent of this file to contain routines that are
42 42 * independent of the specific kernel architecture, and those that are
43 43 * common across kernel architectures.
44 44 * As architectures diverge, and implementations of specific
45 45 * architecture-dependent routines change, the routines should be moved
46 46 * from this file into the respective ../`arch -k`/subr.s file.
47 47 */
48 48
49 49 #include <sys/asm_linkage.h>
50 50 #include <sys/asm_misc.h>
51 51 #include <sys/panic.h>
52 52 #include <sys/ontrap.h>
53 53 #include <sys/regset.h>
54 54 #include <sys/privregs.h>
55 55 #include <sys/reboot.h>
56 56 #include <sys/psw.h>
57 57 #include <sys/x86_archext.h>
58 58
59 59 #if defined(__lint)
60 60 #include <sys/types.h>
61 61 #include <sys/systm.h>
62 62 #include <sys/thread.h>
63 63 #include <sys/archsystm.h>
64 64 #include <sys/byteorder.h>
65 65 #include <sys/dtrace.h>
66 66 #include <sys/ftrace.h>
67 67 #else /* __lint */
68 68 #include "assym.h"
69 69 #endif /* __lint */
70 70 #include <sys/dditypes.h>
71 71
72 72 /*
73 73 * on_fault()
74 74 * Catch lofault faults. Like setjmp except it returns one
75 75 * if code following causes uncorrectable fault. Turned off
76 76 * by calling no_fault().
77 77 */
78 78
79 79 #if defined(__lint)
80 80
81 81 /* ARGSUSED */
82 82 int
83 83 on_fault(label_t *ljb)
84 84 { return (0); }
85 85
86 86 void
87 87 no_fault(void)
88 88 {}
89 89
90 90 #else /* __lint */
91 91
92 92 #if defined(__amd64)
93 93
94 94 ENTRY(on_fault)
95 95 movq %gs:CPU_THREAD, %rsi
96 96 leaq catch_fault(%rip), %rdx
97 97 movq %rdi, T_ONFAULT(%rsi) /* jumpbuf in t_onfault */
98 98 movq %rdx, T_LOFAULT(%rsi) /* catch_fault in t_lofault */
99 99 jmp setjmp /* let setjmp do the rest */
100 100
101 101 catch_fault:
102 102 movq %gs:CPU_THREAD, %rsi
103 103 movq T_ONFAULT(%rsi), %rdi /* address of save area */
104 104 xorl %eax, %eax
105 105 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */
106 106 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */
107 107 jmp longjmp /* let longjmp do the rest */
108 108 SET_SIZE(on_fault)
109 109
110 110 ENTRY(no_fault)
111 111 movq %gs:CPU_THREAD, %rsi
112 112 xorl %eax, %eax
113 113 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */
114 114 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */
115 115 ret
116 116 SET_SIZE(no_fault)
117 117
118 118 #elif defined(__i386)
119 119
120 120 ENTRY(on_fault)
121 121 movl %gs:CPU_THREAD, %edx
122 122 movl 4(%esp), %eax /* jumpbuf address */
123 123 leal catch_fault, %ecx
124 124 movl %eax, T_ONFAULT(%edx) /* jumpbuf in t_onfault */
125 125 movl %ecx, T_LOFAULT(%edx) /* catch_fault in t_lofault */
126 126 jmp setjmp /* let setjmp do the rest */
127 127
128 128 catch_fault:
129 129 movl %gs:CPU_THREAD, %edx
130 130 xorl %eax, %eax
131 131 movl T_ONFAULT(%edx), %ecx /* address of save area */
132 132 movl %eax, T_ONFAULT(%edx) /* turn off onfault */
133 133 movl %eax, T_LOFAULT(%edx) /* turn off lofault */
134 134 pushl %ecx
135 135 call longjmp /* let longjmp do the rest */
136 136 SET_SIZE(on_fault)
137 137
138 138 ENTRY(no_fault)
139 139 movl %gs:CPU_THREAD, %edx
140 140 xorl %eax, %eax
141 141 movl %eax, T_ONFAULT(%edx) /* turn off onfault */
142 142 movl %eax, T_LOFAULT(%edx) /* turn off lofault */
143 143 ret
144 144 SET_SIZE(no_fault)
145 145
146 146 #endif /* __i386 */
147 147 #endif /* __lint */
148 148
149 149 /*
150 150 * Default trampoline code for on_trap() (see <sys/ontrap.h>). We just
151 151 * do a longjmp(&curthread->t_ontrap->ot_jmpbuf) if this is ever called.
152 152 */
153 153
154 154 #if defined(lint)
155 155
156 156 void
157 157 on_trap_trampoline(void)
158 158 {}
159 159
160 160 #else /* __lint */
161 161
162 162 #if defined(__amd64)
163 163
164 164 ENTRY(on_trap_trampoline)
165 165 movq %gs:CPU_THREAD, %rsi
166 166 movq T_ONTRAP(%rsi), %rdi
167 167 addq $OT_JMPBUF, %rdi
168 168 jmp longjmp
169 169 SET_SIZE(on_trap_trampoline)
170 170
171 171 #elif defined(__i386)
172 172
173 173 ENTRY(on_trap_trampoline)
174 174 movl %gs:CPU_THREAD, %eax
175 175 movl T_ONTRAP(%eax), %eax
176 176 addl $OT_JMPBUF, %eax
177 177 pushl %eax
178 178 call longjmp
179 179 SET_SIZE(on_trap_trampoline)
180 180
181 181 #endif /* __i386 */
182 182 #endif /* __lint */
183 183
184 184 /*
185 185 * Push a new element on to the t_ontrap stack. Refer to <sys/ontrap.h> for
186 186 * more information about the on_trap() mechanism. If the on_trap_data is the
187 187 * same as the topmost stack element, we just modify that element.
188 188 */
189 189 #if defined(lint)
190 190
191 191 /*ARGSUSED*/
192 192 int
193 193 on_trap(on_trap_data_t *otp, uint_t prot)
194 194 { return (0); }
195 195
196 196 #else /* __lint */
197 197
198 198 #if defined(__amd64)
199 199
200 200 ENTRY(on_trap)
201 201 movw %si, OT_PROT(%rdi) /* ot_prot = prot */
202 202 movw $0, OT_TRAP(%rdi) /* ot_trap = 0 */
203 203 leaq on_trap_trampoline(%rip), %rdx /* rdx = &on_trap_trampoline */
204 204 movq %rdx, OT_TRAMPOLINE(%rdi) /* ot_trampoline = rdx */
205 205 xorl %ecx, %ecx
206 206 movq %rcx, OT_HANDLE(%rdi) /* ot_handle = NULL */
207 207 movq %rcx, OT_PAD1(%rdi) /* ot_pad1 = NULL */
208 208 movq %gs:CPU_THREAD, %rdx /* rdx = curthread */
209 209 movq T_ONTRAP(%rdx), %rcx /* rcx = curthread->t_ontrap */
210 210 cmpq %rdi, %rcx /* if (otp == %rcx) */
211 211 je 0f /* don't modify t_ontrap */
212 212
213 213 movq %rcx, OT_PREV(%rdi) /* ot_prev = t_ontrap */
214 214 movq %rdi, T_ONTRAP(%rdx) /* curthread->t_ontrap = otp */
215 215
216 216 0: addq $OT_JMPBUF, %rdi /* &ot_jmpbuf */
217 217 jmp setjmp
218 218 SET_SIZE(on_trap)
219 219
220 220 #elif defined(__i386)
221 221
222 222 ENTRY(on_trap)
223 223 movl 4(%esp), %eax /* %eax = otp */
224 224 movl 8(%esp), %edx /* %edx = prot */
225 225
226 226 movw %dx, OT_PROT(%eax) /* ot_prot = prot */
227 227 movw $0, OT_TRAP(%eax) /* ot_trap = 0 */
228 228 leal on_trap_trampoline, %edx /* %edx = &on_trap_trampoline */
229 229 movl %edx, OT_TRAMPOLINE(%eax) /* ot_trampoline = %edx */
230 230 movl $0, OT_HANDLE(%eax) /* ot_handle = NULL */
231 231 movl $0, OT_PAD1(%eax) /* ot_pad1 = NULL */
232 232 movl %gs:CPU_THREAD, %edx /* %edx = curthread */
233 233 movl T_ONTRAP(%edx), %ecx /* %ecx = curthread->t_ontrap */
234 234 cmpl %eax, %ecx /* if (otp == %ecx) */
235 235 je 0f /* don't modify t_ontrap */
236 236
237 237 movl %ecx, OT_PREV(%eax) /* ot_prev = t_ontrap */
238 238 movl %eax, T_ONTRAP(%edx) /* curthread->t_ontrap = otp */
239 239
240 240 0: addl $OT_JMPBUF, %eax /* %eax = &ot_jmpbuf */
241 241 movl %eax, 4(%esp) /* put %eax back on the stack */
242 242 jmp setjmp /* let setjmp do the rest */
243 243 SET_SIZE(on_trap)
244 244
245 245 #endif /* __i386 */
246 246 #endif /* __lint */
247 247
248 248 /*
249 249 * Setjmp and longjmp implement non-local gotos using state vectors
250 250 * type label_t.
251 251 */
252 252
253 253 #if defined(__lint)
254 254
255 255 /* ARGSUSED */
256 256 int
257 257 setjmp(label_t *lp)
258 258 { return (0); }
259 259
260 260 /* ARGSUSED */
261 261 void
262 262 longjmp(label_t *lp)
263 263 {}
264 264
265 265 #else /* __lint */
266 266
267 267 #if LABEL_PC != 0
268 268 #error LABEL_PC MUST be defined as 0 for setjmp/longjmp to work as coded
269 269 #endif /* LABEL_PC != 0 */
270 270
271 271 #if defined(__amd64)
272 272
273 273 ENTRY(setjmp)
274 274 movq %rsp, LABEL_SP(%rdi)
275 275 movq %rbp, LABEL_RBP(%rdi)
276 276 movq %rbx, LABEL_RBX(%rdi)
277 277 movq %r12, LABEL_R12(%rdi)
278 278 movq %r13, LABEL_R13(%rdi)
279 279 movq %r14, LABEL_R14(%rdi)
280 280 movq %r15, LABEL_R15(%rdi)
281 281 movq (%rsp), %rdx /* return address */
282 282 movq %rdx, (%rdi) /* LABEL_PC is 0 */
283 283 xorl %eax, %eax /* return 0 */
284 284 ret
285 285 SET_SIZE(setjmp)
286 286
287 287 ENTRY(longjmp)
288 288 movq LABEL_SP(%rdi), %rsp
289 289 movq LABEL_RBP(%rdi), %rbp
290 290 movq LABEL_RBX(%rdi), %rbx
291 291 movq LABEL_R12(%rdi), %r12
292 292 movq LABEL_R13(%rdi), %r13
293 293 movq LABEL_R14(%rdi), %r14
294 294 movq LABEL_R15(%rdi), %r15
295 295 movq (%rdi), %rdx /* return address; LABEL_PC is 0 */
296 296 movq %rdx, (%rsp)
297 297 xorl %eax, %eax
298 298 incl %eax /* return 1 */
299 299 ret
300 300 SET_SIZE(longjmp)
301 301
302 302 #elif defined(__i386)
303 303
304 304 ENTRY(setjmp)
305 305 movl 4(%esp), %edx /* address of save area */
306 306 movl %ebp, LABEL_EBP(%edx)
307 307 movl %ebx, LABEL_EBX(%edx)
308 308 movl %esi, LABEL_ESI(%edx)
309 309 movl %edi, LABEL_EDI(%edx)
310 310 movl %esp, 4(%edx)
311 311 movl (%esp), %ecx /* %eip (return address) */
312 312 movl %ecx, (%edx) /* LABEL_PC is 0 */
313 313 subl %eax, %eax /* return 0 */
314 314 ret
315 315 SET_SIZE(setjmp)
316 316
317 317 ENTRY(longjmp)
318 318 movl 4(%esp), %edx /* address of save area */
319 319 movl LABEL_EBP(%edx), %ebp
320 320 movl LABEL_EBX(%edx), %ebx
321 321 movl LABEL_ESI(%edx), %esi
322 322 movl LABEL_EDI(%edx), %edi
323 323 movl 4(%edx), %esp
324 324 movl (%edx), %ecx /* %eip (return addr); LABEL_PC is 0 */
325 325 movl $1, %eax
326 326 addl $4, %esp /* pop ret adr */
327 327 jmp *%ecx /* indirect */
328 328 SET_SIZE(longjmp)
329 329
330 330 #endif /* __i386 */
331 331 #endif /* __lint */
332 332
333 333 /*
334 334 * if a() calls b() calls caller(),
335 335 * caller() returns return address in a().
336 336 * (Note: We assume a() and b() are C routines which do the normal entry/exit
337 337 * sequence.)
338 338 */
339 339
340 340 #if defined(__lint)
341 341
342 342 caddr_t
343 343 caller(void)
344 344 { return (0); }
345 345
346 346 #else /* __lint */
347 347
348 348 #if defined(__amd64)
349 349
350 350 ENTRY(caller)
351 351 movq 8(%rbp), %rax /* b()'s return pc, in a() */
352 352 ret
353 353 SET_SIZE(caller)
354 354
355 355 #elif defined(__i386)
356 356
357 357 ENTRY(caller)
358 358 movl 4(%ebp), %eax /* b()'s return pc, in a() */
359 359 ret
360 360 SET_SIZE(caller)
361 361
362 362 #endif /* __i386 */
363 363 #endif /* __lint */
364 364
365 365 /*
366 366 * if a() calls callee(), callee() returns the
367 367 * return address in a();
368 368 */
369 369
370 370 #if defined(__lint)
371 371
372 372 caddr_t
373 373 callee(void)
374 374 { return (0); }
375 375
376 376 #else /* __lint */
377 377
378 378 #if defined(__amd64)
379 379
380 380 ENTRY(callee)
381 381 movq (%rsp), %rax /* callee()'s return pc, in a() */
382 382 ret
383 383 SET_SIZE(callee)
384 384
385 385 #elif defined(__i386)
386 386
387 387 ENTRY(callee)
388 388 movl (%esp), %eax /* callee()'s return pc, in a() */
389 389 ret
390 390 SET_SIZE(callee)
391 391
392 392 #endif /* __i386 */
393 393 #endif /* __lint */
394 394
395 395 /*
396 396 * return the current frame pointer
397 397 */
398 398
399 399 #if defined(__lint)
400 400
401 401 greg_t
402 402 getfp(void)
403 403 { return (0); }
404 404
405 405 #else /* __lint */
406 406
407 407 #if defined(__amd64)
408 408
409 409 ENTRY(getfp)
410 410 movq %rbp, %rax
411 411 ret
412 412 SET_SIZE(getfp)
413 413
414 414 #elif defined(__i386)
415 415
416 416 ENTRY(getfp)
417 417 movl %ebp, %eax
418 418 ret
419 419 SET_SIZE(getfp)
420 420
421 421 #endif /* __i386 */
422 422 #endif /* __lint */
423 423
424 424 /*
425 425 * Invalidate a single page table entry in the TLB
426 426 */
427 427
428 428 #if defined(__lint)
429 429
430 430 /* ARGSUSED */
431 431 void
432 432 mmu_tlbflush_entry(caddr_t m)
433 433 {}
434 434
435 435 #else /* __lint */
436 436
437 437 #if defined(__amd64)
438 438
439 439 ENTRY(mmu_tlbflush_entry)
440 440 invlpg (%rdi)
441 441 ret
442 442 SET_SIZE(mmu_tlbflush_entry)
443 443
444 444 #elif defined(__i386)
445 445
446 446 ENTRY(mmu_tlbflush_entry)
447 447 movl 4(%esp), %eax
448 448 invlpg (%eax)
449 449 ret
450 450 SET_SIZE(mmu_tlbflush_entry)
451 451
452 452 #endif /* __i386 */
453 453 #endif /* __lint */
454 454
455 455
456 456 /*
457 457 * Get/Set the value of various control registers
458 458 */
459 459
460 460 #if defined(__lint)
461 461
462 462 ulong_t
463 463 getcr0(void)
464 464 { return (0); }
465 465
466 466 /* ARGSUSED */
467 467 void
468 468 setcr0(ulong_t value)
469 469 {}
470 470
471 471 ulong_t
472 472 getcr2(void)
473 473 { return (0); }
474 474
475 475 ulong_t
476 476 getcr3(void)
477 477 { return (0); }
478 478
479 479 #if !defined(__xpv)
480 480 /* ARGSUSED */
481 481 void
482 482 setcr3(ulong_t val)
483 483 {}
484 484
485 485 void
486 486 reload_cr3(void)
487 487 {}
488 488 #endif
489 489
490 490 ulong_t
491 491 getcr4(void)
492 492 { return (0); }
493 493
494 494 /* ARGSUSED */
495 495 void
496 496 setcr4(ulong_t val)
497 497 {}
498 498
499 499 #if defined(__amd64)
500 500
501 501 ulong_t
502 502 getcr8(void)
503 503 { return (0); }
504 504
505 505 /* ARGSUSED */
506 506 void
507 507 setcr8(ulong_t val)
508 508 {}
509 509
510 510 #endif /* __amd64 */
511 511
512 512 #else /* __lint */
513 513
514 514 #if defined(__amd64)
515 515
516 516 ENTRY(getcr0)
517 517 movq %cr0, %rax
518 518 ret
519 519 SET_SIZE(getcr0)
520 520
521 521 ENTRY(setcr0)
522 522 movq %rdi, %cr0
523 523 ret
524 524 SET_SIZE(setcr0)
525 525
526 526 ENTRY(getcr2)
527 527 #if defined(__xpv)
528 528 movq %gs:CPU_VCPU_INFO, %rax
529 529 movq VCPU_INFO_ARCH_CR2(%rax), %rax
530 530 #else
531 531 movq %cr2, %rax
532 532 #endif
533 533 ret
534 534 SET_SIZE(getcr2)
535 535
536 536 ENTRY(getcr3)
537 537 movq %cr3, %rax
538 538 ret
539 539 SET_SIZE(getcr3)
540 540
541 541 #if !defined(__xpv)
542 542
543 543 ENTRY(setcr3)
544 544 movq %rdi, %cr3
545 545 ret
546 546 SET_SIZE(setcr3)
547 547
548 548 ENTRY(reload_cr3)
549 549 movq %cr3, %rdi
550 550 movq %rdi, %cr3
551 551 ret
552 552 SET_SIZE(reload_cr3)
553 553
554 554 #endif /* __xpv */
555 555
556 556 ENTRY(getcr4)
557 557 movq %cr4, %rax
558 558 ret
559 559 SET_SIZE(getcr4)
560 560
561 561 ENTRY(setcr4)
562 562 movq %rdi, %cr4
563 563 ret
564 564 SET_SIZE(setcr4)
565 565
566 566 ENTRY(getcr8)
567 567 movq %cr8, %rax
568 568 ret
569 569 SET_SIZE(getcr8)
570 570
571 571 ENTRY(setcr8)
572 572 movq %rdi, %cr8
573 573 ret
574 574 SET_SIZE(setcr8)
575 575
576 576 #elif defined(__i386)
577 577
578 578 ENTRY(getcr0)
579 579 movl %cr0, %eax
580 580 ret
581 581 SET_SIZE(getcr0)
582 582
583 583 ENTRY(setcr0)
584 584 movl 4(%esp), %eax
585 585 movl %eax, %cr0
586 586 ret
587 587 SET_SIZE(setcr0)
588 588
589 589 /*
590 590 * "lock mov %cr0" is used on processors which indicate it is
591 591 * supported via CPUID. Normally the 32 bit TPR is accessed via
592 592 * the local APIC.
593 593 */
594 594 ENTRY(getcr8)
595 595 lock
596 596 movl %cr0, %eax
597 597 ret
598 598 SET_SIZE(getcr8)
599 599
600 600 ENTRY(setcr8)
601 601 movl 4(%esp), %eax
602 602 lock
603 603 movl %eax, %cr0
604 604 ret
605 605 SET_SIZE(setcr8)
606 606
607 607 ENTRY(getcr2)
608 608 #if defined(__xpv)
609 609 movl %gs:CPU_VCPU_INFO, %eax
610 610 movl VCPU_INFO_ARCH_CR2(%eax), %eax
611 611 #else
612 612 movl %cr2, %eax
613 613 #endif
614 614 ret
615 615 SET_SIZE(getcr2)
616 616
617 617 ENTRY(getcr3)
618 618 movl %cr3, %eax
619 619 ret
620 620 SET_SIZE(getcr3)
621 621
622 622 #if !defined(__xpv)
623 623
624 624 ENTRY(setcr3)
625 625 movl 4(%esp), %eax
626 626 movl %eax, %cr3
627 627 ret
628 628 SET_SIZE(setcr3)
629 629
630 630 ENTRY(reload_cr3)
631 631 movl %cr3, %eax
632 632 movl %eax, %cr3
633 633 ret
634 634 SET_SIZE(reload_cr3)
635 635
636 636 #endif /* __xpv */
637 637
638 638 ENTRY(getcr4)
639 639 movl %cr4, %eax
640 640 ret
641 641 SET_SIZE(getcr4)
642 642
643 643 ENTRY(setcr4)
644 644 movl 4(%esp), %eax
645 645 movl %eax, %cr4
646 646 ret
647 647 SET_SIZE(setcr4)
648 648
649 649 #endif /* __i386 */
650 650 #endif /* __lint */
651 651
652 652 #if defined(__lint)
653 653
654 654 /*ARGSUSED*/
655 655 uint32_t
656 656 __cpuid_insn(struct cpuid_regs *regs)
657 657 { return (0); }
658 658
659 659 #else /* __lint */
660 660
661 661 #if defined(__amd64)
662 662
663 663 ENTRY(__cpuid_insn)
664 664 movq %rbx, %r8
665 665 movq %rcx, %r9
666 666 movq %rdx, %r11
667 667 movl (%rdi), %eax /* %eax = regs->cp_eax */
668 668 movl 0x4(%rdi), %ebx /* %ebx = regs->cp_ebx */
669 669 movl 0x8(%rdi), %ecx /* %ecx = regs->cp_ecx */
670 670 movl 0xc(%rdi), %edx /* %edx = regs->cp_edx */
671 671 cpuid
672 672 movl %eax, (%rdi) /* regs->cp_eax = %eax */
673 673 movl %ebx, 0x4(%rdi) /* regs->cp_ebx = %ebx */
674 674 movl %ecx, 0x8(%rdi) /* regs->cp_ecx = %ecx */
675 675 movl %edx, 0xc(%rdi) /* regs->cp_edx = %edx */
676 676 movq %r8, %rbx
677 677 movq %r9, %rcx
678 678 movq %r11, %rdx
679 679 ret
680 680 SET_SIZE(__cpuid_insn)
681 681
682 682 #elif defined(__i386)
683 683
684 684 ENTRY(__cpuid_insn)
685 685 pushl %ebp
686 686 movl 0x8(%esp), %ebp /* %ebp = regs */
687 687 pushl %ebx
688 688 pushl %ecx
689 689 pushl %edx
690 690 movl (%ebp), %eax /* %eax = regs->cp_eax */
691 691 movl 0x4(%ebp), %ebx /* %ebx = regs->cp_ebx */
692 692 movl 0x8(%ebp), %ecx /* %ecx = regs->cp_ecx */
693 693 movl 0xc(%ebp), %edx /* %edx = regs->cp_edx */
694 694 cpuid
695 695 movl %eax, (%ebp) /* regs->cp_eax = %eax */
696 696 movl %ebx, 0x4(%ebp) /* regs->cp_ebx = %ebx */
697 697 movl %ecx, 0x8(%ebp) /* regs->cp_ecx = %ecx */
698 698 movl %edx, 0xc(%ebp) /* regs->cp_edx = %edx */
699 699 popl %edx
700 700 popl %ecx
701 701 popl %ebx
702 702 popl %ebp
703 703 ret
704 704 SET_SIZE(__cpuid_insn)
705 705
706 706 #endif /* __i386 */
707 707 #endif /* __lint */
708 708
709 709 #if defined(__lint)
710 710
711 711 /*ARGSUSED*/
712 712 void
713 713 i86_monitor(volatile uint32_t *addr, uint32_t extensions, uint32_t hints)
714 714 {}
715 715
716 716 #else /* __lint */
717 717
718 718 #if defined(__amd64)
719 719
720 720 ENTRY_NP(i86_monitor)
721 721 pushq %rbp
722 722 movq %rsp, %rbp
723 723 movq %rdi, %rax /* addr */
724 724 movq %rsi, %rcx /* extensions */
725 725 /* rdx contains input arg3: hints */
726 726 clflush (%rax)
727 727 .byte 0x0f, 0x01, 0xc8 /* monitor */
728 728 leave
729 729 ret
730 730 SET_SIZE(i86_monitor)
731 731
732 732 #elif defined(__i386)
733 733
734 734 ENTRY_NP(i86_monitor)
735 735 pushl %ebp
736 736 movl %esp, %ebp
737 737 movl 0x8(%ebp),%eax /* addr */
738 738 movl 0xc(%ebp),%ecx /* extensions */
739 739 movl 0x10(%ebp),%edx /* hints */
740 740 clflush (%eax)
741 741 .byte 0x0f, 0x01, 0xc8 /* monitor */
742 742 leave
743 743 ret
744 744 SET_SIZE(i86_monitor)
745 745
746 746 #endif /* __i386 */
747 747 #endif /* __lint */
748 748
749 749 #if defined(__lint)
750 750
751 751 /*ARGSUSED*/
752 752 void
753 753 i86_mwait(uint32_t data, uint32_t extensions)
754 754 {}
755 755
756 756 #else /* __lint */
757 757
758 758 #if defined(__amd64)
759 759
760 760 ENTRY_NP(i86_mwait)
761 761 pushq %rbp
762 762 movq %rsp, %rbp
763 763 movq %rdi, %rax /* data */
764 764 movq %rsi, %rcx /* extensions */
765 765 .byte 0x0f, 0x01, 0xc9 /* mwait */
766 766 leave
767 767 ret
768 768 SET_SIZE(i86_mwait)
769 769
770 770 #elif defined(__i386)
771 771
772 772 ENTRY_NP(i86_mwait)
773 773 pushl %ebp
774 774 movl %esp, %ebp
775 775 movl 0x8(%ebp),%eax /* data */
776 776 movl 0xc(%ebp),%ecx /* extensions */
777 777 .byte 0x0f, 0x01, 0xc9 /* mwait */
778 778 leave
779 779 ret
780 780 SET_SIZE(i86_mwait)
781 781
782 782 #endif /* __i386 */
783 783 #endif /* __lint */
784 784
785 785 #if defined(__xpv)
786 786 /*
787 787 * Defined in C
788 788 */
789 789 #else
790 790
791 791 #if defined(__lint)
792 792
793 793 hrtime_t
794 794 tsc_read(void)
795 795 {
796 796 return (0);
797 797 }
798 798
799 799 #else /* __lint */
800 800
801 801 #if defined(__amd64)
802 802
803 803 ENTRY_NP(tsc_read)
804 804 movq %rbx, %r11
805 805 movl $0, %eax
806 806 cpuid
807 807 rdtsc
808 808 movq %r11, %rbx
809 809 shlq $32, %rdx
810 810 orq %rdx, %rax
811 811 ret
812 812 .globl _tsc_mfence_start
813 813 _tsc_mfence_start:
814 814 mfence
815 815 rdtsc
816 816 shlq $32, %rdx
817 817 orq %rdx, %rax
818 818 ret
819 819 .globl _tsc_mfence_end
820 820 _tsc_mfence_end:
821 821 .globl _tscp_start
822 822 _tscp_start:
823 823 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */
824 824 shlq $32, %rdx
825 825 orq %rdx, %rax
826 826 ret
827 827 .globl _tscp_end
828 828 _tscp_end:
829 829 .globl _no_rdtsc_start
830 830 _no_rdtsc_start:
831 831 xorl %edx, %edx
832 832 xorl %eax, %eax
833 833 ret
834 834 .globl _no_rdtsc_end
835 835 _no_rdtsc_end:
836 836 .globl _tsc_lfence_start
837 837 _tsc_lfence_start:
838 838 lfence
839 839 rdtsc
840 840 shlq $32, %rdx
841 841 orq %rdx, %rax
842 842 ret
843 843 .globl _tsc_lfence_end
844 844 _tsc_lfence_end:
845 845 SET_SIZE(tsc_read)
846 846
847 847 #else /* __i386 */
848 848
849 849 ENTRY_NP(tsc_read)
850 850 pushl %ebx
851 851 movl $0, %eax
852 852 cpuid
853 853 rdtsc
854 854 popl %ebx
855 855 ret
856 856 .globl _tsc_mfence_start
857 857 _tsc_mfence_start:
858 858 mfence
859 859 rdtsc
860 860 ret
861 861 .globl _tsc_mfence_end
862 862 _tsc_mfence_end:
863 863 .globl _tscp_start
864 864 _tscp_start:
865 865 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */
866 866 ret
867 867 .globl _tscp_end
868 868 _tscp_end:
869 869 .globl _no_rdtsc_start
870 870 _no_rdtsc_start:
871 871 xorl %edx, %edx
872 872 xorl %eax, %eax
873 873 ret
874 874 .globl _no_rdtsc_end
875 875 _no_rdtsc_end:
876 876 .globl _tsc_lfence_start
877 877 _tsc_lfence_start:
878 878 lfence
879 879 rdtsc
880 880 ret
881 881 .globl _tsc_lfence_end
882 882 _tsc_lfence_end:
883 883 SET_SIZE(tsc_read)
884 884
885 885 #endif /* __i386 */
886 886
887 887 #endif /* __lint */
888 888
889 889
890 890 #endif /* __xpv */
891 891
892 892 #ifdef __lint
893 893 /*
894 894 * Do not use this function for obtaining clock tick. This
895 895 * is called by callers who do not need to have a guarenteed
896 896 * correct tick value. The proper routine to use is tsc_read().
897 897 */
898 898 u_longlong_t
899 899 randtick(void)
900 900 {
901 901 return (0);
902 902 }
903 903 #else
904 904 #if defined(__amd64)
905 905 ENTRY_NP(randtick)
906 906 rdtsc
907 907 shlq $32, %rdx
908 908 orq %rdx, %rax
909 909 ret
910 910 SET_SIZE(randtick)
911 911 #else
912 912 ENTRY_NP(randtick)
913 913 rdtsc
914 914 ret
915 915 SET_SIZE(randtick)
916 916 #endif /* __i386 */
917 917 #endif /* __lint */
918 918 /*
919 919 * Insert entryp after predp in a doubly linked list.
920 920 */
921 921
922 922 #if defined(__lint)
923 923
924 924 /*ARGSUSED*/
925 925 void
926 926 _insque(caddr_t entryp, caddr_t predp)
927 927 {}
928 928
929 929 #else /* __lint */
930 930
931 931 #if defined(__amd64)
932 932
933 933 ENTRY(_insque)
934 934 movq (%rsi), %rax /* predp->forw */
935 935 movq %rsi, CPTRSIZE(%rdi) /* entryp->back = predp */
936 936 movq %rax, (%rdi) /* entryp->forw = predp->forw */
937 937 movq %rdi, (%rsi) /* predp->forw = entryp */
938 938 movq %rdi, CPTRSIZE(%rax) /* predp->forw->back = entryp */
939 939 ret
940 940 SET_SIZE(_insque)
941 941
942 942 #elif defined(__i386)
943 943
944 944 ENTRY(_insque)
945 945 movl 8(%esp), %edx
946 946 movl 4(%esp), %ecx
947 947 movl (%edx), %eax /* predp->forw */
948 948 movl %edx, CPTRSIZE(%ecx) /* entryp->back = predp */
949 949 movl %eax, (%ecx) /* entryp->forw = predp->forw */
950 950 movl %ecx, (%edx) /* predp->forw = entryp */
951 951 movl %ecx, CPTRSIZE(%eax) /* predp->forw->back = entryp */
952 952 ret
953 953 SET_SIZE(_insque)
954 954
955 955 #endif /* __i386 */
956 956 #endif /* __lint */
957 957
958 958 /*
959 959 * Remove entryp from a doubly linked list
960 960 */
961 961
962 962 #if defined(__lint)
963 963
964 964 /*ARGSUSED*/
965 965 void
966 966 _remque(caddr_t entryp)
967 967 {}
968 968
969 969 #else /* __lint */
970 970
971 971 #if defined(__amd64)
972 972
973 973 ENTRY(_remque)
974 974 movq (%rdi), %rax /* entry->forw */
975 975 movq CPTRSIZE(%rdi), %rdx /* entry->back */
976 976 movq %rax, (%rdx) /* entry->back->forw = entry->forw */
977 977 movq %rdx, CPTRSIZE(%rax) /* entry->forw->back = entry->back */
978 978 ret
979 979 SET_SIZE(_remque)
980 980
981 981 #elif defined(__i386)
982 982
983 983 ENTRY(_remque)
984 984 movl 4(%esp), %ecx
985 985 movl (%ecx), %eax /* entry->forw */
986 986 movl CPTRSIZE(%ecx), %edx /* entry->back */
987 987 movl %eax, (%edx) /* entry->back->forw = entry->forw */
988 988 movl %edx, CPTRSIZE(%eax) /* entry->forw->back = entry->back */
989 989 ret
990 990 SET_SIZE(_remque)
991 991
992 992 #endif /* __i386 */
993 993 #endif /* __lint */
994 994
995 995 /*
996 996 * Returns the number of
997 997 * non-NULL bytes in string argument.
998 998 */
999 999
1000 1000 #if defined(__lint)
1001 1001
1002 1002 /* ARGSUSED */
1003 1003 size_t
1004 1004 strlen(const char *str)
1005 1005 { return (0); }
1006 1006
1007 1007 #else /* __lint */
1008 1008
1009 1009 #if defined(__amd64)
1010 1010
1011 1011 /*
1012 1012 * This is close to a simple transliteration of a C version of this
1013 1013 * routine. We should either just -make- this be a C version, or
1014 1014 * justify having it in assembler by making it significantly faster.
1015 1015 *
1016 1016 * size_t
1017 1017 * strlen(const char *s)
1018 1018 * {
1019 1019 * const char *s0;
1020 1020 * #if defined(DEBUG)
1021 1021 * if ((uintptr_t)s < KERNELBASE)
1022 1022 * panic(.str_panic_msg);
1023 1023 * #endif
1024 1024 * for (s0 = s; *s; s++)
1025 1025 * ;
1026 1026 * return (s - s0);
1027 1027 * }
1028 1028 */
1029 1029
1030 1030 ENTRY(strlen)
1031 1031 #ifdef DEBUG
1032 1032 movq postbootkernelbase(%rip), %rax
1033 1033 cmpq %rax, %rdi
1034 1034 jae str_valid
1035 1035 pushq %rbp
1036 1036 movq %rsp, %rbp
1037 1037 leaq .str_panic_msg(%rip), %rdi
1038 1038 xorl %eax, %eax
1039 1039 call panic
1040 1040 #endif /* DEBUG */
1041 1041 str_valid:
1042 1042 cmpb $0, (%rdi)
1043 1043 movq %rdi, %rax
1044 1044 je .null_found
1045 1045 .align 4
1046 1046 .strlen_loop:
1047 1047 incq %rdi
1048 1048 cmpb $0, (%rdi)
1049 1049 jne .strlen_loop
1050 1050 .null_found:
1051 1051 subq %rax, %rdi
1052 1052 movq %rdi, %rax
1053 1053 ret
1054 1054 SET_SIZE(strlen)
1055 1055
1056 1056 #elif defined(__i386)
1057 1057
1058 1058 ENTRY(strlen)
1059 1059 #ifdef DEBUG
1060 1060 movl postbootkernelbase, %eax
1061 1061 cmpl %eax, 4(%esp)
1062 1062 jae str_valid
1063 1063 pushl %ebp
1064 1064 movl %esp, %ebp
1065 1065 pushl $.str_panic_msg
1066 1066 call panic
1067 1067 #endif /* DEBUG */
1068 1068
1069 1069 str_valid:
1070 1070 movl 4(%esp), %eax /* %eax = string address */
1071 1071 testl $3, %eax /* if %eax not word aligned */
1072 1072 jnz .not_word_aligned /* goto .not_word_aligned */
1073 1073 .align 4
1074 1074 .word_aligned:
1075 1075 movl (%eax), %edx /* move 1 word from (%eax) to %edx */
1076 1076 movl $0x7f7f7f7f, %ecx
1077 1077 andl %edx, %ecx /* %ecx = %edx & 0x7f7f7f7f */
1078 1078 addl $4, %eax /* next word */
1079 1079 addl $0x7f7f7f7f, %ecx /* %ecx += 0x7f7f7f7f */
1080 1080 orl %edx, %ecx /* %ecx |= %edx */
1081 1081 andl $0x80808080, %ecx /* %ecx &= 0x80808080 */
1082 1082 cmpl $0x80808080, %ecx /* if no null byte in this word */
1083 1083 je .word_aligned /* goto .word_aligned */
1084 1084 subl $4, %eax /* post-incremented */
1085 1085 .not_word_aligned:
1086 1086 cmpb $0, (%eax) /* if a byte in (%eax) is null */
1087 1087 je .null_found /* goto .null_found */
1088 1088 incl %eax /* next byte */
1089 1089 testl $3, %eax /* if %eax not word aligned */
1090 1090 jnz .not_word_aligned /* goto .not_word_aligned */
1091 1091 jmp .word_aligned /* goto .word_aligned */
1092 1092 .align 4
1093 1093 .null_found:
1094 1094 subl 4(%esp), %eax /* %eax -= string address */
1095 1095 ret
1096 1096 SET_SIZE(strlen)
1097 1097
1098 1098 #endif /* __i386 */
1099 1099
1100 1100 #ifdef DEBUG
1101 1101 .text
1102 1102 .str_panic_msg:
1103 1103 .string "strlen: argument below kernelbase"
1104 1104 #endif /* DEBUG */
1105 1105
1106 1106 #endif /* __lint */
1107 1107
1108 1108 /*
1109 1109 * Berkeley 4.3 introduced symbolically named interrupt levels
1110 1110 * as a way deal with priority in a machine independent fashion.
1111 1111 * Numbered priorities are machine specific, and should be
1112 1112 * discouraged where possible.
1113 1113 *
1114 1114 * Note, for the machine specific priorities there are
1115 1115 * examples listed for devices that use a particular priority.
1116 1116 * It should not be construed that all devices of that
1117 1117 * type should be at that priority. It is currently were
1118 1118 * the current devices fit into the priority scheme based
1119 1119 * upon time criticalness.
1120 1120 *
1121 1121 * The underlying assumption of these assignments is that
1122 1122 * IPL 10 is the highest level from which a device
1123 1123 * routine can call wakeup. Devices that interrupt from higher
1124 1124 * levels are restricted in what they can do. If they need
1125 1125 * kernels services they should schedule a routine at a lower
1126 1126 * level (via software interrupt) to do the required
1127 1127 * processing.
1128 1128 *
1129 1129 * Examples of this higher usage:
1130 1130 * Level Usage
1131 1131 * 14 Profiling clock (and PROM uart polling clock)
1132 1132 * 12 Serial ports
1133 1133 *
1134 1134 * The serial ports request lower level processing on level 6.
1135 1135 *
1136 1136 * Also, almost all splN routines (where N is a number or a
1137 1137 * mnemonic) will do a RAISE(), on the assumption that they are
1138 1138 * never used to lower our priority.
1139 1139 * The exceptions are:
1140 1140 * spl8() Because you can't be above 15 to begin with!
1141 1141 * splzs() Because this is used at boot time to lower our
1142 1142 * priority, to allow the PROM to poll the uart.
1143 1143 * spl0() Used to lower priority to 0.
1144 1144 */
1145 1145
1146 1146 #if defined(__lint)
1147 1147
1148 1148 int spl0(void) { return (0); }
1149 1149 int spl6(void) { return (0); }
1150 1150 int spl7(void) { return (0); }
1151 1151 int spl8(void) { return (0); }
1152 1152 int splhigh(void) { return (0); }
1153 1153 int splhi(void) { return (0); }
1154 1154 int splzs(void) { return (0); }
1155 1155
1156 1156 /* ARGSUSED */
1157 1157 void
1158 1158 splx(int level)
1159 1159 {}
1160 1160
1161 1161 #else /* __lint */
1162 1162
1163 1163 #if defined(__amd64)
1164 1164
1165 1165 #define SETPRI(level) \
1166 1166 movl $/**/level, %edi; /* new priority */ \
1167 1167 jmp do_splx /* redirect to do_splx */
1168 1168
1169 1169 #define RAISE(level) \
1170 1170 movl $/**/level, %edi; /* new priority */ \
1171 1171 jmp splr /* redirect to splr */
1172 1172
1173 1173 #elif defined(__i386)
1174 1174
1175 1175 #define SETPRI(level) \
1176 1176 pushl $/**/level; /* new priority */ \
1177 1177 call do_splx; /* invoke common splx code */ \
1178 1178 addl $4, %esp; /* unstack arg */ \
1179 1179 ret
1180 1180
1181 1181 #define RAISE(level) \
1182 1182 pushl $/**/level; /* new priority */ \
1183 1183 call splr; /* invoke common splr code */ \
1184 1184 addl $4, %esp; /* unstack args */ \
1185 1185 ret
1186 1186
1187 1187 #endif /* __i386 */
1188 1188
1189 1189 /* locks out all interrupts, including memory errors */
1190 1190 ENTRY(spl8)
1191 1191 SETPRI(15)
1192 1192 SET_SIZE(spl8)
1193 1193
1194 1194 /* just below the level that profiling runs */
1195 1195 ENTRY(spl7)
1196 1196 RAISE(13)
1197 1197 SET_SIZE(spl7)
1198 1198
1199 1199 /* sun specific - highest priority onboard serial i/o asy ports */
1200 1200 ENTRY(splzs)
1201 1201 SETPRI(12) /* Can't be a RAISE, as it's used to lower us */
1202 1202 SET_SIZE(splzs)
1203 1203
1204 1204 ENTRY(splhi)
1205 1205 ALTENTRY(splhigh)
1206 1206 ALTENTRY(spl6)
1207 1207 ALTENTRY(i_ddi_splhigh)
1208 1208
1209 1209 RAISE(DISP_LEVEL)
1210 1210
1211 1211 SET_SIZE(i_ddi_splhigh)
1212 1212 SET_SIZE(spl6)
1213 1213 SET_SIZE(splhigh)
1214 1214 SET_SIZE(splhi)
1215 1215
1216 1216 /* allow all interrupts */
1217 1217 ENTRY(spl0)
1218 1218 SETPRI(0)
1219 1219 SET_SIZE(spl0)
1220 1220
1221 1221
1222 1222 /* splx implementation */
1223 1223 ENTRY(splx)
1224 1224 jmp do_splx /* redirect to common splx code */
1225 1225 SET_SIZE(splx)
1226 1226
1227 1227 #endif /* __lint */
1228 1228
1229 1229 #if defined(__i386)
1230 1230
1231 1231 /*
1232 1232 * Read and write the %gs register
1233 1233 */
1234 1234
1235 1235 #if defined(__lint)
1236 1236
1237 1237 /*ARGSUSED*/
1238 1238 uint16_t
1239 1239 getgs(void)
1240 1240 { return (0); }
1241 1241
1242 1242 /*ARGSUSED*/
1243 1243 void
1244 1244 setgs(uint16_t sel)
1245 1245 {}
1246 1246
1247 1247 #else /* __lint */
1248 1248
1249 1249 ENTRY(getgs)
1250 1250 clr %eax
1251 1251 movw %gs, %ax
1252 1252 ret
1253 1253 SET_SIZE(getgs)
1254 1254
1255 1255 ENTRY(setgs)
1256 1256 movw 4(%esp), %gs
1257 1257 ret
1258 1258 SET_SIZE(setgs)
1259 1259
1260 1260 #endif /* __lint */
1261 1261 #endif /* __i386 */
1262 1262
1263 1263 #if defined(__lint)
1264 1264
1265 1265 void
1266 1266 pc_reset(void)
1267 1267 {}
1268 1268
1269 1269 void
1270 1270 efi_reset(void)
1271 1271 {}
1272 1272
1273 1273 #else /* __lint */
1274 1274
1275 1275 ENTRY(wait_500ms)
1276 1276 #if defined(__amd64)
1277 1277 pushq %rbx
1278 1278 #elif defined(__i386)
1279 1279 push %ebx
1280 1280 #endif
1281 1281 movl $50000, %ebx
1282 1282 1:
1283 1283 call tenmicrosec
1284 1284 decl %ebx
1285 1285 jnz 1b
1286 1286 #if defined(__amd64)
1287 1287 popq %rbx
1288 1288 #elif defined(__i386)
1289 1289 pop %ebx
1290 1290 #endif
1291 1291 ret
1292 1292 SET_SIZE(wait_500ms)
1293 1293
1294 1294 #define RESET_METHOD_KBC 1
1295 1295 #define RESET_METHOD_PORT92 2
1296 1296 #define RESET_METHOD_PCI 4
1297 1297
1298 1298 DGDEF3(pc_reset_methods, 4, 8)
1299 1299 .long RESET_METHOD_KBC|RESET_METHOD_PORT92|RESET_METHOD_PCI;
1300 1300
1301 1301 ENTRY(pc_reset)
1302 1302
1303 1303 #if defined(__i386)
1304 1304 testl $RESET_METHOD_KBC, pc_reset_methods
1305 1305 #elif defined(__amd64)
1306 1306 testl $RESET_METHOD_KBC, pc_reset_methods(%rip)
1307 1307 #endif
1308 1308 jz 1f
1309 1309
1310 1310 /
1311 1311 / Try the classic keyboard controller-triggered reset.
1312 1312 /
1313 1313 movw $0x64, %dx
1314 1314 movb $0xfe, %al
1315 1315 outb (%dx)
1316 1316
1317 1317 / Wait up to 500 milliseconds here for the keyboard controller
1318 1318 / to pull the reset line. On some systems where the keyboard
1319 1319 / controller is slow to pull the reset line, the next reset method
1320 1320 / may be executed (which may be bad if those systems hang when the
1321 1321 / next reset method is used, e.g. Ferrari 3400 (doesn't like port 92),
1322 1322 / and Ferrari 4000 (doesn't like the cf9 reset method))
1323 1323
1324 1324 call wait_500ms
1325 1325
1326 1326 1:
1327 1327 #if defined(__i386)
1328 1328 testl $RESET_METHOD_PORT92, pc_reset_methods
1329 1329 #elif defined(__amd64)
1330 1330 testl $RESET_METHOD_PORT92, pc_reset_methods(%rip)
1331 1331 #endif
1332 1332 jz 3f
1333 1333
1334 1334 /
1335 1335 / Try port 0x92 fast reset
1336 1336 /
1337 1337 movw $0x92, %dx
1338 1338 inb (%dx)
1339 1339 cmpb $0xff, %al / If port's not there, we should get back 0xFF
1340 1340 je 1f
1341 1341 testb $1, %al / If bit 0
1342 1342 jz 2f / is clear, jump to perform the reset
1343 1343 andb $0xfe, %al / otherwise,
1344 1344 outb (%dx) / clear bit 0 first, then
1345 1345 2:
1346 1346 orb $1, %al / Set bit 0
1347 1347 outb (%dx) / and reset the system
1348 1348 1:
1349 1349
1350 1350 call wait_500ms
1351 1351
1352 1352 3:
1353 1353 #if defined(__i386)
1354 1354 testl $RESET_METHOD_PCI, pc_reset_methods
1355 1355 #elif defined(__amd64)
1356 1356 testl $RESET_METHOD_PCI, pc_reset_methods(%rip)
1357 1357 #endif
1358 1358 jz 4f
1359 1359
1360 1360 / Try the PCI (soft) reset vector (should work on all modern systems,
1361 1361 / but has been shown to cause problems on 450NX systems, and some newer
1362 1362 / systems (e.g. ATI IXP400-equipped systems))
1363 1363 / When resetting via this method, 2 writes are required. The first
1364 1364 / targets bit 1 (0=hard reset without power cycle, 1=hard reset with
1365 1365 / power cycle).
1366 1366 / The reset occurs on the second write, during bit 2's transition from
1367 1367 / 0->1.
1368 1368 movw $0xcf9, %dx
1369 1369 movb $0x2, %al / Reset mode = hard, no power cycle
1370 1370 outb (%dx)
1371 1371 movb $0x6, %al
1372 1372 outb (%dx)
1373 1373
1374 1374 call wait_500ms
1375 1375
1376 1376 4:
1377 1377 /
1378 1378 / port 0xcf9 failed also. Last-ditch effort is to
1379 1379 / triple-fault the CPU.
1380 1380 / Also, use triple fault for EFI firmware
1381 1381 /
1382 1382 ENTRY(efi_reset)
1383 1383 #if defined(__amd64)
1384 1384 pushq $0x0
1385 1385 pushq $0x0 / IDT base of 0, limit of 0 + 2 unused bytes
1386 1386 lidt (%rsp)
1387 1387 #elif defined(__i386)
1388 1388 pushl $0x0
1389 1389 pushl $0x0 / IDT base of 0, limit of 0 + 2 unused bytes
1390 1390 lidt (%esp)
1391 1391 #endif
1392 1392 int $0x0 / Trigger interrupt, generate triple-fault
1393 1393
1394 1394 cli
1395 1395 hlt / Wait forever
1396 1396 /*NOTREACHED*/
1397 1397 SET_SIZE(efi_reset)
1398 1398 SET_SIZE(pc_reset)
1399 1399
1400 1400 #endif /* __lint */
1401 1401
1402 1402 /*
1403 1403 * C callable in and out routines
1404 1404 */
1405 1405
1406 1406 #if defined(__lint)
1407 1407
1408 1408 /* ARGSUSED */
1409 1409 void
1410 1410 outl(int port_address, uint32_t val)
1411 1411 {}
1412 1412
1413 1413 #else /* __lint */
1414 1414
1415 1415 #if defined(__amd64)
1416 1416
1417 1417 ENTRY(outl)
1418 1418 movw %di, %dx
1419 1419 movl %esi, %eax
1420 1420 outl (%dx)
1421 1421 ret
1422 1422 SET_SIZE(outl)
1423 1423
1424 1424 #elif defined(__i386)
1425 1425
1426 1426 .set PORT, 4
1427 1427 .set VAL, 8
1428 1428
1429 1429 ENTRY(outl)
1430 1430 movw PORT(%esp), %dx
1431 1431 movl VAL(%esp), %eax
1432 1432 outl (%dx)
1433 1433 ret
1434 1434 SET_SIZE(outl)
1435 1435
1436 1436 #endif /* __i386 */
1437 1437 #endif /* __lint */
1438 1438
1439 1439 #if defined(__lint)
1440 1440
1441 1441 /* ARGSUSED */
1442 1442 void
1443 1443 outw(int port_address, uint16_t val)
1444 1444 {}
1445 1445
1446 1446 #else /* __lint */
1447 1447
1448 1448 #if defined(__amd64)
1449 1449
1450 1450 ENTRY(outw)
1451 1451 movw %di, %dx
1452 1452 movw %si, %ax
1453 1453 D16 outl (%dx) /* XX64 why not outw? */
1454 1454 ret
1455 1455 SET_SIZE(outw)
1456 1456
1457 1457 #elif defined(__i386)
1458 1458
1459 1459 ENTRY(outw)
1460 1460 movw PORT(%esp), %dx
1461 1461 movw VAL(%esp), %ax
1462 1462 D16 outl (%dx)
1463 1463 ret
1464 1464 SET_SIZE(outw)
1465 1465
1466 1466 #endif /* __i386 */
1467 1467 #endif /* __lint */
1468 1468
1469 1469 #if defined(__lint)
1470 1470
1471 1471 /* ARGSUSED */
1472 1472 void
1473 1473 outb(int port_address, uint8_t val)
1474 1474 {}
1475 1475
1476 1476 #else /* __lint */
1477 1477
1478 1478 #if defined(__amd64)
1479 1479
1480 1480 ENTRY(outb)
1481 1481 movw %di, %dx
1482 1482 movb %sil, %al
1483 1483 outb (%dx)
1484 1484 ret
1485 1485 SET_SIZE(outb)
1486 1486
1487 1487 #elif defined(__i386)
1488 1488
1489 1489 ENTRY(outb)
1490 1490 movw PORT(%esp), %dx
1491 1491 movb VAL(%esp), %al
1492 1492 outb (%dx)
1493 1493 ret
1494 1494 SET_SIZE(outb)
1495 1495
1496 1496 #endif /* __i386 */
1497 1497 #endif /* __lint */
1498 1498
1499 1499 #if defined(__lint)
1500 1500
1501 1501 /* ARGSUSED */
1502 1502 uint32_t
1503 1503 inl(int port_address)
1504 1504 { return (0); }
1505 1505
1506 1506 #else /* __lint */
1507 1507
1508 1508 #if defined(__amd64)
1509 1509
1510 1510 ENTRY(inl)
1511 1511 xorl %eax, %eax
1512 1512 movw %di, %dx
1513 1513 inl (%dx)
1514 1514 ret
1515 1515 SET_SIZE(inl)
1516 1516
1517 1517 #elif defined(__i386)
1518 1518
1519 1519 ENTRY(inl)
1520 1520 movw PORT(%esp), %dx
1521 1521 inl (%dx)
1522 1522 ret
1523 1523 SET_SIZE(inl)
1524 1524
1525 1525 #endif /* __i386 */
1526 1526 #endif /* __lint */
1527 1527
1528 1528 #if defined(__lint)
1529 1529
1530 1530 /* ARGSUSED */
1531 1531 uint16_t
1532 1532 inw(int port_address)
1533 1533 { return (0); }
1534 1534
1535 1535 #else /* __lint */
1536 1536
1537 1537 #if defined(__amd64)
1538 1538
1539 1539 ENTRY(inw)
1540 1540 xorl %eax, %eax
1541 1541 movw %di, %dx
1542 1542 D16 inl (%dx)
1543 1543 ret
1544 1544 SET_SIZE(inw)
1545 1545
1546 1546 #elif defined(__i386)
1547 1547
1548 1548 ENTRY(inw)
1549 1549 subl %eax, %eax
1550 1550 movw PORT(%esp), %dx
1551 1551 D16 inl (%dx)
1552 1552 ret
1553 1553 SET_SIZE(inw)
1554 1554
1555 1555 #endif /* __i386 */
1556 1556 #endif /* __lint */
1557 1557
1558 1558
1559 1559 #if defined(__lint)
1560 1560
1561 1561 /* ARGSUSED */
1562 1562 uint8_t
1563 1563 inb(int port_address)
1564 1564 { return (0); }
1565 1565
1566 1566 #else /* __lint */
1567 1567
1568 1568 #if defined(__amd64)
1569 1569
1570 1570 ENTRY(inb)
1571 1571 xorl %eax, %eax
1572 1572 movw %di, %dx
1573 1573 inb (%dx)
1574 1574 ret
1575 1575 SET_SIZE(inb)
1576 1576
1577 1577 #elif defined(__i386)
1578 1578
1579 1579 ENTRY(inb)
1580 1580 subl %eax, %eax
1581 1581 movw PORT(%esp), %dx
1582 1582 inb (%dx)
1583 1583 ret
1584 1584 SET_SIZE(inb)
1585 1585
1586 1586 #endif /* __i386 */
1587 1587 #endif /* __lint */
1588 1588
1589 1589
1590 1590 #if defined(__lint)
1591 1591
1592 1592 /* ARGSUSED */
1593 1593 void
1594 1594 repoutsw(int port, uint16_t *addr, int cnt)
1595 1595 {}
1596 1596
1597 1597 #else /* __lint */
1598 1598
1599 1599 #if defined(__amd64)
1600 1600
1601 1601 ENTRY(repoutsw)
1602 1602 movl %edx, %ecx
1603 1603 movw %di, %dx
1604 1604 rep
1605 1605 D16 outsl
1606 1606 ret
1607 1607 SET_SIZE(repoutsw)
1608 1608
1609 1609 #elif defined(__i386)
1610 1610
1611 1611 /*
1612 1612 * The arguments and saved registers are on the stack in the
1613 1613 * following order:
1614 1614 * | cnt | +16
1615 1615 * | *addr | +12
1616 1616 * | port | +8
1617 1617 * | eip | +4
1618 1618 * | esi | <-- %esp
1619 1619 * If additional values are pushed onto the stack, make sure
1620 1620 * to adjust the following constants accordingly.
1621 1621 */
1622 1622 .set PORT, 8
1623 1623 .set ADDR, 12
1624 1624 .set COUNT, 16
1625 1625
1626 1626 ENTRY(repoutsw)
1627 1627 pushl %esi
1628 1628 movl PORT(%esp), %edx
1629 1629 movl ADDR(%esp), %esi
1630 1630 movl COUNT(%esp), %ecx
1631 1631 rep
1632 1632 D16 outsl
1633 1633 popl %esi
1634 1634 ret
1635 1635 SET_SIZE(repoutsw)
1636 1636
1637 1637 #endif /* __i386 */
1638 1638 #endif /* __lint */
1639 1639
1640 1640
1641 1641 #if defined(__lint)
1642 1642
1643 1643 /* ARGSUSED */
1644 1644 void
1645 1645 repinsw(int port_addr, uint16_t *addr, int cnt)
1646 1646 {}
1647 1647
1648 1648 #else /* __lint */
1649 1649
1650 1650 #if defined(__amd64)
1651 1651
1652 1652 ENTRY(repinsw)
1653 1653 movl %edx, %ecx
1654 1654 movw %di, %dx
1655 1655 rep
1656 1656 D16 insl
1657 1657 ret
1658 1658 SET_SIZE(repinsw)
1659 1659
1660 1660 #elif defined(__i386)
1661 1661
1662 1662 ENTRY(repinsw)
1663 1663 pushl %edi
1664 1664 movl PORT(%esp), %edx
1665 1665 movl ADDR(%esp), %edi
1666 1666 movl COUNT(%esp), %ecx
1667 1667 rep
1668 1668 D16 insl
1669 1669 popl %edi
1670 1670 ret
1671 1671 SET_SIZE(repinsw)
1672 1672
1673 1673 #endif /* __i386 */
1674 1674 #endif /* __lint */
1675 1675
1676 1676
1677 1677 #if defined(__lint)
1678 1678
1679 1679 /* ARGSUSED */
1680 1680 void
1681 1681 repinsb(int port, uint8_t *addr, int count)
1682 1682 {}
1683 1683
1684 1684 #else /* __lint */
1685 1685
1686 1686 #if defined(__amd64)
1687 1687
1688 1688 ENTRY(repinsb)
1689 1689 movl %edx, %ecx
1690 1690 movw %di, %dx
1691 1691 movq %rsi, %rdi
1692 1692 rep
1693 1693 insb
1694 1694 ret
1695 1695 SET_SIZE(repinsb)
1696 1696
1697 1697 #elif defined(__i386)
1698 1698
1699 1699 /*
1700 1700 * The arguments and saved registers are on the stack in the
1701 1701 * following order:
1702 1702 * | cnt | +16
1703 1703 * | *addr | +12
1704 1704 * | port | +8
1705 1705 * | eip | +4
1706 1706 * | esi | <-- %esp
1707 1707 * If additional values are pushed onto the stack, make sure
1708 1708 * to adjust the following constants accordingly.
1709 1709 */
1710 1710 .set IO_PORT, 8
1711 1711 .set IO_ADDR, 12
1712 1712 .set IO_COUNT, 16
1713 1713
1714 1714 ENTRY(repinsb)
1715 1715 pushl %edi
1716 1716 movl IO_ADDR(%esp), %edi
1717 1717 movl IO_COUNT(%esp), %ecx
1718 1718 movl IO_PORT(%esp), %edx
1719 1719 rep
1720 1720 insb
1721 1721 popl %edi
1722 1722 ret
1723 1723 SET_SIZE(repinsb)
1724 1724
1725 1725 #endif /* __i386 */
1726 1726 #endif /* __lint */
1727 1727
1728 1728
1729 1729 /*
1730 1730 * Input a stream of 32-bit words.
1731 1731 * NOTE: count is a DWORD count.
1732 1732 */
1733 1733 #if defined(__lint)
1734 1734
1735 1735 /* ARGSUSED */
1736 1736 void
1737 1737 repinsd(int port, uint32_t *addr, int count)
1738 1738 {}
1739 1739
1740 1740 #else /* __lint */
1741 1741
1742 1742 #if defined(__amd64)
1743 1743
1744 1744 ENTRY(repinsd)
1745 1745 movl %edx, %ecx
1746 1746 movw %di, %dx
1747 1747 movq %rsi, %rdi
1748 1748 rep
1749 1749 insl
1750 1750 ret
1751 1751 SET_SIZE(repinsd)
1752 1752
1753 1753 #elif defined(__i386)
1754 1754
1755 1755 ENTRY(repinsd)
1756 1756 pushl %edi
1757 1757 movl IO_ADDR(%esp), %edi
1758 1758 movl IO_COUNT(%esp), %ecx
1759 1759 movl IO_PORT(%esp), %edx
1760 1760 rep
1761 1761 insl
1762 1762 popl %edi
1763 1763 ret
1764 1764 SET_SIZE(repinsd)
1765 1765
1766 1766 #endif /* __i386 */
1767 1767 #endif /* __lint */
1768 1768
1769 1769 /*
1770 1770 * Output a stream of bytes
1771 1771 * NOTE: count is a byte count
1772 1772 */
1773 1773 #if defined(__lint)
1774 1774
1775 1775 /* ARGSUSED */
1776 1776 void
1777 1777 repoutsb(int port, uint8_t *addr, int count)
1778 1778 {}
1779 1779
1780 1780 #else /* __lint */
1781 1781
1782 1782 #if defined(__amd64)
1783 1783
1784 1784 ENTRY(repoutsb)
1785 1785 movl %edx, %ecx
1786 1786 movw %di, %dx
1787 1787 rep
1788 1788 outsb
1789 1789 ret
1790 1790 SET_SIZE(repoutsb)
1791 1791
1792 1792 #elif defined(__i386)
1793 1793
1794 1794 ENTRY(repoutsb)
1795 1795 pushl %esi
1796 1796 movl IO_ADDR(%esp), %esi
1797 1797 movl IO_COUNT(%esp), %ecx
1798 1798 movl IO_PORT(%esp), %edx
1799 1799 rep
1800 1800 outsb
1801 1801 popl %esi
1802 1802 ret
1803 1803 SET_SIZE(repoutsb)
1804 1804
1805 1805 #endif /* __i386 */
1806 1806 #endif /* __lint */
1807 1807
1808 1808 /*
1809 1809 * Output a stream of 32-bit words
1810 1810 * NOTE: count is a DWORD count
1811 1811 */
1812 1812 #if defined(__lint)
1813 1813
1814 1814 /* ARGSUSED */
1815 1815 void
1816 1816 repoutsd(int port, uint32_t *addr, int count)
1817 1817 {}
1818 1818
1819 1819 #else /* __lint */
1820 1820
1821 1821 #if defined(__amd64)
1822 1822
1823 1823 ENTRY(repoutsd)
1824 1824 movl %edx, %ecx
1825 1825 movw %di, %dx
1826 1826 rep
1827 1827 outsl
1828 1828 ret
1829 1829 SET_SIZE(repoutsd)
1830 1830
1831 1831 #elif defined(__i386)
1832 1832
1833 1833 ENTRY(repoutsd)
1834 1834 pushl %esi
1835 1835 movl IO_ADDR(%esp), %esi
1836 1836 movl IO_COUNT(%esp), %ecx
1837 1837 movl IO_PORT(%esp), %edx
1838 1838 rep
1839 1839 outsl
1840 1840 popl %esi
1841 1841 ret
1842 1842 SET_SIZE(repoutsd)
1843 1843
1844 1844 #endif /* __i386 */
1845 1845 #endif /* __lint */
1846 1846
1847 1847 /*
1848 1848 * void int3(void)
1849 1849 * void int18(void)
1850 1850 * void int20(void)
1851 1851 * void int_cmci(void)
1852 1852 */
1853 1853
1854 1854 #if defined(__lint)
1855 1855
1856 1856 void
1857 1857 int3(void)
1858 1858 {}
1859 1859
1860 1860 void
1861 1861 int18(void)
1862 1862 {}
1863 1863
1864 1864 void
1865 1865 int20(void)
1866 1866 {}
1867 1867
1868 1868 void
1869 1869 int_cmci(void)
1870 1870 {}
1871 1871
1872 1872 #else /* __lint */
1873 1873
1874 1874 ENTRY(int3)
1875 1875 int $T_BPTFLT
1876 1876 ret
1877 1877 SET_SIZE(int3)
1878 1878
1879 1879 ENTRY(int18)
1880 1880 int $T_MCE
1881 1881 ret
1882 1882 SET_SIZE(int18)
1883 1883
1884 1884 ENTRY(int20)
1885 1885 movl boothowto, %eax
1886 1886 andl $RB_DEBUG, %eax
1887 1887 jz 1f
1888 1888
1889 1889 int $T_DBGENTR
1890 1890 1:
1891 1891 rep; ret /* use 2 byte return instruction when branch target */
1892 1892 /* AMD Software Optimization Guide - Section 6.2 */
1893 1893 SET_SIZE(int20)
1894 1894
1895 1895 ENTRY(int_cmci)
1896 1896 int $T_ENOEXTFLT
1897 1897 ret
1898 1898 SET_SIZE(int_cmci)
1899 1899
1900 1900 #endif /* __lint */
1901 1901
1902 1902 #if defined(__lint)
1903 1903
1904 1904 /* ARGSUSED */
1905 1905 int
1906 1906 scanc(size_t size, uchar_t *cp, uchar_t *table, uchar_t mask)
1907 1907 { return (0); }
1908 1908
1909 1909 #else /* __lint */
1910 1910
1911 1911 #if defined(__amd64)
1912 1912
1913 1913 ENTRY(scanc)
1914 1914 /* rdi == size */
1915 1915 /* rsi == cp */
1916 1916 /* rdx == table */
1917 1917 /* rcx == mask */
1918 1918 addq %rsi, %rdi /* end = &cp[size] */
1919 1919 .scanloop:
1920 1920 cmpq %rdi, %rsi /* while (cp < end */
1921 1921 jnb .scandone
1922 1922 movzbq (%rsi), %r8 /* %r8 = *cp */
1923 1923 incq %rsi /* cp++ */
1924 1924 testb %cl, (%r8, %rdx)
1925 1925 jz .scanloop /* && (table[*cp] & mask) == 0) */
1926 1926 decq %rsi /* (fix post-increment) */
1927 1927 .scandone:
1928 1928 movl %edi, %eax
1929 1929 subl %esi, %eax /* return (end - cp) */
1930 1930 ret
1931 1931 SET_SIZE(scanc)
1932 1932
1933 1933 #elif defined(__i386)
1934 1934
1935 1935 ENTRY(scanc)
1936 1936 pushl %edi
1937 1937 pushl %esi
1938 1938 movb 24(%esp), %cl /* mask = %cl */
1939 1939 movl 16(%esp), %esi /* cp = %esi */
1940 1940 movl 20(%esp), %edx /* table = %edx */
1941 1941 movl %esi, %edi
1942 1942 addl 12(%esp), %edi /* end = &cp[size]; */
1943 1943 .scanloop:
1944 1944 cmpl %edi, %esi /* while (cp < end */
1945 1945 jnb .scandone
1946 1946 movzbl (%esi), %eax /* %al = *cp */
1947 1947 incl %esi /* cp++ */
1948 1948 movb (%edx, %eax), %al /* %al = table[*cp] */
1949 1949 testb %al, %cl
1950 1950 jz .scanloop /* && (table[*cp] & mask) == 0) */
1951 1951 dec %esi /* post-incremented */
1952 1952 .scandone:
1953 1953 movl %edi, %eax
1954 1954 subl %esi, %eax /* return (end - cp) */
1955 1955 popl %esi
1956 1956 popl %edi
1957 1957 ret
1958 1958 SET_SIZE(scanc)
1959 1959
1960 1960 #endif /* __i386 */
1961 1961 #endif /* __lint */
1962 1962
1963 1963 /*
1964 1964 * Replacement functions for ones that are normally inlined.
1965 1965 * In addition to the copy in i86.il, they are defined here just in case.
1966 1966 */
1967 1967
1968 1968 #if defined(__lint)
1969 1969
1970 1970 ulong_t
1971 1971 intr_clear(void)
1972 1972 { return (0); }
1973 1973
1974 1974 ulong_t
1975 1975 clear_int_flag(void)
1976 1976 { return (0); }
1977 1977
1978 1978 #else /* __lint */
1979 1979
1980 1980 #if defined(__amd64)
1981 1981
1982 1982 ENTRY(intr_clear)
1983 1983 ENTRY(clear_int_flag)
1984 1984 pushfq
1985 1985 popq %rax
1986 1986 #if defined(__xpv)
1987 1987 leaq xpv_panicking, %rdi
1988 1988 movl (%rdi), %edi
1989 1989 cmpl $0, %edi
1990 1990 jne 2f
1991 1991 CLIRET(%rdi, %dl) /* returns event mask in %dl */
1992 1992 /*
1993 1993 * Synthesize the PS_IE bit from the event mask bit
1994 1994 */
1995 1995 andq $_BITNOT(PS_IE), %rax
1996 1996 testb $1, %dl
1997 1997 jnz 1f
1998 1998 orq $PS_IE, %rax
1999 1999 1:
2000 2000 ret
2001 2001 2:
2002 2002 #endif
2003 2003 CLI(%rdi)
2004 2004 ret
2005 2005 SET_SIZE(clear_int_flag)
2006 2006 SET_SIZE(intr_clear)
2007 2007
2008 2008 #elif defined(__i386)
2009 2009
2010 2010 ENTRY(intr_clear)
2011 2011 ENTRY(clear_int_flag)
2012 2012 pushfl
2013 2013 popl %eax
2014 2014 #if defined(__xpv)
2015 2015 leal xpv_panicking, %edx
2016 2016 movl (%edx), %edx
2017 2017 cmpl $0, %edx
2018 2018 jne 2f
2019 2019 CLIRET(%edx, %cl) /* returns event mask in %cl */
2020 2020 /*
2021 2021 * Synthesize the PS_IE bit from the event mask bit
2022 2022 */
2023 2023 andl $_BITNOT(PS_IE), %eax
2024 2024 testb $1, %cl
2025 2025 jnz 1f
2026 2026 orl $PS_IE, %eax
2027 2027 1:
2028 2028 ret
2029 2029 2:
2030 2030 #endif
2031 2031 CLI(%edx)
2032 2032 ret
2033 2033 SET_SIZE(clear_int_flag)
2034 2034 SET_SIZE(intr_clear)
2035 2035
2036 2036 #endif /* __i386 */
2037 2037 #endif /* __lint */
2038 2038
2039 2039 #if defined(__lint)
2040 2040
2041 2041 struct cpu *
2042 2042 curcpup(void)
2043 2043 { return 0; }
2044 2044
2045 2045 #else /* __lint */
2046 2046
2047 2047 #if defined(__amd64)
2048 2048
2049 2049 ENTRY(curcpup)
2050 2050 movq %gs:CPU_SELF, %rax
2051 2051 ret
2052 2052 SET_SIZE(curcpup)
2053 2053
2054 2054 #elif defined(__i386)
2055 2055
2056 2056 ENTRY(curcpup)
2057 2057 movl %gs:CPU_SELF, %eax
2058 2058 ret
2059 2059 SET_SIZE(curcpup)
2060 2060
2061 2061 #endif /* __i386 */
2062 2062 #endif /* __lint */
2063 2063
2064 2064 /* htonll(), ntohll(), htonl(), ntohl(), htons(), ntohs()
2065 2065 * These functions reverse the byte order of the input parameter and returns
2066 2066 * the result. This is to convert the byte order from host byte order
2067 2067 * (little endian) to network byte order (big endian), or vice versa.
2068 2068 */
2069 2069
2070 2070 #if defined(__lint)
2071 2071
2072 2072 uint64_t
2073 2073 htonll(uint64_t i)
2074 2074 { return (i); }
2075 2075
2076 2076 uint64_t
2077 2077 ntohll(uint64_t i)
2078 2078 { return (i); }
2079 2079
2080 2080 uint32_t
2081 2081 htonl(uint32_t i)
2082 2082 { return (i); }
2083 2083
2084 2084 uint32_t
2085 2085 ntohl(uint32_t i)
2086 2086 { return (i); }
2087 2087
2088 2088 uint16_t
2089 2089 htons(uint16_t i)
2090 2090 { return (i); }
2091 2091
2092 2092 uint16_t
2093 2093 ntohs(uint16_t i)
2094 2094 { return (i); }
2095 2095
2096 2096 #else /* __lint */
2097 2097
2098 2098 #if defined(__amd64)
2099 2099
2100 2100 ENTRY(htonll)
2101 2101 ALTENTRY(ntohll)
2102 2102 movq %rdi, %rax
2103 2103 bswapq %rax
2104 2104 ret
2105 2105 SET_SIZE(ntohll)
2106 2106 SET_SIZE(htonll)
2107 2107
2108 2108 /* XX64 there must be shorter sequences for this */
2109 2109 ENTRY(htonl)
2110 2110 ALTENTRY(ntohl)
2111 2111 movl %edi, %eax
2112 2112 bswap %eax
2113 2113 ret
2114 2114 SET_SIZE(ntohl)
2115 2115 SET_SIZE(htonl)
2116 2116
2117 2117 /* XX64 there must be better sequences for this */
2118 2118 ENTRY(htons)
2119 2119 ALTENTRY(ntohs)
2120 2120 movl %edi, %eax
2121 2121 bswap %eax
2122 2122 shrl $16, %eax
2123 2123 ret
2124 2124 SET_SIZE(ntohs)
2125 2125 SET_SIZE(htons)
2126 2126
2127 2127 #elif defined(__i386)
2128 2128
2129 2129 ENTRY(htonll)
2130 2130 ALTENTRY(ntohll)
2131 2131 movl 4(%esp), %edx
2132 2132 movl 8(%esp), %eax
2133 2133 bswap %edx
2134 2134 bswap %eax
2135 2135 ret
2136 2136 SET_SIZE(ntohll)
2137 2137 SET_SIZE(htonll)
2138 2138
2139 2139 ENTRY(htonl)
2140 2140 ALTENTRY(ntohl)
2141 2141 movl 4(%esp), %eax
2142 2142 bswap %eax
2143 2143 ret
2144 2144 SET_SIZE(ntohl)
2145 2145 SET_SIZE(htonl)
2146 2146
2147 2147 ENTRY(htons)
2148 2148 ALTENTRY(ntohs)
2149 2149 movl 4(%esp), %eax
2150 2150 bswap %eax
2151 2151 shrl $16, %eax
2152 2152 ret
2153 2153 SET_SIZE(ntohs)
2154 2154 SET_SIZE(htons)
2155 2155
2156 2156 #endif /* __i386 */
2157 2157 #endif /* __lint */
2158 2158
2159 2159
2160 2160 #if defined(__lint)
2161 2161
2162 2162 /* ARGSUSED */
2163 2163 void
2164 2164 intr_restore(ulong_t i)
2165 2165 { return; }
2166 2166
2167 2167 /* ARGSUSED */
2168 2168 void
2169 2169 restore_int_flag(ulong_t i)
2170 2170 { return; }
2171 2171
2172 2172 #else /* __lint */
2173 2173
2174 2174 #if defined(__amd64)
2175 2175
2176 2176 ENTRY(intr_restore)
2177 2177 ENTRY(restore_int_flag)
2178 2178 testq $PS_IE, %rdi
2179 2179 jz 1f
2180 2180 #if defined(__xpv)
2181 2181 leaq xpv_panicking, %rsi
2182 2182 movl (%rsi), %esi
2183 2183 cmpl $0, %esi
2184 2184 jne 1f
2185 2185 /*
2186 2186 * Since we're -really- running unprivileged, our attempt
2187 2187 * to change the state of the IF bit will be ignored.
2188 2188 * The virtual IF bit is tweaked by CLI and STI.
2189 2189 */
2190 2190 IE_TO_EVENT_MASK(%rsi, %rdi)
2191 2191 #else
2192 2192 sti
2193 2193 #endif
2194 2194 1:
2195 2195 ret
2196 2196 SET_SIZE(restore_int_flag)
2197 2197 SET_SIZE(intr_restore)
2198 2198
2199 2199 #elif defined(__i386)
2200 2200
2201 2201 ENTRY(intr_restore)
2202 2202 ENTRY(restore_int_flag)
2203 2203 testl $PS_IE, 4(%esp)
2204 2204 jz 1f
2205 2205 #if defined(__xpv)
2206 2206 leal xpv_panicking, %edx
2207 2207 movl (%edx), %edx
2208 2208 cmpl $0, %edx
2209 2209 jne 1f
2210 2210 /*
2211 2211 * Since we're -really- running unprivileged, our attempt
2212 2212 * to change the state of the IF bit will be ignored.
2213 2213 * The virtual IF bit is tweaked by CLI and STI.
2214 2214 */
2215 2215 IE_TO_EVENT_MASK(%edx, 4(%esp))
2216 2216 #else
2217 2217 sti
2218 2218 #endif
2219 2219 1:
2220 2220 ret
2221 2221 SET_SIZE(restore_int_flag)
2222 2222 SET_SIZE(intr_restore)
2223 2223
2224 2224 #endif /* __i386 */
2225 2225 #endif /* __lint */
2226 2226
2227 2227 #if defined(__lint)
2228 2228
2229 2229 void
2230 2230 sti(void)
2231 2231 {}
2232 2232
2233 2233 void
2234 2234 cli(void)
2235 2235 {}
2236 2236
2237 2237 #else /* __lint */
2238 2238
2239 2239 ENTRY(sti)
2240 2240 STI
2241 2241 ret
2242 2242 SET_SIZE(sti)
2243 2243
2244 2244 ENTRY(cli)
2245 2245 #if defined(__amd64)
2246 2246 CLI(%rax)
2247 2247 #elif defined(__i386)
2248 2248 CLI(%eax)
2249 2249 #endif /* __i386 */
2250 2250 ret
2251 2251 SET_SIZE(cli)
2252 2252
2253 2253 #endif /* __lint */
2254 2254
2255 2255 #if defined(__lint)
2256 2256
2257 2257 dtrace_icookie_t
2258 2258 dtrace_interrupt_disable(void)
2259 2259 { return (0); }
2260 2260
2261 2261 #else /* __lint */
2262 2262
2263 2263 #if defined(__amd64)
2264 2264
2265 2265 ENTRY(dtrace_interrupt_disable)
2266 2266 pushfq
2267 2267 popq %rax
2268 2268 #if defined(__xpv)
2269 2269 leaq xpv_panicking, %rdi
2270 2270 movl (%rdi), %edi
2271 2271 cmpl $0, %edi
2272 2272 jne .dtrace_interrupt_disable_done
2273 2273 CLIRET(%rdi, %dl) /* returns event mask in %dl */
2274 2274 /*
2275 2275 * Synthesize the PS_IE bit from the event mask bit
2276 2276 */
2277 2277 andq $_BITNOT(PS_IE), %rax
2278 2278 testb $1, %dl
2279 2279 jnz .dtrace_interrupt_disable_done
2280 2280 orq $PS_IE, %rax
2281 2281 #else
2282 2282 CLI(%rdx)
2283 2283 #endif
2284 2284 .dtrace_interrupt_disable_done:
2285 2285 ret
2286 2286 SET_SIZE(dtrace_interrupt_disable)
2287 2287
2288 2288 #elif defined(__i386)
2289 2289
2290 2290 ENTRY(dtrace_interrupt_disable)
2291 2291 pushfl
2292 2292 popl %eax
2293 2293 #if defined(__xpv)
2294 2294 leal xpv_panicking, %edx
2295 2295 movl (%edx), %edx
2296 2296 cmpl $0, %edx
2297 2297 jne .dtrace_interrupt_disable_done
2298 2298 CLIRET(%edx, %cl) /* returns event mask in %cl */
2299 2299 /*
2300 2300 * Synthesize the PS_IE bit from the event mask bit
2301 2301 */
2302 2302 andl $_BITNOT(PS_IE), %eax
2303 2303 testb $1, %cl
2304 2304 jnz .dtrace_interrupt_disable_done
2305 2305 orl $PS_IE, %eax
2306 2306 #else
2307 2307 CLI(%edx)
2308 2308 #endif
2309 2309 .dtrace_interrupt_disable_done:
2310 2310 ret
2311 2311 SET_SIZE(dtrace_interrupt_disable)
2312 2312
2313 2313 #endif /* __i386 */
2314 2314 #endif /* __lint */
2315 2315
2316 2316 #if defined(__lint)
2317 2317
2318 2318 /*ARGSUSED*/
2319 2319 void
2320 2320 dtrace_interrupt_enable(dtrace_icookie_t cookie)
2321 2321 {}
2322 2322
2323 2323 #else /* __lint */
2324 2324
2325 2325 #if defined(__amd64)
2326 2326
2327 2327 ENTRY(dtrace_interrupt_enable)
2328 2328 pushq %rdi
2329 2329 popfq
2330 2330 #if defined(__xpv)
2331 2331 leaq xpv_panicking, %rdx
2332 2332 movl (%rdx), %edx
2333 2333 cmpl $0, %edx
2334 2334 jne .dtrace_interrupt_enable_done
2335 2335 /*
2336 2336 * Since we're -really- running unprivileged, our attempt
2337 2337 * to change the state of the IF bit will be ignored. The
2338 2338 * virtual IF bit is tweaked by CLI and STI.
2339 2339 */
2340 2340 IE_TO_EVENT_MASK(%rdx, %rdi)
2341 2341 #endif
2342 2342 .dtrace_interrupt_enable_done:
2343 2343 ret
2344 2344 SET_SIZE(dtrace_interrupt_enable)
2345 2345
2346 2346 #elif defined(__i386)
2347 2347
2348 2348 ENTRY(dtrace_interrupt_enable)
2349 2349 movl 4(%esp), %eax
2350 2350 pushl %eax
2351 2351 popfl
2352 2352 #if defined(__xpv)
2353 2353 leal xpv_panicking, %edx
2354 2354 movl (%edx), %edx
2355 2355 cmpl $0, %edx
2356 2356 jne .dtrace_interrupt_enable_done
2357 2357 /*
2358 2358 * Since we're -really- running unprivileged, our attempt
2359 2359 * to change the state of the IF bit will be ignored. The
2360 2360 * virtual IF bit is tweaked by CLI and STI.
2361 2361 */
2362 2362 IE_TO_EVENT_MASK(%edx, %eax)
2363 2363 #endif
2364 2364 .dtrace_interrupt_enable_done:
2365 2365 ret
2366 2366 SET_SIZE(dtrace_interrupt_enable)
2367 2367
2368 2368 #endif /* __i386 */
2369 2369 #endif /* __lint */
2370 2370
2371 2371
2372 2372 #if defined(lint)
2373 2373
2374 2374 void
2375 2375 dtrace_membar_producer(void)
2376 2376 {}
2377 2377
2378 2378 void
2379 2379 dtrace_membar_consumer(void)
2380 2380 {}
2381 2381
2382 2382 #else /* __lint */
2383 2383
2384 2384 ENTRY(dtrace_membar_producer)
2385 2385 rep; ret /* use 2 byte return instruction when branch target */
2386 2386 /* AMD Software Optimization Guide - Section 6.2 */
2387 2387 SET_SIZE(dtrace_membar_producer)
2388 2388
2389 2389 ENTRY(dtrace_membar_consumer)
2390 2390 rep; ret /* use 2 byte return instruction when branch target */
2391 2391 /* AMD Software Optimization Guide - Section 6.2 */
2392 2392 SET_SIZE(dtrace_membar_consumer)
2393 2393
2394 2394 #endif /* __lint */
2395 2395
2396 2396 #if defined(__lint)
2397 2397
2398 2398 kthread_id_t
2399 2399 threadp(void)
2400 2400 { return ((kthread_id_t)0); }
2401 2401
2402 2402 #else /* __lint */
2403 2403
2404 2404 #if defined(__amd64)
2405 2405
2406 2406 ENTRY(threadp)
2407 2407 movq %gs:CPU_THREAD, %rax
2408 2408 ret
2409 2409 SET_SIZE(threadp)
2410 2410
2411 2411 #elif defined(__i386)
2412 2412
2413 2413 ENTRY(threadp)
2414 2414 movl %gs:CPU_THREAD, %eax
2415 2415 ret
2416 2416 SET_SIZE(threadp)
2417 2417
2418 2418 #endif /* __i386 */
2419 2419 #endif /* __lint */
2420 2420
2421 2421 /*
2422 2422 * Checksum routine for Internet Protocol Headers
2423 2423 */
2424 2424
2425 2425 #if defined(__lint)
2426 2426
2427 2427 /* ARGSUSED */
2428 2428 unsigned int
2429 2429 ip_ocsum(
2430 2430 ushort_t *address, /* ptr to 1st message buffer */
2431 2431 int halfword_count, /* length of data */
2432 2432 unsigned int sum) /* partial checksum */
2433 2433 {
2434 2434 int i;
2435 2435 unsigned int psum = 0; /* partial sum */
2436 2436
2437 2437 for (i = 0; i < halfword_count; i++, address++) {
2438 2438 psum += *address;
2439 2439 }
2440 2440
2441 2441 while ((psum >> 16) != 0) {
2442 2442 psum = (psum & 0xffff) + (psum >> 16);
2443 2443 }
2444 2444
2445 2445 psum += sum;
2446 2446
2447 2447 while ((psum >> 16) != 0) {
2448 2448 psum = (psum & 0xffff) + (psum >> 16);
2449 2449 }
2450 2450
2451 2451 return (psum);
2452 2452 }
2453 2453
2454 2454 #else /* __lint */
2455 2455
2456 2456 #if defined(__amd64)
2457 2457
2458 2458 ENTRY(ip_ocsum)
2459 2459 pushq %rbp
2460 2460 movq %rsp, %rbp
2461 2461 #ifdef DEBUG
2462 2462 movq postbootkernelbase(%rip), %rax
2463 2463 cmpq %rax, %rdi
2464 2464 jnb 1f
2465 2465 xorl %eax, %eax
2466 2466 movq %rdi, %rsi
2467 2467 leaq .ip_ocsum_panic_msg(%rip), %rdi
2468 2468 call panic
2469 2469 /*NOTREACHED*/
2470 2470 .ip_ocsum_panic_msg:
2471 2471 .string "ip_ocsum: address 0x%p below kernelbase\n"
2472 2472 1:
2473 2473 #endif
2474 2474 movl %esi, %ecx /* halfword_count */
2475 2475 movq %rdi, %rsi /* address */
2476 2476 /* partial sum in %edx */
2477 2477 xorl %eax, %eax
2478 2478 testl %ecx, %ecx
2479 2479 jz .ip_ocsum_done
2480 2480 testq $3, %rsi
2481 2481 jnz .ip_csum_notaligned
2482 2482 .ip_csum_aligned: /* XX64 opportunities for 8-byte operations? */
2483 2483 .next_iter:
2484 2484 /* XX64 opportunities for prefetch? */
2485 2485 /* XX64 compute csum with 64 bit quantities? */
2486 2486 subl $32, %ecx
2487 2487 jl .less_than_32
2488 2488
2489 2489 addl 0(%rsi), %edx
2490 2490 .only60:
2491 2491 adcl 4(%rsi), %eax
2492 2492 .only56:
2493 2493 adcl 8(%rsi), %edx
2494 2494 .only52:
2495 2495 adcl 12(%rsi), %eax
2496 2496 .only48:
2497 2497 adcl 16(%rsi), %edx
2498 2498 .only44:
2499 2499 adcl 20(%rsi), %eax
2500 2500 .only40:
2501 2501 adcl 24(%rsi), %edx
2502 2502 .only36:
2503 2503 adcl 28(%rsi), %eax
2504 2504 .only32:
2505 2505 adcl 32(%rsi), %edx
2506 2506 .only28:
2507 2507 adcl 36(%rsi), %eax
2508 2508 .only24:
2509 2509 adcl 40(%rsi), %edx
2510 2510 .only20:
2511 2511 adcl 44(%rsi), %eax
2512 2512 .only16:
2513 2513 adcl 48(%rsi), %edx
2514 2514 .only12:
2515 2515 adcl 52(%rsi), %eax
2516 2516 .only8:
2517 2517 adcl 56(%rsi), %edx
2518 2518 .only4:
2519 2519 adcl 60(%rsi), %eax /* could be adding -1 and -1 with a carry */
2520 2520 .only0:
2521 2521 adcl $0, %eax /* could be adding -1 in eax with a carry */
2522 2522 adcl $0, %eax
2523 2523
2524 2524 addq $64, %rsi
2525 2525 testl %ecx, %ecx
2526 2526 jnz .next_iter
2527 2527
2528 2528 .ip_ocsum_done:
2529 2529 addl %eax, %edx
2530 2530 adcl $0, %edx
2531 2531 movl %edx, %eax /* form a 16 bit checksum by */
2532 2532 shrl $16, %eax /* adding two halves of 32 bit checksum */
2533 2533 addw %dx, %ax
2534 2534 adcw $0, %ax
2535 2535 andl $0xffff, %eax
2536 2536 leave
2537 2537 ret
2538 2538
2539 2539 .ip_csum_notaligned:
2540 2540 xorl %edi, %edi
2541 2541 movw (%rsi), %di
2542 2542 addl %edi, %edx
2543 2543 adcl $0, %edx
2544 2544 addq $2, %rsi
2545 2545 decl %ecx
2546 2546 jmp .ip_csum_aligned
2547 2547
2548 2548 .less_than_32:
2549 2549 addl $32, %ecx
2550 2550 testl $1, %ecx
2551 2551 jz .size_aligned
2552 2552 andl $0xfe, %ecx
2553 2553 movzwl (%rsi, %rcx, 2), %edi
2554 2554 addl %edi, %edx
2555 2555 adcl $0, %edx
2556 2556 .size_aligned:
2557 2557 movl %ecx, %edi
2558 2558 shrl $1, %ecx
2559 2559 shl $1, %edi
2560 2560 subq $64, %rdi
2561 2561 addq %rdi, %rsi
2562 2562 leaq .ip_ocsum_jmptbl(%rip), %rdi
2563 2563 leaq (%rdi, %rcx, 8), %rdi
2564 2564 xorl %ecx, %ecx
2565 2565 clc
2566 2566 jmp *(%rdi)
2567 2567
2568 2568 .align 8
2569 2569 .ip_ocsum_jmptbl:
2570 2570 .quad .only0, .only4, .only8, .only12, .only16, .only20
2571 2571 .quad .only24, .only28, .only32, .only36, .only40, .only44
2572 2572 .quad .only48, .only52, .only56, .only60
2573 2573 SET_SIZE(ip_ocsum)
2574 2574
2575 2575 #elif defined(__i386)
2576 2576
2577 2577 ENTRY(ip_ocsum)
2578 2578 pushl %ebp
2579 2579 movl %esp, %ebp
2580 2580 pushl %ebx
2581 2581 pushl %esi
2582 2582 pushl %edi
2583 2583 movl 12(%ebp), %ecx /* count of half words */
2584 2584 movl 16(%ebp), %edx /* partial checksum */
2585 2585 movl 8(%ebp), %esi
2586 2586 xorl %eax, %eax
2587 2587 testl %ecx, %ecx
2588 2588 jz .ip_ocsum_done
2589 2589
2590 2590 testl $3, %esi
2591 2591 jnz .ip_csum_notaligned
2592 2592 .ip_csum_aligned:
2593 2593 .next_iter:
2594 2594 subl $32, %ecx
2595 2595 jl .less_than_32
2596 2596
2597 2597 addl 0(%esi), %edx
2598 2598 .only60:
2599 2599 adcl 4(%esi), %eax
2600 2600 .only56:
2601 2601 adcl 8(%esi), %edx
2602 2602 .only52:
2603 2603 adcl 12(%esi), %eax
2604 2604 .only48:
2605 2605 adcl 16(%esi), %edx
2606 2606 .only44:
2607 2607 adcl 20(%esi), %eax
2608 2608 .only40:
2609 2609 adcl 24(%esi), %edx
2610 2610 .only36:
2611 2611 adcl 28(%esi), %eax
2612 2612 .only32:
2613 2613 adcl 32(%esi), %edx
2614 2614 .only28:
2615 2615 adcl 36(%esi), %eax
2616 2616 .only24:
2617 2617 adcl 40(%esi), %edx
2618 2618 .only20:
2619 2619 adcl 44(%esi), %eax
2620 2620 .only16:
2621 2621 adcl 48(%esi), %edx
2622 2622 .only12:
2623 2623 adcl 52(%esi), %eax
2624 2624 .only8:
2625 2625 adcl 56(%esi), %edx
2626 2626 .only4:
2627 2627 adcl 60(%esi), %eax /* We could be adding -1 and -1 with a carry */
2628 2628 .only0:
2629 2629 adcl $0, %eax /* we could be adding -1 in eax with a carry */
2630 2630 adcl $0, %eax
2631 2631
2632 2632 addl $64, %esi
2633 2633 andl %ecx, %ecx
2634 2634 jnz .next_iter
2635 2635
2636 2636 .ip_ocsum_done:
2637 2637 addl %eax, %edx
2638 2638 adcl $0, %edx
2639 2639 movl %edx, %eax /* form a 16 bit checksum by */
2640 2640 shrl $16, %eax /* adding two halves of 32 bit checksum */
2641 2641 addw %dx, %ax
2642 2642 adcw $0, %ax
2643 2643 andl $0xffff, %eax
2644 2644 popl %edi /* restore registers */
2645 2645 popl %esi
2646 2646 popl %ebx
2647 2647 leave
2648 2648 ret
2649 2649
2650 2650 .ip_csum_notaligned:
2651 2651 xorl %edi, %edi
2652 2652 movw (%esi), %di
2653 2653 addl %edi, %edx
2654 2654 adcl $0, %edx
2655 2655 addl $2, %esi
2656 2656 decl %ecx
2657 2657 jmp .ip_csum_aligned
2658 2658
2659 2659 .less_than_32:
2660 2660 addl $32, %ecx
2661 2661 testl $1, %ecx
2662 2662 jz .size_aligned
2663 2663 andl $0xfe, %ecx
2664 2664 movzwl (%esi, %ecx, 2), %edi
2665 2665 addl %edi, %edx
2666 2666 adcl $0, %edx
2667 2667 .size_aligned:
2668 2668 movl %ecx, %edi
2669 2669 shrl $1, %ecx
2670 2670 shl $1, %edi
2671 2671 subl $64, %edi
2672 2672 addl %edi, %esi
2673 2673 movl $.ip_ocsum_jmptbl, %edi
2674 2674 lea (%edi, %ecx, 4), %edi
2675 2675 xorl %ecx, %ecx
2676 2676 clc
2677 2677 jmp *(%edi)
2678 2678 SET_SIZE(ip_ocsum)
2679 2679
2680 2680 .data
2681 2681 .align 4
2682 2682
2683 2683 .ip_ocsum_jmptbl:
2684 2684 .long .only0, .only4, .only8, .only12, .only16, .only20
2685 2685 .long .only24, .only28, .only32, .only36, .only40, .only44
2686 2686 .long .only48, .only52, .only56, .only60
2687 2687
2688 2688
2689 2689 #endif /* __i386 */
2690 2690 #endif /* __lint */
2691 2691
2692 2692 /*
2693 2693 * multiply two long numbers and yield a u_longlong_t result, callable from C.
2694 2694 * Provided to manipulate hrtime_t values.
2695 2695 */
2696 2696 #if defined(__lint)
2697 2697
2698 2698 /* result = a * b; */
2699 2699
2700 2700 /* ARGSUSED */
2701 2701 unsigned long long
2702 2702 mul32(uint_t a, uint_t b)
2703 2703 { return (0); }
2704 2704
2705 2705 #else /* __lint */
2706 2706
2707 2707 #if defined(__amd64)
2708 2708
2709 2709 ENTRY(mul32)
2710 2710 xorl %edx, %edx /* XX64 joe, paranoia? */
2711 2711 movl %edi, %eax
2712 2712 mull %esi
2713 2713 shlq $32, %rdx
2714 2714 orq %rdx, %rax
2715 2715 ret
2716 2716 SET_SIZE(mul32)
2717 2717
2718 2718 #elif defined(__i386)
2719 2719
2720 2720 ENTRY(mul32)
2721 2721 movl 8(%esp), %eax
2722 2722 movl 4(%esp), %ecx
2723 2723 mull %ecx
2724 2724 ret
2725 2725 SET_SIZE(mul32)
2726 2726
2727 2727 #endif /* __i386 */
2728 2728 #endif /* __lint */
2729 2729
2730 2730 #if defined(notused)
2731 2731 #if defined(__lint)
2732 2732 /* ARGSUSED */
2733 2733 void
2734 2734 load_pte64(uint64_t *pte, uint64_t pte_value)
2735 2735 {}
2736 2736 #else /* __lint */
2737 2737 .globl load_pte64
2738 2738 load_pte64:
2739 2739 movl 4(%esp), %eax
2740 2740 movl 8(%esp), %ecx
2741 2741 movl 12(%esp), %edx
2742 2742 movl %edx, 4(%eax)
2743 2743 movl %ecx, (%eax)
2744 2744 ret
2745 2745 #endif /* __lint */
2746 2746 #endif /* notused */
2747 2747
2748 2748 #if defined(__lint)
2749 2749
2750 2750 /*ARGSUSED*/
2751 2751 void
2752 2752 scan_memory(caddr_t addr, size_t size)
2753 2753 {}
2754 2754
2755 2755 #else /* __lint */
2756 2756
2757 2757 #if defined(__amd64)
2758 2758
2759 2759 ENTRY(scan_memory)
2760 2760 shrq $3, %rsi /* convert %rsi from byte to quadword count */
2761 2761 jz .scanm_done
2762 2762 movq %rsi, %rcx /* move count into rep control register */
2763 2763 movq %rdi, %rsi /* move addr into lodsq control reg. */
2764 2764 rep lodsq /* scan the memory range */
2765 2765 .scanm_done:
2766 2766 rep; ret /* use 2 byte return instruction when branch target */
2767 2767 /* AMD Software Optimization Guide - Section 6.2 */
2768 2768 SET_SIZE(scan_memory)
2769 2769
2770 2770 #elif defined(__i386)
2771 2771
2772 2772 ENTRY(scan_memory)
2773 2773 pushl %ecx
2774 2774 pushl %esi
2775 2775 movl 16(%esp), %ecx /* move 2nd arg into rep control register */
2776 2776 shrl $2, %ecx /* convert from byte count to word count */
2777 2777 jz .scanm_done
2778 2778 movl 12(%esp), %esi /* move 1st arg into lodsw control register */
2779 2779 .byte 0xf3 /* rep prefix. lame assembler. sigh. */
2780 2780 lodsl
2781 2781 .scanm_done:
2782 2782 popl %esi
2783 2783 popl %ecx
2784 2784 ret
2785 2785 SET_SIZE(scan_memory)
2786 2786
2787 2787 #endif /* __i386 */
2788 2788 #endif /* __lint */
2789 2789
2790 2790
2791 2791 #if defined(__lint)
2792 2792
2793 2793 /*ARGSUSED */
|
↓ open down ↓ |
2793 lines elided |
↑ open up ↑ |
2794 2794 int
2795 2795 lowbit(ulong_t i)
2796 2796 { return (0); }
2797 2797
2798 2798 #else /* __lint */
2799 2799
2800 2800 #if defined(__amd64)
2801 2801
2802 2802 ENTRY(lowbit)
2803 2803 movl $-1, %eax
2804 - bsfq %rdi, %rax
2804 + bsfq %rdi, %rdi
2805 + cmovnz %edi, %eax
2805 2806 incl %eax
2806 2807 ret
2807 2808 SET_SIZE(lowbit)
2808 2809
2809 2810 #elif defined(__i386)
2810 2811
2811 2812 ENTRY(lowbit)
2812 - movl $-1, %eax
2813 2813 bsfl 4(%esp), %eax
2814 + jz 0f
2814 2815 incl %eax
2815 2816 ret
2817 +0:
2818 + xorl %eax, %eax
2819 + ret
2816 2820 SET_SIZE(lowbit)
2817 2821
2818 2822 #endif /* __i386 */
2819 2823 #endif /* __lint */
2820 2824
2821 2825 #if defined(__lint)
2822 2826
2823 2827 /*ARGSUSED*/
2824 2828 int
2825 2829 highbit(ulong_t i)
2826 2830 { return (0); }
2827 2831
2832 +/*ARGSUSED*/
2833 +int
2834 +highbit64(uint64_t i)
2835 +{ return (0); }
2836 +
2828 2837 #else /* __lint */
2829 2838
2830 2839 #if defined(__amd64)
2831 2840
2832 2841 ENTRY(highbit)
2842 + ALTENTRY(highbit64)
2833 2843 movl $-1, %eax
2834 - bsrq %rdi, %rax
2844 + bsrq %rdi, %rdi
2845 + cmovnz %edi, %eax
2835 2846 incl %eax
2836 2847 ret
2848 + SET_SIZE(highbit64)
2837 2849 SET_SIZE(highbit)
2838 2850
2839 2851 #elif defined(__i386)
2840 2852
2841 2853 ENTRY(highbit)
2842 - movl $-1, %eax
2843 2854 bsrl 4(%esp), %eax
2855 + jz 0f
2844 2856 incl %eax
2845 2857 ret
2858 +0:
2859 + xorl %eax, %eax
2860 + ret
2846 2861 SET_SIZE(highbit)
2847 2862
2848 -#endif /* __i386 */
2849 -#endif /* __lint */
2850 -
2851 -#if defined(__lint)
2852 -
2853 -/*ARGSUSED*/
2854 -int
2855 -highbit64(uint64_t i)
2856 -{ return (0); }
2857 -
2858 -#else /* __lint */
2859 -
2860 -#if defined(__amd64)
2861 -
2862 - ENTRY(highbit64)
2863 - movl $-1, %eax
2864 - bsrq %rdi, %rax
2865 - incl %eax
2866 - ret
2867 - SET_SIZE(highbit64)
2868 -
2869 -#elif defined(__i386)
2870 -
2871 2863 ENTRY(highbit64)
2872 2864 bsrl 8(%esp), %eax
2873 - jz .lowbit
2874 - addl $32, %eax
2875 - jmp .done
2876 -
2877 -.lowbit:
2878 - movl $-1, %eax
2879 - bsrl 4(%esp), %eax
2880 -.done:
2881 - incl %eax
2865 + jz highbit
2866 + addl $33, %eax
2882 2867 ret
2883 2868 SET_SIZE(highbit64)
2884 2869
2885 2870 #endif /* __i386 */
2886 2871 #endif /* __lint */
2887 2872
2888 2873 #if defined(__lint)
2889 2874
2890 2875 /*ARGSUSED*/
2891 2876 uint64_t
2892 2877 rdmsr(uint_t r)
2893 2878 { return (0); }
2894 2879
2895 2880 /*ARGSUSED*/
2896 2881 void
2897 2882 wrmsr(uint_t r, const uint64_t val)
2898 2883 {}
2899 2884
2900 2885 /*ARGSUSED*/
2901 2886 uint64_t
2902 2887 xrdmsr(uint_t r)
2903 2888 { return (0); }
2904 2889
2905 2890 /*ARGSUSED*/
2906 2891 void
2907 2892 xwrmsr(uint_t r, const uint64_t val)
2908 2893 {}
2909 2894
2910 2895 void
2911 2896 invalidate_cache(void)
2912 2897 {}
2913 2898
2914 2899 /*ARGSUSED*/
2915 2900 uint64_t
2916 2901 get_xcr(uint_t r)
2917 2902 { return (0); }
2918 2903
2919 2904 /*ARGSUSED*/
2920 2905 void
2921 2906 set_xcr(uint_t r, const uint64_t val)
2922 2907 {}
2923 2908
2924 2909 #else /* __lint */
2925 2910
2926 2911 #define XMSR_ACCESS_VAL $0x9c5a203a
2927 2912
2928 2913 #if defined(__amd64)
2929 2914
2930 2915 ENTRY(rdmsr)
2931 2916 movl %edi, %ecx
2932 2917 rdmsr
2933 2918 shlq $32, %rdx
2934 2919 orq %rdx, %rax
2935 2920 ret
2936 2921 SET_SIZE(rdmsr)
2937 2922
2938 2923 ENTRY(wrmsr)
2939 2924 movq %rsi, %rdx
2940 2925 shrq $32, %rdx
2941 2926 movl %esi, %eax
2942 2927 movl %edi, %ecx
2943 2928 wrmsr
2944 2929 ret
2945 2930 SET_SIZE(wrmsr)
2946 2931
2947 2932 ENTRY(xrdmsr)
2948 2933 pushq %rbp
2949 2934 movq %rsp, %rbp
2950 2935 movl %edi, %ecx
2951 2936 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2952 2937 rdmsr
2953 2938 shlq $32, %rdx
2954 2939 orq %rdx, %rax
2955 2940 leave
2956 2941 ret
2957 2942 SET_SIZE(xrdmsr)
2958 2943
2959 2944 ENTRY(xwrmsr)
2960 2945 pushq %rbp
2961 2946 movq %rsp, %rbp
2962 2947 movl %edi, %ecx
2963 2948 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2964 2949 movq %rsi, %rdx
2965 2950 shrq $32, %rdx
2966 2951 movl %esi, %eax
2967 2952 wrmsr
2968 2953 leave
2969 2954 ret
2970 2955 SET_SIZE(xwrmsr)
2971 2956
2972 2957 ENTRY(get_xcr)
2973 2958 movl %edi, %ecx
2974 2959 #xgetbv
2975 2960 .byte 0x0f,0x01,0xd0
2976 2961 shlq $32, %rdx
2977 2962 orq %rdx, %rax
2978 2963 ret
2979 2964 SET_SIZE(get_xcr)
2980 2965
2981 2966 ENTRY(set_xcr)
2982 2967 movq %rsi, %rdx
2983 2968 shrq $32, %rdx
2984 2969 movl %esi, %eax
2985 2970 movl %edi, %ecx
2986 2971 #xsetbv
2987 2972 .byte 0x0f,0x01,0xd1
2988 2973 ret
2989 2974 SET_SIZE(set_xcr)
2990 2975
2991 2976 #elif defined(__i386)
2992 2977
2993 2978 ENTRY(rdmsr)
2994 2979 movl 4(%esp), %ecx
2995 2980 rdmsr
2996 2981 ret
2997 2982 SET_SIZE(rdmsr)
2998 2983
2999 2984 ENTRY(wrmsr)
3000 2985 movl 4(%esp), %ecx
3001 2986 movl 8(%esp), %eax
3002 2987 movl 12(%esp), %edx
3003 2988 wrmsr
3004 2989 ret
3005 2990 SET_SIZE(wrmsr)
3006 2991
3007 2992 ENTRY(xrdmsr)
3008 2993 pushl %ebp
3009 2994 movl %esp, %ebp
3010 2995 movl 8(%esp), %ecx
3011 2996 pushl %edi
3012 2997 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
3013 2998 rdmsr
3014 2999 popl %edi
3015 3000 leave
3016 3001 ret
3017 3002 SET_SIZE(xrdmsr)
3018 3003
3019 3004 ENTRY(xwrmsr)
3020 3005 pushl %ebp
3021 3006 movl %esp, %ebp
3022 3007 movl 8(%esp), %ecx
3023 3008 movl 12(%esp), %eax
3024 3009 movl 16(%esp), %edx
3025 3010 pushl %edi
3026 3011 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
3027 3012 wrmsr
3028 3013 popl %edi
3029 3014 leave
3030 3015 ret
3031 3016 SET_SIZE(xwrmsr)
3032 3017
3033 3018 ENTRY(get_xcr)
3034 3019 movl 4(%esp), %ecx
3035 3020 #xgetbv
3036 3021 .byte 0x0f,0x01,0xd0
3037 3022 ret
3038 3023 SET_SIZE(get_xcr)
3039 3024
3040 3025 ENTRY(set_xcr)
3041 3026 movl 4(%esp), %ecx
3042 3027 movl 8(%esp), %eax
3043 3028 movl 12(%esp), %edx
3044 3029 #xsetbv
3045 3030 .byte 0x0f,0x01,0xd1
3046 3031 ret
3047 3032 SET_SIZE(set_xcr)
3048 3033
3049 3034 #endif /* __i386 */
3050 3035
3051 3036 ENTRY(invalidate_cache)
3052 3037 wbinvd
3053 3038 ret
3054 3039 SET_SIZE(invalidate_cache)
3055 3040
3056 3041 #endif /* __lint */
3057 3042
3058 3043 #if defined(__lint)
3059 3044
3060 3045 /*ARGSUSED*/
3061 3046 void
3062 3047 getcregs(struct cregs *crp)
3063 3048 {}
3064 3049
3065 3050 #else /* __lint */
3066 3051
3067 3052 #if defined(__amd64)
3068 3053
3069 3054 ENTRY_NP(getcregs)
3070 3055 #if defined(__xpv)
3071 3056 /*
3072 3057 * Only a few of the hardware control registers or descriptor tables
3073 3058 * are directly accessible to us, so just zero the structure.
3074 3059 *
3075 3060 * XXPV Perhaps it would be helpful for the hypervisor to return
3076 3061 * virtualized versions of these for post-mortem use.
3077 3062 * (Need to reevaluate - perhaps it already does!)
3078 3063 */
3079 3064 pushq %rdi /* save *crp */
3080 3065 movq $CREGSZ, %rsi
3081 3066 call bzero
3082 3067 popq %rdi
3083 3068
3084 3069 /*
3085 3070 * Dump what limited information we can
3086 3071 */
3087 3072 movq %cr0, %rax
3088 3073 movq %rax, CREG_CR0(%rdi) /* cr0 */
3089 3074 movq %cr2, %rax
3090 3075 movq %rax, CREG_CR2(%rdi) /* cr2 */
3091 3076 movq %cr3, %rax
3092 3077 movq %rax, CREG_CR3(%rdi) /* cr3 */
3093 3078 movq %cr4, %rax
3094 3079 movq %rax, CREG_CR4(%rdi) /* cr4 */
3095 3080
3096 3081 #else /* __xpv */
3097 3082
3098 3083 #define GETMSR(r, off, d) \
3099 3084 movl $r, %ecx; \
3100 3085 rdmsr; \
3101 3086 movl %eax, off(d); \
3102 3087 movl %edx, off+4(d)
3103 3088
3104 3089 xorl %eax, %eax
3105 3090 movq %rax, CREG_GDT+8(%rdi)
3106 3091 sgdt CREG_GDT(%rdi) /* 10 bytes */
3107 3092 movq %rax, CREG_IDT+8(%rdi)
3108 3093 sidt CREG_IDT(%rdi) /* 10 bytes */
3109 3094 movq %rax, CREG_LDT(%rdi)
3110 3095 sldt CREG_LDT(%rdi) /* 2 bytes */
3111 3096 movq %rax, CREG_TASKR(%rdi)
3112 3097 str CREG_TASKR(%rdi) /* 2 bytes */
3113 3098 movq %cr0, %rax
3114 3099 movq %rax, CREG_CR0(%rdi) /* cr0 */
3115 3100 movq %cr2, %rax
3116 3101 movq %rax, CREG_CR2(%rdi) /* cr2 */
3117 3102 movq %cr3, %rax
3118 3103 movq %rax, CREG_CR3(%rdi) /* cr3 */
3119 3104 movq %cr4, %rax
3120 3105 movq %rax, CREG_CR4(%rdi) /* cr4 */
3121 3106 movq %cr8, %rax
3122 3107 movq %rax, CREG_CR8(%rdi) /* cr8 */
3123 3108 GETMSR(MSR_AMD_KGSBASE, CREG_KGSBASE, %rdi)
3124 3109 GETMSR(MSR_AMD_EFER, CREG_EFER, %rdi)
3125 3110 #endif /* __xpv */
3126 3111 ret
3127 3112 SET_SIZE(getcregs)
3128 3113
3129 3114 #undef GETMSR
3130 3115
3131 3116 #elif defined(__i386)
3132 3117
3133 3118 ENTRY_NP(getcregs)
3134 3119 #if defined(__xpv)
3135 3120 /*
3136 3121 * Only a few of the hardware control registers or descriptor tables
3137 3122 * are directly accessible to us, so just zero the structure.
3138 3123 *
3139 3124 * XXPV Perhaps it would be helpful for the hypervisor to return
3140 3125 * virtualized versions of these for post-mortem use.
3141 3126 * (Need to reevaluate - perhaps it already does!)
3142 3127 */
3143 3128 movl 4(%esp), %edx
3144 3129 pushl $CREGSZ
3145 3130 pushl %edx
3146 3131 call bzero
3147 3132 addl $8, %esp
3148 3133 movl 4(%esp), %edx
3149 3134
3150 3135 /*
3151 3136 * Dump what limited information we can
3152 3137 */
3153 3138 movl %cr0, %eax
3154 3139 movl %eax, CREG_CR0(%edx) /* cr0 */
3155 3140 movl %cr2, %eax
3156 3141 movl %eax, CREG_CR2(%edx) /* cr2 */
3157 3142 movl %cr3, %eax
3158 3143 movl %eax, CREG_CR3(%edx) /* cr3 */
3159 3144 movl %cr4, %eax
3160 3145 movl %eax, CREG_CR4(%edx) /* cr4 */
3161 3146
3162 3147 #else /* __xpv */
3163 3148
3164 3149 movl 4(%esp), %edx
3165 3150 movw $0, CREG_GDT+6(%edx)
3166 3151 movw $0, CREG_IDT+6(%edx)
3167 3152 sgdt CREG_GDT(%edx) /* gdt */
3168 3153 sidt CREG_IDT(%edx) /* idt */
3169 3154 sldt CREG_LDT(%edx) /* ldt */
3170 3155 str CREG_TASKR(%edx) /* task */
3171 3156 movl %cr0, %eax
3172 3157 movl %eax, CREG_CR0(%edx) /* cr0 */
3173 3158 movl %cr2, %eax
3174 3159 movl %eax, CREG_CR2(%edx) /* cr2 */
3175 3160 movl %cr3, %eax
3176 3161 movl %eax, CREG_CR3(%edx) /* cr3 */
3177 3162 bt $X86FSET_LARGEPAGE, x86_featureset
3178 3163 jnc .nocr4
3179 3164 movl %cr4, %eax
3180 3165 movl %eax, CREG_CR4(%edx) /* cr4 */
3181 3166 jmp .skip
3182 3167 .nocr4:
3183 3168 movl $0, CREG_CR4(%edx)
3184 3169 .skip:
3185 3170 #endif
3186 3171 ret
3187 3172 SET_SIZE(getcregs)
3188 3173
3189 3174 #endif /* __i386 */
3190 3175 #endif /* __lint */
3191 3176
3192 3177
3193 3178 /*
3194 3179 * A panic trigger is a word which is updated atomically and can only be set
3195 3180 * once. We atomically store 0xDEFACEDD and load the old value. If the
3196 3181 * previous value was 0, we succeed and return 1; otherwise return 0.
3197 3182 * This allows a partially corrupt trigger to still trigger correctly. DTrace
3198 3183 * has its own version of this function to allow it to panic correctly from
3199 3184 * probe context.
3200 3185 */
3201 3186 #if defined(__lint)
3202 3187
3203 3188 /*ARGSUSED*/
3204 3189 int
3205 3190 panic_trigger(int *tp)
3206 3191 { return (0); }
3207 3192
3208 3193 /*ARGSUSED*/
3209 3194 int
3210 3195 dtrace_panic_trigger(int *tp)
3211 3196 { return (0); }
3212 3197
3213 3198 #else /* __lint */
3214 3199
3215 3200 #if defined(__amd64)
3216 3201
3217 3202 ENTRY_NP(panic_trigger)
3218 3203 xorl %eax, %eax
3219 3204 movl $0xdefacedd, %edx
3220 3205 lock
3221 3206 xchgl %edx, (%rdi)
3222 3207 cmpl $0, %edx
3223 3208 je 0f
3224 3209 movl $0, %eax
3225 3210 ret
3226 3211 0: movl $1, %eax
3227 3212 ret
3228 3213 SET_SIZE(panic_trigger)
3229 3214
3230 3215 ENTRY_NP(dtrace_panic_trigger)
3231 3216 xorl %eax, %eax
3232 3217 movl $0xdefacedd, %edx
3233 3218 lock
3234 3219 xchgl %edx, (%rdi)
3235 3220 cmpl $0, %edx
3236 3221 je 0f
3237 3222 movl $0, %eax
3238 3223 ret
3239 3224 0: movl $1, %eax
3240 3225 ret
3241 3226 SET_SIZE(dtrace_panic_trigger)
3242 3227
3243 3228 #elif defined(__i386)
3244 3229
3245 3230 ENTRY_NP(panic_trigger)
3246 3231 movl 4(%esp), %edx / %edx = address of trigger
3247 3232 movl $0xdefacedd, %eax / %eax = 0xdefacedd
3248 3233 lock / assert lock
3249 3234 xchgl %eax, (%edx) / exchange %eax and the trigger
3250 3235 cmpl $0, %eax / if (%eax == 0x0)
3251 3236 je 0f / return (1);
3252 3237 movl $0, %eax / else
3253 3238 ret / return (0);
3254 3239 0: movl $1, %eax
3255 3240 ret
3256 3241 SET_SIZE(panic_trigger)
3257 3242
3258 3243 ENTRY_NP(dtrace_panic_trigger)
3259 3244 movl 4(%esp), %edx / %edx = address of trigger
3260 3245 movl $0xdefacedd, %eax / %eax = 0xdefacedd
3261 3246 lock / assert lock
3262 3247 xchgl %eax, (%edx) / exchange %eax and the trigger
3263 3248 cmpl $0, %eax / if (%eax == 0x0)
3264 3249 je 0f / return (1);
3265 3250 movl $0, %eax / else
3266 3251 ret / return (0);
3267 3252 0: movl $1, %eax
3268 3253 ret
3269 3254 SET_SIZE(dtrace_panic_trigger)
3270 3255
3271 3256 #endif /* __i386 */
3272 3257 #endif /* __lint */
3273 3258
3274 3259 /*
3275 3260 * The panic() and cmn_err() functions invoke vpanic() as a common entry point
3276 3261 * into the panic code implemented in panicsys(). vpanic() is responsible
3277 3262 * for passing through the format string and arguments, and constructing a
3278 3263 * regs structure on the stack into which it saves the current register
3279 3264 * values. If we are not dying due to a fatal trap, these registers will
3280 3265 * then be preserved in panicbuf as the current processor state. Before
3281 3266 * invoking panicsys(), vpanic() activates the first panic trigger (see
3282 3267 * common/os/panic.c) and switches to the panic_stack if successful. Note that
3283 3268 * DTrace takes a slightly different panic path if it must panic from probe
3284 3269 * context. Instead of calling panic, it calls into dtrace_vpanic(), which
3285 3270 * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and
3286 3271 * branches back into vpanic().
3287 3272 */
3288 3273 #if defined(__lint)
3289 3274
3290 3275 /*ARGSUSED*/
3291 3276 void
3292 3277 vpanic(const char *format, va_list alist)
3293 3278 {}
3294 3279
3295 3280 /*ARGSUSED*/
3296 3281 void
3297 3282 dtrace_vpanic(const char *format, va_list alist)
3298 3283 {}
3299 3284
3300 3285 #else /* __lint */
3301 3286
3302 3287 #if defined(__amd64)
3303 3288
3304 3289 ENTRY_NP(vpanic) /* Initial stack layout: */
3305 3290
3306 3291 pushq %rbp /* | %rip | 0x60 */
3307 3292 movq %rsp, %rbp /* | %rbp | 0x58 */
3308 3293 pushfq /* | rfl | 0x50 */
3309 3294 pushq %r11 /* | %r11 | 0x48 */
3310 3295 pushq %r10 /* | %r10 | 0x40 */
3311 3296 pushq %rbx /* | %rbx | 0x38 */
3312 3297 pushq %rax /* | %rax | 0x30 */
3313 3298 pushq %r9 /* | %r9 | 0x28 */
3314 3299 pushq %r8 /* | %r8 | 0x20 */
3315 3300 pushq %rcx /* | %rcx | 0x18 */
3316 3301 pushq %rdx /* | %rdx | 0x10 */
3317 3302 pushq %rsi /* | %rsi | 0x8 alist */
3318 3303 pushq %rdi /* | %rdi | 0x0 format */
3319 3304
3320 3305 movq %rsp, %rbx /* %rbx = current %rsp */
3321 3306
3322 3307 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */
3323 3308 call panic_trigger /* %eax = panic_trigger() */
3324 3309
3325 3310 vpanic_common:
3326 3311 /*
3327 3312 * The panic_trigger result is in %eax from the call above, and
3328 3313 * dtrace_panic places it in %eax before branching here.
3329 3314 * The rdmsr instructions that follow below will clobber %eax so
3330 3315 * we stash the panic_trigger result in %r11d.
3331 3316 */
3332 3317 movl %eax, %r11d
3333 3318 cmpl $0, %r11d
3334 3319 je 0f
3335 3320
3336 3321 /*
3337 3322 * If panic_trigger() was successful, we are the first to initiate a
3338 3323 * panic: we now switch to the reserved panic_stack before continuing.
3339 3324 */
3340 3325 leaq panic_stack(%rip), %rsp
3341 3326 addq $PANICSTKSIZE, %rsp
3342 3327 0: subq $REGSIZE, %rsp
3343 3328 /*
3344 3329 * Now that we've got everything set up, store the register values as
3345 3330 * they were when we entered vpanic() to the designated location in
3346 3331 * the regs structure we allocated on the stack.
3347 3332 */
3348 3333 movq 0x0(%rbx), %rcx
3349 3334 movq %rcx, REGOFF_RDI(%rsp)
3350 3335 movq 0x8(%rbx), %rcx
3351 3336 movq %rcx, REGOFF_RSI(%rsp)
3352 3337 movq 0x10(%rbx), %rcx
3353 3338 movq %rcx, REGOFF_RDX(%rsp)
3354 3339 movq 0x18(%rbx), %rcx
3355 3340 movq %rcx, REGOFF_RCX(%rsp)
3356 3341 movq 0x20(%rbx), %rcx
3357 3342
3358 3343 movq %rcx, REGOFF_R8(%rsp)
3359 3344 movq 0x28(%rbx), %rcx
3360 3345 movq %rcx, REGOFF_R9(%rsp)
3361 3346 movq 0x30(%rbx), %rcx
3362 3347 movq %rcx, REGOFF_RAX(%rsp)
3363 3348 movq 0x38(%rbx), %rcx
3364 3349 movq %rcx, REGOFF_RBX(%rsp)
3365 3350 movq 0x58(%rbx), %rcx
3366 3351
3367 3352 movq %rcx, REGOFF_RBP(%rsp)
3368 3353 movq 0x40(%rbx), %rcx
3369 3354 movq %rcx, REGOFF_R10(%rsp)
3370 3355 movq 0x48(%rbx), %rcx
3371 3356 movq %rcx, REGOFF_R11(%rsp)
3372 3357 movq %r12, REGOFF_R12(%rsp)
3373 3358
3374 3359 movq %r13, REGOFF_R13(%rsp)
3375 3360 movq %r14, REGOFF_R14(%rsp)
3376 3361 movq %r15, REGOFF_R15(%rsp)
3377 3362
3378 3363 xorl %ecx, %ecx
3379 3364 movw %ds, %cx
3380 3365 movq %rcx, REGOFF_DS(%rsp)
3381 3366 movw %es, %cx
3382 3367 movq %rcx, REGOFF_ES(%rsp)
3383 3368 movw %fs, %cx
3384 3369 movq %rcx, REGOFF_FS(%rsp)
3385 3370 movw %gs, %cx
3386 3371 movq %rcx, REGOFF_GS(%rsp)
3387 3372
3388 3373 movq $0, REGOFF_TRAPNO(%rsp)
3389 3374
3390 3375 movq $0, REGOFF_ERR(%rsp)
3391 3376 leaq vpanic(%rip), %rcx
3392 3377 movq %rcx, REGOFF_RIP(%rsp)
3393 3378 movw %cs, %cx
3394 3379 movzwq %cx, %rcx
3395 3380 movq %rcx, REGOFF_CS(%rsp)
3396 3381 movq 0x50(%rbx), %rcx
3397 3382 movq %rcx, REGOFF_RFL(%rsp)
3398 3383 movq %rbx, %rcx
3399 3384 addq $0x60, %rcx
3400 3385 movq %rcx, REGOFF_RSP(%rsp)
3401 3386 movw %ss, %cx
3402 3387 movzwq %cx, %rcx
3403 3388 movq %rcx, REGOFF_SS(%rsp)
3404 3389
3405 3390 /*
3406 3391 * panicsys(format, alist, rp, on_panic_stack)
3407 3392 */
3408 3393 movq REGOFF_RDI(%rsp), %rdi /* format */
3409 3394 movq REGOFF_RSI(%rsp), %rsi /* alist */
3410 3395 movq %rsp, %rdx /* struct regs */
3411 3396 movl %r11d, %ecx /* on_panic_stack */
3412 3397 call panicsys
3413 3398 addq $REGSIZE, %rsp
3414 3399 popq %rdi
3415 3400 popq %rsi
3416 3401 popq %rdx
3417 3402 popq %rcx
3418 3403 popq %r8
3419 3404 popq %r9
3420 3405 popq %rax
3421 3406 popq %rbx
3422 3407 popq %r10
3423 3408 popq %r11
3424 3409 popfq
3425 3410 leave
3426 3411 ret
3427 3412 SET_SIZE(vpanic)
3428 3413
3429 3414 ENTRY_NP(dtrace_vpanic) /* Initial stack layout: */
3430 3415
3431 3416 pushq %rbp /* | %rip | 0x60 */
3432 3417 movq %rsp, %rbp /* | %rbp | 0x58 */
3433 3418 pushfq /* | rfl | 0x50 */
3434 3419 pushq %r11 /* | %r11 | 0x48 */
3435 3420 pushq %r10 /* | %r10 | 0x40 */
3436 3421 pushq %rbx /* | %rbx | 0x38 */
3437 3422 pushq %rax /* | %rax | 0x30 */
3438 3423 pushq %r9 /* | %r9 | 0x28 */
3439 3424 pushq %r8 /* | %r8 | 0x20 */
3440 3425 pushq %rcx /* | %rcx | 0x18 */
3441 3426 pushq %rdx /* | %rdx | 0x10 */
3442 3427 pushq %rsi /* | %rsi | 0x8 alist */
3443 3428 pushq %rdi /* | %rdi | 0x0 format */
3444 3429
3445 3430 movq %rsp, %rbx /* %rbx = current %rsp */
3446 3431
3447 3432 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */
3448 3433 call dtrace_panic_trigger /* %eax = dtrace_panic_trigger() */
3449 3434 jmp vpanic_common
3450 3435
3451 3436 SET_SIZE(dtrace_vpanic)
3452 3437
3453 3438 #elif defined(__i386)
3454 3439
3455 3440 ENTRY_NP(vpanic) / Initial stack layout:
3456 3441
3457 3442 pushl %ebp / | %eip | 20
3458 3443 movl %esp, %ebp / | %ebp | 16
3459 3444 pushl %eax / | %eax | 12
3460 3445 pushl %ebx / | %ebx | 8
3461 3446 pushl %ecx / | %ecx | 4
3462 3447 pushl %edx / | %edx | 0
3463 3448
3464 3449 movl %esp, %ebx / %ebx = current stack pointer
3465 3450
3466 3451 lea panic_quiesce, %eax / %eax = &panic_quiesce
3467 3452 pushl %eax / push &panic_quiesce
3468 3453 call panic_trigger / %eax = panic_trigger()
3469 3454 addl $4, %esp / reset stack pointer
3470 3455
3471 3456 vpanic_common:
3472 3457 cmpl $0, %eax / if (%eax == 0)
3473 3458 je 0f / goto 0f;
3474 3459
3475 3460 /*
3476 3461 * If panic_trigger() was successful, we are the first to initiate a
3477 3462 * panic: we now switch to the reserved panic_stack before continuing.
3478 3463 */
3479 3464 lea panic_stack, %esp / %esp = panic_stack
3480 3465 addl $PANICSTKSIZE, %esp / %esp += PANICSTKSIZE
3481 3466
3482 3467 0: subl $REGSIZE, %esp / allocate struct regs
3483 3468
3484 3469 /*
3485 3470 * Now that we've got everything set up, store the register values as
3486 3471 * they were when we entered vpanic() to the designated location in
3487 3472 * the regs structure we allocated on the stack.
3488 3473 */
3489 3474 #if !defined(__GNUC_AS__)
3490 3475 movw %gs, %edx
3491 3476 movl %edx, REGOFF_GS(%esp)
3492 3477 movw %fs, %edx
3493 3478 movl %edx, REGOFF_FS(%esp)
3494 3479 movw %es, %edx
3495 3480 movl %edx, REGOFF_ES(%esp)
3496 3481 movw %ds, %edx
3497 3482 movl %edx, REGOFF_DS(%esp)
3498 3483 #else /* __GNUC_AS__ */
3499 3484 mov %gs, %edx
3500 3485 mov %edx, REGOFF_GS(%esp)
3501 3486 mov %fs, %edx
3502 3487 mov %edx, REGOFF_FS(%esp)
3503 3488 mov %es, %edx
3504 3489 mov %edx, REGOFF_ES(%esp)
3505 3490 mov %ds, %edx
3506 3491 mov %edx, REGOFF_DS(%esp)
3507 3492 #endif /* __GNUC_AS__ */
3508 3493 movl %edi, REGOFF_EDI(%esp)
3509 3494 movl %esi, REGOFF_ESI(%esp)
3510 3495 movl 16(%ebx), %ecx
3511 3496 movl %ecx, REGOFF_EBP(%esp)
3512 3497 movl %ebx, %ecx
3513 3498 addl $20, %ecx
3514 3499 movl %ecx, REGOFF_ESP(%esp)
3515 3500 movl 8(%ebx), %ecx
3516 3501 movl %ecx, REGOFF_EBX(%esp)
3517 3502 movl 0(%ebx), %ecx
3518 3503 movl %ecx, REGOFF_EDX(%esp)
3519 3504 movl 4(%ebx), %ecx
3520 3505 movl %ecx, REGOFF_ECX(%esp)
3521 3506 movl 12(%ebx), %ecx
3522 3507 movl %ecx, REGOFF_EAX(%esp)
3523 3508 movl $0, REGOFF_TRAPNO(%esp)
3524 3509 movl $0, REGOFF_ERR(%esp)
3525 3510 lea vpanic, %ecx
3526 3511 movl %ecx, REGOFF_EIP(%esp)
3527 3512 #if !defined(__GNUC_AS__)
3528 3513 movw %cs, %edx
3529 3514 #else /* __GNUC_AS__ */
3530 3515 mov %cs, %edx
3531 3516 #endif /* __GNUC_AS__ */
3532 3517 movl %edx, REGOFF_CS(%esp)
3533 3518 pushfl
3534 3519 popl %ecx
3535 3520 #if defined(__xpv)
3536 3521 /*
3537 3522 * Synthesize the PS_IE bit from the event mask bit
3538 3523 */
3539 3524 CURTHREAD(%edx)
3540 3525 KPREEMPT_DISABLE(%edx)
3541 3526 EVENT_MASK_TO_IE(%edx, %ecx)
3542 3527 CURTHREAD(%edx)
3543 3528 KPREEMPT_ENABLE_NOKP(%edx)
3544 3529 #endif
3545 3530 movl %ecx, REGOFF_EFL(%esp)
3546 3531 movl $0, REGOFF_UESP(%esp)
3547 3532 #if !defined(__GNUC_AS__)
3548 3533 movw %ss, %edx
3549 3534 #else /* __GNUC_AS__ */
3550 3535 mov %ss, %edx
3551 3536 #endif /* __GNUC_AS__ */
3552 3537 movl %edx, REGOFF_SS(%esp)
3553 3538
3554 3539 movl %esp, %ecx / %ecx = ®s
3555 3540 pushl %eax / push on_panic_stack
3556 3541 pushl %ecx / push ®s
3557 3542 movl 12(%ebp), %ecx / %ecx = alist
3558 3543 pushl %ecx / push alist
3559 3544 movl 8(%ebp), %ecx / %ecx = format
3560 3545 pushl %ecx / push format
3561 3546 call panicsys / panicsys();
3562 3547 addl $16, %esp / pop arguments
3563 3548
3564 3549 addl $REGSIZE, %esp
3565 3550 popl %edx
3566 3551 popl %ecx
3567 3552 popl %ebx
3568 3553 popl %eax
3569 3554 leave
3570 3555 ret
3571 3556 SET_SIZE(vpanic)
3572 3557
3573 3558 ENTRY_NP(dtrace_vpanic) / Initial stack layout:
3574 3559
3575 3560 pushl %ebp / | %eip | 20
3576 3561 movl %esp, %ebp / | %ebp | 16
3577 3562 pushl %eax / | %eax | 12
3578 3563 pushl %ebx / | %ebx | 8
3579 3564 pushl %ecx / | %ecx | 4
3580 3565 pushl %edx / | %edx | 0
3581 3566
3582 3567 movl %esp, %ebx / %ebx = current stack pointer
3583 3568
3584 3569 lea panic_quiesce, %eax / %eax = &panic_quiesce
3585 3570 pushl %eax / push &panic_quiesce
3586 3571 call dtrace_panic_trigger / %eax = dtrace_panic_trigger()
3587 3572 addl $4, %esp / reset stack pointer
3588 3573 jmp vpanic_common / jump back to common code
3589 3574
3590 3575 SET_SIZE(dtrace_vpanic)
3591 3576
3592 3577 #endif /* __i386 */
3593 3578 #endif /* __lint */
3594 3579
3595 3580 #if defined(__lint)
3596 3581
3597 3582 void
3598 3583 hres_tick(void)
3599 3584 {}
3600 3585
3601 3586 int64_t timedelta;
3602 3587 hrtime_t hres_last_tick;
3603 3588 volatile timestruc_t hrestime;
3604 3589 int64_t hrestime_adj;
3605 3590 volatile int hres_lock;
3606 3591 hrtime_t hrtime_base;
3607 3592
3608 3593 #else /* __lint */
3609 3594
3610 3595 DGDEF3(hrestime, _MUL(2, CLONGSIZE), 8)
3611 3596 .NWORD 0, 0
3612 3597
3613 3598 DGDEF3(hrestime_adj, 8, 8)
3614 3599 .long 0, 0
3615 3600
3616 3601 DGDEF3(hres_last_tick, 8, 8)
3617 3602 .long 0, 0
3618 3603
3619 3604 DGDEF3(timedelta, 8, 8)
3620 3605 .long 0, 0
3621 3606
3622 3607 DGDEF3(hres_lock, 4, 8)
3623 3608 .long 0
3624 3609
3625 3610 /*
3626 3611 * initialized to a non zero value to make pc_gethrtime()
3627 3612 * work correctly even before clock is initialized
3628 3613 */
3629 3614 DGDEF3(hrtime_base, 8, 8)
3630 3615 .long _MUL(NSEC_PER_CLOCK_TICK, 6), 0
3631 3616
3632 3617 DGDEF3(adj_shift, 4, 4)
3633 3618 .long ADJ_SHIFT
3634 3619
3635 3620 #if defined(__amd64)
3636 3621
3637 3622 ENTRY_NP(hres_tick)
3638 3623 pushq %rbp
3639 3624 movq %rsp, %rbp
3640 3625
3641 3626 /*
3642 3627 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
3643 3628 * hres_last_tick can only be modified while holding CLOCK_LOCK).
3644 3629 * At worst, performing this now instead of under CLOCK_LOCK may
3645 3630 * introduce some jitter in pc_gethrestime().
3646 3631 */
3647 3632 call *gethrtimef(%rip)
3648 3633 movq %rax, %r8
3649 3634
3650 3635 leaq hres_lock(%rip), %rax
3651 3636 movb $-1, %dl
3652 3637 .CL1:
3653 3638 xchgb %dl, (%rax)
3654 3639 testb %dl, %dl
3655 3640 jz .CL3 /* got it */
3656 3641 .CL2:
3657 3642 cmpb $0, (%rax) /* possible to get lock? */
3658 3643 pause
3659 3644 jne .CL2
3660 3645 jmp .CL1 /* yes, try again */
3661 3646 .CL3:
3662 3647 /*
3663 3648 * compute the interval since last time hres_tick was called
3664 3649 * and adjust hrtime_base and hrestime accordingly
3665 3650 * hrtime_base is an 8 byte value (in nsec), hrestime is
3666 3651 * a timestruc_t (sec, nsec)
3667 3652 */
3668 3653 leaq hres_last_tick(%rip), %rax
3669 3654 movq %r8, %r11
3670 3655 subq (%rax), %r8
3671 3656 addq %r8, hrtime_base(%rip) /* add interval to hrtime_base */
3672 3657 addq %r8, hrestime+8(%rip) /* add interval to hrestime.tv_nsec */
3673 3658 /*
3674 3659 * Now that we have CLOCK_LOCK, we can update hres_last_tick
3675 3660 */
3676 3661 movq %r11, (%rax)
3677 3662
3678 3663 call __adj_hrestime
3679 3664
3680 3665 /*
3681 3666 * release the hres_lock
3682 3667 */
3683 3668 incl hres_lock(%rip)
3684 3669 leave
3685 3670 ret
3686 3671 SET_SIZE(hres_tick)
3687 3672
3688 3673 #elif defined(__i386)
3689 3674
3690 3675 ENTRY_NP(hres_tick)
3691 3676 pushl %ebp
3692 3677 movl %esp, %ebp
3693 3678 pushl %esi
3694 3679 pushl %ebx
3695 3680
3696 3681 /*
3697 3682 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
3698 3683 * hres_last_tick can only be modified while holding CLOCK_LOCK).
3699 3684 * At worst, performing this now instead of under CLOCK_LOCK may
3700 3685 * introduce some jitter in pc_gethrestime().
3701 3686 */
3702 3687 call *gethrtimef
3703 3688 movl %eax, %ebx
3704 3689 movl %edx, %esi
3705 3690
3706 3691 movl $hres_lock, %eax
3707 3692 movl $-1, %edx
3708 3693 .CL1:
3709 3694 xchgb %dl, (%eax)
3710 3695 testb %dl, %dl
3711 3696 jz .CL3 / got it
3712 3697 .CL2:
3713 3698 cmpb $0, (%eax) / possible to get lock?
3714 3699 pause
3715 3700 jne .CL2
3716 3701 jmp .CL1 / yes, try again
3717 3702 .CL3:
3718 3703 /*
3719 3704 * compute the interval since last time hres_tick was called
3720 3705 * and adjust hrtime_base and hrestime accordingly
3721 3706 * hrtime_base is an 8 byte value (in nsec), hrestime is
3722 3707 * timestruc_t (sec, nsec)
3723 3708 */
3724 3709
3725 3710 lea hres_last_tick, %eax
3726 3711
3727 3712 movl %ebx, %edx
3728 3713 movl %esi, %ecx
3729 3714
3730 3715 subl (%eax), %edx
3731 3716 sbbl 4(%eax), %ecx
3732 3717
3733 3718 addl %edx, hrtime_base / add interval to hrtime_base
3734 3719 adcl %ecx, hrtime_base+4
3735 3720
3736 3721 addl %edx, hrestime+4 / add interval to hrestime.tv_nsec
3737 3722
3738 3723 /
3739 3724 / Now that we have CLOCK_LOCK, we can update hres_last_tick.
3740 3725 /
3741 3726 movl %ebx, (%eax)
3742 3727 movl %esi, 4(%eax)
3743 3728
3744 3729 / get hrestime at this moment. used as base for pc_gethrestime
3745 3730 /
3746 3731 / Apply adjustment, if any
3747 3732 /
3748 3733 / #define HRES_ADJ (NSEC_PER_CLOCK_TICK >> ADJ_SHIFT)
3749 3734 / (max_hres_adj)
3750 3735 /
3751 3736 / void
3752 3737 / adj_hrestime()
3753 3738 / {
3754 3739 / long long adj;
3755 3740 /
3756 3741 / if (hrestime_adj == 0)
3757 3742 / adj = 0;
3758 3743 / else if (hrestime_adj > 0) {
3759 3744 / if (hrestime_adj < HRES_ADJ)
3760 3745 / adj = hrestime_adj;
3761 3746 / else
3762 3747 / adj = HRES_ADJ;
3763 3748 / }
3764 3749 / else {
3765 3750 / if (hrestime_adj < -(HRES_ADJ))
3766 3751 / adj = -(HRES_ADJ);
3767 3752 / else
3768 3753 / adj = hrestime_adj;
3769 3754 / }
3770 3755 /
3771 3756 / timedelta -= adj;
3772 3757 / hrestime_adj = timedelta;
3773 3758 / hrestime.tv_nsec += adj;
3774 3759 /
3775 3760 / while (hrestime.tv_nsec >= NANOSEC) {
3776 3761 / one_sec++;
3777 3762 / hrestime.tv_sec++;
3778 3763 / hrestime.tv_nsec -= NANOSEC;
3779 3764 / }
3780 3765 / }
3781 3766 __adj_hrestime:
3782 3767 movl hrestime_adj, %esi / if (hrestime_adj == 0)
3783 3768 movl hrestime_adj+4, %edx
3784 3769 andl %esi, %esi
3785 3770 jne .CL4 / no
3786 3771 andl %edx, %edx
3787 3772 jne .CL4 / no
3788 3773 subl %ecx, %ecx / yes, adj = 0;
3789 3774 subl %edx, %edx
3790 3775 jmp .CL5
3791 3776 .CL4:
3792 3777 subl %ecx, %ecx
3793 3778 subl %eax, %eax
3794 3779 subl %esi, %ecx
3795 3780 sbbl %edx, %eax
3796 3781 andl %eax, %eax / if (hrestime_adj > 0)
3797 3782 jge .CL6
3798 3783
3799 3784 / In the following comments, HRES_ADJ is used, while in the code
3800 3785 / max_hres_adj is used.
3801 3786 /
3802 3787 / The test for "hrestime_adj < HRES_ADJ" is complicated because
3803 3788 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely
3804 3789 / on the logical equivalence of:
3805 3790 /
3806 3791 / !(hrestime_adj < HRES_ADJ)
3807 3792 /
3808 3793 / and the two step sequence:
3809 3794 /
3810 3795 / (HRES_ADJ - lsw(hrestime_adj)) generates a Borrow/Carry
3811 3796 /
3812 3797 / which computes whether or not the least significant 32-bits
3813 3798 / of hrestime_adj is greater than HRES_ADJ, followed by:
3814 3799 /
3815 3800 / Previous Borrow/Carry + -1 + msw(hrestime_adj) generates a Carry
3816 3801 /
3817 3802 / which generates a carry whenever step 1 is true or the most
3818 3803 / significant long of the longlong hrestime_adj is non-zero.
3819 3804
3820 3805 movl max_hres_adj, %ecx / hrestime_adj is positive
3821 3806 subl %esi, %ecx
3822 3807 movl %edx, %eax
3823 3808 adcl $-1, %eax
3824 3809 jnc .CL7
3825 3810 movl max_hres_adj, %ecx / adj = HRES_ADJ;
3826 3811 subl %edx, %edx
3827 3812 jmp .CL5
3828 3813
3829 3814 / The following computation is similar to the one above.
3830 3815 /
3831 3816 / The test for "hrestime_adj < -(HRES_ADJ)" is complicated because
3832 3817 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely
3833 3818 / on the logical equivalence of:
3834 3819 /
3835 3820 / (hrestime_adj > -HRES_ADJ)
3836 3821 /
3837 3822 / and the two step sequence:
3838 3823 /
3839 3824 / (HRES_ADJ + lsw(hrestime_adj)) generates a Carry
3840 3825 /
3841 3826 / which means the least significant 32-bits of hrestime_adj is
3842 3827 / greater than -HRES_ADJ, followed by:
3843 3828 /
3844 3829 / Previous Carry + 0 + msw(hrestime_adj) generates a Carry
3845 3830 /
3846 3831 / which generates a carry only when step 1 is true and the most
3847 3832 / significant long of the longlong hrestime_adj is -1.
3848 3833
3849 3834 .CL6: / hrestime_adj is negative
3850 3835 movl %esi, %ecx
3851 3836 addl max_hres_adj, %ecx
3852 3837 movl %edx, %eax
3853 3838 adcl $0, %eax
3854 3839 jc .CL7
3855 3840 xor %ecx, %ecx
3856 3841 subl max_hres_adj, %ecx / adj = -(HRES_ADJ);
3857 3842 movl $-1, %edx
3858 3843 jmp .CL5
3859 3844 .CL7:
3860 3845 movl %esi, %ecx / adj = hrestime_adj;
3861 3846 .CL5:
3862 3847 movl timedelta, %esi
3863 3848 subl %ecx, %esi
3864 3849 movl timedelta+4, %eax
3865 3850 sbbl %edx, %eax
3866 3851 movl %esi, timedelta
3867 3852 movl %eax, timedelta+4 / timedelta -= adj;
3868 3853 movl %esi, hrestime_adj
3869 3854 movl %eax, hrestime_adj+4 / hrestime_adj = timedelta;
3870 3855 addl hrestime+4, %ecx
3871 3856
3872 3857 movl %ecx, %eax / eax = tv_nsec
3873 3858 1:
3874 3859 cmpl $NANOSEC, %eax / if ((unsigned long)tv_nsec >= NANOSEC)
3875 3860 jb .CL8 / no
3876 3861 incl one_sec / yes, one_sec++;
3877 3862 incl hrestime / hrestime.tv_sec++;
3878 3863 addl $-NANOSEC, %eax / tv_nsec -= NANOSEC
3879 3864 jmp 1b / check for more seconds
3880 3865
3881 3866 .CL8:
3882 3867 movl %eax, hrestime+4 / store final into hrestime.tv_nsec
3883 3868 incl hres_lock / release the hres_lock
3884 3869
3885 3870 popl %ebx
3886 3871 popl %esi
3887 3872 leave
3888 3873 ret
3889 3874 SET_SIZE(hres_tick)
3890 3875
3891 3876 #endif /* __i386 */
3892 3877 #endif /* __lint */
3893 3878
3894 3879 /*
3895 3880 * void prefetch_smap_w(void *)
3896 3881 *
3897 3882 * Prefetch ahead within a linear list of smap structures.
3898 3883 * Not implemented for ia32. Stub for compatibility.
3899 3884 */
3900 3885
3901 3886 #if defined(__lint)
3902 3887
3903 3888 /*ARGSUSED*/
3904 3889 void prefetch_smap_w(void *smp)
3905 3890 {}
3906 3891
3907 3892 #else /* __lint */
3908 3893
3909 3894 ENTRY(prefetch_smap_w)
3910 3895 rep; ret /* use 2 byte return instruction when branch target */
3911 3896 /* AMD Software Optimization Guide - Section 6.2 */
3912 3897 SET_SIZE(prefetch_smap_w)
3913 3898
3914 3899 #endif /* __lint */
3915 3900
3916 3901 /*
3917 3902 * prefetch_page_r(page_t *)
3918 3903 * issue prefetch instructions for a page_t
3919 3904 */
3920 3905 #if defined(__lint)
3921 3906
3922 3907 /*ARGSUSED*/
3923 3908 void
3924 3909 prefetch_page_r(void *pp)
3925 3910 {}
3926 3911
3927 3912 #else /* __lint */
3928 3913
3929 3914 ENTRY(prefetch_page_r)
3930 3915 rep; ret /* use 2 byte return instruction when branch target */
3931 3916 /* AMD Software Optimization Guide - Section 6.2 */
3932 3917 SET_SIZE(prefetch_page_r)
3933 3918
3934 3919 #endif /* __lint */
3935 3920
3936 3921 #if defined(__lint)
3937 3922
3938 3923 /*ARGSUSED*/
3939 3924 int
3940 3925 bcmp(const void *s1, const void *s2, size_t count)
3941 3926 { return (0); }
3942 3927
3943 3928 #else /* __lint */
3944 3929
3945 3930 #if defined(__amd64)
3946 3931
3947 3932 ENTRY(bcmp)
3948 3933 pushq %rbp
3949 3934 movq %rsp, %rbp
3950 3935 #ifdef DEBUG
3951 3936 testq %rdx,%rdx
3952 3937 je 1f
3953 3938 movq postbootkernelbase(%rip), %r11
3954 3939 cmpq %r11, %rdi
3955 3940 jb 0f
3956 3941 cmpq %r11, %rsi
3957 3942 jnb 1f
3958 3943 0: leaq .bcmp_panic_msg(%rip), %rdi
3959 3944 xorl %eax, %eax
3960 3945 call panic
3961 3946 1:
3962 3947 #endif /* DEBUG */
3963 3948 call memcmp
3964 3949 testl %eax, %eax
3965 3950 setne %dl
3966 3951 leave
3967 3952 movzbl %dl, %eax
3968 3953 ret
3969 3954 SET_SIZE(bcmp)
3970 3955
3971 3956 #elif defined(__i386)
3972 3957
3973 3958 #define ARG_S1 8
3974 3959 #define ARG_S2 12
3975 3960 #define ARG_LENGTH 16
3976 3961
3977 3962 ENTRY(bcmp)
3978 3963 pushl %ebp
3979 3964 movl %esp, %ebp / create new stack frame
3980 3965 #ifdef DEBUG
3981 3966 cmpl $0, ARG_LENGTH(%ebp)
3982 3967 je 1f
3983 3968 movl postbootkernelbase, %eax
3984 3969 cmpl %eax, ARG_S1(%ebp)
3985 3970 jb 0f
3986 3971 cmpl %eax, ARG_S2(%ebp)
3987 3972 jnb 1f
3988 3973 0: pushl $.bcmp_panic_msg
3989 3974 call panic
3990 3975 1:
3991 3976 #endif /* DEBUG */
3992 3977
3993 3978 pushl %edi / save register variable
3994 3979 movl ARG_S1(%ebp), %eax / %eax = address of string 1
3995 3980 movl ARG_S2(%ebp), %ecx / %ecx = address of string 2
3996 3981 cmpl %eax, %ecx / if the same string
3997 3982 je .equal / goto .equal
3998 3983 movl ARG_LENGTH(%ebp), %edi / %edi = length in bytes
3999 3984 cmpl $4, %edi / if %edi < 4
4000 3985 jb .byte_check / goto .byte_check
4001 3986 .align 4
4002 3987 .word_loop:
4003 3988 movl (%ecx), %edx / move 1 word from (%ecx) to %edx
4004 3989 leal -4(%edi), %edi / %edi -= 4
4005 3990 cmpl (%eax), %edx / compare 1 word from (%eax) with %edx
4006 3991 jne .word_not_equal / if not equal, goto .word_not_equal
4007 3992 leal 4(%ecx), %ecx / %ecx += 4 (next word)
4008 3993 leal 4(%eax), %eax / %eax += 4 (next word)
4009 3994 cmpl $4, %edi / if %edi >= 4
4010 3995 jae .word_loop / goto .word_loop
4011 3996 .byte_check:
4012 3997 cmpl $0, %edi / if %edi == 0
4013 3998 je .equal / goto .equal
4014 3999 jmp .byte_loop / goto .byte_loop (checks in bytes)
4015 4000 .word_not_equal:
4016 4001 leal 4(%edi), %edi / %edi += 4 (post-decremented)
4017 4002 .align 4
4018 4003 .byte_loop:
4019 4004 movb (%ecx), %dl / move 1 byte from (%ecx) to %dl
4020 4005 cmpb %dl, (%eax) / compare %dl with 1 byte from (%eax)
4021 4006 jne .not_equal / if not equal, goto .not_equal
4022 4007 incl %ecx / %ecx++ (next byte)
4023 4008 incl %eax / %eax++ (next byte)
4024 4009 decl %edi / %edi--
4025 4010 jnz .byte_loop / if not zero, goto .byte_loop
4026 4011 .equal:
4027 4012 xorl %eax, %eax / %eax = 0
4028 4013 popl %edi / restore register variable
4029 4014 leave / restore old stack frame
4030 4015 ret / return (NULL)
4031 4016 .align 4
4032 4017 .not_equal:
4033 4018 movl $1, %eax / return 1
4034 4019 popl %edi / restore register variable
4035 4020 leave / restore old stack frame
4036 4021 ret / return (NULL)
4037 4022 SET_SIZE(bcmp)
4038 4023
4039 4024 #endif /* __i386 */
4040 4025
4041 4026 #ifdef DEBUG
4042 4027 .text
4043 4028 .bcmp_panic_msg:
4044 4029 .string "bcmp: arguments below kernelbase"
4045 4030 #endif /* DEBUG */
4046 4031
4047 4032 #endif /* __lint */
4048 4033
4049 4034 #if defined(__lint)
4050 4035
4051 4036 uint_t
4052 4037 bsrw_insn(uint16_t mask)
4053 4038 {
4054 4039 uint_t index = sizeof (mask) * NBBY - 1;
4055 4040
4056 4041 while ((mask & (1 << index)) == 0)
4057 4042 index--;
4058 4043 return (index);
4059 4044 }
4060 4045
4061 4046 #else /* __lint */
4062 4047
4063 4048 #if defined(__amd64)
4064 4049
4065 4050 ENTRY_NP(bsrw_insn)
4066 4051 xorl %eax, %eax
4067 4052 bsrw %di, %ax
4068 4053 ret
4069 4054 SET_SIZE(bsrw_insn)
4070 4055
4071 4056 #elif defined(__i386)
4072 4057
4073 4058 ENTRY_NP(bsrw_insn)
4074 4059 movw 4(%esp), %cx
4075 4060 xorl %eax, %eax
4076 4061 bsrw %cx, %ax
4077 4062 ret
4078 4063 SET_SIZE(bsrw_insn)
4079 4064
4080 4065 #endif /* __i386 */
4081 4066 #endif /* __lint */
4082 4067
4083 4068 #if defined(__lint)
4084 4069
4085 4070 uint_t
4086 4071 atomic_btr32(uint32_t *pending, uint_t pil)
4087 4072 {
4088 4073 return (*pending &= ~(1 << pil));
4089 4074 }
4090 4075
4091 4076 #else /* __lint */
4092 4077
4093 4078 #if defined(__i386)
4094 4079
4095 4080 ENTRY_NP(atomic_btr32)
4096 4081 movl 4(%esp), %ecx
4097 4082 movl 8(%esp), %edx
4098 4083 xorl %eax, %eax
4099 4084 lock
4100 4085 btrl %edx, (%ecx)
4101 4086 setc %al
4102 4087 ret
4103 4088 SET_SIZE(atomic_btr32)
4104 4089
4105 4090 #endif /* __i386 */
4106 4091 #endif /* __lint */
4107 4092
4108 4093 #if defined(__lint)
4109 4094
4110 4095 /*ARGSUSED*/
4111 4096 void
4112 4097 switch_sp_and_call(void *newsp, void (*func)(uint_t, uint_t), uint_t arg1,
4113 4098 uint_t arg2)
4114 4099 {}
4115 4100
4116 4101 #else /* __lint */
4117 4102
4118 4103 #if defined(__amd64)
4119 4104
4120 4105 ENTRY_NP(switch_sp_and_call)
4121 4106 pushq %rbp
4122 4107 movq %rsp, %rbp /* set up stack frame */
4123 4108 movq %rdi, %rsp /* switch stack pointer */
4124 4109 movq %rdx, %rdi /* pass func arg 1 */
4125 4110 movq %rsi, %r11 /* save function to call */
4126 4111 movq %rcx, %rsi /* pass func arg 2 */
4127 4112 call *%r11 /* call function */
4128 4113 leave /* restore stack */
4129 4114 ret
4130 4115 SET_SIZE(switch_sp_and_call)
4131 4116
4132 4117 #elif defined(__i386)
4133 4118
4134 4119 ENTRY_NP(switch_sp_and_call)
4135 4120 pushl %ebp
4136 4121 mov %esp, %ebp /* set up stack frame */
4137 4122 movl 8(%ebp), %esp /* switch stack pointer */
4138 4123 pushl 20(%ebp) /* push func arg 2 */
4139 4124 pushl 16(%ebp) /* push func arg 1 */
4140 4125 call *12(%ebp) /* call function */
4141 4126 addl $8, %esp /* pop arguments */
4142 4127 leave /* restore stack */
4143 4128 ret
4144 4129 SET_SIZE(switch_sp_and_call)
4145 4130
4146 4131 #endif /* __i386 */
4147 4132 #endif /* __lint */
4148 4133
4149 4134 #if defined(__lint)
4150 4135
4151 4136 void
4152 4137 kmdb_enter(void)
4153 4138 {}
4154 4139
4155 4140 #else /* __lint */
4156 4141
4157 4142 #if defined(__amd64)
4158 4143
4159 4144 ENTRY_NP(kmdb_enter)
4160 4145 pushq %rbp
4161 4146 movq %rsp, %rbp
4162 4147
4163 4148 /*
4164 4149 * Save flags, do a 'cli' then return the saved flags
4165 4150 */
4166 4151 call intr_clear
4167 4152
4168 4153 int $T_DBGENTR
4169 4154
4170 4155 /*
4171 4156 * Restore the saved flags
4172 4157 */
4173 4158 movq %rax, %rdi
4174 4159 call intr_restore
4175 4160
4176 4161 leave
4177 4162 ret
4178 4163 SET_SIZE(kmdb_enter)
4179 4164
4180 4165 #elif defined(__i386)
4181 4166
4182 4167 ENTRY_NP(kmdb_enter)
4183 4168 pushl %ebp
4184 4169 movl %esp, %ebp
4185 4170
4186 4171 /*
4187 4172 * Save flags, do a 'cli' then return the saved flags
4188 4173 */
4189 4174 call intr_clear
4190 4175
4191 4176 int $T_DBGENTR
4192 4177
4193 4178 /*
4194 4179 * Restore the saved flags
4195 4180 */
4196 4181 pushl %eax
4197 4182 call intr_restore
4198 4183 addl $4, %esp
4199 4184
4200 4185 leave
4201 4186 ret
4202 4187 SET_SIZE(kmdb_enter)
4203 4188
4204 4189 #endif /* __i386 */
4205 4190 #endif /* __lint */
4206 4191
4207 4192 #if defined(__lint)
4208 4193
4209 4194 void
4210 4195 return_instr(void)
4211 4196 {}
4212 4197
4213 4198 #else /* __lint */
4214 4199
4215 4200 ENTRY_NP(return_instr)
4216 4201 rep; ret /* use 2 byte instruction when branch target */
4217 4202 /* AMD Software Optimization Guide - Section 6.2 */
4218 4203 SET_SIZE(return_instr)
4219 4204
4220 4205 #endif /* __lint */
4221 4206
4222 4207 #if defined(__lint)
4223 4208
4224 4209 ulong_t
4225 4210 getflags(void)
4226 4211 {
4227 4212 return (0);
4228 4213 }
4229 4214
4230 4215 #else /* __lint */
4231 4216
4232 4217 #if defined(__amd64)
4233 4218
4234 4219 ENTRY(getflags)
4235 4220 pushfq
4236 4221 popq %rax
4237 4222 #if defined(__xpv)
4238 4223 CURTHREAD(%rdi)
4239 4224 KPREEMPT_DISABLE(%rdi)
4240 4225 /*
4241 4226 * Synthesize the PS_IE bit from the event mask bit
4242 4227 */
4243 4228 CURVCPU(%r11)
4244 4229 andq $_BITNOT(PS_IE), %rax
4245 4230 XEN_TEST_UPCALL_MASK(%r11)
4246 4231 jnz 1f
4247 4232 orq $PS_IE, %rax
4248 4233 1:
4249 4234 KPREEMPT_ENABLE_NOKP(%rdi)
4250 4235 #endif
4251 4236 ret
4252 4237 SET_SIZE(getflags)
4253 4238
4254 4239 #elif defined(__i386)
4255 4240
4256 4241 ENTRY(getflags)
4257 4242 pushfl
4258 4243 popl %eax
4259 4244 #if defined(__xpv)
4260 4245 CURTHREAD(%ecx)
4261 4246 KPREEMPT_DISABLE(%ecx)
4262 4247 /*
4263 4248 * Synthesize the PS_IE bit from the event mask bit
4264 4249 */
4265 4250 CURVCPU(%edx)
4266 4251 andl $_BITNOT(PS_IE), %eax
4267 4252 XEN_TEST_UPCALL_MASK(%edx)
4268 4253 jnz 1f
4269 4254 orl $PS_IE, %eax
4270 4255 1:
4271 4256 KPREEMPT_ENABLE_NOKP(%ecx)
4272 4257 #endif
4273 4258 ret
4274 4259 SET_SIZE(getflags)
4275 4260
4276 4261 #endif /* __i386 */
4277 4262
4278 4263 #endif /* __lint */
4279 4264
4280 4265 #if defined(__lint)
4281 4266
4282 4267 ftrace_icookie_t
4283 4268 ftrace_interrupt_disable(void)
4284 4269 { return (0); }
4285 4270
4286 4271 #else /* __lint */
4287 4272
4288 4273 #if defined(__amd64)
4289 4274
4290 4275 ENTRY(ftrace_interrupt_disable)
4291 4276 pushfq
4292 4277 popq %rax
4293 4278 CLI(%rdx)
4294 4279 ret
4295 4280 SET_SIZE(ftrace_interrupt_disable)
4296 4281
4297 4282 #elif defined(__i386)
4298 4283
4299 4284 ENTRY(ftrace_interrupt_disable)
4300 4285 pushfl
4301 4286 popl %eax
4302 4287 CLI(%edx)
4303 4288 ret
4304 4289 SET_SIZE(ftrace_interrupt_disable)
4305 4290
4306 4291 #endif /* __i386 */
4307 4292 #endif /* __lint */
4308 4293
4309 4294 #if defined(__lint)
4310 4295
4311 4296 /*ARGSUSED*/
4312 4297 void
4313 4298 ftrace_interrupt_enable(ftrace_icookie_t cookie)
4314 4299 {}
4315 4300
4316 4301 #else /* __lint */
4317 4302
4318 4303 #if defined(__amd64)
4319 4304
4320 4305 ENTRY(ftrace_interrupt_enable)
4321 4306 pushq %rdi
4322 4307 popfq
4323 4308 ret
4324 4309 SET_SIZE(ftrace_interrupt_enable)
4325 4310
4326 4311 #elif defined(__i386)
4327 4312
4328 4313 ENTRY(ftrace_interrupt_enable)
4329 4314 movl 4(%esp), %eax
4330 4315 pushl %eax
4331 4316 popfl
4332 4317 ret
4333 4318 SET_SIZE(ftrace_interrupt_enable)
4334 4319
4335 4320 #endif /* __i386 */
4336 4321 #endif /* __lint */
4337 4322
4338 4323 #if defined (__lint)
4339 4324
4340 4325 /*ARGSUSED*/
4341 4326 void
4342 4327 clflush_insn(caddr_t addr)
4343 4328 {}
4344 4329
4345 4330 #else /* __lint */
4346 4331
4347 4332 #if defined (__amd64)
4348 4333 ENTRY(clflush_insn)
4349 4334 clflush (%rdi)
4350 4335 ret
4351 4336 SET_SIZE(clflush_insn)
4352 4337 #elif defined (__i386)
4353 4338 ENTRY(clflush_insn)
4354 4339 movl 4(%esp), %eax
4355 4340 clflush (%eax)
4356 4341 ret
4357 4342 SET_SIZE(clflush_insn)
4358 4343
4359 4344 #endif /* __i386 */
4360 4345 #endif /* __lint */
4361 4346
4362 4347 #if defined (__lint)
4363 4348 /*ARGSUSED*/
4364 4349 void
4365 4350 mfence_insn(void)
4366 4351 {}
4367 4352
4368 4353 #else /* __lint */
4369 4354
4370 4355 #if defined (__amd64)
4371 4356 ENTRY(mfence_insn)
4372 4357 mfence
4373 4358 ret
4374 4359 SET_SIZE(mfence_insn)
4375 4360 #elif defined (__i386)
4376 4361 ENTRY(mfence_insn)
4377 4362 mfence
4378 4363 ret
4379 4364 SET_SIZE(mfence_insn)
4380 4365
4381 4366 #endif /* __i386 */
4382 4367 #endif /* __lint */
4383 4368
4384 4369 /*
4385 4370 * VMware implements an I/O port that programs can query to detect if software
4386 4371 * is running in a VMware hypervisor. This hypervisor port behaves differently
4387 4372 * depending on magic values in certain registers and modifies some registers
4388 4373 * as a side effect.
4389 4374 *
4390 4375 * References: http://kb.vmware.com/kb/1009458
4391 4376 */
4392 4377
4393 4378 #if defined(__lint)
4394 4379
4395 4380 /* ARGSUSED */
4396 4381 void
4397 4382 vmware_port(int cmd, uint32_t *regs) { return; }
4398 4383
4399 4384 #else
4400 4385
4401 4386 #if defined(__amd64)
4402 4387
4403 4388 ENTRY(vmware_port)
4404 4389 pushq %rbx
4405 4390 movl $VMWARE_HVMAGIC, %eax
4406 4391 movl $0xffffffff, %ebx
4407 4392 movl %edi, %ecx
4408 4393 movl $VMWARE_HVPORT, %edx
4409 4394 inl (%dx)
4410 4395 movl %eax, (%rsi)
4411 4396 movl %ebx, 4(%rsi)
4412 4397 movl %ecx, 8(%rsi)
4413 4398 movl %edx, 12(%rsi)
4414 4399 popq %rbx
4415 4400 ret
4416 4401 SET_SIZE(vmware_port)
4417 4402
4418 4403 #elif defined(__i386)
4419 4404
4420 4405 ENTRY(vmware_port)
4421 4406 pushl %ebx
4422 4407 pushl %esi
4423 4408 movl $VMWARE_HVMAGIC, %eax
4424 4409 movl $0xffffffff, %ebx
4425 4410 movl 12(%esp), %ecx
4426 4411 movl $VMWARE_HVPORT, %edx
4427 4412 inl (%dx)
4428 4413 movl 16(%esp), %esi
4429 4414 movl %eax, (%esi)
4430 4415 movl %ebx, 4(%esi)
4431 4416 movl %ecx, 8(%esi)
4432 4417 movl %edx, 12(%esi)
4433 4418 popl %esi
4434 4419 popl %ebx
4435 4420 ret
4436 4421 SET_SIZE(vmware_port)
4437 4422
4438 4423 #endif /* __i386 */
4439 4424 #endif /* __lint */
|
↓ open down ↓ |
1548 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX