1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2014 Nexenta Systems, Inc. All rights reserved.
25 * Copyright (c) 2014 by Delphix. All rights reserved.
26 */
27
28 /*
29 * Copyright (c) 1990, 1991 UNIX System Laboratories, Inc.
30 * Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T
31 * All Rights Reserved
32 */
33
34 /*
35 * Copyright (c) 2009, Intel Corporation.
36 * All rights reserved.
37 */
38
39 /*
40 * General assembly language routines.
41 * It is the intent of this file to contain routines that are
42 * independent of the specific kernel architecture, and those that are
43 * common across kernel architectures.
44 * As architectures diverge, and implementations of specific
45 * architecture-dependent routines change, the routines should be moved
46 * from this file into the respective ../`arch -k`/subr.s file.
47 */
48
49 #include <sys/asm_linkage.h>
50 #include <sys/asm_misc.h>
51 #include <sys/panic.h>
52 #include <sys/ontrap.h>
53 #include <sys/regset.h>
54 #include <sys/privregs.h>
55 #include <sys/reboot.h>
56 #include <sys/psw.h>
57 #include <sys/x86_archext.h>
58
59 #if defined(__lint)
60 #include <sys/types.h>
61 #include <sys/systm.h>
62 #include <sys/thread.h>
63 #include <sys/archsystm.h>
64 #include <sys/byteorder.h>
65 #include <sys/dtrace.h>
66 #include <sys/ftrace.h>
67 #else /* __lint */
68 #include "assym.h"
69 #endif /* __lint */
70 #include <sys/dditypes.h>
71
72 /*
73 * on_fault()
74 * Catch lofault faults. Like setjmp except it returns one
75 * if code following causes uncorrectable fault. Turned off
76 * by calling no_fault().
77 */
78
79 #if defined(__lint)
80
81 /* ARGSUSED */
82 int
83 on_fault(label_t *ljb)
84 { return (0); }
85
86 void
87 no_fault(void)
88 {}
89
90 #else /* __lint */
91
92 #if defined(__amd64)
93
94 ENTRY(on_fault)
95 movq %gs:CPU_THREAD, %rsi
96 leaq catch_fault(%rip), %rdx
97 movq %rdi, T_ONFAULT(%rsi) /* jumpbuf in t_onfault */
98 movq %rdx, T_LOFAULT(%rsi) /* catch_fault in t_lofault */
99 jmp setjmp /* let setjmp do the rest */
100
101 catch_fault:
102 movq %gs:CPU_THREAD, %rsi
103 movq T_ONFAULT(%rsi), %rdi /* address of save area */
104 xorl %eax, %eax
105 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */
106 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */
107 jmp longjmp /* let longjmp do the rest */
108 SET_SIZE(on_fault)
109
110 ENTRY(no_fault)
111 movq %gs:CPU_THREAD, %rsi
112 xorl %eax, %eax
113 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */
114 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */
115 ret
116 SET_SIZE(no_fault)
117
118 #elif defined(__i386)
119
120 ENTRY(on_fault)
121 movl %gs:CPU_THREAD, %edx
122 movl 4(%esp), %eax /* jumpbuf address */
123 leal catch_fault, %ecx
124 movl %eax, T_ONFAULT(%edx) /* jumpbuf in t_onfault */
125 movl %ecx, T_LOFAULT(%edx) /* catch_fault in t_lofault */
126 jmp setjmp /* let setjmp do the rest */
127
128 catch_fault:
129 movl %gs:CPU_THREAD, %edx
130 xorl %eax, %eax
131 movl T_ONFAULT(%edx), %ecx /* address of save area */
132 movl %eax, T_ONFAULT(%edx) /* turn off onfault */
133 movl %eax, T_LOFAULT(%edx) /* turn off lofault */
134 pushl %ecx
135 call longjmp /* let longjmp do the rest */
136 SET_SIZE(on_fault)
137
138 ENTRY(no_fault)
139 movl %gs:CPU_THREAD, %edx
140 xorl %eax, %eax
141 movl %eax, T_ONFAULT(%edx) /* turn off onfault */
142 movl %eax, T_LOFAULT(%edx) /* turn off lofault */
143 ret
144 SET_SIZE(no_fault)
145
146 #endif /* __i386 */
147 #endif /* __lint */
148
149 /*
150 * Default trampoline code for on_trap() (see <sys/ontrap.h>). We just
151 * do a longjmp(&curthread->t_ontrap->ot_jmpbuf) if this is ever called.
152 */
153
154 #if defined(lint)
155
156 void
157 on_trap_trampoline(void)
158 {}
159
160 #else /* __lint */
161
162 #if defined(__amd64)
163
164 ENTRY(on_trap_trampoline)
165 movq %gs:CPU_THREAD, %rsi
166 movq T_ONTRAP(%rsi), %rdi
167 addq $OT_JMPBUF, %rdi
168 jmp longjmp
169 SET_SIZE(on_trap_trampoline)
170
171 #elif defined(__i386)
172
173 ENTRY(on_trap_trampoline)
174 movl %gs:CPU_THREAD, %eax
175 movl T_ONTRAP(%eax), %eax
176 addl $OT_JMPBUF, %eax
177 pushl %eax
178 call longjmp
179 SET_SIZE(on_trap_trampoline)
180
181 #endif /* __i386 */
182 #endif /* __lint */
183
184 /*
185 * Push a new element on to the t_ontrap stack. Refer to <sys/ontrap.h> for
186 * more information about the on_trap() mechanism. If the on_trap_data is the
187 * same as the topmost stack element, we just modify that element.
188 */
189 #if defined(lint)
190
191 /*ARGSUSED*/
192 int
193 on_trap(on_trap_data_t *otp, uint_t prot)
194 { return (0); }
195
196 #else /* __lint */
197
198 #if defined(__amd64)
199
200 ENTRY(on_trap)
201 movw %si, OT_PROT(%rdi) /* ot_prot = prot */
202 movw $0, OT_TRAP(%rdi) /* ot_trap = 0 */
203 leaq on_trap_trampoline(%rip), %rdx /* rdx = &on_trap_trampoline */
204 movq %rdx, OT_TRAMPOLINE(%rdi) /* ot_trampoline = rdx */
205 xorl %ecx, %ecx
206 movq %rcx, OT_HANDLE(%rdi) /* ot_handle = NULL */
207 movq %rcx, OT_PAD1(%rdi) /* ot_pad1 = NULL */
208 movq %gs:CPU_THREAD, %rdx /* rdx = curthread */
209 movq T_ONTRAP(%rdx), %rcx /* rcx = curthread->t_ontrap */
210 cmpq %rdi, %rcx /* if (otp == %rcx) */
211 je 0f /* don't modify t_ontrap */
212
213 movq %rcx, OT_PREV(%rdi) /* ot_prev = t_ontrap */
214 movq %rdi, T_ONTRAP(%rdx) /* curthread->t_ontrap = otp */
215
216 0: addq $OT_JMPBUF, %rdi /* &ot_jmpbuf */
217 jmp setjmp
218 SET_SIZE(on_trap)
219
220 #elif defined(__i386)
221
222 ENTRY(on_trap)
223 movl 4(%esp), %eax /* %eax = otp */
224 movl 8(%esp), %edx /* %edx = prot */
225
226 movw %dx, OT_PROT(%eax) /* ot_prot = prot */
227 movw $0, OT_TRAP(%eax) /* ot_trap = 0 */
228 leal on_trap_trampoline, %edx /* %edx = &on_trap_trampoline */
229 movl %edx, OT_TRAMPOLINE(%eax) /* ot_trampoline = %edx */
230 movl $0, OT_HANDLE(%eax) /* ot_handle = NULL */
231 movl $0, OT_PAD1(%eax) /* ot_pad1 = NULL */
232 movl %gs:CPU_THREAD, %edx /* %edx = curthread */
233 movl T_ONTRAP(%edx), %ecx /* %ecx = curthread->t_ontrap */
234 cmpl %eax, %ecx /* if (otp == %ecx) */
235 je 0f /* don't modify t_ontrap */
236
237 movl %ecx, OT_PREV(%eax) /* ot_prev = t_ontrap */
238 movl %eax, T_ONTRAP(%edx) /* curthread->t_ontrap = otp */
239
240 0: addl $OT_JMPBUF, %eax /* %eax = &ot_jmpbuf */
241 movl %eax, 4(%esp) /* put %eax back on the stack */
242 jmp setjmp /* let setjmp do the rest */
243 SET_SIZE(on_trap)
244
245 #endif /* __i386 */
246 #endif /* __lint */
247
248 /*
249 * Setjmp and longjmp implement non-local gotos using state vectors
250 * type label_t.
251 */
252
253 #if defined(__lint)
254
255 /* ARGSUSED */
256 int
257 setjmp(label_t *lp)
258 { return (0); }
259
260 /* ARGSUSED */
261 void
262 longjmp(label_t *lp)
263 {}
264
265 #else /* __lint */
266
267 #if LABEL_PC != 0
268 #error LABEL_PC MUST be defined as 0 for setjmp/longjmp to work as coded
269 #endif /* LABEL_PC != 0 */
270
271 #if defined(__amd64)
272
273 ENTRY(setjmp)
274 movq %rsp, LABEL_SP(%rdi)
275 movq %rbp, LABEL_RBP(%rdi)
276 movq %rbx, LABEL_RBX(%rdi)
277 movq %r12, LABEL_R12(%rdi)
278 movq %r13, LABEL_R13(%rdi)
279 movq %r14, LABEL_R14(%rdi)
280 movq %r15, LABEL_R15(%rdi)
281 movq (%rsp), %rdx /* return address */
282 movq %rdx, (%rdi) /* LABEL_PC is 0 */
283 xorl %eax, %eax /* return 0 */
284 ret
285 SET_SIZE(setjmp)
286
287 ENTRY(longjmp)
288 movq LABEL_SP(%rdi), %rsp
289 movq LABEL_RBP(%rdi), %rbp
290 movq LABEL_RBX(%rdi), %rbx
291 movq LABEL_R12(%rdi), %r12
292 movq LABEL_R13(%rdi), %r13
293 movq LABEL_R14(%rdi), %r14
294 movq LABEL_R15(%rdi), %r15
295 movq (%rdi), %rdx /* return address; LABEL_PC is 0 */
296 movq %rdx, (%rsp)
297 xorl %eax, %eax
298 incl %eax /* return 1 */
299 ret
300 SET_SIZE(longjmp)
301
302 #elif defined(__i386)
303
304 ENTRY(setjmp)
305 movl 4(%esp), %edx /* address of save area */
306 movl %ebp, LABEL_EBP(%edx)
307 movl %ebx, LABEL_EBX(%edx)
308 movl %esi, LABEL_ESI(%edx)
309 movl %edi, LABEL_EDI(%edx)
310 movl %esp, 4(%edx)
311 movl (%esp), %ecx /* %eip (return address) */
312 movl %ecx, (%edx) /* LABEL_PC is 0 */
313 subl %eax, %eax /* return 0 */
314 ret
315 SET_SIZE(setjmp)
316
317 ENTRY(longjmp)
318 movl 4(%esp), %edx /* address of save area */
319 movl LABEL_EBP(%edx), %ebp
320 movl LABEL_EBX(%edx), %ebx
321 movl LABEL_ESI(%edx), %esi
322 movl LABEL_EDI(%edx), %edi
323 movl 4(%edx), %esp
324 movl (%edx), %ecx /* %eip (return addr); LABEL_PC is 0 */
325 movl $1, %eax
326 addl $4, %esp /* pop ret adr */
327 jmp *%ecx /* indirect */
328 SET_SIZE(longjmp)
329
330 #endif /* __i386 */
331 #endif /* __lint */
332
333 /*
334 * if a() calls b() calls caller(),
335 * caller() returns return address in a().
336 * (Note: We assume a() and b() are C routines which do the normal entry/exit
337 * sequence.)
338 */
339
340 #if defined(__lint)
341
342 caddr_t
343 caller(void)
344 { return (0); }
345
346 #else /* __lint */
347
348 #if defined(__amd64)
349
350 ENTRY(caller)
351 movq 8(%rbp), %rax /* b()'s return pc, in a() */
352 ret
353 SET_SIZE(caller)
354
355 #elif defined(__i386)
356
357 ENTRY(caller)
358 movl 4(%ebp), %eax /* b()'s return pc, in a() */
359 ret
360 SET_SIZE(caller)
361
362 #endif /* __i386 */
363 #endif /* __lint */
364
365 /*
366 * if a() calls callee(), callee() returns the
367 * return address in a();
368 */
369
370 #if defined(__lint)
371
372 caddr_t
373 callee(void)
374 { return (0); }
375
376 #else /* __lint */
377
378 #if defined(__amd64)
379
380 ENTRY(callee)
381 movq (%rsp), %rax /* callee()'s return pc, in a() */
382 ret
383 SET_SIZE(callee)
384
385 #elif defined(__i386)
386
387 ENTRY(callee)
388 movl (%esp), %eax /* callee()'s return pc, in a() */
389 ret
390 SET_SIZE(callee)
391
392 #endif /* __i386 */
393 #endif /* __lint */
394
395 /*
396 * return the current frame pointer
397 */
398
399 #if defined(__lint)
400
401 greg_t
402 getfp(void)
403 { return (0); }
404
405 #else /* __lint */
406
407 #if defined(__amd64)
408
409 ENTRY(getfp)
410 movq %rbp, %rax
411 ret
412 SET_SIZE(getfp)
413
414 #elif defined(__i386)
415
416 ENTRY(getfp)
417 movl %ebp, %eax
418 ret
419 SET_SIZE(getfp)
420
421 #endif /* __i386 */
422 #endif /* __lint */
423
424 /*
425 * Invalidate a single page table entry in the TLB
426 */
427
428 #if defined(__lint)
429
430 /* ARGSUSED */
431 void
432 mmu_tlbflush_entry(caddr_t m)
433 {}
434
435 #else /* __lint */
436
437 #if defined(__amd64)
438
439 ENTRY(mmu_tlbflush_entry)
440 invlpg (%rdi)
441 ret
442 SET_SIZE(mmu_tlbflush_entry)
443
444 #elif defined(__i386)
445
446 ENTRY(mmu_tlbflush_entry)
447 movl 4(%esp), %eax
448 invlpg (%eax)
449 ret
450 SET_SIZE(mmu_tlbflush_entry)
451
452 #endif /* __i386 */
453 #endif /* __lint */
454
455
456 /*
457 * Get/Set the value of various control registers
458 */
459
460 #if defined(__lint)
461
462 ulong_t
463 getcr0(void)
464 { return (0); }
465
466 /* ARGSUSED */
467 void
468 setcr0(ulong_t value)
469 {}
470
471 ulong_t
472 getcr2(void)
473 { return (0); }
474
475 ulong_t
476 getcr3(void)
477 { return (0); }
478
479 #if !defined(__xpv)
480 /* ARGSUSED */
481 void
482 setcr3(ulong_t val)
483 {}
484
485 void
486 reload_cr3(void)
487 {}
488 #endif
489
490 ulong_t
491 getcr4(void)
492 { return (0); }
493
494 /* ARGSUSED */
495 void
496 setcr4(ulong_t val)
497 {}
498
499 #if defined(__amd64)
500
501 ulong_t
502 getcr8(void)
503 { return (0); }
504
505 /* ARGSUSED */
506 void
507 setcr8(ulong_t val)
508 {}
509
510 #endif /* __amd64 */
511
512 #else /* __lint */
513
514 #if defined(__amd64)
515
516 ENTRY(getcr0)
517 movq %cr0, %rax
518 ret
519 SET_SIZE(getcr0)
520
521 ENTRY(setcr0)
522 movq %rdi, %cr0
523 ret
524 SET_SIZE(setcr0)
525
526 ENTRY(getcr2)
527 #if defined(__xpv)
528 movq %gs:CPU_VCPU_INFO, %rax
529 movq VCPU_INFO_ARCH_CR2(%rax), %rax
530 #else
531 movq %cr2, %rax
532 #endif
533 ret
534 SET_SIZE(getcr2)
535
536 ENTRY(getcr3)
537 movq %cr3, %rax
538 ret
539 SET_SIZE(getcr3)
540
541 #if !defined(__xpv)
542
543 ENTRY(setcr3)
544 movq %rdi, %cr3
545 ret
546 SET_SIZE(setcr3)
547
548 ENTRY(reload_cr3)
549 movq %cr3, %rdi
550 movq %rdi, %cr3
551 ret
552 SET_SIZE(reload_cr3)
553
554 #endif /* __xpv */
555
556 ENTRY(getcr4)
557 movq %cr4, %rax
558 ret
559 SET_SIZE(getcr4)
560
561 ENTRY(setcr4)
562 movq %rdi, %cr4
563 ret
564 SET_SIZE(setcr4)
565
566 ENTRY(getcr8)
567 movq %cr8, %rax
568 ret
569 SET_SIZE(getcr8)
570
571 ENTRY(setcr8)
572 movq %rdi, %cr8
573 ret
574 SET_SIZE(setcr8)
575
576 #elif defined(__i386)
577
578 ENTRY(getcr0)
579 movl %cr0, %eax
580 ret
581 SET_SIZE(getcr0)
582
583 ENTRY(setcr0)
584 movl 4(%esp), %eax
585 movl %eax, %cr0
586 ret
587 SET_SIZE(setcr0)
588
589 /*
590 * "lock mov %cr0" is used on processors which indicate it is
591 * supported via CPUID. Normally the 32 bit TPR is accessed via
592 * the local APIC.
593 */
594 ENTRY(getcr8)
595 lock
596 movl %cr0, %eax
597 ret
598 SET_SIZE(getcr8)
599
600 ENTRY(setcr8)
601 movl 4(%esp), %eax
602 lock
603 movl %eax, %cr0
604 ret
605 SET_SIZE(setcr8)
606
607 ENTRY(getcr2)
608 #if defined(__xpv)
609 movl %gs:CPU_VCPU_INFO, %eax
610 movl VCPU_INFO_ARCH_CR2(%eax), %eax
611 #else
612 movl %cr2, %eax
613 #endif
614 ret
615 SET_SIZE(getcr2)
616
617 ENTRY(getcr3)
618 movl %cr3, %eax
619 ret
620 SET_SIZE(getcr3)
621
622 #if !defined(__xpv)
623
624 ENTRY(setcr3)
625 movl 4(%esp), %eax
626 movl %eax, %cr3
627 ret
628 SET_SIZE(setcr3)
629
630 ENTRY(reload_cr3)
631 movl %cr3, %eax
632 movl %eax, %cr3
633 ret
634 SET_SIZE(reload_cr3)
635
636 #endif /* __xpv */
637
638 ENTRY(getcr4)
639 movl %cr4, %eax
640 ret
641 SET_SIZE(getcr4)
642
643 ENTRY(setcr4)
644 movl 4(%esp), %eax
645 movl %eax, %cr4
646 ret
647 SET_SIZE(setcr4)
648
649 #endif /* __i386 */
650 #endif /* __lint */
651
652 #if defined(__lint)
653
654 /*ARGSUSED*/
655 uint32_t
656 __cpuid_insn(struct cpuid_regs *regs)
657 { return (0); }
658
659 #else /* __lint */
660
661 #if defined(__amd64)
662
663 ENTRY(__cpuid_insn)
664 movq %rbx, %r8
665 movq %rcx, %r9
666 movq %rdx, %r11
667 movl (%rdi), %eax /* %eax = regs->cp_eax */
668 movl 0x4(%rdi), %ebx /* %ebx = regs->cp_ebx */
669 movl 0x8(%rdi), %ecx /* %ecx = regs->cp_ecx */
670 movl 0xc(%rdi), %edx /* %edx = regs->cp_edx */
671 cpuid
672 movl %eax, (%rdi) /* regs->cp_eax = %eax */
673 movl %ebx, 0x4(%rdi) /* regs->cp_ebx = %ebx */
674 movl %ecx, 0x8(%rdi) /* regs->cp_ecx = %ecx */
675 movl %edx, 0xc(%rdi) /* regs->cp_edx = %edx */
676 movq %r8, %rbx
677 movq %r9, %rcx
678 movq %r11, %rdx
679 ret
680 SET_SIZE(__cpuid_insn)
681
682 #elif defined(__i386)
683
684 ENTRY(__cpuid_insn)
685 pushl %ebp
686 movl 0x8(%esp), %ebp /* %ebp = regs */
687 pushl %ebx
688 pushl %ecx
689 pushl %edx
690 movl (%ebp), %eax /* %eax = regs->cp_eax */
691 movl 0x4(%ebp), %ebx /* %ebx = regs->cp_ebx */
692 movl 0x8(%ebp), %ecx /* %ecx = regs->cp_ecx */
693 movl 0xc(%ebp), %edx /* %edx = regs->cp_edx */
694 cpuid
695 movl %eax, (%ebp) /* regs->cp_eax = %eax */
696 movl %ebx, 0x4(%ebp) /* regs->cp_ebx = %ebx */
697 movl %ecx, 0x8(%ebp) /* regs->cp_ecx = %ecx */
698 movl %edx, 0xc(%ebp) /* regs->cp_edx = %edx */
699 popl %edx
700 popl %ecx
701 popl %ebx
702 popl %ebp
703 ret
704 SET_SIZE(__cpuid_insn)
705
706 #endif /* __i386 */
707 #endif /* __lint */
708
709 #if defined(__lint)
710
711 /*ARGSUSED*/
712 void
713 i86_monitor(volatile uint32_t *addr, uint32_t extensions, uint32_t hints)
714 {}
715
716 #else /* __lint */
717
718 #if defined(__amd64)
719
720 ENTRY_NP(i86_monitor)
721 pushq %rbp
722 movq %rsp, %rbp
723 movq %rdi, %rax /* addr */
724 movq %rsi, %rcx /* extensions */
725 /* rdx contains input arg3: hints */
726 clflush (%rax)
727 .byte 0x0f, 0x01, 0xc8 /* monitor */
728 leave
729 ret
730 SET_SIZE(i86_monitor)
731
732 #elif defined(__i386)
733
734 ENTRY_NP(i86_monitor)
735 pushl %ebp
736 movl %esp, %ebp
737 movl 0x8(%ebp),%eax /* addr */
738 movl 0xc(%ebp),%ecx /* extensions */
739 movl 0x10(%ebp),%edx /* hints */
740 clflush (%eax)
741 .byte 0x0f, 0x01, 0xc8 /* monitor */
742 leave
743 ret
744 SET_SIZE(i86_monitor)
745
746 #endif /* __i386 */
747 #endif /* __lint */
748
749 #if defined(__lint)
750
751 /*ARGSUSED*/
752 void
753 i86_mwait(uint32_t data, uint32_t extensions)
754 {}
755
756 #else /* __lint */
757
758 #if defined(__amd64)
759
760 ENTRY_NP(i86_mwait)
761 pushq %rbp
762 movq %rsp, %rbp
763 movq %rdi, %rax /* data */
764 movq %rsi, %rcx /* extensions */
765 .byte 0x0f, 0x01, 0xc9 /* mwait */
766 leave
767 ret
768 SET_SIZE(i86_mwait)
769
770 #elif defined(__i386)
771
772 ENTRY_NP(i86_mwait)
773 pushl %ebp
774 movl %esp, %ebp
775 movl 0x8(%ebp),%eax /* data */
776 movl 0xc(%ebp),%ecx /* extensions */
777 .byte 0x0f, 0x01, 0xc9 /* mwait */
778 leave
779 ret
780 SET_SIZE(i86_mwait)
781
782 #endif /* __i386 */
783 #endif /* __lint */
784
785 #if defined(__xpv)
786 /*
787 * Defined in C
788 */
789 #else
790
791 #if defined(__lint)
792
793 hrtime_t
794 tsc_read(void)
795 {
796 return (0);
797 }
798
799 #else /* __lint */
800
801 #if defined(__amd64)
802
803 ENTRY_NP(tsc_read)
804 movq %rbx, %r11
805 movl $0, %eax
806 cpuid
807 rdtsc
808 movq %r11, %rbx
809 shlq $32, %rdx
810 orq %rdx, %rax
811 ret
812 .globl _tsc_mfence_start
813 _tsc_mfence_start:
814 mfence
815 rdtsc
816 shlq $32, %rdx
817 orq %rdx, %rax
818 ret
819 .globl _tsc_mfence_end
820 _tsc_mfence_end:
821 .globl _tscp_start
822 _tscp_start:
823 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */
824 shlq $32, %rdx
825 orq %rdx, %rax
826 ret
827 .globl _tscp_end
828 _tscp_end:
829 .globl _no_rdtsc_start
830 _no_rdtsc_start:
831 xorl %edx, %edx
832 xorl %eax, %eax
833 ret
834 .globl _no_rdtsc_end
835 _no_rdtsc_end:
836 .globl _tsc_lfence_start
837 _tsc_lfence_start:
838 lfence
839 rdtsc
840 shlq $32, %rdx
841 orq %rdx, %rax
842 ret
843 .globl _tsc_lfence_end
844 _tsc_lfence_end:
845 SET_SIZE(tsc_read)
846
847 #else /* __i386 */
848
849 ENTRY_NP(tsc_read)
850 pushl %ebx
851 movl $0, %eax
852 cpuid
853 rdtsc
854 popl %ebx
855 ret
856 .globl _tsc_mfence_start
857 _tsc_mfence_start:
858 mfence
859 rdtsc
860 ret
861 .globl _tsc_mfence_end
862 _tsc_mfence_end:
863 .globl _tscp_start
864 _tscp_start:
865 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */
866 ret
867 .globl _tscp_end
868 _tscp_end:
869 .globl _no_rdtsc_start
870 _no_rdtsc_start:
871 xorl %edx, %edx
872 xorl %eax, %eax
873 ret
874 .globl _no_rdtsc_end
875 _no_rdtsc_end:
876 .globl _tsc_lfence_start
877 _tsc_lfence_start:
878 lfence
879 rdtsc
880 ret
881 .globl _tsc_lfence_end
882 _tsc_lfence_end:
883 SET_SIZE(tsc_read)
884
885 #endif /* __i386 */
886
887 #endif /* __lint */
888
889
890 #endif /* __xpv */
891
892 #ifdef __lint
893 /*
894 * Do not use this function for obtaining clock tick. This
895 * is called by callers who do not need to have a guarenteed
896 * correct tick value. The proper routine to use is tsc_read().
897 */
898 u_longlong_t
899 randtick(void)
900 {
901 return (0);
902 }
903 #else
904 #if defined(__amd64)
905 ENTRY_NP(randtick)
906 rdtsc
907 shlq $32, %rdx
908 orq %rdx, %rax
909 ret
910 SET_SIZE(randtick)
911 #else
912 ENTRY_NP(randtick)
913 rdtsc
914 ret
915 SET_SIZE(randtick)
916 #endif /* __i386 */
917 #endif /* __lint */
918 /*
919 * Insert entryp after predp in a doubly linked list.
920 */
921
922 #if defined(__lint)
923
924 /*ARGSUSED*/
925 void
926 _insque(caddr_t entryp, caddr_t predp)
927 {}
928
929 #else /* __lint */
930
931 #if defined(__amd64)
932
933 ENTRY(_insque)
934 movq (%rsi), %rax /* predp->forw */
935 movq %rsi, CPTRSIZE(%rdi) /* entryp->back = predp */
936 movq %rax, (%rdi) /* entryp->forw = predp->forw */
937 movq %rdi, (%rsi) /* predp->forw = entryp */
938 movq %rdi, CPTRSIZE(%rax) /* predp->forw->back = entryp */
939 ret
940 SET_SIZE(_insque)
941
942 #elif defined(__i386)
943
944 ENTRY(_insque)
945 movl 8(%esp), %edx
946 movl 4(%esp), %ecx
947 movl (%edx), %eax /* predp->forw */
948 movl %edx, CPTRSIZE(%ecx) /* entryp->back = predp */
949 movl %eax, (%ecx) /* entryp->forw = predp->forw */
950 movl %ecx, (%edx) /* predp->forw = entryp */
951 movl %ecx, CPTRSIZE(%eax) /* predp->forw->back = entryp */
952 ret
953 SET_SIZE(_insque)
954
955 #endif /* __i386 */
956 #endif /* __lint */
957
958 /*
959 * Remove entryp from a doubly linked list
960 */
961
962 #if defined(__lint)
963
964 /*ARGSUSED*/
965 void
966 _remque(caddr_t entryp)
967 {}
968
969 #else /* __lint */
970
971 #if defined(__amd64)
972
973 ENTRY(_remque)
974 movq (%rdi), %rax /* entry->forw */
975 movq CPTRSIZE(%rdi), %rdx /* entry->back */
976 movq %rax, (%rdx) /* entry->back->forw = entry->forw */
977 movq %rdx, CPTRSIZE(%rax) /* entry->forw->back = entry->back */
978 ret
979 SET_SIZE(_remque)
980
981 #elif defined(__i386)
982
983 ENTRY(_remque)
984 movl 4(%esp), %ecx
985 movl (%ecx), %eax /* entry->forw */
986 movl CPTRSIZE(%ecx), %edx /* entry->back */
987 movl %eax, (%edx) /* entry->back->forw = entry->forw */
988 movl %edx, CPTRSIZE(%eax) /* entry->forw->back = entry->back */
989 ret
990 SET_SIZE(_remque)
991
992 #endif /* __i386 */
993 #endif /* __lint */
994
995 /*
996 * Returns the number of
997 * non-NULL bytes in string argument.
998 */
999
1000 #if defined(__lint)
1001
1002 /* ARGSUSED */
1003 size_t
1004 strlen(const char *str)
1005 { return (0); }
1006
1007 #else /* __lint */
1008
1009 #if defined(__amd64)
1010
1011 /*
1012 * This is close to a simple transliteration of a C version of this
1013 * routine. We should either just -make- this be a C version, or
1014 * justify having it in assembler by making it significantly faster.
1015 *
1016 * size_t
1017 * strlen(const char *s)
1018 * {
1019 * const char *s0;
1020 * #if defined(DEBUG)
1021 * if ((uintptr_t)s < KERNELBASE)
1022 * panic(.str_panic_msg);
1023 * #endif
1024 * for (s0 = s; *s; s++)
1025 * ;
1026 * return (s - s0);
1027 * }
1028 */
1029
1030 ENTRY(strlen)
1031 #ifdef DEBUG
1032 movq postbootkernelbase(%rip), %rax
1033 cmpq %rax, %rdi
1034 jae str_valid
1035 pushq %rbp
1036 movq %rsp, %rbp
1037 leaq .str_panic_msg(%rip), %rdi
1038 xorl %eax, %eax
1039 call panic
1040 #endif /* DEBUG */
1041 str_valid:
1042 cmpb $0, (%rdi)
1043 movq %rdi, %rax
1044 je .null_found
1045 .align 4
1046 .strlen_loop:
1047 incq %rdi
1048 cmpb $0, (%rdi)
1049 jne .strlen_loop
1050 .null_found:
1051 subq %rax, %rdi
1052 movq %rdi, %rax
1053 ret
1054 SET_SIZE(strlen)
1055
1056 #elif defined(__i386)
1057
1058 ENTRY(strlen)
1059 #ifdef DEBUG
1060 movl postbootkernelbase, %eax
1061 cmpl %eax, 4(%esp)
1062 jae str_valid
1063 pushl %ebp
1064 movl %esp, %ebp
1065 pushl $.str_panic_msg
1066 call panic
1067 #endif /* DEBUG */
1068
1069 str_valid:
1070 movl 4(%esp), %eax /* %eax = string address */
1071 testl $3, %eax /* if %eax not word aligned */
1072 jnz .not_word_aligned /* goto .not_word_aligned */
1073 .align 4
1074 .word_aligned:
1075 movl (%eax), %edx /* move 1 word from (%eax) to %edx */
1076 movl $0x7f7f7f7f, %ecx
1077 andl %edx, %ecx /* %ecx = %edx & 0x7f7f7f7f */
1078 addl $4, %eax /* next word */
1079 addl $0x7f7f7f7f, %ecx /* %ecx += 0x7f7f7f7f */
1080 orl %edx, %ecx /* %ecx |= %edx */
1081 andl $0x80808080, %ecx /* %ecx &= 0x80808080 */
1082 cmpl $0x80808080, %ecx /* if no null byte in this word */
1083 je .word_aligned /* goto .word_aligned */
1084 subl $4, %eax /* post-incremented */
1085 .not_word_aligned:
1086 cmpb $0, (%eax) /* if a byte in (%eax) is null */
1087 je .null_found /* goto .null_found */
1088 incl %eax /* next byte */
1089 testl $3, %eax /* if %eax not word aligned */
1090 jnz .not_word_aligned /* goto .not_word_aligned */
1091 jmp .word_aligned /* goto .word_aligned */
1092 .align 4
1093 .null_found:
1094 subl 4(%esp), %eax /* %eax -= string address */
1095 ret
1096 SET_SIZE(strlen)
1097
1098 #endif /* __i386 */
1099
1100 #ifdef DEBUG
1101 .text
1102 .str_panic_msg:
1103 .string "strlen: argument below kernelbase"
1104 #endif /* DEBUG */
1105
1106 #endif /* __lint */
1107
1108 /*
1109 * Berkeley 4.3 introduced symbolically named interrupt levels
1110 * as a way deal with priority in a machine independent fashion.
1111 * Numbered priorities are machine specific, and should be
1112 * discouraged where possible.
1113 *
1114 * Note, for the machine specific priorities there are
1115 * examples listed for devices that use a particular priority.
1116 * It should not be construed that all devices of that
1117 * type should be at that priority. It is currently were
1118 * the current devices fit into the priority scheme based
1119 * upon time criticalness.
1120 *
1121 * The underlying assumption of these assignments is that
1122 * IPL 10 is the highest level from which a device
1123 * routine can call wakeup. Devices that interrupt from higher
1124 * levels are restricted in what they can do. If they need
1125 * kernels services they should schedule a routine at a lower
1126 * level (via software interrupt) to do the required
1127 * processing.
1128 *
1129 * Examples of this higher usage:
1130 * Level Usage
1131 * 14 Profiling clock (and PROM uart polling clock)
1132 * 12 Serial ports
1133 *
1134 * The serial ports request lower level processing on level 6.
1135 *
1136 * Also, almost all splN routines (where N is a number or a
1137 * mnemonic) will do a RAISE(), on the assumption that they are
1138 * never used to lower our priority.
1139 * The exceptions are:
1140 * spl8() Because you can't be above 15 to begin with!
1141 * splzs() Because this is used at boot time to lower our
1142 * priority, to allow the PROM to poll the uart.
1143 * spl0() Used to lower priority to 0.
1144 */
1145
1146 #if defined(__lint)
1147
1148 int spl0(void) { return (0); }
1149 int spl6(void) { return (0); }
1150 int spl7(void) { return (0); }
1151 int spl8(void) { return (0); }
1152 int splhigh(void) { return (0); }
1153 int splhi(void) { return (0); }
1154 int splzs(void) { return (0); }
1155
1156 /* ARGSUSED */
1157 void
1158 splx(int level)
1159 {}
1160
1161 #else /* __lint */
1162
1163 #if defined(__amd64)
1164
1165 #define SETPRI(level) \
1166 movl $/**/level, %edi; /* new priority */ \
1167 jmp do_splx /* redirect to do_splx */
1168
1169 #define RAISE(level) \
1170 movl $/**/level, %edi; /* new priority */ \
1171 jmp splr /* redirect to splr */
1172
1173 #elif defined(__i386)
1174
1175 #define SETPRI(level) \
1176 pushl $/**/level; /* new priority */ \
1177 call do_splx; /* invoke common splx code */ \
1178 addl $4, %esp; /* unstack arg */ \
1179 ret
1180
1181 #define RAISE(level) \
1182 pushl $/**/level; /* new priority */ \
1183 call splr; /* invoke common splr code */ \
1184 addl $4, %esp; /* unstack args */ \
1185 ret
1186
1187 #endif /* __i386 */
1188
1189 /* locks out all interrupts, including memory errors */
1190 ENTRY(spl8)
1191 SETPRI(15)
1192 SET_SIZE(spl8)
1193
1194 /* just below the level that profiling runs */
1195 ENTRY(spl7)
1196 RAISE(13)
1197 SET_SIZE(spl7)
1198
1199 /* sun specific - highest priority onboard serial i/o asy ports */
1200 ENTRY(splzs)
1201 SETPRI(12) /* Can't be a RAISE, as it's used to lower us */
1202 SET_SIZE(splzs)
1203
1204 ENTRY(splhi)
1205 ALTENTRY(splhigh)
1206 ALTENTRY(spl6)
1207 ALTENTRY(i_ddi_splhigh)
1208
1209 RAISE(DISP_LEVEL)
1210
1211 SET_SIZE(i_ddi_splhigh)
1212 SET_SIZE(spl6)
1213 SET_SIZE(splhigh)
1214 SET_SIZE(splhi)
1215
1216 /* allow all interrupts */
1217 ENTRY(spl0)
1218 SETPRI(0)
1219 SET_SIZE(spl0)
1220
1221
1222 /* splx implementation */
1223 ENTRY(splx)
1224 jmp do_splx /* redirect to common splx code */
1225 SET_SIZE(splx)
1226
1227 #endif /* __lint */
1228
1229 #if defined(__i386)
1230
1231 /*
1232 * Read and write the %gs register
1233 */
1234
1235 #if defined(__lint)
1236
1237 /*ARGSUSED*/
1238 uint16_t
1239 getgs(void)
1240 { return (0); }
1241
1242 /*ARGSUSED*/
1243 void
1244 setgs(uint16_t sel)
1245 {}
1246
1247 #else /* __lint */
1248
1249 ENTRY(getgs)
1250 clr %eax
1251 movw %gs, %ax
1252 ret
1253 SET_SIZE(getgs)
1254
1255 ENTRY(setgs)
1256 movw 4(%esp), %gs
1257 ret
1258 SET_SIZE(setgs)
1259
1260 #endif /* __lint */
1261 #endif /* __i386 */
1262
1263 #if defined(__lint)
1264
1265 void
1266 pc_reset(void)
1267 {}
1268
1269 void
1270 efi_reset(void)
1271 {}
1272
1273 #else /* __lint */
1274
1275 ENTRY(wait_500ms)
1276 #if defined(__amd64)
1277 pushq %rbx
1278 #elif defined(__i386)
1279 push %ebx
1280 #endif
1281 movl $50000, %ebx
1282 1:
1283 call tenmicrosec
1284 decl %ebx
1285 jnz 1b
1286 #if defined(__amd64)
1287 popq %rbx
1288 #elif defined(__i386)
1289 pop %ebx
1290 #endif
1291 ret
1292 SET_SIZE(wait_500ms)
1293
1294 #define RESET_METHOD_KBC 1
1295 #define RESET_METHOD_PORT92 2
1296 #define RESET_METHOD_PCI 4
1297
1298 DGDEF3(pc_reset_methods, 4, 8)
1299 .long RESET_METHOD_KBC|RESET_METHOD_PORT92|RESET_METHOD_PCI;
1300
1301 ENTRY(pc_reset)
1302
1303 #if defined(__i386)
1304 testl $RESET_METHOD_KBC, pc_reset_methods
1305 #elif defined(__amd64)
1306 testl $RESET_METHOD_KBC, pc_reset_methods(%rip)
1307 #endif
1308 jz 1f
1309
1310 /
1311 / Try the classic keyboard controller-triggered reset.
1312 /
1313 movw $0x64, %dx
1314 movb $0xfe, %al
1315 outb (%dx)
1316
1317 / Wait up to 500 milliseconds here for the keyboard controller
1318 / to pull the reset line. On some systems where the keyboard
1319 / controller is slow to pull the reset line, the next reset method
1320 / may be executed (which may be bad if those systems hang when the
1321 / next reset method is used, e.g. Ferrari 3400 (doesn't like port 92),
1322 / and Ferrari 4000 (doesn't like the cf9 reset method))
1323
1324 call wait_500ms
1325
1326 1:
1327 #if defined(__i386)
1328 testl $RESET_METHOD_PORT92, pc_reset_methods
1329 #elif defined(__amd64)
1330 testl $RESET_METHOD_PORT92, pc_reset_methods(%rip)
1331 #endif
1332 jz 3f
1333
1334 /
1335 / Try port 0x92 fast reset
1336 /
1337 movw $0x92, %dx
1338 inb (%dx)
1339 cmpb $0xff, %al / If port's not there, we should get back 0xFF
1340 je 1f
1341 testb $1, %al / If bit 0
1342 jz 2f / is clear, jump to perform the reset
1343 andb $0xfe, %al / otherwise,
1344 outb (%dx) / clear bit 0 first, then
1345 2:
1346 orb $1, %al / Set bit 0
1347 outb (%dx) / and reset the system
1348 1:
1349
1350 call wait_500ms
1351
1352 3:
1353 #if defined(__i386)
1354 testl $RESET_METHOD_PCI, pc_reset_methods
1355 #elif defined(__amd64)
1356 testl $RESET_METHOD_PCI, pc_reset_methods(%rip)
1357 #endif
1358 jz 4f
1359
1360 / Try the PCI (soft) reset vector (should work on all modern systems,
1361 / but has been shown to cause problems on 450NX systems, and some newer
1362 / systems (e.g. ATI IXP400-equipped systems))
1363 / When resetting via this method, 2 writes are required. The first
1364 / targets bit 1 (0=hard reset without power cycle, 1=hard reset with
1365 / power cycle).
1366 / The reset occurs on the second write, during bit 2's transition from
1367 / 0->1.
1368 movw $0xcf9, %dx
1369 movb $0x2, %al / Reset mode = hard, no power cycle
1370 outb (%dx)
1371 movb $0x6, %al
1372 outb (%dx)
1373
1374 call wait_500ms
1375
1376 4:
1377 /
1378 / port 0xcf9 failed also. Last-ditch effort is to
1379 / triple-fault the CPU.
1380 / Also, use triple fault for EFI firmware
1381 /
1382 ENTRY(efi_reset)
1383 #if defined(__amd64)
1384 pushq $0x0
1385 pushq $0x0 / IDT base of 0, limit of 0 + 2 unused bytes
1386 lidt (%rsp)
1387 #elif defined(__i386)
1388 pushl $0x0
1389 pushl $0x0 / IDT base of 0, limit of 0 + 2 unused bytes
1390 lidt (%esp)
1391 #endif
1392 int $0x0 / Trigger interrupt, generate triple-fault
1393
1394 cli
1395 hlt / Wait forever
1396 /*NOTREACHED*/
1397 SET_SIZE(efi_reset)
1398 SET_SIZE(pc_reset)
1399
1400 #endif /* __lint */
1401
1402 /*
1403 * C callable in and out routines
1404 */
1405
1406 #if defined(__lint)
1407
1408 /* ARGSUSED */
1409 void
1410 outl(int port_address, uint32_t val)
1411 {}
1412
1413 #else /* __lint */
1414
1415 #if defined(__amd64)
1416
1417 ENTRY(outl)
1418 movw %di, %dx
1419 movl %esi, %eax
1420 outl (%dx)
1421 ret
1422 SET_SIZE(outl)
1423
1424 #elif defined(__i386)
1425
1426 .set PORT, 4
1427 .set VAL, 8
1428
1429 ENTRY(outl)
1430 movw PORT(%esp), %dx
1431 movl VAL(%esp), %eax
1432 outl (%dx)
1433 ret
1434 SET_SIZE(outl)
1435
1436 #endif /* __i386 */
1437 #endif /* __lint */
1438
1439 #if defined(__lint)
1440
1441 /* ARGSUSED */
1442 void
1443 outw(int port_address, uint16_t val)
1444 {}
1445
1446 #else /* __lint */
1447
1448 #if defined(__amd64)
1449
1450 ENTRY(outw)
1451 movw %di, %dx
1452 movw %si, %ax
1453 D16 outl (%dx) /* XX64 why not outw? */
1454 ret
1455 SET_SIZE(outw)
1456
1457 #elif defined(__i386)
1458
1459 ENTRY(outw)
1460 movw PORT(%esp), %dx
1461 movw VAL(%esp), %ax
1462 D16 outl (%dx)
1463 ret
1464 SET_SIZE(outw)
1465
1466 #endif /* __i386 */
1467 #endif /* __lint */
1468
1469 #if defined(__lint)
1470
1471 /* ARGSUSED */
1472 void
1473 outb(int port_address, uint8_t val)
1474 {}
1475
1476 #else /* __lint */
1477
1478 #if defined(__amd64)
1479
1480 ENTRY(outb)
1481 movw %di, %dx
1482 movb %sil, %al
1483 outb (%dx)
1484 ret
1485 SET_SIZE(outb)
1486
1487 #elif defined(__i386)
1488
1489 ENTRY(outb)
1490 movw PORT(%esp), %dx
1491 movb VAL(%esp), %al
1492 outb (%dx)
1493 ret
1494 SET_SIZE(outb)
1495
1496 #endif /* __i386 */
1497 #endif /* __lint */
1498
1499 #if defined(__lint)
1500
1501 /* ARGSUSED */
1502 uint32_t
1503 inl(int port_address)
1504 { return (0); }
1505
1506 #else /* __lint */
1507
1508 #if defined(__amd64)
1509
1510 ENTRY(inl)
1511 xorl %eax, %eax
1512 movw %di, %dx
1513 inl (%dx)
1514 ret
1515 SET_SIZE(inl)
1516
1517 #elif defined(__i386)
1518
1519 ENTRY(inl)
1520 movw PORT(%esp), %dx
1521 inl (%dx)
1522 ret
1523 SET_SIZE(inl)
1524
1525 #endif /* __i386 */
1526 #endif /* __lint */
1527
1528 #if defined(__lint)
1529
1530 /* ARGSUSED */
1531 uint16_t
1532 inw(int port_address)
1533 { return (0); }
1534
1535 #else /* __lint */
1536
1537 #if defined(__amd64)
1538
1539 ENTRY(inw)
1540 xorl %eax, %eax
1541 movw %di, %dx
1542 D16 inl (%dx)
1543 ret
1544 SET_SIZE(inw)
1545
1546 #elif defined(__i386)
1547
1548 ENTRY(inw)
1549 subl %eax, %eax
1550 movw PORT(%esp), %dx
1551 D16 inl (%dx)
1552 ret
1553 SET_SIZE(inw)
1554
1555 #endif /* __i386 */
1556 #endif /* __lint */
1557
1558
1559 #if defined(__lint)
1560
1561 /* ARGSUSED */
1562 uint8_t
1563 inb(int port_address)
1564 { return (0); }
1565
1566 #else /* __lint */
1567
1568 #if defined(__amd64)
1569
1570 ENTRY(inb)
1571 xorl %eax, %eax
1572 movw %di, %dx
1573 inb (%dx)
1574 ret
1575 SET_SIZE(inb)
1576
1577 #elif defined(__i386)
1578
1579 ENTRY(inb)
1580 subl %eax, %eax
1581 movw PORT(%esp), %dx
1582 inb (%dx)
1583 ret
1584 SET_SIZE(inb)
1585
1586 #endif /* __i386 */
1587 #endif /* __lint */
1588
1589
1590 #if defined(__lint)
1591
1592 /* ARGSUSED */
1593 void
1594 repoutsw(int port, uint16_t *addr, int cnt)
1595 {}
1596
1597 #else /* __lint */
1598
1599 #if defined(__amd64)
1600
1601 ENTRY(repoutsw)
1602 movl %edx, %ecx
1603 movw %di, %dx
1604 rep
1605 D16 outsl
1606 ret
1607 SET_SIZE(repoutsw)
1608
1609 #elif defined(__i386)
1610
1611 /*
1612 * The arguments and saved registers are on the stack in the
1613 * following order:
1614 * | cnt | +16
1615 * | *addr | +12
1616 * | port | +8
1617 * | eip | +4
1618 * | esi | <-- %esp
1619 * If additional values are pushed onto the stack, make sure
1620 * to adjust the following constants accordingly.
1621 */
1622 .set PORT, 8
1623 .set ADDR, 12
1624 .set COUNT, 16
1625
1626 ENTRY(repoutsw)
1627 pushl %esi
1628 movl PORT(%esp), %edx
1629 movl ADDR(%esp), %esi
1630 movl COUNT(%esp), %ecx
1631 rep
1632 D16 outsl
1633 popl %esi
1634 ret
1635 SET_SIZE(repoutsw)
1636
1637 #endif /* __i386 */
1638 #endif /* __lint */
1639
1640
1641 #if defined(__lint)
1642
1643 /* ARGSUSED */
1644 void
1645 repinsw(int port_addr, uint16_t *addr, int cnt)
1646 {}
1647
1648 #else /* __lint */
1649
1650 #if defined(__amd64)
1651
1652 ENTRY(repinsw)
1653 movl %edx, %ecx
1654 movw %di, %dx
1655 rep
1656 D16 insl
1657 ret
1658 SET_SIZE(repinsw)
1659
1660 #elif defined(__i386)
1661
1662 ENTRY(repinsw)
1663 pushl %edi
1664 movl PORT(%esp), %edx
1665 movl ADDR(%esp), %edi
1666 movl COUNT(%esp), %ecx
1667 rep
1668 D16 insl
1669 popl %edi
1670 ret
1671 SET_SIZE(repinsw)
1672
1673 #endif /* __i386 */
1674 #endif /* __lint */
1675
1676
1677 #if defined(__lint)
1678
1679 /* ARGSUSED */
1680 void
1681 repinsb(int port, uint8_t *addr, int count)
1682 {}
1683
1684 #else /* __lint */
1685
1686 #if defined(__amd64)
1687
1688 ENTRY(repinsb)
1689 movl %edx, %ecx
1690 movw %di, %dx
1691 movq %rsi, %rdi
1692 rep
1693 insb
1694 ret
1695 SET_SIZE(repinsb)
1696
1697 #elif defined(__i386)
1698
1699 /*
1700 * The arguments and saved registers are on the stack in the
1701 * following order:
1702 * | cnt | +16
1703 * | *addr | +12
1704 * | port | +8
1705 * | eip | +4
1706 * | esi | <-- %esp
1707 * If additional values are pushed onto the stack, make sure
1708 * to adjust the following constants accordingly.
1709 */
1710 .set IO_PORT, 8
1711 .set IO_ADDR, 12
1712 .set IO_COUNT, 16
1713
1714 ENTRY(repinsb)
1715 pushl %edi
1716 movl IO_ADDR(%esp), %edi
1717 movl IO_COUNT(%esp), %ecx
1718 movl IO_PORT(%esp), %edx
1719 rep
1720 insb
1721 popl %edi
1722 ret
1723 SET_SIZE(repinsb)
1724
1725 #endif /* __i386 */
1726 #endif /* __lint */
1727
1728
1729 /*
1730 * Input a stream of 32-bit words.
1731 * NOTE: count is a DWORD count.
1732 */
1733 #if defined(__lint)
1734
1735 /* ARGSUSED */
1736 void
1737 repinsd(int port, uint32_t *addr, int count)
1738 {}
1739
1740 #else /* __lint */
1741
1742 #if defined(__amd64)
1743
1744 ENTRY(repinsd)
1745 movl %edx, %ecx
1746 movw %di, %dx
1747 movq %rsi, %rdi
1748 rep
1749 insl
1750 ret
1751 SET_SIZE(repinsd)
1752
1753 #elif defined(__i386)
1754
1755 ENTRY(repinsd)
1756 pushl %edi
1757 movl IO_ADDR(%esp), %edi
1758 movl IO_COUNT(%esp), %ecx
1759 movl IO_PORT(%esp), %edx
1760 rep
1761 insl
1762 popl %edi
1763 ret
1764 SET_SIZE(repinsd)
1765
1766 #endif /* __i386 */
1767 #endif /* __lint */
1768
1769 /*
1770 * Output a stream of bytes
1771 * NOTE: count is a byte count
1772 */
1773 #if defined(__lint)
1774
1775 /* ARGSUSED */
1776 void
1777 repoutsb(int port, uint8_t *addr, int count)
1778 {}
1779
1780 #else /* __lint */
1781
1782 #if defined(__amd64)
1783
1784 ENTRY(repoutsb)
1785 movl %edx, %ecx
1786 movw %di, %dx
1787 rep
1788 outsb
1789 ret
1790 SET_SIZE(repoutsb)
1791
1792 #elif defined(__i386)
1793
1794 ENTRY(repoutsb)
1795 pushl %esi
1796 movl IO_ADDR(%esp), %esi
1797 movl IO_COUNT(%esp), %ecx
1798 movl IO_PORT(%esp), %edx
1799 rep
1800 outsb
1801 popl %esi
1802 ret
1803 SET_SIZE(repoutsb)
1804
1805 #endif /* __i386 */
1806 #endif /* __lint */
1807
1808 /*
1809 * Output a stream of 32-bit words
1810 * NOTE: count is a DWORD count
1811 */
1812 #if defined(__lint)
1813
1814 /* ARGSUSED */
1815 void
1816 repoutsd(int port, uint32_t *addr, int count)
1817 {}
1818
1819 #else /* __lint */
1820
1821 #if defined(__amd64)
1822
1823 ENTRY(repoutsd)
1824 movl %edx, %ecx
1825 movw %di, %dx
1826 rep
1827 outsl
1828 ret
1829 SET_SIZE(repoutsd)
1830
1831 #elif defined(__i386)
1832
1833 ENTRY(repoutsd)
1834 pushl %esi
1835 movl IO_ADDR(%esp), %esi
1836 movl IO_COUNT(%esp), %ecx
1837 movl IO_PORT(%esp), %edx
1838 rep
1839 outsl
1840 popl %esi
1841 ret
1842 SET_SIZE(repoutsd)
1843
1844 #endif /* __i386 */
1845 #endif /* __lint */
1846
1847 /*
1848 * void int3(void)
1849 * void int18(void)
1850 * void int20(void)
1851 * void int_cmci(void)
1852 */
1853
1854 #if defined(__lint)
1855
1856 void
1857 int3(void)
1858 {}
1859
1860 void
1861 int18(void)
1862 {}
1863
1864 void
1865 int20(void)
1866 {}
1867
1868 void
1869 int_cmci(void)
1870 {}
1871
1872 #else /* __lint */
1873
1874 ENTRY(int3)
1875 int $T_BPTFLT
1876 ret
1877 SET_SIZE(int3)
1878
1879 ENTRY(int18)
1880 int $T_MCE
1881 ret
1882 SET_SIZE(int18)
1883
1884 ENTRY(int20)
1885 movl boothowto, %eax
1886 andl $RB_DEBUG, %eax
1887 jz 1f
1888
1889 int $T_DBGENTR
1890 1:
1891 rep; ret /* use 2 byte return instruction when branch target */
1892 /* AMD Software Optimization Guide - Section 6.2 */
1893 SET_SIZE(int20)
1894
1895 ENTRY(int_cmci)
1896 int $T_ENOEXTFLT
1897 ret
1898 SET_SIZE(int_cmci)
1899
1900 #endif /* __lint */
1901
1902 #if defined(__lint)
1903
1904 /* ARGSUSED */
1905 int
1906 scanc(size_t size, uchar_t *cp, uchar_t *table, uchar_t mask)
1907 { return (0); }
1908
1909 #else /* __lint */
1910
1911 #if defined(__amd64)
1912
1913 ENTRY(scanc)
1914 /* rdi == size */
1915 /* rsi == cp */
1916 /* rdx == table */
1917 /* rcx == mask */
1918 addq %rsi, %rdi /* end = &cp[size] */
1919 .scanloop:
1920 cmpq %rdi, %rsi /* while (cp < end */
1921 jnb .scandone
1922 movzbq (%rsi), %r8 /* %r8 = *cp */
1923 incq %rsi /* cp++ */
1924 testb %cl, (%r8, %rdx)
1925 jz .scanloop /* && (table[*cp] & mask) == 0) */
1926 decq %rsi /* (fix post-increment) */
1927 .scandone:
1928 movl %edi, %eax
1929 subl %esi, %eax /* return (end - cp) */
1930 ret
1931 SET_SIZE(scanc)
1932
1933 #elif defined(__i386)
1934
1935 ENTRY(scanc)
1936 pushl %edi
1937 pushl %esi
1938 movb 24(%esp), %cl /* mask = %cl */
1939 movl 16(%esp), %esi /* cp = %esi */
1940 movl 20(%esp), %edx /* table = %edx */
1941 movl %esi, %edi
1942 addl 12(%esp), %edi /* end = &cp[size]; */
1943 .scanloop:
1944 cmpl %edi, %esi /* while (cp < end */
1945 jnb .scandone
1946 movzbl (%esi), %eax /* %al = *cp */
1947 incl %esi /* cp++ */
1948 movb (%edx, %eax), %al /* %al = table[*cp] */
1949 testb %al, %cl
1950 jz .scanloop /* && (table[*cp] & mask) == 0) */
1951 dec %esi /* post-incremented */
1952 .scandone:
1953 movl %edi, %eax
1954 subl %esi, %eax /* return (end - cp) */
1955 popl %esi
1956 popl %edi
1957 ret
1958 SET_SIZE(scanc)
1959
1960 #endif /* __i386 */
1961 #endif /* __lint */
1962
1963 /*
1964 * Replacement functions for ones that are normally inlined.
1965 * In addition to the copy in i86.il, they are defined here just in case.
1966 */
1967
1968 #if defined(__lint)
1969
1970 ulong_t
1971 intr_clear(void)
1972 { return (0); }
1973
1974 ulong_t
1975 clear_int_flag(void)
1976 { return (0); }
1977
1978 #else /* __lint */
1979
1980 #if defined(__amd64)
1981
1982 ENTRY(intr_clear)
1983 ENTRY(clear_int_flag)
1984 pushfq
1985 popq %rax
1986 #if defined(__xpv)
1987 leaq xpv_panicking, %rdi
1988 movl (%rdi), %edi
1989 cmpl $0, %edi
1990 jne 2f
1991 CLIRET(%rdi, %dl) /* returns event mask in %dl */
1992 /*
1993 * Synthesize the PS_IE bit from the event mask bit
1994 */
1995 andq $_BITNOT(PS_IE), %rax
1996 testb $1, %dl
1997 jnz 1f
1998 orq $PS_IE, %rax
1999 1:
2000 ret
2001 2:
2002 #endif
2003 CLI(%rdi)
2004 ret
2005 SET_SIZE(clear_int_flag)
2006 SET_SIZE(intr_clear)
2007
2008 #elif defined(__i386)
2009
2010 ENTRY(intr_clear)
2011 ENTRY(clear_int_flag)
2012 pushfl
2013 popl %eax
2014 #if defined(__xpv)
2015 leal xpv_panicking, %edx
2016 movl (%edx), %edx
2017 cmpl $0, %edx
2018 jne 2f
2019 CLIRET(%edx, %cl) /* returns event mask in %cl */
2020 /*
2021 * Synthesize the PS_IE bit from the event mask bit
2022 */
2023 andl $_BITNOT(PS_IE), %eax
2024 testb $1, %cl
2025 jnz 1f
2026 orl $PS_IE, %eax
2027 1:
2028 ret
2029 2:
2030 #endif
2031 CLI(%edx)
2032 ret
2033 SET_SIZE(clear_int_flag)
2034 SET_SIZE(intr_clear)
2035
2036 #endif /* __i386 */
2037 #endif /* __lint */
2038
2039 #if defined(__lint)
2040
2041 struct cpu *
2042 curcpup(void)
2043 { return 0; }
2044
2045 #else /* __lint */
2046
2047 #if defined(__amd64)
2048
2049 ENTRY(curcpup)
2050 movq %gs:CPU_SELF, %rax
2051 ret
2052 SET_SIZE(curcpup)
2053
2054 #elif defined(__i386)
2055
2056 ENTRY(curcpup)
2057 movl %gs:CPU_SELF, %eax
2058 ret
2059 SET_SIZE(curcpup)
2060
2061 #endif /* __i386 */
2062 #endif /* __lint */
2063
2064 /* htonll(), ntohll(), htonl(), ntohl(), htons(), ntohs()
2065 * These functions reverse the byte order of the input parameter and returns
2066 * the result. This is to convert the byte order from host byte order
2067 * (little endian) to network byte order (big endian), or vice versa.
2068 */
2069
2070 #if defined(__lint)
2071
2072 uint64_t
2073 htonll(uint64_t i)
2074 { return (i); }
2075
2076 uint64_t
2077 ntohll(uint64_t i)
2078 { return (i); }
2079
2080 uint32_t
2081 htonl(uint32_t i)
2082 { return (i); }
2083
2084 uint32_t
2085 ntohl(uint32_t i)
2086 { return (i); }
2087
2088 uint16_t
2089 htons(uint16_t i)
2090 { return (i); }
2091
2092 uint16_t
2093 ntohs(uint16_t i)
2094 { return (i); }
2095
2096 #else /* __lint */
2097
2098 #if defined(__amd64)
2099
2100 ENTRY(htonll)
2101 ALTENTRY(ntohll)
2102 movq %rdi, %rax
2103 bswapq %rax
2104 ret
2105 SET_SIZE(ntohll)
2106 SET_SIZE(htonll)
2107
2108 /* XX64 there must be shorter sequences for this */
2109 ENTRY(htonl)
2110 ALTENTRY(ntohl)
2111 movl %edi, %eax
2112 bswap %eax
2113 ret
2114 SET_SIZE(ntohl)
2115 SET_SIZE(htonl)
2116
2117 /* XX64 there must be better sequences for this */
2118 ENTRY(htons)
2119 ALTENTRY(ntohs)
2120 movl %edi, %eax
2121 bswap %eax
2122 shrl $16, %eax
2123 ret
2124 SET_SIZE(ntohs)
2125 SET_SIZE(htons)
2126
2127 #elif defined(__i386)
2128
2129 ENTRY(htonll)
2130 ALTENTRY(ntohll)
2131 movl 4(%esp), %edx
2132 movl 8(%esp), %eax
2133 bswap %edx
2134 bswap %eax
2135 ret
2136 SET_SIZE(ntohll)
2137 SET_SIZE(htonll)
2138
2139 ENTRY(htonl)
2140 ALTENTRY(ntohl)
2141 movl 4(%esp), %eax
2142 bswap %eax
2143 ret
2144 SET_SIZE(ntohl)
2145 SET_SIZE(htonl)
2146
2147 ENTRY(htons)
2148 ALTENTRY(ntohs)
2149 movl 4(%esp), %eax
2150 bswap %eax
2151 shrl $16, %eax
2152 ret
2153 SET_SIZE(ntohs)
2154 SET_SIZE(htons)
2155
2156 #endif /* __i386 */
2157 #endif /* __lint */
2158
2159
2160 #if defined(__lint)
2161
2162 /* ARGSUSED */
2163 void
2164 intr_restore(ulong_t i)
2165 { return; }
2166
2167 /* ARGSUSED */
2168 void
2169 restore_int_flag(ulong_t i)
2170 { return; }
2171
2172 #else /* __lint */
2173
2174 #if defined(__amd64)
2175
2176 ENTRY(intr_restore)
2177 ENTRY(restore_int_flag)
2178 testq $PS_IE, %rdi
2179 jz 1f
2180 #if defined(__xpv)
2181 leaq xpv_panicking, %rsi
2182 movl (%rsi), %esi
2183 cmpl $0, %esi
2184 jne 1f
2185 /*
2186 * Since we're -really- running unprivileged, our attempt
2187 * to change the state of the IF bit will be ignored.
2188 * The virtual IF bit is tweaked by CLI and STI.
2189 */
2190 IE_TO_EVENT_MASK(%rsi, %rdi)
2191 #else
2192 sti
2193 #endif
2194 1:
2195 ret
2196 SET_SIZE(restore_int_flag)
2197 SET_SIZE(intr_restore)
2198
2199 #elif defined(__i386)
2200
2201 ENTRY(intr_restore)
2202 ENTRY(restore_int_flag)
2203 testl $PS_IE, 4(%esp)
2204 jz 1f
2205 #if defined(__xpv)
2206 leal xpv_panicking, %edx
2207 movl (%edx), %edx
2208 cmpl $0, %edx
2209 jne 1f
2210 /*
2211 * Since we're -really- running unprivileged, our attempt
2212 * to change the state of the IF bit will be ignored.
2213 * The virtual IF bit is tweaked by CLI and STI.
2214 */
2215 IE_TO_EVENT_MASK(%edx, 4(%esp))
2216 #else
2217 sti
2218 #endif
2219 1:
2220 ret
2221 SET_SIZE(restore_int_flag)
2222 SET_SIZE(intr_restore)
2223
2224 #endif /* __i386 */
2225 #endif /* __lint */
2226
2227 #if defined(__lint)
2228
2229 void
2230 sti(void)
2231 {}
2232
2233 void
2234 cli(void)
2235 {}
2236
2237 #else /* __lint */
2238
2239 ENTRY(sti)
2240 STI
2241 ret
2242 SET_SIZE(sti)
2243
2244 ENTRY(cli)
2245 #if defined(__amd64)
2246 CLI(%rax)
2247 #elif defined(__i386)
2248 CLI(%eax)
2249 #endif /* __i386 */
2250 ret
2251 SET_SIZE(cli)
2252
2253 #endif /* __lint */
2254
2255 #if defined(__lint)
2256
2257 dtrace_icookie_t
2258 dtrace_interrupt_disable(void)
2259 { return (0); }
2260
2261 #else /* __lint */
2262
2263 #if defined(__amd64)
2264
2265 ENTRY(dtrace_interrupt_disable)
2266 pushfq
2267 popq %rax
2268 #if defined(__xpv)
2269 leaq xpv_panicking, %rdi
2270 movl (%rdi), %edi
2271 cmpl $0, %edi
2272 jne .dtrace_interrupt_disable_done
2273 CLIRET(%rdi, %dl) /* returns event mask in %dl */
2274 /*
2275 * Synthesize the PS_IE bit from the event mask bit
2276 */
2277 andq $_BITNOT(PS_IE), %rax
2278 testb $1, %dl
2279 jnz .dtrace_interrupt_disable_done
2280 orq $PS_IE, %rax
2281 #else
2282 CLI(%rdx)
2283 #endif
2284 .dtrace_interrupt_disable_done:
2285 ret
2286 SET_SIZE(dtrace_interrupt_disable)
2287
2288 #elif defined(__i386)
2289
2290 ENTRY(dtrace_interrupt_disable)
2291 pushfl
2292 popl %eax
2293 #if defined(__xpv)
2294 leal xpv_panicking, %edx
2295 movl (%edx), %edx
2296 cmpl $0, %edx
2297 jne .dtrace_interrupt_disable_done
2298 CLIRET(%edx, %cl) /* returns event mask in %cl */
2299 /*
2300 * Synthesize the PS_IE bit from the event mask bit
2301 */
2302 andl $_BITNOT(PS_IE), %eax
2303 testb $1, %cl
2304 jnz .dtrace_interrupt_disable_done
2305 orl $PS_IE, %eax
2306 #else
2307 CLI(%edx)
2308 #endif
2309 .dtrace_interrupt_disable_done:
2310 ret
2311 SET_SIZE(dtrace_interrupt_disable)
2312
2313 #endif /* __i386 */
2314 #endif /* __lint */
2315
2316 #if defined(__lint)
2317
2318 /*ARGSUSED*/
2319 void
2320 dtrace_interrupt_enable(dtrace_icookie_t cookie)
2321 {}
2322
2323 #else /* __lint */
2324
2325 #if defined(__amd64)
2326
2327 ENTRY(dtrace_interrupt_enable)
2328 pushq %rdi
2329 popfq
2330 #if defined(__xpv)
2331 leaq xpv_panicking, %rdx
2332 movl (%rdx), %edx
2333 cmpl $0, %edx
2334 jne .dtrace_interrupt_enable_done
2335 /*
2336 * Since we're -really- running unprivileged, our attempt
2337 * to change the state of the IF bit will be ignored. The
2338 * virtual IF bit is tweaked by CLI and STI.
2339 */
2340 IE_TO_EVENT_MASK(%rdx, %rdi)
2341 #endif
2342 .dtrace_interrupt_enable_done:
2343 ret
2344 SET_SIZE(dtrace_interrupt_enable)
2345
2346 #elif defined(__i386)
2347
2348 ENTRY(dtrace_interrupt_enable)
2349 movl 4(%esp), %eax
2350 pushl %eax
2351 popfl
2352 #if defined(__xpv)
2353 leal xpv_panicking, %edx
2354 movl (%edx), %edx
2355 cmpl $0, %edx
2356 jne .dtrace_interrupt_enable_done
2357 /*
2358 * Since we're -really- running unprivileged, our attempt
2359 * to change the state of the IF bit will be ignored. The
2360 * virtual IF bit is tweaked by CLI and STI.
2361 */
2362 IE_TO_EVENT_MASK(%edx, %eax)
2363 #endif
2364 .dtrace_interrupt_enable_done:
2365 ret
2366 SET_SIZE(dtrace_interrupt_enable)
2367
2368 #endif /* __i386 */
2369 #endif /* __lint */
2370
2371
2372 #if defined(lint)
2373
2374 void
2375 dtrace_membar_producer(void)
2376 {}
2377
2378 void
2379 dtrace_membar_consumer(void)
2380 {}
2381
2382 #else /* __lint */
2383
2384 ENTRY(dtrace_membar_producer)
2385 rep; ret /* use 2 byte return instruction when branch target */
2386 /* AMD Software Optimization Guide - Section 6.2 */
2387 SET_SIZE(dtrace_membar_producer)
2388
2389 ENTRY(dtrace_membar_consumer)
2390 rep; ret /* use 2 byte return instruction when branch target */
2391 /* AMD Software Optimization Guide - Section 6.2 */
2392 SET_SIZE(dtrace_membar_consumer)
2393
2394 #endif /* __lint */
2395
2396 #if defined(__lint)
2397
2398 kthread_id_t
2399 threadp(void)
2400 { return ((kthread_id_t)0); }
2401
2402 #else /* __lint */
2403
2404 #if defined(__amd64)
2405
2406 ENTRY(threadp)
2407 movq %gs:CPU_THREAD, %rax
2408 ret
2409 SET_SIZE(threadp)
2410
2411 #elif defined(__i386)
2412
2413 ENTRY(threadp)
2414 movl %gs:CPU_THREAD, %eax
2415 ret
2416 SET_SIZE(threadp)
2417
2418 #endif /* __i386 */
2419 #endif /* __lint */
2420
2421 /*
2422 * Checksum routine for Internet Protocol Headers
2423 */
2424
2425 #if defined(__lint)
2426
2427 /* ARGSUSED */
2428 unsigned int
2429 ip_ocsum(
2430 ushort_t *address, /* ptr to 1st message buffer */
2431 int halfword_count, /* length of data */
2432 unsigned int sum) /* partial checksum */
2433 {
2434 int i;
2435 unsigned int psum = 0; /* partial sum */
2436
2437 for (i = 0; i < halfword_count; i++, address++) {
2438 psum += *address;
2439 }
2440
2441 while ((psum >> 16) != 0) {
2442 psum = (psum & 0xffff) + (psum >> 16);
2443 }
2444
2445 psum += sum;
2446
2447 while ((psum >> 16) != 0) {
2448 psum = (psum & 0xffff) + (psum >> 16);
2449 }
2450
2451 return (psum);
2452 }
2453
2454 #else /* __lint */
2455
2456 #if defined(__amd64)
2457
2458 ENTRY(ip_ocsum)
2459 pushq %rbp
2460 movq %rsp, %rbp
2461 #ifdef DEBUG
2462 movq postbootkernelbase(%rip), %rax
2463 cmpq %rax, %rdi
2464 jnb 1f
2465 xorl %eax, %eax
2466 movq %rdi, %rsi
2467 leaq .ip_ocsum_panic_msg(%rip), %rdi
2468 call panic
2469 /*NOTREACHED*/
2470 .ip_ocsum_panic_msg:
2471 .string "ip_ocsum: address 0x%p below kernelbase\n"
2472 1:
2473 #endif
2474 movl %esi, %ecx /* halfword_count */
2475 movq %rdi, %rsi /* address */
2476 /* partial sum in %edx */
2477 xorl %eax, %eax
2478 testl %ecx, %ecx
2479 jz .ip_ocsum_done
2480 testq $3, %rsi
2481 jnz .ip_csum_notaligned
2482 .ip_csum_aligned: /* XX64 opportunities for 8-byte operations? */
2483 .next_iter:
2484 /* XX64 opportunities for prefetch? */
2485 /* XX64 compute csum with 64 bit quantities? */
2486 subl $32, %ecx
2487 jl .less_than_32
2488
2489 addl 0(%rsi), %edx
2490 .only60:
2491 adcl 4(%rsi), %eax
2492 .only56:
2493 adcl 8(%rsi), %edx
2494 .only52:
2495 adcl 12(%rsi), %eax
2496 .only48:
2497 adcl 16(%rsi), %edx
2498 .only44:
2499 adcl 20(%rsi), %eax
2500 .only40:
2501 adcl 24(%rsi), %edx
2502 .only36:
2503 adcl 28(%rsi), %eax
2504 .only32:
2505 adcl 32(%rsi), %edx
2506 .only28:
2507 adcl 36(%rsi), %eax
2508 .only24:
2509 adcl 40(%rsi), %edx
2510 .only20:
2511 adcl 44(%rsi), %eax
2512 .only16:
2513 adcl 48(%rsi), %edx
2514 .only12:
2515 adcl 52(%rsi), %eax
2516 .only8:
2517 adcl 56(%rsi), %edx
2518 .only4:
2519 adcl 60(%rsi), %eax /* could be adding -1 and -1 with a carry */
2520 .only0:
2521 adcl $0, %eax /* could be adding -1 in eax with a carry */
2522 adcl $0, %eax
2523
2524 addq $64, %rsi
2525 testl %ecx, %ecx
2526 jnz .next_iter
2527
2528 .ip_ocsum_done:
2529 addl %eax, %edx
2530 adcl $0, %edx
2531 movl %edx, %eax /* form a 16 bit checksum by */
2532 shrl $16, %eax /* adding two halves of 32 bit checksum */
2533 addw %dx, %ax
2534 adcw $0, %ax
2535 andl $0xffff, %eax
2536 leave
2537 ret
2538
2539 .ip_csum_notaligned:
2540 xorl %edi, %edi
2541 movw (%rsi), %di
2542 addl %edi, %edx
2543 adcl $0, %edx
2544 addq $2, %rsi
2545 decl %ecx
2546 jmp .ip_csum_aligned
2547
2548 .less_than_32:
2549 addl $32, %ecx
2550 testl $1, %ecx
2551 jz .size_aligned
2552 andl $0xfe, %ecx
2553 movzwl (%rsi, %rcx, 2), %edi
2554 addl %edi, %edx
2555 adcl $0, %edx
2556 .size_aligned:
2557 movl %ecx, %edi
2558 shrl $1, %ecx
2559 shl $1, %edi
2560 subq $64, %rdi
2561 addq %rdi, %rsi
2562 leaq .ip_ocsum_jmptbl(%rip), %rdi
2563 leaq (%rdi, %rcx, 8), %rdi
2564 xorl %ecx, %ecx
2565 clc
2566 jmp *(%rdi)
2567
2568 .align 8
2569 .ip_ocsum_jmptbl:
2570 .quad .only0, .only4, .only8, .only12, .only16, .only20
2571 .quad .only24, .only28, .only32, .only36, .only40, .only44
2572 .quad .only48, .only52, .only56, .only60
2573 SET_SIZE(ip_ocsum)
2574
2575 #elif defined(__i386)
2576
2577 ENTRY(ip_ocsum)
2578 pushl %ebp
2579 movl %esp, %ebp
2580 pushl %ebx
2581 pushl %esi
2582 pushl %edi
2583 movl 12(%ebp), %ecx /* count of half words */
2584 movl 16(%ebp), %edx /* partial checksum */
2585 movl 8(%ebp), %esi
2586 xorl %eax, %eax
2587 testl %ecx, %ecx
2588 jz .ip_ocsum_done
2589
2590 testl $3, %esi
2591 jnz .ip_csum_notaligned
2592 .ip_csum_aligned:
2593 .next_iter:
2594 subl $32, %ecx
2595 jl .less_than_32
2596
2597 addl 0(%esi), %edx
2598 .only60:
2599 adcl 4(%esi), %eax
2600 .only56:
2601 adcl 8(%esi), %edx
2602 .only52:
2603 adcl 12(%esi), %eax
2604 .only48:
2605 adcl 16(%esi), %edx
2606 .only44:
2607 adcl 20(%esi), %eax
2608 .only40:
2609 adcl 24(%esi), %edx
2610 .only36:
2611 adcl 28(%esi), %eax
2612 .only32:
2613 adcl 32(%esi), %edx
2614 .only28:
2615 adcl 36(%esi), %eax
2616 .only24:
2617 adcl 40(%esi), %edx
2618 .only20:
2619 adcl 44(%esi), %eax
2620 .only16:
2621 adcl 48(%esi), %edx
2622 .only12:
2623 adcl 52(%esi), %eax
2624 .only8:
2625 adcl 56(%esi), %edx
2626 .only4:
2627 adcl 60(%esi), %eax /* We could be adding -1 and -1 with a carry */
2628 .only0:
2629 adcl $0, %eax /* we could be adding -1 in eax with a carry */
2630 adcl $0, %eax
2631
2632 addl $64, %esi
2633 andl %ecx, %ecx
2634 jnz .next_iter
2635
2636 .ip_ocsum_done:
2637 addl %eax, %edx
2638 adcl $0, %edx
2639 movl %edx, %eax /* form a 16 bit checksum by */
2640 shrl $16, %eax /* adding two halves of 32 bit checksum */
2641 addw %dx, %ax
2642 adcw $0, %ax
2643 andl $0xffff, %eax
2644 popl %edi /* restore registers */
2645 popl %esi
2646 popl %ebx
2647 leave
2648 ret
2649
2650 .ip_csum_notaligned:
2651 xorl %edi, %edi
2652 movw (%esi), %di
2653 addl %edi, %edx
2654 adcl $0, %edx
2655 addl $2, %esi
2656 decl %ecx
2657 jmp .ip_csum_aligned
2658
2659 .less_than_32:
2660 addl $32, %ecx
2661 testl $1, %ecx
2662 jz .size_aligned
2663 andl $0xfe, %ecx
2664 movzwl (%esi, %ecx, 2), %edi
2665 addl %edi, %edx
2666 adcl $0, %edx
2667 .size_aligned:
2668 movl %ecx, %edi
2669 shrl $1, %ecx
2670 shl $1, %edi
2671 subl $64, %edi
2672 addl %edi, %esi
2673 movl $.ip_ocsum_jmptbl, %edi
2674 lea (%edi, %ecx, 4), %edi
2675 xorl %ecx, %ecx
2676 clc
2677 jmp *(%edi)
2678 SET_SIZE(ip_ocsum)
2679
2680 .data
2681 .align 4
2682
2683 .ip_ocsum_jmptbl:
2684 .long .only0, .only4, .only8, .only12, .only16, .only20
2685 .long .only24, .only28, .only32, .only36, .only40, .only44
2686 .long .only48, .only52, .only56, .only60
2687
2688
2689 #endif /* __i386 */
2690 #endif /* __lint */
2691
2692 /*
2693 * multiply two long numbers and yield a u_longlong_t result, callable from C.
2694 * Provided to manipulate hrtime_t values.
2695 */
2696 #if defined(__lint)
2697
2698 /* result = a * b; */
2699
2700 /* ARGSUSED */
2701 unsigned long long
2702 mul32(uint_t a, uint_t b)
2703 { return (0); }
2704
2705 #else /* __lint */
2706
2707 #if defined(__amd64)
2708
2709 ENTRY(mul32)
2710 xorl %edx, %edx /* XX64 joe, paranoia? */
2711 movl %edi, %eax
2712 mull %esi
2713 shlq $32, %rdx
2714 orq %rdx, %rax
2715 ret
2716 SET_SIZE(mul32)
2717
2718 #elif defined(__i386)
2719
2720 ENTRY(mul32)
2721 movl 8(%esp), %eax
2722 movl 4(%esp), %ecx
2723 mull %ecx
2724 ret
2725 SET_SIZE(mul32)
2726
2727 #endif /* __i386 */
2728 #endif /* __lint */
2729
2730 #if defined(notused)
2731 #if defined(__lint)
2732 /* ARGSUSED */
2733 void
2734 load_pte64(uint64_t *pte, uint64_t pte_value)
2735 {}
2736 #else /* __lint */
2737 .globl load_pte64
2738 load_pte64:
2739 movl 4(%esp), %eax
2740 movl 8(%esp), %ecx
2741 movl 12(%esp), %edx
2742 movl %edx, 4(%eax)
2743 movl %ecx, (%eax)
2744 ret
2745 #endif /* __lint */
2746 #endif /* notused */
2747
2748 #if defined(__lint)
2749
2750 /*ARGSUSED*/
2751 void
2752 scan_memory(caddr_t addr, size_t size)
2753 {}
2754
2755 #else /* __lint */
2756
2757 #if defined(__amd64)
2758
2759 ENTRY(scan_memory)
2760 shrq $3, %rsi /* convert %rsi from byte to quadword count */
2761 jz .scanm_done
2762 movq %rsi, %rcx /* move count into rep control register */
2763 movq %rdi, %rsi /* move addr into lodsq control reg. */
2764 rep lodsq /* scan the memory range */
2765 .scanm_done:
2766 rep; ret /* use 2 byte return instruction when branch target */
2767 /* AMD Software Optimization Guide - Section 6.2 */
2768 SET_SIZE(scan_memory)
2769
2770 #elif defined(__i386)
2771
2772 ENTRY(scan_memory)
2773 pushl %ecx
2774 pushl %esi
2775 movl 16(%esp), %ecx /* move 2nd arg into rep control register */
2776 shrl $2, %ecx /* convert from byte count to word count */
2777 jz .scanm_done
2778 movl 12(%esp), %esi /* move 1st arg into lodsw control register */
2779 .byte 0xf3 /* rep prefix. lame assembler. sigh. */
2780 lodsl
2781 .scanm_done:
2782 popl %esi
2783 popl %ecx
2784 ret
2785 SET_SIZE(scan_memory)
2786
2787 #endif /* __i386 */
2788 #endif /* __lint */
2789
2790
2791 #if defined(__lint)
2792
2793 /*ARGSUSED */
2794 int
2795 lowbit(ulong_t i)
2796 { return (0); }
2797
2798 #else /* __lint */
2799
2800 #if defined(__amd64)
2801
2802 ENTRY(lowbit)
2803 movl $-1, %eax
2804 bsfq %rdi, %rdi
2805 cmovnz %edi, %eax
2806 incl %eax
2807 ret
2808 SET_SIZE(lowbit)
2809
2810 #elif defined(__i386)
2811
2812 ENTRY(lowbit)
2813 bsfl 4(%esp), %eax
2814 jz 0f
2815 incl %eax
2816 ret
2817 0:
2818 xorl %eax, %eax
2819 ret
2820 SET_SIZE(lowbit)
2821
2822 #endif /* __i386 */
2823 #endif /* __lint */
2824
2825 #if defined(__lint)
2826
2827 /*ARGSUSED*/
2828 int
2829 highbit(ulong_t i)
2830 { return (0); }
2831
2832 /*ARGSUSED*/
2833 int
2834 highbit64(uint64_t i)
2835 { return (0); }
2836
2837 #else /* __lint */
2838
2839 #if defined(__amd64)
2840
2841 ENTRY(highbit)
2842 ALTENTRY(highbit64)
2843 movl $-1, %eax
2844 bsrq %rdi, %rdi
2845 cmovnz %edi, %eax
2846 incl %eax
2847 ret
2848 SET_SIZE(highbit64)
2849 SET_SIZE(highbit)
2850
2851 #elif defined(__i386)
2852
2853 ENTRY(highbit)
2854 bsrl 4(%esp), %eax
2855 jz 0f
2856 incl %eax
2857 ret
2858 0:
2859 xorl %eax, %eax
2860 ret
2861 SET_SIZE(highbit)
2862
2863 ENTRY(highbit64)
2864 bsrl 8(%esp), %eax
2865 jz highbit
2866 addl $33, %eax
2867 ret
2868 SET_SIZE(highbit64)
2869
2870 #endif /* __i386 */
2871 #endif /* __lint */
2872
2873 #if defined(__lint)
2874
2875 /*ARGSUSED*/
2876 uint64_t
2877 rdmsr(uint_t r)
2878 { return (0); }
2879
2880 /*ARGSUSED*/
2881 void
2882 wrmsr(uint_t r, const uint64_t val)
2883 {}
2884
2885 /*ARGSUSED*/
2886 uint64_t
2887 xrdmsr(uint_t r)
2888 { return (0); }
2889
2890 /*ARGSUSED*/
2891 void
2892 xwrmsr(uint_t r, const uint64_t val)
2893 {}
2894
2895 void
2896 invalidate_cache(void)
2897 {}
2898
2899 /*ARGSUSED*/
2900 uint64_t
2901 get_xcr(uint_t r)
2902 { return (0); }
2903
2904 /*ARGSUSED*/
2905 void
2906 set_xcr(uint_t r, const uint64_t val)
2907 {}
2908
2909 #else /* __lint */
2910
2911 #define XMSR_ACCESS_VAL $0x9c5a203a
2912
2913 #if defined(__amd64)
2914
2915 ENTRY(rdmsr)
2916 movl %edi, %ecx
2917 rdmsr
2918 shlq $32, %rdx
2919 orq %rdx, %rax
2920 ret
2921 SET_SIZE(rdmsr)
2922
2923 ENTRY(wrmsr)
2924 movq %rsi, %rdx
2925 shrq $32, %rdx
2926 movl %esi, %eax
2927 movl %edi, %ecx
2928 wrmsr
2929 ret
2930 SET_SIZE(wrmsr)
2931
2932 ENTRY(xrdmsr)
2933 pushq %rbp
2934 movq %rsp, %rbp
2935 movl %edi, %ecx
2936 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2937 rdmsr
2938 shlq $32, %rdx
2939 orq %rdx, %rax
2940 leave
2941 ret
2942 SET_SIZE(xrdmsr)
2943
2944 ENTRY(xwrmsr)
2945 pushq %rbp
2946 movq %rsp, %rbp
2947 movl %edi, %ecx
2948 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2949 movq %rsi, %rdx
2950 shrq $32, %rdx
2951 movl %esi, %eax
2952 wrmsr
2953 leave
2954 ret
2955 SET_SIZE(xwrmsr)
2956
2957 ENTRY(get_xcr)
2958 movl %edi, %ecx
2959 #xgetbv
2960 .byte 0x0f,0x01,0xd0
2961 shlq $32, %rdx
2962 orq %rdx, %rax
2963 ret
2964 SET_SIZE(get_xcr)
2965
2966 ENTRY(set_xcr)
2967 movq %rsi, %rdx
2968 shrq $32, %rdx
2969 movl %esi, %eax
2970 movl %edi, %ecx
2971 #xsetbv
2972 .byte 0x0f,0x01,0xd1
2973 ret
2974 SET_SIZE(set_xcr)
2975
2976 #elif defined(__i386)
2977
2978 ENTRY(rdmsr)
2979 movl 4(%esp), %ecx
2980 rdmsr
2981 ret
2982 SET_SIZE(rdmsr)
2983
2984 ENTRY(wrmsr)
2985 movl 4(%esp), %ecx
2986 movl 8(%esp), %eax
2987 movl 12(%esp), %edx
2988 wrmsr
2989 ret
2990 SET_SIZE(wrmsr)
2991
2992 ENTRY(xrdmsr)
2993 pushl %ebp
2994 movl %esp, %ebp
2995 movl 8(%esp), %ecx
2996 pushl %edi
2997 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2998 rdmsr
2999 popl %edi
3000 leave
3001 ret
3002 SET_SIZE(xrdmsr)
3003
3004 ENTRY(xwrmsr)
3005 pushl %ebp
3006 movl %esp, %ebp
3007 movl 8(%esp), %ecx
3008 movl 12(%esp), %eax
3009 movl 16(%esp), %edx
3010 pushl %edi
3011 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
3012 wrmsr
3013 popl %edi
3014 leave
3015 ret
3016 SET_SIZE(xwrmsr)
3017
3018 ENTRY(get_xcr)
3019 movl 4(%esp), %ecx
3020 #xgetbv
3021 .byte 0x0f,0x01,0xd0
3022 ret
3023 SET_SIZE(get_xcr)
3024
3025 ENTRY(set_xcr)
3026 movl 4(%esp), %ecx
3027 movl 8(%esp), %eax
3028 movl 12(%esp), %edx
3029 #xsetbv
3030 .byte 0x0f,0x01,0xd1
3031 ret
3032 SET_SIZE(set_xcr)
3033
3034 #endif /* __i386 */
3035
3036 ENTRY(invalidate_cache)
3037 wbinvd
3038 ret
3039 SET_SIZE(invalidate_cache)
3040
3041 #endif /* __lint */
3042
3043 #if defined(__lint)
3044
3045 /*ARGSUSED*/
3046 void
3047 getcregs(struct cregs *crp)
3048 {}
3049
3050 #else /* __lint */
3051
3052 #if defined(__amd64)
3053
3054 ENTRY_NP(getcregs)
3055 #if defined(__xpv)
3056 /*
3057 * Only a few of the hardware control registers or descriptor tables
3058 * are directly accessible to us, so just zero the structure.
3059 *
3060 * XXPV Perhaps it would be helpful for the hypervisor to return
3061 * virtualized versions of these for post-mortem use.
3062 * (Need to reevaluate - perhaps it already does!)
3063 */
3064 pushq %rdi /* save *crp */
3065 movq $CREGSZ, %rsi
3066 call bzero
3067 popq %rdi
3068
3069 /*
3070 * Dump what limited information we can
3071 */
3072 movq %cr0, %rax
3073 movq %rax, CREG_CR0(%rdi) /* cr0 */
3074 movq %cr2, %rax
3075 movq %rax, CREG_CR2(%rdi) /* cr2 */
3076 movq %cr3, %rax
3077 movq %rax, CREG_CR3(%rdi) /* cr3 */
3078 movq %cr4, %rax
3079 movq %rax, CREG_CR4(%rdi) /* cr4 */
3080
3081 #else /* __xpv */
3082
3083 #define GETMSR(r, off, d) \
3084 movl $r, %ecx; \
3085 rdmsr; \
3086 movl %eax, off(d); \
3087 movl %edx, off+4(d)
3088
3089 xorl %eax, %eax
3090 movq %rax, CREG_GDT+8(%rdi)
3091 sgdt CREG_GDT(%rdi) /* 10 bytes */
3092 movq %rax, CREG_IDT+8(%rdi)
3093 sidt CREG_IDT(%rdi) /* 10 bytes */
3094 movq %rax, CREG_LDT(%rdi)
3095 sldt CREG_LDT(%rdi) /* 2 bytes */
3096 movq %rax, CREG_TASKR(%rdi)
3097 str CREG_TASKR(%rdi) /* 2 bytes */
3098 movq %cr0, %rax
3099 movq %rax, CREG_CR0(%rdi) /* cr0 */
3100 movq %cr2, %rax
3101 movq %rax, CREG_CR2(%rdi) /* cr2 */
3102 movq %cr3, %rax
3103 movq %rax, CREG_CR3(%rdi) /* cr3 */
3104 movq %cr4, %rax
3105 movq %rax, CREG_CR4(%rdi) /* cr4 */
3106 movq %cr8, %rax
3107 movq %rax, CREG_CR8(%rdi) /* cr8 */
3108 GETMSR(MSR_AMD_KGSBASE, CREG_KGSBASE, %rdi)
3109 GETMSR(MSR_AMD_EFER, CREG_EFER, %rdi)
3110 #endif /* __xpv */
3111 ret
3112 SET_SIZE(getcregs)
3113
3114 #undef GETMSR
3115
3116 #elif defined(__i386)
3117
3118 ENTRY_NP(getcregs)
3119 #if defined(__xpv)
3120 /*
3121 * Only a few of the hardware control registers or descriptor tables
3122 * are directly accessible to us, so just zero the structure.
3123 *
3124 * XXPV Perhaps it would be helpful for the hypervisor to return
3125 * virtualized versions of these for post-mortem use.
3126 * (Need to reevaluate - perhaps it already does!)
3127 */
3128 movl 4(%esp), %edx
3129 pushl $CREGSZ
3130 pushl %edx
3131 call bzero
3132 addl $8, %esp
3133 movl 4(%esp), %edx
3134
3135 /*
3136 * Dump what limited information we can
3137 */
3138 movl %cr0, %eax
3139 movl %eax, CREG_CR0(%edx) /* cr0 */
3140 movl %cr2, %eax
3141 movl %eax, CREG_CR2(%edx) /* cr2 */
3142 movl %cr3, %eax
3143 movl %eax, CREG_CR3(%edx) /* cr3 */
3144 movl %cr4, %eax
3145 movl %eax, CREG_CR4(%edx) /* cr4 */
3146
3147 #else /* __xpv */
3148
3149 movl 4(%esp), %edx
3150 movw $0, CREG_GDT+6(%edx)
3151 movw $0, CREG_IDT+6(%edx)
3152 sgdt CREG_GDT(%edx) /* gdt */
3153 sidt CREG_IDT(%edx) /* idt */
3154 sldt CREG_LDT(%edx) /* ldt */
3155 str CREG_TASKR(%edx) /* task */
3156 movl %cr0, %eax
3157 movl %eax, CREG_CR0(%edx) /* cr0 */
3158 movl %cr2, %eax
3159 movl %eax, CREG_CR2(%edx) /* cr2 */
3160 movl %cr3, %eax
3161 movl %eax, CREG_CR3(%edx) /* cr3 */
3162 bt $X86FSET_LARGEPAGE, x86_featureset
3163 jnc .nocr4
3164 movl %cr4, %eax
3165 movl %eax, CREG_CR4(%edx) /* cr4 */
3166 jmp .skip
3167 .nocr4:
3168 movl $0, CREG_CR4(%edx)
3169 .skip:
3170 #endif
3171 ret
3172 SET_SIZE(getcregs)
3173
3174 #endif /* __i386 */
3175 #endif /* __lint */
3176
3177
3178 /*
3179 * A panic trigger is a word which is updated atomically and can only be set
3180 * once. We atomically store 0xDEFACEDD and load the old value. If the
3181 * previous value was 0, we succeed and return 1; otherwise return 0.
3182 * This allows a partially corrupt trigger to still trigger correctly. DTrace
3183 * has its own version of this function to allow it to panic correctly from
3184 * probe context.
3185 */
3186 #if defined(__lint)
3187
3188 /*ARGSUSED*/
3189 int
3190 panic_trigger(int *tp)
3191 { return (0); }
3192
3193 /*ARGSUSED*/
3194 int
3195 dtrace_panic_trigger(int *tp)
3196 { return (0); }
3197
3198 #else /* __lint */
3199
3200 #if defined(__amd64)
3201
3202 ENTRY_NP(panic_trigger)
3203 xorl %eax, %eax
3204 movl $0xdefacedd, %edx
3205 lock
3206 xchgl %edx, (%rdi)
3207 cmpl $0, %edx
3208 je 0f
3209 movl $0, %eax
3210 ret
3211 0: movl $1, %eax
3212 ret
3213 SET_SIZE(panic_trigger)
3214
3215 ENTRY_NP(dtrace_panic_trigger)
3216 xorl %eax, %eax
3217 movl $0xdefacedd, %edx
3218 lock
3219 xchgl %edx, (%rdi)
3220 cmpl $0, %edx
3221 je 0f
3222 movl $0, %eax
3223 ret
3224 0: movl $1, %eax
3225 ret
3226 SET_SIZE(dtrace_panic_trigger)
3227
3228 #elif defined(__i386)
3229
3230 ENTRY_NP(panic_trigger)
3231 movl 4(%esp), %edx / %edx = address of trigger
3232 movl $0xdefacedd, %eax / %eax = 0xdefacedd
3233 lock / assert lock
3234 xchgl %eax, (%edx) / exchange %eax and the trigger
3235 cmpl $0, %eax / if (%eax == 0x0)
3236 je 0f / return (1);
3237 movl $0, %eax / else
3238 ret / return (0);
3239 0: movl $1, %eax
3240 ret
3241 SET_SIZE(panic_trigger)
3242
3243 ENTRY_NP(dtrace_panic_trigger)
3244 movl 4(%esp), %edx / %edx = address of trigger
3245 movl $0xdefacedd, %eax / %eax = 0xdefacedd
3246 lock / assert lock
3247 xchgl %eax, (%edx) / exchange %eax and the trigger
3248 cmpl $0, %eax / if (%eax == 0x0)
3249 je 0f / return (1);
3250 movl $0, %eax / else
3251 ret / return (0);
3252 0: movl $1, %eax
3253 ret
3254 SET_SIZE(dtrace_panic_trigger)
3255
3256 #endif /* __i386 */
3257 #endif /* __lint */
3258
3259 /*
3260 * The panic() and cmn_err() functions invoke vpanic() as a common entry point
3261 * into the panic code implemented in panicsys(). vpanic() is responsible
3262 * for passing through the format string and arguments, and constructing a
3263 * regs structure on the stack into which it saves the current register
3264 * values. If we are not dying due to a fatal trap, these registers will
3265 * then be preserved in panicbuf as the current processor state. Before
3266 * invoking panicsys(), vpanic() activates the first panic trigger (see
3267 * common/os/panic.c) and switches to the panic_stack if successful. Note that
3268 * DTrace takes a slightly different panic path if it must panic from probe
3269 * context. Instead of calling panic, it calls into dtrace_vpanic(), which
3270 * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and
3271 * branches back into vpanic().
3272 */
3273 #if defined(__lint)
3274
3275 /*ARGSUSED*/
3276 void
3277 vpanic(const char *format, va_list alist)
3278 {}
3279
3280 /*ARGSUSED*/
3281 void
3282 dtrace_vpanic(const char *format, va_list alist)
3283 {}
3284
3285 #else /* __lint */
3286
3287 #if defined(__amd64)
3288
3289 ENTRY_NP(vpanic) /* Initial stack layout: */
3290
3291 pushq %rbp /* | %rip | 0x60 */
3292 movq %rsp, %rbp /* | %rbp | 0x58 */
3293 pushfq /* | rfl | 0x50 */
3294 pushq %r11 /* | %r11 | 0x48 */
3295 pushq %r10 /* | %r10 | 0x40 */
3296 pushq %rbx /* | %rbx | 0x38 */
3297 pushq %rax /* | %rax | 0x30 */
3298 pushq %r9 /* | %r9 | 0x28 */
3299 pushq %r8 /* | %r8 | 0x20 */
3300 pushq %rcx /* | %rcx | 0x18 */
3301 pushq %rdx /* | %rdx | 0x10 */
3302 pushq %rsi /* | %rsi | 0x8 alist */
3303 pushq %rdi /* | %rdi | 0x0 format */
3304
3305 movq %rsp, %rbx /* %rbx = current %rsp */
3306
3307 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */
3308 call panic_trigger /* %eax = panic_trigger() */
3309
3310 vpanic_common:
3311 /*
3312 * The panic_trigger result is in %eax from the call above, and
3313 * dtrace_panic places it in %eax before branching here.
3314 * The rdmsr instructions that follow below will clobber %eax so
3315 * we stash the panic_trigger result in %r11d.
3316 */
3317 movl %eax, %r11d
3318 cmpl $0, %r11d
3319 je 0f
3320
3321 /*
3322 * If panic_trigger() was successful, we are the first to initiate a
3323 * panic: we now switch to the reserved panic_stack before continuing.
3324 */
3325 leaq panic_stack(%rip), %rsp
3326 addq $PANICSTKSIZE, %rsp
3327 0: subq $REGSIZE, %rsp
3328 /*
3329 * Now that we've got everything set up, store the register values as
3330 * they were when we entered vpanic() to the designated location in
3331 * the regs structure we allocated on the stack.
3332 */
3333 movq 0x0(%rbx), %rcx
3334 movq %rcx, REGOFF_RDI(%rsp)
3335 movq 0x8(%rbx), %rcx
3336 movq %rcx, REGOFF_RSI(%rsp)
3337 movq 0x10(%rbx), %rcx
3338 movq %rcx, REGOFF_RDX(%rsp)
3339 movq 0x18(%rbx), %rcx
3340 movq %rcx, REGOFF_RCX(%rsp)
3341 movq 0x20(%rbx), %rcx
3342
3343 movq %rcx, REGOFF_R8(%rsp)
3344 movq 0x28(%rbx), %rcx
3345 movq %rcx, REGOFF_R9(%rsp)
3346 movq 0x30(%rbx), %rcx
3347 movq %rcx, REGOFF_RAX(%rsp)
3348 movq 0x38(%rbx), %rcx
3349 movq %rcx, REGOFF_RBX(%rsp)
3350 movq 0x58(%rbx), %rcx
3351
3352 movq %rcx, REGOFF_RBP(%rsp)
3353 movq 0x40(%rbx), %rcx
3354 movq %rcx, REGOFF_R10(%rsp)
3355 movq 0x48(%rbx), %rcx
3356 movq %rcx, REGOFF_R11(%rsp)
3357 movq %r12, REGOFF_R12(%rsp)
3358
3359 movq %r13, REGOFF_R13(%rsp)
3360 movq %r14, REGOFF_R14(%rsp)
3361 movq %r15, REGOFF_R15(%rsp)
3362
3363 xorl %ecx, %ecx
3364 movw %ds, %cx
3365 movq %rcx, REGOFF_DS(%rsp)
3366 movw %es, %cx
3367 movq %rcx, REGOFF_ES(%rsp)
3368 movw %fs, %cx
3369 movq %rcx, REGOFF_FS(%rsp)
3370 movw %gs, %cx
3371 movq %rcx, REGOFF_GS(%rsp)
3372
3373 movq $0, REGOFF_TRAPNO(%rsp)
3374
3375 movq $0, REGOFF_ERR(%rsp)
3376 leaq vpanic(%rip), %rcx
3377 movq %rcx, REGOFF_RIP(%rsp)
3378 movw %cs, %cx
3379 movzwq %cx, %rcx
3380 movq %rcx, REGOFF_CS(%rsp)
3381 movq 0x50(%rbx), %rcx
3382 movq %rcx, REGOFF_RFL(%rsp)
3383 movq %rbx, %rcx
3384 addq $0x60, %rcx
3385 movq %rcx, REGOFF_RSP(%rsp)
3386 movw %ss, %cx
3387 movzwq %cx, %rcx
3388 movq %rcx, REGOFF_SS(%rsp)
3389
3390 /*
3391 * panicsys(format, alist, rp, on_panic_stack)
3392 */
3393 movq REGOFF_RDI(%rsp), %rdi /* format */
3394 movq REGOFF_RSI(%rsp), %rsi /* alist */
3395 movq %rsp, %rdx /* struct regs */
3396 movl %r11d, %ecx /* on_panic_stack */
3397 call panicsys
3398 addq $REGSIZE, %rsp
3399 popq %rdi
3400 popq %rsi
3401 popq %rdx
3402 popq %rcx
3403 popq %r8
3404 popq %r9
3405 popq %rax
3406 popq %rbx
3407 popq %r10
3408 popq %r11
3409 popfq
3410 leave
3411 ret
3412 SET_SIZE(vpanic)
3413
3414 ENTRY_NP(dtrace_vpanic) /* Initial stack layout: */
3415
3416 pushq %rbp /* | %rip | 0x60 */
3417 movq %rsp, %rbp /* | %rbp | 0x58 */
3418 pushfq /* | rfl | 0x50 */
3419 pushq %r11 /* | %r11 | 0x48 */
3420 pushq %r10 /* | %r10 | 0x40 */
3421 pushq %rbx /* | %rbx | 0x38 */
3422 pushq %rax /* | %rax | 0x30 */
3423 pushq %r9 /* | %r9 | 0x28 */
3424 pushq %r8 /* | %r8 | 0x20 */
3425 pushq %rcx /* | %rcx | 0x18 */
3426 pushq %rdx /* | %rdx | 0x10 */
3427 pushq %rsi /* | %rsi | 0x8 alist */
3428 pushq %rdi /* | %rdi | 0x0 format */
3429
3430 movq %rsp, %rbx /* %rbx = current %rsp */
3431
3432 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */
3433 call dtrace_panic_trigger /* %eax = dtrace_panic_trigger() */
3434 jmp vpanic_common
3435
3436 SET_SIZE(dtrace_vpanic)
3437
3438 #elif defined(__i386)
3439
3440 ENTRY_NP(vpanic) / Initial stack layout:
3441
3442 pushl %ebp / | %eip | 20
3443 movl %esp, %ebp / | %ebp | 16
3444 pushl %eax / | %eax | 12
3445 pushl %ebx / | %ebx | 8
3446 pushl %ecx / | %ecx | 4
3447 pushl %edx / | %edx | 0
3448
3449 movl %esp, %ebx / %ebx = current stack pointer
3450
3451 lea panic_quiesce, %eax / %eax = &panic_quiesce
3452 pushl %eax / push &panic_quiesce
3453 call panic_trigger / %eax = panic_trigger()
3454 addl $4, %esp / reset stack pointer
3455
3456 vpanic_common:
3457 cmpl $0, %eax / if (%eax == 0)
3458 je 0f / goto 0f;
3459
3460 /*
3461 * If panic_trigger() was successful, we are the first to initiate a
3462 * panic: we now switch to the reserved panic_stack before continuing.
3463 */
3464 lea panic_stack, %esp / %esp = panic_stack
3465 addl $PANICSTKSIZE, %esp / %esp += PANICSTKSIZE
3466
3467 0: subl $REGSIZE, %esp / allocate struct regs
3468
3469 /*
3470 * Now that we've got everything set up, store the register values as
3471 * they were when we entered vpanic() to the designated location in
3472 * the regs structure we allocated on the stack.
3473 */
3474 #if !defined(__GNUC_AS__)
3475 movw %gs, %edx
3476 movl %edx, REGOFF_GS(%esp)
3477 movw %fs, %edx
3478 movl %edx, REGOFF_FS(%esp)
3479 movw %es, %edx
3480 movl %edx, REGOFF_ES(%esp)
3481 movw %ds, %edx
3482 movl %edx, REGOFF_DS(%esp)
3483 #else /* __GNUC_AS__ */
3484 mov %gs, %edx
3485 mov %edx, REGOFF_GS(%esp)
3486 mov %fs, %edx
3487 mov %edx, REGOFF_FS(%esp)
3488 mov %es, %edx
3489 mov %edx, REGOFF_ES(%esp)
3490 mov %ds, %edx
3491 mov %edx, REGOFF_DS(%esp)
3492 #endif /* __GNUC_AS__ */
3493 movl %edi, REGOFF_EDI(%esp)
3494 movl %esi, REGOFF_ESI(%esp)
3495 movl 16(%ebx), %ecx
3496 movl %ecx, REGOFF_EBP(%esp)
3497 movl %ebx, %ecx
3498 addl $20, %ecx
3499 movl %ecx, REGOFF_ESP(%esp)
3500 movl 8(%ebx), %ecx
3501 movl %ecx, REGOFF_EBX(%esp)
3502 movl 0(%ebx), %ecx
3503 movl %ecx, REGOFF_EDX(%esp)
3504 movl 4(%ebx), %ecx
3505 movl %ecx, REGOFF_ECX(%esp)
3506 movl 12(%ebx), %ecx
3507 movl %ecx, REGOFF_EAX(%esp)
3508 movl $0, REGOFF_TRAPNO(%esp)
3509 movl $0, REGOFF_ERR(%esp)
3510 lea vpanic, %ecx
3511 movl %ecx, REGOFF_EIP(%esp)
3512 #if !defined(__GNUC_AS__)
3513 movw %cs, %edx
3514 #else /* __GNUC_AS__ */
3515 mov %cs, %edx
3516 #endif /* __GNUC_AS__ */
3517 movl %edx, REGOFF_CS(%esp)
3518 pushfl
3519 popl %ecx
3520 #if defined(__xpv)
3521 /*
3522 * Synthesize the PS_IE bit from the event mask bit
3523 */
3524 CURTHREAD(%edx)
3525 KPREEMPT_DISABLE(%edx)
3526 EVENT_MASK_TO_IE(%edx, %ecx)
3527 CURTHREAD(%edx)
3528 KPREEMPT_ENABLE_NOKP(%edx)
3529 #endif
3530 movl %ecx, REGOFF_EFL(%esp)
3531 movl $0, REGOFF_UESP(%esp)
3532 #if !defined(__GNUC_AS__)
3533 movw %ss, %edx
3534 #else /* __GNUC_AS__ */
3535 mov %ss, %edx
3536 #endif /* __GNUC_AS__ */
3537 movl %edx, REGOFF_SS(%esp)
3538
3539 movl %esp, %ecx / %ecx = ®s
3540 pushl %eax / push on_panic_stack
3541 pushl %ecx / push ®s
3542 movl 12(%ebp), %ecx / %ecx = alist
3543 pushl %ecx / push alist
3544 movl 8(%ebp), %ecx / %ecx = format
3545 pushl %ecx / push format
3546 call panicsys / panicsys();
3547 addl $16, %esp / pop arguments
3548
3549 addl $REGSIZE, %esp
3550 popl %edx
3551 popl %ecx
3552 popl %ebx
3553 popl %eax
3554 leave
3555 ret
3556 SET_SIZE(vpanic)
3557
3558 ENTRY_NP(dtrace_vpanic) / Initial stack layout:
3559
3560 pushl %ebp / | %eip | 20
3561 movl %esp, %ebp / | %ebp | 16
3562 pushl %eax / | %eax | 12
3563 pushl %ebx / | %ebx | 8
3564 pushl %ecx / | %ecx | 4
3565 pushl %edx / | %edx | 0
3566
3567 movl %esp, %ebx / %ebx = current stack pointer
3568
3569 lea panic_quiesce, %eax / %eax = &panic_quiesce
3570 pushl %eax / push &panic_quiesce
3571 call dtrace_panic_trigger / %eax = dtrace_panic_trigger()
3572 addl $4, %esp / reset stack pointer
3573 jmp vpanic_common / jump back to common code
3574
3575 SET_SIZE(dtrace_vpanic)
3576
3577 #endif /* __i386 */
3578 #endif /* __lint */
3579
3580 #if defined(__lint)
3581
3582 void
3583 hres_tick(void)
3584 {}
3585
3586 int64_t timedelta;
3587 hrtime_t hres_last_tick;
3588 volatile timestruc_t hrestime;
3589 int64_t hrestime_adj;
3590 volatile int hres_lock;
3591 hrtime_t hrtime_base;
3592
3593 #else /* __lint */
3594
3595 DGDEF3(hrestime, _MUL(2, CLONGSIZE), 8)
3596 .NWORD 0, 0
3597
3598 DGDEF3(hrestime_adj, 8, 8)
3599 .long 0, 0
3600
3601 DGDEF3(hres_last_tick, 8, 8)
3602 .long 0, 0
3603
3604 DGDEF3(timedelta, 8, 8)
3605 .long 0, 0
3606
3607 DGDEF3(hres_lock, 4, 8)
3608 .long 0
3609
3610 /*
3611 * initialized to a non zero value to make pc_gethrtime()
3612 * work correctly even before clock is initialized
3613 */
3614 DGDEF3(hrtime_base, 8, 8)
3615 .long _MUL(NSEC_PER_CLOCK_TICK, 6), 0
3616
3617 DGDEF3(adj_shift, 4, 4)
3618 .long ADJ_SHIFT
3619
3620 #if defined(__amd64)
3621
3622 ENTRY_NP(hres_tick)
3623 pushq %rbp
3624 movq %rsp, %rbp
3625
3626 /*
3627 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
3628 * hres_last_tick can only be modified while holding CLOCK_LOCK).
3629 * At worst, performing this now instead of under CLOCK_LOCK may
3630 * introduce some jitter in pc_gethrestime().
3631 */
3632 call *gethrtimef(%rip)
3633 movq %rax, %r8
3634
3635 leaq hres_lock(%rip), %rax
3636 movb $-1, %dl
3637 .CL1:
3638 xchgb %dl, (%rax)
3639 testb %dl, %dl
3640 jz .CL3 /* got it */
3641 .CL2:
3642 cmpb $0, (%rax) /* possible to get lock? */
3643 pause
3644 jne .CL2
3645 jmp .CL1 /* yes, try again */
3646 .CL3:
3647 /*
3648 * compute the interval since last time hres_tick was called
3649 * and adjust hrtime_base and hrestime accordingly
3650 * hrtime_base is an 8 byte value (in nsec), hrestime is
3651 * a timestruc_t (sec, nsec)
3652 */
3653 leaq hres_last_tick(%rip), %rax
3654 movq %r8, %r11
3655 subq (%rax), %r8
3656 addq %r8, hrtime_base(%rip) /* add interval to hrtime_base */
3657 addq %r8, hrestime+8(%rip) /* add interval to hrestime.tv_nsec */
3658 /*
3659 * Now that we have CLOCK_LOCK, we can update hres_last_tick
3660 */
3661 movq %r11, (%rax)
3662
3663 call __adj_hrestime
3664
3665 /*
3666 * release the hres_lock
3667 */
3668 incl hres_lock(%rip)
3669 leave
3670 ret
3671 SET_SIZE(hres_tick)
3672
3673 #elif defined(__i386)
3674
3675 ENTRY_NP(hres_tick)
3676 pushl %ebp
3677 movl %esp, %ebp
3678 pushl %esi
3679 pushl %ebx
3680
3681 /*
3682 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
3683 * hres_last_tick can only be modified while holding CLOCK_LOCK).
3684 * At worst, performing this now instead of under CLOCK_LOCK may
3685 * introduce some jitter in pc_gethrestime().
3686 */
3687 call *gethrtimef
3688 movl %eax, %ebx
3689 movl %edx, %esi
3690
3691 movl $hres_lock, %eax
3692 movl $-1, %edx
3693 .CL1:
3694 xchgb %dl, (%eax)
3695 testb %dl, %dl
3696 jz .CL3 / got it
3697 .CL2:
3698 cmpb $0, (%eax) / possible to get lock?
3699 pause
3700 jne .CL2
3701 jmp .CL1 / yes, try again
3702 .CL3:
3703 /*
3704 * compute the interval since last time hres_tick was called
3705 * and adjust hrtime_base and hrestime accordingly
3706 * hrtime_base is an 8 byte value (in nsec), hrestime is
3707 * timestruc_t (sec, nsec)
3708 */
3709
3710 lea hres_last_tick, %eax
3711
3712 movl %ebx, %edx
3713 movl %esi, %ecx
3714
3715 subl (%eax), %edx
3716 sbbl 4(%eax), %ecx
3717
3718 addl %edx, hrtime_base / add interval to hrtime_base
3719 adcl %ecx, hrtime_base+4
3720
3721 addl %edx, hrestime+4 / add interval to hrestime.tv_nsec
3722
3723 /
3724 / Now that we have CLOCK_LOCK, we can update hres_last_tick.
3725 /
3726 movl %ebx, (%eax)
3727 movl %esi, 4(%eax)
3728
3729 / get hrestime at this moment. used as base for pc_gethrestime
3730 /
3731 / Apply adjustment, if any
3732 /
3733 / #define HRES_ADJ (NSEC_PER_CLOCK_TICK >> ADJ_SHIFT)
3734 / (max_hres_adj)
3735 /
3736 / void
3737 / adj_hrestime()
3738 / {
3739 / long long adj;
3740 /
3741 / if (hrestime_adj == 0)
3742 / adj = 0;
3743 / else if (hrestime_adj > 0) {
3744 / if (hrestime_adj < HRES_ADJ)
3745 / adj = hrestime_adj;
3746 / else
3747 / adj = HRES_ADJ;
3748 / }
3749 / else {
3750 / if (hrestime_adj < -(HRES_ADJ))
3751 / adj = -(HRES_ADJ);
3752 / else
3753 / adj = hrestime_adj;
3754 / }
3755 /
3756 / timedelta -= adj;
3757 / hrestime_adj = timedelta;
3758 / hrestime.tv_nsec += adj;
3759 /
3760 / while (hrestime.tv_nsec >= NANOSEC) {
3761 / one_sec++;
3762 / hrestime.tv_sec++;
3763 / hrestime.tv_nsec -= NANOSEC;
3764 / }
3765 / }
3766 __adj_hrestime:
3767 movl hrestime_adj, %esi / if (hrestime_adj == 0)
3768 movl hrestime_adj+4, %edx
3769 andl %esi, %esi
3770 jne .CL4 / no
3771 andl %edx, %edx
3772 jne .CL4 / no
3773 subl %ecx, %ecx / yes, adj = 0;
3774 subl %edx, %edx
3775 jmp .CL5
3776 .CL4:
3777 subl %ecx, %ecx
3778 subl %eax, %eax
3779 subl %esi, %ecx
3780 sbbl %edx, %eax
3781 andl %eax, %eax / if (hrestime_adj > 0)
3782 jge .CL6
3783
3784 / In the following comments, HRES_ADJ is used, while in the code
3785 / max_hres_adj is used.
3786 /
3787 / The test for "hrestime_adj < HRES_ADJ" is complicated because
3788 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely
3789 / on the logical equivalence of:
3790 /
3791 / !(hrestime_adj < HRES_ADJ)
3792 /
3793 / and the two step sequence:
3794 /
3795 / (HRES_ADJ - lsw(hrestime_adj)) generates a Borrow/Carry
3796 /
3797 / which computes whether or not the least significant 32-bits
3798 / of hrestime_adj is greater than HRES_ADJ, followed by:
3799 /
3800 / Previous Borrow/Carry + -1 + msw(hrestime_adj) generates a Carry
3801 /
3802 / which generates a carry whenever step 1 is true or the most
3803 / significant long of the longlong hrestime_adj is non-zero.
3804
3805 movl max_hres_adj, %ecx / hrestime_adj is positive
3806 subl %esi, %ecx
3807 movl %edx, %eax
3808 adcl $-1, %eax
3809 jnc .CL7
3810 movl max_hres_adj, %ecx / adj = HRES_ADJ;
3811 subl %edx, %edx
3812 jmp .CL5
3813
3814 / The following computation is similar to the one above.
3815 /
3816 / The test for "hrestime_adj < -(HRES_ADJ)" is complicated because
3817 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely
3818 / on the logical equivalence of:
3819 /
3820 / (hrestime_adj > -HRES_ADJ)
3821 /
3822 / and the two step sequence:
3823 /
3824 / (HRES_ADJ + lsw(hrestime_adj)) generates a Carry
3825 /
3826 / which means the least significant 32-bits of hrestime_adj is
3827 / greater than -HRES_ADJ, followed by:
3828 /
3829 / Previous Carry + 0 + msw(hrestime_adj) generates a Carry
3830 /
3831 / which generates a carry only when step 1 is true and the most
3832 / significant long of the longlong hrestime_adj is -1.
3833
3834 .CL6: / hrestime_adj is negative
3835 movl %esi, %ecx
3836 addl max_hres_adj, %ecx
3837 movl %edx, %eax
3838 adcl $0, %eax
3839 jc .CL7
3840 xor %ecx, %ecx
3841 subl max_hres_adj, %ecx / adj = -(HRES_ADJ);
3842 movl $-1, %edx
3843 jmp .CL5
3844 .CL7:
3845 movl %esi, %ecx / adj = hrestime_adj;
3846 .CL5:
3847 movl timedelta, %esi
3848 subl %ecx, %esi
3849 movl timedelta+4, %eax
3850 sbbl %edx, %eax
3851 movl %esi, timedelta
3852 movl %eax, timedelta+4 / timedelta -= adj;
3853 movl %esi, hrestime_adj
3854 movl %eax, hrestime_adj+4 / hrestime_adj = timedelta;
3855 addl hrestime+4, %ecx
3856
3857 movl %ecx, %eax / eax = tv_nsec
3858 1:
3859 cmpl $NANOSEC, %eax / if ((unsigned long)tv_nsec >= NANOSEC)
3860 jb .CL8 / no
3861 incl one_sec / yes, one_sec++;
3862 incl hrestime / hrestime.tv_sec++;
3863 addl $-NANOSEC, %eax / tv_nsec -= NANOSEC
3864 jmp 1b / check for more seconds
3865
3866 .CL8:
3867 movl %eax, hrestime+4 / store final into hrestime.tv_nsec
3868 incl hres_lock / release the hres_lock
3869
3870 popl %ebx
3871 popl %esi
3872 leave
3873 ret
3874 SET_SIZE(hres_tick)
3875
3876 #endif /* __i386 */
3877 #endif /* __lint */
3878
3879 /*
3880 * void prefetch_smap_w(void *)
3881 *
3882 * Prefetch ahead within a linear list of smap structures.
3883 * Not implemented for ia32. Stub for compatibility.
3884 */
3885
3886 #if defined(__lint)
3887
3888 /*ARGSUSED*/
3889 void prefetch_smap_w(void *smp)
3890 {}
3891
3892 #else /* __lint */
3893
3894 ENTRY(prefetch_smap_w)
3895 rep; ret /* use 2 byte return instruction when branch target */
3896 /* AMD Software Optimization Guide - Section 6.2 */
3897 SET_SIZE(prefetch_smap_w)
3898
3899 #endif /* __lint */
3900
3901 /*
3902 * prefetch_page_r(page_t *)
3903 * issue prefetch instructions for a page_t
3904 */
3905 #if defined(__lint)
3906
3907 /*ARGSUSED*/
3908 void
3909 prefetch_page_r(void *pp)
3910 {}
3911
3912 #else /* __lint */
3913
3914 ENTRY(prefetch_page_r)
3915 rep; ret /* use 2 byte return instruction when branch target */
3916 /* AMD Software Optimization Guide - Section 6.2 */
3917 SET_SIZE(prefetch_page_r)
3918
3919 #endif /* __lint */
3920
3921 #if defined(__lint)
3922
3923 /*ARGSUSED*/
3924 int
3925 bcmp(const void *s1, const void *s2, size_t count)
3926 { return (0); }
3927
3928 #else /* __lint */
3929
3930 #if defined(__amd64)
3931
3932 ENTRY(bcmp)
3933 pushq %rbp
3934 movq %rsp, %rbp
3935 #ifdef DEBUG
3936 testq %rdx,%rdx
3937 je 1f
3938 movq postbootkernelbase(%rip), %r11
3939 cmpq %r11, %rdi
3940 jb 0f
3941 cmpq %r11, %rsi
3942 jnb 1f
3943 0: leaq .bcmp_panic_msg(%rip), %rdi
3944 xorl %eax, %eax
3945 call panic
3946 1:
3947 #endif /* DEBUG */
3948 call memcmp
3949 testl %eax, %eax
3950 setne %dl
3951 leave
3952 movzbl %dl, %eax
3953 ret
3954 SET_SIZE(bcmp)
3955
3956 #elif defined(__i386)
3957
3958 #define ARG_S1 8
3959 #define ARG_S2 12
3960 #define ARG_LENGTH 16
3961
3962 ENTRY(bcmp)
3963 pushl %ebp
3964 movl %esp, %ebp / create new stack frame
3965 #ifdef DEBUG
3966 cmpl $0, ARG_LENGTH(%ebp)
3967 je 1f
3968 movl postbootkernelbase, %eax
3969 cmpl %eax, ARG_S1(%ebp)
3970 jb 0f
3971 cmpl %eax, ARG_S2(%ebp)
3972 jnb 1f
3973 0: pushl $.bcmp_panic_msg
3974 call panic
3975 1:
3976 #endif /* DEBUG */
3977
3978 pushl %edi / save register variable
3979 movl ARG_S1(%ebp), %eax / %eax = address of string 1
3980 movl ARG_S2(%ebp), %ecx / %ecx = address of string 2
3981 cmpl %eax, %ecx / if the same string
3982 je .equal / goto .equal
3983 movl ARG_LENGTH(%ebp), %edi / %edi = length in bytes
3984 cmpl $4, %edi / if %edi < 4
3985 jb .byte_check / goto .byte_check
3986 .align 4
3987 .word_loop:
3988 movl (%ecx), %edx / move 1 word from (%ecx) to %edx
3989 leal -4(%edi), %edi / %edi -= 4
3990 cmpl (%eax), %edx / compare 1 word from (%eax) with %edx
3991 jne .word_not_equal / if not equal, goto .word_not_equal
3992 leal 4(%ecx), %ecx / %ecx += 4 (next word)
3993 leal 4(%eax), %eax / %eax += 4 (next word)
3994 cmpl $4, %edi / if %edi >= 4
3995 jae .word_loop / goto .word_loop
3996 .byte_check:
3997 cmpl $0, %edi / if %edi == 0
3998 je .equal / goto .equal
3999 jmp .byte_loop / goto .byte_loop (checks in bytes)
4000 .word_not_equal:
4001 leal 4(%edi), %edi / %edi += 4 (post-decremented)
4002 .align 4
4003 .byte_loop:
4004 movb (%ecx), %dl / move 1 byte from (%ecx) to %dl
4005 cmpb %dl, (%eax) / compare %dl with 1 byte from (%eax)
4006 jne .not_equal / if not equal, goto .not_equal
4007 incl %ecx / %ecx++ (next byte)
4008 incl %eax / %eax++ (next byte)
4009 decl %edi / %edi--
4010 jnz .byte_loop / if not zero, goto .byte_loop
4011 .equal:
4012 xorl %eax, %eax / %eax = 0
4013 popl %edi / restore register variable
4014 leave / restore old stack frame
4015 ret / return (NULL)
4016 .align 4
4017 .not_equal:
4018 movl $1, %eax / return 1
4019 popl %edi / restore register variable
4020 leave / restore old stack frame
4021 ret / return (NULL)
4022 SET_SIZE(bcmp)
4023
4024 #endif /* __i386 */
4025
4026 #ifdef DEBUG
4027 .text
4028 .bcmp_panic_msg:
4029 .string "bcmp: arguments below kernelbase"
4030 #endif /* DEBUG */
4031
4032 #endif /* __lint */
4033
4034 #if defined(__lint)
4035
4036 uint_t
4037 bsrw_insn(uint16_t mask)
4038 {
4039 uint_t index = sizeof (mask) * NBBY - 1;
4040
4041 while ((mask & (1 << index)) == 0)
4042 index--;
4043 return (index);
4044 }
4045
4046 #else /* __lint */
4047
4048 #if defined(__amd64)
4049
4050 ENTRY_NP(bsrw_insn)
4051 xorl %eax, %eax
4052 bsrw %di, %ax
4053 ret
4054 SET_SIZE(bsrw_insn)
4055
4056 #elif defined(__i386)
4057
4058 ENTRY_NP(bsrw_insn)
4059 movw 4(%esp), %cx
4060 xorl %eax, %eax
4061 bsrw %cx, %ax
4062 ret
4063 SET_SIZE(bsrw_insn)
4064
4065 #endif /* __i386 */
4066 #endif /* __lint */
4067
4068 #if defined(__lint)
4069
4070 uint_t
4071 atomic_btr32(uint32_t *pending, uint_t pil)
4072 {
4073 return (*pending &= ~(1 << pil));
4074 }
4075
4076 #else /* __lint */
4077
4078 #if defined(__i386)
4079
4080 ENTRY_NP(atomic_btr32)
4081 movl 4(%esp), %ecx
4082 movl 8(%esp), %edx
4083 xorl %eax, %eax
4084 lock
4085 btrl %edx, (%ecx)
4086 setc %al
4087 ret
4088 SET_SIZE(atomic_btr32)
4089
4090 #endif /* __i386 */
4091 #endif /* __lint */
4092
4093 #if defined(__lint)
4094
4095 /*ARGSUSED*/
4096 void
4097 switch_sp_and_call(void *newsp, void (*func)(uint_t, uint_t), uint_t arg1,
4098 uint_t arg2)
4099 {}
4100
4101 #else /* __lint */
4102
4103 #if defined(__amd64)
4104
4105 ENTRY_NP(switch_sp_and_call)
4106 pushq %rbp
4107 movq %rsp, %rbp /* set up stack frame */
4108 movq %rdi, %rsp /* switch stack pointer */
4109 movq %rdx, %rdi /* pass func arg 1 */
4110 movq %rsi, %r11 /* save function to call */
4111 movq %rcx, %rsi /* pass func arg 2 */
4112 call *%r11 /* call function */
4113 leave /* restore stack */
4114 ret
4115 SET_SIZE(switch_sp_and_call)
4116
4117 #elif defined(__i386)
4118
4119 ENTRY_NP(switch_sp_and_call)
4120 pushl %ebp
4121 mov %esp, %ebp /* set up stack frame */
4122 movl 8(%ebp), %esp /* switch stack pointer */
4123 pushl 20(%ebp) /* push func arg 2 */
4124 pushl 16(%ebp) /* push func arg 1 */
4125 call *12(%ebp) /* call function */
4126 addl $8, %esp /* pop arguments */
4127 leave /* restore stack */
4128 ret
4129 SET_SIZE(switch_sp_and_call)
4130
4131 #endif /* __i386 */
4132 #endif /* __lint */
4133
4134 #if defined(__lint)
4135
4136 void
4137 kmdb_enter(void)
4138 {}
4139
4140 #else /* __lint */
4141
4142 #if defined(__amd64)
4143
4144 ENTRY_NP(kmdb_enter)
4145 pushq %rbp
4146 movq %rsp, %rbp
4147
4148 /*
4149 * Save flags, do a 'cli' then return the saved flags
4150 */
4151 call intr_clear
4152
4153 int $T_DBGENTR
4154
4155 /*
4156 * Restore the saved flags
4157 */
4158 movq %rax, %rdi
4159 call intr_restore
4160
4161 leave
4162 ret
4163 SET_SIZE(kmdb_enter)
4164
4165 #elif defined(__i386)
4166
4167 ENTRY_NP(kmdb_enter)
4168 pushl %ebp
4169 movl %esp, %ebp
4170
4171 /*
4172 * Save flags, do a 'cli' then return the saved flags
4173 */
4174 call intr_clear
4175
4176 int $T_DBGENTR
4177
4178 /*
4179 * Restore the saved flags
4180 */
4181 pushl %eax
4182 call intr_restore
4183 addl $4, %esp
4184
4185 leave
4186 ret
4187 SET_SIZE(kmdb_enter)
4188
4189 #endif /* __i386 */
4190 #endif /* __lint */
4191
4192 #if defined(__lint)
4193
4194 void
4195 return_instr(void)
4196 {}
4197
4198 #else /* __lint */
4199
4200 ENTRY_NP(return_instr)
4201 rep; ret /* use 2 byte instruction when branch target */
4202 /* AMD Software Optimization Guide - Section 6.2 */
4203 SET_SIZE(return_instr)
4204
4205 #endif /* __lint */
4206
4207 #if defined(__lint)
4208
4209 ulong_t
4210 getflags(void)
4211 {
4212 return (0);
4213 }
4214
4215 #else /* __lint */
4216
4217 #if defined(__amd64)
4218
4219 ENTRY(getflags)
4220 pushfq
4221 popq %rax
4222 #if defined(__xpv)
4223 CURTHREAD(%rdi)
4224 KPREEMPT_DISABLE(%rdi)
4225 /*
4226 * Synthesize the PS_IE bit from the event mask bit
4227 */
4228 CURVCPU(%r11)
4229 andq $_BITNOT(PS_IE), %rax
4230 XEN_TEST_UPCALL_MASK(%r11)
4231 jnz 1f
4232 orq $PS_IE, %rax
4233 1:
4234 KPREEMPT_ENABLE_NOKP(%rdi)
4235 #endif
4236 ret
4237 SET_SIZE(getflags)
4238
4239 #elif defined(__i386)
4240
4241 ENTRY(getflags)
4242 pushfl
4243 popl %eax
4244 #if defined(__xpv)
4245 CURTHREAD(%ecx)
4246 KPREEMPT_DISABLE(%ecx)
4247 /*
4248 * Synthesize the PS_IE bit from the event mask bit
4249 */
4250 CURVCPU(%edx)
4251 andl $_BITNOT(PS_IE), %eax
4252 XEN_TEST_UPCALL_MASK(%edx)
4253 jnz 1f
4254 orl $PS_IE, %eax
4255 1:
4256 KPREEMPT_ENABLE_NOKP(%ecx)
4257 #endif
4258 ret
4259 SET_SIZE(getflags)
4260
4261 #endif /* __i386 */
4262
4263 #endif /* __lint */
4264
4265 #if defined(__lint)
4266
4267 ftrace_icookie_t
4268 ftrace_interrupt_disable(void)
4269 { return (0); }
4270
4271 #else /* __lint */
4272
4273 #if defined(__amd64)
4274
4275 ENTRY(ftrace_interrupt_disable)
4276 pushfq
4277 popq %rax
4278 CLI(%rdx)
4279 ret
4280 SET_SIZE(ftrace_interrupt_disable)
4281
4282 #elif defined(__i386)
4283
4284 ENTRY(ftrace_interrupt_disable)
4285 pushfl
4286 popl %eax
4287 CLI(%edx)
4288 ret
4289 SET_SIZE(ftrace_interrupt_disable)
4290
4291 #endif /* __i386 */
4292 #endif /* __lint */
4293
4294 #if defined(__lint)
4295
4296 /*ARGSUSED*/
4297 void
4298 ftrace_interrupt_enable(ftrace_icookie_t cookie)
4299 {}
4300
4301 #else /* __lint */
4302
4303 #if defined(__amd64)
4304
4305 ENTRY(ftrace_interrupt_enable)
4306 pushq %rdi
4307 popfq
4308 ret
4309 SET_SIZE(ftrace_interrupt_enable)
4310
4311 #elif defined(__i386)
4312
4313 ENTRY(ftrace_interrupt_enable)
4314 movl 4(%esp), %eax
4315 pushl %eax
4316 popfl
4317 ret
4318 SET_SIZE(ftrace_interrupt_enable)
4319
4320 #endif /* __i386 */
4321 #endif /* __lint */
4322
4323 #if defined (__lint)
4324
4325 /*ARGSUSED*/
4326 void
4327 clflush_insn(caddr_t addr)
4328 {}
4329
4330 #else /* __lint */
4331
4332 #if defined (__amd64)
4333 ENTRY(clflush_insn)
4334 clflush (%rdi)
4335 ret
4336 SET_SIZE(clflush_insn)
4337 #elif defined (__i386)
4338 ENTRY(clflush_insn)
4339 movl 4(%esp), %eax
4340 clflush (%eax)
4341 ret
4342 SET_SIZE(clflush_insn)
4343
4344 #endif /* __i386 */
4345 #endif /* __lint */
4346
4347 #if defined (__lint)
4348 /*ARGSUSED*/
4349 void
4350 mfence_insn(void)
4351 {}
4352
4353 #else /* __lint */
4354
4355 #if defined (__amd64)
4356 ENTRY(mfence_insn)
4357 mfence
4358 ret
4359 SET_SIZE(mfence_insn)
4360 #elif defined (__i386)
4361 ENTRY(mfence_insn)
4362 mfence
4363 ret
4364 SET_SIZE(mfence_insn)
4365
4366 #endif /* __i386 */
4367 #endif /* __lint */
4368
4369 /*
4370 * VMware implements an I/O port that programs can query to detect if software
4371 * is running in a VMware hypervisor. This hypervisor port behaves differently
4372 * depending on magic values in certain registers and modifies some registers
4373 * as a side effect.
4374 *
4375 * References: http://kb.vmware.com/kb/1009458
4376 */
4377
4378 #if defined(__lint)
4379
4380 /* ARGSUSED */
4381 void
4382 vmware_port(int cmd, uint32_t *regs) { return; }
4383
4384 #else
4385
4386 #if defined(__amd64)
4387
4388 ENTRY(vmware_port)
4389 pushq %rbx
4390 movl $VMWARE_HVMAGIC, %eax
4391 movl $0xffffffff, %ebx
4392 movl %edi, %ecx
4393 movl $VMWARE_HVPORT, %edx
4394 inl (%dx)
4395 movl %eax, (%rsi)
4396 movl %ebx, 4(%rsi)
4397 movl %ecx, 8(%rsi)
4398 movl %edx, 12(%rsi)
4399 popq %rbx
4400 ret
4401 SET_SIZE(vmware_port)
4402
4403 #elif defined(__i386)
4404
4405 ENTRY(vmware_port)
4406 pushl %ebx
4407 pushl %esi
4408 movl $VMWARE_HVMAGIC, %eax
4409 movl $0xffffffff, %ebx
4410 movl 12(%esp), %ecx
4411 movl $VMWARE_HVPORT, %edx
4412 inl (%dx)
4413 movl 16(%esp), %esi
4414 movl %eax, (%esi)
4415 movl %ebx, 4(%esi)
4416 movl %ecx, 8(%esi)
4417 movl %edx, 12(%esi)
4418 popl %esi
4419 popl %ebx
4420 ret
4421 SET_SIZE(vmware_port)
4422
4423 #endif /* __i386 */
4424 #endif /* __lint */