1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2014 Nexenta Systems, Inc. All rights reserved.
25 * Copyright (c) 2014 by Delphix. All rights reserved.
26 */
27
28 /*
29 * Copyright (c) 1990, 1991 UNIX System Laboratories, Inc.
30 * Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T
31 * All Rights Reserved
32 */
33
34 /*
35 * Copyright (c) 2009, Intel Corporation.
36 * All rights reserved.
37 */
38
39 /*
40 * General assembly language routines.
41 * It is the intent of this file to contain routines that are
42 * independent of the specific kernel architecture, and those that are
43 * common across kernel architectures.
44 * As architectures diverge, and implementations of specific
45 * architecture-dependent routines change, the routines should be moved
46 * from this file into the respective ../`arch -k`/subr.s file.
47 */
48
49 #include <sys/asm_linkage.h>
50 #include <sys/asm_misc.h>
51 #include <sys/panic.h>
52 #include <sys/ontrap.h>
53 #include <sys/regset.h>
54 #include <sys/privregs.h>
55 #include <sys/reboot.h>
56 #include <sys/psw.h>
57 #include <sys/x86_archext.h>
58
59 #if defined(__lint)
60 #include <sys/types.h>
61 #include <sys/systm.h>
62 #include <sys/thread.h>
63 #include <sys/archsystm.h>
64 #include <sys/byteorder.h>
65 #include <sys/dtrace.h>
66 #include <sys/ftrace.h>
67 #else /* __lint */
68 #include "assym.h"
69 #endif /* __lint */
70 #include <sys/dditypes.h>
71
72 /*
73 * on_fault()
74 * Catch lofault faults. Like setjmp except it returns one
75 * if code following causes uncorrectable fault. Turned off
76 * by calling no_fault().
77 */
78
79 #if defined(__lint)
80
81 /* ARGSUSED */
82 int
83 on_fault(label_t *ljb)
84 { return (0); }
85
86 void
87 no_fault(void)
88 {}
89
90 #else /* __lint */
91
92 #if defined(__amd64)
93
94 ENTRY(on_fault)
95 movq %gs:CPU_THREAD, %rsi
96 leaq catch_fault(%rip), %rdx
97 movq %rdi, T_ONFAULT(%rsi) /* jumpbuf in t_onfault */
98 movq %rdx, T_LOFAULT(%rsi) /* catch_fault in t_lofault */
99 jmp setjmp /* let setjmp do the rest */
100
101 catch_fault:
102 movq %gs:CPU_THREAD, %rsi
103 movq T_ONFAULT(%rsi), %rdi /* address of save area */
104 xorl %eax, %eax
105 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */
106 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */
107 jmp longjmp /* let longjmp do the rest */
108 SET_SIZE(on_fault)
109
110 ENTRY(no_fault)
111 movq %gs:CPU_THREAD, %rsi
112 xorl %eax, %eax
113 movq %rax, T_ONFAULT(%rsi) /* turn off onfault */
114 movq %rax, T_LOFAULT(%rsi) /* turn off lofault */
115 ret
116 SET_SIZE(no_fault)
117
118 #elif defined(__i386)
119
120 ENTRY(on_fault)
121 movl %gs:CPU_THREAD, %edx
122 movl 4(%esp), %eax /* jumpbuf address */
123 leal catch_fault, %ecx
124 movl %eax, T_ONFAULT(%edx) /* jumpbuf in t_onfault */
125 movl %ecx, T_LOFAULT(%edx) /* catch_fault in t_lofault */
126 jmp setjmp /* let setjmp do the rest */
127
128 catch_fault:
129 movl %gs:CPU_THREAD, %edx
130 xorl %eax, %eax
131 movl T_ONFAULT(%edx), %ecx /* address of save area */
132 movl %eax, T_ONFAULT(%edx) /* turn off onfault */
133 movl %eax, T_LOFAULT(%edx) /* turn off lofault */
134 pushl %ecx
135 call longjmp /* let longjmp do the rest */
136 SET_SIZE(on_fault)
137
138 ENTRY(no_fault)
139 movl %gs:CPU_THREAD, %edx
140 xorl %eax, %eax
141 movl %eax, T_ONFAULT(%edx) /* turn off onfault */
142 movl %eax, T_LOFAULT(%edx) /* turn off lofault */
143 ret
144 SET_SIZE(no_fault)
145
146 #endif /* __i386 */
147 #endif /* __lint */
148
149 /*
150 * Default trampoline code for on_trap() (see <sys/ontrap.h>). We just
151 * do a longjmp(&curthread->t_ontrap->ot_jmpbuf) if this is ever called.
152 */
153
154 #if defined(lint)
155
156 void
157 on_trap_trampoline(void)
158 {}
159
160 #else /* __lint */
161
162 #if defined(__amd64)
163
164 ENTRY(on_trap_trampoline)
165 movq %gs:CPU_THREAD, %rsi
166 movq T_ONTRAP(%rsi), %rdi
167 addq $OT_JMPBUF, %rdi
168 jmp longjmp
169 SET_SIZE(on_trap_trampoline)
170
171 #elif defined(__i386)
172
173 ENTRY(on_trap_trampoline)
174 movl %gs:CPU_THREAD, %eax
175 movl T_ONTRAP(%eax), %eax
176 addl $OT_JMPBUF, %eax
177 pushl %eax
178 call longjmp
179 SET_SIZE(on_trap_trampoline)
180
181 #endif /* __i386 */
182 #endif /* __lint */
183
184 /*
185 * Push a new element on to the t_ontrap stack. Refer to <sys/ontrap.h> for
186 * more information about the on_trap() mechanism. If the on_trap_data is the
187 * same as the topmost stack element, we just modify that element.
188 */
189 #if defined(lint)
190
191 /*ARGSUSED*/
192 int
193 on_trap(on_trap_data_t *otp, uint_t prot)
194 { return (0); }
195
196 #else /* __lint */
197
198 #if defined(__amd64)
199
200 ENTRY(on_trap)
201 movw %si, OT_PROT(%rdi) /* ot_prot = prot */
202 movw $0, OT_TRAP(%rdi) /* ot_trap = 0 */
203 leaq on_trap_trampoline(%rip), %rdx /* rdx = &on_trap_trampoline */
204 movq %rdx, OT_TRAMPOLINE(%rdi) /* ot_trampoline = rdx */
205 xorl %ecx, %ecx
206 movq %rcx, OT_HANDLE(%rdi) /* ot_handle = NULL */
207 movq %rcx, OT_PAD1(%rdi) /* ot_pad1 = NULL */
208 movq %gs:CPU_THREAD, %rdx /* rdx = curthread */
209 movq T_ONTRAP(%rdx), %rcx /* rcx = curthread->t_ontrap */
210 cmpq %rdi, %rcx /* if (otp == %rcx) */
211 je 0f /* don't modify t_ontrap */
212
213 movq %rcx, OT_PREV(%rdi) /* ot_prev = t_ontrap */
214 movq %rdi, T_ONTRAP(%rdx) /* curthread->t_ontrap = otp */
215
216 0: addq $OT_JMPBUF, %rdi /* &ot_jmpbuf */
217 jmp setjmp
218 SET_SIZE(on_trap)
219
220 #elif defined(__i386)
221
222 ENTRY(on_trap)
223 movl 4(%esp), %eax /* %eax = otp */
224 movl 8(%esp), %edx /* %edx = prot */
225
226 movw %dx, OT_PROT(%eax) /* ot_prot = prot */
227 movw $0, OT_TRAP(%eax) /* ot_trap = 0 */
228 leal on_trap_trampoline, %edx /* %edx = &on_trap_trampoline */
229 movl %edx, OT_TRAMPOLINE(%eax) /* ot_trampoline = %edx */
230 movl $0, OT_HANDLE(%eax) /* ot_handle = NULL */
231 movl $0, OT_PAD1(%eax) /* ot_pad1 = NULL */
232 movl %gs:CPU_THREAD, %edx /* %edx = curthread */
233 movl T_ONTRAP(%edx), %ecx /* %ecx = curthread->t_ontrap */
234 cmpl %eax, %ecx /* if (otp == %ecx) */
235 je 0f /* don't modify t_ontrap */
236
237 movl %ecx, OT_PREV(%eax) /* ot_prev = t_ontrap */
238 movl %eax, T_ONTRAP(%edx) /* curthread->t_ontrap = otp */
239
240 0: addl $OT_JMPBUF, %eax /* %eax = &ot_jmpbuf */
241 movl %eax, 4(%esp) /* put %eax back on the stack */
242 jmp setjmp /* let setjmp do the rest */
243 SET_SIZE(on_trap)
244
245 #endif /* __i386 */
246 #endif /* __lint */
247
248 /*
249 * Setjmp and longjmp implement non-local gotos using state vectors
250 * type label_t.
251 */
252
253 #if defined(__lint)
254
255 /* ARGSUSED */
256 int
257 setjmp(label_t *lp)
258 { return (0); }
259
260 /* ARGSUSED */
261 void
262 longjmp(label_t *lp)
263 {}
264
265 #else /* __lint */
266
267 #if LABEL_PC != 0
268 #error LABEL_PC MUST be defined as 0 for setjmp/longjmp to work as coded
269 #endif /* LABEL_PC != 0 */
270
271 #if defined(__amd64)
272
273 ENTRY(setjmp)
274 movq %rsp, LABEL_SP(%rdi)
275 movq %rbp, LABEL_RBP(%rdi)
276 movq %rbx, LABEL_RBX(%rdi)
277 movq %r12, LABEL_R12(%rdi)
278 movq %r13, LABEL_R13(%rdi)
279 movq %r14, LABEL_R14(%rdi)
280 movq %r15, LABEL_R15(%rdi)
281 movq (%rsp), %rdx /* return address */
282 movq %rdx, (%rdi) /* LABEL_PC is 0 */
283 xorl %eax, %eax /* return 0 */
284 ret
285 SET_SIZE(setjmp)
286
287 ENTRY(longjmp)
288 movq LABEL_SP(%rdi), %rsp
289 movq LABEL_RBP(%rdi), %rbp
290 movq LABEL_RBX(%rdi), %rbx
291 movq LABEL_R12(%rdi), %r12
292 movq LABEL_R13(%rdi), %r13
293 movq LABEL_R14(%rdi), %r14
294 movq LABEL_R15(%rdi), %r15
295 movq (%rdi), %rdx /* return address; LABEL_PC is 0 */
296 movq %rdx, (%rsp)
297 xorl %eax, %eax
298 incl %eax /* return 1 */
299 ret
300 SET_SIZE(longjmp)
301
302 #elif defined(__i386)
303
304 ENTRY(setjmp)
305 movl 4(%esp), %edx /* address of save area */
306 movl %ebp, LABEL_EBP(%edx)
307 movl %ebx, LABEL_EBX(%edx)
308 movl %esi, LABEL_ESI(%edx)
309 movl %edi, LABEL_EDI(%edx)
310 movl %esp, 4(%edx)
311 movl (%esp), %ecx /* %eip (return address) */
312 movl %ecx, (%edx) /* LABEL_PC is 0 */
313 subl %eax, %eax /* return 0 */
314 ret
315 SET_SIZE(setjmp)
316
317 ENTRY(longjmp)
318 movl 4(%esp), %edx /* address of save area */
319 movl LABEL_EBP(%edx), %ebp
320 movl LABEL_EBX(%edx), %ebx
321 movl LABEL_ESI(%edx), %esi
322 movl LABEL_EDI(%edx), %edi
323 movl 4(%edx), %esp
324 movl (%edx), %ecx /* %eip (return addr); LABEL_PC is 0 */
325 movl $1, %eax
326 addl $4, %esp /* pop ret adr */
327 jmp *%ecx /* indirect */
328 SET_SIZE(longjmp)
329
330 #endif /* __i386 */
331 #endif /* __lint */
332
333 /*
334 * if a() calls b() calls caller(),
335 * caller() returns return address in a().
336 * (Note: We assume a() and b() are C routines which do the normal entry/exit
337 * sequence.)
338 */
339
340 #if defined(__lint)
341
342 caddr_t
343 caller(void)
344 { return (0); }
345
346 #else /* __lint */
347
348 #if defined(__amd64)
349
350 ENTRY(caller)
351 movq 8(%rbp), %rax /* b()'s return pc, in a() */
352 ret
353 SET_SIZE(caller)
354
355 #elif defined(__i386)
356
357 ENTRY(caller)
358 movl 4(%ebp), %eax /* b()'s return pc, in a() */
359 ret
360 SET_SIZE(caller)
361
362 #endif /* __i386 */
363 #endif /* __lint */
364
365 /*
366 * if a() calls callee(), callee() returns the
367 * return address in a();
368 */
369
370 #if defined(__lint)
371
372 caddr_t
373 callee(void)
374 { return (0); }
375
376 #else /* __lint */
377
378 #if defined(__amd64)
379
380 ENTRY(callee)
381 movq (%rsp), %rax /* callee()'s return pc, in a() */
382 ret
383 SET_SIZE(callee)
384
385 #elif defined(__i386)
386
387 ENTRY(callee)
388 movl (%esp), %eax /* callee()'s return pc, in a() */
389 ret
390 SET_SIZE(callee)
391
392 #endif /* __i386 */
393 #endif /* __lint */
394
395 /*
396 * return the current frame pointer
397 */
398
399 #if defined(__lint)
400
401 greg_t
402 getfp(void)
403 { return (0); }
404
405 #else /* __lint */
406
407 #if defined(__amd64)
408
409 ENTRY(getfp)
410 movq %rbp, %rax
411 ret
412 SET_SIZE(getfp)
413
414 #elif defined(__i386)
415
416 ENTRY(getfp)
417 movl %ebp, %eax
418 ret
419 SET_SIZE(getfp)
420
421 #endif /* __i386 */
422 #endif /* __lint */
423
424 /*
425 * Invalidate a single page table entry in the TLB
426 */
427
428 #if defined(__lint)
429
430 /* ARGSUSED */
431 void
432 mmu_tlbflush_entry(caddr_t m)
433 {}
434
435 #else /* __lint */
436
437 #if defined(__amd64)
438
439 ENTRY(mmu_tlbflush_entry)
440 invlpg (%rdi)
441 ret
442 SET_SIZE(mmu_tlbflush_entry)
443
444 #elif defined(__i386)
445
446 ENTRY(mmu_tlbflush_entry)
447 movl 4(%esp), %eax
448 invlpg (%eax)
449 ret
450 SET_SIZE(mmu_tlbflush_entry)
451
452 #endif /* __i386 */
453 #endif /* __lint */
454
455
456 /*
457 * Get/Set the value of various control registers
458 */
459
460 #if defined(__lint)
461
462 ulong_t
463 getcr0(void)
464 { return (0); }
465
466 /* ARGSUSED */
467 void
468 setcr0(ulong_t value)
469 {}
470
471 ulong_t
472 getcr2(void)
473 { return (0); }
474
475 ulong_t
476 getcr3(void)
477 { return (0); }
478
479 #if !defined(__xpv)
480 /* ARGSUSED */
481 void
482 setcr3(ulong_t val)
483 {}
484
485 void
486 reload_cr3(void)
487 {}
488 #endif
489
490 ulong_t
491 getcr4(void)
492 { return (0); }
493
494 /* ARGSUSED */
495 void
496 setcr4(ulong_t val)
497 {}
498
499 #if defined(__amd64)
500
501 ulong_t
502 getcr8(void)
503 { return (0); }
504
505 /* ARGSUSED */
506 void
507 setcr8(ulong_t val)
508 {}
509
510 #endif /* __amd64 */
511
512 #else /* __lint */
513
514 #if defined(__amd64)
515
516 ENTRY(getcr0)
517 movq %cr0, %rax
518 ret
519 SET_SIZE(getcr0)
520
521 ENTRY(setcr0)
522 movq %rdi, %cr0
523 ret
524 SET_SIZE(setcr0)
525
526 ENTRY(getcr2)
527 #if defined(__xpv)
528 movq %gs:CPU_VCPU_INFO, %rax
529 movq VCPU_INFO_ARCH_CR2(%rax), %rax
530 #else
531 movq %cr2, %rax
532 #endif
533 ret
534 SET_SIZE(getcr2)
535
536 ENTRY(getcr3)
537 movq %cr3, %rax
538 ret
539 SET_SIZE(getcr3)
540
541 #if !defined(__xpv)
542
543 ENTRY(setcr3)
544 movq %rdi, %cr3
545 ret
546 SET_SIZE(setcr3)
547
548 ENTRY(reload_cr3)
549 movq %cr3, %rdi
550 movq %rdi, %cr3
551 ret
552 SET_SIZE(reload_cr3)
553
554 #endif /* __xpv */
555
556 ENTRY(getcr4)
557 movq %cr4, %rax
558 ret
559 SET_SIZE(getcr4)
560
561 ENTRY(setcr4)
562 movq %rdi, %cr4
563 ret
564 SET_SIZE(setcr4)
565
566 ENTRY(getcr8)
567 movq %cr8, %rax
568 ret
569 SET_SIZE(getcr8)
570
571 ENTRY(setcr8)
572 movq %rdi, %cr8
573 ret
574 SET_SIZE(setcr8)
575
576 #elif defined(__i386)
577
578 ENTRY(getcr0)
579 movl %cr0, %eax
580 ret
581 SET_SIZE(getcr0)
582
583 ENTRY(setcr0)
584 movl 4(%esp), %eax
585 movl %eax, %cr0
586 ret
587 SET_SIZE(setcr0)
588
589 /*
590 * "lock mov %cr0" is used on processors which indicate it is
591 * supported via CPUID. Normally the 32 bit TPR is accessed via
592 * the local APIC.
593 */
594 ENTRY(getcr8)
595 lock
596 movl %cr0, %eax
597 ret
598 SET_SIZE(getcr8)
599
600 ENTRY(setcr8)
601 movl 4(%esp), %eax
602 lock
603 movl %eax, %cr0
604 ret
605 SET_SIZE(setcr8)
606
607 ENTRY(getcr2)
608 #if defined(__xpv)
609 movl %gs:CPU_VCPU_INFO, %eax
610 movl VCPU_INFO_ARCH_CR2(%eax), %eax
611 #else
612 movl %cr2, %eax
613 #endif
614 ret
615 SET_SIZE(getcr2)
616
617 ENTRY(getcr3)
618 movl %cr3, %eax
619 ret
620 SET_SIZE(getcr3)
621
622 #if !defined(__xpv)
623
624 ENTRY(setcr3)
625 movl 4(%esp), %eax
626 movl %eax, %cr3
627 ret
628 SET_SIZE(setcr3)
629
630 ENTRY(reload_cr3)
631 movl %cr3, %eax
632 movl %eax, %cr3
633 ret
634 SET_SIZE(reload_cr3)
635
636 #endif /* __xpv */
637
638 ENTRY(getcr4)
639 movl %cr4, %eax
640 ret
641 SET_SIZE(getcr4)
642
643 ENTRY(setcr4)
644 movl 4(%esp), %eax
645 movl %eax, %cr4
646 ret
647 SET_SIZE(setcr4)
648
649 #endif /* __i386 */
650 #endif /* __lint */
651
652 #if defined(__lint)
653
654 /*ARGSUSED*/
655 uint32_t
656 __cpuid_insn(struct cpuid_regs *regs)
657 { return (0); }
658
659 #else /* __lint */
660
661 #if defined(__amd64)
662
663 ENTRY(__cpuid_insn)
664 movq %rbx, %r8
665 movq %rcx, %r9
666 movq %rdx, %r11
667 movl (%rdi), %eax /* %eax = regs->cp_eax */
668 movl 0x4(%rdi), %ebx /* %ebx = regs->cp_ebx */
669 movl 0x8(%rdi), %ecx /* %ecx = regs->cp_ecx */
670 movl 0xc(%rdi), %edx /* %edx = regs->cp_edx */
671 cpuid
672 movl %eax, (%rdi) /* regs->cp_eax = %eax */
673 movl %ebx, 0x4(%rdi) /* regs->cp_ebx = %ebx */
674 movl %ecx, 0x8(%rdi) /* regs->cp_ecx = %ecx */
675 movl %edx, 0xc(%rdi) /* regs->cp_edx = %edx */
676 movq %r8, %rbx
677 movq %r9, %rcx
678 movq %r11, %rdx
679 ret
680 SET_SIZE(__cpuid_insn)
681
682 #elif defined(__i386)
683
684 ENTRY(__cpuid_insn)
685 pushl %ebp
686 movl 0x8(%esp), %ebp /* %ebp = regs */
687 pushl %ebx
688 pushl %ecx
689 pushl %edx
690 movl (%ebp), %eax /* %eax = regs->cp_eax */
691 movl 0x4(%ebp), %ebx /* %ebx = regs->cp_ebx */
692 movl 0x8(%ebp), %ecx /* %ecx = regs->cp_ecx */
693 movl 0xc(%ebp), %edx /* %edx = regs->cp_edx */
694 cpuid
695 movl %eax, (%ebp) /* regs->cp_eax = %eax */
696 movl %ebx, 0x4(%ebp) /* regs->cp_ebx = %ebx */
697 movl %ecx, 0x8(%ebp) /* regs->cp_ecx = %ecx */
698 movl %edx, 0xc(%ebp) /* regs->cp_edx = %edx */
699 popl %edx
700 popl %ecx
701 popl %ebx
702 popl %ebp
703 ret
704 SET_SIZE(__cpuid_insn)
705
706 #endif /* __i386 */
707 #endif /* __lint */
708
709 #if defined(__lint)
710
711 /*ARGSUSED*/
712 void
713 i86_monitor(volatile uint32_t *addr, uint32_t extensions, uint32_t hints)
714 {}
715
716 #else /* __lint */
717
718 #if defined(__amd64)
719
720 ENTRY_NP(i86_monitor)
721 pushq %rbp
722 movq %rsp, %rbp
723 movq %rdi, %rax /* addr */
724 movq %rsi, %rcx /* extensions */
725 /* rdx contains input arg3: hints */
726 clflush (%rax)
727 .byte 0x0f, 0x01, 0xc8 /* monitor */
728 leave
729 ret
730 SET_SIZE(i86_monitor)
731
732 #elif defined(__i386)
733
734 ENTRY_NP(i86_monitor)
735 pushl %ebp
736 movl %esp, %ebp
737 movl 0x8(%ebp),%eax /* addr */
738 movl 0xc(%ebp),%ecx /* extensions */
739 movl 0x10(%ebp),%edx /* hints */
740 clflush (%eax)
741 .byte 0x0f, 0x01, 0xc8 /* monitor */
742 leave
743 ret
744 SET_SIZE(i86_monitor)
745
746 #endif /* __i386 */
747 #endif /* __lint */
748
749 #if defined(__lint)
750
751 /*ARGSUSED*/
752 void
753 i86_mwait(uint32_t data, uint32_t extensions)
754 {}
755
756 #else /* __lint */
757
758 #if defined(__amd64)
759
760 ENTRY_NP(i86_mwait)
761 pushq %rbp
762 movq %rsp, %rbp
763 movq %rdi, %rax /* data */
764 movq %rsi, %rcx /* extensions */
765 .byte 0x0f, 0x01, 0xc9 /* mwait */
766 leave
767 ret
768 SET_SIZE(i86_mwait)
769
770 #elif defined(__i386)
771
772 ENTRY_NP(i86_mwait)
773 pushl %ebp
774 movl %esp, %ebp
775 movl 0x8(%ebp),%eax /* data */
776 movl 0xc(%ebp),%ecx /* extensions */
777 .byte 0x0f, 0x01, 0xc9 /* mwait */
778 leave
779 ret
780 SET_SIZE(i86_mwait)
781
782 #endif /* __i386 */
783 #endif /* __lint */
784
785 #if defined(__xpv)
786 /*
787 * Defined in C
788 */
789 #else
790
791 #if defined(__lint)
792
793 hrtime_t
794 tsc_read(void)
795 {
796 return (0);
797 }
798
799 #else /* __lint */
800
801 #if defined(__amd64)
802
803 ENTRY_NP(tsc_read)
804 movq %rbx, %r11
805 movl $0, %eax
806 cpuid
807 rdtsc
808 movq %r11, %rbx
809 shlq $32, %rdx
810 orq %rdx, %rax
811 ret
812 .globl _tsc_mfence_start
813 _tsc_mfence_start:
814 mfence
815 rdtsc
816 shlq $32, %rdx
817 orq %rdx, %rax
818 ret
819 .globl _tsc_mfence_end
820 _tsc_mfence_end:
821 .globl _tscp_start
822 _tscp_start:
823 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */
824 shlq $32, %rdx
825 orq %rdx, %rax
826 ret
827 .globl _tscp_end
828 _tscp_end:
829 .globl _no_rdtsc_start
830 _no_rdtsc_start:
831 xorl %edx, %edx
832 xorl %eax, %eax
833 ret
834 .globl _no_rdtsc_end
835 _no_rdtsc_end:
836 .globl _tsc_lfence_start
837 _tsc_lfence_start:
838 lfence
839 rdtsc
840 shlq $32, %rdx
841 orq %rdx, %rax
842 ret
843 .globl _tsc_lfence_end
844 _tsc_lfence_end:
845 SET_SIZE(tsc_read)
846
847 #else /* __i386 */
848
849 ENTRY_NP(tsc_read)
850 pushl %ebx
851 movl $0, %eax
852 cpuid
853 rdtsc
854 popl %ebx
855 ret
856 .globl _tsc_mfence_start
857 _tsc_mfence_start:
858 mfence
859 rdtsc
860 ret
861 .globl _tsc_mfence_end
862 _tsc_mfence_end:
863 .globl _tscp_start
864 _tscp_start:
865 .byte 0x0f, 0x01, 0xf9 /* rdtscp instruction */
866 ret
867 .globl _tscp_end
868 _tscp_end:
869 .globl _no_rdtsc_start
870 _no_rdtsc_start:
871 xorl %edx, %edx
872 xorl %eax, %eax
873 ret
874 .globl _no_rdtsc_end
875 _no_rdtsc_end:
876 .globl _tsc_lfence_start
877 _tsc_lfence_start:
878 lfence
879 rdtsc
880 ret
881 .globl _tsc_lfence_end
882 _tsc_lfence_end:
883 SET_SIZE(tsc_read)
884
885 #endif /* __i386 */
886
887 #endif /* __lint */
888
889
890 #endif /* __xpv */
891
892 #ifdef __lint
893 /*
894 * Do not use this function for obtaining clock tick. This
895 * is called by callers who do not need to have a guarenteed
896 * correct tick value. The proper routine to use is tsc_read().
897 */
898 u_longlong_t
899 randtick(void)
900 {
901 return (0);
902 }
903 #else
904 #if defined(__amd64)
905 ENTRY_NP(randtick)
906 rdtsc
907 shlq $32, %rdx
908 orq %rdx, %rax
909 ret
910 SET_SIZE(randtick)
911 #else
912 ENTRY_NP(randtick)
913 rdtsc
914 ret
915 SET_SIZE(randtick)
916 #endif /* __i386 */
917 #endif /* __lint */
918 /*
919 * Insert entryp after predp in a doubly linked list.
920 */
921
922 #if defined(__lint)
923
924 /*ARGSUSED*/
925 void
926 _insque(caddr_t entryp, caddr_t predp)
927 {}
928
929 #else /* __lint */
930
931 #if defined(__amd64)
932
933 ENTRY(_insque)
934 movq (%rsi), %rax /* predp->forw */
935 movq %rsi, CPTRSIZE(%rdi) /* entryp->back = predp */
936 movq %rax, (%rdi) /* entryp->forw = predp->forw */
937 movq %rdi, (%rsi) /* predp->forw = entryp */
938 movq %rdi, CPTRSIZE(%rax) /* predp->forw->back = entryp */
939 ret
940 SET_SIZE(_insque)
941
942 #elif defined(__i386)
943
944 ENTRY(_insque)
945 movl 8(%esp), %edx
946 movl 4(%esp), %ecx
947 movl (%edx), %eax /* predp->forw */
948 movl %edx, CPTRSIZE(%ecx) /* entryp->back = predp */
949 movl %eax, (%ecx) /* entryp->forw = predp->forw */
950 movl %ecx, (%edx) /* predp->forw = entryp */
951 movl %ecx, CPTRSIZE(%eax) /* predp->forw->back = entryp */
952 ret
953 SET_SIZE(_insque)
954
955 #endif /* __i386 */
956 #endif /* __lint */
957
958 /*
959 * Remove entryp from a doubly linked list
960 */
961
962 #if defined(__lint)
963
964 /*ARGSUSED*/
965 void
966 _remque(caddr_t entryp)
967 {}
968
969 #else /* __lint */
970
971 #if defined(__amd64)
972
973 ENTRY(_remque)
974 movq (%rdi), %rax /* entry->forw */
975 movq CPTRSIZE(%rdi), %rdx /* entry->back */
976 movq %rax, (%rdx) /* entry->back->forw = entry->forw */
977 movq %rdx, CPTRSIZE(%rax) /* entry->forw->back = entry->back */
978 ret
979 SET_SIZE(_remque)
980
981 #elif defined(__i386)
982
983 ENTRY(_remque)
984 movl 4(%esp), %ecx
985 movl (%ecx), %eax /* entry->forw */
986 movl CPTRSIZE(%ecx), %edx /* entry->back */
987 movl %eax, (%edx) /* entry->back->forw = entry->forw */
988 movl %edx, CPTRSIZE(%eax) /* entry->forw->back = entry->back */
989 ret
990 SET_SIZE(_remque)
991
992 #endif /* __i386 */
993 #endif /* __lint */
994
995 /*
996 * Returns the number of
997 * non-NULL bytes in string argument.
998 */
999
1000 #if defined(__lint)
1001
1002 /* ARGSUSED */
1003 size_t
1004 strlen(const char *str)
1005 { return (0); }
1006
1007 #else /* __lint */
1008
1009 #if defined(__amd64)
1010
1011 /*
1012 * This is close to a simple transliteration of a C version of this
1013 * routine. We should either just -make- this be a C version, or
1014 * justify having it in assembler by making it significantly faster.
1015 *
1016 * size_t
1017 * strlen(const char *s)
1018 * {
1019 * const char *s0;
1020 * #if defined(DEBUG)
1021 * if ((uintptr_t)s < KERNELBASE)
1022 * panic(.str_panic_msg);
1023 * #endif
1024 * for (s0 = s; *s; s++)
1025 * ;
1026 * return (s - s0);
1027 * }
1028 */
1029
1030 ENTRY(strlen)
1031 #ifdef DEBUG
1032 movq postbootkernelbase(%rip), %rax
1033 cmpq %rax, %rdi
1034 jae str_valid
1035 pushq %rbp
1036 movq %rsp, %rbp
1037 leaq .str_panic_msg(%rip), %rdi
1038 xorl %eax, %eax
1039 call panic
1040 #endif /* DEBUG */
1041 str_valid:
1042 cmpb $0, (%rdi)
1043 movq %rdi, %rax
1044 je .null_found
1045 .align 4
1046 .strlen_loop:
1047 incq %rdi
1048 cmpb $0, (%rdi)
1049 jne .strlen_loop
1050 .null_found:
1051 subq %rax, %rdi
1052 movq %rdi, %rax
1053 ret
1054 SET_SIZE(strlen)
1055
1056 #elif defined(__i386)
1057
1058 ENTRY(strlen)
1059 #ifdef DEBUG
1060 movl postbootkernelbase, %eax
1061 cmpl %eax, 4(%esp)
1062 jae str_valid
1063 pushl %ebp
1064 movl %esp, %ebp
1065 pushl $.str_panic_msg
1066 call panic
1067 #endif /* DEBUG */
1068
1069 str_valid:
1070 movl 4(%esp), %eax /* %eax = string address */
1071 testl $3, %eax /* if %eax not word aligned */
1072 jnz .not_word_aligned /* goto .not_word_aligned */
1073 .align 4
1074 .word_aligned:
1075 movl (%eax), %edx /* move 1 word from (%eax) to %edx */
1076 movl $0x7f7f7f7f, %ecx
1077 andl %edx, %ecx /* %ecx = %edx & 0x7f7f7f7f */
1078 addl $4, %eax /* next word */
1079 addl $0x7f7f7f7f, %ecx /* %ecx += 0x7f7f7f7f */
1080 orl %edx, %ecx /* %ecx |= %edx */
1081 andl $0x80808080, %ecx /* %ecx &= 0x80808080 */
1082 cmpl $0x80808080, %ecx /* if no null byte in this word */
1083 je .word_aligned /* goto .word_aligned */
1084 subl $4, %eax /* post-incremented */
1085 .not_word_aligned:
1086 cmpb $0, (%eax) /* if a byte in (%eax) is null */
1087 je .null_found /* goto .null_found */
1088 incl %eax /* next byte */
1089 testl $3, %eax /* if %eax not word aligned */
1090 jnz .not_word_aligned /* goto .not_word_aligned */
1091 jmp .word_aligned /* goto .word_aligned */
1092 .align 4
1093 .null_found:
1094 subl 4(%esp), %eax /* %eax -= string address */
1095 ret
1096 SET_SIZE(strlen)
1097
1098 #endif /* __i386 */
1099
1100 #ifdef DEBUG
1101 .text
1102 .str_panic_msg:
1103 .string "strlen: argument below kernelbase"
1104 #endif /* DEBUG */
1105
1106 #endif /* __lint */
1107
1108 /*
1109 * Berkeley 4.3 introduced symbolically named interrupt levels
1110 * as a way deal with priority in a machine independent fashion.
1111 * Numbered priorities are machine specific, and should be
1112 * discouraged where possible.
1113 *
1114 * Note, for the machine specific priorities there are
1115 * examples listed for devices that use a particular priority.
1116 * It should not be construed that all devices of that
1117 * type should be at that priority. It is currently were
1118 * the current devices fit into the priority scheme based
1119 * upon time criticalness.
1120 *
1121 * The underlying assumption of these assignments is that
1122 * IPL 10 is the highest level from which a device
1123 * routine can call wakeup. Devices that interrupt from higher
1124 * levels are restricted in what they can do. If they need
1125 * kernels services they should schedule a routine at a lower
1126 * level (via software interrupt) to do the required
1127 * processing.
1128 *
1129 * Examples of this higher usage:
1130 * Level Usage
1131 * 14 Profiling clock (and PROM uart polling clock)
1132 * 12 Serial ports
1133 *
1134 * The serial ports request lower level processing on level 6.
1135 *
1136 * Also, almost all splN routines (where N is a number or a
1137 * mnemonic) will do a RAISE(), on the assumption that they are
1138 * never used to lower our priority.
1139 * The exceptions are:
1140 * spl8() Because you can't be above 15 to begin with!
1141 * splzs() Because this is used at boot time to lower our
1142 * priority, to allow the PROM to poll the uart.
1143 * spl0() Used to lower priority to 0.
1144 */
1145
1146 #if defined(__lint)
1147
1148 int spl0(void) { return (0); }
1149 int spl6(void) { return (0); }
1150 int spl7(void) { return (0); }
1151 int spl8(void) { return (0); }
1152 int splhigh(void) { return (0); }
1153 int splhi(void) { return (0); }
1154 int splzs(void) { return (0); }
1155
1156 /* ARGSUSED */
1157 void
1158 splx(int level)
1159 {}
1160
1161 #else /* __lint */
1162
1163 #if defined(__amd64)
1164
1165 #define SETPRI(level) \
1166 movl $/**/level, %edi; /* new priority */ \
1167 jmp do_splx /* redirect to do_splx */
1168
1169 #define RAISE(level) \
1170 movl $/**/level, %edi; /* new priority */ \
1171 jmp splr /* redirect to splr */
1172
1173 #elif defined(__i386)
1174
1175 #define SETPRI(level) \
1176 pushl $/**/level; /* new priority */ \
1177 call do_splx; /* invoke common splx code */ \
1178 addl $4, %esp; /* unstack arg */ \
1179 ret
1180
1181 #define RAISE(level) \
1182 pushl $/**/level; /* new priority */ \
1183 call splr; /* invoke common splr code */ \
1184 addl $4, %esp; /* unstack args */ \
1185 ret
1186
1187 #endif /* __i386 */
1188
1189 /* locks out all interrupts, including memory errors */
1190 ENTRY(spl8)
1191 SETPRI(15)
1192 SET_SIZE(spl8)
1193
1194 /* just below the level that profiling runs */
1195 ENTRY(spl7)
1196 RAISE(13)
1197 SET_SIZE(spl7)
1198
1199 /* sun specific - highest priority onboard serial i/o asy ports */
1200 ENTRY(splzs)
1201 SETPRI(12) /* Can't be a RAISE, as it's used to lower us */
1202 SET_SIZE(splzs)
1203
1204 ENTRY(splhi)
1205 ALTENTRY(splhigh)
1206 ALTENTRY(spl6)
1207 ALTENTRY(i_ddi_splhigh)
1208
1209 RAISE(DISP_LEVEL)
1210
1211 SET_SIZE(i_ddi_splhigh)
1212 SET_SIZE(spl6)
1213 SET_SIZE(splhigh)
1214 SET_SIZE(splhi)
1215
1216 /* allow all interrupts */
1217 ENTRY(spl0)
1218 SETPRI(0)
1219 SET_SIZE(spl0)
1220
1221
1222 /* splx implementation */
1223 ENTRY(splx)
1224 jmp do_splx /* redirect to common splx code */
1225 SET_SIZE(splx)
1226
1227 #endif /* __lint */
1228
1229 #if defined(__i386)
1230
1231 /*
1232 * Read and write the %gs register
1233 */
1234
1235 #if defined(__lint)
1236
1237 /*ARGSUSED*/
1238 uint16_t
1239 getgs(void)
1240 { return (0); }
1241
1242 /*ARGSUSED*/
1243 void
1244 setgs(uint16_t sel)
1245 {}
1246
1247 #else /* __lint */
1248
1249 ENTRY(getgs)
1250 clr %eax
1251 movw %gs, %ax
1252 ret
1253 SET_SIZE(getgs)
1254
1255 ENTRY(setgs)
1256 movw 4(%esp), %gs
1257 ret
1258 SET_SIZE(setgs)
1259
1260 #endif /* __lint */
1261 #endif /* __i386 */
1262
1263 #if defined(__lint)
1264
1265 void
1266 pc_reset(void)
1267 {}
1268
1269 void
1270 efi_reset(void)
1271 {}
1272
1273 #else /* __lint */
1274
1275 ENTRY(wait_500ms)
1276 #if defined(__amd64)
1277 pushq %rbx
1278 #elif defined(__i386)
1279 push %ebx
1280 #endif
1281 movl $50000, %ebx
1282 1:
1283 call tenmicrosec
1284 decl %ebx
1285 jnz 1b
1286 #if defined(__amd64)
1287 popq %rbx
1288 #elif defined(__i386)
1289 pop %ebx
1290 #endif
1291 ret
1292 SET_SIZE(wait_500ms)
1293
1294 #define RESET_METHOD_KBC 1
1295 #define RESET_METHOD_PORT92 2
1296 #define RESET_METHOD_PCI 4
1297
1298 DGDEF3(pc_reset_methods, 4, 8)
1299 .long RESET_METHOD_KBC|RESET_METHOD_PORT92|RESET_METHOD_PCI;
1300
1301 ENTRY(pc_reset)
1302
1303 #if defined(__i386)
1304 testl $RESET_METHOD_KBC, pc_reset_methods
1305 #elif defined(__amd64)
1306 testl $RESET_METHOD_KBC, pc_reset_methods(%rip)
1307 #endif
1308 jz 1f
1309
1310 /
1311 / Try the classic keyboard controller-triggered reset.
1312 /
1313 movw $0x64, %dx
1314 movb $0xfe, %al
1315 outb (%dx)
1316
1317 / Wait up to 500 milliseconds here for the keyboard controller
1318 / to pull the reset line. On some systems where the keyboard
1319 / controller is slow to pull the reset line, the next reset method
1320 / may be executed (which may be bad if those systems hang when the
1321 / next reset method is used, e.g. Ferrari 3400 (doesn't like port 92),
1322 / and Ferrari 4000 (doesn't like the cf9 reset method))
1323
1324 call wait_500ms
1325
1326 1:
1327 #if defined(__i386)
1328 testl $RESET_METHOD_PORT92, pc_reset_methods
1329 #elif defined(__amd64)
1330 testl $RESET_METHOD_PORT92, pc_reset_methods(%rip)
1331 #endif
1332 jz 3f
1333
1334 /
1335 / Try port 0x92 fast reset
1336 /
1337 movw $0x92, %dx
1338 inb (%dx)
1339 cmpb $0xff, %al / If port's not there, we should get back 0xFF
1340 je 1f
1341 testb $1, %al / If bit 0
1342 jz 2f / is clear, jump to perform the reset
1343 andb $0xfe, %al / otherwise,
1344 outb (%dx) / clear bit 0 first, then
1345 2:
1346 orb $1, %al / Set bit 0
1347 outb (%dx) / and reset the system
1348 1:
1349
1350 call wait_500ms
1351
1352 3:
1353 #if defined(__i386)
1354 testl $RESET_METHOD_PCI, pc_reset_methods
1355 #elif defined(__amd64)
1356 testl $RESET_METHOD_PCI, pc_reset_methods(%rip)
1357 #endif
1358 jz 4f
1359
1360 / Try the PCI (soft) reset vector (should work on all modern systems,
1361 / but has been shown to cause problems on 450NX systems, and some newer
1362 / systems (e.g. ATI IXP400-equipped systems))
1363 / When resetting via this method, 2 writes are required. The first
1364 / targets bit 1 (0=hard reset without power cycle, 1=hard reset with
1365 / power cycle).
1366 / The reset occurs on the second write, during bit 2's transition from
1367 / 0->1.
1368 movw $0xcf9, %dx
1369 movb $0x2, %al / Reset mode = hard, no power cycle
1370 outb (%dx)
1371 movb $0x6, %al
1372 outb (%dx)
1373
1374 call wait_500ms
1375
1376 4:
1377 /
1378 / port 0xcf9 failed also. Last-ditch effort is to
1379 / triple-fault the CPU.
1380 / Also, use triple fault for EFI firmware
1381 /
1382 ENTRY(efi_reset)
1383 #if defined(__amd64)
1384 pushq $0x0
1385 pushq $0x0 / IDT base of 0, limit of 0 + 2 unused bytes
1386 lidt (%rsp)
1387 #elif defined(__i386)
1388 pushl $0x0
1389 pushl $0x0 / IDT base of 0, limit of 0 + 2 unused bytes
1390 lidt (%esp)
1391 #endif
1392 int $0x0 / Trigger interrupt, generate triple-fault
1393
1394 cli
1395 hlt / Wait forever
1396 /*NOTREACHED*/
1397 SET_SIZE(efi_reset)
1398 SET_SIZE(pc_reset)
1399
1400 #endif /* __lint */
1401
1402 /*
1403 * C callable in and out routines
1404 */
1405
1406 #if defined(__lint)
1407
1408 /* ARGSUSED */
1409 void
1410 outl(int port_address, uint32_t val)
1411 {}
1412
1413 #else /* __lint */
1414
1415 #if defined(__amd64)
1416
1417 ENTRY(outl)
1418 movw %di, %dx
1419 movl %esi, %eax
1420 outl (%dx)
1421 ret
1422 SET_SIZE(outl)
1423
1424 #elif defined(__i386)
1425
1426 .set PORT, 4
1427 .set VAL, 8
1428
1429 ENTRY(outl)
1430 movw PORT(%esp), %dx
1431 movl VAL(%esp), %eax
1432 outl (%dx)
1433 ret
1434 SET_SIZE(outl)
1435
1436 #endif /* __i386 */
1437 #endif /* __lint */
1438
1439 #if defined(__lint)
1440
1441 /* ARGSUSED */
1442 void
1443 outw(int port_address, uint16_t val)
1444 {}
1445
1446 #else /* __lint */
1447
1448 #if defined(__amd64)
1449
1450 ENTRY(outw)
1451 movw %di, %dx
1452 movw %si, %ax
1453 D16 outl (%dx) /* XX64 why not outw? */
1454 ret
1455 SET_SIZE(outw)
1456
1457 #elif defined(__i386)
1458
1459 ENTRY(outw)
1460 movw PORT(%esp), %dx
1461 movw VAL(%esp), %ax
1462 D16 outl (%dx)
1463 ret
1464 SET_SIZE(outw)
1465
1466 #endif /* __i386 */
1467 #endif /* __lint */
1468
1469 #if defined(__lint)
1470
1471 /* ARGSUSED */
1472 void
1473 outb(int port_address, uint8_t val)
1474 {}
1475
1476 #else /* __lint */
1477
1478 #if defined(__amd64)
1479
1480 ENTRY(outb)
1481 movw %di, %dx
1482 movb %sil, %al
1483 outb (%dx)
1484 ret
1485 SET_SIZE(outb)
1486
1487 #elif defined(__i386)
1488
1489 ENTRY(outb)
1490 movw PORT(%esp), %dx
1491 movb VAL(%esp), %al
1492 outb (%dx)
1493 ret
1494 SET_SIZE(outb)
1495
1496 #endif /* __i386 */
1497 #endif /* __lint */
1498
1499 #if defined(__lint)
1500
1501 /* ARGSUSED */
1502 uint32_t
1503 inl(int port_address)
1504 { return (0); }
1505
1506 #else /* __lint */
1507
1508 #if defined(__amd64)
1509
1510 ENTRY(inl)
1511 xorl %eax, %eax
1512 movw %di, %dx
1513 inl (%dx)
1514 ret
1515 SET_SIZE(inl)
1516
1517 #elif defined(__i386)
1518
1519 ENTRY(inl)
1520 movw PORT(%esp), %dx
1521 inl (%dx)
1522 ret
1523 SET_SIZE(inl)
1524
1525 #endif /* __i386 */
1526 #endif /* __lint */
1527
1528 #if defined(__lint)
1529
1530 /* ARGSUSED */
1531 uint16_t
1532 inw(int port_address)
1533 { return (0); }
1534
1535 #else /* __lint */
1536
1537 #if defined(__amd64)
1538
1539 ENTRY(inw)
1540 xorl %eax, %eax
1541 movw %di, %dx
1542 D16 inl (%dx)
1543 ret
1544 SET_SIZE(inw)
1545
1546 #elif defined(__i386)
1547
1548 ENTRY(inw)
1549 subl %eax, %eax
1550 movw PORT(%esp), %dx
1551 D16 inl (%dx)
1552 ret
1553 SET_SIZE(inw)
1554
1555 #endif /* __i386 */
1556 #endif /* __lint */
1557
1558
1559 #if defined(__lint)
1560
1561 /* ARGSUSED */
1562 uint8_t
1563 inb(int port_address)
1564 { return (0); }
1565
1566 #else /* __lint */
1567
1568 #if defined(__amd64)
1569
1570 ENTRY(inb)
1571 xorl %eax, %eax
1572 movw %di, %dx
1573 inb (%dx)
1574 ret
1575 SET_SIZE(inb)
1576
1577 #elif defined(__i386)
1578
1579 ENTRY(inb)
1580 subl %eax, %eax
1581 movw PORT(%esp), %dx
1582 inb (%dx)
1583 ret
1584 SET_SIZE(inb)
1585
1586 #endif /* __i386 */
1587 #endif /* __lint */
1588
1589
1590 #if defined(__lint)
1591
1592 /* ARGSUSED */
1593 void
1594 repoutsw(int port, uint16_t *addr, int cnt)
1595 {}
1596
1597 #else /* __lint */
1598
1599 #if defined(__amd64)
1600
1601 ENTRY(repoutsw)
1602 movl %edx, %ecx
1603 movw %di, %dx
1604 rep
1605 D16 outsl
1606 ret
1607 SET_SIZE(repoutsw)
1608
1609 #elif defined(__i386)
1610
1611 /*
1612 * The arguments and saved registers are on the stack in the
1613 * following order:
1614 * | cnt | +16
1615 * | *addr | +12
1616 * | port | +8
1617 * | eip | +4
1618 * | esi | <-- %esp
1619 * If additional values are pushed onto the stack, make sure
1620 * to adjust the following constants accordingly.
1621 */
1622 .set PORT, 8
1623 .set ADDR, 12
1624 .set COUNT, 16
1625
1626 ENTRY(repoutsw)
1627 pushl %esi
1628 movl PORT(%esp), %edx
1629 movl ADDR(%esp), %esi
1630 movl COUNT(%esp), %ecx
1631 rep
1632 D16 outsl
1633 popl %esi
1634 ret
1635 SET_SIZE(repoutsw)
1636
1637 #endif /* __i386 */
1638 #endif /* __lint */
1639
1640
1641 #if defined(__lint)
1642
1643 /* ARGSUSED */
1644 void
1645 repinsw(int port_addr, uint16_t *addr, int cnt)
1646 {}
1647
1648 #else /* __lint */
1649
1650 #if defined(__amd64)
1651
1652 ENTRY(repinsw)
1653 movl %edx, %ecx
1654 movw %di, %dx
1655 rep
1656 D16 insl
1657 ret
1658 SET_SIZE(repinsw)
1659
1660 #elif defined(__i386)
1661
1662 ENTRY(repinsw)
1663 pushl %edi
1664 movl PORT(%esp), %edx
1665 movl ADDR(%esp), %edi
1666 movl COUNT(%esp), %ecx
1667 rep
1668 D16 insl
1669 popl %edi
1670 ret
1671 SET_SIZE(repinsw)
1672
1673 #endif /* __i386 */
1674 #endif /* __lint */
1675
1676
1677 #if defined(__lint)
1678
1679 /* ARGSUSED */
1680 void
1681 repinsb(int port, uint8_t *addr, int count)
1682 {}
1683
1684 #else /* __lint */
1685
1686 #if defined(__amd64)
1687
1688 ENTRY(repinsb)
1689 movl %edx, %ecx
1690 movw %di, %dx
1691 movq %rsi, %rdi
1692 rep
1693 insb
1694 ret
1695 SET_SIZE(repinsb)
1696
1697 #elif defined(__i386)
1698
1699 /*
1700 * The arguments and saved registers are on the stack in the
1701 * following order:
1702 * | cnt | +16
1703 * | *addr | +12
1704 * | port | +8
1705 * | eip | +4
1706 * | esi | <-- %esp
1707 * If additional values are pushed onto the stack, make sure
1708 * to adjust the following constants accordingly.
1709 */
1710 .set IO_PORT, 8
1711 .set IO_ADDR, 12
1712 .set IO_COUNT, 16
1713
1714 ENTRY(repinsb)
1715 pushl %edi
1716 movl IO_ADDR(%esp), %edi
1717 movl IO_COUNT(%esp), %ecx
1718 movl IO_PORT(%esp), %edx
1719 rep
1720 insb
1721 popl %edi
1722 ret
1723 SET_SIZE(repinsb)
1724
1725 #endif /* __i386 */
1726 #endif /* __lint */
1727
1728
1729 /*
1730 * Input a stream of 32-bit words.
1731 * NOTE: count is a DWORD count.
1732 */
1733 #if defined(__lint)
1734
1735 /* ARGSUSED */
1736 void
1737 repinsd(int port, uint32_t *addr, int count)
1738 {}
1739
1740 #else /* __lint */
1741
1742 #if defined(__amd64)
1743
1744 ENTRY(repinsd)
1745 movl %edx, %ecx
1746 movw %di, %dx
1747 movq %rsi, %rdi
1748 rep
1749 insl
1750 ret
1751 SET_SIZE(repinsd)
1752
1753 #elif defined(__i386)
1754
1755 ENTRY(repinsd)
1756 pushl %edi
1757 movl IO_ADDR(%esp), %edi
1758 movl IO_COUNT(%esp), %ecx
1759 movl IO_PORT(%esp), %edx
1760 rep
1761 insl
1762 popl %edi
1763 ret
1764 SET_SIZE(repinsd)
1765
1766 #endif /* __i386 */
1767 #endif /* __lint */
1768
1769 /*
1770 * Output a stream of bytes
1771 * NOTE: count is a byte count
1772 */
1773 #if defined(__lint)
1774
1775 /* ARGSUSED */
1776 void
1777 repoutsb(int port, uint8_t *addr, int count)
1778 {}
1779
1780 #else /* __lint */
1781
1782 #if defined(__amd64)
1783
1784 ENTRY(repoutsb)
1785 movl %edx, %ecx
1786 movw %di, %dx
1787 rep
1788 outsb
1789 ret
1790 SET_SIZE(repoutsb)
1791
1792 #elif defined(__i386)
1793
1794 ENTRY(repoutsb)
1795 pushl %esi
1796 movl IO_ADDR(%esp), %esi
1797 movl IO_COUNT(%esp), %ecx
1798 movl IO_PORT(%esp), %edx
1799 rep
1800 outsb
1801 popl %esi
1802 ret
1803 SET_SIZE(repoutsb)
1804
1805 #endif /* __i386 */
1806 #endif /* __lint */
1807
1808 /*
1809 * Output a stream of 32-bit words
1810 * NOTE: count is a DWORD count
1811 */
1812 #if defined(__lint)
1813
1814 /* ARGSUSED */
1815 void
1816 repoutsd(int port, uint32_t *addr, int count)
1817 {}
1818
1819 #else /* __lint */
1820
1821 #if defined(__amd64)
1822
1823 ENTRY(repoutsd)
1824 movl %edx, %ecx
1825 movw %di, %dx
1826 rep
1827 outsl
1828 ret
1829 SET_SIZE(repoutsd)
1830
1831 #elif defined(__i386)
1832
1833 ENTRY(repoutsd)
1834 pushl %esi
1835 movl IO_ADDR(%esp), %esi
1836 movl IO_COUNT(%esp), %ecx
1837 movl IO_PORT(%esp), %edx
1838 rep
1839 outsl
1840 popl %esi
1841 ret
1842 SET_SIZE(repoutsd)
1843
1844 #endif /* __i386 */
1845 #endif /* __lint */
1846
1847 /*
1848 * void int3(void)
1849 * void int18(void)
1850 * void int20(void)
1851 * void int_cmci(void)
1852 */
1853
1854 #if defined(__lint)
1855
1856 void
1857 int3(void)
1858 {}
1859
1860 void
1861 int18(void)
1862 {}
1863
1864 void
1865 int20(void)
1866 {}
1867
1868 void
1869 int_cmci(void)
1870 {}
1871
1872 #else /* __lint */
1873
1874 ENTRY(int3)
1875 int $T_BPTFLT
1876 ret
1877 SET_SIZE(int3)
1878
1879 ENTRY(int18)
1880 int $T_MCE
1881 ret
1882 SET_SIZE(int18)
1883
1884 ENTRY(int20)
1885 movl boothowto, %eax
1886 andl $RB_DEBUG, %eax
1887 jz 1f
1888
1889 int $T_DBGENTR
1890 1:
1891 rep; ret /* use 2 byte return instruction when branch target */
1892 /* AMD Software Optimization Guide - Section 6.2 */
1893 SET_SIZE(int20)
1894
1895 ENTRY(int_cmci)
1896 int $T_ENOEXTFLT
1897 ret
1898 SET_SIZE(int_cmci)
1899
1900 #endif /* __lint */
1901
1902 #if defined(__lint)
1903
1904 /* ARGSUSED */
1905 int
1906 scanc(size_t size, uchar_t *cp, uchar_t *table, uchar_t mask)
1907 { return (0); }
1908
1909 #else /* __lint */
1910
1911 #if defined(__amd64)
1912
1913 ENTRY(scanc)
1914 /* rdi == size */
1915 /* rsi == cp */
1916 /* rdx == table */
1917 /* rcx == mask */
1918 addq %rsi, %rdi /* end = &cp[size] */
1919 .scanloop:
1920 cmpq %rdi, %rsi /* while (cp < end */
1921 jnb .scandone
1922 movzbq (%rsi), %r8 /* %r8 = *cp */
1923 incq %rsi /* cp++ */
1924 testb %cl, (%r8, %rdx)
1925 jz .scanloop /* && (table[*cp] & mask) == 0) */
1926 decq %rsi /* (fix post-increment) */
1927 .scandone:
1928 movl %edi, %eax
1929 subl %esi, %eax /* return (end - cp) */
1930 ret
1931 SET_SIZE(scanc)
1932
1933 #elif defined(__i386)
1934
1935 ENTRY(scanc)
1936 pushl %edi
1937 pushl %esi
1938 movb 24(%esp), %cl /* mask = %cl */
1939 movl 16(%esp), %esi /* cp = %esi */
1940 movl 20(%esp), %edx /* table = %edx */
1941 movl %esi, %edi
1942 addl 12(%esp), %edi /* end = &cp[size]; */
1943 .scanloop:
1944 cmpl %edi, %esi /* while (cp < end */
1945 jnb .scandone
1946 movzbl (%esi), %eax /* %al = *cp */
1947 incl %esi /* cp++ */
1948 movb (%edx, %eax), %al /* %al = table[*cp] */
1949 testb %al, %cl
1950 jz .scanloop /* && (table[*cp] & mask) == 0) */
1951 dec %esi /* post-incremented */
1952 .scandone:
1953 movl %edi, %eax
1954 subl %esi, %eax /* return (end - cp) */
1955 popl %esi
1956 popl %edi
1957 ret
1958 SET_SIZE(scanc)
1959
1960 #endif /* __i386 */
1961 #endif /* __lint */
1962
1963 /*
1964 * Replacement functions for ones that are normally inlined.
1965 * In addition to the copy in i86.il, they are defined here just in case.
1966 */
1967
1968 #if defined(__lint)
1969
1970 ulong_t
1971 intr_clear(void)
1972 { return (0); }
1973
1974 ulong_t
1975 clear_int_flag(void)
1976 { return (0); }
1977
1978 #else /* __lint */
1979
1980 #if defined(__amd64)
1981
1982 ENTRY(intr_clear)
1983 ENTRY(clear_int_flag)
1984 pushfq
1985 popq %rax
1986 #if defined(__xpv)
1987 leaq xpv_panicking, %rdi
1988 movl (%rdi), %edi
1989 cmpl $0, %edi
1990 jne 2f
1991 CLIRET(%rdi, %dl) /* returns event mask in %dl */
1992 /*
1993 * Synthesize the PS_IE bit from the event mask bit
1994 */
1995 andq $_BITNOT(PS_IE), %rax
1996 testb $1, %dl
1997 jnz 1f
1998 orq $PS_IE, %rax
1999 1:
2000 ret
2001 2:
2002 #endif
2003 CLI(%rdi)
2004 ret
2005 SET_SIZE(clear_int_flag)
2006 SET_SIZE(intr_clear)
2007
2008 #elif defined(__i386)
2009
2010 ENTRY(intr_clear)
2011 ENTRY(clear_int_flag)
2012 pushfl
2013 popl %eax
2014 #if defined(__xpv)
2015 leal xpv_panicking, %edx
2016 movl (%edx), %edx
2017 cmpl $0, %edx
2018 jne 2f
2019 CLIRET(%edx, %cl) /* returns event mask in %cl */
2020 /*
2021 * Synthesize the PS_IE bit from the event mask bit
2022 */
2023 andl $_BITNOT(PS_IE), %eax
2024 testb $1, %cl
2025 jnz 1f
2026 orl $PS_IE, %eax
2027 1:
2028 ret
2029 2:
2030 #endif
2031 CLI(%edx)
2032 ret
2033 SET_SIZE(clear_int_flag)
2034 SET_SIZE(intr_clear)
2035
2036 #endif /* __i386 */
2037 #endif /* __lint */
2038
2039 #if defined(__lint)
2040
2041 struct cpu *
2042 curcpup(void)
2043 { return 0; }
2044
2045 #else /* __lint */
2046
2047 #if defined(__amd64)
2048
2049 ENTRY(curcpup)
2050 movq %gs:CPU_SELF, %rax
2051 ret
2052 SET_SIZE(curcpup)
2053
2054 #elif defined(__i386)
2055
2056 ENTRY(curcpup)
2057 movl %gs:CPU_SELF, %eax
2058 ret
2059 SET_SIZE(curcpup)
2060
2061 #endif /* __i386 */
2062 #endif /* __lint */
2063
2064 /* htonll(), ntohll(), htonl(), ntohl(), htons(), ntohs()
2065 * These functions reverse the byte order of the input parameter and returns
2066 * the result. This is to convert the byte order from host byte order
2067 * (little endian) to network byte order (big endian), or vice versa.
2068 */
2069
2070 #if defined(__lint)
2071
2072 uint64_t
2073 htonll(uint64_t i)
2074 { return (i); }
2075
2076 uint64_t
2077 ntohll(uint64_t i)
2078 { return (i); }
2079
2080 uint32_t
2081 htonl(uint32_t i)
2082 { return (i); }
2083
2084 uint32_t
2085 ntohl(uint32_t i)
2086 { return (i); }
2087
2088 uint16_t
2089 htons(uint16_t i)
2090 { return (i); }
2091
2092 uint16_t
2093 ntohs(uint16_t i)
2094 { return (i); }
2095
2096 #else /* __lint */
2097
2098 #if defined(__amd64)
2099
2100 ENTRY(htonll)
2101 ALTENTRY(ntohll)
2102 movq %rdi, %rax
2103 bswapq %rax
2104 ret
2105 SET_SIZE(ntohll)
2106 SET_SIZE(htonll)
2107
2108 /* XX64 there must be shorter sequences for this */
2109 ENTRY(htonl)
2110 ALTENTRY(ntohl)
2111 movl %edi, %eax
2112 bswap %eax
2113 ret
2114 SET_SIZE(ntohl)
2115 SET_SIZE(htonl)
2116
2117 /* XX64 there must be better sequences for this */
2118 ENTRY(htons)
2119 ALTENTRY(ntohs)
2120 movl %edi, %eax
2121 bswap %eax
2122 shrl $16, %eax
2123 ret
2124 SET_SIZE(ntohs)
2125 SET_SIZE(htons)
2126
2127 #elif defined(__i386)
2128
2129 ENTRY(htonll)
2130 ALTENTRY(ntohll)
2131 movl 4(%esp), %edx
2132 movl 8(%esp), %eax
2133 bswap %edx
2134 bswap %eax
2135 ret
2136 SET_SIZE(ntohll)
2137 SET_SIZE(htonll)
2138
2139 ENTRY(htonl)
2140 ALTENTRY(ntohl)
2141 movl 4(%esp), %eax
2142 bswap %eax
2143 ret
2144 SET_SIZE(ntohl)
2145 SET_SIZE(htonl)
2146
2147 ENTRY(htons)
2148 ALTENTRY(ntohs)
2149 movl 4(%esp), %eax
2150 bswap %eax
2151 shrl $16, %eax
2152 ret
2153 SET_SIZE(ntohs)
2154 SET_SIZE(htons)
2155
2156 #endif /* __i386 */
2157 #endif /* __lint */
2158
2159
2160 #if defined(__lint)
2161
2162 /* ARGSUSED */
2163 void
2164 intr_restore(ulong_t i)
2165 { return; }
2166
2167 /* ARGSUSED */
2168 void
2169 restore_int_flag(ulong_t i)
2170 { return; }
2171
2172 #else /* __lint */
2173
2174 #if defined(__amd64)
2175
2176 ENTRY(intr_restore)
2177 ENTRY(restore_int_flag)
2178 testq $PS_IE, %rdi
2179 jz 1f
2180 #if defined(__xpv)
2181 leaq xpv_panicking, %rsi
2182 movl (%rsi), %esi
2183 cmpl $0, %esi
2184 jne 1f
2185 /*
2186 * Since we're -really- running unprivileged, our attempt
2187 * to change the state of the IF bit will be ignored.
2188 * The virtual IF bit is tweaked by CLI and STI.
2189 */
2190 IE_TO_EVENT_MASK(%rsi, %rdi)
2191 #else
2192 sti
2193 #endif
2194 1:
2195 ret
2196 SET_SIZE(restore_int_flag)
2197 SET_SIZE(intr_restore)
2198
2199 #elif defined(__i386)
2200
2201 ENTRY(intr_restore)
2202 ENTRY(restore_int_flag)
2203 testl $PS_IE, 4(%esp)
2204 jz 1f
2205 #if defined(__xpv)
2206 leal xpv_panicking, %edx
2207 movl (%edx), %edx
2208 cmpl $0, %edx
2209 jne 1f
2210 /*
2211 * Since we're -really- running unprivileged, our attempt
2212 * to change the state of the IF bit will be ignored.
2213 * The virtual IF bit is tweaked by CLI and STI.
2214 */
2215 IE_TO_EVENT_MASK(%edx, 4(%esp))
2216 #else
2217 sti
2218 #endif
2219 1:
2220 ret
2221 SET_SIZE(restore_int_flag)
2222 SET_SIZE(intr_restore)
2223
2224 #endif /* __i386 */
2225 #endif /* __lint */
2226
2227 #if defined(__lint)
2228
2229 void
2230 sti(void)
2231 {}
2232
2233 void
2234 cli(void)
2235 {}
2236
2237 #else /* __lint */
2238
2239 ENTRY(sti)
2240 STI
2241 ret
2242 SET_SIZE(sti)
2243
2244 ENTRY(cli)
2245 #if defined(__amd64)
2246 CLI(%rax)
2247 #elif defined(__i386)
2248 CLI(%eax)
2249 #endif /* __i386 */
2250 ret
2251 SET_SIZE(cli)
2252
2253 #endif /* __lint */
2254
2255 #if defined(__lint)
2256
2257 dtrace_icookie_t
2258 dtrace_interrupt_disable(void)
2259 { return (0); }
2260
2261 #else /* __lint */
2262
2263 #if defined(__amd64)
2264
2265 ENTRY(dtrace_interrupt_disable)
2266 pushfq
2267 popq %rax
2268 #if defined(__xpv)
2269 leaq xpv_panicking, %rdi
2270 movl (%rdi), %edi
2271 cmpl $0, %edi
2272 jne .dtrace_interrupt_disable_done
2273 CLIRET(%rdi, %dl) /* returns event mask in %dl */
2274 /*
2275 * Synthesize the PS_IE bit from the event mask bit
2276 */
2277 andq $_BITNOT(PS_IE), %rax
2278 testb $1, %dl
2279 jnz .dtrace_interrupt_disable_done
2280 orq $PS_IE, %rax
2281 #else
2282 CLI(%rdx)
2283 #endif
2284 .dtrace_interrupt_disable_done:
2285 ret
2286 SET_SIZE(dtrace_interrupt_disable)
2287
2288 #elif defined(__i386)
2289
2290 ENTRY(dtrace_interrupt_disable)
2291 pushfl
2292 popl %eax
2293 #if defined(__xpv)
2294 leal xpv_panicking, %edx
2295 movl (%edx), %edx
2296 cmpl $0, %edx
2297 jne .dtrace_interrupt_disable_done
2298 CLIRET(%edx, %cl) /* returns event mask in %cl */
2299 /*
2300 * Synthesize the PS_IE bit from the event mask bit
2301 */
2302 andl $_BITNOT(PS_IE), %eax
2303 testb $1, %cl
2304 jnz .dtrace_interrupt_disable_done
2305 orl $PS_IE, %eax
2306 #else
2307 CLI(%edx)
2308 #endif
2309 .dtrace_interrupt_disable_done:
2310 ret
2311 SET_SIZE(dtrace_interrupt_disable)
2312
2313 #endif /* __i386 */
2314 #endif /* __lint */
2315
2316 #if defined(__lint)
2317
2318 /*ARGSUSED*/
2319 void
2320 dtrace_interrupt_enable(dtrace_icookie_t cookie)
2321 {}
2322
2323 #else /* __lint */
2324
2325 #if defined(__amd64)
2326
2327 ENTRY(dtrace_interrupt_enable)
2328 pushq %rdi
2329 popfq
2330 #if defined(__xpv)
2331 leaq xpv_panicking, %rdx
2332 movl (%rdx), %edx
2333 cmpl $0, %edx
2334 jne .dtrace_interrupt_enable_done
2335 /*
2336 * Since we're -really- running unprivileged, our attempt
2337 * to change the state of the IF bit will be ignored. The
2338 * virtual IF bit is tweaked by CLI and STI.
2339 */
2340 IE_TO_EVENT_MASK(%rdx, %rdi)
2341 #endif
2342 .dtrace_interrupt_enable_done:
2343 ret
2344 SET_SIZE(dtrace_interrupt_enable)
2345
2346 #elif defined(__i386)
2347
2348 ENTRY(dtrace_interrupt_enable)
2349 movl 4(%esp), %eax
2350 pushl %eax
2351 popfl
2352 #if defined(__xpv)
2353 leal xpv_panicking, %edx
2354 movl (%edx), %edx
2355 cmpl $0, %edx
2356 jne .dtrace_interrupt_enable_done
2357 /*
2358 * Since we're -really- running unprivileged, our attempt
2359 * to change the state of the IF bit will be ignored. The
2360 * virtual IF bit is tweaked by CLI and STI.
2361 */
2362 IE_TO_EVENT_MASK(%edx, %eax)
2363 #endif
2364 .dtrace_interrupt_enable_done:
2365 ret
2366 SET_SIZE(dtrace_interrupt_enable)
2367
2368 #endif /* __i386 */
2369 #endif /* __lint */
2370
2371
2372 #if defined(lint)
2373
2374 void
2375 dtrace_membar_producer(void)
2376 {}
2377
2378 void
2379 dtrace_membar_consumer(void)
2380 {}
2381
2382 #else /* __lint */
2383
2384 ENTRY(dtrace_membar_producer)
2385 rep; ret /* use 2 byte return instruction when branch target */
2386 /* AMD Software Optimization Guide - Section 6.2 */
2387 SET_SIZE(dtrace_membar_producer)
2388
2389 ENTRY(dtrace_membar_consumer)
2390 rep; ret /* use 2 byte return instruction when branch target */
2391 /* AMD Software Optimization Guide - Section 6.2 */
2392 SET_SIZE(dtrace_membar_consumer)
2393
2394 #endif /* __lint */
2395
2396 #if defined(__lint)
2397
2398 kthread_id_t
2399 threadp(void)
2400 { return ((kthread_id_t)0); }
2401
2402 #else /* __lint */
2403
2404 #if defined(__amd64)
2405
2406 ENTRY(threadp)
2407 movq %gs:CPU_THREAD, %rax
2408 ret
2409 SET_SIZE(threadp)
2410
2411 #elif defined(__i386)
2412
2413 ENTRY(threadp)
2414 movl %gs:CPU_THREAD, %eax
2415 ret
2416 SET_SIZE(threadp)
2417
2418 #endif /* __i386 */
2419 #endif /* __lint */
2420
2421 /*
2422 * Checksum routine for Internet Protocol Headers
2423 */
2424
2425 #if defined(__lint)
2426
2427 /* ARGSUSED */
2428 unsigned int
2429 ip_ocsum(
2430 ushort_t *address, /* ptr to 1st message buffer */
2431 int halfword_count, /* length of data */
2432 unsigned int sum) /* partial checksum */
2433 {
2434 int i;
2435 unsigned int psum = 0; /* partial sum */
2436
2437 for (i = 0; i < halfword_count; i++, address++) {
2438 psum += *address;
2439 }
2440
2441 while ((psum >> 16) != 0) {
2442 psum = (psum & 0xffff) + (psum >> 16);
2443 }
2444
2445 psum += sum;
2446
2447 while ((psum >> 16) != 0) {
2448 psum = (psum & 0xffff) + (psum >> 16);
2449 }
2450
2451 return (psum);
2452 }
2453
2454 #else /* __lint */
2455
2456 #if defined(__amd64)
2457
2458 ENTRY(ip_ocsum)
2459 pushq %rbp
2460 movq %rsp, %rbp
2461 #ifdef DEBUG
2462 movq postbootkernelbase(%rip), %rax
2463 cmpq %rax, %rdi
2464 jnb 1f
2465 xorl %eax, %eax
2466 movq %rdi, %rsi
2467 leaq .ip_ocsum_panic_msg(%rip), %rdi
2468 call panic
2469 /*NOTREACHED*/
2470 .ip_ocsum_panic_msg:
2471 .string "ip_ocsum: address 0x%p below kernelbase\n"
2472 1:
2473 #endif
2474 movl %esi, %ecx /* halfword_count */
2475 movq %rdi, %rsi /* address */
2476 /* partial sum in %edx */
2477 xorl %eax, %eax
2478 testl %ecx, %ecx
2479 jz .ip_ocsum_done
2480 testq $3, %rsi
2481 jnz .ip_csum_notaligned
2482 .ip_csum_aligned: /* XX64 opportunities for 8-byte operations? */
2483 .next_iter:
2484 /* XX64 opportunities for prefetch? */
2485 /* XX64 compute csum with 64 bit quantities? */
2486 subl $32, %ecx
2487 jl .less_than_32
2488
2489 addl 0(%rsi), %edx
2490 .only60:
2491 adcl 4(%rsi), %eax
2492 .only56:
2493 adcl 8(%rsi), %edx
2494 .only52:
2495 adcl 12(%rsi), %eax
2496 .only48:
2497 adcl 16(%rsi), %edx
2498 .only44:
2499 adcl 20(%rsi), %eax
2500 .only40:
2501 adcl 24(%rsi), %edx
2502 .only36:
2503 adcl 28(%rsi), %eax
2504 .only32:
2505 adcl 32(%rsi), %edx
2506 .only28:
2507 adcl 36(%rsi), %eax
2508 .only24:
2509 adcl 40(%rsi), %edx
2510 .only20:
2511 adcl 44(%rsi), %eax
2512 .only16:
2513 adcl 48(%rsi), %edx
2514 .only12:
2515 adcl 52(%rsi), %eax
2516 .only8:
2517 adcl 56(%rsi), %edx
2518 .only4:
2519 adcl 60(%rsi), %eax /* could be adding -1 and -1 with a carry */
2520 .only0:
2521 adcl $0, %eax /* could be adding -1 in eax with a carry */
2522 adcl $0, %eax
2523
2524 addq $64, %rsi
2525 testl %ecx, %ecx
2526 jnz .next_iter
2527
2528 .ip_ocsum_done:
2529 addl %eax, %edx
2530 adcl $0, %edx
2531 movl %edx, %eax /* form a 16 bit checksum by */
2532 shrl $16, %eax /* adding two halves of 32 bit checksum */
2533 addw %dx, %ax
2534 adcw $0, %ax
2535 andl $0xffff, %eax
2536 leave
2537 ret
2538
2539 .ip_csum_notaligned:
2540 xorl %edi, %edi
2541 movw (%rsi), %di
2542 addl %edi, %edx
2543 adcl $0, %edx
2544 addq $2, %rsi
2545 decl %ecx
2546 jmp .ip_csum_aligned
2547
2548 .less_than_32:
2549 addl $32, %ecx
2550 testl $1, %ecx
2551 jz .size_aligned
2552 andl $0xfe, %ecx
2553 movzwl (%rsi, %rcx, 2), %edi
2554 addl %edi, %edx
2555 adcl $0, %edx
2556 .size_aligned:
2557 movl %ecx, %edi
2558 shrl $1, %ecx
2559 shl $1, %edi
2560 subq $64, %rdi
2561 addq %rdi, %rsi
2562 leaq .ip_ocsum_jmptbl(%rip), %rdi
2563 leaq (%rdi, %rcx, 8), %rdi
2564 xorl %ecx, %ecx
2565 clc
2566 jmp *(%rdi)
2567
2568 .align 8
2569 .ip_ocsum_jmptbl:
2570 .quad .only0, .only4, .only8, .only12, .only16, .only20
2571 .quad .only24, .only28, .only32, .only36, .only40, .only44
2572 .quad .only48, .only52, .only56, .only60
2573 SET_SIZE(ip_ocsum)
2574
2575 #elif defined(__i386)
2576
2577 ENTRY(ip_ocsum)
2578 pushl %ebp
2579 movl %esp, %ebp
2580 pushl %ebx
2581 pushl %esi
2582 pushl %edi
2583 movl 12(%ebp), %ecx /* count of half words */
2584 movl 16(%ebp), %edx /* partial checksum */
2585 movl 8(%ebp), %esi
2586 xorl %eax, %eax
2587 testl %ecx, %ecx
2588 jz .ip_ocsum_done
2589
2590 testl $3, %esi
2591 jnz .ip_csum_notaligned
2592 .ip_csum_aligned:
2593 .next_iter:
2594 subl $32, %ecx
2595 jl .less_than_32
2596
2597 addl 0(%esi), %edx
2598 .only60:
2599 adcl 4(%esi), %eax
2600 .only56:
2601 adcl 8(%esi), %edx
2602 .only52:
2603 adcl 12(%esi), %eax
2604 .only48:
2605 adcl 16(%esi), %edx
2606 .only44:
2607 adcl 20(%esi), %eax
2608 .only40:
2609 adcl 24(%esi), %edx
2610 .only36:
2611 adcl 28(%esi), %eax
2612 .only32:
2613 adcl 32(%esi), %edx
2614 .only28:
2615 adcl 36(%esi), %eax
2616 .only24:
2617 adcl 40(%esi), %edx
2618 .only20:
2619 adcl 44(%esi), %eax
2620 .only16:
2621 adcl 48(%esi), %edx
2622 .only12:
2623 adcl 52(%esi), %eax
2624 .only8:
2625 adcl 56(%esi), %edx
2626 .only4:
2627 adcl 60(%esi), %eax /* We could be adding -1 and -1 with a carry */
2628 .only0:
2629 adcl $0, %eax /* we could be adding -1 in eax with a carry */
2630 adcl $0, %eax
2631
2632 addl $64, %esi
2633 andl %ecx, %ecx
2634 jnz .next_iter
2635
2636 .ip_ocsum_done:
2637 addl %eax, %edx
2638 adcl $0, %edx
2639 movl %edx, %eax /* form a 16 bit checksum by */
2640 shrl $16, %eax /* adding two halves of 32 bit checksum */
2641 addw %dx, %ax
2642 adcw $0, %ax
2643 andl $0xffff, %eax
2644 popl %edi /* restore registers */
2645 popl %esi
2646 popl %ebx
2647 leave
2648 ret
2649
2650 .ip_csum_notaligned:
2651 xorl %edi, %edi
2652 movw (%esi), %di
2653 addl %edi, %edx
2654 adcl $0, %edx
2655 addl $2, %esi
2656 decl %ecx
2657 jmp .ip_csum_aligned
2658
2659 .less_than_32:
2660 addl $32, %ecx
2661 testl $1, %ecx
2662 jz .size_aligned
2663 andl $0xfe, %ecx
2664 movzwl (%esi, %ecx, 2), %edi
2665 addl %edi, %edx
2666 adcl $0, %edx
2667 .size_aligned:
2668 movl %ecx, %edi
2669 shrl $1, %ecx
2670 shl $1, %edi
2671 subl $64, %edi
2672 addl %edi, %esi
2673 movl $.ip_ocsum_jmptbl, %edi
2674 lea (%edi, %ecx, 4), %edi
2675 xorl %ecx, %ecx
2676 clc
2677 jmp *(%edi)
2678 SET_SIZE(ip_ocsum)
2679
2680 .data
2681 .align 4
2682
2683 .ip_ocsum_jmptbl:
2684 .long .only0, .only4, .only8, .only12, .only16, .only20
2685 .long .only24, .only28, .only32, .only36, .only40, .only44
2686 .long .only48, .only52, .only56, .only60
2687
2688
2689 #endif /* __i386 */
2690 #endif /* __lint */
2691
2692 /*
2693 * multiply two long numbers and yield a u_longlong_t result, callable from C.
2694 * Provided to manipulate hrtime_t values.
2695 */
2696 #if defined(__lint)
2697
2698 /* result = a * b; */
2699
2700 /* ARGSUSED */
2701 unsigned long long
2702 mul32(uint_t a, uint_t b)
2703 { return (0); }
2704
2705 #else /* __lint */
2706
2707 #if defined(__amd64)
2708
2709 ENTRY(mul32)
2710 xorl %edx, %edx /* XX64 joe, paranoia? */
2711 movl %edi, %eax
2712 mull %esi
2713 shlq $32, %rdx
2714 orq %rdx, %rax
2715 ret
2716 SET_SIZE(mul32)
2717
2718 #elif defined(__i386)
2719
2720 ENTRY(mul32)
2721 movl 8(%esp), %eax
2722 movl 4(%esp), %ecx
2723 mull %ecx
2724 ret
2725 SET_SIZE(mul32)
2726
2727 #endif /* __i386 */
2728 #endif /* __lint */
2729
2730 #if defined(notused)
2731 #if defined(__lint)
2732 /* ARGSUSED */
2733 void
2734 load_pte64(uint64_t *pte, uint64_t pte_value)
2735 {}
2736 #else /* __lint */
2737 .globl load_pte64
2738 load_pte64:
2739 movl 4(%esp), %eax
2740 movl 8(%esp), %ecx
2741 movl 12(%esp), %edx
2742 movl %edx, 4(%eax)
2743 movl %ecx, (%eax)
2744 ret
2745 #endif /* __lint */
2746 #endif /* notused */
2747
2748 #if defined(__lint)
2749
2750 /*ARGSUSED*/
2751 void
2752 scan_memory(caddr_t addr, size_t size)
2753 {}
2754
2755 #else /* __lint */
2756
2757 #if defined(__amd64)
2758
2759 ENTRY(scan_memory)
2760 shrq $3, %rsi /* convert %rsi from byte to quadword count */
2761 jz .scanm_done
2762 movq %rsi, %rcx /* move count into rep control register */
2763 movq %rdi, %rsi /* move addr into lodsq control reg. */
2764 rep lodsq /* scan the memory range */
2765 .scanm_done:
2766 rep; ret /* use 2 byte return instruction when branch target */
2767 /* AMD Software Optimization Guide - Section 6.2 */
2768 SET_SIZE(scan_memory)
2769
2770 #elif defined(__i386)
2771
2772 ENTRY(scan_memory)
2773 pushl %ecx
2774 pushl %esi
2775 movl 16(%esp), %ecx /* move 2nd arg into rep control register */
2776 shrl $2, %ecx /* convert from byte count to word count */
2777 jz .scanm_done
2778 movl 12(%esp), %esi /* move 1st arg into lodsw control register */
2779 .byte 0xf3 /* rep prefix. lame assembler. sigh. */
2780 lodsl
2781 .scanm_done:
2782 popl %esi
2783 popl %ecx
2784 ret
2785 SET_SIZE(scan_memory)
2786
2787 #endif /* __i386 */
2788 #endif /* __lint */
2789
2790
2791 #if defined(__lint)
2792
2793 /*ARGSUSED */
2794 int
2795 lowbit(ulong_t i)
2796 { return (0); }
2797
2798 #else /* __lint */
2799
2800 #if defined(__amd64)
2801
2802 ENTRY(lowbit)
2803 movl $-1, %eax
2804 bsfq %rdi, %rax
2805 incl %eax
2806 ret
2807 SET_SIZE(lowbit)
2808
2809 #elif defined(__i386)
2810
2811 ENTRY(lowbit)
2812 movl $-1, %eax
2813 bsfl 4(%esp), %eax
2814 incl %eax
2815 ret
2816 SET_SIZE(lowbit)
2817
2818 #endif /* __i386 */
2819 #endif /* __lint */
2820
2821 #if defined(__lint)
2822
2823 /*ARGSUSED*/
2824 int
2825 highbit(ulong_t i)
2826 { return (0); }
2827
2828 #else /* __lint */
2829
2830 #if defined(__amd64)
2831
2832 ENTRY(highbit)
2833 movl $-1, %eax
2834 bsrq %rdi, %rax
2835 incl %eax
2836 ret
2837 SET_SIZE(highbit)
2838
2839 #elif defined(__i386)
2840
2841 ENTRY(highbit)
2842 movl $-1, %eax
2843 bsrl 4(%esp), %eax
2844 incl %eax
2845 ret
2846 SET_SIZE(highbit)
2847
2848 #endif /* __i386 */
2849 #endif /* __lint */
2850
2851 #if defined(__lint)
2852
2853 /*ARGSUSED*/
2854 int
2855 highbit64(uint64_t i)
2856 { return (0); }
2857
2858 #else /* __lint */
2859
2860 #if defined(__amd64)
2861
2862 ENTRY(highbit64)
2863 movl $-1, %eax
2864 bsrq %rdi, %rax
2865 incl %eax
2866 ret
2867 SET_SIZE(highbit64)
2868
2869 #elif defined(__i386)
2870
2871 ENTRY(highbit64)
2872 bsrl 8(%esp), %eax
2873 jz .lowbit
2874 addl $32, %eax
2875 jmp .done
2876
2877 .lowbit:
2878 movl $-1, %eax
2879 bsrl 4(%esp), %eax
2880 .done:
2881 incl %eax
2882 ret
2883 SET_SIZE(highbit64)
2884
2885 #endif /* __i386 */
2886 #endif /* __lint */
2887
2888 #if defined(__lint)
2889
2890 /*ARGSUSED*/
2891 uint64_t
2892 rdmsr(uint_t r)
2893 { return (0); }
2894
2895 /*ARGSUSED*/
2896 void
2897 wrmsr(uint_t r, const uint64_t val)
2898 {}
2899
2900 /*ARGSUSED*/
2901 uint64_t
2902 xrdmsr(uint_t r)
2903 { return (0); }
2904
2905 /*ARGSUSED*/
2906 void
2907 xwrmsr(uint_t r, const uint64_t val)
2908 {}
2909
2910 void
2911 invalidate_cache(void)
2912 {}
2913
2914 /*ARGSUSED*/
2915 uint64_t
2916 get_xcr(uint_t r)
2917 { return (0); }
2918
2919 /*ARGSUSED*/
2920 void
2921 set_xcr(uint_t r, const uint64_t val)
2922 {}
2923
2924 #else /* __lint */
2925
2926 #define XMSR_ACCESS_VAL $0x9c5a203a
2927
2928 #if defined(__amd64)
2929
2930 ENTRY(rdmsr)
2931 movl %edi, %ecx
2932 rdmsr
2933 shlq $32, %rdx
2934 orq %rdx, %rax
2935 ret
2936 SET_SIZE(rdmsr)
2937
2938 ENTRY(wrmsr)
2939 movq %rsi, %rdx
2940 shrq $32, %rdx
2941 movl %esi, %eax
2942 movl %edi, %ecx
2943 wrmsr
2944 ret
2945 SET_SIZE(wrmsr)
2946
2947 ENTRY(xrdmsr)
2948 pushq %rbp
2949 movq %rsp, %rbp
2950 movl %edi, %ecx
2951 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2952 rdmsr
2953 shlq $32, %rdx
2954 orq %rdx, %rax
2955 leave
2956 ret
2957 SET_SIZE(xrdmsr)
2958
2959 ENTRY(xwrmsr)
2960 pushq %rbp
2961 movq %rsp, %rbp
2962 movl %edi, %ecx
2963 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
2964 movq %rsi, %rdx
2965 shrq $32, %rdx
2966 movl %esi, %eax
2967 wrmsr
2968 leave
2969 ret
2970 SET_SIZE(xwrmsr)
2971
2972 ENTRY(get_xcr)
2973 movl %edi, %ecx
2974 #xgetbv
2975 .byte 0x0f,0x01,0xd0
2976 shlq $32, %rdx
2977 orq %rdx, %rax
2978 ret
2979 SET_SIZE(get_xcr)
2980
2981 ENTRY(set_xcr)
2982 movq %rsi, %rdx
2983 shrq $32, %rdx
2984 movl %esi, %eax
2985 movl %edi, %ecx
2986 #xsetbv
2987 .byte 0x0f,0x01,0xd1
2988 ret
2989 SET_SIZE(set_xcr)
2990
2991 #elif defined(__i386)
2992
2993 ENTRY(rdmsr)
2994 movl 4(%esp), %ecx
2995 rdmsr
2996 ret
2997 SET_SIZE(rdmsr)
2998
2999 ENTRY(wrmsr)
3000 movl 4(%esp), %ecx
3001 movl 8(%esp), %eax
3002 movl 12(%esp), %edx
3003 wrmsr
3004 ret
3005 SET_SIZE(wrmsr)
3006
3007 ENTRY(xrdmsr)
3008 pushl %ebp
3009 movl %esp, %ebp
3010 movl 8(%esp), %ecx
3011 pushl %edi
3012 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
3013 rdmsr
3014 popl %edi
3015 leave
3016 ret
3017 SET_SIZE(xrdmsr)
3018
3019 ENTRY(xwrmsr)
3020 pushl %ebp
3021 movl %esp, %ebp
3022 movl 8(%esp), %ecx
3023 movl 12(%esp), %eax
3024 movl 16(%esp), %edx
3025 pushl %edi
3026 movl XMSR_ACCESS_VAL, %edi /* this value is needed to access MSR */
3027 wrmsr
3028 popl %edi
3029 leave
3030 ret
3031 SET_SIZE(xwrmsr)
3032
3033 ENTRY(get_xcr)
3034 movl 4(%esp), %ecx
3035 #xgetbv
3036 .byte 0x0f,0x01,0xd0
3037 ret
3038 SET_SIZE(get_xcr)
3039
3040 ENTRY(set_xcr)
3041 movl 4(%esp), %ecx
3042 movl 8(%esp), %eax
3043 movl 12(%esp), %edx
3044 #xsetbv
3045 .byte 0x0f,0x01,0xd1
3046 ret
3047 SET_SIZE(set_xcr)
3048
3049 #endif /* __i386 */
3050
3051 ENTRY(invalidate_cache)
3052 wbinvd
3053 ret
3054 SET_SIZE(invalidate_cache)
3055
3056 #endif /* __lint */
3057
3058 #if defined(__lint)
3059
3060 /*ARGSUSED*/
3061 void
3062 getcregs(struct cregs *crp)
3063 {}
3064
3065 #else /* __lint */
3066
3067 #if defined(__amd64)
3068
3069 ENTRY_NP(getcregs)
3070 #if defined(__xpv)
3071 /*
3072 * Only a few of the hardware control registers or descriptor tables
3073 * are directly accessible to us, so just zero the structure.
3074 *
3075 * XXPV Perhaps it would be helpful for the hypervisor to return
3076 * virtualized versions of these for post-mortem use.
3077 * (Need to reevaluate - perhaps it already does!)
3078 */
3079 pushq %rdi /* save *crp */
3080 movq $CREGSZ, %rsi
3081 call bzero
3082 popq %rdi
3083
3084 /*
3085 * Dump what limited information we can
3086 */
3087 movq %cr0, %rax
3088 movq %rax, CREG_CR0(%rdi) /* cr0 */
3089 movq %cr2, %rax
3090 movq %rax, CREG_CR2(%rdi) /* cr2 */
3091 movq %cr3, %rax
3092 movq %rax, CREG_CR3(%rdi) /* cr3 */
3093 movq %cr4, %rax
3094 movq %rax, CREG_CR4(%rdi) /* cr4 */
3095
3096 #else /* __xpv */
3097
3098 #define GETMSR(r, off, d) \
3099 movl $r, %ecx; \
3100 rdmsr; \
3101 movl %eax, off(d); \
3102 movl %edx, off+4(d)
3103
3104 xorl %eax, %eax
3105 movq %rax, CREG_GDT+8(%rdi)
3106 sgdt CREG_GDT(%rdi) /* 10 bytes */
3107 movq %rax, CREG_IDT+8(%rdi)
3108 sidt CREG_IDT(%rdi) /* 10 bytes */
3109 movq %rax, CREG_LDT(%rdi)
3110 sldt CREG_LDT(%rdi) /* 2 bytes */
3111 movq %rax, CREG_TASKR(%rdi)
3112 str CREG_TASKR(%rdi) /* 2 bytes */
3113 movq %cr0, %rax
3114 movq %rax, CREG_CR0(%rdi) /* cr0 */
3115 movq %cr2, %rax
3116 movq %rax, CREG_CR2(%rdi) /* cr2 */
3117 movq %cr3, %rax
3118 movq %rax, CREG_CR3(%rdi) /* cr3 */
3119 movq %cr4, %rax
3120 movq %rax, CREG_CR4(%rdi) /* cr4 */
3121 movq %cr8, %rax
3122 movq %rax, CREG_CR8(%rdi) /* cr8 */
3123 GETMSR(MSR_AMD_KGSBASE, CREG_KGSBASE, %rdi)
3124 GETMSR(MSR_AMD_EFER, CREG_EFER, %rdi)
3125 #endif /* __xpv */
3126 ret
3127 SET_SIZE(getcregs)
3128
3129 #undef GETMSR
3130
3131 #elif defined(__i386)
3132
3133 ENTRY_NP(getcregs)
3134 #if defined(__xpv)
3135 /*
3136 * Only a few of the hardware control registers or descriptor tables
3137 * are directly accessible to us, so just zero the structure.
3138 *
3139 * XXPV Perhaps it would be helpful for the hypervisor to return
3140 * virtualized versions of these for post-mortem use.
3141 * (Need to reevaluate - perhaps it already does!)
3142 */
3143 movl 4(%esp), %edx
3144 pushl $CREGSZ
3145 pushl %edx
3146 call bzero
3147 addl $8, %esp
3148 movl 4(%esp), %edx
3149
3150 /*
3151 * Dump what limited information we can
3152 */
3153 movl %cr0, %eax
3154 movl %eax, CREG_CR0(%edx) /* cr0 */
3155 movl %cr2, %eax
3156 movl %eax, CREG_CR2(%edx) /* cr2 */
3157 movl %cr3, %eax
3158 movl %eax, CREG_CR3(%edx) /* cr3 */
3159 movl %cr4, %eax
3160 movl %eax, CREG_CR4(%edx) /* cr4 */
3161
3162 #else /* __xpv */
3163
3164 movl 4(%esp), %edx
3165 movw $0, CREG_GDT+6(%edx)
3166 movw $0, CREG_IDT+6(%edx)
3167 sgdt CREG_GDT(%edx) /* gdt */
3168 sidt CREG_IDT(%edx) /* idt */
3169 sldt CREG_LDT(%edx) /* ldt */
3170 str CREG_TASKR(%edx) /* task */
3171 movl %cr0, %eax
3172 movl %eax, CREG_CR0(%edx) /* cr0 */
3173 movl %cr2, %eax
3174 movl %eax, CREG_CR2(%edx) /* cr2 */
3175 movl %cr3, %eax
3176 movl %eax, CREG_CR3(%edx) /* cr3 */
3177 bt $X86FSET_LARGEPAGE, x86_featureset
3178 jnc .nocr4
3179 movl %cr4, %eax
3180 movl %eax, CREG_CR4(%edx) /* cr4 */
3181 jmp .skip
3182 .nocr4:
3183 movl $0, CREG_CR4(%edx)
3184 .skip:
3185 #endif
3186 ret
3187 SET_SIZE(getcregs)
3188
3189 #endif /* __i386 */
3190 #endif /* __lint */
3191
3192
3193 /*
3194 * A panic trigger is a word which is updated atomically and can only be set
3195 * once. We atomically store 0xDEFACEDD and load the old value. If the
3196 * previous value was 0, we succeed and return 1; otherwise return 0.
3197 * This allows a partially corrupt trigger to still trigger correctly. DTrace
3198 * has its own version of this function to allow it to panic correctly from
3199 * probe context.
3200 */
3201 #if defined(__lint)
3202
3203 /*ARGSUSED*/
3204 int
3205 panic_trigger(int *tp)
3206 { return (0); }
3207
3208 /*ARGSUSED*/
3209 int
3210 dtrace_panic_trigger(int *tp)
3211 { return (0); }
3212
3213 #else /* __lint */
3214
3215 #if defined(__amd64)
3216
3217 ENTRY_NP(panic_trigger)
3218 xorl %eax, %eax
3219 movl $0xdefacedd, %edx
3220 lock
3221 xchgl %edx, (%rdi)
3222 cmpl $0, %edx
3223 je 0f
3224 movl $0, %eax
3225 ret
3226 0: movl $1, %eax
3227 ret
3228 SET_SIZE(panic_trigger)
3229
3230 ENTRY_NP(dtrace_panic_trigger)
3231 xorl %eax, %eax
3232 movl $0xdefacedd, %edx
3233 lock
3234 xchgl %edx, (%rdi)
3235 cmpl $0, %edx
3236 je 0f
3237 movl $0, %eax
3238 ret
3239 0: movl $1, %eax
3240 ret
3241 SET_SIZE(dtrace_panic_trigger)
3242
3243 #elif defined(__i386)
3244
3245 ENTRY_NP(panic_trigger)
3246 movl 4(%esp), %edx / %edx = address of trigger
3247 movl $0xdefacedd, %eax / %eax = 0xdefacedd
3248 lock / assert lock
3249 xchgl %eax, (%edx) / exchange %eax and the trigger
3250 cmpl $0, %eax / if (%eax == 0x0)
3251 je 0f / return (1);
3252 movl $0, %eax / else
3253 ret / return (0);
3254 0: movl $1, %eax
3255 ret
3256 SET_SIZE(panic_trigger)
3257
3258 ENTRY_NP(dtrace_panic_trigger)
3259 movl 4(%esp), %edx / %edx = address of trigger
3260 movl $0xdefacedd, %eax / %eax = 0xdefacedd
3261 lock / assert lock
3262 xchgl %eax, (%edx) / exchange %eax and the trigger
3263 cmpl $0, %eax / if (%eax == 0x0)
3264 je 0f / return (1);
3265 movl $0, %eax / else
3266 ret / return (0);
3267 0: movl $1, %eax
3268 ret
3269 SET_SIZE(dtrace_panic_trigger)
3270
3271 #endif /* __i386 */
3272 #endif /* __lint */
3273
3274 /*
3275 * The panic() and cmn_err() functions invoke vpanic() as a common entry point
3276 * into the panic code implemented in panicsys(). vpanic() is responsible
3277 * for passing through the format string and arguments, and constructing a
3278 * regs structure on the stack into which it saves the current register
3279 * values. If we are not dying due to a fatal trap, these registers will
3280 * then be preserved in panicbuf as the current processor state. Before
3281 * invoking panicsys(), vpanic() activates the first panic trigger (see
3282 * common/os/panic.c) and switches to the panic_stack if successful. Note that
3283 * DTrace takes a slightly different panic path if it must panic from probe
3284 * context. Instead of calling panic, it calls into dtrace_vpanic(), which
3285 * sets up the initial stack as vpanic does, calls dtrace_panic_trigger(), and
3286 * branches back into vpanic().
3287 */
3288 #if defined(__lint)
3289
3290 /*ARGSUSED*/
3291 void
3292 vpanic(const char *format, va_list alist)
3293 {}
3294
3295 /*ARGSUSED*/
3296 void
3297 dtrace_vpanic(const char *format, va_list alist)
3298 {}
3299
3300 #else /* __lint */
3301
3302 #if defined(__amd64)
3303
3304 ENTRY_NP(vpanic) /* Initial stack layout: */
3305
3306 pushq %rbp /* | %rip | 0x60 */
3307 movq %rsp, %rbp /* | %rbp | 0x58 */
3308 pushfq /* | rfl | 0x50 */
3309 pushq %r11 /* | %r11 | 0x48 */
3310 pushq %r10 /* | %r10 | 0x40 */
3311 pushq %rbx /* | %rbx | 0x38 */
3312 pushq %rax /* | %rax | 0x30 */
3313 pushq %r9 /* | %r9 | 0x28 */
3314 pushq %r8 /* | %r8 | 0x20 */
3315 pushq %rcx /* | %rcx | 0x18 */
3316 pushq %rdx /* | %rdx | 0x10 */
3317 pushq %rsi /* | %rsi | 0x8 alist */
3318 pushq %rdi /* | %rdi | 0x0 format */
3319
3320 movq %rsp, %rbx /* %rbx = current %rsp */
3321
3322 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */
3323 call panic_trigger /* %eax = panic_trigger() */
3324
3325 vpanic_common:
3326 /*
3327 * The panic_trigger result is in %eax from the call above, and
3328 * dtrace_panic places it in %eax before branching here.
3329 * The rdmsr instructions that follow below will clobber %eax so
3330 * we stash the panic_trigger result in %r11d.
3331 */
3332 movl %eax, %r11d
3333 cmpl $0, %r11d
3334 je 0f
3335
3336 /*
3337 * If panic_trigger() was successful, we are the first to initiate a
3338 * panic: we now switch to the reserved panic_stack before continuing.
3339 */
3340 leaq panic_stack(%rip), %rsp
3341 addq $PANICSTKSIZE, %rsp
3342 0: subq $REGSIZE, %rsp
3343 /*
3344 * Now that we've got everything set up, store the register values as
3345 * they were when we entered vpanic() to the designated location in
3346 * the regs structure we allocated on the stack.
3347 */
3348 movq 0x0(%rbx), %rcx
3349 movq %rcx, REGOFF_RDI(%rsp)
3350 movq 0x8(%rbx), %rcx
3351 movq %rcx, REGOFF_RSI(%rsp)
3352 movq 0x10(%rbx), %rcx
3353 movq %rcx, REGOFF_RDX(%rsp)
3354 movq 0x18(%rbx), %rcx
3355 movq %rcx, REGOFF_RCX(%rsp)
3356 movq 0x20(%rbx), %rcx
3357
3358 movq %rcx, REGOFF_R8(%rsp)
3359 movq 0x28(%rbx), %rcx
3360 movq %rcx, REGOFF_R9(%rsp)
3361 movq 0x30(%rbx), %rcx
3362 movq %rcx, REGOFF_RAX(%rsp)
3363 movq 0x38(%rbx), %rcx
3364 movq %rcx, REGOFF_RBX(%rsp)
3365 movq 0x58(%rbx), %rcx
3366
3367 movq %rcx, REGOFF_RBP(%rsp)
3368 movq 0x40(%rbx), %rcx
3369 movq %rcx, REGOFF_R10(%rsp)
3370 movq 0x48(%rbx), %rcx
3371 movq %rcx, REGOFF_R11(%rsp)
3372 movq %r12, REGOFF_R12(%rsp)
3373
3374 movq %r13, REGOFF_R13(%rsp)
3375 movq %r14, REGOFF_R14(%rsp)
3376 movq %r15, REGOFF_R15(%rsp)
3377
3378 xorl %ecx, %ecx
3379 movw %ds, %cx
3380 movq %rcx, REGOFF_DS(%rsp)
3381 movw %es, %cx
3382 movq %rcx, REGOFF_ES(%rsp)
3383 movw %fs, %cx
3384 movq %rcx, REGOFF_FS(%rsp)
3385 movw %gs, %cx
3386 movq %rcx, REGOFF_GS(%rsp)
3387
3388 movq $0, REGOFF_TRAPNO(%rsp)
3389
3390 movq $0, REGOFF_ERR(%rsp)
3391 leaq vpanic(%rip), %rcx
3392 movq %rcx, REGOFF_RIP(%rsp)
3393 movw %cs, %cx
3394 movzwq %cx, %rcx
3395 movq %rcx, REGOFF_CS(%rsp)
3396 movq 0x50(%rbx), %rcx
3397 movq %rcx, REGOFF_RFL(%rsp)
3398 movq %rbx, %rcx
3399 addq $0x60, %rcx
3400 movq %rcx, REGOFF_RSP(%rsp)
3401 movw %ss, %cx
3402 movzwq %cx, %rcx
3403 movq %rcx, REGOFF_SS(%rsp)
3404
3405 /*
3406 * panicsys(format, alist, rp, on_panic_stack)
3407 */
3408 movq REGOFF_RDI(%rsp), %rdi /* format */
3409 movq REGOFF_RSI(%rsp), %rsi /* alist */
3410 movq %rsp, %rdx /* struct regs */
3411 movl %r11d, %ecx /* on_panic_stack */
3412 call panicsys
3413 addq $REGSIZE, %rsp
3414 popq %rdi
3415 popq %rsi
3416 popq %rdx
3417 popq %rcx
3418 popq %r8
3419 popq %r9
3420 popq %rax
3421 popq %rbx
3422 popq %r10
3423 popq %r11
3424 popfq
3425 leave
3426 ret
3427 SET_SIZE(vpanic)
3428
3429 ENTRY_NP(dtrace_vpanic) /* Initial stack layout: */
3430
3431 pushq %rbp /* | %rip | 0x60 */
3432 movq %rsp, %rbp /* | %rbp | 0x58 */
3433 pushfq /* | rfl | 0x50 */
3434 pushq %r11 /* | %r11 | 0x48 */
3435 pushq %r10 /* | %r10 | 0x40 */
3436 pushq %rbx /* | %rbx | 0x38 */
3437 pushq %rax /* | %rax | 0x30 */
3438 pushq %r9 /* | %r9 | 0x28 */
3439 pushq %r8 /* | %r8 | 0x20 */
3440 pushq %rcx /* | %rcx | 0x18 */
3441 pushq %rdx /* | %rdx | 0x10 */
3442 pushq %rsi /* | %rsi | 0x8 alist */
3443 pushq %rdi /* | %rdi | 0x0 format */
3444
3445 movq %rsp, %rbx /* %rbx = current %rsp */
3446
3447 leaq panic_quiesce(%rip), %rdi /* %rdi = &panic_quiesce */
3448 call dtrace_panic_trigger /* %eax = dtrace_panic_trigger() */
3449 jmp vpanic_common
3450
3451 SET_SIZE(dtrace_vpanic)
3452
3453 #elif defined(__i386)
3454
3455 ENTRY_NP(vpanic) / Initial stack layout:
3456
3457 pushl %ebp / | %eip | 20
3458 movl %esp, %ebp / | %ebp | 16
3459 pushl %eax / | %eax | 12
3460 pushl %ebx / | %ebx | 8
3461 pushl %ecx / | %ecx | 4
3462 pushl %edx / | %edx | 0
3463
3464 movl %esp, %ebx / %ebx = current stack pointer
3465
3466 lea panic_quiesce, %eax / %eax = &panic_quiesce
3467 pushl %eax / push &panic_quiesce
3468 call panic_trigger / %eax = panic_trigger()
3469 addl $4, %esp / reset stack pointer
3470
3471 vpanic_common:
3472 cmpl $0, %eax / if (%eax == 0)
3473 je 0f / goto 0f;
3474
3475 /*
3476 * If panic_trigger() was successful, we are the first to initiate a
3477 * panic: we now switch to the reserved panic_stack before continuing.
3478 */
3479 lea panic_stack, %esp / %esp = panic_stack
3480 addl $PANICSTKSIZE, %esp / %esp += PANICSTKSIZE
3481
3482 0: subl $REGSIZE, %esp / allocate struct regs
3483
3484 /*
3485 * Now that we've got everything set up, store the register values as
3486 * they were when we entered vpanic() to the designated location in
3487 * the regs structure we allocated on the stack.
3488 */
3489 #if !defined(__GNUC_AS__)
3490 movw %gs, %edx
3491 movl %edx, REGOFF_GS(%esp)
3492 movw %fs, %edx
3493 movl %edx, REGOFF_FS(%esp)
3494 movw %es, %edx
3495 movl %edx, REGOFF_ES(%esp)
3496 movw %ds, %edx
3497 movl %edx, REGOFF_DS(%esp)
3498 #else /* __GNUC_AS__ */
3499 mov %gs, %edx
3500 mov %edx, REGOFF_GS(%esp)
3501 mov %fs, %edx
3502 mov %edx, REGOFF_FS(%esp)
3503 mov %es, %edx
3504 mov %edx, REGOFF_ES(%esp)
3505 mov %ds, %edx
3506 mov %edx, REGOFF_DS(%esp)
3507 #endif /* __GNUC_AS__ */
3508 movl %edi, REGOFF_EDI(%esp)
3509 movl %esi, REGOFF_ESI(%esp)
3510 movl 16(%ebx), %ecx
3511 movl %ecx, REGOFF_EBP(%esp)
3512 movl %ebx, %ecx
3513 addl $20, %ecx
3514 movl %ecx, REGOFF_ESP(%esp)
3515 movl 8(%ebx), %ecx
3516 movl %ecx, REGOFF_EBX(%esp)
3517 movl 0(%ebx), %ecx
3518 movl %ecx, REGOFF_EDX(%esp)
3519 movl 4(%ebx), %ecx
3520 movl %ecx, REGOFF_ECX(%esp)
3521 movl 12(%ebx), %ecx
3522 movl %ecx, REGOFF_EAX(%esp)
3523 movl $0, REGOFF_TRAPNO(%esp)
3524 movl $0, REGOFF_ERR(%esp)
3525 lea vpanic, %ecx
3526 movl %ecx, REGOFF_EIP(%esp)
3527 #if !defined(__GNUC_AS__)
3528 movw %cs, %edx
3529 #else /* __GNUC_AS__ */
3530 mov %cs, %edx
3531 #endif /* __GNUC_AS__ */
3532 movl %edx, REGOFF_CS(%esp)
3533 pushfl
3534 popl %ecx
3535 #if defined(__xpv)
3536 /*
3537 * Synthesize the PS_IE bit from the event mask bit
3538 */
3539 CURTHREAD(%edx)
3540 KPREEMPT_DISABLE(%edx)
3541 EVENT_MASK_TO_IE(%edx, %ecx)
3542 CURTHREAD(%edx)
3543 KPREEMPT_ENABLE_NOKP(%edx)
3544 #endif
3545 movl %ecx, REGOFF_EFL(%esp)
3546 movl $0, REGOFF_UESP(%esp)
3547 #if !defined(__GNUC_AS__)
3548 movw %ss, %edx
3549 #else /* __GNUC_AS__ */
3550 mov %ss, %edx
3551 #endif /* __GNUC_AS__ */
3552 movl %edx, REGOFF_SS(%esp)
3553
3554 movl %esp, %ecx / %ecx = ®s
3555 pushl %eax / push on_panic_stack
3556 pushl %ecx / push ®s
3557 movl 12(%ebp), %ecx / %ecx = alist
3558 pushl %ecx / push alist
3559 movl 8(%ebp), %ecx / %ecx = format
3560 pushl %ecx / push format
3561 call panicsys / panicsys();
3562 addl $16, %esp / pop arguments
3563
3564 addl $REGSIZE, %esp
3565 popl %edx
3566 popl %ecx
3567 popl %ebx
3568 popl %eax
3569 leave
3570 ret
3571 SET_SIZE(vpanic)
3572
3573 ENTRY_NP(dtrace_vpanic) / Initial stack layout:
3574
3575 pushl %ebp / | %eip | 20
3576 movl %esp, %ebp / | %ebp | 16
3577 pushl %eax / | %eax | 12
3578 pushl %ebx / | %ebx | 8
3579 pushl %ecx / | %ecx | 4
3580 pushl %edx / | %edx | 0
3581
3582 movl %esp, %ebx / %ebx = current stack pointer
3583
3584 lea panic_quiesce, %eax / %eax = &panic_quiesce
3585 pushl %eax / push &panic_quiesce
3586 call dtrace_panic_trigger / %eax = dtrace_panic_trigger()
3587 addl $4, %esp / reset stack pointer
3588 jmp vpanic_common / jump back to common code
3589
3590 SET_SIZE(dtrace_vpanic)
3591
3592 #endif /* __i386 */
3593 #endif /* __lint */
3594
3595 #if defined(__lint)
3596
3597 void
3598 hres_tick(void)
3599 {}
3600
3601 int64_t timedelta;
3602 hrtime_t hres_last_tick;
3603 volatile timestruc_t hrestime;
3604 int64_t hrestime_adj;
3605 volatile int hres_lock;
3606 hrtime_t hrtime_base;
3607
3608 #else /* __lint */
3609
3610 DGDEF3(hrestime, _MUL(2, CLONGSIZE), 8)
3611 .NWORD 0, 0
3612
3613 DGDEF3(hrestime_adj, 8, 8)
3614 .long 0, 0
3615
3616 DGDEF3(hres_last_tick, 8, 8)
3617 .long 0, 0
3618
3619 DGDEF3(timedelta, 8, 8)
3620 .long 0, 0
3621
3622 DGDEF3(hres_lock, 4, 8)
3623 .long 0
3624
3625 /*
3626 * initialized to a non zero value to make pc_gethrtime()
3627 * work correctly even before clock is initialized
3628 */
3629 DGDEF3(hrtime_base, 8, 8)
3630 .long _MUL(NSEC_PER_CLOCK_TICK, 6), 0
3631
3632 DGDEF3(adj_shift, 4, 4)
3633 .long ADJ_SHIFT
3634
3635 #if defined(__amd64)
3636
3637 ENTRY_NP(hres_tick)
3638 pushq %rbp
3639 movq %rsp, %rbp
3640
3641 /*
3642 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
3643 * hres_last_tick can only be modified while holding CLOCK_LOCK).
3644 * At worst, performing this now instead of under CLOCK_LOCK may
3645 * introduce some jitter in pc_gethrestime().
3646 */
3647 call *gethrtimef(%rip)
3648 movq %rax, %r8
3649
3650 leaq hres_lock(%rip), %rax
3651 movb $-1, %dl
3652 .CL1:
3653 xchgb %dl, (%rax)
3654 testb %dl, %dl
3655 jz .CL3 /* got it */
3656 .CL2:
3657 cmpb $0, (%rax) /* possible to get lock? */
3658 pause
3659 jne .CL2
3660 jmp .CL1 /* yes, try again */
3661 .CL3:
3662 /*
3663 * compute the interval since last time hres_tick was called
3664 * and adjust hrtime_base and hrestime accordingly
3665 * hrtime_base is an 8 byte value (in nsec), hrestime is
3666 * a timestruc_t (sec, nsec)
3667 */
3668 leaq hres_last_tick(%rip), %rax
3669 movq %r8, %r11
3670 subq (%rax), %r8
3671 addq %r8, hrtime_base(%rip) /* add interval to hrtime_base */
3672 addq %r8, hrestime+8(%rip) /* add interval to hrestime.tv_nsec */
3673 /*
3674 * Now that we have CLOCK_LOCK, we can update hres_last_tick
3675 */
3676 movq %r11, (%rax)
3677
3678 call __adj_hrestime
3679
3680 /*
3681 * release the hres_lock
3682 */
3683 incl hres_lock(%rip)
3684 leave
3685 ret
3686 SET_SIZE(hres_tick)
3687
3688 #elif defined(__i386)
3689
3690 ENTRY_NP(hres_tick)
3691 pushl %ebp
3692 movl %esp, %ebp
3693 pushl %esi
3694 pushl %ebx
3695
3696 /*
3697 * We need to call *gethrtimef before picking up CLOCK_LOCK (obviously,
3698 * hres_last_tick can only be modified while holding CLOCK_LOCK).
3699 * At worst, performing this now instead of under CLOCK_LOCK may
3700 * introduce some jitter in pc_gethrestime().
3701 */
3702 call *gethrtimef
3703 movl %eax, %ebx
3704 movl %edx, %esi
3705
3706 movl $hres_lock, %eax
3707 movl $-1, %edx
3708 .CL1:
3709 xchgb %dl, (%eax)
3710 testb %dl, %dl
3711 jz .CL3 / got it
3712 .CL2:
3713 cmpb $0, (%eax) / possible to get lock?
3714 pause
3715 jne .CL2
3716 jmp .CL1 / yes, try again
3717 .CL3:
3718 /*
3719 * compute the interval since last time hres_tick was called
3720 * and adjust hrtime_base and hrestime accordingly
3721 * hrtime_base is an 8 byte value (in nsec), hrestime is
3722 * timestruc_t (sec, nsec)
3723 */
3724
3725 lea hres_last_tick, %eax
3726
3727 movl %ebx, %edx
3728 movl %esi, %ecx
3729
3730 subl (%eax), %edx
3731 sbbl 4(%eax), %ecx
3732
3733 addl %edx, hrtime_base / add interval to hrtime_base
3734 adcl %ecx, hrtime_base+4
3735
3736 addl %edx, hrestime+4 / add interval to hrestime.tv_nsec
3737
3738 /
3739 / Now that we have CLOCK_LOCK, we can update hres_last_tick.
3740 /
3741 movl %ebx, (%eax)
3742 movl %esi, 4(%eax)
3743
3744 / get hrestime at this moment. used as base for pc_gethrestime
3745 /
3746 / Apply adjustment, if any
3747 /
3748 / #define HRES_ADJ (NSEC_PER_CLOCK_TICK >> ADJ_SHIFT)
3749 / (max_hres_adj)
3750 /
3751 / void
3752 / adj_hrestime()
3753 / {
3754 / long long adj;
3755 /
3756 / if (hrestime_adj == 0)
3757 / adj = 0;
3758 / else if (hrestime_adj > 0) {
3759 / if (hrestime_adj < HRES_ADJ)
3760 / adj = hrestime_adj;
3761 / else
3762 / adj = HRES_ADJ;
3763 / }
3764 / else {
3765 / if (hrestime_adj < -(HRES_ADJ))
3766 / adj = -(HRES_ADJ);
3767 / else
3768 / adj = hrestime_adj;
3769 / }
3770 /
3771 / timedelta -= adj;
3772 / hrestime_adj = timedelta;
3773 / hrestime.tv_nsec += adj;
3774 /
3775 / while (hrestime.tv_nsec >= NANOSEC) {
3776 / one_sec++;
3777 / hrestime.tv_sec++;
3778 / hrestime.tv_nsec -= NANOSEC;
3779 / }
3780 / }
3781 __adj_hrestime:
3782 movl hrestime_adj, %esi / if (hrestime_adj == 0)
3783 movl hrestime_adj+4, %edx
3784 andl %esi, %esi
3785 jne .CL4 / no
3786 andl %edx, %edx
3787 jne .CL4 / no
3788 subl %ecx, %ecx / yes, adj = 0;
3789 subl %edx, %edx
3790 jmp .CL5
3791 .CL4:
3792 subl %ecx, %ecx
3793 subl %eax, %eax
3794 subl %esi, %ecx
3795 sbbl %edx, %eax
3796 andl %eax, %eax / if (hrestime_adj > 0)
3797 jge .CL6
3798
3799 / In the following comments, HRES_ADJ is used, while in the code
3800 / max_hres_adj is used.
3801 /
3802 / The test for "hrestime_adj < HRES_ADJ" is complicated because
3803 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely
3804 / on the logical equivalence of:
3805 /
3806 / !(hrestime_adj < HRES_ADJ)
3807 /
3808 / and the two step sequence:
3809 /
3810 / (HRES_ADJ - lsw(hrestime_adj)) generates a Borrow/Carry
3811 /
3812 / which computes whether or not the least significant 32-bits
3813 / of hrestime_adj is greater than HRES_ADJ, followed by:
3814 /
3815 / Previous Borrow/Carry + -1 + msw(hrestime_adj) generates a Carry
3816 /
3817 / which generates a carry whenever step 1 is true or the most
3818 / significant long of the longlong hrestime_adj is non-zero.
3819
3820 movl max_hres_adj, %ecx / hrestime_adj is positive
3821 subl %esi, %ecx
3822 movl %edx, %eax
3823 adcl $-1, %eax
3824 jnc .CL7
3825 movl max_hres_adj, %ecx / adj = HRES_ADJ;
3826 subl %edx, %edx
3827 jmp .CL5
3828
3829 / The following computation is similar to the one above.
3830 /
3831 / The test for "hrestime_adj < -(HRES_ADJ)" is complicated because
3832 / hrestime_adj is 64-bits, while HRES_ADJ is 32-bits. We rely
3833 / on the logical equivalence of:
3834 /
3835 / (hrestime_adj > -HRES_ADJ)
3836 /
3837 / and the two step sequence:
3838 /
3839 / (HRES_ADJ + lsw(hrestime_adj)) generates a Carry
3840 /
3841 / which means the least significant 32-bits of hrestime_adj is
3842 / greater than -HRES_ADJ, followed by:
3843 /
3844 / Previous Carry + 0 + msw(hrestime_adj) generates a Carry
3845 /
3846 / which generates a carry only when step 1 is true and the most
3847 / significant long of the longlong hrestime_adj is -1.
3848
3849 .CL6: / hrestime_adj is negative
3850 movl %esi, %ecx
3851 addl max_hres_adj, %ecx
3852 movl %edx, %eax
3853 adcl $0, %eax
3854 jc .CL7
3855 xor %ecx, %ecx
3856 subl max_hres_adj, %ecx / adj = -(HRES_ADJ);
3857 movl $-1, %edx
3858 jmp .CL5
3859 .CL7:
3860 movl %esi, %ecx / adj = hrestime_adj;
3861 .CL5:
3862 movl timedelta, %esi
3863 subl %ecx, %esi
3864 movl timedelta+4, %eax
3865 sbbl %edx, %eax
3866 movl %esi, timedelta
3867 movl %eax, timedelta+4 / timedelta -= adj;
3868 movl %esi, hrestime_adj
3869 movl %eax, hrestime_adj+4 / hrestime_adj = timedelta;
3870 addl hrestime+4, %ecx
3871
3872 movl %ecx, %eax / eax = tv_nsec
3873 1:
3874 cmpl $NANOSEC, %eax / if ((unsigned long)tv_nsec >= NANOSEC)
3875 jb .CL8 / no
3876 incl one_sec / yes, one_sec++;
3877 incl hrestime / hrestime.tv_sec++;
3878 addl $-NANOSEC, %eax / tv_nsec -= NANOSEC
3879 jmp 1b / check for more seconds
3880
3881 .CL8:
3882 movl %eax, hrestime+4 / store final into hrestime.tv_nsec
3883 incl hres_lock / release the hres_lock
3884
3885 popl %ebx
3886 popl %esi
3887 leave
3888 ret
3889 SET_SIZE(hres_tick)
3890
3891 #endif /* __i386 */
3892 #endif /* __lint */
3893
3894 /*
3895 * void prefetch_smap_w(void *)
3896 *
3897 * Prefetch ahead within a linear list of smap structures.
3898 * Not implemented for ia32. Stub for compatibility.
3899 */
3900
3901 #if defined(__lint)
3902
3903 /*ARGSUSED*/
3904 void prefetch_smap_w(void *smp)
3905 {}
3906
3907 #else /* __lint */
3908
3909 ENTRY(prefetch_smap_w)
3910 rep; ret /* use 2 byte return instruction when branch target */
3911 /* AMD Software Optimization Guide - Section 6.2 */
3912 SET_SIZE(prefetch_smap_w)
3913
3914 #endif /* __lint */
3915
3916 /*
3917 * prefetch_page_r(page_t *)
3918 * issue prefetch instructions for a page_t
3919 */
3920 #if defined(__lint)
3921
3922 /*ARGSUSED*/
3923 void
3924 prefetch_page_r(void *pp)
3925 {}
3926
3927 #else /* __lint */
3928
3929 ENTRY(prefetch_page_r)
3930 rep; ret /* use 2 byte return instruction when branch target */
3931 /* AMD Software Optimization Guide - Section 6.2 */
3932 SET_SIZE(prefetch_page_r)
3933
3934 #endif /* __lint */
3935
3936 #if defined(__lint)
3937
3938 /*ARGSUSED*/
3939 int
3940 bcmp(const void *s1, const void *s2, size_t count)
3941 { return (0); }
3942
3943 #else /* __lint */
3944
3945 #if defined(__amd64)
3946
3947 ENTRY(bcmp)
3948 pushq %rbp
3949 movq %rsp, %rbp
3950 #ifdef DEBUG
3951 testq %rdx,%rdx
3952 je 1f
3953 movq postbootkernelbase(%rip), %r11
3954 cmpq %r11, %rdi
3955 jb 0f
3956 cmpq %r11, %rsi
3957 jnb 1f
3958 0: leaq .bcmp_panic_msg(%rip), %rdi
3959 xorl %eax, %eax
3960 call panic
3961 1:
3962 #endif /* DEBUG */
3963 call memcmp
3964 testl %eax, %eax
3965 setne %dl
3966 leave
3967 movzbl %dl, %eax
3968 ret
3969 SET_SIZE(bcmp)
3970
3971 #elif defined(__i386)
3972
3973 #define ARG_S1 8
3974 #define ARG_S2 12
3975 #define ARG_LENGTH 16
3976
3977 ENTRY(bcmp)
3978 pushl %ebp
3979 movl %esp, %ebp / create new stack frame
3980 #ifdef DEBUG
3981 cmpl $0, ARG_LENGTH(%ebp)
3982 je 1f
3983 movl postbootkernelbase, %eax
3984 cmpl %eax, ARG_S1(%ebp)
3985 jb 0f
3986 cmpl %eax, ARG_S2(%ebp)
3987 jnb 1f
3988 0: pushl $.bcmp_panic_msg
3989 call panic
3990 1:
3991 #endif /* DEBUG */
3992
3993 pushl %edi / save register variable
3994 movl ARG_S1(%ebp), %eax / %eax = address of string 1
3995 movl ARG_S2(%ebp), %ecx / %ecx = address of string 2
3996 cmpl %eax, %ecx / if the same string
3997 je .equal / goto .equal
3998 movl ARG_LENGTH(%ebp), %edi / %edi = length in bytes
3999 cmpl $4, %edi / if %edi < 4
4000 jb .byte_check / goto .byte_check
4001 .align 4
4002 .word_loop:
4003 movl (%ecx), %edx / move 1 word from (%ecx) to %edx
4004 leal -4(%edi), %edi / %edi -= 4
4005 cmpl (%eax), %edx / compare 1 word from (%eax) with %edx
4006 jne .word_not_equal / if not equal, goto .word_not_equal
4007 leal 4(%ecx), %ecx / %ecx += 4 (next word)
4008 leal 4(%eax), %eax / %eax += 4 (next word)
4009 cmpl $4, %edi / if %edi >= 4
4010 jae .word_loop / goto .word_loop
4011 .byte_check:
4012 cmpl $0, %edi / if %edi == 0
4013 je .equal / goto .equal
4014 jmp .byte_loop / goto .byte_loop (checks in bytes)
4015 .word_not_equal:
4016 leal 4(%edi), %edi / %edi += 4 (post-decremented)
4017 .align 4
4018 .byte_loop:
4019 movb (%ecx), %dl / move 1 byte from (%ecx) to %dl
4020 cmpb %dl, (%eax) / compare %dl with 1 byte from (%eax)
4021 jne .not_equal / if not equal, goto .not_equal
4022 incl %ecx / %ecx++ (next byte)
4023 incl %eax / %eax++ (next byte)
4024 decl %edi / %edi--
4025 jnz .byte_loop / if not zero, goto .byte_loop
4026 .equal:
4027 xorl %eax, %eax / %eax = 0
4028 popl %edi / restore register variable
4029 leave / restore old stack frame
4030 ret / return (NULL)
4031 .align 4
4032 .not_equal:
4033 movl $1, %eax / return 1
4034 popl %edi / restore register variable
4035 leave / restore old stack frame
4036 ret / return (NULL)
4037 SET_SIZE(bcmp)
4038
4039 #endif /* __i386 */
4040
4041 #ifdef DEBUG
4042 .text
4043 .bcmp_panic_msg:
4044 .string "bcmp: arguments below kernelbase"
4045 #endif /* DEBUG */
4046
4047 #endif /* __lint */
4048
4049 #if defined(__lint)
4050
4051 uint_t
4052 bsrw_insn(uint16_t mask)
4053 {
4054 uint_t index = sizeof (mask) * NBBY - 1;
4055
4056 while ((mask & (1 << index)) == 0)
4057 index--;
4058 return (index);
4059 }
4060
4061 #else /* __lint */
4062
4063 #if defined(__amd64)
4064
4065 ENTRY_NP(bsrw_insn)
4066 xorl %eax, %eax
4067 bsrw %di, %ax
4068 ret
4069 SET_SIZE(bsrw_insn)
4070
4071 #elif defined(__i386)
4072
4073 ENTRY_NP(bsrw_insn)
4074 movw 4(%esp), %cx
4075 xorl %eax, %eax
4076 bsrw %cx, %ax
4077 ret
4078 SET_SIZE(bsrw_insn)
4079
4080 #endif /* __i386 */
4081 #endif /* __lint */
4082
4083 #if defined(__lint)
4084
4085 uint_t
4086 atomic_btr32(uint32_t *pending, uint_t pil)
4087 {
4088 return (*pending &= ~(1 << pil));
4089 }
4090
4091 #else /* __lint */
4092
4093 #if defined(__i386)
4094
4095 ENTRY_NP(atomic_btr32)
4096 movl 4(%esp), %ecx
4097 movl 8(%esp), %edx
4098 xorl %eax, %eax
4099 lock
4100 btrl %edx, (%ecx)
4101 setc %al
4102 ret
4103 SET_SIZE(atomic_btr32)
4104
4105 #endif /* __i386 */
4106 #endif /* __lint */
4107
4108 #if defined(__lint)
4109
4110 /*ARGSUSED*/
4111 void
4112 switch_sp_and_call(void *newsp, void (*func)(uint_t, uint_t), uint_t arg1,
4113 uint_t arg2)
4114 {}
4115
4116 #else /* __lint */
4117
4118 #if defined(__amd64)
4119
4120 ENTRY_NP(switch_sp_and_call)
4121 pushq %rbp
4122 movq %rsp, %rbp /* set up stack frame */
4123 movq %rdi, %rsp /* switch stack pointer */
4124 movq %rdx, %rdi /* pass func arg 1 */
4125 movq %rsi, %r11 /* save function to call */
4126 movq %rcx, %rsi /* pass func arg 2 */
4127 call *%r11 /* call function */
4128 leave /* restore stack */
4129 ret
4130 SET_SIZE(switch_sp_and_call)
4131
4132 #elif defined(__i386)
4133
4134 ENTRY_NP(switch_sp_and_call)
4135 pushl %ebp
4136 mov %esp, %ebp /* set up stack frame */
4137 movl 8(%ebp), %esp /* switch stack pointer */
4138 pushl 20(%ebp) /* push func arg 2 */
4139 pushl 16(%ebp) /* push func arg 1 */
4140 call *12(%ebp) /* call function */
4141 addl $8, %esp /* pop arguments */
4142 leave /* restore stack */
4143 ret
4144 SET_SIZE(switch_sp_and_call)
4145
4146 #endif /* __i386 */
4147 #endif /* __lint */
4148
4149 #if defined(__lint)
4150
4151 void
4152 kmdb_enter(void)
4153 {}
4154
4155 #else /* __lint */
4156
4157 #if defined(__amd64)
4158
4159 ENTRY_NP(kmdb_enter)
4160 pushq %rbp
4161 movq %rsp, %rbp
4162
4163 /*
4164 * Save flags, do a 'cli' then return the saved flags
4165 */
4166 call intr_clear
4167
4168 int $T_DBGENTR
4169
4170 /*
4171 * Restore the saved flags
4172 */
4173 movq %rax, %rdi
4174 call intr_restore
4175
4176 leave
4177 ret
4178 SET_SIZE(kmdb_enter)
4179
4180 #elif defined(__i386)
4181
4182 ENTRY_NP(kmdb_enter)
4183 pushl %ebp
4184 movl %esp, %ebp
4185
4186 /*
4187 * Save flags, do a 'cli' then return the saved flags
4188 */
4189 call intr_clear
4190
4191 int $T_DBGENTR
4192
4193 /*
4194 * Restore the saved flags
4195 */
4196 pushl %eax
4197 call intr_restore
4198 addl $4, %esp
4199
4200 leave
4201 ret
4202 SET_SIZE(kmdb_enter)
4203
4204 #endif /* __i386 */
4205 #endif /* __lint */
4206
4207 #if defined(__lint)
4208
4209 void
4210 return_instr(void)
4211 {}
4212
4213 #else /* __lint */
4214
4215 ENTRY_NP(return_instr)
4216 rep; ret /* use 2 byte instruction when branch target */
4217 /* AMD Software Optimization Guide - Section 6.2 */
4218 SET_SIZE(return_instr)
4219
4220 #endif /* __lint */
4221
4222 #if defined(__lint)
4223
4224 ulong_t
4225 getflags(void)
4226 {
4227 return (0);
4228 }
4229
4230 #else /* __lint */
4231
4232 #if defined(__amd64)
4233
4234 ENTRY(getflags)
4235 pushfq
4236 popq %rax
4237 #if defined(__xpv)
4238 CURTHREAD(%rdi)
4239 KPREEMPT_DISABLE(%rdi)
4240 /*
4241 * Synthesize the PS_IE bit from the event mask bit
4242 */
4243 CURVCPU(%r11)
4244 andq $_BITNOT(PS_IE), %rax
4245 XEN_TEST_UPCALL_MASK(%r11)
4246 jnz 1f
4247 orq $PS_IE, %rax
4248 1:
4249 KPREEMPT_ENABLE_NOKP(%rdi)
4250 #endif
4251 ret
4252 SET_SIZE(getflags)
4253
4254 #elif defined(__i386)
4255
4256 ENTRY(getflags)
4257 pushfl
4258 popl %eax
4259 #if defined(__xpv)
4260 CURTHREAD(%ecx)
4261 KPREEMPT_DISABLE(%ecx)
4262 /*
4263 * Synthesize the PS_IE bit from the event mask bit
4264 */
4265 CURVCPU(%edx)
4266 andl $_BITNOT(PS_IE), %eax
4267 XEN_TEST_UPCALL_MASK(%edx)
4268 jnz 1f
4269 orl $PS_IE, %eax
4270 1:
4271 KPREEMPT_ENABLE_NOKP(%ecx)
4272 #endif
4273 ret
4274 SET_SIZE(getflags)
4275
4276 #endif /* __i386 */
4277
4278 #endif /* __lint */
4279
4280 #if defined(__lint)
4281
4282 ftrace_icookie_t
4283 ftrace_interrupt_disable(void)
4284 { return (0); }
4285
4286 #else /* __lint */
4287
4288 #if defined(__amd64)
4289
4290 ENTRY(ftrace_interrupt_disable)
4291 pushfq
4292 popq %rax
4293 CLI(%rdx)
4294 ret
4295 SET_SIZE(ftrace_interrupt_disable)
4296
4297 #elif defined(__i386)
4298
4299 ENTRY(ftrace_interrupt_disable)
4300 pushfl
4301 popl %eax
4302 CLI(%edx)
4303 ret
4304 SET_SIZE(ftrace_interrupt_disable)
4305
4306 #endif /* __i386 */
4307 #endif /* __lint */
4308
4309 #if defined(__lint)
4310
4311 /*ARGSUSED*/
4312 void
4313 ftrace_interrupt_enable(ftrace_icookie_t cookie)
4314 {}
4315
4316 #else /* __lint */
4317
4318 #if defined(__amd64)
4319
4320 ENTRY(ftrace_interrupt_enable)
4321 pushq %rdi
4322 popfq
4323 ret
4324 SET_SIZE(ftrace_interrupt_enable)
4325
4326 #elif defined(__i386)
4327
4328 ENTRY(ftrace_interrupt_enable)
4329 movl 4(%esp), %eax
4330 pushl %eax
4331 popfl
4332 ret
4333 SET_SIZE(ftrace_interrupt_enable)
4334
4335 #endif /* __i386 */
4336 #endif /* __lint */
4337
4338 #if defined (__lint)
4339
4340 /*ARGSUSED*/
4341 void
4342 clflush_insn(caddr_t addr)
4343 {}
4344
4345 #else /* __lint */
4346
4347 #if defined (__amd64)
4348 ENTRY(clflush_insn)
4349 clflush (%rdi)
4350 ret
4351 SET_SIZE(clflush_insn)
4352 #elif defined (__i386)
4353 ENTRY(clflush_insn)
4354 movl 4(%esp), %eax
4355 clflush (%eax)
4356 ret
4357 SET_SIZE(clflush_insn)
4358
4359 #endif /* __i386 */
4360 #endif /* __lint */
4361
4362 #if defined (__lint)
4363 /*ARGSUSED*/
4364 void
4365 mfence_insn(void)
4366 {}
4367
4368 #else /* __lint */
4369
4370 #if defined (__amd64)
4371 ENTRY(mfence_insn)
4372 mfence
4373 ret
4374 SET_SIZE(mfence_insn)
4375 #elif defined (__i386)
4376 ENTRY(mfence_insn)
4377 mfence
4378 ret
4379 SET_SIZE(mfence_insn)
4380
4381 #endif /* __i386 */
4382 #endif /* __lint */
4383
4384 /*
4385 * VMware implements an I/O port that programs can query to detect if software
4386 * is running in a VMware hypervisor. This hypervisor port behaves differently
4387 * depending on magic values in certain registers and modifies some registers
4388 * as a side effect.
4389 *
4390 * References: http://kb.vmware.com/kb/1009458
4391 */
4392
4393 #if defined(__lint)
4394
4395 /* ARGSUSED */
4396 void
4397 vmware_port(int cmd, uint32_t *regs) { return; }
4398
4399 #else
4400
4401 #if defined(__amd64)
4402
4403 ENTRY(vmware_port)
4404 pushq %rbx
4405 movl $VMWARE_HVMAGIC, %eax
4406 movl $0xffffffff, %ebx
4407 movl %edi, %ecx
4408 movl $VMWARE_HVPORT, %edx
4409 inl (%dx)
4410 movl %eax, (%rsi)
4411 movl %ebx, 4(%rsi)
4412 movl %ecx, 8(%rsi)
4413 movl %edx, 12(%rsi)
4414 popq %rbx
4415 ret
4416 SET_SIZE(vmware_port)
4417
4418 #elif defined(__i386)
4419
4420 ENTRY(vmware_port)
4421 pushl %ebx
4422 pushl %esi
4423 movl $VMWARE_HVMAGIC, %eax
4424 movl $0xffffffff, %ebx
4425 movl 12(%esp), %ecx
4426 movl $VMWARE_HVPORT, %edx
4427 inl (%dx)
4428 movl 16(%esp), %esi
4429 movl %eax, (%esi)
4430 movl %ebx, 4(%esi)
4431 movl %ecx, 8(%esi)
4432 movl %edx, 12(%esi)
4433 popl %esi
4434 popl %ebx
4435 ret
4436 SET_SIZE(vmware_port)
4437
4438 #endif /* __i386 */
4439 #endif /* __lint */