6583 remove whole-process swapping
1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 /*
28 * Copyright (c) 2012 Joyent, Inc. All rights reserved.
29 */
30
31 #include <sys/mmu.h>
32 #include <sys/systm.h>
33 #include <sys/trap.h>
34 #include <sys/machtrap.h>
35 #include <sys/vtrace.h>
36 #include <sys/prsystm.h>
37 #include <sys/archsystm.h>
38 #include <sys/machsystm.h>
39 #include <sys/fpu/fpusystm.h>
40 #include <sys/tnf.h>
41 #include <sys/tnf_probe.h>
42 #include <sys/simulate.h>
43 #include <sys/ftrace.h>
44 #include <sys/ontrap.h>
45 #include <sys/kcpc.h>
46 #include <sys/kobj.h>
47 #include <sys/procfs.h>
48 #include <sys/sun4asi.h>
49 #include <sys/sdt.h>
50 #include <sys/fpras.h>
51 #include <sys/contract/process_impl.h>
52
53 #ifdef TRAPTRACE
54 #include <sys/traptrace.h>
55 #endif
56
57 int tudebug = 0;
58 static int tudebugbpt = 0;
59 static int tudebugfpe = 0;
60
61 static int alignfaults = 0;
62
63 #if defined(TRAPDEBUG) || defined(lint)
64 static int lodebug = 0;
65 #else
66 #define lodebug 0
67 #endif /* defined(TRAPDEBUG) || defined(lint) */
68
69
70 int vis1_partial_support(struct regs *rp, k_siginfo_t *siginfo, uint_t *fault);
71 #pragma weak vis1_partial_support
72
73 void showregs(unsigned, struct regs *, caddr_t, uint_t);
74 #pragma weak showregs
75
76 void trap_async_hwerr(void);
77 #pragma weak trap_async_hwerr
78
79 void trap_async_berr_bto(int, struct regs *);
80 #pragma weak trap_async_berr_bto
81
82 static enum seg_rw get_accesstype(struct regs *);
83 static int nfload(struct regs *, int *);
84 static int swap_nc(struct regs *, int);
85 static int ldstub_nc(struct regs *, int);
86 void trap_cleanup(struct regs *, uint_t, k_siginfo_t *, int);
87 void trap_rtt(void);
88
89 static int
90 die(unsigned type, struct regs *rp, caddr_t addr, uint_t mmu_fsr)
91 {
92 struct panic_trap_info ti;
93
94 #ifdef TRAPTRACE
95 TRAPTRACE_FREEZE;
96 #endif
97
98 ti.trap_regs = rp;
99 ti.trap_type = type;
100 ti.trap_addr = addr;
101 ti.trap_mmu_fsr = mmu_fsr;
102
103 curthread->t_panic_trap = &ti;
104
105 if (type == T_DATA_MMU_MISS && addr < (caddr_t)KERNELBASE) {
106 panic("BAD TRAP: type=%x rp=%p addr=%p mmu_fsr=%x "
107 "occurred in module \"%s\" due to %s",
108 type, (void *)rp, (void *)addr, mmu_fsr,
109 mod_containing_pc((caddr_t)rp->r_pc),
110 addr < (caddr_t)PAGESIZE ?
111 "a NULL pointer dereference" :
112 "an illegal access to a user address");
113 } else {
114 panic("BAD TRAP: type=%x rp=%p addr=%p mmu_fsr=%x",
115 type, (void *)rp, (void *)addr, mmu_fsr);
116 }
117
118 return (0); /* avoid optimization of restore in call's delay slot */
119 }
120
121 #if defined(SF_ERRATA_23) || defined(SF_ERRATA_30) /* call ... illegal-insn */
122 int ill_calls;
123 #endif
124
125 /*
126 * Currently, the only PREFETCH/PREFETCHA instructions which cause traps
127 * are the "strong" prefetches (fcn=20-23). But we check for all flavors of
128 * PREFETCH, in case some future variant also causes a DATA_MMU_MISS.
129 */
130 #define IS_PREFETCH(i) (((i) & 0xc1780000) == 0xc1680000)
131
132 #define IS_FLUSH(i) (((i) & 0xc1f80000) == 0x81d80000)
133 #define IS_SWAP(i) (((i) & 0xc1f80000) == 0xc0780000)
134 #define IS_LDSTUB(i) (((i) & 0xc1f80000) == 0xc0680000)
135 #define IS_FLOAT(i) (((i) & 0x1000000) != 0)
136 #define IS_STORE(i) (((i) >> 21) & 1)
137
138 /*
139 * Called from the trap handler when a processor trap occurs.
140 */
141 /*VARARGS2*/
142 void
143 trap(struct regs *rp, caddr_t addr, uint32_t type, uint32_t mmu_fsr)
144 {
145 proc_t *p = ttoproc(curthread);
146 klwp_id_t lwp = ttolwp(curthread);
147 struct machpcb *mpcb = NULL;
148 k_siginfo_t siginfo;
149 uint_t op3, fault = 0;
150 int stepped = 0;
151 greg_t oldpc;
152 int mstate;
153 char *badaddr;
154 faultcode_t res;
155 enum fault_type fault_type;
156 enum seg_rw rw;
157 uintptr_t lofault;
158 label_t *onfault;
159 int instr;
160 int iskernel;
161 int watchcode;
162 int watchpage;
163 extern faultcode_t pagefault(caddr_t, enum fault_type,
164 enum seg_rw, int);
165 #ifdef sun4v
166 extern boolean_t tick_stick_emulation_active;
167 #endif /* sun4v */
168
169 CPU_STATS_ADDQ(CPU, sys, trap, 1);
170
171 if (USERMODE(rp->r_tstate) || (type & T_USER)) {
172 /*
173 * Set lwp_state before trying to acquire any
174 * adaptive lock
175 */
176 ASSERT(lwp != NULL);
177 lwp->lwp_state = LWP_SYS;
178 /*
179 * Set up the current cred to use during this trap. u_cred
180 * no longer exists. t_cred is used instead.
181 * The current process credential applies to the thread for
182 * the entire trap. If trapping from the kernel, this
183 * should already be set up.
184 */
185 if (curthread->t_cred != p->p_cred) {
186 cred_t *oldcred = curthread->t_cred;
187 /*
188 * DTrace accesses t_cred in probe context. t_cred
189 * must always be either NULL, or point to a valid,
190 * allocated cred structure.
191 */
192 curthread->t_cred = crgetcred();
193 crfree(oldcred);
194 }
195 type |= T_USER;
196 ASSERT((type == (T_SYS_RTT_PAGE | T_USER)) ||
197 (type == (T_SYS_RTT_ALIGN | T_USER)) ||
198 lwp->lwp_regs == rp);
199 mpcb = lwptompcb(lwp);
200 switch (type) {
201 case T_WIN_OVERFLOW + T_USER:
202 case T_WIN_UNDERFLOW + T_USER:
203 case T_SYS_RTT_PAGE + T_USER:
204 case T_DATA_MMU_MISS + T_USER:
205 mstate = LMS_DFAULT;
206 break;
207 case T_INSTR_MMU_MISS + T_USER:
208 mstate = LMS_TFAULT;
209 break;
210 default:
211 mstate = LMS_TRAP;
212 break;
213 }
214 /* Kernel probe */
215 TNF_PROBE_1(thread_state, "thread", /* CSTYLED */,
216 tnf_microstate, state, (char)mstate);
217 mstate = new_mstate(curthread, mstate);
218 siginfo.si_signo = 0;
219 stepped =
220 lwp->lwp_pcb.pcb_step != STEP_NONE &&
221 ((oldpc = rp->r_pc), prundostep()) &&
222 mmu_btop((uintptr_t)addr) == mmu_btop((uintptr_t)oldpc);
223 /* this assignment must not precede call to prundostep() */
224 oldpc = rp->r_pc;
225 }
226
227 TRACE_1(TR_FAC_TRAP, TR_C_TRAP_HANDLER_ENTER,
228 "C_trap_handler_enter:type %x", type);
229
230 #ifdef F_DEFERRED
231 /*
232 * Take any pending floating point exceptions now.
233 * If the floating point unit has an exception to handle,
234 * just return to user-level to let the signal handler run.
235 * The instruction that got us to trap() will be reexecuted on
236 * return from the signal handler and we will trap to here again.
237 * This is necessary to disambiguate simultaneous traps which
238 * happen when a floating-point exception is pending and a
239 * machine fault is incurred.
240 */
241 if (type & USER) {
242 /*
243 * FP_TRAPPED is set only by sendsig() when it copies
244 * out the floating-point queue for the signal handler.
245 * It is set there so we can test it here and in syscall().
246 */
247 mpcb->mpcb_flags &= ~FP_TRAPPED;
248 syncfpu();
249 if (mpcb->mpcb_flags & FP_TRAPPED) {
250 /*
251 * trap() has have been called recursively and may
252 * have stopped the process, so do single step
253 * support for /proc.
254 */
255 mpcb->mpcb_flags &= ~FP_TRAPPED;
256 goto out;
257 }
258 }
259 #endif
260 switch (type) {
261 case T_DATA_MMU_MISS:
262 case T_INSTR_MMU_MISS + T_USER:
263 case T_DATA_MMU_MISS + T_USER:
264 case T_DATA_PROT + T_USER:
265 case T_AST + T_USER:
266 case T_SYS_RTT_PAGE + T_USER:
267 case T_FLUSH_PCB + T_USER:
268 case T_FLUSHW + T_USER:
269 break;
270
271 default:
272 FTRACE_3("trap(): type=0x%lx, regs=0x%lx, addr=0x%lx",
273 (ulong_t)type, (ulong_t)rp, (ulong_t)addr);
274 break;
275 }
276
277 switch (type) {
278
279 default:
280 /*
281 * Check for user software trap.
282 */
283 if (type & T_USER) {
284 if (tudebug)
285 showregs(type, rp, (caddr_t)0, 0);
286 if ((type & ~T_USER) >= T_SOFTWARE_TRAP) {
287 bzero(&siginfo, sizeof (siginfo));
288 siginfo.si_signo = SIGILL;
289 siginfo.si_code = ILL_ILLTRP;
290 siginfo.si_addr = (caddr_t)rp->r_pc;
291 siginfo.si_trapno = type &~ T_USER;
292 fault = FLTILL;
293 break;
294 }
295 }
296 addr = (caddr_t)rp->r_pc;
297 (void) die(type, rp, addr, 0);
298 /*NOTREACHED*/
299
300 case T_ALIGNMENT: /* supv alignment error */
301 if (nfload(rp, NULL))
302 goto cleanup;
303
304 if (curthread->t_lofault) {
305 if (lodebug) {
306 showregs(type, rp, addr, 0);
307 traceback((caddr_t)rp->r_sp);
308 }
309 rp->r_g1 = EFAULT;
310 rp->r_pc = curthread->t_lofault;
311 rp->r_npc = rp->r_pc + 4;
312 goto cleanup;
313 }
314 (void) die(type, rp, addr, 0);
315 /*NOTREACHED*/
316
317 case T_INSTR_EXCEPTION: /* sys instruction access exception */
318 addr = (caddr_t)rp->r_pc;
319 (void) die(type, rp, addr, mmu_fsr);
320 /*NOTREACHED*/
321
322 case T_INSTR_MMU_MISS: /* sys instruction mmu miss */
323 addr = (caddr_t)rp->r_pc;
324 (void) die(type, rp, addr, 0);
325 /*NOTREACHED*/
326
327 case T_DATA_EXCEPTION: /* system data access exception */
328 switch (X_FAULT_TYPE(mmu_fsr)) {
329 case FT_RANGE:
330 /*
331 * This happens when we attempt to dereference an
332 * address in the address hole. If t_ontrap is set,
333 * then break and fall through to T_DATA_MMU_MISS /
334 * T_DATA_PROT case below. If lofault is set, then
335 * honour it (perhaps the user gave us a bogus
336 * address in the hole to copyin from or copyout to?)
337 */
338
339 if (curthread->t_ontrap != NULL)
340 break;
341
342 addr = (caddr_t)((uintptr_t)addr & TAGACC_VADDR_MASK);
343 if (curthread->t_lofault) {
344 if (lodebug) {
345 showregs(type, rp, addr, 0);
346 traceback((caddr_t)rp->r_sp);
347 }
348 rp->r_g1 = EFAULT;
349 rp->r_pc = curthread->t_lofault;
350 rp->r_npc = rp->r_pc + 4;
351 goto cleanup;
352 }
353 (void) die(type, rp, addr, mmu_fsr);
354 /*NOTREACHED*/
355
356 case FT_PRIV:
357 /*
358 * This can happen if we access ASI_USER from a kernel
359 * thread. To support pxfs, we need to honor lofault if
360 * we're doing a copyin/copyout from a kernel thread.
361 */
362
363 if (nfload(rp, NULL))
364 goto cleanup;
365 addr = (caddr_t)((uintptr_t)addr & TAGACC_VADDR_MASK);
366 if (curthread->t_lofault) {
367 if (lodebug) {
368 showregs(type, rp, addr, 0);
369 traceback((caddr_t)rp->r_sp);
370 }
371 rp->r_g1 = EFAULT;
372 rp->r_pc = curthread->t_lofault;
373 rp->r_npc = rp->r_pc + 4;
374 goto cleanup;
375 }
376 (void) die(type, rp, addr, mmu_fsr);
377 /*NOTREACHED*/
378
379 default:
380 if (nfload(rp, NULL))
381 goto cleanup;
382 addr = (caddr_t)((uintptr_t)addr & TAGACC_VADDR_MASK);
383 (void) die(type, rp, addr, mmu_fsr);
384 /*NOTREACHED*/
385
386 case FT_NFO:
387 break;
388 }
389 /* fall into ... */
390
391 case T_DATA_MMU_MISS: /* system data mmu miss */
392 case T_DATA_PROT: /* system data protection fault */
393 if (nfload(rp, &instr))
394 goto cleanup;
395
396 /*
397 * If we're under on_trap() protection (see <sys/ontrap.h>),
398 * set ot_trap and return from the trap to the trampoline.
399 */
400 if (curthread->t_ontrap != NULL) {
401 on_trap_data_t *otp = curthread->t_ontrap;
402
403 TRACE_0(TR_FAC_TRAP, TR_C_TRAP_HANDLER_EXIT,
404 "C_trap_handler_exit");
405 TRACE_0(TR_FAC_TRAP, TR_TRAP_END, "trap_end");
406
407 if (otp->ot_prot & OT_DATA_ACCESS) {
408 otp->ot_trap |= OT_DATA_ACCESS;
409 rp->r_pc = otp->ot_trampoline;
410 rp->r_npc = rp->r_pc + 4;
411 goto cleanup;
412 }
413 }
414 lofault = curthread->t_lofault;
415 onfault = curthread->t_onfault;
416 curthread->t_lofault = 0;
417
418 mstate = new_mstate(curthread, LMS_KFAULT);
419
420 switch (type) {
421 case T_DATA_PROT:
422 fault_type = F_PROT;
423 rw = S_WRITE;
424 break;
425 case T_INSTR_MMU_MISS:
426 fault_type = F_INVAL;
427 rw = S_EXEC;
428 break;
429 case T_DATA_MMU_MISS:
430 case T_DATA_EXCEPTION:
431 /*
432 * The hardware doesn't update the sfsr on mmu
433 * misses so it is not easy to find out whether
434 * the access was a read or a write so we need
435 * to decode the actual instruction.
436 */
437 fault_type = F_INVAL;
438 rw = get_accesstype(rp);
439 break;
440 default:
441 cmn_err(CE_PANIC, "trap: unknown type %x", type);
442 break;
443 }
444 /*
445 * We determine if access was done to kernel or user
446 * address space. The addr passed into trap is really the
447 * tag access register.
448 */
449 iskernel = (((uintptr_t)addr & TAGACC_CTX_MASK) == KCONTEXT);
450 addr = (caddr_t)((uintptr_t)addr & TAGACC_VADDR_MASK);
451
452 res = pagefault(addr, fault_type, rw, iskernel);
453 if (!iskernel && res == FC_NOMAP &&
454 addr < p->p_usrstack && grow(addr))
455 res = 0;
456
457 (void) new_mstate(curthread, mstate);
458
459 /*
460 * Restore lofault and onfault. If we resolved the fault, exit.
461 * If we didn't and lofault wasn't set, die.
462 */
463 curthread->t_lofault = lofault;
464 curthread->t_onfault = onfault;
465
466 if (res == 0)
467 goto cleanup;
468
469 if (IS_PREFETCH(instr)) {
470 /* skip prefetch instructions in kernel-land */
471 rp->r_pc = rp->r_npc;
472 rp->r_npc += 4;
473 goto cleanup;
474 }
475
476 if ((lofault == 0 || lodebug) &&
477 (calc_memaddr(rp, &badaddr) == SIMU_SUCCESS))
478 addr = badaddr;
479 if (lofault == 0)
480 (void) die(type, rp, addr, 0);
481 /*
482 * Cannot resolve fault. Return to lofault.
483 */
484 if (lodebug) {
485 showregs(type, rp, addr, 0);
486 traceback((caddr_t)rp->r_sp);
487 }
488 if (FC_CODE(res) == FC_OBJERR)
489 res = FC_ERRNO(res);
490 else
491 res = EFAULT;
492 rp->r_g1 = res;
493 rp->r_pc = curthread->t_lofault;
494 rp->r_npc = curthread->t_lofault + 4;
495 goto cleanup;
496
497 case T_INSTR_EXCEPTION + T_USER: /* user insn access exception */
498 bzero(&siginfo, sizeof (siginfo));
499 siginfo.si_addr = (caddr_t)rp->r_pc;
500 siginfo.si_signo = SIGSEGV;
501 siginfo.si_code = X_FAULT_TYPE(mmu_fsr) == FT_PRIV ?
502 SEGV_ACCERR : SEGV_MAPERR;
503 fault = FLTBOUNDS;
504 break;
505
506 case T_WIN_OVERFLOW + T_USER: /* window overflow in ??? */
507 case T_WIN_UNDERFLOW + T_USER: /* window underflow in ??? */
508 case T_SYS_RTT_PAGE + T_USER: /* window underflow in user_rtt */
509 case T_INSTR_MMU_MISS + T_USER: /* user instruction mmu miss */
510 case T_DATA_MMU_MISS + T_USER: /* user data mmu miss */
511 case T_DATA_PROT + T_USER: /* user data protection fault */
512 switch (type) {
513 case T_INSTR_MMU_MISS + T_USER:
514 addr = (caddr_t)rp->r_pc;
515 fault_type = F_INVAL;
516 rw = S_EXEC;
517 break;
518
519 case T_DATA_MMU_MISS + T_USER:
520 addr = (caddr_t)((uintptr_t)addr & TAGACC_VADDR_MASK);
521 fault_type = F_INVAL;
522 /*
523 * The hardware doesn't update the sfsr on mmu misses
524 * so it is not easy to find out whether the access
525 * was a read or a write so we need to decode the
526 * actual instruction. XXX BUGLY HW
527 */
528 rw = get_accesstype(rp);
529 break;
530
531 case T_DATA_PROT + T_USER:
532 addr = (caddr_t)((uintptr_t)addr & TAGACC_VADDR_MASK);
533 fault_type = F_PROT;
534 rw = S_WRITE;
535 break;
536
537 case T_WIN_OVERFLOW + T_USER:
538 addr = (caddr_t)((uintptr_t)addr & TAGACC_VADDR_MASK);
539 fault_type = F_INVAL;
540 rw = S_WRITE;
541 break;
542
543 case T_WIN_UNDERFLOW + T_USER:
544 case T_SYS_RTT_PAGE + T_USER:
545 addr = (caddr_t)((uintptr_t)addr & TAGACC_VADDR_MASK);
546 fault_type = F_INVAL;
547 rw = S_READ;
548 break;
549
550 default:
551 cmn_err(CE_PANIC, "trap: unknown type %x", type);
552 break;
553 }
554
555 /*
556 * If we are single stepping do not call pagefault
557 */
558 if (stepped) {
559 res = FC_NOMAP;
560 } else {
561 caddr_t vaddr = addr;
562 size_t sz;
563 int ta;
564
565 ASSERT(!(curthread->t_flag & T_WATCHPT));
566 watchpage = (pr_watch_active(p) &&
567 type != T_WIN_OVERFLOW + T_USER &&
568 type != T_WIN_UNDERFLOW + T_USER &&
569 type != T_SYS_RTT_PAGE + T_USER &&
570 pr_is_watchpage(addr, rw));
571
572 if (!watchpage ||
573 (sz = instr_size(rp, &vaddr, rw)) <= 0)
574 /* EMPTY */;
575 else if ((watchcode = pr_is_watchpoint(&vaddr, &ta,
576 sz, NULL, rw)) != 0) {
577 if (ta) {
578 do_watch_step(vaddr, sz, rw,
579 watchcode, rp->r_pc);
580 fault_type = F_INVAL;
581 } else {
582 bzero(&siginfo, sizeof (siginfo));
583 siginfo.si_signo = SIGTRAP;
584 siginfo.si_code = watchcode;
585 siginfo.si_addr = vaddr;
586 siginfo.si_trapafter = 0;
587 siginfo.si_pc = (caddr_t)rp->r_pc;
588 fault = FLTWATCH;
589 break;
590 }
591 } else {
592 if (rw != S_EXEC &&
593 pr_watch_emul(rp, vaddr, rw))
594 goto out;
595 do_watch_step(vaddr, sz, rw, 0, 0);
596 fault_type = F_INVAL;
597 }
598
599 if (pr_watch_active(p) &&
600 (type == T_WIN_OVERFLOW + T_USER ||
601 type == T_WIN_UNDERFLOW + T_USER ||
602 type == T_SYS_RTT_PAGE + T_USER)) {
603 int dotwo = (type == T_WIN_UNDERFLOW + T_USER);
604 if (copy_return_window(dotwo))
605 goto out;
606 fault_type = F_INVAL;
607 }
608
609 res = pagefault(addr, fault_type, rw, 0);
610
611 /*
612 * If pagefault succeed, ok.
613 * Otherwise grow the stack automatically.
614 */
615 if (res == 0 ||
616 (res == FC_NOMAP &&
617 type != T_INSTR_MMU_MISS + T_USER &&
618 addr < p->p_usrstack &&
619 grow(addr))) {
620 int ismem = prismember(&p->p_fltmask, FLTPAGE);
621
622 /*
623 * instr_size() is used to get the exact
624 * address of the fault, instead of the
625 * page of the fault. Unfortunately it is
626 * very slow, and this is an important
627 * code path. Don't call it unless
628 * correctness is needed. ie. if FLTPAGE
629 * is set, or we're profiling.
630 */
631
632 if (curthread->t_rprof != NULL || ismem)
633 (void) instr_size(rp, &addr, rw);
634
635 lwp->lwp_lastfault = FLTPAGE;
636 lwp->lwp_lastfaddr = addr;
637
638 if (ismem) {
639 bzero(&siginfo, sizeof (siginfo));
640 siginfo.si_addr = addr;
641 (void) stop_on_fault(FLTPAGE, &siginfo);
642 }
643 goto out;
644 }
645
646 if (type != (T_INSTR_MMU_MISS + T_USER)) {
647 /*
648 * check for non-faulting loads, also
649 * fetch the instruction to check for
650 * flush
651 */
652 if (nfload(rp, &instr))
653 goto out;
654
655 /* skip userland prefetch instructions */
656 if (IS_PREFETCH(instr)) {
657 rp->r_pc = rp->r_npc;
658 rp->r_npc += 4;
659 goto out;
660 /*NOTREACHED*/
661 }
662
663 /*
664 * check if the instruction was a
665 * flush. ABI allows users to specify
666 * an illegal address on the flush
667 * instruction so we simply return in
668 * this case.
669 *
670 * NB: the hardware should set a bit
671 * indicating this trap was caused by
672 * a flush instruction. Instruction
673 * decoding is bugly!
674 */
675 if (IS_FLUSH(instr)) {
676 /* skip the flush instruction */
677 rp->r_pc = rp->r_npc;
678 rp->r_npc += 4;
679 goto out;
680 /*NOTREACHED*/
681 }
682 } else if (res == FC_PROT) {
683 report_stack_exec(p, addr);
684 }
685
686 if (tudebug)
687 showregs(type, rp, addr, 0);
688 }
689
690 /*
691 * In the case where both pagefault and grow fail,
692 * set the code to the value provided by pagefault.
693 */
694 (void) instr_size(rp, &addr, rw);
695 bzero(&siginfo, sizeof (siginfo));
696 siginfo.si_addr = addr;
697 if (FC_CODE(res) == FC_OBJERR) {
698 siginfo.si_errno = FC_ERRNO(res);
699 if (siginfo.si_errno != EINTR) {
700 siginfo.si_signo = SIGBUS;
701 siginfo.si_code = BUS_OBJERR;
702 fault = FLTACCESS;
703 }
704 } else { /* FC_NOMAP || FC_PROT */
705 siginfo.si_signo = SIGSEGV;
706 siginfo.si_code = (res == FC_NOMAP) ?
707 SEGV_MAPERR : SEGV_ACCERR;
708 fault = FLTBOUNDS;
709 }
710 /*
711 * If this is the culmination of a single-step,
712 * reset the addr, code, signal and fault to
713 * indicate a hardware trace trap.
714 */
715 if (stepped) {
716 pcb_t *pcb = &lwp->lwp_pcb;
717
718 siginfo.si_signo = 0;
719 fault = 0;
720 if (pcb->pcb_step == STEP_WASACTIVE) {
721 pcb->pcb_step = STEP_NONE;
722 pcb->pcb_tracepc = NULL;
723 oldpc = rp->r_pc - 4;
724 }
725 /*
726 * If both NORMAL_STEP and WATCH_STEP are in
727 * effect, give precedence to WATCH_STEP.
728 * One or the other must be set at this point.
729 */
730 ASSERT(pcb->pcb_flags & (NORMAL_STEP|WATCH_STEP));
731 if ((fault = undo_watch_step(&siginfo)) == 0 &&
732 (pcb->pcb_flags & NORMAL_STEP)) {
733 siginfo.si_signo = SIGTRAP;
734 siginfo.si_code = TRAP_TRACE;
735 siginfo.si_addr = (caddr_t)rp->r_pc;
736 fault = FLTTRACE;
737 }
738 pcb->pcb_flags &= ~(NORMAL_STEP|WATCH_STEP);
739 }
740 break;
741
742 case T_DATA_EXCEPTION + T_USER: /* user data access exception */
743
744 if (&vis1_partial_support != NULL) {
745 bzero(&siginfo, sizeof (siginfo));
746 if (vis1_partial_support(rp,
747 &siginfo, &fault) == 0)
748 goto out;
749 }
750
751 if (nfload(rp, &instr))
752 goto out;
753 if (IS_FLUSH(instr)) {
754 /* skip the flush instruction */
755 rp->r_pc = rp->r_npc;
756 rp->r_npc += 4;
757 goto out;
758 /*NOTREACHED*/
759 }
760 bzero(&siginfo, sizeof (siginfo));
761 siginfo.si_addr = addr;
762 switch (X_FAULT_TYPE(mmu_fsr)) {
763 case FT_ATOMIC_NC:
764 if ((IS_SWAP(instr) && swap_nc(rp, instr)) ||
765 (IS_LDSTUB(instr) && ldstub_nc(rp, instr))) {
766 /* skip the atomic */
767 rp->r_pc = rp->r_npc;
768 rp->r_npc += 4;
769 goto out;
770 }
771 /* fall into ... */
772 case FT_PRIV:
773 siginfo.si_signo = SIGSEGV;
774 siginfo.si_code = SEGV_ACCERR;
775 fault = FLTBOUNDS;
776 break;
777 case FT_SPEC_LD:
778 case FT_ILL_ALT:
779 siginfo.si_signo = SIGILL;
780 siginfo.si_code = ILL_ILLADR;
781 fault = FLTILL;
782 break;
783 default:
784 siginfo.si_signo = SIGSEGV;
785 siginfo.si_code = SEGV_MAPERR;
786 fault = FLTBOUNDS;
787 break;
788 }
789 break;
790
791 case T_SYS_RTT_ALIGN + T_USER: /* user alignment error */
792 case T_ALIGNMENT + T_USER: /* user alignment error */
793 if (tudebug)
794 showregs(type, rp, addr, 0);
795 /*
796 * If the user has to do unaligned references
797 * the ugly stuff gets done here.
798 */
799 alignfaults++;
800 if (&vis1_partial_support != NULL) {
801 bzero(&siginfo, sizeof (siginfo));
802 if (vis1_partial_support(rp,
803 &siginfo, &fault) == 0)
804 goto out;
805 }
806
807 bzero(&siginfo, sizeof (siginfo));
808 if (type == T_SYS_RTT_ALIGN + T_USER) {
809 if (nfload(rp, NULL))
810 goto out;
811 /*
812 * Can't do unaligned stack access
813 */
814 siginfo.si_signo = SIGBUS;
815 siginfo.si_code = BUS_ADRALN;
816 siginfo.si_addr = addr;
817 fault = FLTACCESS;
818 break;
819 }
820
821 /*
822 * Try to fix alignment before non-faulting load test.
823 */
824 if (p->p_fixalignment) {
825 if (do_unaligned(rp, &badaddr) == SIMU_SUCCESS) {
826 rp->r_pc = rp->r_npc;
827 rp->r_npc += 4;
828 goto out;
829 }
830 if (nfload(rp, NULL))
831 goto out;
832 siginfo.si_signo = SIGSEGV;
833 siginfo.si_code = SEGV_MAPERR;
834 siginfo.si_addr = badaddr;
835 fault = FLTBOUNDS;
836 } else {
837 if (nfload(rp, NULL))
838 goto out;
839 siginfo.si_signo = SIGBUS;
840 siginfo.si_code = BUS_ADRALN;
841 if (rp->r_pc & 3) { /* offending address, if pc */
842 siginfo.si_addr = (caddr_t)rp->r_pc;
843 } else {
844 if (calc_memaddr(rp, &badaddr) == SIMU_UNALIGN)
845 siginfo.si_addr = badaddr;
846 else
847 siginfo.si_addr = (caddr_t)rp->r_pc;
848 }
849 fault = FLTACCESS;
850 }
851 break;
852
853 case T_PRIV_INSTR + T_USER: /* privileged instruction fault */
854 if (tudebug)
855 showregs(type, rp, (caddr_t)0, 0);
856
857 bzero(&siginfo, sizeof (siginfo));
858 #ifdef sun4v
859 /*
860 * If this instruction fault is a non-privileged %tick
861 * or %stick trap, and %tick/%stick user emulation is
862 * enabled as a result of an OS suspend, then simulate
863 * the register read. We rely on simulate_rdtick to fail
864 * if the instruction is not a %tick or %stick read,
865 * causing us to fall through to the normal privileged
866 * instruction handling.
867 */
868 if (tick_stick_emulation_active &&
869 (X_FAULT_TYPE(mmu_fsr) == FT_NEW_PRVACT) &&
870 simulate_rdtick(rp) == SIMU_SUCCESS) {
871 /* skip the successfully simulated instruction */
872 rp->r_pc = rp->r_npc;
873 rp->r_npc += 4;
874 goto out;
875 }
876 #endif
877 siginfo.si_signo = SIGILL;
878 siginfo.si_code = ILL_PRVOPC;
879 siginfo.si_addr = (caddr_t)rp->r_pc;
880 fault = FLTILL;
881 break;
882
883 case T_UNIMP_INSTR: /* priv illegal instruction fault */
884 if (fpras_implemented) {
885 /*
886 * Call fpras_chktrap indicating that
887 * we've come from a trap handler and pass
888 * the regs. That function may choose to panic
889 * (in which case it won't return) or it may
890 * determine that a reboot is desired. In the
891 * latter case it must alter pc/npc to skip
892 * the illegal instruction and continue at
893 * a controlled address.
894 */
895 if (&fpras_chktrap) {
896 if (fpras_chktrap(rp))
897 goto cleanup;
898 }
899 }
900 #if defined(SF_ERRATA_23) || defined(SF_ERRATA_30) /* call ... illegal-insn */
901 instr = *(int *)rp->r_pc;
902 if ((instr & 0xc0000000) == 0x40000000) {
903 long pc;
904
905 rp->r_o7 = (long long)rp->r_pc;
906 pc = rp->r_pc + ((instr & 0x3fffffff) << 2);
907 rp->r_pc = rp->r_npc;
908 rp->r_npc = pc;
909 ill_calls++;
910 goto cleanup;
911 }
912 #endif /* SF_ERRATA_23 || SF_ERRATA_30 */
913 /*
914 * It's not an fpras failure and it's not SF_ERRATA_23 - die
915 */
916 addr = (caddr_t)rp->r_pc;
917 (void) die(type, rp, addr, 0);
918 /*NOTREACHED*/
919
920 case T_UNIMP_INSTR + T_USER: /* illegal instruction fault */
921 #if defined(SF_ERRATA_23) || defined(SF_ERRATA_30) /* call ... illegal-insn */
922 instr = fetch_user_instr((caddr_t)rp->r_pc);
923 if ((instr & 0xc0000000) == 0x40000000) {
924 long pc;
925
926 rp->r_o7 = (long long)rp->r_pc;
927 pc = rp->r_pc + ((instr & 0x3fffffff) << 2);
928 rp->r_pc = rp->r_npc;
929 rp->r_npc = pc;
930 ill_calls++;
931 goto out;
932 }
933 #endif /* SF_ERRATA_23 || SF_ERRATA_30 */
934 if (tudebug)
935 showregs(type, rp, (caddr_t)0, 0);
936 bzero(&siginfo, sizeof (siginfo));
937 /*
938 * Try to simulate the instruction.
939 */
940 switch (simulate_unimp(rp, &badaddr)) {
941 case SIMU_RETRY:
942 goto out; /* regs are already set up */
943 /*NOTREACHED*/
944
945 case SIMU_SUCCESS:
946 /* skip the successfully simulated instruction */
947 rp->r_pc = rp->r_npc;
948 rp->r_npc += 4;
949 goto out;
950 /*NOTREACHED*/
951
952 case SIMU_FAULT:
953 siginfo.si_signo = SIGSEGV;
954 siginfo.si_code = SEGV_MAPERR;
955 siginfo.si_addr = badaddr;
956 fault = FLTBOUNDS;
957 break;
958
959 case SIMU_DZERO:
960 siginfo.si_signo = SIGFPE;
961 siginfo.si_code = FPE_INTDIV;
962 siginfo.si_addr = (caddr_t)rp->r_pc;
963 fault = FLTIZDIV;
964 break;
965
966 case SIMU_UNALIGN:
967 siginfo.si_signo = SIGBUS;
968 siginfo.si_code = BUS_ADRALN;
969 siginfo.si_addr = badaddr;
970 fault = FLTACCESS;
971 break;
972
973 case SIMU_ILLEGAL:
974 default:
975 siginfo.si_signo = SIGILL;
976 op3 = (instr >> 19) & 0x3F;
977 if ((IS_FLOAT(instr) && (op3 == IOP_V8_STQFA) ||
978 (op3 == IOP_V8_STDFA)))
979 siginfo.si_code = ILL_ILLADR;
980 else
981 siginfo.si_code = ILL_ILLOPC;
982 siginfo.si_addr = (caddr_t)rp->r_pc;
983 fault = FLTILL;
984 break;
985 }
986 break;
987
988 case T_UNIMP_LDD + T_USER:
989 case T_UNIMP_STD + T_USER:
990 if (tudebug)
991 showregs(type, rp, (caddr_t)0, 0);
992 switch (simulate_lddstd(rp, &badaddr)) {
993 case SIMU_SUCCESS:
994 /* skip the successfully simulated instruction */
995 rp->r_pc = rp->r_npc;
996 rp->r_npc += 4;
997 goto out;
998 /*NOTREACHED*/
999
1000 case SIMU_FAULT:
1001 if (nfload(rp, NULL))
1002 goto out;
1003 siginfo.si_signo = SIGSEGV;
1004 siginfo.si_code = SEGV_MAPERR;
1005 siginfo.si_addr = badaddr;
1006 fault = FLTBOUNDS;
1007 break;
1008
1009 case SIMU_UNALIGN:
1010 if (nfload(rp, NULL))
1011 goto out;
1012 siginfo.si_signo = SIGBUS;
1013 siginfo.si_code = BUS_ADRALN;
1014 siginfo.si_addr = badaddr;
1015 fault = FLTACCESS;
1016 break;
1017
1018 case SIMU_ILLEGAL:
1019 default:
1020 siginfo.si_signo = SIGILL;
1021 siginfo.si_code = ILL_ILLOPC;
1022 siginfo.si_addr = (caddr_t)rp->r_pc;
1023 fault = FLTILL;
1024 break;
1025 }
1026 break;
1027
1028 case T_UNIMP_LDD:
1029 case T_UNIMP_STD:
1030 if (simulate_lddstd(rp, &badaddr) == SIMU_SUCCESS) {
1031 /* skip the successfully simulated instruction */
1032 rp->r_pc = rp->r_npc;
1033 rp->r_npc += 4;
1034 goto cleanup;
1035 /*NOTREACHED*/
1036 }
1037 /*
1038 * A third party driver executed an {LDD,STD,LDDA,STDA}
1039 * that we couldn't simulate.
1040 */
1041 if (nfload(rp, NULL))
1042 goto cleanup;
1043
1044 if (curthread->t_lofault) {
1045 if (lodebug) {
1046 showregs(type, rp, addr, 0);
1047 traceback((caddr_t)rp->r_sp);
1048 }
1049 rp->r_g1 = EFAULT;
1050 rp->r_pc = curthread->t_lofault;
1051 rp->r_npc = rp->r_pc + 4;
1052 goto cleanup;
1053 }
1054 (void) die(type, rp, addr, 0);
1055 /*NOTREACHED*/
1056
1057 case T_IDIV0 + T_USER: /* integer divide by zero */
1058 case T_DIV0 + T_USER: /* integer divide by zero */
1059 if (tudebug && tudebugfpe)
1060 showregs(type, rp, (caddr_t)0, 0);
1061 bzero(&siginfo, sizeof (siginfo));
1062 siginfo.si_signo = SIGFPE;
1063 siginfo.si_code = FPE_INTDIV;
1064 siginfo.si_addr = (caddr_t)rp->r_pc;
1065 fault = FLTIZDIV;
1066 break;
1067
1068 case T_INT_OVERFLOW + T_USER: /* integer overflow */
1069 if (tudebug && tudebugfpe)
1070 showregs(type, rp, (caddr_t)0, 0);
1071 bzero(&siginfo, sizeof (siginfo));
1072 siginfo.si_signo = SIGFPE;
1073 siginfo.si_code = FPE_INTOVF;
1074 siginfo.si_addr = (caddr_t)rp->r_pc;
1075 fault = FLTIOVF;
1076 break;
1077
1078 case T_BREAKPOINT + T_USER: /* breakpoint trap (t 1) */
1079 if (tudebug && tudebugbpt)
1080 showregs(type, rp, (caddr_t)0, 0);
1081 bzero(&siginfo, sizeof (siginfo));
1082 siginfo.si_signo = SIGTRAP;
1083 siginfo.si_code = TRAP_BRKPT;
1084 siginfo.si_addr = (caddr_t)rp->r_pc;
1085 fault = FLTBPT;
1086 break;
1087
1088 case T_TAG_OVERFLOW + T_USER: /* tag overflow (taddcctv, tsubcctv) */
1089 if (tudebug)
1090 showregs(type, rp, (caddr_t)0, 0);
1091 bzero(&siginfo, sizeof (siginfo));
1092 siginfo.si_signo = SIGEMT;
1093 siginfo.si_code = EMT_TAGOVF;
1094 siginfo.si_addr = (caddr_t)rp->r_pc;
1095 fault = FLTACCESS;
1096 break;
1097
1098 case T_FLUSH_PCB + T_USER: /* finish user window overflow */
1099 case T_FLUSHW + T_USER: /* finish user window flush */
1100 /*
1101 * This trap is entered from sys_rtt in locore.s when,
1102 * upon return to user is is found that there are user
1103 * windows in pcb_wbuf. This happens because they could
1104 * not be saved on the user stack, either because it
1105 * wasn't resident or because it was misaligned.
1106 */
1107 {
1108 int error;
1109 caddr_t sp;
1110
1111 error = flush_user_windows_to_stack(&sp);
1112 /*
1113 * Possible errors:
1114 * error copying out
1115 * unaligned stack pointer
1116 * The first is given to us as the return value
1117 * from flush_user_windows_to_stack(). The second
1118 * results in residual windows in the pcb.
1119 */
1120 if (error != 0) {
1121 /*
1122 * EINTR comes from a signal during copyout;
1123 * we should not post another signal.
1124 */
1125 if (error != EINTR) {
1126 /*
1127 * Zap the process with a SIGSEGV - process
1128 * may be managing its own stack growth by
1129 * taking SIGSEGVs on a different signal stack.
1130 */
1131 bzero(&siginfo, sizeof (siginfo));
1132 siginfo.si_signo = SIGSEGV;
1133 siginfo.si_code = SEGV_MAPERR;
1134 siginfo.si_addr = sp;
1135 fault = FLTBOUNDS;
1136 }
1137 break;
1138 } else if (mpcb->mpcb_wbcnt) {
1139 bzero(&siginfo, sizeof (siginfo));
1140 siginfo.si_signo = SIGILL;
1141 siginfo.si_code = ILL_BADSTK;
1142 siginfo.si_addr = (caddr_t)rp->r_pc;
1143 fault = FLTILL;
1144 break;
1145 }
1146 }
1147
1148 /*
1149 * T_FLUSHW is used when handling a ta 0x3 -- the old flush
1150 * window trap -- which is implemented by executing the
1151 * flushw instruction. The flushw can trap if any of the
1152 * stack pages are not writable for whatever reason. In this
1153 * case only, we advance the pc to the next instruction so
1154 * that the user thread doesn't needlessly execute the trap
1155 * again. Normally this wouldn't be a problem -- we'll
1156 * usually only end up here if this is the first touch to a
1157 * stack page -- since the second execution won't trap, but
1158 * if there's a watchpoint on the stack page the user thread
1159 * would spin, continuously executing the trap instruction.
1160 */
1161 if (type == T_FLUSHW + T_USER) {
1162 rp->r_pc = rp->r_npc;
1163 rp->r_npc += 4;
1164 }
1165 goto out;
1166
1167 case T_AST + T_USER: /* profiling or resched pseudo trap */
1168 if (lwp->lwp_pcb.pcb_flags & CPC_OVERFLOW) {
1169 lwp->lwp_pcb.pcb_flags &= ~CPC_OVERFLOW;
1170 if (kcpc_overflow_ast()) {
1171 /*
1172 * Signal performance counter overflow
1173 */
1174 if (tudebug)
1175 showregs(type, rp, (caddr_t)0, 0);
1176 bzero(&siginfo, sizeof (siginfo));
1177 siginfo.si_signo = SIGEMT;
1178 siginfo.si_code = EMT_CPCOVF;
1179 siginfo.si_addr = (caddr_t)rp->r_pc;
1180 /* for trap_cleanup(), below */
1181 oldpc = rp->r_pc - 4;
1182 fault = FLTCPCOVF;
1183 }
1184 }
1185
1186 /*
1187 * The CPC_OVERFLOW check above may already have populated
1188 * siginfo and set fault, so the checks below must not
1189 * touch these and the functions they call must use
1190 * trapsig() directly.
1191 */
1192
1193 if (lwp->lwp_pcb.pcb_flags & ASYNC_HWERR) {
1194 lwp->lwp_pcb.pcb_flags &= ~ASYNC_HWERR;
1195 trap_async_hwerr();
1196 }
1197
1198 if (lwp->lwp_pcb.pcb_flags & ASYNC_BERR) {
1199 lwp->lwp_pcb.pcb_flags &= ~ASYNC_BERR;
1200 trap_async_berr_bto(ASYNC_BERR, rp);
1201 }
1202
1203 if (lwp->lwp_pcb.pcb_flags & ASYNC_BTO) {
1204 lwp->lwp_pcb.pcb_flags &= ~ASYNC_BTO;
1205 trap_async_berr_bto(ASYNC_BTO, rp);
1206 }
1207
1208 break;
1209 }
1210
1211 if (fault) {
1212 /* We took a fault so abort single step. */
1213 lwp->lwp_pcb.pcb_flags &= ~(NORMAL_STEP|WATCH_STEP);
1214 }
1215 trap_cleanup(rp, fault, &siginfo, oldpc == rp->r_pc);
1216
1217 out: /* We can't get here from a system trap */
1218 ASSERT(type & T_USER);
1219 trap_rtt();
1220 (void) new_mstate(curthread, mstate);
1221 /* Kernel probe */
1222 TNF_PROBE_1(thread_state, "thread", /* CSTYLED */,
1223 tnf_microstate, state, LMS_USER);
1224
1225 TRACE_0(TR_FAC_TRAP, TR_C_TRAP_HANDLER_EXIT, "C_trap_handler_exit");
1226 return;
1227
1228 cleanup: /* system traps end up here */
1229 ASSERT(!(type & T_USER));
1230
1231 TRACE_0(TR_FAC_TRAP, TR_C_TRAP_HANDLER_EXIT, "C_trap_handler_exit");
1232 }
1233
1234 void
1235 trap_cleanup(
1236 struct regs *rp,
1237 uint_t fault,
1238 k_siginfo_t *sip,
1239 int restartable)
1240 {
1241 extern void aio_cleanup();
1242 proc_t *p = ttoproc(curthread);
1243 klwp_id_t lwp = ttolwp(curthread);
1244
1245 if (fault) {
1246 /*
1247 * Remember the fault and fault address
1248 * for real-time (SIGPROF) profiling.
1249 */
1250 lwp->lwp_lastfault = fault;
1251 lwp->lwp_lastfaddr = sip->si_addr;
1252
1253 DTRACE_PROC2(fault, int, fault, ksiginfo_t *, sip);
1254
1255 /*
1256 * If a debugger has declared this fault to be an
1257 * event of interest, stop the lwp. Otherwise just
1258 * deliver the associated signal.
1259 */
1260 if (sip->si_signo != SIGKILL &&
1261 prismember(&p->p_fltmask, fault) &&
1262 stop_on_fault(fault, sip) == 0)
1263 sip->si_signo = 0;
1264 }
1265
1266 if (sip->si_signo)
1267 trapsig(sip, restartable);
1268
1269 if (lwp->lwp_oweupc)
1270 profil_tick(rp->r_pc);
1271
1272 if (curthread->t_astflag | curthread->t_sig_check) {
1273 /*
1274 * Turn off the AST flag before checking all the conditions that
1275 * may have caused an AST. This flag is on whenever a signal or
1276 * unusual condition should be handled after the next trap or
1277 * syscall.
1278 */
1279 astoff(curthread);
1280 curthread->t_sig_check = 0;
1281
1282 /*
1283 * The following check is legal for the following reasons:
1284 * 1) The thread we are checking, is ourselves, so there is
1285 * no way the proc can go away.
1286 * 2) The only time we need to be protected by the
1287 * lock is if the binding is changed.
1288 *
1289 * Note we will still take the lock and check the binding
1290 * if the condition was true without the lock held. This
1291 * prevents lock contention among threads owned by the
1292 * same proc.
1293 */
1294
1295 if (curthread->t_proc_flag & TP_CHANGEBIND) {
1296 mutex_enter(&p->p_lock);
1297 if (curthread->t_proc_flag & TP_CHANGEBIND) {
1298 timer_lwpbind();
1299 curthread->t_proc_flag &= ~TP_CHANGEBIND;
1300 }
1301 mutex_exit(&p->p_lock);
1302 }
1303
1304 /*
1305 * for kaio requests that are on the per-process poll queue,
1306 * aiop->aio_pollq, they're AIO_POLL bit is set, the kernel
1307 * should copyout their result_t to user memory. by copying
1308 * out the result_t, the user can poll on memory waiting
1309 * for the kaio request to complete.
1310 */
1311 if (p->p_aio)
1312 aio_cleanup(0);
1313
1314 /*
1315 * If this LWP was asked to hold, call holdlwp(), which will
1316 * stop. holdlwps() sets this up and calls pokelwps() which
1317 * sets the AST flag.
1318 *
1319 * Also check TP_EXITLWP, since this is used by fresh new LWPs
1320 * through lwp_rtt(). That flag is set if the lwp_create(2)
1321 * syscall failed after creating the LWP.
1322 */
1323 if (ISHOLD(p))
1324 holdlwp();
1325
1326 /*
1327 * All code that sets signals and makes ISSIG evaluate true must
1328 * set t_astflag afterwards.
1329 */
1330 if (ISSIG_PENDING(curthread, lwp, p)) {
1331 if (issig(FORREAL))
1332 psig();
1333 curthread->t_sig_check = 1;
1334 }
1335
1336 if (curthread->t_rprof != NULL) {
1337 realsigprof(0, 0, 0);
1338 curthread->t_sig_check = 1;
1339 }
1340 }
1341 }
1342
1343 /*
1344 * Called from fp_traps when a floating point trap occurs.
1345 * Note that the T_DATA_EXCEPTION case does not use X_FAULT_TYPE(mmu_fsr),
1346 * because mmu_fsr (now changed to code) is always 0.
1347 * Note that the T_UNIMP_INSTR case does not call simulate_unimp(),
1348 * because the simulator only simulates multiply and divide instructions,
1349 * which would not cause floating point traps in the first place.
1350 * XXX - Supervisor mode floating point traps?
1351 */
1352 void
1353 fpu_trap(struct regs *rp, caddr_t addr, uint32_t type, uint32_t code)
1354 {
1355 proc_t *p = ttoproc(curthread);
1356 klwp_id_t lwp = ttolwp(curthread);
1357 k_siginfo_t siginfo;
1358 uint_t op3, fault = 0;
1359 int mstate;
1360 char *badaddr;
1361 kfpu_t *fp;
1362 struct _fpq *pfpq;
1363 uint32_t inst;
1364 utrap_handler_t *utrapp;
1365
1366 CPU_STATS_ADDQ(CPU, sys, trap, 1);
1367
1368 if (USERMODE(rp->r_tstate)) {
1369 /*
1370 * Set lwp_state before trying to acquire any
1371 * adaptive lock
1372 */
1373 ASSERT(lwp != NULL);
1374 lwp->lwp_state = LWP_SYS;
1375 /*
1376 * Set up the current cred to use during this trap. u_cred
1377 * no longer exists. t_cred is used instead.
1378 * The current process credential applies to the thread for
1379 * the entire trap. If trapping from the kernel, this
1380 * should already be set up.
1381 */
1382 if (curthread->t_cred != p->p_cred) {
1383 cred_t *oldcred = curthread->t_cred;
1384 /*
1385 * DTrace accesses t_cred in probe context. t_cred
1386 * must always be either NULL, or point to a valid,
1387 * allocated cred structure.
1388 */
1389 curthread->t_cred = crgetcred();
1390 crfree(oldcred);
1391 }
1392 ASSERT(lwp->lwp_regs == rp);
1393 mstate = new_mstate(curthread, LMS_TRAP);
1394 siginfo.si_signo = 0;
1395 type |= T_USER;
1396 }
1397
1398 TRACE_1(TR_FAC_TRAP, TR_C_TRAP_HANDLER_ENTER,
1399 "C_fpu_trap_handler_enter:type %x", type);
1400
1401 if (tudebug && tudebugfpe)
1402 showregs(type, rp, addr, 0);
1403
1404 bzero(&siginfo, sizeof (siginfo));
1405 siginfo.si_code = code;
1406 siginfo.si_addr = addr;
1407
1408 switch (type) {
1409
1410 case T_FP_EXCEPTION_IEEE + T_USER: /* FPU arithmetic exception */
1411 /*
1412 * FPU arithmetic exception - fake up a fpq if we
1413 * came here directly from _fp_ieee_exception,
1414 * which is indicated by a zero fpu_qcnt.
1415 */
1416 fp = lwptofpu(curthread->t_lwp);
1417 utrapp = curthread->t_procp->p_utraps;
1418 if (fp->fpu_qcnt == 0) {
1419 inst = fetch_user_instr((caddr_t)rp->r_pc);
1420 lwp->lwp_state = LWP_SYS;
1421 pfpq = &fp->fpu_q->FQu.fpq;
1422 pfpq->fpq_addr = (uint32_t *)rp->r_pc;
1423 pfpq->fpq_instr = inst;
1424 fp->fpu_qcnt = 1;
1425 fp->fpu_q_entrysize = sizeof (struct _fpq);
1426 #ifdef SF_V9_TABLE_28
1427 /*
1428 * Spitfire and blackbird followed the SPARC V9 manual
1429 * paragraph 3 of section 5.1.7.9 FSR_current_exception
1430 * (cexc) for setting fsr.cexc bits on underflow and
1431 * overflow traps when the fsr.tem.inexact bit is set,
1432 * instead of following Table 28. Bugid 1263234.
1433 */
1434 {
1435 extern int spitfire_bb_fsr_bug;
1436
1437 if (spitfire_bb_fsr_bug &&
1438 (fp->fpu_fsr & FSR_TEM_NX)) {
1439 if (((fp->fpu_fsr & FSR_TEM_OF) == 0) &&
1440 (fp->fpu_fsr & FSR_CEXC_OF)) {
1441 fp->fpu_fsr &= ~FSR_CEXC_OF;
1442 fp->fpu_fsr |= FSR_CEXC_NX;
1443 _fp_write_pfsr(&fp->fpu_fsr);
1444 siginfo.si_code = FPE_FLTRES;
1445 }
1446 if (((fp->fpu_fsr & FSR_TEM_UF) == 0) &&
1447 (fp->fpu_fsr & FSR_CEXC_UF)) {
1448 fp->fpu_fsr &= ~FSR_CEXC_UF;
1449 fp->fpu_fsr |= FSR_CEXC_NX;
1450 _fp_write_pfsr(&fp->fpu_fsr);
1451 siginfo.si_code = FPE_FLTRES;
1452 }
1453 }
1454 }
1455 #endif /* SF_V9_TABLE_28 */
1456 rp->r_pc = rp->r_npc;
1457 rp->r_npc += 4;
1458 } else if (utrapp && utrapp[UT_FP_EXCEPTION_IEEE_754]) {
1459 /*
1460 * The user had a trap handler installed. Jump to
1461 * the trap handler instead of signalling the process.
1462 */
1463 rp->r_pc = (long)utrapp[UT_FP_EXCEPTION_IEEE_754];
1464 rp->r_npc = rp->r_pc + 4;
1465 break;
1466 }
1467 siginfo.si_signo = SIGFPE;
1468 fault = FLTFPE;
1469 break;
1470
1471 case T_DATA_EXCEPTION + T_USER: /* user data access exception */
1472 siginfo.si_signo = SIGSEGV;
1473 fault = FLTBOUNDS;
1474 break;
1475
1476 case T_LDDF_ALIGN + T_USER: /* 64 bit user lddfa alignment error */
1477 case T_STDF_ALIGN + T_USER: /* 64 bit user stdfa alignment error */
1478 alignfaults++;
1479 lwp->lwp_state = LWP_SYS;
1480 if (&vis1_partial_support != NULL) {
1481 bzero(&siginfo, sizeof (siginfo));
1482 if (vis1_partial_support(rp,
1483 &siginfo, &fault) == 0)
1484 goto out;
1485 }
1486 if (do_unaligned(rp, &badaddr) == SIMU_SUCCESS) {
1487 rp->r_pc = rp->r_npc;
1488 rp->r_npc += 4;
1489 goto out;
1490 }
1491 fp = lwptofpu(curthread->t_lwp);
1492 fp->fpu_qcnt = 0;
1493 siginfo.si_signo = SIGSEGV;
1494 siginfo.si_code = SEGV_MAPERR;
1495 siginfo.si_addr = badaddr;
1496 fault = FLTBOUNDS;
1497 break;
1498
1499 case T_ALIGNMENT + T_USER: /* user alignment error */
1500 /*
1501 * If the user has to do unaligned references
1502 * the ugly stuff gets done here.
1503 * Only handles vanilla loads and stores.
1504 */
1505 alignfaults++;
1506 if (p->p_fixalignment) {
1507 if (do_unaligned(rp, &badaddr) == SIMU_SUCCESS) {
1508 rp->r_pc = rp->r_npc;
1509 rp->r_npc += 4;
1510 goto out;
1511 }
1512 siginfo.si_signo = SIGSEGV;
1513 siginfo.si_code = SEGV_MAPERR;
1514 siginfo.si_addr = badaddr;
1515 fault = FLTBOUNDS;
1516 } else {
1517 siginfo.si_signo = SIGBUS;
1518 siginfo.si_code = BUS_ADRALN;
1519 if (rp->r_pc & 3) { /* offending address, if pc */
1520 siginfo.si_addr = (caddr_t)rp->r_pc;
1521 } else {
1522 if (calc_memaddr(rp, &badaddr) == SIMU_UNALIGN)
1523 siginfo.si_addr = badaddr;
1524 else
1525 siginfo.si_addr = (caddr_t)rp->r_pc;
1526 }
1527 fault = FLTACCESS;
1528 }
1529 break;
1530
1531 case T_UNIMP_INSTR + T_USER: /* illegal instruction fault */
1532 siginfo.si_signo = SIGILL;
1533 inst = fetch_user_instr((caddr_t)rp->r_pc);
1534 op3 = (inst >> 19) & 0x3F;
1535 if ((op3 == IOP_V8_STQFA) || (op3 == IOP_V8_STDFA))
1536 siginfo.si_code = ILL_ILLADR;
1537 else
1538 siginfo.si_code = ILL_ILLTRP;
1539 fault = FLTILL;
1540 break;
1541
1542 default:
1543 (void) die(type, rp, addr, 0);
1544 /*NOTREACHED*/
1545 }
1546
1547 /*
1548 * We can't get here from a system trap
1549 * Never restart any instruction which got here from an fp trap.
1550 */
1551 ASSERT(type & T_USER);
1552
1553 trap_cleanup(rp, fault, &siginfo, 0);
1554 out:
1555 trap_rtt();
1556 (void) new_mstate(curthread, mstate);
1557 }
1558
1559 void
1560 trap_rtt(void)
1561 {
1562 klwp_id_t lwp = ttolwp(curthread);
1563
1564 /*
1565 * Restore register window if a debugger modified it.
1566 * Set up to perform a single-step if a debugger requested it.
1567 */
1568 if (lwp->lwp_pcb.pcb_xregstat != XREGNONE)
1569 xregrestore(lwp, 0);
1570
1571 /*
1572 * Set state to LWP_USER here so preempt won't give us a kernel
1573 * priority if it occurs after this point. Call CL_TRAPRET() to
1574 * restore the user-level priority.
1575 *
1576 * It is important that no locks (other than spinlocks) be entered
1577 * after this point before returning to user mode (unless lwp_state
1578 * is set back to LWP_SYS).
1579 */
1580 lwp->lwp_state = LWP_USER;
1581 if (curthread->t_trapret) {
1582 curthread->t_trapret = 0;
1583 thread_lock(curthread);
1584 CL_TRAPRET(curthread);
1585 thread_unlock(curthread);
1586 }
1587 if (CPU->cpu_runrun || curthread->t_schedflag & TS_ANYWAITQ)
1588 preempt();
1589 prunstop();
1590 if (lwp->lwp_pcb.pcb_step != STEP_NONE)
1591 prdostep();
1592
1593 TRACE_0(TR_FAC_TRAP, TR_C_TRAP_HANDLER_EXIT, "C_trap_handler_exit");
1594 }
1595
1596 #define IS_LDASI(o) \
1597 ((o) == (uint32_t)0xC0C00000 || (o) == (uint32_t)0xC0800000 || \
1598 (o) == (uint32_t)0xC1800000)
1599 #define IS_IMM_ASI(i) (((i) & 0x2000) == 0)
1600 #define IS_ASINF(a) (((a) & 0xF6) == 0x82)
1601 #define IS_LDDA(i) (((i) & 0xC1F80000) == 0xC0980000)
1602
1603 static int
1604 nfload(struct regs *rp, int *instrp)
1605 {
1606 uint_t instr, asi, op3, rd;
1607 size_t len;
1608 struct as *as;
1609 caddr_t addr;
1610 FPU_DREGS_TYPE zero;
1611 extern int segnf_create();
1612
1613 if (USERMODE(rp->r_tstate))
1614 instr = fetch_user_instr((caddr_t)rp->r_pc);
1615 else
1616 instr = *(int *)rp->r_pc;
1617
1618 if (instrp)
1619 *instrp = instr;
1620
1621 op3 = (uint_t)(instr & 0xC1E00000);
1622 if (!IS_LDASI(op3))
1623 return (0);
1624 if (IS_IMM_ASI(instr))
1625 asi = (instr & 0x1FE0) >> 5;
1626 else
1627 asi = (uint_t)((rp->r_tstate >> TSTATE_ASI_SHIFT) &
1628 TSTATE_ASI_MASK);
1629 if (!IS_ASINF(asi))
1630 return (0);
1631 if (calc_memaddr(rp, &addr) == SIMU_SUCCESS) {
1632 len = 1;
1633 as = USERMODE(rp->r_tstate) ? ttoproc(curthread)->p_as : &kas;
1634 as_rangelock(as);
1635 if (as_gap(as, len, &addr, &len, 0, addr) == 0)
1636 (void) as_map(as, addr, len, segnf_create, NULL);
1637 as_rangeunlock(as);
1638 }
1639 zero = 0;
1640 rd = (instr >> 25) & 0x1f;
1641 if (IS_FLOAT(instr)) {
1642 uint_t dbflg = ((instr >> 19) & 3) == 3;
1643
1644 if (dbflg) { /* clever v9 reg encoding */
1645 if (rd & 1)
1646 rd = (rd & 0x1e) | 0x20;
1647 rd >>= 1;
1648 }
1649 if (fpu_exists) {
1650 if (!(_fp_read_fprs() & FPRS_FEF))
1651 fp_enable();
1652
1653 if (dbflg)
1654 _fp_write_pdreg(&zero, rd);
1655 else
1656 _fp_write_pfreg((uint_t *)&zero, rd);
1657 } else {
1658 kfpu_t *fp = lwptofpu(curthread->t_lwp);
1659
1660 if (!fp->fpu_en)
1661 fp_enable();
1662
1663 if (dbflg)
1664 fp->fpu_fr.fpu_dregs[rd] = zero;
1665 else
1666 fp->fpu_fr.fpu_regs[rd] = 0;
1667 }
1668 } else {
1669 (void) putreg(&zero, rp, rd, &addr);
1670 if (IS_LDDA(instr))
1671 (void) putreg(&zero, rp, rd + 1, &addr);
1672 }
1673 rp->r_pc = rp->r_npc;
1674 rp->r_npc += 4;
1675 return (1);
1676 }
1677
1678 kmutex_t atomic_nc_mutex;
1679
1680 /*
1681 * The following couple of routines are for userland drivers which
1682 * do atomics to noncached addresses. This sort of worked on previous
1683 * platforms -- the operation really wasn't atomic, but it didn't generate
1684 * a trap as sun4u systems do.
1685 */
1686 static int
1687 swap_nc(struct regs *rp, int instr)
1688 {
1689 uint64_t rdata, mdata;
1690 caddr_t addr, badaddr;
1691 uint_t tmp, rd;
1692
1693 (void) flush_user_windows_to_stack(NULL);
1694 rd = (instr >> 25) & 0x1f;
1695 if (calc_memaddr(rp, &addr) != SIMU_SUCCESS)
1696 return (0);
1697 if (getreg(rp, rd, &rdata, &badaddr))
1698 return (0);
1699 mutex_enter(&atomic_nc_mutex);
1700 if (fuword32(addr, &tmp) == -1) {
1701 mutex_exit(&atomic_nc_mutex);
1702 return (0);
1703 }
1704 mdata = (u_longlong_t)tmp;
1705 if (suword32(addr, (uint32_t)rdata) == -1) {
1706 mutex_exit(&atomic_nc_mutex);
1707 return (0);
1708 }
1709 (void) putreg(&mdata, rp, rd, &badaddr);
1710 mutex_exit(&atomic_nc_mutex);
1711 return (1);
1712 }
1713
1714 static int
1715 ldstub_nc(struct regs *rp, int instr)
1716 {
1717 uint64_t mdata;
1718 caddr_t addr, badaddr;
1719 uint_t rd;
1720 uint8_t tmp;
1721
1722 (void) flush_user_windows_to_stack(NULL);
1723 rd = (instr >> 25) & 0x1f;
1724 if (calc_memaddr(rp, &addr) != SIMU_SUCCESS)
1725 return (0);
1726 mutex_enter(&atomic_nc_mutex);
1727 if (fuword8(addr, &tmp) == -1) {
1728 mutex_exit(&atomic_nc_mutex);
1729 return (0);
1730 }
1731 mdata = (u_longlong_t)tmp;
1732 if (suword8(addr, (uint8_t)0xff) == -1) {
1733 mutex_exit(&atomic_nc_mutex);
1734 return (0);
1735 }
1736 (void) putreg(&mdata, rp, rd, &badaddr);
1737 mutex_exit(&atomic_nc_mutex);
1738 return (1);
1739 }
1740
1741 /*
1742 * This function helps instr_size() determine the operand size.
1743 * It is called for the extended ldda/stda asi's.
1744 */
1745 int
1746 extended_asi_size(int asi)
1747 {
1748 switch (asi) {
1749 case ASI_PST8_P:
1750 case ASI_PST8_S:
1751 case ASI_PST16_P:
1752 case ASI_PST16_S:
1753 case ASI_PST32_P:
1754 case ASI_PST32_S:
1755 case ASI_PST8_PL:
1756 case ASI_PST8_SL:
1757 case ASI_PST16_PL:
1758 case ASI_PST16_SL:
1759 case ASI_PST32_PL:
1760 case ASI_PST32_SL:
1761 return (8);
1762 case ASI_FL8_P:
1763 case ASI_FL8_S:
1764 case ASI_FL8_PL:
1765 case ASI_FL8_SL:
1766 return (1);
1767 case ASI_FL16_P:
1768 case ASI_FL16_S:
1769 case ASI_FL16_PL:
1770 case ASI_FL16_SL:
1771 return (2);
1772 case ASI_BLK_P:
1773 case ASI_BLK_S:
1774 case ASI_BLK_PL:
1775 case ASI_BLK_SL:
1776 case ASI_BLK_COMMIT_P:
1777 case ASI_BLK_COMMIT_S:
1778 return (64);
1779 }
1780
1781 return (0);
1782 }
1783
1784 /*
1785 * Patch non-zero to disable preemption of threads in the kernel.
1786 */
1787 int IGNORE_KERNEL_PREEMPTION = 0; /* XXX - delete this someday */
1788
1789 struct kpreempt_cnts { /* kernel preemption statistics */
1790 int kpc_idle; /* executing idle thread */
1791 int kpc_intr; /* executing interrupt thread */
1792 int kpc_clock; /* executing clock thread */
1793 int kpc_blocked; /* thread has blocked preemption (t_preempt) */
1794 int kpc_notonproc; /* thread is surrendering processor */
1795 int kpc_inswtch; /* thread has ratified scheduling decision */
1796 int kpc_prilevel; /* processor interrupt level is too high */
1797 int kpc_apreempt; /* asynchronous preemption */
1798 int kpc_spreempt; /* synchronous preemption */
1799 } kpreempt_cnts;
1800
1801 /*
1802 * kernel preemption: forced rescheduling
1803 * preempt the running kernel thread.
1804 */
1805 void
1806 kpreempt(int asyncspl)
1807 {
1808 if (IGNORE_KERNEL_PREEMPTION) {
1809 aston(CPU->cpu_dispthread);
1810 return;
1811 }
1812 /*
1813 * Check that conditions are right for kernel preemption
1814 */
1815 do {
1816 if (curthread->t_preempt) {
1817 /*
1818 * either a privileged thread (idle, panic, interrupt)
1819 * or will check when t_preempt is lowered
1820 * We need to specifically handle the case where
1821 * the thread is in the middle of swtch (resume has
1822 * been called) and has its t_preempt set
1823 * [idle thread and a thread which is in kpreempt
1824 * already] and then a high priority thread is
1825 * available in the local dispatch queue.
1826 * In this case the resumed thread needs to take a
1827 * trap so that it can call kpreempt. We achieve
1828 * this by using siron().
1829 * How do we detect this condition:
1830 * idle thread is running and is in the midst of
1831 * resume: curthread->t_pri == -1 && CPU->dispthread
1832 * != CPU->thread
1833 * Need to ensure that this happens only at high pil
1834 * resume is called at high pil
1835 * Only in resume_from_idle is the pil changed.
1836 */
1837 if (curthread->t_pri < 0) {
1838 kpreempt_cnts.kpc_idle++;
1839 if (CPU->cpu_dispthread != CPU->cpu_thread)
1840 siron();
1841 } else if (curthread->t_flag & T_INTR_THREAD) {
1842 kpreempt_cnts.kpc_intr++;
1843 if (curthread->t_pil == CLOCK_LEVEL)
1844 kpreempt_cnts.kpc_clock++;
1845 } else {
1846 kpreempt_cnts.kpc_blocked++;
1847 if (CPU->cpu_dispthread != CPU->cpu_thread)
1848 siron();
1849 }
1850 aston(CPU->cpu_dispthread);
1851 return;
1852 }
1853 if (curthread->t_state != TS_ONPROC ||
1854 curthread->t_disp_queue != CPU->cpu_disp) {
1855 /* this thread will be calling swtch() shortly */
1856 kpreempt_cnts.kpc_notonproc++;
1857 if (CPU->cpu_thread != CPU->cpu_dispthread) {
1858 /* already in swtch(), force another */
1859 kpreempt_cnts.kpc_inswtch++;
1860 siron();
1861 }
1862 return;
1863 }
1864
1865 if (((asyncspl != KPREEMPT_SYNC) ? spltoipl(asyncspl) :
1866 getpil()) >= DISP_LEVEL) {
1867 /*
1868 * We can't preempt this thread if it is at
1869 * a PIL >= DISP_LEVEL since it may be holding
1870 * a spin lock (like sched_lock).
1871 */
1872 siron(); /* check back later */
1873 kpreempt_cnts.kpc_prilevel++;
1874 return;
1875 }
1876
1877 /*
1878 * block preemption so we don't have multiple preemptions
1879 * pending on the interrupt stack
1880 */
1881 curthread->t_preempt++;
1882 if (asyncspl != KPREEMPT_SYNC) {
1883 splx(asyncspl);
1884 kpreempt_cnts.kpc_apreempt++;
1885 } else
1886 kpreempt_cnts.kpc_spreempt++;
1887
1888 preempt();
1889 curthread->t_preempt--;
1890 } while (CPU->cpu_kprunrun);
1891 }
1892
1893 static enum seg_rw
1894 get_accesstype(struct regs *rp)
1895 {
1896 uint32_t instr;
1897
1898 if (USERMODE(rp->r_tstate))
1899 instr = fetch_user_instr((caddr_t)rp->r_pc);
1900 else
1901 instr = *(uint32_t *)rp->r_pc;
1902
1903 if (IS_FLUSH(instr))
1904 return (S_OTHER);
1905
1906 if (IS_STORE(instr))
1907 return (S_WRITE);
1908 else
1909 return (S_READ);
1910 }
1911
1912 /*
1913 * Handle an asynchronous hardware error.
1914 * The policy is currently to send a hardware error contract event to
1915 * the process's process contract and to kill the process. Eventually
1916 * we may want to instead send a special signal whose default
1917 * disposition is to generate the contract event.
1918 */
1919 void
1920 trap_async_hwerr(void)
1921 {
1922 k_siginfo_t si;
1923 proc_t *p = ttoproc(curthread);
1924 extern void print_msg_hwerr(ctid_t ct_id, proc_t *p);
1925
1926 errorq_drain(ue_queue); /* flush pending async error messages */
1927
1928 print_msg_hwerr(p->p_ct_process->conp_contract.ct_id, p);
1929
1930 contract_process_hwerr(p->p_ct_process, p);
1931
1932 bzero(&si, sizeof (k_siginfo_t));
1933 si.si_signo = SIGKILL;
1934 si.si_code = SI_NOINFO;
1935 trapsig(&si, 1);
1936 }
1937
1938 /*
1939 * Handle bus error and bus timeout for a user process by sending SIGBUS
1940 * The type is either ASYNC_BERR or ASYNC_BTO.
1941 */
1942 void
1943 trap_async_berr_bto(int type, struct regs *rp)
1944 {
1945 k_siginfo_t si;
1946
1947 errorq_drain(ue_queue); /* flush pending async error messages */
1948 bzero(&si, sizeof (k_siginfo_t));
1949
1950 si.si_signo = SIGBUS;
1951 si.si_code = (type == ASYNC_BERR ? BUS_OBJERR : BUS_ADRERR);
1952 si.si_addr = (caddr_t)rp->r_pc; /* AFAR unavailable - future RFE */
1953 si.si_errno = ENXIO;
1954
1955 trapsig(&si, 1);
1956 }
--- EOF ---