1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright (c) 2012, Joyent, Inc.  All rights reserverd.
  25  */
  26 
  27 /*
  28  * To understand the present state of interrupt handling on i86pc, we must
  29  * first consider the history of interrupt controllers and our way of handling
  30  * interrupts.
  31  *
  32  * History of Interrupt Controllers on i86pc
  33  * -----------------------------------------
  34  *
  35  *    Intel 8259 and 8259A
  36  *
  37  * The first interrupt controller that attained widespread use on i86pc was
  38  * the Intel 8259(A) Programmable Interrupt Controller that first saw use with
  39  * the 8086. It took up to 8 interrupt sources and combined them into one
  40  * output wire. Up to 8 8259s could be slaved together providing up to 64 IRQs.
  41  * With the switch to the 8259A, level mode interrupts became possible. For a
  42  * long time on i86pc the 8259A was the only way to handle interrupts and it
  43  * had its own set of quirks. The 8259A and its corresponding interval timer
  44  * the 8254 are programmed using outb and inb instructions.
  45  *
  46  *    Intel Advanced Programmable Interrupt Controller (APIC)
  47  *
  48  * Starting around the time of the introduction of the P6 family
  49  * microarchitecture (i686) Intel introduced a new interrupt controller.
  50  * Instead of having the series of slaved 8259A devices, Intel opted to outfit
  51  * each processor with a Local APIC (lapic) and to outfit the system with at
  52  * least one, but potentially more, I/O APICs (ioapic). The lapics and ioapics
  53  * initially communicated over a dedicated bus, but this has since been
  54  * replaced. Each physical core and even hyperthread currently contains its
  55  * own local apic, which is not shared. There are a few exceptions for
  56  * hyperthreads, but that does not usually concern us.
  57  *
  58  * Instead of talking directly to 8259 for status, sending End Of Interrupt
  59  * (EOI), etc. a microprocessor now communicates directly to the lapic. This
  60  * also allows for each microprocessor to be able to have independent controls.
  61  * The programming method is different from the 8259. Consumers map the lapic
  62  * registers into uncacheable memory to read and manipulate the state.
  63  *
  64  * The number of addressable interrupt vectors was increased to 256. However
  65  * vectors 0-31 are reserved for the processor exception handling, leaving the
  66  * remaining vectors for general use. In addition to hardware generated
  67  * interrupts, the lapic provides a way for generating inter-processor
  68  * interrupts (IPI) which are the basis for CPU cross calls and CPU pokes.
  69  *
  70  * AMD ended up implementing the Intel APIC architecture in lieu of their work
  71  * with Cyrix.
  72  *
  73  *    Intel x2apic
  74  *
  75  * The x2apic is an extension to the lapic which started showing up around the
  76  * same time as the Sandy Bridge chipsets. It provides a new programming mode
  77  * as well as new features. The goal of the x2apic is to solve a few problems
  78  * with the previous generation of lapic and the x2apic is backwards compatible
  79  * with the previous programming and model. The only downsides to using the
  80  * backwards compatibility is that you are not able to take advantage of the new
  81  * x2apic features.
  82  *
  83  *    o The APIC ID is increased from an 8-bit value to a 32-bit value. This
  84  *    increases the maximum number of addressable physical processors beyond
  85  *    256. This new ID is assembled in a similar manner as the information that
  86  *    is obtainable by the extended cpuid topology leaves.
  87  *
  88  *    o A new means of generating IPIs was introduced.
  89  *
  90  *    o Instead of memory mapping the registers, the x2apic only allows for
  91  *    programming it through a series of wrmsrs. This has important semantic
  92  *    side effects. Recall that the registers were previously all mapped to
  93  *    uncachable memory which meant that all operations to the local apic were
  94  *    serializing instructions. With the switch to using wrmsrs this has been
  95  *    relaxed and these operations can no longer be assumed to be serializing
  96  *    instructions.
  97  *
  98  * Note for the rest of this we are only going to concern ourselves with the
  99  * apic and x2apic which practically all of i86pc has been using now for
 100  * quite some time.
 101  *
 102  * Interrupt Priority Levels
 103  * -------------------------
 104  *
 105  * On i86pc systems there are a total of fifteen interrupt priority levels
 106  * (ipls) which range from 1-15. Level 0 is for normal processing and
 107  * non-interrupt processing. To manipulate these values the family of spl
 108  * functions (which date back to UNIX on the PDP-11) are used. Specifically,
 109  * splr() to raise the priority level and splx() to lower it. One should not
 110  * generally call setspl() directly.
 111  *
 112  * Both i86pc and the supported SPARC platforms honor the same conventions for
 113  * the meaning behind these IPLs. The most important IPL is the platform's
 114  * LOCK_LEVEL (0xa on i86pc). If a thread is above LOCK_LEVEL it _must_ not
 115  * sleep on any synchronization object. The only allowed synchronization
 116  * primitive is a mutex that has been specifically initialized to be a spin
 117  * lock (see mutex_init(9F)). Another important level is DISP_LEVEL (0xb on
 118  * i86pc). You must be at DISP_LEVEL if you want to control the dispatcher.
 119  * The XC_HI_PIL is the highest level (0xf) and is used during cross-calls.
 120  *
 121  * Each interrupt that is registered in the system fires at a specific IPL.
 122  * Generally most interrupts fire below LOCK_LEVEL.
 123  *
 124  * PSM Drivers
 125  * -----------
 126  *
 127  * We currently have three sets of PSM (platform specific module) drivers
 128  * available. uppc, pcplusmp, and apix. uppc (uni-processor PC) is the original
 129  * driver that interacts with the 8259A and 8254. In general, it is not used
 130  * anymore given the prevalence of the apic.
 131  *
 132  * The system prefers to use the apix driver over the pcplusmp driver. The apix
 133  * driver requires HW support for an x2apic. If there is no x2apic HW, apix
 134  * will not be used. In general we prefer using the apix driver over the
 135  * pcplusmp driver because it gives us much more flexibility with respect to
 136  * interrupts. In the apix driver each local apic has its own independent set
 137  * of  interrupts, whereas the pcplusmp driver only has a single global set of
 138  * interrupts. This is why pcplusmp only supports a finite number of interrupts
 139  * per IPL -- generally 16, often less. The apix driver supports using either
 140  * the x2apic or the local apic programing modes. The programming mode does not
 141  * change the number of interrupts available, just the number of processors
 142  * that we can address. For the apix driver, the x2apic mode is enabled if the
 143  * system supports interrupt re-mapping, otherwise the module manages the
 144  * x2apic in local mode.
 145  *
 146  * When there is no x2apic present, we default back to the pcplusmp PSM driver.
 147  * In general, this is not problematic unless you have more than 256
 148  * processors in the machine or you do not have enough interrupts available.
 149  *
 150  * Controlling Interrupt Generation on i86pc
 151  * -----------------------------------------
 152  *
 153  * There are two different ways to manipulate which interrupts will be
 154  * generated on i86pc. Each offers different degrees of control.
 155  *
 156  * The first is through the flags register (eflags and rflags on i386 and amd64
 157  * respectively). The IF bit determines whether or not interrupts are enabled
 158  * or disabled. This is manipulated in one of several ways. The most common way
 159  * is through the cli and sti instructions. These clear the IF flag and set it,
 160  * respectively, for the current processor. The other common way is through the
 161  * use of the intr_clear and intr_restore functions.
 162  *
 163  * Assuming interrupts are not blocked by the IF flag, then the second form is
 164  * through the Processor-Priority Register (PPR). The PPR is used to determine
 165  * whether or not a pending interrupt should be delivered. If the ipl of the
 166  * new interrupt is higher than the current value in the PPR, then the lapic
 167  * will either deliver it immediately (if interrupts are not in progress) or it
 168  * will deliver it once the current interrupt processing has issued an EOI. The
 169  * highest unmasked interrupt will be the one delivered.
 170  *
 171  * The PPR register is based upon the max of the following two registers in the
 172  * lapic, the TPR register (also known as CR8 on amd64) that can be used to
 173  * mask interrupt levels, and the current vector. Because the pcplusmp module
 174  * always sets TPR appropriately early in the do_interrupt path, we can usually
 175  * just think that the PPR is the TPR. The pcplusmp module also issues an EOI
 176  * once it has set the TPR, so higher priority interrupts can come in while
 177  * we're servicing a lower priority interrupt.
 178  *
 179  * Handling Interrupts
 180  * -------------------
 181  *
 182  * Interrupts can be broken down into three categories based on priority and
 183  * source:
 184  *
 185  *   o High level interrupts
 186  *   o Low level hardware interrupts
 187  *   o Low level software interrupts
 188  *
 189  *   High Level Interrupts
 190  *
 191  * High level interrupts encompasses both hardware-sourced and software-sourced
 192  * interrupts. Examples of high level hardware interrupts include the serial
 193  * console. High level software-sourced interrupts are still delivered through
 194  * the local apic through IPIs. This is primarily cross calls.
 195  *
 196  * When a high level interrupt comes in, we will raise the SPL and then pin the
 197  * current lwp to the processor. We will use its lwp, but our own interrupt
 198  * stack and process the high level interrupt in-situ. These handlers are
 199  * designed to be very short in nature and cannot go to sleep, only block on a
 200  * spin lock. If the interrupt has a lot of work to do, it must generate a
 201  * low-priority software interrupt that will be processed later.
 202  *
 203  *   Low level hardware interrupts
 204  *
 205  * Low level hardware interrupts start off like their high-level cousins. The
 206  * current CPU contains a number of kernel threads (kthread_t) that can be used
 207  * to process low level interrupts. These are shared between both low level
 208  * hardware and software interrupts. Note that while we run with our
 209  * kthread_t, we borrow the pinned threads lwp_t until such a time as we hit a
 210  * synchronization object. If we hit one and need to sleep, then the scheduler
 211  * will instead create the rest of what we need.
 212  *
 213  *   Low level software interrupts
 214  *
 215  * Low level software interrupts are handled in a similar way as hardware
 216  * interrupts, but the notification vector is different. Each CPU has a bitmask
 217  * of pending software interrupts. We can notify a CPU to process software
 218  * interrupts through a specific trap vector as well as through several
 219  * checks that are performed throughout the code. These checks will look at
 220  * processing software interrupts as we lower our spl.
 221  *
 222  * We attempt to process the highest pending software interrupt that we can
 223  * which is greater than our current IPL. If none currently exist, then we move
 224  * on. We process a software interrupt in a similar fashion to a hardware
 225  * interrupt.
 226  *
 227  * Traditional Interrupt Flow
 228  * --------------------------
 229  *
 230  * The following diagram tracks the flow of the traditional uppc and pcplusmp
 231  * interrupt handlers. The apix driver has its own version of do_interrupt().
 232  * We come into the interrupt handler with all interrupts masked by the IF
 233  * flag. This is because we set up the handler using an interrupt-gate, which
 234  * is defined architecturally to have cleared the IF flag for us.
 235  *
 236  * +--------------+    +----------------+    +-----------+
 237  * | _interrupt() |--->| do_interrupt() |--->| *setlvl() |
 238  * +--------------+    +----------------+    +-----------+
 239  *                       |      |     |
 240  *                       |      |     |
 241  *              low-level|      |     | softint
 242  *                HW int |      |     +---------------------------------------+
 243  * +--------------+      |      |                                             |
 244  * | intr_thread_ |<-----+      | hi-level int                                |
 245  * | prolog()     |             |    +----------+                             |
 246  * +--------------+             +--->| hilevel_ |      Not on intr stack      |
 247  *       |                           | intr_    |-----------------+           |
 248  *       |                           | prolog() |                 |           |
 249  * +------------+                    +----------+                 |           |
 250  * | switch_sp_ |                        | On intr                v           |
 251  * | and_call() |                        | Stack          +------------+      |
 252  * +------------+                        |                | switch_sp_ |      |
 253  *       |                               v                | and_call() |      |
 254  *       v                             +-----------+      +------------+      |
 255  * +-----------+                       | dispatch_ |             |            |
 256  * | dispatch_ |   +-------------------| hilevel() |<------------+            |
 257  * | hardint() |   |                   +-----------+                          |
 258  * +-----------+   |                                                          |
 259  *       |         v                                                          |
 260  *       |     +-----+  +----------------------+  +-----+  hi-level           |
 261  *       +---->| sti |->| av_dispatch_autovect |->| cli |---------+           |
 262  *             +-----+  +----------------------+  +-----+         |           |
 263  *                                |                |              |           |
 264  *                                v                |              |           |
 265  *                         +----------+            |              |           |
 266  *                         | for each |            |              |           |
 267  *                         | handler  |            |              |           |
 268  *                         |  *intr() |            |              v           |
 269  * +--------------+        +----------+            |      +----------------+  |
 270  * | intr_thread_ |                      low-level |      | hilevel_intr_  |  |
 271  * | epilog()     |<-------------------------------+      | epilog()       |  |
 272  * +--------------+                                       +----------------+  |
 273  *   |       |                                                   |            |
 274  *   |       +----------------------v      v---------------------+            |
 275  *   |                           +------------+                               |
 276  *   |   +---------------------->| *setlvlx() |                               |
 277  *   |   |                       +------------+                               |
 278  *   |   |                              |                                     |
 279  *   |   |                              v                                     |
 280  *   |   |      +--------+     +------------------+      +-------------+      |
 281  *   |   |      | return |<----| softint pending? |----->| dosoftint() |<-----+
 282  *   |   |      +--------+  no +------------------+ yes  +-------------+
 283  *   |   |           ^                                      |     |
 284  *   |   |           |  softint pil too low                 |     |
 285  *   |   |           +--------------------------------------+     |
 286  *   |   |                                                        v
 287  *   |   |    +-----------+      +------------+          +-----------+
 288  *   |   |    | dispatch_ |<-----| switch_sp_ |<---------| *setspl() |
 289  *   |   |    | softint() |      | and_call() |          +-----------+
 290  *   |   |    +-----------+      +------------+
 291  *   |   |        |
 292  *   |   |        v
 293  *   |   |      +-----+  +----------------------+  +-----+  +------------+
 294  *   |   |      | sti |->| av_dispatch_autovect |->| cli |->| dosoftint_ |
 295  *   |   |      +-----+  +----------------------+  +-----+  | epilog()   |
 296  *   |   |                                                  +------------+
 297  *   |   |                                                    |     |
 298  *   |   +----------------------------------------------------+     |
 299  *   v                                                              |
 300  * +-----------+                                                    |
 301  * | interrupt |                                                    |
 302  * | thread    |<---------------------------------------------------+
 303  * | blocked   |
 304  * +-----------+
 305  *      |
 306  *      v
 307  *  +----------------+  +------------+  +-----------+  +-------+  +---------+
 308  *  | set_base_spl() |->| *setlvlx() |->| splhigh() |->| sti() |->| swtch() |
 309  *  +----------------+  +------------+  +-----------+  +-------+  +---------+
 310  *
 311  *    Calls made on Interrupt Stacks and Epilogue routines
 312  *
 313  * We use the switch_sp_and_call() assembly routine to switch our sp to the
 314  * interrupt stacks and then call the appropriate dispatch function. In the
 315  * case of interrupts which may block, softints and hardints, we always ensure
 316  * that we are still on the interrupt thread when we call the epilog routine.
 317  * This is not just important, it's necessary. If the interrupt thread blocked,
 318  * we won't return from our switch_sp_and_call() function and instead we'll go
 319  * through and set ourselves up to swtch() directly.
 320  *
 321  * New Interrupt Flow
 322  * ------------------
 323  *
 324  * The apix module has its own interrupt path. This is done for various
 325  * reasons. The first is that rather than having global interrupt vectors, we
 326  * now have per-cpu vectors.
 327  *
 328  * The other substantial change is that the apix design does not use the TPR to
 329  * mask interrupts below the current level. In fact, except for one special
 330  * case, it does not use the TPR at all. Instead, it only uses the IF flag
 331  * (cli/sti) to either block all interrupts or allow any interrupts to come in.
 332  * The design is such that when interrupts are allowed to come in, if we are
 333  * currently servicing a higher priority interupt, the new interrupt is treated
 334  * as pending and serviced later. Specifically, in the pcplusmp module's
 335  * apic_intr_enter() the code masks interrupts at or below the current
 336  * IPL using the TPR before sending EOI, whereas the apix module's
 337  * apix_intr_enter() simply sends EOI.
 338  *
 339  * The one special case where the apix code uses the TPR is when it calls
 340  * through the apic_reg_ops function pointer apic_write_task_reg in
 341  * apix_init_intr() to initially mask all levels and then finally to enable all
 342  * levels.
 343  *
 344  * Recall that we come into the interrupt handler with all interrupts masked
 345  * by the IF flag. This is because we set up the handler using an
 346  * interrupt-gate which is defined architecturally to have cleared the IF flag
 347  * for us.
 348  *
 349  * +--------------+    +---------------------+
 350  * | _interrupt() |--->| apix_do_interrupt() |
 351  * +--------------+    +---------------------+
 352  *                               |
 353  *                hard int? +----+--------+ softint?
 354  *                          |             | (but no low-level looping)
 355  *                   +-----------+        |
 356  *                   | *setlvl() |        |
 357  * +---------+       +-----------+        +----------------------------------+
 358  * |apix_add_|    check IPL |                                                |
 359  * |pending_ |<-------------+------+----------------------+                  |
 360  * |hardint()|        low-level int|          hi-level int|                  |
 361  * +---------+                     v                      v                  |
 362  *     | check IPL       +-----------------+     +---------------+           |
 363  *  +--+-----+           | apix_intr_      |     | apix_hilevel_ |           |
 364  *  |        |           | thread_prolog() |     | intr_prolog() |           |
 365  *  |      return        +-----------------+     +---------------+           |
 366  *  |                         |                    | On intr                 |
 367  *  |                   +------------+             | stack?  +------------+  |
 368  *  |                   | switch_sp_ |             +---------| switch_sp_ |  |
 369  *  |                   | and_call() |             |         | and_call() |  |
 370  *  |                   +------------+             |         +------------+  |
 371  *  |                         |                    |          |              |
 372  *  |                   +----------------+     +----------------+            |
 373  *  |                   | apix_dispatch_ |     | apix_dispatch_ |            |
 374  *  |                   | lowlevel()     |     | hilevel()      |            |
 375  *  |                   +----------------+     +----------------+            |
 376  *  |                                |             |                         |
 377  *  |                                v             v                         |
 378  *  |                       +-------------------------+                      |
 379  *  |                       |apix_dispatch_by_vector()|----+                 |
 380  *  |                       +-------------------------+    |                 |
 381  *  |               !XC_HI_PIL|         |         |        |                 |
 382  *  |                       +---+   +-------+   +---+      |                 |
 383  *  |                       |sti|   |*intr()|   |cli|      |                 |
 384  *  |                       +---+   +-------+   +---+      |  hi-level?      |
 385  *  |                          +---------------------------+----+            |
 386  *  |                          v                low-level?      v            |
 387  *  |                  +----------------+               +----------------+   |
 388  *  |                  | apix_intr_     |               | apix_hilevel_  |   |
 389  *  |                  | thread_epilog()|               | intr_epilog()  |   |
 390  *  |                  +----------------+               +----------------+   |
 391  *  |                          |                                |            |
 392  *  |        v-----------------+--------------------------------+            |
 393  *  |  +------------+                                                        |
 394  *  |  | *setlvlx() |   +----------------------------------------------------+
 395  *  |  +------------+   |
 396  *  |      |            |            +--------------------------------+ low
 397  *  v      v     v------+            v                                | level
 398  * +------------------+      +------------------+      +-----------+  | pending?
 399  * | apix_do_pending_ |----->| apix_do_pending_ |----->| apix_do_  |--+
 400  * | hilevel()        |      | hardint()        |      | softint() |  |
 401  * +------------------+      +------------------+      +-----------+    return
 402  *     |                       |                         |
 403  *     | while pending         | while pending           | while pending
 404  *     | hi-level              | low-level               | softint
 405  *     |                       |                         |
 406  *  +---------------+        +-----------------+       +-----------------+
 407  *  | apix_hilevel_ |        | apix_intr_      |       | apix_do_        |
 408  *  | intr_prolog() |        | thread_prolog() |       | softint_prolog()|
 409  *  +---------------+        +-----------------+       +-----------------+
 410  *     | On intr                       |                      |
 411  *     | stack? +------------+    +------------+        +------------+
 412  *     +--------| switch_sp_ |    | switch_sp_ |        | switch_sp_ |
 413  *     |        | and_call() |    | and_call() |        | and_call() |
 414  *     |        +------------+    +------------+        +------------+
 415  *     |           |                   |                      |
 416  *  +------------------+   +------------------+   +------------------------+
 417  *  | apix_dispatch_   |   | apix_dispatch_   |   | apix_dispatch_softint()|
 418  *  | pending_hilevel()|   | pending_hardint()|   +------------------------+
 419  *  +------------------+   +------------------+      |    |      |      |
 420  *    |         |           |         |              |    |      |      |
 421  *    | +----------------+  | +----------------+     |    |      |      |
 422  *    | | apix_hilevel_  |  | | apix_intr_     |     |    |      |      |
 423  *    | | intr_epilog()  |  | | thread_epilog()|     |    |      |      |
 424  *    | +----------------+  | +----------------+     |    |      |      |
 425  *    |         |           |       |                |    |      |      |
 426  *    |   +------------+    |  +----------+   +------+    |      |      |
 427  *    |   | *setlvlx() |    |  |*setlvlx()|   |           |      |      |
 428  *    |   +------------+    |  +----------+   |   +----------+   |   +---------+
 429  *    |                     |               +---+ |av_       | +---+ |apix_do_ |
 430  * +---------------------------------+      |sti| |dispatch_ | |cli| |softint_ |
 431  * | apix_dispatch_pending_autovect()|      +---+ |softvect()| +---+ |epilog() |
 432  * +---------------------------------+            +----------+       +---------+
 433  *  |!XC_HI_PIL  |       |         |                    |
 434  * +---+  +-------+    +---+  +----------+          +-------+
 435  * |sti|  |*intr()|    |cli|  |apix_post_|          |*intr()|
 436  * +---+  +-------+    +---+  |hardint() |          +-------+
 437  *                            +----------+
 438  */
 439 
 440 #include <sys/cpuvar.h>
 441 #include <sys/cpu_event.h>
 442 #include <sys/regset.h>
 443 #include <sys/psw.h>
 444 #include <sys/types.h>
 445 #include <sys/thread.h>
 446 #include <sys/systm.h>
 447 #include <sys/segments.h>
 448 #include <sys/pcb.h>
 449 #include <sys/trap.h>
 450 #include <sys/ftrace.h>
 451 #include <sys/traptrace.h>
 452 #include <sys/clock.h>
 453 #include <sys/panic.h>
 454 #include <sys/disp.h>
 455 #include <vm/seg_kp.h>
 456 #include <sys/stack.h>
 457 #include <sys/sysmacros.h>
 458 #include <sys/cmn_err.h>
 459 #include <sys/kstat.h>
 460 #include <sys/smp_impldefs.h>
 461 #include <sys/pool_pset.h>
 462 #include <sys/zone.h>
 463 #include <sys/bitmap.h>
 464 #include <sys/archsystm.h>
 465 #include <sys/machsystm.h>
 466 #include <sys/ontrap.h>
 467 #include <sys/x86_archext.h>
 468 #include <sys/promif.h>
 469 #include <vm/hat_i86.h>
 470 #if defined(__xpv)
 471 #include <sys/hypervisor.h>
 472 #endif
 473 
 474 
 475 #if defined(__xpv) && defined(DEBUG)
 476 
 477 /*
 478  * This panic message is intended as an aid to interrupt debugging.
 479  *
 480  * The associated assertion tests the condition of enabling
 481  * events when events are already enabled.  The implication
 482  * being that whatever code the programmer thought was
 483  * protected by having events disabled until the second
 484  * enable happened really wasn't protected at all ..
 485  */
 486 
 487 int stistipanic = 1;    /* controls the debug panic check */
 488 const char *stistimsg = "stisti";
 489 ulong_t laststi[NCPU];
 490 
 491 /*
 492  * This variable tracks the last place events were disabled on each cpu
 493  * it assists in debugging when asserts that interrupts are enabled trip.
 494  */
 495 ulong_t lastcli[NCPU];
 496 
 497 #endif
 498 
 499 void do_interrupt(struct regs *rp, trap_trace_rec_t *ttp);
 500 
 501 void (*do_interrupt_common)(struct regs *, trap_trace_rec_t *) = do_interrupt;
 502 uintptr_t (*get_intr_handler)(int, short) = NULL;
 503 
 504 /*
 505  * Set cpu's base SPL level to the highest active interrupt level
 506  */
 507 void
 508 set_base_spl(void)
 509 {
 510         struct cpu *cpu = CPU;
 511         uint16_t active = (uint16_t)cpu->cpu_intr_actv;
 512 
 513         cpu->cpu_base_spl = active == 0 ? 0 : bsrw_insn(active);
 514 }
 515 
 516 /*
 517  * Do all the work necessary to set up the cpu and thread structures
 518  * to dispatch a high-level interrupt.
 519  *
 520  * Returns 0 if we're -not- already on the high-level interrupt stack,
 521  * (and *must* switch to it), non-zero if we are already on that stack.
 522  *
 523  * Called with interrupts masked.
 524  * The 'pil' is already set to the appropriate level for rp->r_trapno.
 525  */
 526 static int
 527 hilevel_intr_prolog(struct cpu *cpu, uint_t pil, uint_t oldpil, struct regs *rp)
 528 {
 529         struct machcpu *mcpu = &cpu->cpu_m;
 530         uint_t mask;
 531         hrtime_t intrtime;
 532         hrtime_t now = tsc_read();
 533 
 534         ASSERT(pil > LOCK_LEVEL);
 535 
 536         if (pil == CBE_HIGH_PIL) {
 537                 cpu->cpu_profile_pil = oldpil;
 538                 if (USERMODE(rp->r_cs)) {
 539                         cpu->cpu_profile_pc = 0;
 540                         cpu->cpu_profile_upc = rp->r_pc;
 541                         cpu->cpu_cpcprofile_pc = 0;
 542                         cpu->cpu_cpcprofile_upc = rp->r_pc;
 543                 } else {
 544                         cpu->cpu_profile_pc = rp->r_pc;
 545                         cpu->cpu_profile_upc = 0;
 546                         cpu->cpu_cpcprofile_pc = rp->r_pc;
 547                         cpu->cpu_cpcprofile_upc = 0;
 548                 }
 549         }
 550 
 551         mask = cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK;
 552         if (mask != 0) {
 553                 int nestpil;
 554 
 555                 /*
 556                  * We have interrupted another high-level interrupt.
 557                  * Load starting timestamp, compute interval, update
 558                  * cumulative counter.
 559                  */
 560                 nestpil = bsrw_insn((uint16_t)mask);
 561                 ASSERT(nestpil < pil);
 562                 intrtime = now -
 563                     mcpu->pil_high_start[nestpil - (LOCK_LEVEL + 1)];
 564                 mcpu->intrstat[nestpil][0] += intrtime;
 565                 cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
 566                 /*
 567                  * Another high-level interrupt is active below this one, so
 568                  * there is no need to check for an interrupt thread.  That
 569                  * will be done by the lowest priority high-level interrupt
 570                  * active.
 571                  */
 572         } else {
 573                 kthread_t *t = cpu->cpu_thread;
 574 
 575                 /*
 576                  * See if we are interrupting a low-level interrupt thread.
 577                  * If so, account for its time slice only if its time stamp
 578                  * is non-zero.
 579                  */
 580                 if ((t->t_flag & T_INTR_THREAD) != 0 && t->t_intr_start != 0) {
 581                         intrtime = now - t->t_intr_start;
 582                         mcpu->intrstat[t->t_pil][0] += intrtime;
 583                         cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
 584                         t->t_intr_start = 0;
 585                 }
 586         }
 587 
 588         /*
 589          * Store starting timestamp in CPU structure for this PIL.
 590          */
 591         mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)] = now;
 592 
 593         ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0);
 594 
 595         if (pil == 15) {
 596                 /*
 597                  * To support reentrant level 15 interrupts, we maintain a
 598                  * recursion count in the top half of cpu_intr_actv.  Only
 599                  * when this count hits zero do we clear the PIL 15 bit from
 600                  * the lower half of cpu_intr_actv.
 601                  */
 602                 uint16_t *refcntp = (uint16_t *)&cpu->cpu_intr_actv + 1;
 603                 (*refcntp)++;
 604         }
 605 
 606         mask = cpu->cpu_intr_actv;
 607 
 608         cpu->cpu_intr_actv |= (1 << pil);
 609 
 610         return (mask & CPU_INTR_ACTV_HIGH_LEVEL_MASK);
 611 }
 612 
 613 /*
 614  * Does most of the work of returning from a high level interrupt.
 615  *
 616  * Returns 0 if there are no more high level interrupts (in which
 617  * case we must switch back to the interrupted thread stack) or
 618  * non-zero if there are more (in which case we should stay on it).
 619  *
 620  * Called with interrupts masked
 621  */
 622 static int
 623 hilevel_intr_epilog(struct cpu *cpu, uint_t pil, uint_t oldpil, uint_t vecnum)
 624 {
 625         struct machcpu *mcpu = &cpu->cpu_m;
 626         uint_t mask;
 627         hrtime_t intrtime;
 628         hrtime_t now = tsc_read();
 629 
 630         ASSERT(mcpu->mcpu_pri == pil);
 631 
 632         cpu->cpu_stats.sys.intr[pil - 1]++;
 633 
 634         ASSERT(cpu->cpu_intr_actv & (1 << pil));
 635 
 636         if (pil == 15) {
 637                 /*
 638                  * To support reentrant level 15 interrupts, we maintain a
 639                  * recursion count in the top half of cpu_intr_actv.  Only
 640                  * when this count hits zero do we clear the PIL 15 bit from
 641                  * the lower half of cpu_intr_actv.
 642                  */
 643                 uint16_t *refcntp = (uint16_t *)&cpu->cpu_intr_actv + 1;
 644 
 645                 ASSERT(*refcntp > 0);
 646 
 647                 if (--(*refcntp) == 0)
 648                         cpu->cpu_intr_actv &= ~(1 << pil);
 649         } else {
 650                 cpu->cpu_intr_actv &= ~(1 << pil);
 651         }
 652 
 653         ASSERT(mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)] != 0);
 654 
 655         intrtime = now - mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)];
 656         mcpu->intrstat[pil][0] += intrtime;
 657         cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
 658 
 659         /*
 660          * Check for lower-pil nested high-level interrupt beneath
 661          * current one.  If so, place a starting timestamp in its
 662          * pil_high_start entry.
 663          */
 664         mask = cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK;
 665         if (mask != 0) {
 666                 int nestpil;
 667 
 668                 /*
 669                  * find PIL of nested interrupt
 670                  */
 671                 nestpil = bsrw_insn((uint16_t)mask);
 672                 ASSERT(nestpil < pil);
 673                 mcpu->pil_high_start[nestpil - (LOCK_LEVEL + 1)] = now;
 674                 /*
 675                  * (Another high-level interrupt is active below this one,
 676                  * so there is no need to check for an interrupt
 677                  * thread.  That will be done by the lowest priority
 678                  * high-level interrupt active.)
 679                  */
 680         } else {
 681                 /*
 682                  * Check to see if there is a low-level interrupt active.
 683                  * If so, place a starting timestamp in the thread
 684                  * structure.
 685                  */
 686                 kthread_t *t = cpu->cpu_thread;
 687 
 688                 if (t->t_flag & T_INTR_THREAD)
 689                         t->t_intr_start = now;
 690         }
 691 
 692         mcpu->mcpu_pri = oldpil;
 693         (void) (*setlvlx)(oldpil, vecnum);
 694 
 695         return (cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK);
 696 }
 697 
 698 /*
 699  * Set up the cpu, thread and interrupt thread structures for
 700  * executing an interrupt thread.  The new stack pointer of the
 701  * interrupt thread (which *must* be switched to) is returned.
 702  */
 703 static caddr_t
 704 intr_thread_prolog(struct cpu *cpu, caddr_t stackptr, uint_t pil)
 705 {
 706         struct machcpu *mcpu = &cpu->cpu_m;
 707         kthread_t *t, *volatile it;
 708         hrtime_t now = tsc_read();
 709 
 710         ASSERT(pil > 0);
 711         ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0);
 712         cpu->cpu_intr_actv |= (1 << pil);
 713 
 714         /*
 715          * Get set to run an interrupt thread.
 716          * There should always be an interrupt thread, since we
 717          * allocate one for each level on each CPU.
 718          *
 719          * t_intr_start could be zero due to cpu_intr_swtch_enter.
 720          */
 721         t = cpu->cpu_thread;
 722         if ((t->t_flag & T_INTR_THREAD) && t->t_intr_start != 0) {
 723                 hrtime_t intrtime = now - t->t_intr_start;
 724                 mcpu->intrstat[t->t_pil][0] += intrtime;
 725                 cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
 726                 t->t_intr_start = 0;
 727         }
 728 
 729         ASSERT(SA((uintptr_t)stackptr) == (uintptr_t)stackptr);
 730 
 731         t->t_sp = (uintptr_t)stackptr;       /* mark stack in curthread for resume */
 732 
 733         /*
 734          * unlink the interrupt thread off the cpu
 735          *
 736          * Note that the code in kcpc_overflow_intr -relies- on the
 737          * ordering of events here - in particular that t->t_lwp of
 738          * the interrupt thread is set to the pinned thread *before*
 739          * curthread is changed.
 740          */
 741         it = cpu->cpu_intr_thread;
 742         cpu->cpu_intr_thread = it->t_link;
 743         it->t_intr = t;
 744         it->t_lwp = t->t_lwp;
 745 
 746         /*
 747          * (threads on the interrupt thread free list could have state
 748          * preset to TS_ONPROC, but it helps in debugging if
 749          * they're TS_FREE.)
 750          */
 751         it->t_state = TS_ONPROC;
 752 
 753         cpu->cpu_thread = it;                /* new curthread on this cpu */
 754         it->t_pil = (uchar_t)pil;
 755         it->t_pri = intr_pri + (pri_t)pil;
 756         it->t_intr_start = now;
 757 
 758         return (it->t_stk);
 759 }
 760 
 761 
 762 #ifdef DEBUG
 763 int intr_thread_cnt;
 764 #endif
 765 
 766 /*
 767  * Called with interrupts disabled
 768  */
 769 static void
 770 intr_thread_epilog(struct cpu *cpu, uint_t vec, uint_t oldpil)
 771 {
 772         struct machcpu *mcpu = &cpu->cpu_m;
 773         kthread_t *t;
 774         kthread_t *it = cpu->cpu_thread;     /* curthread */
 775         uint_t pil, basespl;
 776         hrtime_t intrtime;
 777         hrtime_t now = tsc_read();
 778 
 779         pil = it->t_pil;
 780         cpu->cpu_stats.sys.intr[pil - 1]++;
 781 
 782         ASSERT(it->t_intr_start != 0);
 783         intrtime = now - it->t_intr_start;
 784         mcpu->intrstat[pil][0] += intrtime;
 785         cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
 786 
 787         ASSERT(cpu->cpu_intr_actv & (1 << pil));
 788         cpu->cpu_intr_actv &= ~(1 << pil);
 789 
 790         /*
 791          * If there is still an interrupted thread underneath this one
 792          * then the interrupt was never blocked and the return is
 793          * fairly simple.  Otherwise it isn't.
 794          */
 795         if ((t = it->t_intr) == NULL) {
 796                 /*
 797                  * The interrupted thread is no longer pinned underneath
 798                  * the interrupt thread.  This means the interrupt must
 799                  * have blocked, and the interrupted thread has been
 800                  * unpinned, and has probably been running around the
 801                  * system for a while.
 802                  *
 803                  * Since there is no longer a thread under this one, put
 804                  * this interrupt thread back on the CPU's free list and
 805                  * resume the idle thread which will dispatch the next
 806                  * thread to run.
 807                  */
 808 #ifdef DEBUG
 809                 intr_thread_cnt++;
 810 #endif
 811                 cpu->cpu_stats.sys.intrblk++;
 812                 /*
 813                  * Set CPU's base SPL based on active interrupts bitmask
 814                  */
 815                 set_base_spl();
 816                 basespl = cpu->cpu_base_spl;
 817                 mcpu->mcpu_pri = basespl;
 818                 (*setlvlx)(basespl, vec);
 819                 (void) splhigh();
 820                 sti();
 821                 it->t_state = TS_FREE;
 822                 /*
 823                  * Return interrupt thread to pool
 824                  */
 825                 it->t_link = cpu->cpu_intr_thread;
 826                 cpu->cpu_intr_thread = it;
 827                 swtch();
 828                 panic("intr_thread_epilog: swtch returned");
 829                 /*NOTREACHED*/
 830         }
 831 
 832         /*
 833          * Return interrupt thread to the pool
 834          */
 835         it->t_link = cpu->cpu_intr_thread;
 836         cpu->cpu_intr_thread = it;
 837         it->t_state = TS_FREE;
 838 
 839         basespl = cpu->cpu_base_spl;
 840         pil = MAX(oldpil, basespl);
 841         mcpu->mcpu_pri = pil;
 842         (*setlvlx)(pil, vec);
 843         t->t_intr_start = now;
 844         cpu->cpu_thread = t;
 845 }
 846 
 847 /*
 848  * intr_get_time() is a resource for interrupt handlers to determine how
 849  * much time has been spent handling the current interrupt. Such a function
 850  * is needed because higher level interrupts can arrive during the
 851  * processing of an interrupt.  intr_get_time() only returns time spent in the
 852  * current interrupt handler.
 853  *
 854  * The caller must be calling from an interrupt handler running at a pil
 855  * below or at lock level. Timings are not provided for high-level
 856  * interrupts.
 857  *
 858  * The first time intr_get_time() is called while handling an interrupt,
 859  * it returns the time since the interrupt handler was invoked. Subsequent
 860  * calls will return the time since the prior call to intr_get_time(). Time
 861  * is returned as ticks. Use scalehrtimef() to convert ticks to nsec.
 862  *
 863  * Theory Of Intrstat[][]:
 864  *
 865  * uint64_t intrstat[pil][0..1] is an array indexed by pil level, with two
 866  * uint64_ts per pil.
 867  *
 868  * intrstat[pil][0] is a cumulative count of the number of ticks spent
 869  * handling all interrupts at the specified pil on this CPU. It is
 870  * exported via kstats to the user.
 871  *
 872  * intrstat[pil][1] is always a count of ticks less than or equal to the
 873  * value in [0]. The difference between [1] and [0] is the value returned
 874  * by a call to intr_get_time(). At the start of interrupt processing,
 875  * [0] and [1] will be equal (or nearly so). As the interrupt consumes
 876  * time, [0] will increase, but [1] will remain the same. A call to
 877  * intr_get_time() will return the difference, then update [1] to be the
 878  * same as [0]. Future calls will return the time since the last call.
 879  * Finally, when the interrupt completes, [1] is updated to the same as [0].
 880  *
 881  * Implementation:
 882  *
 883  * intr_get_time() works much like a higher level interrupt arriving. It
 884  * "checkpoints" the timing information by incrementing intrstat[pil][0]
 885  * to include elapsed running time, and by setting t_intr_start to rdtsc.
 886  * It then sets the return value to intrstat[pil][0] - intrstat[pil][1],
 887  * and updates intrstat[pil][1] to be the same as the new value of
 888  * intrstat[pil][0].
 889  *
 890  * In the normal handling of interrupts, after an interrupt handler returns
 891  * and the code in intr_thread() updates intrstat[pil][0], it then sets
 892  * intrstat[pil][1] to the new value of intrstat[pil][0]. When [0] == [1],
 893  * the timings are reset, i.e. intr_get_time() will return [0] - [1] which
 894  * is 0.
 895  *
 896  * Whenever interrupts arrive on a CPU which is handling a lower pil
 897  * interrupt, they update the lower pil's [0] to show time spent in the
 898  * handler that they've interrupted. This results in a growing discrepancy
 899  * between [0] and [1], which is returned the next time intr_get_time() is
 900  * called. Time spent in the higher-pil interrupt will not be returned in
 901  * the next intr_get_time() call from the original interrupt, because
 902  * the higher-pil interrupt's time is accumulated in intrstat[higherpil][].
 903  */
 904 uint64_t
 905 intr_get_time(void)
 906 {
 907         struct cpu *cpu;
 908         struct machcpu *mcpu;
 909         kthread_t *t;
 910         uint64_t time, delta, ret;
 911         uint_t pil;
 912 
 913         cli();
 914         cpu = CPU;
 915         mcpu = &cpu->cpu_m;
 916         t = cpu->cpu_thread;
 917         pil = t->t_pil;
 918         ASSERT((cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK) == 0);
 919         ASSERT(t->t_flag & T_INTR_THREAD);
 920         ASSERT(pil != 0);
 921         ASSERT(t->t_intr_start != 0);
 922 
 923         time = tsc_read();
 924         delta = time - t->t_intr_start;
 925         t->t_intr_start = time;
 926 
 927         time = mcpu->intrstat[pil][0] + delta;
 928         ret = time - mcpu->intrstat[pil][1];
 929         mcpu->intrstat[pil][0] = time;
 930         mcpu->intrstat[pil][1] = time;
 931         cpu->cpu_intracct[cpu->cpu_mstate] += delta;
 932 
 933         sti();
 934         return (ret);
 935 }
 936 
 937 static caddr_t
 938 dosoftint_prolog(
 939         struct cpu *cpu,
 940         caddr_t stackptr,
 941         uint32_t st_pending,
 942         uint_t oldpil)
 943 {
 944         kthread_t *t, *volatile it;
 945         struct machcpu *mcpu = &cpu->cpu_m;
 946         uint_t pil;
 947         hrtime_t now;
 948 
 949 top:
 950         ASSERT(st_pending == mcpu->mcpu_softinfo.st_pending);
 951 
 952         pil = bsrw_insn((uint16_t)st_pending);
 953         if (pil <= oldpil || pil <= cpu->cpu_base_spl)
 954                 return (0);
 955 
 956         /*
 957          * XX64 Sigh.
 958          *
 959          * This is a transliteration of the i386 assembler code for
 960          * soft interrupts.  One question is "why does this need
 961          * to be atomic?"  One possible race is -other- processors
 962          * posting soft interrupts to us in set_pending() i.e. the
 963          * CPU might get preempted just after the address computation,
 964          * but just before the atomic transaction, so another CPU would
 965          * actually set the original CPU's st_pending bit.  However,
 966          * it looks like it would be simpler to disable preemption there.
 967          * Are there other races for which preemption control doesn't work?
 968          *
 969          * The i386 assembler version -also- checks to see if the bit
 970          * being cleared was actually set; if it wasn't, it rechecks
 971          * for more.  This seems a bit strange, as the only code that
 972          * ever clears the bit is -this- code running with interrupts
 973          * disabled on -this- CPU.  This code would probably be cheaper:
 974          *
 975          * atomic_and_32((uint32_t *)&mcpu->mcpu_softinfo.st_pending,
 976          *   ~(1 << pil));
 977          *
 978          * and t->t_preempt--/++ around set_pending() even cheaper,
 979          * but at this point, correctness is critical, so we slavishly
 980          * emulate the i386 port.
 981          */
 982         if (atomic_btr32((uint32_t *)
 983             &mcpu->mcpu_softinfo.st_pending, pil) == 0) {
 984                 st_pending = mcpu->mcpu_softinfo.st_pending;
 985                 goto top;
 986         }
 987 
 988         mcpu->mcpu_pri = pil;
 989         (*setspl)(pil);
 990 
 991         now = tsc_read();
 992 
 993         /*
 994          * Get set to run interrupt thread.
 995          * There should always be an interrupt thread since we
 996          * allocate one for each level on the CPU.
 997          */
 998         it = cpu->cpu_intr_thread;
 999         cpu->cpu_intr_thread = it->t_link;
1000 
1001         /* t_intr_start could be zero due to cpu_intr_swtch_enter. */
1002         t = cpu->cpu_thread;
1003         if ((t->t_flag & T_INTR_THREAD) && t->t_intr_start != 0) {
1004                 hrtime_t intrtime = now - t->t_intr_start;
1005                 mcpu->intrstat[pil][0] += intrtime;
1006                 cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
1007                 t->t_intr_start = 0;
1008         }
1009 
1010         /*
1011          * Note that the code in kcpc_overflow_intr -relies- on the
1012          * ordering of events here - in particular that t->t_lwp of
1013          * the interrupt thread is set to the pinned thread *before*
1014          * curthread is changed.
1015          */
1016         it->t_lwp = t->t_lwp;
1017         it->t_state = TS_ONPROC;
1018 
1019         /*
1020          * Push interrupted thread onto list from new thread.
1021          * Set the new thread as the current one.
1022          * Set interrupted thread's T_SP because if it is the idle thread,
1023          * resume() may use that stack between threads.
1024          */
1025 
1026         ASSERT(SA((uintptr_t)stackptr) == (uintptr_t)stackptr);
1027         t->t_sp = (uintptr_t)stackptr;
1028 
1029         it->t_intr = t;
1030         cpu->cpu_thread = it;
1031 
1032         /*
1033          * Set bit for this pil in CPU's interrupt active bitmask.
1034          */
1035         ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0);
1036         cpu->cpu_intr_actv |= (1 << pil);
1037 
1038         /*
1039          * Initialize thread priority level from intr_pri
1040          */
1041         it->t_pil = (uchar_t)pil;
1042         it->t_pri = (pri_t)pil + intr_pri;
1043         it->t_intr_start = now;
1044 
1045         return (it->t_stk);
1046 }
1047 
1048 static void
1049 dosoftint_epilog(struct cpu *cpu, uint_t oldpil)
1050 {
1051         struct machcpu *mcpu = &cpu->cpu_m;
1052         kthread_t *t, *it;
1053         uint_t pil, basespl;
1054         hrtime_t intrtime;
1055         hrtime_t now = tsc_read();
1056 
1057         it = cpu->cpu_thread;
1058         pil = it->t_pil;
1059 
1060         cpu->cpu_stats.sys.intr[pil - 1]++;
1061 
1062         ASSERT(cpu->cpu_intr_actv & (1 << pil));
1063         cpu->cpu_intr_actv &= ~(1 << pil);
1064         intrtime = now - it->t_intr_start;
1065         mcpu->intrstat[pil][0] += intrtime;
1066         cpu->cpu_intracct[cpu->cpu_mstate] += intrtime;
1067 
1068         /*
1069          * If there is still an interrupted thread underneath this one
1070          * then the interrupt was never blocked and the return is
1071          * fairly simple.  Otherwise it isn't.
1072          */
1073         if ((t = it->t_intr) == NULL) {
1074                 /*
1075                  * Put thread back on the interrupt thread list.
1076                  * This was an interrupt thread, so set CPU's base SPL.
1077                  */
1078                 set_base_spl();
1079                 it->t_state = TS_FREE;
1080                 it->t_link = cpu->cpu_intr_thread;
1081                 cpu->cpu_intr_thread = it;
1082                 (void) splhigh();
1083                 sti();
1084                 swtch();
1085                 /*NOTREACHED*/
1086                 panic("dosoftint_epilog: swtch returned");
1087         }
1088         it->t_link = cpu->cpu_intr_thread;
1089         cpu->cpu_intr_thread = it;
1090         it->t_state = TS_FREE;
1091         cpu->cpu_thread = t;
1092         if (t->t_flag & T_INTR_THREAD)
1093                 t->t_intr_start = now;
1094         basespl = cpu->cpu_base_spl;
1095         pil = MAX(oldpil, basespl);
1096         mcpu->mcpu_pri = pil;
1097         (*setspl)(pil);
1098 }
1099 
1100 
1101 /*
1102  * Make the interrupted thread 'to' be runnable.
1103  *
1104  * Since t->t_sp has already been saved, t->t_pc is all
1105  * that needs to be set in this function.
1106  *
1107  * Returns the interrupt level of the interrupt thread.
1108  */
1109 int
1110 intr_passivate(
1111         kthread_t *it,          /* interrupt thread */
1112         kthread_t *t)           /* interrupted thread */
1113 {
1114         extern void _sys_rtt();
1115 
1116         ASSERT(it->t_flag & T_INTR_THREAD);
1117         ASSERT(SA(t->t_sp) == t->t_sp);
1118 
1119         t->t_pc = (uintptr_t)_sys_rtt;
1120         return (it->t_pil);
1121 }
1122 
1123 /*
1124  * Create interrupt kstats for this CPU.
1125  */
1126 void
1127 cpu_create_intrstat(cpu_t *cp)
1128 {
1129         int             i;
1130         kstat_t         *intr_ksp;
1131         kstat_named_t   *knp;
1132         char            name[KSTAT_STRLEN];
1133         zoneid_t        zoneid;
1134 
1135         ASSERT(MUTEX_HELD(&cpu_lock));
1136 
1137         if (pool_pset_enabled())
1138                 zoneid = GLOBAL_ZONEID;
1139         else
1140                 zoneid = ALL_ZONES;
1141 
1142         intr_ksp = kstat_create_zone("cpu", cp->cpu_id, "intrstat", "misc",
1143             KSTAT_TYPE_NAMED, PIL_MAX * 2, NULL, zoneid);
1144 
1145         /*
1146          * Initialize each PIL's named kstat
1147          */
1148         if (intr_ksp != NULL) {
1149                 intr_ksp->ks_update = cpu_kstat_intrstat_update;
1150                 knp = (kstat_named_t *)intr_ksp->ks_data;
1151                 intr_ksp->ks_private = cp;
1152                 for (i = 0; i < PIL_MAX; i++) {
1153                         (void) snprintf(name, KSTAT_STRLEN, "level-%d-time",
1154                             i + 1);
1155                         kstat_named_init(&knp[i * 2], name, KSTAT_DATA_UINT64);
1156                         (void) snprintf(name, KSTAT_STRLEN, "level-%d-count",
1157                             i + 1);
1158                         kstat_named_init(&knp[(i * 2) + 1], name,
1159                             KSTAT_DATA_UINT64);
1160                 }
1161                 kstat_install(intr_ksp);
1162         }
1163 }
1164 
1165 /*
1166  * Delete interrupt kstats for this CPU.
1167  */
1168 void
1169 cpu_delete_intrstat(cpu_t *cp)
1170 {
1171         kstat_delete_byname_zone("cpu", cp->cpu_id, "intrstat", ALL_ZONES);
1172 }
1173 
1174 /*
1175  * Convert interrupt statistics from CPU ticks to nanoseconds and
1176  * update kstat.
1177  */
1178 int
1179 cpu_kstat_intrstat_update(kstat_t *ksp, int rw)
1180 {
1181         kstat_named_t   *knp = ksp->ks_data;
1182         cpu_t           *cpup = (cpu_t *)ksp->ks_private;
1183         int             i;
1184         hrtime_t        hrt;
1185 
1186         if (rw == KSTAT_WRITE)
1187                 return (EACCES);
1188 
1189         for (i = 0; i < PIL_MAX; i++) {
1190                 hrt = (hrtime_t)cpup->cpu_m.intrstat[i + 1][0];
1191                 scalehrtimef(&hrt);
1192                 knp[i * 2].value.ui64 = (uint64_t)hrt;
1193                 knp[(i * 2) + 1].value.ui64 = cpup->cpu_stats.sys.intr[i];
1194         }
1195 
1196         return (0);
1197 }
1198 
1199 /*
1200  * An interrupt thread is ending a time slice, so compute the interval it
1201  * ran for and update the statistic for its PIL.
1202  */
1203 void
1204 cpu_intr_swtch_enter(kthread_id_t t)
1205 {
1206         uint64_t        interval;
1207         uint64_t        start;
1208         cpu_t           *cpu;
1209 
1210         ASSERT((t->t_flag & T_INTR_THREAD) != 0);
1211         ASSERT(t->t_pil > 0 && t->t_pil <= LOCK_LEVEL);
1212 
1213         /*
1214          * We could be here with a zero timestamp. This could happen if:
1215          * an interrupt thread which no longer has a pinned thread underneath
1216          * it (i.e. it blocked at some point in its past) has finished running
1217          * its handler. intr_thread() updated the interrupt statistic for its
1218          * PIL and zeroed its timestamp. Since there was no pinned thread to
1219          * return to, swtch() gets called and we end up here.
1220          *
1221          * Note that we use atomic ops below (atomic_cas_64 and
1222          * atomic_add_64), which we don't use in the functions above,
1223          * because we're not called with interrupts blocked, but the
1224          * epilog/prolog functions are.
1225          */
1226         if (t->t_intr_start) {
1227                 do {
1228                         start = t->t_intr_start;
1229                         interval = tsc_read() - start;
1230                 } while (atomic_cas_64(&t->t_intr_start, start, 0) != start);
1231                 cpu = CPU;
1232                 cpu->cpu_m.intrstat[t->t_pil][0] += interval;
1233 
1234                 atomic_add_64((uint64_t *)&cpu->cpu_intracct[cpu->cpu_mstate],
1235                     interval);
1236         } else
1237                 ASSERT(t->t_intr == NULL);
1238 }
1239 
1240 /*
1241  * An interrupt thread is returning from swtch(). Place a starting timestamp
1242  * in its thread structure.
1243  */
1244 void
1245 cpu_intr_swtch_exit(kthread_id_t t)
1246 {
1247         uint64_t ts;
1248 
1249         ASSERT((t->t_flag & T_INTR_THREAD) != 0);
1250         ASSERT(t->t_pil > 0 && t->t_pil <= LOCK_LEVEL);
1251 
1252         do {
1253                 ts = t->t_intr_start;
1254         } while (atomic_cas_64(&t->t_intr_start, ts, tsc_read()) != ts);
1255 }
1256 
1257 /*
1258  * Dispatch a hilevel interrupt (one above LOCK_LEVEL)
1259  */
1260 /*ARGSUSED*/
1261 static void
1262 dispatch_hilevel(uint_t vector, uint_t arg2)
1263 {
1264         sti();
1265         av_dispatch_autovect(vector);
1266         cli();
1267 }
1268 
1269 /*
1270  * Dispatch a soft interrupt
1271  */
1272 /*ARGSUSED*/
1273 static void
1274 dispatch_softint(uint_t oldpil, uint_t arg2)
1275 {
1276         struct cpu *cpu = CPU;
1277 
1278         sti();
1279         av_dispatch_softvect((int)cpu->cpu_thread->t_pil);
1280         cli();
1281 
1282         /*
1283          * Must run softint_epilog() on the interrupt thread stack, since
1284          * there may not be a return from it if the interrupt thread blocked.
1285          */
1286         dosoftint_epilog(cpu, oldpil);
1287 }
1288 
1289 /*
1290  * Dispatch a normal interrupt
1291  */
1292 static void
1293 dispatch_hardint(uint_t vector, uint_t oldipl)
1294 {
1295         struct cpu *cpu = CPU;
1296 
1297         sti();
1298         av_dispatch_autovect(vector);
1299         cli();
1300 
1301         /*
1302          * Must run intr_thread_epilog() on the interrupt thread stack, since
1303          * there may not be a return from it if the interrupt thread blocked.
1304          */
1305         intr_thread_epilog(cpu, vector, oldipl);
1306 }
1307 
1308 /*
1309  * Deliver any softints the current interrupt priority allows.
1310  * Called with interrupts disabled.
1311  */
1312 void
1313 dosoftint(struct regs *regs)
1314 {
1315         struct cpu *cpu = CPU;
1316         int oldipl;
1317         caddr_t newsp;
1318 
1319         while (cpu->cpu_softinfo.st_pending) {
1320                 oldipl = cpu->cpu_pri;
1321                 newsp = dosoftint_prolog(cpu, (caddr_t)regs,
1322                     cpu->cpu_softinfo.st_pending, oldipl);
1323                 /*
1324                  * If returned stack pointer is NULL, priority is too high
1325                  * to run any of the pending softints now.
1326                  * Break out and they will be run later.
1327                  */
1328                 if (newsp == NULL)
1329                         break;
1330                 switch_sp_and_call(newsp, dispatch_softint, oldipl, 0);
1331         }
1332 }
1333 
1334 /*
1335  * Interrupt service routine, called with interrupts disabled.
1336  */
1337 /*ARGSUSED*/
1338 void
1339 do_interrupt(struct regs *rp, trap_trace_rec_t *ttp)
1340 {
1341         struct cpu *cpu = CPU;
1342         int newipl, oldipl = cpu->cpu_pri;
1343         uint_t vector;
1344         caddr_t newsp;
1345 
1346 #ifdef TRAPTRACE
1347         ttp->ttr_marker = TT_INTERRUPT;
1348         ttp->ttr_ipl = 0xff;
1349         ttp->ttr_pri = oldipl;
1350         ttp->ttr_spl = cpu->cpu_base_spl;
1351         ttp->ttr_vector = 0xff;
1352 #endif  /* TRAPTRACE */
1353 
1354         cpu_idle_exit(CPU_IDLE_CB_FLAG_INTR);
1355 
1356         ++*(uint16_t *)&cpu->cpu_m.mcpu_istamp;
1357 
1358         /*
1359          * If it's a softint go do it now.
1360          */
1361         if (rp->r_trapno == T_SOFTINT) {
1362                 dosoftint(rp);
1363                 ASSERT(!interrupts_enabled());
1364                 return;
1365         }
1366 
1367         /*
1368          * Raise the interrupt priority.
1369          */
1370         newipl = (*setlvl)(oldipl, (int *)&rp->r_trapno);
1371 #ifdef TRAPTRACE
1372         ttp->ttr_ipl = newipl;
1373 #endif  /* TRAPTRACE */
1374 
1375         /*
1376          * Bail if it is a spurious interrupt
1377          */
1378         if (newipl == -1)
1379                 return;
1380         cpu->cpu_pri = newipl;
1381         vector = rp->r_trapno;
1382 #ifdef TRAPTRACE
1383         ttp->ttr_vector = vector;
1384 #endif  /* TRAPTRACE */
1385         if (newipl > LOCK_LEVEL) {
1386                 /*
1387                  * High priority interrupts run on this cpu's interrupt stack.
1388                  */
1389                 if (hilevel_intr_prolog(cpu, newipl, oldipl, rp) == 0) {
1390                         newsp = cpu->cpu_intr_stack;
1391                         switch_sp_and_call(newsp, dispatch_hilevel, vector, 0);
1392                 } else { /* already on the interrupt stack */
1393                         dispatch_hilevel(vector, 0);
1394                 }
1395                 (void) hilevel_intr_epilog(cpu, newipl, oldipl, vector);
1396         } else {
1397                 /*
1398                  * Run this interrupt in a separate thread.
1399                  */
1400                 newsp = intr_thread_prolog(cpu, (caddr_t)rp, newipl);
1401                 switch_sp_and_call(newsp, dispatch_hardint, vector, oldipl);
1402         }
1403 
1404 #if !defined(__xpv)
1405         /*
1406          * Deliver any pending soft interrupts.
1407          */
1408         if (cpu->cpu_softinfo.st_pending)
1409                 dosoftint(rp);
1410 #endif  /* !__xpv */
1411 }
1412 
1413 
1414 /*
1415  * Common tasks always done by _sys_rtt, called with interrupts disabled.
1416  * Returns 1 if returning to userland, 0 if returning to system mode.
1417  */
1418 int
1419 sys_rtt_common(struct regs *rp)
1420 {
1421         kthread_t *tp;
1422         extern void mutex_exit_critical_start();
1423         extern long mutex_exit_critical_size;
1424         extern void mutex_owner_running_critical_start();
1425         extern long mutex_owner_running_critical_size;
1426 
1427 loop:
1428 
1429         /*
1430          * Check if returning to user
1431          */
1432         tp = CPU->cpu_thread;
1433         if (USERMODE(rp->r_cs)) {
1434                 /*
1435                  * Check if AST pending.
1436                  */
1437                 if (tp->t_astflag) {
1438                         /*
1439                          * Let trap() handle the AST
1440                          */
1441                         sti();
1442                         rp->r_trapno = T_AST;
1443                         trap(rp, (caddr_t)0, CPU->cpu_id);
1444                         cli();
1445                         goto loop;
1446                 }
1447 
1448 #if defined(__amd64)
1449                 /*
1450                  * We are done if segment registers do not need updating.
1451                  */
1452                 if (tp->t_lwp->lwp_pcb.pcb_rupdate == 0)
1453                         return (1);
1454 
1455                 if (update_sregs(rp, tp->t_lwp)) {
1456                         /*
1457                          * 1 or more of the selectors is bad.
1458                          * Deliver a SIGSEGV.
1459                          */
1460                         proc_t *p = ttoproc(tp);
1461 
1462                         sti();
1463                         mutex_enter(&p->p_lock);
1464                         tp->t_lwp->lwp_cursig = SIGSEGV;
1465                         mutex_exit(&p->p_lock);
1466                         psig();
1467                         tp->t_sig_check = 1;
1468                         cli();
1469                 }
1470                 tp->t_lwp->lwp_pcb.pcb_rupdate = 0;
1471 
1472 #endif  /* __amd64 */
1473                 return (1);
1474         }
1475 
1476         /*
1477          * Here if we are returning to supervisor mode.
1478          * Check for a kernel preemption request.
1479          */
1480         if (CPU->cpu_kprunrun && (rp->r_ps & PS_IE)) {
1481 
1482                 /*
1483                  * Do nothing if already in kpreempt
1484                  */
1485                 if (!tp->t_preempt_lk) {
1486                         tp->t_preempt_lk = 1;
1487                         sti();
1488                         kpreempt(1); /* asynchronous kpreempt call */
1489                         cli();
1490                         tp->t_preempt_lk = 0;
1491                 }
1492         }
1493 
1494         /*
1495          * If we interrupted the mutex_exit() critical region we must
1496          * reset the PC back to the beginning to prevent missed wakeups
1497          * See the comments in mutex_exit() for details.
1498          */
1499         if ((uintptr_t)rp->r_pc - (uintptr_t)mutex_exit_critical_start <
1500             mutex_exit_critical_size) {
1501                 rp->r_pc = (greg_t)mutex_exit_critical_start;
1502         }
1503 
1504         /*
1505          * If we interrupted the mutex_owner_running() critical region we
1506          * must reset the PC back to the beginning to prevent dereferencing
1507          * of a freed thread pointer. See the comments in mutex_owner_running
1508          * for details.
1509          */
1510         if ((uintptr_t)rp->r_pc -
1511             (uintptr_t)mutex_owner_running_critical_start <
1512             mutex_owner_running_critical_size) {
1513                 rp->r_pc = (greg_t)mutex_owner_running_critical_start;
1514         }
1515 
1516         return (0);
1517 }
1518 
1519 void
1520 send_dirint(int cpuid, int int_level)
1521 {
1522         (*send_dirintf)(cpuid, int_level);
1523 }
1524 
1525 #define IS_FAKE_SOFTINT(flag, newpri)           \
1526         (((flag) & PS_IE) &&                                \
1527             (((*get_pending_spl)() > (newpri)) ||    \
1528             bsrw_insn((uint16_t)cpu->cpu_softinfo.st_pending) > (newpri)))
1529 
1530 /*
1531  * do_splx routine, takes new ipl to set
1532  * returns the old ipl.
1533  * We are careful not to set priority lower than CPU->cpu_base_pri,
1534  * even though it seems we're raising the priority, it could be set
1535  * higher at any time by an interrupt routine, so we must block interrupts
1536  * and look at CPU->cpu_base_pri
1537  */
1538 int
1539 do_splx(int newpri)
1540 {
1541         ulong_t flag;
1542         cpu_t   *cpu;
1543         int     curpri, basepri;
1544 
1545         flag = intr_clear();
1546         cpu = CPU; /* ints are disabled, now safe to cache cpu ptr */
1547         curpri = cpu->cpu_m.mcpu_pri;
1548         basepri = cpu->cpu_base_spl;
1549         if (newpri < basepri)
1550                 newpri = basepri;
1551         cpu->cpu_m.mcpu_pri = newpri;
1552         (*setspl)(newpri);
1553         /*
1554          * If we are going to reenable interrupts see if new priority level
1555          * allows pending softint delivery.
1556          */
1557         if (IS_FAKE_SOFTINT(flag, newpri))
1558                 fakesoftint();
1559         ASSERT(!interrupts_enabled());
1560         intr_restore(flag);
1561         return (curpri);
1562 }
1563 
1564 /*
1565  * Common spl raise routine, takes new ipl to set
1566  * returns the old ipl, will not lower ipl.
1567  */
1568 int
1569 splr(int newpri)
1570 {
1571         ulong_t flag;
1572         cpu_t   *cpu;
1573         int     curpri, basepri;
1574 
1575         flag = intr_clear();
1576         cpu = CPU; /* ints are disabled, now safe to cache cpu ptr */
1577         curpri = cpu->cpu_m.mcpu_pri;
1578         /*
1579          * Only do something if new priority is larger
1580          */
1581         if (newpri > curpri) {
1582                 basepri = cpu->cpu_base_spl;
1583                 if (newpri < basepri)
1584                         newpri = basepri;
1585                 cpu->cpu_m.mcpu_pri = newpri;
1586                 (*setspl)(newpri);
1587                 /*
1588                  * See if new priority level allows pending softint delivery
1589                  */
1590                 if (IS_FAKE_SOFTINT(flag, newpri))
1591                         fakesoftint();
1592         }
1593         intr_restore(flag);
1594         return (curpri);
1595 }
1596 
1597 int
1598 getpil(void)
1599 {
1600         return (CPU->cpu_m.mcpu_pri);
1601 }
1602 
1603 int
1604 spl_xcall(void)
1605 {
1606         return (splr(ipltospl(XCALL_PIL)));
1607 }
1608 
1609 int
1610 interrupts_enabled(void)
1611 {
1612         ulong_t flag;
1613 
1614         flag = getflags();
1615         return ((flag & PS_IE) == PS_IE);
1616 }
1617 
1618 #ifdef DEBUG
1619 void
1620 assert_ints_enabled(void)
1621 {
1622         ASSERT(!interrupts_unleashed || interrupts_enabled());
1623 }
1624 #endif  /* DEBUG */