1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
  24  */
  25 /*
  26  * Copyright (c) 2010, Intel Corporation.
  27  * All rights reserved.
  28  */
  29 /*
  30  * Copyright (c) 2013, Joyent, Inc.  All rights reserved.
  31  */
  32 
  33 /*
  34  * To understand how the apix module interacts with the interrupt subsystem read
  35  * the theory statement in uts/i86pc/os/intr.c.
  36  */
  37 
  38 /*
  39  * PSMI 1.1 extensions are supported only in 2.6 and later versions.
  40  * PSMI 1.2 extensions are supported only in 2.7 and later versions.
  41  * PSMI 1.3 and 1.4 extensions are supported in Solaris 10.
  42  * PSMI 1.5 extensions are supported in Solaris Nevada.
  43  * PSMI 1.6 extensions are supported in Solaris Nevada.
  44  * PSMI 1.7 extensions are supported in Solaris Nevada.
  45  */
  46 #define PSMI_1_7
  47 
  48 #include <sys/processor.h>
  49 #include <sys/time.h>
  50 #include <sys/psm.h>
  51 #include <sys/smp_impldefs.h>
  52 #include <sys/cram.h>
  53 #include <sys/acpi/acpi.h>
  54 #include <sys/acpica.h>
  55 #include <sys/psm_common.h>
  56 #include <sys/pit.h>
  57 #include <sys/ddi.h>
  58 #include <sys/sunddi.h>
  59 #include <sys/ddi_impldefs.h>
  60 #include <sys/pci.h>
  61 #include <sys/promif.h>
  62 #include <sys/x86_archext.h>
  63 #include <sys/cpc_impl.h>
  64 #include <sys/uadmin.h>
  65 #include <sys/panic.h>
  66 #include <sys/debug.h>
  67 #include <sys/archsystm.h>
  68 #include <sys/trap.h>
  69 #include <sys/machsystm.h>
  70 #include <sys/sysmacros.h>
  71 #include <sys/cpuvar.h>
  72 #include <sys/rm_platter.h>
  73 #include <sys/privregs.h>
  74 #include <sys/note.h>
  75 #include <sys/pci_intr_lib.h>
  76 #include <sys/spl.h>
  77 #include <sys/clock.h>
  78 #include <sys/cyclic.h>
  79 #include <sys/dditypes.h>
  80 #include <sys/sunddi.h>
  81 #include <sys/x_call.h>
  82 #include <sys/reboot.h>
  83 #include <sys/mach_intr.h>
  84 #include <sys/apix.h>
  85 #include <sys/apix_irm_impl.h>
  86 
  87 static int apix_probe();
  88 static void apix_init();
  89 static void apix_picinit(void);
  90 static int apix_intr_enter(int, int *);
  91 static void apix_intr_exit(int, int);
  92 static void apix_setspl(int);
  93 static int apix_disable_intr(processorid_t);
  94 static void apix_enable_intr(processorid_t);
  95 static int apix_get_clkvect(int);
  96 static int apix_get_ipivect(int, int);
  97 static void apix_post_cyclic_setup(void *);
  98 static int apix_post_cpu_start();
  99 static int apix_intr_ops(dev_info_t *, ddi_intr_handle_impl_t *,
 100     psm_intr_op_t, int *);
 101 
 102 /*
 103  * Helper functions for apix_intr_ops()
 104  */
 105 static void apix_redistribute_compute(void);
 106 static int apix_get_pending(apix_vector_t *);
 107 static apix_vector_t *apix_get_req_vector(ddi_intr_handle_impl_t *, ushort_t);
 108 static int apix_get_intr_info(ddi_intr_handle_impl_t *, apic_get_intr_t *);
 109 static char *apix_get_apic_type(void);
 110 static int apix_intx_get_pending(int);
 111 static void apix_intx_set_mask(int irqno);
 112 static void apix_intx_clear_mask(int irqno);
 113 static int apix_intx_get_shared(int irqno);
 114 static void apix_intx_set_shared(int irqno, int delta);
 115 static apix_vector_t *apix_intx_xlate_vector(dev_info_t *, int,
 116     struct intrspec *);
 117 static int apix_intx_alloc_vector(dev_info_t *, int, struct intrspec *);
 118 
 119 extern int apic_clkinit(int);
 120 
 121 /* IRM initialization for APIX PSM module */
 122 extern void apix_irm_init(void);
 123 
 124 extern int irm_enable;
 125 
 126 /*
 127  *      Local static data
 128  */
 129 static struct   psm_ops apix_ops = {
 130         apix_probe,
 131 
 132         apix_init,
 133         apix_picinit,
 134         apix_intr_enter,
 135         apix_intr_exit,
 136         apix_setspl,
 137         apix_addspl,
 138         apix_delspl,
 139         apix_disable_intr,
 140         apix_enable_intr,
 141         NULL,                   /* psm_softlvl_to_irq */
 142         NULL,                   /* psm_set_softintr */
 143 
 144         apic_set_idlecpu,
 145         apic_unset_idlecpu,
 146 
 147         apic_clkinit,
 148         apix_get_clkvect,
 149         NULL,                   /* psm_hrtimeinit */
 150         apic_gethrtime,
 151 
 152         apic_get_next_processorid,
 153         apic_cpu_start,
 154         apix_post_cpu_start,
 155         apic_shutdown,
 156         apix_get_ipivect,
 157         apic_send_ipi,
 158 
 159         NULL,                   /* psm_translate_irq */
 160         NULL,                   /* psm_notify_error */
 161         NULL,                   /* psm_notify_func */
 162         apic_timer_reprogram,
 163         apic_timer_enable,
 164         apic_timer_disable,
 165         apix_post_cyclic_setup,
 166         apic_preshutdown,
 167         apix_intr_ops,          /* Advanced DDI Interrupt framework */
 168         apic_state,             /* save, restore apic state for S3 */
 169         apic_cpu_ops,           /* CPU control interface. */
 170 };
 171 
 172 struct psm_ops *psmops = &apix_ops;
 173 
 174 static struct   psm_info apix_psm_info = {
 175         PSM_INFO_VER01_7,                       /* version */
 176         PSM_OWN_EXCLUSIVE,                      /* ownership */
 177         &apix_ops,                          /* operation */
 178         APIX_NAME,                              /* machine name */
 179         "apix MPv1.4 compatible",
 180 };
 181 
 182 static void *apix_hdlp;
 183 
 184 static int apix_is_enabled = 0;
 185 
 186 /*
 187  * Flag to indicate if APIX is to be enabled only for platforms
 188  * with specific hw feature(s).
 189  */
 190 int apix_hw_chk_enable = 1;
 191 
 192 /*
 193  * Hw features that are checked for enabling APIX support.
 194  */
 195 #define APIX_SUPPORT_X2APIC     0x00000001
 196 uint_t apix_supported_hw = APIX_SUPPORT_X2APIC;
 197 
 198 /*
 199  * apix_lock is used for cpu selection and vector re-binding
 200  */
 201 lock_t apix_lock;
 202 apix_impl_t *apixs[NCPU];
 203 /*
 204  * Mapping between device interrupt and the allocated vector. Indexed
 205  * by major number.
 206  */
 207 apix_dev_vector_t **apix_dev_vector;
 208 /*
 209  * Mapping between device major number and cpu id. It gets used
 210  * when interrupt binding policy round robin with affinity is
 211  * applied. With that policy, devices with the same major number
 212  * will be bound to the same CPU.
 213  */
 214 processorid_t *apix_major_to_cpu;       /* major to cpu mapping */
 215 kmutex_t apix_mutex;    /* for apix_dev_vector & apix_major_to_cpu */
 216 
 217 int apix_nipis = 16;    /* Maximum number of IPIs */
 218 /*
 219  * Maximum number of vectors in a CPU that can be used for interrupt
 220  * allocation (including IPIs and the reserved vectors).
 221  */
 222 int apix_cpu_nvectors = APIX_NVECTOR;
 223 
 224 /* gcpu.h */
 225 
 226 extern void apic_do_interrupt(struct regs *rp, trap_trace_rec_t *ttp);
 227 extern void apic_change_eoi();
 228 
 229 /*
 230  *      This is the loadable module wrapper
 231  */
 232 
 233 int
 234 _init(void)
 235 {
 236         if (apic_coarse_hrtime)
 237                 apix_ops.psm_gethrtime = &apic_gettime;
 238         return (psm_mod_init(&apix_hdlp, &apix_psm_info));
 239 }
 240 
 241 int
 242 _fini(void)
 243 {
 244         return (psm_mod_fini(&apix_hdlp, &apix_psm_info));
 245 }
 246 
 247 int
 248 _info(struct modinfo *modinfop)
 249 {
 250         return (psm_mod_info(&apix_hdlp, &apix_psm_info, modinfop));
 251 }
 252 
 253 static int
 254 apix_probe()
 255 {
 256         int rval;
 257 
 258         if (apix_enable == 0)
 259                 return (PSM_FAILURE);
 260 
 261         /* check for hw features if specified  */
 262         if (apix_hw_chk_enable) {
 263                 /* check if x2APIC mode is supported */
 264                 if ((apix_supported_hw & APIX_SUPPORT_X2APIC) ==
 265                     APIX_SUPPORT_X2APIC) {
 266                         if (!((apic_local_mode() == LOCAL_X2APIC) ||
 267                             apic_detect_x2apic())) {
 268                                 /* x2APIC mode is not supported in the hw */
 269                                 apix_enable = 0;
 270                         }
 271                 }
 272                 if (apix_enable == 0)
 273                         return (PSM_FAILURE);
 274         }
 275 
 276         rval = apic_probe_common(apix_psm_info.p_mach_idstring);
 277         if (rval == PSM_SUCCESS)
 278                 apix_is_enabled = 1;
 279         else
 280                 apix_is_enabled = 0;
 281         return (rval);
 282 }
 283 
 284 /*
 285  * Initialize the data structures needed by pcplusmpx module.
 286  * Specifically, the data structures used by addspl() and delspl()
 287  * routines.
 288  */
 289 static void
 290 apix_softinit()
 291 {
 292         int i, *iptr;
 293         apix_impl_t *hdlp;
 294         int nproc;
 295 
 296         nproc = max(apic_nproc, apic_max_nproc);
 297 
 298         hdlp = kmem_zalloc(nproc * sizeof (apix_impl_t), KM_SLEEP);
 299         for (i = 0; i < nproc; i++) {
 300                 apixs[i] = &hdlp[i];
 301                 apixs[i]->x_cpuid = i;
 302                 LOCK_INIT_CLEAR(&apixs[i]->x_lock);
 303         }
 304 
 305         /* cpu 0 is always up (for now) */
 306         apic_cpus[0].aci_status = APIC_CPU_ONLINE | APIC_CPU_INTR_ENABLE;
 307 
 308         iptr = (int *)&apic_irq_table[0];
 309         for (i = 0; i <= APIC_MAX_VECTOR; i++) {
 310                 apic_level_intr[i] = 0;
 311                 *iptr++ = NULL;
 312         }
 313         mutex_init(&airq_mutex, NULL, MUTEX_DEFAULT, NULL);
 314 
 315         apix_dev_vector = kmem_zalloc(sizeof (apix_dev_vector_t *) * devcnt,
 316             KM_SLEEP);
 317 
 318         if (apic_intr_policy == INTR_ROUND_ROBIN_WITH_AFFINITY) {
 319                 apix_major_to_cpu = kmem_zalloc(sizeof (int) * devcnt,
 320                     KM_SLEEP);
 321                 for (i = 0; i < devcnt; i++)
 322                         apix_major_to_cpu[i] = IRQ_UNINIT;
 323         }
 324 
 325         mutex_init(&apix_mutex, NULL, MUTEX_DEFAULT, NULL);
 326 }
 327 
 328 static int
 329 apix_get_pending_spl(void)
 330 {
 331         int cpuid = CPU->cpu_id;
 332 
 333         return (bsrw_insn(apixs[cpuid]->x_intr_pending));
 334 }
 335 
 336 static uintptr_t
 337 apix_get_intr_handler(int cpu, short vec)
 338 {
 339         apix_vector_t *apix_vector;
 340 
 341         ASSERT(cpu < apic_nproc && vec < APIX_NVECTOR);
 342         if (cpu >= apic_nproc)
 343                 return (NULL);
 344 
 345         apix_vector = apixs[cpu]->x_vectbl[vec];
 346 
 347         return ((uintptr_t)(apix_vector->v_autovect));
 348 }
 349 
 350 #if defined(__amd64)
 351 static unsigned char dummy_cpu_pri[MAXIPL + 1] = {
 352         0, 0, 0, 0, 0, 0, 0, 0,
 353         0, 0, 0, 0, 0, 0, 0, 0, 0
 354 };
 355 #endif
 356 
 357 static void
 358 apix_init()
 359 {
 360         extern void (*do_interrupt_common)(struct regs *, trap_trace_rec_t *);
 361 
 362         APIC_VERBOSE(INIT, (CE_CONT, "apix: psm_softinit\n"));
 363 
 364         do_interrupt_common = apix_do_interrupt;
 365         addintr = apix_add_avintr;
 366         remintr = apix_rem_avintr;
 367         get_pending_spl = apix_get_pending_spl;
 368         get_intr_handler = apix_get_intr_handler;
 369         psm_get_localapicid = apic_get_localapicid;
 370         psm_get_ioapicid = apic_get_ioapicid;
 371 
 372         apix_softinit();
 373 #if defined(__amd64)
 374         CPU->cpu_pri_data = dummy_cpu_pri;
 375 #else
 376         if (cpuid_have_cr8access(CPU))
 377                 apic_have_32bit_cr8 = 1;
 378 #endif  /* __amd64 */
 379 
 380         /*
 381          * Initialize IRM pool parameters
 382          */
 383         if (irm_enable) {
 384                 int     i;
 385                 int     lowest_irq;
 386                 int     highest_irq;
 387 
 388                 /* number of CPUs present */
 389                 apix_irminfo.apix_ncpus = apic_nproc;
 390                 /* total number of entries in all of the IOAPICs present */
 391                 lowest_irq = apic_io_vectbase[0];
 392                 highest_irq = apic_io_vectend[0];
 393                 for (i = 1; i < apic_io_max; i++) {
 394                         if (apic_io_vectbase[i] < lowest_irq)
 395                                 lowest_irq = apic_io_vectbase[i];
 396                         if (apic_io_vectend[i] > highest_irq)
 397                                 highest_irq = apic_io_vectend[i];
 398                 }
 399                 apix_irminfo.apix_ioapic_max_vectors =
 400                     highest_irq - lowest_irq + 1;
 401                 /*
 402                  * Number of available per-CPU vectors excluding
 403                  * reserved vectors for Dtrace, int80, system-call,
 404                  * fast-trap, etc.
 405                  */
 406                 apix_irminfo.apix_per_cpu_vectors = APIX_NAVINTR -
 407                     APIX_SW_RESERVED_VECTORS;
 408 
 409                 /* Number of vectors (pre) allocated (SCI and HPET) */
 410                 apix_irminfo.apix_vectors_allocated = 0;
 411                 if (apic_hpet_vect != -1)
 412                         apix_irminfo.apix_vectors_allocated++;
 413                 if (apic_sci_vect != -1)
 414                         apix_irminfo.apix_vectors_allocated++;
 415         }
 416 }
 417 
 418 static void
 419 apix_init_intr()
 420 {
 421         processorid_t   cpun = psm_get_cpu_id();
 422         uint_t nlvt;
 423         uint32_t svr = AV_UNIT_ENABLE | APIC_SPUR_INTR;
 424         extern void cmi_cmci_trap(void);
 425 
 426         apic_reg_ops->apic_write_task_reg(APIC_MASK_ALL);
 427 
 428         if (apic_mode == LOCAL_APIC) {
 429                 /*
 430                  * We are running APIC in MMIO mode.
 431                  */
 432                 if (apic_flat_model) {
 433                         apic_reg_ops->apic_write(APIC_FORMAT_REG,
 434                             APIC_FLAT_MODEL);
 435                 } else {
 436                         apic_reg_ops->apic_write(APIC_FORMAT_REG,
 437                             APIC_CLUSTER_MODEL);
 438                 }
 439 
 440                 apic_reg_ops->apic_write(APIC_DEST_REG,
 441                     AV_HIGH_ORDER >> cpun);
 442         }
 443 
 444         if (apic_directed_EOI_supported()) {
 445                 /*
 446                  * Setting the 12th bit in the Spurious Interrupt Vector
 447                  * Register suppresses broadcast EOIs generated by the local
 448                  * APIC. The suppression of broadcast EOIs happens only when
 449                  * interrupts are level-triggered.
 450                  */
 451                 svr |= APIC_SVR_SUPPRESS_BROADCAST_EOI;
 452         }
 453 
 454         /* need to enable APIC before unmasking NMI */
 455         apic_reg_ops->apic_write(APIC_SPUR_INT_REG, svr);
 456 
 457         /*
 458          * Presence of an invalid vector with delivery mode AV_FIXED can
 459          * cause an error interrupt, even if the entry is masked...so
 460          * write a valid vector to LVT entries along with the mask bit
 461          */
 462 
 463         /* All APICs have timer and LINT0/1 */
 464         apic_reg_ops->apic_write(APIC_LOCAL_TIMER, AV_MASK|APIC_RESV_IRQ);
 465         apic_reg_ops->apic_write(APIC_INT_VECT0, AV_MASK|APIC_RESV_IRQ);
 466         apic_reg_ops->apic_write(APIC_INT_VECT1, AV_NMI);    /* enable NMI */
 467 
 468         /*
 469          * On integrated APICs, the number of LVT entries is
 470          * 'Max LVT entry' + 1; on 82489DX's (non-integrated
 471          * APICs), nlvt is "3" (LINT0, LINT1, and timer)
 472          */
 473 
 474         if (apic_cpus[cpun].aci_local_ver < APIC_INTEGRATED_VERS) {
 475                 nlvt = 3;
 476         } else {
 477                 nlvt = ((apic_reg_ops->apic_read(APIC_VERS_REG) >> 16) &
 478                     0xFF) + 1;
 479         }
 480 
 481         if (nlvt >= 5) {
 482                 /* Enable performance counter overflow interrupt */
 483 
 484                 if (!is_x86_feature(x86_featureset, X86FSET_MSR))
 485                         apic_enable_cpcovf_intr = 0;
 486                 if (apic_enable_cpcovf_intr) {
 487                         if (apic_cpcovf_vect == 0) {
 488                                 int ipl = APIC_PCINT_IPL;
 489 
 490                                 apic_cpcovf_vect = apix_get_ipivect(ipl, -1);
 491                                 ASSERT(apic_cpcovf_vect);
 492 
 493                                 (void) add_avintr(NULL, ipl,
 494                                     (avfunc)kcpc_hw_overflow_intr,
 495                                     "apic pcint", apic_cpcovf_vect,
 496                                     NULL, NULL, NULL, NULL);
 497                                 kcpc_hw_overflow_intr_installed = 1;
 498                                 kcpc_hw_enable_cpc_intr =
 499                                     apic_cpcovf_mask_clear;
 500                         }
 501                         apic_reg_ops->apic_write(APIC_PCINT_VECT,
 502                             apic_cpcovf_vect);
 503                 }
 504         }
 505 
 506         if (nlvt >= 6) {
 507                 /* Only mask TM intr if the BIOS apparently doesn't use it */
 508 
 509                 uint32_t lvtval;
 510 
 511                 lvtval = apic_reg_ops->apic_read(APIC_THERM_VECT);
 512                 if (((lvtval & AV_MASK) == AV_MASK) ||
 513                     ((lvtval & AV_DELIV_MODE) != AV_SMI)) {
 514                         apic_reg_ops->apic_write(APIC_THERM_VECT,
 515                             AV_MASK|APIC_RESV_IRQ);
 516                 }
 517         }
 518 
 519         /* Enable error interrupt */
 520 
 521         if (nlvt >= 4 && apic_enable_error_intr) {
 522                 if (apic_errvect == 0) {
 523                         int ipl = 0xf;  /* get highest priority intr */
 524                         apic_errvect = apix_get_ipivect(ipl, -1);
 525                         ASSERT(apic_errvect);
 526                         /*
 527                          * Not PSMI compliant, but we are going to merge
 528                          * with ON anyway
 529                          */
 530                         (void) add_avintr(NULL, ipl,
 531                             (avfunc)apic_error_intr, "apic error intr",
 532                             apic_errvect, NULL, NULL, NULL, NULL);
 533                 }
 534                 apic_reg_ops->apic_write(APIC_ERR_VECT, apic_errvect);
 535                 apic_reg_ops->apic_write(APIC_ERROR_STATUS, 0);
 536                 apic_reg_ops->apic_write(APIC_ERROR_STATUS, 0);
 537         }
 538 
 539         /* Enable CMCI interrupt */
 540         if (cmi_enable_cmci) {
 541                 mutex_enter(&cmci_cpu_setup_lock);
 542                 if (cmci_cpu_setup_registered == 0) {
 543                         mutex_enter(&cpu_lock);
 544                         register_cpu_setup_func(cmci_cpu_setup, NULL);
 545                         mutex_exit(&cpu_lock);
 546                         cmci_cpu_setup_registered = 1;
 547                 }
 548                 mutex_exit(&cmci_cpu_setup_lock);
 549 
 550                 if (apic_cmci_vect == 0) {
 551                         int ipl = 0x2;
 552                         apic_cmci_vect = apix_get_ipivect(ipl, -1);
 553                         ASSERT(apic_cmci_vect);
 554 
 555                         (void) add_avintr(NULL, ipl,
 556                             (avfunc)cmi_cmci_trap, "apic cmci intr",
 557                             apic_cmci_vect, NULL, NULL, NULL, NULL);
 558                 }
 559                 apic_reg_ops->apic_write(APIC_CMCI_VECT, apic_cmci_vect);
 560         }
 561 
 562         apic_reg_ops->apic_write_task_reg(0);
 563 }
 564 
 565 static void
 566 apix_picinit(void)
 567 {
 568         int i, j;
 569         uint_t isr;
 570 
 571         APIC_VERBOSE(INIT, (CE_CONT, "apix: psm_picinit\n"));
 572 
 573         /*
 574          * initialize interrupt remapping before apic
 575          * hardware initialization
 576          */
 577         apic_intrmap_init(apic_mode);
 578         if (apic_vt_ops == psm_vt_ops)
 579                 apix_mul_ioapic_method = APIC_MUL_IOAPIC_IIR;
 580 
 581         /*
 582          * On UniSys Model 6520, the BIOS leaves vector 0x20 isr
 583          * bit on without clearing it with EOI.  Since softint
 584          * uses vector 0x20 to interrupt itself, so softint will
 585          * not work on this machine.  In order to fix this problem
 586          * a check is made to verify all the isr bits are clear.
 587          * If not, EOIs are issued to clear the bits.
 588          */
 589         for (i = 7; i >= 1; i--) {
 590                 isr = apic_reg_ops->apic_read(APIC_ISR_REG + (i * 4));
 591                 if (isr != 0)
 592                         for (j = 0; ((j < 32) && (isr != 0)); j++)
 593                                 if (isr & (1 << j)) {
 594                                         apic_reg_ops->apic_write(
 595                                             APIC_EOI_REG, 0);
 596                                         isr &= ~(1 << j);
 597                                         apic_error |= APIC_ERR_BOOT_EOI;
 598                                 }
 599         }
 600 
 601         /* set a flag so we know we have run apic_picinit() */
 602         apic_picinit_called = 1;
 603         LOCK_INIT_CLEAR(&apic_gethrtime_lock);
 604         LOCK_INIT_CLEAR(&apic_ioapic_lock);
 605         LOCK_INIT_CLEAR(&apic_error_lock);
 606         LOCK_INIT_CLEAR(&apic_mode_switch_lock);
 607 
 608         picsetup();      /* initialise the 8259 */
 609 
 610         /* add nmi handler - least priority nmi handler */
 611         LOCK_INIT_CLEAR(&apic_nmi_lock);
 612 
 613         if (!psm_add_nmintr(0, (avfunc) apic_nmi_intr,
 614             "apix NMI handler", (caddr_t)NULL))
 615                 cmn_err(CE_WARN, "apix: Unable to add nmi handler");
 616 
 617         apix_init_intr();
 618 
 619         /* enable apic mode if imcr present */
 620         if (apic_imcrp) {
 621                 outb(APIC_IMCR_P1, (uchar_t)APIC_IMCR_SELECT);
 622                 outb(APIC_IMCR_P2, (uchar_t)APIC_IMCR_APIC);
 623         }
 624 
 625         ioapix_init_intr(IOAPIC_MASK);
 626 
 627         /* setup global IRM pool if applicable */
 628         if (irm_enable)
 629                 apix_irm_init();
 630 }
 631 
 632 static __inline__ void
 633 apix_send_eoi(void)
 634 {
 635         if (apic_mode == LOCAL_APIC)
 636                 LOCAL_APIC_WRITE_REG(APIC_EOI_REG, 0);
 637         else
 638                 X2APIC_WRITE(APIC_EOI_REG, 0);
 639 }
 640 
 641 /*
 642  * platform_intr_enter
 643  *
 644  *      Called at the beginning of the interrupt service routine, but unlike
 645  *      pcplusmp, does not mask interrupts. An EOI is given to the interrupt
 646  *      controller to enable other HW interrupts but interrupts are still
 647  *      masked by the IF flag.
 648  *
 649  *      Return -1 for spurious interrupts
 650  *
 651  */
 652 static int
 653 apix_intr_enter(int ipl, int *vectorp)
 654 {
 655         struct cpu *cpu = CPU;
 656         uint32_t cpuid = CPU->cpu_id;
 657         apic_cpus_info_t *cpu_infop;
 658         uchar_t vector;
 659         apix_vector_t *vecp;
 660         int nipl = -1;
 661 
 662         /*
 663          * The real vector delivered is (*vectorp + 0x20), but our caller
 664          * subtracts 0x20 from the vector before passing it to us.
 665          * (That's why APIC_BASE_VECT is 0x20.)
 666          */
 667         vector = *vectorp = (uchar_t)*vectorp + APIC_BASE_VECT;
 668 
 669         cpu_infop = &apic_cpus[cpuid];
 670         if (vector == APIC_SPUR_INTR) {
 671                 cpu_infop->aci_spur_cnt++;
 672                 return (APIC_INT_SPURIOUS);
 673         }
 674 
 675         vecp = xv_vector(cpuid, vector);
 676         if (vecp == NULL) {
 677                 if (APIX_IS_FAKE_INTR(vector))
 678                         nipl = apix_rebindinfo.i_pri;
 679                 apix_send_eoi();
 680                 return (nipl);
 681         }
 682         nipl = vecp->v_pri;
 683 
 684         /* if interrupted by the clock, increment apic_nsec_since_boot */
 685         if (vector == (apic_clkvect + APIC_BASE_VECT)) {
 686                 if (!apic_oneshot) {
 687                         /* NOTE: this is not MT aware */
 688                         apic_hrtime_stamp++;
 689                         apic_nsec_since_boot += apic_nsec_per_intr;
 690                         apic_hrtime_stamp++;
 691                         last_count_read = apic_hertz_count;
 692                         apix_redistribute_compute();
 693                 }
 694 
 695                 apix_send_eoi();
 696 
 697                 return (nipl);
 698         }
 699 
 700         ASSERT(vecp->v_state != APIX_STATE_OBSOLETED);
 701 
 702         /* pre-EOI handling for level-triggered interrupts */
 703         if (!APIX_IS_DIRECTED_EOI(apix_mul_ioapic_method) &&
 704             (vecp->v_type & APIX_TYPE_FIXED) && apic_level_intr[vecp->v_inum])
 705                 apix_level_intr_pre_eoi(vecp->v_inum);
 706 
 707         /* send back EOI */
 708         apix_send_eoi();
 709 
 710         cpu_infop->aci_current[nipl] = vector;
 711         if ((nipl > ipl) && (nipl > cpu->cpu_base_spl)) {
 712                 cpu_infop->aci_curipl = (uchar_t)nipl;
 713                 cpu_infop->aci_ISR_in_progress |= 1 << nipl;
 714         }
 715 
 716 #ifdef  DEBUG
 717         if (vector >= APIX_IPI_MIN)
 718                 return (nipl);  /* skip IPI */
 719 
 720         APIC_DEBUG_BUF_PUT(vector);
 721         APIC_DEBUG_BUF_PUT(vecp->v_inum);
 722         APIC_DEBUG_BUF_PUT(nipl);
 723         APIC_DEBUG_BUF_PUT(psm_get_cpu_id());
 724         if ((apic_stretch_interrupts) && (apic_stretch_ISR & (1 << nipl)))
 725                 drv_usecwait(apic_stretch_interrupts);
 726 #endif /* DEBUG */
 727 
 728         return (nipl);
 729 }
 730 
 731 /*
 732  * Any changes made to this function must also change X2APIC
 733  * version of intr_exit.
 734  */
 735 static void
 736 apix_intr_exit(int prev_ipl, int arg2)
 737 {
 738         int cpuid = psm_get_cpu_id();
 739         apic_cpus_info_t *cpu_infop = &apic_cpus[cpuid];
 740         apix_impl_t *apixp = apixs[cpuid];
 741 
 742         UNREFERENCED_1PARAMETER(arg2);
 743 
 744         cpu_infop->aci_curipl = (uchar_t)prev_ipl;
 745         /* ISR above current pri could not be in progress */
 746         cpu_infop->aci_ISR_in_progress &= (2 << prev_ipl) - 1;
 747 
 748         if (apixp->x_obsoletes != NULL) {
 749                 if (APIX_CPU_LOCK_HELD(cpuid))
 750                         return;
 751 
 752                 APIX_ENTER_CPU_LOCK(cpuid);
 753                 (void) apix_obsolete_vector(apixp->x_obsoletes);
 754                 APIX_LEAVE_CPU_LOCK(cpuid);
 755         }
 756 }
 757 
 758 /*
 759  * The pcplusmp setspl code uses the TPR to mask all interrupts at or below the
 760  * given ipl, but apix never uses the TPR and we never mask a subset of the
 761  * interrupts. They are either all blocked by the IF flag or all can come in.
 762  *
 763  * For setspl, we mask all interrupts for XC_HI_PIL (15), otherwise, interrupts
 764  * can come in if currently enabled by the IF flag. This table shows the state
 765  * of the IF flag when we leave this function.
 766  *
 767  *    curr IF | ipl == 15       ipl != 15
 768  *    --------+---------------------------
 769  *       0    |    0                0
 770  *       1    |    0                1
 771  */
 772 static void
 773 apix_setspl(int ipl)
 774 {
 775         /*
 776          * Interrupts at ipl above this cannot be in progress, so the following
 777          * mask is ok.
 778          */
 779         apic_cpus[psm_get_cpu_id()].aci_ISR_in_progress &= (2 << ipl) - 1;
 780 
 781         if (ipl == XC_HI_PIL)
 782                 cli();
 783 }
 784 
 785 int
 786 apix_addspl(int virtvec, int ipl, int min_ipl, int max_ipl)
 787 {
 788         uint32_t cpuid = APIX_VIRTVEC_CPU(virtvec);
 789         uchar_t vector = (uchar_t)APIX_VIRTVEC_VECTOR(virtvec);
 790         apix_vector_t *vecp = xv_vector(cpuid, vector);
 791 
 792         UNREFERENCED_3PARAMETER(ipl, min_ipl, max_ipl);
 793         ASSERT(vecp != NULL && LOCK_HELD(&apix_lock));
 794 
 795         if (vecp->v_type == APIX_TYPE_FIXED)
 796                 apix_intx_set_shared(vecp->v_inum, 1);
 797 
 798         /* There are more interrupts, so it's already been enabled */
 799         if (vecp->v_share > 1)
 800                 return (PSM_SUCCESS);
 801 
 802         /* return if it is not hardware interrupt */
 803         if (vecp->v_type == APIX_TYPE_IPI)
 804                 return (PSM_SUCCESS);
 805 
 806         /*
 807          * if apix_picinit() has not been called yet, just return.
 808          * At the end of apic_picinit(), we will call setup_io_intr().
 809          */
 810         if (!apic_picinit_called)
 811                 return (PSM_SUCCESS);
 812 
 813         (void) apix_setup_io_intr(vecp);
 814 
 815         return (PSM_SUCCESS);
 816 }
 817 
 818 int
 819 apix_delspl(int virtvec, int ipl, int min_ipl, int max_ipl)
 820 {
 821         uint32_t cpuid = APIX_VIRTVEC_CPU(virtvec);
 822         uchar_t vector = (uchar_t)APIX_VIRTVEC_VECTOR(virtvec);
 823         apix_vector_t *vecp = xv_vector(cpuid, vector);
 824 
 825         UNREFERENCED_3PARAMETER(ipl, min_ipl, max_ipl);
 826         ASSERT(vecp != NULL && LOCK_HELD(&apix_lock));
 827 
 828         if (vecp->v_type == APIX_TYPE_FIXED)
 829                 apix_intx_set_shared(vecp->v_inum, -1);
 830 
 831         /* There are more interrupts */
 832         if (vecp->v_share > 1)
 833                 return (PSM_SUCCESS);
 834 
 835         /* return if it is not hardware interrupt */
 836         if (vecp->v_type == APIX_TYPE_IPI)
 837                 return (PSM_SUCCESS);
 838 
 839         if (!apic_picinit_called) {
 840                 cmn_err(CE_WARN, "apix: delete 0x%x before apic init",
 841                     virtvec);
 842                 return (PSM_SUCCESS);
 843         }
 844 
 845         apix_disable_vector(vecp);
 846 
 847         return (PSM_SUCCESS);
 848 }
 849 
 850 /*
 851  * Try and disable all interrupts. We just assign interrupts to other
 852  * processors based on policy. If any were bound by user request, we
 853  * let them continue and return failure. We do not bother to check
 854  * for cache affinity while rebinding.
 855  */
 856 static int
 857 apix_disable_intr(processorid_t cpun)
 858 {
 859         apix_impl_t *apixp = apixs[cpun];
 860         apix_vector_t *vecp, *newp;
 861         int bindcpu, i, hardbound = 0, errbound = 0, ret, loop, type;
 862 
 863         lock_set(&apix_lock);
 864 
 865         apic_cpus[cpun].aci_status &= ~APIC_CPU_INTR_ENABLE;
 866         apic_cpus[cpun].aci_curipl = 0;
 867 
 868         /* if this is for SUSPEND operation, skip rebinding */
 869         if (apic_cpus[cpun].aci_status & APIC_CPU_SUSPEND) {
 870                 for (i = APIX_AVINTR_MIN; i <= APIX_AVINTR_MAX; i++) {
 871                         vecp = apixp->x_vectbl[i];
 872                         if (!IS_VECT_ENABLED(vecp))
 873                                 continue;
 874 
 875                         apix_disable_vector(vecp);
 876                 }
 877                 lock_clear(&apix_lock);
 878                 return (PSM_SUCCESS);
 879         }
 880 
 881         for (i = APIX_AVINTR_MIN; i <= APIX_AVINTR_MAX; i++) {
 882                 vecp = apixp->x_vectbl[i];
 883                 if (!IS_VECT_ENABLED(vecp))
 884                         continue;
 885 
 886                 if (vecp->v_flags & APIX_VECT_USER_BOUND) {
 887                         hardbound++;
 888                         continue;
 889                 }
 890                 type = vecp->v_type;
 891 
 892                 /*
 893                  * If there are bound interrupts on this cpu, then
 894                  * rebind them to other processors.
 895                  */
 896                 loop = 0;
 897                 do {
 898                         bindcpu = apic_find_cpu(APIC_CPU_INTR_ENABLE);
 899 
 900                         if (type != APIX_TYPE_MSI)
 901                                 newp = apix_set_cpu(vecp, bindcpu, &ret);
 902                         else
 903                                 newp = apix_grp_set_cpu(vecp, bindcpu, &ret);
 904                 } while ((newp == NULL) && (loop++ < apic_nproc));
 905 
 906                 if (loop >= apic_nproc) {
 907                         errbound++;
 908                         cmn_err(CE_WARN, "apix: failed to rebind vector %x/%x",
 909                             vecp->v_cpuid, vecp->v_vector);
 910                 }
 911         }
 912 
 913         lock_clear(&apix_lock);
 914 
 915         if (hardbound || errbound) {
 916                 cmn_err(CE_WARN, "Could not disable interrupts on %d"
 917                     "due to user bound interrupts or failed operation",
 918                     cpun);
 919                 return (PSM_FAILURE);
 920         }
 921 
 922         return (PSM_SUCCESS);
 923 }
 924 
 925 /*
 926  * Bind interrupts to specified CPU
 927  */
 928 static void
 929 apix_enable_intr(processorid_t cpun)
 930 {
 931         apix_vector_t *vecp;
 932         int i, ret;
 933         processorid_t n;
 934 
 935         lock_set(&apix_lock);
 936 
 937         apic_cpus[cpun].aci_status |= APIC_CPU_INTR_ENABLE;
 938 
 939         /* interrupt enabling for system resume */
 940         if (apic_cpus[cpun].aci_status & APIC_CPU_SUSPEND) {
 941                 for (i = APIX_AVINTR_MIN; i <= APIX_AVINTR_MAX; i++) {
 942                         vecp = xv_vector(cpun, i);
 943                         if (!IS_VECT_ENABLED(vecp))
 944                                 continue;
 945 
 946                         apix_enable_vector(vecp);
 947                 }
 948                 apic_cpus[cpun].aci_status &= ~APIC_CPU_SUSPEND;
 949         }
 950 
 951         for (n = 0; n < apic_nproc; n++) {
 952                 if (!apic_cpu_in_range(n) || n == cpun ||
 953                     (apic_cpus[n].aci_status & APIC_CPU_INTR_ENABLE) == 0)
 954                         continue;
 955 
 956                 for (i = APIX_AVINTR_MIN; i <= APIX_AVINTR_MAX; i++) {
 957                         vecp = xv_vector(n, i);
 958                         if (!IS_VECT_ENABLED(vecp) ||
 959                             vecp->v_bound_cpuid != cpun)
 960                                 continue;
 961 
 962                         if (vecp->v_type != APIX_TYPE_MSI)
 963                                 (void) apix_set_cpu(vecp, cpun, &ret);
 964                         else
 965                                 (void) apix_grp_set_cpu(vecp, cpun, &ret);
 966                 }
 967         }
 968 
 969         lock_clear(&apix_lock);
 970 }
 971 
 972 /*
 973  * Allocate vector for IPI
 974  * type == -1 indicates it is an internal request. Do not change
 975  * resv_vector for these requests.
 976  */
 977 static int
 978 apix_get_ipivect(int ipl, int type)
 979 {
 980         uchar_t vector;
 981 
 982         if ((vector = apix_alloc_ipi(ipl)) > 0) {
 983                 if (type != -1)
 984                         apic_resv_vector[ipl] = vector;
 985                 return (vector);
 986         }
 987         apic_error |= APIC_ERR_GET_IPIVECT_FAIL;
 988         return (-1);    /* shouldn't happen */
 989 }
 990 
 991 static int
 992 apix_get_clkvect(int ipl)
 993 {
 994         int vector;
 995 
 996         if ((vector = apix_get_ipivect(ipl, -1)) == -1)
 997                 return (-1);
 998 
 999         apic_clkvect = vector - APIC_BASE_VECT;
1000         APIC_VERBOSE(IPI, (CE_CONT, "apix: clock vector = %x\n",
1001             apic_clkvect));
1002         return (vector);
1003 }
1004 
1005 static int
1006 apix_post_cpu_start()
1007 {
1008         int cpun;
1009         static int cpus_started = 1;
1010 
1011         /* We know this CPU + BSP  started successfully. */
1012         cpus_started++;
1013 
1014         /*
1015          * On BSP we would have enabled X2APIC, if supported by processor,
1016          * in acpi_probe(), but on AP we do it here.
1017          *
1018          * We enable X2APIC mode only if BSP is running in X2APIC & the
1019          * local APIC mode of the current CPU is MMIO (xAPIC).
1020          */
1021         if (apic_mode == LOCAL_X2APIC && apic_detect_x2apic() &&
1022             apic_local_mode() == LOCAL_APIC) {
1023                 apic_enable_x2apic();
1024         }
1025 
1026         /*
1027          * Switch back to x2apic IPI sending method for performance when target
1028          * CPU has entered x2apic mode.
1029          */
1030         if (apic_mode == LOCAL_X2APIC) {
1031                 apic_switch_ipi_callback(B_FALSE);
1032         }
1033 
1034         splx(ipltospl(LOCK_LEVEL));
1035         apix_init_intr();
1036 
1037         /*
1038          * since some systems don't enable the internal cache on the non-boot
1039          * cpus, so we have to enable them here
1040          */
1041         setcr0(getcr0() & ~(CR0_CD | CR0_NW));
1042 
1043 #ifdef  DEBUG
1044         APIC_AV_PENDING_SET();
1045 #else
1046         if (apic_mode == LOCAL_APIC)
1047                 APIC_AV_PENDING_SET();
1048 #endif  /* DEBUG */
1049 
1050         /*
1051          * We may be booting, or resuming from suspend; aci_status will
1052          * be APIC_CPU_INTR_ENABLE if coming from suspend, so we add the
1053          * APIC_CPU_ONLINE flag here rather than setting aci_status completely.
1054          */
1055         cpun = psm_get_cpu_id();
1056         apic_cpus[cpun].aci_status |= APIC_CPU_ONLINE;
1057 
1058         apic_reg_ops->apic_write(APIC_DIVIDE_REG, apic_divide_reg_init);
1059 
1060         return (PSM_SUCCESS);
1061 }
1062 
1063 /*
1064  * If this module needs a periodic handler for the interrupt distribution, it
1065  * can be added here. The argument to the periodic handler is not currently
1066  * used, but is reserved for future.
1067  */
1068 static void
1069 apix_post_cyclic_setup(void *arg)
1070 {
1071         UNREFERENCED_1PARAMETER(arg);
1072 
1073         cyc_handler_t cyh;
1074         cyc_time_t cyt;
1075 
1076         /* cpu_lock is held */
1077         /* set up a periodic handler for intr redistribution */
1078 
1079         /*
1080          * In peridoc mode intr redistribution processing is done in
1081          * apic_intr_enter during clk intr processing
1082          */
1083         if (!apic_oneshot)
1084                 return;
1085 
1086         /*
1087          * Register a periodical handler for the redistribution processing.
1088          * Though we would generally prefer to use the DDI interface for
1089          * periodic handler invocation, ddi_periodic_add(9F), we are
1090          * unfortunately already holding cpu_lock, which ddi_periodic_add will
1091          * attempt to take for us.  Thus, we add our own cyclic directly:
1092          */
1093         cyh.cyh_func = (void (*)(void *))apix_redistribute_compute;
1094         cyh.cyh_arg = NULL;
1095         cyh.cyh_level = CY_LOW_LEVEL;
1096 
1097         cyt.cyt_when = 0;
1098         cyt.cyt_interval = apic_redistribute_sample_interval;
1099 
1100         apic_cyclic_id = cyclic_add(&cyh, &cyt);
1101 }
1102 
1103 /*
1104  * Called the first time we enable x2apic mode on this cpu.
1105  * Update some of the function pointers to use x2apic routines.
1106  */
1107 void
1108 x2apic_update_psm()
1109 {
1110         struct psm_ops *pops = &apix_ops;
1111 
1112         ASSERT(pops != NULL);
1113 
1114         /*
1115          * The pcplusmp module's version of x2apic_update_psm makes additional
1116          * changes that we do not have to make here. It needs to make those
1117          * changes because pcplusmp relies on the TPR register and the means of
1118          * addressing that changes when using the local apic versus the x2apic.
1119          * It's also worth noting that the apix driver specific function end up
1120          * being apix_foo as opposed to apic_foo and x2apic_foo.
1121          */
1122         pops->psm_send_ipi = x2apic_send_ipi;
1123 
1124         send_dirintf = pops->psm_send_ipi;
1125 
1126         apic_mode = LOCAL_X2APIC;
1127         apic_change_ops();
1128 }
1129 
1130 /*
1131  * This function provides external interface to the nexus for all
1132  * functionalities related to the new DDI interrupt framework.
1133  *
1134  * Input:
1135  * dip     - pointer to the dev_info structure of the requested device
1136  * hdlp    - pointer to the internal interrupt handle structure for the
1137  *           requested interrupt
1138  * intr_op - opcode for this call
1139  * result  - pointer to the integer that will hold the result to be
1140  *           passed back if return value is PSM_SUCCESS
1141  *
1142  * Output:
1143  * return value is either PSM_SUCCESS or PSM_FAILURE
1144  */
1145 static int
1146 apix_intr_ops(dev_info_t *dip, ddi_intr_handle_impl_t *hdlp,
1147     psm_intr_op_t intr_op, int *result)
1148 {
1149         int             cap;
1150         apix_vector_t   *vecp, *newvecp;
1151         struct intrspec *ispec, intr_spec;
1152         processorid_t target;
1153 
1154         ispec = &intr_spec;
1155         ispec->intrspec_pri = hdlp->ih_pri;
1156         ispec->intrspec_vec = hdlp->ih_inum;
1157         ispec->intrspec_func = hdlp->ih_cb_func;
1158 
1159         switch (intr_op) {
1160         case PSM_INTR_OP_ALLOC_VECTORS:
1161                 switch (hdlp->ih_type) {
1162                 case DDI_INTR_TYPE_MSI:
1163                         /* allocate MSI vectors */
1164                         *result = apix_alloc_msi(dip, hdlp->ih_inum,
1165                             hdlp->ih_scratch1,
1166                             (int)(uintptr_t)hdlp->ih_scratch2);
1167                         break;
1168                 case DDI_INTR_TYPE_MSIX:
1169                         /* allocate MSI-X vectors */
1170                         *result = apix_alloc_msix(dip, hdlp->ih_inum,
1171                             hdlp->ih_scratch1,
1172                             (int)(uintptr_t)hdlp->ih_scratch2);
1173                         break;
1174                 case DDI_INTR_TYPE_FIXED:
1175                         /* allocate or share vector for fixed */
1176                         if ((ihdl_plat_t *)hdlp->ih_private == NULL) {
1177                                 return (PSM_FAILURE);
1178                         }
1179                         ispec = ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp;
1180                         *result = apix_intx_alloc_vector(dip, hdlp->ih_inum,
1181                             ispec);
1182                         break;
1183                 default:
1184                         return (PSM_FAILURE);
1185                 }
1186                 break;
1187         case PSM_INTR_OP_FREE_VECTORS:
1188                 apix_free_vectors(dip, hdlp->ih_inum, hdlp->ih_scratch1,
1189                     hdlp->ih_type);
1190                 break;
1191         case PSM_INTR_OP_XLATE_VECTOR:
1192                 /*
1193                  * Vectors are allocated by ALLOC and freed by FREE.
1194                  * XLATE finds and returns APIX_VIRTVEC_VECTOR(cpu, vector).
1195                  */
1196                 *result = APIX_INVALID_VECT;
1197                 vecp = apix_get_dev_map(dip, hdlp->ih_inum, hdlp->ih_type);
1198                 if (vecp != NULL) {
1199                         *result = APIX_VIRTVECTOR(vecp->v_cpuid,
1200                             vecp->v_vector);
1201                         break;
1202                 }
1203 
1204                 /*
1205                  * No vector to device mapping exists. If this is FIXED type
1206                  * then check if this IRQ is already mapped for another device
1207                  * then return the vector number for it (i.e. shared IRQ case).
1208                  * Otherwise, return PSM_FAILURE.
1209                  */
1210                 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) {
1211                         vecp = apix_intx_xlate_vector(dip, hdlp->ih_inum,
1212                             ispec);
1213                         *result = (vecp == NULL) ? APIX_INVALID_VECT :
1214                             APIX_VIRTVECTOR(vecp->v_cpuid, vecp->v_vector);
1215                 }
1216                 if (*result == APIX_INVALID_VECT)
1217                         return (PSM_FAILURE);
1218                 break;
1219         case PSM_INTR_OP_GET_PENDING:
1220                 vecp = apix_get_dev_map(dip, hdlp->ih_inum, hdlp->ih_type);
1221                 if (vecp == NULL)
1222                         return (PSM_FAILURE);
1223 
1224                 *result = apix_get_pending(vecp);
1225                 break;
1226         case PSM_INTR_OP_CLEAR_MASK:
1227                 if (hdlp->ih_type != DDI_INTR_TYPE_FIXED)
1228                         return (PSM_FAILURE);
1229 
1230                 vecp = apix_get_dev_map(dip, hdlp->ih_inum, hdlp->ih_type);
1231                 if (vecp == NULL)
1232                         return (PSM_FAILURE);
1233 
1234                 apix_intx_clear_mask(vecp->v_inum);
1235                 break;
1236         case PSM_INTR_OP_SET_MASK:
1237                 if (hdlp->ih_type != DDI_INTR_TYPE_FIXED)
1238                         return (PSM_FAILURE);
1239 
1240                 vecp = apix_get_dev_map(dip, hdlp->ih_inum, hdlp->ih_type);
1241                 if (vecp == NULL)
1242                         return (PSM_FAILURE);
1243 
1244                 apix_intx_set_mask(vecp->v_inum);
1245                 break;
1246         case PSM_INTR_OP_GET_SHARED:
1247                 if (hdlp->ih_type != DDI_INTR_TYPE_FIXED)
1248                         return (PSM_FAILURE);
1249 
1250                 vecp = apix_get_dev_map(dip, hdlp->ih_inum, hdlp->ih_type);
1251                 if (vecp == NULL)
1252                         return (PSM_FAILURE);
1253 
1254                 *result = apix_intx_get_shared(vecp->v_inum);
1255                 break;
1256         case PSM_INTR_OP_SET_PRI:
1257                 /*
1258                  * Called prior to adding the interrupt handler or when
1259                  * an interrupt handler is unassigned.
1260                  */
1261                 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED)
1262                         return (PSM_SUCCESS);
1263 
1264                 if (apix_get_dev_map(dip, hdlp->ih_inum, hdlp->ih_type) == NULL)
1265                         return (PSM_FAILURE);
1266 
1267                 break;
1268         case PSM_INTR_OP_SET_CPU:
1269         case PSM_INTR_OP_GRP_SET_CPU:
1270                 /*
1271                  * The interrupt handle given here has been allocated
1272                  * specifically for this command, and ih_private carries
1273                  * a CPU value.
1274                  */
1275                 *result = EINVAL;
1276                 target = (int)(intptr_t)hdlp->ih_private;
1277                 if (!apic_cpu_in_range(target)) {
1278                         DDI_INTR_IMPLDBG((CE_WARN,
1279                             "[grp_]set_cpu: cpu out of range: %d\n", target));
1280                         return (PSM_FAILURE);
1281                 }
1282 
1283                 lock_set(&apix_lock);
1284 
1285                 vecp = apix_get_req_vector(hdlp, hdlp->ih_flags);
1286                 if (!IS_VECT_ENABLED(vecp)) {
1287                         DDI_INTR_IMPLDBG((CE_WARN,
1288                             "[grp]_set_cpu: invalid vector 0x%x\n",
1289                             hdlp->ih_vector));
1290                         lock_clear(&apix_lock);
1291                         return (PSM_FAILURE);
1292                 }
1293 
1294                 *result = 0;
1295 
1296                 if (intr_op == PSM_INTR_OP_SET_CPU)
1297                         newvecp = apix_set_cpu(vecp, target, result);
1298                 else
1299                         newvecp = apix_grp_set_cpu(vecp, target, result);
1300 
1301                 lock_clear(&apix_lock);
1302 
1303                 if (newvecp == NULL) {
1304                         *result = EIO;
1305                         return (PSM_FAILURE);
1306                 }
1307                 newvecp->v_bound_cpuid = target;
1308                 hdlp->ih_vector = APIX_VIRTVECTOR(newvecp->v_cpuid,
1309                     newvecp->v_vector);
1310                 break;
1311 
1312         case PSM_INTR_OP_GET_INTR:
1313                 /*
1314                  * The interrupt handle given here has been allocated
1315                  * specifically for this command, and ih_private carries
1316                  * a pointer to a apic_get_intr_t.
1317                  */
1318                 if (apix_get_intr_info(hdlp, hdlp->ih_private) != PSM_SUCCESS)
1319                         return (PSM_FAILURE);
1320                 break;
1321 
1322         case PSM_INTR_OP_CHECK_MSI:
1323                 /*
1324                  * Check MSI/X is supported or not at APIC level and
1325                  * masked off the MSI/X bits in hdlp->ih_type if not
1326                  * supported before return.  If MSI/X is supported,
1327                  * leave the ih_type unchanged and return.
1328                  *
1329                  * hdlp->ih_type passed in from the nexus has all the
1330                  * interrupt types supported by the device.
1331                  */
1332                 if (apic_support_msi == 0) {    /* uninitialized */
1333                         /*
1334                          * if apic_support_msi is not set, call
1335                          * apic_check_msi_support() to check whether msi
1336                          * is supported first
1337                          */
1338                         if (apic_check_msi_support() == PSM_SUCCESS)
1339                                 apic_support_msi = 1;   /* supported */
1340                         else
1341                                 apic_support_msi = -1;  /* not-supported */
1342                 }
1343                 if (apic_support_msi == 1) {
1344                         if (apic_msix_enable)
1345                                 *result = hdlp->ih_type;
1346                         else
1347                                 *result = hdlp->ih_type & ~DDI_INTR_TYPE_MSIX;
1348                 } else
1349                         *result = hdlp->ih_type & ~(DDI_INTR_TYPE_MSI |
1350                             DDI_INTR_TYPE_MSIX);
1351                 break;
1352         case PSM_INTR_OP_GET_CAP:
1353                 cap = DDI_INTR_FLAG_PENDING;
1354                 if (hdlp->ih_type == DDI_INTR_TYPE_FIXED)
1355                         cap |= DDI_INTR_FLAG_MASKABLE;
1356                 *result = cap;
1357                 break;
1358         case PSM_INTR_OP_APIC_TYPE:
1359                 ((apic_get_type_t *)(hdlp->ih_private))->avgi_type =
1360                     apix_get_apic_type();
1361                 ((apic_get_type_t *)(hdlp->ih_private))->avgi_num_intr =
1362                     APIX_IPI_MIN;
1363                 ((apic_get_type_t *)(hdlp->ih_private))->avgi_num_cpu =
1364                     apic_nproc;
1365                 hdlp->ih_ver = apic_get_apic_version();
1366                 break;
1367         case PSM_INTR_OP_SET_CAP:
1368         default:
1369                 return (PSM_FAILURE);
1370         }
1371 
1372         return (PSM_SUCCESS);
1373 }
1374 
1375 static void
1376 apix_cleanup_busy(void)
1377 {
1378         int i, j;
1379         apix_vector_t *vecp;
1380 
1381         for (i = 0; i < apic_nproc; i++) {
1382                 if (!apic_cpu_in_range(i))
1383                         continue;
1384                 apic_cpus[i].aci_busy = 0;
1385                 for (j = APIX_AVINTR_MIN; j < APIX_AVINTR_MAX; j++) {
1386                         if ((vecp = xv_vector(i, j)) != NULL)
1387                                 vecp->v_busy = 0;
1388                 }
1389         }
1390 }
1391 
1392 static void
1393 apix_redistribute_compute(void)
1394 {
1395         int     i, j, max_busy;
1396 
1397         if (!apic_enable_dynamic_migration)
1398                 return;
1399 
1400         if (++apic_nticks == apic_sample_factor_redistribution) {
1401                 /*
1402                  * Time to call apic_intr_redistribute().
1403                  * reset apic_nticks. This will cause max_busy
1404                  * to be calculated below and if it is more than
1405                  * apic_int_busy, we will do the whole thing
1406                  */
1407                 apic_nticks = 0;
1408         }
1409         max_busy = 0;
1410         for (i = 0; i < apic_nproc; i++) {
1411                 if (!apic_cpu_in_range(i))
1412                         continue;
1413                 /*
1414                  * Check if curipl is non zero & if ISR is in
1415                  * progress
1416                  */
1417                 if (((j = apic_cpus[i].aci_curipl) != 0) &&
1418                     (apic_cpus[i].aci_ISR_in_progress & (1 << j))) {
1419 
1420                         int     vect;
1421                         apic_cpus[i].aci_busy++;
1422                         vect = apic_cpus[i].aci_current[j];
1423                         apixs[i]->x_vectbl[vect]->v_busy++;
1424                 }
1425 
1426                 if (!apic_nticks &&
1427                     (apic_cpus[i].aci_busy > max_busy))
1428                         max_busy = apic_cpus[i].aci_busy;
1429         }
1430         if (!apic_nticks) {
1431                 if (max_busy > apic_int_busy_mark) {
1432                 /*
1433                  * We could make the following check be
1434                  * skipped > 1 in which case, we get a
1435                  * redistribution at half the busy mark (due to
1436                  * double interval). Need to be able to collect
1437                  * more empirical data to decide if that is a
1438                  * good strategy. Punt for now.
1439                  */
1440                         apix_cleanup_busy();
1441                         apic_skipped_redistribute = 0;
1442                 } else
1443                         apic_skipped_redistribute++;
1444         }
1445 }
1446 
1447 /*
1448  * intr_ops() service routines
1449  */
1450 
1451 static int
1452 apix_get_pending(apix_vector_t *vecp)
1453 {
1454         int bit, index, irr, pending;
1455 
1456         /* need to get on the bound cpu */
1457         mutex_enter(&cpu_lock);
1458         affinity_set(vecp->v_cpuid);
1459 
1460         index = vecp->v_vector / 32;
1461         bit = vecp->v_vector % 32;
1462         irr = apic_reg_ops->apic_read(APIC_IRR_REG + index);
1463 
1464         affinity_clear();
1465         mutex_exit(&cpu_lock);
1466 
1467         pending = (irr & (1 << bit)) ? 1 : 0;
1468         if (!pending && vecp->v_type == APIX_TYPE_FIXED)
1469                 pending = apix_intx_get_pending(vecp->v_inum);
1470 
1471         return (pending);
1472 }
1473 
1474 static apix_vector_t *
1475 apix_get_req_vector(ddi_intr_handle_impl_t *hdlp, ushort_t flags)
1476 {
1477         apix_vector_t *vecp;
1478         processorid_t cpuid;
1479         int32_t virt_vec = 0;
1480 
1481         switch (flags & PSMGI_INTRBY_FLAGS) {
1482         case PSMGI_INTRBY_IRQ:
1483                 return (apix_intx_get_vector(hdlp->ih_vector));
1484         case PSMGI_INTRBY_VEC:
1485                 virt_vec = (virt_vec == 0) ? hdlp->ih_vector : virt_vec;
1486 
1487                 cpuid = APIX_VIRTVEC_CPU(virt_vec);
1488                 if (!apic_cpu_in_range(cpuid))
1489                         return (NULL);
1490 
1491                 vecp = xv_vector(cpuid, APIX_VIRTVEC_VECTOR(virt_vec));
1492                 break;
1493         case PSMGI_INTRBY_DEFAULT:
1494                 vecp = apix_get_dev_map(hdlp->ih_dip, hdlp->ih_inum,
1495                     hdlp->ih_type);
1496                 break;
1497         default:
1498                 return (NULL);
1499         }
1500 
1501         return (vecp);
1502 }
1503 
1504 static int
1505 apix_get_intr_info(ddi_intr_handle_impl_t *hdlp,
1506     apic_get_intr_t *intr_params_p)
1507 {
1508         apix_vector_t *vecp;
1509         struct autovec *av_dev;
1510         int i;
1511 
1512         vecp = apix_get_req_vector(hdlp, intr_params_p->avgi_req_flags);
1513         if (IS_VECT_FREE(vecp)) {
1514                 intr_params_p->avgi_num_devs = 0;
1515                 intr_params_p->avgi_cpu_id = 0;
1516                 intr_params_p->avgi_req_flags = 0;
1517                 return (PSM_SUCCESS);
1518         }
1519 
1520         if (intr_params_p->avgi_req_flags & PSMGI_REQ_CPUID) {
1521                 intr_params_p->avgi_cpu_id = vecp->v_cpuid;
1522 
1523                 /* Return user bound info for intrd. */
1524                 if (intr_params_p->avgi_cpu_id & IRQ_USER_BOUND) {
1525                         intr_params_p->avgi_cpu_id &= ~IRQ_USER_BOUND;
1526                         intr_params_p->avgi_cpu_id |= PSMGI_CPU_USER_BOUND;
1527                 }
1528         }
1529 
1530         if (intr_params_p->avgi_req_flags & PSMGI_REQ_VECTOR)
1531                 intr_params_p->avgi_vector = vecp->v_vector;
1532 
1533         if (intr_params_p->avgi_req_flags &
1534             (PSMGI_REQ_NUM_DEVS | PSMGI_REQ_GET_DEVS))
1535                 /* Get number of devices from apic_irq table shared field. */
1536                 intr_params_p->avgi_num_devs = vecp->v_share;
1537 
1538         if (intr_params_p->avgi_req_flags &  PSMGI_REQ_GET_DEVS) {
1539 
1540                 intr_params_p->avgi_req_flags  |= PSMGI_REQ_NUM_DEVS;
1541 
1542                 /* Some devices have NULL dip.  Don't count these. */
1543                 if (intr_params_p->avgi_num_devs > 0) {
1544                         for (i = 0, av_dev = vecp->v_autovect; av_dev;
1545                             av_dev = av_dev->av_link) {
1546                                 if (av_dev->av_vector && av_dev->av_dip)
1547                                         i++;
1548                         }
1549                         intr_params_p->avgi_num_devs =
1550                             (uint8_t)MIN(intr_params_p->avgi_num_devs, i);
1551                 }
1552 
1553                 /* There are no viable dips to return. */
1554                 if (intr_params_p->avgi_num_devs == 0) {
1555                         intr_params_p->avgi_dip_list = NULL;
1556 
1557                 } else {        /* Return list of dips */
1558 
1559                         /* Allocate space in array for that number of devs. */
1560                         intr_params_p->avgi_dip_list = kmem_zalloc(
1561                             intr_params_p->avgi_num_devs *
1562                             sizeof (dev_info_t *),
1563                             KM_NOSLEEP);
1564                         if (intr_params_p->avgi_dip_list == NULL) {
1565                                 DDI_INTR_IMPLDBG((CE_WARN,
1566                                     "apix_get_vector_intr_info: no memory"));
1567                                 return (PSM_FAILURE);
1568                         }
1569 
1570                         /*
1571                          * Loop through the device list of the autovec table
1572                          * filling in the dip array.
1573                          *
1574                          * Note that the autovect table may have some special
1575                          * entries which contain NULL dips.  These will be
1576                          * ignored.
1577                          */
1578                         for (i = 0, av_dev = vecp->v_autovect; av_dev;
1579                             av_dev = av_dev->av_link) {
1580                                 if (av_dev->av_vector && av_dev->av_dip)
1581                                         intr_params_p->avgi_dip_list[i++] =
1582                                             av_dev->av_dip;
1583                         }
1584                 }
1585         }
1586 
1587         return (PSM_SUCCESS);
1588 }
1589 
1590 static char *
1591 apix_get_apic_type(void)
1592 {
1593         return (apix_psm_info.p_mach_idstring);
1594 }
1595 
1596 apix_vector_t *
1597 apix_set_cpu(apix_vector_t *vecp, int new_cpu, int *result)
1598 {
1599         apix_vector_t *newp = NULL;
1600         dev_info_t *dip;
1601         int inum, cap_ptr;
1602         ddi_acc_handle_t handle;
1603         ddi_intr_msix_t *msix_p = NULL;
1604         ushort_t msix_ctrl;
1605         uintptr_t off;
1606         uint32_t mask;
1607 
1608         ASSERT(LOCK_HELD(&apix_lock));
1609         *result = ENXIO;
1610 
1611         /* Fail if this is an MSI intr and is part of a group. */
1612         if (vecp->v_type == APIX_TYPE_MSI) {
1613                 if (i_ddi_intr_get_current_nintrs(APIX_GET_DIP(vecp)) > 1)
1614                         return (NULL);
1615                 else
1616                         return (apix_grp_set_cpu(vecp, new_cpu, result));
1617         }
1618 
1619         /*
1620          * Mask MSI-X. It's unmasked when MSI-X gets enabled.
1621          */
1622         if (vecp->v_type == APIX_TYPE_MSIX && IS_VECT_ENABLED(vecp)) {
1623                 if ((dip = APIX_GET_DIP(vecp)) == NULL)
1624                         return (NULL);
1625                 inum = vecp->v_devp->dv_inum;
1626 
1627                 handle = i_ddi_get_pci_config_handle(dip);
1628                 cap_ptr = i_ddi_get_msi_msix_cap_ptr(dip);
1629                 msix_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSIX_CTRL);
1630                 if ((msix_ctrl & PCI_MSIX_FUNCTION_MASK) == 0) {
1631                         /*
1632                          * Function is not masked, then mask "inum"th
1633                          * entry in the MSI-X table
1634                          */
1635                         msix_p = i_ddi_get_msix(dip);
1636                         off = (uintptr_t)msix_p->msix_tbl_addr + (inum *
1637                             PCI_MSIX_VECTOR_SIZE) + PCI_MSIX_VECTOR_CTRL_OFFSET;
1638                         mask = ddi_get32(msix_p->msix_tbl_hdl, (uint32_t *)off);
1639                         ddi_put32(msix_p->msix_tbl_hdl, (uint32_t *)off,
1640                             mask | 1);
1641                 }
1642         }
1643 
1644         *result = 0;
1645         if ((newp = apix_rebind(vecp, new_cpu, 1)) == NULL)
1646                 *result = EIO;
1647 
1648         /* Restore mask bit */
1649         if (msix_p != NULL)
1650                 ddi_put32(msix_p->msix_tbl_hdl, (uint32_t *)off, mask);
1651 
1652         return (newp);
1653 }
1654 
1655 /*
1656  * Set cpu for MSIs
1657  */
1658 apix_vector_t *
1659 apix_grp_set_cpu(apix_vector_t *vecp, int new_cpu, int *result)
1660 {
1661         apix_vector_t *newp, *vp;
1662         uint32_t orig_cpu = vecp->v_cpuid;
1663         int orig_vect = vecp->v_vector;
1664         int i, num_vectors, cap_ptr, msi_mask_off;
1665         uint32_t msi_pvm;
1666         ushort_t msi_ctrl;
1667         ddi_acc_handle_t handle;
1668         dev_info_t *dip;
1669 
1670         APIC_VERBOSE(INTR, (CE_CONT, "apix_grp_set_cpu: oldcpu: %x, vector: %x,"
1671             " newcpu:%x\n", vecp->v_cpuid, vecp->v_vector, new_cpu));
1672 
1673         ASSERT(LOCK_HELD(&apix_lock));
1674 
1675         *result = ENXIO;
1676 
1677         if (vecp->v_type != APIX_TYPE_MSI) {
1678                 DDI_INTR_IMPLDBG((CE_WARN, "set_grp: intr not MSI\n"));
1679                 return (NULL);
1680         }
1681 
1682         if ((dip = APIX_GET_DIP(vecp)) == NULL)
1683                 return (NULL);
1684 
1685         num_vectors = i_ddi_intr_get_current_nintrs(dip);
1686         if ((num_vectors < 1) || ((num_vectors - 1) & orig_vect)) {
1687                 APIC_VERBOSE(INTR, (CE_WARN,
1688                     "set_grp: base vec not part of a grp or not aligned: "
1689                     "vec:0x%x, num_vec:0x%x\n", orig_vect, num_vectors));
1690                 return (NULL);
1691         }
1692 
1693         if (vecp->v_inum != apix_get_min_dev_inum(dip, vecp->v_type))
1694                 return (NULL);
1695 
1696         *result = EIO;
1697         for (i = 1; i < num_vectors; i++) {
1698                 if ((vp = xv_vector(orig_cpu, orig_vect + i)) == NULL)
1699                         return (NULL);
1700 #ifdef DEBUG
1701                 /*
1702                  * Sanity check: CPU and dip is the same for all entries.
1703                  * May be called when first msi to be enabled, at this time
1704                  * add_avintr() is not called for other msi
1705                  */
1706                 if ((vp->v_share != 0) &&
1707                     ((APIX_GET_DIP(vp) != dip) ||
1708                     (vp->v_cpuid != vecp->v_cpuid))) {
1709                         APIC_VERBOSE(INTR, (CE_WARN,
1710                             "set_grp: cpu or dip for vec 0x%x difft than for "
1711                             "vec 0x%x\n", orig_vect, orig_vect + i));
1712                         APIC_VERBOSE(INTR, (CE_WARN,
1713                             "  cpu: %d vs %d, dip: 0x%p vs 0x%p\n", orig_cpu,
1714                             vp->v_cpuid, (void *)dip,
1715                             (void *)APIX_GET_DIP(vp)));
1716                         return (NULL);
1717                 }
1718 #endif /* DEBUG */
1719         }
1720 
1721         cap_ptr = i_ddi_get_msi_msix_cap_ptr(dip);
1722         handle = i_ddi_get_pci_config_handle(dip);
1723         msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSI_CTRL);
1724 
1725         /* MSI Per vector masking is supported. */
1726         if (msi_ctrl & PCI_MSI_PVM_MASK) {
1727                 if (msi_ctrl &  PCI_MSI_64BIT_MASK)
1728                         msi_mask_off = cap_ptr + PCI_MSI_64BIT_MASKBITS;
1729                 else
1730                         msi_mask_off = cap_ptr + PCI_MSI_32BIT_MASK;
1731                 msi_pvm = pci_config_get32(handle, msi_mask_off);
1732                 pci_config_put32(handle, msi_mask_off, (uint32_t)-1);
1733                 APIC_VERBOSE(INTR, (CE_CONT,
1734                     "set_grp: pvm supported.  Mask set to 0x%x\n",
1735                     pci_config_get32(handle, msi_mask_off)));
1736         }
1737 
1738         if ((newp = apix_rebind(vecp, new_cpu, num_vectors)) != NULL)
1739                 *result = 0;
1740 
1741         /* Reenable vectors if per vector masking is supported. */
1742         if (msi_ctrl & PCI_MSI_PVM_MASK) {
1743                 pci_config_put32(handle, msi_mask_off, msi_pvm);
1744                 APIC_VERBOSE(INTR, (CE_CONT,
1745                     "set_grp: pvm supported.  Mask restored to 0x%x\n",
1746                     pci_config_get32(handle, msi_mask_off)));
1747         }
1748 
1749         return (newp);
1750 }
1751 
1752 void
1753 apix_intx_set_vector(int irqno, uint32_t cpuid, uchar_t vector)
1754 {
1755         apic_irq_t *irqp;
1756 
1757         mutex_enter(&airq_mutex);
1758         irqp = apic_irq_table[irqno];
1759         irqp->airq_cpu = cpuid;
1760         irqp->airq_vector = vector;
1761         apic_record_rdt_entry(irqp, irqno);
1762         mutex_exit(&airq_mutex);
1763 }
1764 
1765 apix_vector_t *
1766 apix_intx_get_vector(int irqno)
1767 {
1768         apic_irq_t *irqp;
1769         uint32_t cpuid;
1770         uchar_t vector;
1771 
1772         mutex_enter(&airq_mutex);
1773         irqp = apic_irq_table[irqno & 0xff];
1774         if (IS_IRQ_FREE(irqp) || (irqp->airq_cpu == IRQ_UNINIT)) {
1775                 mutex_exit(&airq_mutex);
1776                 return (NULL);
1777         }
1778         cpuid = irqp->airq_cpu;
1779         vector = irqp->airq_vector;
1780         mutex_exit(&airq_mutex);
1781 
1782         return (xv_vector(cpuid, vector));
1783 }
1784 
1785 /*
1786  * Must called with interrupts disabled and apic_ioapic_lock held
1787  */
1788 void
1789 apix_intx_enable(int irqno)
1790 {
1791         uchar_t ioapicindex, intin;
1792         apic_irq_t *irqp = apic_irq_table[irqno];
1793         ioapic_rdt_t irdt;
1794         apic_cpus_info_t *cpu_infop;
1795         apix_vector_t *vecp = xv_vector(irqp->airq_cpu, irqp->airq_vector);
1796 
1797         ASSERT(LOCK_HELD(&apic_ioapic_lock) && !IS_IRQ_FREE(irqp));
1798 
1799         ioapicindex = irqp->airq_ioapicindex;
1800         intin = irqp->airq_intin_no;
1801         cpu_infop =  &apic_cpus[irqp->airq_cpu];
1802 
1803         irdt.ir_lo = AV_PDEST | AV_FIXED | irqp->airq_rdt_entry;
1804         irdt.ir_hi = cpu_infop->aci_local_id;
1805 
1806         apic_vt_ops->apic_intrmap_alloc_entry(&vecp->v_intrmap_private, NULL,
1807             vecp->v_type, 1, ioapicindex);
1808         apic_vt_ops->apic_intrmap_map_entry(vecp->v_intrmap_private,
1809             (void *)&irdt, vecp->v_type, 1);
1810         apic_vt_ops->apic_intrmap_record_rdt(vecp->v_intrmap_private, &irdt);
1811 
1812         /* write RDT entry high dword - destination */
1813         WRITE_IOAPIC_RDT_ENTRY_HIGH_DWORD(ioapicindex, intin,
1814             irdt.ir_hi);
1815 
1816         /* Write the vector, trigger, and polarity portion of the RDT */
1817         WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapicindex, intin, irdt.ir_lo);
1818 
1819         vecp->v_state = APIX_STATE_ENABLED;
1820 
1821         APIC_VERBOSE_IOAPIC((CE_CONT, "apix_intx_enable: ioapic 0x%x"
1822             " intin 0x%x rdt_low 0x%x rdt_high 0x%x\n",
1823             ioapicindex, intin, irdt.ir_lo, irdt.ir_hi));
1824 }
1825 
1826 /*
1827  * Must called with interrupts disabled and apic_ioapic_lock held
1828  */
1829 void
1830 apix_intx_disable(int irqno)
1831 {
1832         apic_irq_t *irqp = apic_irq_table[irqno];
1833         int ioapicindex, intin;
1834 
1835         ASSERT(LOCK_HELD(&apic_ioapic_lock) && !IS_IRQ_FREE(irqp));
1836         /*
1837          * The assumption here is that this is safe, even for
1838          * systems with IOAPICs that suffer from the hardware
1839          * erratum because all devices have been quiesced before
1840          * they unregister their interrupt handlers.  If that
1841          * assumption turns out to be false, this mask operation
1842          * can induce the same erratum result we're trying to
1843          * avoid.
1844          */
1845         ioapicindex = irqp->airq_ioapicindex;
1846         intin = irqp->airq_intin_no;
1847         ioapic_write(ioapicindex, APIC_RDT_CMD + 2 * intin, AV_MASK);
1848 
1849         APIC_VERBOSE_IOAPIC((CE_CONT, "apix_intx_disable: ioapic 0x%x"
1850             " intin 0x%x\n", ioapicindex, intin));
1851 }
1852 
1853 void
1854 apix_intx_free(int irqno)
1855 {
1856         apic_irq_t *irqp;
1857 
1858         mutex_enter(&airq_mutex);
1859         irqp = apic_irq_table[irqno];
1860 
1861         if (IS_IRQ_FREE(irqp)) {
1862                 mutex_exit(&airq_mutex);
1863                 return;
1864         }
1865 
1866         irqp->airq_mps_intr_index = FREE_INDEX;
1867         irqp->airq_cpu = IRQ_UNINIT;
1868         irqp->airq_vector = APIX_INVALID_VECT;
1869         mutex_exit(&airq_mutex);
1870 }
1871 
1872 #ifdef DEBUG
1873 int apix_intr_deliver_timeouts = 0;
1874 int apix_intr_rirr_timeouts = 0;
1875 int apix_intr_rirr_reset_failure = 0;
1876 #endif
1877 int apix_max_reps_irr_pending = 10;
1878 
1879 #define GET_RDT_BITS(ioapic, intin, bits)       \
1880         (READ_IOAPIC_RDT_ENTRY_LOW_DWORD((ioapic), (intin)) & (bits))
1881 #define APIX_CHECK_IRR_DELAY    drv_usectohz(5000)
1882 
1883 int
1884 apix_intx_rebind(int irqno, processorid_t cpuid, uchar_t vector)
1885 {
1886         apic_irq_t *irqp = apic_irq_table[irqno];
1887         ulong_t iflag;
1888         int waited, ioapic_ix, intin_no, level, repeats, rdt_entry, masked;
1889 
1890         ASSERT(irqp != NULL);
1891 
1892         iflag = intr_clear();
1893         lock_set(&apic_ioapic_lock);
1894 
1895         ioapic_ix = irqp->airq_ioapicindex;
1896         intin_no = irqp->airq_intin_no;
1897         level = apic_level_intr[irqno];
1898 
1899         /*
1900          * Wait for the delivery status bit to be cleared. This should
1901          * be a very small amount of time.
1902          */
1903         repeats = 0;
1904         do {
1905                 repeats++;
1906 
1907                 for (waited = 0; waited < apic_max_reps_clear_pending;
1908                     waited++) {
1909                         if (GET_RDT_BITS(ioapic_ix, intin_no, AV_PENDING) == 0)
1910                                 break;
1911                 }
1912                 if (!level)
1913                         break;
1914 
1915                 /*
1916                  * Mask the RDT entry for level-triggered interrupts.
1917                  */
1918                 irqp->airq_rdt_entry |= AV_MASK;
1919                 rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1920                     intin_no);
1921                 if ((masked = (rdt_entry & AV_MASK)) == 0) {
1922                         /* Mask it */
1923                         WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, intin_no,
1924                             AV_MASK | rdt_entry);
1925                 }
1926 
1927                 /*
1928                  * If there was a race and an interrupt was injected
1929                  * just before we masked, check for that case here.
1930                  * Then, unmask the RDT entry and try again.  If we're
1931                  * on our last try, don't unmask (because we want the
1932                  * RDT entry to remain masked for the rest of the
1933                  * function).
1934                  */
1935                 rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1936                     intin_no);
1937                 if ((masked == 0) && ((rdt_entry & AV_PENDING) != 0) &&
1938                     (repeats < apic_max_reps_clear_pending)) {
1939                         /* Unmask it */
1940                         WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
1941                             intin_no, rdt_entry & ~AV_MASK);
1942                         irqp->airq_rdt_entry &= ~AV_MASK;
1943                 }
1944         } while ((rdt_entry & AV_PENDING) &&
1945             (repeats < apic_max_reps_clear_pending));
1946 
1947 #ifdef DEBUG
1948         if (GET_RDT_BITS(ioapic_ix, intin_no, AV_PENDING) != 0)
1949                 apix_intr_deliver_timeouts++;
1950 #endif
1951 
1952         if (!level || !APIX_IS_MASK_RDT(apix_mul_ioapic_method))
1953                 goto done;
1954 
1955         /*
1956          * wait for remote IRR to be cleared for level-triggered
1957          * interrupts
1958          */
1959         repeats = 0;
1960         do {
1961                 repeats++;
1962 
1963                 for (waited = 0; waited < apic_max_reps_clear_pending;
1964                     waited++) {
1965                         if (GET_RDT_BITS(ioapic_ix, intin_no, AV_REMOTE_IRR)
1966                             == 0)
1967                                 break;
1968                 }
1969 
1970                 if (GET_RDT_BITS(ioapic_ix, intin_no, AV_REMOTE_IRR) != 0) {
1971                         lock_clear(&apic_ioapic_lock);
1972                         intr_restore(iflag);
1973 
1974                         delay(APIX_CHECK_IRR_DELAY);
1975 
1976                         iflag = intr_clear();
1977                         lock_set(&apic_ioapic_lock);
1978                 }
1979         } while (repeats < apix_max_reps_irr_pending);
1980 
1981         if (repeats >= apix_max_reps_irr_pending) {
1982 #ifdef DEBUG
1983                 apix_intr_rirr_timeouts++;
1984 #endif
1985 
1986                 /*
1987                  * If we waited and the Remote IRR bit is still not cleared,
1988                  * AND if we've invoked the timeout APIC_REPROGRAM_MAX_TIMEOUTS
1989                  * times for this interrupt, try the last-ditch workaround:
1990                  */
1991                 if (GET_RDT_BITS(ioapic_ix, intin_no, AV_REMOTE_IRR) != 0) {
1992                         /*
1993                          * Trying to clear the bit through normal
1994                          * channels has failed.  So as a last-ditch
1995                          * effort, try to set the trigger mode to
1996                          * edge, then to level.  This has been
1997                          * observed to work on many systems.
1998                          */
1999                         WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
2000                             intin_no,
2001                             READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
2002                             intin_no) & ~AV_LEVEL);
2003                         WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
2004                             intin_no,
2005                             READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix,
2006                             intin_no) | AV_LEVEL);
2007                 }
2008 
2009                 if (GET_RDT_BITS(ioapic_ix, intin_no, AV_REMOTE_IRR) != 0) {
2010 #ifdef DEBUG
2011                         apix_intr_rirr_reset_failure++;
2012 #endif
2013                         lock_clear(&apic_ioapic_lock);
2014                         intr_restore(iflag);
2015                         prom_printf("apix: Remote IRR still "
2016                             "not clear for IOAPIC %d intin %d.\n"
2017                             "\tInterrupts to this pin may cease "
2018                             "functioning.\n", ioapic_ix, intin_no);
2019                         return (1);     /* return failure */
2020                 }
2021         }
2022 
2023 done:
2024         /* change apic_irq_table */
2025         lock_clear(&apic_ioapic_lock);
2026         intr_restore(iflag);
2027         apix_intx_set_vector(irqno, cpuid, vector);
2028         iflag = intr_clear();
2029         lock_set(&apic_ioapic_lock);
2030 
2031         /* reprogramme IO-APIC RDT entry */
2032         apix_intx_enable(irqno);
2033 
2034         lock_clear(&apic_ioapic_lock);
2035         intr_restore(iflag);
2036 
2037         return (0);
2038 }
2039 
2040 static int
2041 apix_intx_get_pending(int irqno)
2042 {
2043         apic_irq_t *irqp;
2044         int intin, ioapicindex, pending;
2045         ulong_t iflag;
2046 
2047         mutex_enter(&airq_mutex);
2048         irqp = apic_irq_table[irqno];
2049         if (IS_IRQ_FREE(irqp)) {
2050                 mutex_exit(&airq_mutex);
2051                 return (0);
2052         }
2053 
2054         /* check IO-APIC delivery status */
2055         intin = irqp->airq_intin_no;
2056         ioapicindex = irqp->airq_ioapicindex;
2057         mutex_exit(&airq_mutex);
2058 
2059         iflag = intr_clear();
2060         lock_set(&apic_ioapic_lock);
2061 
2062         pending = (READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapicindex, intin) &
2063             AV_PENDING) ? 1 : 0;
2064 
2065         lock_clear(&apic_ioapic_lock);
2066         intr_restore(iflag);
2067 
2068         return (pending);
2069 }
2070 
2071 /*
2072  * This function will mask the interrupt on the I/O APIC
2073  */
2074 static void
2075 apix_intx_set_mask(int irqno)
2076 {
2077         int intin, ioapixindex, rdt_entry;
2078         ulong_t iflag;
2079         apic_irq_t *irqp;
2080 
2081         mutex_enter(&airq_mutex);
2082         irqp = apic_irq_table[irqno];
2083 
2084         ASSERT(irqp->airq_mps_intr_index != FREE_INDEX);
2085 
2086         intin = irqp->airq_intin_no;
2087         ioapixindex = irqp->airq_ioapicindex;
2088         mutex_exit(&airq_mutex);
2089 
2090         iflag = intr_clear();
2091         lock_set(&apic_ioapic_lock);
2092 
2093         rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapixindex, intin);
2094 
2095         /* clear mask */
2096         WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapixindex, intin,
2097             (AV_MASK | rdt_entry));
2098 
2099         lock_clear(&apic_ioapic_lock);
2100         intr_restore(iflag);
2101 }
2102 
2103 /*
2104  * This function will clear the mask for the interrupt on the I/O APIC
2105  */
2106 static void
2107 apix_intx_clear_mask(int irqno)
2108 {
2109         int intin, ioapixindex, rdt_entry;
2110         ulong_t iflag;
2111         apic_irq_t *irqp;
2112 
2113         mutex_enter(&airq_mutex);
2114         irqp = apic_irq_table[irqno];
2115 
2116         ASSERT(irqp->airq_mps_intr_index != FREE_INDEX);
2117 
2118         intin = irqp->airq_intin_no;
2119         ioapixindex = irqp->airq_ioapicindex;
2120         mutex_exit(&airq_mutex);
2121 
2122         iflag = intr_clear();
2123         lock_set(&apic_ioapic_lock);
2124 
2125         rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapixindex, intin);
2126 
2127         /* clear mask */
2128         WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapixindex, intin,
2129             ((~AV_MASK) & rdt_entry));
2130 
2131         lock_clear(&apic_ioapic_lock);
2132         intr_restore(iflag);
2133 }
2134 
2135 /*
2136  * For level-triggered interrupt, mask the IRQ line. Mask means
2137  * new interrupts will not be delivered. The interrupt already
2138  * accepted by a local APIC is not affected
2139  */
2140 void
2141 apix_level_intr_pre_eoi(int irq)
2142 {
2143         apic_irq_t *irqp = apic_irq_table[irq];
2144         int apic_ix, intin_ix;
2145 
2146         if (irqp == NULL)
2147                 return;
2148 
2149         ASSERT(apic_level_intr[irq] == TRIGGER_MODE_LEVEL);
2150 
2151         lock_set(&apic_ioapic_lock);
2152 
2153         intin_ix = irqp->airq_intin_no;
2154         apic_ix = irqp->airq_ioapicindex;
2155 
2156         if (irqp->airq_cpu != CPU->cpu_id) {
2157                 if (!APIX_IS_MASK_RDT(apix_mul_ioapic_method))
2158                         ioapic_write_eoi(apic_ix, irqp->airq_vector);
2159                 lock_clear(&apic_ioapic_lock);
2160                 return;
2161         }
2162 
2163         if (apix_mul_ioapic_method == APIC_MUL_IOAPIC_IOXAPIC) {
2164                 /*
2165                  * This is a IOxAPIC and there is EOI register:
2166                  *      Change the vector to reserved unused vector, so that
2167                  *      the EOI from Local APIC won't clear the Remote IRR for
2168                  *      this level trigger interrupt. Instead, we'll manually
2169                  *      clear it in apix_post_hardint() after ISR handling.
2170                  */
2171                 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(apic_ix, intin_ix,
2172                     (irqp->airq_rdt_entry & (~0xff)) | APIX_RESV_VECTOR);
2173         } else {
2174                 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(apic_ix, intin_ix,
2175                     AV_MASK | irqp->airq_rdt_entry);
2176         }
2177 
2178         lock_clear(&apic_ioapic_lock);
2179 }
2180 
2181 /*
2182  * For level-triggered interrupt, unmask the IRQ line
2183  * or restore the original vector number.
2184  */
2185 void
2186 apix_level_intr_post_dispatch(int irq)
2187 {
2188         apic_irq_t *irqp = apic_irq_table[irq];
2189         int apic_ix, intin_ix;
2190 
2191         if (irqp == NULL)
2192                 return;
2193 
2194         lock_set(&apic_ioapic_lock);
2195 
2196         intin_ix = irqp->airq_intin_no;
2197         apic_ix = irqp->airq_ioapicindex;
2198 
2199         if (APIX_IS_DIRECTED_EOI(apix_mul_ioapic_method)) {
2200                 /*
2201                  * Already sent EOI back to Local APIC.
2202                  * Send EOI to IO-APIC
2203                  */
2204                 ioapic_write_eoi(apic_ix, irqp->airq_vector);
2205         } else {
2206                 /* clear the mask or restore the vector */
2207                 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(apic_ix, intin_ix,
2208                     irqp->airq_rdt_entry);
2209 
2210                 /* send EOI to IOxAPIC */
2211                 if (apix_mul_ioapic_method == APIC_MUL_IOAPIC_IOXAPIC)
2212                         ioapic_write_eoi(apic_ix, irqp->airq_vector);
2213         }
2214 
2215         lock_clear(&apic_ioapic_lock);
2216 }
2217 
2218 static int
2219 apix_intx_get_shared(int irqno)
2220 {
2221         apic_irq_t *irqp;
2222         int share;
2223 
2224         mutex_enter(&airq_mutex);
2225         irqp = apic_irq_table[irqno];
2226         if (IS_IRQ_FREE(irqp) || (irqp->airq_cpu == IRQ_UNINIT)) {
2227                 mutex_exit(&airq_mutex);
2228                 return (0);
2229         }
2230         share = irqp->airq_share;
2231         mutex_exit(&airq_mutex);
2232 
2233         return (share);
2234 }
2235 
2236 static void
2237 apix_intx_set_shared(int irqno, int delta)
2238 {
2239         apic_irq_t *irqp;
2240 
2241         mutex_enter(&airq_mutex);
2242         irqp = apic_irq_table[irqno];
2243         if (IS_IRQ_FREE(irqp)) {
2244                 mutex_exit(&airq_mutex);
2245                 return;
2246         }
2247         irqp->airq_share += delta;
2248         mutex_exit(&airq_mutex);
2249 }
2250 
2251 /*
2252  * Setup IRQ table. Return IRQ no or -1 on failure
2253  */
2254 static int
2255 apix_intx_setup(dev_info_t *dip, int inum, int irqno,
2256     struct apic_io_intr *intrp, struct intrspec *ispec, iflag_t *iflagp)
2257 {
2258         int origirq = ispec->intrspec_vec;
2259         int newirq;
2260         short intr_index;
2261         uchar_t ipin, ioapic, ioapicindex;
2262         apic_irq_t *irqp;
2263 
2264         UNREFERENCED_1PARAMETER(inum);
2265 
2266         if (intrp != NULL) {
2267                 intr_index = (short)(intrp - apic_io_intrp);
2268                 ioapic = intrp->intr_destid;
2269                 ipin = intrp->intr_destintin;
2270 
2271                 /* Find ioapicindex. If destid was ALL, we will exit with 0. */
2272                 for (ioapicindex = apic_io_max - 1; ioapicindex; ioapicindex--)
2273                         if (apic_io_id[ioapicindex] == ioapic)
2274                                 break;
2275                 ASSERT((ioapic == apic_io_id[ioapicindex]) ||
2276                     (ioapic == INTR_ALL_APIC));
2277 
2278                 /* check whether this intin# has been used by another irqno */
2279                 if ((newirq = apic_find_intin(ioapicindex, ipin)) != -1)
2280                         return (newirq);
2281 
2282         } else if (iflagp != NULL) {    /* ACPI */
2283                 intr_index = ACPI_INDEX;
2284                 ioapicindex = acpi_find_ioapic(irqno);
2285                 ASSERT(ioapicindex != 0xFF);
2286                 ioapic = apic_io_id[ioapicindex];
2287                 ipin = irqno - apic_io_vectbase[ioapicindex];
2288 
2289                 if (apic_irq_table[irqno] &&
2290                     apic_irq_table[irqno]->airq_mps_intr_index == ACPI_INDEX) {
2291                         ASSERT(apic_irq_table[irqno]->airq_intin_no == ipin &&
2292                             apic_irq_table[irqno]->airq_ioapicindex ==
2293                             ioapicindex);
2294                         return (irqno);
2295                 }
2296 
2297         } else {        /* default configuration */
2298                 intr_index = DEFAULT_INDEX;
2299                 ioapicindex = 0;
2300                 ioapic = apic_io_id[ioapicindex];
2301                 ipin = (uchar_t)irqno;
2302         }
2303 
2304         /* allocate a new IRQ no */
2305         if ((irqp = apic_irq_table[irqno]) == NULL) {
2306                 irqp = kmem_zalloc(sizeof (apic_irq_t), KM_SLEEP);
2307                 apic_irq_table[irqno] = irqp;
2308         } else {
2309                 if (irqp->airq_mps_intr_index != FREE_INDEX) {
2310                         newirq = apic_allocate_irq(apic_first_avail_irq);
2311                         if (newirq == -1) {
2312                                 return (-1);
2313                         }
2314                         irqno = newirq;
2315                         irqp = apic_irq_table[irqno];
2316                         ASSERT(irqp != NULL);
2317                 }
2318         }
2319         apic_max_device_irq = max(irqno, apic_max_device_irq);
2320         apic_min_device_irq = min(irqno, apic_min_device_irq);
2321 
2322         irqp->airq_mps_intr_index = intr_index;
2323         irqp->airq_ioapicindex = ioapicindex;
2324         irqp->airq_intin_no = ipin;
2325         irqp->airq_dip = dip;
2326         irqp->airq_origirq = (uchar_t)origirq;
2327         if (iflagp != NULL)
2328                 irqp->airq_iflag = *iflagp;
2329         irqp->airq_cpu = IRQ_UNINIT;
2330         irqp->airq_vector = 0;
2331 
2332         return (irqno);
2333 }
2334 
2335 /*
2336  * Setup IRQ table for non-pci devices. Return IRQ no or -1 on error
2337  */
2338 static int
2339 apix_intx_setup_nonpci(dev_info_t *dip, int inum, int bustype,
2340     struct intrspec *ispec)
2341 {
2342         int irqno = ispec->intrspec_vec;
2343         int newirq, i;
2344         iflag_t intr_flag;
2345         ACPI_SUBTABLE_HEADER    *hp;
2346         ACPI_MADT_INTERRUPT_OVERRIDE *isop;
2347         struct apic_io_intr *intrp;
2348 
2349         if (!apic_enable_acpi || apic_use_acpi_madt_only) {
2350                 int busid;
2351 
2352                 if (bustype == 0)
2353                         bustype = eisa_level_intr_mask ? BUS_EISA : BUS_ISA;
2354 
2355                 /* loop checking BUS_ISA/BUS_EISA */
2356                 for (i = 0; i < 2; i++) {
2357                         if (((busid = apic_find_bus_id(bustype)) != -1) &&
2358                             ((intrp = apic_find_io_intr_w_busid(irqno, busid))
2359                             != NULL)) {
2360                                 return (apix_intx_setup(dip, inum, irqno,
2361                                     intrp, ispec, NULL));
2362                         }
2363                         bustype = (bustype == BUS_EISA) ? BUS_ISA : BUS_EISA;
2364                 }
2365 
2366                 /* fall back to default configuration */
2367                 return (-1);
2368         }
2369 
2370         /* search iso entries first */
2371         if (acpi_iso_cnt != 0) {
2372                 hp = (ACPI_SUBTABLE_HEADER *)acpi_isop;
2373                 i = 0;
2374                 while (i < acpi_iso_cnt) {
2375                         if (hp->Type == ACPI_MADT_TYPE_INTERRUPT_OVERRIDE) {
2376                                 isop = (ACPI_MADT_INTERRUPT_OVERRIDE *) hp;
2377                                 if (isop->Bus == 0 &&
2378                                     isop->SourceIrq == irqno) {
2379                                         newirq = isop->GlobalIrq;
2380                                         intr_flag.intr_po = isop->IntiFlags &
2381                                             ACPI_MADT_POLARITY_MASK;
2382                                         intr_flag.intr_el = (isop->IntiFlags &
2383                                             ACPI_MADT_TRIGGER_MASK) >> 2;
2384                                         intr_flag.bustype = BUS_ISA;
2385 
2386                                         return (apix_intx_setup(dip, inum,
2387                                             newirq, NULL, ispec, &intr_flag));
2388                                 }
2389                                 i++;
2390                         }
2391                         hp = (ACPI_SUBTABLE_HEADER *)(((char *)hp) +
2392                             hp->Length);
2393                 }
2394         }
2395         intr_flag.intr_po = INTR_PO_ACTIVE_HIGH;
2396         intr_flag.intr_el = INTR_EL_EDGE;
2397         intr_flag.bustype = BUS_ISA;
2398         return (apix_intx_setup(dip, inum, irqno, NULL, ispec, &intr_flag));
2399 }
2400 
2401 
2402 /*
2403  * Setup IRQ table for pci devices. Return IRQ no or -1 on error
2404  */
2405 static int
2406 apix_intx_setup_pci(dev_info_t *dip, int inum, int bustype,
2407     struct intrspec *ispec)
2408 {
2409         int busid, devid, pci_irq;
2410         ddi_acc_handle_t cfg_handle;
2411         uchar_t ipin;
2412         iflag_t intr_flag;
2413         struct apic_io_intr *intrp;
2414 
2415         if (acpica_get_bdf(dip, &busid, &devid, NULL) != 0)
2416                 return (-1);
2417 
2418         if (busid == 0 && apic_pci_bus_total == 1)
2419                 busid = (int)apic_single_pci_busid;
2420 
2421         if (pci_config_setup(dip, &cfg_handle) != DDI_SUCCESS)
2422                 return (-1);
2423         ipin = pci_config_get8(cfg_handle, PCI_CONF_IPIN) - PCI_INTA;
2424         pci_config_teardown(&cfg_handle);
2425 
2426         if (apic_enable_acpi && !apic_use_acpi_madt_only) {     /* ACPI */
2427                 if (apic_acpi_translate_pci_irq(dip, busid, devid,
2428                     ipin, &pci_irq, &intr_flag) != ACPI_PSM_SUCCESS)
2429                         return (-1);
2430 
2431                 intr_flag.bustype = (uchar_t)bustype;
2432                 return (apix_intx_setup(dip, inum, pci_irq, NULL, ispec,
2433                     &intr_flag));
2434         }
2435 
2436         /* MP configuration table */
2437         pci_irq = ((devid & 0x1f) << 2) | (ipin & 0x3);
2438         if ((intrp = apic_find_io_intr_w_busid(pci_irq, busid)) == NULL) {
2439                 pci_irq = apic_handle_pci_pci_bridge(dip, devid, ipin, &intrp);
2440                 if (pci_irq == -1)
2441                         return (-1);
2442         }
2443 
2444         return (apix_intx_setup(dip, inum, pci_irq, intrp, ispec, NULL));
2445 }
2446 
2447 /*
2448  * Translate and return IRQ no
2449  */
2450 static int
2451 apix_intx_xlate_irq(dev_info_t *dip, int inum, struct intrspec *ispec)
2452 {
2453         int newirq, irqno = ispec->intrspec_vec;
2454         int parent_is_pci_or_pciex = 0, child_is_pciex = 0;
2455         int bustype = 0, dev_len;
2456         char dev_type[16];
2457 
2458         if (apic_defconf) {
2459                 mutex_enter(&airq_mutex);
2460                 goto defconf;
2461         }
2462 
2463         if ((dip == NULL) || (!apic_irq_translate && !apic_enable_acpi)) {
2464                 mutex_enter(&airq_mutex);
2465                 goto nonpci;
2466         }
2467 
2468         /*
2469          * use ddi_getlongprop_buf() instead of ddi_prop_lookup_string()
2470          * to avoid extra buffer allocation.
2471          */
2472         dev_len = sizeof (dev_type);
2473         if (ddi_getlongprop_buf(DDI_DEV_T_ANY, ddi_get_parent(dip),
2474             DDI_PROP_DONTPASS, "device_type", (caddr_t)dev_type,
2475             &dev_len) == DDI_PROP_SUCCESS) {
2476                 if ((strcmp(dev_type, "pci") == 0) ||
2477                     (strcmp(dev_type, "pciex") == 0))
2478                         parent_is_pci_or_pciex = 1;
2479         }
2480 
2481         if (ddi_getlongprop_buf(DDI_DEV_T_ANY, dip,
2482             DDI_PROP_DONTPASS, "compatible", (caddr_t)dev_type,
2483             &dev_len) == DDI_PROP_SUCCESS) {
2484                 if (strstr(dev_type, "pciex"))
2485                         child_is_pciex = 1;
2486         }
2487 
2488         mutex_enter(&airq_mutex);
2489 
2490         if (parent_is_pci_or_pciex) {
2491                 bustype = child_is_pciex ? BUS_PCIE : BUS_PCI;
2492                 newirq = apix_intx_setup_pci(dip, inum, bustype, ispec);
2493                 if (newirq != -1)
2494                         goto done;
2495                 bustype = 0;
2496         } else if (strcmp(dev_type, "isa") == 0)
2497                 bustype = BUS_ISA;
2498         else if (strcmp(dev_type, "eisa") == 0)
2499                 bustype = BUS_EISA;
2500 
2501 nonpci:
2502         newirq = apix_intx_setup_nonpci(dip, inum, bustype, ispec);
2503         if (newirq != -1)
2504                 goto done;
2505 
2506 defconf:
2507         newirq = apix_intx_setup(dip, inum, irqno, NULL, ispec, NULL);
2508         if (newirq == -1) {
2509                 mutex_exit(&airq_mutex);
2510                 return (-1);
2511         }
2512 done:
2513         ASSERT(apic_irq_table[newirq]);
2514         mutex_exit(&airq_mutex);
2515         return (newirq);
2516 }
2517 
2518 static int
2519 apix_intx_alloc_vector(dev_info_t *dip, int inum, struct intrspec *ispec)
2520 {
2521         int irqno;
2522         apix_vector_t *vecp;
2523 
2524         if ((irqno = apix_intx_xlate_irq(dip, inum, ispec)) == -1)
2525                 return (0);
2526 
2527         if ((vecp = apix_alloc_intx(dip, inum, irqno)) == NULL)
2528                 return (0);
2529 
2530         DDI_INTR_IMPLDBG((CE_CONT, "apix_intx_alloc_vector: dip=0x%p name=%s "
2531             "irqno=0x%x cpuid=%d vector=0x%x\n",
2532             (void *)dip, ddi_driver_name(dip), irqno,
2533             vecp->v_cpuid, vecp->v_vector));
2534 
2535         return (1);
2536 }
2537 
2538 /*
2539  * Return the vector number if the translated IRQ for this device
2540  * has a vector mapping setup. If no IRQ setup exists or no vector is
2541  * allocated to it then return 0.
2542  */
2543 static apix_vector_t *
2544 apix_intx_xlate_vector(dev_info_t *dip, int inum, struct intrspec *ispec)
2545 {
2546         int irqno;
2547         apix_vector_t *vecp;
2548 
2549         /* get the IRQ number */
2550         if ((irqno = apix_intx_xlate_irq(dip, inum, ispec)) == -1)
2551                 return (NULL);
2552 
2553         /* get the vector number if a vector is allocated to this irqno */
2554         vecp = apix_intx_get_vector(irqno);
2555 
2556         return (vecp);
2557 }
2558 
2559 /* stub function */
2560 int
2561 apix_loaded(void)
2562 {
2563         return (apix_is_enabled);
2564 }