1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25 
  26 #include <sys/cpuvar.h>
  27 #include <sys/psm.h>
  28 #include <sys/archsystm.h>
  29 #include <sys/apic.h>
  30 #include <sys/sunddi.h>
  31 #include <sys/ddi_impldefs.h>
  32 #include <sys/mach_intr.h>
  33 #include <sys/sysmacros.h>
  34 #include <sys/trap.h>
  35 #include <sys/x86_archext.h>
  36 #include <sys/privregs.h>
  37 #include <sys/psm_common.h>
  38 
  39 /* Function prototypes of local apic and X2APIC */
  40 static uint64_t local_apic_read(uint32_t reg);
  41 static void local_apic_write(uint32_t reg, uint64_t value);
  42 static int get_local_apic_pri(void);
  43 static void local_apic_write_task_reg(uint64_t value);
  44 static void local_apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1);
  45 static uint64_t local_x2apic_read(uint32_t msr);
  46 static void local_x2apic_write(uint32_t msr, uint64_t value);
  47 static int get_local_x2apic_pri(void);
  48 static void local_x2apic_write_task_reg(uint64_t value);
  49 static void local_x2apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1);
  50 
  51 /*
  52  * According to the X2APIC specification:
  53  *
  54  *   xAPIC global enable    X2APIC enable         Description
  55  *   (IA32_APIC_BASE[11])   (IA32_APIC_BASE[10])
  56  * -----------------------------------------------------------
  57  *      0                       0       APIC is disabled
  58  *      0                       1       Invalid
  59  *      1                       0       APIC is enabled in xAPIC mode
  60  *      1                       1       APIC is enabled in X2APIC mode
  61  * -----------------------------------------------------------
  62  */
  63 int     x2apic_enable = 1;
  64 enum apic_mode apic_mode = LOCAL_APIC;  /* Default mode is Local APIC */
  65 
  66 /* Uses MMIO (Memory Mapped IO) */
  67 static apic_reg_ops_t local_apic_regs_ops = {
  68         local_apic_read,
  69         local_apic_write,
  70         get_local_apic_pri,
  71         local_apic_write_task_reg,
  72         local_apic_write_int_cmd,
  73         apic_send_EOI,
  74 };
  75 
  76 /* X2APIC : Uses RDMSR/WRMSR instructions to access APIC registers */
  77 static apic_reg_ops_t x2apic_regs_ops = {
  78         local_x2apic_read,
  79         local_x2apic_write,
  80         get_local_x2apic_pri,
  81         local_x2apic_write_task_reg,
  82         local_x2apic_write_int_cmd,
  83         apic_send_EOI,
  84 };
  85 
  86 int apic_have_32bit_cr8 = 0;
  87 
  88 /* The default ops is local APIC (Memory Mapped IO) */
  89 apic_reg_ops_t *apic_reg_ops = &local_apic_regs_ops;
  90 
  91 /*
  92  * APIC register ops related data sturctures and functions.
  93  */
  94 void apic_send_EOI();
  95 void apic_send_directed_EOI(uint32_t irq);
  96 
  97 #define X2APIC_CPUID_BIT        21
  98 #define X2APIC_ENABLE_BIT       10
  99 
 100 /*
 101  * Local APIC Implementation
 102  */
 103 static uint64_t
 104 local_apic_read(uint32_t reg)
 105 {
 106         return ((uint32_t)apicadr[reg]);
 107 }
 108 
 109 static void
 110 local_apic_write(uint32_t reg, uint64_t value)
 111 {
 112         apicadr[reg] = (uint32_t)value;
 113 }
 114 
 115 static int
 116 get_local_apic_pri(void)
 117 {
 118 #if defined(__amd64)
 119         return ((int)getcr8());
 120 #else
 121         if (apic_have_32bit_cr8)
 122                 return ((int)getcr8());
 123         return (apicadr[APIC_TASK_REG]);
 124 #endif
 125 }
 126 
 127 static void
 128 local_apic_write_task_reg(uint64_t value)
 129 {
 130 #if defined(__amd64)
 131         setcr8((ulong_t)(value >> APIC_IPL_SHIFT));
 132 #else
 133         if (apic_have_32bit_cr8)
 134                 setcr8((ulong_t)(value >> APIC_IPL_SHIFT));
 135         else
 136                 apicadr[APIC_TASK_REG] = (uint32_t)value;
 137 #endif
 138 }
 139 
 140 static void
 141 local_apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1)
 142 {
 143         apicadr[APIC_INT_CMD2] = cpu_id << APIC_ICR_ID_BIT_OFFSET;
 144         apicadr[APIC_INT_CMD1] = cmd1;
 145 }
 146 
 147 /*
 148  * X2APIC Implementation.
 149  */
 150 static uint64_t
 151 local_x2apic_read(uint32_t msr)
 152 {
 153         uint64_t i;
 154 
 155         i = (uint64_t)(rdmsr(REG_X2APIC_BASE_MSR + (msr >> 2)) & 0xffffffff);
 156         return (i);
 157 }
 158 
 159 static void
 160 local_x2apic_write(uint32_t msr, uint64_t value)
 161 {
 162         uint64_t tmp;
 163 
 164         if (msr != APIC_EOI_REG) {
 165                 tmp = rdmsr(REG_X2APIC_BASE_MSR + (msr >> 2));
 166                 tmp = (tmp & 0xffffffff00000000) | value;
 167         } else {
 168                 tmp = 0;
 169         }
 170 
 171         wrmsr((REG_X2APIC_BASE_MSR + (msr >> 2)), tmp);
 172 }
 173 
 174 static int
 175 get_local_x2apic_pri(void)
 176 {
 177         return (rdmsr(REG_X2APIC_BASE_MSR + (APIC_TASK_REG >> 2)));
 178 }
 179 
 180 static void
 181 local_x2apic_write_task_reg(uint64_t value)
 182 {
 183         X2APIC_WRITE(APIC_TASK_REG, value);
 184 }
 185 
 186 static void
 187 local_x2apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1)
 188 {
 189         wrmsr((REG_X2APIC_BASE_MSR + (APIC_INT_CMD1 >> 2)),
 190             (((uint64_t)cpu_id << 32) | cmd1));
 191 }
 192 
 193 /*ARGSUSED*/
 194 void
 195 apic_send_EOI(uint32_t irq)
 196 {
 197         apic_reg_ops->apic_write(APIC_EOI_REG, 0);
 198 }
 199 
 200 /*
 201  * Support for Directed EOI capability is available in both the xAPIC
 202  * and x2APIC mode.
 203  */
 204 void
 205 apic_send_directed_EOI(uint32_t irq)
 206 {
 207         uchar_t ioapicindex;
 208         uchar_t vector;
 209         apic_irq_t *apic_irq;
 210         short intr_index;
 211 
 212         /*
 213          * Following the EOI to the local APIC unit, perform a directed
 214          * EOI to the IOxAPIC generating the interrupt by writing to its
 215          * EOI register.
 216          *
 217          * A broadcast EOI is not generated.
 218          */
 219         apic_reg_ops->apic_write(APIC_EOI_REG, 0);
 220 
 221         apic_irq = apic_irq_table[irq];
 222         while (apic_irq) {
 223                 intr_index = apic_irq->airq_mps_intr_index;
 224                 if (intr_index == ACPI_INDEX || intr_index >= 0) {
 225                         ioapicindex = apic_irq->airq_ioapicindex;
 226                         vector = apic_irq->airq_vector;
 227                         ioapic_write_eoi(ioapicindex, vector);
 228                 }
 229                 apic_irq = apic_irq->airq_next;
 230         }
 231 }
 232 
 233 int
 234 apic_detect_x2apic(void)
 235 {
 236         struct cpuid_regs cp;
 237 
 238         if (x2apic_enable == 0)
 239                 return (0);
 240 
 241         cp.cp_eax = 1;
 242         (void) __cpuid_insn(&cp);
 243 
 244         return ((cp.cp_ecx & (0x1 << X2APIC_CPUID_BIT)) ? 1 : 0);
 245 }
 246 
 247 void
 248 apic_enable_x2apic(void)
 249 {
 250         uint64_t apic_base_msr;
 251 
 252         if (apic_local_mode() == LOCAL_X2APIC) {
 253                 /* BIOS apparently has enabled X2APIC */
 254                 if (apic_mode != LOCAL_X2APIC)
 255                         x2apic_update_psm();
 256                 return;
 257         }
 258 
 259         /*
 260          * This is the first time we are enabling X2APIC on this CPU
 261          */
 262         apic_base_msr = rdmsr(REG_APIC_BASE_MSR);
 263         apic_base_msr = apic_base_msr | (0x1 << X2APIC_ENABLE_BIT);
 264         wrmsr(REG_APIC_BASE_MSR, apic_base_msr);
 265 
 266         if (apic_mode != LOCAL_X2APIC)
 267                 x2apic_update_psm();
 268 }
 269 
 270 /*
 271  * Determine which mode the current CPU is in. See the table above.
 272  * (IA32_APIC_BASE[11])   (IA32_APIC_BASE[10])
 273  */
 274 int
 275 apic_local_mode(void)
 276 {
 277         uint64_t apic_base_msr;
 278         int bit = ((0x1 << (X2APIC_ENABLE_BIT + 1)) |
 279             (0x1 << X2APIC_ENABLE_BIT));
 280 
 281         apic_base_msr = rdmsr(REG_APIC_BASE_MSR);
 282 
 283         if ((apic_base_msr & bit) == bit)
 284                 return (LOCAL_X2APIC);
 285         else
 286                 return (LOCAL_APIC);
 287 }
 288 
 289 void
 290 apic_set_directed_EOI_handler()
 291 {
 292         apic_reg_ops->apic_send_eoi = apic_send_directed_EOI;
 293 }
 294 
 295 int
 296 apic_directed_EOI_supported()
 297 {
 298         uint32_t ver;
 299 
 300         ver = apic_reg_ops->apic_read(APIC_VERS_REG);
 301         if (ver & APIC_DIRECTED_EOI_BIT)
 302                 return (1);
 303 
 304         return (0);
 305 }
 306 
 307 /*
 308  * Change apic_reg_ops depending upon the apic_mode.
 309  */
 310 void
 311 apic_change_ops()
 312 {
 313         if (apic_mode == LOCAL_APIC)
 314                 apic_reg_ops = &local_apic_regs_ops;
 315         else if (apic_mode == LOCAL_X2APIC)
 316                 apic_reg_ops = &x2apic_regs_ops;
 317 }
 318 
 319 /*
 320  * Generates an interprocessor interrupt to another CPU when X2APIC mode is
 321  * enabled.
 322  */
 323 void
 324 x2apic_send_ipi(int cpun, int ipl)
 325 {
 326         int vector;
 327         ulong_t flag;
 328 
 329         ASSERT(apic_mode == LOCAL_X2APIC);
 330 
 331         /*
 332          * With X2APIC, Intel relaxed the semantics of the
 333          * WRMSR instruction such that references to the X2APIC
 334          * MSR registers are no longer serializing instructions.
 335          * The code that initiates IPIs assumes that some sort
 336          * of memory serialization occurs. The old APIC code
 337          * did a write to uncachable memory mapped registers.
 338          * Any reference to uncached memory is a serializing
 339          * operation. To mimic those semantics here, we do an
 340          * atomic operation, which translates to a LOCK OR instruction,
 341          * which is serializing.
 342          */
 343         atomic_or_ulong(&flag, 1);
 344 
 345         vector = apic_resv_vector[ipl];
 346 
 347         flag = intr_clear();
 348 
 349         /*
 350          * According to X2APIC specification in section '2.3.5.1' of
 351          * Interrupt Command Register Semantics, the semantics of
 352          * programming Interrupt Command Register to dispatch an interrupt
 353          * is simplified. A single MSR write to the 64-bit ICR is required
 354          * for dispatching an interrupt. Specifically with the 64-bit MSR
 355          * interface to ICR, system software is not required to check the
 356          * status of the delivery status bit prior to writing to the ICR
 357          * to send an IPI. With the removal of the Delivery Status bit,
 358          * system software no longer has a reason to read the ICR. It remains
 359          * readable only to aid in debugging.
 360          */
 361 #ifdef  DEBUG
 362         APIC_AV_PENDING_SET();
 363 #endif  /* DEBUG */
 364 
 365         if ((cpun == psm_get_cpu_id())) {
 366                 X2APIC_WRITE(X2APIC_SELF_IPI, vector);
 367         } else {
 368                 apic_reg_ops->apic_write_int_cmd(
 369                     apic_cpus[cpun].aci_local_id, vector);
 370         }
 371 
 372         intr_restore(flag);
 373 }
 374 
 375 /*
 376  * Generates IPI to another CPU depending on the local APIC mode.
 377  * apic_send_ipi() and x2apic_send_ipi() depends on the configured
 378  * mode of the local APIC, but that may not match the actual mode
 379  * early in CPU startup.
 380  *
 381  * Any changes made to this routine must be accompanied by similar
 382  * changes to apic_send_ipi().
 383  */
 384 void
 385 apic_common_send_ipi(int cpun, int ipl)
 386 {
 387         int vector;
 388         ulong_t flag;
 389         int mode = apic_local_mode();
 390 
 391         if (mode == LOCAL_X2APIC) {
 392                 x2apic_send_ipi(cpun, ipl);
 393                 return;
 394         }
 395 
 396         ASSERT(mode == LOCAL_APIC);
 397 
 398         vector = apic_resv_vector[ipl];
 399         ASSERT((vector >= APIC_BASE_VECT) && (vector <= APIC_SPUR_INTR));
 400         flag = intr_clear();
 401         while (local_apic_regs_ops.apic_read(APIC_INT_CMD1) & AV_PENDING)
 402                 apic_ret();
 403         local_apic_regs_ops.apic_write_int_cmd(apic_cpus[cpun].aci_local_id,
 404             vector);
 405         intr_restore(flag);
 406 }