1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25 
  26 #include <sys/cpuvar.h>
  27 #include <sys/psm.h>
  28 #include <sys/archsystm.h>
  29 #include <sys/apic.h>
  30 #include <sys/sunddi.h>
  31 #include <sys/ddi_impldefs.h>
  32 #include <sys/mach_intr.h>
  33 #include <sys/sysmacros.h>
  34 #include <sys/trap.h>
  35 #include <sys/x86_archext.h>
  36 #include <sys/privregs.h>
  37 #include <sys/psm_common.h>
  38 
  39 /* Function prototypes of local apic and X2APIC */
  40 static uint64_t local_apic_read(uint32_t reg);
  41 static void local_apic_write(uint32_t reg, uint64_t value);
  42 static int get_local_apic_pri(void);
  43 static void local_apic_write_task_reg(uint64_t value);
  44 static void local_apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1);
  45 static uint64_t local_x2apic_read(uint32_t msr);
  46 static void local_x2apic_write(uint32_t msr, uint64_t value);
  47 static int get_local_x2apic_pri(void);
  48 static void local_x2apic_write_task_reg(uint64_t value);
  49 static void local_x2apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1);
  50 
  51 /*
  52  * According to the X2APIC specification:
  53  *
  54  *   xAPIC global enable    X2APIC enable         Description
  55  *   (IA32_APIC_BASE[11])   (IA32_APIC_BASE[10])
  56  * -----------------------------------------------------------
  57  *      0                       0       APIC is disabled
  58  *      0                       1       Invalid
  59  *      1                       0       APIC is enabled in xAPIC mode
  60  *      1                       1       APIC is enabled in X2APIC mode
  61  * -----------------------------------------------------------
  62  */
  63 int     x2apic_enable = 1;
  64 int     apic_mode = LOCAL_APIC;         /* Default mode is Local APIC */
  65 
  66 /* Uses MMIO (Memory Mapped IO) */
  67 static apic_reg_ops_t local_apic_regs_ops = {
  68         local_apic_read,
  69         local_apic_write,
  70         get_local_apic_pri,
  71         local_apic_write_task_reg,
  72         local_apic_write_int_cmd,
  73         apic_send_EOI,
  74 };
  75 
  76 /* X2APIC : Uses RDMSR/WRMSR instructions to access APIC registers */
  77 static apic_reg_ops_t x2apic_regs_ops = {
  78         local_x2apic_read,
  79         local_x2apic_write,
  80         get_local_x2apic_pri,
  81         local_x2apic_write_task_reg,
  82         local_x2apic_write_int_cmd,
  83         apic_send_EOI,
  84 };
  85 
  86 int apic_have_32bit_cr8 = 0;
  87 
  88 /* The default ops is local APIC (Memory Mapped IO) */
  89 apic_reg_ops_t *apic_reg_ops = &local_apic_regs_ops;
  90 
  91 /*
  92  * APIC register ops related data sturctures and functions.
  93  */
  94 void apic_send_EOI();
  95 void apic_send_directed_EOI(uint32_t irq);
  96 
  97 #define X2APIC_ENABLE_BIT       10
  98 
  99 /*
 100  * Local APIC Implementation
 101  */
 102 static uint64_t
 103 local_apic_read(uint32_t reg)
 104 {
 105         return ((uint32_t)apicadr[reg]);
 106 }
 107 
 108 static void
 109 local_apic_write(uint32_t reg, uint64_t value)
 110 {
 111         LOCAL_APIC_WRITE_REG(reg, (uint32_t)value);
 112 }
 113 
 114 static int
 115 get_local_apic_pri(void)
 116 {
 117 #if defined(__amd64)
 118         return ((int)getcr8());
 119 #else
 120         if (apic_have_32bit_cr8)
 121                 return ((int)getcr8());
 122         return (apicadr[APIC_TASK_REG]);
 123 #endif
 124 }
 125 
 126 static void
 127 local_apic_write_task_reg(uint64_t value)
 128 {
 129 #if defined(__amd64)
 130         setcr8((ulong_t)(value >> APIC_IPL_SHIFT));
 131 #else
 132         if (apic_have_32bit_cr8)
 133                 setcr8((ulong_t)(value >> APIC_IPL_SHIFT));
 134         else
 135                 LOCAL_APIC_WRITE_REG(APIC_TASK_REG, (uint32_t)value);
 136 #endif
 137 }
 138 
 139 static void
 140 local_apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1)
 141 {
 142         apicadr[APIC_INT_CMD2] = cpu_id << APIC_ICR_ID_BIT_OFFSET;
 143         apicadr[APIC_INT_CMD1] = cmd1;
 144 }
 145 
 146 /*
 147  * X2APIC Implementation.
 148  */
 149 static uint64_t
 150 local_x2apic_read(uint32_t msr)
 151 {
 152         uint64_t i;
 153 
 154         i = (uint64_t)(rdmsr(REG_X2APIC_BASE_MSR + (msr >> 2)) & 0xffffffff);
 155         return (i);
 156 }
 157 
 158 static void
 159 local_x2apic_write(uint32_t msr, uint64_t value)
 160 {
 161         uint64_t tmp;
 162 
 163         if (msr != APIC_EOI_REG) {
 164                 tmp = rdmsr(REG_X2APIC_BASE_MSR + (msr >> 2));
 165                 tmp = (tmp & 0xffffffff00000000) | value;
 166         } else {
 167                 tmp = 0;
 168         }
 169 
 170         wrmsr((REG_X2APIC_BASE_MSR + (msr >> 2)), tmp);
 171 }
 172 
 173 static int
 174 get_local_x2apic_pri(void)
 175 {
 176         return (rdmsr(REG_X2APIC_BASE_MSR + (APIC_TASK_REG >> 2)));
 177 }
 178 
 179 static void
 180 local_x2apic_write_task_reg(uint64_t value)
 181 {
 182         X2APIC_WRITE(APIC_TASK_REG, value);
 183 }
 184 
 185 static void
 186 local_x2apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1)
 187 {
 188         wrmsr((REG_X2APIC_BASE_MSR + (APIC_INT_CMD1 >> 2)),
 189             (((uint64_t)cpu_id << 32) | cmd1));
 190 }
 191 
 192 /*ARGSUSED*/
 193 void
 194 apic_send_EOI(uint32_t irq)
 195 {
 196         apic_reg_ops->apic_write(APIC_EOI_REG, 0);
 197 }
 198 
 199 /*
 200  * Support for Directed EOI capability is available in both the xAPIC
 201  * and x2APIC mode.
 202  */
 203 void
 204 apic_send_directed_EOI(uint32_t irq)
 205 {
 206         uchar_t ioapicindex;
 207         uchar_t vector;
 208         apic_irq_t *apic_irq;
 209         short intr_index;
 210 
 211         /*
 212          * Following the EOI to the local APIC unit, perform a directed
 213          * EOI to the IOxAPIC generating the interrupt by writing to its
 214          * EOI register.
 215          *
 216          * A broadcast EOI is not generated.
 217          */
 218         apic_reg_ops->apic_write(APIC_EOI_REG, 0);
 219 
 220         apic_irq = apic_irq_table[irq];
 221         while (apic_irq) {
 222                 intr_index = apic_irq->airq_mps_intr_index;
 223                 if (intr_index == ACPI_INDEX || intr_index >= 0) {
 224                         ioapicindex = apic_irq->airq_ioapicindex;
 225                         vector = apic_irq->airq_vector;
 226                         ioapic_write_eoi(ioapicindex, vector);
 227                 }
 228                 apic_irq = apic_irq->airq_next;
 229         }
 230 }
 231 
 232 int
 233 apic_detect_x2apic(void)
 234 {
 235         if (x2apic_enable == 0)
 236                 return (0);
 237 
 238         return is_x86_feature(x86_featureset, X86FSET_X2APIC);
 239 }
 240 
 241 void
 242 apic_enable_x2apic(void)
 243 {
 244         uint64_t apic_base_msr;
 245 
 246         if (apic_local_mode() == LOCAL_X2APIC) {
 247                 /* BIOS apparently has enabled X2APIC */
 248                 if (apic_mode != LOCAL_X2APIC)
 249                         x2apic_update_psm();
 250                 return;
 251         }
 252 
 253         /*
 254          * This is the first time we are enabling X2APIC on this CPU
 255          */
 256         apic_base_msr = rdmsr(REG_APIC_BASE_MSR);
 257         apic_base_msr = apic_base_msr | (0x1 << X2APIC_ENABLE_BIT);
 258         wrmsr(REG_APIC_BASE_MSR, apic_base_msr);
 259 
 260         if (apic_mode != LOCAL_X2APIC)
 261                 x2apic_update_psm();
 262 }
 263 
 264 /*
 265  * Determine which mode the current CPU is in. See the table above.
 266  * (IA32_APIC_BASE[11])   (IA32_APIC_BASE[10])
 267  */
 268 int
 269 apic_local_mode(void)
 270 {
 271         uint64_t apic_base_msr;
 272         int bit = ((0x1 << (X2APIC_ENABLE_BIT + 1)) |
 273             (0x1 << X2APIC_ENABLE_BIT));
 274 
 275         apic_base_msr = rdmsr(REG_APIC_BASE_MSR);
 276 
 277         if ((apic_base_msr & bit) == bit)
 278                 return (LOCAL_X2APIC);
 279         else
 280                 return (LOCAL_APIC);
 281 }
 282 
 283 void
 284 apic_set_directed_EOI_handler()
 285 {
 286         apic_reg_ops->apic_send_eoi = apic_send_directed_EOI;
 287 }
 288 
 289 int
 290 apic_directed_EOI_supported()
 291 {
 292         uint32_t ver;
 293 
 294         ver = apic_reg_ops->apic_read(APIC_VERS_REG);
 295         if (ver & APIC_DIRECTED_EOI_BIT)
 296                 return (1);
 297 
 298         return (0);
 299 }
 300 
 301 /*
 302  * Change apic_reg_ops depending upon the apic_mode.
 303  */
 304 void
 305 apic_change_ops()
 306 {
 307         if (apic_mode == LOCAL_APIC)
 308                 apic_reg_ops = &local_apic_regs_ops;
 309         else if (apic_mode == LOCAL_X2APIC)
 310                 apic_reg_ops = &x2apic_regs_ops;
 311 }
 312 
 313 /*
 314  * Generates an interprocessor interrupt to another CPU when X2APIC mode is
 315  * enabled.
 316  */
 317 void
 318 x2apic_send_ipi(int cpun, int ipl)
 319 {
 320         int vector;
 321         ulong_t flag;
 322 
 323         ASSERT(apic_mode == LOCAL_X2APIC);
 324 
 325         /*
 326          * With X2APIC, Intel relaxed the semantics of the
 327          * WRMSR instruction such that references to the X2APIC
 328          * MSR registers are no longer serializing instructions.
 329          * The code that initiates IPIs assumes that some sort
 330          * of memory serialization occurs. The old APIC code
 331          * did a write to uncachable memory mapped registers.
 332          * Any reference to uncached memory is a serializing
 333          * operation. To mimic those semantics here, we do an
 334          * atomic operation, which translates to a LOCK OR instruction,
 335          * which is serializing.
 336          */
 337         atomic_or_ulong(&flag, 1);
 338 
 339         vector = apic_resv_vector[ipl];
 340 
 341         flag = intr_clear();
 342 
 343         /*
 344          * According to X2APIC specification in section '2.3.5.1' of
 345          * Interrupt Command Register Semantics, the semantics of
 346          * programming Interrupt Command Register to dispatch an interrupt
 347          * is simplified. A single MSR write to the 64-bit ICR is required
 348          * for dispatching an interrupt. Specifically with the 64-bit MSR
 349          * interface to ICR, system software is not required to check the
 350          * status of the delivery status bit prior to writing to the ICR
 351          * to send an IPI. With the removal of the Delivery Status bit,
 352          * system software no longer has a reason to read the ICR. It remains
 353          * readable only to aid in debugging.
 354          */
 355 #ifdef  DEBUG
 356         APIC_AV_PENDING_SET();
 357 #endif  /* DEBUG */
 358 
 359         if ((cpun == psm_get_cpu_id())) {
 360                 X2APIC_WRITE(X2APIC_SELF_IPI, vector);
 361         } else {
 362                 apic_reg_ops->apic_write_int_cmd(
 363                     apic_cpus[cpun].aci_local_id, vector);
 364         }
 365 
 366         intr_restore(flag);
 367 }
 368 
 369 /*
 370  * Generates IPI to another CPU depending on the local APIC mode.
 371  * apic_send_ipi() and x2apic_send_ipi() depends on the configured
 372  * mode of the local APIC, but that may not match the actual mode
 373  * early in CPU startup.
 374  *
 375  * Any changes made to this routine must be accompanied by similar
 376  * changes to apic_send_ipi().
 377  */
 378 void
 379 apic_common_send_ipi(int cpun, int ipl)
 380 {
 381         int vector;
 382         ulong_t flag;
 383         int mode = apic_local_mode();
 384 
 385         if (mode == LOCAL_X2APIC) {
 386                 x2apic_send_ipi(cpun, ipl);
 387                 return;
 388         }
 389 
 390         ASSERT(mode == LOCAL_APIC);
 391 
 392         vector = apic_resv_vector[ipl];
 393         ASSERT((vector >= APIC_BASE_VECT) && (vector <= APIC_SPUR_INTR));
 394         flag = intr_clear();
 395         while (local_apic_regs_ops.apic_read(APIC_INT_CMD1) & AV_PENDING)
 396                 apic_ret();
 397         local_apic_regs_ops.apic_write_int_cmd(apic_cpus[cpun].aci_local_id,
 398             vector);
 399         intr_restore(flag);
 400 }