1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25 /*
  26  * Copyright 2014 Josef 'Jeff' Sipek <jeffpc@josefsipek.net>
  27  */
  28 
  29 #include <sys/cpuvar.h>
  30 #include <sys/psm.h>
  31 #include <sys/archsystm.h>
  32 #include <sys/apic.h>
  33 #include <sys/sunddi.h>
  34 #include <sys/ddi_impldefs.h>
  35 #include <sys/mach_intr.h>
  36 #include <sys/sysmacros.h>
  37 #include <sys/trap.h>
  38 #include <sys/x86_archext.h>
  39 #include <sys/privregs.h>
  40 #include <sys/psm_common.h>
  41 
  42 /* Function prototypes of local apic and X2APIC */
  43 static uint64_t local_apic_read(uint32_t reg);
  44 static void local_apic_write(uint32_t reg, uint64_t value);
  45 static int get_local_apic_pri(void);
  46 static void local_apic_write_task_reg(uint64_t value);
  47 static void local_apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1);
  48 static uint64_t local_x2apic_read(uint32_t msr);
  49 static void local_x2apic_write(uint32_t msr, uint64_t value);
  50 static int get_local_x2apic_pri(void);
  51 static void local_x2apic_write_task_reg(uint64_t value);
  52 static void local_x2apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1);
  53 
  54 /*
  55  * According to the X2APIC specification:
  56  *
  57  *   xAPIC global enable    X2APIC enable         Description
  58  *   (IA32_APIC_BASE[11])   (IA32_APIC_BASE[10])
  59  * -----------------------------------------------------------
  60  *      0                       0       APIC is disabled
  61  *      0                       1       Invalid
  62  *      1                       0       APIC is enabled in xAPIC mode
  63  *      1                       1       APIC is enabled in X2APIC mode
  64  * -----------------------------------------------------------
  65  */
  66 int     x2apic_enable = 1;
  67 apic_mode_t apic_mode = LOCAL_APIC;     /* Default mode is Local APIC */
  68 
  69 /* Uses MMIO (Memory Mapped IO) */
  70 static apic_reg_ops_t local_apic_regs_ops = {
  71         local_apic_read,
  72         local_apic_write,
  73         get_local_apic_pri,
  74         local_apic_write_task_reg,
  75         local_apic_write_int_cmd,
  76         apic_send_EOI,
  77 };
  78 
  79 /* X2APIC : Uses RDMSR/WRMSR instructions to access APIC registers */
  80 static apic_reg_ops_t x2apic_regs_ops = {
  81         local_x2apic_read,
  82         local_x2apic_write,
  83         get_local_x2apic_pri,
  84         local_x2apic_write_task_reg,
  85         local_x2apic_write_int_cmd,
  86         apic_send_EOI,
  87 };
  88 
  89 int apic_have_32bit_cr8 = 0;
  90 
  91 /* The default ops is local APIC (Memory Mapped IO) */
  92 apic_reg_ops_t *apic_reg_ops = &local_apic_regs_ops;
  93 
  94 /*
  95  * APIC register ops related data sturctures and functions.
  96  */
  97 void apic_send_EOI();
  98 void apic_send_directed_EOI(uint32_t irq);
  99 
 100 #define X2APIC_ENABLE_BIT       10
 101 
 102 /*
 103  * Local APIC Implementation
 104  */
 105 static uint64_t
 106 local_apic_read(uint32_t reg)
 107 {
 108         return ((uint32_t)apicadr[reg]);
 109 }
 110 
 111 static void
 112 local_apic_write(uint32_t reg, uint64_t value)
 113 {
 114         apicadr[reg] = (uint32_t)value;
 115 }
 116 
 117 static int
 118 get_local_apic_pri(void)
 119 {
 120 #if defined(__amd64)
 121         return ((int)getcr8());
 122 #else
 123         if (apic_have_32bit_cr8)
 124                 return ((int)getcr8());
 125         return (apicadr[APIC_TASK_REG]);
 126 #endif
 127 }
 128 
 129 static void
 130 local_apic_write_task_reg(uint64_t value)
 131 {
 132 #if defined(__amd64)
 133         setcr8((ulong_t)(value >> APIC_IPL_SHIFT));
 134 #else
 135         if (apic_have_32bit_cr8)
 136                 setcr8((ulong_t)(value >> APIC_IPL_SHIFT));
 137         else
 138                 apicadr[APIC_TASK_REG] = (uint32_t)value;
 139 #endif
 140 }
 141 
 142 static void
 143 local_apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1)
 144 {
 145         apicadr[APIC_INT_CMD2] = cpu_id << APIC_ICR_ID_BIT_OFFSET;
 146         apicadr[APIC_INT_CMD1] = cmd1;
 147 }
 148 
 149 /*
 150  * X2APIC Implementation.
 151  */
 152 static uint64_t
 153 local_x2apic_read(uint32_t msr)
 154 {
 155         uint64_t i;
 156 
 157         i = (uint64_t)(rdmsr(REG_X2APIC_BASE_MSR + (msr >> 2)) & 0xffffffff);
 158         return (i);
 159 }
 160 
 161 static void
 162 local_x2apic_write(uint32_t msr, uint64_t value)
 163 {
 164         uint64_t tmp;
 165 
 166         if (msr != APIC_EOI_REG) {
 167                 tmp = rdmsr(REG_X2APIC_BASE_MSR + (msr >> 2));
 168                 tmp = (tmp & 0xffffffff00000000) | value;
 169         } else {
 170                 tmp = 0;
 171         }
 172 
 173         wrmsr((REG_X2APIC_BASE_MSR + (msr >> 2)), tmp);
 174 }
 175 
 176 static int
 177 get_local_x2apic_pri(void)
 178 {
 179         return (rdmsr(REG_X2APIC_BASE_MSR + (APIC_TASK_REG >> 2)));
 180 }
 181 
 182 static void
 183 local_x2apic_write_task_reg(uint64_t value)
 184 {
 185         X2APIC_WRITE(APIC_TASK_REG, value);
 186 }
 187 
 188 static void
 189 local_x2apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1)
 190 {
 191         wrmsr((REG_X2APIC_BASE_MSR + (APIC_INT_CMD1 >> 2)),
 192             (((uint64_t)cpu_id << 32) | cmd1));
 193 }
 194 
 195 /*ARGSUSED*/
 196 void
 197 apic_send_EOI(uint32_t irq)
 198 {
 199         apic_reg_ops->apic_write(APIC_EOI_REG, 0);
 200 }
 201 
 202 /*
 203  * Support for Directed EOI capability is available in both the xAPIC
 204  * and x2APIC mode.
 205  */
 206 void
 207 apic_send_directed_EOI(uint32_t irq)
 208 {
 209         uchar_t ioapicindex;
 210         uchar_t vector;
 211         apic_irq_t *apic_irq;
 212         short intr_index;
 213 
 214         /*
 215          * Following the EOI to the local APIC unit, perform a directed
 216          * EOI to the IOxAPIC generating the interrupt by writing to its
 217          * EOI register.
 218          *
 219          * A broadcast EOI is not generated.
 220          */
 221         apic_reg_ops->apic_write(APIC_EOI_REG, 0);
 222 
 223         apic_irq = apic_irq_table[irq];
 224         while (apic_irq) {
 225                 intr_index = apic_irq->airq_mps_intr_index;
 226                 if (intr_index == ACPI_INDEX || intr_index >= 0) {
 227                         ioapicindex = apic_irq->airq_ioapicindex;
 228                         vector = apic_irq->airq_vector;
 229                         ioapic_write_eoi(ioapicindex, vector);
 230                 }
 231                 apic_irq = apic_irq->airq_next;
 232         }
 233 }
 234 
 235 int
 236 apic_detect_x2apic(void)
 237 {
 238         if (x2apic_enable == 0)
 239                 return (0);
 240 
 241         return (is_x86_feature(x86_featureset, X86FSET_X2APIC));
 242 }
 243 
 244 void
 245 apic_enable_x2apic(void)
 246 {
 247         uint64_t apic_base_msr;
 248 
 249         if (apic_local_mode() == LOCAL_X2APIC) {
 250                 /* BIOS apparently has enabled X2APIC */
 251                 if (apic_mode != LOCAL_X2APIC)
 252                         x2apic_update_psm();
 253                 return;
 254         }
 255 
 256         /*
 257          * This is the first time we are enabling X2APIC on this CPU
 258          */
 259         apic_base_msr = rdmsr(REG_APIC_BASE_MSR);
 260         apic_base_msr = apic_base_msr | (0x1 << X2APIC_ENABLE_BIT);
 261         wrmsr(REG_APIC_BASE_MSR, apic_base_msr);
 262 
 263         if (apic_mode != LOCAL_X2APIC)
 264                 x2apic_update_psm();
 265 }
 266 
 267 /*
 268  * Determine which mode the current CPU is in. See the table above.
 269  * (IA32_APIC_BASE[11])   (IA32_APIC_BASE[10])
 270  */
 271 int
 272 apic_local_mode(void)
 273 {
 274         uint64_t apic_base_msr;
 275         int bit = ((0x1 << (X2APIC_ENABLE_BIT + 1)) |
 276             (0x1 << X2APIC_ENABLE_BIT));
 277 
 278         apic_base_msr = rdmsr(REG_APIC_BASE_MSR);
 279 
 280         if ((apic_base_msr & bit) == bit)
 281                 return (LOCAL_X2APIC);
 282         else
 283                 return (LOCAL_APIC);
 284 }
 285 
 286 void
 287 apic_set_directed_EOI_handler()
 288 {
 289         apic_reg_ops->apic_send_eoi = apic_send_directed_EOI;
 290 }
 291 
 292 int
 293 apic_directed_EOI_supported()
 294 {
 295         uint32_t ver;
 296 
 297         ver = apic_reg_ops->apic_read(APIC_VERS_REG);
 298         if (ver & APIC_DIRECTED_EOI_BIT)
 299                 return (1);
 300 
 301         return (0);
 302 }
 303 
 304 /*
 305  * Change apic_reg_ops depending upon the apic_mode.
 306  */
 307 void
 308 apic_change_ops()
 309 {
 310         if (apic_mode == LOCAL_APIC)
 311                 apic_reg_ops = &local_apic_regs_ops;
 312         else if (apic_mode == LOCAL_X2APIC)
 313                 apic_reg_ops = &x2apic_regs_ops;
 314 }
 315 
 316 /*
 317  * Generates an interprocessor interrupt to another CPU when X2APIC mode is
 318  * enabled.
 319  */
 320 void
 321 x2apic_send_ipi(int cpun, int ipl)
 322 {
 323         int vector;
 324         ulong_t flag;
 325 
 326         ASSERT(apic_mode == LOCAL_X2APIC);
 327 
 328         /*
 329          * With X2APIC, Intel relaxed the semantics of the
 330          * WRMSR instruction such that references to the X2APIC
 331          * MSR registers are no longer serializing instructions.
 332          * The code that initiates IPIs assumes that some sort
 333          * of memory serialization occurs. The old APIC code
 334          * did a write to uncachable memory mapped registers.
 335          * Any reference to uncached memory is a serializing
 336          * operation. To mimic those semantics here, we do an
 337          * atomic operation, which translates to a LOCK OR instruction,
 338          * which is serializing.
 339          */
 340         atomic_or_ulong(&flag, 1);
 341 
 342         vector = apic_resv_vector[ipl];
 343 
 344         flag = intr_clear();
 345 
 346         /*
 347          * According to X2APIC specification in section '2.3.5.1' of
 348          * Interrupt Command Register Semantics, the semantics of
 349          * programming Interrupt Command Register to dispatch an interrupt
 350          * is simplified. A single MSR write to the 64-bit ICR is required
 351          * for dispatching an interrupt. Specifically with the 64-bit MSR
 352          * interface to ICR, system software is not required to check the
 353          * status of the delivery status bit prior to writing to the ICR
 354          * to send an IPI. With the removal of the Delivery Status bit,
 355          * system software no longer has a reason to read the ICR. It remains
 356          * readable only to aid in debugging.
 357          */
 358 #ifdef  DEBUG
 359         APIC_AV_PENDING_SET();
 360 #endif  /* DEBUG */
 361 
 362         if ((cpun == psm_get_cpu_id())) {
 363                 X2APIC_WRITE(X2APIC_SELF_IPI, vector);
 364         } else {
 365                 apic_reg_ops->apic_write_int_cmd(
 366                     apic_cpus[cpun].aci_local_id, vector);
 367         }
 368 
 369         intr_restore(flag);
 370 }
 371 
 372 /*
 373  * Generates IPI to another CPU depending on the local APIC mode.
 374  * apic_send_ipi() and x2apic_send_ipi() depends on the configured
 375  * mode of the local APIC, but that may not match the actual mode
 376  * early in CPU startup.
 377  *
 378  * Any changes made to this routine must be accompanied by similar
 379  * changes to apic_send_ipi().
 380  */
 381 void
 382 apic_common_send_ipi(int cpun, int ipl)
 383 {
 384         int vector;
 385         ulong_t flag;
 386         int mode = apic_local_mode();
 387 
 388         if (mode == LOCAL_X2APIC) {
 389                 x2apic_send_ipi(cpun, ipl);
 390                 return;
 391         }
 392 
 393         ASSERT(mode == LOCAL_APIC);
 394 
 395         vector = apic_resv_vector[ipl];
 396         ASSERT((vector >= APIC_BASE_VECT) && (vector <= APIC_SPUR_INTR));
 397         flag = intr_clear();
 398         while (local_apic_regs_ops.apic_read(APIC_INT_CMD1) & AV_PENDING)
 399                 apic_ret();
 400         local_apic_regs_ops.apic_write_int_cmd(apic_cpus[cpun].aci_local_id,
 401             vector);
 402         intr_restore(flag);
 403 }